diff --git a/DeFi-Data-Engine/Api-Handler/.classpath b/DeFi-Data-Engine/Api-Handler/.classpath index 14cb2803..b045a086 100644 --- a/DeFi-Data-Engine/Api-Handler/.classpath +++ b/DeFi-Data-Engine/Api-Handler/.classpath @@ -11,6 +11,7 @@ + diff --git a/DeFi-Data-Engine/Api-Handler/pom.xml b/DeFi-Data-Engine/Api-Handler/pom.xml index e05f00da..d5c23ec0 100644 --- a/DeFi-Data-Engine/Api-Handler/pom.xml +++ b/DeFi-Data-Engine/Api-Handler/pom.xml @@ -46,7 +46,6 @@ reflections 0.10.2 - diff --git a/DeFi-Data-Engine/Api-Handler/src/main/java/org/stream/external/request/types/RequestFramework.java b/DeFi-Data-Engine/Api-Handler/src/main/java/org/stream/external/request/types/RequestFramework.java index cd437ddd..6413d6af 100644 --- a/DeFi-Data-Engine/Api-Handler/src/main/java/org/stream/external/request/types/RequestFramework.java +++ b/DeFi-Data-Engine/Api-Handler/src/main/java/org/stream/external/request/types/RequestFramework.java @@ -19,7 +19,7 @@ public abstract class RequestFramework { - private final static DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + public final static DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); private final String name; private String url; @@ -32,9 +32,9 @@ public abstract class RequestFramework { private final String[] path; private final boolean is_dated; private final String date_location; - private final String date_start_var; - private final String date_end_var; - private final DateTimeFormatter date_format; + protected final String date_start_var; + protected final String date_end_var; + protected final DateTimeFormatter date_format; // default constructor used for templating public RequestFramework() { @@ -305,7 +305,7 @@ protected String process(String url, HashMap properties, HashMap return processRequest(url, all_properties, all_headers); } - private String processRequest(String url, HashMap properties, HashMap headers) { + protected String processRequest(String url, HashMap properties, HashMap headers) { OkHttpClient client = new OkHttpClient(); Request request = getRequest(url, properties, headers); if(request == null) { diff --git a/DeFi-Data-Engine/Api-Handler/src/main/java/org/stream/external/request/types/RequestGraphQL.java b/DeFi-Data-Engine/Api-Handler/src/main/java/org/stream/external/request/types/RequestGraphQL.java new file mode 100644 index 00000000..6c3cca54 --- /dev/null +++ b/DeFi-Data-Engine/Api-Handler/src/main/java/org/stream/external/request/types/RequestGraphQL.java @@ -0,0 +1,256 @@ +package org.stream.external.request.types; + +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.LocalDate; +import java.util.HashMap; + +import org.application.apihandler.ApiHandlerApplication; +import org.json.JSONArray; +import org.json.JSONObject; + +public class RequestGraphQL extends RequestFramework { + + public RequestGraphQL() { + super(); + } + + public RequestGraphQL(String name, String url, String[] url_path, HashMap properties, HashMap headers, + HashMap tags, String[] recursive_location, String recursive_replacement, String[] path, + boolean is_dated, String date_location, String date_start_var, String date_end_var, String date_format) { + super(name, url, url_path, properties, headers, tags, recursive_location, recursive_replacement, path, + is_dated, date_location, date_start_var, date_end_var, date_format); + } + + @Override + public String getType() { + return "graphql"; + } + + @Override + protected String processRequest(String url, HashMap properties, HashMap headers) { + // validate properties and url and not empty + if(url.isEmpty() || properties.isEmpty()) + return "Key parameter is empty"; + + // check for required tag -l + if(!hasTag("-l")) { + System.err.println(String.format("Missing required recursive parameter <-l>")); + return "Missing required recursive parameter <-l>"; + } + + // check for -l being an integer + try { + Integer.parseInt(getTag("-l")); + } catch(Exception e) { + e.printStackTrace(); + System.err.println(String.format("Value following <-l> must be an integer.")); + return "Value following <-l> must be an integer."; + } + + // validate all required properties exist + String[] req_properties = {"values", "method"}; + for(String key : req_properties) { + if(!properties.containsKey(key)) + return String.format("Required property <%s> not found.", key); + } + + // build query from properties: + + // generate values + String[] values_arr = properties.get("values").split(":"); + StringBuilder values = new StringBuilder(); + for(int i = 0; i < values_arr.length; i++) { + values.append(values_arr[i]); + if(i != values_arr.length - 1) + values.append(","); + } + + // generate timestamp/recursive values + String recursive_location = getRecursiveLocation()[0]; + StringBuilder where = new StringBuilder(); + if(properties.containsKey("gt")) { + where.append(String.format("{%s_gt:%s", recursive_location, properties.get("gt"))); + if(properties.containsKey("lt")) + where.append(String.format(" %s_lt:%s", recursive_location, properties.get("lt"))); + } + + // if no gt or lt detected, check if dated + else { + // if dated + if(properties.containsKey(date_start_var) && properties.containsKey(date_end_var)) { + // define timestamp definition based on recursive parameter for gt and lt + LocalDate start_date = LocalDate.parse(properties.get(date_start_var), formatter); + LocalDate end_date = LocalDate.parse(properties.get(date_end_var), formatter); + long start_epoch = start_date.toEpochDay() * 86400L; + long end_epoch = end_date.toEpochDay() * 86400L; + + // append + where.append(String.format("{%s_gt:%s %s_lt:%s", + recursive_location, start_epoch, + recursive_location, end_epoch)); + + // push gt and lt properties + properties.put("gt", "" + start_epoch); + properties.put("lt", "" + end_epoch); + } + + // if not dated then apply basic lt + else { + where.append(String.format("{%s_gt:0", recursive_location)); + properties.put("gt", "0"); + } + } + + // close recursion + where.append("}"); + + // append + String query = String.format("query {" + + "%s(first:%s orderBy:%s where:%s){%s}}", + properties.get("method"), + getTag("-l"), + recursive_location, + where, + values); + + System.out.println(query); + // make request to server: + + // create client + HttpClient client = HttpClient.newHttpClient(); + + // create http request with POST method + HttpRequest.Builder builder = HttpRequest.newBuilder() + .uri(URI.create(getUrl())) + .POST(HttpRequest.BodyPublishers.ofString(new JSONObject().put("query", query).toString())); + + // add all headers + for(String key : headers.keySet()) { + builder = builder.header(key, headers.get(key)); + } + + // generate request + HttpRequest request = builder.build(); + + // submit request + HttpResponse response; + try { + response = client.send(request, HttpResponse.BodyHandlers.ofString()); + } catch(Exception e) { + e.printStackTrace(); + return "Invalid client request to server"; + } + + if(response.statusCode() != 200) { + return "GraphQL request failed with status code: " + response.statusCode(); + } + + // extract json body + String body = response.body(); + JSONObject json = new JSONObject(body); + if(json.has("errors")) { + return json.toString(); + } + + // process in handler + return handle(body, properties, headers); + } + + @Override + protected String handle(String json, HashMap properties, HashMap headers) { + // parse json formatting + JSONObject obj = new JSONObject(json); + + // validate all required parameters are present: + // check for required tag -l + if(!hasTag("-l")) { + System.err.println(String.format("Missing required recursive parameter <-l>")); + return "Missing required recursive parameter <-l>"; + } + + // check for -l being an integer + int limit; + try { + limit = Integer.parseInt(getTag("-l")); + } catch(Exception e) { + e.printStackTrace(); + System.err.println(String.format("Value following <-l> must be an integer.")); + return "Value following <-l> must be an integer."; + } + + // validate that the base has the proper data path + String[] path = getPath(); + JSONArray data = null; + for(int i = 0; i < path.length; i++) { + if(i == path.length - 1) { + if(obj.has(path[i])) { + try { + data = obj.getJSONArray(path[i]); + } catch(Exception e) { + System.err.println("obj path type is not of type . Cannot parse"); + return "obj path type is not of type . Cannot parse"; + } + } + } + + else if(obj.has(path[i])) { + try { + obj = obj.getJSONObject(path[i]); + } catch(Exception e) { + System.err.println("obj path type step is not of type . Cannot parse."); + return "obj path type step is not of type . Cannot parse."; + } + } + + else { + return "Data path is invalid. Please revise configuration."; + } + } + + // validate that data is non-empty + if(data == null) { + System.err.println("Data array retrieval had fatal error, killing process."); + return "Data array retrieval had fatal error, killing process."; + } + + // define recursive location + String recursive_location = getRecursiveLocation()[0]; + + // extract and print data + // validate recursive parameter with first data point and store value from last + // note that if dated then use date restrictive query + for(int i = 0; i < data.length(); i++) { + // retrieve values + HashMap point = parse(data.getJSONObject(i)); + + // if i == 0 then parse recursive parameter + if(i == 0 && !point.containsKey(recursive_location)) { + System.err.println("Point does not contain recursive location."); + return "Point does not contain recursive location."; + } + + // push data + ApiHandlerApplication.output(this.getName(), point); + + // if last data point then retrieve recursive parameter + if(i == data.length() - 1) { + if(!point.containsKey(recursive_location)) { + System.err.println("Final point does not contain recursive location. Data collection may not be complete."); + return "Final point does not contain recursive location. Data collection may not be complete."; + } + + // update gt property with new value + properties.put("gt", String.format("\"%s\"", point.get(recursive_location))); + } + } + + // if data is less than provided limit then return + if(data.length() < limit) + return null; + + return process(getUrl(), properties, headers); + } +} diff --git a/DeFi-Data-Engine/Api-Handler/src/main/resources/requests/graph-reserve-params-hist-items.properties b/DeFi-Data-Engine/Api-Handler/src/main/resources/requests/graph-reserve-params-hist-items.properties new file mode 100644 index 00000000..9b30bffb --- /dev/null +++ b/DeFi-Data-Engine/Api-Handler/src/main/resources/requests/graph-reserve-params-hist-items.properties @@ -0,0 +1,58 @@ +request.name= graph-reserve-params-hist-items + +url.base= https://api.thegraph.com/subgraphs/name/aave/protocol-v2 + +url.properties= method,reserveParamsHistoryItems,\ + values,\ + id:\ + reserve{id}:\ + variableBorrowRate:\ + variableBorrowIndex:\ + utilizationRate:\ + stableBorrowRate:\ + averageStableBorrowRate:\ + liquidityIndex:\ + liquidityRate:\ + totalLiquidity:\ + totalATokenSupply:\ + totalLiquidityAsCollateral:\ + availableLiquidity:\ + priceInEth:\ + priceInUsd:\ + timestamp:\ + totalScaledVariableDebt:\ + totalCurrentVariableDebt:\ + totalPrincipalStableDebt:\ + lifetimePrincipalStableDebt:\ + lifetimeScaledVariableDebt:\ + lifetimeCurrentVariableDebt:\ + lifetimeLiquidity:\ + lifetimeRepayments:\ + lifetimeWithdrawals:\ + lifetimeBorrows:\ + lifetimeLiquidated:\ + lifetimeFlashLoans:\ + lifetimeFlashLoanPremium:\ + lifetimeReserveFactorAccrued:\ + lifetimeDepositorsInterestEarned + +url.headers= Content-Type,application/json + +data.path= data,\ + reserveParamsHistoryItems + +recursion.type= graphql + +recursion.tags= -l,1000 + +recursion.location= timestamp + +date.valid= true + +date.location= properties + +date.start= startDate + +date.end= endDate + +date.format= yyyy-MM-dd \ No newline at end of file diff --git a/DeFi-Data-Engine/Api-Handler/src/main/resources/requests/graph-users.properties b/DeFi-Data-Engine/Api-Handler/src/main/resources/requests/graph-users.properties new file mode 100644 index 00000000..b1cb27a6 --- /dev/null +++ b/DeFi-Data-Engine/Api-Handler/src/main/resources/requests/graph-users.properties @@ -0,0 +1,32 @@ +request.name= graph-users + +url.base= https://api.thegraph.com/subgraphs/name/aave/protocol-v2 + +url.properties= method,users,\ + values,\ + id:\ + borrowedReservesCount:\ + unclaimedRewards:\ + lifetimeRewards:\ + incentivesLastUpdated + +url.headers= Content-Type,application/json + +data.path= data,\ + users + +recursion.type= graphql + +recursion.tags= -l,1000 + +recursion.location= id + +date.valid= true + +date.location= properties + +date.start= startDate + +date.end= endDate + +date.format= yyyy-MM-dd \ No newline at end of file diff --git a/DeFi-Data-Engine/Api-Handler/src/test/java/test/sample/TestGraphGL.java b/DeFi-Data-Engine/Api-Handler/src/test/java/test/sample/TestGraphGL.java new file mode 100644 index 00000000..df099436 --- /dev/null +++ b/DeFi-Data-Engine/Api-Handler/src/test/java/test/sample/TestGraphGL.java @@ -0,0 +1,61 @@ +package test.sample; + +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +public class TestGraphGL { + +// public static void main(String[] args) throws IOException, InterruptedException { +// execute(); +// } + + public static void execute() throws IOException, InterruptedException, JSONException { + // utc to epoch + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + LocalDate date = LocalDate.parse("2023-01-01", formatter); + LocalDate tmr = date.plusDays(1); + long s_epoch = date.toEpochDay() * 86400L; + long e_epoch = tmr.toEpochDay() * 86400L; + + String query = String.format("query {reserveParamsHistoryItems(first:1000 orderBy: timestamp where: {timestamp_gt:%d, timestamp_lt:%d}){id, timestamp}}", s_epoch, e_epoch); + + // Define the GraphQL endpoint URL + String url = "https://api.thegraph.com/subgraphs/name/aave/protocol-v2"; + + // Create an HTTP client + HttpClient httpClient = HttpClient.newHttpClient(); + + // Create an HTTP request with POST method and JSON payload + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(url)) + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString("{\"query\":\"" + query + "\"}")) + .build(); + + // Send the HTTP request and get the response + HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); + + // Check if the request was successful (status code 200) + if (response.statusCode() == 200) { + // Extract the response body as JSON + String responseBody = response.body(); + // Process the response body as needed + JSONObject obj = new JSONObject(responseBody); + JSONObject data = obj.getJSONObject("data"); + JSONArray reserve = data.getJSONArray("reserveParamsHistoryItems"); + System.out.println(reserve.get(1).toString()); + } else { + // Print an error message if the request failed + System.out.println("GraphQL request failed with status code: " + response.statusCode()); + } + } +} \ No newline at end of file