From 38b9c96eb73e91c304f0d0a5c923406f9d0966e0 Mon Sep 17 00:00:00 2001 From: Conor Flynn Date: Wed, 12 Apr 2023 11:59:50 -0400 Subject: [PATCH 1/3] stable graphql api-handler integration --- DeFi-Data-Engine/Api-Handler/.classpath | 1 + DeFi-Data-Engine/Api-Handler/pom.xml | 1 - .../request/types/RequestFramework.java | 10 +- .../request/types/RequestGraphQL.java | 256 ++++++++++++++++++ ...graph-reserve-params-hist-items.properties | 58 ++++ .../resources/requests/graph-users.properties | 32 +++ .../test/java/test/sample/TestGraphGL.java | 61 +++++ 7 files changed, 413 insertions(+), 6 deletions(-) create mode 100644 DeFi-Data-Engine/Api-Handler/src/main/java/org/stream/external/request/types/RequestGraphQL.java create mode 100644 DeFi-Data-Engine/Api-Handler/src/main/resources/requests/graph-reserve-params-hist-items.properties create mode 100644 DeFi-Data-Engine/Api-Handler/src/main/resources/requests/graph-users.properties create mode 100644 DeFi-Data-Engine/Api-Handler/src/test/java/test/sample/TestGraphGL.java diff --git a/DeFi-Data-Engine/Api-Handler/.classpath b/DeFi-Data-Engine/Api-Handler/.classpath index 14cb2803..b045a086 100644 --- a/DeFi-Data-Engine/Api-Handler/.classpath +++ b/DeFi-Data-Engine/Api-Handler/.classpath @@ -11,6 +11,7 @@ + diff --git a/DeFi-Data-Engine/Api-Handler/pom.xml b/DeFi-Data-Engine/Api-Handler/pom.xml index e05f00da..d5c23ec0 100644 --- a/DeFi-Data-Engine/Api-Handler/pom.xml +++ b/DeFi-Data-Engine/Api-Handler/pom.xml @@ -46,7 +46,6 @@ reflections 0.10.2 - diff --git a/DeFi-Data-Engine/Api-Handler/src/main/java/org/stream/external/request/types/RequestFramework.java b/DeFi-Data-Engine/Api-Handler/src/main/java/org/stream/external/request/types/RequestFramework.java index cd437ddd..6413d6af 100644 --- a/DeFi-Data-Engine/Api-Handler/src/main/java/org/stream/external/request/types/RequestFramework.java +++ b/DeFi-Data-Engine/Api-Handler/src/main/java/org/stream/external/request/types/RequestFramework.java @@ -19,7 +19,7 @@ public abstract class RequestFramework { - private final static DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + public final static DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); private final String name; private String url; @@ -32,9 +32,9 @@ public abstract class RequestFramework { private final String[] path; private final boolean is_dated; private final String date_location; - private final String date_start_var; - private final String date_end_var; - private final DateTimeFormatter date_format; + protected final String date_start_var; + protected final String date_end_var; + protected final DateTimeFormatter date_format; // default constructor used for templating public RequestFramework() { @@ -305,7 +305,7 @@ protected String process(String url, HashMap properties, HashMap return processRequest(url, all_properties, all_headers); } - private String processRequest(String url, HashMap properties, HashMap headers) { + protected String processRequest(String url, HashMap properties, HashMap headers) { OkHttpClient client = new OkHttpClient(); Request request = getRequest(url, properties, headers); if(request == null) { diff --git a/DeFi-Data-Engine/Api-Handler/src/main/java/org/stream/external/request/types/RequestGraphQL.java b/DeFi-Data-Engine/Api-Handler/src/main/java/org/stream/external/request/types/RequestGraphQL.java new file mode 100644 index 00000000..6c3cca54 --- /dev/null +++ b/DeFi-Data-Engine/Api-Handler/src/main/java/org/stream/external/request/types/RequestGraphQL.java @@ -0,0 +1,256 @@ +package org.stream.external.request.types; + +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.LocalDate; +import java.util.HashMap; + +import org.application.apihandler.ApiHandlerApplication; +import org.json.JSONArray; +import org.json.JSONObject; + +public class RequestGraphQL extends RequestFramework { + + public RequestGraphQL() { + super(); + } + + public RequestGraphQL(String name, String url, String[] url_path, HashMap properties, HashMap headers, + HashMap tags, String[] recursive_location, String recursive_replacement, String[] path, + boolean is_dated, String date_location, String date_start_var, String date_end_var, String date_format) { + super(name, url, url_path, properties, headers, tags, recursive_location, recursive_replacement, path, + is_dated, date_location, date_start_var, date_end_var, date_format); + } + + @Override + public String getType() { + return "graphql"; + } + + @Override + protected String processRequest(String url, HashMap properties, HashMap headers) { + // validate properties and url and not empty + if(url.isEmpty() || properties.isEmpty()) + return "Key parameter is empty"; + + // check for required tag -l + if(!hasTag("-l")) { + System.err.println(String.format("Missing required recursive parameter <-l>")); + return "Missing required recursive parameter <-l>"; + } + + // check for -l being an integer + try { + Integer.parseInt(getTag("-l")); + } catch(Exception e) { + e.printStackTrace(); + System.err.println(String.format("Value following <-l> must be an integer.")); + return "Value following <-l> must be an integer."; + } + + // validate all required properties exist + String[] req_properties = {"values", "method"}; + for(String key : req_properties) { + if(!properties.containsKey(key)) + return String.format("Required property <%s> not found.", key); + } + + // build query from properties: + + // generate values + String[] values_arr = properties.get("values").split(":"); + StringBuilder values = new StringBuilder(); + for(int i = 0; i < values_arr.length; i++) { + values.append(values_arr[i]); + if(i != values_arr.length - 1) + values.append(","); + } + + // generate timestamp/recursive values + String recursive_location = getRecursiveLocation()[0]; + StringBuilder where = new StringBuilder(); + if(properties.containsKey("gt")) { + where.append(String.format("{%s_gt:%s", recursive_location, properties.get("gt"))); + if(properties.containsKey("lt")) + where.append(String.format(" %s_lt:%s", recursive_location, properties.get("lt"))); + } + + // if no gt or lt detected, check if dated + else { + // if dated + if(properties.containsKey(date_start_var) && properties.containsKey(date_end_var)) { + // define timestamp definition based on recursive parameter for gt and lt + LocalDate start_date = LocalDate.parse(properties.get(date_start_var), formatter); + LocalDate end_date = LocalDate.parse(properties.get(date_end_var), formatter); + long start_epoch = start_date.toEpochDay() * 86400L; + long end_epoch = end_date.toEpochDay() * 86400L; + + // append + where.append(String.format("{%s_gt:%s %s_lt:%s", + recursive_location, start_epoch, + recursive_location, end_epoch)); + + // push gt and lt properties + properties.put("gt", "" + start_epoch); + properties.put("lt", "" + end_epoch); + } + + // if not dated then apply basic lt + else { + where.append(String.format("{%s_gt:0", recursive_location)); + properties.put("gt", "0"); + } + } + + // close recursion + where.append("}"); + + // append + String query = String.format("query {" + + "%s(first:%s orderBy:%s where:%s){%s}}", + properties.get("method"), + getTag("-l"), + recursive_location, + where, + values); + + System.out.println(query); + // make request to server: + + // create client + HttpClient client = HttpClient.newHttpClient(); + + // create http request with POST method + HttpRequest.Builder builder = HttpRequest.newBuilder() + .uri(URI.create(getUrl())) + .POST(HttpRequest.BodyPublishers.ofString(new JSONObject().put("query", query).toString())); + + // add all headers + for(String key : headers.keySet()) { + builder = builder.header(key, headers.get(key)); + } + + // generate request + HttpRequest request = builder.build(); + + // submit request + HttpResponse response; + try { + response = client.send(request, HttpResponse.BodyHandlers.ofString()); + } catch(Exception e) { + e.printStackTrace(); + return "Invalid client request to server"; + } + + if(response.statusCode() != 200) { + return "GraphQL request failed with status code: " + response.statusCode(); + } + + // extract json body + String body = response.body(); + JSONObject json = new JSONObject(body); + if(json.has("errors")) { + return json.toString(); + } + + // process in handler + return handle(body, properties, headers); + } + + @Override + protected String handle(String json, HashMap properties, HashMap headers) { + // parse json formatting + JSONObject obj = new JSONObject(json); + + // validate all required parameters are present: + // check for required tag -l + if(!hasTag("-l")) { + System.err.println(String.format("Missing required recursive parameter <-l>")); + return "Missing required recursive parameter <-l>"; + } + + // check for -l being an integer + int limit; + try { + limit = Integer.parseInt(getTag("-l")); + } catch(Exception e) { + e.printStackTrace(); + System.err.println(String.format("Value following <-l> must be an integer.")); + return "Value following <-l> must be an integer."; + } + + // validate that the base has the proper data path + String[] path = getPath(); + JSONArray data = null; + for(int i = 0; i < path.length; i++) { + if(i == path.length - 1) { + if(obj.has(path[i])) { + try { + data = obj.getJSONArray(path[i]); + } catch(Exception e) { + System.err.println("obj path type is not of type . Cannot parse"); + return "obj path type is not of type . Cannot parse"; + } + } + } + + else if(obj.has(path[i])) { + try { + obj = obj.getJSONObject(path[i]); + } catch(Exception e) { + System.err.println("obj path type step is not of type . Cannot parse."); + return "obj path type step is not of type . Cannot parse."; + } + } + + else { + return "Data path is invalid. Please revise configuration."; + } + } + + // validate that data is non-empty + if(data == null) { + System.err.println("Data array retrieval had fatal error, killing process."); + return "Data array retrieval had fatal error, killing process."; + } + + // define recursive location + String recursive_location = getRecursiveLocation()[0]; + + // extract and print data + // validate recursive parameter with first data point and store value from last + // note that if dated then use date restrictive query + for(int i = 0; i < data.length(); i++) { + // retrieve values + HashMap point = parse(data.getJSONObject(i)); + + // if i == 0 then parse recursive parameter + if(i == 0 && !point.containsKey(recursive_location)) { + System.err.println("Point does not contain recursive location."); + return "Point does not contain recursive location."; + } + + // push data + ApiHandlerApplication.output(this.getName(), point); + + // if last data point then retrieve recursive parameter + if(i == data.length() - 1) { + if(!point.containsKey(recursive_location)) { + System.err.println("Final point does not contain recursive location. Data collection may not be complete."); + return "Final point does not contain recursive location. Data collection may not be complete."; + } + + // update gt property with new value + properties.put("gt", String.format("\"%s\"", point.get(recursive_location))); + } + } + + // if data is less than provided limit then return + if(data.length() < limit) + return null; + + return process(getUrl(), properties, headers); + } +} diff --git a/DeFi-Data-Engine/Api-Handler/src/main/resources/requests/graph-reserve-params-hist-items.properties b/DeFi-Data-Engine/Api-Handler/src/main/resources/requests/graph-reserve-params-hist-items.properties new file mode 100644 index 00000000..9b30bffb --- /dev/null +++ b/DeFi-Data-Engine/Api-Handler/src/main/resources/requests/graph-reserve-params-hist-items.properties @@ -0,0 +1,58 @@ +request.name= graph-reserve-params-hist-items + +url.base= https://api.thegraph.com/subgraphs/name/aave/protocol-v2 + +url.properties= method,reserveParamsHistoryItems,\ + values,\ + id:\ + reserve{id}:\ + variableBorrowRate:\ + variableBorrowIndex:\ + utilizationRate:\ + stableBorrowRate:\ + averageStableBorrowRate:\ + liquidityIndex:\ + liquidityRate:\ + totalLiquidity:\ + totalATokenSupply:\ + totalLiquidityAsCollateral:\ + availableLiquidity:\ + priceInEth:\ + priceInUsd:\ + timestamp:\ + totalScaledVariableDebt:\ + totalCurrentVariableDebt:\ + totalPrincipalStableDebt:\ + lifetimePrincipalStableDebt:\ + lifetimeScaledVariableDebt:\ + lifetimeCurrentVariableDebt:\ + lifetimeLiquidity:\ + lifetimeRepayments:\ + lifetimeWithdrawals:\ + lifetimeBorrows:\ + lifetimeLiquidated:\ + lifetimeFlashLoans:\ + lifetimeFlashLoanPremium:\ + lifetimeReserveFactorAccrued:\ + lifetimeDepositorsInterestEarned + +url.headers= Content-Type,application/json + +data.path= data,\ + reserveParamsHistoryItems + +recursion.type= graphql + +recursion.tags= -l,1000 + +recursion.location= timestamp + +date.valid= true + +date.location= properties + +date.start= startDate + +date.end= endDate + +date.format= yyyy-MM-dd \ No newline at end of file diff --git a/DeFi-Data-Engine/Api-Handler/src/main/resources/requests/graph-users.properties b/DeFi-Data-Engine/Api-Handler/src/main/resources/requests/graph-users.properties new file mode 100644 index 00000000..b1cb27a6 --- /dev/null +++ b/DeFi-Data-Engine/Api-Handler/src/main/resources/requests/graph-users.properties @@ -0,0 +1,32 @@ +request.name= graph-users + +url.base= https://api.thegraph.com/subgraphs/name/aave/protocol-v2 + +url.properties= method,users,\ + values,\ + id:\ + borrowedReservesCount:\ + unclaimedRewards:\ + lifetimeRewards:\ + incentivesLastUpdated + +url.headers= Content-Type,application/json + +data.path= data,\ + users + +recursion.type= graphql + +recursion.tags= -l,1000 + +recursion.location= id + +date.valid= true + +date.location= properties + +date.start= startDate + +date.end= endDate + +date.format= yyyy-MM-dd \ No newline at end of file diff --git a/DeFi-Data-Engine/Api-Handler/src/test/java/test/sample/TestGraphGL.java b/DeFi-Data-Engine/Api-Handler/src/test/java/test/sample/TestGraphGL.java new file mode 100644 index 00000000..df099436 --- /dev/null +++ b/DeFi-Data-Engine/Api-Handler/src/test/java/test/sample/TestGraphGL.java @@ -0,0 +1,61 @@ +package test.sample; + +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +public class TestGraphGL { + +// public static void main(String[] args) throws IOException, InterruptedException { +// execute(); +// } + + public static void execute() throws IOException, InterruptedException, JSONException { + // utc to epoch + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + LocalDate date = LocalDate.parse("2023-01-01", formatter); + LocalDate tmr = date.plusDays(1); + long s_epoch = date.toEpochDay() * 86400L; + long e_epoch = tmr.toEpochDay() * 86400L; + + String query = String.format("query {reserveParamsHistoryItems(first:1000 orderBy: timestamp where: {timestamp_gt:%d, timestamp_lt:%d}){id, timestamp}}", s_epoch, e_epoch); + + // Define the GraphQL endpoint URL + String url = "https://api.thegraph.com/subgraphs/name/aave/protocol-v2"; + + // Create an HTTP client + HttpClient httpClient = HttpClient.newHttpClient(); + + // Create an HTTP request with POST method and JSON payload + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(url)) + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString("{\"query\":\"" + query + "\"}")) + .build(); + + // Send the HTTP request and get the response + HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); + + // Check if the request was successful (status code 200) + if (response.statusCode() == 200) { + // Extract the response body as JSON + String responseBody = response.body(); + // Process the response body as needed + JSONObject obj = new JSONObject(responseBody); + JSONObject data = obj.getJSONObject("data"); + JSONArray reserve = data.getJSONArray("reserveParamsHistoryItems"); + System.out.println(reserve.get(1).toString()); + } else { + // Print an error message if the request failed + System.out.println("GraphQL request failed with status code: " + response.statusCode()); + } + } +} \ No newline at end of file From 17a3b9161f9440e87ed77ea18adf859b68395c3e Mon Sep 17 00:00:00 2001 From: Conor Flynn Date: Thu, 13 Apr 2023 13:36:33 -0400 Subject: [PATCH 2/3] graphql integration into primary engine --- .../requests/ExternalRequestFramework.java | 8 +- .../requests/ExternalRequestGraphQL.java | 260 ++++++++++++++++++ ...graph-reserve-params-hist-items.properties | 58 ++++ .../resources/requests/graph-users.properties | 32 +++ 4 files changed, 354 insertions(+), 4 deletions(-) create mode 100644 DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/requests/ExternalRequestGraphQL.java create mode 100644 DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-reserve-params-hist-items.properties create mode 100644 DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-users.properties diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/requests/ExternalRequestFramework.java b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/requests/ExternalRequestFramework.java index e9fe300b..56320192 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/requests/ExternalRequestFramework.java +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/requests/ExternalRequestFramework.java @@ -22,7 +22,7 @@ public abstract class ExternalRequestFramework { -private final static DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + protected final static DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); protected final ExternalStreamManager manager; @@ -39,8 +39,8 @@ public abstract class ExternalRequestFramework { private final String[] path; private final boolean is_dated; private final String date_location; - private final String date_start_var; - private final String date_end_var; + protected final String date_start_var; + protected final String date_end_var; private final DateTimeFormatter date_format; // default constructor used for templating @@ -342,7 +342,7 @@ protected String process(String url, HashMap properties, HashMap return processRequest(url, all_properties, all_headers); } - private String processRequest(String url, HashMap properties, HashMap headers) { + protected String processRequest(String url, HashMap properties, HashMap headers) { OkHttpClient client = new OkHttpClient(); Request request = getRequest(url, properties, headers); if(request == null) { diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/requests/ExternalRequestGraphQL.java b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/requests/ExternalRequestGraphQL.java new file mode 100644 index 00000000..02b379fb --- /dev/null +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/requests/ExternalRequestGraphQL.java @@ -0,0 +1,260 @@ +package org.stream.external.requests; + +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.LocalDate; +import java.util.HashMap; + +import org.json.JSONArray; +import org.json.JSONObject; +import org.stream.external.handler.ExternalStreamManager; + +public class ExternalRequestGraphQL extends ExternalRequestFramework { + + public ExternalRequestGraphQL() { + super(); + } + + public ExternalRequestGraphQL(ExternalStreamManager manager, String name, String url, String[] url_path, HashMap properties, HashMap headers, + HashMap tags, String[] recursive_location, String recursive_replacement, String[] path, + boolean is_dated, String date_location, String date_start_var, String date_end_var, String date_format) { + + super(manager, name, url, url_path, properties, headers, tags, recursive_location, recursive_replacement, path, + is_dated, date_location, date_start_var, date_end_var, date_format); + } + + @Override + public String getType() { + return "graphql"; + } + + @Override + protected String processRequest(String url, HashMap properties, HashMap headers) { + // validate properties and url and not empty + if(url.isEmpty() || properties.isEmpty()) + return "Key parameter is empty"; + + // check for required tag -l + if(!hasTag("-l")) { + System.err.println(String.format("Missing required recursive parameter <-l>")); + return "Missing required recursive parameter <-l>"; + } + + // check for -l being an integer + try { + Integer.parseInt(getTag("-l")); + } catch(Exception e) { + e.printStackTrace(); + System.err.println(String.format("Value following <-l> must be an integer.")); + return "Value following <-l> must be an integer."; + } + + // validate all required properties exist + String[] req_properties = {"values", "method"}; + for(String key : req_properties) { + if(!properties.containsKey(key)) + return String.format("Required property <%s> not found.", key); + } + + // build query from properties: + + // generate values + String[] values_arr = properties.get("values").split(":"); + StringBuilder values = new StringBuilder(); + for(int i = 0; i < values_arr.length; i++) { + values.append(values_arr[i]); + if(i != values_arr.length - 1) + values.append(","); + } + + // generate timestamp/recursive values + String recursive_location = getRecursiveLocation()[0]; + StringBuilder where = new StringBuilder(); + if(properties.containsKey("gt")) { + where.append(String.format("{%s_gt:%s", recursive_location, properties.get("gt"))); + if(properties.containsKey("lt")) + where.append(String.format(" %s_lt:%s", recursive_location, properties.get("lt"))); + } + + // if no gt or lt detected, check if dated + else { + // if dated + if(properties.containsKey(date_start_var) && properties.containsKey(date_end_var)) { + // define timestamp definition based on recursive parameter for gt and lt + LocalDate start_date = LocalDate.parse(properties.get(date_start_var), formatter); + LocalDate end_date = LocalDate.parse(properties.get(date_end_var), formatter); + long start_epoch = start_date.toEpochDay() * 86400L; + long end_epoch = end_date.toEpochDay() * 86400L; + + // append + where.append(String.format("{%s_gt:%s %s_lt:%s", + recursive_location, start_epoch, + recursive_location, end_epoch)); + + // push gt and lt properties + properties.put("gt", "" + start_epoch); + properties.put("lt", "" + end_epoch); + } + + // if not dated then apply basic lt + else { + where.append(String.format("{%s_gt:0", recursive_location)); + properties.put("gt", "0"); + } + } + + // close recursion + where.append("}"); + + // append + String query = String.format("query {" + + "%s(first:%s orderBy:%s where:%s){%s}}", + properties.get("method"), + getTag("-l"), + recursive_location, + where, + values); + + // make request to server: + + // create client + HttpClient client = HttpClient.newHttpClient(); + + // create http request with POST method + HttpRequest.Builder builder = HttpRequest.newBuilder() + .uri(URI.create(getUrl())) + .POST(HttpRequest.BodyPublishers.ofString(new JSONObject().put("query", query).toString())); + + // add all headers + for(String key : headers.keySet()) { + builder = builder.header(key, headers.get(key)); + } + + // generate request + HttpRequest request = builder.build(); + + // submit request + HttpResponse response; + try { + response = client.send(request, HttpResponse.BodyHandlers.ofString()); + } catch(Exception e) { + e.printStackTrace(); + return "Invalid client request to server"; + } + + if(response.statusCode() != 200) { + return "GraphQL request failed with status code: " + response.statusCode(); + } + + // extract json body + String body = response.body(); + JSONObject json = new JSONObject(body); + if(json.has("errors")) { + return json.toString(); + } + + // process in handler + return handle(body, properties, headers); + } + + @Override + protected String handle(String json, HashMap properties, HashMap headers) { + // parse json formatting + JSONObject obj = new JSONObject(json); + + // validate all required parameters are present: + // check for required tag -l + if(!hasTag("-l")) { + System.err.println(String.format("Missing required recursive parameter <-l>")); + return "Missing required recursive parameter <-l>"; + } + + // check for -l being an integer + int limit; + try { + limit = Integer.parseInt(getTag("-l")); + } catch(Exception e) { + e.printStackTrace(); + System.err.println(String.format("Value following <-l> must be an integer.")); + return "Value following <-l> must be an integer."; + } + + // validate that the base has the proper data path + String[] path = getPath(); + JSONArray data = null; + for(int i = 0; i < path.length; i++) { + if(i == path.length - 1) { + if(obj.has(path[i])) { + try { + data = obj.getJSONArray(path[i]); + } catch(Exception e) { + System.err.println("obj path type is not of type . Cannot parse"); + return "obj path type is not of type . Cannot parse"; + } + } + } + + else if(obj.has(path[i])) { + try { + obj = obj.getJSONObject(path[i]); + } catch(Exception e) { + System.err.println("obj path type step is not of type . Cannot parse."); + return "obj path type step is not of type . Cannot parse."; + } + } + + else { + return "Data path is invalid. Please revise configuration."; + } + } + + // validate that data is non-empty + if(data == null) { + System.err.println("Data array retrieval had fatal error, killing process."); + return "Data array retrieval had fatal error, killing process."; + } + + // define recursive location + String recursive_location = getRecursiveLocation()[0]; + + // extract and print data + // validate recursive parameter with first data point and store value from last + // note that if dated then use date restrictive query + for(int i = 0; i < data.length(); i++) { + // retrieve values + HashMap point = parse(data.getJSONObject(i)); + + // if i == 0 then parse recursive parameter + if(i == 0 && !point.containsKey(recursive_location)) { + System.err.println("Point does not contain recursive location."); + return "Point does not contain recursive location."; + } + + // push data + manager.processRequest(getCollection(), point); + + // if last data point then retrieve recursive parameter + if(i == data.length() - 1) { + if(!point.containsKey(recursive_location)) { + System.err.println("Final point does not contain recursive location. Data collection may not be complete."); + return "Final point does not contain recursive location. Data collection may not be complete."; + } + + // update gt property with new value + // check for if gt uses a string parsing parameter. if so then add with parameter + if(properties.get("gt").charAt(0) == '\"') + properties.put("gt", String.format("\"%s\"", point.get(recursive_location))); + else + properties.put("gt", point.get(recursive_location)); + } + } + + // if data is less than provided limit then return + if(data.length() < limit) + return null; + + return process(getUrl(), properties, headers); + } +} diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-reserve-params-hist-items.properties b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-reserve-params-hist-items.properties new file mode 100644 index 00000000..9b30bffb --- /dev/null +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-reserve-params-hist-items.properties @@ -0,0 +1,58 @@ +request.name= graph-reserve-params-hist-items + +url.base= https://api.thegraph.com/subgraphs/name/aave/protocol-v2 + +url.properties= method,reserveParamsHistoryItems,\ + values,\ + id:\ + reserve{id}:\ + variableBorrowRate:\ + variableBorrowIndex:\ + utilizationRate:\ + stableBorrowRate:\ + averageStableBorrowRate:\ + liquidityIndex:\ + liquidityRate:\ + totalLiquidity:\ + totalATokenSupply:\ + totalLiquidityAsCollateral:\ + availableLiquidity:\ + priceInEth:\ + priceInUsd:\ + timestamp:\ + totalScaledVariableDebt:\ + totalCurrentVariableDebt:\ + totalPrincipalStableDebt:\ + lifetimePrincipalStableDebt:\ + lifetimeScaledVariableDebt:\ + lifetimeCurrentVariableDebt:\ + lifetimeLiquidity:\ + lifetimeRepayments:\ + lifetimeWithdrawals:\ + lifetimeBorrows:\ + lifetimeLiquidated:\ + lifetimeFlashLoans:\ + lifetimeFlashLoanPremium:\ + lifetimeReserveFactorAccrued:\ + lifetimeDepositorsInterestEarned + +url.headers= Content-Type,application/json + +data.path= data,\ + reserveParamsHistoryItems + +recursion.type= graphql + +recursion.tags= -l,1000 + +recursion.location= timestamp + +date.valid= true + +date.location= properties + +date.start= startDate + +date.end= endDate + +date.format= yyyy-MM-dd \ No newline at end of file diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-users.properties b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-users.properties new file mode 100644 index 00000000..b1cb27a6 --- /dev/null +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-users.properties @@ -0,0 +1,32 @@ +request.name= graph-users + +url.base= https://api.thegraph.com/subgraphs/name/aave/protocol-v2 + +url.properties= method,users,\ + values,\ + id:\ + borrowedReservesCount:\ + unclaimedRewards:\ + lifetimeRewards:\ + incentivesLastUpdated + +url.headers= Content-Type,application/json + +data.path= data,\ + users + +recursion.type= graphql + +recursion.tags= -l,1000 + +recursion.location= id + +date.valid= true + +date.location= properties + +date.start= startDate + +date.end= endDate + +date.format= yyyy-MM-dd \ No newline at end of file From 84a0fc802a04a1660a3eeb588eaac5f7e3fdb922 Mon Sep 17 00:00:00 2001 From: Conor Flynn Date: Thu, 13 Apr 2023 14:36:18 -0400 Subject: [PATCH 3/3] add graph protocol configuration files --- .../requests/graph-borrows.properties | 39 ++++++++ .../requests/graph-collaterals.properties | 35 ++++++++ .../requests/graph-deposits.properties | 35 ++++++++ .../requests/graph-flash-loans.properties | 35 ++++++++ .../requests/graph-liquidations.properties | 38 ++++++++ .../graph-price-history-items.properties | 31 +++++++ .../requests/graph-redeems.properties | 35 ++++++++ .../requests/graph-repays.properties | 35 ++++++++ .../requests/graph-reserves.properties | 37 ++++++++ .../resources/requests/graph-swaps.properties | 90 +++++++++++++++++++ .../requests/graph-user-reserves.properties | 51 +++++++++++ .../resources/requests/graph-users.properties | 2 +- 12 files changed, 462 insertions(+), 1 deletion(-) create mode 100644 DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-borrows.properties create mode 100644 DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-collaterals.properties create mode 100644 DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-deposits.properties create mode 100644 DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-flash-loans.properties create mode 100644 DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-liquidations.properties create mode 100644 DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-price-history-items.properties create mode 100644 DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-redeems.properties create mode 100644 DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-repays.properties create mode 100644 DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-reserves.properties create mode 100644 DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-swaps.properties create mode 100644 DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-user-reserves.properties diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-borrows.properties b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-borrows.properties new file mode 100644 index 00000000..5c183fdc --- /dev/null +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-borrows.properties @@ -0,0 +1,39 @@ +request.name= graph-borrows + +url.base= https://api.thegraph.com/subgraphs/name/aave/protocol-v2 + +url.properties= method,borrows,\ + values,\ + id:\ + user:\ + caller:\ + reserve:\ + pool:\ + userReserve:\ + timestamp:\ + amount:\ + borrowRate:\ + borrowRateMode:\ + stableTokenDebt:\ + variableTokenDebt + +url.headers= Content-Type,application/json + +data.path= data,\ + borrows + +recursion.type= graphql + +recursion.tags= -l,1000 + +recursion.location= timestamp + +date.valid= true + +date.location= properties + +date.start= startDate + +date.end= endDate + +date.format= yyyy-MM-dd \ No newline at end of file diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-collaterals.properties b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-collaterals.properties new file mode 100644 index 00000000..f97e7f82 --- /dev/null +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-collaterals.properties @@ -0,0 +1,35 @@ +request.name= graph-collaterals + +url.base= https://api.thegraph.com/subgraphs/name/aave/protocol-v2 + +url.properties= method,usageAsCollaterals,\ + values,\ + id:\ + user:\ + reserve:\ + pool:\ + userReserve:\ + timestamp:\ + fromState:\ + toState + +url.headers= Content-Type,application/json + +data.path= data,\ + usageAsCollaterals + +recursion.type= graphql + +recursion.tags= -l,1000 + +recursion.location= timestamp + +date.valid= true + +date.location= properties + +date.start= startDate + +date.end= endDate + +date.format= yyyy-MM-dd \ No newline at end of file diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-deposits.properties b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-deposits.properties new file mode 100644 index 00000000..c9c5be4c --- /dev/null +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-deposits.properties @@ -0,0 +1,35 @@ +request.name= graph-deposits + +url.base= https://api.thegraph.com/subgraphs/name/aave/protocol-v2 + +url.properties= method,deposits,\ + values,\ + id:\ + user:\ + caller:\ + reserve:\ + pool:\ + userReserve:\ + timestamp:\ + amount + +url.headers= Content-Type,application/json + +data.path= data,\ + deposits + +recursion.type= graphql + +recursion.tags= -l,1000 + +recursion.location= timestamp + +date.valid= true + +date.location= properties + +date.start= startDate + +date.end= endDate + +date.format= yyyy-MM-dd \ No newline at end of file diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-flash-loans.properties b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-flash-loans.properties new file mode 100644 index 00000000..346426e6 --- /dev/null +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-flash-loans.properties @@ -0,0 +1,35 @@ +request.name= graph-flash-loans + +url.base= https://api.thegraph.com/subgraphs/name/aave/protocol-v2 + +url.properties= method,flashLoans,\ + values,\ + id:\ + pool:\ + reserve:\ + target:\ + amount:\ + totalFee:\ + initiator:\ + timestamp + +url.headers= Content-Type,application/json + +data.path= data,\ + flashLoans + +recursion.type= graphql + +recursion.tags= -l,1000 + +recursion.location= timestamp + +date.valid= true + +date.location= properties + +date.start= startDate + +date.end= endDate + +date.format= yyyy-MM-dd \ No newline at end of file diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-liquidations.properties b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-liquidations.properties new file mode 100644 index 00000000..7b02888d --- /dev/null +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-liquidations.properties @@ -0,0 +1,38 @@ +request.name= graph-liquidations + +url.base= https://api.thegraph.com/subgraphs/name/aave/protocol-v2 + +url.properties= method,liquidationCalls,\ + values,\ + id:\ + user:\ + pool:\ + collateralReserve:\ + collateralUserReserve:\ + principalReserve:\ + principalUserReserve:\ + principalAmount:\ + collateralAmount:\ + timestamp:\ + liquidator + +url.headers= Content-Type,application/json + +data.path= data,\ + liquidationCalls + +recursion.type= graphql + +recursion.tags= -l,1000 + +recursion.location= timestamp + +date.valid= true + +date.location= properties + +date.start= startDate + +date.end= endDate + +date.format= yyyy-MM-dd \ No newline at end of file diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-price-history-items.properties b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-price-history-items.properties new file mode 100644 index 00000000..745986e0 --- /dev/null +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-price-history-items.properties @@ -0,0 +1,31 @@ +request.name= graph-price-history-items + +url.base= https://api.thegraph.com/subgraphs/name/aave/protocol-v2 + +url.properties= method,priceHistoryItems,\ + values,\ + id:\ + asset:\ + price:\ + timestamp + +url.headers= Content-Type,application/json + +data.path= data,\ + priceHistoryItems + +recursion.type= graphql + +recursion.tags= -l,1000 + +recursion.location= timestamp + +date.valid= true + +date.location= properties + +date.start= startDate + +date.end= endDate + +date.format= yyyy-MM-dd \ No newline at end of file diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-redeems.properties b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-redeems.properties new file mode 100644 index 00000000..d7ecd7dc --- /dev/null +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-redeems.properties @@ -0,0 +1,35 @@ +request.name= graph-redeems + +url.base= https://api.thegraph.com/subgraphs/name/aave/protocol-v2 + +url.properties= method,redeemUnderlyings,\ + values,\ + id:\ + user:\ + to:\ + reserve:\ + pool:\ + userReserve:\ + timestamp:\ + amount + +url.headers= Content-Type,application/json + +data.path= data,\ + redeemUnderlyings + +recursion.type= graphql + +recursion.tags= -l,1000 + +recursion.location= timestamp + +date.valid= true + +date.location= properties + +date.start= startDate + +date.end= endDate + +date.format= yyyy-MM-dd \ No newline at end of file diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-repays.properties b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-repays.properties new file mode 100644 index 00000000..129c34a0 --- /dev/null +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-repays.properties @@ -0,0 +1,35 @@ +request.name= graph-repays + +url.base= https://api.thegraph.com/subgraphs/name/aave/protocol-v2 + +url.properties= method,repays,\ + values,\ + id:\ + user:\ + repayer:\ + reserve:\ + pool:\ + userReserve:\ + timestamp:\ + amount + +url.headers= Content-Type,application/json + +data.path= data,\ + repays + +recursion.type= graphql + +recursion.tags= -l,1000 + +recursion.location= timestamp + +date.valid= true + +date.location= properties + +date.start= startDate + +date.end= endDate + +date.format= yyyy-MM-dd \ No newline at end of file diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-reserves.properties b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-reserves.properties new file mode 100644 index 00000000..15dcd2e9 --- /dev/null +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-reserves.properties @@ -0,0 +1,37 @@ +request.name= graph-swaps + +url.base= https://api.thegraph.com/subgraphs/name/aave/protocol-v2 + +url.properties= method,swaps,\ + values,\ + id:\ + user:\ + reserve:\ + pool:\ + userReserve:\ + timestamp:\ + borrowRateModeTo:\ + borrowRateModeFrom:\ + stableBorrowRate:\ + variableBorrowRate + +url.headers= Content-Type,application/json + +data.path= data,\ + swaps + +recursion.type= graphql + +recursion.tags= -l,1000 + +recursion.location= timestamp + +date.valid= true + +date.location= properties + +date.start= startDate + +date.end= endDate + +date.format= yyyy-MM-dd \ No newline at end of file diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-swaps.properties b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-swaps.properties new file mode 100644 index 00000000..6711c348 --- /dev/null +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-swaps.properties @@ -0,0 +1,90 @@ +request.name= graph-reserves + +url.base= https://api.thegraph.com/subgraphs/name/aave/protocol-v2 + +url.properties= method,reserves,\ + values,\ + id:\ + underlyingAsset:\ + pool:\ + symbol:\ + name:\ + decimals:\ + usageAsCollateralEnabled:\ + borrowingEnabled:\ + stableBorrowRateEnabled:\ + isActive:\ + isFrozen:\ + price:\ + reserveInterestRateStrategy:\ + optimalUtilisationRate:\ + variableRateSlope1:\ + variableRateSlope2:\ + stableRateSlope1:\ + stableRateSlope2:\ + baseVariableBorrowRate:\ + baseLTVasCollateral:\ + reserveLiquidationThreshold:\ + reserveLiquidationBonus:\ + utilizationRate:\ + totalLiquidity:\ + totalATokenSupply:\ + totalLiquidityAsCollateral:\ + totalPrincipalStableDebt:\ + totalScaledVariableDebt:\ + totalCurrentVariableDebt:\ + totalDeposits:\ + liquidityRate:\ + averageStableRate:\ + variableBorrowRate:\ + stableBorrowRate:\ + liquidityIndex:\ + variableBorrowIndex:\ + aToken{id}:\ + vToken{id}:\ + sToken{id}:\ + reserveFactor:\ + lastUpdateTimestamp:\ + stableDebtLastUpdateTimestamp:\ + aEmissionPerSecond:\ + vEmissionPerSecond:\ + sEmissionPerSecond:\ + aTokenIncentivesIndex:\ + vTokenIncentivesIndex:\ + sTokenIncentivesIndex:\ + aIncentivesLastUpdateTimestamp:\ + vIncentivesLastUpdateTimestamp:\ + sIncentivesLastUpdateTimestamp:\ + lifetimeLiquidity:\ + lifetimePrincipalStableDebt:\ + lifetimeScaledVariableDebt:\ + lifetimeCurrentVariableDebt:\ + lifetimeRepayments:\ + lifetimeWithdrawals:\ + lifetimeBorrows:\ + lifetimeLiquidated:\ + lifetimeFlashLoans:\ + lifetimeFlashLoanPremium:\ + lifetimeDepositorsInterestEarned:\ + ifetimeReserveFactorAccrued + +url.headers= Content-Type,application/json + +data.path= data,\ + reserves + +recursion.type= graphql + +recursion.tags= -l,1000 + +recursion.location= id + +date.valid= false + +date.location= properties + +date.start= startDate + +date.end= endDate + +date.format= yyyy-MM-dd \ No newline at end of file diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-user-reserves.properties b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-user-reserves.properties new file mode 100644 index 00000000..3732930c --- /dev/null +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-user-reserves.properties @@ -0,0 +1,51 @@ +request.name= graph-swaps + +url.base= https://api.thegraph.com/subgraphs/name/aave/protocol-v2 + +url.properties= method,userReserves,\ + values,\ + id:\ + pool:\ + reserve:\ + user:\ + usageAsCollateralEnabledOnUser:\ + scaledATokenBalance:\ + currentATokenBalance:\ + scaledVariableDebt:\ + currentVariableDebt:\ + principalStableDebt:\ + currentStableDebt:\ + currentTotalDebt:\ + stableBorrowRate:\ + oldStableBorrowRate:\ + liquiditiyRate:\ + stableBorrowLastUpdateTimestamp:\ + variableBorrowIndex:\ + aTokenincentivesUserIndex:\ + vTokenincentivesUserIndex:\ + sTokenincentivesUserIndex:\ + aIncentivesLastUpdateTimestamp:\ + vIncentivesLastUpdateTimestamp:\ + sIncentivesLastUpdateTimestamp:\ + lastUpdateTimestamp + +url.headers= Content-Type,application/json + +data.path= data,\ + swaps + +recursion.type= graphql + +recursion.tags= -l,1000 + +recursion.location= timestamp + +date.valid= true + +date.location= properties + +date.start= startDate + +date.end= endDate + +date.format= yyyy-MM-dd \ No newline at end of file diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-users.properties b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-users.properties index b1cb27a6..d8b1abb7 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-users.properties +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-users.properties @@ -21,7 +21,7 @@ recursion.tags= -l,1000 recursion.location= id -date.valid= true +date.valid= false date.location= properties