From 17a3b9161f9440e87ed77ea18adf859b68395c3e Mon Sep 17 00:00:00 2001 From: Conor Flynn Date: Thu, 13 Apr 2023 13:36:33 -0400 Subject: [PATCH] graphql integration into primary engine --- .../requests/ExternalRequestFramework.java | 8 +- .../requests/ExternalRequestGraphQL.java | 260 ++++++++++++++++++ ...graph-reserve-params-hist-items.properties | 58 ++++ .../resources/requests/graph-users.properties | 32 +++ 4 files changed, 354 insertions(+), 4 deletions(-) create mode 100644 DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/requests/ExternalRequestGraphQL.java create mode 100644 DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-reserve-params-hist-items.properties create mode 100644 DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-users.properties diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/requests/ExternalRequestFramework.java b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/requests/ExternalRequestFramework.java index e9fe300b..56320192 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/requests/ExternalRequestFramework.java +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/requests/ExternalRequestFramework.java @@ -22,7 +22,7 @@ public abstract class ExternalRequestFramework { -private final static DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + protected final static DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); protected final ExternalStreamManager manager; @@ -39,8 +39,8 @@ public abstract class ExternalRequestFramework { private final String[] path; private final boolean is_dated; private final String date_location; - private final String date_start_var; - private final String date_end_var; + protected final String date_start_var; + protected final String date_end_var; private final DateTimeFormatter date_format; // default constructor used for templating @@ -342,7 +342,7 @@ protected String process(String url, HashMap properties, HashMap return processRequest(url, all_properties, all_headers); } - private String processRequest(String url, HashMap properties, HashMap headers) { + protected String processRequest(String url, HashMap properties, HashMap headers) { OkHttpClient client = new OkHttpClient(); Request request = getRequest(url, properties, headers); if(request == null) { diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/requests/ExternalRequestGraphQL.java b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/requests/ExternalRequestGraphQL.java new file mode 100644 index 00000000..02b379fb --- /dev/null +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/requests/ExternalRequestGraphQL.java @@ -0,0 +1,260 @@ +package org.stream.external.requests; + +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.LocalDate; +import java.util.HashMap; + +import org.json.JSONArray; +import org.json.JSONObject; +import org.stream.external.handler.ExternalStreamManager; + +public class ExternalRequestGraphQL extends ExternalRequestFramework { + + public ExternalRequestGraphQL() { + super(); + } + + public ExternalRequestGraphQL(ExternalStreamManager manager, String name, String url, String[] url_path, HashMap properties, HashMap headers, + HashMap tags, String[] recursive_location, String recursive_replacement, String[] path, + boolean is_dated, String date_location, String date_start_var, String date_end_var, String date_format) { + + super(manager, name, url, url_path, properties, headers, tags, recursive_location, recursive_replacement, path, + is_dated, date_location, date_start_var, date_end_var, date_format); + } + + @Override + public String getType() { + return "graphql"; + } + + @Override + protected String processRequest(String url, HashMap properties, HashMap headers) { + // validate properties and url and not empty + if(url.isEmpty() || properties.isEmpty()) + return "Key parameter is empty"; + + // check for required tag -l + if(!hasTag("-l")) { + System.err.println(String.format("Missing required recursive parameter <-l>")); + return "Missing required recursive parameter <-l>"; + } + + // check for -l being an integer + try { + Integer.parseInt(getTag("-l")); + } catch(Exception e) { + e.printStackTrace(); + System.err.println(String.format("Value following <-l> must be an integer.")); + return "Value following <-l> must be an integer."; + } + + // validate all required properties exist + String[] req_properties = {"values", "method"}; + for(String key : req_properties) { + if(!properties.containsKey(key)) + return String.format("Required property <%s> not found.", key); + } + + // build query from properties: + + // generate values + String[] values_arr = properties.get("values").split(":"); + StringBuilder values = new StringBuilder(); + for(int i = 0; i < values_arr.length; i++) { + values.append(values_arr[i]); + if(i != values_arr.length - 1) + values.append(","); + } + + // generate timestamp/recursive values + String recursive_location = getRecursiveLocation()[0]; + StringBuilder where = new StringBuilder(); + if(properties.containsKey("gt")) { + where.append(String.format("{%s_gt:%s", recursive_location, properties.get("gt"))); + if(properties.containsKey("lt")) + where.append(String.format(" %s_lt:%s", recursive_location, properties.get("lt"))); + } + + // if no gt or lt detected, check if dated + else { + // if dated + if(properties.containsKey(date_start_var) && properties.containsKey(date_end_var)) { + // define timestamp definition based on recursive parameter for gt and lt + LocalDate start_date = LocalDate.parse(properties.get(date_start_var), formatter); + LocalDate end_date = LocalDate.parse(properties.get(date_end_var), formatter); + long start_epoch = start_date.toEpochDay() * 86400L; + long end_epoch = end_date.toEpochDay() * 86400L; + + // append + where.append(String.format("{%s_gt:%s %s_lt:%s", + recursive_location, start_epoch, + recursive_location, end_epoch)); + + // push gt and lt properties + properties.put("gt", "" + start_epoch); + properties.put("lt", "" + end_epoch); + } + + // if not dated then apply basic lt + else { + where.append(String.format("{%s_gt:0", recursive_location)); + properties.put("gt", "0"); + } + } + + // close recursion + where.append("}"); + + // append + String query = String.format("query {" + + "%s(first:%s orderBy:%s where:%s){%s}}", + properties.get("method"), + getTag("-l"), + recursive_location, + where, + values); + + // make request to server: + + // create client + HttpClient client = HttpClient.newHttpClient(); + + // create http request with POST method + HttpRequest.Builder builder = HttpRequest.newBuilder() + .uri(URI.create(getUrl())) + .POST(HttpRequest.BodyPublishers.ofString(new JSONObject().put("query", query).toString())); + + // add all headers + for(String key : headers.keySet()) { + builder = builder.header(key, headers.get(key)); + } + + // generate request + HttpRequest request = builder.build(); + + // submit request + HttpResponse response; + try { + response = client.send(request, HttpResponse.BodyHandlers.ofString()); + } catch(Exception e) { + e.printStackTrace(); + return "Invalid client request to server"; + } + + if(response.statusCode() != 200) { + return "GraphQL request failed with status code: " + response.statusCode(); + } + + // extract json body + String body = response.body(); + JSONObject json = new JSONObject(body); + if(json.has("errors")) { + return json.toString(); + } + + // process in handler + return handle(body, properties, headers); + } + + @Override + protected String handle(String json, HashMap properties, HashMap headers) { + // parse json formatting + JSONObject obj = new JSONObject(json); + + // validate all required parameters are present: + // check for required tag -l + if(!hasTag("-l")) { + System.err.println(String.format("Missing required recursive parameter <-l>")); + return "Missing required recursive parameter <-l>"; + } + + // check for -l being an integer + int limit; + try { + limit = Integer.parseInt(getTag("-l")); + } catch(Exception e) { + e.printStackTrace(); + System.err.println(String.format("Value following <-l> must be an integer.")); + return "Value following <-l> must be an integer."; + } + + // validate that the base has the proper data path + String[] path = getPath(); + JSONArray data = null; + for(int i = 0; i < path.length; i++) { + if(i == path.length - 1) { + if(obj.has(path[i])) { + try { + data = obj.getJSONArray(path[i]); + } catch(Exception e) { + System.err.println("obj path type is not of type . Cannot parse"); + return "obj path type is not of type . Cannot parse"; + } + } + } + + else if(obj.has(path[i])) { + try { + obj = obj.getJSONObject(path[i]); + } catch(Exception e) { + System.err.println("obj path type step is not of type . Cannot parse."); + return "obj path type step is not of type . Cannot parse."; + } + } + + else { + return "Data path is invalid. Please revise configuration."; + } + } + + // validate that data is non-empty + if(data == null) { + System.err.println("Data array retrieval had fatal error, killing process."); + return "Data array retrieval had fatal error, killing process."; + } + + // define recursive location + String recursive_location = getRecursiveLocation()[0]; + + // extract and print data + // validate recursive parameter with first data point and store value from last + // note that if dated then use date restrictive query + for(int i = 0; i < data.length(); i++) { + // retrieve values + HashMap point = parse(data.getJSONObject(i)); + + // if i == 0 then parse recursive parameter + if(i == 0 && !point.containsKey(recursive_location)) { + System.err.println("Point does not contain recursive location."); + return "Point does not contain recursive location."; + } + + // push data + manager.processRequest(getCollection(), point); + + // if last data point then retrieve recursive parameter + if(i == data.length() - 1) { + if(!point.containsKey(recursive_location)) { + System.err.println("Final point does not contain recursive location. Data collection may not be complete."); + return "Final point does not contain recursive location. Data collection may not be complete."; + } + + // update gt property with new value + // check for if gt uses a string parsing parameter. if so then add with parameter + if(properties.get("gt").charAt(0) == '\"') + properties.put("gt", String.format("\"%s\"", point.get(recursive_location))); + else + properties.put("gt", point.get(recursive_location)); + } + } + + // if data is less than provided limit then return + if(data.length() < limit) + return null; + + return process(getUrl(), properties, headers); + } +} diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-reserve-params-hist-items.properties b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-reserve-params-hist-items.properties new file mode 100644 index 00000000..9b30bffb --- /dev/null +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-reserve-params-hist-items.properties @@ -0,0 +1,58 @@ +request.name= graph-reserve-params-hist-items + +url.base= https://api.thegraph.com/subgraphs/name/aave/protocol-v2 + +url.properties= method,reserveParamsHistoryItems,\ + values,\ + id:\ + reserve{id}:\ + variableBorrowRate:\ + variableBorrowIndex:\ + utilizationRate:\ + stableBorrowRate:\ + averageStableBorrowRate:\ + liquidityIndex:\ + liquidityRate:\ + totalLiquidity:\ + totalATokenSupply:\ + totalLiquidityAsCollateral:\ + availableLiquidity:\ + priceInEth:\ + priceInUsd:\ + timestamp:\ + totalScaledVariableDebt:\ + totalCurrentVariableDebt:\ + totalPrincipalStableDebt:\ + lifetimePrincipalStableDebt:\ + lifetimeScaledVariableDebt:\ + lifetimeCurrentVariableDebt:\ + lifetimeLiquidity:\ + lifetimeRepayments:\ + lifetimeWithdrawals:\ + lifetimeBorrows:\ + lifetimeLiquidated:\ + lifetimeFlashLoans:\ + lifetimeFlashLoanPremium:\ + lifetimeReserveFactorAccrued:\ + lifetimeDepositorsInterestEarned + +url.headers= Content-Type,application/json + +data.path= data,\ + reserveParamsHistoryItems + +recursion.type= graphql + +recursion.tags= -l,1000 + +recursion.location= timestamp + +date.valid= true + +date.location= properties + +date.start= startDate + +date.end= endDate + +date.format= yyyy-MM-dd \ No newline at end of file diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-users.properties b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-users.properties new file mode 100644 index 00000000..b1cb27a6 --- /dev/null +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-users.properties @@ -0,0 +1,32 @@ +request.name= graph-users + +url.base= https://api.thegraph.com/subgraphs/name/aave/protocol-v2 + +url.properties= method,users,\ + values,\ + id:\ + borrowedReservesCount:\ + unclaimedRewards:\ + lifetimeRewards:\ + incentivesLastUpdated + +url.headers= Content-Type,application/json + +data.path= data,\ + users + +recursion.type= graphql + +recursion.tags= -l,1000 + +recursion.location= id + +date.valid= true + +date.location= properties + +date.start= startDate + +date.end= endDate + +date.format= yyyy-MM-dd \ No newline at end of file