From 18d1fa142d0b6d96be80c7525769c368a1467225 Mon Sep 17 00:00:00 2001 From: Conor Flynn Date: Mon, 17 Apr 2023 13:16:00 -0400 Subject: [PATCH] modify properties files and make minor fixes to SRC RQST external handlers --- .../requests/ExternalRequestFramework.java | 19 ++++++++++++++----- .../requests/ExternalRequestGraphQL.java | 8 +++++--- .../registry/StreamRegistryController.java | 3 +++ .../requests/graph-aave-borrows.properties | 10 +++++----- .../graph-aave-collaterals.properties | 8 ++++---- .../requests/graph-aave-deposits.properties | 10 +++++----- .../graph-aave-flash-loans.properties | 4 ++-- .../graph-aave-liquidations.properties | 12 ++++++------ .../graph-aave-price-history-items.properties | 2 +- .../requests/graph-aave-redeems.properties | 10 +++++----- .../requests/graph-aave-repays.properties | 10 +++++----- .../graph-aave-user-reserves.properties | 6 +++--- 12 files changed, 58 insertions(+), 44 deletions(-) diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/requests/ExternalRequestFramework.java b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/requests/ExternalRequestFramework.java index faca19a8..a6bf2b3e 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/requests/ExternalRequestFramework.java +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/requests/ExternalRequestFramework.java @@ -387,11 +387,14 @@ protected String processRequest(String url, HashMap properties, public abstract String getType(); protected final HashMap parse(Object input) throws JSONException { - + return parse(input, null); + } + + protected final HashMap parse(Object input, String parent) throws JSONException { + HashMap out = new HashMap(); if (input instanceof JSONObject) { - Iterator keys = ((JSONObject) input).keys(); while (keys.hasNext()) { @@ -400,12 +403,18 @@ protected final HashMap parse(Object input) throws JSONException if (!(((JSONObject) input).get(key) instanceof JSONArray)) { if (((JSONObject) input).get(key) instanceof JSONObject) { - out.putAll(parse(((JSONObject) input).get(key))); + HashMap parsed = parse(((JSONObject) input).get(key), key); + for(String parsedKey : parsed.keySet()) { + out.put(parent == null ? parsedKey : parent, parsed.get(parsedKey)); + } } else { - out.put(key, ((JSONObject) input).get(key).toString()); + out.put(parent == null ? key : parent, ((JSONObject) input).get(key).toString()); } } else { - out.putAll(parse(new JSONArray(((JSONObject) input).get(key).toString()))); + HashMap parsed = parse(new JSONArray(((JSONObject) input).get(key).toString()), key); + for(String parsedKey : parsed.keySet()) { + out.put(parent == null ? parsedKey : parent, parsed.get(parsedKey)); + } } } } diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/requests/ExternalRequestGraphQL.java b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/requests/ExternalRequestGraphQL.java index 02b379fb..76cafb8f 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/requests/ExternalRequestGraphQL.java +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/requests/ExternalRequestGraphQL.java @@ -244,10 +244,12 @@ else if(obj.has(path[i])) { // update gt property with new value // check for if gt uses a string parsing parameter. if so then add with parameter - if(properties.get("gt").charAt(0) == '\"') + try { + long gt = Long.parseLong(point.get(recursive_location)); + properties.put("gt", "" + gt); + } catch(Exception e) { properties.put("gt", String.format("\"%s\"", point.get(recursive_location))); - else - properties.put("gt", point.get(recursive_location)); + } } } diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/registry/StreamRegistryController.java b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/registry/StreamRegistryController.java index 275bedcf..4da3e2ab 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/registry/StreamRegistryController.java +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/registry/StreamRegistryController.java @@ -76,6 +76,9 @@ public Response processRQST(Packet packet) { data.put("query", String.format("%s", sb)); Response lsh_response = send("LSH", "RQST", data); + if(lsh_response.code() == 200) + return lsh_response; + // if data does not exist send request to external stream handler if(lsh_response.code() == 446) send("ESH", "RQST", data); diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-aave-borrows.properties b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-aave-borrows.properties index 4dce6cad..4e1aef9c 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-aave-borrows.properties +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-aave-borrows.properties @@ -5,11 +5,11 @@ url.base= https://api.thegraph.com/subgraphs/name/aave/protocol-v2 url.properties= method,borrows,\ values,\ id:\ - user:\ - caller:\ - reserve:\ - pool:\ - userReserve:\ + user{id}:\ + caller{id}:\ + reserve{id}:\ + pool{id}:\ + userReserve{id}:\ timestamp:\ amount:\ borrowRate:\ diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-aave-collaterals.properties b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-aave-collaterals.properties index 82cc879e..70c8cf19 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-aave-collaterals.properties +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-aave-collaterals.properties @@ -5,10 +5,10 @@ url.base= https://api.thegraph.com/subgraphs/name/aave/protocol-v2 url.properties= method,usageAsCollaterals,\ values,\ id:\ - user:\ - reserve:\ - pool:\ - userReserve:\ + user{id}:\ + reserve{id}:\ + pool{id}:\ + userReserve{id}:\ timestamp:\ fromState:\ toState diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-aave-deposits.properties b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-aave-deposits.properties index 88fdec39..73978331 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-aave-deposits.properties +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-aave-deposits.properties @@ -5,11 +5,11 @@ url.base= https://api.thegraph.com/subgraphs/name/aave/protocol-v2 url.properties= method,deposits,\ values,\ id:\ - user:\ - caller:\ - reserve:\ - pool:\ - userReserve:\ + user{id}:\ + caller{id}:\ + reserve{id}:\ + pool{id}:\ + userReserve{id}:\ timestamp:\ amount diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-aave-flash-loans.properties b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-aave-flash-loans.properties index 80c737d2..be909b1b 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-aave-flash-loans.properties +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-aave-flash-loans.properties @@ -5,8 +5,8 @@ url.base= https://api.thegraph.com/subgraphs/name/aave/protocol-v2 url.properties= method,flashLoans,\ values,\ id:\ - pool:\ - reserve:\ + pool{id}:\ + reserve{id}:\ target:\ amount:\ totalFee:\ diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-aave-liquidations.properties b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-aave-liquidations.properties index b0f88bfb..c5eecf7f 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-aave-liquidations.properties +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-aave-liquidations.properties @@ -5,12 +5,12 @@ url.base= https://api.thegraph.com/subgraphs/name/aave/protocol-v2 url.properties= method,liquidationCalls,\ values,\ id:\ - user:\ - pool:\ - collateralReserve:\ - collateralUserReserve:\ - principalReserve:\ - principalUserReserve:\ + user{id}:\ + pool{id}:\ + collateralReserve{id}:\ + collateralUserReserve{id}:\ + principalReserve{id}:\ + principalUserReserve{id}:\ principalAmount:\ collateralAmount:\ timestamp:\ diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-aave-price-history-items.properties b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-aave-price-history-items.properties index 40aa86b5..9a2da1af 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-aave-price-history-items.properties +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-aave-price-history-items.properties @@ -5,7 +5,7 @@ url.base= https://api.thegraph.com/subgraphs/name/aave/protocol-v2 url.properties= method,priceHistoryItems,\ values,\ id:\ - asset:\ + asset{id}:\ price:\ timestamp diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-aave-redeems.properties b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-aave-redeems.properties index 7d744810..90f70c38 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-aave-redeems.properties +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-aave-redeems.properties @@ -5,11 +5,11 @@ url.base= https://api.thegraph.com/subgraphs/name/aave/protocol-v2 url.properties= method,redeemUnderlyings,\ values,\ id:\ - user:\ - to:\ - reserve:\ - pool:\ - userReserve:\ + user{id}:\ + to{id}:\ + reserve{id}:\ + pool{id}:\ + userReserve{id}:\ timestamp:\ amount diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-aave-repays.properties b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-aave-repays.properties index 29a4f6cd..c2d44c4f 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-aave-repays.properties +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-aave-repays.properties @@ -5,11 +5,11 @@ url.base= https://api.thegraph.com/subgraphs/name/aave/protocol-v2 url.properties= method,repays,\ values,\ id:\ - user:\ - repayer:\ - reserve:\ - pool:\ - userReserve:\ + user{id}:\ + repayer{id}:\ + reserve{id}:\ + pool{id}:\ + userReserve{id}:\ timestamp:\ amount diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-aave-user-reserves.properties b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-aave-user-reserves.properties index 07affa4c..faf7a11d 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-aave-user-reserves.properties +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-aave-user-reserves.properties @@ -5,9 +5,9 @@ url.base= https://api.thegraph.com/subgraphs/name/aave/protocol-v2 url.properties= method,userReserves,\ values,\ id:\ - pool:\ - reserve:\ - user:\ + pool{id}:\ + reserve{id}:\ + user{id}:\ usageAsCollateralEnabledOnUser:\ scaledATokenBalance:\ currentATokenBalance:\