Merge pull request #14 from DataINCITE/flynnc3-temp

modify properties files and make minor fixes to SRC RQST external han…
DataINCITE · Apr 17, 2023 · 2cab88b · 2cab88b
2 parents 4b4c1b0 + 7469618
commit 2cab88b
Show file tree

Hide file tree

Showing 12 changed files with 58 additions and 44 deletions.
diff --git a/...DeFi Data Engine/src/main/java/org/stream/external/requests/ExternalRequestFramework.java b/...DeFi Data Engine/src/main/java/org/stream/external/requests/ExternalRequestFramework.java
@@ -387,11 +387,14 @@ protected String processRequest(String url, HashMap<String, String> properties,
 	public abstract String getType();
 
 	protected final HashMap<String, String> parse(Object input) throws JSONException {
-
+		return parse(input, null);
+	}
+
+	protected final HashMap<String, String> parse(Object input, String parent) throws JSONException {
+
 		HashMap<String, String> out = new HashMap<String, String>();
 
 	    if (input instanceof JSONObject) {
-
 	        Iterator<?> keys = ((JSONObject) input).keys();
 
 	        while (keys.hasNext()) {
@@ -400,12 +403,18 @@ protected final HashMap<String, String> parse(Object input) throws JSONException
 
 	            if (!(((JSONObject) input).get(key) instanceof JSONArray)) {
 	                if (((JSONObject) input).get(key) instanceof JSONObject) {
-	                	out.putAll(parse(((JSONObject) input).get(key)));
+	                	HashMap<String, String> parsed = parse(((JSONObject) input).get(key), key);
+	                	for(String parsedKey : parsed.keySet()) {
+	                		out.put(parent == null ? parsedKey : parent, parsed.get(parsedKey));
+	                	}
 	                } else {
-	                    out.put(key, ((JSONObject) input).get(key).toString());
+                		out.put(parent == null ? key : parent, ((JSONObject) input).get(key).toString());
 	                }
 	            } else {
-	            	out.putAll(parse(new JSONArray(((JSONObject) input).get(key).toString())));
+	            	HashMap<String, String> parsed = parse(new JSONArray(((JSONObject) input).get(key).toString()), key);
+	            	for(String parsedKey : parsed.keySet()) {
+                		out.put(parent == null ? parsedKey : parent, parsed.get(parsedKey));
+	            	}
 	            }
 	        }
 	    }

diff --git a/...e/DeFi Data Engine/src/main/java/org/stream/external/requests/ExternalRequestGraphQL.java b/...e/DeFi Data Engine/src/main/java/org/stream/external/requests/ExternalRequestGraphQL.java
@@ -244,10 +244,12 @@ else if(obj.has(path[i])) {
 
 				// update gt property with new value
 				// check for if gt uses a string parsing parameter. if so then add with parameter
-				if(properties.get("gt").charAt(0) == '\"')
+				try {
+					long gt = Long.parseLong(point.get(recursive_location));
+					properties.put("gt", "" + gt);
+				} catch(Exception e) {
 					properties.put("gt", String.format("\"%s\"", point.get(recursive_location)));
-				else
-					properties.put("gt", point.get(recursive_location));
+				}
 			}
 		}
 

diff --git a/...a-Engine/DeFi Data Engine/src/main/java/org/stream/registry/StreamRegistryController.java b/...a-Engine/DeFi Data Engine/src/main/java/org/stream/registry/StreamRegistryController.java
@@ -76,6 +76,9 @@ public Response processRQST(Packet packet) {
 			data.put("query", String.format("%s", sb));
 
 			Response lsh_response = send("LSH", "RQST", data);
+			if(lsh_response.code() == 200)
+				return lsh_response;
+
 			// if data does not exist send request to external stream handler
 			if(lsh_response.code() == 446)
 				send("ESH", "RQST", data);

diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-aave-borrows.properties b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-aave-borrows.properties
@@ -5,11 +5,11 @@ url.base=		https://api.thegraph.com/subgraphs/name/aave/protocol-v2
 url.properties= method,borrows,\
 				values,\
 				id:\
-              	user:\
-              	caller:\
-              	reserve:\
-              	pool:\
-              	userReserve:\
+              	user{id}:\
+              	caller{id}:\
+              	reserve{id}:\
+              	pool{id}:\
+              	userReserve{id}:\
               	timestamp:\
               	amount:\
               	borrowRate:\

diff --git a/...ata-Engine/DeFi Data Engine/src/main/resources/requests/graph-aave-collaterals.properties b/...ata-Engine/DeFi Data Engine/src/main/resources/requests/graph-aave-collaterals.properties
@@ -5,10 +5,10 @@ url.base=		https://api.thegraph.com/subgraphs/name/aave/protocol-v2
 url.properties= method,usageAsCollaterals,\
 				values,\
 				id:\
-              	user:\
-				reserve:\
-				pool:\
-				userReserve:\
+              	user{id}:\
+				reserve{id}:\
+				pool{id}:\
+				userReserve{id}:\
 				timestamp:\
 				fromState:\
 				toState

diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-aave-deposits.properties b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-aave-deposits.properties
@@ -5,11 +5,11 @@ url.base=		https://api.thegraph.com/subgraphs/name/aave/protocol-v2
 url.properties= method,deposits,\
 				values,\
 				id:\
-              	user:\
-				caller:\
-				reserve:\
-				pool:\
-				userReserve:\
+              	user{id}:\
+				caller{id}:\
+				reserve{id}:\
+				pool{id}:\
+				userReserve{id}:\
 				timestamp:\
 				amount
 

diff --git a/...ata-Engine/DeFi Data Engine/src/main/resources/requests/graph-aave-flash-loans.properties b/...ata-Engine/DeFi Data Engine/src/main/resources/requests/graph-aave-flash-loans.properties
@@ -5,8 +5,8 @@ url.base=		https://api.thegraph.com/subgraphs/name/aave/protocol-v2
 url.properties= method,flashLoans,\
 				values,\
 				id:\
-              	pool:\
-              	reserve:\
+              	pool{id}:\
+              	reserve{id}:\
               	target:\
               	amount:\
               	totalFee:\

diff --git a/...ta-Engine/DeFi Data Engine/src/main/resources/requests/graph-aave-liquidations.properties b/...ta-Engine/DeFi Data Engine/src/main/resources/requests/graph-aave-liquidations.properties
@@ -5,12 +5,12 @@ url.base=		https://api.thegraph.com/subgraphs/name/aave/protocol-v2
 url.properties= method,liquidationCalls,\
 				values,\
 				id:\
-				user:\
-              	pool:\
-              	collateralReserve:\
-              	collateralUserReserve:\
-              	principalReserve:\
-              	principalUserReserve:\
+				user{id}:\
+              	pool{id}:\
+              	collateralReserve{id}:\
+              	collateralUserReserve{id}:\
+              	principalReserve{id}:\
+              	principalUserReserve{id}:\
               	principalAmount:\
               	collateralAmount:\
               	timestamp:\

diff --git a/...ne/DeFi Data Engine/src/main/resources/requests/graph-aave-price-history-items.properties b/...ne/DeFi Data Engine/src/main/resources/requests/graph-aave-price-history-items.properties
@@ -5,7 +5,7 @@ url.base=		https://api.thegraph.com/subgraphs/name/aave/protocol-v2
 url.properties= method,priceHistoryItems,\
 				values,\
 				id:\
-				asset:\
+				asset{id}:\
 				price:\
 				timestamp
 

diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-aave-redeems.properties b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-aave-redeems.properties
@@ -5,11 +5,11 @@ url.base=		https://api.thegraph.com/subgraphs/name/aave/protocol-v2
 url.properties= method,redeemUnderlyings,\
 				values,\
 				id:\
-				user:\
-				to:\
-				reserve:\
-				pool:\
-				userReserve:\
+				user{id}:\
+				to{id}:\
+				reserve{id}:\
+				pool{id}:\
+				userReserve{id}:\
 				timestamp:\
 				amount
 

diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-aave-repays.properties b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/graph-aave-repays.properties
@@ -5,11 +5,11 @@ url.base=		https://api.thegraph.com/subgraphs/name/aave/protocol-v2
 url.properties= method,repays,\
 				values,\
 				id:\
-				user:\
-				repayer:\
-				reserve:\
-				pool:\
-				userReserve:\
+				user{id}:\
+				repayer{id}:\
+				reserve{id}:\
+				pool{id}:\
+				userReserve{id}:\
 				timestamp:\
 				amount
 

diff --git a/...a-Engine/DeFi Data Engine/src/main/resources/requests/graph-aave-user-reserves.properties b/...a-Engine/DeFi Data Engine/src/main/resources/requests/graph-aave-user-reserves.properties
@@ -5,9 +5,9 @@ url.base=		https://api.thegraph.com/subgraphs/name/aave/protocol-v2
 url.properties= method,userReserves,\
 				values,\
 				id:\
-				pool:\
-				reserve:\
-				user:\
+				pool{id}:\
+				reserve{id}:\
+				user{id}:\
 				usageAsCollateralEnabledOnUser:\
 				scaledATokenBalance:\
 				currentATokenBalance:\