From 056c91bcac8b5f4a239f1acffb1ade3756c3f745 Mon Sep 17 00:00:00 2001 From: Conor Flynn Date: Tue, 18 Apr 2023 16:59:51 -0400 Subject: [PATCH] increase engine external stability and introduce defi-llama protocol --- .../requests/ExternalRequestFramework.java | 49 +++++++++------ .../requests/ExternalRequestGraphQL.java | 24 ++++--- .../requests/ExternalRequestREST.java | 63 +++++++++++-------- .../registry/StreamRegistryController.java | 10 ++- .../requests/llama-stablecoins.properties | 17 +++++ 5 files changed, 107 insertions(+), 56 deletions(-) create mode 100644 DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/llama-stablecoins.properties diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/requests/ExternalRequestFramework.java b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/requests/ExternalRequestFramework.java index a6bf2b3e..18d68043 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/requests/ExternalRequestFramework.java +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/requests/ExternalRequestFramework.java @@ -10,6 +10,7 @@ import java.util.TreeMap; import java.util.stream.Stream; +import org.core.logger.Logger; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; @@ -141,13 +142,13 @@ protected final Request getRequest(String url, HashMap propertie // check if empty if(value == null || value.equals("")) { - System.err.println(String.format("Property cannot be empty <%s>", property)); + Logger.warn(String.format("Property cannot be empty <%s>", property)); return null; } // check if required if(value.equals(".") && this.properties.containsKey(property)) { - System.err.println(String.format("Required property <%s> not defined.", property)); + Logger.warn(String.format("Required property <%s> not defined.", property)); return null; } @@ -166,13 +167,13 @@ protected final Request getRequest(String url, HashMap propertie // check if empty if(value == null || value.equals("")) { - System.err.println(String.format("Header cannot be empty <%s>", header)); + Logger.warn(String.format("Header cannot be empty <%s>", header)); return null; } // check if required if(value.equals(".") && this.properties.containsKey(header)) { - System.err.println(String.format("Required header <%s> not defined.", header)); + Logger.warn(String.format("Required header <%s> not defined.", header)); return null; } @@ -185,13 +186,13 @@ protected final Request getRequest(String url, HashMap propertie public final synchronized String request(String[] url_path, HashMap properties, HashMap headers) { // validate that url_path is correctly formatted if(url_path.length % 2 != 0) { - System.err.println("Url path is not formatted properly and must be in pairs."); + Logger.warn("Url path is not formatted properly and must be in pairs."); return "Url path is not formatted properly and must be in pairs."; } // validate that url_path parameter is valid if(this.url_path.length != url_path.length) { - System.err.println("Url path does not match defined url path."); + Logger.warn("Url path does not match defined url path."); return "Url path does not match defined url path."; } @@ -200,7 +201,7 @@ public final synchronized String request(String[] url_path, HashMap 0) { for(int i = 0; i < url_path.length; i+=2) { if(!url_path[i].equals(this.url_path[i])) { - System.err.println(String.format("Url path key <%s> does not match defined key <%s>", url_path[i], this.url_path[i])); + Logger.warn(String.format("Url path key <%s> does not match defined key <%s>", url_path[i], this.url_path[i])); return String.format("Url path key <%s> does not match defined key <%s>", url_path[i], this.url_path[i]); } @@ -229,7 +230,7 @@ public final synchronized String request(String[] url_path, HashMap properties, OkHttpClient client = new OkHttpClient(); Request request = getRequest(url, properties, headers); if(request == null) { - System.err.println("Malformed request, killing process."); + Logger.warn("Malformed request, killing process."); return "Malformed request, killing process."; } @@ -359,15 +360,15 @@ protected String processRequest(String url, HashMap properties, response = client.newCall(request).execute(); body = response.body().string().toString(); if(response.code() != 200) { - System.err.println(String.format("Request Failure code <%d> url <%s>\nbody:\n%s", response.code(), request.url().toString(), body)); - return String.format("Request Failure code <%d> url <%s>\nbody:\n%s", response.code(), request.url().toString(), body); + Logger.warn(String.format("Request Failure code=<%d> url=<%s> body=<%s>", response.code(), request.url().toString(), body)); + return String.format("Request Failure code=<%d> url=<%s> body=<%s>", response.code(), request.url().toString(), body); } } catch (IOException e) { e.printStackTrace(); } if(body == null) { - System.err.println("Response had fatal issue, killing process."); + Logger.warn("Response had fatal issue, killing process."); return "Response had fatal issue, killing process."; } @@ -420,10 +421,20 @@ protected final HashMap parse(Object input, String parent) throw } if (input instanceof JSONArray) { - for (int i = 0; i < ((JSONArray) input).length(); i++) { - JSONObject a = ((JSONArray) input).getJSONObject(i); - out.putAll(parse(a)); + // check to see if objects are json, if not then create list + JSONArray arr = (JSONArray)input; + StringBuilder list = new StringBuilder(); + for (int i = 0; i < arr.length(); i++) { + try { + JSONObject a = arr.getJSONObject(i); + out.putAll(parse(a, parent)); + } catch(Exception e) { + list.append(arr.get(i).toString()).append(","); + } } + + if(!list.isEmpty()) + out.put(parent == null ? "_placeholder" : parent, list.substring(0, list.length()-1)); } return out; diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/requests/ExternalRequestGraphQL.java b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/requests/ExternalRequestGraphQL.java index 76cafb8f..9bda2826 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/requests/ExternalRequestGraphQL.java +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/requests/ExternalRequestGraphQL.java @@ -7,6 +7,7 @@ import java.time.LocalDate; import java.util.HashMap; +import org.core.logger.Logger; import org.json.JSONArray; import org.json.JSONObject; import org.stream.external.handler.ExternalStreamManager; @@ -38,7 +39,7 @@ protected String processRequest(String url, HashMap properties, // check for required tag -l if(!hasTag("-l")) { - System.err.println(String.format("Missing required recursive parameter <-l>")); + Logger.warn(String.format("Missing required recursive parameter <-l>")); return "Missing required recursive parameter <-l>"; } @@ -47,7 +48,7 @@ protected String processRequest(String url, HashMap properties, Integer.parseInt(getTag("-l")); } catch(Exception e) { e.printStackTrace(); - System.err.println(String.format("Value following <-l> must be an integer.")); + Logger.warn(String.format("Value following <-l> must be an integer.")); return "Value following <-l> must be an integer."; } @@ -167,7 +168,7 @@ protected String handle(String json, HashMap properties, HashMap // validate all required parameters are present: // check for required tag -l if(!hasTag("-l")) { - System.err.println(String.format("Missing required recursive parameter <-l>")); + Logger.warn(String.format("Missing required recursive parameter <-l>")); return "Missing required recursive parameter <-l>"; } @@ -177,7 +178,7 @@ protected String handle(String json, HashMap properties, HashMap limit = Integer.parseInt(getTag("-l")); } catch(Exception e) { e.printStackTrace(); - System.err.println(String.format("Value following <-l> must be an integer.")); + Logger.warn(String.format("Value following <-l> must be an integer.")); return "Value following <-l> must be an integer."; } @@ -190,7 +191,7 @@ protected String handle(String json, HashMap properties, HashMap try { data = obj.getJSONArray(path[i]); } catch(Exception e) { - System.err.println("obj path type is not of type . Cannot parse"); + Logger.warn("obj path type is not of type . Cannot parse"); return "obj path type is not of type . Cannot parse"; } } @@ -200,7 +201,7 @@ else if(obj.has(path[i])) { try { obj = obj.getJSONObject(path[i]); } catch(Exception e) { - System.err.println("obj path type step is not of type . Cannot parse."); + Logger.warn("obj path type step is not of type . Cannot parse."); return "obj path type step is not of type . Cannot parse."; } } @@ -212,10 +213,15 @@ else if(obj.has(path[i])) { // validate that data is non-empty if(data == null) { - System.err.println("Data array retrieval had fatal error, killing process."); + Logger.warn("Data array retrieval had fatal error, killing process."); return "Data array retrieval had fatal error, killing process."; } + // if data is empty push empty data point + if(data.length() == 0) { + manager.processRequest(getCollection(), new HashMap()); + } + // define recursive location String recursive_location = getRecursiveLocation()[0]; @@ -228,7 +234,7 @@ else if(obj.has(path[i])) { // if i == 0 then parse recursive parameter if(i == 0 && !point.containsKey(recursive_location)) { - System.err.println("Point does not contain recursive location."); + Logger.warn("Point does not contain recursive location."); return "Point does not contain recursive location."; } @@ -238,7 +244,7 @@ else if(obj.has(path[i])) { // if last data point then retrieve recursive parameter if(i == data.length() - 1) { if(!point.containsKey(recursive_location)) { - System.err.println("Final point does not contain recursive location. Data collection may not be complete."); + Logger.warn("Final point does not contain recursive location. Data collection may not be complete."); return "Final point does not contain recursive location. Data collection may not be complete."; } diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/requests/ExternalRequestREST.java b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/requests/ExternalRequestREST.java index 91ad7edc..9b03c947 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/requests/ExternalRequestREST.java +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/requests/ExternalRequestREST.java @@ -2,6 +2,7 @@ import java.util.HashMap; +import org.core.logger.Logger; import org.json.JSONArray; import org.json.JSONObject; import org.stream.external.handler.ExternalStreamManager; @@ -29,28 +30,31 @@ public String handle(String json, HashMap properties, HashMap")); - return "Missing required recursive parameter <-l>"; - } - // check for required tag -t if(!hasTag("-t")) { - System.err.println(String.format("Missing required recursive parameter <-t>")); + Logger.warn(String.format("Missing required recursive parameter <-t>")); return "Missing required recursive parameter <-t>"; } - // check for -l being an integer + // validate all required parameters are present: + // check for required tag -l int limit = 0; - try { - limit = Integer.parseInt(getTag("-l")); - } catch(Exception e) { - e.printStackTrace(); - System.err.println(String.format("Value following <-l> must be an integer.")); - return "Value following <-l> must be an integer."; + if(!getTag("-t").equals("single")) { + if(!hasTag("-l")) { + Logger.warn(String.format("Missing required recursive parameter <-l>")); + return "Missing required recursive parameter <-l>"; + } else { + try { + limit = Integer.parseInt(getTag("-l")); + } catch(Exception e) { + e.printStackTrace(); + Logger.warn(String.format("Value following <-l> must be an integer.")); + return "Value following <-l> must be an integer."; + } + } } + // check for -l being an integer + // validate that recursion location is valid // should exist within json if url or static @@ -63,7 +67,7 @@ public String handle(String json, HashMap properties, HashMap properties, HashMap for tag <-t> is not valid. Must be: url, incremental, static, or single", getTag("-t"))); + return String.format("Provided value <%s> for tag <-t> is not valid. Must be: url, incremental, static, or single", getTag("-t")); + } + // validate that recursive parameter has been set if(recursive_parameter == null) { - System.err.println("Fatal error. Recursive parameter is null after successful initialization."); + Logger.warn("Fatal error. Recursive parameter is null after successful initialization."); return "Fatal error. Recursive parameter is null after successful initialization."; } @@ -96,7 +109,7 @@ else if(getTag("-t").equals("incremental")) { try { data = obj.getJSONArray(path[i]); } catch(Exception e) { - System.err.println("obj path type is not of type . Cannot parse."); + Logger.warn("obj path type is not of type . Cannot parse."); return "obj path type is not of type . Cannot parse."; } } @@ -106,7 +119,7 @@ else if(obj.has(path[i])) { try { obj = obj.getJSONObject(path[i]); } catch(Exception e) { - System.err.println("obj path type step is not of type . Cannot parse."); + Logger.warn("obj path type step is not of type . Cannot parse."); return "obj path type step is not of type . Cannot parse."; } } @@ -114,7 +127,7 @@ else if(obj.has(path[i])) { // validate that data is non-empty if(data == null) { - System.err.println("Data array retrieval had fatal error, killing process."); + Logger.warn("Data array retrieval had fatal error, killing process."); return "Data array retrieval had fatal error, killing process."; } @@ -130,8 +143,8 @@ else if(obj.has(path[i])) { } // initiate recursive call - // if under limit requested, terminate call - if(data.length() < limit) + // if under limit requested or designated single call, terminate call + if(data.length() < limit || getTag("-t").equals("single")) return null; // extract recursive parameter and apply to next call @@ -157,12 +170,12 @@ else if(obj.has(path[i])) { param = Integer.parseInt(recursive_parameter); } catch(Exception e) { e.printStackTrace(); - System.err.println(String.format("Recursive parameter <%s> is not of type integer.", recursive_parameter)); + Logger.warn(String.format("Recursive parameter <%s> is not of type integer.", recursive_parameter)); return String.format("Recursive parameter <%s> is not of type integer.", recursive_parameter); } if(param == -1) { - System.err.println("Fatal parsing error occured."); + Logger.warn("Fatal parsing error occured."); return "Fatal parsing error occured."; } diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/registry/StreamRegistryController.java b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/registry/StreamRegistryController.java index 4da3e2ab..1ee8759e 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/registry/StreamRegistryController.java +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/registry/StreamRegistryController.java @@ -75,13 +75,17 @@ public Response processRQST(Packet packet) { } data.put("query", String.format("%s", sb)); - Response lsh_response = send("LSH", "RQST", data); + Response lsh_response, esh_response; + lsh_response = send("LSH", "RQST", data); if(lsh_response.code() == 200) return lsh_response; // if data does not exist send request to external stream handler - if(lsh_response.code() == 446) - send("ESH", "RQST", data); + if(lsh_response.code() == 446) { + esh_response = send("ESH", "RQST", data); + if(esh_response.code() != 200) + return esh_response; + } return send("LSH", "RQST", data); } diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/llama-stablecoins.properties b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/llama-stablecoins.properties new file mode 100644 index 00000000..a56a8c37 --- /dev/null +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/llama-stablecoins.properties @@ -0,0 +1,17 @@ +request.name= llama-stablecoins + +url.base= https://stablecoins.llama.fi/stablecoins + +url.properties= includePrices,false + +url.headers= accept,*/* + +data.path= peggedAssets + +recursion.type= rest + +recursion.tags= -t,single + +recursion.location= n/a + +date.valid= false \ No newline at end of file