Skip to content

Commit

Permalink
Merge pull request #15 from DataINCITE/flynnc3-temp
Browse files Browse the repository at this point in the history 
increase engine external stability and introduce defi-llama protocol
  • Loading branch information
@flynnc3
flynnc3 authored Apr 18, 2023
2 parents 2cab88b + 55f2457 commit 9d8c9b5
Show file tree
Hide file tree
Showing 5 changed files with 107 additions and 56 deletions.
49 changes: 30 additions & 19 deletions ...DeFi Data Engine/src/main/java/org/stream/external/requests/ExternalRequestFramework.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import java.util.TreeMap;
import java.util.stream.Stream;

import org.core.logger.Logger;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
Expand Down Expand Up @@ -141,13 +142,13 @@ protected final Request getRequest(String url, HashMap<String, String> propertie

// check if empty
if(value == null || value.equals("")) {
System.err.println(String.format("Property cannot be empty <%s>", property));
Logger.warn(String.format("Property cannot be empty <%s>", property));
return null;
}

// check if required
if(value.equals(".") && this.properties.containsKey(property)) {
System.err.println(String.format("Required property <%s> not defined.", property));
Logger.warn(String.format("Required property <%s> not defined.", property));
return null;
}

Expand All @@ -166,13 +167,13 @@ protected final Request getRequest(String url, HashMap<String, String> propertie

// check if empty
if(value == null || value.equals("")) {
System.err.println(String.format("Header cannot be empty <%s>", header));
Logger.warn(String.format("Header cannot be empty <%s>", header));
return null;
}

// check if required
if(value.equals(".") && this.properties.containsKey(header)) {
System.err.println(String.format("Required header <%s> not defined.", header));
Logger.warn(String.format("Required header <%s> not defined.", header));
return null;
}

Expand All @@ -185,13 +186,13 @@ protected final Request getRequest(String url, HashMap<String, String> propertie
public final synchronized String request(String[] url_path, HashMap<String, String> properties, HashMap<String, String> headers) {
// validate that url_path is correctly formatted
if(url_path.length % 2 != 0) {
System.err.println("Url path is not formatted properly and must be in <key, value> pairs.");
Logger.warn("Url path is not formatted properly and must be in <key, value> pairs.");
return "Url path is not formatted properly and must be in <key, value> pairs.";
}

// validate that url_path parameter is valid
if(this.url_path.length != url_path.length) {
System.err.println("Url path does not match defined url path.");
Logger.warn("Url path does not match defined url path.");
return "Url path does not match defined url path.";
}

Expand All @@ -200,7 +201,7 @@ public final synchronized String request(String[] url_path, HashMap<String, Stri
if(url_path.length > 0) {
for(int i = 0; i < url_path.length; i+=2) {
if(!url_path[i].equals(this.url_path[i])) {
System.err.println(String.format("Url path key <%s> does not match defined key <%s>", url_path[i], this.url_path[i]));
Logger.warn(String.format("Url path key <%s> does not match defined key <%s>", url_path[i], this.url_path[i]));
return String.format("Url path key <%s> does not match defined key <%s>", url_path[i], this.url_path[i]);
}

Expand Down Expand Up @@ -229,7 +230,7 @@ public final synchronized String request(String[] url_path, HashMap<String, Stri

// validate that the request can be dated
if(!is_dated) {
System.err.println("Request cannot be dated.");
Logger.warn("Request cannot be dated.");
return "Request cannot be dated.";
}

Expand All @@ -243,7 +244,7 @@ public final synchronized String request(String[] url_path, HashMap<String, Stri
}

if(start == null || end == null) {
System.err.println("Fatal error parsing dates.");
Logger.warn("Fatal error parsing dates.");
return "Fatal error parsing dates.";
}

Expand All @@ -253,7 +254,7 @@ public final synchronized String request(String[] url_path, HashMap<String, Stri
try {
dates = start.datesUntil(end);
} catch(Exception e) {
System.err.println("End date must be after start date.");
Logger.warn("End date must be after start date.");
return "End date must be after start date.";
}

Expand All @@ -272,7 +273,7 @@ public final synchronized String request(String[] url_path, HashMap<String, Stri
headers.put(date_start_var, date.format(date_format));
break;
default:
System.err.println("Invalid date.location parameter value.");
Logger.warn("Invalid date.location parameter value.");
System.exit(1);
}

Expand All @@ -287,7 +288,7 @@ public final synchronized String request(String[] url_path, HashMap<String, Stri
headers.put(date_end_var, tmr.format(date_format));
break;
default:
System.err.println("Invalid date.location parameter value.");
Logger.warn("Invalid date.location parameter value.");
System.exit(1);
}
}
Expand Down Expand Up @@ -349,7 +350,7 @@ protected String processRequest(String url, HashMap<String, String> properties,
OkHttpClient client = new OkHttpClient();
Request request = getRequest(url, properties, headers);
if(request == null) {
System.err.println("Malformed request, killing process.");
Logger.warn("Malformed request, killing process.");
return "Malformed request, killing process.";
}

Expand All @@ -359,15 +360,15 @@ protected String processRequest(String url, HashMap<String, String> properties,
response = client.newCall(request).execute();
body = response.body().string().toString();
if(response.code() != 200) {
System.err.println(String.format("Request Failure code <%d> url <%s>\nbody:\n%s", response.code(), request.url().toString(), body));
return String.format("Request Failure code <%d> url <%s>\nbody:\n%s", response.code(), request.url().toString(), body);
Logger.warn(String.format("Request Failure code=<%d> url=<%s> body=<%s>", response.code(), request.url().toString(), body));
return String.format("Request Failure code=<%d> url=<%s> body=<%s>", response.code(), request.url().toString(), body);
}
} catch (IOException e) {
e.printStackTrace();
}

if(body == null) {
System.err.println("Response had fatal issue, killing process.");
Logger.warn("Response had fatal issue, killing process.");
return "Response had fatal issue, killing process.";
}

Expand Down Expand Up @@ -420,10 +421,20 @@ protected final HashMap<String, String> parse(Object input, String parent) throw
}

if (input instanceof JSONArray) {
for (int i = 0; i < ((JSONArray) input).length(); i++) {
JSONObject a = ((JSONArray) input).getJSONObject(i);
out.putAll(parse(a));
// check to see if objects are json, if not then create list
JSONArray arr = (JSONArray)input;
StringBuilder list = new StringBuilder();
for (int i = 0; i < arr.length(); i++) {
try {
JSONObject a = arr.getJSONObject(i);
out.putAll(parse(a, parent));
} catch(Exception e) {
list.append(arr.get(i).toString()).append(",");
}
}

if(!list.isEmpty())
out.put(parent == null ? "_placeholder" : parent, list.substring(0, list.length()-1));
}

return out;
Expand Down
24 changes: 15 additions & 9 deletions ...e/DeFi Data Engine/src/main/java/org/stream/external/requests/ExternalRequestGraphQL.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import java.time.LocalDate;
import java.util.HashMap;

import org.core.logger.Logger;
import org.json.JSONArray;
import org.json.JSONObject;
import org.stream.external.handler.ExternalStreamManager;
Expand Down Expand Up @@ -38,7 +39,7 @@ protected String processRequest(String url, HashMap<String, String> properties,

// check for required tag -l
if(!hasTag("-l")) {
System.err.println(String.format("Missing required recursive parameter <-l>"));
Logger.warn(String.format("Missing required recursive parameter <-l>"));
return "Missing required recursive parameter <-l>";
}

Expand All @@ -47,7 +48,7 @@ protected String processRequest(String url, HashMap<String, String> properties,
Integer.parseInt(getTag("-l"));
} catch(Exception e) {
e.printStackTrace();
System.err.println(String.format("Value following <-l> must be an integer."));
Logger.warn(String.format("Value following <-l> must be an integer."));
return "Value following <-l> must be an integer.";
}

Expand Down Expand Up @@ -167,7 +168,7 @@ protected String handle(String json, HashMap<String, String> properties, HashMap
// validate all required parameters are present:
// check for required tag -l
if(!hasTag("-l")) {
System.err.println(String.format("Missing required recursive parameter <-l>"));
Logger.warn(String.format("Missing required recursive parameter <-l>"));
return "Missing required recursive parameter <-l>";
}

Expand All @@ -177,7 +178,7 @@ protected String handle(String json, HashMap<String, String> properties, HashMap
limit = Integer.parseInt(getTag("-l"));
} catch(Exception e) {
e.printStackTrace();
System.err.println(String.format("Value following <-l> must be an integer."));
Logger.warn(String.format("Value following <-l> must be an integer."));
return "Value following <-l> must be an integer.";
}

Expand All @@ -190,7 +191,7 @@ protected String handle(String json, HashMap<String, String> properties, HashMap
try {
data = obj.getJSONArray(path[i]);
} catch(Exception e) {
System.err.println("obj path type is not of type <JSONArray>. Cannot parse");
Logger.warn("obj path type is not of type <JSONArray>. Cannot parse");
return "obj path type is not of type <JSONArray>. Cannot parse";
}
}
Expand All @@ -200,7 +201,7 @@ else if(obj.has(path[i])) {
try {
obj = obj.getJSONObject(path[i]);
} catch(Exception e) {
System.err.println("obj path type step is not of type <JSONObject>. Cannot parse.");
Logger.warn("obj path type step is not of type <JSONObject>. Cannot parse.");
return "obj path type step is not of type <JSONObject>. Cannot parse.";
}
}
Expand All @@ -212,10 +213,15 @@ else if(obj.has(path[i])) {

// validate that data is non-empty
if(data == null) {
System.err.println("Data array retrieval had fatal error, killing process.");
Logger.warn("Data array retrieval had fatal error, killing process.");
return "Data array retrieval had fatal error, killing process.";
}

// if data is empty push empty data point
if(data.length() == 0) {
manager.processRequest(getCollection(), new HashMap<String, String>());
}

// define recursive location
String recursive_location = getRecursiveLocation()[0];

Expand All @@ -228,7 +234,7 @@ else if(obj.has(path[i])) {

// if i == 0 then parse recursive parameter
if(i == 0 && !point.containsKey(recursive_location)) {
System.err.println("Point does not contain recursive location.");
Logger.warn("Point does not contain recursive location.");
return "Point does not contain recursive location.";
}

Expand All @@ -238,7 +244,7 @@ else if(obj.has(path[i])) {
// if last data point then retrieve recursive parameter
if(i == data.length() - 1) {
if(!point.containsKey(recursive_location)) {
System.err.println("Final point does not contain recursive location. Data collection may not be complete.");
Logger.warn("Final point does not contain recursive location. Data collection may not be complete.");
return "Final point does not contain recursive location. Data collection may not be complete.";
}

Expand Down
Loading

0 comments on commit 9d8c9b5

Please sign in to comment.