Skip to content
Jump to bottom

increase engine external stability and introduce defi-llama protocol #15

Merged
merged 1 commit into from
Apr 18, 2023
+107 −56
Merged

increase engine external stability and introduce defi-llama protocol #15

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 30 additions & 19 deletions ...DeFi Data Engine/src/main/java/org/stream/external/requests/ExternalRequestFramework.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import java.util.TreeMap;
import java.util.stream.Stream;

import org.core.logger.Logger;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
Expand Down Expand Up @@ -141,13 +142,13 @@ protected final Request getRequest(String url, HashMap<String, String> propertie

// check if empty
if(value == null || value.equals("")) {
System.err.println(String.format("Property cannot be empty <%s>", property));
Logger.warn(String.format("Property cannot be empty <%s>", property));
return null;
}

// check if required
if(value.equals(".") && this.properties.containsKey(property)) {
System.err.println(String.format("Required property <%s> not defined.", property));
Logger.warn(String.format("Required property <%s> not defined.", property));
return null;
}

Expand All @@ -166,13 +167,13 @@ protected final Request getRequest(String url, HashMap<String, String> propertie

// check if empty
if(value == null || value.equals("")) {
System.err.println(String.format("Header cannot be empty <%s>", header));
Logger.warn(String.format("Header cannot be empty <%s>", header));
return null;
}

// check if required
if(value.equals(".") && this.properties.containsKey(header)) {
System.err.println(String.format("Required header <%s> not defined.", header));
Logger.warn(String.format("Required header <%s> not defined.", header));
return null;
}

Expand All @@ -185,13 +186,13 @@ protected final Request getRequest(String url, HashMap<String, String> propertie
public final synchronized String request(String[] url_path, HashMap<String, String> properties, HashMap<String, String> headers) {
// validate that url_path is correctly formatted
if(url_path.length % 2 != 0) {
System.err.println("Url path is not formatted properly and must be in <key, value> pairs.");
Logger.warn("Url path is not formatted properly and must be in <key, value> pairs.");
return "Url path is not formatted properly and must be in <key, value> pairs.";
}

// validate that url_path parameter is valid
if(this.url_path.length != url_path.length) {
System.err.println("Url path does not match defined url path.");
Logger.warn("Url path does not match defined url path.");
return "Url path does not match defined url path.";
}

Expand All @@ -200,7 +201,7 @@ public final synchronized String request(String[] url_path, HashMap<String, Stri
if(url_path.length > 0) {
for(int i = 0; i < url_path.length; i+=2) {
if(!url_path[i].equals(this.url_path[i])) {
System.err.println(String.format("Url path key <%s> does not match defined key <%s>", url_path[i], this.url_path[i]));
Logger.warn(String.format("Url path key <%s> does not match defined key <%s>", url_path[i], this.url_path[i]));
return String.format("Url path key <%s> does not match defined key <%s>", url_path[i], this.url_path[i]);
}

Expand Down Expand Up @@ -229,7 +230,7 @@ public final synchronized String request(String[] url_path, HashMap<String, Stri

// validate that the request can be dated
if(!is_dated) {
System.err.println("Request cannot be dated.");
Logger.warn("Request cannot be dated.");
return "Request cannot be dated.";
}

Expand All @@ -243,7 +244,7 @@ public final synchronized String request(String[] url_path, HashMap<String, Stri
}

if(start == null || end == null) {
System.err.println("Fatal error parsing dates.");
Logger.warn("Fatal error parsing dates.");
return "Fatal error parsing dates.";
}

Expand All @@ -253,7 +254,7 @@ public final synchronized String request(String[] url_path, HashMap<String, Stri
try {
dates = start.datesUntil(end);
} catch(Exception e) {
System.err.println("End date must be after start date.");
Logger.warn("End date must be after start date.");
return "End date must be after start date.";
}

Expand All @@ -272,7 +273,7 @@ public final synchronized String request(String[] url_path, HashMap<String, Stri
headers.put(date_start_var, date.format(date_format));
break;
default:
System.err.println("Invalid date.location parameter value.");
Logger.warn("Invalid date.location parameter value.");
System.exit(1);
}

Expand All @@ -287,7 +288,7 @@ public final synchronized String request(String[] url_path, HashMap<String, Stri
headers.put(date_end_var, tmr.format(date_format));
break;
default:
System.err.println("Invalid date.location parameter value.");
Logger.warn("Invalid date.location parameter value.");
System.exit(1);
}
}
Expand Down Expand Up @@ -349,7 +350,7 @@ protected String processRequest(String url, HashMap<String, String> properties,
OkHttpClient client = new OkHttpClient();
Request request = getRequest(url, properties, headers);
if(request == null) {
System.err.println("Malformed request, killing process.");
Logger.warn("Malformed request, killing process.");
return "Malformed request, killing process.";
}

Expand All @@ -359,15 +360,15 @@ protected String processRequest(String url, HashMap<String, String> properties,
response = client.newCall(request).execute();
body = response.body().string().toString();
if(response.code() != 200) {
System.err.println(String.format("Request Failure code <%d> url <%s>\nbody:\n%s", response.code(), request.url().toString(), body));
return String.format("Request Failure code <%d> url <%s>\nbody:\n%s", response.code(), request.url().toString(), body);
Logger.warn(String.format("Request Failure code=<%d> url=<%s> body=<%s>", response.code(), request.url().toString(), body));
return String.format("Request Failure code=<%d> url=<%s> body=<%s>", response.code(), request.url().toString(), body);
}
} catch (IOException e) {
e.printStackTrace();
}

if(body == null) {
System.err.println("Response had fatal issue, killing process.");
Logger.warn("Response had fatal issue, killing process.");
return "Response had fatal issue, killing process.";
}

Expand Down Expand Up @@ -420,10 +421,20 @@ protected final HashMap<String, String> parse(Object input, String parent) throw
}

if (input instanceof JSONArray) {
for (int i = 0; i < ((JSONArray) input).length(); i++) {
JSONObject a = ((JSONArray) input).getJSONObject(i);
out.putAll(parse(a));
// check to see if objects are json, if not then create list
JSONArray arr = (JSONArray)input;
StringBuilder list = new StringBuilder();
for (int i = 0; i < arr.length(); i++) {
try {
JSONObject a = arr.getJSONObject(i);
out.putAll(parse(a, parent));
} catch(Exception e) {
list.append(arr.get(i).toString()).append(",");
}
}

if(!list.isEmpty())
out.put(parent == null ? "_placeholder" : parent, list.substring(0, list.length()-1));
}

return out;
Expand Down
24 changes: 15 additions & 9 deletions ...e/DeFi Data Engine/src/main/java/org/stream/external/requests/ExternalRequestGraphQL.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import java.time.LocalDate;
import java.util.HashMap;

import org.core.logger.Logger;
import org.json.JSONArray;
import org.json.JSONObject;
import org.stream.external.handler.ExternalStreamManager;
Expand Down Expand Up @@ -38,7 +39,7 @@ protected String processRequest(String url, HashMap<String, String> properties,

// check for required tag -l
if(!hasTag("-l")) {
System.err.println(String.format("Missing required recursive parameter <-l>"));
Logger.warn(String.format("Missing required recursive parameter <-l>"));
return "Missing required recursive parameter <-l>";
}

Expand All @@ -47,7 +48,7 @@ protected String processRequest(String url, HashMap<String, String> properties,
Integer.parseInt(getTag("-l"));
} catch(Exception e) {
e.printStackTrace();
System.err.println(String.format("Value following <-l> must be an integer."));
Logger.warn(String.format("Value following <-l> must be an integer."));
return "Value following <-l> must be an integer.";
}

Expand Down Expand Up @@ -167,7 +168,7 @@ protected String handle(String json, HashMap<String, String> properties, HashMap
// validate all required parameters are present:
// check for required tag -l
if(!hasTag("-l")) {
System.err.println(String.format("Missing required recursive parameter <-l>"));
Logger.warn(String.format("Missing required recursive parameter <-l>"));
return "Missing required recursive parameter <-l>";
}

Expand All @@ -177,7 +178,7 @@ protected String handle(String json, HashMap<String, String> properties, HashMap
limit = Integer.parseInt(getTag("-l"));
} catch(Exception e) {
e.printStackTrace();
System.err.println(String.format("Value following <-l> must be an integer."));
Logger.warn(String.format("Value following <-l> must be an integer."));
return "Value following <-l> must be an integer.";
}

Expand All @@ -190,7 +191,7 @@ protected String handle(String json, HashMap<String, String> properties, HashMap
try {
data = obj.getJSONArray(path[i]);
} catch(Exception e) {
System.err.println("obj path type is not of type <JSONArray>. Cannot parse");
Logger.warn("obj path type is not of type <JSONArray>. Cannot parse");
return "obj path type is not of type <JSONArray>. Cannot parse";
}
}
Expand All @@ -200,7 +201,7 @@ else if(obj.has(path[i])) {
try {
obj = obj.getJSONObject(path[i]);
} catch(Exception e) {
System.err.println("obj path type step is not of type <JSONObject>. Cannot parse.");
Logger.warn("obj path type step is not of type <JSONObject>. Cannot parse.");
return "obj path type step is not of type <JSONObject>. Cannot parse.";
}
}
Expand All @@ -212,10 +213,15 @@ else if(obj.has(path[i])) {

// validate that data is non-empty
if(data == null) {
System.err.println("Data array retrieval had fatal error, killing process.");
Logger.warn("Data array retrieval had fatal error, killing process.");
return "Data array retrieval had fatal error, killing process.";
}

// if data is empty push empty data point
if(data.length() == 0) {
manager.processRequest(getCollection(), new HashMap<String, String>());
}

// define recursive location
String recursive_location = getRecursiveLocation()[0];

Expand All @@ -228,7 +234,7 @@ else if(obj.has(path[i])) {

// if i == 0 then parse recursive parameter
if(i == 0 && !point.containsKey(recursive_location)) {
System.err.println("Point does not contain recursive location.");
Logger.warn("Point does not contain recursive location.");
return "Point does not contain recursive location.";
}

Expand All @@ -238,7 +244,7 @@ else if(obj.has(path[i])) {
// if last data point then retrieve recursive parameter
if(i == data.length() - 1) {
if(!point.containsKey(recursive_location)) {
System.err.println("Final point does not contain recursive location. Data collection may not be complete.");
Logger.warn("Final point does not contain recursive location. Data collection may not be complete.");
return "Final point does not contain recursive location. Data collection may not be complete.";
}

Expand Down
Loading