diff --git a/DeFi-Data-Engine/Api-Handler/src/main/java/org/stream/external/requester/RequestFramework.java b/DeFi-Data-Engine/Api-Handler/src/main/java/org/stream/external/requester/RequestFramework.java index a27dddbd..5702b08c 100644 --- a/DeFi-Data-Engine/Api-Handler/src/main/java/org/stream/external/requester/RequestFramework.java +++ b/DeFi-Data-Engine/Api-Handler/src/main/java/org/stream/external/requester/RequestFramework.java @@ -1,23 +1,34 @@ package org.stream.external.requester; +import java.io.IOException; import java.util.HashMap; -import java.util.List; -import java.util.stream.Collectors; +import java.util.Iterator; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +import okhttp3.OkHttpClient; import okhttp3.Request; import okhttp3.Request.Builder; +import okhttp3.Response; public abstract class RequestFramework { + private final String name; private final String url; private final HashMap properties; private final HashMap headers; - private final String[] tags; + private final HashMap tags; + private final String[] path; - public RequestFramework(String url, HashMap properties, HashMap headers, String... tags) { + public RequestFramework(String name, String url, HashMap properties, HashMap headers, + HashMap tags, String[] path) { + this.name = name; this.url = url; this.properties = properties; this.headers = headers; this.tags = tags; + this.path = path; } protected final Request getRequest(HashMap properties, HashMap headers) { @@ -89,9 +100,25 @@ protected final Request getRequest(HashMap properties, HashMap getTags() { return this.tags; } + + public final String getName() { + return name; + } + + public final String[] getPath() { + return path; + } @SuppressWarnings("unchecked") public final void request(HashMap properties, HashMap headers) { @@ -102,5 +129,76 @@ public final void request(HashMap properties, HashMap properties, HashMap headers); + protected void process(HashMap properties, HashMap headers) { + OkHttpClient client = new OkHttpClient(); + Request request = getRequest(properties, headers); + if(request == null) { + System.err.println("Malformed request, killing process."); + return; + } + + Response response = null; + try { + response = client.newCall(request).execute(); + if(response.code() != 200) { + System.err.println(String.format("Request Failure code <%d> url <%s>\nmsg <%s>", response.code(), request.url().toString(), response.message())); + System.exit(1); + } + } catch (IOException e) { + e.printStackTrace(); + } + + if(response == null) { + System.err.println("Response had fatal issue, killing process."); + return; + } + + // send to specific request handler + process(response.body().toString(), properties, headers); + } + + protected abstract void process(String json, HashMap properties, HashMap headers); + + public abstract String getType(); + + + protected final StringBuilder parse(Object input) throws JSONException { + StringBuilder out = new StringBuilder(); + out = out.append(parseJsonObject(input)); + return out.deleteCharAt(out.length() - 1); + } + + private final StringBuilder parseJsonObject(Object input) throws JSONException { + + StringBuilder out = new StringBuilder(); + + if (input instanceof JSONObject) { + + Iterator keys = ((JSONObject) input).keys(); + + while (keys.hasNext()) { + + String key = (String) keys.next(); + + if (!(((JSONObject) input).get(key) instanceof JSONArray)) { + if (((JSONObject) input).get(key) instanceof JSONObject) { + out = out.append(parseJsonObject(((JSONObject) input).get(key))); + } else { + out = out.append(key + "=" + ((JSONObject) input).get(key) + ","); + } + } else { + out = out.append(parseJsonObject(new JSONArray(((JSONObject) input).get(key).toString()))); + } + } + } + + if (input instanceof JSONArray) { + for (int i = 0; i < ((JSONArray) input).length(); i++) { + JSONObject a = ((JSONArray) input).getJSONObject(i); + out = out.append(parseJsonObject(a)); + } + } + + return out; + } } \ No newline at end of file diff --git a/DeFi-Data-Engine/Api-Handler/src/main/java/org/stream/external/requester/RequestParameterized.java b/DeFi-Data-Engine/Api-Handler/src/main/java/org/stream/external/requester/RequestParameterized.java new file mode 100644 index 00000000..759ba651 --- /dev/null +++ b/DeFi-Data-Engine/Api-Handler/src/main/java/org/stream/external/requester/RequestParameterized.java @@ -0,0 +1,108 @@ +package org.stream.external.requester; + +import java.util.HashMap; + +import org.json.JSONArray; +import org.json.JSONObject; + +public class RequestParameterized extends RequestFramework { + + public RequestParameterized(String name, String url, HashMap properties, HashMap headers, + HashMap tags, String[] path) { + super(name, url, properties, headers, tags, path); + } + + @Override + public String getType() { + return "parameterized"; + } + + public void process(String json, HashMap properties, HashMap headers) { + // parse json formatting + JSONObject obj = new JSONObject(json); + + // validate all required parameters are present: + // check for recursive parameter + if(hasTag("-rp") && !properties.containsKey(getTag("-rp"))) { + System.err.println(String.format("Missing required recursive parameter <%s>", getTag("-rp"))); + return; + } + + // check for required tag -l + if(!hasTag("-l")) { + System.err.println(String.format("Missing required recursive parameter <-l>")); + return; + } + + // check for -l being an integer + int limit = 0; + try { + limit = Integer.parseInt(getTag("-l")); + } catch(Exception e) { + e.printStackTrace(); + System.err.println(String.format("Value following <-l> must be an integer.")); + return; + } + + // validate that the base has the proper obj path + String[] path = getPath(); + JSONArray data = null; + for(int i = 0; i < path.length; i++) { + if(i == path.length - 1) { + if(obj.has(path[i])) { + try { + data = obj.getJSONArray(path[i]); + } catch(Exception e) { + System.err.println("obj path type is not of type . Cannot parse."); + return; + } + } + } + + else if(obj.has(path[i])) { + try { + obj = obj.getJSONObject(path[i]); + } catch(Exception e) { + System.err.println("obj path type step is not of type . Cannot parse."); + return; + } + } + } + + if(data == null) { + System.err.println("Data array retrieval had fatal error, killing process."); + return; + } + + // extract and print data + for(int i = 0; i < data.length(); i++) { + StringBuilder point = parse(data.getJSONObject(i)); + System.out.println(point); + } + + // initiate recursive call + // if under limit requested, terminate call + if(data.length() < limit) + return; + + // extract recursive parameter + int param = -1; + try { + param = Integer.parseInt(properties.get(getTag("-rp"))); + } catch(Exception e) { + e.printStackTrace(); + System.err.println(String.format("Recursive parameter <%s> is not of type integer.", getTag("-rp"))); + return; + } + + if(param == -1) { + System.err.println("Fatal parsing error occured."); + return; + } + + // increment param, replace, and execute + param += 1; + properties.put(getTag("-rp"), "" + param); + process(properties, headers); + } +} \ No newline at end of file diff --git a/DeFi-Data-Engine/Api-Handler/src/main/java/org/stream/external/requester/RequestRecursive.java b/DeFi-Data-Engine/Api-Handler/src/main/java/org/stream/external/requester/RequestRecursive.java deleted file mode 100644 index d28af63a..00000000 --- a/DeFi-Data-Engine/Api-Handler/src/main/java/org/stream/external/requester/RequestRecursive.java +++ /dev/null @@ -1,17 +0,0 @@ -package org.stream.external.requester; - -import java.util.HashMap; - -import okhttp3.Request; - -public class RequestRecursive extends RequestFramework { - - public RequestRecursive(String url, HashMap properties, HashMap headers, - String[] tags) { - super(url, properties, headers, tags); - } - - public void process(HashMap properties, HashMap headers) { - - } -} diff --git a/DeFi-Data-Engine/Api-Handler/src/main/resources/requests/template.properties b/DeFi-Data-Engine/Api-Handler/src/main/resources/requests/template.properties index 5dd16153..65544255 100644 --- a/DeFi-Data-Engine/Api-Handler/src/main/resources/requests/template.properties +++ b/DeFi-Data-Engine/Api-Handler/src/main/resources/requests/template.properties @@ -34,6 +34,8 @@ url.headers= header1,value1,\ # which have specific properties and handlers. Please review documentation to get a full # list of these tags. To default with no recursive call, set this property to . # This property we will set to for a clearer example. +# - parameterized: one of the parameters in the url call +# - embedded: parameter included within the response url.recursion.type= parameterized # This property sets all tags pertaining to the type of recursive call. Please refer to @@ -43,16 +45,15 @@ url.recursion.type= parameterized # example we will set the tags for which are as follows: # -rp: recursive parameter # -l: limit on items from request -# -t: type of parameter (url or incremental) +# -t: type of parameter (url or incremental) [embedded only required] url.recursion.tags= -rp,next,\ - -l,1000,\ - -t,url + -l,1000 # This property sets the location of the data points to be retrieved from the call. This # should be a JSONArray which the handler can iterate through. To access these data points # directly, the direct path must be specified (consisting of all JSONObject values). In # the example below, we point to the path located at response->data. Note that for storing # all non-array values and just recording all base values returned by the call, please set -# the value of this variable to '-b'. (i.e. url.data.path=-b) +# the value of this variable to '.') url.data.path= response,\ data \ No newline at end of file diff --git a/DeFi-Data-Engine/Api-Handler/target/classes/org/stream/external/requester/RequestFramework.class b/DeFi-Data-Engine/Api-Handler/target/classes/org/stream/external/requester/RequestFramework.class index 3ed11161..18aafaea 100644 Binary files a/DeFi-Data-Engine/Api-Handler/target/classes/org/stream/external/requester/RequestFramework.class and b/DeFi-Data-Engine/Api-Handler/target/classes/org/stream/external/requester/RequestFramework.class differ diff --git a/DeFi-Data-Engine/Api-Handler/target/classes/requests/template.properties b/DeFi-Data-Engine/Api-Handler/target/classes/requests/template.properties index 5dd16153..65544255 100644 --- a/DeFi-Data-Engine/Api-Handler/target/classes/requests/template.properties +++ b/DeFi-Data-Engine/Api-Handler/target/classes/requests/template.properties @@ -34,6 +34,8 @@ url.headers= header1,value1,\ # which have specific properties and handlers. Please review documentation to get a full # list of these tags. To default with no recursive call, set this property to . # This property we will set to for a clearer example. +# - parameterized: one of the parameters in the url call +# - embedded: parameter included within the response url.recursion.type= parameterized # This property sets all tags pertaining to the type of recursive call. Please refer to @@ -43,16 +45,15 @@ url.recursion.type= parameterized # example we will set the tags for which are as follows: # -rp: recursive parameter # -l: limit on items from request -# -t: type of parameter (url or incremental) +# -t: type of parameter (url or incremental) [embedded only required] url.recursion.tags= -rp,next,\ - -l,1000,\ - -t,url + -l,1000 # This property sets the location of the data points to be retrieved from the call. This # should be a JSONArray which the handler can iterate through. To access these data points # directly, the direct path must be specified (consisting of all JSONObject values). In # the example below, we point to the path located at response->data. Note that for storing # all non-array values and just recording all base values returned by the call, please set -# the value of this variable to '-b'. (i.e. url.data.path=-b) +# the value of this variable to '.') url.data.path= response,\ data \ No newline at end of file