Integrate request parameterized url

DeFi-Data-Engine/Api-Handler/src/main/java/org/stream/external/requester/RequestFramework.java

@@ -1,23 +1,34 @@
 package org.stream.external.requester;
 
+import java.io.IOException;
 import java.util.HashMap;
-import java.util.List;
-import java.util.stream.Collectors;
+import java.util.Iterator;
 
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+import okhttp3.OkHttpClient;
 import okhttp3.Request;
 import okhttp3.Request.Builder;
+import okhttp3.Response;
 
 public abstract class RequestFramework {
+	private final String name;
 	private final String url;
 	private final HashMap<String, String> properties;
 	private final HashMap<String, String> headers;
-	private final String[] tags;
+	private final HashMap<String, String> tags;
+	private final String[] path;
 
-	public RequestFramework(String url, HashMap<String, String> properties, HashMap<String, String> headers, String... tags) {
+	public RequestFramework(String name, String url, HashMap<String, String> properties, HashMap<String, String> headers, 
+							HashMap<String, String> tags, String[] path) {
+		this.name = name;
 		this.url = url;
 		this.properties = properties;
 		this.headers = headers;
 		this.tags = tags;
+		this.path = path;
 	}
 
 	protected final Request getRequest(HashMap<String, String> properties, HashMap<String, String> headers) {
@@ -89,9 +100,25 @@ protected final Request getRequest(HashMap<String, String> properties, HashMap<S
 		return builder.build();
 	}
 
-	public final String[] getTags() {
+	public final boolean hasTag(String tag) {
+		return this.tags.containsKey(tag);
+	}
+
+	public final String getTag(String tag) {
+		return this.tags.get(tag);
+	}
+
+	public final HashMap<String, String> getTags() {
 		return this.tags;
 	}
+
+	public final String getName() {
+		return name;
+	}
+
+	public final String[] getPath() {
+		return path;
+	}
 
 	@SuppressWarnings("unchecked")
 	public final void request(HashMap<String, String> properties, HashMap<String, String> headers) {
@@ -102,5 +129,76 @@ public final void request(HashMap<String, String> properties, HashMap<String, St
 		process(cloned_properties, cloned_headers);
 	}
 
-	protected abstract void process(HashMap<String, String> properties, HashMap<String, String> headers);
+	protected void process(HashMap<String, String> properties, HashMap<String, String> headers) {
+		OkHttpClient client = new OkHttpClient();
+		Request request = getRequest(properties, headers);
+		if(request == null) {
+			System.err.println("Malformed request, killing process.");
+			return;
+		}
+
+		Response response = null;
+		try {
+			response = client.newCall(request).execute();
+			if(response.code() != 200) {
+				System.err.println(String.format("Request Failure code <%d> url <%s>\nmsg <%s>", response.code(), request.url().toString(), response.message()));
+				System.exit(1);
+			}
+		} catch (IOException e) {
+			e.printStackTrace();
+		}
+
+		if(response == null) {
+			System.err.println("Response had fatal issue, killing process.");
+			return;
+		}
+
+		// send to specific request handler
+		process(response.body().toString(), properties, headers);
+	}
+
+	protected abstract void process(String json, HashMap<String, String> properties, HashMap<String, String> headers);
+
+	public abstract String getType();
+
+
+	protected final StringBuilder parse(Object input) throws JSONException {
+		StringBuilder out = new StringBuilder();
+		out = out.append(parseJsonObject(input));
+		return out.deleteCharAt(out.length() - 1);
+	}
+
+	private final StringBuilder parseJsonObject(Object input) throws JSONException {
+
+		StringBuilder out = new StringBuilder();
+
+	    if (input instanceof JSONObject) {
+
+	        Iterator<?> keys = ((JSONObject) input).keys();
+
+	        while (keys.hasNext()) {
+
+	            String key = (String) keys.next();
+
+	            if (!(((JSONObject) input).get(key) instanceof JSONArray)) {
+	                if (((JSONObject) input).get(key) instanceof JSONObject) {
+	                	out = out.append(parseJsonObject(((JSONObject) input).get(key)));
+	                } else {
+	                    out = out.append(key + "=" + ((JSONObject) input).get(key) + ",");
+	                }
+	            } else {
+	            	out = out.append(parseJsonObject(new JSONArray(((JSONObject) input).get(key).toString())));
+	            }
+	        }
+	    }
+
+	    if (input instanceof JSONArray) {
+	        for (int i = 0; i < ((JSONArray) input).length(); i++) {
+	            JSONObject a = ((JSONArray) input).getJSONObject(i);
+	            out = out.append(parseJsonObject(a));
+	        }
+	    }
+
+	    return out;
+	}
 }

DeFi-Data-Engine/Api-Handler/src/main/java/org/stream/external/requester/RequestParameterized.java

@@ -0,0 +1,108 @@
+package org.stream.external.requester;
+
+import java.util.HashMap;
+
+import org.json.JSONArray;
+import org.json.JSONObject;
+
+public class RequestParameterized extends RequestFramework {
+
+	public RequestParameterized(String name, String url, HashMap<String, String> properties, HashMap<String, String> headers,
+			HashMap<String, String> tags, String[] path) {
+		super(name, url, properties, headers, tags, path);
+	}
+
+	@Override
+	public String getType() {
+		return "parameterized";
+	}
+
+	public void process(String json, HashMap<String, String> properties, HashMap<String, String> headers) {
+		// parse json formatting
+		JSONObject obj = new JSONObject(json);
+
+		// validate all required parameters are present:
+		// check for recursive parameter
+		if(hasTag("-rp") && !properties.containsKey(getTag("-rp"))) {
+			System.err.println(String.format("Missing required recursive parameter <%s>", getTag("-rp")));
+			return;
+		}
+
+		// check for required tag -l
+		if(!hasTag("-l")) {
+			System.err.println(String.format("Missing required recursive parameter <-l>"));
+			return;
+		}
+
+		// check for -l being an integer
+		int limit = 0;
+		try {
+			limit = Integer.parseInt(getTag("-l"));
+		} catch(Exception e) {
+			e.printStackTrace();
+			System.err.println(String.format("Value following <-l> must be an integer."));
+			return;
+		}
+
+		// validate that the base has the proper obj path
+		String[] path = getPath();
+		JSONArray data = null;
+		for(int i = 0; i < path.length; i++) {
+			if(i == path.length - 1) {
+				if(obj.has(path[i])) {
+					try {
+						data = obj.getJSONArray(path[i]);
+					} catch(Exception e) {
+						System.err.println("obj path type is not of type <JSONArray>. Cannot parse.");
+						return;
+					}
+				}
+			}
+
+			else if(obj.has(path[i])) {
+				try {
+					obj = obj.getJSONObject(path[i]);
+				} catch(Exception e) {
+					System.err.println("obj path type step is not of type <JSONObject>. Cannot parse.");
+					return;
+				}
+			}
+		}
+
+		if(data == null) {
+			System.err.println("Data array retrieval had fatal error, killing process.");
+			return;
+		}
+
+		// extract and print data
+		for(int i = 0; i < data.length(); i++) {
+			StringBuilder point = parse(data.getJSONObject(i));
+			System.out.println(point);
+		}
+
+		// initiate recursive call
+		// if under limit requested, terminate call
+		if(data.length() < limit)
+			return;
+
+		// extract recursive parameter
+		int param = -1;
+		try {
+			param = Integer.parseInt(properties.get(getTag("-rp")));
+		} catch(Exception e) {
+			e.printStackTrace();
+			System.err.println(String.format("Recursive parameter <%s> is not of type integer.", getTag("-rp")));
+			return;
+		}
+
+		if(param == -1) {
+			System.err.println("Fatal parsing error occured.");
+			return;
+		}
+
+		// increment param, replace, and execute
+		param += 1;
+		properties.put(getTag("-rp"), "" + param);
+		process(properties, headers);
+	}
+}

DeFi-Data-Engine/Api-Handler/src/main/resources/requests/template.properties

@@ -34,6 +34,8 @@ url.headers=	header1,value1,\
 # which have specific properties and handlers. Please review documentation to get a full
 # list of these tags. To default with no recursive call, set this property to <single>.
 # This property we will set to <parameterized> for a clearer example.
+# - parameterized: 	one of the parameters in the url call
+# - embedded: 		parameter included within the response
 url.recursion.type=		parameterized
 
 # This property sets all tags pertaining to the type of recursive call. Please refer to
@@ -43,16 +45,15 @@ url.recursion.type=		parameterized
 # example we will set the tags for <parameterized> which are as follows:
 # -rp: 	recursive parameter
 # -l:	limit on items from request
-# -t:	type of parameter (url or incremental)
+# -t:	type of parameter (url or incremental) [embedded only required]
 url.recursion.tags=		-rp,next,\
-						-l,1000,\
-						-t,url
+						-l,1000
 
 # This property sets the location of the data points to be retrieved from the call. This
 # should be a JSONArray which the handler can iterate through. To access these data points
 # directly, the direct path must be specified (consisting of all JSONObject values). In 
 # the example below, we point to the path located at response->data. Note that for storing
 # all non-array values and just recording all base values returned by the call, please set
-# the value of this variable to '-b'. (i.e. url.data.path=-b)
+# the value of this variable to '.')
 url.data.path=			response,\
 						data

DeFi-Data-Engine/Api-Handler/target/classes/requests/template.properties

@@ -34,6 +34,8 @@ url.headers=	header1,value1,\
 # which have specific properties and handlers. Please review documentation to get a full
 # list of these tags. To default with no recursive call, set this property to <single>.
 # This property we will set to <parameterized> for a clearer example.
+# - parameterized: 	one of the parameters in the url call
+# - embedded: 		parameter included within the response
 url.recursion.type=		parameterized
 
 # This property sets all tags pertaining to the type of recursive call. Please refer to
@@ -43,16 +45,15 @@ url.recursion.type=		parameterized
 # example we will set the tags for <parameterized> which are as follows:
 # -rp: 	recursive parameter
 # -l:	limit on items from request
-# -t:	type of parameter (url or incremental)
+# -t:	type of parameter (url or incremental) [embedded only required]
 url.recursion.tags=		-rp,next,\
-						-l,1000,\
-						-t,url
+						-l,1000
 
 # This property sets the location of the data points to be retrieved from the call. This
 # should be a JSONArray which the handler can iterate through. To access these data points
 # directly, the direct path must be specified (consisting of all JSONObject values). In 
 # the example below, we point to the path located at response->data. Note that for storing
 # all non-array values and just recording all base values returned by the call, please set
-# the value of this variable to '-b'. (i.e. url.data.path=-b)
+# the value of this variable to '.')
 url.data.path=			response,\
 						data