From 80abb81e85afc17b35b42a816e13d5121b1faf17 Mon Sep 17 00:00:00 2001 From: Conor Flynn Date: Sat, 4 Mar 2023 00:07:38 -0500 Subject: [PATCH] add new api handler files --- .../requests/ExternalRequestFramework.java | 382 ++++++++++++++++++ .../requests/ExternalRequestManager.java | 226 +++++++++++ .../amberdata-aave-governance.properties | 28 ++ .../amberdata-aave-protocol.properties | 29 ++ .../amberdata-blockchain-addresses.properties | 30 ++ .../amberdata-compound-protocol.properties | 28 ++ .../amberdata-makerdao-asset.properties | 31 ++ .../amberdata-makerdao-protocol.properties | 29 ++ .../amberdata-makerdao-wallet.properties | 30 ++ .../amberdata-uniswap-pool.properties | 30 ++ .../resources/requests/template.properties | 142 +++++++ 11 files changed, 985 insertions(+) create mode 100644 DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/requests/ExternalRequestFramework.java create mode 100644 DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/requests/ExternalRequestManager.java create mode 100644 DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/amberdata-aave-governance.properties create mode 100644 DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/amberdata-aave-protocol.properties create mode 100644 DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/amberdata-blockchain-addresses.properties create mode 100644 DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/amberdata-compound-protocol.properties create mode 100644 DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/amberdata-makerdao-asset.properties create mode 100644 DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/amberdata-makerdao-protocol.properties create mode 100644 DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/amberdata-makerdao-wallet.properties create mode 100644 DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/amberdata-uniswap-pool.properties create mode 100644 DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/template.properties diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/requests/ExternalRequestFramework.java b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/requests/ExternalRequestFramework.java new file mode 100644 index 00000000..06ecd699 --- /dev/null +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/requests/ExternalRequestFramework.java @@ -0,0 +1,382 @@ +package org.stream.external.requests; + +import java.io.IOException; +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.util.HashMap; +import java.util.Iterator; +import java.util.stream.Stream; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.Response; +import okhttp3.Request.Builder; + +public abstract class ExternalRequestFramework { + +private final static DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + + private final String name; + private String url; + private final String[] url_path; + private final HashMap properties; + private final HashMap headers; + private final HashMap tags; + private final String[] recursive_location; + private final String recursive_replacement; + private final String[] path; + private final boolean is_dated; + private final String date_location; + private final String date_start_var; + private final String date_end_var; + private final DateTimeFormatter date_format; + + // default constructor used for templating + public ExternalRequestFramework() { + this.name = null; + this.url = null; + this.url_path = null; + this.properties = null; + this.headers = null; + this.tags = null; + this.recursive_location = null; + this.recursive_replacement = null; + this.path = null; + this.is_dated = false; + this.date_location = null; + this.date_start_var = null; + this.date_end_var = null; + this.date_format = null; + } + + public ExternalRequestFramework(String name, String url, String[] url_path, HashMap properties, HashMap headers, + HashMap tags, String[] recursive_location, String recursive_replacement, String[] path, + boolean is_dated, String date_location, String date_start_var, String date_end_var, String date_format) { + this.name = name; + this.url = url; + this.url_path = url_path; + this.properties = properties; + this.headers = headers; + this.tags = tags; + this.recursive_location = recursive_location; + this.recursive_replacement = recursive_replacement; + this.path = path; + this.is_dated = is_dated; + this.date_location = date_location; + this.date_start_var = date_start_var; + this.date_end_var = date_end_var; + this.date_format = DateTimeFormatter.ofPattern(date_format); + } + + public final boolean hasTag(String tag) { + return this.tags.containsKey(tag); + } + + public final String getTag(String tag) { + return this.tags.get(tag); + } + + public final HashMap getTags() { + return this.tags; + } + + public final String getName() { + return name; + } + + public final String getUrl() { + return url; + } + + public final String[] getRecursiveLocation() { + return recursive_location; + } + + public final String getRecursiveReplacement() { + return recursive_replacement; + } + + public final String[] getPath() { + return path; + } + + protected final Request getRequest(HashMap properties, HashMap headers) { + return this.getRequest(this.url, properties, headers); + } + + protected final Request getRequest(String url, HashMap properties, HashMap headers) { + // define builder + Builder builder = new Builder(); + StringBuilder url_builder = new StringBuilder(url); + + // add properties delim if there are properties + if(!properties.isEmpty()) + url_builder.append("?"); + + // add all required/optional properties + for(String property : properties.keySet()) { + String value = properties.get(property); + + // check if empty + if(value == null || value.equals("")) { + System.err.println(String.format("Property cannot be empty <%s>", property)); + return null; + } + + // check if required + if(value.equals(".") && this.properties.containsKey(property)) { + System.err.println(String.format("Required property <%s> not defined.", property)); + return null; + } + + // add to url + url_builder.append(String.format("%s=%s&", property, value)); + } + + // remove final & and update builder + if(!properties.isEmpty()) + url_builder.deleteCharAt(url_builder.length() - 1); + builder = builder.url(url_builder.toString()); + + // add all headers + for(String header : headers.keySet()) { + String value = headers.get(header); + + // check if empty + if(value == null || value.equals("")) { + System.err.println(String.format("Header cannot be empty <%s>", header)); + return null; + } + + // check if required + if(value.equals(".") && this.properties.containsKey(header)) { + System.err.println(String.format("Required header <%s> not defined.", header)); + return null; + } + + builder = builder.addHeader(header, value); + } + + return builder.build(); + } + + + public final synchronized String request(String[] url_path, HashMap properties, HashMap headers) { + // validate that url_path is correctly formatted + if(url_path.length % 2 != 0) { + System.err.println("Url path is not formatted properly and must be in pairs."); + return "Url path is not formatted properly and must be in pairs."; + } + + // validate that url_path parameter is valid + if(this.url_path.length != url_path.length) { + System.err.println("Url path does not match defined url path."); + return "Url path does not match defined url path."; + } + + // update path if there are url path extensions + StringBuilder url_builder = new StringBuilder(url); + if(url_path.length > 0) { + for(int i = 0; i < url_path.length; i+=2) { + if(!url_path[i].equals(this.url_path[i])) { + System.err.println(String.format("Url path key <%s> does not match defined key <%s>", url_path[i], this.url_path[i])); + return String.format("Url path key <%s> does not match defined key <%s>", url_path[i], this.url_path[i]); + } + + url_builder.append("/").append(url_path[i+1]); + } + } + + return process(url_builder.toString(), properties, headers); + } + + public final synchronized String request(String[] url_path, HashMap properties, HashMap headers, + String startDate, String endDate) { + + // validate that the request can be dated + if(!is_dated) { + System.err.println("Request cannot be dated."); + return "Request cannot be dated."; + } + + // parse passed dates + LocalDate start, end; + try { + start = LocalDate.parse(startDate, formatter); + end = LocalDate.parse(endDate, formatter); + } catch (DateTimeParseException e) { + return "Unable to parse dates."; + } + + if(start == null || end == null) { + System.err.println("Fatal error parsing dates."); + return "Fatal error parsing dates."; + } + + // retrieve all dates in-between + Stream dates = null; + + try { + dates = start.datesUntil(end); + } catch(Exception e) { + System.err.println("End date must be after start date."); + return "End date must be after start date."; + } + + // submit requests for each date + Iterator itr = dates.iterator(); + while(itr.hasNext()) { + // retrieve date + LocalDate date = itr.next(); + + // retrieve start date var and update to location + switch(date_location) { + case "properties": + properties.put(date_start_var, date.format(date_format)); + break; + case "headers": + headers.put(date_start_var, date.format(date_format)); + break; + default: + System.err.println("Invalid date.location parameter value."); + System.exit(1); + } + + // if end date is required add one day and push to properties + if(!date_end_var.equals(".")) { + LocalDate tmr = date.plusDays(1); + switch(date_location) { + case "properties": + properties.put(date_end_var, tmr.format(date_format)); + break; + case "headers": + headers.put(date_end_var, tmr.format(date_format)); + break; + default: + System.err.println("Invalid date.location parameter value."); + System.exit(1); + } + } + + // submit request with updated properties + System.out.println(String.format("Requesting: name=[%s] date=[%s]", getName(), date.toString())); + String request = request(url_path, properties, headers); + if(request != null) + return request; + } + + return null; + } + + protected String processUrl(String url, HashMap headers) { + HashMap all_headers = new HashMap(); + + for(String header : this.headers.keySet()) + all_headers.put(header, this.headers.get(header)); + + for(String header : headers.keySet()) + all_headers.put(header, headers.get(header)); + + return processRequest(url, new HashMap(), all_headers); + } + + protected String process(String url, HashMap properties, HashMap headers) { + HashMap all_properties = new HashMap(); + HashMap all_headers = new HashMap(); + + // add all properties and headers + for(String property : this.properties.keySet()) + all_properties.put(property, this.properties.get(property)); + + for(String property : properties.keySet()) + all_properties.put(property, properties.get(property)); + + for(String header : this.headers.keySet()) + all_headers.put(header, this.headers.get(header)); + + for(String header : headers.keySet()) + all_headers.put(header, headers.get(header)); + + return processRequest(url, all_properties, all_headers); + } + + private String processRequest(String url, HashMap properties, HashMap headers) { + OkHttpClient client = new OkHttpClient(); + Request request = getRequest(url, properties, headers); + if(request == null) { + System.err.println("Malformed request, killing process."); + return "Malformed request, killing process."; + } + + Response response = null; + String body = null; + try { + response = client.newCall(request).execute(); + body = response.body().string().toString(); + if(response.code() != 200) { + System.err.println(String.format("Request Failure code <%d> url <%s>\nbody:\n%s", response.code(), request.url().toString(), body)); + return String.format("Request Failure code <%d> url <%s>\nbody:\n%s", response.code(), request.url().toString(), body); + } + } catch (IOException e) { + e.printStackTrace(); + } + + if(body == null) { + System.err.println("Response had fatal issue, killing process."); + return "Response had fatal issue, killing process."; + } + + // send to specific request handler + try { + handle(body, properties, headers); + } catch(Exception e) { + e.printStackTrace(); + return e.toString(); + } + + return null; + } + + protected abstract String handle(String json, HashMap properties, HashMap headers); + + public abstract String getType(); + + protected final HashMap parse(Object input) throws JSONException { + + HashMap out = new HashMap(); + + if (input instanceof JSONObject) { + + Iterator keys = ((JSONObject) input).keys(); + + while (keys.hasNext()) { + + String key = (String) keys.next(); + + if (!(((JSONObject) input).get(key) instanceof JSONArray)) { + if (((JSONObject) input).get(key) instanceof JSONObject) { + out.putAll(parse(((JSONObject) input).get(key))); + } else { + out.put(key, ((JSONObject) input).get(key).toString()); + } + } else { + out.putAll(parse(new JSONArray(((JSONObject) input).get(key).toString()))); + } + } + } + + if (input instanceof JSONArray) { + for (int i = 0; i < ((JSONArray) input).length(); i++) { + JSONObject a = ((JSONArray) input).getJSONObject(i); + out.putAll(parse(a)); + } + } + + return out; + } +} diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/requests/ExternalRequestManager.java b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/requests/ExternalRequestManager.java new file mode 100644 index 00000000..cd4d132f --- /dev/null +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/requests/ExternalRequestManager.java @@ -0,0 +1,226 @@ +package org.stream.external.requests; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.net.URISyntaxException; +import java.util.HashMap; +import java.util.Properties; +import java.util.Set; + +import org.reflections.Reflections; +import org.reflections.util.ConfigurationBuilder; + +public class ExternalRequestManager { + + private static final HashMap> templates = new HashMap>(); + private static final HashMap requests = new HashMap(); + + public static void initialize() throws InstantiationException, IllegalAccessException, IllegalArgumentException, InvocationTargetException, NoSuchMethodException, SecurityException, URISyntaxException, IOException { + // load all templates + Reflections reflection = new Reflections(new ConfigurationBuilder().forPackages(ExternalRequestManager.class.getPackageName())); + Set> types = reflection.getSubTypesOf(ExternalRequestFramework.class); + for(Class c : types) { + templates.put(c.getDeclaredConstructor().newInstance().getType(), c); + } + + // load all property files + File directory = new File("target/classes/requests/"); + if(!directory.isDirectory()) { + System.err.println("Missing /target/classes/requests/ directory."); + System.exit(1); + } + + String[] requests = directory.list(); + for(String r : requests) { + // create file + Properties p = new Properties(); + p.load(new FileInputStream(directory.getPath() + "/" + r)); + + // extract all properties and parse + String name = null; + String url = null; + String[] url_path = {}; + HashMap properties = new HashMap(); + HashMap headers = new HashMap(); + String recursive_type = null; + HashMap tags = new HashMap(); + String[] recursive_location = {}; + String recursive_replacement = null; + String[] path = {}; + boolean is_dated = false; + String date_location = null; + String date_start_var = null; + String date_end_var = null; + String date_format = null; + + // parse name + if(p.containsKey("request.name")) { + name = p.getProperty("request.name"); + } else { + System.err.println(String.format("Missing required property <%s>.", "request.name")); + System.exit(1); + } + + // parse url base + if(p.containsKey("url.base")) { + url = p.getProperty("url.base"); + } else { + System.err.println(String.format("Missing required property <%s>.", "url.base")); + System.exit(1); + } + + // parse url base properties + if(p.containsKey("url.base.path")) { + url_path = p.getProperty("url.base.path").split(","); + if(url_path.length % 2 != 0) { + System.err.println("Url path must be formatted with pairs."); + System.exit(1); + } + } + + // parse url properties + if(p.containsKey("url.properties")) { + String[] arr = p.getProperty("url.properties").split(","); + if(arr.length % 2 != 0) { + System.err.println("Each property must be a pair."); + System.exit(1); + } + + for(int i = 0; i < arr.length; i+=2) + properties.put(arr[i], arr[i + 1]); + } else { + System.err.println(String.format("Missing required property <%s>.", "url.properties")); + System.exit(1); + } + + // parse url headers + if(p.containsKey("url.headers")) { + String[] arr = p.getProperty("url.headers").split(","); + if(arr.length % 2 != 0) { + System.err.println("Each property must be a pair."); + System.exit(1); + } + + for(int i = 0; i < arr.length; i+=2) + headers.put(arr[i], arr[i + 1]); + } else { + System.err.println(String.format("Missing required property <%s>.", "url.headers")); + System.exit(1); + } + + // parse data path + if(p.containsKey("data.path")) { + path = p.getProperty("data.path").split(","); + } else { + System.err.println(String.format("Missing required property <%s>.", "data.path")); + System.exit(1); + } + + // parse recursion type + if(p.containsKey("recursion.type")) { + recursive_type = p.getProperty("recursion.type"); + } else { + System.err.println(String.format("Missing required property <%s>.", "url.recursive.type")); + System.exit(1); + } + + // parse recursion tags + if(p.containsKey("recursion.tags")) { + String[] arr = p.getProperty("recursion.tags").split(","); + if(arr.length % 2 != 0) { + System.err.println("Each property must be a pair."); + System.exit(1); + } + + for(int i = 0; i < arr.length; i+=2) + tags.put(arr[i], arr[i + 1]); + } + + // parse recursive location + if(p.containsKey("recursion.location")) { + recursive_location = p.getProperty("recursion.location").split(","); + } + + // parse recursive replacement location + if(p.containsKey("recursion.replacement")) { + recursive_replacement = p.getProperty("recursive.replacement"); + } + + // parse date valid + if(p.containsKey("date.valid")) { + try { + is_dated = Boolean.parseBoolean(p.getProperty("date.valid")); + } catch(Exception e) { + System.err.println("Property date.valid is not of type boolean."); + System.exit(1); + } + } else { + System.err.println(String.format("Missing required property <%s>.", "date.valid")); + System.exit(1); + } + + // parse date location + if(p.containsKey("date.location")) { + date_location = p.getProperty("date.location"); + } else if(!p.containsKey("date.location") && is_dated) { + System.err.println("Missing required property date.location if date.valid=true."); + System.exit(1); + } + + // parse date start + if(p.containsKey("date.start")) { + date_start_var = p.getProperty("date.start"); + } else if(!p.containsKey("date.start") && is_dated) { + System.err.println("Missing required property date.start if date.valid=true."); + System.exit(1); + } + + // parse date end + if(p.containsKey("date.end")) { + date_end_var = p.getProperty("date.end"); + } else if(!p.containsKey("date.end") && is_dated) { + System.err.println("Missing required property date.end if date.valid=true."); + System.exit(1); + } + + // parse date format + if(p.containsKey("date.format")) { + date_format = p.getProperty("date.format"); + } else if(!p.containsKey("date.format") && is_dated) { + System.err.println("Missing required property date.format if date.valid=true."); + System.exit(1); + } + + // validate templates contains proper key + if(!templates.containsKey(recursive_type)) { + System.err.println(String.format("Template not found for <%s>.", recursive_type)); + System.exit(1); + } + + // validate that duplicate request doesn't exist + if(ExternalRequestManager.requests.containsKey(name)) { + System.err.println(String.format("Duplicate name found <%s>. Please modify one of the names to be unique.", name)); + System.exit(1); + } + + ExternalRequestManager.requests.put(name, templates.get(recursive_type).getDeclaredConstructor( + String.class, String.class, String[].class, HashMap.class, HashMap.class, + HashMap.class, String[].class, String.class, String[].class, + boolean.class, String.class, String.class, String.class, String.class).newInstance( + name, url, url_path, properties, headers, tags, recursive_location, recursive_replacement, path, + is_dated, date_location, date_start_var, date_end_var, date_format)); + + System.out.println(String.format("Successfully added type [%s]", name)); + } + } + + public static final boolean hasRequestFormat(String name) { + return requests.containsKey(name); + } + + public static final ExternalRequestFramework getRequestFormat(String name) { + return requests.get(name); + } +} diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/amberdata-aave-governance.properties b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/amberdata-aave-governance.properties new file mode 100644 index 00000000..394e20c3 --- /dev/null +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/amberdata-aave-governance.properties @@ -0,0 +1,28 @@ +request.name= amberdata-aave-governance + +url.base= https://web3api.io/api/v2/defi/lending/aavev3/governance + +url.properties= size,900 + +url.headers= accept,application/json,\ + x-api-key,. + +data.path= payload,\ + data + +recursion.type= parameterized + +recursion.tags= -l,900,\ + -t,url + +recursion.location= payload,metadata,next + +date.valid= true + +date.location= properties + +date.start= startDate + +date.end= endDate + +date.format= yyyy-MM-dd \ No newline at end of file diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/amberdata-aave-protocol.properties b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/amberdata-aave-protocol.properties new file mode 100644 index 00000000..cf803956 --- /dev/null +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/amberdata-aave-protocol.properties @@ -0,0 +1,29 @@ +request.name= amberdata-aave-protocol + +url.base= https://web3api.io/api/v2/defi/lending/aavev3/protocol + +url.properties= size,900 + +url.headers= accept,application/json,\ + x-amberdata-blockchain-id,polygon-mainnet,\ + x-api-key,. + +data.path= payload,\ + data + +recursion.type= parameterized + +recursion.tags= -l,900,\ + -t,url + +recursion.location= payload,metadata,next + +date.valid= true + +date.location= properties + +date.start= startDate + +date.end= endDate + +date.format= yyyy-MM-dd \ No newline at end of file diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/amberdata-blockchain-addresses.properties b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/amberdata-blockchain-addresses.properties new file mode 100644 index 00000000..823fad7a --- /dev/null +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/amberdata-blockchain-addresses.properties @@ -0,0 +1,30 @@ +request.name= amberdata-blockchain-addresses + +url.base= https://web3api.io/api/v2/addresses + +url.properties= page,0,\ + size,900 + +url.headers= accept,application/json,\ + x-amberdata-blockchain-id,ethereum-mainnet,\ + x-api-key,. + +data.path= payload,\ + records + +recursion.type= parameterized + +recursion.tags= -l,900,\ + -t,incremental + +recursion.location= page + +date.valid= true + +date.location= properties + +date.start= startDate + +date.end= endDate + +date.format= yyyy-MM-dd \ No newline at end of file diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/amberdata-compound-protocol.properties b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/amberdata-compound-protocol.properties new file mode 100644 index 00000000..c7a78364 --- /dev/null +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/amberdata-compound-protocol.properties @@ -0,0 +1,28 @@ +request.name= amberdata-compound-protocol + +url.base= https://web3api.io/api/v2/defi/lending/compoundv2/protocol + +url.properties= size,900 + +url.headers= accept,application/json,\ + x-api-key,. + +data.path= payload,\ + data + +recursion.type= parameterized + +recursion.tags= -l,900,\ + -t,url + +recursion.location= payload,metadata,next + +date.valid= true + +date.location= properties + +date.start= startDate + +date.end= endDate + +date.format= yyyy-MM-dd \ No newline at end of file diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/amberdata-makerdao-asset.properties b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/amberdata-makerdao-asset.properties new file mode 100644 index 00000000..385c25c4 --- /dev/null +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/amberdata-makerdao-asset.properties @@ -0,0 +1,31 @@ +request.name= amberdata-makerdao-asset + +url.base= https://web3api.io/api/v2/defi/lending/makerdao/assets + +url.base.path= asset,. + +url.properties= size,900 + +url.headers= accept,application/json,\ + x-amberdata-blockchain-id,ethereum-mainnet,\ + x-api-key,. + +data.path= payload,\ + data + +recursion.type= parameterized + +recursion.tags= -l,900,\ + -t,url + +recursion.location= payload,metadata,next + +date.valid= true + +date.location= properties + +date.start= startDate + +date.end= endDate + +date.format= yyyy-MM-dd \ No newline at end of file diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/amberdata-makerdao-protocol.properties b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/amberdata-makerdao-protocol.properties new file mode 100644 index 00000000..37d841e4 --- /dev/null +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/amberdata-makerdao-protocol.properties @@ -0,0 +1,29 @@ +request.name= amberdata-makerdao-protocol + +url.base= https://web3api.io/api/v2/defi/lending/makerdao/protocol + +url.properties= size,900 + +url.headers= accept,application/json,\ + x-amberdata-blockchain-id,ethereum-mainnet,\ + x-api-key,. + +data.path= payload,\ + data + +recursion.type= parameterized + +recursion.tags= -l,900,\ + -t,url + +recursion.location= payload,metadata,next + +date.valid= true + +date.location= properties + +date.start= startDate + +date.end= endDate + +date.format= yyyy-MM-dd \ No newline at end of file diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/amberdata-makerdao-wallet.properties b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/amberdata-makerdao-wallet.properties new file mode 100644 index 00000000..0669c075 --- /dev/null +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/amberdata-makerdao-wallet.properties @@ -0,0 +1,30 @@ +request.name= amberdata-makerdao-wallet + +url.base= https://web3api.io/api/v2/defi/lending/makerdao/wallets + +url.base.path= walletAddress,. + +url.properties= size,900 + +url.headers= accept,application/json,\ + x-api-key,. + +data.path= payload,\ + data + +recursion.type= parameterized + +recursion.tags= -l,900,\ + -t,url + +recursion.location= payload,metadata,next + +date.valid= true + +date.location= properties + +date.start= startDate + +date.end= endDate + +date.format= yyyy-MM-dd \ No newline at end of file diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/amberdata-uniswap-pool.properties b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/amberdata-uniswap-pool.properties new file mode 100644 index 00000000..26c80cf1 --- /dev/null +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/amberdata-uniswap-pool.properties @@ -0,0 +1,30 @@ +request.name= amberdata-uniswap-pool + +url.base= https://web3api.io/api/v2/defi/dex/uniswapv2/pools + +url.base.path= poolAddress,. + +url.properties= size,900 + +url.headers= accept,application/json,\ + x-api-key,. + +data.path= payload,\ + data + +recursion.type= parameterized + +recursion.tags= -l,900,\ + -t,url + +recursion.location= payload,metadata,next + +date.valid= true + +date.location= properties + +date.start= startDate + +date.end= endDate + +date.format= yyyy-MM-dd \ No newline at end of file diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/template.properties b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/template.properties new file mode 100644 index 00000000..fcd8b2dc --- /dev/null +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/template.properties @@ -0,0 +1,142 @@ +# === Required Properties === +# These properties are required regardless of the request and should maintain a uniform syntax. + +# [REQUIRED] +# Name of the call to be referenced in the engine +request.name= template + +# [REQUIRED] +# Base url of the REST API call +url.base= http://localhost:8080 + +# [OPTIONAL] +# Url PATH properties that are required for parsing of the system. For example, if the url is +# https://localhost:8080/api/v1// where asset is dictated by the parameter 'asset', +# then this variable will be set to 'asset,.' Note all properties are in sequential order, such that +# asset must be defined before value. +url.base.path= asset,.,\ + value,. + +# [REQUIRED] +# This property details all required properties to be passed on runtime when called. Optional +# properties are not required to be specified here. Note that properties can be given a default +# value or can be forced to be specified. The list is in (, ) pairs, with each property +# having both a key and value specified. Default values can be placed in the property. +# Key's with no default property that are required on runtime can be filled as '.'. In this example +# has the default value , is required on runtime, and +# has the default value . +url.properties= property1,value1,\ + property2,.,\ + property3,value3 + +# [REQUIRED] +# This property details all headers to be passed on runtime when generating the request. +# Optional headers can be passed on runtime and are not required to be specified here. Note +# that headers can be given a default value or can be forced to be specified. The list is in +# (, ) pairs, with each property having both a key and value specified. Default +# values can be placed in the property. Key's with no default property that are required +# on runtime can be filled as '.'. In this example has the default value , +# and headers and are required on runtime. +url.headers= header1,value1,\ + header2,.,\ + header3,. + +# [REQUIRED] +# This property sets the location of the data points to be retrieved from the call. This +# should be a JSONArray which the handler can iterate through. To access these data points +# directly, the direct path must be specified (consisting of all JSONObject values). In +# the example below, we point to the path located at response->data. Note that for storing +# all non-array values and just recording all base values returned by the call, please set +# the value of this variable to '-b'. (i.e. url.data.path=-b) +data.path= response,\ + data + +# [REQUIRED] +# This property determines if the call is recursive, meaning that all data points required +# cannot be obtained in a single request. There are several integrated recursive types +# which have specific properties and handlers. Please review documentation to get a full +# list of these tags. To default with no recursive call, set this property to . +# This property we will set to for a clearer example. +recursion.type= parameterized + +# [OPTIONAL]: +# - url.recursion.type = static +# [REQUIRED]: +# - url.recursion.type = parameterized +# This property sets all tags pertaining to the type of recursive call. Please refer to +# the documentation for the full list of all tags and specified recursive types. All tags +# are in (, ) pairs, with each property having both a key and value specified. +# Should a tag not require an accompanying , please set it to '.'. In this +# example we will set the tags for which are as follows: +# -l: limit on items from request +# -t: type of recursive parameter (url, incremental, static) +recursion.tags= -l,1000,\ + -t,url + +# [OPTIONAL]: +# - url.recursion.type = static +# [REQUIRED]: +# - url.recursion.type = parameterized +# This property sets the recurisve parameters location within the response. Should '-t' in +# url.recursion.tags be of type 'incremental', this property should be the property to be +# incremented in the url. Otherwise if it is of type 'url' or 'static', point to the exact +# location in the response which will retrieve this property. For example if the next url +# is contained in: +# { +# "payload": { +# "metadata": { +# "next": "https://..." +# } +# } +# } +# this property would be set to 'payload,metadata,next'. For properties with no recursive call +# (with recursion type 'single') this parameter can remain blank. +recursion.location= payload,metadata,next + +# [OPTIONAL]: +# - url.recursion.type = static +# [REQUIRED]: +# - url.recursion.type = parameterized +# If the property defined in url.recursion.location is not the same as the property to replace +# in the url, define it here. If they are the same, this property can remain blank. This property +# will primarily be used if url.recursion.tags '-t' is set to 'static'. For '-t' being set to +# 'url', this property can remain blank. +recursion.replacement= + +# [REQUIRED] +# This parameter is used to determine whether the protocol can be dated or not. If so, the following +# properties are required: date.location, date.start, date.end, and date.format. +date.valid= true + +# [OPTIONAL]: +# - date.enable = false +# [REQUIRED]: +# - date.enable = true +# This property tells the location of the date variable, whether in the 'properties' or 'header' (note add +# path at a later date). This property will default to properties. +date.location= properties + +# [OPTIONAL]: +# - date.enable = false +# [REQUIRED]: +# - date.enable = true +# This property tells what key value will point to the start date variable. This is required and if no +# end date is required, use this as the primary date. +date.start= startDate + +# [OPTIONAL]: +# - date.enable = false +# [REQUIRED]: +# - date.enable = true +# This property tells what key value will point to the end date variable. This is not required if there +# is only one variable needed to reference the date. If so, set this value to '.' +date.end= endDate + +# [OPTIONAL]: +# - date.enable = false +# [REQUIRED]: +# - date.enable = true +# This property states the format for the date to be pushed to the parameter. Note this value will be extracted +# from the original call (which will be of the form yyyy-MM-dd and will always default to midnight should a time +# be required). +date.format= yyyy-MM-dd \ No newline at end of file