From da480cd036564793f4e4a0e461c4465b57599f3d Mon Sep 17 00:00:00 2001 From: Conor Flynn Date: Sun, 12 Mar 2023 23:13:04 -0400 Subject: [PATCH 1/3] add external stream handler and framework with formatting --- .../amberdata/AmberDataRequestHandler.java | 392 ------------------ .../amberdata/AmberDataRequestPacket.java | 37 -- .../connections/AmberDataConnection.java | 139 ------- .../TemplateExternalConnection.java | 98 ----- .../handler/ExternalStreamConnection.java | 130 +++--- .../handler/ExternalStreamHandler.java | 358 ++++++++-------- .../handler/ExternalStreamManager.java | 65 ++- .../requests/ExternalRequestFramework.java | 21 +- .../requests/ExternalRequestManager.java | 7 +- .../requests/ExternalRequestREST.java | 184 ++++++++ .../amberdata-aave-governance.properties | 2 +- .../amberdata-aave-protocol.properties | 2 +- .../amberdata-blockchain-addresses.properties | 2 +- .../amberdata-compound-protocol.properties | 2 +- .../amberdata-makerdao-asset.properties | 2 +- .../amberdata-makerdao-protocol.properties | 2 +- .../amberdata-makerdao-wallet.properties | 2 +- .../amberdata-uniswap-pool.properties | 2 +- 18 files changed, 522 insertions(+), 925 deletions(-) delete mode 100644 DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/connected/amberdata/AmberDataRequestHandler.java delete mode 100644 DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/connected/amberdata/AmberDataRequestPacket.java delete mode 100644 DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/connected/connections/AmberDataConnection.java delete mode 100644 DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/connected/connections/TemplateExternalConnection.java create mode 100644 DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/requests/ExternalRequestREST.java diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/connected/amberdata/AmberDataRequestHandler.java b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/connected/amberdata/AmberDataRequestHandler.java deleted file mode 100644 index e7aca6bf..00000000 --- a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/connected/amberdata/AmberDataRequestHandler.java +++ /dev/null @@ -1,392 +0,0 @@ -package org.stream.external.connected.amberdata; - -import java.io.IOException; -import java.lang.reflect.Method; -import java.time.LocalDate; -import java.time.format.DateTimeFormatter; -import java.util.HashMap; - -import org.core.logger.Logger; -import org.framework.router.ResponseFactory; -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; -import org.properties.Config; - -import okhttp3.OkHttpClient; -import okhttp3.Request; - -public class AmberDataRequestHandler { - - private volatile static OkHttpClient client; - private final static HashMap requests; - private static final int request_size = 999; - - private static final DateTimeFormatter formatter = DateTimeFormatter.ofPattern(Config.getProperty("app", "general.data.dateformat")); - -static { - client = new OkHttpClient(); - - requests = new HashMap(); - - Class classobj = AmberDataRequestHandler.class; - - try { - // add all methods to the handler - requests.put("lending-latest", classobj.getMethod("requestLendingLatest", AmberDataRequestPacket.class)); - requests.put("aave-protocol-dated", classobj.getMethod("requestAaveProtocolDated", AmberDataRequestPacket.class)); - requests.put("aave-asset-dated", classobj.getMethod("requestAaveAssetDated", AmberDataRequestPacket.class)); - requests.put("blockchain-addresses-dated", classobj.getMethod("requestBlockchainAddresses", AmberDataRequestPacket.class)); - } catch (Exception e) { - e.printStackTrace(); - ResponseFactory.responseNotHandled("Irregular method call for AmberDataRequestHandler."); - } -} - - public static Object[] request(AmberDataRequestPacket packet) { - if(!requests.containsKey(packet.getData("request"))) - return new Object[] {false, "Request does not exist."}; - - if(packet.getKey() == null) - return new Object[] {false, " cannot be null."}; - - try { - return (Object[])requests.get(packet.getData("request")).invoke(null, packet); - } catch (Exception e) { - e.printStackTrace(); - ResponseFactory.responseNotHandled("Irregular method call for AmberDataRequestHandler."); - return null; - } - } - - public static Object[] requestLendingLatest(AmberDataRequestPacket packet) { - Request request = new Request.Builder() - .url("https://web3api.io/api/v2/market/defi/lending/exchanges/aave/latest") - .get() - .addHeader("Accept", "application/json") - .addHeader("x-api-key", packet.getData("key")) - .build(); - - try { - okhttp3.Response response = client.newCall(request).execute(); - JSONObject json = new JSONObject(response.body().string()); - - // validate successful request - if(!response.isSuccessful() || json.getInt("status") != 200 - || !json.getString("description").equals("Successful request")) - return new Object[]{false, json.getString("description")}; - - return new Object[] {true, json.toString()}; - } catch (Exception e) { - e.printStackTrace(); - ResponseFactory.responseNotHandled("Unhandled exception cost."); - return new Object[] {false, null}; - } - } - - public static Object[] requestAaveProtocolDated(AmberDataRequestPacket packet) { - LocalDate next = LocalDate.parse(packet.getData("date"), formatter); - next = next.plusDays(1); - String tmr = next.format(formatter); - - String url = String.format("https://web3api.io/api/v2/defi/lending/aavev2/protocol?startDate=%s&endDate=%s", - packet.getData("date") + "T01:00:00", - tmr + "T01:00:00"); - - return requestAaveURL(packet, url); - } - - public static Object[] requestAaveAssetDated(AmberDataRequestPacket packet) { - if(packet.getData("asset") == null) - return new Object[] {false, "Missing parameter ."}; - - LocalDate next = LocalDate.parse(packet.getData("date"), formatter); - next = next.plusDays(1); - String tmr = next.format(formatter); - - String url = String.format("https://web3api.io/api/v2/defi/lending/aavev2/assets/%s?startDate=%s&endDate=%s", - packet.getData("asset"), - packet.getData("date") + "T01:00:00", - tmr + "T01:00:00"); - - return requestAaveURL(packet, url, packet.getData("asset")); - } - - private static Object[] requestAaveURL(AmberDataRequestPacket packet, String url) { - return requestAaveURL(packet, url, null); - } - - private static Object[] requestAaveURL(AmberDataRequestPacket packet, String url, String request_specifier) { - OkHttpClient client = new OkHttpClient(); - - Request request = new Request.Builder() - .url(url + "&size=" + request_size) - .get() - .addHeader("accept", "application/json") - .addHeader("x-api-key", packet.getKey()) - .build(); - - okhttp3.Response response; - try { - response = client.newCall(request).execute(); - JSONObject json = new JSONObject(response.body().string()); - - if(json.toString().equals("") || !json.has("description")) - return new Object[] {false, "JSON Object returned empty or invalid contents."}; - - if(!response.isSuccessful() || json.getInt("status") != 200 - || !json.getString("description").equals("Successful request")) - return new Object[]{false, json.getString("description")}; - - if(!json.has("payload") || !json.getJSONObject("payload").has("data")) - return new Object[] {false, "Malformed Aave packet"}; - - JSONObject payload = json.getJSONObject("payload"); - JSONArray arr = payload.getJSONArray("data"); - for(int i = 0; i < arr.length(); i++) { - if(request_specifier == null) { - processAaveJsonRequest(packet, arr.getJSONObject(i)); - } else { - processAaveJsonRequest(packet, arr.getJSONObject(i), request_specifier); - } - } - - if(arr.length() >= request_size) { - JSONObject metadata = payload.getJSONObject("metadata"); - if(metadata.has("next")) - return requestAaveURL(packet, metadata.getString("next")); - else - return new Object[] {false, "Response did not contain next API cursor"}; - } else { - return new Object[] {true, ""}; - } - } catch (IOException e) { - e.printStackTrace(); - ResponseFactory.responseNotHandled("Unhandled exception cost."); - return new Object[] {false, null}; - } - } - - private static void processAaveJsonRequest(AmberDataRequestPacket packet, JSONObject obj) throws JSONException { - processAaveJsonRequest(packet, obj, null); - } - - private static void processAaveJsonRequest(AmberDataRequestPacket packet, JSONObject obj, String request_specifier) throws JSONException { - if(request_specifier != null) - request_specifier = "-" + request_specifier; - else - request_specifier = ""; - - String action = obj.getString("action"); - switch(action) { - - case "UseReserveAsCollateral": - packet.getConnection().processRequest( - packet.getData("request") + request_specifier, - packet.getData("date"), - format("action", obj.getString("action"), - "timestamp", obj.get("timestamp").toString(), - "blockNumber", obj.getLong("blockNumber"), - "transactionHash", obj.getString("transactionHash"), - "logIndex", obj.getInt("logIndex"), - "assetId", obj.getString("assetId"), - "assetSymbol", obj.getString("assetSymbol"), - "marketId", obj.getString("marketId"), - "market", obj.getString("market"), - "reserveAsCollateralEnabled", obj.getBoolean("reserveAsCollateralEnabled"), - "user", obj.getString("user"))); - break; - - case "Deposit": - packet.getConnection().processRequest( - packet.getData("request") + request_specifier, - packet.getData("date"), - format("action", obj.getString("action"), - "timestamp", obj.get("timestamp").toString(), - "blockNumber", obj.getLong("blockNumber"), - "transactionHash", obj.getString("transactionHash"), - "logIndex", obj.getInt("logIndex"), - "assetId", obj.getString("assetId"), - "assetSymbol", obj.getString("assetSymbol"), - "marketId", obj.getString("marketId"), - "market", obj.getString("market"), - "user", obj.getString("user"), - "onBehalfOf", obj.getString("onBehalfOf"))); - break; - - case "Withdraw": - packet.getConnection().processRequest( - packet.getData("request") + request_specifier, - packet.getData("date"), - format("action", obj.getString("action"), - "timestamp", obj.get("timestamp").toString(), - "blockNumber", obj.getLong("blockNumber"), - "transactionHash", obj.getString("transactionHash"), - "logIndex", obj.getInt("logIndex"), - "assetId", obj.getString("assetId"), - "assetSymbol", obj.getString("assetSymbol"), - "marketId", obj.getString("marketId"), - "market", obj.getString("market"), - "amount", obj.get("amount").toString(), - "user", obj.getString("user"), - "to", obj.getString("to"))); - break; - - case "LiquidationCall": - packet.getConnection().processRequest( - packet.getData("request") + request_specifier, - packet.getData("date"), - format("action", obj.getString("action"), - "timestamp", obj.get("timestamp").toString(), - "blockNumber", obj.getLong("blockNumber"), - "transactionHash", obj.getString("transactionHash"), - "logIndex", obj.getInt("logIndex"), - "collateralAssetId", obj.getString("collateralAssetId"), - "collateralAssetSymbol", obj.getString("collateralAssetSymbol"), - "principalAssetId", obj.getString("principalAssetId"), - "principalAssetSymbol", obj.getString("principalAssetSymbol"), - "principalAmount", obj.get("principalAmount").toString(), - "marketId", obj.getString("marketId"), - "market", obj.getString("market"), - "liquidator", obj.getString("liquidator"))); - break; - - case "Repay": - packet.getConnection().processRequest( - packet.getData("request") + request_specifier, - packet.getData("date"), - format("action", obj.getString("action"), - "timestamp", obj.get("timestamp").toString(), - "blockNumber", obj.getLong("blockNumber"), - "transactionHash", obj.getString("transactionHash"), - "logIndex", obj.getInt("logIndex"), - "assetId", obj.getString("assetId"), - "assetSymbol", obj.getString("assetSymbol"), - "marketId", obj.getString("marketId"), - "market", obj.getString("market"), - "amount", obj.get("amount").toString(), - "user", obj.getString("user"), - "repayer", obj.getString("repayer"))); - break; - - case "Borrow": - packet.getConnection().processRequest( - packet.getData("request") + request_specifier, - packet.getData("date"), - format("action", obj.getString("action"), - "timestamp", obj.get("timestamp").toString(), - "blockNumber", obj.getLong("blockNumber"), - "transactionHash", obj.getString("transactionHash"), - "logIndex", obj.getInt("logIndex"), - "assetId", obj.getString("assetId"), - "assetSymbol", obj.getString("assetSymbol"), - "marketId", obj.getString("marketId"), - "market", obj.getString("market"), - "amount", obj.get("amount").toString(), - "borrowRate", obj.get("borrowRate").toString(), - "debt", obj.get("debt"), - "user", obj.getString("user"), - "onBehalfOf", obj.getString("onBehalfOf"))); - break; - - case "FlashLoan": - packet.getConnection().processRequest( - packet.getData("request") + request_specifier, - packet.getData("date"), - format("action", obj.getString("action"), - "timestamp", obj.get("timestamp").toString(), - "blockNumber", obj.getLong("blockNumber"), - "transactionHash", obj.getString("transactionHash"), - "logIndex", obj.getInt("logIndex"), - "assetId", obj.getString("assetId"), - "assetSymbol", obj.getString("assetSymbol"), - "marketId", obj.getString("marketId"), - "market", obj.getString("market"), - "amount", obj.get("amount").toString(), - "borrowRate", obj.get("totalFee").toString(), - "target", obj.get("target"), - "initiator", obj.getString("initiator"))); - - default: - Logger.warn("Unrecognized transaction type in : " + action); - } - } - - public static Object[] requestBlockchainAddresses(AmberDataRequestPacket packet) { - return processBlockchainAddresses(packet, 0); - } - - private static Object[] processBlockchainAddresses(AmberDataRequestPacket packet, int page) { - OkHttpClient client = new OkHttpClient(); - - LocalDate next = LocalDate.parse(packet.getData("date"), formatter); - next = next.plusDays(1); - String tmr = next.format(formatter); - - String url = String.format("https://web3api.io/api/v2/defi/lending/aavev2/assets/%s?startDate=%s&endDate=%s&page=%s", - packet.getData("asset"), - packet.getData("date") + "T01:00:00", - tmr + "T01:00:00", - page); - - Request request = new Request.Builder() - .url(url + "&size=" + request_size) - .get() - .addHeader("x-amberdata-blockchain-id", "ethereum-mainnet") - .addHeader("accept", "application/json") - .addHeader("x-api-key", "UAK7ed69235426c360be22bfc2bde1809b6") - .build(); - - okhttp3.Response response; - try { - response = client.newCall(request).execute(); - JSONObject json = new JSONObject(response.body().string()); - - if(json.toString().equals("") || !json.has("description")) - return new Object[] {false, "JSON Object returned empty or invalid contents."}; - - if(!response.isSuccessful() || json.getInt("status") != 200 - || !json.getString("description").equals("Successful request")) { - System.out.println(json.getString("message")); - return new Object[]{false, json.getString("description")}; - } - - if(!json.has("payload") || !json.getJSONObject("payload").has("records")) - return new Object[] {false, "Malformed Aave packet"}; - - JSONObject payload = json.getJSONObject("payload"); - JSONArray arr = payload.getJSONArray("records"); - for(int i = 0; i < arr.length(); i++) { - JSONObject obj = arr.getJSONObject(i); - packet.getConnection().processRequest( - packet.getData("request"), - packet.getData("date"), - format("creator", obj.getString("creator"), - "firstBlockNumber", obj.getString("firstBlockNumber"), - "firstTransactionHash", obj.getString("firstTransactionHash"), - "address", obj.getJSONObject("hash").getString("address"), - "timestamp", obj.get("timestamp").toString(), - "type", obj.getString("type"))); - } - - if(arr.length() >= request_size) { - return processBlockchainAddresses(packet, page + 1); - } else { - return new Object[] {true, ""}; - } - } catch (IOException e) { - e.printStackTrace(); - return new Object[] {false, "Exception."}; - } - } - - private static String format(Object... data) { - StringBuilder out = new StringBuilder(); - for(int i = 0; i < data.length; i++) { - out.append(data[i].toString()); - if(i != data.length - 1) - out.append(","); - } - return out.toString(); - } -} \ No newline at end of file diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/connected/amberdata/AmberDataRequestPacket.java b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/connected/amberdata/AmberDataRequestPacket.java deleted file mode 100644 index d6af5459..00000000 --- a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/connected/amberdata/AmberDataRequestPacket.java +++ /dev/null @@ -1,37 +0,0 @@ -package org.stream.external.connected.amberdata; - -import java.util.HashMap; - -import org.stream.external.connected.connections.AmberDataConnection; - -public class AmberDataRequestPacket { - - private final String key; - private final AmberDataConnection connection; - private final HashMap request; - - private AmberDataRequestPacket(String key, AmberDataConnection connection, HashMap request) { - this.key = key; - this.connection = connection; - this.request = request; - } - - public final String getKey() { - return key; - } - - public final AmberDataConnection getConnection() { - return connection; - } - - public final String getData(String key) { - if(!request.containsKey(key)) - return null; - - return request.get(key); - } - - public static AmberDataRequestPacket create(String key, AmberDataConnection connection, HashMap request) { - return new AmberDataRequestPacket(key, connection, request); - } -} diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/connected/connections/AmberDataConnection.java b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/connected/connections/AmberDataConnection.java deleted file mode 100644 index c292886d..00000000 --- a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/connected/connections/AmberDataConnection.java +++ /dev/null @@ -1,139 +0,0 @@ -package org.stream.external.connected.connections; - -import java.math.BigInteger; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.util.HashMap; -import java.util.HashSet; - -import org.framework.router.ResponseFactory; -import org.json.JSONObject; -import org.stream.external.connected.amberdata.AmberDataRequestHandler; -import org.stream.external.connected.amberdata.AmberDataRequestPacket; -import org.stream.external.handler.ExternalStreamConnection; -import org.stream.external.handler.ExternalStreamManager; - -import okhttp3.OkHttpClient; -import okhttp3.Request; - -public class AmberDataConnection extends ExternalStreamConnection { - - private static final HashSet requestTypes; - -static { - requestTypes = new HashSet<>(); - requestTypes.add("aave-protocol-dated"); - requestTypes.add("aave-asset-dated"); - requestTypes.add("blockchain-addresses-dated"); -} - - private boolean authorized = false; - - public AmberDataConnection(ExternalStreamManager manager, HashMap data) { - super(manager, data); - } - - public String getUUID() { - return "amber_data"; - } - - public void init() { - - } - - public String getHash(HashMap data) { - if(!data.containsKey("key")) { - return null; - } - - try { - MessageDigest md = MessageDigest.getInstance("SHA-512"); - byte[] bytes = md.digest(("salt" + data).getBytes()); - BigInteger signum = new BigInteger(1, bytes); - String hashed = signum.toString(16); - while(hashed.length() < 32) - hashed = "0" + hashed; - return hashed; - } catch (NoSuchAlgorithmException e) { - e.printStackTrace(); - } - System.err.println("Internal Hash Algorithm Failure. Not handled exception, contact external"); - ResponseFactory.responseNotHandled("Internal hash failure: Exception not handled."); - return null; - } - - @Override - public boolean authorize() { - OkHttpClient client = new OkHttpClient(); - - // request using default latest lending exchange protocol - Request request = new Request.Builder() - .url("https://web3api.io/api/v2/market/defi/lending/exchanges/aave/latest") - .get() - .addHeader("Accept", "application/json") - .addHeader("x-api-key", data.get("key")) - .build(); - - try { - okhttp3.Response response = client.newCall(request).execute(); - JSONObject json = new JSONObject(response.body().string()); - - // validate successful request - if(!response.isSuccessful() || json.getInt("status") != 200 - || !json.getString("description").equals("Successful request")) - return false; - - } catch (Exception e) { - e.printStackTrace(); - return false; - } - authorized = true; - return true; - } - - @Override - public boolean isAuthorized() { - return authorized; - } - - @Override - public Object[] subscribe(String data) { - return new Object[] {false, null}; - } - - @Override - public boolean isActive() { - return false; - } - - @Override - public boolean isReady() { - return authorized; - } - - @Override - public boolean start() { - return false; - } - - @Override - public boolean stop() { - return false; - } - - @Override - public boolean containsSubscriptionType(String type) { - // TODO Auto-generated method stub - return false; - } - - @Override - public boolean containsRequestType(String type) { - return requestTypes.contains(type); - } - - @Override - public Object[] request(HashMap request) { - return AmberDataRequestHandler.request(AmberDataRequestPacket.create(data.get("key"), this, request)); - } -} diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/connected/connections/TemplateExternalConnection.java b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/connected/connections/TemplateExternalConnection.java deleted file mode 100644 index 5396f5de..00000000 --- a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/connected/connections/TemplateExternalConnection.java +++ /dev/null @@ -1,98 +0,0 @@ -package org.stream.external.connected.connections; - -import java.util.HashMap; - -import org.stream.external.handler.ExternalStreamConnection; -import org.stream.external.handler.ExternalStreamManager; - -public class TemplateExternalConnection extends ExternalStreamConnection { - - private boolean active = false; - private boolean authorized = false; - private boolean override = false; - - public TemplateExternalConnection(ExternalStreamManager manager, HashMap data) { - super(manager, data); - - if(data.containsKey("override")) - override = true; - } - - public String getUUID() { - return "external_template"; - } - - public String getHash(String data) { - return data; - } - - @Override - public boolean authorize() { - if(this.data.containsKey("key")) - authorized = data.get("key").equals("key") || data.get("key").equals("not_ready"); - return authorized; - } - - @Override - public boolean isAuthorized() { - return authorized; - } - - @Override - public Object[] subscribe(String data) { - if(data.equals("correct")) - return new Object[] {true, "subscribed"}; - else - return new Object[] {false, "Subscription handled irregularly"}; - } - - @Override - public boolean isActive() { - return active; - } - - @Override - public boolean isReady() { - if(override) - return false; - return authorized; - } - - @Override - public boolean start() { - active = true; - return active; - } - - @Override - public boolean stop() { - active = false; - return true; - } - - @Override - public String getHash(HashMap data) { - return data.get("key"); - } - - @Override - public boolean containsSubscriptionType(String type) { - return type.equals("correct") || type.equals("irregular"); - } - - @Override - public boolean containsRequestType(String type) { - return type.equals("valid"); - } - - @Override - public Object[] request(HashMap request) { - if(request.get("key").equals("valid")) { - this.processRequest("valid", "col1", "data1"); - this.processRequest("valid", "col2", "data2"); - return new Object[] {true, ""}; - } - - return new Object[] {false, ""}; - } -} diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/handler/ExternalStreamConnection.java b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/handler/ExternalStreamConnection.java index 5f9af17f..f3bff397 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/handler/ExternalStreamConnection.java +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/handler/ExternalStreamConnection.java @@ -1,65 +1,65 @@ -package org.stream.external.handler; - -import java.util.HashMap; -import java.util.TreeSet; - -import org.framework.interfaces.Hash; -import org.framework.interfaces.UUID; -import org.properties.Config; - -public abstract class ExternalStreamConnection implements UUID, Hash { - - private final String hash; - private final ExternalStreamManager manager; - protected final HashMap data; - protected final TreeSet subscriptionTypes; - protected final TreeSet requestTypes; - - public ExternalStreamConnection(ExternalStreamManager manager, HashMap data) { - this.hash = getHash(data); - this.manager = manager; - this.data = data; - this.subscriptionTypes = new TreeSet(); - this.requestTypes = new TreeSet(); - - if(data != null) { - init(); - } - } - - public void init() {} - - public void processSubscription(String subscription, String data) { - manager.processSubscription(hash, subscription, data); - } - - public void processRequest(String request, String data) { - this.processRequest(request, null, data); - } - - public void processRequest(String request, String date, String data) { - if(date != null) - manager.processRequest(getUUID() + Config.getProperty("app", "general.collection.delim") + request + Config.getProperty("app", "general.collection.delim") + date, data); - - else - manager.processRequest(getUUID() + Config.getProperty("app", "general.collection.delim") + request, data); - } - - public final String getHash() { - return hash; - } - - public abstract boolean authorize(); - public abstract boolean isAuthorized(); - public abstract boolean isReady(); - public abstract boolean isActive(); - - public abstract boolean containsSubscriptionType(String type); - public abstract Object[] subscribe(String data); - - public abstract boolean containsRequestType(String type); - public abstract Object[] request(HashMap request); - - public abstract boolean start(); - public abstract boolean stop(); -} \ No newline at end of file +//package org.stream.external.handler; +// +//import java.util.HashMap; +//import java.util.TreeSet; +// +//import org.framework.interfaces.Hash; +//import org.framework.interfaces.UUID; +//import org.properties.Config; +// +//public abstract class ExternalStreamConnection implements UUID, Hash { +// +// private final String hash; +// private final ExternalStreamManager manager; +// protected final HashMap data; +// protected final TreeSet subscriptionTypes; +// protected final TreeSet requestTypes; +// +// public ExternalStreamConnection(ExternalStreamManager manager, HashMap data) { +// this.hash = getHash(data); +// this.manager = manager; +// this.data = data; +// this.subscriptionTypes = new TreeSet(); +// this.requestTypes = new TreeSet(); +// +// if(data != null) { +// init(); +// } +// } +// +// public void init() {} +// +// public void processSubscription(String subscription, String data) { +// manager.processSubscription(hash, subscription, data); +// } +// +// public void processRequest(String request, String data) { +// this.processRequest(request, null, data); +// } +// +// public void processRequest(String request, String date, String data) { +// if(date != null) +// manager.processRequest(getUUID() + Config.getProperty("app", "general.collection.delim") + request + Config.getProperty("app", "general.collection.delim") + date, data); +// +// else +// manager.processRequest(getUUID() + Config.getProperty("app", "general.collection.delim") + request, data); +// } +// +// public final String getHash() { +// return hash; +// } +// +// public abstract boolean authorize(); +// public abstract boolean isAuthorized(); +// public abstract boolean isReady(); +// public abstract boolean isActive(); +// +// public abstract boolean containsSubscriptionType(String type); +// public abstract Object[] subscribe(String data); +// +// public abstract boolean containsRequestType(String type); +// public abstract Object[] request(HashMap request); +// +// public abstract boolean start(); +// public abstract boolean stop(); +//} \ No newline at end of file diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/handler/ExternalStreamHandler.java b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/handler/ExternalStreamHandler.java index 127cf0d8..cf1053e2 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/handler/ExternalStreamHandler.java +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/handler/ExternalStreamHandler.java @@ -15,183 +15,183 @@ public ExternalStreamHandler() { manager = new ExternalStreamManager(this); } - // source: source of data - public Response processEXSR(Packet packet) { - String validate; - if((validate = packet.validate("source")) != null) - return ResponseFactory.response500("ExternalStreamHandler", validate); - - return ResponseFactory.response200(String.format("%s", manager.containsTemplate(packet.getData("source")))); - } - - // key: key of the given stream - public Response processEXST(Packet packet) { - String validate; - if((validate = packet.validate("key")) != null) - return ResponseFactory.response500("ExternalStreamHandler", validate); - - return ResponseFactory.response200(String.format("%s", manager.containsStream(packet.getData("key")))); - } - - // source: source of the given stream - // auth_data[]: all keys associated with data - public Response processINIT(Packet packet) { - String validate; - if((validate = packet.validate("source")) != null) - return ResponseFactory.response500("ExternalStreamHandler", validate); - - String source = packet.getData("source"); - - // validate data - String tempHash = manager.getHash(source, packet.getData()); - if(tempHash == null) - return ResponseFactory.response430(packet.getData()); - if(manager.containsStream(tempHash)) - return ResponseFactory.response220(tempHash); - - // attempt to add stream - Object[] output = manager.addStream(source, packet.getData()); - boolean success = (Boolean) output[0]; - String hash = (String) output[1]; - - if(!success) - return ResponseFactory.response420(source); - - boolean authorized = manager.authorizeStream(hash); - - // if successful authorize - if(authorized) - return ResponseFactory.response200(String.format("%s", hash)); - - manager.removeStream(hash); - return ResponseFactory.response422(source); - } - - // key: stream key - public Response processIATH(Packet packet) { - String validate; - if((validate = packet.validate("key")) != null) - return ResponseFactory.response500("ExternalStreamHandler", validate); - - if(!manager.containsStream(packet.getData("key"))) - return ResponseFactory.response421(packet.getData("key")); - - return ResponseFactory.response200(String.format("%s", manager.isStreamAuthorized(packet.getData("key")))); - } - - public Response processIATV(Packet packet) { - String validate; - if((validate = packet.validate("key")) != null) - return ResponseFactory.response500("ExternalStreamHandler", validate); - - String key = packet.getData("key"); - if(!manager.containsStream(key)) - return ResponseFactory.response421(key); - - return ResponseFactory.response200(String.format("%s", manager.isStreamActive(key))); - } - - public Response processEXEC(Packet packet) { - String validate; - if((validate = packet.validate("key")) != null) - return ResponseFactory.response500("ExternalStreamHandler", validate); - - String key = packet.getData("key"); - if(!manager.containsStream(key)) - return ResponseFactory.response421(key); - - if(!manager.isStreamReady(key)) - return ResponseFactory.response423(key); - - if(manager.isStreamActive(key)) - return ResponseFactory.response424(key); - - return ResponseFactory.response200(String.format("%s", manager.executeStream(key))); - } - - public Response processKILL(Packet packet) { - String validate; - if((validate = packet.validate("key")) != null) - return ResponseFactory.response500("ExternalStreamHandler", validate); - - String key = packet.getData("key"); - if(!manager.containsStream(key)) - return ResponseFactory.response421(key); - - if(!manager.isStreamActive(key)) - return ResponseFactory.response425(key); - - return ResponseFactory.response200(String.format("%s", manager.killStream(key))); - } - - public Response processSUBS(Packet packet) { - String validate; - if((validate = packet.validate("key", "subscription")) != null) - return ResponseFactory.response500("ExternalStreamHandler", validate); - - String key = packet.getData("key"); - String subscription = packet.getData("subscription"); - - if(!manager.containsStream(key)) - return ResponseFactory.response421(key); - - if(!manager.isStreamActive(key)) - return ResponseFactory.response425(key); - - if(!manager.containsSubscriptionType(key, subscription)) - return ResponseFactory.response426(key, subscription); - - Object[] response = manager.subscribe(key, subscription); - - if((Boolean)response[0]) - return ResponseFactory.response200(String.format("%s", "true")); - - return ResponseFactory.response427(key, response[0].toString(), response[1].toString()); - } - - public Response processRQST(Packet packet) { - String validate; - if((validate = packet.validate("key", "request", "query", "destination")) != null) - return ResponseFactory.response500("ExternalStreamHandler", validate); - - String key = packet.getData("key"); - String request = packet.getData("request"); - - if(!manager.containsStream(key)) - return ResponseFactory.response421(key); - - if(!manager.isStreamReady(key)) - return ResponseFactory.response423(key); - - if(!manager.containsRequestType(key, request)) - return ResponseFactory.response428(key, request); - - Object[] response = manager.request(key, packet.getData()); - - if(response == null) - return ResponseFactory.response501("Response from Manager.request should never be null. " - + "Improper implementation of ExternalStreamConnection."); - - if((Boolean)response[0]) - return ResponseFactory.response200(String.format("%s", (String)response[1])); - - return ResponseFactory.response429(key, request, response[1].toString()); - } - - public Response processTYPE(Packet packet) { - String validate; - if((validate = packet.validate("key")) != null) - return ResponseFactory.response500("ExternalStreamHandler", validate); - - String key = packet.getData("key"); - if(!manager.containsStream(key)) - return ResponseFactory.response421(key); - - String uuid = manager.getStreamType(key); - if(uuid == null) - return ResponseFactory.response501("Stream was removed in different thread mid observation."); - - return ResponseFactory.response200(uuid); - - } +// // source: source of data +// public Response processEXSR(Packet packet) { +// String validate; +// if((validate = packet.validate("source")) != null) +// return ResponseFactory.response500("ExternalStreamHandler", validate); +// +// return ResponseFactory.response200(String.format("%s", manager.containsTemplate(packet.getData("source")))); +// } +// +// // key: key of the given stream +// public Response processEXST(Packet packet) { +// String validate; +// if((validate = packet.validate("key")) != null) +// return ResponseFactory.response500("ExternalStreamHandler", validate); +// +// return ResponseFactory.response200(String.format("%s", manager.containsStream(packet.getData("key")))); +// } +// +// // source: source of the given stream +// // auth_data[]: all keys associated with data +// public Response processINIT(Packet packet) { +// String validate; +// if((validate = packet.validate("source")) != null) +// return ResponseFactory.response500("ExternalStreamHandler", validate); +// +// String source = packet.getData("source"); +// +// // validate data +// String tempHash = manager.getHash(source, packet.getData()); +// if(tempHash == null) +// return ResponseFactory.response430(packet.getData()); +// if(manager.containsStream(tempHash)) +// return ResponseFactory.response220(tempHash); +// +// // attempt to add stream +// Object[] output = manager.addStream(source, packet.getData()); +// boolean success = (Boolean) output[0]; +// String hash = (String) output[1]; +// +// if(!success) +// return ResponseFactory.response420(source); +// +// boolean authorized = manager.authorizeStream(hash); +// +// // if successful authorize +// if(authorized) +// return ResponseFactory.response200(String.format("%s", hash)); +// +// manager.removeStream(hash); +// return ResponseFactory.response422(source); +// } +// +// // key: stream key +// public Response processIATH(Packet packet) { +// String validate; +// if((validate = packet.validate("key")) != null) +// return ResponseFactory.response500("ExternalStreamHandler", validate); +// +// if(!manager.containsStream(packet.getData("key"))) +// return ResponseFactory.response421(packet.getData("key")); +// +// return ResponseFactory.response200(String.format("%s", manager.isStreamAuthorized(packet.getData("key")))); +// } +// +// public Response processIATV(Packet packet) { +// String validate; +// if((validate = packet.validate("key")) != null) +// return ResponseFactory.response500("ExternalStreamHandler", validate); +// +// String key = packet.getData("key"); +// if(!manager.containsStream(key)) +// return ResponseFactory.response421(key); +// +// return ResponseFactory.response200(String.format("%s", manager.isStreamActive(key))); +// } +// +// public Response processEXEC(Packet packet) { +// String validate; +// if((validate = packet.validate("key")) != null) +// return ResponseFactory.response500("ExternalStreamHandler", validate); +// +// String key = packet.getData("key"); +// if(!manager.containsStream(key)) +// return ResponseFactory.response421(key); +// +// if(!manager.isStreamReady(key)) +// return ResponseFactory.response423(key); +// +// if(manager.isStreamActive(key)) +// return ResponseFactory.response424(key); +// +// return ResponseFactory.response200(String.format("%s", manager.executeStream(key))); +// } +// +// public Response processKILL(Packet packet) { +// String validate; +// if((validate = packet.validate("key")) != null) +// return ResponseFactory.response500("ExternalStreamHandler", validate); +// +// String key = packet.getData("key"); +// if(!manager.containsStream(key)) +// return ResponseFactory.response421(key); +// +// if(!manager.isStreamActive(key)) +// return ResponseFactory.response425(key); +// +// return ResponseFactory.response200(String.format("%s", manager.killStream(key))); +// } +// +// public Response processSUBS(Packet packet) { +// String validate; +// if((validate = packet.validate("key", "subscription")) != null) +// return ResponseFactory.response500("ExternalStreamHandler", validate); +// +// String key = packet.getData("key"); +// String subscription = packet.getData("subscription"); +// +// if(!manager.containsStream(key)) +// return ResponseFactory.response421(key); +// +// if(!manager.isStreamActive(key)) +// return ResponseFactory.response425(key); +// +// if(!manager.containsSubscriptionType(key, subscription)) +// return ResponseFactory.response426(key, subscription); +// +// Object[] response = manager.subscribe(key, subscription); +// +// if((Boolean)response[0]) +// return ResponseFactory.response200(String.format("%s", "true")); +// +// return ResponseFactory.response427(key, response[0].toString(), response[1].toString()); +// } +// +// public Response processRQST(Packet packet) { +// String validate; +// if((validate = packet.validate("key", "request", "query", "destination")) != null) +// return ResponseFactory.response500("ExternalStreamHandler", validate); +// +// String key = packet.getData("key"); +// String request = packet.getData("request"); +// +// if(!manager.containsStream(key)) +// return ResponseFactory.response421(key); +// +// if(!manager.isStreamReady(key)) +// return ResponseFactory.response423(key); +// +// if(!manager.containsRequestType(key, request)) +// return ResponseFactory.response428(key, request); +// +// Object[] response = manager.request(key, packet.getData()); +// +// if(response == null) +// return ResponseFactory.response501("Response from Manager.request should never be null. " +// + "Improper implementation of ExternalStreamConnection."); +// +// if((Boolean)response[0]) +// return ResponseFactory.response200(String.format("%s", (String)response[1])); +// +// return ResponseFactory.response429(key, request, response[1].toString()); +// } +// +// public Response processTYPE(Packet packet) { +// String validate; +// if((validate = packet.validate("key")) != null) +// return ResponseFactory.response500("ExternalStreamHandler", validate); +// +// String key = packet.getData("key"); +// if(!manager.containsStream(key)) +// return ResponseFactory.response421(key); +// +// String uuid = manager.getStreamType(key); +// if(uuid == null) +// return ResponseFactory.response501("Stream was removed in different thread mid observation."); +// +// return ResponseFactory.response200(uuid); +// +// } } \ No newline at end of file diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/handler/ExternalStreamManager.java b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/handler/ExternalStreamManager.java index 4d8cd95b..a9d28480 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/handler/ExternalStreamManager.java +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/handler/ExternalStreamManager.java @@ -2,6 +2,8 @@ import java.util.HashMap; +import org.framework.router.Response; +import org.stream.external.requests.ExternalRequestFramework; import org.stream.external.requests.ExternalRequestManager; public class ExternalStreamManager { @@ -9,13 +11,15 @@ public class ExternalStreamManager { private final ExternalStreamHandler handler; private final ExternalRequestManager manager; + private final HashMap collections = new HashMap(); + protected ExternalStreamManager(ExternalStreamHandler handler) { this.handler = handler; this.manager = new ExternalRequestManager(); // initialize all available connections try { - this.manager.initialize(); + this.manager.initialize(this); } catch(Exception e) { e.printStackTrace(); System.exit(1); @@ -23,6 +27,65 @@ protected ExternalStreamManager(ExternalStreamHandler handler) { } protected Object[] request(String type, HashMap data) { + return request(type, data, null, null); + } + + protected Object[] request(String type, HashMap data, String startDate, String endDate) { + // validate that type exists + if(!manager.hasRequestFormat(type)) + return new Object[]{false, String.format("Missing request type <%s>", type)}; + + // parse data into required formatting + // retrieve url path + String[] url_path; + if(!data.containsKey("url_path")) { + url_path = new String[] {}; + } else { + url_path = data.get("url_path").split(","); + } + + // retrieve properties + HashMap properties = new HashMap(); + if(data.containsKey("properties")) { + String[] raw_properties = data.get("properties").split(","); + if(raw_properties.length % 2 != 0) + return new Object[] {false, String.format("Properties must be in pairs.")}; + + for(int i = 0; i < raw_properties.length; i++) + properties.put(raw_properties[i], raw_properties[i + 1]); + } + + // retrieve headers + HashMap headers = new HashMap(); + if(data.containsKey("headers")) { + String[] raw_headers = data.get("headers").split(","); + if(raw_headers.length % 2 != 0) + return new Object[] {false, String.format("Headers must be in pairs.")}; + + for(int i = 0; i < raw_headers.length; i++) + properties.put(raw_headers[i], raw_headers[i + 1]); + } + + // retrieve request framework + ExternalRequestFramework request = manager.getRequestFormat(type); + + // submit response + String response; + if(startDate == null && endDate == null) + response = request.request(url_path, properties, headers); + else + response = request.request(url_path, properties, headers, startDate, endDate); + + if(response != null) + return new Object[] {false, response}; + + return new Object[] {true, ""}; + } + + public void processRequest(String collection, HashMap data) { + if(data == null || data.isEmpty()) + return; + } } diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/requests/ExternalRequestFramework.java b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/requests/ExternalRequestFramework.java index 06ecd699..f5cfb1b2 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/requests/ExternalRequestFramework.java +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/requests/ExternalRequestFramework.java @@ -11,16 +11,21 @@ import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; +import org.stream.external.handler.ExternalStreamManager; import okhttp3.OkHttpClient; import okhttp3.Request; -import okhttp3.Response; import okhttp3.Request.Builder; +import okhttp3.Response; public abstract class ExternalRequestFramework { private final static DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + protected final ExternalStreamManager manager; + + private String collection; + private final String name; private String url; private final String[] url_path; @@ -38,6 +43,7 @@ public abstract class ExternalRequestFramework { // default constructor used for templating public ExternalRequestFramework() { + this.manager = null; this.name = null; this.url = null; this.url_path = null; @@ -54,9 +60,12 @@ public ExternalRequestFramework() { this.date_format = null; } - public ExternalRequestFramework(String name, String url, String[] url_path, HashMap properties, HashMap headers, + public ExternalRequestFramework(ExternalStreamManager manager, String name, String url, String[] url_path, + HashMap properties, HashMap headers, HashMap tags, String[] recursive_location, String recursive_replacement, String[] path, boolean is_dated, String date_location, String date_start_var, String date_end_var, String date_format) { + this.manager = manager; + this.collection = name; this.name = name; this.url = url; this.url_path = url_path; @@ -89,6 +98,10 @@ public final String getName() { return name; } + public final String getCollection() { + return collection; + } + public final String getUrl() { return url; } @@ -264,8 +277,10 @@ public final synchronized String request(String[] url_path, HashMap> templates = new HashMap>(); private static final HashMap requests = new HashMap(); - public void initialize() throws InstantiationException, IllegalAccessException, IllegalArgumentException, InvocationTargetException, NoSuchMethodException, SecurityException, URISyntaxException, IOException { + public void initialize(ExternalStreamManager manager) throws InstantiationException, IllegalAccessException, IllegalArgumentException, InvocationTargetException, NoSuchMethodException, SecurityException, URISyntaxException, IOException { // load all templates Reflections reflection = new Reflections(new ConfigurationBuilder().forPackages(ExternalRequestManager.class.getPackageName())); Set> types = reflection.getSubTypesOf(ExternalRequestFramework.class); @@ -206,10 +207,10 @@ public void initialize() throws InstantiationException, IllegalAccessException, } ExternalRequestManager.requests.put(name, templates.get(recursive_type).getDeclaredConstructor( - String.class, String.class, String[].class, HashMap.class, HashMap.class, + ExternalStreamManager.class, String.class, String.class, String[].class, HashMap.class, HashMap.class, HashMap.class, String[].class, String.class, String[].class, boolean.class, String.class, String.class, String.class, String.class).newInstance( - name, url, url_path, properties, headers, tags, recursive_location, recursive_replacement, path, + manager, name, url, url_path, properties, headers, tags, recursive_location, recursive_replacement, path, is_dated, date_location, date_start_var, date_end_var, date_format)); System.out.println(String.format("Successfully added type [%s]", name)); diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/requests/ExternalRequestREST.java b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/requests/ExternalRequestREST.java new file mode 100644 index 00000000..618d562f --- /dev/null +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/requests/ExternalRequestREST.java @@ -0,0 +1,184 @@ +package org.stream.external.requests; + +import java.util.HashMap; + +import org.json.JSONArray; +import org.json.JSONObject; +import org.stream.external.handler.ExternalStreamManager; + +public class ExternalRequestREST extends ExternalRequestFramework { + + public ExternalRequestREST() { + super(); + } + + public ExternalRequestREST(ExternalStreamManager manager, String name, String url, String[] url_path, HashMap properties, HashMap headers, + HashMap tags, String[] recursive_location, String recursive_replacement, String[] path, + boolean is_dated, String date_location, String date_start_var, String date_end_var, String date_format) { + + super(manager, name, url, url_path, properties, headers, tags, recursive_location, recursive_replacement, path, + is_dated, date_location, date_start_var, date_end_var, date_format); + } + + @Override + public String getType() { + return "rest"; + } + + public String handle(String json, HashMap properties, HashMap headers) { + // parse json formatting + JSONObject obj = new JSONObject(json); + + // validate all required parameters are present: + // check for required tag -l + if(!hasTag("-l")) { + System.err.println(String.format("Missing required recursive parameter <-l>")); + return "Missing required recursive parameter <-l>"; + } + + // check for required tag -t + if(!hasTag("-t")) { + System.err.println(String.format("Missing required recursive parameter <-t>")); + return "Missing required recursive parameter <-t>"; + } + + // check for -l being an integer + int limit = 0; + try { + limit = Integer.parseInt(getTag("-l")); + } catch(Exception e) { + e.printStackTrace(); + System.err.println(String.format("Value following <-l> must be an integer.")); + return "Value following <-l> must be an integer."; + } + + // validate that recursion location is valid + // should exist within json if url or static + // should exist within properties if type incremental + String recursive_parameter = null; + if(getTag("-t").equals("url") || getTag("-t").equals("static")) { + JSONObject base = obj; + String[] location = getRecursiveLocation(); + for(int i = 0; i < location.length - 1; i++) { + if(base.has(location[i])) { + base = base.getJSONObject(location[i]); + } else { + System.err.println("Response does not contain proper recursive parameter location at: " + location[i]); + return "Response does not contain proper recursive parameter location at: " + location[i]; + } + } + + recursive_parameter = base.get(location[location.length - 1]).toString(); + } + + else if(getTag("-t").equals("incremental")) { + if(!properties.containsKey(getRecursiveLocation()[0])) { + System.err.println("Properties do not contain incremental parameter listed in: " + getRecursiveLocation()[0]); + return "Properties do not contain incremental parameter listed in: " + getRecursiveLocation()[0]; + } + + recursive_parameter = properties.get(getRecursiveLocation()[0]); + } + + // validate that recursive parameter has been set + if(recursive_parameter == null) { + System.err.println("Fatal error. Recursive parameter is null after successful initialization."); + return "Fatal error. Recursive parameter is null after successful initialization."; + } + + + // validate that the base has the proper obj path + String[] path = getPath(); + JSONArray data = null; + for(int i = 0; i < path.length; i++) { + if(i == path.length - 1) { + if(obj.has(path[i])) { + try { + data = obj.getJSONArray(path[i]); + } catch(Exception e) { + System.err.println("obj path type is not of type . Cannot parse."); + return "obj path type is not of type . Cannot parse."; + } + } + } + + else if(obj.has(path[i])) { + try { + obj = obj.getJSONObject(path[i]); + } catch(Exception e) { + System.err.println("obj path type step is not of type . Cannot parse."); + return "obj path type step is not of type . Cannot parse."; + } + } + } + + // validate that data is non-empty + if(data == null) { + System.err.println("Data array retrieval had fatal error, killing process."); + return "Data array retrieval had fatal error, killing process."; + } + + // extract and print data + for(int i = 0; i < data.length(); i++) { + HashMap point = parse(data.getJSONObject(i)); + manager.processRequest(getCollection(), point); + } + + // initiate recursive call + // if under limit requested, terminate call + if(data.length() < limit) + return null; + + // extract recursive parameter and apply to next call + String rec_type = getTag("-t"); + String url = getUrl(); + switch(rec_type) { + + // type url: + // - clear properties and set base to extracted url + // - execute + case "url": + properties.clear(); + url = recursive_parameter; + return processUrl(url, headers); + + // type incremental: + // - assert that parameter is an integer + // - increment parameter and update + // - execute + case "incremental": + int param = -1; + try { + param = Integer.parseInt(recursive_parameter); + } catch(Exception e) { + e.printStackTrace(); + System.err.println(String.format("Recursive parameter <%s> is not of type integer.", recursive_parameter)); + return String.format("Recursive parameter <%s> is not of type integer.", recursive_parameter); + } + + if(param == -1) { + System.err.println("Fatal parsing error occured."); + return "Fatal parsing error occured."; + } + + param += 1; + properties.put(getRecursiveLocation()[0], "" + param); + break; + + // type static: + // - check to see if replacement location exists + // - if so update replacement + // - execute + case "static": + if(getRecursiveReplacement() != null) { + properties.put(getRecursiveReplacement(), recursive_parameter); + } else { + properties.put(getRecursiveLocation()[0], recursive_parameter); + } + break; + } + + // increment param, replace, and execute + return process(url, properties, headers); + } +} \ No newline at end of file diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/amberdata-aave-governance.properties b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/amberdata-aave-governance.properties index 394e20c3..a8d7136a 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/amberdata-aave-governance.properties +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/amberdata-aave-governance.properties @@ -10,7 +10,7 @@ url.headers= accept,application/json,\ data.path= payload,\ data -recursion.type= parameterized +recursion.type= rest recursion.tags= -l,900,\ -t,url diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/amberdata-aave-protocol.properties b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/amberdata-aave-protocol.properties index cf803956..2d608648 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/amberdata-aave-protocol.properties +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/amberdata-aave-protocol.properties @@ -11,7 +11,7 @@ url.headers= accept,application/json,\ data.path= payload,\ data -recursion.type= parameterized +recursion.type= rest recursion.tags= -l,900,\ -t,url diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/amberdata-blockchain-addresses.properties b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/amberdata-blockchain-addresses.properties index 823fad7a..fe33840f 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/amberdata-blockchain-addresses.properties +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/amberdata-blockchain-addresses.properties @@ -12,7 +12,7 @@ url.headers= accept,application/json,\ data.path= payload,\ records -recursion.type= parameterized +recursion.type= rest recursion.tags= -l,900,\ -t,incremental diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/amberdata-compound-protocol.properties b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/amberdata-compound-protocol.properties index c7a78364..fb15c727 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/amberdata-compound-protocol.properties +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/amberdata-compound-protocol.properties @@ -10,7 +10,7 @@ url.headers= accept,application/json,\ data.path= payload,\ data -recursion.type= parameterized +recursion.type= rest recursion.tags= -l,900,\ -t,url diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/amberdata-makerdao-asset.properties b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/amberdata-makerdao-asset.properties index 385c25c4..4df62c4e 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/amberdata-makerdao-asset.properties +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/amberdata-makerdao-asset.properties @@ -13,7 +13,7 @@ url.headers= accept,application/json,\ data.path= payload,\ data -recursion.type= parameterized +recursion.type= rest recursion.tags= -l,900,\ -t,url diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/amberdata-makerdao-protocol.properties b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/amberdata-makerdao-protocol.properties index 37d841e4..eea45002 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/amberdata-makerdao-protocol.properties +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/amberdata-makerdao-protocol.properties @@ -11,7 +11,7 @@ url.headers= accept,application/json,\ data.path= payload,\ data -recursion.type= parameterized +recursion.type= rest recursion.tags= -l,900,\ -t,url diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/amberdata-makerdao-wallet.properties b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/amberdata-makerdao-wallet.properties index 0669c075..6f7999fa 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/amberdata-makerdao-wallet.properties +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/amberdata-makerdao-wallet.properties @@ -12,7 +12,7 @@ url.headers= accept,application/json,\ data.path= payload,\ data -recursion.type= parameterized +recursion.type= rest recursion.tags= -l,900,\ -t,url diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/amberdata-uniswap-pool.properties b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/amberdata-uniswap-pool.properties index 26c80cf1..ee6d84a6 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/amberdata-uniswap-pool.properties +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/amberdata-uniswap-pool.properties @@ -12,7 +12,7 @@ url.headers= accept,application/json,\ data.path= payload,\ data -recursion.type= parameterized +recursion.type= rest recursion.tags= -l,900,\ -t,url From 40fb3989e53f57413e6dcd950b338f21decbb789 Mon Sep 17 00:00:00 2001 From: Conor Flynn Date: Mon, 13 Mar 2023 13:00:57 -0400 Subject: [PATCH 2/3] Integrate ESH-EXSR --- .../src/main/java/org/core/engine/Engine.java | 7 ++-- .../src/main/java/org/properties/Config.java | 12 +++---- .../handler/ExternalStreamHandler.java | 35 +++++++++++++++++++ .../handler/ExternalStreamManager.java | 6 +++- .../resources/requests/template.properties | 2 +- .../connections/TestAmberDataConnection.java | 1 - .../requests/TestExternalRequestManager.java | 25 +++++++++++++ 7 files changed, 76 insertions(+), 12 deletions(-) create mode 100644 DeFi-Data-Engine/DeFi Data Engine/src/test/java/test/stream/external/requests/TestExternalRequestManager.java diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/core/engine/Engine.java b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/core/engine/Engine.java index d5315ff0..27996b03 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/core/engine/Engine.java +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/core/engine/Engine.java @@ -15,9 +15,10 @@ public Engine() { // source: source of the local stream to initialize public Response processSTRT(Packet packet) { // start output processes: - Response out_response = send("OUT", "STRT"); - if(out_response.code() != 200) - return out_response; + //TODO: RE-ENABLE +// Response out_response = send("OUT", "STRT"); +// if(out_response.code() != 200) +// return out_response; // start local stream handler processes: String lsh_type = Config.getProperty("stream", "local.stream.type"); diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/properties/Config.java b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/properties/Config.java index fba63efd..9ac97dc8 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/properties/Config.java +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/properties/Config.java @@ -23,15 +23,15 @@ public class Config { Properties stream_properties = new Properties(); stream_properties.put("general.consumer.types", "socket_consumer"); stream_properties.put("general.producer.types", "socket_producer"); - stream_properties.put("rest.socket.address", "DataEngine"); - //stream_properties.put("rest.socket.address", "localhost"); + //stream_properties.put("rest.socket.address", "DataEngine"); + stream_properties.put("rest.socket.address", "localhost"); stream_properties.put("rest.socket.port", "61100"); stream_properties.put("rest.socket.key", "rest-key-reserved"); - // stream_properties.put("output.socket.address", "defi-de.idea.rpi.edu"); - stream_properties.put("output.socket.address", "RestApp"); - // stream_properties.put("output.socket.address", "localhost"); + //stream_properties.put("output.socket.address", "RestApp"); + stream_properties.put("output.socket.address", "localhost"); stream_properties.put("output.socket.port", "61200"); - stream_properties.put("local.stream.type", "mongo_db"); + //stream_properties.put("local.stream.type", "mongo_db"); + stream_properties.put("local.stream.type", "null"); stream_properties.put("mongodb.properties.uri", "mongodb://MONGO:27017"); stream_properties.put("mongodb.database.state", "main-state-db"); stream_properties.put("mongodb.database.main", "main-db"); diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/handler/ExternalStreamHandler.java b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/handler/ExternalStreamHandler.java index cf1053e2..763ea1c5 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/handler/ExternalStreamHandler.java +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/handler/ExternalStreamHandler.java @@ -15,6 +15,41 @@ public ExternalStreamHandler() { manager = new ExternalStreamManager(this); } + // type: type of data + public Response processEXSR(Packet packet) { + String validate; + if((validate = packet.validate("type")) != null) + return ResponseFactory.response500("ExternalStreamHandler", validate); + + return ResponseFactory.response200(String.format("%s", manager.containsType(packet.getData("type")))); + } + + // type: type of data + // url_path (opt): path of the url + // properties (opt):properties required for call + // headers (opt): headers required for call + public Response processRQST(Packet packet) { + String validate; + if((validate = packet.validate("type")) != null) + return ResponseFactory.response500("ExternalStreamHandler", validate); + + Object[] response; + + // check to see if dated + // if not + if((validate = packet.validate("startDate", "endDate")) != null) + response = manager.request(packet.getData("type"), packet.getData()); + + // if dated + else + response = manager.request(packet.getData("type"), packet.getData(), packet.getData("startDate"), packet.getData("endDate")); + + // check to see if valid + if(!((boolean)response[0])) { + + } + } + // // source: source of data // public Response processEXSR(Packet packet) { // String validate; diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/handler/ExternalStreamManager.java b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/handler/ExternalStreamManager.java index a9d28480..b56c7ecf 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/handler/ExternalStreamManager.java +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/handler/ExternalStreamManager.java @@ -26,6 +26,10 @@ protected ExternalStreamManager(ExternalStreamHandler handler) { } } + public boolean containsType(String type) { + return manager.hasRequestFormat(type); + } + protected Object[] request(String type, HashMap data) { return request(type, data, null, null); } @@ -86,6 +90,6 @@ public void processRequest(String collection, HashMap data) { if(data == null || data.isEmpty()) return; - + // TODO: FINISH } } diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/template.properties b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/template.properties index fcd8b2dc..8cd4bd82 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/template.properties +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/requests/template.properties @@ -57,7 +57,7 @@ data.path= response,\ # which have specific properties and handlers. Please review documentation to get a full # list of these tags. To default with no recursive call, set this property to . # This property we will set to for a clearer example. -recursion.type= parameterized +recursion.type= rest # [OPTIONAL]: # - url.recursion.type = static diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/test/java/test/stream/external/connected/connections/TestAmberDataConnection.java b/DeFi-Data-Engine/DeFi Data Engine/src/test/java/test/stream/external/connected/connections/TestAmberDataConnection.java index b0c41ae4..536182a5 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/src/test/java/test/stream/external/connected/connections/TestAmberDataConnection.java +++ b/DeFi-Data-Engine/DeFi Data Engine/src/test/java/test/stream/external/connected/connections/TestAmberDataConnection.java @@ -13,7 +13,6 @@ import org.framework.router.Response; import org.junit.BeforeClass; import org.junit.Test; -import org.stream.external.connected.amberdata.AmberDataRequestHandler; public class TestAmberDataConnection { diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/test/java/test/stream/external/requests/TestExternalRequestManager.java b/DeFi-Data-Engine/DeFi Data Engine/src/test/java/test/stream/external/requests/TestExternalRequestManager.java new file mode 100644 index 00000000..50985f64 --- /dev/null +++ b/DeFi-Data-Engine/DeFi Data Engine/src/test/java/test/stream/external/requests/TestExternalRequestManager.java @@ -0,0 +1,25 @@ +package test.stream.external.requests; + +import static org.junit.Assert.assertEquals; + +import java.util.HashMap; + +import org.core.core.Core; +import org.junit.Test; + +public class TestExternalRequestManager { + + @Test + public void TestEXSR() { + Core core = new Core(); + + HashMap data = new HashMap(); + data.put("type", "amberdata-aave-protocol"); + + assertEquals("true", core.send("ESH", "EXSR", data).data()); + + data.put("type", "does-not-exist"); + + assertEquals("false", core.send("ESH", "EXSR", data).data()); + } +} From 61b628241d31bdeebf236734e1f2b1bb5a87e89e Mon Sep 17 00:00:00 2001 From: Conor Flynn Date: Mon, 13 Mar 2023 20:56:27 -0400 Subject: [PATCH 3/3] add ESH-RQST --- .../org/framework/router/ResponseFactory.java | 4 +- .../handler/ExternalStreamConnection.java | 65 ------------------- .../handler/ExternalStreamHandler.java | 9 ++- .../handler/ExternalStreamManager.java | 7 +- .../requests/TestExternalRequestManager.java | 22 ++++++- 5 files changed, 34 insertions(+), 73 deletions(-) delete mode 100644 DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/handler/ExternalStreamConnection.java diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/framework/router/ResponseFactory.java b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/framework/router/ResponseFactory.java index 0bbc2991..d90fc193 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/framework/router/ResponseFactory.java +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/framework/router/ResponseFactory.java @@ -85,8 +85,8 @@ public static Response response426(String hash, String subscription) { return Response.create(426, String.format("Stream with hash <%s> does not contain a subscription request of type <%s>.", hash, subscription)); } - public static Response response427(String hash, String subscription, String response) { - return Response.create(427, String.format("Stream with hash <%s> returned an irregular response when attempting to subscribe to <%s>. Response returned is: <%s>", hash, subscription, response)); + public static Response response427(String type, String response) { + return Response.create(427, String.format("Stream of type <%s> returned an irregular response when attempting to send request. Response returned is: <%s>", type, response)); } public static Response response428(String hash, String request) { diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/handler/ExternalStreamConnection.java b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/handler/ExternalStreamConnection.java deleted file mode 100644 index f3bff397..00000000 --- a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/handler/ExternalStreamConnection.java +++ /dev/null @@ -1,65 +0,0 @@ -//package org.stream.external.handler; -// -//import java.util.HashMap; -//import java.util.TreeSet; -// -//import org.framework.interfaces.Hash; -//import org.framework.interfaces.UUID; -//import org.properties.Config; -// -//public abstract class ExternalStreamConnection implements UUID, Hash { -// -// private final String hash; -// private final ExternalStreamManager manager; -// protected final HashMap data; -// protected final TreeSet subscriptionTypes; -// protected final TreeSet requestTypes; -// -// public ExternalStreamConnection(ExternalStreamManager manager, HashMap data) { -// this.hash = getHash(data); -// this.manager = manager; -// this.data = data; -// this.subscriptionTypes = new TreeSet(); -// this.requestTypes = new TreeSet(); -// -// if(data != null) { -// init(); -// } -// } -// -// public void init() {} -// -// public void processSubscription(String subscription, String data) { -// manager.processSubscription(hash, subscription, data); -// } -// -// public void processRequest(String request, String data) { -// this.processRequest(request, null, data); -// } -// -// public void processRequest(String request, String date, String data) { -// if(date != null) -// manager.processRequest(getUUID() + Config.getProperty("app", "general.collection.delim") + request + Config.getProperty("app", "general.collection.delim") + date, data); -// -// else -// manager.processRequest(getUUID() + Config.getProperty("app", "general.collection.delim") + request, data); -// } -// -// public final String getHash() { -// return hash; -// } -// -// public abstract boolean authorize(); -// public abstract boolean isAuthorized(); -// public abstract boolean isReady(); -// public abstract boolean isActive(); -// -// public abstract boolean containsSubscriptionType(String type); -// public abstract Object[] subscribe(String data); -// -// public abstract boolean containsRequestType(String type); -// public abstract Object[] request(HashMap request); -// -// public abstract boolean start(); -// public abstract boolean stop(); -//} \ No newline at end of file diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/handler/ExternalStreamHandler.java b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/handler/ExternalStreamHandler.java index 763ea1c5..037eb634 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/handler/ExternalStreamHandler.java +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/handler/ExternalStreamHandler.java @@ -33,6 +33,10 @@ public Response processRQST(Packet packet) { if((validate = packet.validate("type")) != null) return ResponseFactory.response500("ExternalStreamHandler", validate); + // validate type exists + if(!manager.containsType(packet.getData("type"))) + return ResponseFactory.response420(packet.getData("type")); + Object[] response; // check to see if dated @@ -46,8 +50,11 @@ public Response processRQST(Packet packet) { // check to see if valid if(!((boolean)response[0])) { - + return ResponseFactory.response427(packet.getData("type"), response[1].toString()); } + + // return valid + return ResponseFactory.response200(); } // // source: source of data diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/handler/ExternalStreamManager.java b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/handler/ExternalStreamManager.java index b56c7ecf..af48a451 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/handler/ExternalStreamManager.java +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/handler/ExternalStreamManager.java @@ -55,7 +55,7 @@ protected Object[] request(String type, HashMap data, String sta if(raw_properties.length % 2 != 0) return new Object[] {false, String.format("Properties must be in pairs.")}; - for(int i = 0; i < raw_properties.length; i++) + for(int i = 0; i < raw_properties.length; i+=2) properties.put(raw_properties[i], raw_properties[i + 1]); } @@ -66,8 +66,8 @@ protected Object[] request(String type, HashMap data, String sta if(raw_headers.length % 2 != 0) return new Object[] {false, String.format("Headers must be in pairs.")}; - for(int i = 0; i < raw_headers.length; i++) - properties.put(raw_headers[i], raw_headers[i + 1]); + for(int i = 0; i < raw_headers.length; i+=2) + headers.put(raw_headers[i], raw_headers[i + 1]); } // retrieve request framework @@ -90,6 +90,7 @@ public void processRequest(String collection, HashMap data) { if(data == null || data.isEmpty()) return; + System.out.println(collection + ": " + data.toString()); // TODO: FINISH } } diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/test/java/test/stream/external/requests/TestExternalRequestManager.java b/DeFi-Data-Engine/DeFi Data Engine/src/test/java/test/stream/external/requests/TestExternalRequestManager.java index 50985f64..8c0b50cd 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/src/test/java/test/stream/external/requests/TestExternalRequestManager.java +++ b/DeFi-Data-Engine/DeFi Data Engine/src/test/java/test/stream/external/requests/TestExternalRequestManager.java @@ -5,14 +5,20 @@ import java.util.HashMap; import org.core.core.Core; +import org.junit.BeforeClass; import org.junit.Test; public class TestExternalRequestManager { + public static Core core; + + @BeforeClass + public static void init() { + core = new Core(); + } + @Test public void TestEXSR() { - Core core = new Core(); - HashMap data = new HashMap(); data.put("type", "amberdata-aave-protocol"); @@ -22,4 +28,16 @@ public void TestEXSR() { assertEquals("false", core.send("ESH", "EXSR", data).data()); } + + @Test + public void TestRQST() { + HashMap data = new HashMap(); + data.put("type", "amberdata-uniswap-pool"); + data.put("url_path", "poolAddress,0xb4e16d0168e52d35cacd2c6185b44281ec28c9dc"); + data.put("headers", "x-api-key,UAK7ed69235426c360be22bfc2bde1809b6"); + data.put("startDate", "2022-09-01"); + data.put("endDate", "2022-09-02"); + + assertEquals(200, core.send("ESH", "RQST", data).code()); + } }