diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/properties/Config.java b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/properties/Config.java index 0ddad2b8..fba63efd 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/properties/Config.java +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/properties/Config.java @@ -24,10 +24,12 @@ public class Config { stream_properties.put("general.consumer.types", "socket_consumer"); stream_properties.put("general.producer.types", "socket_producer"); stream_properties.put("rest.socket.address", "DataEngine"); + //stream_properties.put("rest.socket.address", "localhost"); stream_properties.put("rest.socket.port", "61100"); stream_properties.put("rest.socket.key", "rest-key-reserved"); // stream_properties.put("output.socket.address", "defi-de.idea.rpi.edu"); stream_properties.put("output.socket.address", "RestApp"); + // stream_properties.put("output.socket.address", "localhost"); stream_properties.put("output.socket.port", "61200"); stream_properties.put("local.stream.type", "mongo_db"); stream_properties.put("mongodb.properties.uri", "mongodb://MONGO:27017"); diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/connected/amberdata/AmberDataRequestHandler.java b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/connected/amberdata/AmberDataRequestHandler.java index b9ec167e..e7aca6bf 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/connected/amberdata/AmberDataRequestHandler.java +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/connected/amberdata/AmberDataRequestHandler.java @@ -36,6 +36,7 @@ public class AmberDataRequestHandler { requests.put("lending-latest", classobj.getMethod("requestLendingLatest", AmberDataRequestPacket.class)); requests.put("aave-protocol-dated", classobj.getMethod("requestAaveProtocolDated", AmberDataRequestPacket.class)); requests.put("aave-asset-dated", classobj.getMethod("requestAaveAssetDated", AmberDataRequestPacket.class)); + requests.put("blockchain-addresses-dated", classobj.getMethod("requestBlockchainAddresses", AmberDataRequestPacket.class)); } catch (Exception e) { e.printStackTrace(); ResponseFactory.responseNotHandled("Irregular method call for AmberDataRequestHandler."); @@ -311,6 +312,74 @@ private static void processAaveJsonRequest(AmberDataRequestPacket packet, JSONOb } } + public static Object[] requestBlockchainAddresses(AmberDataRequestPacket packet) { + return processBlockchainAddresses(packet, 0); + } + + private static Object[] processBlockchainAddresses(AmberDataRequestPacket packet, int page) { + OkHttpClient client = new OkHttpClient(); + + LocalDate next = LocalDate.parse(packet.getData("date"), formatter); + next = next.plusDays(1); + String tmr = next.format(formatter); + + String url = String.format("https://web3api.io/api/v2/defi/lending/aavev2/assets/%s?startDate=%s&endDate=%s&page=%s", + packet.getData("asset"), + packet.getData("date") + "T01:00:00", + tmr + "T01:00:00", + page); + + Request request = new Request.Builder() + .url(url + "&size=" + request_size) + .get() + .addHeader("x-amberdata-blockchain-id", "ethereum-mainnet") + .addHeader("accept", "application/json") + .addHeader("x-api-key", "UAK7ed69235426c360be22bfc2bde1809b6") + .build(); + + okhttp3.Response response; + try { + response = client.newCall(request).execute(); + JSONObject json = new JSONObject(response.body().string()); + + if(json.toString().equals("") || !json.has("description")) + return new Object[] {false, "JSON Object returned empty or invalid contents."}; + + if(!response.isSuccessful() || json.getInt("status") != 200 + || !json.getString("description").equals("Successful request")) { + System.out.println(json.getString("message")); + return new Object[]{false, json.getString("description")}; + } + + if(!json.has("payload") || !json.getJSONObject("payload").has("records")) + return new Object[] {false, "Malformed Aave packet"}; + + JSONObject payload = json.getJSONObject("payload"); + JSONArray arr = payload.getJSONArray("records"); + for(int i = 0; i < arr.length(); i++) { + JSONObject obj = arr.getJSONObject(i); + packet.getConnection().processRequest( + packet.getData("request"), + packet.getData("date"), + format("creator", obj.getString("creator"), + "firstBlockNumber", obj.getString("firstBlockNumber"), + "firstTransactionHash", obj.getString("firstTransactionHash"), + "address", obj.getJSONObject("hash").getString("address"), + "timestamp", obj.get("timestamp").toString(), + "type", obj.getString("type"))); + } + + if(arr.length() >= request_size) { + return processBlockchainAddresses(packet, page + 1); + } else { + return new Object[] {true, ""}; + } + } catch (IOException e) { + e.printStackTrace(); + return new Object[] {false, "Exception."}; + } + } + private static String format(Object... data) { StringBuilder out = new StringBuilder(); for(int i = 0; i < data.length; i++) { diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/connected/connections/AmberDataConnection.java b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/connected/connections/AmberDataConnection.java index 96f7327d..c292886d 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/connected/connections/AmberDataConnection.java +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/connected/connections/AmberDataConnection.java @@ -24,8 +24,7 @@ public class AmberDataConnection extends ExternalStreamConnection { requestTypes = new HashSet<>(); requestTypes.add("aave-protocol-dated"); requestTypes.add("aave-asset-dated"); - //requestTypes.add("aave-wallet-dated"); - //requestTypes.add("aave-governance-dated"); + requestTypes.add("blockchain-addresses-dated"); } private boolean authorized = false; diff --git a/DeFi-Data-Engine/Rest Application/src/main/java/org/properties/Config.java b/DeFi-Data-Engine/Rest Application/src/main/java/org/properties/Config.java index 91ec6a33..a5e4ee12 100644 --- a/DeFi-Data-Engine/Rest Application/src/main/java/org/properties/Config.java +++ b/DeFi-Data-Engine/Rest Application/src/main/java/org/properties/Config.java @@ -20,7 +20,9 @@ public class Config { app_properties.put("spring.server.port", "8080"); app_properties.put("spring.server.address", "RestApp"); // app_properties.put("spring.server.address", "defi-de.idea.rpi.edu"); + // app_properties.put("spring.server.address", "localhost"); app_properties.put("rest.socket.address", "DataEngine"); + // app_properties.put("rest.socket.address", "localhost"); app_properties.put("rest.socket.port", "61100"); app_properties.put("rest.socket.key", "rest-key-reserved"); properties.put("app", app_properties); diff --git a/DeFi-Data-Engine/Testing Environment/src/test/connection/socket/LocalTest.java b/DeFi-Data-Engine/Testing Environment/src/test/connection/socket/LocalTest.java index 8b1558eb..4e766a85 100644 --- a/DeFi-Data-Engine/Testing Environment/src/test/connection/socket/LocalTest.java +++ b/DeFi-Data-Engine/Testing Environment/src/test/connection/socket/LocalTest.java @@ -16,7 +16,7 @@ public class LocalTest { //private static final String host = "localhost"; - private static final String host = "defi-de.idea.rpi.edu"; + private static final String host = "localhost"; public static void main(String[] args) throws UnknownHostException, IOException { final Socket socket = SocketFactory.getDefault().createSocket(host, 61200); @@ -39,8 +39,8 @@ public static void main(String[] args) throws UnknownHostException, IOException + "key&&&%s&&&" + "start_date&&&2022-08-01&&&" + "end_date&&&2022-09-01&&&" - + "query&&&aave-protocol-dated&&&" - + "request&&&aave-protocol-dated\n", + + "query&&&blockchain-addresses-dated&&&" + + "request&&&blockchain-addresses-dated\n", destination, key)); diff --git a/DeFi-Data-Engine/Testing Environment/src/test/misc/Testing1.java b/DeFi-Data-Engine/Testing Environment/src/test/misc/Testing1.java index 0d3e9f68..2823b092 100644 --- a/DeFi-Data-Engine/Testing Environment/src/test/misc/Testing1.java +++ b/DeFi-Data-Engine/Testing Environment/src/test/misc/Testing1.java @@ -10,47 +10,66 @@ public class Testing1 { + private static final int request_size = 999; + public static void main(String args[]) { - String url = String.format("https://web3api.io/api/v2/defi/lending/aavev2/protocol?startDate=%s&endDate=%s", + String url = String.format("https://web3api.io/api/v2/addresses?startDate=%s&endDate=%s", "2022-09-01T01:00:00", - "2022-09-02T01:00:00" - + "&size=999"); + "2022-09-02T01:00:00"); - recursiveCall(url); + Object[] obj = recursiveCall(url, 0); + if(!(boolean)obj[0]) + System.out.println(obj[1]); System.out.println("Exit"); System.exit(0); } - public static void recursiveCall(String url) { - System.out.println(url); - OkHttpClient client = new OkHttpClient(); + public static Object[] recursiveCall(String base_url, int page) { + System.out.println(page); + OkHttpClient client = new OkHttpClient(); + String url = base_url + "&page=" + page; + Request request = new Request.Builder() - .url(url) + .url(url + "&size=" + request_size) .get() + .addHeader("x-amberdata-blockchain-id", "ethereum-mainnet") .addHeader("accept", "application/json") .addHeader("x-api-key", "UAK7ed69235426c360be22bfc2bde1809b6") .build(); - + okhttp3.Response response; try { response = client.newCall(request).execute(); JSONObject json = new JSONObject(response.body().string()); - JSONObject payload = json.getJSONObject("payload"); - JSONArray arr = payload.getJSONArray("data"); - for(int i = 0; i < arr.length(); i++) { - System.out.println(arr.get(i).toString()); + if(json.toString().equals("") || !json.has("description")) + return new Object[] {false, "JSON Object returned empty or invalid contents."}; + + if(!response.isSuccessful() || json.getInt("status") != 200 + || !json.getString("description").equals("Successful request")) { + System.out.println(json.getString("message")); + return new Object[]{false, json.getString("description")}; } - System.out.println(arr.length()); - if(arr.length() >= 999) { - JSONObject metadata = payload.getJSONObject("metadata"); - if(metadata.has("next")) - recursiveCall(metadata.getString("next")+"&size=999"); + + if(!json.has("payload") || !json.getJSONObject("payload").has("records")) + return new Object[] {false, "Malformed Aave packet"}; + + JSONObject payload = json.getJSONObject("payload"); + JSONArray arr = payload.getJSONArray("records"); +// for(int i = 0; i < arr.length(); i++) { +// System.out.println(arr.getJSONObject(i).getJSONObject("hash").getString("address")); +// } + + if(arr.length() >= request_size) { + return recursiveCall(base_url, page + 1); + } else { + return new Object[] {true, ""}; } } catch (IOException e) { e.printStackTrace(); + return new Object[] {false, "Exception."}; } } }