Integrate Blockchain Addresses

DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/properties/Config.java

@@ -24,10 +24,12 @@ public class Config {
 	stream_properties.put("general.consumer.types", "socket_consumer");
 	stream_properties.put("general.producer.types", "socket_producer");
 	stream_properties.put("rest.socket.address", "DataEngine");
+	//stream_properties.put("rest.socket.address", "localhost");
 	stream_properties.put("rest.socket.port", "61100");
 	stream_properties.put("rest.socket.key", "rest-key-reserved");
 	// stream_properties.put("output.socket.address", "defi-de.idea.rpi.edu");
 	stream_properties.put("output.socket.address", "RestApp");
+	// stream_properties.put("output.socket.address", "localhost");
 	stream_properties.put("output.socket.port", "61200");
 	stream_properties.put("local.stream.type", "mongo_db");
 	stream_properties.put("mongodb.properties.uri", "mongodb://MONGO:27017");

DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/connected/amberdata/AmberDataRequestHandler.java

@@ -36,6 +36,7 @@ public class AmberDataRequestHandler {
 		requests.put("lending-latest", classobj.getMethod("requestLendingLatest", AmberDataRequestPacket.class));
 		requests.put("aave-protocol-dated", classobj.getMethod("requestAaveProtocolDated", AmberDataRequestPacket.class));
 		requests.put("aave-asset-dated", classobj.getMethod("requestAaveAssetDated", AmberDataRequestPacket.class));
+		requests.put("blockchain-addresses-dated", classobj.getMethod("requestBlockchainAddresses", AmberDataRequestPacket.class));
 	} catch (Exception e) {
 		e.printStackTrace();
 		ResponseFactory.responseNotHandled("Irregular method call for AmberDataRequestHandler.");
@@ -311,6 +312,74 @@ private static void processAaveJsonRequest(AmberDataRequestPacket packet, JSONOb
 		}
 	}
 
+	public static Object[] requestBlockchainAddresses(AmberDataRequestPacket packet) {
+		return processBlockchainAddresses(packet, 0);
+	}
+
+	private static Object[] processBlockchainAddresses(AmberDataRequestPacket packet, int page) {
+		OkHttpClient client = new OkHttpClient();
+
+		LocalDate next = LocalDate.parse(packet.getData("date"), formatter);
+		next = next.plusDays(1);
+		String tmr = next.format(formatter);
+
+		String url = String.format("https://web3api.io/api/v2/defi/lending/aavev2/assets/%s?startDate=%s&endDate=%s&page=%s",
+				packet.getData("asset"),
+				packet.getData("date") + "T01:00:00", 
+				tmr + "T01:00:00",
+				page);
+
+		Request request = new Request.Builder()
+		  .url(url + "&size=" + request_size)
+		  .get()
+		  .addHeader("x-amberdata-blockchain-id", "ethereum-mainnet")
+		  .addHeader("accept", "application/json")
+		  .addHeader("x-api-key", "UAK7ed69235426c360be22bfc2bde1809b6")
+		  .build();
+
+		okhttp3.Response response;
+		try {
+			response = client.newCall(request).execute();
+			JSONObject json = new JSONObject(response.body().string());
+
+			if(json.toString().equals("") || !json.has("description"))
+				return new Object[] {false, "JSON Object returned empty or invalid contents."};
+
+			if(!response.isSuccessful() || json.getInt("status") != 200
+				|| !json.getString("description").equals("Successful request")) {
+				System.out.println(json.getString("message"));
+				return new Object[]{false, json.getString("description")};
+			}
+
+			if(!json.has("payload") || !json.getJSONObject("payload").has("records"))
+				return new Object[] {false, "Malformed Aave packet"};
+
+			JSONObject payload = json.getJSONObject("payload");
+			JSONArray arr = payload.getJSONArray("records");
+			for(int i = 0; i < arr.length(); i++) {
+				JSONObject obj = arr.getJSONObject(i);
+				packet.getConnection().processRequest(
+						packet.getData("request"),
+						packet.getData("date"),
+						format("creator", obj.getString("creator"),
+								"firstBlockNumber", obj.getString("firstBlockNumber"),
+								"firstTransactionHash", obj.getString("firstTransactionHash"),
+								"address", obj.getJSONObject("hash").getString("address"),
+								"timestamp", obj.get("timestamp").toString(),
+								"type", obj.getString("type")));
+			}
+
+			if(arr.length() >= request_size) {
+				return processBlockchainAddresses(packet, page + 1);
+			} else {
+				return new Object[] {true, ""};
+			}
+		} catch (IOException e) {
+			e.printStackTrace();
+			return new Object[] {false, "Exception."};
+		}
+	}
+
 	private static String format(Object... data) {
 		StringBuilder out = new StringBuilder();
 		for(int i = 0; i < data.length; i++) {

DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/connected/connections/AmberDataConnection.java

@@ -24,8 +24,7 @@ public class AmberDataConnection extends ExternalStreamConnection {
 	requestTypes = new HashSet<>();
 	requestTypes.add("aave-protocol-dated");
 	requestTypes.add("aave-asset-dated");
-	//requestTypes.add("aave-wallet-dated");
-	//requestTypes.add("aave-governance-dated");
+	requestTypes.add("blockchain-addresses-dated");
 }
 
 	private boolean authorized = false;

DeFi-Data-Engine/Rest Application/src/main/java/org/properties/Config.java

@@ -20,7 +20,9 @@ public class Config {
 	app_properties.put("spring.server.port", "8080");
 	app_properties.put("spring.server.address", "RestApp");
 	// app_properties.put("spring.server.address", "defi-de.idea.rpi.edu");
+	// app_properties.put("spring.server.address", "localhost");
 	app_properties.put("rest.socket.address", "DataEngine");
+	// app_properties.put("rest.socket.address", "localhost");
 	app_properties.put("rest.socket.port", "61100");
 	app_properties.put("rest.socket.key", "rest-key-reserved");
 	properties.put("app", app_properties);

DeFi-Data-Engine/Testing Environment/src/test/connection/socket/LocalTest.java

@@ -16,7 +16,7 @@
 public class LocalTest {
 
 	//private static final String host = "localhost";
-	private static final String host = "defi-de.idea.rpi.edu";
+	private static final String host = "localhost";
 
 	public static void main(String[] args) throws UnknownHostException, IOException {
 		final Socket socket = SocketFactory.getDefault().createSocket(host, 61200);
@@ -39,8 +39,8 @@ public static void main(String[] args) throws UnknownHostException, IOException
 				+ "key&&&%s&&&"
 				+ "start_date&&&2022-08-01&&&"
 				+ "end_date&&&2022-09-01&&&"
-				+ "query&&&aave-protocol-dated&&&"
-				+ "request&&&aave-protocol-dated\n",
+				+ "query&&&blockchain-addresses-dated&&&"
+				+ "request&&&blockchain-addresses-dated\n",
 				destination,
 				key));
 

DeFi-Data-Engine/Testing Environment/src/test/misc/Testing1.java

@@ -10,47 +10,66 @@
 
 public class Testing1 {
 
+	private static final int request_size = 999;
+
 	 public static void main(String args[])
 	 {
-		 String url = String.format("https://web3api.io/api/v2/defi/lending/aavev2/protocol?startDate=%s&endDate=%s",
+		 String url = String.format("https://web3api.io/api/v2/addresses?startDate=%s&endDate=%s",
 				"2022-09-01T01:00:00", 
-				"2022-09-02T01:00:00"
-				+ "&size=999");
+				"2022-09-02T01:00:00");
 
-		recursiveCall(url);
+		Object[] obj = recursiveCall(url, 0);
+		if(!(boolean)obj[0])
+			System.out.println(obj[1]);
 		System.out.println("Exit");
 		System.exit(0);
 	 }
 
-	 public static void recursiveCall(String url) {
-		 System.out.println(url);
-		 OkHttpClient client = new OkHttpClient();
+	 public static Object[] recursiveCall(String base_url, int page) {
+		System.out.println(page);
+		OkHttpClient client = new OkHttpClient();
 
+		String url = base_url + "&page=" + page;
+
 		Request request = new Request.Builder()
-		  .url(url)
+		  .url(url + "&size=" + request_size)
 		  .get()
+		  .addHeader("x-amberdata-blockchain-id", "ethereum-mainnet")
 		  .addHeader("accept", "application/json")
 		  .addHeader("x-api-key", "UAK7ed69235426c360be22bfc2bde1809b6")
 		  .build();
-		
+
 		okhttp3.Response response;
 		try {
 			response = client.newCall(request).execute();
 			JSONObject json = new JSONObject(response.body().string());
 
-			JSONObject payload = json.getJSONObject("payload");
-			JSONArray arr = payload.getJSONArray("data");
-			for(int i = 0; i < arr.length(); i++) {
-				System.out.println(arr.get(i).toString());
+			if(json.toString().equals("") || !json.has("description"))
+				return new Object[] {false, "JSON Object returned empty or invalid contents."};
+
+			if(!response.isSuccessful() || json.getInt("status") != 200
+				|| !json.getString("description").equals("Successful request")) {
+				System.out.println(json.getString("message"));
+				return new Object[]{false, json.getString("description")};
 			}
-			System.out.println(arr.length());
-			if(arr.length() >= 999) {
-				JSONObject metadata = payload.getJSONObject("metadata");
-				if(metadata.has("next"))
-					recursiveCall(metadata.getString("next")+"&size=999");
+
+			if(!json.has("payload") || !json.getJSONObject("payload").has("records"))
+				return new Object[] {false, "Malformed Aave packet"};
+
+			JSONObject payload = json.getJSONObject("payload");
+			JSONArray arr = payload.getJSONArray("records");
+//			for(int i = 0; i < arr.length(); i++) {
+//				System.out.println(arr.getJSONObject(i).getJSONObject("hash").getString("address"));
+//			}
+
+			if(arr.length() >= request_size) {
+				return recursiveCall(base_url, page + 1);
+			} else {
+				return new Object[] {true, ""};
 			}
 		} catch (IOException e) {
 			e.printStackTrace();
+			return new Object[] {false, "Exception."};
 		}
 	 }
 }