diff --git a/.github/workflows/dockerpushandrun.yml b/.github/workflows/dockerpushandrun.yml new file mode 100644 index 00000000..8b5d80b4 --- /dev/null +++ b/.github/workflows/dockerpushandrun.yml @@ -0,0 +1,40 @@ +name: CD to Docker +on: + push: + branches: [ main ] + paths: 'DeFi-Data-Engine/**' + workflow_dispatch: + +jobs: + buildDocker: + runs-on: [ self-hosted ] + defaults: + run: + working-directory: "DeFi-Data-Engine/" + + steps: + - uses: actions/checkout@v3 + + - name: Stop Existing Images + run: sudo docker compose down + + - name: Build + run: sudo docker compose build + + - name: Push to Docker + run: sudo docker compose push + + runDocker: + needs: buildDocker + runs-on: [ self-hosted ] + + defaults: + run: + working-directory: "DeFi-Data-Engine/" + + steps: + - name: Create Containers + run: sudo docker compose create + + - name: Start services + run: sudo docker compose start diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml deleted file mode 100644 index bad8b187..00000000 --- a/.github/workflows/maven.yml +++ /dev/null @@ -1,47 +0,0 @@ -name: CI with mvn -on: - push: - branches: [ main ] - paths: 'DeFi-Data-Engine/**' - pull_request: - branches: [ main ] - paths: 'DeFi-Data-Engine/**' - workflow_dispatch: - -jobs: - buildandDeployDataEngine: - - runs-on: [ self-hosted ] - defaults: - run: - working-directory: "DeFi-Data-Engine/DeFi Data Engine/" - - steps: - - uses: actions/checkout@v3 - - - name: Build with Maven - run: mvn clean compile assembly:single -Dmaven.skip.test=true - -# #Deployment will fail if there are deployments which did not come from this workflow -# - name: Check if existing deployments -# run: pkill -f 'java -jar target/defi-data-engine-0.0.1-jar-with-dependencies.jar' - - - name: Deploy data engine - run: RUNNER_TRACKING_ID="" && (java -jar target/defi-data-engine-0.0.1-jar-with-dependencies.jar &) - - buildRestAPI: - needs: buildandDeployDataEngine - runs-on: [ self-hosted ] - defaults: - run: - working-directory: "DeFi-Data-Engine/Rest Application/" - - steps: - - uses: actions/checkout@v3 - - - name: Build with Maven - run: mvn clean compile assembly:single -Dmaven.skip.test=true - - - name: Deploy rest application - run: RUNNER_TRACKING_ID="" && (java -jar target/rest-connection-4.3.3-jar-with-dependencies.jar &) - diff --git a/DeFi-Data-Engine/DeFi Data Engine/.classpath b/DeFi-Data-Engine/DeFi Data Engine/.classpath index 4d455a8c..b2e8bc05 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/.classpath +++ b/DeFi-Data-Engine/DeFi Data Engine/.classpath @@ -25,12 +25,33 @@ - + + + + + + + + + + + + + + + + + + + + + + diff --git a/DeFi-Data-Engine/DeFi Data Engine/.settings/org.eclipse.jdt.apt.core.prefs b/DeFi-Data-Engine/DeFi Data Engine/.settings/org.eclipse.jdt.apt.core.prefs new file mode 100644 index 00000000..d4313d4b --- /dev/null +++ b/DeFi-Data-Engine/DeFi Data Engine/.settings/org.eclipse.jdt.apt.core.prefs @@ -0,0 +1,2 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.apt.aptEnabled=false diff --git a/DeFi-Data-Engine/DeFi Data Engine/.settings/org.eclipse.jdt.core.prefs b/DeFi-Data-Engine/DeFi Data Engine/.settings/org.eclipse.jdt.core.prefs index ecb498c8..b5e07ce6 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/.settings/org.eclipse.jdt.core.prefs +++ b/DeFi-Data-Engine/DeFi Data Engine/.settings/org.eclipse.jdt.core.prefs @@ -1,9 +1,9 @@ eclipse.preferences.version=1 org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate -org.eclipse.jdt.core.compiler.codegen.targetPlatform=17 +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve -org.eclipse.jdt.core.compiler.compliance=17 +org.eclipse.jdt.core.compiler.compliance=1.8 org.eclipse.jdt.core.compiler.debug.lineNumber=generate org.eclipse.jdt.core.compiler.debug.localVariable=generate org.eclipse.jdt.core.compiler.debug.sourceFile=generate @@ -12,5 +12,6 @@ org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled org.eclipse.jdt.core.compiler.problem.enumIdentifier=error org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=warning -org.eclipse.jdt.core.compiler.release=enabled -org.eclipse.jdt.core.compiler.source=17 +org.eclipse.jdt.core.compiler.processAnnotations=disabled +org.eclipse.jdt.core.compiler.release=disabled +org.eclipse.jdt.core.compiler.source=1.8 diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/framework/router/Router.java b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/framework/router/Router.java index f503c2e1..f839548e 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/framework/router/Router.java +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/framework/router/Router.java @@ -223,11 +223,11 @@ public final Response receive(Packet packet) { * All {@link Method} objects must contain a single parameter, a {@link Packet} object, * and return a {@link Response} object. * - * @param subtag Subtag of the process to handle the incoming {@link Packet} object. + * @param sub_tag Subtag of the process to handle the incoming {@link Packet} object. * @param method {@link Method} to pass the {@link Packet} object to. */ - public final void addProcess(String subtag, Method method) { - processes.put(subtag, method); + public final void addProcess(String sub_tag, Method method) { + processes.put(sub_tag, method); } /** diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/out/consumers/SocketConsumer.java b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/out/consumers/SocketConsumer.java index 4bd7b97e..366a0ad8 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/out/consumers/SocketConsumer.java +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/out/consumers/SocketConsumer.java @@ -49,32 +49,45 @@ public void run() { while(true) { String[] input = ((String)in.readUTF()).split(Config.getProperty("app", "general.transfer.delim")); - // validate length is greater than 2 - if(input.length <= 2) { - out.writeUTF(new JSONObject() - .put("response", "502") - .put("message", "Packet processed from REST API does not contain a TAG or SUB_TAG. Review REST API endpoint code.") - .toString()); - } + Thread thread = new Thread() { + public void run() { + try { + // validate length is greater than 2 + if(input.length <= 2) { + out.writeUTF(new JSONObject() + .put("response", "502") + .put("message", "Packet processed from REST API does not contain a TAG or SUB_TAG. Review REST API endpoint code.") + .toString()); + } + + // extract non-essential data + String[] data = Arrays.copyOfRange(input, 2, input.length); + String tag = input[0]; + String sub_tag = input[1]; + + // execute valid response to engine + Response response = send(tag, sub_tag, data); + out.writeUTF(new JSONObject() + .put("response", "200") + .put("code", response.code()) + .put("message", response.message()) + .put("data", response.data()) + .toString()); + out.flush(); + + } catch(SocketException e) { + System.err.println("Rest Application has unexpectedly closed."); + System.exit(1); + } catch (IOException e) { + e.printStackTrace(); + System.exit(1); + } + } + }; - // extract non-essential data - String[] data = Arrays.copyOfRange(input, 2, input.length); - String tag = input[0]; - String sub_tag = input[1]; - - // execute valid response to engine - Response response = send(tag, sub_tag, data); - out.writeUTF(new JSONObject() - .put("response", "200") - .put("code", response.code()) - .put("message", response.message()) - .put("data", response.data()) - .toString()); + thread.start(); } - } catch(SocketException e) { - System.err.println("Rest Application has unexpectedly closed."); - System.exit(1); - } catch (IOException e) { + } catch(Exception e) { e.printStackTrace(); System.exit(1); } diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/out/destinations/SocketDestination.java b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/out/destinations/SocketDestination.java index 239bdd4a..2b23eab4 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/out/destinations/SocketDestination.java +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/out/destinations/SocketDestination.java @@ -21,6 +21,7 @@ public final synchronized boolean send(Packet packet) { try { out.write(packet.getData("data").getBytes()); out.write(10); + out.flush(); } catch (JSONException | IOException e) { return false; } diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/out/handler/OutputManager.java b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/out/handler/OutputManager.java index 65ca05de..fc834178 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/out/handler/OutputManager.java +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/out/handler/OutputManager.java @@ -111,7 +111,7 @@ public Object[] producerListen() { return new Object[] {true, ""}; } - protected final Response send(String tag, String sub_tag, String... data) { + public final Response send(String tag, String sub_tag, String... data) { return handler.send(tag, sub_tag, data); } diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/out/producers/SocketProducer.java b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/out/producers/SocketProducer.java index d2fd265f..e29be73a 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/out/producers/SocketProducer.java +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/out/producers/SocketProducer.java @@ -1,5 +1,6 @@ package org.out.producers; +import org.framework.router.Response; import org.out.destinations.SocketDestination; import org.out.handler.OutputManager; import org.out.handler.OutputProducer; @@ -9,6 +10,7 @@ public class SocketProducer extends OutputProducer { private Thread listener; + public final SocketProducer producer = this; public SocketProducer(OutputManager manager) { super(manager); @@ -24,7 +26,7 @@ protected boolean init() { listener = new Thread() { public void run() { while(true) { - String key = SocketManager.accept(Integer.parseInt(Config.getProperty("stream", "output.socket.port"))); + String key = SocketManager.accept(Integer.parseInt(Config.getProperty("stream", "output.socket.port")), producer); if(key == null) { System.err.println("SocketProducer: Could not create connection to socket port."); System.exit(1); @@ -55,4 +57,8 @@ protected boolean kill() { return true; } + + public Response send(String tag, String sub_tag, String... data) { + return manager.send(tag, sub_tag, data); + } } diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/out/socket/SocketManager.java b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/out/socket/SocketManager.java index 7148a737..0d923738 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/out/socket/SocketManager.java +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/out/socket/SocketManager.java @@ -5,11 +5,16 @@ import java.io.IOException; import java.net.ServerSocket; import java.net.Socket; +import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.UUID; import org.core.logger.Logger; +import org.framework.router.Response; +import org.json.JSONObject; +import org.out.producers.SocketProducer; +import org.properties.Config; public class SocketManager { @@ -38,8 +43,69 @@ public synchronized static boolean exists(String key) { return connections.containsKey(key); } + public synchronized static void createThread(String key, SocketProducer producer) { + Thread thread = new Thread() { + public void run() { + // perform logger verifications + Logger.log(String.format("Starting thread for Socket with key <%s>", key)); + if(!inflow.containsKey(key)) { + Logger.terminate(String.format("Key <%s> not found within inflow thread configuration, manual review recommended.", key)); + return; + } + + if(!outflow.containsKey(key)) { + Logger.terminate(String.format("Key <%s> not found within outflow thread configuration, manual review recommended.", key)); + return; + } + + // retrieve inflow stream and listen + DataInputStream in = inflow.get(key); + DataOutputStream out = outflow.get(key); + String str; + while(true) + try { + str = readLine(in); + + // parse input + String[] input = str.split(Config.getProperty("app", "general.transfer.delim")); + + // validate input + if(input.length <= 2) { + out.writeUTF(new JSONObject() + .put("response", "502") + .put("message", "Packet processed from REST API does not contain a TAG or SUB_TAG. Review REST API endpoint code.") + .toString()); + } + + // extract non-essential data + String[] data = Arrays.copyOfRange(input, 2, input.length); + String tag = input[0]; + String sub_tag = input[1]; + + // execute valid response to engine + Response response = producer.send(tag, sub_tag, data); + out.writeUTF(new JSONObject() + .put("response", "200") + .put("code", response.code()) + .put("message", response.message()) + .put("data", response.data()) + .toString()); + out.flush(); + System.out.println("<<>>"); + + } catch(Exception e) { + break; + } + + Logger.log(String.format("Terminating thread for Socket with key <%s>", key)); + } + }; + + thread.start(); + } + // used for generic channel accepting - public synchronized static String accept(int port) { + public synchronized static String accept(int port, SocketProducer producer) { if(!servers.containsKey(port)) if(!createServer(port)) return null; @@ -68,6 +134,9 @@ public synchronized static String accept(int port) { if(!synced(key)) throw new Exception("Connection inflow and outflow not synchronized"); + // start internal thread for socket information parsing + createThread(key, producer); + return key; } catch(Exception e) { e.printStackTrace(); @@ -132,4 +201,13 @@ public static DataInputStream read(String key) { private static boolean synced(String key) { return connections.containsKey(key) && inflow.containsKey(key) && outflow.containsKey(key); } + + private static final String readLine(DataInputStream in) throws IOException { + StringBuilder out = new StringBuilder(); + char c = 0; + while((c = (char)in.read()) != 10) + out.append(c); + + return out.toString(); + } } diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/properties/Config.java b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/properties/Config.java index c0587a74..0879bcde 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/properties/Config.java +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/properties/Config.java @@ -41,14 +41,14 @@ public class Config { Properties stream_properties = new Properties(); stream_properties.put("general.consumer.types", "socket_consumer"); stream_properties.put("general.producer.types", "socket_producer"); - stream_properties.put("rest.socket.address", "localhost"); + stream_properties.put("rest.socket.address", "DataEngine"); stream_properties.put("rest.socket.port", "61100"); stream_properties.put("rest.socket.key", "rest-key-reserved"); - //stream_properties.put("output.socket.address", "defi-de.idea.rpi.edu"); - stream_properties.put("output.socket.address", "localhost"); + // stream_properties.put("output.socket.address", "defi-de.idea.rpi.edu"); + stream_properties.put("output.socket.address", "RestApp"); stream_properties.put("output.socket.port", "61200"); stream_properties.put("local.stream.type", "mongo_db"); - stream_properties.put("mongodb.properties.uri", "mongodb://localhost:27017"); + stream_properties.put("mongodb.properties.uri", "mongodb://MONGO:27017"); stream_properties.put("mongodb.database.state", "main-state-db"); stream_properties.put("mongodb.database.main", "main-db"); stream_properties.put("mongodb.auth.collection", "auth-collection"); diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/connected/amberdata/AmberDataRequestHandler.java b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/connected/amberdata/AmberDataRequestHandler.java index 0fdda531..b9ec167e 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/connected/amberdata/AmberDataRequestHandler.java +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/stream/external/connected/amberdata/AmberDataRequestHandler.java @@ -20,6 +20,7 @@ public class AmberDataRequestHandler { private volatile static OkHttpClient client; private final static HashMap requests; + private static final int request_size = 999; private static final DateTimeFormatter formatter = DateTimeFormatter.ofPattern(Config.getProperty("app", "general.data.dateformat")); @@ -91,55 +92,7 @@ public static Object[] requestAaveProtocolDated(AmberDataRequestPacket packet) { packet.getData("date") + "T01:00:00", tmr + "T01:00:00"); - return requestAaveProtocolDated(packet, url); - } - - private static final int request_size = 999; - private static Object[] requestAaveProtocolDated(AmberDataRequestPacket packet, String url) { - OkHttpClient client = new OkHttpClient(); - - Request request = new Request.Builder() - .url(url + "&size=" + request_size) - .get() - .addHeader("accept", "application/json") - .addHeader("x-api-key", packet.getKey()) - .build(); - - okhttp3.Response response; - try { - response = client.newCall(request).execute(); - JSONObject json = new JSONObject(response.body().string()); - - if(json.toString().equals("") || !json.has("description")) - return new Object[] {false, "JSON Object returned empty or invalid contents."}; - - if(!response.isSuccessful() || json.getInt("status") != 200 - || !json.getString("description").equals("Successful request")) - return new Object[]{false, json.getString("description")}; - - if(!json.has("payload") || !json.getJSONObject("payload").has("data")) - return new Object[] {false, "Malformed Aave packet"}; - - JSONObject payload = json.getJSONObject("payload"); - JSONArray arr = payload.getJSONArray("data"); - for(int i = 0; i < arr.length(); i++) { - processAaveJsonRequest(packet, arr.getJSONObject(i)); - } - - if(arr.length() >= request_size) { - JSONObject metadata = payload.getJSONObject("metadata"); - if(metadata.has("next")) - return requestAaveProtocolDated(packet, metadata.getString("next")); - else - return new Object[] {false, "Response did not contain next API cursor"}; - } else { - return new Object[] {true, ""}; - } - } catch (IOException e) { - e.printStackTrace(); - ResponseFactory.responseNotHandled("Unhandled exception cost."); - return new Object[] {false, null}; - } + return requestAaveURL(packet, url); } public static Object[] requestAaveAssetDated(AmberDataRequestPacket packet) { @@ -155,10 +108,18 @@ public static Object[] requestAaveAssetDated(AmberDataRequestPacket packet) { packet.getData("date") + "T01:00:00", tmr + "T01:00:00"); + return requestAaveURL(packet, url, packet.getData("asset")); + } + + private static Object[] requestAaveURL(AmberDataRequestPacket packet, String url) { + return requestAaveURL(packet, url, null); + } + + private static Object[] requestAaveURL(AmberDataRequestPacket packet, String url, String request_specifier) { OkHttpClient client = new OkHttpClient(); Request request = new Request.Builder() - .url(url) + .url(url + "&size=" + request_size) .get() .addHeader("accept", "application/json") .addHeader("x-api-key", packet.getKey()) @@ -181,10 +142,23 @@ public static Object[] requestAaveAssetDated(AmberDataRequestPacket packet) { JSONObject payload = json.getJSONObject("payload"); JSONArray arr = payload.getJSONArray("data"); - for(int i = 0; i < arr.length(); i++) - processAaveJsonRequest(packet, arr.getJSONObject(i), packet.getData("asset")); + for(int i = 0; i < arr.length(); i++) { + if(request_specifier == null) { + processAaveJsonRequest(packet, arr.getJSONObject(i)); + } else { + processAaveJsonRequest(packet, arr.getJSONObject(i), request_specifier); + } + } - return new Object[] {true, ""}; + if(arr.length() >= request_size) { + JSONObject metadata = payload.getJSONObject("metadata"); + if(metadata.has("next")) + return requestAaveURL(packet, metadata.getString("next")); + else + return new Object[] {false, "Response did not contain next API cursor"}; + } else { + return new Object[] {true, ""}; + } } catch (IOException e) { e.printStackTrace(); ResponseFactory.responseNotHandled("Unhandled exception cost."); diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/config/stream.properties b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/config/stream.properties index 4bb2109a..17e1b268 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/config/stream.properties +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/config/stream.properties @@ -9,7 +9,7 @@ general.producer.types=socket_producer # === REST SOCKET PROPERTIES === # Rest socket address -rest.socket.address=localhost +rest.socket.address=DataEngine # Rest socket port rest.socket.port=61100 @@ -20,7 +20,7 @@ rest.socket.key=rest-key-reserved # === OUTPUT SOCKET PROPERTIES === # Output socket address -output.socket.address=localhost +output.socket.address=RestApp # Output socket port output.socket.port=61200 @@ -33,7 +33,7 @@ local.stream.type=mongo_db # === MONGO DB PROPERTIES === # local MongoDB client URI -mongodb.properties.uri=mongodb://localhost:27017 +mongodb.properties.uri=mongodb://MONGO:27017 # local MongoDB state database name mongodb.database.state=main-state-db diff --git a/DeFi-Data-Engine/Documentation/Internal-Documentation/Documentation.xlsx b/DeFi-Data-Engine/Documentation/Internal-Documentation/Documentation.xlsx index 801e6381..a5b44e77 100644 Binary files a/DeFi-Data-Engine/Documentation/Internal-Documentation/Documentation.xlsx and b/DeFi-Data-Engine/Documentation/Internal-Documentation/Documentation.xlsx differ diff --git a/DeFi-Data-Engine/Executables/DeFi-Data-Engine.jar b/DeFi-Data-Engine/Executables/DeFi-Data-Engine.jar new file mode 100644 index 00000000..53654496 Binary files /dev/null and b/DeFi-Data-Engine/Executables/DeFi-Data-Engine.jar differ diff --git a/DeFi-Data-Engine/Executables/rest-connection-4.3.3.jar b/DeFi-Data-Engine/Executables/rest-connection-4.3.3.jar new file mode 100644 index 00000000..7c15f46f Binary files /dev/null and b/DeFi-Data-Engine/Executables/rest-connection-4.3.3.jar differ diff --git a/DeFi-Data-Engine/Rest Application/.classpath b/DeFi-Data-Engine/Rest Application/.classpath index d118e897..e90905c2 100644 --- a/DeFi-Data-Engine/Rest Application/.classpath +++ b/DeFi-Data-Engine/Rest Application/.classpath @@ -18,21 +18,31 @@ - + - - + - - + + + + + + + + + + + + + diff --git a/DeFi-Data-Engine/Rest Application/.settings/org.eclipse.core.resources.prefs b/DeFi-Data-Engine/Rest Application/.settings/org.eclipse.core.resources.prefs index 609d3cab..29abf999 100644 --- a/DeFi-Data-Engine/Rest Application/.settings/org.eclipse.core.resources.prefs +++ b/DeFi-Data-Engine/Rest Application/.settings/org.eclipse.core.resources.prefs @@ -1,4 +1,6 @@ eclipse.preferences.version=1 +encoding//src/main/java=UTF-8 encoding//src/main/resources=UTF-8 +encoding//src/test/java=UTF-8 encoding//src/test/resources=UTF-8 encoding/=UTF-8 diff --git a/DeFi-Data-Engine/Rest Application/.settings/org.eclipse.jdt.apt.core.prefs b/DeFi-Data-Engine/Rest Application/.settings/org.eclipse.jdt.apt.core.prefs new file mode 100644 index 00000000..d4313d4b --- /dev/null +++ b/DeFi-Data-Engine/Rest Application/.settings/org.eclipse.jdt.apt.core.prefs @@ -0,0 +1,2 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.apt.aptEnabled=false diff --git a/DeFi-Data-Engine/Rest Application/.settings/org.eclipse.jdt.core.prefs b/DeFi-Data-Engine/Rest Application/.settings/org.eclipse.jdt.core.prefs index b041b83d..23731ecc 100644 --- a/DeFi-Data-Engine/Rest Application/.settings/org.eclipse.jdt.core.prefs +++ b/DeFi-Data-Engine/Rest Application/.settings/org.eclipse.jdt.core.prefs @@ -12,5 +12,6 @@ org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled org.eclipse.jdt.core.compiler.problem.enumIdentifier=error org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=warning +org.eclipse.jdt.core.compiler.processAnnotations=disabled org.eclipse.jdt.core.compiler.release=disabled org.eclipse.jdt.core.compiler.source=17 diff --git a/DeFi-Data-Engine/Rest Application/Dockerfile b/DeFi-Data-Engine/Rest Application/Dockerfile index bdbe8984..c87bd6cf 100644 --- a/DeFi-Data-Engine/Rest Application/Dockerfile +++ b/DeFi-Data-Engine/Rest Application/Dockerfile @@ -1,9 +1,4 @@ FROM maven:3.8.6-eclipse-temurin-17 - COPY ./ ./ - -# package our application code RUN mvn clean package -Dmaven.test.skip -EXPOSE 8080 -# set the startup command to execute the jar CMD ["java", "-jar", "target/rest-connection-4.3.3.jar"] \ No newline at end of file diff --git a/DeFi-Data-Engine/Rest Application/src/main/java/org/properties/Config.java b/DeFi-Data-Engine/Rest Application/src/main/java/org/properties/Config.java index 3cfee27f..c6210aef 100644 --- a/DeFi-Data-Engine/Rest Application/src/main/java/org/properties/Config.java +++ b/DeFi-Data-Engine/Rest Application/src/main/java/org/properties/Config.java @@ -36,9 +36,9 @@ public class Config { app_properties.put("general.transfer.delim", "&&&"); app_properties.put("general.data.dateformat", "yyyy-MM-dd"); app_properties.put("spring.server.port", "8080"); - app_properties.put("spring.server.address", "localhost"); - //app_properties.put("spring.server.address", "defi-de.idea.rpi.edu"); - app_properties.put("rest.socket.address", "localhost"); + app_properties.put("spring.server.address", "RestApp"); + // app_properties.put("spring.server.address", "defi-de.idea.rpi.edu"); + app_properties.put("rest.socket.address", "DataEngine"); app_properties.put("rest.socket.port", "61100"); app_properties.put("rest.socket.key", "rest-key-reserved"); properties.put("app", app_properties); diff --git a/DeFi-Data-Engine/Rest Application/src/main/java/org/rest/application/Endpoint.java b/DeFi-Data-Engine/Rest Application/src/main/java/org/rest/application/Endpoint.java index 4213b4f3..e382e6d6 100644 --- a/DeFi-Data-Engine/Rest Application/src/main/java/org/rest/application/Endpoint.java +++ b/DeFi-Data-Engine/Rest Application/src/main/java/org/rest/application/Endpoint.java @@ -150,8 +150,6 @@ private final String request(String tag, String sub_tag, String... data) { formatted_data.append(delim); } - // static request - System.out.println(String.format("%s%s%s%s%s", tag, delim, sub_tag, delim, formatted_data)); out.writeUTF(String.format("%s%s%s%s%s", tag, delim, sub_tag, delim, formatted_data)); return in.readUTF(); } catch (IOException e) { diff --git a/DeFi-Data-Engine/Rest Application/src/main/resources/config/app.properties b/DeFi-Data-Engine/Rest Application/src/main/resources/config/app.properties index 11c5215b..049af529 100644 --- a/DeFi-Data-Engine/Rest Application/src/main/resources/config/app.properties +++ b/DeFi-Data-Engine/Rest Application/src/main/resources/config/app.properties @@ -23,7 +23,7 @@ spring.server.port=8080 # === SOCKET PROPERTIES === # Rest socket address -rest.socket.address=localhost +rest.socket.address=DataEngine # Rest socket port rest.socket.port=61100 diff --git a/DeFi-Data-Engine/Testing Environment/.classpath b/DeFi-Data-Engine/Testing Environment/.classpath index cddbf79a..58cbceb6 100644 --- a/DeFi-Data-Engine/Testing Environment/.classpath +++ b/DeFi-Data-Engine/Testing Environment/.classpath @@ -6,9 +6,8 @@ - + - @@ -17,5 +16,21 @@ + + + + + + + + + + + + + + + + diff --git a/DeFi-Data-Engine/Testing Environment/.settings/org.eclipse.jdt.apt.core.prefs b/DeFi-Data-Engine/Testing Environment/.settings/org.eclipse.jdt.apt.core.prefs new file mode 100644 index 00000000..d4313d4b --- /dev/null +++ b/DeFi-Data-Engine/Testing Environment/.settings/org.eclipse.jdt.apt.core.prefs @@ -0,0 +1,2 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.apt.aptEnabled=false diff --git a/DeFi-Data-Engine/Testing Environment/.settings/org.eclipse.jdt.core.prefs b/DeFi-Data-Engine/Testing Environment/.settings/org.eclipse.jdt.core.prefs index 43c8d716..b2ef4d6e 100644 --- a/DeFi-Data-Engine/Testing Environment/.settings/org.eclipse.jdt.core.prefs +++ b/DeFi-Data-Engine/Testing Environment/.settings/org.eclipse.jdt.core.prefs @@ -11,5 +11,6 @@ org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled org.eclipse.jdt.core.compiler.problem.enumIdentifier=error org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=warning +org.eclipse.jdt.core.compiler.processAnnotations=disabled org.eclipse.jdt.core.compiler.release=enabled org.eclipse.jdt.core.compiler.source=17 diff --git a/DeFi-Data-Engine/Testing Environment/src/test/connection/socket/LocalTest.java b/DeFi-Data-Engine/Testing Environment/src/test/connection/socket/LocalTest.java index 5dd556ae..6d7c931f 100644 --- a/DeFi-Data-Engine/Testing Environment/src/test/connection/socket/LocalTest.java +++ b/DeFi-Data-Engine/Testing Environment/src/test/connection/socket/LocalTest.java @@ -2,6 +2,7 @@ import java.io.BufferedReader; import java.io.DataInputStream; +import java.io.DataOutputStream; import java.io.IOException; import java.io.InputStreamReader; import java.net.HttpURLConnection; @@ -15,28 +16,15 @@ public class LocalTest { - private static final String host = "defi-de.idea.rpi.edu"; + private static final String host = "localhost"; public static void main(String[] args) throws UnknownHostException, IOException { final Socket socket = SocketFactory.getDefault().createSocket(host, 61200); - final DataInputStream in = new DataInputStream(socket.getInputStream()); + final BufferedReader in = new BufferedReader(new InputStreamReader(socket.getInputStream())); + final DataOutputStream out = new DataOutputStream(socket.getOutputStream()); String destination = readLine(in); - Thread thread = new Thread() { - public void run() { - try { - while(true) { - System.out.println(readLine(in)); - } - } catch(Exception e) { - e.printStackTrace(); - System.exit(1); - } - } - }; - thread.start(); - String init = request(String.format("http://%s:8080/defi/v1/rest/initialize?" + "source=amber_data&" + "auth_data=key,UAK7ed69235426c360be22bfc2bde1809b6", @@ -45,22 +33,38 @@ public void run() { JSONObject json_init = new JSONObject(init); String key = json_init.getString("data"); - String rqst = request(String.format("http://%s:8080/defi/v1/rest/request_dated?" - + "destination=%s&" - + "key=%s&" - + "request=request,aave-asset-dated,asset,WETH&" - + "query=aave-asset-dated-WETH&" - + "start_date=%s&" - + "end_date=%s", - host, + out.writeBytes(String.format( + "SRC&&&RQST&&&" + + "destination&&&%s&&&" + + "key&&&%s&&&" + + "start_date&&&2022-08-01&&&" + + "end_date&&&2022-08-02&&&" + + "query&&&aave-protocol-dated&&&" + + "request&&&aave-protocol-dated\n", destination, - key, - "2022-08-01", - "2022-08-02")); -// String rqst = request(String.format("http://%s:8080/defi/v1/rest/request_dated?" + key)); + + while(true) { + System.out.print(readLine(in)); + } + +// String rqst_protocol_dated = String.format("http://%s:8080/defi/v1/rest/request_dated?" +// + "destination=%s&" +// + "key=%s&" +// + "request=request,aave-protocol-dated&" +// + "query=aave-protocol-dated&" +// + "start_date=%s&" +// + "end_date=%s", +// host, +// destination, +// key, +// "2022-08-01", +// "2022-08-02"); +// +// String rqst_asset_dated = String.format("http://%s:8080/defi/v1/rest/request_dated?" // + "destination=%s&" // + "key=%s&" -// + "request=aave-protocol-dated&" +// + "request=request,aave-protocol-dated&" // + "query=aave-protocol-dated&" // + "start_date=%s&" // + "end_date=%s", @@ -68,8 +72,10 @@ public void run() { // destination, // key, // "2022-08-01", -// "2022-08-02")); - System.out.println(rqst); +// "2022-08-02"); +// +// String rqst = request(rqst_asset_dated); +// System.out.println(rqst); } public static String request(String str) throws IOException { @@ -96,7 +102,7 @@ public static String request(String str) throws IOException { return ""; } - public static final String readLine(DataInputStream in) throws IOException { + public static final String readLine(BufferedReader in) throws IOException { StringBuilder out = new StringBuilder(); char c = 0; while((c = (char)in.read()) != 10) diff --git a/DeFi-Data-Engine/docker-compose.yml b/DeFi-Data-Engine/docker-compose.yml new file mode 100644 index 00000000..8f24ad09 --- /dev/null +++ b/DeFi-Data-Engine/docker-compose.yml @@ -0,0 +1,33 @@ +version: "3.9" +services: + mongodb_container: + image: mongo:latest + ports: + - 27017:27017 + volumes: + - mongodb_data_container:/data/db + hostname: MONGO + dataengine: + build: + context: "DeFi Data Engine" + image: dataincite/defi-data-engine:latest + depends_on: + - mongodb_container + ports: + - 61100:61100 + - 61200:61200 + hostname: DataEngine + restapp: + build: + context: "Rest Application" + image: dataincite/data-engine-rest-app:latest + depends_on: + - dataengine + ports: + - 8080:8080 + hostname: RestApp + + + +volumes: + mongodb_data_container: