From 2a2f7b37c1d83f18aec124ad51fc38d3f69ab59b Mon Sep 17 00:00:00 2001 From: Kadams35 Date: Fri, 9 Dec 2022 11:21:43 -0500 Subject: [PATCH 01/14] add docker-compose --- DeFi-Data-Engine/DeFi Data Engine/.classpath | 23 +++++++++++++- .../.settings/org.eclipse.jdt.apt.core.prefs | 2 ++ .../.settings/org.eclipse.jdt.core.prefs | 9 +++--- .../src/main/java/org/properties/Config.java | 8 ++--- .../main/resources/config/stream.properties | 6 ++-- DeFi-Data-Engine/Rest Application/.classpath | 20 +++++++++--- .../org.eclipse.core.resources.prefs | 2 ++ .../.settings/org.eclipse.jdt.apt.core.prefs | 2 ++ .../.settings/org.eclipse.jdt.core.prefs | 1 + DeFi-Data-Engine/Rest Application/Dockerfile | 5 --- .../src/main/java/org/properties/Config.java | 6 ++-- .../src/main/resources/config/app.properties | 2 +- .../Testing Environment/.classpath | 19 ++++++++++-- .../.settings/org.eclipse.jdt.apt.core.prefs | 2 ++ .../.settings/org.eclipse.jdt.core.prefs | 1 + DeFi-Data-Engine/docker-compose.yml | 31 +++++++++++++++++++ 16 files changed, 111 insertions(+), 28 deletions(-) create mode 100644 DeFi-Data-Engine/DeFi Data Engine/.settings/org.eclipse.jdt.apt.core.prefs create mode 100644 DeFi-Data-Engine/Rest Application/.settings/org.eclipse.jdt.apt.core.prefs create mode 100644 DeFi-Data-Engine/Testing Environment/.settings/org.eclipse.jdt.apt.core.prefs create mode 100644 DeFi-Data-Engine/docker-compose.yml diff --git a/DeFi-Data-Engine/DeFi Data Engine/.classpath b/DeFi-Data-Engine/DeFi Data Engine/.classpath index 4d455a8c..b2e8bc05 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/.classpath +++ b/DeFi-Data-Engine/DeFi Data Engine/.classpath @@ -25,12 +25,33 @@ - + + + + + + + + + + + + + + + + + + + + + + diff --git a/DeFi-Data-Engine/DeFi Data Engine/.settings/org.eclipse.jdt.apt.core.prefs b/DeFi-Data-Engine/DeFi Data Engine/.settings/org.eclipse.jdt.apt.core.prefs new file mode 100644 index 00000000..d4313d4b --- /dev/null +++ b/DeFi-Data-Engine/DeFi Data Engine/.settings/org.eclipse.jdt.apt.core.prefs @@ -0,0 +1,2 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.apt.aptEnabled=false diff --git a/DeFi-Data-Engine/DeFi Data Engine/.settings/org.eclipse.jdt.core.prefs b/DeFi-Data-Engine/DeFi Data Engine/.settings/org.eclipse.jdt.core.prefs index ecb498c8..b5e07ce6 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/.settings/org.eclipse.jdt.core.prefs +++ b/DeFi-Data-Engine/DeFi Data Engine/.settings/org.eclipse.jdt.core.prefs @@ -1,9 +1,9 @@ eclipse.preferences.version=1 org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate -org.eclipse.jdt.core.compiler.codegen.targetPlatform=17 +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve -org.eclipse.jdt.core.compiler.compliance=17 +org.eclipse.jdt.core.compiler.compliance=1.8 org.eclipse.jdt.core.compiler.debug.lineNumber=generate org.eclipse.jdt.core.compiler.debug.localVariable=generate org.eclipse.jdt.core.compiler.debug.sourceFile=generate @@ -12,5 +12,6 @@ org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled org.eclipse.jdt.core.compiler.problem.enumIdentifier=error org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=warning -org.eclipse.jdt.core.compiler.release=enabled -org.eclipse.jdt.core.compiler.source=17 +org.eclipse.jdt.core.compiler.processAnnotations=disabled +org.eclipse.jdt.core.compiler.release=disabled +org.eclipse.jdt.core.compiler.source=1.8 diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/properties/Config.java b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/properties/Config.java index 5b90b025..921989b4 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/properties/Config.java +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/properties/Config.java @@ -41,14 +41,14 @@ public class Config { Properties stream_properties = new Properties(); stream_properties.put("general.consumer.types", "socket_consumer"); stream_properties.put("general.producer.types", "socket_producer"); - stream_properties.put("rest.socket.address", "localhost"); + stream_properties.put("rest.socket.address", "RestApp"); stream_properties.put("rest.socket.port", "61100"); stream_properties.put("rest.socket.key", "rest-key-reserved"); - stream_properties.put("output.socket.address", "defi-de.idea.rpi.edu"); - //stream_properties.put("output.socket.address", "localhost"); + // stream_properties.put("output.socket.address", "defi-de.idea.rpi.edu"); + stream_properties.put("output.socket.address", "RestApp"); stream_properties.put("output.socket.port", "61200"); stream_properties.put("local.stream.type", "mongo_db"); - stream_properties.put("mongodb.properties.uri", "mongodb://localhost:27017"); + stream_properties.put("mongodb.properties.uri", "mongodb://MONGO:27017"); stream_properties.put("mongodb.database.state", "main-state-db"); stream_properties.put("mongodb.database.main", "main-db"); stream_properties.put("mongodb.auth.collection", "auth-collection"); diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/config/stream.properties b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/config/stream.properties index 4bb2109a..be4e956d 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/config/stream.properties +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/config/stream.properties @@ -9,7 +9,7 @@ general.producer.types=socket_producer # === REST SOCKET PROPERTIES === # Rest socket address -rest.socket.address=localhost +rest.socket.address=RestApp # Rest socket port rest.socket.port=61100 @@ -20,7 +20,7 @@ rest.socket.key=rest-key-reserved # === OUTPUT SOCKET PROPERTIES === # Output socket address -output.socket.address=localhost +output.socket.address=RestApp # Output socket port output.socket.port=61200 @@ -33,7 +33,7 @@ local.stream.type=mongo_db # === MONGO DB PROPERTIES === # local MongoDB client URI -mongodb.properties.uri=mongodb://localhost:27017 +mongodb.properties.uri=mongodb://MONGO:27017 # local MongoDB state database name mongodb.database.state=main-state-db diff --git a/DeFi-Data-Engine/Rest Application/.classpath b/DeFi-Data-Engine/Rest Application/.classpath index d118e897..e90905c2 100644 --- a/DeFi-Data-Engine/Rest Application/.classpath +++ b/DeFi-Data-Engine/Rest Application/.classpath @@ -18,21 +18,31 @@ - + - - + - - + + + + + + + + + + + + + diff --git a/DeFi-Data-Engine/Rest Application/.settings/org.eclipse.core.resources.prefs b/DeFi-Data-Engine/Rest Application/.settings/org.eclipse.core.resources.prefs index 609d3cab..29abf999 100644 --- a/DeFi-Data-Engine/Rest Application/.settings/org.eclipse.core.resources.prefs +++ b/DeFi-Data-Engine/Rest Application/.settings/org.eclipse.core.resources.prefs @@ -1,4 +1,6 @@ eclipse.preferences.version=1 +encoding//src/main/java=UTF-8 encoding//src/main/resources=UTF-8 +encoding//src/test/java=UTF-8 encoding//src/test/resources=UTF-8 encoding/=UTF-8 diff --git a/DeFi-Data-Engine/Rest Application/.settings/org.eclipse.jdt.apt.core.prefs b/DeFi-Data-Engine/Rest Application/.settings/org.eclipse.jdt.apt.core.prefs new file mode 100644 index 00000000..d4313d4b --- /dev/null +++ b/DeFi-Data-Engine/Rest Application/.settings/org.eclipse.jdt.apt.core.prefs @@ -0,0 +1,2 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.apt.aptEnabled=false diff --git a/DeFi-Data-Engine/Rest Application/.settings/org.eclipse.jdt.core.prefs b/DeFi-Data-Engine/Rest Application/.settings/org.eclipse.jdt.core.prefs index b041b83d..23731ecc 100644 --- a/DeFi-Data-Engine/Rest Application/.settings/org.eclipse.jdt.core.prefs +++ b/DeFi-Data-Engine/Rest Application/.settings/org.eclipse.jdt.core.prefs @@ -12,5 +12,6 @@ org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled org.eclipse.jdt.core.compiler.problem.enumIdentifier=error org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=warning +org.eclipse.jdt.core.compiler.processAnnotations=disabled org.eclipse.jdt.core.compiler.release=disabled org.eclipse.jdt.core.compiler.source=17 diff --git a/DeFi-Data-Engine/Rest Application/Dockerfile b/DeFi-Data-Engine/Rest Application/Dockerfile index bdbe8984..c87bd6cf 100644 --- a/DeFi-Data-Engine/Rest Application/Dockerfile +++ b/DeFi-Data-Engine/Rest Application/Dockerfile @@ -1,9 +1,4 @@ FROM maven:3.8.6-eclipse-temurin-17 - COPY ./ ./ - -# package our application code RUN mvn clean package -Dmaven.test.skip -EXPOSE 8080 -# set the startup command to execute the jar CMD ["java", "-jar", "target/rest-connection-4.3.3.jar"] \ No newline at end of file diff --git a/DeFi-Data-Engine/Rest Application/src/main/java/org/properties/Config.java b/DeFi-Data-Engine/Rest Application/src/main/java/org/properties/Config.java index 7bb68a6b..c6210aef 100644 --- a/DeFi-Data-Engine/Rest Application/src/main/java/org/properties/Config.java +++ b/DeFi-Data-Engine/Rest Application/src/main/java/org/properties/Config.java @@ -36,9 +36,9 @@ public class Config { app_properties.put("general.transfer.delim", "&&&"); app_properties.put("general.data.dateformat", "yyyy-MM-dd"); app_properties.put("spring.server.port", "8080"); - //app_properties.put("spring.server.address", "localhost"); - app_properties.put("spring.server.address", "defi-de.idea.rpi.edu"); - app_properties.put("rest.socket.address", "localhost"); + app_properties.put("spring.server.address", "RestApp"); + // app_properties.put("spring.server.address", "defi-de.idea.rpi.edu"); + app_properties.put("rest.socket.address", "DataEngine"); app_properties.put("rest.socket.port", "61100"); app_properties.put("rest.socket.key", "rest-key-reserved"); properties.put("app", app_properties); diff --git a/DeFi-Data-Engine/Rest Application/src/main/resources/config/app.properties b/DeFi-Data-Engine/Rest Application/src/main/resources/config/app.properties index 11c5215b..21da7603 100644 --- a/DeFi-Data-Engine/Rest Application/src/main/resources/config/app.properties +++ b/DeFi-Data-Engine/Rest Application/src/main/resources/config/app.properties @@ -23,7 +23,7 @@ spring.server.port=8080 # === SOCKET PROPERTIES === # Rest socket address -rest.socket.address=localhost +rest.socket.address=RestApp # Rest socket port rest.socket.port=61100 diff --git a/DeFi-Data-Engine/Testing Environment/.classpath b/DeFi-Data-Engine/Testing Environment/.classpath index cddbf79a..58cbceb6 100644 --- a/DeFi-Data-Engine/Testing Environment/.classpath +++ b/DeFi-Data-Engine/Testing Environment/.classpath @@ -6,9 +6,8 @@ - + - @@ -17,5 +16,21 @@ + + + + + + + + + + + + + + + + diff --git a/DeFi-Data-Engine/Testing Environment/.settings/org.eclipse.jdt.apt.core.prefs b/DeFi-Data-Engine/Testing Environment/.settings/org.eclipse.jdt.apt.core.prefs new file mode 100644 index 00000000..d4313d4b --- /dev/null +++ b/DeFi-Data-Engine/Testing Environment/.settings/org.eclipse.jdt.apt.core.prefs @@ -0,0 +1,2 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.apt.aptEnabled=false diff --git a/DeFi-Data-Engine/Testing Environment/.settings/org.eclipse.jdt.core.prefs b/DeFi-Data-Engine/Testing Environment/.settings/org.eclipse.jdt.core.prefs index 43c8d716..b2ef4d6e 100644 --- a/DeFi-Data-Engine/Testing Environment/.settings/org.eclipse.jdt.core.prefs +++ b/DeFi-Data-Engine/Testing Environment/.settings/org.eclipse.jdt.core.prefs @@ -11,5 +11,6 @@ org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled org.eclipse.jdt.core.compiler.problem.enumIdentifier=error org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=warning +org.eclipse.jdt.core.compiler.processAnnotations=disabled org.eclipse.jdt.core.compiler.release=enabled org.eclipse.jdt.core.compiler.source=17 diff --git a/DeFi-Data-Engine/docker-compose.yml b/DeFi-Data-Engine/docker-compose.yml new file mode 100644 index 00000000..b1a7662c --- /dev/null +++ b/DeFi-Data-Engine/docker-compose.yml @@ -0,0 +1,31 @@ +version: "3.9" +services: + mongodb_container: + image: mongo:latest + ports: + - 27017:27017 + volumes: + - mongodb_data_container:/data/db + hostname: MONGO + dataengine: + build: + context: "./Defi Data Engine" + depends_on: + - mongodb_container + ports: + - 61100:61100 + - 61200:61200 + hostname: DataEngine + restapp: + build: + context: "./Rest Application" + depends_on: + - dataengine + ports: + - 8080:8080 + hostname: RestApp + + + +volumes: + mongodb_data_container: From 840e5e78385607200b51c281c0011768731f0ca0 Mon Sep 17 00:00:00 2001 From: "Adams, Kacy" Date: Fri, 9 Dec 2022 11:40:17 -0500 Subject: [PATCH 02/14] Update docker-compose.yml --- DeFi-Data-Engine/docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DeFi-Data-Engine/docker-compose.yml b/DeFi-Data-Engine/docker-compose.yml index b1a7662c..2dab1c72 100644 --- a/DeFi-Data-Engine/docker-compose.yml +++ b/DeFi-Data-Engine/docker-compose.yml @@ -9,7 +9,7 @@ services: hostname: MONGO dataengine: build: - context: "./Defi Data Engine" + context: "/Defi Data Engine" depends_on: - mongodb_container ports: @@ -18,7 +18,7 @@ services: hostname: DataEngine restapp: build: - context: "./Rest Application" + context: "/Rest Application" depends_on: - dataengine ports: From 1d812c4d0b584c2a6b73bd235da182ce10d23889 Mon Sep 17 00:00:00 2001 From: "Adams, Kacy" Date: Fri, 9 Dec 2022 11:42:27 -0500 Subject: [PATCH 03/14] Update docker-compose.yml --- DeFi-Data-Engine/docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DeFi-Data-Engine/docker-compose.yml b/DeFi-Data-Engine/docker-compose.yml index 2dab1c72..66e0ad77 100644 --- a/DeFi-Data-Engine/docker-compose.yml +++ b/DeFi-Data-Engine/docker-compose.yml @@ -9,7 +9,7 @@ services: hostname: MONGO dataengine: build: - context: "/Defi Data Engine" + context: ./Defi\ Data\ Engine depends_on: - mongodb_container ports: @@ -18,7 +18,7 @@ services: hostname: DataEngine restapp: build: - context: "/Rest Application" + context: ./Rest\ Application depends_on: - dataengine ports: From 8b4122489f4c2d40b85fa4597d0442495c6e528d Mon Sep 17 00:00:00 2001 From: "Adams, Kacy" Date: Fri, 9 Dec 2022 11:44:32 -0500 Subject: [PATCH 04/14] Update docker-compose.yml --- DeFi-Data-Engine/docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DeFi-Data-Engine/docker-compose.yml b/DeFi-Data-Engine/docker-compose.yml index 66e0ad77..b90b28a8 100644 --- a/DeFi-Data-Engine/docker-compose.yml +++ b/DeFi-Data-Engine/docker-compose.yml @@ -9,7 +9,7 @@ services: hostname: MONGO dataengine: build: - context: ./Defi\ Data\ Engine + context: ./Defi\ Data\ Engine/ depends_on: - mongodb_container ports: @@ -18,7 +18,7 @@ services: hostname: DataEngine restapp: build: - context: ./Rest\ Application + context: ./Rest\ Application/ depends_on: - dataengine ports: From 0ff652528fbb4fa2bc3d04aeed6411c493a59361 Mon Sep 17 00:00:00 2001 From: "Adams, Kacy" Date: Fri, 9 Dec 2022 11:45:56 -0500 Subject: [PATCH 05/14] Update docker-compose.yml --- DeFi-Data-Engine/docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DeFi-Data-Engine/docker-compose.yml b/DeFi-Data-Engine/docker-compose.yml index b90b28a8..8b679ae4 100644 --- a/DeFi-Data-Engine/docker-compose.yml +++ b/DeFi-Data-Engine/docker-compose.yml @@ -9,7 +9,7 @@ services: hostname: MONGO dataengine: build: - context: ./Defi\ Data\ Engine/ + context: "DeFi Data Engine" depends_on: - mongodb_container ports: @@ -18,7 +18,7 @@ services: hostname: DataEngine restapp: build: - context: ./Rest\ Application/ + context: "Rest Application" depends_on: - dataengine ports: From bfad867fade5c9393aefb32a9ce8c9a8ec83018b Mon Sep 17 00:00:00 2001 From: Kadams35 Date: Fri, 9 Dec 2022 11:51:26 -0500 Subject: [PATCH 06/14] adjust vm config --- .../Rest Application/src/main/java/org/properties/Config.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DeFi-Data-Engine/Rest Application/src/main/java/org/properties/Config.java b/DeFi-Data-Engine/Rest Application/src/main/java/org/properties/Config.java index c6210aef..bd0525d0 100644 --- a/DeFi-Data-Engine/Rest Application/src/main/java/org/properties/Config.java +++ b/DeFi-Data-Engine/Rest Application/src/main/java/org/properties/Config.java @@ -36,8 +36,8 @@ public class Config { app_properties.put("general.transfer.delim", "&&&"); app_properties.put("general.data.dateformat", "yyyy-MM-dd"); app_properties.put("spring.server.port", "8080"); - app_properties.put("spring.server.address", "RestApp"); - // app_properties.put("spring.server.address", "defi-de.idea.rpi.edu"); + // app_properties.put("spring.server.address", "RestApp"); + app_properties.put("spring.server.address", "defi-de.idea.rpi.edu"); app_properties.put("rest.socket.address", "DataEngine"); app_properties.put("rest.socket.port", "61100"); app_properties.put("rest.socket.key", "rest-key-reserved"); From e199729f0058308bf6aec088d82281a1e0f1e46a Mon Sep 17 00:00:00 2001 From: Kadams35 Date: Tue, 13 Dec 2022 01:52:03 -0500 Subject: [PATCH 07/14] fix host specs for docker --- .../DeFi Data Engine/src/main/java/org/properties/Config.java | 4 ++-- .../src/main/resources/config/stream.properties | 4 ++-- .../Rest Application/src/main/java/org/properties/Config.java | 4 ++-- .../Rest Application/src/main/resources/config/app.properties | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/properties/Config.java b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/properties/Config.java index 921989b4..250e572b 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/properties/Config.java +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/properties/Config.java @@ -41,11 +41,11 @@ public class Config { Properties stream_properties = new Properties(); stream_properties.put("general.consumer.types", "socket_consumer"); stream_properties.put("general.producer.types", "socket_producer"); - stream_properties.put("rest.socket.address", "RestApp"); + stream_properties.put("rest.socket.address", "DataEngine"); stream_properties.put("rest.socket.port", "61100"); stream_properties.put("rest.socket.key", "rest-key-reserved"); // stream_properties.put("output.socket.address", "defi-de.idea.rpi.edu"); - stream_properties.put("output.socket.address", "RestApp"); + stream_properties.put("output.socket.address", "DataEngine"); stream_properties.put("output.socket.port", "61200"); stream_properties.put("local.stream.type", "mongo_db"); stream_properties.put("mongodb.properties.uri", "mongodb://MONGO:27017"); diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/config/stream.properties b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/config/stream.properties index be4e956d..3bd10e99 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/config/stream.properties +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/config/stream.properties @@ -9,7 +9,7 @@ general.producer.types=socket_producer # === REST SOCKET PROPERTIES === # Rest socket address -rest.socket.address=RestApp +rest.socket.address=DataEngine # Rest socket port rest.socket.port=61100 @@ -20,7 +20,7 @@ rest.socket.key=rest-key-reserved # === OUTPUT SOCKET PROPERTIES === # Output socket address -output.socket.address=RestApp +output.socket.address=DataEngine # Output socket port output.socket.port=61200 diff --git a/DeFi-Data-Engine/Rest Application/src/main/java/org/properties/Config.java b/DeFi-Data-Engine/Rest Application/src/main/java/org/properties/Config.java index bd0525d0..c6210aef 100644 --- a/DeFi-Data-Engine/Rest Application/src/main/java/org/properties/Config.java +++ b/DeFi-Data-Engine/Rest Application/src/main/java/org/properties/Config.java @@ -36,8 +36,8 @@ public class Config { app_properties.put("general.transfer.delim", "&&&"); app_properties.put("general.data.dateformat", "yyyy-MM-dd"); app_properties.put("spring.server.port", "8080"); - // app_properties.put("spring.server.address", "RestApp"); - app_properties.put("spring.server.address", "defi-de.idea.rpi.edu"); + app_properties.put("spring.server.address", "RestApp"); + // app_properties.put("spring.server.address", "defi-de.idea.rpi.edu"); app_properties.put("rest.socket.address", "DataEngine"); app_properties.put("rest.socket.port", "61100"); app_properties.put("rest.socket.key", "rest-key-reserved"); diff --git a/DeFi-Data-Engine/Rest Application/src/main/resources/config/app.properties b/DeFi-Data-Engine/Rest Application/src/main/resources/config/app.properties index 21da7603..049af529 100644 --- a/DeFi-Data-Engine/Rest Application/src/main/resources/config/app.properties +++ b/DeFi-Data-Engine/Rest Application/src/main/resources/config/app.properties @@ -23,7 +23,7 @@ spring.server.port=8080 # === SOCKET PROPERTIES === # Rest socket address -rest.socket.address=RestApp +rest.socket.address=DataEngine # Rest socket port rest.socket.port=61100 From 7d893f6016d7c5062283a703ef90a744b7d5e1d1 Mon Sep 17 00:00:00 2001 From: Kadams35 Date: Tue, 13 Dec 2022 18:00:17 -0500 Subject: [PATCH 08/14] change configs --- .../DeFi Data Engine/src/main/java/org/properties/Config.java | 2 +- .../src/main/resources/config/stream.properties | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/properties/Config.java b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/properties/Config.java index 250e572b..0879bcde 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/properties/Config.java +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/properties/Config.java @@ -45,7 +45,7 @@ public class Config { stream_properties.put("rest.socket.port", "61100"); stream_properties.put("rest.socket.key", "rest-key-reserved"); // stream_properties.put("output.socket.address", "defi-de.idea.rpi.edu"); - stream_properties.put("output.socket.address", "DataEngine"); + stream_properties.put("output.socket.address", "RestApp"); stream_properties.put("output.socket.port", "61200"); stream_properties.put("local.stream.type", "mongo_db"); stream_properties.put("mongodb.properties.uri", "mongodb://MONGO:27017"); diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/config/stream.properties b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/config/stream.properties index 3bd10e99..17e1b268 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/config/stream.properties +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/config/stream.properties @@ -20,7 +20,7 @@ rest.socket.key=rest-key-reserved # === OUTPUT SOCKET PROPERTIES === # Output socket address -output.socket.address=DataEngine +output.socket.address=RestApp # Output socket port output.socket.port=61200 From baba5fe009ecd9b10aaceca34c36644dff99f7fa Mon Sep 17 00:00:00 2001 From: "Adams, Kacy" Date: Tue, 13 Dec 2022 20:22:24 -0500 Subject: [PATCH 09/14] Update docker-compose.yml --- DeFi-Data-Engine/docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/DeFi-Data-Engine/docker-compose.yml b/DeFi-Data-Engine/docker-compose.yml index 8b679ae4..8f24ad09 100644 --- a/DeFi-Data-Engine/docker-compose.yml +++ b/DeFi-Data-Engine/docker-compose.yml @@ -10,6 +10,7 @@ services: dataengine: build: context: "DeFi Data Engine" + image: dataincite/defi-data-engine:latest depends_on: - mongodb_container ports: @@ -19,6 +20,7 @@ services: restapp: build: context: "Rest Application" + image: dataincite/data-engine-rest-app:latest depends_on: - dataengine ports: From 48e4f50ea027de56b8ba1b46e949f2383cc7e765 Mon Sep 17 00:00:00 2001 From: "Adams, Kacy" Date: Tue, 13 Dec 2022 20:47:14 -0500 Subject: [PATCH 10/14] Create dockerpushandrun.yml --- .github/workflows/dockerpushandrun.yml | 32 ++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 .github/workflows/dockerpushandrun.yml diff --git a/.github/workflows/dockerpushandrun.yml b/.github/workflows/dockerpushandrun.yml new file mode 100644 index 00000000..b9c3422a --- /dev/null +++ b/.github/workflows/dockerpushandrun.yml @@ -0,0 +1,32 @@ +name: CD to Docker +on: + push: + branches: [ main ] + paths: 'DeFi-Data-Engine/**' + workflow_dispatch: + +jobs: + buildDocker: + runs-on: [ self-hosted ] + defaults: + run: + working-directory: "DeFi-Data-Engine/" + + steps: + - uses: actions/checkout@v3 + + - name: Stop Existing Images + run: sudo docker compose down + + - name: Build + run: sudo docker compose build + + - name: Push to Docker + run: sudo docker compose push + + runDocker: + needs: buildDocker + runs-on: [ self-hosted ] + + - name: Run Docker Compose + run: sudo docker compose up From 8e0c9038880e3f14ba7d646e664386303afaacb4 Mon Sep 17 00:00:00 2001 From: "Adams, Kacy" Date: Tue, 13 Dec 2022 20:48:16 -0500 Subject: [PATCH 11/14] Delete maven.yml --- .github/workflows/maven.yml | 47 ------------------------------------- 1 file changed, 47 deletions(-) delete mode 100644 .github/workflows/maven.yml diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml deleted file mode 100644 index bad8b187..00000000 --- a/.github/workflows/maven.yml +++ /dev/null @@ -1,47 +0,0 @@ -name: CI with mvn -on: - push: - branches: [ main ] - paths: 'DeFi-Data-Engine/**' - pull_request: - branches: [ main ] - paths: 'DeFi-Data-Engine/**' - workflow_dispatch: - -jobs: - buildandDeployDataEngine: - - runs-on: [ self-hosted ] - defaults: - run: - working-directory: "DeFi-Data-Engine/DeFi Data Engine/" - - steps: - - uses: actions/checkout@v3 - - - name: Build with Maven - run: mvn clean compile assembly:single -Dmaven.skip.test=true - -# #Deployment will fail if there are deployments which did not come from this workflow -# - name: Check if existing deployments -# run: pkill -f 'java -jar target/defi-data-engine-0.0.1-jar-with-dependencies.jar' - - - name: Deploy data engine - run: RUNNER_TRACKING_ID="" && (java -jar target/defi-data-engine-0.0.1-jar-with-dependencies.jar &) - - buildRestAPI: - needs: buildandDeployDataEngine - runs-on: [ self-hosted ] - defaults: - run: - working-directory: "DeFi-Data-Engine/Rest Application/" - - steps: - - uses: actions/checkout@v3 - - - name: Build with Maven - run: mvn clean compile assembly:single -Dmaven.skip.test=true - - - name: Deploy rest application - run: RUNNER_TRACKING_ID="" && (java -jar target/rest-connection-4.3.3-jar-with-dependencies.jar &) - From 6dfb2b4918470a48bdddd1d98763630f40c1c4e7 Mon Sep 17 00:00:00 2001 From: "Adams, Kacy" Date: Tue, 13 Dec 2022 20:51:07 -0500 Subject: [PATCH 12/14] Update dockerpushandrun.yml --- .github/workflows/dockerpushandrun.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/dockerpushandrun.yml b/.github/workflows/dockerpushandrun.yml index b9c3422a..24f89512 100644 --- a/.github/workflows/dockerpushandrun.yml +++ b/.github/workflows/dockerpushandrun.yml @@ -28,5 +28,10 @@ jobs: needs: buildDocker runs-on: [ self-hosted ] + defaults: + run: + working-directory: "DeFi-Data-Engine/" + + steps: - name: Run Docker Compose run: sudo docker compose up From d6e1de8db615ca93603a9aa744537af318e911eb Mon Sep 17 00:00:00 2001 From: "Adams, Kacy" Date: Tue, 13 Dec 2022 21:09:01 -0500 Subject: [PATCH 13/14] Update dockerpushandrun.yml --- .github/workflows/dockerpushandrun.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dockerpushandrun.yml b/.github/workflows/dockerpushandrun.yml index 24f89512..8b5d80b4 100644 --- a/.github/workflows/dockerpushandrun.yml +++ b/.github/workflows/dockerpushandrun.yml @@ -33,5 +33,8 @@ jobs: working-directory: "DeFi-Data-Engine/" steps: - - name: Run Docker Compose - run: sudo docker compose up + - name: Create Containers + run: sudo docker compose create + + - name: Start services + run: sudo docker compose start From f09e27df79f08be90a0f3d65c90a252c308b538e Mon Sep 17 00:00:00 2001 From: Kacy Adams Date: Wed, 14 Dec 2022 18:03:33 -0500 Subject: [PATCH 14/14] update r code engine samples --- R-Code-Samples/aave-protocol-dated-func.Rmd | 77 +++++++++++---------- 1 file changed, 40 insertions(+), 37 deletions(-) diff --git a/R-Code-Samples/aave-protocol-dated-func.Rmd b/R-Code-Samples/aave-protocol-dated-func.Rmd index 277806c9..d6636421 100644 --- a/R-Code-Samples/aave-protocol-dated-func.Rmd +++ b/R-Code-Samples/aave-protocol-dated-func.Rmd @@ -4,55 +4,46 @@ library("httr") library("jsonlite") library("lubridate") -``` - -This function sets up an Engine socket and makes the get request. The engine feeds the socket while -the function listens and parses the data, and then returns it as a data frame. -```{R} - -getDF <- function(startdate, enddate) { - engine_key <- "b6c810a7f35f4fa0d28258278325b4b5ab82ba79868ab33d01d5c878e13872ec129a91a3fbf702e59c2404f0fb4a53420a3ffb50130c35b4d06b32d81e56c1f4" +getJson <- function(startdate, enddate) { - socket <- make.socket("defi-de.idea.rpi.edu", 61200) - - ss <- read.socket(socket) - ss <- gsub("[\r\n]", "", ss) + httr::GET(url = "http://defi-de.idea.rpi.edu:8080/defi/v1/rest/initialize?source=amber_data&auth_data=key,UAK7ed69235426c360be22bfc2bde1809b6") + engine_key <- "b6c810a7f35f4fa0d28258278325b4b5ab82ba79868ab33d01d5c878e13872ec129a91a3fbf702e59c2404f0fb4a53420a3ffb50130c35b4d06b32d81e56c1f4" + socket <- socketConnection("defi-de.idea.rpi.edu", 61200, blocking=TRUE) + ss <- readLines(socket, 1) + ss - httr::GET( - url = "http://defi-de.idea.rpi.edu:8080/defi/v1/rest/request_dated", - query = list( - destination = ss, - key = engine_key, - request = "aave-protocol-dated", - query = "get_all,aave-protocol_dated", - start_date = startdate, - end_date = enddate - ) - ) + reqString <- paste( + "SRC&&&RQST&&&destination&&&", ss, + "&&&key&&&", engine_key, "&&&", + "start_date&&&", startdate, "&&&", + "end_date&&&", enddate, "&&&", + "query&&&aave-protocol-dated&&&", + "request&&&aave-protocol-dated\n", sep="") - data <- read.socket(socket) + writeLines(reqString, socket) + data <- readLines(socket, 1) while (TRUE) { - temp <- read.socket(socket) + temp <- readLines(socket, 1) data <- paste(data, temp, "") if (grepl("<<>>", temp, fixed=TRUE)) {break} } + data - data <- str_replace(data, "\n<<>>\n ", "") - data <- str_replace_all(data, "\n", ",") - data <- sub(',', '', data) + data <- str_replace(data, " <<>> ", "") data <- str_replace_all(data, " ", "") + data <- str_replace_all(data, "\\}\\{", "\\},\\{") data <- paste("[", data, "]", sep="") output <- fromJSON(data) output <- output[,-(colnames(output) == "_id")] colnames(output)[colnames(output) == "action"] = "type" output$type <- as.factor(output$type) - output <- output %>% mutate(datetime = as_datetime(as.numeric(timestamp))) - + output <- output %>% mutate(datetime = as_datetime(as.numeric(substr(timestamp, 1, nchar(timestamp)-3)))) + colnames(output)[colnames(output) == "assetId"] = "reserveId" colnames(output)[colnames(output) == "assetSymbol"] = "reserve" @@ -62,8 +53,8 @@ getDF <- function(startdate, enddate) { colnames(output)[colnames(output) == "user"] = "userId" output <- output %>% unite(col = "onBehalfOfId", onBehalfOf,repayer,initiator, na.rm = TRUE, sep = "") - - + + colnames(output)[colnames(output) == "collateralAssetId"] = "collateralReserveId" colnames(output)[colnames(output) == "collateralAssetSymbol"] = "collateralReserve" colnames(output)[colnames(output) == "principalAssetId"] = "principalReserveId" @@ -71,19 +62,31 @@ getDF <- function(startdate, enddate) { output$principalAmount <- as.double(output$principalAmount) output <- output[,-15] - + output return(output) } +temp <- getJson("2022-08-01", "2022-09-01") +temp + ``` +```{r} +library(dplyr) -We make a sample call to the function which takes the start date and end date and receive back -a data frame containing the aave data. -```{R} +weeklySummaries <- temp %>% + mutate(week = floor_date(datetime, unit = "day")) %>% # Add a new column that rounds the date of each transaction down to the nearest week + group_by(week) %>% # Group the transactions together by the week they were performed. + summarise(transactionCount = n()) # Count the number of transactions in each group. +``` -temp <- getDF("2022-08-01", "2022-08-04") +With these weekly summaries computed, we can simply plot the week on the x-axis and the transaction count on the y-axis to visualize this new representation: +```{r} +weeklyTransactionsPlot <- ggplot(data = weeklySummaries, aes(x = week, y = transactionCount)) + geom_line() +weeklyTransactionsPlot ``` + +