diff --git a/.github/workflows/dockerpushandrun.yml b/.github/workflows/dockerpushandrun.yml new file mode 100644 index 00000000..8b5d80b4 --- /dev/null +++ b/.github/workflows/dockerpushandrun.yml @@ -0,0 +1,40 @@ +name: CD to Docker +on: + push: + branches: [ main ] + paths: 'DeFi-Data-Engine/**' + workflow_dispatch: + +jobs: + buildDocker: + runs-on: [ self-hosted ] + defaults: + run: + working-directory: "DeFi-Data-Engine/" + + steps: + - uses: actions/checkout@v3 + + - name: Stop Existing Images + run: sudo docker compose down + + - name: Build + run: sudo docker compose build + + - name: Push to Docker + run: sudo docker compose push + + runDocker: + needs: buildDocker + runs-on: [ self-hosted ] + + defaults: + run: + working-directory: "DeFi-Data-Engine/" + + steps: + - name: Create Containers + run: sudo docker compose create + + - name: Start services + run: sudo docker compose start diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml deleted file mode 100644 index bad8b187..00000000 --- a/.github/workflows/maven.yml +++ /dev/null @@ -1,47 +0,0 @@ -name: CI with mvn -on: - push: - branches: [ main ] - paths: 'DeFi-Data-Engine/**' - pull_request: - branches: [ main ] - paths: 'DeFi-Data-Engine/**' - workflow_dispatch: - -jobs: - buildandDeployDataEngine: - - runs-on: [ self-hosted ] - defaults: - run: - working-directory: "DeFi-Data-Engine/DeFi Data Engine/" - - steps: - - uses: actions/checkout@v3 - - - name: Build with Maven - run: mvn clean compile assembly:single -Dmaven.skip.test=true - -# #Deployment will fail if there are deployments which did not come from this workflow -# - name: Check if existing deployments -# run: pkill -f 'java -jar target/defi-data-engine-0.0.1-jar-with-dependencies.jar' - - - name: Deploy data engine - run: RUNNER_TRACKING_ID="" && (java -jar target/defi-data-engine-0.0.1-jar-with-dependencies.jar &) - - buildRestAPI: - needs: buildandDeployDataEngine - runs-on: [ self-hosted ] - defaults: - run: - working-directory: "DeFi-Data-Engine/Rest Application/" - - steps: - - uses: actions/checkout@v3 - - - name: Build with Maven - run: mvn clean compile assembly:single -Dmaven.skip.test=true - - - name: Deploy rest application - run: RUNNER_TRACKING_ID="" && (java -jar target/rest-connection-4.3.3-jar-with-dependencies.jar &) - diff --git a/DeFi-Data-Engine/DeFi Data Engine/.classpath b/DeFi-Data-Engine/DeFi Data Engine/.classpath index 4d455a8c..b2e8bc05 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/.classpath +++ b/DeFi-Data-Engine/DeFi Data Engine/.classpath @@ -25,12 +25,33 @@ - + + + + + + + + + + + + + + + + + + + + + + diff --git a/DeFi-Data-Engine/DeFi Data Engine/.settings/org.eclipse.jdt.apt.core.prefs b/DeFi-Data-Engine/DeFi Data Engine/.settings/org.eclipse.jdt.apt.core.prefs new file mode 100644 index 00000000..d4313d4b --- /dev/null +++ b/DeFi-Data-Engine/DeFi Data Engine/.settings/org.eclipse.jdt.apt.core.prefs @@ -0,0 +1,2 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.apt.aptEnabled=false diff --git a/DeFi-Data-Engine/DeFi Data Engine/.settings/org.eclipse.jdt.core.prefs b/DeFi-Data-Engine/DeFi Data Engine/.settings/org.eclipse.jdt.core.prefs index ecb498c8..b5e07ce6 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/.settings/org.eclipse.jdt.core.prefs +++ b/DeFi-Data-Engine/DeFi Data Engine/.settings/org.eclipse.jdt.core.prefs @@ -1,9 +1,9 @@ eclipse.preferences.version=1 org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate -org.eclipse.jdt.core.compiler.codegen.targetPlatform=17 +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve -org.eclipse.jdt.core.compiler.compliance=17 +org.eclipse.jdt.core.compiler.compliance=1.8 org.eclipse.jdt.core.compiler.debug.lineNumber=generate org.eclipse.jdt.core.compiler.debug.localVariable=generate org.eclipse.jdt.core.compiler.debug.sourceFile=generate @@ -12,5 +12,6 @@ org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled org.eclipse.jdt.core.compiler.problem.enumIdentifier=error org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=warning -org.eclipse.jdt.core.compiler.release=enabled -org.eclipse.jdt.core.compiler.source=17 +org.eclipse.jdt.core.compiler.processAnnotations=disabled +org.eclipse.jdt.core.compiler.release=disabled +org.eclipse.jdt.core.compiler.source=1.8 diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/properties/Config.java b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/properties/Config.java index c0587a74..d7ba8960 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/properties/Config.java +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/java/org/properties/Config.java @@ -41,14 +41,19 @@ public class Config { Properties stream_properties = new Properties(); stream_properties.put("general.consumer.types", "socket_consumer"); stream_properties.put("general.producer.types", "socket_producer"); - stream_properties.put("rest.socket.address", "localhost"); + stream_properties.put("rest.socket.address", "DataEngine"); stream_properties.put("rest.socket.port", "61100"); stream_properties.put("rest.socket.key", "rest-key-reserved"); +<<<<<<< HEAD //stream_properties.put("output.socket.address", "defi-de.idea.rpi.edu"); stream_properties.put("output.socket.address", "localhost"); +======= + // stream_properties.put("output.socket.address", "defi-de.idea.rpi.edu"); + stream_properties.put("output.socket.address", "RestApp"); +>>>>>>> branch 'main' of https://github.rpi.edu/DataINCITE/IDEA-DeFi-CRAFT stream_properties.put("output.socket.port", "61200"); stream_properties.put("local.stream.type", "mongo_db"); - stream_properties.put("mongodb.properties.uri", "mongodb://localhost:27017"); + stream_properties.put("mongodb.properties.uri", "mongodb://MONGO:27017"); stream_properties.put("mongodb.database.state", "main-state-db"); stream_properties.put("mongodb.database.main", "main-db"); stream_properties.put("mongodb.auth.collection", "auth-collection"); diff --git a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/config/stream.properties b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/config/stream.properties index 4bb2109a..17e1b268 100644 --- a/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/config/stream.properties +++ b/DeFi-Data-Engine/DeFi Data Engine/src/main/resources/config/stream.properties @@ -9,7 +9,7 @@ general.producer.types=socket_producer # === REST SOCKET PROPERTIES === # Rest socket address -rest.socket.address=localhost +rest.socket.address=DataEngine # Rest socket port rest.socket.port=61100 @@ -20,7 +20,7 @@ rest.socket.key=rest-key-reserved # === OUTPUT SOCKET PROPERTIES === # Output socket address -output.socket.address=localhost +output.socket.address=RestApp # Output socket port output.socket.port=61200 @@ -33,7 +33,7 @@ local.stream.type=mongo_db # === MONGO DB PROPERTIES === # local MongoDB client URI -mongodb.properties.uri=mongodb://localhost:27017 +mongodb.properties.uri=mongodb://MONGO:27017 # local MongoDB state database name mongodb.database.state=main-state-db diff --git a/DeFi-Data-Engine/Rest Application/.classpath b/DeFi-Data-Engine/Rest Application/.classpath index d118e897..e90905c2 100644 --- a/DeFi-Data-Engine/Rest Application/.classpath +++ b/DeFi-Data-Engine/Rest Application/.classpath @@ -18,21 +18,31 @@ - + - - + - - + + + + + + + + + + + + + diff --git a/DeFi-Data-Engine/Rest Application/.settings/org.eclipse.core.resources.prefs b/DeFi-Data-Engine/Rest Application/.settings/org.eclipse.core.resources.prefs index 609d3cab..29abf999 100644 --- a/DeFi-Data-Engine/Rest Application/.settings/org.eclipse.core.resources.prefs +++ b/DeFi-Data-Engine/Rest Application/.settings/org.eclipse.core.resources.prefs @@ -1,4 +1,6 @@ eclipse.preferences.version=1 +encoding//src/main/java=UTF-8 encoding//src/main/resources=UTF-8 +encoding//src/test/java=UTF-8 encoding//src/test/resources=UTF-8 encoding/=UTF-8 diff --git a/DeFi-Data-Engine/Rest Application/.settings/org.eclipse.jdt.apt.core.prefs b/DeFi-Data-Engine/Rest Application/.settings/org.eclipse.jdt.apt.core.prefs new file mode 100644 index 00000000..d4313d4b --- /dev/null +++ b/DeFi-Data-Engine/Rest Application/.settings/org.eclipse.jdt.apt.core.prefs @@ -0,0 +1,2 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.apt.aptEnabled=false diff --git a/DeFi-Data-Engine/Rest Application/.settings/org.eclipse.jdt.core.prefs b/DeFi-Data-Engine/Rest Application/.settings/org.eclipse.jdt.core.prefs index b041b83d..23731ecc 100644 --- a/DeFi-Data-Engine/Rest Application/.settings/org.eclipse.jdt.core.prefs +++ b/DeFi-Data-Engine/Rest Application/.settings/org.eclipse.jdt.core.prefs @@ -12,5 +12,6 @@ org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled org.eclipse.jdt.core.compiler.problem.enumIdentifier=error org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=warning +org.eclipse.jdt.core.compiler.processAnnotations=disabled org.eclipse.jdt.core.compiler.release=disabled org.eclipse.jdt.core.compiler.source=17 diff --git a/DeFi-Data-Engine/Rest Application/Dockerfile b/DeFi-Data-Engine/Rest Application/Dockerfile index bdbe8984..c87bd6cf 100644 --- a/DeFi-Data-Engine/Rest Application/Dockerfile +++ b/DeFi-Data-Engine/Rest Application/Dockerfile @@ -1,9 +1,4 @@ FROM maven:3.8.6-eclipse-temurin-17 - COPY ./ ./ - -# package our application code RUN mvn clean package -Dmaven.test.skip -EXPOSE 8080 -# set the startup command to execute the jar CMD ["java", "-jar", "target/rest-connection-4.3.3.jar"] \ No newline at end of file diff --git a/DeFi-Data-Engine/Rest Application/src/main/java/org/properties/Config.java b/DeFi-Data-Engine/Rest Application/src/main/java/org/properties/Config.java index 3cfee27f..3b2872cd 100644 --- a/DeFi-Data-Engine/Rest Application/src/main/java/org/properties/Config.java +++ b/DeFi-Data-Engine/Rest Application/src/main/java/org/properties/Config.java @@ -36,9 +36,15 @@ public class Config { app_properties.put("general.transfer.delim", "&&&"); app_properties.put("general.data.dateformat", "yyyy-MM-dd"); app_properties.put("spring.server.port", "8080"); +<<<<<<< HEAD app_properties.put("spring.server.address", "localhost"); //app_properties.put("spring.server.address", "defi-de.idea.rpi.edu"); app_properties.put("rest.socket.address", "localhost"); +======= + app_properties.put("spring.server.address", "RestApp"); + // app_properties.put("spring.server.address", "defi-de.idea.rpi.edu"); + app_properties.put("rest.socket.address", "DataEngine"); +>>>>>>> branch 'main' of https://github.rpi.edu/DataINCITE/IDEA-DeFi-CRAFT app_properties.put("rest.socket.port", "61100"); app_properties.put("rest.socket.key", "rest-key-reserved"); properties.put("app", app_properties); diff --git a/DeFi-Data-Engine/Rest Application/src/main/resources/config/app.properties b/DeFi-Data-Engine/Rest Application/src/main/resources/config/app.properties index 11c5215b..049af529 100644 --- a/DeFi-Data-Engine/Rest Application/src/main/resources/config/app.properties +++ b/DeFi-Data-Engine/Rest Application/src/main/resources/config/app.properties @@ -23,7 +23,7 @@ spring.server.port=8080 # === SOCKET PROPERTIES === # Rest socket address -rest.socket.address=localhost +rest.socket.address=DataEngine # Rest socket port rest.socket.port=61100 diff --git a/DeFi-Data-Engine/Testing Environment/.classpath b/DeFi-Data-Engine/Testing Environment/.classpath index cddbf79a..58cbceb6 100644 --- a/DeFi-Data-Engine/Testing Environment/.classpath +++ b/DeFi-Data-Engine/Testing Environment/.classpath @@ -6,9 +6,8 @@ - + - @@ -17,5 +16,21 @@ + + + + + + + + + + + + + + + + diff --git a/DeFi-Data-Engine/Testing Environment/.settings/org.eclipse.jdt.apt.core.prefs b/DeFi-Data-Engine/Testing Environment/.settings/org.eclipse.jdt.apt.core.prefs new file mode 100644 index 00000000..d4313d4b --- /dev/null +++ b/DeFi-Data-Engine/Testing Environment/.settings/org.eclipse.jdt.apt.core.prefs @@ -0,0 +1,2 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.apt.aptEnabled=false diff --git a/DeFi-Data-Engine/Testing Environment/.settings/org.eclipse.jdt.core.prefs b/DeFi-Data-Engine/Testing Environment/.settings/org.eclipse.jdt.core.prefs index 43c8d716..b2ef4d6e 100644 --- a/DeFi-Data-Engine/Testing Environment/.settings/org.eclipse.jdt.core.prefs +++ b/DeFi-Data-Engine/Testing Environment/.settings/org.eclipse.jdt.core.prefs @@ -11,5 +11,6 @@ org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled org.eclipse.jdt.core.compiler.problem.enumIdentifier=error org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=warning +org.eclipse.jdt.core.compiler.processAnnotations=disabled org.eclipse.jdt.core.compiler.release=enabled org.eclipse.jdt.core.compiler.source=17 diff --git a/DeFi-Data-Engine/docker-compose.yml b/DeFi-Data-Engine/docker-compose.yml new file mode 100644 index 00000000..8f24ad09 --- /dev/null +++ b/DeFi-Data-Engine/docker-compose.yml @@ -0,0 +1,33 @@ +version: "3.9" +services: + mongodb_container: + image: mongo:latest + ports: + - 27017:27017 + volumes: + - mongodb_data_container:/data/db + hostname: MONGO + dataengine: + build: + context: "DeFi Data Engine" + image: dataincite/defi-data-engine:latest + depends_on: + - mongodb_container + ports: + - 61100:61100 + - 61200:61200 + hostname: DataEngine + restapp: + build: + context: "Rest Application" + image: dataincite/data-engine-rest-app:latest + depends_on: + - dataengine + ports: + - 8080:8080 + hostname: RestApp + + + +volumes: + mongodb_data_container: diff --git a/R-Code-Samples/aave-protocol-dated-func.Rmd b/R-Code-Samples/aave-protocol-dated-func.Rmd index 277806c9..d6636421 100644 --- a/R-Code-Samples/aave-protocol-dated-func.Rmd +++ b/R-Code-Samples/aave-protocol-dated-func.Rmd @@ -4,55 +4,46 @@ library("httr") library("jsonlite") library("lubridate") -``` - -This function sets up an Engine socket and makes the get request. The engine feeds the socket while -the function listens and parses the data, and then returns it as a data frame. -```{R} - -getDF <- function(startdate, enddate) { - engine_key <- "b6c810a7f35f4fa0d28258278325b4b5ab82ba79868ab33d01d5c878e13872ec129a91a3fbf702e59c2404f0fb4a53420a3ffb50130c35b4d06b32d81e56c1f4" +getJson <- function(startdate, enddate) { - socket <- make.socket("defi-de.idea.rpi.edu", 61200) - - ss <- read.socket(socket) - ss <- gsub("[\r\n]", "", ss) + httr::GET(url = "http://defi-de.idea.rpi.edu:8080/defi/v1/rest/initialize?source=amber_data&auth_data=key,UAK7ed69235426c360be22bfc2bde1809b6") + engine_key <- "b6c810a7f35f4fa0d28258278325b4b5ab82ba79868ab33d01d5c878e13872ec129a91a3fbf702e59c2404f0fb4a53420a3ffb50130c35b4d06b32d81e56c1f4" + socket <- socketConnection("defi-de.idea.rpi.edu", 61200, blocking=TRUE) + ss <- readLines(socket, 1) + ss - httr::GET( - url = "http://defi-de.idea.rpi.edu:8080/defi/v1/rest/request_dated", - query = list( - destination = ss, - key = engine_key, - request = "aave-protocol-dated", - query = "get_all,aave-protocol_dated", - start_date = startdate, - end_date = enddate - ) - ) + reqString <- paste( + "SRC&&&RQST&&&destination&&&", ss, + "&&&key&&&", engine_key, "&&&", + "start_date&&&", startdate, "&&&", + "end_date&&&", enddate, "&&&", + "query&&&aave-protocol-dated&&&", + "request&&&aave-protocol-dated\n", sep="") - data <- read.socket(socket) + writeLines(reqString, socket) + data <- readLines(socket, 1) while (TRUE) { - temp <- read.socket(socket) + temp <- readLines(socket, 1) data <- paste(data, temp, "") if (grepl("<<>>", temp, fixed=TRUE)) {break} } + data - data <- str_replace(data, "\n<<>>\n ", "") - data <- str_replace_all(data, "\n", ",") - data <- sub(',', '', data) + data <- str_replace(data, " <<>> ", "") data <- str_replace_all(data, " ", "") + data <- str_replace_all(data, "\\}\\{", "\\},\\{") data <- paste("[", data, "]", sep="") output <- fromJSON(data) output <- output[,-(colnames(output) == "_id")] colnames(output)[colnames(output) == "action"] = "type" output$type <- as.factor(output$type) - output <- output %>% mutate(datetime = as_datetime(as.numeric(timestamp))) - + output <- output %>% mutate(datetime = as_datetime(as.numeric(substr(timestamp, 1, nchar(timestamp)-3)))) + colnames(output)[colnames(output) == "assetId"] = "reserveId" colnames(output)[colnames(output) == "assetSymbol"] = "reserve" @@ -62,8 +53,8 @@ getDF <- function(startdate, enddate) { colnames(output)[colnames(output) == "user"] = "userId" output <- output %>% unite(col = "onBehalfOfId", onBehalfOf,repayer,initiator, na.rm = TRUE, sep = "") - - + + colnames(output)[colnames(output) == "collateralAssetId"] = "collateralReserveId" colnames(output)[colnames(output) == "collateralAssetSymbol"] = "collateralReserve" colnames(output)[colnames(output) == "principalAssetId"] = "principalReserveId" @@ -71,19 +62,31 @@ getDF <- function(startdate, enddate) { output$principalAmount <- as.double(output$principalAmount) output <- output[,-15] - + output return(output) } +temp <- getJson("2022-08-01", "2022-09-01") +temp + ``` +```{r} +library(dplyr) -We make a sample call to the function which takes the start date and end date and receive back -a data frame containing the aave data. -```{R} +weeklySummaries <- temp %>% + mutate(week = floor_date(datetime, unit = "day")) %>% # Add a new column that rounds the date of each transaction down to the nearest week + group_by(week) %>% # Group the transactions together by the week they were performed. + summarise(transactionCount = n()) # Count the number of transactions in each group. +``` -temp <- getDF("2022-08-01", "2022-08-04") +With these weekly summaries computed, we can simply plot the week on the x-axis and the transaction count on the y-axis to visualize this new representation: +```{r} +weeklyTransactionsPlot <- ggplot(data = weeklySummaries, aes(x = week, y = transactionCount)) + geom_line() +weeklyTransactionsPlot ``` + +