From 70fac2fc6a4902353e729339d40f7afdaa970437 Mon Sep 17 00:00:00 2001 From: Conor Flynn Date: Thu, 6 Apr 2023 13:29:30 -0400 Subject: [PATCH] Update R files and clean directory --- .../apihandler/ApiHandlerApplication.java | 2 +- .../request/types/RequestFramework.java | 33 +- .../request/types/RequestParameterized.java | 3 +- .../apihandler/SampleBlockchainAddresses.java | 32 -- .../apihandler/SampleMakerDAOProtocol.java | 32 -- .../amberdata-uniswap-pool.properties | 30 ++ .../~$Documentation.xlsx | Bin 0 -> 165 bytes DeFi-Data-Engine/Rest Application/.classpath | 49 --- DeFi-Data-Engine/Rest Application/.gitignore | 2 - DeFi-Data-Engine/Rest Application/.project | 23 - .../org.eclipse.core.resources.prefs | 6 - .../.settings/org.eclipse.jdt.apt.core.prefs | 2 - .../.settings/org.eclipse.jdt.core.prefs | 17 - .../.settings/org.eclipse.m2e.core.prefs | 4 - DeFi-Data-Engine/Rest Application/Dockerfile | 4 - DeFi-Data-Engine/Rest Application/pom.xml | 76 ---- .../src/main/java/org/properties/Config.java | 60 --- .../java/org/rest/application/Endpoint.java | 165 ------- .../org/rest/application/RestApplication.java | 22 - .../rest/application/ServletInitializer.java | 13 - .../src/main/resources/config/app.properties | 32 -- .../RestconnectionApplicationTests.java | 13 - ...ons.Rmd => DataEnginePrimaryFunctions.Rmd} | 85 ++-- .../ExampleUserClusteringStarter.Rmd | 413 ++++++++++++++++++ 24 files changed, 514 insertions(+), 604 deletions(-) delete mode 100644 DeFi-Data-Engine/Api-Handler/src/main/java/test/application/apihandler/SampleBlockchainAddresses.java delete mode 100644 DeFi-Data-Engine/Api-Handler/src/main/java/test/application/apihandler/SampleMakerDAOProtocol.java create mode 100644 DeFi-Data-Engine/Api-Handler/src/main/resources/requests/amberdata-uniswap-pool.properties create mode 100644 DeFi-Data-Engine/Documentation/Internal-Documentation/~$Documentation.xlsx delete mode 100644 DeFi-Data-Engine/Rest Application/.classpath delete mode 100644 DeFi-Data-Engine/Rest Application/.gitignore delete mode 100644 DeFi-Data-Engine/Rest Application/.project delete mode 100644 DeFi-Data-Engine/Rest Application/.settings/org.eclipse.core.resources.prefs delete mode 100644 DeFi-Data-Engine/Rest Application/.settings/org.eclipse.jdt.apt.core.prefs delete mode 100644 DeFi-Data-Engine/Rest Application/.settings/org.eclipse.jdt.core.prefs delete mode 100644 DeFi-Data-Engine/Rest Application/.settings/org.eclipse.m2e.core.prefs delete mode 100644 DeFi-Data-Engine/Rest Application/Dockerfile delete mode 100644 DeFi-Data-Engine/Rest Application/pom.xml delete mode 100644 DeFi-Data-Engine/Rest Application/src/main/java/org/properties/Config.java delete mode 100644 DeFi-Data-Engine/Rest Application/src/main/java/org/rest/application/Endpoint.java delete mode 100644 DeFi-Data-Engine/Rest Application/src/main/java/org/rest/application/RestApplication.java delete mode 100644 DeFi-Data-Engine/Rest Application/src/main/java/org/rest/application/ServletInitializer.java delete mode 100644 DeFi-Data-Engine/Rest Application/src/main/resources/config/app.properties delete mode 100644 DeFi-Data-Engine/Rest Application/src/test/java/test/rest/application/RestconnectionApplicationTests.java rename R-Code-Samples/{data-engine-r-functions.Rmd => DataEnginePrimaryFunctions.Rmd} (66%) create mode 100644 R-Code-Samples/ExampleUserClusteringStarter.Rmd diff --git a/DeFi-Data-Engine/Api-Handler/src/main/java/org/application/apihandler/ApiHandlerApplication.java b/DeFi-Data-Engine/Api-Handler/src/main/java/org/application/apihandler/ApiHandlerApplication.java index d3c54b7a..19994bba 100644 --- a/DeFi-Data-Engine/Api-Handler/src/main/java/org/application/apihandler/ApiHandlerApplication.java +++ b/DeFi-Data-Engine/Api-Handler/src/main/java/org/application/apihandler/ApiHandlerApplication.java @@ -89,7 +89,7 @@ public final static void unlock(String name) { // loop through all headers and format for(String header : headers.get(name)) { if(buffer.get(i).containsKey(header)) { - line.append(buffer.get(i).get(header)); + line.append(buffer.get(i).get(header).replaceAll(",", "|")); } line.append(","); diff --git a/DeFi-Data-Engine/Api-Handler/src/main/java/org/stream/external/request/types/RequestFramework.java b/DeFi-Data-Engine/Api-Handler/src/main/java/org/stream/external/request/types/RequestFramework.java index 99e6b12d..cd437ddd 100644 --- a/DeFi-Data-Engine/Api-Handler/src/main/java/org/stream/external/request/types/RequestFramework.java +++ b/DeFi-Data-Engine/Api-Handler/src/main/java/org/stream/external/request/types/RequestFramework.java @@ -139,7 +139,8 @@ protected final Request getRequest(String url, HashMap propertie } // remove final & and update builder - url_builder.deleteCharAt(url_builder.length() - 1); + if(!properties.isEmpty()) + url_builder.deleteCharAt(url_builder.length() - 1); builder = builder.url(url_builder.toString()); // add all headers @@ -263,6 +264,7 @@ public final synchronized String request(String[] url_path, HashMap headers) { + HashMap all_headers = new HashMap(); + + for(String header : this.headers.keySet()) + all_headers.put(header, this.headers.get(header)); + + for(String header : headers.keySet()) + all_headers.put(header, headers.get(header)); + + return processRequest(url, new HashMap(), all_headers); + } + protected String process(String url, HashMap properties, HashMap headers) { HashMap all_properties = new HashMap(); HashMap all_headers = new HashMap(); @@ -279,17 +293,21 @@ protected String process(String url, HashMap properties, HashMap for(String property : this.properties.keySet()) all_properties.put(property, this.properties.get(property)); - for(String header : this.headers.keySet()) - all_headers.put(header, this.headers.get(header)); - for(String property : properties.keySet()) all_properties.put(property, properties.get(property)); + for(String header : this.headers.keySet()) + all_headers.put(header, this.headers.get(header)); + for(String header : headers.keySet()) all_headers.put(header, headers.get(header)); + return processRequest(url, all_properties, all_headers); + } + + private String processRequest(String url, HashMap properties, HashMap headers) { OkHttpClient client = new OkHttpClient(); - Request request = getRequest(url, all_properties, all_headers); + Request request = getRequest(url, properties, headers); if(request == null) { System.err.println("Malformed request, killing process."); return "Malformed request, killing process."; @@ -301,8 +319,7 @@ protected String process(String url, HashMap properties, HashMap response = client.newCall(request).execute(); body = response.body().string().toString(); if(response.code() != 200) { - System.err.println(String.format("Request Failure code <%d> url <%s>\nbody:\n%s", response.code(), request.url().toString(), body)); - return String.format("Request Failure code <%d> url <%s>\nbody:\n%s", response.code(), request.url().toString(), body); + return String.format("Request Failure code=<%d> url=<%s> body=<%s>", response.code(), request.url().toString(), body); } } catch (IOException e) { e.printStackTrace(); @@ -315,7 +332,7 @@ protected String process(String url, HashMap properties, HashMap // send to specific request handler try { - handle(body, all_properties, all_headers); + handle(body, properties, headers); } catch(Exception e) { e.printStackTrace(); return e.toString(); diff --git a/DeFi-Data-Engine/Api-Handler/src/main/java/org/stream/external/request/types/RequestParameterized.java b/DeFi-Data-Engine/Api-Handler/src/main/java/org/stream/external/request/types/RequestParameterized.java index f7072ef9..20332884 100644 --- a/DeFi-Data-Engine/Api-Handler/src/main/java/org/stream/external/request/types/RequestParameterized.java +++ b/DeFi-Data-Engine/Api-Handler/src/main/java/org/stream/external/request/types/RequestParameterized.java @@ -139,7 +139,7 @@ else if(obj.has(path[i])) { case "url": properties.clear(); url = recursive_parameter; - break; + return processUrl(url, headers); // type incremental: // - assert that parameter is an integer @@ -174,6 +174,7 @@ else if(obj.has(path[i])) { } else { properties.put(getRecursiveLocation()[0], recursive_parameter); } + break; } // increment param, replace, and execute diff --git a/DeFi-Data-Engine/Api-Handler/src/main/java/test/application/apihandler/SampleBlockchainAddresses.java b/DeFi-Data-Engine/Api-Handler/src/main/java/test/application/apihandler/SampleBlockchainAddresses.java deleted file mode 100644 index 793ff991..00000000 --- a/DeFi-Data-Engine/Api-Handler/src/main/java/test/application/apihandler/SampleBlockchainAddresses.java +++ /dev/null @@ -1,32 +0,0 @@ -package test.application.apihandler; - -public class SampleBlockchainAddresses { - - // disabled for docker composition - -// public static void main(String[] args) { -// HashMap properties = new HashMap(); -// HashMap headers = new HashMap(); -// HashMap tags = new HashMap(); -// -// properties.put("page", "0"); -// properties.put("size", "50"); -// -// headers.put("accept", "application/json"); -// headers.put("x-amberdata-blockchain-id", "ethereum-mainnet"); -// headers.put("x-api-key", "UAK7ed69235426c360be22bfc2bde1809b6"); -// -// tags.put("-l", "50"); -// tags.put("-t", "incremental"); -// -// String[] path = {"payload", "records"}; -// String[] recursive_location = {"page"}; -// String recursive_replacement = null; -// -// RequestParameterized request = new RequestParameterized("test", -// "https://web3api.io/api/v2/addresses", new String[]{}, properties, headers, tags, -// recursive_location, recursive_replacement, path, false, "", "", "", ""); -// -// request.request(new String[] {}, properties, headers); -// } -} diff --git a/DeFi-Data-Engine/Api-Handler/src/main/java/test/application/apihandler/SampleMakerDAOProtocol.java b/DeFi-Data-Engine/Api-Handler/src/main/java/test/application/apihandler/SampleMakerDAOProtocol.java deleted file mode 100644 index fc81cc2d..00000000 --- a/DeFi-Data-Engine/Api-Handler/src/main/java/test/application/apihandler/SampleMakerDAOProtocol.java +++ /dev/null @@ -1,32 +0,0 @@ -package test.application.apihandler; - -public class SampleMakerDAOProtocol { - - // disabled for docker composition - -// public static void main(String[] args) { -// HashMap properties = new HashMap(); -// HashMap headers = new HashMap(); -// HashMap tags = new HashMap(); -// -// properties.put("startDate", "2020-09-01T01:00:00"); -// properties.put("endDate", "2020-09-03T01:00:00"); -// properties.put("size", "900"); -// -// headers.put("accept", "application/json"); -// headers.put("x-api-key", "UAK7ed69235426c360be22bfc2bde1809b6"); -// -// tags.put("-l", "900"); -// tags.put("-t", "url"); -// -// String[] path = {"payload", "data"}; -// String[] recursive_location = {"payload", "metadata", "next"}; -// String recursive_replacement = null; -// -// RequestParameterized request = new RequestParameterized("test", -// "https://web3api.io/api/v2/defi/lending/makerdao/protocol", new String[]{}, properties, headers, tags, -// recursive_location, recursive_replacement, path, false, "", "", "", ""); -// -// request.request(new String[] {}, properties, headers); -// } -} diff --git a/DeFi-Data-Engine/Api-Handler/src/main/resources/requests/amberdata-uniswap-pool.properties b/DeFi-Data-Engine/Api-Handler/src/main/resources/requests/amberdata-uniswap-pool.properties new file mode 100644 index 00000000..26c80cf1 --- /dev/null +++ b/DeFi-Data-Engine/Api-Handler/src/main/resources/requests/amberdata-uniswap-pool.properties @@ -0,0 +1,30 @@ +request.name= amberdata-uniswap-pool + +url.base= https://web3api.io/api/v2/defi/dex/uniswapv2/pools + +url.base.path= poolAddress,. + +url.properties= size,900 + +url.headers= accept,application/json,\ + x-api-key,. + +data.path= payload,\ + data + +recursion.type= parameterized + +recursion.tags= -l,900,\ + -t,url + +recursion.location= payload,metadata,next + +date.valid= true + +date.location= properties + +date.start= startDate + +date.end= endDate + +date.format= yyyy-MM-dd \ No newline at end of file diff --git a/DeFi-Data-Engine/Documentation/Internal-Documentation/~$Documentation.xlsx b/DeFi-Data-Engine/Documentation/Internal-Documentation/~$Documentation.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..11bacf8159c39393b8696b836d85c7c180e7eb9c GIT binary patch literal 165 ucmd;f&d - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/DeFi-Data-Engine/Rest Application/.gitignore b/DeFi-Data-Engine/Rest Application/.gitignore deleted file mode 100644 index 09e3bc9b..00000000 --- a/DeFi-Data-Engine/Rest Application/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -/bin/ -/target/ diff --git a/DeFi-Data-Engine/Rest Application/.project b/DeFi-Data-Engine/Rest Application/.project deleted file mode 100644 index 10c2ba0f..00000000 --- a/DeFi-Data-Engine/Rest Application/.project +++ /dev/null @@ -1,23 +0,0 @@ - - - Rest Application - - - - - - org.eclipse.jdt.core.javabuilder - - - - - org.eclipse.m2e.core.maven2Builder - - - - - - org.eclipse.m2e.core.maven2Nature - org.eclipse.jdt.core.javanature - - diff --git a/DeFi-Data-Engine/Rest Application/.settings/org.eclipse.core.resources.prefs b/DeFi-Data-Engine/Rest Application/.settings/org.eclipse.core.resources.prefs deleted file mode 100644 index 29abf999..00000000 --- a/DeFi-Data-Engine/Rest Application/.settings/org.eclipse.core.resources.prefs +++ /dev/null @@ -1,6 +0,0 @@ -eclipse.preferences.version=1 -encoding//src/main/java=UTF-8 -encoding//src/main/resources=UTF-8 -encoding//src/test/java=UTF-8 -encoding//src/test/resources=UTF-8 -encoding/=UTF-8 diff --git a/DeFi-Data-Engine/Rest Application/.settings/org.eclipse.jdt.apt.core.prefs b/DeFi-Data-Engine/Rest Application/.settings/org.eclipse.jdt.apt.core.prefs deleted file mode 100644 index d4313d4b..00000000 --- a/DeFi-Data-Engine/Rest Application/.settings/org.eclipse.jdt.apt.core.prefs +++ /dev/null @@ -1,2 +0,0 @@ -eclipse.preferences.version=1 -org.eclipse.jdt.apt.aptEnabled=false diff --git a/DeFi-Data-Engine/Rest Application/.settings/org.eclipse.jdt.core.prefs b/DeFi-Data-Engine/Rest Application/.settings/org.eclipse.jdt.core.prefs deleted file mode 100644 index 23731ecc..00000000 --- a/DeFi-Data-Engine/Rest Application/.settings/org.eclipse.jdt.core.prefs +++ /dev/null @@ -1,17 +0,0 @@ -eclipse.preferences.version=1 -org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled -org.eclipse.jdt.core.compiler.codegen.methodParameters=generate -org.eclipse.jdt.core.compiler.codegen.targetPlatform=17 -org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve -org.eclipse.jdt.core.compiler.compliance=17 -org.eclipse.jdt.core.compiler.debug.lineNumber=generate -org.eclipse.jdt.core.compiler.debug.localVariable=generate -org.eclipse.jdt.core.compiler.debug.sourceFile=generate -org.eclipse.jdt.core.compiler.problem.assertIdentifier=error -org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled -org.eclipse.jdt.core.compiler.problem.enumIdentifier=error -org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning -org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=warning -org.eclipse.jdt.core.compiler.processAnnotations=disabled -org.eclipse.jdt.core.compiler.release=disabled -org.eclipse.jdt.core.compiler.source=17 diff --git a/DeFi-Data-Engine/Rest Application/.settings/org.eclipse.m2e.core.prefs b/DeFi-Data-Engine/Rest Application/.settings/org.eclipse.m2e.core.prefs deleted file mode 100644 index f897a7f1..00000000 --- a/DeFi-Data-Engine/Rest Application/.settings/org.eclipse.m2e.core.prefs +++ /dev/null @@ -1,4 +0,0 @@ -activeProfiles= -eclipse.preferences.version=1 -resolveWorkspaceProjects=true -version=1 diff --git a/DeFi-Data-Engine/Rest Application/Dockerfile b/DeFi-Data-Engine/Rest Application/Dockerfile deleted file mode 100644 index c87bd6cf..00000000 --- a/DeFi-Data-Engine/Rest Application/Dockerfile +++ /dev/null @@ -1,4 +0,0 @@ -FROM maven:3.8.6-eclipse-temurin-17 -COPY ./ ./ -RUN mvn clean package -Dmaven.test.skip -CMD ["java", "-jar", "target/rest-connection-4.3.3.jar"] \ No newline at end of file diff --git a/DeFi-Data-Engine/Rest Application/pom.xml b/DeFi-Data-Engine/Rest Application/pom.xml deleted file mode 100644 index 359964a2..00000000 --- a/DeFi-Data-Engine/Rest Application/pom.xml +++ /dev/null @@ -1,76 +0,0 @@ - - - 4.0.0 - - org.springframework.boot - spring-boot-starter-parent - 2.7.2 - - - org.out.connections - rest-connection - 4.3.3 - jar - restconnection - Rest Connection for DeFi Data Engine - - 17 - 17 - 17 - 0.8.3.Final - - - - org.springframework.boot - spring-boot-starter-web - - - org.springframework.boot - spring-boot-devtools - runtime - true - - - org.springframework.boot - spring-boot-starter-tomcat - provided - - - org.springframework.boot - spring-boot-starter-test - test - - - org.json - json - 20220320 - - - org.springframework.integration - spring-integration-ip - - - - - - - org.springframework.boot - spring-boot-maven-plugin - - - maven-assembly-plugin - - - - org.rest.application.RestApplication - - - - jar-with-dependencies - - - - - - diff --git a/DeFi-Data-Engine/Rest Application/src/main/java/org/properties/Config.java b/DeFi-Data-Engine/Rest Application/src/main/java/org/properties/Config.java deleted file mode 100644 index a5e4ee12..00000000 --- a/DeFi-Data-Engine/Rest Application/src/main/java/org/properties/Config.java +++ /dev/null @@ -1,60 +0,0 @@ -package org.properties; - -import java.util.HashMap; -import java.util.Properties; - -public class Config { - - private static final HashMap properties; - -static { - properties = new HashMap(); - - Properties app_properties = new Properties(); - app_properties.put("", ""); - app_properties.put("general.internal.delim", ":::"); - app_properties.put("general.data.delim", ","); - app_properties.put("general.collection.delim", "="); - app_properties.put("general.transfer.delim", "&&&"); - app_properties.put("general.data.dateformat", "yyyy-MM-dd"); - app_properties.put("spring.server.port", "8080"); - app_properties.put("spring.server.address", "RestApp"); - // app_properties.put("spring.server.address", "defi-de.idea.rpi.edu"); - // app_properties.put("spring.server.address", "localhost"); - app_properties.put("rest.socket.address", "DataEngine"); - // app_properties.put("rest.socket.address", "localhost"); - app_properties.put("rest.socket.port", "61100"); - app_properties.put("rest.socket.key", "rest-key-reserved"); - properties.put("app", app_properties); - - //properties.put("testing", new Properties()); -} - - public static final Properties getProperties(String name) { - validate(name); - return properties.get(name); - } - - public static final String getProperty(String name, String property) { - validate(name, property); - return properties.get(name).getProperty(property); - } - - public static final void setProperty(String name, String property, String value) { - validate(name, property); - properties.get(name).setProperty(property, value); - } - - public static final void validate(String name, String... keys) { - if(!properties.containsKey(name)) { - new IllegalArgumentException(String.format("Property file <%s> does not exist. Program terminating.", name)).printStackTrace(); - System.exit(1); - } - - for(String key : keys) - if(!properties.get(name).containsKey(key)) { - new IllegalArgumentException(String.format("Missing property <%s> in file <%s>. Program terminating.", key, name)).printStackTrace(); - System.exit(1); - } - } -} diff --git a/DeFi-Data-Engine/Rest Application/src/main/java/org/rest/application/Endpoint.java b/DeFi-Data-Engine/Rest Application/src/main/java/org/rest/application/Endpoint.java deleted file mode 100644 index e382e6d6..00000000 --- a/DeFi-Data-Engine/Rest Application/src/main/java/org/rest/application/Endpoint.java +++ /dev/null @@ -1,165 +0,0 @@ -package org.rest.application; - -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.IOException; -import java.net.Socket; - -import javax.net.SocketFactory; - -import org.json.JSONObject; -import org.properties.Config; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RestController; - -@RestController -public class Endpoint { - - private static Socket socket; - private static DataInputStream in; - private static DataOutputStream out; - -static { - try { - socket = SocketFactory.getDefault().createSocket( - Config.getProperty("app", "rest.socket.address"), - Integer.parseInt(Config.getProperty("app", "rest.socket.port"))); - in = new DataInputStream(socket.getInputStream()); - out = new DataOutputStream(socket.getOutputStream()); - - out.writeUTF(Config.getProperty("app", "rest.socket.key")); - } catch (IOException e) { - e.printStackTrace(); - System.exit(1); - } -} - - @GetMapping(path="/defi/v1/rest/source_exists") - public String getSourceExists(@RequestParam String source) { - return request("SRC", "EXSR", "source", source); - } - - @GetMapping(path="/defi/v1/rest/stream_exists") - public String getStreamExists(@RequestParam String key) { - return request("SRC", "EXST", "key", key); - } - - @GetMapping(path="/defi/v1/rest/initialize") - public String getInitialize(@RequestParam String source, @RequestParam String auth_data) { - String[] auth = auth_data.split(","); - if(auth.length % 2 != 0) - return new JSONObject() - .put("response", "500") - .put("message", "Malformed parameters. must be a list of " - + " pairs. Cannot process given response.") - .toString(); - - String[] params = new String[2 + auth.length]; - params[0] = "source"; - params[1] = source; - for(int i = 2; i < params.length; i++) - params[i] = auth[i - 2]; - return request("SRC", "INIT", params); - } - - @GetMapping(path="/defi/v1/rest/is_authorized") - public String getIsAuthorized(@RequestParam String key) { - return request("SRC", "IATH", "key", key); - } - - @GetMapping(path="/defi/v1/rest/is_active") - public String getIsActive(@RequestParam String key) { - return request("SRC", "IATV", "key", key); - } - -// @GetMapping(path="/defi/v1/rest/execute") -// @Deprecated -// public String getExecute(@RequestParam String key) { -// return request("SRC", "EXEC", "key", key); -// } -// -// @GetMapping(path="/defi/v1/rest/kill") -// @Deprecated -// public String getKill(@RequestParam String key) { -// return request("SRC", "KILL", "key", key); -// } - -// @GetMapping(path="/defi/v1/rest/subscribe") -// @Deprecated -// public String getSubscription(@RequestParam String key, @RequestParam String subscription) { -// return request("SRC", "SUBS", "key", key, "subscription", subscription); -// } - -// @GetMapping(path="/defi/v1/rest/request") -// public String getRequest(@RequestParam String destination, @RequestParam String key, @RequestParam String request, @RequestParam String query) { -// return request("SRC", "RQST", "destination", destination, "key", key, "request", request, "query", query); -// } - - @GetMapping(path="/defi/v1/rest/request_dated") - public String getRequest(@RequestParam String destination, @RequestParam String key, @RequestParam String request, @RequestParam String query, @RequestParam String start_date, @RequestParam String end_date) { - String[] split = request.split(","); - if(split.length % 2 != 0 || split.length < 2) - return new JSONObject() - .put("response", "500") - .put("message", String.format("Request must contain an even number of parameters listed delimited by <,>. Must contain the pair >.")) - .toString(); - - String[] requests = new String[10 + split.length]; - requests[0] = "destination"; - requests[1] = destination; - requests[2] = "key"; - requests[3] = key; - requests[4] = "start_date"; - requests[5] = start_date; - requests[6] = "end_date"; - requests[7] = end_date; - requests[8] = "query"; - requests[9] = query; - boolean contains_request = false; - for(int i = 0; i < split.length; i+=2) { - if(split[i].equals("request")) - contains_request = true; - requests[i + 10] = split[i]; - requests[i + 11] = split[i + 1]; - } - - if(!contains_request) { - return new JSONObject() - .put("response", "500") - .put("message", String.format("Request is missing the required parameter .")) - .toString(); - } - - return request("SRC", "RQST", requests); - } - - private final String request(String tag, String sub_tag, String... data) { - try { - String delim = Config.getProperty("app", "general.transfer.delim"); - - StringBuilder formatted_data = new StringBuilder(); - for(int i = 0; i < data.length; i++) { - if(data[i].contains(delim)) - return new JSONObject() - .put("response", "406") - .put("message", String.format("Parameters cannot contain the character sequence <%s>. Please request with a different character.", delim)) - .toString(); - formatted_data.append(data[i]); - if(i != data.length - 1) - formatted_data.append(delim); - } - - out.writeUTF(String.format("%s%s%s%s%s", tag, delim, sub_tag, delim, formatted_data)); - return in.readUTF(); - } catch (IOException e) { - e.printStackTrace(); - System.exit(1); - } - - return new JSONObject() - .put("response", "500") - .put("message", "Unexpected invalid request. Cannot process given response.") - .toString(); - } -} \ No newline at end of file diff --git a/DeFi-Data-Engine/Rest Application/src/main/java/org/rest/application/RestApplication.java b/DeFi-Data-Engine/Rest Application/src/main/java/org/rest/application/RestApplication.java deleted file mode 100644 index 777f290f..00000000 --- a/DeFi-Data-Engine/Rest Application/src/main/java/org/rest/application/RestApplication.java +++ /dev/null @@ -1,22 +0,0 @@ -package org.rest.application; - -import java.io.IOException; -import java.net.UnknownHostException; -import java.util.HashMap; - -import org.properties.Config; -import org.springframework.boot.SpringApplication; -import org.springframework.boot.autoconfigure.SpringBootApplication; - -@SpringBootApplication -public class RestApplication { - - public static void main(String[] args) throws InterruptedException, UnknownHostException, IOException { - SpringApplication app = new SpringApplication(RestApplication.class); - HashMap properties = new HashMap(); - properties.put("server.port", Config.getProperty("app", "spring.server.port")); - properties.put("server.address", Config.getProperty("app", "spring.server.address")); - app.setDefaultProperties(properties); - app.run(args); - } -} \ No newline at end of file diff --git a/DeFi-Data-Engine/Rest Application/src/main/java/org/rest/application/ServletInitializer.java b/DeFi-Data-Engine/Rest Application/src/main/java/org/rest/application/ServletInitializer.java deleted file mode 100644 index edb513bb..00000000 --- a/DeFi-Data-Engine/Rest Application/src/main/java/org/rest/application/ServletInitializer.java +++ /dev/null @@ -1,13 +0,0 @@ -package org.rest.application; - -import org.springframework.boot.builder.SpringApplicationBuilder; -import org.springframework.boot.web.servlet.support.SpringBootServletInitializer; - -public class ServletInitializer extends SpringBootServletInitializer { - - @Override - protected SpringApplicationBuilder configure(SpringApplicationBuilder application) { - return application.sources(RestApplication.class); - } - -} diff --git a/DeFi-Data-Engine/Rest Application/src/main/resources/config/app.properties b/DeFi-Data-Engine/Rest Application/src/main/resources/config/app.properties deleted file mode 100644 index 049af529..00000000 --- a/DeFi-Data-Engine/Rest Application/src/main/resources/config/app.properties +++ /dev/null @@ -1,32 +0,0 @@ -# === GENERAL PROPERTIES === - -# Wait-time in between checking for responses from engine: -rest.wait.ms=10 - -# delimiter used for internal processing -general.internal.delim=::: - -# data delimiter -general.data.delim=, - -# collection delimiter -general.collection.delim== - -# transfer delimiter -general.transfer.delim=&&& - -# === SPRING PROPERTIES === - -# Port of Tomcat server -spring.server.port=8080 - -# === SOCKET PROPERTIES === - -# Rest socket address -rest.socket.address=DataEngine - -# Rest socket port -rest.socket.port=61100 - -# Rest socket key -rest.socket.key=rest-key-reserved \ No newline at end of file diff --git a/DeFi-Data-Engine/Rest Application/src/test/java/test/rest/application/RestconnectionApplicationTests.java b/DeFi-Data-Engine/Rest Application/src/test/java/test/rest/application/RestconnectionApplicationTests.java deleted file mode 100644 index aa6c8cc2..00000000 --- a/DeFi-Data-Engine/Rest Application/src/test/java/test/rest/application/RestconnectionApplicationTests.java +++ /dev/null @@ -1,13 +0,0 @@ -package test.rest.application; - -import org.junit.jupiter.api.Test; -import org.springframework.boot.test.context.SpringBootTest; - -@SpringBootTest -class RestconnectionApplicationTests { - - @Test - void contextLoads() { - } - -} diff --git a/R-Code-Samples/data-engine-r-functions.Rmd b/R-Code-Samples/DataEnginePrimaryFunctions.Rmd similarity index 66% rename from R-Code-Samples/data-engine-r-functions.Rmd rename to R-Code-Samples/DataEnginePrimaryFunctions.Rmd index 81c9e42a..fded60e7 100644 --- a/R-Code-Samples/data-engine-r-functions.Rmd +++ b/R-Code-Samples/DataEnginePrimaryFunctions.Rmd @@ -21,42 +21,17 @@ local({r <- getOption("repos") # Set code chunk defaults knitr::opts_chunk$set(echo = TRUE) +library(plyr) +library(dplyr) +library(httr) +library(jsonlite) +library(lubridate) +library(stringr) +# library(tidyr) +library(knitr) + # Load required packages; install if necessary # CAUTION: DO NOT interrupt R as it installs packages!! -if (!require("ggplot2")) { - install.packages("ggplot2") - library(ggplot2) -} - -if (!require("httr")) { - install.packages("httr") - library(httr) -} -if (!require("jsonlite")) { - install.packages("jsonlite") - library(jsonlite) -} - -if (!require("lubridate")) { - install.packages("lubridate") - library(lubridate) -} -if(!require("dplyr")){ - install.packages("dplyr") - library(dplyr) -} -if(!require("stringr")){ - install.packages("stringr") - library(stringr) -} -if(!require("tidyr")){ - install.packages("tidyr") - library(tidyr) -} -if(!require("knitr")){ - install.packages("knitr") - library(knitr) -} ``` @@ -65,7 +40,8 @@ We provide a function to request and parse data from our DeFi data engine living request <- function(protocol, properties = "", headers = "", startdate = "", enddate = "") { #Create socket and get destination which tells the engine where to put the data - socket <- socketConnection("localhost", 61200, blocking=TRUE) + socket <- socketConnection("defi-de.idea.rpi.edu", 61200, blocking=TRUE) + #socket <- socketConnection("localhost", 61200, blocking=TRUE) destination <- readLines(socket, 1) formatted_properties = "" @@ -107,28 +83,53 @@ request <- function(protocol, properties = "", headers = "", startdate = "", end #Now the engine will begin feeding us data #We grab the first to initialize the data var and then we continue listening\ - data <- readLines(socket, 1) counter <- 0 + response <- "" while (TRUE) { temp <- readLines(socket, 1) + + # if line is heartbeat then acknowledge and continue + if (grepl("<<>>", temp, fixed=TRUE)) + { + print(paste("Heartbeat read")) + next + } + + # if line is response then process and terminate + else if (grepl("<<>>", temp, fixed=TRUE)) + { + temp <- readLines(socket, 1) + response <- fromJSON(temp) + break + } + + # increment processed line counter counter <- counter + 1 if(counter %% 1000 == 0) print(paste("Processed", counter, "lines")) - if (grepl("<<>>", temp, fixed=TRUE)) {break} + + # add data point line to data frame temp.df <- as.data.frame(fromJSON(temp)) df <- rbind.fill(df, temp.df) } - return(df) + output <- list("response"=response, "df"=df) + return(output) } - ``` With this function, we can now get our data. ```{R} - #We make a sample call to the function which will return all transactions 2022 August 1st to August 3rd -df <- request("amberdata-aave-protocol", "", "x-api-key,UAK7ed69235426c360be22bfc2bde1809b6", "2022-08-02", "2022-08-03") -kable(head(df), "simple") +# aave <- request("amberdata-aave-protocol", "", "x-api-key,UAK7ed69235426c360be22bfc2bde1809b6", "2022-08-01", "2022-08-03") +# aave.df <- aave$df +# aave.response <- aave$response + +aave.liquidations <- request("amberdata-aave-protocol", "action,LiquidationCall", "x-api-key,UAK7ed69235426c360be22bfc2bde1809b6", "2022-07-01", "2023-09-01") +aave.liquidations.df <- aave.liquidations$df +aave.liquidations.response <- aave.liquidations$response +# sushiswap <- request("amberdata-sushiswap-protocol", "", "x-api-key,UAK7ed69235426c360be22bfc2bde1809b6", "2022-08-01", "2022-08-03") +# sushiswap.df <- sushiswap$df +# sushiswap.response <- sushiswap$response ``` \ No newline at end of file diff --git a/R-Code-Samples/ExampleUserClusteringStarter.Rmd b/R-Code-Samples/ExampleUserClusteringStarter.Rmd new file mode 100644 index 00000000..bac82c6f --- /dev/null +++ b/R-Code-Samples/ExampleUserClusteringStarter.Rmd @@ -0,0 +1,413 @@ +--- +title: 'DAR F22 - User Clustering Starter' +author: "Your Name Here" +subtitle: "Making Users from Transaction Data and Clustering" +output: + pdf_document: default + html_document: default + header-includes: \usepackage{color} + toc: yes +--- + +```{r setup, include=FALSE, warning=FALSE} +# Set the default CRAN repository +local({r <- getOption("repos") + r["CRAN"] <- "http://cran.r-project.org" + options(repos=r) +}) + +# Set code chunk defaults +knitr::opts_chunk$set(echo = TRUE) + +# Load required packages; install if necessary +# CAUTION: DO NOT interrupt R as it installs packages!! +if (!require("ggplot2")) { + install.packages("ggplot2") + +} + +library(ggplot2) +library(knitr) +library(plyr) +library(dplyr) +library(jsonlite) +library(RColorBrewer) +library(tidyverse) +library(beeswarm) +library(ggbeeswarm) +library(xts) +library(plotly) +library(lubridate) +library(survival) +library(survminer) +library(ranger) +library(ggfortify) +library(factoextra) +library(cluster) +library(fclust) +library(ppclust) +library(e1071) +library(randomNames) +``` + +We provide a function to request and parse data from our DeFi data engine living on the IDEA Cluster. This initializes a data stream from the Amber Data API, opens a socket, requests data, listens on the socket, and then parses the received data. The finished dataframe is as close as possible to the schema of the cold-storage data we currently use. +```{R} +request <- function(protocol, properties = "", headers = "", startdate = "", enddate = "") { + #Create socket and get destination which tells the engine where to put the data + socket <- socketConnection("defi-de.idea.rpi.edu", 61200, blocking=TRUE) + destination <- readLines(socket, 1) + + formatted_properties = "" + if(properties != "") + formatted_properties = paste("properties", "&&&", properties, "&&&") + + formatted_headers = "" + if(headers != "") + formatted_headers = paste("headers", "&&&", headers, "&&&") + + formatted_startdate = "" + formatted_enddate = "" + if(startdate != "" && enddate != "") { + formatted_startdate = paste("start_date", "&&&", startdate, "&&&") + formatted_enddate = paste("end_date", "&&&", enddate, "&&&") + } + + #Build the request delimited by &&& + #Similar to a GET request in the way we handle parameters + request.raw <- paste( + "SRC", "&&&", "RQST", "&&&", + "type", "&&&", protocol, "&&&", + formatted_properties, + formatted_headers, + formatted_startdate, + formatted_enddate, + "destination", "&&&", destination, "&&&", + "\n", sep="") + + # remove all spaces from request + request.data <- str_replace_all(request.raw, " ", "") + + #Write this request back to the socket to tell engine what we want + writeLines(request.data, socket) + + # define data frame + df = data.frame() + temp.df = data.frame() + + #Now the engine will begin feeding us data + #We grab the first to initialize the data var and then we continue listening\ + counter <- 0 + response <- "" + while (TRUE) { + temp <- readLines(socket, 1) + counter <- counter + 1 + if(counter %% 1000 == 0) + print(paste("Processed", counter, "lines")) + if (grepl("<<>>", temp, fixed=TRUE)) + { + temp <- readLines(socket, 1) + response <- fromJSON(temp) + break + } + temp.df <- as.data.frame(fromJSON(temp)) + df <- rbind.fill(df, temp.df) + } + + output <- list("response"=response, "df"=df) + return(output) +} +``` + +```{r} +#We make a sample call to the function which will return all transactions 2022 August 1st to August 3rd +# aave <- request("amberdata-aave-protocol", "", "x-api-key,UAK7ed69235426c360be22bfc2bde1809b6", "2022-08-01", "2022-08-02") +# aave.df <- aave$df +# aave.response <- aave$response + +sushiswap <- request("amberdata-sushiswap-protocol", "", "x-api-key,UAK7ed69235426c360be22bfc2bde1809b6", "2022-08-01", "2022-08-03") +sushiswap.df <- sushiswap$df +sushiswap.response <- sushiswap$response +``` + +# Set df to the loaded Aave set: +```{r, warning=FALSE} +df <- aave.df +``` + +# Set user ailas': +```{r} + +``` + +# Clustering Users: + + One goal of this project is to characterize users by their behavior within the DeFi ecosystem. For now, that just means characterizing them by their behavior within AAVE. Since we only have transaction data, we need to use this transaction data to create user-level data that can be actually be used for clustering, or perhaps other behavior-characterization methods. To help with the creation of these features, we start by separating our transactions into different dataframes for each transaction type and doing some aggregation of liquidations to help better characterize liquidation events. + +## Build some helper dataframes: +```{r, warning=FALSE} +not_all_na <- function(x) any(!is.na(x)) +`%notin%` <- Negate(`%in%`) + +borrows <- df %>% + filter(action == "Borrow") %>% + select(where(not_all_na)) + +repays <- df %>% + filter(action == "Repay") %>% + select(where(not_all_na)) + +deposits <- df %>% + filter(action == "Deposit") %>% + select(where(not_all_na)) + +redeems <- df %>% + filter(action == "Redeem") %>% + select(where(not_all_na)) + +liquidations <- df %>% + filter(action == "LiquidationCall") %>% + select(where(not_all_na)) + +swaps <- df %>% + filter(action == "SwapBorrowRateMode") %>% + select(where(not_all_na)) + +collaterals <- df %>% + filter(action == "UseReserveAsCollateral") %>% + select(where(not_all_na)) + +liquidationsPerformed <- liquidations %>% + mutate(liquidatee = user, liquidateeAlias = userAlias) %>% + select(-user, -userAlias) %>% + rename(user = liquidator, userAlias = liquidatorAlias) + +reserveTypes <- mainnetReserveInfo %>% + select(reserve = symbol, stable) %>% + mutate(reserveType = case_when(stable ~ "Stable", + TRUE ~ "Non-Stable")) %>% + select(-stable) + +df2 <- left_join(df, reserveTypes, by="reserve") %>% + distinct() + +numLiqPerUser <- liquidations %>% + group_by(user) %>% + dplyr::summarise(numLiquidations = n()) + + +aggregateLiquidations <- df2 %>% + filter(user %in% numLiqPerUser$user) %>% # First, let's filter out all users who have never been liquidated. + group_by(user) %>% # The next set of logic is to sort users' transactions by timestamp and pull out all liquidations that are + arrange(timestamp) %>% # part of a consecutive set of liquidations. + mutate(nextTransaction = lead(type)) %>% + mutate(prevTransaction = lag(type)) %>% + filter(type == "liquidation" & (nextTransaction == "liquidation" | prevTransaction == "liquidation")) %>% + mutate(liquidationDay = floor_date(as_datetime(timestamp), unit = "day")) %>% # Then we want to use some approximation for the timeframe of this liquidation event, so we naively group consecutive liquidations by the day on which they took place. + group_by(user,liquidationDay) %>% # Doing this means that we can group by user and liquidationDay, which is functionally grouping by "liquidation event" + mutate(liquidationDuration = max(timestamp) - min(timestamp)) %>% # Now we can compute some basic stats about the event. + mutate(liquidationStart = min(timestamp), liquidationEnd = max(timestamp)) %>% + mutate(liquidationStartDatetime = as_datetime(liquidationStart), liquidationEndDatetime = as_datetime(liquidationEnd)) %>% + mutate(reserve = collateralReserve) %>% + left_join(reserveTypes, by = "reserve") %>% + dplyr::rename(collateralType = reserveType.y) %>% + mutate(reserve = principalReserve) %>% + left_join(reserveTypes, by = "reserve") %>% + dplyr::rename(principalType = reserveType) %>% + mutate(totalCollateralUSD = sum(amountUSDCollateral), totalPrincipalUSD = sum(amountUSDPrincipal))%>% + dplyr::mutate(numLiquidations = n()) %>% + dplyr::summarise(userAlias, numLiquidations, liquidationDuration, liquidationStart, liquidationEnd, liquidationStartDatetime, liquidationEndDatetime, + collateralReserves = str_flatten(str_sort(unique(collateralReserve)), collapse = ","), + collateralTypes = str_flatten(str_sort(unique(collateralType)), collapse= ","), + principalReserves = str_flatten(str_sort(unique(principalReserve)), collapse = ","), + principalTypes = str_flatten(str_sort(unique(principalType)), collapse = ","), + totalCollateralUSD, totalPrincipalUSD, liquidationType = str_c(principalTypes, collateralTypes, sep = ":")) %>% + distinct() + +rm(df2) + +``` + + +## Build features to cluster the users: +```{r, warning=FALSE} +timeFinal <- max(df$timestamp) +userActiveTime <- df %>% + group_by(user) %>% + dplyr::summarise(firstTransactionTimestamp = min(timestamp), finalTimestamp = max(timestamp), daysSinceFirstTransaction = max((timeFinal-min(timestamp))/86400, 1)) + +userDailyTransactions <- df %>% + group_by(user) %>% + mutate(transactionDay = floor_date(as_datetime(timestamp), unit = "day")) %>% + group_by(user, transactionDay) %>% + dplyr::summarise(transactionsPerActiveDay = n()) + +userActiveDays <- userDailyTransactions %>% + group_by(user) %>% + dplyr::summarise(activeDays = n()) + +userBorrowCounts <- borrows %>% + group_by(user) %>% + dplyr::summarise(borrowCount = n(), borrowValue = sum(amountUSD)) %>% + mutate(logBorrowValue = case_when(borrowValue != 0 ~ log(borrowValue, 10), + TRUE ~ 0)) + +userDepositCounts <- deposits %>% + group_by(user) %>% + dplyr::summarise(depositCount = n(), depositValue = sum(amountUSD))%>% + mutate(logDepositValue = case_when(depositValue != 0 ~ log(depositValue, 10), + TRUE ~ 0)) + +userRedeemCounts <- redeems %>% + group_by(user) %>% + dplyr::summarise(redeemCount = n(), redeemValue = sum(amountUSD)) %>% + mutate(logRedeemValue = case_when(redeemValue != 0 ~ log(redeemValue, 10), + TRUE ~ 0)) + +userRepayCounts <- repays %>% + group_by(user) %>% + dplyr::summarise(repayCount = n(), repayValue = sum(amountUSD)) %>% + mutate(logRepayValue = case_when(repayValue != 0 ~ log(repayValue, 10), + TRUE ~ 0)) + +userLiquidatedCounts <- aggregateLiquidations %>% + group_by(user) %>% + dplyr::summarise(liquidatedCount = n(), liquidatedValue = sum(totalPrincipalUSD)) + +userLiquidationCounts <- liquidationsPerformed %>% + group_by(user) %>% + dplyr::summarise(liquidationsPerformed = n(), liquidationsPerformedValue = sum(amountUSDCollateral)) + +userSwapCounts <- swaps %>% + group_by(user) %>% + dplyr::summarise(swapCount = n()) + +userCollateralCounts <- collaterals %>% + group_by(user) %>% + dplyr::summarise(collateralCount = n()) + +userReservesUsed <- df %>% + filter(type == "deposit" | type == "borrow") %>% + group_by(user) %>% + dplyr::summarise(reservesUsed = n_distinct(reserve)) + +transactionsMadeOnBehalfOf <- df %>% + filter(user != onBehalfOf) %>% + select(onBehalfOf) %>% + rename(user = onBehalfOf) %>% + group_by(user) %>% + summarise(onBehalfOfCount = n()) + +transactionsPerformedForOther <- df %>% + filter(user != onBehalfOf) %>% + select(user) %>% + group_by(user) %>% + summarise(performedForOtherCount = n()) + + +userTransactionCounts <- df %>% + select(user) %>% + distinct() %>% + full_join(userBorrowCounts, by = "user") %>% + full_join(userDepositCounts, by = "user") %>% + full_join(userRedeemCounts, by = "user") %>% + full_join(userRepayCounts, by = "user") %>% + full_join(userLiquidatedCounts, by = "user") %>% + full_join(userLiquidationCounts, by = "user") %>% + full_join(userSwapCounts, by = "user") %>% + full_join(userCollateralCounts, by = "user") + +userTransactionCounts[is.na(userTransactionCounts)] = 0 + +userTransactionCounts <- userTransactionCounts %>% + mutate(totalTransactionCount = borrowCount + depositCount + redeemCount + repayCount + liquidatedCount + liquidationsPerformed + swapCount + collateralCount) + +userActiveCollaterals <- collaterals %>% + group_by(user, reserve) %>% + slice_max(timestamp) %>% + filter(toState == TRUE) %>% + ungroup() %>% + group_by(user) %>% + dplyr::summarise(numActiveCollaterals=n()) + +userClusteringData <- userTransactionCounts %>% + mutate(percentDepositRedeem = (depositCount + redeemCount) / totalTransactionCount) %>% + mutate(averageUSDPerTransaction = (depositValue + redeemValue + repayValue + liquidatedValue + liquidationsPerformedValue + borrowValue) / totalTransactionCount) %>% + mutate(logUSDPerTransaction = case_when(averageUSDPerTransaction != 0 ~ log(averageUSDPerTransaction, 10), TRUE ~ 0)) %>% + mutate(timesLiquidated = liquidatedCount) %>% + mutate(liquidationsPerformed = liquidationsPerformed) %>% + left_join(userActiveTime, by="user") %>% + mutate(averageTransactionsPerDay = totalTransactionCount / daysSinceFirstTransaction) %>% + left_join(userActiveDays, by="user") %>% + mutate(percentageDaysActive = activeDays / daysSinceFirstTransaction) %>% + left_join(userReservesUsed, by = "user") %>% + left_join(userActiveCollaterals, by="user") %>% + mutate(percentDeposit = depositCount / totalTransactionCount, percentRedeems = redeemCount / totalTransactionCount, + percentBorrow = borrowCount / totalTransactionCount, percentRepay = repayCount / totalTransactionCount, + percentSwap = swapCount / totalTransactionCount, percentCollateral = collateralCount / totalTransactionCount, + percentLiquidations = liquidationsPerformed / totalTransactionCount) %>% + left_join(transactionsMadeOnBehalfOf, by="user") %>% + left_join(transactionsPerformedForOther, by = "user") + +userClusteringData[is.na(userClusteringData)] = 0 +``` +# Computing K-Means clusters for these users: + +```{r, warning=FALSE} +# First, let's select the features we want to use, because we might not want to use them all. +clusteringFeatures <- userClusteringData %>% + select(percentDeposit, percentRedeems, percentCollateral, percentBorrow, percentRepay, percentLiquidations, percentSwap, daysSinceFirstTransaction, logDepositValue, logRedeemValue, logBorrowValue, logRepayValue) + +# Don't forget to scale the data ahead of time: +scaledData <- clusteringFeatures %>% mutate_all(scale) + +kmeansClusters <- kmeans(scaledData, centers = 4, nstart = 25) + +fviz_cluster(kmeansClusters, data = scaledData) +``` + +```{r} +clusteredUsers <- userClusteringData %>% + bind_cols(cluster = kmeansClusters$cluster) %>% + select(user, cluster) + +transactionsWithClusters <- mainnetTransactions %>% + left_join(clusteredUsers, by = "user") %>% + filter(type %in% c("liquidation", "redeem", "borrow", "repay", "deposit")) + +transactionsByClusterPlot <- ggplot(data = transactionsWithClusters, aes(x = datetime, group = type, color = type)) + + geom_density() + facet_wrap( ~ cluster) + + ggtitle("Transaction Types Over Time by User Cluster") + +transactionsByClusterPlot + +usersPerCluster <- clusteredUsers %>% + group_by(cluster) %>% + summarize(count = n()) +``` + + +# Choose the appropriate amount of clusters using the "elbow" method: +```{r, warning=FALSE} +set.seed(123) + +fviz_nbclust(head(scaledData, 10000), kmeans, method = "wss") +``` + It looks like the "elbow" occurs at 3, so let's run k-means with 3 centers and see what the clusters look like with respect to the first two principal components. +```{r, warning=FALSE} + +kmeansClusters <- kmeans(scaledData, centers = 3, nstart=25) + +fviz_cluster(kmeansClusters, data = scaledData) +``` + + It's hard to know from this visualization whether these clusters are meaningful. The principal components do not explain that much variance of the data. Given that K-means is among the most naive clustering methods, it's likely a good idea to explore other clustering methods. Additionally, the selected features for the above clustering were not necessarily selected in a smart way. + +# Future Work and Questions: + + 1. Can you think of other features that could be meaningful for clustering and characterizing users? If so, can you build those features from the transaction data? + + 2. Try some feature selection techniques to figure out which features of those computed above should actually be used in the clustering. + + 3. Implement more complex clustering methods for clustering the users. + + 4. Compare user behavior by clusters across deployments of AAVE. How do users behave differently between the Ethereum mainnet deployment and the Matic deployment? \ No newline at end of file