Skip to content

Commit

Permalink
Add in finalized example files
Browse files Browse the repository at this point in the history
  • Loading branch information
Conor Flynn committed Apr 18, 2023
1 parent 056c91b commit ae809ee
Show file tree
Hide file tree
Showing 3 changed files with 660 additions and 111 deletions.
249 changes: 170 additions & 79 deletions R-Code-Samples/DataEnginePrimaryFunctions.Rmd
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,100 +21,129 @@ local({r <- getOption("repos")
# Set code chunk defaults
knitr::opts_chunk$set(echo = TRUE)
library(ggplot2)
library(knitr)
library(plyr)
library(dplyr)
library(httr)
library(jsonlite)
library(RColorBrewer)
library(tidyverse)
library(beeswarm)
library(ggbeeswarm)
library(xts)
library(plotly)
library(lubridate)
library(stringr)
# library(tidyr)
library(knitr)
library(survival)
library(survminer)
library(ranger)
library(ggfortify)
library(factoextra)
library(cluster)
library(fclust)
library(ppclust)
library(e1071)
library(randomNames)
# Load required packages; install if necessary
# CAUTION: DO NOT interrupt R as it installs packages!!
```

We provide a function to request and parse data from our DeFi data engine living on the IDEA Cluster. This initializes a data stream from the Amber Data API, opens a socket, requests data, listens on the socket, and then parses the received data. The finished dataframe is as close as possible to the schema of the cold-storage data we currently use.
```{R}
request <- function(protocol, properties = "", headers = "", startdate = "", enddate = "") {
#Create socket and get destination which tells the engine where to put the data
socket <- socketConnection("defi-de.idea.rpi.edu", 61200, blocking=TRUE)
#socket <- socketConnection("localhost", 61200, blocking=TRUE)
destination <- readLines(socket, 1)
formatted_properties = ""
if(properties != "")
formatted_properties = paste("properties", "&&&", properties, "&&&")
formatted_headers = ""
if(headers != "")
formatted_headers = paste("headers", "&&&", headers, "&&&")
formatted_startdate = ""
formatted_enddate = ""
if(startdate != "" && enddate != "") {
formatted_startdate = paste("start_date", "&&&", startdate, "&&&")
formatted_enddate = paste("end_date", "&&&", enddate, "&&&")
}
#Build the request delimited by &&&
#Similar to a GET request in the way we handle parameters
request.raw <- paste(
"SRC", "&&&", "RQST", "&&&",
"type", "&&&", protocol, "&&&",
formatted_properties,
formatted_headers,
formatted_startdate,
formatted_enddate,
"destination", "&&&", destination, "&&&",
"\n", sep="")
# remove all spaces from request
request.data <- str_replace_all(request.raw, " ", "")
#Write this request back to the socket to tell engine what we want
writeLines(request.data, socket)
# define data frame
df = data.frame()
temp.df = data.frame()
#Now the engine will begin feeding us data
#We grab the first to initialize the data var and then we continue listening\
counter <- 0
response <- ""
while (TRUE) {
temp <- readLines(socket, 1)
```{r}
request_deprecated <- function(protocol, properties = "", headers = "", startdate = "", enddate = "") {
suppressWarnings({
#Create socket and get destination which tells the engine where to put the data
socket <- socketConnection("defi-de.idea.rpi.edu", 61200, blocking=TRUE)
# socket <- socketConnection("localhost", 61200, blocking=TRUE)
destination <- readLines(socket, 1)
formatted_properties = ""
if(properties != "")
formatted_properties = paste("properties", "&&&", properties, "&&&")
# if line is heartbeat then acknowledge and continue
if (grepl("<<<heartbeat>>>", temp, fixed=TRUE))
{
print(paste("Heartbeat read"))
next
formatted_headers = ""
if(headers != "")
formatted_headers = paste("headers", "&&&", headers, "&&&")
formatted_startdate = ""
formatted_enddate = ""
if(startdate != "" && enddate != "") {
formatted_startdate = paste("start_date", "&&&", startdate, "&&&")
formatted_enddate = paste("end_date", "&&&", enddate, "&&&")
}
# if line is response then process and terminate
else if (grepl("<<<response>>>", temp, fixed=TRUE))
{
#Build the request delimited by &&&
#Similar to a GET request in the way we handle parameters
request.raw <- paste(
"SRC", "&&&", "RQST", "&&&",
"type", "&&&", protocol, "&&&",
formatted_properties,
formatted_headers,
formatted_startdate,
formatted_enddate,
"destination", "&&&", destination, "&&&",
"\n", sep="")
# remove all spaces from request
request.data <- str_replace_all(request.raw, " ", "")
#Write this request back to the socket to tell engine what we want
writeLines(request.data, socket)
# define data frame
df = data.frame()
# temp.df = data.frame()
#Now the engine will begin feeding us data
#We grab the first to initialize the data var and then we continue listening\
counter <- 0
response <- ""
json <- "{\"data\":["
while (TRUE) {
temp <- readLines(socket, 1)
response <- fromJSON(temp)
break
if(temp == '')
{
print("Read empty string. Check engine logs or refresh configuration.")
next
}
# if line is heartbeat then acknowledge and continue
if (grepl("<<<heartbeat>>>", temp, fixed=TRUE))
{
print(paste("Heartbeat read for", protocol, sep=" "))
next
}
# if line is response then process and terminate
else if (grepl("<<<response>>>", temp, fixed=TRUE))
{
temp <- readLines(socket, 1)
response <- fromJSON(temp)
break
}
# increment processed line counter
counter <- counter + 1
if(counter %% 1000 == 0)
print(paste("Processed", counter, "lines for", protocol))
# add data point line to data frame
# temp.df <- as.data.frame(fromJSON(temp))
# df <- rbind.fill(df, temp.df)
json <- paste(json, temp, ",", sep="")
}
# increment processed line counter
counter <- counter + 1
if(counter %% 1000 == 0)
print(paste("Processed", counter, "lines"))
json <- substr(json, 0, str_length(json) - 1)
json <- paste(json, "]}")
# add data point line to data frame
temp.df <- as.data.frame(fromJSON(temp))
df <- rbind.fill(df, temp.df)
}
output <- list("response"=response, "df"=df)
return(output)
df <- as.data.frame(fromJSON(json))
output <- list("response"=response, "df"=df)
close(socket)
return(output)
})
}
```

Expand All @@ -125,11 +154,73 @@ With this function, we can now get our data.
# aave.df <- aave$df
# aave.response <- aave$response
aave.liquidations <- request("amberdata-aave-protocol", "action,LiquidationCall", "x-api-key,UAK7ed69235426c360be22bfc2bde1809b6", "2022-07-01", "2023-09-01")
aave.liquidations.df <- aave.liquidations$df
aave.liquidations.response <- aave.liquidations$response
# aave.liquidations <- request("amberdata-aave-protocol", "action,LiquidationCall", "x-api-key,UAK7ed69235426c360be22bfc2bde1809b6", "2022-07-01", "2023-09-01")
# aave.liquidations.df <- aave.liquidations$df
# aave.liquidations.response <- aave.liquidations$response
# sushiswap <- request("amberdata-sushiswap-protocol", "", "x-api-key,UAK7ed69235426c360be22bfc2bde1809b6", "2022-08-01", "2022-08-03")
# sushiswap.df <- sushiswap$df
# sushiswap.response <- sushiswap$response
start_date <- "2022-01-01"
end_date <- "2022-01-10"
graph.borrows.df <- request("graph-aave-borrows", "", "", start_date, end_date)$df
graph.collaterals.df <- request("graph-aave-collaterals", "", "", start_date, end_date)$df
graph.deposits.df <- request("graph-aave-deposits", "", "", start_date, end_date)$df
graph.flashloans.df <- request("graph-aave-flash-loans", "", "", start_date, end_date)$df
graph.liquidations.df <- request("graph-aave-liquidations", "", "", start_date, end_date)$df
graph.pricehist.df <- request("graph-aave-price-history-items", "", "", start_date, end_date)$df
graph.redeems.df <- request("graph-aave-redeems", "", "", start_date, end_date)$df
graph.repays.df <- request("graph-aave-repays", "", "", start_date, end_date)$df
graph.reservehist.df <- request("graph-aave-reserve-params-hist-items", "", "", start_date, end_date)$df
graph.reserves.df <- request("graph-aave-reserves", "", "")$df
graph.swaps.df <- request("graph-aave-swaps", "", "", start_date, end_date)$df
graph.userreserves.df <- request("graph-aave-user-reserves", "", "", start_date, end_date)$df
graph.users.df <- request("graph-aave-users", "", "")$df
graph.users.df <- addAliases(graph.users.df, graph.users.df$id)
```

```{r}
amberdata.addresses.test <- request("amberdata-blockchain-addresses", "hash,0xfaf3af4f551f76af4cde17c3d3708c4f3a69d21e", "x-api-key,UAK7ed69235426c360be22bfc2bde1809b6")
```

```{r}
stablecoins <- request("llama-stablecoins", "", "")$df
```

```{r}
graph.users.sample <- graph.users.df %>%
select(-alias)
aliases = NULL
set.seed(69420)
while(length(aliases[,1]) < length(graph.users.sample$id)){
alias <- randomNames(1000, name.order = "first.last", name.sep = " ", sample.with.replacement = FALSE)
aliases <- aliases %>%
bind_rows(data.frame(alias)) %>%
distinct()
}
aliases <- aliases %>%
head(length(graph.users.sample$id))
userAliases <- bind_cols(graph.users.sample, aliases) %>%
mutate(version = "V2",
deployment = "Mainnet")
graph.users.sample <- userAliases
```
Loading

0 comments on commit ae809ee

Please sign in to comment.