From 2afdbb028b5f883eaf8ac1e1cad0f62e652d6db1 Mon Sep 17 00:00:00 2001 From: Conor Flynn Date: Wed, 1 Feb 2023 13:51:32 -0500 Subject: [PATCH] Integrate request parameterized url --- .../external/requester/RequestFramework.java | 110 +++++++++++++++++- .../requester/RequestParameterized.java | 108 +++++++++++++++++ .../external/requester/RequestRecursive.java | 17 --- .../resources/requests/template.properties | 9 +- .../external/requester/RequestFramework.class | Bin 4510 -> 7746 bytes .../classes/requests/template.properties | 9 +- 6 files changed, 222 insertions(+), 31 deletions(-) create mode 100644 DeFi-Data-Engine/Api-Handler/src/main/java/org/stream/external/requester/RequestParameterized.java delete mode 100644 DeFi-Data-Engine/Api-Handler/src/main/java/org/stream/external/requester/RequestRecursive.java diff --git a/DeFi-Data-Engine/Api-Handler/src/main/java/org/stream/external/requester/RequestFramework.java b/DeFi-Data-Engine/Api-Handler/src/main/java/org/stream/external/requester/RequestFramework.java index a27dddbd..5702b08c 100644 --- a/DeFi-Data-Engine/Api-Handler/src/main/java/org/stream/external/requester/RequestFramework.java +++ b/DeFi-Data-Engine/Api-Handler/src/main/java/org/stream/external/requester/RequestFramework.java @@ -1,23 +1,34 @@ package org.stream.external.requester; +import java.io.IOException; import java.util.HashMap; -import java.util.List; -import java.util.stream.Collectors; +import java.util.Iterator; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +import okhttp3.OkHttpClient; import okhttp3.Request; import okhttp3.Request.Builder; +import okhttp3.Response; public abstract class RequestFramework { + private final String name; private final String url; private final HashMap properties; private final HashMap headers; - private final String[] tags; + private final HashMap tags; + private final String[] path; - public RequestFramework(String url, HashMap properties, HashMap headers, String... tags) { + public RequestFramework(String name, String url, HashMap properties, HashMap headers, + HashMap tags, String[] path) { + this.name = name; this.url = url; this.properties = properties; this.headers = headers; this.tags = tags; + this.path = path; } protected final Request getRequest(HashMap properties, HashMap headers) { @@ -89,9 +100,25 @@ protected final Request getRequest(HashMap properties, HashMap getTags() { return this.tags; } + + public final String getName() { + return name; + } + + public final String[] getPath() { + return path; + } @SuppressWarnings("unchecked") public final void request(HashMap properties, HashMap headers) { @@ -102,5 +129,76 @@ public final void request(HashMap properties, HashMap properties, HashMap headers); + protected void process(HashMap properties, HashMap headers) { + OkHttpClient client = new OkHttpClient(); + Request request = getRequest(properties, headers); + if(request == null) { + System.err.println("Malformed request, killing process."); + return; + } + + Response response = null; + try { + response = client.newCall(request).execute(); + if(response.code() != 200) { + System.err.println(String.format("Request Failure code <%d> url <%s>\nmsg <%s>", response.code(), request.url().toString(), response.message())); + System.exit(1); + } + } catch (IOException e) { + e.printStackTrace(); + } + + if(response == null) { + System.err.println("Response had fatal issue, killing process."); + return; + } + + // send to specific request handler + process(response.body().toString(), properties, headers); + } + + protected abstract void process(String json, HashMap properties, HashMap headers); + + public abstract String getType(); + + + protected final StringBuilder parse(Object input) throws JSONException { + StringBuilder out = new StringBuilder(); + out = out.append(parseJsonObject(input)); + return out.deleteCharAt(out.length() - 1); + } + + private final StringBuilder parseJsonObject(Object input) throws JSONException { + + StringBuilder out = new StringBuilder(); + + if (input instanceof JSONObject) { + + Iterator keys = ((JSONObject) input).keys(); + + while (keys.hasNext()) { + + String key = (String) keys.next(); + + if (!(((JSONObject) input).get(key) instanceof JSONArray)) { + if (((JSONObject) input).get(key) instanceof JSONObject) { + out = out.append(parseJsonObject(((JSONObject) input).get(key))); + } else { + out = out.append(key + "=" + ((JSONObject) input).get(key) + ","); + } + } else { + out = out.append(parseJsonObject(new JSONArray(((JSONObject) input).get(key).toString()))); + } + } + } + + if (input instanceof JSONArray) { + for (int i = 0; i < ((JSONArray) input).length(); i++) { + JSONObject a = ((JSONArray) input).getJSONObject(i); + out = out.append(parseJsonObject(a)); + } + } + + return out; + } } \ No newline at end of file diff --git a/DeFi-Data-Engine/Api-Handler/src/main/java/org/stream/external/requester/RequestParameterized.java b/DeFi-Data-Engine/Api-Handler/src/main/java/org/stream/external/requester/RequestParameterized.java new file mode 100644 index 00000000..759ba651 --- /dev/null +++ b/DeFi-Data-Engine/Api-Handler/src/main/java/org/stream/external/requester/RequestParameterized.java @@ -0,0 +1,108 @@ +package org.stream.external.requester; + +import java.util.HashMap; + +import org.json.JSONArray; +import org.json.JSONObject; + +public class RequestParameterized extends RequestFramework { + + public RequestParameterized(String name, String url, HashMap properties, HashMap headers, + HashMap tags, String[] path) { + super(name, url, properties, headers, tags, path); + } + + @Override + public String getType() { + return "parameterized"; + } + + public void process(String json, HashMap properties, HashMap headers) { + // parse json formatting + JSONObject obj = new JSONObject(json); + + // validate all required parameters are present: + // check for recursive parameter + if(hasTag("-rp") && !properties.containsKey(getTag("-rp"))) { + System.err.println(String.format("Missing required recursive parameter <%s>", getTag("-rp"))); + return; + } + + // check for required tag -l + if(!hasTag("-l")) { + System.err.println(String.format("Missing required recursive parameter <-l>")); + return; + } + + // check for -l being an integer + int limit = 0; + try { + limit = Integer.parseInt(getTag("-l")); + } catch(Exception e) { + e.printStackTrace(); + System.err.println(String.format("Value following <-l> must be an integer.")); + return; + } + + // validate that the base has the proper obj path + String[] path = getPath(); + JSONArray data = null; + for(int i = 0; i < path.length; i++) { + if(i == path.length - 1) { + if(obj.has(path[i])) { + try { + data = obj.getJSONArray(path[i]); + } catch(Exception e) { + System.err.println("obj path type is not of type . Cannot parse."); + return; + } + } + } + + else if(obj.has(path[i])) { + try { + obj = obj.getJSONObject(path[i]); + } catch(Exception e) { + System.err.println("obj path type step is not of type . Cannot parse."); + return; + } + } + } + + if(data == null) { + System.err.println("Data array retrieval had fatal error, killing process."); + return; + } + + // extract and print data + for(int i = 0; i < data.length(); i++) { + StringBuilder point = parse(data.getJSONObject(i)); + System.out.println(point); + } + + // initiate recursive call + // if under limit requested, terminate call + if(data.length() < limit) + return; + + // extract recursive parameter + int param = -1; + try { + param = Integer.parseInt(properties.get(getTag("-rp"))); + } catch(Exception e) { + e.printStackTrace(); + System.err.println(String.format("Recursive parameter <%s> is not of type integer.", getTag("-rp"))); + return; + } + + if(param == -1) { + System.err.println("Fatal parsing error occured."); + return; + } + + // increment param, replace, and execute + param += 1; + properties.put(getTag("-rp"), "" + param); + process(properties, headers); + } +} \ No newline at end of file diff --git a/DeFi-Data-Engine/Api-Handler/src/main/java/org/stream/external/requester/RequestRecursive.java b/DeFi-Data-Engine/Api-Handler/src/main/java/org/stream/external/requester/RequestRecursive.java deleted file mode 100644 index d28af63a..00000000 --- a/DeFi-Data-Engine/Api-Handler/src/main/java/org/stream/external/requester/RequestRecursive.java +++ /dev/null @@ -1,17 +0,0 @@ -package org.stream.external.requester; - -import java.util.HashMap; - -import okhttp3.Request; - -public class RequestRecursive extends RequestFramework { - - public RequestRecursive(String url, HashMap properties, HashMap headers, - String[] tags) { - super(url, properties, headers, tags); - } - - public void process(HashMap properties, HashMap headers) { - - } -} diff --git a/DeFi-Data-Engine/Api-Handler/src/main/resources/requests/template.properties b/DeFi-Data-Engine/Api-Handler/src/main/resources/requests/template.properties index 5dd16153..65544255 100644 --- a/DeFi-Data-Engine/Api-Handler/src/main/resources/requests/template.properties +++ b/DeFi-Data-Engine/Api-Handler/src/main/resources/requests/template.properties @@ -34,6 +34,8 @@ url.headers= header1,value1,\ # which have specific properties and handlers. Please review documentation to get a full # list of these tags. To default with no recursive call, set this property to . # This property we will set to for a clearer example. +# - parameterized: one of the parameters in the url call +# - embedded: parameter included within the response url.recursion.type= parameterized # This property sets all tags pertaining to the type of recursive call. Please refer to @@ -43,16 +45,15 @@ url.recursion.type= parameterized # example we will set the tags for which are as follows: # -rp: recursive parameter # -l: limit on items from request -# -t: type of parameter (url or incremental) +# -t: type of parameter (url or incremental) [embedded only required] url.recursion.tags= -rp,next,\ - -l,1000,\ - -t,url + -l,1000 # This property sets the location of the data points to be retrieved from the call. This # should be a JSONArray which the handler can iterate through. To access these data points # directly, the direct path must be specified (consisting of all JSONObject values). In # the example below, we point to the path located at response->data. Note that for storing # all non-array values and just recording all base values returned by the call, please set -# the value of this variable to '-b'. (i.e. url.data.path=-b) +# the value of this variable to '.') url.data.path= response,\ data \ No newline at end of file diff --git a/DeFi-Data-Engine/Api-Handler/target/classes/org/stream/external/requester/RequestFramework.class b/DeFi-Data-Engine/Api-Handler/target/classes/org/stream/external/requester/RequestFramework.class index 3ed11161c1de27cd11b02b55f89bb1e82bc769ff..18aafaea2761f4f02b9c496485ed72be9425a896 100644 GIT binary patch literal 7746 zcmcIo33yyp75?w!&AdtSx+F~+rX}sP(2}(QS%jnwOKEK&O`$0*fdalJugSENnJ_bf zMiB&QSwx@$BDese1eL8orzsU=2T^eWM3yS3;L4)7AOi6}@4cD4$xNX7eSB@^-FNRf z=bm%!IsZBLJ@NiM_W?La?XSb9uppl3ZAzvRmf7EAZAw{*m>F$KSf>wKNxn3#bVf@P zX1{f2Jh4%SrZ9Dbd4|~(HDkR^E7onWx>E{Tj06hh?PE)wsYE2!+oDi3n20Kr3?$+M zRw5O#k_yuty1`T=+O*6}_ANIDNLbt%>5ZAG!2~58mQS%TUwHmZE9K~Ymf2$^xQ~`H zd*v};ruw++$@%mOfrXJ+B-N^rtjS-VXeLKtm3T$4xTH%%dbJsoeM!s$zHRA+! zYK!ANRtbtwX<#a*Gc0OqR~JKJF9UuAWHiHo?u_;}5Ja)e%`#v(qp*QV&S9taB zYA{sY6a26MsBMpL>`SEv4$Nc~ci3Ej_oF_bd_8nDD3sZ$>ouYT^{AAeCIkE90H(@D zYpBy=x|h||dIF16Eu}aR2kV3tOd6BJ1p|lR;|jqD(Zo!}6Le}0Z<{+$0f(VQM>9cW zOh9JUz(On%lp#bZRAeN0Ac(Ao=OB{T#w7-p z;uB;X7$oDN1sFZ>FtI4Htj)kNI98y^Om^5tF#J!_v0R~QTvY6P)L!3 zit(VDrC5QLl1-fom0s;!=&fBXR-9m9wY_yDxuky}H59~&3hM9@tiVYIPR3fEt2r=W z#d;JP3rX=9=?HeWZwca51(cu_pEO`%9R;ymm{BHMjyf6FoK%b+tk=Or^^}m#Ay%4x z1HB+b7g-60NTsu7B;Ir!YeZH+gWG{IrkPb&VIKGV0~o*;0Z5By^n4(qSJ#u0jTq;(2{Qbd1J=7^Ghp zh|AA3uu0re+8vLj%t$PGtVOqfN?{)XjYz`k3A@$MQ9CS#^jPZ&XFUsoIE%{9OD>w1 zoG*uTwt;i-Sx!Z*Sg({ZMoyb7oM+&3IA6i&v7%PWTHI$Qj-Y+A@S~39atrmM&wjB8# zcDpD$9cQ;@l{K~Alyuc2ZNh0l6>8lx_iowZsGl<>J=I_V+C5Y+U*aAzI$*oaz@1X5 z{N2%bjO~uX-3GSfHd?|^>9&&eYMH&(lM_~x-8nb1bnIkOX2NmB#${|97DppiELDOL zsmTi8Gf;!tAnqfoEH|S9c7zqzT@B%lk!Y0dx8RKIN?01f{RSSugH$DEow=BC%oMB{ zAEK;e*2cBtC@v)l1{LztLW=7|;C z=bQ^B+^~^rv`GJPsr81$(6L9%gdBReQ)Nw%g#zivl5Fz)U3h%;~8n z>@_ky6)(!!NJ#2R(TQ?x*^rFKnvUsQ(UD!0N-jX4G(-9Ez66ZycMJ)gO-P`0Hh3zi zaG=*e(kOIFvc zG$cznQO-VGO&iyN7bryT7P_irB;*J`GLbNc*yE`xX1UsHoG6zszxinisu_%*MdA_O zEKkP!K{bmL4JE1qEBT}57;3KChc%W3NxbZcl8hmCvhgMFWEWoylRtn66dC(j64o! zIi_*;NxtiR&ac~nin{vksH$t&j!<3WcFe2`Y1>g<7xHh%?7C23JLYU7WfAt_?_6H* zfxVz%26vmu>-^r7SB(&6VK)EGLm2y#VwoeI@ALV_eQaxZ>EKB0@!2D>NUWW2+oIVc zvBGbUME?Nx<2dY61!;GNzLYu$=OoVgQ6V)oYh#_*s69n)+U)Dvh5D`?XzXyG&05{v zn3=|co%onSb6^;ALxG(*RN;QKHv1bxepwIs(>Q_?X&k*qmX|49JNi6Vs=b{1csWCw z&01ZM^|^X4-)t`*cdReSv%A5|6ZZ1RUJV7AUfhLtn%mLr--Q)jR5-K)$2SKW(^z#a zN<)G2HEDFM;j3$Za2V%@f?50Z#*pq<-dr3ieiV~J#pS1@aoRNq-HGn~f<|1=l{@&nho5Dh#sPSR&o^)o-p0YK1&1hu zxLd(bDGSh|4kKJHng5d<$`^uN9`R4J&<}Y-%Jmzbvre>&m1iM-H&aoIr{XLErJDlGH5o zEyFb&Y1Dffu4O{`X-@^dVvnvTusCy1(`sk#ZDr!C_!|7muU6sfxDEkYGicN4v}pr< z5I`|C?z1^`>b=&Pvs1lJ(%ROkl)&ZaRGQUE(%;cZ(*Fip^E#%D5|wlvN=JS6KSAa& zVAhP;RWqyh9;No<33RmUEW{XNYQRcvToZsXTu$RNP zaFZ+Q2v?NhiW20L@b*fJdZZayX+~DsFm5KjQ)RwwE7(NoqK5$L?D66TY;lG|;EF55 zcW|Mrl9#z^Jaeib^LKGe0p>a5nKu<=z7@A+)xX_Vf0)rGr6@$--f=(bI|#?}WC3Xu zG`ef7E|yE!;%uhUIrQtfjE3{U~W$`d96b`9RT3nMHC_jJ_NQ~h1dT1a~ghCunfgzs%i3Lb6w z{!To^V(1rO`#~8VzYeWJ^JKt5YndIvPdka(B>qJjFLZXMHwiK6;@=|8+ zWz6Qwu^wB=cLn0O5-D7bA)?f|#HaI!Hx2S}Z8~@Q1-ld5kkiLiaDQ=WwjW+_=Vb@L7q|2$6`n^N>mN(SZ z?@&c)6?T69P zAc8~Mh?>r(WSR=G70FVi)y!#%-Bp)e@w=7}V|GK`E>+>7z?(bNY*!i;u{(y-sbj@8 zs4r?A#!L#Us7|YJv(~^i)}=hZNL9;1TJ3wo=t~XaIw$ZtY=7TMx7^O6xD8dfi+$~O zx@`xG@(31VCr)6oJ`uZcKJH~`-be7gizxFDe;+2rBX|&x@{7pBc!9?~7J$36uHT(? z{WGrXt8kT?s^;<7=J6^rpDvg9n6LIzHJp18rK(m*_DcW?JUaCCT$8X1BlsR>uy(Qd pyo*JwKMUT$%y$rcmyA{KFvfhUfpk9A$igdM`4vB!KC7Bg{9l^~W=a46 literal 4510 zcmcImTXY*&72Q{sG?F|?ZP|`HN!mC~9sG!`l%$Z@uG7@5Yd4OYM8U>s+Ax+TwqjXQ zN24T$LZLLJ58jW`hC%~{hF2l5h(ePEt6%-#m;S(l4?g$=e*3|~?0aYAj4U}A)?(R9 zbLZZ3&fVv==lt$p*MAFOFMh5eAkd$;=MqK7w#>zZb>6Y;oS9A7){`Zx$d|-1cQtC8 zi`Kcky`Ui|&~(N;YbLU0ZZ7fY>=`TN2sD)JtU!F?=1J1AGr74zfmp%L7cAS!SVe&r zSEA%(vWYRXIDgnIP$Zhn%;ij{WLpCJYbAzi&DQP=(~dT8nQ6=BJweBulh-{~D<}{i z%H%T6u)t(j?XkO41+dOm9lPCA0vFe1aGn0tUb{}kHNXW&@@XrE2DIqdh*l=8t9vR6 zflWF>2+P&&Iy6M&YO{_;cePcAjtv60O=NP`WNC5MvX7gyS*9vJkx!Y~Dbvo#cW*Q3 z%x4(+-4p*^VKIA~s?9AGDkfudmg5y7>%$B2f67qzM1Eo3aSD4XHOw0>)X`mMO#kO( z)Z);vQ=nOOirKMX(EG{NZV5h$!T(TJSobj8JiU)MukAqY!l&H9`j-nd`zg`_v(z6jA58@%o2WD|n zIl)ptq2XbH_ElAJ_XDL&mYfm8C|Wc;LW$KJwjwc(3E4S^1={@RJf`cOl3qNjgAsgG05R;t2_28&Bn^?A z%`BU(3Qi@;C!%-)AJcHEI#EmH^+g>fW*KVBW{I@9S7!2wBcyDHxAJ4a>xl4eoyDCpN#}Je;p5!OTDdugjYrTf4^nMOd9OsrP9epTRTiNhj|fuI?x=MzR2(l|+1Dt-@HNa+#^m32YftpT4?z zir@u-?JL1>KawhgRiTgwUefV-e1TtNGo5xV3LL0YrnT9t;kim@F|VWel8!IqD*~Ze z*>21*NoAEkcrwmT~cbbA5wv-_YMv8pXOF_h}a z{@*`IimAS*;|)pkp;R`X<6I!{108STb=70}lvO0LG^?|8RZcc~dn;*)CG#aaWsPR! zOwi_is8=S&ei>`F&3G`=23IlDpljnO~OW1}EzDaqBkj%`L#28R3iB5X3Sk(}nM z=YQlfo;4yX{%d`P=6XI5HKOlggAt8Smof7yj9=pMcq;fCG|V&@TJqPh%E*W-JkP1~ zP8BvP7d+Wwt!y}63RMiu1dXU`B{?kts0g1o!bZ4^bHQ{OpO`j6^5y9=K06RHf_}*7 z{E!)pTVZ~#7RQ>HpRbh-8JZC-<3&QKpu8X>|HUgT$e=3tH_^@Y?JQV>rMDIR+&#$W zA(r?&i~K1*pJRz$Vv%0q^EyAd?_nqY%I80*`#yR^i1Vt>iflzf^kcW!k3C|H)8$Fr zE1uw}=wLuRk3sPw4v04>^A<+Lbv!8EW>Zv}k1LlJv+gai4VeRD<{ME{Q}o zI0->vlCADylCADyic#)cjQQWuvdn;L8ia<jg. # This property we will set to for a clearer example. +# - parameterized: one of the parameters in the url call +# - embedded: parameter included within the response url.recursion.type= parameterized # This property sets all tags pertaining to the type of recursive call. Please refer to @@ -43,16 +45,15 @@ url.recursion.type= parameterized # example we will set the tags for which are as follows: # -rp: recursive parameter # -l: limit on items from request -# -t: type of parameter (url or incremental) +# -t: type of parameter (url or incremental) [embedded only required] url.recursion.tags= -rp,next,\ - -l,1000,\ - -t,url + -l,1000 # This property sets the location of the data points to be retrieved from the call. This # should be a JSONArray which the handler can iterate through. To access these data points # directly, the direct path must be specified (consisting of all JSONObject values). In # the example below, we point to the path located at response->data. Note that for storing # all non-array values and just recording all base values returned by the call, please set -# the value of this variable to '-b'. (i.e. url.data.path=-b) +# the value of this variable to '.') url.data.path= response,\ data \ No newline at end of file