From 7301ddab7689f1e6e8537e886f9074efd550d930 Mon Sep 17 00:00:00 2001 From: Conor Flynn Date: Tue, 14 Feb 2023 16:59:42 -0500 Subject: [PATCH] Stable version for csv agnostic parsing --- .../apihandler/ApiHandlerApplication.java | 75 +++++++++++++++---- .../application/apihandler/Controller.java | 2 + .../request/types/RequestFramework.java | 18 ++--- .../request/types/RequestParameterized.java | 4 +- 4 files changed, 69 insertions(+), 30 deletions(-) diff --git a/DeFi-Data-Engine/Api-Handler/src/main/java/org/application/apihandler/ApiHandlerApplication.java b/DeFi-Data-Engine/Api-Handler/src/main/java/org/application/apihandler/ApiHandlerApplication.java index c5e733c6..d3c54b7a 100644 --- a/DeFi-Data-Engine/Api-Handler/src/main/java/org/application/apihandler/ApiHandlerApplication.java +++ b/DeFi-Data-Engine/Api-Handler/src/main/java/org/application/apihandler/ApiHandlerApplication.java @@ -5,7 +5,10 @@ import java.io.FileNotFoundException; import java.io.FileWriter; import java.io.IOException; +import java.util.ArrayList; import java.util.HashMap; +import java.util.Iterator; +import java.util.TreeSet; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; @@ -23,7 +26,9 @@ public class ApiHandlerApplication { // System.getProperty("user.home") + private static final String DEFAULT_OUTPUT = "/data";//String.format("/Users/%s/Documents/DeFi-Data", System.getProperty("user.name")); - private static final HashMap file_lock = new HashMap(); + private static final HashMap writers = new HashMap(); + private static final HashMap>> buffers = new HashMap>>(); + private static final HashMap> headers = new HashMap>(); private static final void initialize() { // retrieve home directory and path to default output @@ -34,7 +39,7 @@ private static final void initialize() { } public final static void lock(String name) { - if(!file_lock.containsKey(name)) { + if(!writers.containsKey(name)) { try { String file_name = DEFAULT_OUTPUT + "/" + name + ".csv"; File file = new File(file_name); @@ -46,8 +51,9 @@ public final static void lock(String name) { file.delete(); file.createNewFile(); } - System.out.println("writing to: " + file.getAbsolutePath()); - file_lock.put(name, new BufferedWriter(new FileWriter(file))); + writers.put(name, new BufferedWriter(new FileWriter(file))); + buffers.put(name, new ArrayList>()); + headers.put(name, new TreeSet()); } catch (FileNotFoundException e) { e.printStackTrace(); System.exit(1); @@ -59,30 +65,67 @@ public final static void lock(String name) { } public final static void unlock(String name) { - if(file_lock.containsKey(name)) { + if(writers.containsKey(name)) { try { - file_lock.get(name).close(); + // retrieve writer + BufferedWriter writer = writers.get(name); + + // write all column headers + Iterator header_itr = headers.get(name).iterator(); + while(header_itr.hasNext()) { + writer.write(header_itr.next()); + if(header_itr.hasNext()) + writer.write(","); + } + + // new line + writer.write("\n"); + + // write to file using formatted headers + ArrayList> buffer = buffers.get(name); + for(int i = 0; i < buffer.size(); i++) { + StringBuilder line = new StringBuilder(); + + // loop through all headers and format + for(String header : headers.get(name)) { + if(buffer.get(i).containsKey(header)) { + line.append(buffer.get(i).get(header)); + } + + line.append(","); + } + + // write line to file + line.delete(line.length() - 1, line.length()); + writer.write(line.toString()); + + // new line + if(i != buffer.size() - 1) + writer.write("\n"); + } + + // close output loop + writers.get(name).close(); } catch (IOException e) { e.printStackTrace(); System.exit(1); } } - file_lock.remove(name); + writers.remove(name); } - public final static void output(String name, String data) { - if(!file_lock.containsKey(name)) { + public final static void output(String name, HashMap data) { + if(!writers.containsKey(name) || !buffers.containsKey(name) || !headers.containsKey(name)) { lock(name); } - try { - file_lock.get(name).write(data); - file_lock.get(name).write("\n"); - } catch (IOException e) { - e.printStackTrace(); - System.exit(1); - } + // add all headers + for(String header : data.keySet()) + headers.get(name).add(header); + + // push data to buffers + buffers.get(name).add(data); } public static void main(String[] args) { diff --git a/DeFi-Data-Engine/Api-Handler/src/main/java/org/application/apihandler/Controller.java b/DeFi-Data-Engine/Api-Handler/src/main/java/org/application/apihandler/Controller.java index c7bfb68b..118f4f3e 100644 --- a/DeFi-Data-Engine/Api-Handler/src/main/java/org/application/apihandler/Controller.java +++ b/DeFi-Data-Engine/Api-Handler/src/main/java/org/application/apihandler/Controller.java @@ -134,7 +134,9 @@ public ResponseEntity handleRequestDated( } // submit request + ApiHandlerApplication.lock(name); String response = request.request(path_map, properties_map, headers_map, start_date, end_date); + ApiHandlerApplication.unlock(name); if(response != null) return new ResponseEntity(response, HttpStatus.SERVICE_UNAVAILABLE); diff --git a/DeFi-Data-Engine/Api-Handler/src/main/java/org/stream/external/request/types/RequestFramework.java b/DeFi-Data-Engine/Api-Handler/src/main/java/org/stream/external/request/types/RequestFramework.java index f723b004..99e6b12d 100644 --- a/DeFi-Data-Engine/Api-Handler/src/main/java/org/stream/external/request/types/RequestFramework.java +++ b/DeFi-Data-Engine/Api-Handler/src/main/java/org/stream/external/request/types/RequestFramework.java @@ -328,15 +328,9 @@ protected String process(String url, HashMap properties, HashMap public abstract String getType(); - protected final StringBuilder parse(Object input) throws JSONException { - StringBuilder out = new StringBuilder(); - out = out.append(parseJsonObject(input)); - return out.deleteCharAt(out.length() - 1); - } - - private final StringBuilder parseJsonObject(Object input) throws JSONException { + protected final HashMap parse(Object input) throws JSONException { - StringBuilder out = new StringBuilder(); + HashMap out = new HashMap(); if (input instanceof JSONObject) { @@ -348,12 +342,12 @@ private final StringBuilder parseJsonObject(Object input) throws JSONException { if (!(((JSONObject) input).get(key) instanceof JSONArray)) { if (((JSONObject) input).get(key) instanceof JSONObject) { - out = out.append(parseJsonObject(((JSONObject) input).get(key))); + out.putAll(parse(((JSONObject) input).get(key))); } else { - out = out.append(key + "=" + ((JSONObject) input).get(key) + ","); + out.put(key, ((JSONObject) input).get(key).toString()); } } else { - out = out.append(parseJsonObject(new JSONArray(((JSONObject) input).get(key).toString()))); + out.putAll(parse(new JSONArray(((JSONObject) input).get(key).toString()))); } } } @@ -361,7 +355,7 @@ private final StringBuilder parseJsonObject(Object input) throws JSONException { if (input instanceof JSONArray) { for (int i = 0; i < ((JSONArray) input).length(); i++) { JSONObject a = ((JSONArray) input).getJSONObject(i); - out = out.append(parseJsonObject(a)); + out.putAll(parse(a)); } } diff --git a/DeFi-Data-Engine/Api-Handler/src/main/java/org/stream/external/request/types/RequestParameterized.java b/DeFi-Data-Engine/Api-Handler/src/main/java/org/stream/external/request/types/RequestParameterized.java index eef878ac..f7072ef9 100644 --- a/DeFi-Data-Engine/Api-Handler/src/main/java/org/stream/external/request/types/RequestParameterized.java +++ b/DeFi-Data-Engine/Api-Handler/src/main/java/org/stream/external/request/types/RequestParameterized.java @@ -119,8 +119,8 @@ else if(obj.has(path[i])) { // extract and print data for(int i = 0; i < data.length(); i++) { - StringBuilder point = parse(data.getJSONObject(i)); - ApiHandlerApplication.output(this.getName(), point.toString()); + HashMap point = parse(data.getJSONObject(i)); + ApiHandlerApplication.output(this.getName(), point); } // initiate recursive call