Stable version for csv agnostic parsing

DataINCITE · Feb 14, 2023 · f3ccdb7 · f3ccdb7
1 parent 484df2d
commit f3ccdb7
Show file tree

Hide file tree

Showing 4 changed files with 69 additions and 30 deletions.
diff --git a/...ta-Engine/Api-Handler/src/main/java/org/application/apihandler/ApiHandlerApplication.java b/...ta-Engine/Api-Handler/src/main/java/org/application/apihandler/ApiHandlerApplication.java
@@ -5,7 +5,10 @@
 import java.io.FileNotFoundException;
 import java.io.FileWriter;
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.Iterator;
+import java.util.TreeSet;
 
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
@@ -23,7 +26,9 @@ public class ApiHandlerApplication {
 	// System.getProperty("user.home") + 
 	private static final String DEFAULT_OUTPUT = "/data";//String.format("/Users/%s/Documents/DeFi-Data", System.getProperty("user.name"));
 
-	private static final HashMap<String, BufferedWriter> file_lock = new HashMap<String, BufferedWriter>();
+	private static final HashMap<String, BufferedWriter> writers = new HashMap<String, BufferedWriter>();
+	private static final HashMap<String, ArrayList<HashMap<String, String>>> buffers = new HashMap<String, ArrayList<HashMap<String, String>>>();
+	private static final HashMap<String, TreeSet<String>> headers = new HashMap<String, TreeSet<String>>();
 
 	private static final void initialize() {
 		// retrieve home directory and path to default output
@@ -34,7 +39,7 @@ private static final void initialize() {
 	}
 
 	public final static void lock(String name) {
-		if(!file_lock.containsKey(name)) {
+		if(!writers.containsKey(name)) {
 			try {
 				String file_name = DEFAULT_OUTPUT + "/" + name + ".csv";
 				File file = new File(file_name);
@@ -46,8 +51,9 @@ public final static void lock(String name) {
 					file.delete();
 					file.createNewFile();
 				}
-				System.out.println("writing to: " + file.getAbsolutePath());
-				file_lock.put(name, new BufferedWriter(new FileWriter(file)));
+				writers.put(name, new BufferedWriter(new FileWriter(file)));
+				buffers.put(name, new ArrayList<HashMap<String, String>>());
+				headers.put(name, new TreeSet<String>());
 			} catch (FileNotFoundException e) {
 				e.printStackTrace();
 				System.exit(1);
@@ -59,30 +65,67 @@ public final static void lock(String name) {
 	}
 
 	public final static void unlock(String name) {
-		if(file_lock.containsKey(name)) {
+		if(writers.containsKey(name)) {
 			try {
-				file_lock.get(name).close();
+				// retrieve writer
+				BufferedWriter writer = writers.get(name);
+
+				// write all column headers
+				Iterator<String> header_itr = headers.get(name).iterator();
+				while(header_itr.hasNext()) {
+					writer.write(header_itr.next());
+					if(header_itr.hasNext())
+						writer.write(",");
+				}
+
+				// new line
+				writer.write("\n");
+
+				// write to file using formatted headers
+				ArrayList<HashMap<String, String>> buffer = buffers.get(name);
+				for(int i = 0; i < buffer.size(); i++) {
+					StringBuilder line = new StringBuilder();
+
+					// loop through all headers and format
+					for(String header : headers.get(name)) {
+						if(buffer.get(i).containsKey(header)) {
+							line.append(buffer.get(i).get(header));
+						}
+
+						line.append(",");
+					}
+
+					// write line to file
+					line.delete(line.length() - 1, line.length());
+					writer.write(line.toString());
+
+					// new line
+					if(i != buffer.size() - 1)
+						writer.write("\n");
+				}
+
+				// close output loop
+				writers.get(name).close();
 			} catch (IOException e) {
 				e.printStackTrace();
 				System.exit(1);
 			}
 		}
 
-		file_lock.remove(name);
+		writers.remove(name);
 	}
 
-	public final static void output(String name, String data) {
-		if(!file_lock.containsKey(name)) {
+	public final static void output(String name, HashMap<String, String> data) {
+		if(!writers.containsKey(name) || !buffers.containsKey(name) || !headers.containsKey(name)) {
 			lock(name);
 		}
 
-		try {
-			file_lock.get(name).write(data);
-			file_lock.get(name).write("\n");
-		} catch (IOException e) {
-			e.printStackTrace();
-			System.exit(1);
-		}
+		// add all headers
+		for(String header : data.keySet())
+			headers.get(name).add(header);
+
+		// push data to buffers
+		buffers.get(name).add(data);
 	}
 
 	public static void main(String[] args) {

diff --git a/DeFi-Data-Engine/Api-Handler/src/main/java/org/application/apihandler/Controller.java b/DeFi-Data-Engine/Api-Handler/src/main/java/org/application/apihandler/Controller.java
@@ -134,7 +134,9 @@ public ResponseEntity<String> handleRequestDated(
 		}
 
 		// submit request
+		ApiHandlerApplication.lock(name);
 		String response = request.request(path_map, properties_map, headers_map, start_date, end_date);
+		ApiHandlerApplication.unlock(name);
 		if(response != null)
 			return new ResponseEntity<String>(response, HttpStatus.SERVICE_UNAVAILABLE);
 

diff --git a/...-Engine/Api-Handler/src/main/java/org/stream/external/request/types/RequestFramework.java b/...-Engine/Api-Handler/src/main/java/org/stream/external/request/types/RequestFramework.java
@@ -328,15 +328,9 @@ protected String process(String url, HashMap<String, String> properties, HashMap
 
 	public abstract String getType();
 
-	protected final StringBuilder parse(Object input) throws JSONException {
-		StringBuilder out = new StringBuilder();
-		out = out.append(parseJsonObject(input));
-		return out.deleteCharAt(out.length() - 1);
-	}
-
-	private final StringBuilder parseJsonObject(Object input) throws JSONException {
+	protected final HashMap<String, String> parse(Object input) throws JSONException {
 
-		StringBuilder out = new StringBuilder();
+		HashMap<String, String> out = new HashMap<String, String>();
 
 	    if (input instanceof JSONObject) {
 
@@ -348,20 +342,20 @@ private final StringBuilder parseJsonObject(Object input) throws JSONException {
 
 	            if (!(((JSONObject) input).get(key) instanceof JSONArray)) {
 	                if (((JSONObject) input).get(key) instanceof JSONObject) {
-	                	out = out.append(parseJsonObject(((JSONObject) input).get(key)));
+	                	out.putAll(parse(((JSONObject) input).get(key)));
 	                } else {
-	                    out = out.append(key + "=" + ((JSONObject) input).get(key) + ",");
+	                    out.put(key, ((JSONObject) input).get(key).toString());
 	                }
 	            } else {
-	            	out = out.append(parseJsonObject(new JSONArray(((JSONObject) input).get(key).toString())));
+	            	out.putAll(parse(new JSONArray(((JSONObject) input).get(key).toString())));
 	            }
 	        }
 	    }
 
 	    if (input instanceof JSONArray) {
 	        for (int i = 0; i < ((JSONArray) input).length(); i++) {
 	            JSONObject a = ((JSONArray) input).getJSONObject(i);
-	            out = out.append(parseJsonObject(a));
+	            out.putAll(parse(a));
 	        }
 	    }
 

diff --git a/...ine/Api-Handler/src/main/java/org/stream/external/request/types/RequestParameterized.java b/...ine/Api-Handler/src/main/java/org/stream/external/request/types/RequestParameterized.java
@@ -119,8 +119,8 @@ else if(obj.has(path[i])) {
 
 		// extract and print data
 		for(int i = 0; i < data.length(); i++) {
-			StringBuilder point = parse(data.getJSONObject(i));
-			ApiHandlerApplication.output(this.getName(), point.toString());
+			HashMap<String, String> point = parse(data.getJSONObject(i));
+			ApiHandlerApplication.output(this.getName(), point);
 		}
 
 		// initiate recursive call