From 5406113dbb0a0618a2dc56b58b415d2cd41f7ed5 Mon Sep 17 00:00:00 2001
From: Nulo <git@nulo.in>
Date: Tue, 19 Dec 2023 00:55:47 -0300
Subject: [PATCH] refactor get json

---
 downloader/download_json.js | 30 ++++++++++++++++++------------
 1 file changed, 18 insertions(+), 12 deletions(-)

diff --git a/downloader/download_json.js b/downloader/download_json.js
index 9aeb241..3d7de24 100644
--- a/downloader/download_json.js
+++ b/downloader/download_json.js
@@ -34,20 +34,11 @@ for (const target of targets)
  */
 async function downloadFromData(target) {
   const outputPath = generateOutputPath(target.url);
-  let json;
-  if (target.type === "ckan") {
-    json = await generateDataJsonFromCkan(target.url);
-  } else if (target.type === "datajson") {
-    const jsonRes = await customRequestWithLimitsAndRetries(
-      new URL(target.url)
-    );
-    json = await jsonRes.body.json();
-  }
-
-  const parsed = zData.parse(json);
+  const json = await getDataJsonForTarget(target);
+  const parsed = zData.parse(JSON.parse(json));
 
   await mkdir(outputPath, { recursive: true });
-  await writeFile(join(outputPath, "data.json"), JSON.stringify(json));
+  await writeFile(join(outputPath, "data.json"), json);
   await writeFile(join(outputPath, "url.txt"), `${target.type}+${target.url}`);
   const errorFile = createWriteStream(join(outputPath, "errors.jsonl"), {
     flags: "w",
@@ -117,6 +108,21 @@ async function downloadFromData(target) {
   }
 }
 
+/**
+ * @param {Target} target
+ * @returns {Promise<string>}
+ */
+async function getDataJsonForTarget(target) {
+  if (target.type === "ckan") {
+    return JSON.stringify(await generateDataJsonFromCkan(target.url));
+  } else if (target.type === "datajson") {
+    const jsonRes = await customRequestWithLimitsAndRetries(
+      new URL(target.url)
+    );
+    return await jsonRes.body.text();
+  } else throw new Error("?????????????");
+}
+
 /**
  * @param {string} jsonUrlString
  */