Nulo/transicion-desordenada-diablo
1
1
Fork 0
mirror of https://github.com/catdevnull/transicion-desordenada-diablo synced 2024-11-14 18:21:38 +00:00
Code Issues Projects Releases Packages Wiki Activity

refactor get json

Browse source
This commit is contained in:
Cat /dev/Nulo 2023-12-19 00:55:47 -03:00
parent eb18951686
commit 5406113dbb
1 changed files with 18 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
downloader/download_json.js
View file

@ -34,20 +34,11 @@ for (const target of targets)
*/ */
async function downloadFromData(target) { async function downloadFromData(target) {
const outputPath = generateOutputPath(target.url); const outputPath = generateOutputPath(target.url);
let json; const json = await getDataJsonForTarget(target);
if (target.type === "ckan") { const parsed = zData.parse(JSON.parse(json));
json = await generateDataJsonFromCkan(target.url);
} else if (target.type === "datajson") {
const jsonRes = await customRequestWithLimitsAndRetries(
new URL(target.url)
);
json = await jsonRes.body.json();
}
const parsed = zData.parse(json);
await mkdir(outputPath, { recursive: true }); await mkdir(outputPath, { recursive: true });
await writeFile(join(outputPath, "data.json"), JSON.stringify(json)); await writeFile(join(outputPath, "data.json"), json);
await writeFile(join(outputPath, "url.txt"), `${target.type}+${target.url}`); await writeFile(join(outputPath, "url.txt"), `${target.type}+${target.url}`);
const errorFile = createWriteStream(join(outputPath, "errors.jsonl"), { const errorFile = createWriteStream(join(outputPath, "errors.jsonl"), {
flags: "w", flags: "w",
@ -117,6 +108,21 @@ async function downloadFromData(target) {
} }
} }
/**
* @param {Target} target
* @returns {Promise<string>}
*/
async function getDataJsonForTarget(target) {
if (target.type === "ckan") {
return JSON.stringify(await generateDataJsonFromCkan(target.url));
} else if (target.type === "datajson") {
const jsonRes = await customRequestWithLimitsAndRetries(
new URL(target.url)
);
return await jsonRes.body.text();
} else throw new Error("?????????????");
}
/** /**
* @param {string} jsonUrlString * @param {string} jsonUrlString
*/ */