Nulo/transicion-desordenada-diablo
1
1
Fork 0
mirror of https://github.com/catdevnull/transicion-desordenada-diablo synced 2024-11-15 10:31:38 +00:00
Code Issues Projects Releases Packages Wiki Activity

downloader: reportar correctamente n errores

Browse source
This commit is contained in:
Cat /dev/Nulo 2023-12-10 01:37:08 -03:00
parent 49bde3947c
commit 5607d46726
1 changed files with 4 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
downloader/download_json.js
View file

@ -68,6 +68,8 @@ async function downloadFromData(target) {
await open(join(outputPath, "errors.jsonl"), "w") await open(join(outputPath, "errors.jsonl"), "w")
).createWriteStream(); ).createWriteStream();
try { try {
let nFinished = 0;
let nErrors = 0;
/** @type {DownloadJob[]} */ /** @type {DownloadJob[]} */
const jobs = parsed.dataset.flatMap((dataset) => const jobs = parsed.dataset.flatMap((dataset) =>
dataset.distribution dataset.distribution
@ -82,8 +84,9 @@ async function downloadFromData(target) {
return true; return true;
} catch (error) { } catch (error) {
errorFile.write( errorFile.write(
JSON.stringify(encodeError({ dataset, dist }, error)) + "\n" JSON.stringify(encodeError({ dataset, dist }, error)) + "\n",
); );
nErrors++;
return false; return false;
} }
}, },
@ -97,8 +100,6 @@ async function downloadFromData(target) {
})), })),
); );
const totalJobs = jobs.length; const totalJobs = jobs.length;
let nFinished = 0;
let nErrors = 0;
// por las dudas verificar que no hayan archivos duplicados // por las dudas verificar que no hayan archivos duplicados
chequearIdsDuplicados(jobs, outputPath); chequearIdsDuplicados(jobs, outputPath);