From 5607d46726de676e91b9c6a06c42cf0c1c54b6f7 Mon Sep 17 00:00:00 2001 From: Nulo Date: Sun, 10 Dec 2023 01:37:08 -0300 Subject: [PATCH] downloader: reportar correctamente n errores --- downloader/download_json.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/downloader/download_json.js b/downloader/download_json.js index 415ea92..477ca87 100644 --- a/downloader/download_json.js +++ b/downloader/download_json.js @@ -68,6 +68,8 @@ async function downloadFromData(target) { await open(join(outputPath, "errors.jsonl"), "w") ).createWriteStream(); try { + let nFinished = 0; + let nErrors = 0; /** @type {DownloadJob[]} */ const jobs = parsed.dataset.flatMap((dataset) => dataset.distribution @@ -82,8 +84,9 @@ async function downloadFromData(target) { return true; } catch (error) { errorFile.write( - JSON.stringify(encodeError({ dataset, dist }, error)) + "\n" + JSON.stringify(encodeError({ dataset, dist }, error)) + "\n", ); + nErrors++; return false; } }, @@ -97,8 +100,6 @@ async function downloadFromData(target) { })), ); const totalJobs = jobs.length; - let nFinished = 0; - let nErrors = 0; // por las dudas verificar que no hayan archivos duplicados chequearIdsDuplicados(jobs, outputPath);