diff --git a/.gitignore b/.gitignore index de7af75..608b29d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ node_modules/ dataJsons/ log +prueba \ No newline at end of file diff --git a/download_json.js b/download_json.js index 6f0ff10..d914866 100644 --- a/download_json.js +++ b/download_json.js @@ -36,13 +36,16 @@ const jsonString = json.join(""); const parsed = JSON.parse(jsonString); const jobs = parsed.dataset.flatMap((dataset) => - dataset.distribution.map((dist) => ({ dataset, dist })) + dataset.distribution.map((dist) => ({ + dataset, + dist, + url: new URL(dist.downloadURL), + })) ); -// forma barrani de distribuir carga entre servidores -shuffleArray(jobs); const totalJobs = jobs.length; let nFinished = 0; +// por las dudas verificar que no hayan archivos duplicados const duplicated = hasDuplicates( jobs.map((j) => `${j.dataset.identifier}/${j.dist.identifier}`) ); @@ -152,19 +155,6 @@ function hasDuplicates(array) { return new Set(array).size !== array.length; } -// https://stackoverflow.com/a/12646864 -/** - * @argument {any[]} array - */ -function shuffleArray(array) { - for (var i = array.length - 1; i > 0; i--) { - var j = Math.floor(Math.random() * (i + 1)); - var temp = array[i]; - array[i] = array[j]; - array[j] = temp; - } -} - /** * @argument {number} ms */