.gitignore

@@ -1,5 +1,3 @@
 node_modules/
 dataJsons/
 log
-prueba
-datos.gob.ar*

download_json.js

@@ -1,17 +1,28 @@
 // @ts-check
-import { mkdir, open, writeFile } from "node:fs/promises";
+import { mkdir, open } from "node:fs/promises";
-import { Agent, fetch, request, setGlobalDispatcher } from "undici";
+import { Agent, fetch } from "undici";
 import { join, normalize } from "node:path";
 import { pipeline } from "node:stream/promises";
 
+// lista de dominios que permitimos usar http: porque tienen HTTPS roto..
+const brokenHttps = [
+  "datos.mindef.gov.ar", // cert para otro dominio
+  "datos.energia.gob.ar", // cert para otro dominio
+  "datos.minem.gob.ar", // vencido 2022-17-06
+  "datos.agroindustria.gob.ar", // vencido 2022-03-10
+  "andino.siu.edu.ar", // self signed, igual parece que todo tira 404 en este..
+  "datos.salud.gob.ar", // timeout en HTTPS
+  "datos.jus.gob.ar", // HTTPS redirige incorrectamente a URLs inexistentes
+  "www.hidro.gob.ar", // no HTTPS
+];
+
 // FYI: al menos los siguientes dominios no tienen la cadena completa de certificados en HTTPS. tenemos que usar un hack (node_extra_ca_certs_mozilla_bundle) para conectarnos a estos sitios. (se puede ver con ssllabs.com) ojalá lxs administradorxs de estos servidores lo arreglen.
 // www.enargas.gov.ar, transparencia.enargas.gov.ar, www.energia.gob.ar, www.economia.gob.ar, datos.yvera.gob.ar
 
-setGlobalDispatcher(
+const dispatcher = new Agent({
-  new Agent({
+  pipelining: 10,
-    pipelining: 0,
+  maxRedirections: 20,
-  })
+});
-);
 
 class StatusCodeError extends Error {
   /**
@@ -22,142 +33,107 @@ class StatusCodeError extends Error {
     this.code = code;
   }
 }
-class TooManyRedirectsError extends Error {}
 
-let jsonUrlString = process.argv[2];
+const outputPath = process.argv[2];
-if (!jsonUrlString) {
+if (!outputPath) {
-  console.error("Especificamente el url al json porfa");
+  console.error("Especificamente el output porfa");
   process.exit(1);
 }
-const jsonUrl = new URL(jsonUrlString);
-const outputPath = jsonUrl.host;
 await mkdir(outputPath, { recursive: true });
-const errorFile = await open(join(outputPath, "errors.jsonl"), "w");
 
-const jsonRes = await fetch(jsonUrl);
+// Leer JSON de stdin
-// prettier-ignore
+const json = await process.stdin.toArray();
-const parsed = /** @type {{ dataset: Dataset[] }} */(await jsonRes.json())
+const jsonString = json.join("");
-await writeFile(join(outputPath, "data.json"), JSON.stringify(parsed));
+/** @type {{ dataset: Dataset[] }} */
+const parsed = JSON.parse(jsonString);
 
 const jobs = parsed.dataset.flatMap((dataset) =>
-  dataset.distribution.map((dist) => ({
+  dataset.distribution.map((dist) => ({ dataset, dist })),
-    dataset,
-    dist,
-    url: patchUrl(new URL(dist.downloadURL)),
-  }))
 );
+// forma barrani de distribuir carga entre servidores
+shuffleArray(jobs);
 const totalJobs = jobs.length;
 let nFinished = 0;
-let nErrors = 0;
 
-// por las dudas verificar que no hayan archivos duplicados
+const duplicated = hasDuplicates(
-chequearIdsDuplicados();
+  jobs.map((j) => `${j.dataset.identifier}/${j.dist.identifier}`),
-
+);
-/** @type {Map< string, DownloadJob[] >} */
+if (duplicated) {
-let jobsPerHost = new Map();
+  console.error(
-for (const job of jobs) {
+    "ADVERTENCIA: ¡encontré duplicados! es posible que se pisen archivos entre si",
-  jobsPerHost.set(job.url.host, [
+  );
-    ...(jobsPerHost.get(job.url.host) || []),
-    job,
-  ]);
 }
 
-const greens = [...jobsPerHost.entries()].flatMap(([host, jobs]) => {
+const greens = Array(128)
-  const nThreads = 8;
-  return Array(nThreads)
   .fill(0)
   .map(() =>
     (async () => {
       let job;
       while ((job = jobs.pop())) {
+        const { dataset, dist } = job;
+        request: do {
           try {
-            await downloadDistWithRetries(job);
+            await downloadDist(dataset, dist);
           } catch (error) {
-            await errorFile.write(
+            if (error instanceof StatusCodeError) {
-              JSON.stringify({
+              // algunos servidores usan 403 como coso para decir "calmate"
-                url: job.url.toString(),
+              if (error.code === 403) {
-                ...encodeError(error),
+                console.debug(
-              }) + "\n"
+                  `debug: reintentando ${dist.downloadURL} porque tiró 403`,
                 );
-            nErrors++;
+                await wait(15000);
+                continue request;
+              }
+              error = error.toString();
+            }
+            console.error(
+              `error: Failed to download URL ${dist.downloadURL} (${dataset.identifier}/${dist.identifier}):`,
+              error,
+            );
+            if (!(error instanceof StatusCodeError)) continue request;
           } finally {
             nFinished++;
           }
+        } while (0);
       }
-      })()
+    })(),
   );
-});
-process.stderr.write(`greens: ${greens.length}\n`);
 
 const interval = setInterval(() => {
-  process.stderr.write(`info: ${nFinished}/${totalJobs} done\n`);
+  console.info(`info: ${nFinished}/${totalJobs} done`);
-}, 30000);
+}, 15000);
 await Promise.all(greens);
 clearInterval(interval);
-if (nErrors > 0) console.error(`Finished with ${nErrors} errors`);
 
 /**
- * @argument {DownloadJob} job
+ * @argument {Dataset} dataset
- * @argument {number} tries
+ * @argument {Distribution} dist
  */
-async function downloadDistWithRetries(job, tries = 0) {
+async function downloadDist(dataset, dist) {
-  const { url } = job;
+  const url = new URL(dist.downloadURL);
-  try {
-    await downloadDist(job);
-  } catch (error) {
-    // algunos servidores usan 403 como coso para decir "calmate"
-    // intentar hasta 15 veces con 15 segundos de por medio
-    if (
-      error instanceof StatusCodeError &&
-      error.code === 403 &&
-      url.host === "minsegar-my.sharepoint.com" &&
-      tries < 15
-    ) {
-      await wait(15000);
-      return await downloadDistWithRetries(job, tries + 1);
-    }
-    // si no fue un error de http, reintentar hasta 5 veces con 5 segundos de por medio
-    else if (
-      !(error instanceof StatusCodeError) &&
-      !(error instanceof TooManyRedirectsError) &&
-      tries < 5
-    ) {
-      await wait(5000);
-      return await downloadDistWithRetries(job, tries + 1);
-    } else throw error;
-  }
-}
 
-/**
+  // Siempre usar HTTPS excepto cuando está roto
- * @argument {DownloadJob} job
+  if (brokenHttps.includes(url.host)) {
- */
+    url.protocol = "http:";
-async function downloadDist({ dist, dataset, url }) {
+    // console.debug(url);
-  // sharepoint no le gusta compartir a bots lol
+  } else url.protocol = "https:";
-  const spoofUserAgent = url.host.endsWith("sharepoint.com");
 
-  const res = await request(url.toString(), {
+  const res = await fetch(url.toString(), {
-    maxRedirections: 20,
+    dispatcher,
-    headers: {
-      "User-Agent": spoofUserAgent
-        ? "Mozilla/5.0 (X11; Linux x86_64; rv:120.0) Gecko/20100101 Firefox/120.0"
-        : "transicion-desordenada (https://nulo.ar)",
-    },
   });
-  if (res.statusCode >= 300 && res.statusCode <= 399)
+  if (res.status >= 400) {
-    throw new TooManyRedirectsError();
+    throw new StatusCodeError(res.status);
-  if (res.statusCode < 200 || res.statusCode > 299) {
-    throw new StatusCodeError(res.statusCode);
   }
 
   const fileDirPath = join(
     outputPath,
     sanitizeSuffix(dataset.identifier),
-    sanitizeSuffix(dist.identifier)
+    sanitizeSuffix(dist.identifier),
   );
   await mkdir(fileDirPath, { recursive: true });
   const filePath = join(
     fileDirPath,
-    sanitizeSuffix(dist.fileName || dist.identifier)
+    sanitizeSuffix(dist.fileName || dist.identifier),
   );
   const outputFile = await open(filePath, "w");
 
@@ -165,16 +141,11 @@ async function downloadDist({ dist, dataset, url }) {
   await pipeline(res.body, outputFile.createWriteStream());
 }
 
-/** @typedef DownloadJob
+/** @typedef {object} Dataset
- * @prop {Dataset} dataset
- * @prop {Distribution} dist
- * @prop {URL} url
- */
-/** @typedef Dataset
  * @prop {string} identifier
  * @prop {Distribution[]} distribution
  */
-/** @typedef Distribution
+/** @typedef {object} Distribution
  * @prop {string} identifier
  * @prop {string} fileName
  * @prop {string} downloadURL
@@ -188,46 +159,31 @@ function sanitizeSuffix(path) {
   return normalize(path).replace(/^(\.\.(\/|\\|$))+/, "");
 }
 
-function chequearIdsDuplicados() {
-  const duplicated = hasDuplicates(
-    jobs.map((j) => `${j.dataset.identifier}/${j.dist.identifier}`)
-  );
-  if (duplicated) {
-    console.error(
-      "ADVERTENCIA: ¡encontré duplicados! es posible que se pisen archivos entre si"
-    );
-  }
-}
 // https://stackoverflow.com/a/7376645
-/** @argument {any[]} array */
+/**
+ * @argument {any[]} array
+ */
 function hasDuplicates(array) {
   return new Set(array).size !== array.length;
 }
 
-/** @argument {number} ms */
+// https://stackoverflow.com/a/12646864
-function wait(ms) {
+/**
-  if (ms < 0) return Promise.resolve();
+ * @argument {any[]} array
-  return new Promise((resolve) => setTimeout(resolve, ms));
+ */
-}
+function shuffleArray(array) {
-
+  for (var i = array.length - 1; i > 0; i--) {
-function encodeError(error) {
+    var j = Math.floor(Math.random() * (i + 1));
-  if (error instanceof StatusCodeError)
+    var temp = array[i];
-    return { kind: "http_error", status_code: error.code };
+    array[i] = array[j];
-  else if (error instanceof TooManyRedirectsError)
+    array[j] = temp;
-    return { kind: "infinite_redirect" };
-  else {
-    return { kind: "generic_error", error: error.code || error.message };
   }
 }
 
 /**
- * parchea URLs que se rompen solas
+ * @argument {number} ms
- * @param {URL} url
  */
-function patchUrl(url) {
+function wait(ms) {
-  if (url.host === "www.ign.gob.ar") {
+  if (ms < 0) return Promise.resolve();
-    // por defecto, 'http://www.ign.gob.ar' redirige a 'https://ign.gob.ar' pero su certificado solo aplica para '*.ign.gob.ar'. se sirve todo el contenido correctamente en 'https://www.ign.gob.ar', así que vamos para ahí.
+  return new Promise((resolve) => setTimeout(resolve, ms));
-    url.protocol = "https:";
-  }
-  return url;
 }

package.json

@@ -13,8 +13,5 @@
   "dependencies": {
     "node_extra_ca_certs_mozilla_bundle": "^1.0.5",
     "undici": "^5.28.0"
-  },
-  "devDependencies": {
-    "@types/node": "^20.10.0"
   }
 }

pnpm-lock.yaml

@@ -12,11 +12,6 @@ dependencies:
     specifier: ^5.28.0
     version: 5.28.0
 
-devDependencies:
-  '@types/node':
-    specifier: ^20.10.0
-    version: 20.10.0
-
 packages:
 
   /@fastify/busboy@2.1.0:
@@ -24,12 +19,6 @@ packages:
     engines: {node: '>=14'}
     dev: false
 
-  /@types/node@20.10.0:
-    resolution: {integrity: sha512-D0WfRmU9TQ8I9PFx9Yc+EBHw+vSpIub4IDvQivcp26PtPrdMGAq5SDcpXEo/epqa/DXotVpekHiLNTg3iaKXBQ==}
-    dependencies:
-      undici-types: 5.26.5
-    dev: true
-
   /asynckit@0.4.0:
     resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==}
     dev: false
@@ -164,10 +153,6 @@ packages:
       is-utf8: 0.2.1
     dev: false
 
-  /undici-types@5.26.5:
-    resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==}
-    dev: true
-
   /undici@5.28.0:
     resolution: {integrity: sha512-gM12DkXhlAc5+/TPe60iy9P6ETgVfqTuRJ6aQ4w8RYu0MqKuXhaq3/b86GfzDQnNA3NUO6aUNdvevrKH59D0Nw==}
     engines: {node: '>=14.0'}

readme.md

@@ -9,15 +9,5 @@ pnpm install
 ## correr
 
 ```
-pnpm run run download_json.js https://datos.gob.ar/data.json
+pnpm run run download_json.js carpeta_output < dataJsons/datos.gob.ar.data.json
-# guarda en ./datos.gob.ar
 ```
-
-## formato de repo guardado
-
-- `{dominio de repo}`
-  - `data.json`
-  - `errors.jsonl`
-  - `{identifier de dataset}`
-    - `{identifier de distribution}`
-      - `{fileName (o, si no existe, identifier de distribution)}`