mirror of
https://github.com/catdevnull/transicion-desordenada-diablo
synced 2024-11-15 02:21:39 +00:00
init
This commit is contained in:
commit
9516fcc674
4 changed files with 378 additions and 0 deletions
3
.gitignore
vendored
Normal file
3
.gitignore
vendored
Normal file
|
|
@ -0,0 +1,3 @@
|
||||||
|
node_modules/
|
||||||
|
dataJsons/
|
||||||
|
log
|
||||||
189
download_json.js
Normal file
189
download_json.js
Normal file
|
|
@ -0,0 +1,189 @@
|
||||||
|
// @ts-check
|
||||||
|
import { mkdir, open } from "node:fs/promises";
|
||||||
|
import { Agent, fetch } from "undici";
|
||||||
|
import { join, normalize } from "node:path";
|
||||||
|
import { pipeline } from "node:stream/promises";
|
||||||
|
|
||||||
|
// lista de dominios que permitimos usar http: porque tienen HTTPS roto..
|
||||||
|
const brokenHttps = [
|
||||||
|
"datos.mindef.gov.ar", // cert para otro dominio
|
||||||
|
"datos.energia.gob.ar", // cert para otro dominio
|
||||||
|
"datos.minem.gob.ar", // vencido 2022-17-06
|
||||||
|
"datos.agroindustria.gob.ar", // vencido 2022-03-10
|
||||||
|
"andino.siu.edu.ar", // self signed, igual parece que todo tira 404 en este..
|
||||||
|
"datos.salud.gob.ar", // timeout en HTTPS
|
||||||
|
"datos.jus.gob.ar", // HTTPS redirige incorrectamente a URLs inexistentes
|
||||||
|
"www.hidro.gob.ar", // no HTTPS
|
||||||
|
];
|
||||||
|
|
||||||
|
// FYI: al menos los siguientes dominios no tienen la cadena completa de certificados en HTTPS. tenemos que usar un hack (node_extra_ca_certs_mozilla_bundle) para conectarnos a estos sitios. (se puede ver con ssllabs.com) ojalá lxs administradorxs de estos servidores lo arreglen.
|
||||||
|
// www.enargas.gov.ar, transparencia.enargas.gov.ar, www.energia.gob.ar, www.economia.gob.ar, datos.yvera.gob.ar
|
||||||
|
|
||||||
|
const dispatcher = new Agent({
|
||||||
|
pipelining: 10,
|
||||||
|
maxRedirections: 20,
|
||||||
|
});
|
||||||
|
|
||||||
|
class StatusCodeError extends Error {
|
||||||
|
/**
|
||||||
|
* @param {number} code
|
||||||
|
*/
|
||||||
|
constructor(code) {
|
||||||
|
super(`Status code: ${code}`);
|
||||||
|
this.code = code;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const outputPath = process.argv[2];
|
||||||
|
if (!outputPath) {
|
||||||
|
console.error("Especificamente el output porfa");
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
await mkdir(outputPath, { recursive: true });
|
||||||
|
|
||||||
|
// Leer JSON de stdin
|
||||||
|
const json = await process.stdin.toArray();
|
||||||
|
const jsonString = json.join("");
|
||||||
|
/** @type {{ dataset: Dataset[] }} */
|
||||||
|
const parsed = JSON.parse(jsonString);
|
||||||
|
|
||||||
|
const jobs = parsed.dataset.flatMap((dataset) =>
|
||||||
|
dataset.distribution.map((dist) => ({ dataset, dist })),
|
||||||
|
);
|
||||||
|
// forma barrani de distribuir carga entre servidores
|
||||||
|
shuffleArray(jobs);
|
||||||
|
const totalJobs = jobs.length;
|
||||||
|
let nFinished = 0;
|
||||||
|
|
||||||
|
const duplicated = hasDuplicates(
|
||||||
|
jobs.map((j) => `${j.dataset.identifier}/${j.dist.identifier}`),
|
||||||
|
);
|
||||||
|
if (duplicated) {
|
||||||
|
console.error(
|
||||||
|
"ADVERTENCIA: ¡encontré duplicados! es posible que se pisen archivos entre si",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const greens = Array(128)
|
||||||
|
.fill(0)
|
||||||
|
.map(() =>
|
||||||
|
(async () => {
|
||||||
|
let job;
|
||||||
|
while ((job = jobs.pop())) {
|
||||||
|
const { dataset, dist } = job;
|
||||||
|
request: do {
|
||||||
|
try {
|
||||||
|
await downloadDist(dataset, dist);
|
||||||
|
} catch (error) {
|
||||||
|
if (error instanceof StatusCodeError) {
|
||||||
|
// algunos servidores usan 403 como coso para decir "calmate"
|
||||||
|
if (error.code === 403) {
|
||||||
|
console.debug(
|
||||||
|
`debug: reintentando ${dist.downloadURL} porque tiró 403`,
|
||||||
|
);
|
||||||
|
await wait(15000);
|
||||||
|
continue request;
|
||||||
|
}
|
||||||
|
error = error.toString();
|
||||||
|
}
|
||||||
|
console.error(
|
||||||
|
`error: Failed to download URL ${dist.downloadURL} (${dataset.identifier}/${dist.identifier}):`,
|
||||||
|
error,
|
||||||
|
);
|
||||||
|
if (!(error instanceof StatusCodeError)) continue request;
|
||||||
|
} finally {
|
||||||
|
nFinished++;
|
||||||
|
}
|
||||||
|
} while (0);
|
||||||
|
}
|
||||||
|
})(),
|
||||||
|
);
|
||||||
|
|
||||||
|
const interval = setInterval(() => {
|
||||||
|
console.info(`info: ${nFinished}/${totalJobs} done`);
|
||||||
|
}, 15000);
|
||||||
|
await Promise.all(greens);
|
||||||
|
clearInterval(interval);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @argument {Dataset} dataset
|
||||||
|
* @argument {Distribution} dist
|
||||||
|
*/
|
||||||
|
async function downloadDist(dataset, dist) {
|
||||||
|
const url = new URL(dist.downloadURL);
|
||||||
|
|
||||||
|
// Siempre usar HTTPS excepto cuando está roto
|
||||||
|
if (brokenHttps.includes(url.host)) {
|
||||||
|
url.protocol = "http:";
|
||||||
|
// console.debug(url);
|
||||||
|
} else url.protocol = "https:";
|
||||||
|
|
||||||
|
const res = await fetch(url.toString(), {
|
||||||
|
dispatcher,
|
||||||
|
});
|
||||||
|
if (res.status >= 400) {
|
||||||
|
throw new StatusCodeError(res.status);
|
||||||
|
}
|
||||||
|
|
||||||
|
const fileDirPath = join(
|
||||||
|
outputPath,
|
||||||
|
sanitizeSuffix(dataset.identifier),
|
||||||
|
sanitizeSuffix(dist.identifier),
|
||||||
|
);
|
||||||
|
await mkdir(fileDirPath, { recursive: true });
|
||||||
|
const filePath = join(
|
||||||
|
fileDirPath,
|
||||||
|
sanitizeSuffix(dist.fileName || dist.identifier),
|
||||||
|
);
|
||||||
|
const outputFile = await open(filePath, "w");
|
||||||
|
|
||||||
|
if (!res.body) throw new Error("no body");
|
||||||
|
await pipeline(res.body, outputFile.createWriteStream());
|
||||||
|
}
|
||||||
|
|
||||||
|
/** @typedef {object} Dataset
|
||||||
|
* @prop {string} identifier
|
||||||
|
* @prop {Distribution[]} distribution
|
||||||
|
*/
|
||||||
|
/** @typedef {object} Distribution
|
||||||
|
* @prop {string} identifier
|
||||||
|
* @prop {string} fileName
|
||||||
|
* @prop {string} downloadURL
|
||||||
|
*/
|
||||||
|
|
||||||
|
// https://security.stackexchange.com/a/123723
|
||||||
|
/**
|
||||||
|
* @argument {string} path
|
||||||
|
*/
|
||||||
|
function sanitizeSuffix(path) {
|
||||||
|
return normalize(path).replace(/^(\.\.(\/|\\|$))+/, "");
|
||||||
|
}
|
||||||
|
|
||||||
|
// https://stackoverflow.com/a/7376645
|
||||||
|
/**
|
||||||
|
* @argument {any[]} array
|
||||||
|
*/
|
||||||
|
function hasDuplicates(array) {
|
||||||
|
return new Set(array).size !== array.length;
|
||||||
|
}
|
||||||
|
|
||||||
|
// https://stackoverflow.com/a/12646864
|
||||||
|
/**
|
||||||
|
* @argument {any[]} array
|
||||||
|
*/
|
||||||
|
function shuffleArray(array) {
|
||||||
|
for (var i = array.length - 1; i > 0; i--) {
|
||||||
|
var j = Math.floor(Math.random() * (i + 1));
|
||||||
|
var temp = array[i];
|
||||||
|
array[i] = array[j];
|
||||||
|
array[j] = temp;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @argument {number} ms
|
||||||
|
*/
|
||||||
|
function wait(ms) {
|
||||||
|
if (ms < 0) return Promise.resolve();
|
||||||
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||||
|
}
|
||||||
17
package.json
Normal file
17
package.json
Normal file
|
|
@ -0,0 +1,17 @@
|
||||||
|
{
|
||||||
|
"name": "js",
|
||||||
|
"type": "module",
|
||||||
|
"version": "1.0.0",
|
||||||
|
"description": "",
|
||||||
|
"main": "index.js",
|
||||||
|
"scripts": {
|
||||||
|
"run": "env NODE_EXTRA_CA_CERTS=node_modules/node_extra_ca_certs_mozilla_bundle/ca_bundle/ca_intermediate_root_bundle.pem node download_json.js"
|
||||||
|
},
|
||||||
|
"keywords": [],
|
||||||
|
"author": "",
|
||||||
|
"license": "ISC",
|
||||||
|
"dependencies": {
|
||||||
|
"node_extra_ca_certs_mozilla_bundle": "^1.0.5",
|
||||||
|
"undici": "^5.28.0"
|
||||||
|
}
|
||||||
|
}
|
||||||
169
pnpm-lock.yaml
Normal file
169
pnpm-lock.yaml
Normal file
|
|
@ -0,0 +1,169 @@
|
||||||
|
lockfileVersion: '6.0'
|
||||||
|
|
||||||
|
settings:
|
||||||
|
autoInstallPeers: true
|
||||||
|
excludeLinksFromLockfile: false
|
||||||
|
|
||||||
|
dependencies:
|
||||||
|
node_extra_ca_certs_mozilla_bundle:
|
||||||
|
specifier: ^1.0.5
|
||||||
|
version: 1.0.5
|
||||||
|
undici:
|
||||||
|
specifier: ^5.28.0
|
||||||
|
version: 5.28.0
|
||||||
|
|
||||||
|
packages:
|
||||||
|
|
||||||
|
/@fastify/busboy@2.1.0:
|
||||||
|
resolution: {integrity: sha512-+KpH+QxZU7O4675t3mnkQKcZZg56u+K/Ct2K+N2AZYNVK8kyeo/bI18tI8aPm3tvNNRyTWfj6s5tnGNlcbQRsA==}
|
||||||
|
engines: {node: '>=14'}
|
||||||
|
dev: false
|
||||||
|
|
||||||
|
/asynckit@0.4.0:
|
||||||
|
resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==}
|
||||||
|
dev: false
|
||||||
|
|
||||||
|
/axios@0.27.2:
|
||||||
|
resolution: {integrity: sha512-t+yRIyySRTp/wua5xEr+z1q60QmLq8ABsS5O9Me1AsE5dfKqgnCFzwiCZZ/cGNd1lq4/7akDWMxdhVlucjmnOQ==}
|
||||||
|
dependencies:
|
||||||
|
follow-redirects: 1.15.3
|
||||||
|
form-data: 4.0.0
|
||||||
|
transitivePeerDependencies:
|
||||||
|
- debug
|
||||||
|
dev: false
|
||||||
|
|
||||||
|
/bluebird@3.7.2:
|
||||||
|
resolution: {integrity: sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==}
|
||||||
|
dev: false
|
||||||
|
|
||||||
|
/combined-stream@1.0.8:
|
||||||
|
resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==}
|
||||||
|
engines: {node: '>= 0.8'}
|
||||||
|
dependencies:
|
||||||
|
delayed-stream: 1.0.0
|
||||||
|
dev: false
|
||||||
|
|
||||||
|
/cross-env@6.0.3:
|
||||||
|
resolution: {integrity: sha512-+KqxF6LCvfhWvADcDPqo64yVIB31gv/jQulX2NGzKS/g3GEVz6/pt4wjHFtFWsHMddebWD/sDthJemzM4MaAag==}
|
||||||
|
engines: {node: '>=8.0'}
|
||||||
|
hasBin: true
|
||||||
|
dependencies:
|
||||||
|
cross-spawn: 7.0.3
|
||||||
|
dev: false
|
||||||
|
|
||||||
|
/cross-spawn@7.0.3:
|
||||||
|
resolution: {integrity: sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==}
|
||||||
|
engines: {node: '>= 8'}
|
||||||
|
dependencies:
|
||||||
|
path-key: 3.1.1
|
||||||
|
shebang-command: 2.0.0
|
||||||
|
which: 2.0.2
|
||||||
|
dev: false
|
||||||
|
|
||||||
|
/csvtojson@2.0.10:
|
||||||
|
resolution: {integrity: sha512-lUWFxGKyhraKCW8Qghz6Z0f2l/PqB1W3AO0HKJzGIQ5JRSlR651ekJDiGJbBT4sRNNv5ddnSGVEnsxP9XRCVpQ==}
|
||||||
|
engines: {node: '>=4.0.0'}
|
||||||
|
hasBin: true
|
||||||
|
dependencies:
|
||||||
|
bluebird: 3.7.2
|
||||||
|
lodash: 4.17.21
|
||||||
|
strip-bom: 2.0.0
|
||||||
|
dev: false
|
||||||
|
|
||||||
|
/delayed-stream@1.0.0:
|
||||||
|
resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==}
|
||||||
|
engines: {node: '>=0.4.0'}
|
||||||
|
dev: false
|
||||||
|
|
||||||
|
/follow-redirects@1.15.3:
|
||||||
|
resolution: {integrity: sha512-1VzOtuEM8pC9SFU1E+8KfTjZyMztRsgEfwQl44z8A25uy13jSzTj6dyK2Df52iV0vgHCfBwLhDWevLn95w5v6Q==}
|
||||||
|
engines: {node: '>=4.0'}
|
||||||
|
peerDependencies:
|
||||||
|
debug: '*'
|
||||||
|
peerDependenciesMeta:
|
||||||
|
debug:
|
||||||
|
optional: true
|
||||||
|
dev: false
|
||||||
|
|
||||||
|
/form-data@4.0.0:
|
||||||
|
resolution: {integrity: sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==}
|
||||||
|
engines: {node: '>= 6'}
|
||||||
|
dependencies:
|
||||||
|
asynckit: 0.4.0
|
||||||
|
combined-stream: 1.0.8
|
||||||
|
mime-types: 2.1.35
|
||||||
|
dev: false
|
||||||
|
|
||||||
|
/is-utf8@0.2.1:
|
||||||
|
resolution: {integrity: sha512-rMYPYvCzsXywIsldgLaSoPlw5PfoB/ssr7hY4pLfcodrA5M/eArza1a9VmTiNIBNMjOGr1Ow9mTyU2o69U6U9Q==}
|
||||||
|
dev: false
|
||||||
|
|
||||||
|
/isexe@2.0.0:
|
||||||
|
resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==}
|
||||||
|
dev: false
|
||||||
|
|
||||||
|
/lodash@4.17.21:
|
||||||
|
resolution: {integrity: sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==}
|
||||||
|
dev: false
|
||||||
|
|
||||||
|
/mime-db@1.52.0:
|
||||||
|
resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==}
|
||||||
|
engines: {node: '>= 0.6'}
|
||||||
|
dev: false
|
||||||
|
|
||||||
|
/mime-types@2.1.35:
|
||||||
|
resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==}
|
||||||
|
engines: {node: '>= 0.6'}
|
||||||
|
dependencies:
|
||||||
|
mime-db: 1.52.0
|
||||||
|
dev: false
|
||||||
|
|
||||||
|
/node_extra_ca_certs_mozilla_bundle@1.0.5:
|
||||||
|
resolution: {integrity: sha512-Y+wek3qK8WYybCIxArGTmCEJCJ/6uGud/HCJECBZPIgagF9ba90nhnQMxBcMUAwQaR53iphGYp0JzlVPpUBsjg==}
|
||||||
|
requiresBuild: true
|
||||||
|
dependencies:
|
||||||
|
axios: 0.27.2
|
||||||
|
cross-env: 6.0.3
|
||||||
|
csvtojson: 2.0.10
|
||||||
|
transitivePeerDependencies:
|
||||||
|
- debug
|
||||||
|
dev: false
|
||||||
|
|
||||||
|
/path-key@3.1.1:
|
||||||
|
resolution: {integrity: sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==}
|
||||||
|
engines: {node: '>=8'}
|
||||||
|
dev: false
|
||||||
|
|
||||||
|
/shebang-command@2.0.0:
|
||||||
|
resolution: {integrity: sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==}
|
||||||
|
engines: {node: '>=8'}
|
||||||
|
dependencies:
|
||||||
|
shebang-regex: 3.0.0
|
||||||
|
dev: false
|
||||||
|
|
||||||
|
/shebang-regex@3.0.0:
|
||||||
|
resolution: {integrity: sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==}
|
||||||
|
engines: {node: '>=8'}
|
||||||
|
dev: false
|
||||||
|
|
||||||
|
/strip-bom@2.0.0:
|
||||||
|
resolution: {integrity: sha512-kwrX1y7czp1E69n2ajbG65mIo9dqvJ+8aBQXOGVxqwvNbsXdFM6Lq37dLAY3mknUwru8CfcCbfOLL/gMo+fi3g==}
|
||||||
|
engines: {node: '>=0.10.0'}
|
||||||
|
dependencies:
|
||||||
|
is-utf8: 0.2.1
|
||||||
|
dev: false
|
||||||
|
|
||||||
|
/undici@5.28.0:
|
||||||
|
resolution: {integrity: sha512-gM12DkXhlAc5+/TPe60iy9P6ETgVfqTuRJ6aQ4w8RYu0MqKuXhaq3/b86GfzDQnNA3NUO6aUNdvevrKH59D0Nw==}
|
||||||
|
engines: {node: '>=14.0'}
|
||||||
|
dependencies:
|
||||||
|
'@fastify/busboy': 2.1.0
|
||||||
|
dev: false
|
||||||
|
|
||||||
|
/which@2.0.2:
|
||||||
|
resolution: {integrity: sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==}
|
||||||
|
engines: {node: '>= 8'}
|
||||||
|
hasBin: true
|
||||||
|
dependencies:
|
||||||
|
isexe: 2.0.0
|
||||||
|
dev: false
|
||||||
Loading…
Reference in a new issue