Nulo/preciazo
1
0
Fork 0
mirror of https://github.com/catdevnull/preciazo.git synced 2025-02-22 13:46:24 +00:00
Code Issues Projects Releases Packages Wiki Activity

use socks proxy instead

Browse source
This commit is contained in:
Cat /dev/Nulo 2025-01-30 20:43:39 -03:00
parent 57aaad78b0
commit f69eb125f6
2 changed files with 6 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.github/workflows/sepa-precios-archiver.yml vendored
View file

@ -27,7 +27,7 @@ jobs:
B2_BUCKET_NAME: ${{ secrets.B2_BUCKET_NAME }} B2_BUCKET_NAME: ${{ secrets.B2_BUCKET_NAME }}
B2_BUCKET_KEY_ID: ${{ secrets.B2_BUCKET_KEY_ID }} B2_BUCKET_KEY_ID: ${{ secrets.B2_BUCKET_KEY_ID }}
B2_BUCKET_KEY: ${{ secrets.B2_BUCKET_KEY }} B2_BUCKET_KEY: ${{ secrets.B2_BUCKET_KEY }}
DATOS_PRODUCCION_GOB_AR: https://proxy-datos-produccion-gob-ar.nulo.in PROXY_URI: ${{ secrets.PROXY_URI }}
run: | run: |
cd sepa cd sepa
bun install --frozen-lockfile bun install --frozen-lockfile

8
sepa/archiver.ts
View file

@ -25,6 +25,8 @@ const B2_BUCKET_KEY = checkEnvVariable("B2_BUCKET_KEY");
const DATOS_PRODUCCION_GOB_AR = const DATOS_PRODUCCION_GOB_AR =
process.env.DATOS_PRODUCCION_GOB_AR || "https://datos.produccion.gob.ar"; process.env.DATOS_PRODUCCION_GOB_AR || "https://datos.produccion.gob.ar";
const PROXY_URI = process.env.PROXY_URI;
const CURL_PROXY_ARG = PROXY_URI ? `-x${PROXY_URI}` : "";
const processUrl = (url: string) => const processUrl = (url: string) =>
url.replace(/^https:\/\/datos\.produccion\.gob\.ar/, DATOS_PRODUCCION_GOB_AR); url.replace(/^https:\/\/datos\.produccion\.gob\.ar/, DATOS_PRODUCCION_GOB_AR);
@ -42,7 +44,7 @@ async function getRawDatasetInfo(attempts = 0) {
const url = processUrl( const url = processUrl(
"https://datos.produccion.gob.ar/api/3/action/package_show?id=sepa-precios" "https://datos.produccion.gob.ar/api/3/action/package_show?id=sepa-precios"
); );
return await $`curl -L ${url}`.json(); return await $`curl ${CURL_PROXY_ARG} -L ${url}`.json();
} catch (error) { } catch (error) {
if (attempts >= 4) { if (attempts >= 4) {
console.error( console.error(
@ -146,7 +148,7 @@ for (const resource of datasetInfo.result.resources) {
try { try {
const zip = join(dir, "zip"); const zip = join(dir, "zip");
const url = processUrl(resource.url); const url = processUrl(resource.url);
await $`curl --retry 8 --retry-delay 5 --retry-all-errors -L -o ${zip} ${url}`; await $`curl ${CURL_PROXY_ARG} --retry 8 --retry-delay 5 --retry-all-errors -L -o ${zip} ${url}`;
await $`unzip ${zip} -d ${dir}`; await $`unzip ${zip} -d ${dir}`;
await rm(zip); await rm(zip);
@ -174,7 +176,7 @@ for (const resource of datasetInfo.result.resources) {
const fileName = `${resource.id}-${basename(resource.url)}`; const fileName = `${resource.id}-${basename(resource.url)}`;
if (await checkFileExistsInB2(fileName)) continue; if (await checkFileExistsInB2(fileName)) continue;
console.log(`⬇️ Downloading and reuploading ${resource.url}`); console.log(`⬇️ Downloading and reuploading ${resource.url}`);
const response = await $`curl -L ${resource.url}`.blob(); const response = await $`curl ${CURL_PROXY_ARG} -L ${resource.url}`.blob();
await uploadToB2Bucket(fileName, response); await uploadToB2Bucket(fileName, response);
} }