Nulo/preciazo
1
0
Fork 0
mirror of https://github.com/catdevnull/preciazo.git synced 2025-02-22 21:06:28 +00:00
Code Issues Projects Releases Packages Wiki Activity

usar socks proxy

Browse source
This commit is contained in:
Cat /dev/Nulo 2025-01-28 15:28:47 -03:00
parent 9688910edc
commit 67623acb31
2 changed files with 9 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
.github/workflows/sepa-precios-archiver.yml vendored
View file

@ -27,7 +27,8 @@ jobs:
B2_BUCKET_NAME: ${{ secrets.B2_BUCKET_NAME }} B2_BUCKET_NAME: ${{ secrets.B2_BUCKET_NAME }}
B2_BUCKET_KEY_ID: ${{ secrets.B2_BUCKET_KEY_ID }} B2_BUCKET_KEY_ID: ${{ secrets.B2_BUCKET_KEY_ID }}
B2_BUCKET_KEY: ${{ secrets.B2_BUCKET_KEY }} B2_BUCKET_KEY: ${{ secrets.B2_BUCKET_KEY }}
DATOS_PRODUCCION_GOB_AR: https://proxy-datos-produccion-gob-ar.nulo.in # DATOS_PRODUCCION_GOB_AR: https://proxy-datos-produccion-gob-ar.nulo.in
PROXY_URI: ${{ secrets.PROXY_URI }}
run: | run: |
cd sepa cd sepa
bun install --frozen-lockfile bun install --frozen-lockfile

10
sepa/archiver.ts
View file

@ -37,12 +37,16 @@ const s3 = new S3Client({
}, },
}); });
const CURL_PROXY_ARG = process.env.PROXY_URI
? { raw: `-x ${process.env.PROXY_URI}` }
: "";
async function getRawDatasetInfo(attempts = 0) { async function getRawDatasetInfo(attempts = 0) {
try { try {
const url = processUrl( const url = processUrl(
"https://datos.produccion.gob.ar/api/3/action/package_show?id=sepa-precios" "https://datos.produccion.gob.ar/api/3/action/package_show?id=sepa-precios"
); );
return await $`curl -L ${url}`.json(); return await $`curl ${CURL_PROXY_ARG} -L ${url}`.json();
} catch (error) { } catch (error) {
if (attempts >= 4) { if (attempts >= 4) {
console.error(`❌ Error fetching dataset info`, error); console.error(`❌ Error fetching dataset info`, error);
@ -142,7 +146,7 @@ for (const resource of datasetInfo.result.resources) {
try { try {
const zip = join(dir, "zip"); const zip = join(dir, "zip");
const url = processUrl(resource.url); const url = processUrl(resource.url);
await $`curl --retry 8 --retry-delay 5 --retry-all-errors -L -o ${zip} ${url}`; await $`curl ${CURL_PROXY_ARG} --retry 8 --retry-delay 5 --retry-all-errors -L -o ${zip} ${url}`;
await $`unzip ${zip} -d ${dir}`; await $`unzip ${zip} -d ${dir}`;
await rm(zip); await rm(zip);
@ -170,7 +174,7 @@ for (const resource of datasetInfo.result.resources) {
const fileName = `${resource.id}-${basename(resource.url)}`; const fileName = `${resource.id}-${basename(resource.url)}`;
if (await checkFileExistsInB2(fileName)) continue; if (await checkFileExistsInB2(fileName)) continue;
console.log(`⬇️ Downloading and reuploading ${resource.url}`); console.log(`⬇️ Downloading and reuploading ${resource.url}`);
const response = await $`curl -L ${resource.url}`.blob(); const response = await $`curl ${CURL_PROXY_ARG} -L ${resource.url}`.blob();
await uploadToB2Bucket(fileName, response); await uploadToB2Bucket(fileName, response);
} }