Nulo/preciazo
1
0
Fork 0
mirror of https://github.com/catdevnull/preciazo.git synced 2024-11-22 14:16:19 +00:00
Code Issues Projects Releases Packages Wiki Activity

usar curll lmfaooooo

Browse source
This commit is contained in:
Cat /dev/Nulo 2024-09-02 09:39:39 -03:00
parent 9c14f98c5c
commit 56cd3d87a7
1 changed files with 3 additions and 34 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

37
sepa-precios-archiver/index.ts
View file

@ -33,10 +33,7 @@ const s3 = new S3Client({
async function getRawDatasetInfo() { async function getRawDatasetInfo() {
try { try {
const response = await fetchWithRetry( return await $`curl -L https://datos.produccion.gob.ar/api/3/action/package_show?id=sepa-precios`.json();
"https://datos.produccion.gob.ar/api/3/action/package_show?id=sepa-precios",
);
return await response.json();
} catch (error) { } catch (error) {
console.error( console.error(
`❌ Error fetching dataset info`, `❌ Error fetching dataset info`,
@ -105,33 +102,6 @@ await saveDatasetInfoIntoRepo(rawDatasetInfo);
let errored = false; let errored = false;
async function fetchWithRetry(
url: string,
maxRetries = 3,
waitTime = 15000,
): Promise<Response> {
let retries = 0;
while (retries < maxRetries) {
try {
const response = await fetch(url, {
signal: AbortSignal.timeout(waitTime),
});
if (!response.ok) {
throw new Error(`HTTP error! status: ${response.status}`);
}
return response;
} catch (error) {
console.error(`Attempt ${retries + 1} failed: ${error}`);
retries++;
if (retries >= maxRetries) {
throw error;
}
await new Promise((resolve) => setTimeout(resolve, 1000 * retries));
}
}
throw new Error("Max retries reached");
}
function checkRes( function checkRes(
res: Response, res: Response,
): res is Response & { body: ReadableStream<Uint8Array> } { ): res is Response & { body: ReadableStream<Uint8Array> } {
@ -187,10 +157,9 @@ for (const resource of datasetInfo.result.resources) {
const fileName = `${resource.id}-${basename(resource.url)}`; const fileName = `${resource.id}-${basename(resource.url)}`;
if (await checkFileExistsInB2(fileName)) continue; if (await checkFileExistsInB2(fileName)) continue;
console.log(`⬇️ Downloading and reuploading ${resource.url}`); console.log(`⬇️ Downloading and reuploading ${resource.url}`);
const response = await fetchWithRetry(resource.url, 3, 60 * 1000); const response = await $`curl -L ${resource.url}`.blob();
if (!checkRes(response)) continue;
await uploadToB2Bucket(fileName, response.body); await uploadToB2Bucket(fileName, response);
} }
} }