Nulo/preciazo
1
0
Fork 0
mirror of https://github.com/catdevnull/preciazo.git synced 2024-11-22 06:16:18 +00:00
Code Issues Projects Releases Packages Wiki Activity

fetch with retry

Browse source
This commit is contained in:
Cat /dev/Nulo 2024-08-28 20:59:03 -03:00
parent dd6fc0b172
commit 814041eaf5
1 changed files with 23 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
sepa-precios-archiver/index.ts
View file

@ -32,7 +32,7 @@ const s3 = new S3Client({
});
async function getRawDatasetInfo() {
const response = await fetch(
const response = await fetchWithRetry(
"https://datos.produccion.gob.ar/api/3/action/package_show?id=sepa-precios",
);
const json = await response.json();
@ -96,6 +96,27 @@ await saveDatasetInfoIntoRepo(rawDatasetInfo);
let errored = false;
async function fetchWithRetry(url: string, maxRetries = 3): Promise<Response> {
let retries = 0;
while (retries < maxRetries) {
try {
const response = await fetch(url);
if (!response.ok) {
throw new Error(`HTTP error! status: ${response.status}`);
}
return response;
} catch (error) {
console.error(`Attempt ${retries + 1} failed: ${error}`);
retries++;
if (retries >= maxRetries) {
throw error;
}
await new Promise((resolve) => setTimeout(resolve, 1000 * retries));
}
}
throw new Error("Max retries reached");
}
function checkRes(
res: Response,
): res is Response & { body: ReadableStream<Uint8Array> } {
@ -151,7 +172,7 @@ for (const resource of datasetInfo.result.resources) {
const fileName = `${resource.id}-${basename(resource.url)}`;
if (await checkFileExistsInB2(fileName)) continue;
console.log(`⬇️ Downloading and reuploading ${resource.url}`);
const response = await fetch(resource.url);
const response = await fetchWithRetry(resource.url);
if (!checkRes(response)) continue;
await uploadToB2Bucket(fileName, response.body);