.github/workflows/sepa-precios-archiver.yml

@@ -15,10 +15,10 @@ jobs:
       - uses: oven-sh/setup-bun@v2
         with:
           bun-version: latest
-      # - name: Setup tmate session
+      - name: Setup tmate session
-      #   uses: mxschmitt/action-tmate@v3
+        uses: mxschmitt/action-tmate@v3
-      #   with:
+        with:
-      #     limit-access-to-actor: true
+          limit-access-to-actor: true
 
       - name: Run archiver script
         env:
@@ -27,6 +27,6 @@ jobs:
           B2_BUCKET_KEY_ID: ${{ secrets.B2_BUCKET_KEY_ID }}
           B2_BUCKET_KEY: ${{ secrets.B2_BUCKET_KEY }}
         run: |
-          cd sepa/sepa-precios-archiver
+          cd sepa-precios-archiver
           bun install --frozen-lockfile
           bun index.ts

sepa-precios-archiver/index.ts

@@ -1,5 +1,5 @@
 import { z } from "zod";
-import { zDatasetInfo } from "ckan/schemas";
+import { zDatasetInfo } from "./schemas";
 import { mkdtemp, writeFile, readdir, mkdir, rm } from "fs/promises";
 import { basename, extname, join } from "path";
 import { $, write } from "bun";
@@ -33,7 +33,10 @@ const s3 = new S3Client({
 
 async function getRawDatasetInfo() {
   try {
-    return await $`curl -L https://datos.produccion.gob.ar/api/3/action/package_show?id=sepa-precios`.json();
+    const response = await fetchWithRetry(
+      "https://datos.produccion.gob.ar/api/3/action/package_show?id=sepa-precios",
+    );
+    return await response.json();
   } catch (error) {
     console.error(
       `❌ Error fetching dataset info`,
@@ -102,6 +105,33 @@ await saveDatasetInfoIntoRepo(rawDatasetInfo);
 
 let errored = false;
 
+async function fetchWithRetry(
+  url: string,
+  maxRetries = 3,
+  waitTime = 15000,
+): Promise<Response> {
+  let retries = 0;
+  while (retries < maxRetries) {
+    try {
+      const response = await fetch(url, {
+        signal: AbortSignal.timeout(waitTime),
+      });
+      if (!response.ok) {
+        throw new Error(`HTTP error! status: ${response.status}`);
+      }
+      return response;
+    } catch (error) {
+      console.error(`Attempt ${retries + 1} failed: ${error}`);
+      retries++;
+      if (retries >= maxRetries) {
+        throw error;
+      }
+      await new Promise((resolve) => setTimeout(resolve, 1000 * retries));
+    }
+  }
+  throw new Error("Max retries reached");
+}
+
 function checkRes(
   res: Response,
 ): res is Response & { body: ReadableStream<Uint8Array> } {
@@ -122,7 +152,7 @@ await uploadToB2Bucket(
 const datasetInfo = z.object({ result: zDatasetInfo }).parse(rawDatasetInfo);
 for (const resource of datasetInfo.result.resources) {
   if (extname(resource.url) === ".zip") {
-    const fileName = `${resource.id}-revID-${resource.revision_id}-${basename(resource.url)}-repackaged.tar.zst`;
+    const fileName = `${resource.id}-${basename(resource.url)}-repackaged.tar.zst`;
     if (await checkFileExistsInB2(fileName)) continue;
     console.log(`⬇️ Downloading, repackaging and uploading ${resource.url}`);
     const dir = await mkdtemp("/tmp/sepa-precios-archiver-repackage-");
@@ -157,9 +187,10 @@ for (const resource of datasetInfo.result.resources) {
     const fileName = `${resource.id}-${basename(resource.url)}`;
     if (await checkFileExistsInB2(fileName)) continue;
     console.log(`⬇️ Downloading and reuploading ${resource.url}`);
-    const response = await $`curl -L ${resource.url}`.blob();
+    const response = await fetchWithRetry(resource.url, 3, 60 * 1000);
+    if (!checkRes(response)) continue;
 
-    await uploadToB2Bucket(fileName, response);
+    await uploadToB2Bucket(fileName, response.body);
   }
 }
 

sepa-precios-archiver/schemas.ts

@@ -0,0 +1,17 @@
+import { z } from "zod";
+
+export const zDatasetInfo = z.object({
+  metadata_modified: z.coerce.date(),
+  metadata_created: z.coerce.date(),
+  resources: z.array(
+    z.object({
+      id: z.string(),
+      size: z.number(),
+      format: z.string(),
+      created: z.coerce.date(),
+      url: z.string(),
+      modified: z.coerce.date().optional(),
+      description: z.string(),
+    }),
+  ),
+});

sepa-precios-importer/package.json

@@ -12,7 +12,6 @@
   "dependencies": {
     "p-queue": "^8.0.1",
     "papaparse": "^5.4.1",
-    "postgres": "^3.4.4",
+    "postgres": "^3.4.4"
-    "ckan": "workspace:*"
   }
 }

sepa/ckan/README.md

@@ -1,15 +0,0 @@
-# ckan
-
-To install dependencies:
-
-```bash
-bun install
-```
-
-To run:
-
-```bash
-bun run index.ts
-```
-
-This project was created using `bun init` in bun v1.1.26. [Bun](https://bun.sh) is a fast all-in-one JavaScript runtime.

sepa/ckan/package.json

@@ -1,13 +0,0 @@
-{
-  "name": "ckan",
-  "type": "module",
-  "devDependencies": {
-    "@types/bun": "latest"
-  },
-  "peerDependencies": {
-    "typescript": "^5.0.0"
-  },
-  "dependencies": {
-    "zod": "^3.23.8"
-  }
-}

sepa/ckan/schemas.ts

@@ -1,19 +0,0 @@
-import { z } from "zod";
-
-export const zResource = z.object({
-  id: z.string(),
-  revision_id: z.string(),
-  size: z.number(),
-  format: z.string(),
-  created: z.coerce.date(),
-  url: z.string(),
-  modified: z.coerce.date().optional(),
-  description: z.string(),
-});
-export type Resource = z.infer<typeof zResource>;
-export const zDatasetInfo = z.object({
-  metadata_modified: z.coerce.date(),
-  metadata_created: z.coerce.date(),
-  resources: z.array(zResource),
-});
-export type DatasetInfo = z.infer<typeof zDatasetInfo>;

sepa/package.json

@@ -1,9 +0,0 @@
-{
-  "name": "sepa",
-  "private": true,
-  "workspaces": [
-    "sepa-precios-archiver",
-    "sepa-precios-importer",
-    "ckan"
-  ]
-}

sepa/sepa-precios-importer/.gitignore

@@ -1,175 +0,0 @@
-# Based on https://raw.githubusercontent.com/github/gitignore/main/Node.gitignore
-
-# Logs
-
-logs
-_.log
-npm-debug.log_
-yarn-debug.log*
-yarn-error.log*
-lerna-debug.log*
-.pnpm-debug.log*
-
-# Caches
-
-.cache
-
-# Diagnostic reports (https://nodejs.org/api/report.html)
-
-report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json
-
-# Runtime data
-
-pids
-_.pid
-_.seed
-*.pid.lock
-
-# Directory for instrumented libs generated by jscoverage/JSCover
-
-lib-cov
-
-# Coverage directory used by tools like istanbul
-
-coverage
-*.lcov
-
-# nyc test coverage
-
-.nyc_output
-
-# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
-
-.grunt
-
-# Bower dependency directory (https://bower.io/)
-
-bower_components
-
-# node-waf configuration
-
-.lock-wscript
-
-# Compiled binary addons (https://nodejs.org/api/addons.html)
-
-build/Release
-
-# Dependency directories
-
-node_modules/
-jspm_packages/
-
-# Snowpack dependency directory (https://snowpack.dev/)
-
-web_modules/
-
-# TypeScript cache
-
-*.tsbuildinfo
-
-# Optional npm cache directory
-
-.npm
-
-# Optional eslint cache
-
-.eslintcache
-
-# Optional stylelint cache
-
-.stylelintcache
-
-# Microbundle cache
-
-.rpt2_cache/
-.rts2_cache_cjs/
-.rts2_cache_es/
-.rts2_cache_umd/
-
-# Optional REPL history
-
-.node_repl_history
-
-# Output of 'npm pack'
-
-*.tgz
-
-# Yarn Integrity file
-
-.yarn-integrity
-
-# dotenv environment variable files
-
-.env
-.env.development.local
-.env.test.local
-.env.production.local
-.env.local
-
-# parcel-bundler cache (https://parceljs.org/)
-
-.parcel-cache
-
-# Next.js build output
-
-.next
-out
-
-# Nuxt.js build / generate output
-
-.nuxt
-dist
-
-# Gatsby files
-
-# Comment in the public line in if your project uses Gatsby and not Next.js
-
-# https://nextjs.org/blog/next-9-1#public-directory-support
-
-# public
-
-# vuepress build output
-
-.vuepress/dist
-
-# vuepress v2.x temp and cache directory
-
-.temp
-
-# Docusaurus cache and generated files
-
-.docusaurus
-
-# Serverless directories
-
-.serverless/
-
-# FuseBox cache
-
-.fusebox/
-
-# DynamoDB Local files
-
-.dynamodb/
-
-# TernJS port file
-
-.tern-port
-
-# Stores VSCode versions used for testing VSCode extensions
-
-.vscode-test
-
-# yarn v2
-
-.yarn/cache
-.yarn/unplugged
-.yarn/build-state.yml
-.yarn/install-state.gz
-.pnp.*
-
-# IntelliJ based IDEs
-.idea
-
-# Finder (MacOS) folder config
-.DS_Store

sepa/sepa-precios-importer/tsconfig.json

@@ -1,27 +0,0 @@
-{
-  "compilerOptions": {
-    // Enable latest features
-    "lib": ["ESNext", "DOM"],
-    "target": "ESNext",
-    "module": "ESNext",
-    "moduleDetection": "force",
-    "jsx": "react-jsx",
-    "allowJs": true,
-
-    // Bundler mode
-    "moduleResolution": "bundler",
-    "allowImportingTsExtensions": true,
-    "verbatimModuleSyntax": true,
-    "noEmit": true,
-
-    // Best practices
-    "strict": true,
-    "skipLibCheck": true,
-    "noFallthroughCasesInSwitch": true,
-
-    // Some stricter flags (disabled by default)
-    "noUnusedLocals": false,
-    "noUnusedParameters": false,
-    "noPropertyAccessFromIndexSignature": false
-  }
-}