lpm - usar revision id

.github/workflows/sepa-precios-archiver.yml

@@ -15,10 +15,10 @@ jobs:
       - uses: oven-sh/setup-bun@v2
         with:
           bun-version: latest
-      - name: Setup tmate session
+      # - name: Setup tmate session
-        uses: mxschmitt/action-tmate@v3
+      #   uses: mxschmitt/action-tmate@v3
-        with:
+      #   with:
-          limit-access-to-actor: true
+      #     limit-access-to-actor: true
 
       - name: Run archiver script
         env:
@@ -27,6 +27,6 @@ jobs:
           B2_BUCKET_KEY_ID: ${{ secrets.B2_BUCKET_KEY_ID }}
           B2_BUCKET_KEY: ${{ secrets.B2_BUCKET_KEY }}
         run: |
-          cd sepa-precios-archiver
+          cd sepa/sepa-precios-archiver
           bun install --frozen-lockfile
           bun index.ts

sepa-precios-archiver/schemas.ts

@@ -1,17 +0,0 @@
-import { z } from "zod";
-
-export const zDatasetInfo = z.object({
-  metadata_modified: z.coerce.date(),
-  metadata_created: z.coerce.date(),
-  resources: z.array(
-    z.object({
-      id: z.string(),
-      size: z.number(),
-      format: z.string(),
-      created: z.coerce.date(),
-      url: z.string(),
-      modified: z.coerce.date().optional(),
-      description: z.string(),
-    }),
-  ),
-});

sepa/ckan/README.md

@@ -0,0 +1,15 @@
+# ckan
+
+To install dependencies:
+
+```bash
+bun install
+```
+
+To run:
+
+```bash
+bun run index.ts
+```
+
+This project was created using `bun init` in bun v1.1.26. [Bun](https://bun.sh) is a fast all-in-one JavaScript runtime.

sepa/ckan/package.json

@@ -0,0 +1,13 @@
+{
+  "name": "ckan",
+  "type": "module",
+  "devDependencies": {
+    "@types/bun": "latest"
+  },
+  "peerDependencies": {
+    "typescript": "^5.0.0"
+  },
+  "dependencies": {
+    "zod": "^3.23.8"
+  }
+}

sepa/ckan/schemas.ts

@@ -0,0 +1,19 @@
+import { z } from "zod";
+
+export const zResource = z.object({
+  id: z.string(),
+  revision_id: z.string(),
+  size: z.number(),
+  format: z.string(),
+  created: z.coerce.date(),
+  url: z.string(),
+  modified: z.coerce.date().optional(),
+  description: z.string(),
+});
+export type Resource = z.infer<typeof zResource>;
+export const zDatasetInfo = z.object({
+  metadata_modified: z.coerce.date(),
+  metadata_created: z.coerce.date(),
+  resources: z.array(zResource),
+});
+export type DatasetInfo = z.infer<typeof zDatasetInfo>;

sepa/package.json

@@ -0,0 +1,9 @@
+{
+  "name": "sepa",
+  "private": true,
+  "workspaces": [
+    "sepa-precios-archiver",
+    "sepa-precios-importer",
+    "ckan"
+  ]
+}

sepa/sepa-precios-archiver/index.ts

@@ -1,5 +1,5 @@
 import { z } from "zod";
-import { zDatasetInfo } from "./schemas";
+import { zDatasetInfo } from "ckan/schemas";
 import { mkdtemp, writeFile, readdir, mkdir, rm } from "fs/promises";
 import { basename, extname, join } from "path";
 import { $, write } from "bun";
@@ -33,10 +33,7 @@ const s3 = new S3Client({
 
 async function getRawDatasetInfo() {
   try {
-    const response = await fetchWithRetry(
+    return await $`curl -L https://datos.produccion.gob.ar/api/3/action/package_show?id=sepa-precios`.json();
-      "https://datos.produccion.gob.ar/api/3/action/package_show?id=sepa-precios",
-    );
-    return await response.json();
   } catch (error) {
     console.error(
       `❌ Error fetching dataset info`,
@@ -105,33 +102,6 @@ await saveDatasetInfoIntoRepo(rawDatasetInfo);
 
 let errored = false;
 
-async function fetchWithRetry(
-  url: string,
-  maxRetries = 3,
-  waitTime = 15000,
-): Promise<Response> {
-  let retries = 0;
-  while (retries < maxRetries) {
-    try {
-      const response = await fetch(url, {
-        signal: AbortSignal.timeout(waitTime),
-      });
-      if (!response.ok) {
-        throw new Error(`HTTP error! status: ${response.status}`);
-      }
-      return response;
-    } catch (error) {
-      console.error(`Attempt ${retries + 1} failed: ${error}`);
-      retries++;
-      if (retries >= maxRetries) {
-        throw error;
-      }
-      await new Promise((resolve) => setTimeout(resolve, 1000 * retries));
-    }
-  }
-  throw new Error("Max retries reached");
-}
-
 function checkRes(
   res: Response,
 ): res is Response & { body: ReadableStream<Uint8Array> } {
@@ -152,7 +122,7 @@ await uploadToB2Bucket(
 const datasetInfo = z.object({ result: zDatasetInfo }).parse(rawDatasetInfo);
 for (const resource of datasetInfo.result.resources) {
   if (extname(resource.url) === ".zip") {
-    const fileName = `${resource.id}-${basename(resource.url)}-repackaged.tar.zst`;
+    const fileName = `${resource.id}-revID-${resource.revision_id}-${basename(resource.url)}-repackaged.tar.zst`;
     if (await checkFileExistsInB2(fileName)) continue;
     console.log(`⬇️ Downloading, repackaging and uploading ${resource.url}`);
     const dir = await mkdtemp("/tmp/sepa-precios-archiver-repackage-");
@@ -187,10 +157,9 @@ for (const resource of datasetInfo.result.resources) {
     const fileName = `${resource.id}-${basename(resource.url)}`;
     if (await checkFileExistsInB2(fileName)) continue;
     console.log(`⬇️ Downloading and reuploading ${resource.url}`);
-    const response = await fetchWithRetry(resource.url, 3, 60 * 1000);
+    const response = await $`curl -L ${resource.url}`.blob();
-    if (!checkRes(response)) continue;
 
-    await uploadToB2Bucket(fileName, response.body);
+    await uploadToB2Bucket(fileName, response);
   }
 }
 

sepa/sepa-precios-importer/.gitignore

@@ -0,0 +1,175 @@
+# Based on https://raw.githubusercontent.com/github/gitignore/main/Node.gitignore
+
+# Logs
+
+logs
+_.log
+npm-debug.log_
+yarn-debug.log*
+yarn-error.log*
+lerna-debug.log*
+.pnpm-debug.log*
+
+# Caches
+
+.cache
+
+# Diagnostic reports (https://nodejs.org/api/report.html)
+
+report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json
+
+# Runtime data
+
+pids
+_.pid
+_.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+
+lib-cov
+
+# Coverage directory used by tools like istanbul
+
+coverage
+*.lcov
+
+# nyc test coverage
+
+.nyc_output
+
+# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
+
+.grunt
+
+# Bower dependency directory (https://bower.io/)
+
+bower_components
+
+# node-waf configuration
+
+.lock-wscript
+
+# Compiled binary addons (https://nodejs.org/api/addons.html)
+
+build/Release
+
+# Dependency directories
+
+node_modules/
+jspm_packages/
+
+# Snowpack dependency directory (https://snowpack.dev/)
+
+web_modules/
+
+# TypeScript cache
+
+*.tsbuildinfo
+
+# Optional npm cache directory
+
+.npm
+
+# Optional eslint cache
+
+.eslintcache
+
+# Optional stylelint cache
+
+.stylelintcache
+
+# Microbundle cache
+
+.rpt2_cache/
+.rts2_cache_cjs/
+.rts2_cache_es/
+.rts2_cache_umd/
+
+# Optional REPL history
+
+.node_repl_history
+
+# Output of 'npm pack'
+
+*.tgz
+
+# Yarn Integrity file
+
+.yarn-integrity
+
+# dotenv environment variable files
+
+.env
+.env.development.local
+.env.test.local
+.env.production.local
+.env.local
+
+# parcel-bundler cache (https://parceljs.org/)
+
+.parcel-cache
+
+# Next.js build output
+
+.next
+out
+
+# Nuxt.js build / generate output
+
+.nuxt
+dist
+
+# Gatsby files
+
+# Comment in the public line in if your project uses Gatsby and not Next.js
+
+# https://nextjs.org/blog/next-9-1#public-directory-support
+
+# public
+
+# vuepress build output
+
+.vuepress/dist
+
+# vuepress v2.x temp and cache directory
+
+.temp
+
+# Docusaurus cache and generated files
+
+.docusaurus
+
+# Serverless directories
+
+.serverless/
+
+# FuseBox cache
+
+.fusebox/
+
+# DynamoDB Local files
+
+.dynamodb/
+
+# TernJS port file
+
+.tern-port
+
+# Stores VSCode versions used for testing VSCode extensions
+
+.vscode-test
+
+# yarn v2
+
+.yarn/cache
+.yarn/unplugged
+.yarn/build-state.yml
+.yarn/install-state.gz
+.pnp.*
+
+# IntelliJ based IDEs
+.idea
+
+# Finder (MacOS) folder config
+.DS_Store

sepa/sepa-precios-importer/package.json

@@ -12,6 +12,7 @@
   "dependencies": {
     "p-queue": "^8.0.1",
     "papaparse": "^5.4.1",
-    "postgres": "^3.4.4"
+    "postgres": "^3.4.4",
+    "ckan": "workspace:*"
   }
 }

sepa/sepa-precios-importer/tsconfig.json

@@ -0,0 +1,27 @@
+{
+  "compilerOptions": {
+    // Enable latest features
+    "lib": ["ESNext", "DOM"],
+    "target": "ESNext",
+    "module": "ESNext",
+    "moduleDetection": "force",
+    "jsx": "react-jsx",
+    "allowJs": true,
+
+    // Bundler mode
+    "moduleResolution": "bundler",
+    "allowImportingTsExtensions": true,
+    "verbatimModuleSyntax": true,
+    "noEmit": true,
+
+    // Best practices
+    "strict": true,
+    "skipLibCheck": true,
+    "noFallthroughCasesInSwitch": true,
+
+    // Some stricter flags (disabled by default)
+    "noUnusedLocals": false,
+    "noUnusedParameters": false,
+    "noPropertyAccessFromIndexSignature": false
+  }
+}