.github/workflows/sepa-precios-archiver.yml

@@ -27,6 +27,6 @@ jobs:
           B2_BUCKET_KEY_ID: ${{ secrets.B2_BUCKET_KEY_ID }}
           B2_BUCKET_KEY: ${{ secrets.B2_BUCKET_KEY }}
         run: |
-          cd sepa
+          cd sepa/sepa-precios-archiver
           bun install --frozen-lockfile
-          bun archiver.ts
+          bun index.ts

sepa/README.md

@@ -1,36 +0,0 @@
-# sepa
-
-## sepa-precios-importer
-
-Importador de [datasets de precios de SEPA](https://datos.produccion.gob.ar/dataset/sepa-precios/archivo/d076720f-a7f0-4af8-b1d6-1b99d5a90c14) a una base de datos PostgreSQL.
-
-Vease [Errores en el formato de los datos SEPA](https://gist.github.com/catdevnull/587d5c63c4bab11b9798861c917db93b)
-
-```bash
-bun install
-bun run importer.ts ~/carpeta-con-datasets-descomprimidos
-```
-
-## sepa-precios-archiver
-
-Archivador del dataset de precios de [Precios Claros - Base SEPA](https://datos.produccion.gob.ar/dataset/sepa-precios). Recomprime para utilizar ~8 veces menos espacio, y resube a un bucket mio de Backblaze B2.
-
-```bash
-bun install
-bun run archiver.ts
-```
-
-## sepa-dataset-validator
-
-un script para validar los datasets de SEPA automaticamente
-
-basado en [la lista de problemas](https://gist.github.com/catdevnull/587d5c63c4bab11b9798861c917db93b) que encontramos
-
-para ejecutar, necesitas [Bun](https://bun.sh)
-
-```bash
-bun install
-bun run dataset-validator/index.ts [ruta/al/dataset]
-```
-
-podes descargar un dump de [nuestro index](https://github.com/catdevnull/sepa-precios-metadata/blob/main/index.md) para analizar (la descarga pesa mucho menos que los oficiales :). para descomprimir, necesitas tener `zstd` y `tar`. después solo tenes que ejecutar `tar xvf ARCHIVO.tar.zst` y listo.

sepa/ckan/README.md

@@ -0,0 +1,15 @@
+# ckan
+
+To install dependencies:
+
+```bash
+bun install
+```
+
+To run:
+
+```bash
+bun run index.ts
+```
+
+This project was created using `bun init` in bun v1.1.26. [Bun](https://bun.sh) is a fast all-in-one JavaScript runtime.

sepa/ckan/package.json

@@ -0,0 +1,13 @@
+{
+  "name": "ckan",
+  "type": "module",
+  "devDependencies": {
+    "@types/bun": "latest"
+  },
+  "peerDependencies": {
+    "typescript": "^5.0.0"
+  },
+  "dependencies": {
+    "zod": "^3.23.8"
+  }
+}

sepa/package.json

@@ -1,22 +1,11 @@
 {
   "name": "sepa",
   "private": true,
-  "type": "module",
+  "workspaces": [
-  "devDependencies": {
+    "sepa-precios-archiver",
-    "@types/bun": "^1.1.7",
+    "sepa-precios-importer",
-    "@types/papaparse": "^5.3.14"
+    "sepa-index-gen",
-  },
+    "sepa-dataset-validator",
-  "peerDependencies": {
+    "ckan"
-    "typescript": "^5.0.0"
+  ]
-  },
-  "dependencies": {
-    "@aws-sdk/client-s3": "^3.651.0",
-    "@aws-sdk/lib-storage": "^3.651.0",
-    "date-fns": "^3.6.0",
-    "jschardet": "^3.1.3",
-    "p-queue": "^8.0.1",
-    "papaparse": "^5.4.1",
-    "postgres": "^3.4.4",
-    "zod": "^3.23.8"
-  }
 }

sepa/scripts/downloadAll.ts

@@ -1,35 +0,0 @@
-import PQueue from "p-queue";
-import { IndexEntry, IndexJson } from "../index-gen";
-import { $ } from "bun";
-import { existsSync } from "fs";
-
-async function getIndex() {
-  const res = await fetch(
-    "https://raw.githubusercontent.com/catdevnull/sepa-precios-metadata/main/index.json"
-  );
-  return IndexJson.parse(await res.json());
-}
-
-const index = await getIndex();
-
-const latestResources = Object.values(index)
-  .filter((a) => a.length > 0)
-  .map(
-    (a) =>
-      a
-        .filter(
-          (r): r is IndexEntry & { link: string } => !!(!r.warnings && r.link)
-        )
-        .sort((a, b) => +b.firstSeenAt - +a.firstSeenAt)[0]
-  );
-
-const queue = new PQueue({ concurrency: 10 });
-
-for (const resource of latestResources) {
-  queue.add(async () => {
-    const filename = resource.link.split("/").pop()!;
-    if (existsSync(filename)) return;
-    await $`curl ${resource.link} -o ${filename}.temp`;
-    await $`mv ${filename}.temp ${filename}`;
-  });
-}

sepa/sepa-dataset-validator/.gitignore

@@ -0,0 +1,175 @@
+# Based on https://raw.githubusercontent.com/github/gitignore/main/Node.gitignore
+
+# Logs
+
+logs
+_.log
+npm-debug.log_
+yarn-debug.log*
+yarn-error.log*
+lerna-debug.log*
+.pnpm-debug.log*
+
+# Caches
+
+.cache
+
+# Diagnostic reports (https://nodejs.org/api/report.html)
+
+report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json
+
+# Runtime data
+
+pids
+_.pid
+_.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+
+lib-cov
+
+# Coverage directory used by tools like istanbul
+
+coverage
+*.lcov
+
+# nyc test coverage
+
+.nyc_output
+
+# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
+
+.grunt
+
+# Bower dependency directory (https://bower.io/)
+
+bower_components
+
+# node-waf configuration
+
+.lock-wscript
+
+# Compiled binary addons (https://nodejs.org/api/addons.html)
+
+build/Release
+
+# Dependency directories
+
+node_modules/
+jspm_packages/
+
+# Snowpack dependency directory (https://snowpack.dev/)
+
+web_modules/
+
+# TypeScript cache
+
+*.tsbuildinfo
+
+# Optional npm cache directory
+
+.npm
+
+# Optional eslint cache
+
+.eslintcache
+
+# Optional stylelint cache
+
+.stylelintcache
+
+# Microbundle cache
+
+.rpt2_cache/
+.rts2_cache_cjs/
+.rts2_cache_es/
+.rts2_cache_umd/
+
+# Optional REPL history
+
+.node_repl_history
+
+# Output of 'npm pack'
+
+*.tgz
+
+# Yarn Integrity file
+
+.yarn-integrity
+
+# dotenv environment variable files
+
+.env
+.env.development.local
+.env.test.local
+.env.production.local
+.env.local
+
+# parcel-bundler cache (https://parceljs.org/)
+
+.parcel-cache
+
+# Next.js build output
+
+.next
+out
+
+# Nuxt.js build / generate output
+
+.nuxt
+dist
+
+# Gatsby files
+
+# Comment in the public line in if your project uses Gatsby and not Next.js
+
+# https://nextjs.org/blog/next-9-1#public-directory-support
+
+# public
+
+# vuepress build output
+
+.vuepress/dist
+
+# vuepress v2.x temp and cache directory
+
+.temp
+
+# Docusaurus cache and generated files
+
+.docusaurus
+
+# Serverless directories
+
+.serverless/
+
+# FuseBox cache
+
+.fusebox/
+
+# DynamoDB Local files
+
+.dynamodb/
+
+# TernJS port file
+
+.tern-port
+
+# Stores VSCode versions used for testing VSCode extensions
+
+.vscode-test
+
+# yarn v2
+
+.yarn/cache
+.yarn/unplugged
+.yarn/build-state.yml
+.yarn/install-state.gz
+.pnp.*
+
+# IntelliJ based IDEs
+.idea
+
+# Finder (MacOS) folder config
+.DS_Store

sepa/sepa-dataset-validator/README.md

@@ -0,0 +1,14 @@
+# sepa-dataset-validator
+
+un script para validar los datasets de SEPA automaticamente
+
+basado en [la lista de problemas](https://gist.github.com/catdevnull/587d5c63c4bab11b9798861c917db93b) que encontramos
+
+para ejecutar, necesitas [Bun](https://bun.sh)
+
+```bash
+bun install
+bun run . [ruta/al/dataset]
+```
+
+podes descargar un dump de [nuestro index](https://github.com/catdevnull/sepa-precios-metadata/blob/main/index.md) para analizar (la descarga pesa mucho menos que los oficiales :). para descomprimir, necesitas tener `zstd` y `tar`. después solo tenes que ejecutar `tar xvf ARCHIVO.tar.zst` y listo.

sepa/sepa-dataset-validator/index.ts

@@ -100,27 +100,6 @@ const checkers: Record<string, (files: Files) => boolean | string> = {
     }
     return false;
   },
-  ["[productos.csv] hay id_productos no numéricos"](files) {
-    const productos = files["productos.csv"].data;
-    for (const producto of productos) {
-      if (!(producto as any).id_producto) continue;
-      try {
-        const n = Number((producto as any).id_producto);
-        if (isNaN(n)) {
-          console.error(
-            `    El id_producto ${(producto as any).id_producto} parsea a NaN`
-          );
-          return true;
-        }
-      } catch {
-        console.error(
-          `    El id_producto ${(producto as any).id_producto} no es un número`
-        );
-        return true;
-      }
-    }
-    return false;
-  },
   ["Sucursales mencionadas en productos.csv existen en sucursales.csv"](files) {
     const productos = new Set(
       files["productos.csv"].data.map((row) => (row as any).id_sucursal)

sepa/sepa-dataset-validator/package.json

@@ -0,0 +1,16 @@
+{
+  "name": "sepa-dataset-validator",
+  "module": "index.ts",
+  "type": "module",
+  "devDependencies": {
+    "@types/bun": "latest"
+  },
+  "peerDependencies": {
+    "typescript": "^5.5.4"
+  },
+  "dependencies": {
+    "jschardet": "^3.1.3",
+    "papaparse": "^5.4.1",
+    "zod": "^3.23.8"
+  }
+}

sepa/sepa-dataset-validator/tsconfig.json

@@ -0,0 +1,27 @@
+{
+  "compilerOptions": {
+    // Enable latest features
+    "lib": ["ESNext", "DOM"],
+    "target": "ESNext",
+    "module": "ESNext",
+    "moduleDetection": "force",
+    "jsx": "react-jsx",
+    "allowJs": true,
+
+    // Bundler mode
+    "moduleResolution": "bundler",
+    "allowImportingTsExtensions": true,
+    "verbatimModuleSyntax": true,
+    "noEmit": true,
+
+    // Best practices
+    "strict": true,
+    "skipLibCheck": true,
+    "noFallthroughCasesInSwitch": true,
+
+    // Some stricter flags (disabled by default)
+    "noUnusedLocals": false,
+    "noUnusedParameters": false,
+    "noPropertyAccessFromIndexSignature": false
+  }
+}

sepa/sepa-index-gen/.gitignore

@@ -0,0 +1,175 @@
+# Based on https://raw.githubusercontent.com/github/gitignore/main/Node.gitignore
+
+# Logs
+
+logs
+_.log
+npm-debug.log_
+yarn-debug.log*
+yarn-error.log*
+lerna-debug.log*
+.pnpm-debug.log*
+
+# Caches
+
+.cache
+
+# Diagnostic reports (https://nodejs.org/api/report.html)
+
+report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json
+
+# Runtime data
+
+pids
+_.pid
+_.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+
+lib-cov
+
+# Coverage directory used by tools like istanbul
+
+coverage
+*.lcov
+
+# nyc test coverage
+
+.nyc_output
+
+# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
+
+.grunt
+
+# Bower dependency directory (https://bower.io/)
+
+bower_components
+
+# node-waf configuration
+
+.lock-wscript
+
+# Compiled binary addons (https://nodejs.org/api/addons.html)
+
+build/Release
+
+# Dependency directories
+
+node_modules/
+jspm_packages/
+
+# Snowpack dependency directory (https://snowpack.dev/)
+
+web_modules/
+
+# TypeScript cache
+
+*.tsbuildinfo
+
+# Optional npm cache directory
+
+.npm
+
+# Optional eslint cache
+
+.eslintcache
+
+# Optional stylelint cache
+
+.stylelintcache
+
+# Microbundle cache
+
+.rpt2_cache/
+.rts2_cache_cjs/
+.rts2_cache_es/
+.rts2_cache_umd/
+
+# Optional REPL history
+
+.node_repl_history
+
+# Output of 'npm pack'
+
+*.tgz
+
+# Yarn Integrity file
+
+.yarn-integrity
+
+# dotenv environment variable files
+
+.env
+.env.development.local
+.env.test.local
+.env.production.local
+.env.local
+
+# parcel-bundler cache (https://parceljs.org/)
+
+.parcel-cache
+
+# Next.js build output
+
+.next
+out
+
+# Nuxt.js build / generate output
+
+.nuxt
+dist
+
+# Gatsby files
+
+# Comment in the public line in if your project uses Gatsby and not Next.js
+
+# https://nextjs.org/blog/next-9-1#public-directory-support
+
+# public
+
+# vuepress build output
+
+.vuepress/dist
+
+# vuepress v2.x temp and cache directory
+
+.temp
+
+# Docusaurus cache and generated files
+
+.docusaurus
+
+# Serverless directories
+
+.serverless/
+
+# FuseBox cache
+
+.fusebox/
+
+# DynamoDB Local files
+
+.dynamodb/
+
+# TernJS port file
+
+.tern-port
+
+# Stores VSCode versions used for testing VSCode extensions
+
+.vscode-test
+
+# yarn v2
+
+.yarn/cache
+.yarn/unplugged
+.yarn/build-state.yml
+.yarn/install-state.gz
+.pnp.*
+
+# IntelliJ based IDEs
+.idea
+
+# Finder (MacOS) folder config
+.DS_Store

sepa/sepa-index-gen/README.md

@@ -0,0 +1,15 @@
+# sepa-index-gen
+
+To install dependencies:
+
+```bash
+bun install
+```
+
+To run:
+
+```bash
+bun run index.ts
+```
+
+This project was created using `bun init` in bun v1.1.26. [Bun](https://bun.sh) is a fast all-in-one JavaScript runtime.

sepa/sepa-index-gen/b2.ts

@@ -4,42 +4,31 @@ import {
   ListObjectsV2Command,
   GetObjectCommand,
 } from "@aws-sdk/client-s3";
+import { Upload } from "@aws-sdk/lib-storage";
 
 function checkEnvVariable(variableName: string) {
   const value = process.env[variableName];
   if (value) {
+    console.log(`✅ ${variableName} is set`);
     return value;
   } else {
     console.log(`❌ ${variableName} is not set`);
     process.exit(1);
   }
 }
-function getVariables() {
+export const B2_BUCKET_NAME = checkEnvVariable("B2_BUCKET_NAME");
-  const B2_BUCKET_NAME = checkEnvVariable("B2_BUCKET_NAME");
+const B2_BUCKET_KEY_ID = checkEnvVariable("B2_BUCKET_KEY_ID");
-  const B2_BUCKET_KEY_ID = checkEnvVariable("B2_BUCKET_KEY_ID");
+const B2_BUCKET_KEY = checkEnvVariable("B2_BUCKET_KEY");
-  const B2_BUCKET_KEY = checkEnvVariable("B2_BUCKET_KEY");
+export const s3 = new S3Client({
-
-  return {
-    B2_BUCKET_NAME,
-    B2_BUCKET_KEY_ID,
-    B2_BUCKET_KEY,
-  };
-}
-function getS3Client() {
-  const { B2_BUCKET_NAME, B2_BUCKET_KEY_ID, B2_BUCKET_KEY } = getVariables();
-  return new S3Client({
   endpoint: "https://s3.us-west-004.backblazeb2.com",
   region: "us-west-004",
   credentials: {
     accessKeyId: B2_BUCKET_KEY_ID,
     secretAccessKey: B2_BUCKET_KEY,
   },
-  });
+});
-}
 
 export async function listDirectory(directoryName: string) {
-  const { B2_BUCKET_NAME } = getVariables();
-  const s3 = getS3Client();
   const command = new ListObjectsV2Command({
     Bucket: B2_BUCKET_NAME,
     Prefix: directoryName ? `${directoryName}/` : "",
@@ -51,8 +40,6 @@ export async function listDirectory(directoryName: string) {
 }
 
 export async function getFileContent(fileName: string) {
-  const { B2_BUCKET_NAME } = getVariables();
-  const s3 = getS3Client();
   const fileContent = await s3.send(
     new GetObjectCommand({
       Bucket: B2_BUCKET_NAME,
@@ -63,8 +50,6 @@ export async function getFileContent(fileName: string) {
 }
 
 export async function checkFileExists(fileName: string): Promise<boolean> {
-  const { B2_BUCKET_NAME } = getVariables();
-  const s3 = getS3Client();
   try {
     await s3.send(
       new HeadObjectCommand({

sepa/sepa-index-gen/index-resources.ts

@@ -1,4 +1,4 @@
-import { zDatasetInfo, type DatasetInfo, type Resource } from "../ckan/schemas";
+import { zDatasetInfo, type DatasetInfo, type Resource } from "ckan/schemas";
 import { getFileContent, listDirectory } from "./b2";
 
 import { basename } from "path";

sepa/sepa-index-gen/index.ts

@@ -1,22 +1,10 @@
-import { zResource, type Resource } from "../ckan/schemas";
+import { zResource, type Resource } from "ckan/schemas";
 import { z } from "zod";
 import { listDirectory } from "./b2";
 import { isSameDay } from "date-fns";
 import { indexResources } from "./index-resources";
 
-export const IndexEntry = z.object({
+export async function generateMarkdown() {
-  id: z.string(),
-  warnings: z.string(),
-  name: z.string().optional(),
-  link: z.string().optional(),
-  firstSeenAt: z.coerce.date(),
-});
-export type IndexEntry = z.infer<typeof IndexEntry>;
-
-export const IndexJson = z.record(z.string(), z.array(IndexEntry));
-export type IndexJson = z.infer<typeof IndexJson>;
-
-export async function generateIndexes() {
   const resourcesIndex = await indexResources();
 
   const datasets = z
@@ -123,9 +111,6 @@ esto esta automáticamente generado por sepa-index-gen dentro de preciazo.`;
     hour: "2-digit",
     minute: "2-digit",
   });
-
-  let jsonIndex: IndexJson = {};
-
   for (const dateStr of dates) {
     const date = new Date(dateStr);
     markdown += `\n* ${formatter.format(date)}:`;
@@ -135,7 +120,6 @@ esto esta automáticamente generado por sepa-index-gen dentro de preciazo.`;
     if (!resourcesInDate.length) {
       markdown += " ❌ no tengo recursos para esta fecha";
     }
-    jsonIndex[dateStr] = [];
     for (const resource of resourcesInDate) {
       const id = `${resource.id}-revID-${resource.revision_id}`;
       const fileExists = fileList.find((file) => file.startsWith(id));
@@ -151,16 +135,8 @@ esto esta automáticamente generado por sepa-index-gen dentro de preciazo.`;
           "⁉️⚠️ dia de semana incorrecto, puede haberse subido incorrectamente ";
       }
       markdown += `\n  * ${id} ${warnings} ${fileExists ? `[✅ descargar](${link})` : "❌"} (primera vez visto: ${dateTimeFormatter.format(resource.firstSeenAt)})`;
-
-      jsonIndex[dateStr].push({
-        id,
-        warnings: warnings.trim(),
-        name: fileExists,
-        link,
-        firstSeenAt: resource.firstSeenAt,
-      });
     }
   }
 
-  return { markdown, jsonIndex };
+  return markdown;
 }

sepa/sepa-index-gen/package.json

@@ -0,0 +1,19 @@
+{
+  "name": "sepa-index-gen",
+  "module": "index.ts",
+  "type": "module",
+  "devDependencies": {
+    "@types/bun": "latest",
+    "prettier": "^3.3.3"
+  },
+  "peerDependencies": {
+    "typescript": "^5.0.0"
+  },
+  "dependencies": {
+    "@aws-sdk/client-s3": "^3.637.0",
+    "@aws-sdk/lib-storage": "^3.637.0",
+    "ckan": "workspace:*",
+    "date-fns": "^3.6.0",
+    "zod": "^3.23.8"
+  }
+}

sepa/sepa-index-gen/tsconfig.json

@@ -0,0 +1,27 @@
+{
+  "compilerOptions": {
+    // Enable latest features
+    "lib": ["ESNext", "DOM"],
+    "target": "ESNext",
+    "module": "ESNext",
+    "moduleDetection": "force",
+    "jsx": "react-jsx",
+    "allowJs": true,
+
+    // Bundler mode
+    "moduleResolution": "bundler",
+    "allowImportingTsExtensions": true,
+    "verbatimModuleSyntax": true,
+    "noEmit": true,
+
+    // Best practices
+    "strict": true,
+    "skipLibCheck": true,
+    "noFallthroughCasesInSwitch": true,
+
+    // Some stricter flags (disabled by default)
+    "noUnusedLocals": false,
+    "noUnusedParameters": false,
+    "noPropertyAccessFromIndexSignature": false
+  }
+}

sepa/sepa-precios-archiver/.gitignore

@@ -0,0 +1,175 @@
+# Based on https://raw.githubusercontent.com/github/gitignore/main/Node.gitignore
+
+# Logs
+
+logs
+_.log
+npm-debug.log_
+yarn-debug.log*
+yarn-error.log*
+lerna-debug.log*
+.pnpm-debug.log*
+
+# Caches
+
+.cache
+
+# Diagnostic reports (https://nodejs.org/api/report.html)
+
+report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json
+
+# Runtime data
+
+pids
+_.pid
+_.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+
+lib-cov
+
+# Coverage directory used by tools like istanbul
+
+coverage
+*.lcov
+
+# nyc test coverage
+
+.nyc_output
+
+# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
+
+.grunt
+
+# Bower dependency directory (https://bower.io/)
+
+bower_components
+
+# node-waf configuration
+
+.lock-wscript
+
+# Compiled binary addons (https://nodejs.org/api/addons.html)
+
+build/Release
+
+# Dependency directories
+
+node_modules/
+jspm_packages/
+
+# Snowpack dependency directory (https://snowpack.dev/)
+
+web_modules/
+
+# TypeScript cache
+
+*.tsbuildinfo
+
+# Optional npm cache directory
+
+.npm
+
+# Optional eslint cache
+
+.eslintcache
+
+# Optional stylelint cache
+
+.stylelintcache
+
+# Microbundle cache
+
+.rpt2_cache/
+.rts2_cache_cjs/
+.rts2_cache_es/
+.rts2_cache_umd/
+
+# Optional REPL history
+
+.node_repl_history
+
+# Output of 'npm pack'
+
+*.tgz
+
+# Yarn Integrity file
+
+.yarn-integrity
+
+# dotenv environment variable files
+
+.env
+.env.development.local
+.env.test.local
+.env.production.local
+.env.local
+
+# parcel-bundler cache (https://parceljs.org/)
+
+.parcel-cache
+
+# Next.js build output
+
+.next
+out
+
+# Nuxt.js build / generate output
+
+.nuxt
+dist
+
+# Gatsby files
+
+# Comment in the public line in if your project uses Gatsby and not Next.js
+
+# https://nextjs.org/blog/next-9-1#public-directory-support
+
+# public
+
+# vuepress build output
+
+.vuepress/dist
+
+# vuepress v2.x temp and cache directory
+
+.temp
+
+# Docusaurus cache and generated files
+
+.docusaurus
+
+# Serverless directories
+
+.serverless/
+
+# FuseBox cache
+
+.fusebox/
+
+# DynamoDB Local files
+
+.dynamodb/
+
+# TernJS port file
+
+.tern-port
+
+# Stores VSCode versions used for testing VSCode extensions
+
+.vscode-test
+
+# yarn v2
+
+.yarn/cache
+.yarn/unplugged
+.yarn/build-state.yml
+.yarn/install-state.gz
+.pnp.*
+
+# IntelliJ based IDEs
+.idea
+
+# Finder (MacOS) folder config
+.DS_Store

sepa/sepa-precios-archiver/README.md

@@ -0,0 +1,19 @@
+# sepa-precios-archiver
+
+Archivador del dataset de precios de [Precios Claros - Base SEPA](https://datos.produccion.gob.ar/dataset/sepa-precios). Recomprime para utilizar ~8 veces menos espacio, y resube a un bucket mio de Backblaze B2.
+
+## Instalación
+
+Para instalar las dependencias:
+
+```bash
+bun install
+```
+
+Para ejecutarlo:
+
+```bash
+bun run index.ts
+```
+
+This project was created using `bun init` in bun v1.1.25. [Bun](https://bun.sh) is a fast all-in-one JavaScript runtime.

sepa/sepa-precios-archiver/index.ts

@@ -1,11 +1,11 @@
 import { z } from "zod";
-import { zDatasetInfo } from "./ckan/schemas";
+import { zDatasetInfo } from "ckan/schemas";
 import { mkdtemp, writeFile, readdir, mkdir, rm } from "fs/promises";
 import { basename, extname, join } from "path";
-import { $ } from "bun";
+import { $, write } from "bun";
 import { S3Client, HeadObjectCommand } from "@aws-sdk/client-s3";
 import { Upload } from "@aws-sdk/lib-storage";
-import { generateIndexes } from "./index-gen";
+import { generateMarkdown } from "sepa-index-gen";
 
 function checkEnvVariable(variableName: string) {
   const value = process.env[variableName];
@@ -162,9 +162,8 @@ for (const resource of datasetInfo.result.resources) {
     await uploadToB2Bucket(fileName, response);
   }
 }
-const { markdown, jsonIndex } = await generateIndexes();
+
-await saveFileIntoRepo("index.md", markdown);
+await saveFileIntoRepo("index.md", await generateMarkdown());
-await saveFileIntoRepo("index.json", JSON.stringify(jsonIndex, null, 2));
 
 if (errored) {
   process.exit(1);

sepa/sepa-precios-archiver/package.json

@@ -0,0 +1,17 @@
+{
+  "name": "sepa-precios-archiver",
+  "module": "index.ts",
+  "type": "module",
+  "devDependencies": {
+    "@types/bun": "latest"
+  },
+  "peerDependencies": {
+    "typescript": "^5.5.4"
+  },
+  "dependencies": {
+    "@aws-sdk/client-s3": "^3.637.0",
+    "@aws-sdk/lib-storage": "^3.637.0",
+    "zod": "^3.23.8",
+    "sepa-index-gen": "workspace:*"
+  }
+}

sepa/sepa-precios-archiver/tsconfig.json

@@ -0,0 +1,27 @@
+{
+  "compilerOptions": {
+    // Enable latest features
+    "lib": ["ESNext", "DOM"],
+    "target": "ESNext",
+    "module": "ESNext",
+    "moduleDetection": "force",
+    "jsx": "react-jsx",
+    "allowJs": true,
+
+    // Bundler mode
+    "moduleResolution": "bundler",
+    "allowImportingTsExtensions": true,
+    "verbatimModuleSyntax": true,
+    "noEmit": true,
+
+    // Best practices
+    "strict": true,
+    "skipLibCheck": true,
+    "noFallthroughCasesInSwitch": true,
+
+    // Some stricter flags (disabled by default)
+    "noUnusedLocals": false,
+    "noUnusedParameters": false,
+    "noPropertyAccessFromIndexSignature": false
+  }
+}

sepa/sepa-precios-importer/.gitignore

@@ -0,0 +1,175 @@
+# Based on https://raw.githubusercontent.com/github/gitignore/main/Node.gitignore
+
+# Logs
+
+logs
+_.log
+npm-debug.log_
+yarn-debug.log*
+yarn-error.log*
+lerna-debug.log*
+.pnpm-debug.log*
+
+# Caches
+
+.cache
+
+# Diagnostic reports (https://nodejs.org/api/report.html)
+
+report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json
+
+# Runtime data
+
+pids
+_.pid
+_.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+
+lib-cov
+
+# Coverage directory used by tools like istanbul
+
+coverage
+*.lcov
+
+# nyc test coverage
+
+.nyc_output
+
+# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
+
+.grunt
+
+# Bower dependency directory (https://bower.io/)
+
+bower_components
+
+# node-waf configuration
+
+.lock-wscript
+
+# Compiled binary addons (https://nodejs.org/api/addons.html)
+
+build/Release
+
+# Dependency directories
+
+node_modules/
+jspm_packages/
+
+# Snowpack dependency directory (https://snowpack.dev/)
+
+web_modules/
+
+# TypeScript cache
+
+*.tsbuildinfo
+
+# Optional npm cache directory
+
+.npm
+
+# Optional eslint cache
+
+.eslintcache
+
+# Optional stylelint cache
+
+.stylelintcache
+
+# Microbundle cache
+
+.rpt2_cache/
+.rts2_cache_cjs/
+.rts2_cache_es/
+.rts2_cache_umd/
+
+# Optional REPL history
+
+.node_repl_history
+
+# Output of 'npm pack'
+
+*.tgz
+
+# Yarn Integrity file
+
+.yarn-integrity
+
+# dotenv environment variable files
+
+.env
+.env.development.local
+.env.test.local
+.env.production.local
+.env.local
+
+# parcel-bundler cache (https://parceljs.org/)
+
+.parcel-cache
+
+# Next.js build output
+
+.next
+out
+
+# Nuxt.js build / generate output
+
+.nuxt
+dist
+
+# Gatsby files
+
+# Comment in the public line in if your project uses Gatsby and not Next.js
+
+# https://nextjs.org/blog/next-9-1#public-directory-support
+
+# public
+
+# vuepress build output
+
+.vuepress/dist
+
+# vuepress v2.x temp and cache directory
+
+.temp
+
+# Docusaurus cache and generated files
+
+.docusaurus
+
+# Serverless directories
+
+.serverless/
+
+# FuseBox cache
+
+.fusebox/
+
+# DynamoDB Local files
+
+.dynamodb/
+
+# TernJS port file
+
+.tern-port
+
+# Stores VSCode versions used for testing VSCode extensions
+
+.vscode-test
+
+# yarn v2
+
+.yarn/cache
+.yarn/unplugged
+.yarn/build-state.yml
+.yarn/install-state.gz
+.pnp.*
+
+# IntelliJ based IDEs
+.idea
+
+# Finder (MacOS) folder config
+.DS_Store

sepa/sepa-precios-importer/README.md

@@ -0,0 +1,14 @@
+# sepa-precios-importer
+
+Importador de [datasets de precios de SEPA](https://datos.produccion.gob.ar/dataset/sepa-precios/archivo/d076720f-a7f0-4af8-b1d6-1b99d5a90c14) a una base de datos PostgreSQL.
+
+Vease [Errores en el formato de los datos SEPA](https://gist.github.com/catdevnull/587d5c63c4bab11b9798861c917db93b)
+
+To install dependencies:
+
+```bash
+bun install
+bun run index.ts ~/carpeta-con-datasets-descomprimidos
+```
+
+This project was created using `bun init` in bun v1.1.26. [Bun](https://bun.sh) is a fast all-in-one JavaScript runtime.

sepa/sepa-precios-importer/index.ts

@@ -4,7 +4,7 @@ import { basename, join, dirname } from "path";
 import postgres from "postgres";
 import { Readable } from "stream";
 import { pipeline } from "node:stream/promises";
-import { $, Glob } from "bun";
+import { Glob } from "bun";
 import PQueue from "p-queue";
 
 // TODO: verificar que pasa cuando hay varios datasets del mismo día (como los suele haber cuando actualizan el dataset con nuevos comercios)
@@ -161,13 +161,16 @@ async function importDataset(dir: string) {
         // Alberdi S.A.
         file = file.replaceAll(";", "|");
       }
-      if (["30707429468", "30589621499"].includes(comercioCuit)) {
+      if (
+        ["33504047089", "30707429468", "30589621499"].includes(comercioCuit)
+      ) {
         // TODO: si tienen los valores, pero con otros nombres, por ejemplo
         // productos_precio_lista seria precio_unitario_bulto_por_unidad_venta_con_iva.
         // pero no quiero mentir, asi que por ahora no lo importo
-        throw new Error(
+        console.error(
           `No voy a importar el dataset ${dir} porque el formato está mal. Pero se podría importar. Pero por ahora no lo voy a hacer. Véase https://gist.github.com/catdevnull/587d5c63c4bab11b9798861c917db93b`
         );
+        return;
       }
 
       console.time("parse");
@@ -244,38 +247,15 @@ async function importDataset(dir: string) {
   }
 }
 
-async function importDatasetTar(tarPath: string) {
+const pQueue = new PQueue({ concurrency: 2 });
-  console.log(`importing tar ${tarPath}`);
-  const dir = await fs.mkdtemp("/tmp/sepa-precios-importer-");
-  try {
-    await $`tar -x -C ${dir} -f ${tarPath}`;
-    await importDump(dir);
-  } finally {
-    await fs.rm(dir, { recursive: true });
-  }
-}
-async function importDump(dumpDir: string) {
-  const pQueue = new PQueue({ concurrency: 2 });
-  const glob = new Glob("**/productos.csv");
-  for await (const file of glob.scan(dumpDir)) {
-    const dir = join(dumpDir, dirname(file));
-    pQueue.add(() => importDataset(dir));
-  }
-  await pQueue.onIdle();
-}
 
 try {
-  const tarGlob = new Glob("**/*.tar.zst");
+  const glob = new Glob("**/productos.csv");
-  let hasTars = false;
+  for await (const file of glob.scan(process.argv[2])) {
-  for await (const file of tarGlob.scan(process.argv[2])) {
+    const dir = join(process.argv[2], dirname(file));
-    hasTars = true;
+    pQueue.add(() => importDataset(dir));
-    const tar = join(process.argv[2], file);
-    await importDatasetTar(tar);
-  }
-
-  if (!hasTars) {
-    await importDump(process.argv[2]);
   }
 } finally {
+  await pQueue.onIdle();
   await sql.end();
 }

sepa/sepa-precios-importer/package.json

@@ -0,0 +1,18 @@
+{
+  "name": "sepa-precios-importer",
+  "module": "index.ts",
+  "type": "module",
+  "devDependencies": {
+    "@types/bun": "^1.1.7",
+    "@types/papaparse": "^5.3.14"
+  },
+  "peerDependencies": {
+    "typescript": "^5.0.0"
+  },
+  "dependencies": {
+    "ckan": "workspace:*",
+    "p-queue": "^8.0.1",
+    "papaparse": "^5.4.1",
+    "postgres": "^3.4.4"
+  }
+}

sepa/sepa-precios-importer/tsconfig.json

@@ -0,0 +1,27 @@
+{
+  "compilerOptions": {
+    // Enable latest features
+    "lib": ["ESNext", "DOM"],
+    "target": "ESNext",
+    "module": "ESNext",
+    "moduleDetection": "force",
+    "jsx": "react-jsx",
+    "allowJs": true,
+
+    // Bundler mode
+    "moduleResolution": "bundler",
+    "allowImportingTsExtensions": true,
+    "verbatimModuleSyntax": true,
+    "noEmit": true,
+
+    // Best practices
+    "strict": true,
+    "skipLibCheck": true,
+    "noFallthroughCasesInSwitch": true,
+
+    // Some stricter flags (disabled by default)
+    "noUnusedLocals": false,
+    "noUnusedParameters": false,
+    "noPropertyAccessFromIndexSignature": false
+  }
+}