.github/workflows/sepa-precios-archiver.yml

@@ -2,7 +2,7 @@ name: Sepa Precios Archiver
 
 on:
   schedule:
-    - cron: "0 */3 * * *" # Run every 3 hours
+    - cron: "0 */12 * * *" # Run every 6 hours
   workflow_dispatch: # Allow manual trigger
 
 jobs:

sepa/package.json

@@ -4,7 +4,6 @@
   "workspaces": [
     "sepa-precios-archiver",
     "sepa-precios-importer",
-    "sepa-index-gen",
     "ckan"
   ]
 }

sepa/sepa-index-gen/.gitignore

@@ -1,175 +0,0 @@
-# Based on https://raw.githubusercontent.com/github/gitignore/main/Node.gitignore
-
-# Logs
-
-logs
-_.log
-npm-debug.log_
-yarn-debug.log*
-yarn-error.log*
-lerna-debug.log*
-.pnpm-debug.log*
-
-# Caches
-
-.cache
-
-# Diagnostic reports (https://nodejs.org/api/report.html)
-
-report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json
-
-# Runtime data
-
-pids
-_.pid
-_.seed
-*.pid.lock
-
-# Directory for instrumented libs generated by jscoverage/JSCover
-
-lib-cov
-
-# Coverage directory used by tools like istanbul
-
-coverage
-*.lcov
-
-# nyc test coverage
-
-.nyc_output
-
-# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
-
-.grunt
-
-# Bower dependency directory (https://bower.io/)
-
-bower_components
-
-# node-waf configuration
-
-.lock-wscript
-
-# Compiled binary addons (https://nodejs.org/api/addons.html)
-
-build/Release
-
-# Dependency directories
-
-node_modules/
-jspm_packages/
-
-# Snowpack dependency directory (https://snowpack.dev/)
-
-web_modules/
-
-# TypeScript cache
-
-*.tsbuildinfo
-
-# Optional npm cache directory
-
-.npm
-
-# Optional eslint cache
-
-.eslintcache
-
-# Optional stylelint cache
-
-.stylelintcache
-
-# Microbundle cache
-
-.rpt2_cache/
-.rts2_cache_cjs/
-.rts2_cache_es/
-.rts2_cache_umd/
-
-# Optional REPL history
-
-.node_repl_history
-
-# Output of 'npm pack'
-
-*.tgz
-
-# Yarn Integrity file
-
-.yarn-integrity
-
-# dotenv environment variable files
-
-.env
-.env.development.local
-.env.test.local
-.env.production.local
-.env.local
-
-# parcel-bundler cache (https://parceljs.org/)
-
-.parcel-cache
-
-# Next.js build output
-
-.next
-out
-
-# Nuxt.js build / generate output
-
-.nuxt
-dist
-
-# Gatsby files
-
-# Comment in the public line in if your project uses Gatsby and not Next.js
-
-# https://nextjs.org/blog/next-9-1#public-directory-support
-
-# public
-
-# vuepress build output
-
-.vuepress/dist
-
-# vuepress v2.x temp and cache directory
-
-.temp
-
-# Docusaurus cache and generated files
-
-.docusaurus
-
-# Serverless directories
-
-.serverless/
-
-# FuseBox cache
-
-.fusebox/
-
-# DynamoDB Local files
-
-.dynamodb/
-
-# TernJS port file
-
-.tern-port
-
-# Stores VSCode versions used for testing VSCode extensions
-
-.vscode-test
-
-# yarn v2
-
-.yarn/cache
-.yarn/unplugged
-.yarn/build-state.yml
-.yarn/install-state.gz
-.pnp.*
-
-# IntelliJ based IDEs
-.idea
-
-# Finder (MacOS) folder config
-.DS_Store

sepa/sepa-index-gen/README.md

@@ -1,15 +0,0 @@
-# sepa-index-gen
-
-To install dependencies:
-
-```bash
-bun install
-```
-
-To run:
-
-```bash
-bun run index.ts
-```
-
-This project was created using `bun init` in bun v1.1.26. [Bun](https://bun.sh) is a fast all-in-one JavaScript runtime.

sepa/sepa-index-gen/b2.ts

@@ -1,67 +0,0 @@
-import {
-  S3Client,
-  HeadObjectCommand,
-  ListObjectsV2Command,
-  GetObjectCommand,
-} from "@aws-sdk/client-s3";
-import { Upload } from "@aws-sdk/lib-storage";
-
-function checkEnvVariable(variableName: string) {
-  const value = process.env[variableName];
-  if (value) {
-    console.log(`✅ ${variableName} is set`);
-    return value;
-  } else {
-    console.log(`❌ ${variableName} is not set`);
-    process.exit(1);
-  }
-}
-export const B2_BUCKET_NAME = checkEnvVariable("B2_BUCKET_NAME");
-const B2_BUCKET_KEY_ID = checkEnvVariable("B2_BUCKET_KEY_ID");
-const B2_BUCKET_KEY = checkEnvVariable("B2_BUCKET_KEY");
-export const s3 = new S3Client({
-  endpoint: "https://s3.us-west-004.backblazeb2.com",
-  region: "us-west-004",
-  credentials: {
-    accessKeyId: B2_BUCKET_KEY_ID,
-    secretAccessKey: B2_BUCKET_KEY,
-  },
-});
-
-export async function listDirectory(directoryName: string) {
-  const command = new ListObjectsV2Command({
-    Bucket: B2_BUCKET_NAME,
-    Prefix: directoryName ? `${directoryName}/` : "",
-    Delimiter: "/",
-  });
-
-  const response = await s3.send(command);
-  return response.Contents?.map((item) => item.Key ?? "") ?? [];
-}
-
-export async function getFileContent(fileName: string) {
-  const fileContent = await s3.send(
-    new GetObjectCommand({
-      Bucket: B2_BUCKET_NAME,
-      Key: fileName,
-    })
-  );
-  return (await fileContent.Body?.transformToString()) ?? "";
-}
-
-export async function checkFileExists(fileName: string): Promise<boolean> {
-  try {
-    await s3.send(
-      new HeadObjectCommand({
-        Bucket: B2_BUCKET_NAME,
-        Key: fileName,
-      })
-    );
-    return true;
-  } catch (error) {
-    if ((error as any).name === "NotFound") {
-      return false;
-    }
-    throw error;
-  }
-}

sepa/sepa-index-gen/index-resources.ts

@@ -1,53 +0,0 @@
-import { zDatasetInfo, type DatasetInfo, type Resource } from "ckan/schemas";
-import { getFileContent, listDirectory } from "./b2";
-
-import { basename } from "path";
-
-export async function indexResources() {
-  const fileNames = await listDirectory("timestamped-metadata");
-
-  let datasetInfos = new Map<string, Map<string, Resource>>();
-
-  await Promise.all(
-    fileNames.map(async (fileName) => {
-      let resources = new Map<string, Resource>();
-
-      const fileContentText = await getFileContent(fileName);
-      const json = JSON.parse(fileContentText);
-      if (!json.success) {
-        console.log(`skipping ${fileName} because of ${json.error.message}`);
-        return;
-      }
-      let datasetInfo: DatasetInfo;
-      try {
-        datasetInfo = zDatasetInfo.parse(json.result);
-      } catch (e) {
-        console.log(fileName, e);
-        throw e;
-      }
-
-      for (const resource of datasetInfo.resources) {
-        const id = `${resource.id}-${resource.revision_id}`;
-        const existingResource = resources.get(id);
-        if (
-          existingResource &&
-          existingResource.modified &&
-          existingResource.modified > (resource.modified ?? 0)
-        ) {
-          continue;
-        }
-        resources.set(id, resource);
-      }
-      const base = basename(fileName, ".json");
-      datasetInfos.set(base, resources);
-    })
-  );
-
-  return Array.from(datasetInfos.entries()).reduce(
-    (acc, [fileName, resources]) => {
-      acc[fileName] = Array.from(resources.values());
-      return acc;
-    },
-    {} as Record<string, Resource[]>
-  );
-}

sepa/sepa-index-gen/index.ts

@@ -1,141 +0,0 @@
-import { zResource, type Resource } from "ckan/schemas";
-import { z } from "zod";
-import { listDirectory } from "./b2";
-import { isSameDay } from "date-fns";
-import { indexResources } from "./index-resources";
-
-export async function generateMarkdown() {
-  const resourcesIndex = await indexResources();
-
-  const datasets = z
-    .record(z.string(), z.array(zResource))
-    .parse(resourcesIndex);
-  const datasetsArray = Object.entries(datasets)
-    .map(([date, resources]) => ({
-      date: new Date(date),
-      resources,
-    }))
-    .sort((a, b) => +b.date - +a.date);
-
-  let latestResources = new Map<string, Resource>();
-
-  for (const { date, resources } of datasetsArray) {
-    for (const resource of resources) {
-      const id = `${resource.id}-revID-${resource.revision_id}`;
-      if (latestResources.has(id)) continue;
-      latestResources.set(id, resource);
-    }
-  }
-
-  function getDate(resource: Resource) {
-    {
-      const matches = resource.name.match(/precios_(\d{4})(\d{2})(\d{2})/);
-      if (matches) {
-        return new Date(
-          parseInt(matches[1]),
-          parseInt(matches[2]) - 1,
-          parseInt(matches[3])
-        );
-      }
-    }
-    {
-      const matches = resource.description.match(
-        /(?<day>\d{2})\/(?<month>\d{2})\/(?<year>\d{4})|(?<year2>\d{4})-(?<month2>\d{2})-(?<day2>\d{2})/
-      );
-      if (matches) {
-        const { day, month, year, day2, month2, year2 } = matches.groups!;
-        return new Date(
-          parseInt(year || year2),
-          parseInt(month || month2) - 1,
-          parseInt(day || day2)
-        );
-      }
-    }
-    console.debug(resource);
-    throw new Error(`No date found for ${resource.name}`);
-  }
-
-  const fileList = await listDirectory("");
-
-  const zipResources = [...latestResources.values()].filter(
-    (r) => r.format === "ZIP"
-  );
-  const dates = [
-    ...new Set(
-      zipResources.map((r) => getDate(r).toISOString().split("T")[0]).sort()
-    ),
-  ];
-
-  // check if dates are missing in between min and max date
-  const minDate = new Date(
-    Math.min(...[...dates].map((d) => new Date(d).getTime()))
-  );
-  const maxDate = new Date(
-    Math.max(...[...dates].map((d) => new Date(d).getTime()))
-  );
-  for (let d = minDate; d <= maxDate; d.setDate(d.getDate() + 1)) {
-    const dateStr = d.toISOString().split("T")[0];
-    if (!dates.includes(dateStr)) dates.push(dateStr);
-  }
-  dates.sort();
-
-  function getWeekDay(searchIn: string) {
-    if (/domingo/iu.test(searchIn)) return 0;
-    if (/lunes/iu.test(searchIn)) return 1;
-    if (/martes/iu.test(searchIn)) return 2;
-    if (/mi[eé]rcoles/iu.test(searchIn)) return 3;
-    if (/jueves/iu.test(searchIn)) return 4;
-    if (/viernes/iu.test(searchIn)) return 5;
-    if (/s[aá]bado/iu.test(searchIn)) return 6;
-    return null;
-  }
-  function getWeekDayInResource(resource: Resource) {
-    return getWeekDay(resource.description) ?? getWeekDay(resource.name);
-  }
-
-  let markdown = `# index de archivo de datasets de precios SEPA
-
-esto esta automáticamente generado por sepa-index-gen dentro de preciazo.`;
-
-  const formatter = Intl.DateTimeFormat("es-AR", {
-    year: "numeric",
-    month: "2-digit",
-    day: "2-digit",
-    weekday: "long",
-  });
-  const dateTimeFormatter = Intl.DateTimeFormat("es-AR", {
-    year: "numeric",
-    month: "2-digit",
-    day: "2-digit",
-    hour: "2-digit",
-    minute: "2-digit",
-  });
-  for (const dateStr of dates) {
-    const date = new Date(dateStr);
-    markdown += `\n* ${formatter.format(date)}:`;
-    const resourcesInDate = zipResources.filter((r) =>
-      isSameDay(getDate(r), date)
-    );
-    if (!resourcesInDate.length) {
-      markdown += " ❌ no tengo recursos para esta fecha";
-    }
-    for (const resource of resourcesInDate) {
-      const id = `${resource.id}-revID-${resource.revision_id}`;
-      const fileExists = fileList.find((file) => file.startsWith(id));
-      const link =
-        fileExists ??
-        `https://f004.backblazeb2.com/file/precios-justos-datasets/${fileExists}`;
-      let warnings = "";
-      if (
-        getWeekDayInResource(resource) &&
-        date.getDay() !== getWeekDayInResource(resource)
-      ) {
-        warnings +=
-          "⁉️⚠️ dia de semana incorrecto, puede haberse subido incorrectamente ";
-      }
-      markdown += `\n  * ${id} ${warnings} ${fileExists ? `[✅ descargar](${link})` : "❌"} (${dateTimeFormatter.format(resource.modified)})`;
-    }
-  }
-
-  return markdown;
-}

sepa/sepa-index-gen/package.json

@@ -1,19 +0,0 @@
-{
-  "name": "sepa-index-gen",
-  "module": "index.ts",
-  "type": "module",
-  "devDependencies": {
-    "@types/bun": "latest",
-    "prettier": "^3.3.3"
-  },
-  "peerDependencies": {
-    "typescript": "^5.0.0"
-  },
-  "dependencies": {
-    "@aws-sdk/client-s3": "^3.637.0",
-    "@aws-sdk/lib-storage": "^3.637.0",
-    "ckan": "workspace:*",
-    "date-fns": "^3.6.0",
-    "zod": "^3.23.8"
-  }
-}

sepa/sepa-index-gen/tsconfig.json

@@ -1,27 +0,0 @@
-{
-  "compilerOptions": {
-    // Enable latest features
-    "lib": ["ESNext", "DOM"],
-    "target": "ESNext",
-    "module": "ESNext",
-    "moduleDetection": "force",
-    "jsx": "react-jsx",
-    "allowJs": true,
-
-    // Bundler mode
-    "moduleResolution": "bundler",
-    "allowImportingTsExtensions": true,
-    "verbatimModuleSyntax": true,
-    "noEmit": true,
-
-    // Best practices
-    "strict": true,
-    "skipLibCheck": true,
-    "noFallthroughCasesInSwitch": true,
-
-    // Some stricter flags (disabled by default)
-    "noUnusedLocals": false,
-    "noUnusedParameters": false,
-    "noPropertyAccessFromIndexSignature": false
-  }
-}

sepa/sepa-precios-archiver/index.ts

@@ -5,7 +5,6 @@ import { basename, extname, join } from "path";
 import { $, write } from "bun";
 import { S3Client, HeadObjectCommand } from "@aws-sdk/client-s3";
 import { Upload } from "@aws-sdk/lib-storage";
-import { generateMarkdown } from "sepa-index-gen";
 
 function checkEnvVariable(variableName: string) {
   const value = process.env[variableName];
@@ -39,25 +38,29 @@ async function getRawDatasetInfo() {
     console.error(
       `❌ Error fetching dataset info`,
       error,
-      `retrying in 5min...`
+      `retrying in 5min...`,
     );
     await new Promise((resolve) => setTimeout(resolve, 5 * 60 * 1000));
     return await getRawDatasetInfo();
   }
 }
-async function saveFileIntoRepo(fileName: string, fileContent: string) {
+
+async function saveDatasetInfoIntoRepo(datasetInfo: any) {
   const dir = await mkdtemp("/tmp/sepa-precios-archiver-metadata-repo-");
   try {
     await $`git clone https://catdevnull:${GITHUB_TOKEN}@github.com/catdevnull/sepa-precios-metadata.git ${dir}`;
-    await writeFile(join(dir, fileName), fileContent);
+    await writeFile(
+      dir + "/dataset-info.json",
+      JSON.stringify(datasetInfo, null, 2),
+    );
+    await $`cd ${dir} && git add dataset-info.json`;
     await $`cd ${dir} && git config user.email "git@nulo.in" && git config user.name "github actions"`;
-    await $`cd ${dir} && git add ${fileName}`;
+    await $`cd ${dir} && git diff --staged --quiet || git commit -m "Update dataset info"`;
-    await $`cd ${dir} && git diff --staged --quiet || git commit -m "Update ${fileName}"`;
     await $`cd ${dir} && git push origin main`;
   } finally {
     await $`rm -rf ${dir}`;
   }
-  console.log(`✅ Saved ${fileName} into repo`);
+  console.log(`✅ Saved dataset info into repo`);
 }
 
 async function checkFileExistsInB2(fileName: string): Promise<boolean> {
@@ -66,7 +69,7 @@ async function checkFileExistsInB2(fileName: string): Promise<boolean> {
       new HeadObjectCommand({
         Bucket: B2_BUCKET_NAME,
         Key: fileName,
-      })
+      }),
     );
     return true;
   } catch (error) {
@@ -79,7 +82,7 @@ async function checkFileExistsInB2(fileName: string): Promise<boolean> {
 
 async function uploadToB2Bucket(
   fileName: string,
-  fileContent: ReadableStream | Blob | string
+  fileContent: ReadableStream | Blob | string,
 ) {
   const upload = new Upload({
     client: s3,
@@ -95,15 +98,12 @@ async function uploadToB2Bucket(
 
 const rawDatasetInfo = await getRawDatasetInfo();
 
-await saveFileIntoRepo(
+await saveDatasetInfoIntoRepo(rawDatasetInfo);
-  "dataset-info.json",
-  JSON.stringify(rawDatasetInfo, null, 2)
-);
 
 let errored = false;
 
 function checkRes(
-  res: Response
+  res: Response,
 ): res is Response & { body: ReadableStream<Uint8Array> } {
   if (!res.ok) {
     console.error(`❌ Error downloading ${res.url}`);
@@ -116,7 +116,7 @@ function checkRes(
 
 await uploadToB2Bucket(
   `timestamped-metadata/${new Date().toISOString()}.json`,
-  JSON.stringify(rawDatasetInfo, null, 2)
+  JSON.stringify(rawDatasetInfo, null, 2),
 );
 
 const datasetInfo = z.object({ result: zDatasetInfo }).parse(rawDatasetInfo);
@@ -144,7 +144,7 @@ for (const resource of datasetInfo.result.resources) {
 
       await writeFile(
         join(dir, "dataset-info.json"),
-        JSON.stringify(rawDatasetInfo, null, 2)
+        JSON.stringify(rawDatasetInfo, null, 2),
       );
 
       const compressed =
@@ -163,8 +163,6 @@ for (const resource of datasetInfo.result.resources) {
   }
 }
 
-await saveFileIntoRepo("index.md", await generateMarkdown());
-
 if (errored) {
   process.exit(1);
 }

sepa/sepa-precios-archiver/package.json

@@ -11,7 +11,6 @@
   "dependencies": {
     "@aws-sdk/client-s3": "^3.637.0",
     "@aws-sdk/lib-storage": "^3.637.0",
-    "zod": "^3.23.8",
+    "zod": "^3.23.8"
-    "sepa-index-gen": "workspace:*"
   }
 }

sepa/sepa-precios-importer/index.ts

@@ -126,7 +126,6 @@ async function importSucursales(
 }
 
 async function importDataset(dir: string) {
-  console.log(dir);
   const date = basename(dir).match(/(\d{4}-\d{2}-\d{2})/)![1];
   // TODO: parsear "Ultima actualizacion" al final del CSV y insertarlo en la tabla datasets
 
@@ -247,12 +246,13 @@ async function importDataset(dir: string) {
   }
 }
 
-const pQueue = new PQueue({ concurrency: 2 });
+const pQueue = new PQueue({ concurrency: 4 });
 
 try {
   const glob = new Glob("**/productos.csv");
   for await (const file of glob.scan(process.argv[2])) {
     const dir = join(process.argv[2], dirname(file));
+    console.log(dir);
     pQueue.add(() => importDataset(dir));
   }
 } finally {