.gitignore

@@ -12,4 +12,5 @@ scraper/debug/
 scraper/x.tsv
 *.tmp
 target/
-.env.*
+.env
+.env*

db-datos/.env

@@ -1 +0,0 @@
-DB_PATH=../scraper/sqlite.db

db-datos/drizzle.config.ts

@@ -1,14 +1,10 @@
 import type { Config } from "drizzle-kit";
 
-if (!process.env.DB_PATH) throw new Error("no hay DB_PATH");
-
-export const DB_PATH = process.env.DB_PATH;
-
 export default {
   schema: "./schema.ts",
   out: "./drizzle",
   driver: "better-sqlite",
   dbCredentials: {
-    url: process.env.DB_PATH,
+    url: "../scraper/sqlite.db",
   },
 } satisfies Config;

db-datos/drizzle/0003_abandoned_landau.sql

@@ -1 +0,0 @@
-CREATE INDEX `precios_ean_idx` ON `precios` (`ean`);

db-datos/drizzle/meta/0003_snapshot.json

@@ -1,101 +0,0 @@
-{
-  "version": "5",
-  "dialect": "sqlite",
-  "id": "e1217fdb-6f54-44c5-a04b-c5aebf202102",
-  "prevId": "cbd90a60-7568-489f-ac45-95bd8818ffbd",
-  "tables": {
-    "precios": {
-      "name": "precios",
-      "columns": {
-        "id": {
-          "name": "id",
-          "type": "integer",
-          "primaryKey": true,
-          "notNull": true,
-          "autoincrement": true
-        },
-        "ean": {
-          "name": "ean",
-          "type": "text",
-          "primaryKey": false,
-          "notNull": true,
-          "autoincrement": false
-        },
-        "fetched_at": {
-          "name": "fetched_at",
-          "type": "integer",
-          "primaryKey": false,
-          "notNull": true,
-          "autoincrement": false
-        },
-        "precio_centavos": {
-          "name": "precio_centavos",
-          "type": "integer",
-          "primaryKey": false,
-          "notNull": false,
-          "autoincrement": false
-        },
-        "in_stock": {
-          "name": "in_stock",
-          "type": "integer",
-          "primaryKey": false,
-          "notNull": false,
-          "autoincrement": false
-        },
-        "url": {
-          "name": "url",
-          "type": "text",
-          "primaryKey": false,
-          "notNull": true,
-          "autoincrement": false
-        },
-        "warc_record_id": {
-          "name": "warc_record_id",
-          "type": "text",
-          "primaryKey": false,
-          "notNull": false,
-          "autoincrement": false
-        },
-        "parser_version": {
-          "name": "parser_version",
-          "type": "integer",
-          "primaryKey": false,
-          "notNull": false,
-          "autoincrement": false
-        },
-        "name": {
-          "name": "name",
-          "type": "text",
-          "primaryKey": false,
-          "notNull": false,
-          "autoincrement": false
-        },
-        "image_url": {
-          "name": "image_url",
-          "type": "text",
-          "primaryKey": false,
-          "notNull": false,
-          "autoincrement": false
-        }
-      },
-      "indexes": {
-        "precios_ean_idx": {
-          "name": "precios_ean_idx",
-          "columns": [
-            "ean"
-          ],
-          "isUnique": false
-        }
-      },
-      "foreignKeys": {},
-      "compositePrimaryKeys": {},
-      "uniqueConstraints": {}
-    }
-  },
-  "enums": {},
-  "_meta": {
-    "schemas": {},
-    "tables": {},
-    "columns": {}
-  }
-}

db-datos/drizzle/meta/_journal.json

@@ -22,13 +22,6 @@
       "when": 1703452301821,
       "tag": "0002_wild_amazoness",
       "breakpoints": true
-    },
-    {
-      "idx": 3,
-      "version": "5",
-      "when": 1703521964385,
-      "tag": "0003_abandoned_landau",
-      "breakpoints": true
     }
   ]
 }

db-datos/migrate.ts

@@ -2,9 +2,8 @@ import Database from "bun:sqlite";
 import { drizzle } from "drizzle-orm/bun-sqlite";
 import { migrate } from "drizzle-orm/bun-sqlite/migrator";
 import * as schema from "./schema.js";
-import { DB_PATH } from "./drizzle.config.js";
 
-const sqlite = new Database(DB_PATH);
+const sqlite = new Database("../scraper/sqlite.db");
 const db = drizzle(sqlite, { schema });
 
 migrate(db, { migrationsFolder: "./drizzle" });

db-datos/schema.ts

@@ -1,24 +1,16 @@
-import { index, integer, sqliteTable, text } from "drizzle-orm/sqlite-core";
+import { integer, sqliteTable, text } from "drizzle-orm/sqlite-core";
 
-export const precios = sqliteTable(
+export const precios = sqliteTable("precios", {
-  "precios",
+  id: integer("id", { mode: "number" }).primaryKey({ autoIncrement: true }),
-  {
+  ean: text("ean").notNull(),
-    id: integer("id", { mode: "number" }).primaryKey({ autoIncrement: true }),
+  fetchedAt: integer("fetched_at", { mode: "timestamp" }).notNull(),
-    ean: text("ean").notNull(),
+  precioCentavos: integer("precio_centavos"),
-    fetchedAt: integer("fetched_at", { mode: "timestamp" }).notNull(),
+  inStock: integer("in_stock", { mode: "boolean" }),
-    precioCentavos: integer("precio_centavos"),
+  url: text("url").notNull(),
-    inStock: integer("in_stock", { mode: "boolean" }),
+  warcRecordId: text("warc_record_id"),
-    url: text("url").notNull(),
+  parserVersion: integer("parser_version"),
-    warcRecordId: text("warc_record_id"),
+  name: text("name"),
-    parserVersion: integer("parser_version"),
+  imageUrl: text("image_url"),
-    name: text("name"),
+});
-    imageUrl: text("image_url"),
-  },
-  (precios) => {
-    return {
-      preciosEanIdx: index("precios_ean_idx").on(precios.ean),
-    };
-  }
-);
 
 export type Precio = typeof precios.$inferSelect;

scraper/.env

@@ -1 +0,0 @@
-DB_PATH=../scraper/sqlite.db

scraper/auto.ts

@@ -1,3 +1,4 @@
+import "dotenv/config";
 import { mkdtemp, access } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join, resolve } from "node:path";
@@ -49,13 +50,13 @@ supermercados.forEach(downloadList);
 
 async function downloadList(supermercado: Supermercado) {
   const listPath = resolve(
-    join(process.env.LISTS_DIR ?? "../data", `${supermercado}.txt`)
+    join(process.env.LISTS_DIR ?? "../data", `${supermercado}.txt`),
   );
   const date = new Date();
   const ctxPath = await mkdtemp(join(tmpdir(), "preciazo-scraper-wget-"));
   const zstdWarcName = `${supermercado}-${format(
     date,
-    "yyyy-MM-dd-HH:mm"
+    "yyyy-MM-dd-HH:mm",
   )}.warc.zst`;
   const zstdWarcPath = join(ctxPath, zstdWarcName);
   const subproc = Bun.spawn({
@@ -121,7 +122,7 @@ async function scrapAndInform({
     inform(
       `Procesado ${zstdWarcName} (${progress.done} ok, ${
         progress.errors.length
-      } errores) (tardó ${formatMs(took)})`
+      } errores) (tardó ${formatMs(took)})`,
     );
   } else {
     inform(`Algo falló en ${zstdWarcName}`);
@@ -158,7 +159,7 @@ function recompress(inputPath: string, outputPath: string) {
       ["-T0", "-15", "--long", "-o", outputPath],
       {
         stdio: ["pipe", null, null],
-      }
+      },
     );
     // @ts-expect-error a los types de bun no le gusta????
     decompressor.stdout.pipe(compressor.stdin);
@@ -214,7 +215,7 @@ async function sendTelegramMsg(text: string) {
   if (!process.env.TELEGRAM_BOT_TOKEN || !process.env.TELEGRAM_BOT_CHAT_ID)
     return;
   const url = new URL(
-    `https://api.telegram.org/bot${process.env.TELEGRAM_BOT_TOKEN}/sendMessage`
+    `https://api.telegram.org/bot${process.env.TELEGRAM_BOT_TOKEN}/sendMessage`,
   );
   url.searchParams.set("chat_id", process.env.TELEGRAM_BOT_CHAT_ID);
   url.searchParams.set("text", text);

scraper/package.json

@@ -15,6 +15,7 @@
     "@aws-sdk/lib-storage": "^3.478.0",
     "date-fns": "^3.0.6",
     "db-datos": "workspace:^",
+    "dotenv": "^16.3.1",
     "drizzle-orm": "^0.29.1",
     "linkedom": "^0.16.5",
     "nanoid": "^5.0.4",

scraper/scrap.ts

@@ -9,12 +9,11 @@ import { getDiaProduct } from "./parsers/dia.js";
 import { getCotoProduct } from "./parsers/coto.js";
 import { join } from "path";
 import { and, eq, sql } from "drizzle-orm";
-import { DB_PATH } from "db-datos/drizzle.config.js";
 
 const DEBUG = false;
 const PARSER_VERSION = 2;
 
-const sqlite = new Database(DB_PATH);
+const sqlite = new Database("sqlite.db");
 const db = drizzle(sqlite, { schema });
 
 sqlite.run(`

sitio/src/lib/server/db.ts

@@ -3,7 +3,7 @@ import { drizzle } from "drizzle-orm/bun-sqlite";
 import * as schema from "db-datos/schema.js";
 import { env } from "$env/dynamic/private";
 
-const sqlite = new Database(env.DB_PATH);
+const sqlite = new Database(env.DB_PATH ?? "../scraper/sqlite.db");
 
 export const db = drizzle(sqlite, { schema });
 export * as schema from "db-datos/schema.js";