From 4589eee5c09756abe0dec8043cfa059861fbf783 Mon Sep 17 00:00:00 2001 From: Nulo Date: Mon, 25 Dec 2023 23:05:50 -0300 Subject: [PATCH] siempre tomar BD de DB_PATH --- .gitignore | 3 +-- db-datos/.env | 1 + db-datos/drizzle.config.ts | 6 +++++- db-datos/migrate.ts | 3 ++- scraper/.env | 1 + scraper/auto.ts | 11 +++++------ scraper/package.json | 1 - scraper/scrap.ts | 3 ++- sitio/src/lib/server/db.ts | 2 +- 9 files changed, 18 insertions(+), 13 deletions(-) create mode 100644 db-datos/.env create mode 100644 scraper/.env diff --git a/.gitignore b/.gitignore index 86aaf67..7d67771 100644 --- a/.gitignore +++ b/.gitignore @@ -12,5 +12,4 @@ scraper/debug/ scraper/x.tsv *.tmp target/ -.env -.env* \ No newline at end of file +.env.* \ No newline at end of file diff --git a/db-datos/.env b/db-datos/.env new file mode 100644 index 0000000..a537cb1 --- /dev/null +++ b/db-datos/.env @@ -0,0 +1 @@ +DB_PATH=../scraper/sqlite.db \ No newline at end of file diff --git a/db-datos/drizzle.config.ts b/db-datos/drizzle.config.ts index 5130bd0..7de3342 100644 --- a/db-datos/drizzle.config.ts +++ b/db-datos/drizzle.config.ts @@ -1,10 +1,14 @@ import type { Config } from "drizzle-kit"; +if (!process.env.DB_PATH) throw new Error("no hay DB_PATH"); + +export const DB_PATH = process.env.DB_PATH; + export default { schema: "./schema.ts", out: "./drizzle", driver: "better-sqlite", dbCredentials: { - url: "../scraper/sqlite.db", + url: process.env.DB_PATH, }, } satisfies Config; diff --git a/db-datos/migrate.ts b/db-datos/migrate.ts index 632dc67..5a99b70 100644 --- a/db-datos/migrate.ts +++ b/db-datos/migrate.ts @@ -2,8 +2,9 @@ import Database from "bun:sqlite"; import { drizzle } from "drizzle-orm/bun-sqlite"; import { migrate } from "drizzle-orm/bun-sqlite/migrator"; import * as schema from "./schema.js"; +import { DB_PATH } from "./drizzle.config.js"; -const sqlite = new Database("../scraper/sqlite.db"); +const sqlite = new Database(DB_PATH); const db = drizzle(sqlite, { schema }); migrate(db, { migrationsFolder: "./drizzle" }); diff --git a/scraper/.env b/scraper/.env new file mode 100644 index 0000000..a537cb1 --- /dev/null +++ b/scraper/.env @@ -0,0 +1 @@ +DB_PATH=../scraper/sqlite.db \ No newline at end of file diff --git a/scraper/auto.ts b/scraper/auto.ts index 494a8df..dd5986d 100644 --- a/scraper/auto.ts +++ b/scraper/auto.ts @@ -1,4 +1,3 @@ -import "dotenv/config"; import { mkdtemp, access } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; @@ -50,13 +49,13 @@ supermercados.forEach(downloadList); async function downloadList(supermercado: Supermercado) { const listPath = resolve( - join(process.env.LISTS_DIR ?? "../data", `${supermercado}.txt`), + join(process.env.LISTS_DIR ?? "../data", `${supermercado}.txt`) ); const date = new Date(); const ctxPath = await mkdtemp(join(tmpdir(), "preciazo-scraper-wget-")); const zstdWarcName = `${supermercado}-${format( date, - "yyyy-MM-dd-HH:mm", + "yyyy-MM-dd-HH:mm" )}.warc.zst`; const zstdWarcPath = join(ctxPath, zstdWarcName); const subproc = Bun.spawn({ @@ -122,7 +121,7 @@ async function scrapAndInform({ inform( `Procesado ${zstdWarcName} (${progress.done} ok, ${ progress.errors.length - } errores) (tardó ${formatMs(took)})`, + } errores) (tardó ${formatMs(took)})` ); } else { inform(`Algo falló en ${zstdWarcName}`); @@ -159,7 +158,7 @@ function recompress(inputPath: string, outputPath: string) { ["-T0", "-15", "--long", "-o", outputPath], { stdio: ["pipe", null, null], - }, + } ); // @ts-expect-error a los types de bun no le gusta???? decompressor.stdout.pipe(compressor.stdin); @@ -215,7 +214,7 @@ async function sendTelegramMsg(text: string) { if (!process.env.TELEGRAM_BOT_TOKEN || !process.env.TELEGRAM_BOT_CHAT_ID) return; const url = new URL( - `https://api.telegram.org/bot${process.env.TELEGRAM_BOT_TOKEN}/sendMessage`, + `https://api.telegram.org/bot${process.env.TELEGRAM_BOT_TOKEN}/sendMessage` ); url.searchParams.set("chat_id", process.env.TELEGRAM_BOT_CHAT_ID); url.searchParams.set("text", text); diff --git a/scraper/package.json b/scraper/package.json index 610b395..1db67ac 100644 --- a/scraper/package.json +++ b/scraper/package.json @@ -15,7 +15,6 @@ "@aws-sdk/lib-storage": "^3.478.0", "date-fns": "^3.0.6", "db-datos": "workspace:^", - "dotenv": "^16.3.1", "drizzle-orm": "^0.29.1", "linkedom": "^0.16.5", "nanoid": "^5.0.4", diff --git a/scraper/scrap.ts b/scraper/scrap.ts index fe794ab..d4b771d 100644 --- a/scraper/scrap.ts +++ b/scraper/scrap.ts @@ -9,11 +9,12 @@ import { getDiaProduct } from "./parsers/dia.js"; import { getCotoProduct } from "./parsers/coto.js"; import { join } from "path"; import { and, eq, sql } from "drizzle-orm"; +import { DB_PATH } from "db-datos/drizzle.config.js"; const DEBUG = false; const PARSER_VERSION = 2; -const sqlite = new Database("sqlite.db"); +const sqlite = new Database(DB_PATH); const db = drizzle(sqlite, { schema }); sqlite.run(` diff --git a/sitio/src/lib/server/db.ts b/sitio/src/lib/server/db.ts index 7462d9c..c15c0f0 100644 --- a/sitio/src/lib/server/db.ts +++ b/sitio/src/lib/server/db.ts @@ -3,7 +3,7 @@ import { drizzle } from "drizzle-orm/bun-sqlite"; import * as schema from "db-datos/schema.js"; import { env } from "$env/dynamic/private"; -const sqlite = new Database(env.DB_PATH ?? "../scraper/sqlite.db"); +const sqlite = new Database(env.DB_PATH); export const db = drizzle(sqlite, { schema }); export * as schema from "db-datos/schema.js";