container scraper

.dockerignore

@@ -0,0 +1,6 @@
+data/warcs/
+data/carrefour/
+*/*.db*
+downloader/
+node_modules/
+*/node_modules/

.gitignore

@@ -12,4 +12,4 @@ scraper/debug/
 scraper/x.tsv
 *.tmp
 target/
-.env.*
+.env.*

db-datos/.env

@@ -1 +0,0 @@
-DB_PATH=../scraper/sqlite.db

db-datos/drizzle.config.ts

@@ -7,7 +7,6 @@ export const DB_PATH = process.env.DB_PATH;
 export default {
   schema: "./schema.ts",
   out: "./drizzle",
-  driver: "better-sqlite",
   dbCredentials: {
     url: process.env.DB_PATH,
   },

db-datos/migrate.ts

@@ -4,9 +4,11 @@ import { migrate } from "drizzle-orm/bun-sqlite/migrator";
 import * as schema from "./schema.js";
 import { DB_PATH } from "./drizzle.config.js";
 
-const sqlite = new Database(DB_PATH);
+export function migrateDb() {
-const db = drizzle(sqlite, { schema });
+  const sqlite = new Database(DB_PATH);
+  const db = drizzle(sqlite, { schema });
 
-migrate(db, { migrationsFolder: "./drizzle" });
+  migrate(db, { migrationsFolder: "./drizzle" });
 
-sqlite.close();
+  sqlite.close();
+}

db-datos/package.json

@@ -15,9 +15,7 @@
     "drizzle-orm": "^0.29.1"
   },
   "devDependencies": {
-    "better-sqlite3": "^9.2.2",
+    "@types/bun": "^1.0.0",
-    "drizzle-kit": "^0.20.7",
+    "drizzle-kit": "^0.20.7"
-    "tsx": "^4.7.0",
-    "@types/better-sqlite3": "^7.6.8"
   }
 }

scraper/.env

@@ -1 +0,0 @@
-DB_PATH=../scraper/sqlite.db

scraper/Containerfile

@@ -0,0 +1,18 @@
+FROM oven/bun:1-alpine AS base
+WORKDIR /usr/src/app
+
+FROM base AS builder
+ENV NODE_ENV=production
+COPY . .
+RUN bun install --frozen-lockfile
+RUN bun build scraper/auto.ts --target=bun --outfile=/tmp/auto.build.js
+
+FROM base
+COPY --from=builder /tmp/auto.build.js /app/built.js
+COPY --from=builder /usr/src/app/db-datos/drizzle /app/drizzle
+WORKDIR /app
+
+VOLUME /db
+ENV DB_PATH=/db/db.db
+
+CMD ["bun", "/app/built.js"]

scraper/package.json

@@ -5,7 +5,7 @@
   "description": "",
   "main": "index.js",
   "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1"
+    "build:container": "podman build -t gitea.nulo.in/nulo/preciazo/scraper -f ./Containerfile .."
   },
   "keywords": [],
   "author": "",

scraper/scrap.ts

@@ -10,10 +10,13 @@ import { getCotoProduct } from "./parsers/coto.js";
 import { join } from "path";
 import { and, eq, sql } from "drizzle-orm";
 import { DB_PATH } from "db-datos/drizzle.config.js";
+import { migrateDb } from "db-datos/migrate.js";
 
 const DEBUG = false;
 const PARSER_VERSION = 2;
 
+migrateDb();
+
 const sqlite = new Database(DB_PATH);
 const db = drizzle(sqlite, { schema });