.dockerignore

@@ -2,8 +2,6 @@ data/warcs/
 data/carrefour/
 */*.db*
 sqlite.db
-db.db
-db.db-wal
 downloader/
 node_modules/
 */node_modules/

db-datos/.env.development

@@ -1 +1 @@
-DB_PATH=../db.db
+DB_PATH=../sqlite.db

db-datos/drizzle.config.js

@@ -1,4 +1,4 @@
-export const DB_PATH = process.env.DB_PATH ?? "../db.db";
+export const DB_PATH = process.env.DB_PATH ?? "../sqlite.db";
 
 /** @type { import("drizzle-kit").Config } */
 export default {

db-datos/drizzle/0013_harsh_starbolt.sql

@@ -1 +0,0 @@
-CREATE INDEX `precios_ean_fetched_at_idx` ON `precios` (`ean`,`fetched_at`);

db-datos/drizzle/meta/0000_snapshot.json

@@ -1,6 +1,8 @@
 {
-  "version": "6",
+  "version": "5",
   "dialect": "sqlite",
+  "id": "88aa0254-106e-424e-ab66-417ff44bbf0b",
+  "prevId": "00000000-0000-0000-0000-000000000000",
   "tables": {
     "precios": {
       "name": "precios",
@@ -56,9 +58,8 @@
   },
   "enums": {},
   "_meta": {
+    "schemas": {},
     "tables": {},
     "columns": {}
-  },
-  "id": "88aa0254-106e-424e-ab66-417ff44bbf0b",
-  "prevId": "00000000-0000-0000-0000-000000000000"
+  }
 }

db-datos/drizzle/meta/0001_snapshot.json

@@ -1,6 +1,8 @@
 {
-  "version": "6",
+  "version": "5",
   "dialect": "sqlite",
+  "id": "a565621c-046e-4f4d-b505-104e2c4f2b6c",
+  "prevId": "88aa0254-106e-424e-ab66-417ff44bbf0b",
   "tables": {
     "precios": {
       "name": "precios",
@@ -70,9 +72,8 @@
   },
   "enums": {},
   "_meta": {
+    "schemas": {},
     "tables": {},
     "columns": {}
-  },
-  "id": "a565621c-046e-4f4d-b505-104e2c4f2b6c",
-  "prevId": "88aa0254-106e-424e-ab66-417ff44bbf0b"
+  }
 }

db-datos/drizzle/meta/0002_snapshot.json

@@ -1,6 +1,8 @@
 {
-  "version": "6",
+  "version": "5",
   "dialect": "sqlite",
+  "id": "cbd90a60-7568-489f-ac45-95bd8818ffbd",
+  "prevId": "a565621c-046e-4f4d-b505-104e2c4f2b6c",
   "tables": {
     "precios": {
       "name": "precios",
@@ -84,9 +86,8 @@
   },
   "enums": {},
   "_meta": {
+    "schemas": {},
     "tables": {},
     "columns": {}
-  },
-  "id": "cbd90a60-7568-489f-ac45-95bd8818ffbd",
-  "prevId": "a565621c-046e-4f4d-b505-104e2c4f2b6c"
+  }
 }

db-datos/drizzle/meta/0003_snapshot.json

@@ -1,6 +1,8 @@
 {
-  "version": "6",
+  "version": "5",
   "dialect": "sqlite",
+  "id": "e1217fdb-6f54-44c5-a04b-c5aebf202102",
+  "prevId": "cbd90a60-7568-489f-ac45-95bd8818ffbd",
   "tables": {
     "precios": {
       "name": "precios",
@@ -92,9 +94,8 @@
   },
   "enums": {},
   "_meta": {
+    "schemas": {},
     "tables": {},
     "columns": {}
-  },
-  "id": "e1217fdb-6f54-44c5-a04b-c5aebf202102",
-  "prevId": "cbd90a60-7568-489f-ac45-95bd8818ffbd"
+  }
 }

db-datos/drizzle/meta/0004_snapshot.json

@@ -1,5 +1,7 @@
 {
-  "version": "6",
+  "id": "bf90a1cd-ae6a-4dba-a1aa-79f14a11d958",
+  "prevId": "e1217fdb-6f54-44c5-a04b-c5aebf202102",
+  "version": "5",
   "dialect": "sqlite",
   "tables": {
     "precios": {
@@ -92,9 +94,8 @@
   },
   "enums": {},
   "_meta": {
-    "tables": {},
-    "columns": {}
-  },
-  "id": "bf90a1cd-ae6a-4dba-a1aa-79f14a11d958",
-  "prevId": "e1217fdb-6f54-44c5-a04b-c5aebf202102"
+    "columns": {},
+    "schemas": {},
+    "tables": {}
+  }
 }

db-datos/drizzle/meta/0005_snapshot.json

@@ -1,5 +1,7 @@
 {
-  "version": "6",
+  "id": "f2cf47b9-e137-41c9-b7fb-6bc016588db0",
+  "prevId": "bf90a1cd-ae6a-4dba-a1aa-79f14a11d958",
+  "version": "5",
   "dialect": "sqlite",
   "tables": {
     "precios": {
@@ -92,9 +94,8 @@
   },
   "enums": {},
   "_meta": {
-    "tables": {},
-    "columns": {}
-  },
-  "id": "f2cf47b9-e137-41c9-b7fb-6bc016588db0",
-  "prevId": "bf90a1cd-ae6a-4dba-a1aa-79f14a11d958"
+    "columns": {},
+    "schemas": {},
+    "tables": {}
+  }
 }

db-datos/drizzle/meta/0006_snapshot.json

@@ -1,5 +1,7 @@
 {
-  "version": "6",
+  "id": "ac099405-ecd0-4637-ae5e-fb29c9847e45",
+  "prevId": "f2cf47b9-e137-41c9-b7fb-6bc016588db0",
+  "version": "5",
   "dialect": "sqlite",
   "tables": {
     "precios": {
@@ -92,9 +94,8 @@
   },
   "enums": {},
   "_meta": {
-    "tables": {},
-    "columns": {}
-  },
-  "id": "ac099405-ecd0-4637-ae5e-fb29c9847e45",
-  "prevId": "f2cf47b9-e137-41c9-b7fb-6bc016588db0"
+    "columns": {},
+    "schemas": {},
+    "tables": {}
+  }
 }

db-datos/drizzle/meta/0007_snapshot.json

@@ -1,5 +1,7 @@
 {
-  "version": "6",
+  "id": "9d2f23bf-dc60-4adb-b1bd-ec75e90dda25",
+  "prevId": "ac099405-ecd0-4637-ae5e-fb29c9847e45",
+  "version": "5",
   "dialect": "sqlite",
   "tables": {
     "precios": {
@@ -92,9 +94,8 @@
   },
   "enums": {},
   "_meta": {
-    "tables": {},
-    "columns": {}
-  },
-  "id": "9d2f23bf-dc60-4adb-b1bd-ec75e90dda25",
-  "prevId": "ac099405-ecd0-4637-ae5e-fb29c9847e45"
+    "columns": {},
+    "schemas": {},
+    "tables": {}
+  }
 }

db-datos/drizzle/meta/0008_snapshot.json

@@ -1,5 +1,7 @@
 {
-  "version": "6",
+  "id": "082630a9-3744-4e33-bde5-06045ca57d36",
+  "prevId": "9d2f23bf-dc60-4adb-b1bd-ec75e90dda25",
+  "version": "5",
   "dialect": "sqlite",
   "tables": {
     "precios": {
@@ -92,9 +94,8 @@
   },
   "enums": {},
   "_meta": {
-    "tables": {},
-    "columns": {}
-  },
-  "id": "082630a9-3744-4e33-bde5-06045ca57d36",
-  "prevId": "9d2f23bf-dc60-4adb-b1bd-ec75e90dda25"
+    "columns": {},
+    "schemas": {},
+    "tables": {}
+  }
 }

db-datos/drizzle/meta/0009_snapshot.json

@@ -1,6 +1,8 @@
 {
-  "version": "6",
+  "version": "5",
   "dialect": "sqlite",
+  "id": "2e398920-ffaf-4d55-ae13-d906cb9e0efa",
+  "prevId": "082630a9-3744-4e33-bde5-06045ca57d36",
   "tables": {
     "precios": {
       "name": "precios",
@@ -137,9 +139,8 @@
   },
   "enums": {},
   "_meta": {
+    "schemas": {},
     "tables": {},
     "columns": {}
-  },
-  "id": "2e398920-ffaf-4d55-ae13-d906cb9e0efa",
-  "prevId": "082630a9-3744-4e33-bde5-06045ca57d36"
+  }
 }

db-datos/drizzle/meta/0010_snapshot.json

@@ -1,6 +1,8 @@
 {
-  "version": "6",
+  "version": "5",
   "dialect": "sqlite",
+  "id": "c8297337-4ed8-432e-8782-65d41be42e00",
+  "prevId": "2e398920-ffaf-4d55-ae13-d906cb9e0efa",
   "tables": {
     "db_best_selling": {
       "name": "db_best_selling",
@@ -174,9 +176,8 @@
   },
   "enums": {},
   "_meta": {
+    "schemas": {},
     "tables": {},
     "columns": {}
-  },
-  "id": "c8297337-4ed8-432e-8782-65d41be42e00",
-  "prevId": "2e398920-ffaf-4d55-ae13-d906cb9e0efa"
+  }
 }

db-datos/drizzle/meta/0011_snapshot.json

@@ -1,6 +1,8 @@
 {
-  "version": "6",
+  "version": "5",
   "dialect": "sqlite",
+  "id": "8b4921b5-6ecd-4d69-ba64-9b0bfb53db84",
+  "prevId": "c8297337-4ed8-432e-8782-65d41be42e00",
   "tables": {
     "db_best_selling": {
       "name": "db_best_selling",
@@ -181,9 +183,8 @@
   },
   "enums": {},
   "_meta": {
+    "schemas": {},
     "tables": {},
     "columns": {}
-  },
-  "id": "8b4921b5-6ecd-4d69-ba64-9b0bfb53db84",
-  "prevId": "c8297337-4ed8-432e-8782-65d41be42e00"
+  }
 }

db-datos/drizzle/meta/0012_snapshot.json

@@ -1,6 +1,8 @@
 {
-  "version": "6",
+  "version": "5",
   "dialect": "sqlite",
+  "id": "16046188-ab24-4bd4-bfb4-8a81f24c6f28",
+  "prevId": "8b4921b5-6ecd-4d69-ba64-9b0bfb53db84",
   "tables": {
     "db_best_selling": {
       "name": "db_best_selling",
@@ -188,9 +190,8 @@
   },
   "enums": {},
   "_meta": {
+    "schemas": {},
     "tables": {},
     "columns": {}
-  },
-  "id": "16046188-ab24-4bd4-bfb4-8a81f24c6f28",
-  "prevId": "8b4921b5-6ecd-4d69-ba64-9b0bfb53db84"
+  }
 }

db-datos/drizzle/meta/0013_snapshot.json

@@ -1,208 +0,0 @@
-{
-  "version": "6",
-  "dialect": "sqlite",
-  "id": "c95c6547-d540-45cf-aa9d-9d828efb468e",
-  "prevId": "16046188-ab24-4bd4-bfb4-8a81f24c6f28",
-  "tables": {
-    "db_best_selling": {
-      "name": "db_best_selling",
-      "columns": {
-        "id": {
-          "name": "id",
-          "type": "integer",
-          "primaryKey": true,
-          "notNull": true,
-          "autoincrement": true
-        },
-        "fetched_at": {
-          "name": "fetched_at",
-          "type": "integer",
-          "primaryKey": false,
-          "notNull": true,
-          "autoincrement": false
-        },
-        "category": {
-          "name": "category",
-          "type": "text",
-          "primaryKey": false,
-          "notNull": true,
-          "autoincrement": false
-        },
-        "eans_json": {
-          "name": "eans_json",
-          "type": "text",
-          "primaryKey": false,
-          "notNull": true,
-          "autoincrement": false
-        }
-      },
-      "indexes": {},
-      "foreignKeys": {},
-      "compositePrimaryKeys": {},
-      "uniqueConstraints": {}
-    },
-    "precios": {
-      "name": "precios",
-      "columns": {
-        "id": {
-          "name": "id",
-          "type": "integer",
-          "primaryKey": true,
-          "notNull": true,
-          "autoincrement": true
-        },
-        "ean": {
-          "name": "ean",
-          "type": "text",
-          "primaryKey": false,
-          "notNull": true,
-          "autoincrement": false
-        },
-        "fetched_at": {
-          "name": "fetched_at",
-          "type": "integer",
-          "primaryKey": false,
-          "notNull": true,
-          "autoincrement": false
-        },
-        "precio_centavos": {
-          "name": "precio_centavos",
-          "type": "integer",
-          "primaryKey": false,
-          "notNull": false,
-          "autoincrement": false
-        },
-        "in_stock": {
-          "name": "in_stock",
-          "type": "integer",
-          "primaryKey": false,
-          "notNull": false,
-          "autoincrement": false
-        },
-        "url": {
-          "name": "url",
-          "type": "text",
-          "primaryKey": false,
-          "notNull": true,
-          "autoincrement": false
-        },
-        "warc_record_id": {
-          "name": "warc_record_id",
-          "type": "text",
-          "primaryKey": false,
-          "notNull": false,
-          "autoincrement": false
-        },
-        "parser_version": {
-          "name": "parser_version",
-          "type": "integer",
-          "primaryKey": false,
-          "notNull": false,
-          "autoincrement": false
-        },
-        "name": {
-          "name": "name",
-          "type": "text",
-          "primaryKey": false,
-          "notNull": false,
-          "autoincrement": false
-        },
-        "image_url": {
-          "name": "image_url",
-          "type": "text",
-          "primaryKey": false,
-          "notNull": false,
-          "autoincrement": false
-        }
-      },
-      "indexes": {
-        "precios_ean_idx": {
-          "name": "precios_ean_idx",
-          "columns": [
-            "ean"
-          ],
-          "isUnique": false
-        },
-        "precios_url_idx": {
-          "name": "precios_url_idx",
-          "columns": [
-            "url"
-          ],
-          "isUnique": false
-        },
-        "precios_fetched_at_idx": {
-          "name": "precios_fetched_at_idx",
-          "columns": [
-            "fetched_at"
-          ],
-          "isUnique": false
-        },
-        "precios_ean_fetched_at_idx": {
-          "name": "precios_ean_fetched_at_idx",
-          "columns": [
-            "ean",
-            "fetched_at"
-          ],
-          "isUnique": false
-        }
-      },
-      "foreignKeys": {},
-      "compositePrimaryKeys": {},
-      "uniqueConstraints": {}
-    },
-    "producto_urls": {
-      "name": "producto_urls",
-      "columns": {
-        "id": {
-          "name": "id",
-          "type": "integer",
-          "primaryKey": true,
-          "notNull": true,
-          "autoincrement": true
-        },
-        "url": {
-          "name": "url",
-          "type": "text",
-          "primaryKey": false,
-          "notNull": true,
-          "autoincrement": false
-        },
-        "first_seen": {
-          "name": "first_seen",
-          "type": "integer",
-          "primaryKey": false,
-          "notNull": true,
-          "autoincrement": false
-        },
-        "last_seen": {
-          "name": "last_seen",
-          "type": "integer",
-          "primaryKey": false,
-          "notNull": true,
-          "autoincrement": false
-        }
-      },
-      "indexes": {
-        "producto_urls_url_unique": {
-          "name": "producto_urls_url_unique",
-          "columns": [
-            "url"
-          ],
-          "isUnique": true
-        }
-      },
-      "foreignKeys": {},
-      "compositePrimaryKeys": {},
-      "uniqueConstraints": {}
-    }
-  },
-  "enums": {},
-  "_meta": {
-    "schemas": {},
-    "tables": {},
-    "columns": {}
-  },
-  "internal": {
-    "indexes": {}
-  }
-}

db-datos/drizzle/meta/_journal.json

@@ -92,13 +92,6 @@
       "when": 1719680946811,
       "tag": "0012_hard_red_wolf",
       "breakpoints": true
-    },
-    {
-      "idx": 13,
-      "version": "6",
-      "when": 1722796469056,
-      "tag": "0013_harsh_starbolt",
-      "breakpoints": true
     }
   ]
 }

db-datos/package.json

@@ -5,7 +5,7 @@
   "description": "",
   "main": "index.js",
   "scripts": {
-    "generate": "drizzle-kit generate",
+    "generate": "drizzle-kit generate:sqlite",
     "migrate": "node migrate-cli.js"
   },
   "keywords": [],

db-datos/schema.js

@@ -22,10 +22,6 @@ export const precios = sqliteTable(
       preciosFetchedAtIdx: index("precios_fetched_at_idx").on(
         precios.fetchedAt
       ),
-      preciosEanFetchedAtIdx: index("precios_ean_fetched_at_idx").on(
-        precios.ean,
-        precios.fetchedAt
-      ),
     };
   }
 );

rust/.env

@@ -1 +1 @@
-DATABASE_URL=sqlite://../db.db
+DATABASE_URL=sqlite://../sqlite.db

rust/src/api/main.rs

@@ -282,7 +282,7 @@ async fn main() {
         .connect_with(
             SqliteConnectOptions::from_str(&format!(
                 "sqlite://{}",
-                env::var("DB_PATH").unwrap_or("../db.db".to_string())
+                env::var("DB_PATH").unwrap_or("../sqlite.db".to_string())
             ))
             .unwrap()
             .journal_mode(sqlx::sqlite::SqliteJournalMode::Wal)

rust/src/scraper/auto.rs

@@ -1,5 +1,3 @@
-use std::env;
-
 use super::now_sec;
 use super::AutoArgs;
 use super::AutoTelegram;
@@ -63,16 +61,7 @@ impl Auto {
         // }
         {
             let t0 = now_sec();
-
-            let n_coroutines = if supermercado == Supermercado::Coto {
-                50
-            } else {
-                env::var("N_COROUTINES")
-                    .map_or(Ok(24), |s| s.parse::<usize>())
-                    .expect("N_COROUTINES no es un número")
-            };
-
-            let counters = self.scraper.fetch_list(&self.db, links, n_coroutines).await;
+            let counters = self.scraper.fetch_list(&self.db, links).await;
             self.inform(&format!(
                 "Downloaded {:?}: {:?} (took {})",
                 &supermercado,

rust/src/scraper/db.rs

@@ -17,7 +17,7 @@ pub struct Db {
 
 impl Db {
     pub async fn connect() -> anyhow::Result<Self> {
-        let db_path = env::var("DB_PATH").unwrap_or("../db.db".to_string());
+        let db_path = env::var("DB_PATH").unwrap_or("../sqlite.db".to_string());
         info!("Opening DB at {}", db_path);
         let read_pool = connect_to_db(&db_path, 32).await?;
         let write_pool = connect_to_db(&db_path, 1).await?;

rust/src/scraper/scraper.rs

@@ -128,7 +128,11 @@ impl Scraper {
         counters
     }
 
-    pub async fn fetch_list(&self, db: &Db, links: Vec<String>, n_coroutines: usize) -> Counters {
+    pub async fn fetch_list(&self, db: &Db, links: Vec<String>) -> Counters {
+        let n_coroutines = env::var("N_COROUTINES")
+            .map_or(Ok(24), |s| s.parse::<usize>())
+            .expect("N_COROUTINES no es un número");
+
         stream::iter(links)
             .map(|url| {
                 let db = db.clone();

rust/src/scraper/sites/coto.rs

@@ -56,11 +56,7 @@ pub fn parse(url: String, dom: &tl::VDom) -> Result<PrecioPoint, anyhow::Error>
         .find_map(|n| n.as_tag())
         .map(|t| t.inner_text(dom.parser()))
         // https://github.com/catdevnull/preciazo/issues/24
-        .map(|s| {
-            html_escape::decode_html_entities(s.trim())
-                .trim()
-                .to_string()
-        });
+        .map(|s| html_escape::decode_html_entities(s.trim()).to_string());
 
     let image_url = dom
         .query_selector(".zoomImage1")

sitio/.env.development

@@ -1,2 +1,2 @@
-DB_PATH=../db.db
+DB_PATH=../sqlite.db
 VITE_API_HOST=http://localhost:8000