usar 50 coroutinas en coto

closes #44

.dockerignore

@@ -2,6 +2,8 @@ data/warcs/
 data/carrefour/
 */*.db*
 sqlite.db
+db.db
+db.db-wal
 downloader/
 node_modules/
 */node_modules/

db-datos/.env.development

@@ -1 +1 @@
-DB_PATH=../sqlite.db
+DB_PATH=../db.db

db-datos/drizzle.config.js

@@ -1,4 +1,4 @@
-export const DB_PATH = process.env.DB_PATH ?? "../sqlite.db";
+export const DB_PATH = process.env.DB_PATH ?? "../db.db";
 
 /** @type { import("drizzle-kit").Config } */
 export default {

db-datos/drizzle/0013_harsh_starbolt.sql

@@ -0,0 +1 @@
+CREATE INDEX `precios_ean_fetched_at_idx` ON `precios` (`ean`,`fetched_at`);

db-datos/drizzle/meta/0000_snapshot.json

@@ -1,8 +1,6 @@
 {
-  "version": "5",
+  "version": "6",
   "dialect": "sqlite",
-  "id": "88aa0254-106e-424e-ab66-417ff44bbf0b",
-  "prevId": "00000000-0000-0000-0000-000000000000",
   "tables": {
     "precios": {
       "name": "precios",
@@ -58,8 +56,9 @@
   },
   "enums": {},
   "_meta": {
-    "schemas": {},
     "tables": {},
     "columns": {}
-  }
+  },
+  "id": "88aa0254-106e-424e-ab66-417ff44bbf0b",
+  "prevId": "00000000-0000-0000-0000-000000000000"
 }

db-datos/drizzle/meta/0001_snapshot.json

@@ -1,8 +1,6 @@
 {
-  "version": "5",
+  "version": "6",
   "dialect": "sqlite",
-  "id": "a565621c-046e-4f4d-b505-104e2c4f2b6c",
-  "prevId": "88aa0254-106e-424e-ab66-417ff44bbf0b",
   "tables": {
     "precios": {
       "name": "precios",
@@ -72,8 +70,9 @@
   },
   "enums": {},
   "_meta": {
-    "schemas": {},
     "tables": {},
     "columns": {}
-  }
+  },
+  "id": "a565621c-046e-4f4d-b505-104e2c4f2b6c",
+  "prevId": "88aa0254-106e-424e-ab66-417ff44bbf0b"
 }

db-datos/drizzle/meta/0002_snapshot.json

@@ -1,8 +1,6 @@
 {
-  "version": "5",
+  "version": "6",
   "dialect": "sqlite",
-  "id": "cbd90a60-7568-489f-ac45-95bd8818ffbd",
-  "prevId": "a565621c-046e-4f4d-b505-104e2c4f2b6c",
   "tables": {
     "precios": {
       "name": "precios",
@@ -86,8 +84,9 @@
   },
   "enums": {},
   "_meta": {
-    "schemas": {},
     "tables": {},
     "columns": {}
-  }
+  },
+  "id": "cbd90a60-7568-489f-ac45-95bd8818ffbd",
+  "prevId": "a565621c-046e-4f4d-b505-104e2c4f2b6c"
 }

db-datos/drizzle/meta/0003_snapshot.json

@@ -1,8 +1,6 @@
 {
-  "version": "5",
+  "version": "6",
   "dialect": "sqlite",
-  "id": "e1217fdb-6f54-44c5-a04b-c5aebf202102",
-  "prevId": "cbd90a60-7568-489f-ac45-95bd8818ffbd",
   "tables": {
     "precios": {
       "name": "precios",
@@ -94,8 +92,9 @@
   },
   "enums": {},
   "_meta": {
-    "schemas": {},
     "tables": {},
     "columns": {}
-  }
+  },
+  "id": "e1217fdb-6f54-44c5-a04b-c5aebf202102",
+  "prevId": "cbd90a60-7568-489f-ac45-95bd8818ffbd"
 }

db-datos/drizzle/meta/0004_snapshot.json

@@ -1,7 +1,5 @@
 {
-  "id": "bf90a1cd-ae6a-4dba-a1aa-79f14a11d958",
+  "version": "6",
-  "prevId": "e1217fdb-6f54-44c5-a04b-c5aebf202102",
-  "version": "5",
   "dialect": "sqlite",
   "tables": {
     "precios": {
@@ -94,8 +92,9 @@
   },
   "enums": {},
   "_meta": {
-    "columns": {},
+    "tables": {},
-    "schemas": {},
+    "columns": {}
-    "tables": {}
+  },
-  }
+  "id": "bf90a1cd-ae6a-4dba-a1aa-79f14a11d958",
+  "prevId": "e1217fdb-6f54-44c5-a04b-c5aebf202102"
 }

db-datos/drizzle/meta/0005_snapshot.json

@@ -1,7 +1,5 @@
 {
-  "id": "f2cf47b9-e137-41c9-b7fb-6bc016588db0",
+  "version": "6",
-  "prevId": "bf90a1cd-ae6a-4dba-a1aa-79f14a11d958",
-  "version": "5",
   "dialect": "sqlite",
   "tables": {
     "precios": {
@@ -94,8 +92,9 @@
   },
   "enums": {},
   "_meta": {
-    "columns": {},
+    "tables": {},
-    "schemas": {},
+    "columns": {}
-    "tables": {}
+  },
-  }
+  "id": "f2cf47b9-e137-41c9-b7fb-6bc016588db0",
+  "prevId": "bf90a1cd-ae6a-4dba-a1aa-79f14a11d958"
 }

db-datos/drizzle/meta/0006_snapshot.json

@@ -1,7 +1,5 @@
 {
-  "id": "ac099405-ecd0-4637-ae5e-fb29c9847e45",
+  "version": "6",
-  "prevId": "f2cf47b9-e137-41c9-b7fb-6bc016588db0",
-  "version": "5",
   "dialect": "sqlite",
   "tables": {
     "precios": {
@@ -94,8 +92,9 @@
   },
   "enums": {},
   "_meta": {
-    "columns": {},
+    "tables": {},
-    "schemas": {},
+    "columns": {}
-    "tables": {}
+  },
-  }
+  "id": "ac099405-ecd0-4637-ae5e-fb29c9847e45",
+  "prevId": "f2cf47b9-e137-41c9-b7fb-6bc016588db0"
 }

db-datos/drizzle/meta/0007_snapshot.json

@@ -1,7 +1,5 @@
 {
-  "id": "9d2f23bf-dc60-4adb-b1bd-ec75e90dda25",
+  "version": "6",
-  "prevId": "ac099405-ecd0-4637-ae5e-fb29c9847e45",
-  "version": "5",
   "dialect": "sqlite",
   "tables": {
     "precios": {
@@ -94,8 +92,9 @@
   },
   "enums": {},
   "_meta": {
-    "columns": {},
+    "tables": {},
-    "schemas": {},
+    "columns": {}
-    "tables": {}
+  },
-  }
+  "id": "9d2f23bf-dc60-4adb-b1bd-ec75e90dda25",
+  "prevId": "ac099405-ecd0-4637-ae5e-fb29c9847e45"
 }

db-datos/drizzle/meta/0008_snapshot.json

@@ -1,7 +1,5 @@
 {
-  "id": "082630a9-3744-4e33-bde5-06045ca57d36",
+  "version": "6",
-  "prevId": "9d2f23bf-dc60-4adb-b1bd-ec75e90dda25",
-  "version": "5",
   "dialect": "sqlite",
   "tables": {
     "precios": {
@@ -94,8 +92,9 @@
   },
   "enums": {},
   "_meta": {
-    "columns": {},
+    "tables": {},
-    "schemas": {},
+    "columns": {}
-    "tables": {}
+  },
-  }
+  "id": "082630a9-3744-4e33-bde5-06045ca57d36",
+  "prevId": "9d2f23bf-dc60-4adb-b1bd-ec75e90dda25"
 }

db-datos/drizzle/meta/0009_snapshot.json

@@ -1,8 +1,6 @@
 {
-  "version": "5",
+  "version": "6",
   "dialect": "sqlite",
-  "id": "2e398920-ffaf-4d55-ae13-d906cb9e0efa",
-  "prevId": "082630a9-3744-4e33-bde5-06045ca57d36",
   "tables": {
     "precios": {
       "name": "precios",
@@ -139,8 +137,9 @@
   },
   "enums": {},
   "_meta": {
-    "schemas": {},
     "tables": {},
     "columns": {}
-  }
+  },
+  "id": "2e398920-ffaf-4d55-ae13-d906cb9e0efa",
+  "prevId": "082630a9-3744-4e33-bde5-06045ca57d36"
 }

db-datos/drizzle/meta/0010_snapshot.json

@@ -1,8 +1,6 @@
 {
-  "version": "5",
+  "version": "6",
   "dialect": "sqlite",
-  "id": "c8297337-4ed8-432e-8782-65d41be42e00",
-  "prevId": "2e398920-ffaf-4d55-ae13-d906cb9e0efa",
   "tables": {
     "db_best_selling": {
       "name": "db_best_selling",
@@ -176,8 +174,9 @@
   },
   "enums": {},
   "_meta": {
-    "schemas": {},
     "tables": {},
     "columns": {}
-  }
+  },
+  "id": "c8297337-4ed8-432e-8782-65d41be42e00",
+  "prevId": "2e398920-ffaf-4d55-ae13-d906cb9e0efa"
 }

db-datos/drizzle/meta/0011_snapshot.json

@@ -1,8 +1,6 @@
 {
-  "version": "5",
+  "version": "6",
   "dialect": "sqlite",
-  "id": "8b4921b5-6ecd-4d69-ba64-9b0bfb53db84",
-  "prevId": "c8297337-4ed8-432e-8782-65d41be42e00",
   "tables": {
     "db_best_selling": {
       "name": "db_best_selling",
@@ -183,8 +181,9 @@
   },
   "enums": {},
   "_meta": {
-    "schemas": {},
     "tables": {},
     "columns": {}
-  }
+  },
+  "id": "8b4921b5-6ecd-4d69-ba64-9b0bfb53db84",
+  "prevId": "c8297337-4ed8-432e-8782-65d41be42e00"
 }

db-datos/drizzle/meta/0012_snapshot.json

@@ -1,8 +1,6 @@
 {
-  "version": "5",
+  "version": "6",
   "dialect": "sqlite",
-  "id": "16046188-ab24-4bd4-bfb4-8a81f24c6f28",
-  "prevId": "8b4921b5-6ecd-4d69-ba64-9b0bfb53db84",
   "tables": {
     "db_best_selling": {
       "name": "db_best_selling",
@@ -190,8 +188,9 @@
   },
   "enums": {},
   "_meta": {
-    "schemas": {},
     "tables": {},
     "columns": {}
-  }
+  },
+  "id": "16046188-ab24-4bd4-bfb4-8a81f24c6f28",
+  "prevId": "8b4921b5-6ecd-4d69-ba64-9b0bfb53db84"
 }

db-datos/drizzle/meta/0013_snapshot.json

@@ -0,0 +1,208 @@
+{
+  "version": "6",
+  "dialect": "sqlite",
+  "id": "c95c6547-d540-45cf-aa9d-9d828efb468e",
+  "prevId": "16046188-ab24-4bd4-bfb4-8a81f24c6f28",
+  "tables": {
+    "db_best_selling": {
+      "name": "db_best_selling",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "integer",
+          "primaryKey": true,
+          "notNull": true,
+          "autoincrement": true
+        },
+        "fetched_at": {
+          "name": "fetched_at",
+          "type": "integer",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "category": {
+          "name": "category",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "eans_json": {
+          "name": "eans_json",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {}
+    },
+    "precios": {
+      "name": "precios",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "integer",
+          "primaryKey": true,
+          "notNull": true,
+          "autoincrement": true
+        },
+        "ean": {
+          "name": "ean",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "fetched_at": {
+          "name": "fetched_at",
+          "type": "integer",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "precio_centavos": {
+          "name": "precio_centavos",
+          "type": "integer",
+          "primaryKey": false,
+          "notNull": false,
+          "autoincrement": false
+        },
+        "in_stock": {
+          "name": "in_stock",
+          "type": "integer",
+          "primaryKey": false,
+          "notNull": false,
+          "autoincrement": false
+        },
+        "url": {
+          "name": "url",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "warc_record_id": {
+          "name": "warc_record_id",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false,
+          "autoincrement": false
+        },
+        "parser_version": {
+          "name": "parser_version",
+          "type": "integer",
+          "primaryKey": false,
+          "notNull": false,
+          "autoincrement": false
+        },
+        "name": {
+          "name": "name",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false,
+          "autoincrement": false
+        },
+        "image_url": {
+          "name": "image_url",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false,
+          "autoincrement": false
+        }
+      },
+      "indexes": {
+        "precios_ean_idx": {
+          "name": "precios_ean_idx",
+          "columns": [
+            "ean"
+          ],
+          "isUnique": false
+        },
+        "precios_url_idx": {
+          "name": "precios_url_idx",
+          "columns": [
+            "url"
+          ],
+          "isUnique": false
+        },
+        "precios_fetched_at_idx": {
+          "name": "precios_fetched_at_idx",
+          "columns": [
+            "fetched_at"
+          ],
+          "isUnique": false
+        },
+        "precios_ean_fetched_at_idx": {
+          "name": "precios_ean_fetched_at_idx",
+          "columns": [
+            "ean",
+            "fetched_at"
+          ],
+          "isUnique": false
+        }
+      },
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {}
+    },
+    "producto_urls": {
+      "name": "producto_urls",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "integer",
+          "primaryKey": true,
+          "notNull": true,
+          "autoincrement": true
+        },
+        "url": {
+          "name": "url",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "first_seen": {
+          "name": "first_seen",
+          "type": "integer",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "last_seen": {
+          "name": "last_seen",
+          "type": "integer",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        }
+      },
+      "indexes": {
+        "producto_urls_url_unique": {
+          "name": "producto_urls_url_unique",
+          "columns": [
+            "url"
+          ],
+          "isUnique": true
+        }
+      },
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {}
+    }
+  },
+  "enums": {},
+  "_meta": {
+    "schemas": {},
+    "tables": {},
+    "columns": {}
+  },
+  "internal": {
+    "indexes": {}
+  }
+}

db-datos/drizzle/meta/_journal.json

@@ -92,6 +92,13 @@
       "when": 1719680946811,
       "tag": "0012_hard_red_wolf",
       "breakpoints": true
+    },
+    {
+      "idx": 13,
+      "version": "6",
+      "when": 1722796469056,
+      "tag": "0013_harsh_starbolt",
+      "breakpoints": true
     }
   ]
 }

db-datos/package.json

@@ -5,7 +5,7 @@
   "description": "",
   "main": "index.js",
   "scripts": {
-    "generate": "drizzle-kit generate:sqlite",
+    "generate": "drizzle-kit generate",
     "migrate": "node migrate-cli.js"
   },
   "keywords": [],

db-datos/schema.js

@@ -22,6 +22,10 @@ export const precios = sqliteTable(
       preciosFetchedAtIdx: index("precios_fetched_at_idx").on(
         precios.fetchedAt
       ),
+      preciosEanFetchedAtIdx: index("precios_ean_fetched_at_idx").on(
+        precios.ean,
+        precios.fetchedAt
+      ),
     };
   }
 );

rust/.env

@@ -1 +1 @@
-DATABASE_URL=sqlite://../sqlite.db
+DATABASE_URL=sqlite://../db.db

rust/src/api/main.rs

@@ -282,7 +282,7 @@ async fn main() {
         .connect_with(
             SqliteConnectOptions::from_str(&format!(
                 "sqlite://{}",
-                env::var("DB_PATH").unwrap_or("../sqlite.db".to_string())
+                env::var("DB_PATH").unwrap_or("../db.db".to_string())
             ))
             .unwrap()
             .journal_mode(sqlx::sqlite::SqliteJournalMode::Wal)

rust/src/scraper/auto.rs

@@ -1,3 +1,5 @@
+use std::env;
+
 use super::now_sec;
 use super::AutoArgs;
 use super::AutoTelegram;
@@ -61,7 +63,16 @@ impl Auto {
         // }
         {
             let t0 = now_sec();
-            let counters = self.scraper.fetch_list(&self.db, links).await;
+
+            let n_coroutines = if supermercado == Supermercado::Coto {
+                50
+            } else {
+                env::var("N_COROUTINES")
+                    .map_or(Ok(24), |s| s.parse::<usize>())
+                    .expect("N_COROUTINES no es un número")
+            };
+
+            let counters = self.scraper.fetch_list(&self.db, links, n_coroutines).await;
             self.inform(&format!(
                 "Downloaded {:?}: {:?} (took {})",
                 &supermercado,

rust/src/scraper/db.rs

@@ -17,7 +17,7 @@ pub struct Db {
 
 impl Db {
     pub async fn connect() -> anyhow::Result<Self> {
-        let db_path = env::var("DB_PATH").unwrap_or("../sqlite.db".to_string());
+        let db_path = env::var("DB_PATH").unwrap_or("../db.db".to_string());
         info!("Opening DB at {}", db_path);
         let read_pool = connect_to_db(&db_path, 32).await?;
         let write_pool = connect_to_db(&db_path, 1).await?;

rust/src/scraper/scraper.rs

@@ -128,11 +128,7 @@ impl Scraper {
         counters
     }
 
-    pub async fn fetch_list(&self, db: &Db, links: Vec<String>) -> Counters {
+    pub async fn fetch_list(&self, db: &Db, links: Vec<String>, n_coroutines: usize) -> Counters {
-        let n_coroutines = env::var("N_COROUTINES")
-            .map_or(Ok(24), |s| s.parse::<usize>())
-            .expect("N_COROUTINES no es un número");
-
         stream::iter(links)
             .map(|url| {
                 let db = db.clone();

rust/src/scraper/sites/coto.rs

@@ -56,7 +56,11 @@ pub fn parse(url: String, dom: &tl::VDom) -> Result<PrecioPoint, anyhow::Error>
         .find_map(|n| n.as_tag())
         .map(|t| t.inner_text(dom.parser()))
         // https://github.com/catdevnull/preciazo/issues/24
-        .map(|s| html_escape::decode_html_entities(s.trim()).to_string());
+        .map(|s| {
+            html_escape::decode_html_entities(s.trim())
+                .trim()
+                .to_string()
+        });
 
     let image_url = dom
         .query_selector(".zoomImage1")

sitio/.env.development

@@ -1,2 +1,2 @@
-DB_PATH=../sqlite.db
+DB_PATH=../db.db
 VITE_API_HOST=http://localhost:8000