rust/.sqlx/query-aa5c2a04aec149d88f6e25a9bd7df4e257f3c9b0efa62c8342d077d69d826a69.json

@@ -1,6 +1,6 @@
 {
   "db_name": "SQLite",
-  "query": "SELECT url FROM producto_urls WHERE url LIKE ?1 AND last_seen > ?2;",
+  "query": "SELECT url FROM producto_urls WHERE url LIKE ?1;",
   "describe": {
     "columns": [
       {
@@ -10,11 +10,11 @@
       }
     ],
     "parameters": {
-      "Right": 2
+      "Right": 1
     },
     "nullable": [
       false
     ]
   },
-  "hash": "8dac7e96d5dab0d6f48c5fa7d4844e9d4f9b3bf38e55cb56e3941bb8fbf0a9ff"
+  "hash": "aa5c2a04aec149d88f6e25a9bd7df4e257f3c9b0efa62c8342d077d69d826a69"
 }

rust/src/scraper/auto.rs

@@ -41,10 +41,7 @@ impl Auto {
             }
         }
         let links: Vec<String> = {
-            let mut links = self
+            let mut links = self.db.get_urls_by_domain(supermercado.host()).await?;
-                .db
-                .get_recent_urls_by_domain(supermercado.host())
-                .await?;
             if let Some(n) = self.args.n_products {
                 links.truncate(n);
             }

rust/src/scraper/db.rs

@@ -11,20 +11,29 @@ use crate::{best_selling::BestSellingRecord, PrecioPoint};
 
 #[derive(Clone)]
 pub struct Db {
-    read_pool: SqlitePool,
+    pool: SqlitePool,
-    write_pool: SqlitePool,
 }
 
 impl Db {
     pub async fn connect() -> anyhow::Result<Self> {
         let db_path = env::var("DB_PATH").unwrap_or("../sqlite.db".to_string());
         info!("Opening DB at {}", db_path);
-        let read_pool = connect_to_db(&db_path, 32).await?;
+        let pool = sqlx::pool::PoolOptions::new()
-        let write_pool = connect_to_db(&db_path, 1).await?;
+            .max_connections(1)
-        Ok(Self {
+            .connect_with(
-            read_pool,
+                SqliteConnectOptions::from_str(&format!("sqlite://{}", db_path))?
-            write_pool,
+                    // https://fractaledmind.github.io/2023/09/07/enhancing-rails-sqlite-fine-tuning/
-        })
+                    .journal_mode(sqlx::sqlite::SqliteJournalMode::Wal)
+                    .pragma("journal_size_limit", "67108864")
+                    .pragma("mmap_size", "134217728")
+                    .synchronous(sqlx::sqlite::SqliteSynchronous::Normal)
+                    .busy_timeout(Duration::from_secs(15))
+                    .pragma("cache_size", "2000")
+                    .pragma("temp_store", "memory")
+                    .optimize_on_close(true, None),
+            )
+            .await?;
+        Ok(Self { pool })
     }
 
     pub async fn insert_precio(&self, point: PrecioPoint) -> anyhow::Result<()> {
@@ -38,37 +47,32 @@ impl Db {
             point.parser_version,
             point.name,
             point.image_url,
-        ).execute(&self.write_pool).await?;
+        ).execute(&self.pool).await?;
         Ok(())
     }
 
     pub async fn get_ean_by_url(&self, url: &str) -> anyhow::Result<Option<String>> {
         Ok(sqlx::query!("SELECT ean FROM precios WHERE url = ?1;", url)
-            .fetch_optional(&self.read_pool)
+            .fetch_optional(&self.pool)
             .await?
             .map(|r| r.ean))
     }
 
-    pub async fn get_recent_urls_by_domain(&self, domain: &str) -> anyhow::Result<Vec<String>> {
+    pub async fn get_urls_by_domain(&self, domain: &str) -> anyhow::Result<Vec<String>> {
         let query = format!("%{}%", domain);
-        let last_60_days: i64 = (now() - Duration::from_secs(60 * 60 * 24 * 60))
+        Ok(
-            .as_millis()
+            sqlx::query!("SELECT url FROM producto_urls WHERE url LIKE ?1;", query)
-            .try_into()?;
+                .fetch_all(&self.pool)
-        Ok(sqlx::query!(
+                .await?
-            "SELECT url FROM producto_urls WHERE url LIKE ?1 AND last_seen > ?2;",
+                .into_iter()
-            query,
+                .map(|r| r.url)
-            last_60_days
+                .collect(),
         )
-        .fetch_all(&self.read_pool)
-        .await?
-        .into_iter()
-        .map(|r| r.url)
-        .collect())
     }
 
     pub async fn save_producto_urls(&self, urls: Vec<String>) -> anyhow::Result<()> {
         let now: i64 = now_ms().try_into()?;
-        let mut tx = self.write_pool.begin().await?;
+        let mut tx = self.pool.begin().await?;
         for url in urls {
             sqlx::query!(
                 r#"INSERT INTO producto_urls(url, first_seen, last_seen)
@@ -85,7 +89,7 @@ impl Db {
     }
 
     pub async fn save_best_selling(&self, records: Vec<BestSellingRecord>) -> anyhow::Result<()> {
-        let mut tx = self.write_pool.begin().await?;
+        let mut tx = self.pool.begin().await?;
         for record in records {
             let fetched_at = record.fetched_at.timestamp_millis();
             let category = record.category.id();
@@ -105,33 +109,9 @@ impl Db {
     }
 }
 
-async fn connect_to_db(
-    db_path: &str,
-    max_connections: u32,
-) -> Result<sqlx::Pool<sqlx::Sqlite>, anyhow::Error> {
-    Ok(sqlx::pool::PoolOptions::new()
-        .max_connections(max_connections)
-        .connect_with(
-            SqliteConnectOptions::from_str(&format!("sqlite://{}", db_path))?
-                // https://fractaledmind.github.io/2023/09/07/enhancing-rails-sqlite-fine-tuning/
-                .journal_mode(sqlx::sqlite::SqliteJournalMode::Wal)
-                .pragma("journal_size_limit", "67108864")
-                .pragma("mmap_size", "134217728")
-                .synchronous(sqlx::sqlite::SqliteSynchronous::Normal)
-                .busy_timeout(Duration::from_secs(15))
-                .pragma("cache_size", "2000")
-                .pragma("temp_store", "memory")
-                .optimize_on_close(true, None),
-        )
-        .await?)
-}
-
 fn now_ms() -> u128 {
-    now().as_millis()
-}
-
-fn now() -> Duration {
     SystemTime::now()
         .duration_since(UNIX_EPOCH)
         .expect("Time went backwards")
+        .as_millis()
 }