From 710170c46b7761f5f09cda91c83ad397ff3e5396 Mon Sep 17 00:00:00 2001
From: Nulo <git@nulo.in>
Date: Tue, 6 Feb 2024 18:54:40 -0300
Subject: [PATCH] permitir scrapear solo n productos para testear

---
 scraper-rs/src/main.rs | 26 ++++++++++++++++++++------
 1 file changed, 20 insertions(+), 6 deletions(-)

diff --git a/scraper-rs/src/main.rs b/scraper-rs/src/main.rs
index 3e61f9c..43e03c6 100644
--- a/scraper-rs/src/main.rs
+++ b/scraper-rs/src/main.rs
@@ -47,7 +47,10 @@ struct ScrapUrlArgs {
     url: String,
 }
 #[derive(clap::Args)]
-struct AutoArgs {}
+struct AutoArgs {
+    #[arg(long)]
+    n_products: Option<usize>,
+}
 
 #[tokio::main]
 async fn main() -> anyhow::Result<()> {
@@ -59,7 +62,7 @@ async fn main() -> anyhow::Result<()> {
         Args::GetUrlList(a) => get_url_list_cli(a.supermercado).await,
         Args::ScrapUrl(a) => scrap_url_cli(a.url).await,
         Args::ScrapBestSelling => scrap_best_selling_cli().await,
-        Args::Auto(_) => auto_cli().await,
+        Args::Auto(a) => auto_cli(a).await,
         Args::Cron(_) => cron_cli().await,
     }
 }
@@ -287,6 +290,7 @@ struct AutoTelegram {
 struct Auto {
     db: Db,
     telegram: Option<AutoTelegram>,
+    limit_n_products: Option<usize>,
 }
 impl Auto {
     async fn download_supermercado(self, supermercado: Supermercado) -> anyhow::Result<()> {
@@ -300,7 +304,13 @@ impl Auto {
             ))
             .await;
         }
-        let links: Vec<String> = self.db.get_urls_by_domain(supermercado.host()).await?;
+        let links: Vec<String> = {
+            let mut links = self.db.get_urls_by_domain(supermercado.host()).await?;
+            if let Some(n) = self.limit_n_products {
+                links.truncate(n);
+            }
+            links
+        };
         // {
         //     let debug_path = PathBuf::from("debug/");
         //     tokio::fs::create_dir_all(&debug_path).await.unwrap();
@@ -355,7 +365,7 @@ impl Auto {
     }
 }
 
-async fn auto_cli() -> anyhow::Result<()> {
+async fn auto_cli(args: AutoArgs) -> anyhow::Result<()> {
     let auto = {
         let db = Db::connect().await?;
         let telegram = {
@@ -370,7 +380,11 @@ async fn auto_cli() -> anyhow::Result<()> {
                 }
             }
         };
-        Auto { db, telegram }
+        Auto {
+            db,
+            telegram,
+            limit_n_products: args.n_products,
+        }
     };
     auto.inform("[auto] Empezando scrap").await;
     let handles: Vec<_> = Supermercado::value_variants()
@@ -405,7 +419,7 @@ async fn cron_cli() -> anyhow::Result<()> {
             .unwrap();
         println!("Waiting for {:?}", t);
         tokio::time::sleep(t).await;
-        auto_cli().await.unwrap();
+        auto_cli(AutoArgs { n_products: None }).await.unwrap();
     }
 }