db-datos/db.js

@@ -9,7 +9,7 @@ import { migrateDb } from "./migrate.js";
 let db = null;
 export function getDb() {
   const sqlite = new Database(DB_PATH);
-  db = drizzle(sqlite, { schema, logger: true });
+  db = drizzle(sqlite, { schema });
   migrateDb(db);
   return db;
 }

db-datos/supermercado.ts

@@ -3,31 +3,25 @@ export enum Supermercado {
   Carrefour = "Carrefour",
   Coto = "Coto",
   Jumbo = "Jumbo",
-  Farmacity = "Farmacity",
 }
 export const supermercados: Supermercado[] = [
   Supermercado.Carrefour,
   Supermercado.Coto,
   Supermercado.Dia,
   Supermercado.Jumbo,
-  Supermercado.Farmacity,
 ];
 export const hosts: { [host: string]: Supermercado } = {
   "diaonline.supermercadosdia.com.ar": Supermercado.Dia,
   "www.carrefour.com.ar": Supermercado.Carrefour,
   "www.cotodigital3.com.ar": Supermercado.Coto,
   "www.jumbo.com.ar": Supermercado.Jumbo,
-  "www.farmacity.com": Supermercado.Farmacity,
 };
 export const hostBySupermercado = Object.fromEntries(
   Object.entries(hosts).map(([a, b]) => [b, a])
 ) as Record<Supermercado, string>;
-
-// también actualizar en sitio/src/routes/ean/[ean]/+page.svelte
 export const colorBySupermercado: { [supermercado in Supermercado]: string } = {
   [Supermercado.Dia]: "#d52b1e",
   [Supermercado.Carrefour]: "#19549d",
   [Supermercado.Coto]: "#e20025",
   [Supermercado.Jumbo]: "#2dc850",
-  [Supermercado.Farmacity]: "#EF7603",
 };

scraper-rs/src/main.rs

@@ -46,12 +46,10 @@ struct GetUrlListArgs {
 struct ScrapUrlArgs {
     url: String,
 }
-#[derive(clap::Args, Clone, Copy)]
+#[derive(clap::Args)]
 struct AutoArgs {
     #[arg(long)]
     n_products: Option<usize>,
-    #[arg(long)]
-    only_supermercado: Option<Supermercado>,
 }
 
 #[tokio::main]
@@ -186,25 +184,18 @@ async fn request_and_body(client: &reqwest::Client, url: &str) -> reqwest::Resul
     res.text().await
 }
 pub async fn fetch_body(client: &reqwest::Client, url: &str) -> reqwest::Result<String> {
-    get_fetch_retry_policy()
+    get_retry_policy()
         .retry_if(|| request_and_body(client, url), retry_if_wasnt_not_found)
         .await
 }
 
-pub fn get_fetch_retry_policy() -> again::RetryPolicy {
+pub fn get_retry_policy() -> again::RetryPolicy {
     RetryPolicy::exponential(Duration::from_millis(300))
         .with_max_retries(20)
         .with_max_delay(Duration::from_secs(40))
         .with_jitter(true)
 }
 
-pub fn get_parse_retry_policy() -> again::RetryPolicy {
-    RetryPolicy::exponential(Duration::from_millis(1500))
-        .with_max_retries(5)
-        .with_max_delay(Duration::from_secs(5))
-        .with_jitter(true)
-}
-
 pub fn retry_if_wasnt_not_found(err: &reqwest::Error) -> bool {
     !err.status().is_some_and(|s| s == StatusCode::NOT_FOUND)
 }
@@ -214,11 +205,8 @@ async fn fetch_and_parse(
     client: &reqwest::Client,
     url: String,
 ) -> Result<PrecioPoint, anyhow::Error> {
-    async fn fetch_and_scrap(
-        client: &reqwest::Client,
-        url: String,
-    ) -> Result<PrecioPoint, anyhow::Error> {
     let body = fetch_body(client, &url).await?;
+
     let maybe_point = { scrap_url(client, url, &body).await };
 
     let point = match maybe_point {
@@ -235,17 +223,6 @@ async fn fetch_and_parse(
     }?;
 
     Ok(point)
-    }
-
-    get_parse_retry_policy()
-        .retry_if(
-            || fetch_and_scrap(client, url.clone()),
-            |err: &anyhow::Error| match err.downcast_ref::<reqwest::Error>() {
-                Some(e) => !e.status().is_some_and(|s| s == StatusCode::NOT_FOUND),
-                None => true,
-            },
-        )
-        .await
 }
 
 async fn parse_file_cli(file_path: String) -> anyhow::Result<()> {
@@ -285,7 +262,6 @@ async fn get_urls(supermercado: &Supermercado) -> Result<Vec<String>, anyhow::Er
         Supermercado::Jumbo => sites::jumbo::get_urls().await?,
         Supermercado::Carrefour => sites::carrefour::get_urls().await?,
         Supermercado::Coto => sites::coto::get_urls().await?,
-        Supermercado::Farmacity => sites::farmacity::get_urls().await?,
     })
 }
 
@@ -306,9 +282,6 @@ async fn scrap_url(
             sites::coto::parse(url, &tl::parse(body, tl::ParserOptions::default())?)
         }
         "www.jumbo.com.ar" => sites::jumbo::scrap(client, url, body).await,
-        "www.farmacity.com" => {
-            sites::farmacity::parse(url, &tl::parse(body, tl::ParserOptions::default())?)
-        }
         s => bail!("Unknown host {}", s),
     }
 }
@@ -323,7 +296,7 @@ struct AutoTelegram {
 struct Auto {
     db: Db,
     telegram: Option<AutoTelegram>,
-    args: AutoArgs,
+    limit_n_products: Option<usize>,
 }
 impl Auto {
     async fn download_supermercado(self, supermercado: Supermercado) -> anyhow::Result<()> {
@@ -339,7 +312,7 @@ impl Auto {
         }
         let links: Vec<String> = {
             let mut links = self.db.get_urls_by_domain(supermercado.host()).await?;
-            if let Some(n) = self.args.n_products {
+            if let Some(n) = self.limit_n_products {
                 links.truncate(n);
             }
             links
@@ -413,16 +386,14 @@ async fn auto_cli(args: AutoArgs) -> anyhow::Result<()> {
                 }
             }
         };
-        Auto { db, telegram, args }
+        Auto {
+            db,
+            telegram,
+            limit_n_products: args.n_products,
+        }
     };
     auto.inform("[auto] Empezando scrap").await;
-
+    let handles: Vec<_> = Supermercado::value_variants()
-    let supermercados = match args.only_supermercado {
-        Some(supermercado) => [supermercado].to_vec(),
-        None => Supermercado::value_variants().to_vec(),
-    };
-
-    let handles: Vec<_> = supermercados
         .iter()
         .map(|s| tokio::spawn(auto.clone().download_supermercado(s.to_owned())))
         .collect();
@@ -454,12 +425,7 @@ async fn cron_cli() -> anyhow::Result<()> {
             .unwrap();
         println!("Waiting for {:?}", t);
         tokio::time::sleep(t).await;
-        auto_cli(AutoArgs {
+        auto_cli(AutoArgs { n_products: None }).await.unwrap();
-            n_products: None,
-            only_supermercado: None,
-        })
-        .await
-        .unwrap();
     }
 }
 

scraper-rs/src/sites/coto.rs

@@ -3,9 +3,7 @@ use futures::{stream, StreamExt, TryFutureExt, TryStreamExt};
 use itertools::Itertools;
 use reqwest::Url;
 
-use crate::{
+use crate::{build_client, do_request, get_retry_policy, retry_if_wasnt_not_found, PrecioPoint};
-    build_client, do_request, get_fetch_retry_policy, retry_if_wasnt_not_found, PrecioPoint,
-};
 
 pub fn parse(url: String, dom: &tl::VDom) -> Result<PrecioPoint, anyhow::Error> {
     let ean = dom
@@ -92,7 +90,7 @@ pub async fn get_urls() -> anyhow::Result<Vec<String>> {
                 .finish();
             let client = &client;
             async move {
-                let text = get_fetch_retry_policy()
+                let text = get_retry_policy()
                     .retry_if(
                         || do_request(client, u.as_str()).and_then(|r| r.text()),
                         retry_if_wasnt_not_found,

scraper-rs/src/sites/farmacity.rs

@@ -1,50 +0,0 @@
-use anyhow::Context;
-use simple_error::bail;
-
-use crate::sites::common;
-use crate::PrecioPoint;
-
-use super::vtex;
-use super::vtex::find_product_ld;
-use super::vtex::AvailabilityLd;
-
-pub fn parse(url: String, dom: &tl::VDom) -> Result<PrecioPoint, anyhow::Error> {
-    let ean = common::get_meta_content(dom, "product:retailer_item_id")
-        .context("Parsing EAN")?
-        .to_string();
-    let precio_centavos = common::price_from_meta(dom)?;
-
-    let (name, image_url, in_stock) = match find_product_ld(dom) {
-        Some(pm) => {
-            let p = pm?;
-            (
-                Some(p.name),
-                Some(p.image),
-                Some(
-                    p.offers.offers.first().context("No offer")?.availability
-                        == AvailabilityLd::InStock,
-                ),
-            )
-        }
-        None => bail!("No JSON/LD"),
-    };
-
-    Ok(PrecioPoint {
-        ean,
-        fetched_at: crate::now_sec(),
-        in_stock,
-        name,
-        image_url,
-        parser_version: 5,
-        precio_centavos,
-        url,
-    })
-}
-
-pub async fn get_urls() -> anyhow::Result<Vec<String>> {
-    let urls = vec![
-        "https://www.farmacity.com/sitemap/product-0.xml",
-        "https://www.farmacity.com/sitemap/product-1.xml",
-    ];
-    vtex::get_urls_from_sitemap(urls).await
-}

scraper-rs/src/sites/jumbo.rs

@@ -7,7 +7,7 @@ use serde::Deserialize;
 use simple_error::bail;
 
 use crate::sites::common;
-use crate::{do_request, get_fetch_retry_policy, PrecioPoint};
+use crate::{do_request, get_retry_policy, PrecioPoint};
 
 use super::vtex;
 
@@ -31,7 +31,7 @@ async fn get_ean_from_search(
         url.set_query(Some(&format!("fq=skuId:{}", retailer_sku)));
         url
     };
-    let s = get_fetch_retry_policy()
+    let s = get_retry_policy()
         .retry(|| do_request(client, url.as_str()).and_then(|r| r.text()))
         .await?;
     let ean = {

scraper-rs/src/sites/mod.rs

@@ -2,6 +2,5 @@ pub mod carrefour;
 mod common;
 pub mod coto;
 pub mod dia;
-pub mod farmacity;
 pub mod jumbo;
 pub mod vtex;

scraper-rs/src/sites/vtex.rs

@@ -10,7 +10,7 @@ use serde_json::json;
 use simple_error::SimpleError;
 use tl::VDom;
 
-use crate::{build_client, do_request, get_fetch_retry_policy, retry_if_wasnt_not_found};
+use crate::{build_client, do_request, get_retry_policy, retry_if_wasnt_not_found};
 
 use super::common;
 
@@ -132,7 +132,7 @@ pub async fn get_urls_from_sitemap(sitemaps: Vec<&str>) -> anyhow::Result<Vec<St
             let url = url.to_string();
             async move {
                 let client = client;
-                let text = get_fetch_retry_policy()
+                let text = get_retry_policy()
                     .retry_if(|| do_request(&client, &url), retry_if_wasnt_not_found)
                     .await?
                     .text()
@@ -152,7 +152,7 @@ pub async fn get_urls_from_sitemap(sitemaps: Vec<&str>) -> anyhow::Result<Vec<St
 }
 
 async fn fetch_body<'a>(client: &reqwest::Client, url: &str) -> anyhow::Result<String> {
-    let body = get_fetch_retry_policy()
+    let body = get_retry_policy()
         .retry_if(|| do_request(client, url), retry_if_wasnt_not_found)
         .await?
         .text()

scraper-rs/src/supermercado.rs

@@ -1,12 +1,11 @@
 use clap::ValueEnum;
 
-#[derive(ValueEnum, Clone, Debug, Copy)]
+#[derive(ValueEnum, Clone, Debug)]
 pub enum Supermercado {
     Dia,
     Jumbo,
     Carrefour,
     Coto,
-    Farmacity,
 }
 impl Supermercado {
     pub fn host(&self) -> &'static str {
@@ -15,7 +14,6 @@ impl Supermercado {
             Self::Carrefour => "www.carrefour.com.ar",
             Self::Coto => "www.cotodigital3.com.ar",
             Self::Jumbo => "www.jumbo.com.ar",
-            Self::Farmacity => "www.farmacity.com",
         }
     }
 }

sitio/src/routes/ean/[ean]/+page.svelte

@@ -1,50 +1,38 @@
 <script lang="ts">
   import { Supermercado, hosts } from "db-datos/supermercado";
-  import * as schema from "db-datos/schema";
   import type { PageData } from "./$types";
   import Chart from "./Chart.svelte";
 
   export let data: PageData;
 
-  let urls: Map<Supermercado, schema.Precio>;
+  let urls: Map<Supermercado, string>;
-  $: urls = data.precios.reduce((prev, curr) => {
+  $: urls = data.precios.toReversed().reduce((prev, curr) => {
     const url = new URL(curr.url);
     const supermercado = hosts[url.hostname];
-    prev.set(supermercado, curr);
+    prev.set(supermercado, curr.url);
     return prev;
-  }, new Map<Supermercado, schema.Precio>());
+  }, new Map<Supermercado, string>());
 
   const classBySupermercado: { [supermercado in Supermercado]: string } = {
     [Supermercado.Dia]: "bg-[#d52b1e] focus:ring-[#d52b1e]",
     [Supermercado.Carrefour]: "bg-[#19549d] focus:ring-[#19549d]",
     [Supermercado.Coto]: "bg-[#e20025] focus:ring-[#e20025]",
     [Supermercado.Jumbo]: "bg-[#2dc850] focus:ring-[#2dc850]",
-    [Supermercado.Farmacity]: "bg-[#EF7603] focus:ring-[#EF7603]",
   };
-
-  const formatter = new Intl.NumberFormat("es-AR", {
-    style: "currency",
-    currency: "ARS",
-  });
 </script>
 
 {#if data.meta}
   <h1 class="text-3xl font-bold">{data.meta.name}</h1>
   <img src={data.meta.imageUrl} alt={data.meta.name} class="max-h-48" />
   <div class="flex gap-2">
-    {#each urls as [supermercado, { url, precioCentavos }]}
+    {#each urls as [supermercado, url]}
       <a
         href={url}
         rel="noreferrer noopener"
         target="_blank"
-        class={`focus:shadow-outline inline-flex flex-col items-center justify-center rounded-md ${classBySupermercado[supermercado]} px-4 py-2 font-medium tracking-wide text-white transition-colors duration-200 hover:bg-opacity-80 focus:outline-none focus:ring-2 focus:ring-offset-2`}
+        class={`focus:shadow-outline inline-flex items-center justify-center rounded-md ${classBySupermercado[supermercado]} px-4 py-2 text-sm font-medium tracking-wide text-white transition-colors duration-200 hover:bg-opacity-80 focus:outline-none focus:ring-2 focus:ring-offset-2`}
       >
-        {#if precioCentavos}
+        Ver en {supermercado}
-          <span class="text-lg font-bold"
-            >{formatter.format(precioCentavos / 100)}</span
-          >
-        {/if}
-        <span class="text-sm">{supermercado}</span>
       </a>
     {/each}
   </div>