Compare commits

..

No commits in common. "eca98c616ec79a032a328cc5ffa1de8601ba6f9f" and "ed7af7621d9ff2b46bf9a01ff70acaa87cdbf721" have entirely different histories.

10 changed files with 43 additions and 150 deletions

View file

@ -9,7 +9,7 @@ import { migrateDb } from "./migrate.js";
let db = null;
export function getDb() {
const sqlite = new Database(DB_PATH);
db = drizzle(sqlite, { schema, logger: true });
db = drizzle(sqlite, { schema });
migrateDb(db);
return db;
}

View file

@ -3,31 +3,25 @@ export enum Supermercado {
Carrefour = "Carrefour",
Coto = "Coto",
Jumbo = "Jumbo",
Farmacity = "Farmacity",
}
export const supermercados: Supermercado[] = [
Supermercado.Carrefour,
Supermercado.Coto,
Supermercado.Dia,
Supermercado.Jumbo,
Supermercado.Farmacity,
];
export const hosts: { [host: string]: Supermercado } = {
"diaonline.supermercadosdia.com.ar": Supermercado.Dia,
"www.carrefour.com.ar": Supermercado.Carrefour,
"www.cotodigital3.com.ar": Supermercado.Coto,
"www.jumbo.com.ar": Supermercado.Jumbo,
"www.farmacity.com": Supermercado.Farmacity,
};
export const hostBySupermercado = Object.fromEntries(
Object.entries(hosts).map(([a, b]) => [b, a])
) as Record<Supermercado, string>;
// también actualizar en sitio/src/routes/ean/[ean]/+page.svelte
export const colorBySupermercado: { [supermercado in Supermercado]: string } = {
[Supermercado.Dia]: "#d52b1e",
[Supermercado.Carrefour]: "#19549d",
[Supermercado.Coto]: "#e20025",
[Supermercado.Jumbo]: "#2dc850",
[Supermercado.Farmacity]: "#EF7603",
};

View file

@ -46,12 +46,10 @@ struct GetUrlListArgs {
struct ScrapUrlArgs {
url: String,
}
#[derive(clap::Args, Clone, Copy)]
#[derive(clap::Args)]
struct AutoArgs {
#[arg(long)]
n_products: Option<usize>,
#[arg(long)]
only_supermercado: Option<Supermercado>,
}
#[tokio::main]
@ -186,25 +184,18 @@ async fn request_and_body(client: &reqwest::Client, url: &str) -> reqwest::Resul
res.text().await
}
pub async fn fetch_body(client: &reqwest::Client, url: &str) -> reqwest::Result<String> {
get_fetch_retry_policy()
get_retry_policy()
.retry_if(|| request_and_body(client, url), retry_if_wasnt_not_found)
.await
}
pub fn get_fetch_retry_policy() -> again::RetryPolicy {
pub fn get_retry_policy() -> again::RetryPolicy {
RetryPolicy::exponential(Duration::from_millis(300))
.with_max_retries(20)
.with_max_delay(Duration::from_secs(40))
.with_jitter(true)
}
pub fn get_parse_retry_policy() -> again::RetryPolicy {
RetryPolicy::exponential(Duration::from_millis(1500))
.with_max_retries(5)
.with_max_delay(Duration::from_secs(5))
.with_jitter(true)
}
pub fn retry_if_wasnt_not_found(err: &reqwest::Error) -> bool {
!err.status().is_some_and(|s| s == StatusCode::NOT_FOUND)
}
@ -214,38 +205,24 @@ async fn fetch_and_parse(
client: &reqwest::Client,
url: String,
) -> Result<PrecioPoint, anyhow::Error> {
async fn fetch_and_scrap(
client: &reqwest::Client,
url: String,
) -> Result<PrecioPoint, anyhow::Error> {
let body = fetch_body(client, &url).await?;
let maybe_point = { scrap_url(client, url, &body).await };
let body = fetch_body(client, &url).await?;
let point = match maybe_point {
Ok(p) => Ok(p),
Err(err) => {
let now: DateTime<Utc> = Utc::now();
let debug_path = PathBuf::from(format!("debug-{}/", now.format("%Y-%m-%d")));
tokio::fs::create_dir_all(&debug_path).await.unwrap();
let file_path = debug_path.join(format!("{}.html", nanoid!()));
tokio::fs::write(&file_path, &body).await.unwrap();
tracing::debug!(error=%err, "Failed to parse, saved body at {}",file_path.display());
Err(err)
}
}?;
let maybe_point = { scrap_url(client, url, &body).await };
Ok(point)
}
let point = match maybe_point {
Ok(p) => Ok(p),
Err(err) => {
let now: DateTime<Utc> = Utc::now();
let debug_path = PathBuf::from(format!("debug-{}/", now.format("%Y-%m-%d")));
tokio::fs::create_dir_all(&debug_path).await.unwrap();
let file_path = debug_path.join(format!("{}.html", nanoid!()));
tokio::fs::write(&file_path, &body).await.unwrap();
tracing::debug!(error=%err, "Failed to parse, saved body at {}",file_path.display());
Err(err)
}
}?;
get_parse_retry_policy()
.retry_if(
|| fetch_and_scrap(client, url.clone()),
|err: &anyhow::Error| match err.downcast_ref::<reqwest::Error>() {
Some(e) => !e.status().is_some_and(|s| s == StatusCode::NOT_FOUND),
None => true,
},
)
.await
Ok(point)
}
async fn parse_file_cli(file_path: String) -> anyhow::Result<()> {
@ -285,7 +262,6 @@ async fn get_urls(supermercado: &Supermercado) -> Result<Vec<String>, anyhow::Er
Supermercado::Jumbo => sites::jumbo::get_urls().await?,
Supermercado::Carrefour => sites::carrefour::get_urls().await?,
Supermercado::Coto => sites::coto::get_urls().await?,
Supermercado::Farmacity => sites::farmacity::get_urls().await?,
})
}
@ -306,9 +282,6 @@ async fn scrap_url(
sites::coto::parse(url, &tl::parse(body, tl::ParserOptions::default())?)
}
"www.jumbo.com.ar" => sites::jumbo::scrap(client, url, body).await,
"www.farmacity.com" => {
sites::farmacity::parse(url, &tl::parse(body, tl::ParserOptions::default())?)
}
s => bail!("Unknown host {}", s),
}
}
@ -323,7 +296,7 @@ struct AutoTelegram {
struct Auto {
db: Db,
telegram: Option<AutoTelegram>,
args: AutoArgs,
limit_n_products: Option<usize>,
}
impl Auto {
async fn download_supermercado(self, supermercado: Supermercado) -> anyhow::Result<()> {
@ -339,7 +312,7 @@ impl Auto {
}
let links: Vec<String> = {
let mut links = self.db.get_urls_by_domain(supermercado.host()).await?;
if let Some(n) = self.args.n_products {
if let Some(n) = self.limit_n_products {
links.truncate(n);
}
links
@ -413,16 +386,14 @@ async fn auto_cli(args: AutoArgs) -> anyhow::Result<()> {
}
}
};
Auto { db, telegram, args }
Auto {
db,
telegram,
limit_n_products: args.n_products,
}
};
auto.inform("[auto] Empezando scrap").await;
let supermercados = match args.only_supermercado {
Some(supermercado) => [supermercado].to_vec(),
None => Supermercado::value_variants().to_vec(),
};
let handles: Vec<_> = supermercados
let handles: Vec<_> = Supermercado::value_variants()
.iter()
.map(|s| tokio::spawn(auto.clone().download_supermercado(s.to_owned())))
.collect();
@ -454,12 +425,7 @@ async fn cron_cli() -> anyhow::Result<()> {
.unwrap();
println!("Waiting for {:?}", t);
tokio::time::sleep(t).await;
auto_cli(AutoArgs {
n_products: None,
only_supermercado: None,
})
.await
.unwrap();
auto_cli(AutoArgs { n_products: None }).await.unwrap();
}
}

View file

@ -3,9 +3,7 @@ use futures::{stream, StreamExt, TryFutureExt, TryStreamExt};
use itertools::Itertools;
use reqwest::Url;
use crate::{
build_client, do_request, get_fetch_retry_policy, retry_if_wasnt_not_found, PrecioPoint,
};
use crate::{build_client, do_request, get_retry_policy, retry_if_wasnt_not_found, PrecioPoint};
pub fn parse(url: String, dom: &tl::VDom) -> Result<PrecioPoint, anyhow::Error> {
let ean = dom
@ -92,7 +90,7 @@ pub async fn get_urls() -> anyhow::Result<Vec<String>> {
.finish();
let client = &client;
async move {
let text = get_fetch_retry_policy()
let text = get_retry_policy()
.retry_if(
|| do_request(client, u.as_str()).and_then(|r| r.text()),
retry_if_wasnt_not_found,

View file

@ -1,50 +0,0 @@
use anyhow::Context;
use simple_error::bail;
use crate::sites::common;
use crate::PrecioPoint;
use super::vtex;
use super::vtex::find_product_ld;
use super::vtex::AvailabilityLd;
pub fn parse(url: String, dom: &tl::VDom) -> Result<PrecioPoint, anyhow::Error> {
let ean = common::get_meta_content(dom, "product:retailer_item_id")
.context("Parsing EAN")?
.to_string();
let precio_centavos = common::price_from_meta(dom)?;
let (name, image_url, in_stock) = match find_product_ld(dom) {
Some(pm) => {
let p = pm?;
(
Some(p.name),
Some(p.image),
Some(
p.offers.offers.first().context("No offer")?.availability
== AvailabilityLd::InStock,
),
)
}
None => bail!("No JSON/LD"),
};
Ok(PrecioPoint {
ean,
fetched_at: crate::now_sec(),
in_stock,
name,
image_url,
parser_version: 5,
precio_centavos,
url,
})
}
pub async fn get_urls() -> anyhow::Result<Vec<String>> {
let urls = vec![
"https://www.farmacity.com/sitemap/product-0.xml",
"https://www.farmacity.com/sitemap/product-1.xml",
];
vtex::get_urls_from_sitemap(urls).await
}

View file

@ -7,7 +7,7 @@ use serde::Deserialize;
use simple_error::bail;
use crate::sites::common;
use crate::{do_request, get_fetch_retry_policy, PrecioPoint};
use crate::{do_request, get_retry_policy, PrecioPoint};
use super::vtex;
@ -31,7 +31,7 @@ async fn get_ean_from_search(
url.set_query(Some(&format!("fq=skuId:{}", retailer_sku)));
url
};
let s = get_fetch_retry_policy()
let s = get_retry_policy()
.retry(|| do_request(client, url.as_str()).and_then(|r| r.text()))
.await?;
let ean = {

View file

@ -2,6 +2,5 @@ pub mod carrefour;
mod common;
pub mod coto;
pub mod dia;
pub mod farmacity;
pub mod jumbo;
pub mod vtex;

View file

@ -10,7 +10,7 @@ use serde_json::json;
use simple_error::SimpleError;
use tl::VDom;
use crate::{build_client, do_request, get_fetch_retry_policy, retry_if_wasnt_not_found};
use crate::{build_client, do_request, get_retry_policy, retry_if_wasnt_not_found};
use super::common;
@ -132,7 +132,7 @@ pub async fn get_urls_from_sitemap(sitemaps: Vec<&str>) -> anyhow::Result<Vec<St
let url = url.to_string();
async move {
let client = client;
let text = get_fetch_retry_policy()
let text = get_retry_policy()
.retry_if(|| do_request(&client, &url), retry_if_wasnt_not_found)
.await?
.text()
@ -152,7 +152,7 @@ pub async fn get_urls_from_sitemap(sitemaps: Vec<&str>) -> anyhow::Result<Vec<St
}
async fn fetch_body<'a>(client: &reqwest::Client, url: &str) -> anyhow::Result<String> {
let body = get_fetch_retry_policy()
let body = get_retry_policy()
.retry_if(|| do_request(client, url), retry_if_wasnt_not_found)
.await?
.text()

View file

@ -1,12 +1,11 @@
use clap::ValueEnum;
#[derive(ValueEnum, Clone, Debug, Copy)]
#[derive(ValueEnum, Clone, Debug)]
pub enum Supermercado {
Dia,
Jumbo,
Carrefour,
Coto,
Farmacity,
}
impl Supermercado {
pub fn host(&self) -> &'static str {
@ -15,7 +14,6 @@ impl Supermercado {
Self::Carrefour => "www.carrefour.com.ar",
Self::Coto => "www.cotodigital3.com.ar",
Self::Jumbo => "www.jumbo.com.ar",
Self::Farmacity => "www.farmacity.com",
}
}
}

View file

@ -1,50 +1,38 @@
<script lang="ts">
import { Supermercado, hosts } from "db-datos/supermercado";
import * as schema from "db-datos/schema";
import type { PageData } from "./$types";
import Chart from "./Chart.svelte";
export let data: PageData;
let urls: Map<Supermercado, schema.Precio>;
$: urls = data.precios.reduce((prev, curr) => {
let urls: Map<Supermercado, string>;
$: urls = data.precios.toReversed().reduce((prev, curr) => {
const url = new URL(curr.url);
const supermercado = hosts[url.hostname];
prev.set(supermercado, curr);
prev.set(supermercado, curr.url);
return prev;
}, new Map<Supermercado, schema.Precio>());
}, new Map<Supermercado, string>());
const classBySupermercado: { [supermercado in Supermercado]: string } = {
[Supermercado.Dia]: "bg-[#d52b1e] focus:ring-[#d52b1e]",
[Supermercado.Carrefour]: "bg-[#19549d] focus:ring-[#19549d]",
[Supermercado.Coto]: "bg-[#e20025] focus:ring-[#e20025]",
[Supermercado.Jumbo]: "bg-[#2dc850] focus:ring-[#2dc850]",
[Supermercado.Farmacity]: "bg-[#EF7603] focus:ring-[#EF7603]",
};
const formatter = new Intl.NumberFormat("es-AR", {
style: "currency",
currency: "ARS",
});
</script>
{#if data.meta}
<h1 class="text-3xl font-bold">{data.meta.name}</h1>
<img src={data.meta.imageUrl} alt={data.meta.name} class="max-h-48" />
<div class="flex gap-2">
{#each urls as [supermercado, { url, precioCentavos }]}
{#each urls as [supermercado, url]}
<a
href={url}
rel="noreferrer noopener"
target="_blank"
class={`focus:shadow-outline inline-flex flex-col items-center justify-center rounded-md ${classBySupermercado[supermercado]} px-4 py-2 font-medium tracking-wide text-white transition-colors duration-200 hover:bg-opacity-80 focus:outline-none focus:ring-2 focus:ring-offset-2`}
class={`focus:shadow-outline inline-flex items-center justify-center rounded-md ${classBySupermercado[supermercado]} px-4 py-2 text-sm font-medium tracking-wide text-white transition-colors duration-200 hover:bg-opacity-80 focus:outline-none focus:ring-2 focus:ring-offset-2`}
>
{#if precioCentavos}
<span class="text-lg font-bold"
>{formatter.format(precioCentavos / 100)}</span
>
{/if}
<span class="text-sm">{supermercado}</span>
Ver en {supermercado}
</a>
{/each}
</div>