Nulo/preciazo
1
0
Fork 0
mirror of https://github.com/catdevnull/preciazo.git synced 2024-11-26 11:36:20 +00:00
Code Issues Projects Releases Packages Wiki Activity

solo pedir urls del supermercado siendo fetcheado

Browse source
This commit is contained in:
Cat /dev/Nulo 2024-01-12 18:09:30 -03:00
parent c7a578d8cc
commit 1108951c79
1 changed files with 27 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

31
scraper-rs/src/main.rs
View file

@ -21,6 +21,16 @@ enum Supermercado {
Carrefour, Carrefour,
Coto, Coto,
} }
impl Supermercado {
fn host(self: &Self) -> &'static str {
match self {
Self::Dia => "diaonline.supermercadosdia.com.ar",
Self::Carrefour => "www.carrefour.com.ar",
Self::Coto => "www.cotodigital3.com.ar",
Self::Jumbo => "www.jumbo.com.ar",
}
}
}
#[derive(Parser)] // requires `derive` feature #[derive(Parser)] // requires `derive` feature
enum Args { enum Args {
@ -126,7 +136,7 @@ async fn fetch_and_save(client: reqwest::Client, url: String, pool: Pool) -> Cou
counters.success += 1; counters.success += 1;
pool.get().await.unwrap().interact(move |conn| conn.execute( pool.get().await.unwrap().interact(move |conn| conn.execute(
"INSERT INTO precios(ean, fetched_at, precio_centavos, in_stock, url, warc_record_id, parser_version, name, image_url) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9);", "INSERT INTO precios(ean, fetched_at, precio_centavos, in_stock, url, warc_record_id, parser_version, name, image_url) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9);",
rusqlite::params![ rusqlite::params![
res.ean, res.ean,
res.fetched_at, res.fetched_at,
res.precio_centavos, res.precio_centavos,
@ -289,19 +299,32 @@ impl Auto {
.await; .await;
} }
let links: Vec<String> = { let links: Vec<String> = {
let search = format!("%{}%", supermercado.host());
self.pool self.pool
.get() .get()
.await? .await?
.interact(|conn| -> anyhow::Result<Vec<String>> { .interact(move |conn| -> anyhow::Result<Vec<String>> {
Ok(conn Ok(conn
.prepare(r#"SELECT url FROM producto_urls;"#)? .prepare(
.query_map([], |r| r.get::<_, String>(0))? r#"SELECT url FROM producto_urls
WHERE url LIKE ?1;"#,
)?
.query_map(rusqlite::params![search], |r| r.get::<_, String>(0))?
.map(|r| r.unwrap()) .map(|r| r.unwrap())
.collect()) .collect())
}) })
.await .await
.unwrap()? .unwrap()?
}; };
// {
// let debug_path = PathBuf::from("debug/");
// tokio::fs::create_dir_all(&debug_path).await.unwrap();
// let file_path = debug_path.join(format!("{}.txt", nanoid!()));
// tokio::fs::write(&file_path, &links.join("\n"))
// .await
// .unwrap();
// tracing::info!("Lista de {:?}: {:?}", &supermercado, file_path.display());
// }
{ {
let t0 = now_sec(); let t0 = now_sec();
let counters = fetch_list(&self.pool, links).await; let counters = fetch_list(&self.pool, links).await;