Nulo/preciazo
1
0
Fork 0
mirror of https://github.com/catdevnull/preciazo.git synced 2024-11-22 14:16:19 +00:00
Code Issues Projects Releases Packages Wiki Activity

solo usar últimos 60 días de urls

Browse source
This commit is contained in:
Cat /dev/Nulo 2024-07-07 13:03:07 -03:00
parent 1230f34aca
commit abf501bd5e
2 changed files with 21 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
rust/src/scraper/auto.rs
View file

@ -41,7 +41,10 @@ impl Auto {
} }
} }
let links: Vec<String> = { let links: Vec<String> = {
let mut links = self.db.get_urls_by_domain(supermercado.host()).await?; let mut links = self
.db
.get_recent_urls_by_domain(supermercado.host())
.await?;
if let Some(n) = self.args.n_products { if let Some(n) = self.args.n_products {
links.truncate(n); links.truncate(n);
} }

26
rust/src/scraper/db.rs
View file

@ -49,16 +49,21 @@ impl Db {
.map(|r| r.ean)) .map(|r| r.ean))
} }
pub async fn get_urls_by_domain(&self, domain: &str) -> anyhow::Result<Vec<String>> { pub async fn get_recent_urls_by_domain(&self, domain: &str) -> anyhow::Result<Vec<String>> {
let query = format!("%{}%", domain); let query = format!("%{}%", domain);
Ok( let last_60_days: i64 = (now() - Duration::from_secs(60 * 60 * 24 * 60))
sqlx::query!("SELECT url FROM producto_urls WHERE url LIKE ?1;", query) .as_millis()
.fetch_all(&self.read_pool) .try_into()?;
.await? Ok(sqlx::query!(
.into_iter() "SELECT url FROM producto_urls WHERE url LIKE ?1 AND last_seen > ?2;",
.map(|r| r.url) query,
.collect(), last_60_days
) )
.fetch_all(&self.read_pool)
.await?
.into_iter()
.map(|r| r.url)
.collect())
} }
pub async fn save_producto_urls(&self, urls: Vec<String>) -> anyhow::Result<()> { pub async fn save_producto_urls(&self, urls: Vec<String>) -> anyhow::Result<()> {
@ -122,8 +127,11 @@ async fn connect_to_db(
} }
fn now_ms() -> u128 { fn now_ms() -> u128 {
now().as_millis()
}
fn now() -> Duration {
SystemTime::now() SystemTime::now()
.duration_since(UNIX_EPOCH) .duration_since(UNIX_EPOCH)
.expect("Time went backwards") .expect("Time went backwards")
.as_millis()
} }