Nulo/preciazo
1
0
Fork 0
mirror of https://github.com/catdevnull/preciazo.git synced 2024-11-22 14:16:19 +00:00
Code Issues Projects Releases Packages Wiki Activity

solo usar últimos 60 días de urls

Browse source
This commit is contained in:
Cat /dev/Nulo 2024-07-07 13:03:07 -03:00
parent 1230f34aca
commit abf501bd5e
2 changed files with 21 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
rust/src/scraper/auto.rs
View file

@ -41,7 +41,10 @@ impl Auto {
}
}
let links: Vec<String> = {
let mut links = self.db.get_urls_by_domain(supermercado.host()).await?;
let mut links = self
.db
.get_recent_urls_by_domain(supermercado.host())
.await?;
if let Some(n) = self.args.n_products {
links.truncate(n);
}

26
rust/src/scraper/db.rs
View file

@ -49,16 +49,21 @@ impl Db {
.map(|r| r.ean))
}
pub async fn get_urls_by_domain(&self, domain: &str) -> anyhow::Result<Vec<String>> {
pub async fn get_recent_urls_by_domain(&self, domain: &str) -> anyhow::Result<Vec<String>> {
let query = format!("%{}%", domain);
Ok(
sqlx::query!("SELECT url FROM producto_urls WHERE url LIKE ?1;", query)
.fetch_all(&self.read_pool)
.await?
.into_iter()
.map(|r| r.url)
.collect(),
let last_60_days: i64 = (now() - Duration::from_secs(60 * 60 * 24 * 60))
.as_millis()
.try_into()?;
Ok(sqlx::query!(
"SELECT url FROM producto_urls WHERE url LIKE ?1 AND last_seen > ?2;",
query,
last_60_days
)
.fetch_all(&self.read_pool)
.await?
.into_iter()
.map(|r| r.url)
.collect())
}
pub async fn save_producto_urls(&self, urls: Vec<String>) -> anyhow::Result<()> {
@ -122,8 +127,11 @@ async fn connect_to_db(
}
fn now_ms() -> u128 {
now().as_millis()
}
fn now() -> Duration {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.expect("Time went backwards")
.as_millis()
}