Nulo/preciazo
1
0
Fork 0
mirror of https://github.com/catdevnull/preciazo.git synced 2024-11-22 22:26:19 +00:00
Code Issues Projects Releases Packages Wiki Activity

para testear: solo fetchear un supermercado

Browse source
This commit is contained in:
Cat /dev/Nulo 2024-02-06 22:04:05 -03:00
parent ed7af7621d
commit d55fc8f603
1 changed files with 19 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

29
scraper-rs/src/main.rs
View file

@ -46,10 +46,12 @@ struct GetUrlListArgs {
struct ScrapUrlArgs { struct ScrapUrlArgs {
url: String, url: String,
} }
#[derive(clap::Args)] #[derive(clap::Args, Clone, Copy)]
struct AutoArgs { struct AutoArgs {
#[arg(long)] #[arg(long)]
n_products: Option<usize>, n_products: Option<usize>,
#[arg(long)]
only_supermercado: Option<Supermercado>,
} }
#[tokio::main] #[tokio::main]
@ -296,7 +298,7 @@ struct AutoTelegram {
struct Auto { struct Auto {
db: Db, db: Db,
telegram: Option<AutoTelegram>, telegram: Option<AutoTelegram>,
limit_n_products: Option<usize>, args: AutoArgs,
} }
impl Auto { impl Auto {
async fn download_supermercado(self, supermercado: Supermercado) -> anyhow::Result<()> { async fn download_supermercado(self, supermercado: Supermercado) -> anyhow::Result<()> {
@ -312,7 +314,7 @@ impl Auto {
} }
let links: Vec<String> = { let links: Vec<String> = {
let mut links = self.db.get_urls_by_domain(supermercado.host()).await?; let mut links = self.db.get_urls_by_domain(supermercado.host()).await?;
if let Some(n) = self.limit_n_products { if let Some(n) = self.args.n_products {
links.truncate(n); links.truncate(n);
} }
links links
@ -386,14 +388,16 @@ async fn auto_cli(args: AutoArgs) -> anyhow::Result<()> {
} }
} }
}; };
Auto { Auto { db, telegram, args }
db,
telegram,
limit_n_products: args.n_products,
}
}; };
auto.inform("[auto] Empezando scrap").await; auto.inform("[auto] Empezando scrap").await;
let handles: Vec<_> = Supermercado::value_variants()
let supermercados = match args.only_supermercado {
Some(supermercado) => [supermercado].to_vec(),
None => Supermercado::value_variants().to_vec(),
};
let handles: Vec<_> = supermercados
.iter() .iter()
.map(|s| tokio::spawn(auto.clone().download_supermercado(s.to_owned()))) .map(|s| tokio::spawn(auto.clone().download_supermercado(s.to_owned())))
.collect(); .collect();
@ -425,7 +429,12 @@ async fn cron_cli() -> anyhow::Result<()> {
.unwrap(); .unwrap();
println!("Waiting for {:?}", t); println!("Waiting for {:?}", t);
tokio::time::sleep(t).await; tokio::time::sleep(t).await;
auto_cli(AutoArgs { n_products: None }).await.unwrap(); auto_cli(AutoArgs {
n_products: None,
only_supermercado: None,
})
.await
.unwrap();
} }
} }