Nulo/preciazo
1
0
Fork 0
mirror of https://github.com/catdevnull/preciazo.git synced 2024-11-26 03:26:19 +00:00
Code Issues Projects Releases Packages Wiki Activity

probemos así

Browse source
This commit is contained in:
Cat /dev/Nulo 2024-01-12 16:21:20 -03:00
parent 75b2297a07
commit 6b315f9af2
1 changed files with 50 additions and 55 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

59
scraper-rs/src/main.rs
View file

@ -1,7 +1,6 @@
use again::RetryPolicy; use again::RetryPolicy;
use async_channel::Receiver;
use clap::{Parser, ValueEnum}; use clap::{Parser, ValueEnum};
use futures::future; use futures::{future, stream, StreamExt};
use nanoid::nanoid; use nanoid::nanoid;
use r2d2::Pool; use r2d2::Pool;
use r2d2_sqlite::SqliteConnectionManager; use r2d2_sqlite::SqliteConnectionManager;
@ -78,32 +77,30 @@ async fn fetch_list_cli(links_list_path: String) -> anyhow::Result<()> {
} }
async fn fetch_list(pool: &Pool<SqliteConnectionManager>, links: Vec<String>) -> Counters { async fn fetch_list(pool: &Pool<SqliteConnectionManager>, links: Vec<String>) -> Counters {
let (sender, receiver) = async_channel::bounded::<String>(1);
let n_coroutines = env::var("N_COROUTINES") let n_coroutines = env::var("N_COROUTINES")
.map_or(Ok(24), |s| s.parse::<usize>()) .map_or(Ok(24), |s| s.parse::<usize>())
.expect("N_COROUTINES no es un número"); .expect("N_COROUTINES no es un número");
let handles = (1..n_coroutines)
.map(|_| { let client = build_client();
let rx = receiver.clone();
stream::iter(links)
.map(|url| {
let pool = pool.clone(); let pool = pool.clone();
tokio::spawn(worker(rx, pool)) let client = client.clone();
tokio::spawn(fetch_and_save(client, url, pool))
}) })
.collect::<Vec<_>>(); // https://github.com/rust-lang/rust/issues/89976#issuecomment-1073115246
.boxed()
for link in links { .buffer_unordered(n_coroutines)
sender.send_blocking(link).unwrap(); .fold(Counters::default(), move |x, y| {
} let ret = y.unwrap();
sender.close(); future::ready(Counters {
success: x.success + ret.success,
let mut counters = Counters::default(); errored: x.errored + ret.errored,
for handle in handles { skipped: x.skipped + ret.skipped,
let c = handle.await.unwrap(); })
counters.success += c.success; })
counters.errored += c.errored; .await
counters.skipped += c.skipped;
}
counters
} }
fn connect_db() -> Pool<SqliteConnectionManager> { fn connect_db() -> Pool<SqliteConnectionManager> {
@ -124,13 +121,13 @@ struct Counters {
skipped: u64, skipped: u64,
} }
async fn worker(rx: Receiver<String>, pool: Pool<SqliteConnectionManager>) -> Counters { async fn fetch_and_save(
let client = build_client(); client: reqwest::Client,
url: String,
pool: Pool<SqliteConnectionManager>,
) -> Counters {
let res = fetch_and_parse(&client, url.clone()).await;
let mut counters = Counters::default(); let mut counters = Counters::default();
while let Ok(url) = rx.recv().await {
let client = &client;
let res = fetch_and_parse(client, url.clone()).await;
match res { match res {
Ok(res) => { Ok(res) => {
counters.success += 1; counters.success += 1;
@ -158,8 +155,6 @@ async fn worker(rx: Receiver<String>, pool: Pool<SqliteConnectionManager>) -> Co
tracing::error!(error=%err, url=url); tracing::error!(error=%err, url=url);
} }
} }
}
counters counters
} }
@ -370,8 +365,8 @@ async fn cron_cli() -> anyhow::Result<()> {
let mut interval = time::interval(std::time::Duration::from_secs(60 * 60 * 24)); let mut interval = time::interval(std::time::Duration::from_secs(60 * 60 * 24));
loop { loop {
interval.tick().await;
auto_cli().await.unwrap(); auto_cli().await.unwrap();
interval.tick().await;
} }
} }