Nulo/preciazo
1
0
Fork 0
mirror of https://github.com/catdevnull/preciazo.git synced 2024-11-22 14:16:19 +00:00
Code Issues Projects Releases Packages Wiki Activity

reducir timeouts y subir retries

Browse source
This commit is contained in:
Cat /dev/Nulo 2024-02-06 16:20:05 -03:00
parent c53897891b
commit f6429d09bc
1 changed files with 4 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
scraper-rs/src/main.rs
View file

@ -165,8 +165,8 @@ fn build_client() -> reqwest::Client {
let mut headers = HeaderMap::new(); let mut headers = HeaderMap::new();
headers.append("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36".parse().unwrap()); headers.append("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36".parse().unwrap());
reqwest::ClientBuilder::default() reqwest::ClientBuilder::default()
.timeout(Duration::from_secs(60 * 5)) .timeout(Duration::from_secs(30))
.connect_timeout(Duration::from_secs(60)) .connect_timeout(Duration::from_secs(30))
.default_headers(headers) .default_headers(headers)
.build() .build()
.unwrap() .unwrap()
@ -179,7 +179,7 @@ pub async fn do_request(client: &reqwest::Client, url: &str) -> reqwest::Result<
pub fn get_retry_policy() -> again::RetryPolicy { pub fn get_retry_policy() -> again::RetryPolicy {
RetryPolicy::exponential(Duration::from_millis(300)) RetryPolicy::exponential(Duration::from_millis(300))
.with_max_retries(10) .with_max_retries(20)
.with_jitter(true) .with_jitter(true)
} }
@ -378,6 +378,7 @@ async fn auto_cli() -> anyhow::Result<()> {
.map(|s| tokio::spawn(auto.clone().download_supermercado(s.to_owned()))) .map(|s| tokio::spawn(auto.clone().download_supermercado(s.to_owned())))
.collect(); .collect();
future::try_join_all(handles).await?; future::try_join_all(handles).await?;
auto.inform("[auto] Download supermercados finished").await;
let best_selling = auto let best_selling = auto
.inform_time( .inform_time(