Nulo/preciazo
1
0
Fork 0
mirror of https://github.com/catdevnull/preciazo.git synced 2024-11-25 19:16:19 +00:00
Code Issues Projects Releases Packages Wiki Activity

no reintentar si no se encontró

Browse source
This commit is contained in:
Cat /dev/Nulo 2024-01-12 17:24:00 -03:00
parent a697f5cb0c
commit 26b9f4b17f
3 changed files with 12 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
scraper-rs/src/main.rs
View file

@ -178,13 +178,17 @@ pub fn get_retry_policy() -> again::RetryPolicy {
.with_jitter(true) .with_jitter(true)
} }
pub fn retry_if_wasnt_not_found(err: &reqwest::Error) -> bool {
!err.status().is_some_and(|s| s == StatusCode::NOT_FOUND)
}
#[tracing::instrument(skip(client))] #[tracing::instrument(skip(client))]
async fn fetch_and_parse( async fn fetch_and_parse(
client: &reqwest::Client, client: &reqwest::Client,
url: String, url: String,
) -> Result<PrecioPoint, anyhow::Error> { ) -> Result<PrecioPoint, anyhow::Error> {
let body = get_retry_policy() let body = get_retry_policy()
.retry(|| do_request(client, &url)) .retry_if(|| do_request(client, &url), retry_if_wasnt_not_found)
.await? .await?
.text() .text()
.await .await

7
scraper-rs/src/sites/coto.rs
View file

@ -3,7 +3,7 @@ use futures::{stream, StreamExt, TryFutureExt, TryStreamExt};
use itertools::Itertools; use itertools::Itertools;
use reqwest::Url; use reqwest::Url;
use crate::{build_client, do_request, get_retry_policy, PrecioPoint}; use crate::{build_client, do_request, get_retry_policy, retry_if_wasnt_not_found, PrecioPoint};
pub fn parse(url: String, dom: &tl::VDom) -> Result<PrecioPoint, anyhow::Error> { pub fn parse(url: String, dom: &tl::VDom) -> Result<PrecioPoint, anyhow::Error> {
let ean = dom let ean = dom
@ -90,7 +90,10 @@ pub async fn get_urls() -> anyhow::Result<Vec<String>> {
let client = &client; let client = &client;
async move { async move {
let text = get_retry_policy() let text = get_retry_policy()
.retry(|| do_request(client, u.as_str()).and_then(|r| r.text())) .retry_if(
|| do_request(client, u.as_str()).and_then(|r| r.text()),
retry_if_wasnt_not_found,
)
.await?; .await?;
let dom = tl::parse(&text, tl::ParserOptions::default())?; let dom = tl::parse(&text, tl::ParserOptions::default())?;

4
scraper-rs/src/sites/vtex.rs
View file

@ -5,7 +5,7 @@ use serde::Deserialize;
use simple_error::SimpleError; use simple_error::SimpleError;
use tl::VDom; use tl::VDom;
use crate::{build_client, do_request, get_retry_policy}; use crate::{build_client, do_request, get_retry_policy, retry_if_wasnt_not_found};
use super::common; use super::common;
@ -128,7 +128,7 @@ pub async fn get_urls_from_sitemap(sitemaps: Vec<&str>) -> anyhow::Result<Vec<St
let client = client; let client = client;
let url = url; let url = url;
let text = get_retry_policy() let text = get_retry_policy()
.retry(|| do_request(&client, &url)) .retry_if(|| do_request(&client, &url), retry_if_wasnt_not_found)
.await? .await?
.text() .text()
.await?; .await?;