Nulo/preciazo
1
0
Fork 0
mirror of https://github.com/catdevnull/preciazo.git synced 2024-11-26 03:26:19 +00:00
Code Issues Projects Releases Packages Wiki Activity

ignore forbidden errors

Browse source 
some Carrefour pages return forbidden for a few products

2024-07-06T11:32:47.059230Z ERROR scraper::scraper: error=HTTP status client error (403 Forbidden) for url (https://www.carrefour.com.ar/apple-macbook-pro-14--m1-pro-chip-10%E2%80%91core-cpu-16%E2%80%91core-gpu---1tb-ssd---silver--1656560/p) url="https://www.carrefour.com.ar/apple-macbook-pro-14--m1-pro-chip-10‑core-cpu-16‑core-gpu---1tb-ssd---silver--1656560/p"
2024-07-06T11:33:07.881310Z ERROR scraper::scraper: error=HTTP status client error (403 Forbidden) for url (https://www.carrefour.com.ar/apple-macbook-pro-14--m1-pro-chip-10%E2%80%91core-cpu-16%E2%80%91core-gpu---1tb-ssd---space-grey-1657067/p) url="https://www.carrefour.com.ar/apple-macbook-pro-14--m1-pro-chip-10‑core-cpu-16‑core-gpu---1tb-ssd---space-grey-1657067/p"
2024-07-06T11:33:56.483980Z ERROR scraper::scraper: error=HTTP status client error (403 Forbidden) for url (https://www.carrefour.com.ar/apple-macbook-pro-16--m1-pro-chip-10%E2%80%91core-cpu-16%E2%80%91core-gpu---1tb-ssd---silver--1656993/p) url="https://www.carrefour.com.ar/apple-macbook-pro-16--m1-pro-chip-10‑core-cpu-16‑core-gpu---1tb-ssd---silver--1656993/p"
2024-07-06T11:34:16.992339Z ERROR scraper::scraper: error=HTTP status client error (403 Forbidden) for url (https://www.carrefour.com.ar/apple-macbook-pro-16--m1-pro-chip-10%E2%80%91core-cpu-16%E2%80%91core-gpu---1tb-ssd---space-grey-1656986/p) url="https://www.carrefour.com.ar/apple-macbook-pro-16--m1-pro-chip-10‑core-cpu-16‑core-gpu---1tb-ssd---space-grey-1656986/p"
2024-07-06T11:41:00.997947Z ERROR scraper::scraper: error=HTTP status client error (403 Forbidden) for url (https://www.carrefour.com.ar/leche-de-coco-coco-do-vale-reducida-en-calor%E2%88%9A-as-500-cc/p) url="https://www.carrefour.com.ar/leche-de-coco-coco-do-vale-reducida-en-calor√-as-500-cc/p"
This commit is contained in:
Cat /dev/Nulo 2024-07-06 08:46:04 -03:00
parent c4e11f05b6
commit 17c7e5701a
1 changed files with 2 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
rust/src/scraper/main.rs
View file

@ -163,7 +163,8 @@ pub fn get_parse_retry_policy() -> again::RetryPolicy {
} }
pub fn retry_if_wasnt_not_found(err: &reqwest::Error) -> bool { pub fn retry_if_wasnt_not_found(err: &reqwest::Error) -> bool {
!err.status().is_some_and(|s| s == StatusCode::NOT_FOUND) !err.status()
.is_some_and(|s| s == StatusCode::NOT_FOUND || s == StatusCode::FORBIDDEN)
} }
pub fn anyhow_retry_if_wasnt_not_found(err: &anyhow::Error) -> bool { pub fn anyhow_retry_if_wasnt_not_found(err: &anyhow::Error) -> bool {
match err.downcast_ref::<reqwest::Error>() { match err.downcast_ref::<reqwest::Error>() {