diff --git a/scraper-rs/src/main.rs b/scraper-rs/src/main.rs index 076b1ab..2eb5f08 100644 --- a/scraper-rs/src/main.rs +++ b/scraper-rs/src/main.rs @@ -238,7 +238,13 @@ async fn fetch_and_parse( } get_parse_retry_policy() - .retry(|| fetch_and_scrap(client, url.clone())) + .retry_if( + || fetch_and_scrap(client, url.clone()), + |err: &anyhow::Error| match err.downcast_ref::() { + Some(e) => !e.status().is_some_and(|s| s == StatusCode::NOT_FOUND), + None => true, + }, + ) .await }