coto client

scraper-rs/src/main.rs

@@ -165,13 +165,28 @@ enum FetchError {
     Tl(#[from] tl::ParseError),
 }
 
-fn build_client() -> reqwest::Client {
+fn get_user_agent() -> &'static str {
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36"
+}
+fn build_header_map() -> HeaderMap {
     let mut headers = HeaderMap::new();
-    headers.append("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36".parse().unwrap());
+    headers.append("User-Agent", get_user_agent().parse().unwrap());
+    headers
+}
+
+fn build_client() -> reqwest::Client {
     reqwest::ClientBuilder::default()
         .timeout(Duration::from_secs(60))
         .connect_timeout(Duration::from_secs(30))
-        .default_headers(headers)
+        .default_headers(build_header_map())
+        .build()
+        .unwrap()
+}
+fn build_coto_client() -> reqwest::Client {
+    reqwest::ClientBuilder::default()
+        .timeout(Duration::from_secs(300))
+        .connect_timeout(Duration::from_secs(150))
+        .default_headers(build_header_map())
         .build()
         .unwrap()
 }
@@ -329,13 +344,25 @@ impl Auto {
     async fn download_supermercado(self, supermercado: Supermercado) -> anyhow::Result<()> {
         {
             let t0 = now_sec();
-            self.get_and_save_urls(&supermercado).await?;
-            self.inform(&format!(
-                "Downloaded url list {:?} (took {})",
-                &supermercado,
-                now_sec() - t0
-            ))
-            .await;
+            match self.get_and_save_urls(&supermercado).await {
+                Ok(_) => {
+                    self.inform(&format!(
+                        "Downloaded url list {:?} (took {})",
+                        &supermercado,
+                        now_sec() - t0
+                    ))
+                    .await
+                }
+                Err(err) => {
+                    self.inform(&format!(
+                        "[{:?}] FAILED url list: {:?} (took {})",
+                        &supermercado,
+                        err,
+                        now_sec() - t0
+                    ))
+                    .await
+                }
+            }
         }
         let links: Vec<String> = {
             let mut links = self.db.get_urls_by_domain(supermercado.host()).await?;

scraper-rs/src/sites/coto.rs

@@ -4,7 +4,7 @@ use itertools::Itertools;
 use reqwest::Url;
 
 use crate::{
-    build_client, do_request, get_fetch_retry_policy, retry_if_wasnt_not_found, PrecioPoint,
+    build_client, build_coto_client, do_request, get_fetch_retry_policy, retry_if_wasnt_not_found, PrecioPoint
 };
 
 pub fn parse(url: String, dom: &tl::VDom) -> Result<PrecioPoint, anyhow::Error> {
@@ -79,7 +79,7 @@ pub fn parse(url: String, dom: &tl::VDom) -> Result<PrecioPoint, anyhow::Error>
 }
 
 pub async fn get_urls() -> anyhow::Result<Vec<String>> {
-    let client = build_client();
+    let client = build_coto_client();
     let initial = Url::parse("https://www.cotodigital3.com.ar/sitios/cdigi/browse?Nf=product.endDate%7CGTEQ+1.7032032E12%7C%7Cproduct.startDate%7CLTEQ+1.7032032E12&Nr=AND%28product.sDisp_200%3A1004%2Cproduct.language%3Aespa%C3%B1ol%2COR%28product.siteId%3ACotoDigital%29%29")?;
 
     let page_size = 50;