paralelizar

scraper-rs/Cargo.lock

@@ -1165,6 +1165,7 @@ dependencies = [
  "anyhow",
  "async-channel",
  "clap",
+ "futures",
  "itertools",
  "nanoid",
  "r2d2",

scraper-rs/Cargo.toml

@@ -10,6 +10,7 @@ again = "0.1.2"
 anyhow = "1.0.79"
 async-channel = "2.1.1"
 clap = { version = "4.4.15", features = ["derive"] }
+futures = "0.3.30"
 itertools = "0.12.0"
 nanoid = "0.4.0"
 r2d2 = "0.8.10"

scraper-rs/src/sites/vtex.rs

@@ -1,4 +1,5 @@
 use anyhow::{bail, Context};
+use futures::{stream, StreamExt, TryStreamExt};
 use itertools::Itertools;
 use serde::Deserialize;
 use simple_error::SimpleError;
@@ -116,16 +117,25 @@ pub fn parse_urls_from_sitemap(sitemap: &str) -> anyhow::Result<Vec<String>> {
         .collect())
 }
 
-pub async fn get_urls_from_sitemap(sitemaps: &[&str]) -> anyhow::Result<Vec<String>> {
+pub async fn get_urls_from_sitemap<'a>(sitemaps: &[&str]) -> anyhow::Result<Vec<String>> {
-    let mut total = vec![];
+    let mut total: Vec<String> = vec![];
-    let client = &build_client();
+    let client = build_client();
-    for url in sitemaps {
+    let handles = stream::iter(sitemaps)
-        let text = get_retry_policy()
+        .map(|url| {
-            .retry(|| do_request(client, url))
+            let client = &client;
-            .await?
+            async move {
-            .text()
+                let text = get_retry_policy()
-            .await?;
+                    .retry(|| do_request(client, &url))
-        let mut urls = parse_urls_from_sitemap(&text)?;
+                    .await?
+                    .text()
+                    .await?;
+                parse_urls_from_sitemap(&text)
+            }
+        })
+        .buffer_unordered(8)
+        .try_collect::<Vec<_>>()
+        .await?;
+    for mut urls in handles {
         total.append(&mut urls);
     }
     Ok(total.into_iter().unique().collect())