parsear correctamente urls en sitemaps xml

scraper-rs/Cargo.lock

@@ -902,6 +902,15 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "quick-xml"
+version = "0.31.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "quote"
 version = "1.0.35"
@@ -1132,6 +1141,7 @@ dependencies = [
  "futures",
  "itertools",
  "nanoid",
+ "quick-xml",
  "rand 0.8.5",
  "reqwest",
  "rusqlite",

scraper-rs/Cargo.toml

@@ -14,6 +14,7 @@ deadpool-sqlite = "0.7.0"
 futures = "0.3.30"
 itertools = "0.12.0"
 nanoid = "0.4.0"
+quick-xml = "0.31.0"
 rand = "0.8.5"
 reqwest = { version = "0.11.23", default-features = false, features = [
     "rustls-tls",

scraper-rs/src/sites/vtex.rs

@@ -107,14 +107,34 @@ pub fn in_stock_from_meta(dom: &VDom) -> anyhow::Result<bool> {
 
 pub fn parse_urls_from_sitemap(sitemap: &str) -> anyhow::Result<Vec<String>> {
     let dom = tl::parse(sitemap, tl::ParserOptions::default())?;
-    Ok(dom
-        .query_selector("loc")
+    dom.query_selector("loc")
         .unwrap()
         .filter_map(|h| h.get(dom.parser()))
         .filter_map(|n| n.as_tag())
         .map(|t| t.inner_text(dom.parser()))
-        .map(|s| s.to_string())
-        .collect())
+        .map(|s| -> anyhow::Result<String> {
+            Ok(quick_xml::escape::unescape(s.as_ref())?.to_string())
+        })
+        .try_collect()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_decode_url() -> anyhow::Result<()> {
+        let links = parse_urls_from_sitemap(
+            r#"
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:xhtml="http://www.w3.org/1999/xhtml">
+<url>
+    <loc>https://www.carrefour.com.ar/postre-danette-mousse-dulce-de-leche-80-g&#x200B;-684952/p</loc>
+    <lastmod>2024-01-12T10:41:25.962Z</lastmod>
+</url>"#,
+        )?;
+        assert_eq!(links[0], "https://www.carrefour.com.ar/postre-danette-mousse-dulce-de-leche-80-g\u{200b}-684952/p");
+        Ok(())
+    }
 }
 
 pub async fn get_urls_from_sitemap(sitemaps: Vec<&str>) -> anyhow::Result<Vec<String>> {