juntar por ean busqueda

fixes #11

.dockerignore

@@ -4,4 +4,7 @@ data/carrefour/
 downloader/
 node_modules/
 */node_modules/
-*/Containerfile
+*/Containerfile
+*.warc.zst
+.git
+scraper/debug/

carrefour-link-scraper/index.ts

@@ -1,4 +1,5 @@
 import pMap from "p-map";
+import { decodeXML } from "entities";
 import { saveUrls } from "db-datos/urlHelpers.js";
 
 export async function scrapCarrefourProducts() {
@@ -31,7 +32,7 @@ async function scrapBySitemap() {
           text(element) {
             const txt = element.text.trim();
             if (!txt) return;
-            urls.add(txt);
+            urls.add(decodeXML(txt));
           },
         })
         .transform(new Response(xml));

dia-link-scraper/index.ts

@@ -1,4 +1,5 @@
 import pMap from "p-map";
+import { decodeXML } from "entities";
 import { parseHTML } from "linkedom";
 import { getHtml } from "../scraper/fetch.js";
 import { saveUrls } from "db-datos/urlHelpers.js";
@@ -90,7 +91,7 @@ async function scrapBySitemap() {
         text(element) {
           const txt = element.text.trim();
           if (!txt) return;
-          urls.add(txt);
+          urls.add(decodeXML(txt));
         },
       })
       .transform(new Response(xml));

scraper/package.json

@@ -17,6 +17,7 @@
     "date-fns": "^3.0.6",
     "db-datos": "workspace:^",
     "drizzle-orm": "=0.29.1",
+    "entities": "^4.5.0",
     "linkedom": "^0.16.5",
     "nanoid": "^5.0.4",
     "p-map": "^7.0.1",

sitio/src/routes/search/+page.server.ts

@@ -11,7 +11,8 @@ export const load: PageServerLoad = async ({ url }) => {
     results = db.all(
       sql`select p.ean, p.name, p.image_url as imageUrl from precios_fts f
       join precios p on p.ean = f.ean
-      where f.name match ${query};`,
+      where f.name match ${query}
+      group by p.ean;`,
     );
   }