Nulo/preciazo
1
0
Fork 0
mirror of https://github.com/catdevnull/preciazo.git synced 2024-11-22 14:16:19 +00:00
Code Issues Projects Releases Packages Wiki Activity

parsear urls xml

Browse source
This commit is contained in:
Cat /dev/Nulo 2024-01-02 10:28:55 -03:00
parent 4d3793ddad
commit 97d94037e3
4 changed files with 5 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

BIN
bun.lockb
View file

Binary file not shown.

3
carrefour-link-scraper/index.ts
View file

@ -1,4 +1,5 @@
import pMap from "p-map";
import { decodeXML } from "entities";
import { saveUrls } from "db-datos/urlHelpers.js";
export async function scrapCarrefourProducts() {
@ -31,7 +32,7 @@ async function scrapBySitemap() {
text(element) {
const txt = element.text.trim();
if (!txt) return;
urls.add(txt);
urls.add(decodeXML(txt));
},
})
.transform(new Response(xml));

3
dia-link-scraper/index.ts
View file

@ -1,4 +1,5 @@
import pMap from "p-map";
import { decodeXML } from "entities";
import { parseHTML } from "linkedom";
import { getHtml } from "../scraper/fetch.js";
import { saveUrls } from "db-datos/urlHelpers.js";
@ -90,7 +91,7 @@ async function scrapBySitemap() {
text(element) {
const txt = element.text.trim();
if (!txt) return;
urls.add(txt);
urls.add(decodeXML(txt));
},
})
.transform(new Response(xml));

1
scraper/package.json
View file

@ -17,6 +17,7 @@
"date-fns": "^3.0.6",
"db-datos": "workspace:^",
"drizzle-orm": "=0.29.1",
"entities": "^4.5.0",
"linkedom": "^0.16.5",
"nanoid": "^5.0.4",
"p-map": "^7.0.1",