Nulo/preciazo
1
0
Fork 0
mirror of https://github.com/catdevnull/preciazo.git synced 2024-11-22 22:26:19 +00:00
Code Issues Projects Releases Packages Wiki Activity

parsear urls xml

Browse source
This commit is contained in:
Cat /dev/Nulo 2024-01-02 10:28:55 -03:00
parent 4d3793ddad
commit 97d94037e3
4 changed files with 5 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

BIN
bun.lockb
View file

Binary file not shown.

3
carrefour-link-scraper/index.ts
View file

@ -1,4 +1,5 @@
import pMap from "p-map"; import pMap from "p-map";
import { decodeXML } from "entities";
import { saveUrls } from "db-datos/urlHelpers.js"; import { saveUrls } from "db-datos/urlHelpers.js";
export async function scrapCarrefourProducts() { export async function scrapCarrefourProducts() {
@ -31,7 +32,7 @@ async function scrapBySitemap() {
text(element) { text(element) {
const txt = element.text.trim(); const txt = element.text.trim();
if (!txt) return; if (!txt) return;
urls.add(txt); urls.add(decodeXML(txt));
}, },
}) })
.transform(new Response(xml)); .transform(new Response(xml));

3
dia-link-scraper/index.ts
View file

@ -1,4 +1,5 @@
import pMap from "p-map"; import pMap from "p-map";
import { decodeXML } from "entities";
import { parseHTML } from "linkedom"; import { parseHTML } from "linkedom";
import { getHtml } from "../scraper/fetch.js"; import { getHtml } from "../scraper/fetch.js";
import { saveUrls } from "db-datos/urlHelpers.js"; import { saveUrls } from "db-datos/urlHelpers.js";
@ -90,7 +91,7 @@ async function scrapBySitemap() {
text(element) { text(element) {
const txt = element.text.trim(); const txt = element.text.trim();
if (!txt) return; if (!txt) return;
urls.add(txt); urls.add(decodeXML(txt));
}, },
}) })
.transform(new Response(xml)); .transform(new Response(xml));

1
scraper/package.json
View file

@ -17,6 +17,7 @@
"date-fns": "^3.0.6", "date-fns": "^3.0.6",
"db-datos": "workspace:^", "db-datos": "workspace:^",
"drizzle-orm": "=0.29.1", "drizzle-orm": "=0.29.1",
"entities": "^4.5.0",
"linkedom": "^0.16.5", "linkedom": "^0.16.5",
"nanoid": "^5.0.4", "nanoid": "^5.0.4",
"p-map": "^7.0.1", "p-map": "^7.0.1",