Nulo/preciazo
1
0
Fork 0
mirror of https://github.com/catdevnull/preciazo.git synced 2024-11-29 21:16:19 +00:00
Code Issues Projects Releases Packages Wiki Activity

Compare commits

base: Nulo:5c52a12fdfb17d6e5e34a8d052704e1746f7a064
Branches Tags
Nulo:master
Nulo:new-rust-api
Nulo:postgres
..
compare: Nulo:c4b49814fb90051b8f15dd7587b7aa1753c98ae1
Branches Tags
Nulo:master
Nulo:new-rust-api
Nulo:postgres

No commits in common. "5c52a12fdfb17d6e5e34a8d052704e1746f7a064" and "c4b49814fb90051b8f15dd7587b7aa1753c98ae1" have entirely different histories.
5c52a12fdf ... c4b49814fb

6 changed files with 4 additions and 11 deletions
Show stats Expand all files Collapse all files

5
.dockerignore
View file

@ -4,7 +4,4 @@ data/carrefour/
downloader/ downloader/
node_modules/ node_modules/
*/node_modules/ */node_modules/
*/Containerfile */Containerfile
*.warc.zst
.git
scraper/debug/

BIN
bun.lockb
View file

Binary file not shown.

3
carrefour-link-scraper/index.ts
View file

@ -1,5 +1,4 @@
import pMap from "p-map"; import pMap from "p-map";
import { decodeXML } from "entities";
import { saveUrls } from "db-datos/urlHelpers.js"; import { saveUrls } from "db-datos/urlHelpers.js";
export async function scrapCarrefourProducts() { export async function scrapCarrefourProducts() {
@ -32,7 +31,7 @@ async function scrapBySitemap() {
text(element) { text(element) {
const txt = element.text.trim(); const txt = element.text.trim();
if (!txt) return; if (!txt) return;
urls.add(decodeXML(txt)); urls.add(txt);
}, },
}) })
.transform(new Response(xml)); .transform(new Response(xml));

3
dia-link-scraper/index.ts
View file

@ -1,5 +1,4 @@
import pMap from "p-map"; import pMap from "p-map";
import { decodeXML } from "entities";
import { parseHTML } from "linkedom"; import { parseHTML } from "linkedom";
import { getHtml } from "../scraper/fetch.js"; import { getHtml } from "../scraper/fetch.js";
import { saveUrls } from "db-datos/urlHelpers.js"; import { saveUrls } from "db-datos/urlHelpers.js";
@ -91,7 +90,7 @@ async function scrapBySitemap() {
text(element) { text(element) {
const txt = element.text.trim(); const txt = element.text.trim();
if (!txt) return; if (!txt) return;
urls.add(decodeXML(txt)); urls.add(txt);
}, },
}) })
.transform(new Response(xml)); .transform(new Response(xml));

1
scraper/package.json
View file

@ -17,7 +17,6 @@
"date-fns": "^3.0.6", "date-fns": "^3.0.6",
"db-datos": "workspace:^", "db-datos": "workspace:^",
"drizzle-orm": "=0.29.1", "drizzle-orm": "=0.29.1",
"entities": "^4.5.0",
"linkedom": "^0.16.5", "linkedom": "^0.16.5",
"nanoid": "^5.0.4", "nanoid": "^5.0.4",
"p-map": "^7.0.1", "p-map": "^7.0.1",

3
sitio/src/routes/search/+page.server.ts
View file

@ -11,8 +11,7 @@ export const load: PageServerLoad = async ({ url }) => {
results = db.all( results = db.all(
sql`select p.ean, p.name, p.image_url as imageUrl from precios_fts f sql`select p.ean, p.name, p.image_url as imageUrl from precios_fts f
join precios p on p.ean = f.ean join precios p on p.ean = f.ean
where f.name match ${query} where f.name match ${query};`,
group by p.ean;`,
); );
} }