Nulo/preciazo
1
0
Fork 0
mirror of https://github.com/catdevnull/preciazo.git synced 2024-11-26 11:36:20 +00:00
Code Issues Projects Releases Packages Wiki Activity

multiroutine scrap

Browse source
This commit is contained in:
Cat /dev/Nulo 2023-12-22 17:01:16 -03:00
parent f0eeb01b66
commit d5e7726185
1 changed files with 4 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
scraper/scrap.ts
View file

@ -12,6 +12,7 @@ import { getCarrefourProduct } from "./carrefour.js";
import { getDiaProduct } from "./dia.js";
import { getCotoProduct } from "./coto.js";
import { join } from "path";
import pMap from "p-map";
const sqlite = new Database("sqlite.db");
const db = drizzle(sqlite);
@ -33,9 +34,9 @@ create table precios(
);
`);
for (const path of process.argv.slice(2)) {
await parseWarc(path);
}
await pMap(process.argv.slice(2), (path) => parseWarc(path), {
concurrency: 40,
});
const DEBUG = false;