Nulo/preciazo
1
0
Fork 0
mirror of https://github.com/catdevnull/preciazo.git synced 2024-11-29 21:16:19 +00:00
Code Issues Projects Releases Packages Wiki Activity

retornar progres

Browse source
This commit is contained in:
Cat /dev/Nulo 2024-01-04 16:55:48 -03:00
parent df845acc66
commit 6256817ee1
1 changed files with 20 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

27
scraper/scrap.ts
View file

@ -18,17 +18,11 @@ export type Precioish = Omit<
>; >;
export async function downloadList(path: string) { export async function downloadList(path: string) {
let progress: {
done: number;
skipped: number;
errors: { error: any; url: string; path: string }[];
} = { done: 0, skipped: 0, errors: [] };
let list = (await Bun.file(path).text()) let list = (await Bun.file(path).text())
.split("\n") .split("\n")
.filter((s) => s.length > 0); .filter((s) => s.length > 0);
await pMap( const results = await pMap(
list, list,
async (urlS) => { async (urlS) => {
let res: ScrapResult = { type: "skipped" }; let res: ScrapResult = { type: "skipped" };
@ -40,10 +34,29 @@ export async function downloadList(path: string) {
} }
} }
if (res.type === "error") console.error(res); if (res.type === "error") console.error(res);
return res;
}, },
{ concurrency: 32 } { concurrency: 32 }
); );
let progress: {
done: number;
skipped: number;
errors: { error: any; url: string; debugPath: string }[];
} = { done: 0, skipped: 0, errors: [] };
for (const result of results) {
switch (result.type) {
case "done":
progress.done++;
break;
case "error":
progress.errors.push(result);
break;
case "skipped":
progress.skipped++;
break;
}
}
return progress; return progress;
} }