Nulo/preciazo
1
0
Fork 0
mirror of https://github.com/catdevnull/preciazo.git synced 2024-11-26 11:36:20 +00:00
Code Issues Projects Releases Packages Wiki Activity

scrap: trackear proceso zstd

Browse source
This commit is contained in:
Cat /dev/Nulo 2024-01-01 15:32:42 -03:00
parent eb0e62447b
commit 8482d7ceea
1 changed files with 6 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
scraper/scrap.ts
View file

@ -37,9 +37,8 @@ export async function parseWarc(path: string) {
errors: { error: any; warcRecordId: string; path: string }[];
} = { done: 0, errors: [] };
const warc = Bun.spawn(["zstd", "-do", "/dev/stdout", path], {
stderr: "ignore",
}).stdout;
const proc = Bun.spawn(["zstdcat", "-d", path], {});
const warc = proc.stdout;
// TODO: tirar error si falla zstd
const parser = new WARCParser(warc);
@ -99,5 +98,9 @@ export async function parseWarc(path: string) {
}
}
if ((await proc.exited) !== 0) {
throw new Error("zstd tiró un error");
}
return progress;
}