Nulo/preciazo
1
0
Fork 0
mirror of https://github.com/catdevnull/preciazo.git synced 2024-11-22 22:26:19 +00:00
Code Issues Projects Releases Packages Wiki Activity

ignorar paginas status!=200

Browse source
This commit is contained in:
Cat /dev/Nulo 2024-01-01 16:50:17 -03:00
parent 7b989f0ea0
commit 405502877c
1 changed files with 6 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
scraper/scrap.ts
View file

@ -52,6 +52,12 @@ export async function parseWarc(path: string) {
console.debug(`skipped ${warcRecordId}`); console.debug(`skipped ${warcRecordId}`);
continue; continue;
} }
if (record.httpHeaders?.statusCode !== 200) {
console.debug(
`skipped ${warcRecordId} because status=${record.httpHeaders?.statusCode} (!=200)`
);
continue;
}
// TODO: sobreescribir si existe el mismo record-id pero con version mas bajo? // TODO: sobreescribir si existe el mismo record-id pero con version mas bajo?
const html = await record.contentText(); const html = await record.contentText();