Nulo/preciazo
1
0
Fork 0
mirror of https://github.com/catdevnull/preciazo.git synced 2025-02-23 19:16:23 +00:00
Code Issues Projects Releases Packages Wiki Activity

TODO

Browse source
This commit is contained in:
Cat /dev/Nulo 2023-12-23 20:43:53 -03:00
parent b66cb6782c
commit 2fdd89b4f8
1 changed files with 3 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
scraper/scrap.ts
View file

@ -49,6 +49,9 @@ async function parseWarc(path: string) {
for await (const record of parser) { for await (const record of parser) {
if (record.warcType === "response") { if (record.warcType === "response") {
if (!record.warcTargetURI) continue; if (!record.warcTargetURI) continue;
// TODO: saltear si ya existe el record-id con el mismo parser version
// y sobreescribir si existe el mismo record-id pero con version mas bajo?
const html = await record.contentText(); const html = await record.contentText();
const url = new URL(record.warcTargetURI); const url = new URL(record.warcTargetURI);