mirror of
https://github.com/catdevnull/preciazo.git
synced 2024-11-29 13:06:19 +00:00
Compare commits
No commits in common. "88e3aef8adb402e62dccbe10a57170eeb8609120" and "c75f8c335cc5b1c0b42bf03265b6c2053fb62b4f" have entirely different histories.
88e3aef8ad
...
c75f8c335c
3 changed files with 6 additions and 9 deletions
BIN
bun.lockb
BIN
bun.lockb
Binary file not shown.
|
@ -50,13 +50,13 @@ supermercados.forEach(downloadList);
|
|||
|
||||
async function downloadList(supermercado: Supermercado) {
|
||||
const listPath = resolve(
|
||||
join(process.env.LISTS_DIR ?? "../data", `${supermercado}.txt`),
|
||||
join(process.env.LISTS_DIR ?? "../data", `${supermercado}.txt`)
|
||||
);
|
||||
const date = new Date();
|
||||
const ctxPath = await mkdtemp(join(tmpdir(), "preciazo-scraper-wget-"));
|
||||
const zstdWarcName = `${supermercado}-${format(
|
||||
date,
|
||||
"yyyy-MM-dd-HH:mm",
|
||||
"yyyy-MM-dd-HH:mm"
|
||||
)}.warc.zst`;
|
||||
const zstdWarcPath = join(ctxPath, zstdWarcName);
|
||||
const subproc = Bun.spawn({
|
||||
|
@ -100,8 +100,6 @@ async function downloadList(supermercado: Supermercado) {
|
|||
inform(`Falló subir ${zstdWarcName} a S3; ${error}`);
|
||||
console.error(error);
|
||||
}
|
||||
|
||||
// TODO: borrar archivos temporales
|
||||
}
|
||||
|
||||
async function scrapAndInform({
|
||||
|
@ -122,7 +120,7 @@ async function scrapAndInform({
|
|||
inform(
|
||||
`Procesado ${zstdWarcName} (${progress.done} ok, ${
|
||||
progress.errors.length
|
||||
} errores) (tardó ${formatMs(took)})`,
|
||||
} errores) (tardó ${formatMs(took)})`
|
||||
);
|
||||
} else {
|
||||
inform(`Algo falló en ${zstdWarcName}`);
|
||||
|
@ -159,7 +157,7 @@ function recompress(inputPath: string, outputPath: string) {
|
|||
["-T0", "-15", "--long", "-o", outputPath],
|
||||
{
|
||||
stdio: ["pipe", null, null],
|
||||
},
|
||||
}
|
||||
);
|
||||
// @ts-expect-error a los types de bun no le gusta????
|
||||
decompressor.stdout.pipe(compressor.stdin);
|
||||
|
@ -215,7 +213,7 @@ async function sendTelegramMsg(text: string) {
|
|||
if (!process.env.TELEGRAM_BOT_TOKEN || !process.env.TELEGRAM_BOT_CHAT_ID)
|
||||
return;
|
||||
const url = new URL(
|
||||
`https://api.telegram.org/bot${process.env.TELEGRAM_BOT_TOKEN}/sendMessage`,
|
||||
`https://api.telegram.org/bot${process.env.TELEGRAM_BOT_TOKEN}/sendMessage`
|
||||
);
|
||||
url.searchParams.set("chat_id", process.env.TELEGRAM_BOT_CHAT_ID);
|
||||
url.searchParams.set("text", text);
|
||||
|
|
|
@ -10,8 +10,7 @@ export const load: PageServerLoad = async ({ params }) => {
|
|||
.from(precios)
|
||||
.where(eq(precios.ean, params.ean))
|
||||
.groupBy(precios.warcRecordId)
|
||||
.having(max(precios.parserVersion))
|
||||
.orderBy(precios.fetchedAt);
|
||||
.having(max(precios.parserVersion));
|
||||
const res = await q;
|
||||
if (res.length === 0) return error(404, "Not Found");
|
||||
|
||||
|
|
Loading…
Reference in a new issue