Nulo/preciazo
1
0
Fork 0
mirror of https://github.com/catdevnull/preciazo.git synced 2024-11-29 13:06:19 +00:00
Code Issues Projects Releases Packages Wiki Activity

Compare commits

base: Nulo:c75f8c335cc5b1c0b42bf03265b6c2053fb62b4f
Branches Tags
Nulo:master
Nulo:new-rust-api
Nulo:postgres
..
compare: Nulo:88e3aef8adb402e62dccbe10a57170eeb8609120
Branches Tags
Nulo:master
Nulo:new-rust-api
Nulo:postgres

2 commits
c75f8c335c ... 88e3aef8ad

Author SHA1 Message Date
Nulo
88e3aef8ad arreglar ordenado de precios 2023-12-25 13:21:33 -03:00
Nulo
e7d8ac8854 limpiar auto 2023-12-25 13:21:20 -03:00
3 changed files with 9 additions and 6 deletions
Show stats Expand all files Collapse all files

BIN
bun.lockb
View file

Binary file not shown.

12
scraper/auto.ts
View file

@ -50,13 +50,13 @@ supermercados.forEach(downloadList);
async function downloadList(supermercado: Supermercado) { async function downloadList(supermercado: Supermercado) {
const listPath = resolve( const listPath = resolve(
join(process.env.LISTS_DIR ?? "../data", `${supermercado}.txt`) join(process.env.LISTS_DIR ?? "../data", `${supermercado}.txt`),
); );
const date = new Date(); const date = new Date();
const ctxPath = await mkdtemp(join(tmpdir(), "preciazo-scraper-wget-")); const ctxPath = await mkdtemp(join(tmpdir(), "preciazo-scraper-wget-"));
const zstdWarcName = `${supermercado}-${format( const zstdWarcName = `${supermercado}-${format(
date, date,
"yyyy-MM-dd-HH:mm" "yyyy-MM-dd-HH:mm",
)}.warc.zst`; )}.warc.zst`;
const zstdWarcPath = join(ctxPath, zstdWarcName); const zstdWarcPath = join(ctxPath, zstdWarcName);
const subproc = Bun.spawn({ const subproc = Bun.spawn({
@ -100,6 +100,8 @@ async function downloadList(supermercado: Supermercado) {
inform(`Falló subir ${zstdWarcName} a S3; ${error}`); inform(`Falló subir ${zstdWarcName} a S3; ${error}`);
console.error(error); console.error(error);
} }
// TODO: borrar archivos temporales
} }
async function scrapAndInform({ async function scrapAndInform({
@ -120,7 +122,7 @@ async function scrapAndInform({
inform( inform(
`Procesado ${zstdWarcName} (${progress.done} ok, ${ `Procesado ${zstdWarcName} (${progress.done} ok, ${
progress.errors.length progress.errors.length
} errores) (tardó ${formatMs(took)})` } errores) (tardó ${formatMs(took)})`,
); );
} else { } else {
inform(`Algo falló en ${zstdWarcName}`); inform(`Algo falló en ${zstdWarcName}`);
@ -157,7 +159,7 @@ function recompress(inputPath: string, outputPath: string) {
["-T0", "-15", "--long", "-o", outputPath], ["-T0", "-15", "--long", "-o", outputPath],
{ {
stdio: ["pipe", null, null], stdio: ["pipe", null, null],
} },
); );
// @ts-expect-error a los types de bun no le gusta???? // @ts-expect-error a los types de bun no le gusta????
decompressor.stdout.pipe(compressor.stdin); decompressor.stdout.pipe(compressor.stdin);
@ -213,7 +215,7 @@ async function sendTelegramMsg(text: string) {
if (!process.env.TELEGRAM_BOT_TOKEN || !process.env.TELEGRAM_BOT_CHAT_ID) if (!process.env.TELEGRAM_BOT_TOKEN || !process.env.TELEGRAM_BOT_CHAT_ID)
return; return;
const url = new URL( const url = new URL(
`https://api.telegram.org/bot${process.env.TELEGRAM_BOT_TOKEN}/sendMessage` `https://api.telegram.org/bot${process.env.TELEGRAM_BOT_TOKEN}/sendMessage`,
); );
url.searchParams.set("chat_id", process.env.TELEGRAM_BOT_CHAT_ID); url.searchParams.set("chat_id", process.env.TELEGRAM_BOT_CHAT_ID);
url.searchParams.set("text", text); url.searchParams.set("text", text);

3
sitio/src/routes/ean/[ean]/+page.server.ts
View file

@ -10,7 +10,8 @@ export const load: PageServerLoad = async ({ params }) => {
.from(precios) .from(precios)
.where(eq(precios.ean, params.ean)) .where(eq(precios.ean, params.ean))
.groupBy(precios.warcRecordId) .groupBy(precios.warcRecordId)
.having(max(precios.parserVersion)); .having(max(precios.parserVersion))
.orderBy(precios.fetchedAt);
const res = await q; const res = await q;
if (res.length === 0) return error(404, "Not Found"); if (res.length === 0) return error(404, "Not Found");