container arreglado para scraper

This commit is contained in:
Cat /dev/Nulo 2023-12-26 14:43:10 -03:00
parent e6ae2d629e
commit 8a49ddab7d
5 changed files with 17 additions and 7 deletions

View file

@ -3,4 +3,5 @@ data/carrefour/
*/*.db*
downloader/
node_modules/
*/node_modules/
*/node_modules/
*/Containerfile

BIN
bun.lockb

Binary file not shown.

View file

@ -4,15 +4,23 @@ WORKDIR /usr/src/app
FROM base AS builder
ENV NODE_ENV=production
COPY . .
RUN bun install --frozen-lockfile
RUN bun build scraper/auto.ts --target=bun --outfile=/tmp/auto.build.js
RUN bun install --frozen-lockfile \
&& bun build scraper/auto.ts --target=bun --outfile=/tmp/auto.build.js \
&& rm -rf node_modules/
FROM base
RUN apk add --no-cache wget zstd cronie tini
RUN printf "#!/bin/sh\nexec bun /app/built.js" > /etc/periodic/daily/scraper \
&& chmod +x /etc/periodic/daily/scraper
COPY --from=builder /tmp/auto.build.js /app/built.js
COPY --from=builder /usr/src/app/db-datos/drizzle /app/drizzle
COPY --from=builder /usr/src/app/data /listas
WORKDIR /app
VOLUME /db
ENV DB_PATH=/db/db.db
ENV LISTS_DIR=/listas/
CMD ["bun", "/app/built.js"]
CMD ["tini", "/usr/sbin/crond", "-n"]
# CMD ["bun", "/app/built.js"]

View file

@ -77,13 +77,13 @@ async function downloadList(supermercado: Supermercado) {
inform(`wget para ${zstdWarcName} tardó ${formatMs(performance.now() - t0)}`);
const gzippedWarcPath = join(ctxPath, "temp.warc.gz");
if (!(await exists(gzippedWarcPath))) {
if (!(await fileExists(gzippedWarcPath))) {
const err = report(`no encontré el ${gzippedWarcPath}`);
throw err;
}
await compressionQueue.add(() => recompress(gzippedWarcPath, zstdWarcPath));
if (!(await exists(zstdWarcPath))) {
if (!(await fileExists(zstdWarcPath))) {
const err = report(`no encontré el ${zstdWarcPath}`);
throw err;
}
@ -201,7 +201,8 @@ function report(msg: string) {
return error;
}
async function exists(path: string) {
// no se llama exists porque bun tiene un bug en el que usa fs.exists por mas que exista una funcion llamada exists
async function fileExists(path: string) {
try {
access(path);
return true;

0
scraper/cli.ts Normal file
View file