printear errores asap

.github/workflows/container.yml

@@ -70,8 +70,6 @@ jobs:
       contents: read
       packages: write
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
       - name: Log in to the Container registry
@@ -85,41 +83,11 @@ jobs:
         uses: docker/metadata-action@v5
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/scraper
-      - name: Cache usr/src/app/target
-        uses: actions/cache@v4
-        with:
-          path: usr/src/app/target
-          key: usr/src/app/target-${{ hashFiles('Dockerfile.scraper') }}
-      - name: inject usr/src/app/target into docker
-        uses: reproducible-containers/buildkit-cache-dance@v2.1.3
-        with:
-          cache-source: usr/src/app/target
-          cache-target: /usr/src/app/target
-      - name: Cache root/.cargo/registry
-        uses: actions/cache@v4
-        with:
-          path: root/.cargo/registry
-          key: root/.cargo/registry-${{ hashFiles('Dockerfile.scraper') }}
-      - name: inject root/.cargo/registry into docker
-        uses: reproducible-containers/buildkit-cache-dance@v2.1.3
-        with:
-          cache-source: root/.cargo/registry
-          cache-target: /root/.cargo/registry
-      - name: Cache root/.cargo/git
-        uses: actions/cache@v4
-        with:
-          path: root/.cargo/git
-          key: root/.cargo/git-${{ hashFiles('Dockerfile.scraper') }}
-      - name: inject root/.cargo/git into docker
-        uses: reproducible-containers/buildkit-cache-dance@v2.1.3
-        with:
-          cache-source: root/.cargo/git
-          cache-target: /root/.cargo/git
       - name: Build and push Docker image
         uses: docker/build-push-action@v5
         with:
-          context: .
-          file: Dockerfile.scraper
+          context: "{{defaultContext}}:scraper-rs/"
+          file: Dockerfile
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}

Dockerfile.scraper

@@ -1,31 +0,0 @@
-FROM cgr.dev/chainguard/wolfi-base AS base
-WORKDIR /usr/src/app
-RUN apk add --no-cache libgcc
-
-# tenemos que generar una DB con las migraciones aplicadas para compilar el codigo por sqlx::query!()
-FROM base as db-build
-RUN apk add --no-cache nodejs npm
-RUN npm install --global pnpm
-COPY db-datos/ .
-RUN pnpm install
-RUN DB_PATH=db.db pnpm migrate
-
-FROM base as rs-build
-RUN apk add --no-cache rust build-base sqlite-dev
-
-COPY scraper-rs/ .
-COPY --from=db-build /usr/src/app/db.db .
-RUN --mount=type=cache,sharing=locked,target=/root/.cargo/git \
-    --mount=type=cache,sharing=locked,target=/root/.cargo/registry \
-    --mount=type=cache,sharing=locked,target=/usr/src/app/target \
-	DATABASE_URL=sqlite:db.db cargo install --locked --path .
-
-FROM base
-RUN apk add --no-cache sqlite sqlite-libs
-
-# Scraper
-COPY --from=rs-build /root/.cargo/bin/scraper-rs /usr/local/bin/scraper-rs
-
-ENV DB_PATH=/db/db.db
-
-CMD ["scraper-rs", "cron"]

devbox.json

@@ -4,7 +4,8 @@
     "cargo@latest",
     "sqlite@latest",
     "git-lfs@latest",
-    "nodePackages.pnpm@latest"
+    "nodePackages.pnpm@latest",
+    "sqlx-cli@latest"
   ],
   "shell": {
     "init_hook": [

devbox.lock

@@ -100,6 +100,26 @@
           "store_path": "/nix/store/b3pbwqqpfllg2lxwhhv4iyw3m9xxbgld-sqlite-3.45.1-bin"
         }
       }
+    },
+    "sqlx-cli@latest": {
+      "last_modified": "2024-02-24T23:06:34Z",
+      "resolved": "github:NixOS/nixpkgs/9a9dae8f6319600fa9aebde37f340975cab4b8c0#sqlx-cli",
+      "source": "devbox-search",
+      "version": "0.7.3",
+      "systems": {
+        "aarch64-darwin": {
+          "store_path": "/nix/store/llc00frrxnz2fp2xgp089d947rw7inq5-sqlx-cli-0.7.3"
+        },
+        "aarch64-linux": {
+          "store_path": "/nix/store/l9k33dxzg7qnhdzqmlw70xcqf8dshkvc-sqlx-cli-0.7.3"
+        },
+        "x86_64-darwin": {
+          "store_path": "/nix/store/3wpq5f4yva6llf9zfbskq1nyy06f94xj-sqlx-cli-0.7.3"
+        },
+        "x86_64-linux": {
+          "store_path": "/nix/store/hnzrhv4043xra2s8zmbxif8c9ff87lc2-sqlx-cli-0.7.3"
+        }
+      }
     }
   }
 }

scraper-rs/.dockerignore

@@ -0,0 +1,2 @@
+.env
+target

scraper-rs/.env

@@ -1 +0,0 @@
-DATABASE_URL=sqlite:../sqlite.db

scraper-rs/Dockerfile

@@ -0,0 +1,23 @@
+FROM cgr.dev/chainguard/wolfi-base AS base
+WORKDIR /usr/src/app
+RUN apk add --no-cache libgcc
+
+FROM docker.io/rust:1.76 as rs-build
+# RUN apt-get update && apt-get install -y libsqlite3-dev && rm -rf /var/lib/apt/lists/*
+WORKDIR /usr/src/app
+
+COPY . .
+RUN --mount=type=cache,sharing=locked,target=/root/.cargo/git \
+    --mount=type=cache,sharing=locked,target=/root/.cargo/registry \
+    --mount=type=cache,sharing=locked,target=/usr/src/app/target \
+	cargo install --locked --path .
+
+FROM base
+RUN apk add --no-cache sqlite sqlite-libs
+
+# Scraper
+COPY --from=rs-build /usr/local/cargo/bin/scraper-rs /usr/local/bin/scraper-rs
+
+ENV DB_PATH=/db/db.db
+
+CMD ["scraper-rs", "cron"]

scraper-rs/src/main.rs

@@ -3,14 +3,13 @@ use chrono::{DateTime, Utc};
 use clap::{Parser, ValueEnum};
 use cron::Schedule;
 use db::Db;
-use futures::{future, stream, Future, StreamExt};
-use nanoid::nanoid;
+use futures::{future, stream, Future, StreamExt, TryFutureExt};
+
 use reqwest::{header::HeaderMap, StatusCode, Url};
 use simple_error::{bail, SimpleError};
 use std::{
     env::{self},
     fs,
-    path::PathBuf,
     str::FromStr,
     time::{Duration, SystemTime, UNIX_EPOCH},
 };
@@ -425,7 +424,16 @@ async fn auto_cli(args: AutoArgs) -> anyhow::Result<()> {
 
     let handles: Vec<_> = supermercados
         .iter()
-        .map(|s| tokio::spawn(auto.clone().download_supermercado(s.to_owned())))
+        .map(|s| {
+            let x = s.clone();
+            tokio::spawn(
+                auto.clone()
+                    .download_supermercado(s.to_owned())
+                    .inspect_err(move |err| {
+                        tracing::error!(error=%err, supermercado=?x);
+                    }),
+            )
+        })
         .collect();
     future::try_join_all(handles).await?;
     auto.inform("[auto] Download supermercados finished").await;