Nulo/preciazo
1
0
Fork 0
mirror of https://github.com/catdevnull/preciazo.git synced 2024-11-25 11:06:20 +00:00
Code Issues Projects Releases Packages Wiki Activity

Compare commits

base: Nulo:ed7af7621d9ff2b46bf9a01ff70acaa87cdbf721
Branches Tags
Nulo:master
Nulo:new-rust-api
Nulo:postgres
..
compare: Nulo:28662978aef8de83f9c5d44aef3f6d50ffc880e3
Branches Tags
Nulo:master
Nulo:new-rust-api
Nulo:postgres

No commits in common. "ed7af7621d9ff2b46bf9a01ff70acaa87cdbf721" and "28662978aef8de83f9c5d44aef3f6d50ffc880e3" have entirely different histories.
ed7af7621d ... 28662978ae

38 changed files with 326 additions and 1880 deletions
Show stats Expand all files Collapse all files

6
.github/workflows/container.yml vendored
View file

@ -85,7 +85,7 @@ jobs:
with:
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/scraper
- name: Cache usr/src/app/target
uses: actions/cache@v4
uses: actions/cache@v3
with:
path: usr/src/app/target
key: usr/src/app/target-${{ hashFiles('Dockerfile.scraper') }}
@ -95,7 +95,7 @@ jobs:
cache-source: usr/src/app/target
cache-target: /usr/src/app/target
- name: Cache root/.cargo/registry
uses: actions/cache@v4
uses: actions/cache@v3
with:
path: root/.cargo/registry
key: root/.cargo/registry-${{ hashFiles('Dockerfile.scraper') }}
@ -105,7 +105,7 @@ jobs:
cache-source: root/.cargo/registry
cache-target: /root/.cargo/registry
- name: Cache root/.cargo/git
uses: actions/cache@v4
uses: actions/cache@v3
with:
path: root/.cargo/git
key: root/.cargo/git-${{ hashFiles('Dockerfile.scraper') }}

4
.gitignore vendored
View file

@ -5,7 +5,7 @@ data/carrefour
p.*
p
node_modules/
*.db*
*.db
*.db-shm
*.db-wal
scraper/debug/
@ -16,4 +16,4 @@ target/
*/flamegraph.svg
*/perf.data*
scraper-rs/debug*/
scraper-rs/debug/

2
Dockerfile
View file

@ -23,7 +23,7 @@ COPY --from=build /usr/src/app/sitio/package.json package.real.json
RUN sh -c 'echo {\"name\":\"sitio\",\"type\":\"module\",\"dependencies\":$(jq .dependencies < package.real.json)} > package.json' && npm install
COPY --from=build /usr/src/app/db-datos node_modules/db-datos
COPY --from=build /usr/src/app/sitio/build .
COPY --from=build /usr/src/app/db-datos/drizzle drizzle
COPY --from=build /usr/src/app/db-datos/drizzle .
ENV DB_PATH=/db/db.db
EXPOSE 3000

11
Dockerfile.scraper
View file

@ -2,23 +2,14 @@ FROM cgr.dev/chainguard/wolfi-base AS base
WORKDIR /usr/src/app
RUN apk add --no-cache libgcc
# tenemos que generar una DB con las migraciones aplicadas para compilar el codigo por sqlx::query!()
FROM base as db-build
RUN apk add --no-cache nodejs npm
RUN npm install --global pnpm
COPY db-datos/ .
RUN pnpm install
RUN DB_PATH=db.db pnpm migrate
FROM base as rs-build
RUN apk add --no-cache rust build-base sqlite-dev
COPY scraper-rs/ .
COPY --from=db-build /usr/src/app/db.db .
RUN --mount=type=cache,sharing=locked,target=/root/.cargo/git \
--mount=type=cache,sharing=locked,target=/root/.cargo/registry \
--mount=type=cache,sharing=locked,target=/usr/src/app/target \
DATABASE_URL=sqlite:db.db cargo install --locked --path .
cargo install --locked --path .
FROM base
RUN apk add --no-cache sqlite sqlite-libs

6
db-datos/drizzle/0010_true_black_tom.sql
View file

@ -1,6 +0,0 @@
CREATE TABLE `db_best_selling` (
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
`fetched_at` integer NOT NULL,
`category` text NOT NULL,
`eans_json` text NOT NULL
);

1
db-datos/drizzle/0011_huge_next_avengers.sql
View file

@ -1 +0,0 @@
CREATE INDEX `precios_url_idx` ON `precios` (`url`);

183
db-datos/drizzle/meta/0010_snapshot.json
View file

@ -1,183 +0,0 @@
{
"version": "5",
"dialect": "sqlite",
"id": "c8297337-4ed8-432e-8782-65d41be42e00",
"prevId": "2e398920-ffaf-4d55-ae13-d906cb9e0efa",
"tables": {
"db_best_selling": {
"name": "db_best_selling",
"columns": {
"id": {
"name": "id",
"type": "integer",
"primaryKey": true,
"notNull": true,
"autoincrement": true
},
"fetched_at": {
"name": "fetched_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"category": {
"name": "category",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"eans_json": {
"name": "eans_json",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
}
},
"indexes": {},
"foreignKeys": {},
"compositePrimaryKeys": {},
"uniqueConstraints": {}
},
"precios": {
"name": "precios",
"columns": {
"id": {
"name": "id",
"type": "integer",
"primaryKey": true,
"notNull": true,
"autoincrement": true
},
"ean": {
"name": "ean",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"fetched_at": {
"name": "fetched_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"precio_centavos": {
"name": "precio_centavos",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"in_stock": {
"name": "in_stock",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"url": {
"name": "url",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"warc_record_id": {
"name": "warc_record_id",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"parser_version": {
"name": "parser_version",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"name": {
"name": "name",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"image_url": {
"name": "image_url",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
}
},
"indexes": {
"precios_ean_idx": {
"name": "precios_ean_idx",
"columns": [
"ean"
],
"isUnique": false
}
},
"foreignKeys": {},
"compositePrimaryKeys": {},
"uniqueConstraints": {}
},
"producto_urls": {
"name": "producto_urls",
"columns": {
"id": {
"name": "id",
"type": "integer",
"primaryKey": true,
"notNull": true,
"autoincrement": true
},
"url": {
"name": "url",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"first_seen": {
"name": "first_seen",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"last_seen": {
"name": "last_seen",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
}
},
"indexes": {
"producto_urls_url_unique": {
"name": "producto_urls_url_unique",
"columns": [
"url"
],
"isUnique": true
}
},
"foreignKeys": {},
"compositePrimaryKeys": {},
"uniqueConstraints": {}
}
},
"enums": {},
"_meta": {
"schemas": {},
"tables": {},
"columns": {}
}
}

190
db-datos/drizzle/meta/0011_snapshot.json
View file

@ -1,190 +0,0 @@
{
"version": "5",
"dialect": "sqlite",
"id": "8b4921b5-6ecd-4d69-ba64-9b0bfb53db84",
"prevId": "c8297337-4ed8-432e-8782-65d41be42e00",
"tables": {
"db_best_selling": {
"name": "db_best_selling",
"columns": {
"id": {
"name": "id",
"type": "integer",
"primaryKey": true,
"notNull": true,
"autoincrement": true
},
"fetched_at": {
"name": "fetched_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"category": {
"name": "category",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"eans_json": {
"name": "eans_json",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
}
},
"indexes": {},
"foreignKeys": {},
"compositePrimaryKeys": {},
"uniqueConstraints": {}
},
"precios": {
"name": "precios",
"columns": {
"id": {
"name": "id",
"type": "integer",
"primaryKey": true,
"notNull": true,
"autoincrement": true
},
"ean": {
"name": "ean",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"fetched_at": {
"name": "fetched_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"precio_centavos": {
"name": "precio_centavos",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"in_stock": {
"name": "in_stock",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"url": {
"name": "url",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"warc_record_id": {
"name": "warc_record_id",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"parser_version": {
"name": "parser_version",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"name": {
"name": "name",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"image_url": {
"name": "image_url",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
}
},
"indexes": {
"precios_ean_idx": {
"name": "precios_ean_idx",
"columns": [
"ean"
],
"isUnique": false
},
"precios_url_idx": {
"name": "precios_url_idx",
"columns": [
"url"
],
"isUnique": false
}
},
"foreignKeys": {},
"compositePrimaryKeys": {},
"uniqueConstraints": {}
},
"producto_urls": {
"name": "producto_urls",
"columns": {
"id": {
"name": "id",
"type": "integer",
"primaryKey": true,
"notNull": true,
"autoincrement": true
},
"url": {
"name": "url",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"first_seen": {
"name": "first_seen",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"last_seen": {
"name": "last_seen",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
}
},
"indexes": {
"producto_urls_url_unique": {
"name": "producto_urls_url_unique",
"columns": [
"url"
],
"isUnique": true
}
},
"foreignKeys": {},
"compositePrimaryKeys": {},
"uniqueConstraints": {}
}
},
"enums": {},
"_meta": {
"schemas": {},
"tables": {},
"columns": {}
}
}

14
db-datos/drizzle/meta/_journal.json
View file

@ -71,20 +71,6 @@
"when": 1703895109501,
"tag": "0009_breezy_forge",
"breakpoints": true
},
{
"idx": 10,
"version": "5",
"when": 1706540751931,
"tag": "0010_true_black_tom",
"breakpoints": true
},
{
"idx": 11,
"version": "5",
"when": 1706628184254,
"tag": "0011_huge_next_avengers",
"breakpoints": true
}
]
}

3
db-datos/migrate-cli.js
View file

@ -1,3 +0,0 @@
import { getDb } from "./db.js";
getDb();

7
db-datos/migrate.js
View file

@ -2,17 +2,12 @@
import { migrate } from "drizzle-orm/better-sqlite3/migrator";
import * as schema from "./schema.js";
import { sql } from "drizzle-orm";
import { existsSync } from "node:fs";
import { join } from "node:path";
/**
* @param {import("drizzle-orm/better-sqlite3").BetterSQLite3Database<schema>} db
*/
export function migrateDb(db) {
let path = "drizzle/";
if (!existsSync(join(path, "meta/_journal.json")))
path = "node_modules/db-datos/drizzle";
migrate(db, { migrationsFolder: path });
migrate(db, { migrationsFolder: "node_modules/db-datos/drizzle" });
db.run(sql`pragma journal_mode = WAL;`);
db.run(sql`PRAGMA synchronous = NORMAL;`);
}

2
db-datos/package.json
View file

@ -6,7 +6,7 @@
"main": "index.js",
"scripts": {
"generate": "drizzle-kit generate:sqlite",
"migrate": "node migrate-cli.js"
"migrate": "node db.js"
},
"keywords": [],
"author": "",

12
db-datos/schema.js
View file

@ -18,9 +18,8 @@ export const precios = sqliteTable(
(precios) => {
return {
preciosEanIdx: index("precios_ean_idx").on(precios.ean),
preciosUrlIdx: index("precios_url_idx").on(precios.url),
};
}
},
);
/** @typedef {typeof precios.$inferSelect} Precio */
@ -33,12 +32,3 @@ export const productoUrls = sqliteTable("producto_urls", {
});
/** @typedef {typeof productoUrls.$inferSelect} ProductUrl */
export const bestSelling = sqliteTable("db_best_selling", {
id: integer("id", { mode: "number" }).primaryKey({ autoIncrement: true }),
fetchedAt: integer("fetched_at", { mode: "timestamp" }).notNull(),
category: text("category").notNull(),
eansJson: text("eans_json").notNull(),
});
/** @typedef {typeof bestSelling.$inferSelect} BestSelling */

3
db-datos/supermercado.ts
View file

@ -16,9 +16,6 @@ export const hosts: { [host: string]: Supermercado } = {
"www.cotodigital3.com.ar": Supermercado.Coto,
"www.jumbo.com.ar": Supermercado.Jumbo,
};
export const hostBySupermercado = Object.fromEntries(
Object.entries(hosts).map(([a, b]) => [b, a])
) as Record<Supermercado, string>;
export const colorBySupermercado: { [supermercado in Supermercado]: string } = {
[Supermercado.Dia]: "#d52b1e",
[Supermercado.Carrefour]: "#19549d",

4
pnpm-lock.yaml
View file

@ -42,9 +42,6 @@ importers:
drizzle-orm:
specifier: ^0.29.1
version: 0.29.3(@types/better-sqlite3@7.6.8)(better-sqlite3@9.2.2)
zod:
specifier: ^3.22.4
version: 3.22.4
devDependencies:
'@sveltejs/adapter-node':
specifier: ^2.0.2
@ -2862,3 +2859,4 @@ packages:
/zod@3.22.4:
resolution: {integrity: sha512-iC+8Io04lddc+mVqQ9AZ7OQ2MrUKGN+oIQyq1vemgt46jwCwLfhq7/pwnBnNXXXZb8VTVLKwp9EDkx+ryxIWmg==}
dev: true

1
scraper-rs/.env
View file

@ -1 +0,0 @@
DATABASE_URL=sqlite:../sqlite.db

12
scraper-rs/.sqlx/query-08d55fc80c8a6ad73d311e8b1cd535425e5c2f39cf98735b5f67cb91d01937ce.json
View file

@ -1,12 +0,0 @@
{
"db_name": "SQLite",
"query": "INSERT INTO producto_urls(url, first_seen, last_seen)\n VALUES (?1, ?2, ?2)\n ON CONFLICT(url) DO UPDATE SET last_seen=?2;",
"describe": {
"columns": [],
"parameters": {
"Right": 2
},
"nullable": []
},
"hash": "08d55fc80c8a6ad73d311e8b1cd535425e5c2f39cf98735b5f67cb91d01937ce"
}

12
scraper-rs/.sqlx/query-144f4622ac9a937aa4885ceb5a67f6c0a78e7e025cf152a8c176b9fd1de241da.json
View file

@ -1,12 +0,0 @@
{
"db_name": "SQLite",
"query": "INSERT INTO db_best_selling(fetched_at, category, eans_json)\n VALUES (?1, ?2, ?3);",
"describe": {
"columns": [],
"parameters": {
"Right": 3
},
"nullable": []
},
"hash": "144f4622ac9a937aa4885ceb5a67f6c0a78e7e025cf152a8c176b9fd1de241da"
}

20
scraper-rs/.sqlx/query-aa5c2a04aec149d88f6e25a9bd7df4e257f3c9b0efa62c8342d077d69d826a69.json
View file

@ -1,20 +0,0 @@
{
"db_name": "SQLite",
"query": "SELECT url FROM producto_urls WHERE url LIKE ?1;",
"describe": {
"columns": [
{
"name": "url",
"ordinal": 0,
"type_info": "Text"
}
],
"parameters": {
"Right": 1
},
"nullable": [
false
]
},
"hash": "aa5c2a04aec149d88f6e25a9bd7df4e257f3c9b0efa62c8342d077d69d826a69"
}

12
scraper-rs/.sqlx/query-d0c3a557a81f6685b242ed0be8e8c67b47ec8a575d2a14a487b3294e0faec438.json
View file

@ -1,12 +0,0 @@
{
"db_name": "SQLite",
"query": "INSERT INTO precios(ean, fetched_at, precio_centavos, in_stock, url, warc_record_id, parser_version, name, image_url) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9);",
"describe": {
"columns": [],
"parameters": {
"Right": 9
},
"nullable": []
},
"hash": "d0c3a557a81f6685b242ed0be8e8c67b47ec8a575d2a14a487b3294e0faec438"
}

20
scraper-rs/.sqlx/query-f249765f2fb013a81a4157a6ce19744a8d5f70c83ed9ddddfd55009136088a52.json
View file

@ -1,20 +0,0 @@
{
"db_name": "SQLite",
"query": "SELECT ean FROM precios WHERE url = ?1;",
"describe": {
"columns": [
{
"name": "ean",
"ordinal": 0,
"type_info": "Text"
}
],
"parameters": {
"Right": 1
},
"nullable": [
false
]
},
"hash": "f249765f2fb013a81a4157a6ce19744a8d5f70c83ed9ddddfd55009136088a52"
}

904
scraper-rs/Cargo.lock generated
View file

File diff suppressed because it is too large Load diff

4
scraper-rs/Cargo.toml
View file

@ -8,11 +8,11 @@ edition = "2021"
[dependencies]
again = "0.1.2"
anyhow = "1.0.79"
base64 = "0.21.7"
chrono = "0.4.32"
clap = { version = "4.4.15", features = ["derive"] }
cron = "0.12.0"
sqlx = { version = "0.7", features = [ "runtime-tokio", "sqlite" ] }
deadpool = "0.10.0"
deadpool-sqlite = "0.7.0"
futures = "0.3.30"
html-escape = "0.2.13"
itertools = "0.12.0"

131
scraper-rs/src/best_selling.rs
View file

@ -1,131 +0,0 @@
use std::collections::HashMap;
use crate::{build_client, db::Db, sites::vtex, supermercado::Supermercado};
use chrono::{DateTime, Utc};
use clap::ValueEnum;
use futures::{stream, FutureExt, StreamExt, TryStreamExt};
use itertools::Itertools;
use tracing::warn;
#[derive(ValueEnum, Clone, Debug)]
pub enum Category {
Almacen,
Bebidas,
FrutasYVerduras,
}
impl Category {
fn query(&self, supermercado: &Supermercado) -> Option<&'static str> {
match self {
Self::Almacen => match supermercado {
Supermercado::Jumbo | Supermercado::Dia => Some("almacen"),
_ => None,
},
Self::Bebidas => match supermercado {
Supermercado::Jumbo | Supermercado::Dia => Some("bebidas"),
_ => None,
},
Self::FrutasYVerduras => match supermercado {
Supermercado::Jumbo => Some("frutas-y-verduras"),
Supermercado::Dia => Some("frescos/frutas-y-verduras"),
_ => None,
},
}
}
pub fn id(&self) -> &'static str {
match self {
Self::Almacen => "almacen",
Self::Bebidas => "bebidas",
Self::FrutasYVerduras => "frutas-y-verduras",
}
}
}
#[derive(Debug)]
pub struct BestSellingRecord {
pub fetched_at: DateTime<Utc>,
pub category: Category,
pub eans: Vec<String>,
}
async fn get_best_selling_eans(db: &Db, urls: Vec<String>) -> anyhow::Result<Vec<String>> {
let mut eans: Vec<String> = Vec::new();
for url in urls {
let ean = db.get_ean_by_url(&url).await?;
match ean {
Some(e) => eans.push(e),
None => warn!("No encontré EAN para {}", url),
}
}
Ok(eans)
}
async fn try_get_best_selling_eans(
client: reqwest::Client,
db: Db,
supermercado: &Supermercado,
category: &Category,
) -> anyhow::Result<Option<Vec<String>>> {
if let Some(query) = category.query(supermercado) {
let urls = vtex::get_best_selling_by_category(&client, supermercado.host(), query).await?;
let eans = get_best_selling_eans(&db, urls).await?;
Ok(Some(eans))
} else {
Ok(None)
}
}
async fn noop<T>(t: T) -> anyhow::Result<T> {
Ok(t)
}
fn rank_eans(eans: Vec<Vec<String>>) -> Vec<String> {
let mut map: HashMap<String, usize> = HashMap::new();
for eans in eans {
for (i, ean) in eans.into_iter().enumerate() {
let base = map.get(&ean).unwrap_or(&0);
let score = base + 1000 / (i + 1);
map.insert(ean, score);
}
}
map.into_iter()
.sorted_by(|a, b| Ord::cmp(&b.1, &a.1))
.map(|t| t.0)
.collect_vec()
}
pub async fn get_all_best_selling(db: &Db) -> anyhow::Result<Vec<BestSellingRecord>> {
let client = &build_client();
stream::iter(Category::value_variants())
.map(|category| {
stream::iter(Supermercado::value_variants())
.map(|supermercado| {
let db = db.clone();
let client = client.clone();
tokio::spawn(try_get_best_selling_eans(
client,
db,
supermercado,
category,
))
})
.buffer_unordered(5)
.map(|f| f.unwrap())
.try_filter_map(noop)
.try_collect::<Vec<Vec<String>>>()
.map(|r| {
r.map(rank_eans).map(|eans| BestSellingRecord {
fetched_at: Utc::now(),
category: category.clone(),
eans,
})
})
})
.buffer_unordered(5)
.boxed()
.try_collect()
.await
}

111
scraper-rs/src/db.rs
View file

@ -1,111 +0,0 @@
use std::{
env,
str::FromStr,
time::{SystemTime, UNIX_EPOCH},
};
use sqlx::{sqlite::SqliteConnectOptions, SqlitePool};
use tracing::info;
use crate::{best_selling::BestSellingRecord, PrecioPoint};
#[derive(Clone)]
pub struct Db {
pool: SqlitePool,
}
impl Db {
pub async fn connect() -> anyhow::Result<Self> {
let db_path = env::var("DB_PATH").unwrap_or("../sqlite.db".to_string());
info!("Opening DB at {}", db_path);
let pool = sqlx::pool::PoolOptions::new()
.max_connections(1)
.connect_with(
SqliteConnectOptions::from_str(&format!("sqlite://{}", db_path))?
.journal_mode(sqlx::sqlite::SqliteJournalMode::Wal)
.synchronous(sqlx::sqlite::SqliteSynchronous::Normal)
.optimize_on_close(true, None),
)
.await?;
Ok(Self { pool })
}
pub async fn insert_precio(&self, point: PrecioPoint) -> anyhow::Result<()> {
sqlx::query!("INSERT INTO precios(ean, fetched_at, precio_centavos, in_stock, url, warc_record_id, parser_version, name, image_url) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9);",
point.ean,
point.fetched_at,
point.precio_centavos,
point.in_stock,
point.url,
None::<String>,
point.parser_version,
point.name,
point.image_url,
).execute(&self.pool).await?;
Ok(())
}
pub async fn get_ean_by_url(&self, url: &str) -> anyhow::Result<Option<String>> {
Ok(sqlx::query!("SELECT ean FROM precios WHERE url = ?1;", url)
.fetch_optional(&self.pool)
.await?
.map(|r| r.ean))
}
pub async fn get_urls_by_domain(&self, domain: &str) -> anyhow::Result<Vec<String>> {
let query = format!("%{}%", domain);
Ok(
sqlx::query!("SELECT url FROM producto_urls WHERE url LIKE ?1;", query)
.fetch_all(&self.pool)
.await?
.into_iter()
.map(|r| r.url)
.collect(),
)
}
pub async fn save_producto_urls(&self, urls: Vec<String>) -> anyhow::Result<()> {
let now: i64 = now_ms().try_into()?;
let mut tx = self.pool.begin().await?;
for url in urls {
sqlx::query!(
r#"INSERT INTO producto_urls(url, first_seen, last_seen)
VALUES (?1, ?2, ?2)
ON CONFLICT(url) DO UPDATE SET last_seen=?2;"#,
url,
now
)
.execute(&mut *tx)
.await?;
}
tx.commit().await?;
Ok(())
}
pub async fn save_best_selling(&self, records: Vec<BestSellingRecord>) -> anyhow::Result<()> {
let mut tx = self.pool.begin().await?;
for record in records {
let fetched_at = record.fetched_at.timestamp_millis();
let category = record.category.id();
let eans_json = serde_json::Value::from(record.eans).to_string();
sqlx::query!(
r#"INSERT INTO db_best_selling(fetched_at, category, eans_json)
VALUES (?1, ?2, ?3);"#,
fetched_at,
category,
eans_json
)
.execute(&mut *tx)
.await?;
}
tx.commit().await?;
Ok(())
}
}
fn now_ms() -> u128 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.expect("Time went backwards")
.as_millis()
}

228
scraper-rs/src/main.rs
View file

@ -1,11 +1,10 @@
use again::RetryPolicy;
use chrono::{DateTime, Utc};
use clap::{Parser, ValueEnum};
use cron::Schedule;
use db::Db;
use futures::{future, stream, Future, StreamExt};
use deadpool_sqlite::Pool;
use futures::{future, stream, StreamExt};
use nanoid::nanoid;
use reqwest::{header::HeaderMap, StatusCode, Url};
use reqwest::{StatusCode, Url};
use simple_error::{bail, SimpleError};
use std::{
env::{self},
@ -16,8 +15,23 @@ use std::{
};
use thiserror::Error;
mod supermercado;
use supermercado::Supermercado;
#[derive(ValueEnum, Clone, Debug)]
enum Supermercado {
Dia,
Jumbo,
Carrefour,
Coto,
}
impl Supermercado {
fn host(&self) -> &'static str {
match self {
Self::Dia => "diaonline.supermercadosdia.com.ar",
Self::Carrefour => "www.carrefour.com.ar",
Self::Coto => "www.cotodigital3.com.ar",
Self::Jumbo => "www.jumbo.com.ar",
}
}
}
#[derive(Parser)] // requires `derive` feature
enum Args {
@ -25,7 +39,6 @@ enum Args {
ParseFile(ParseFileArgs),
GetUrlList(GetUrlListArgs),
ScrapUrl(ScrapUrlArgs),
ScrapBestSelling,
Auto(AutoArgs),
Cron(AutoArgs),
}
@ -47,10 +60,7 @@ struct ScrapUrlArgs {
url: String,
}
#[derive(clap::Args)]
struct AutoArgs {
#[arg(long)]
n_products: Option<usize>,
}
struct AutoArgs {}
#[tokio::main]
async fn main() -> anyhow::Result<()> {
@ -61,8 +71,7 @@ async fn main() -> anyhow::Result<()> {
Args::ParseFile(a) => parse_file_cli(a.file_path).await,
Args::GetUrlList(a) => get_url_list_cli(a.supermercado).await,
Args::ScrapUrl(a) => scrap_url_cli(a.url).await,
Args::ScrapBestSelling => scrap_best_selling_cli().await,
Args::Auto(a) => auto_cli(a).await,
Args::Auto(_) => auto_cli().await,
Args::Cron(_) => cron_cli().await,
}
}
@ -74,14 +83,6 @@ async fn scrap_url_cli(url: String) -> anyhow::Result<()> {
println!("Result: {:#?}", res);
res.map(|_| ())
}
mod best_selling;
async fn scrap_best_selling_cli() -> anyhow::Result<()> {
let db = Db::connect().await?;
let res = best_selling::get_all_best_selling(&db).await;
println!("Result: {:#?}", res);
res.map(|_| ())
}
async fn fetch_list_cli(links_list_path: String) -> anyhow::Result<()> {
let links_str = fs::read_to_string(links_list_path).unwrap();
@ -92,14 +93,14 @@ async fn fetch_list_cli(links_list_path: String) -> anyhow::Result<()> {
.map(|s| s.to_owned())
.collect::<Vec<_>>();
let db = Db::connect().await?;
let counters = fetch_list(&db, links).await;
let pool = connect_db();
let counters = fetch_list(&pool, links).await;
println!("Finished: {:?}", counters);
Ok(())
}
async fn fetch_list(db: &Db, links: Vec<String>) -> Counters {
async fn fetch_list(pool: &Pool, links: Vec<String>) -> Counters {
let n_coroutines = env::var("N_COROUTINES")
.map_or(Ok(24), |s| s.parse::<usize>())
.expect("N_COROUTINES no es un número");
@ -108,9 +109,9 @@ async fn fetch_list(db: &Db, links: Vec<String>) -> Counters {
stream::iter(links)
.map(|url| {
let db = db.clone();
let pool = pool.clone();
let client = client.clone();
tokio::spawn(fetch_and_save(client, url, db))
tokio::spawn(fetch_and_save(client, url, pool))
})
.buffer_unordered(n_coroutines)
.fold(Counters::default(), move |x, y| {
@ -124,7 +125,19 @@ async fn fetch_list(db: &Db, links: Vec<String>) -> Counters {
.await
}
mod db;
fn connect_db() -> Pool {
let db_path = env::var("DB_PATH").unwrap_or("../sqlite.db".to_string());
let cfg = deadpool_sqlite::Config::new(db_path);
cfg.create_pool(deadpool_sqlite::Runtime::Tokio1).unwrap()
}
fn build_client() -> reqwest::Client {
reqwest::ClientBuilder::default()
.timeout(Duration::from_secs(60 * 5))
.connect_timeout(Duration::from_secs(60))
.build()
.unwrap()
}
#[derive(Default, Debug)]
struct Counters {
@ -133,13 +146,26 @@ struct Counters {
skipped: u64,
}
async fn fetch_and_save(client: reqwest::Client, url: String, db: Db) -> Counters {
async fn fetch_and_save(client: reqwest::Client, url: String, pool: Pool) -> Counters {
let res = fetch_and_parse(&client, url.clone()).await;
let mut counters = Counters::default();
match res {
Ok(res) => {
counters.success += 1;
db.insert_precio(res).await.unwrap();
pool.get().await.unwrap().interact(move |conn| conn.execute(
"INSERT INTO precios(ean, fetched_at, precio_centavos, in_stock, url, warc_record_id, parser_version, name, image_url) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9);",
rusqlite::params![
res.ean,
res.fetched_at,
res.precio_centavos,
res.in_stock,
res.url,
None::<String>,
res.parser_version,
res.name,
res.image_url,
]
)).await.unwrap().unwrap();
}
Err(err) => {
match err.downcast_ref::<reqwest::Error>() {
@ -164,35 +190,15 @@ enum FetchError {
Tl(#[from] tl::ParseError),
}
fn build_client() -> reqwest::Client {
let mut headers = HeaderMap::new();
headers.append("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36".parse().unwrap());
reqwest::ClientBuilder::default()
.timeout(Duration::from_secs(60))
.connect_timeout(Duration::from_secs(30))
.default_headers(headers)
.build()
.unwrap()
}
pub async fn do_request(client: &reqwest::Client, url: &str) -> reqwest::Result<reqwest::Response> {
let request = client.get(url).build()?;
let response = client.execute(request).await?.error_for_status()?;
Ok(response)
}
async fn request_and_body(client: &reqwest::Client, url: &str) -> reqwest::Result<String> {
let res = do_request(client, url).await?;
res.text().await
}
pub async fn fetch_body(client: &reqwest::Client, url: &str) -> reqwest::Result<String> {
get_retry_policy()
.retry_if(|| request_and_body(client, url), retry_if_wasnt_not_found)
.await
}
pub fn get_retry_policy() -> again::RetryPolicy {
RetryPolicy::exponential(Duration::from_millis(300))
.with_max_retries(20)
.with_max_delay(Duration::from_secs(40))
.with_max_retries(10)
.with_jitter(true)
}
@ -205,15 +211,18 @@ async fn fetch_and_parse(
client: &reqwest::Client,
url: String,
) -> Result<PrecioPoint, anyhow::Error> {
let body = fetch_body(client, &url).await?;
let body = get_retry_policy()
.retry_if(|| do_request(client, &url), retry_if_wasnt_not_found)
.await?
.text()
.await?;
let maybe_point = { scrap_url(client, url, &body).await };
let point = match maybe_point {
Ok(p) => Ok(p),
Err(err) => {
let now: DateTime<Utc> = Utc::now();
let debug_path = PathBuf::from(format!("debug-{}/", now.format("%Y-%m-%d")));
let debug_path = PathBuf::from("debug/");
tokio::fs::create_dir_all(&debug_path).await.unwrap();
let file_path = debug_path.join(format!("{}.html", nanoid!()));
tokio::fs::write(&file_path, &body).await.unwrap();
@ -294,9 +303,8 @@ struct AutoTelegram {
#[derive(Clone)]
struct Auto {
db: Db,
pool: Pool,
telegram: Option<AutoTelegram>,
limit_n_products: Option<usize>,
}
impl Auto {
async fn download_supermercado(self, supermercado: Supermercado) -> anyhow::Result<()> {
@ -311,11 +319,22 @@ impl Auto {
.await;
}
let links: Vec<String> = {
let mut links = self.db.get_urls_by_domain(supermercado.host()).await?;
if let Some(n) = self.limit_n_products {
links.truncate(n);
}
links
let search = format!("%{}%", supermercado.host());
self.pool
.get()
.await?
.interact(move |conn| -> anyhow::Result<Vec<String>> {
Ok(conn
.prepare(
r#"SELECT url FROM producto_urls
WHERE url LIKE ?1;"#,
)?
.query_map(rusqlite::params![search], |r| r.get::<_, String>(0))?
.map(|r| r.unwrap())
.collect())
})
.await
.unwrap()?
};
// {
// let debug_path = PathBuf::from("debug/");
@ -328,7 +347,7 @@ impl Auto {
// }
{
let t0 = now_sec();
let counters = fetch_list(&self.db, links).await;
let counters = fetch_list(&self.pool, links).await;
self.inform(&format!(
"Downloaded {:?}: {:?} (took {})",
&supermercado,
@ -337,26 +356,37 @@ impl Auto {
))
.await;
}
Ok(())
}
async fn inform_time<T: Future<Output = R>, R>(&self, msg: &str, action: T) -> R {
let t0 = now_sec();
let res = action.await;
self.inform(&format!("{} (took {})", msg, now_sec() - t0))
.await;
res
}
async fn get_and_save_urls(&self, supermercado: &Supermercado) -> anyhow::Result<()> {
let urls = get_urls(supermercado).await?;
self.db.save_producto_urls(urls).await?;
self.pool
.get()
.await?
.interact(|conn| -> Result<(), anyhow::Error> {
let tx = conn.transaction()?;
{
let mut stmt = tx.prepare(
r#"INSERT INTO producto_urls(url, first_seen, last_seen)
VALUES (?1, ?2, ?2)
ON CONFLICT(url) DO UPDATE SET last_seen=?2;"#,
)?;
let now: u64 = now_ms().try_into()?;
for url in urls {
stmt.execute(rusqlite::params![url, now])?;
}
}
tx.commit()?;
Ok(())
})
.await
.unwrap()?;
Ok(())
}
async fn inform(&self, msg: &str) {
tracing::info!("{}", msg);
println!("{}", msg);
if let Some(telegram) = &self.telegram {
let u = Url::parse_with_params(
&format!("https://api.telegram.org/bot{}/sendMessage", telegram.token),
@ -371,43 +401,27 @@ impl Auto {
}
}
async fn auto_cli(args: AutoArgs) -> anyhow::Result<()> {
let auto = {
let db = Db::connect().await?;
let telegram = {
match (
env::var("TELEGRAM_BOT_TOKEN"),
env::var("TELEGRAM_BOT_CHAT_ID"),
) {
(Ok(token), Ok(chat_id)) => Some(AutoTelegram { token, chat_id }),
_ => {
tracing::warn!("No token or chat_id for telegram");
None
}
async fn auto_cli() -> anyhow::Result<()> {
let db = connect_db();
let telegram = {
match (
env::var("TELEGRAM_BOT_TOKEN"),
env::var("TELEGRAM_BOT_CHAT_ID"),
) {
(Ok(token), Ok(chat_id)) => Some(AutoTelegram { token, chat_id }),
_ => {
tracing::warn!("No token or chat_id for telegram");
None
}
};
Auto {
db,
telegram,
limit_n_products: args.n_products,
}
};
let auto = Auto { pool: db, telegram };
auto.inform("[auto] Empezando scrap").await;
let handles: Vec<_> = Supermercado::value_variants()
.iter()
.map(|s| tokio::spawn(auto.clone().download_supermercado(s.to_owned())))
.collect();
future::try_join_all(handles).await?;
auto.inform("[auto] Download supermercados finished").await;
let best_selling = auto
.inform_time(
"Downloaded best selling",
best_selling::get_all_best_selling(&auto.db),
)
.await?;
auto.db.save_best_selling(best_selling).await?;
Ok(())
}
async fn cron_cli() -> anyhow::Result<()> {
@ -425,7 +439,7 @@ async fn cron_cli() -> anyhow::Result<()> {
.unwrap();
println!("Waiting for {:?}", t);
tokio::time::sleep(t).await;
auto_cli(AutoArgs { n_products: None }).await.unwrap();
auto_cli().await.unwrap();
}
}
@ -435,8 +449,8 @@ mod sites;
struct PrecioPoint {
ean: String,
// unix
fetched_at: i64,
precio_centavos: Option<i64>,
fetched_at: u64,
precio_centavos: Option<u64>,
in_stock: Option<bool>,
url: String,
parser_version: u16,
@ -444,9 +458,13 @@ struct PrecioPoint {
image_url: Option<String>,
}
fn now_sec() -> i64 {
since_the_epoch().as_secs().try_into().unwrap()
fn now_sec() -> u64 {
since_the_epoch().as_secs()
}
fn now_ms() -> u128 {
since_the_epoch().as_millis()
}
fn since_the_epoch() -> Duration {
SystemTime::now()
.duration_since(UNIX_EPOCH)

4
scraper-rs/src/sites/common.rs
View file

@ -11,9 +11,9 @@ pub fn get_meta_content<'a>(dom: &'a VDom<'a>, prop: &str) -> Option<Cow<'a, str
.map(|s| s.as_utf8_str())
}
pub fn price_from_meta(dom: &tl::VDom<'_>) -> Result<Option<i64>, anyhow::Error> {
pub fn price_from_meta(dom: &tl::VDom<'_>) -> Result<Option<u64>, anyhow::Error> {
let precio_centavos = get_meta_content(dom, "product:price:amount")
.map(|s| s.parse::<f64>().map(|f| (f * 100.0) as i64))
.map(|s| s.parse::<f64>().map(|f| (f * 100.0) as u64))
.transpose()?;
Ok(precio_centavos)
}

2
scraper-rs/src/sites/coto.rs
View file

@ -37,7 +37,7 @@ pub fn parse(url: String, dom: &tl::VDom) -> Result<PrecioPoint, anyhow::Error>
})
.transpose()
.context("Parseando precio")?
.map(|f| (f * 100.0) as i64);
.map(|f| (f * 100.0) as u64);
let in_stock = Some(
dom.query_selector(".product_not_available")

2
scraper-rs/src/sites/mod.rs
View file

@ -3,4 +3,4 @@ mod common;
pub mod coto;
pub mod dia;
pub mod jumbo;
pub mod vtex;
mod vtex;

109
scraper-rs/src/sites/vtex.rs
View file

@ -1,12 +1,7 @@
use std::str::FromStr;
use anyhow::{bail, Context};
use base64::Engine;
use futures::{stream, StreamExt, TryStreamExt};
use itertools::Itertools;
use reqwest::Url;
use serde::Deserialize;
use serde_json::json;
use simple_error::SimpleError;
use tl::VDom;
@ -105,7 +100,7 @@ pub fn in_stock_from_meta(dom: &VDom) -> anyhow::Result<bool> {
"instock" => true,
_ => bail!("Not a valid product:availability"),
},
None => bail!("No product:availability in vtex"),
None => bail!("No product:availability in carrefour"),
},
)
}
@ -151,101 +146,6 @@ pub async fn get_urls_from_sitemap(sitemaps: Vec<&str>) -> anyhow::Result<Vec<St
Ok(total.into_iter().unique().collect())
}
async fn fetch_body<'a>(client: &reqwest::Client, url: &str) -> anyhow::Result<String> {
let body = get_retry_policy()
.retry_if(|| do_request(client, url), retry_if_wasnt_not_found)
.await?
.text()
.await?;
Ok(body)
}
async fn get_binding_id(client: &reqwest::Client, url: &str) -> anyhow::Result<String> {
let body = fetch_body(client, url).await?;
let dom = tl::parse(&body, tl::ParserOptions::default())?;
let json = parse_script_json(&dom, "__RUNTIME__")?;
let id = json
.as_object()
.ok_or(SimpleError::new("RUNTIME not an object"))?
.get("binding")
.and_then(|v| v.as_object())
.and_then(|o| o.get("id"))
.and_then(|v| v.as_str())
.ok_or(SimpleError::new("binding.id does not exist"))?
.to_string();
Ok(id)
}
/// Returns a vec of product URLs
///
/// Estos parametros se consiguen yendo a una página como `https://www.jumbo.com.ar/almacen` y extrayendo:
/// * `domain` - www.jumbo.com.ar
/// * `query` - almacen
///
/// También `https://diaonline.supermercadosdia.com.ar/frescos/frutas-y-verduras`:
/// * `domain` - diaonline.supermercadosdia.com.ar
/// * `query` - frescos/frutas-y-verduras
pub async fn get_best_selling_by_category(
client: &reqwest::Client,
domain: &str,
query: &str,
) -> anyhow::Result<Vec<String>> {
let base_url = { Url::from_str(&format!("https://{}/{}", domain, query)).unwrap() };
let binding_id = get_binding_id(client, base_url.as_str()).await?;
let url = {
let mut url = base_url.clone();
url.set_path("/_v/segment/graphql/v1");
url.query_pairs_mut().append_pair("workspace", "master")
.append_pair("maxAge", "short")
.append_pair("appsEtag", "remove")
.append_pair("domain", "store")
.append_pair("locale", "es-AR")
.append_pair("__bindingId", &binding_id)
.append_pair("operationName", "productSearchV3")
.append_pair("variables", "%7B%7D")
.append_pair("extensions", &{
let variables_obj = json!({"hideUnavailableItems":true,"skusFilter":"FIRST_AVAILABLE","simulationBehavior":"default","installmentCriteria":"MAX_WITHOUT_INTEREST","productOriginVtex":false,"map":"c","query":query,"orderBy":"OrderByTopSaleDESC","from":0,"to":99,"selectedFacets":
query.split('/').map(|f| json!({"key":"c","value":f})).collect::<Vec<_>>()
,"facetsBehavior":"Static","categoryTreeBehavior":"default","withFacets":false});
let b64=base64::prelude::BASE64_STANDARD.encode(variables_obj.to_string());
format!(
r#"{{
"persistedQuery": {{
"version": 1,
"sha256Hash": "40b843ca1f7934d20d05d334916220a0c2cae3833d9f17bcb79cdd2185adceac",
"sender": "vtex.store-resources@0.x",
"provider": "vtex.search-graphql@0.x"
}},
"variables": "{}"
}}"#, b64
)
});
url
};
let body = fetch_body(client, url.as_str()).await?;
let urls: Vec<String> = serde_json::from_str::<serde_json::Value>(&body)?
.pointer("/data/productSearch/products")
.and_then(|v| v.as_array())
.map(|a| {
a.iter()
.filter_map(|p| {
p.get("link")
.and_then(|v| v.as_str())
.map(|s| format!("https://{}{}", domain, s))
})
.collect()
})
.ok_or(SimpleError::new("failed to get best selling product urls"))?;
if urls.len() < 2 {
bail!("Too few best selling");
}
Ok(urls)
}
#[cfg(test)]
mod tests {
use super::*;
@ -263,11 +163,4 @@ mod tests {
assert_eq!(links[0], "https://www.carrefour.com.ar/postre-danette-mousse-dulce-de-leche-80-g\u{200b}-684952/p");
Ok(())
}
#[tokio::test]
async fn test_jumbo_best_selling() -> anyhow::Result<()> {
get_best_selling_by_category(&build_client(), "www.jumbo.com.ar", "almacen").await?;
// assert_eq!(links[0], "https://www.carrefour.com.ar/postre-danette-mousse-dulce-de-leche-80-g\u{200b}-684952/p");
Ok(())
}
}

19
scraper-rs/src/supermercado.rs
View file

@ -1,19 +0,0 @@
use clap::ValueEnum;
#[derive(ValueEnum, Clone, Debug)]
pub enum Supermercado {
Dia,
Jumbo,
Carrefour,
Coto,
}
impl Supermercado {
pub fn host(&self) -> &'static str {
match self {
Self::Dia => "diaonline.supermercadosdia.com.ar",
Self::Carrefour => "www.carrefour.com.ar",
Self::Coto => "www.cotodigital3.com.ar",
Self::Jumbo => "www.jumbo.com.ar",
}
}
}

11
sitio/package.json
View file

@ -14,11 +14,8 @@
"format": "prettier --write ."
},
"devDependencies": {
"@sveltejs/adapter-node": "^2.0.2",
"@sveltejs/kit": "^2.0.0",
"@sveltejs/vite-plugin-svelte": "^3.0.0",
"@types/better-sqlite3": "^7.6.8",
"@types/node": "^20.10.6",
"autoprefixer": "^10.4.16",
"db-datos": "workspace:^",
"postcss": "^8.4.32",
@ -31,7 +28,10 @@
"tailwindcss": "^3.3.6",
"tslib": "^2.4.1",
"typescript": "^5.0.0",
"vite": "^5.0.3"
"vite": "^5.0.3",
"@sveltejs/adapter-node": "^2.0.2",
"@types/better-sqlite3": "^7.6.8",
"@types/node": "^20.10.6"
},
"type": "module",
"dependencies": {
@ -39,7 +39,6 @@
"chart.js": "^4.4.1",
"chartjs-adapter-dayjs-4": "^1.0.4",
"dayjs": "^1.11.10",
"drizzle-orm": "^0.29.1",
"zod": "^3.22.4"
"drizzle-orm": "^0.29.1"
}
}

4
sitio/src/app.pcss
View file

@ -2,7 +2,3 @@
@tailwind base;
@tailwind components;
@tailwind utilities;
:root {
color-scheme: light dark;
}

15
sitio/src/lib/ProductPreview.svelte
View file

@ -1,19 +1,10 @@
<script lang="ts" context="module">
export type Product = { ean: string; name: string; imageUrl: string | null };
</script>
<script lang="ts">
export let product: Product;
export let product: { ean: string; name: string; imageUrl?: string | null };
</script>
<a href={`/ean/${product.ean}`} class="flex gap-2">
<a href={`/ean/${product.ean}`} class="flex">
{#if product.imageUrl}
<img
src={product.imageUrl}
alt={product.name}
class="max-h-48"
loading="lazy"
/>
<img src={product.imageUrl} alt={product.name} class="max-h-48" />
{/if}
<p class="text-xl">{product.name}</p>
</a>

71
sitio/src/routes/+page.server.ts
View file

@ -1,55 +1,26 @@
import type { PageServerLoad } from "./$types";
import type { PageData, PageServerLoad } from "./$types";
import { getDb, schema } from "$lib/server/db";
const { precios, bestSelling } = schema;
import { max, sql } from "drizzle-orm";
import z from "zod";
import type { Product } from "$lib/ProductPreview.svelte";
const { precios } = schema;
import { sql } from "drizzle-orm";
type Data = {
category: string;
products: Product[];
}[];
let cache: Promise<{ key: Date; data: Data }> = doQuery();
let cache: Promise<{ key: Date; data: { precios: Precios } }> = doQuery();
async function doQuery() {
const db = await getDb();
const categories = await db
const q = db
.select({
fetchedAt: bestSelling.fetchedAt,
category: bestSelling.category,
eansJson: bestSelling.eansJson,
ean: precios.ean,
name: precios.name,
imageUrl: precios.imageUrl,
})
.from(bestSelling)
.groupBy(bestSelling.category)
.having(max(bestSelling.fetchedAt));
const categoriesWithProducts = await Promise.all(
categories.map(async (category) => {
const eans = z.array(z.string()).parse(JSON.parse(category.eansJson));
const products = await db
.select({
ean: precios.ean,
name: precios.name,
imageUrl: precios.imageUrl,
})
.from(precios)
.where(sql`${precios.ean} in ${eans}`)
.groupBy(precios.ean)
.having(max(precios.fetchedAt));
return {
category: category.category,
products: eans
.map((ean) => products.find((p) => p.ean === ean))
.filter((x): x is Product => !!x && !!x.name),
};
}),
);
return { key: new Date(), data: categoriesWithProducts };
.from(precios)
.groupBy(precios.ean)
.having(sql`max(length(name)) and max(parser_version) and in_stock`)
.orderBy(sql`random()`)
.limit(150);
const res = await q;
const data = { precios: res };
return { key: new Date(), data };
}
setInterval(
@ -60,8 +31,14 @@ setInterval(
4 * 60 * 60 * 1000,
);
type Precios = {
ean: string;
name: string | null;
imageUrl: string | null;
}[];
export const load: PageServerLoad = async ({
params,
}): Promise<{ data: Data }> => {
return { data: (await cache).data };
}): Promise<{ precios: Precios }> => {
return (await cache).data;
};

63
sitio/src/routes/+page.svelte
View file

@ -3,27 +3,46 @@
import type { PageData } from "./$types";
export let data: PageData;
const categoryLabels: { [key in string]: string } = {
almacen: "Almacen",
bebidas: "Bebidas",
"frutas-y-verduras": "Frutas y Verduras",
};
$: precios = data.precios.filter(
(d): d is { ean: string; name: string; imageUrl: string | null } =>
!!d.name,
);
</script>
{#each data.data as { category, products }}
<section class="my-6">
<h2 class="text-2xl font-bold">
{categoryLabels[category] ?? category}
</h2>
<ul
class="grid max-w-full grid-flow-col grid-rows-2 gap-x-8 gap-y-4 overflow-x-auto"
>
{#each products as product}
<li class="w-96">
<ProductPreview {product} />
</li>
{/each}
</ul>
</section>
{/each}
<h1 class="text-xl">WIP</h1>
<section>
<h2 class="text-lg font-bold">Ejemplos</h2>
<ul>
<li>
<a href="/ean/7790070410795">
Cookies Sabor Vainilla Con Chips De Chocolate Exquisita Paq 300 Grm
</a>
</li>
<li>
<a href="/ean/7794000006911">
Sopa Instantánea KNORR QUICK Zapallo Romero Sobres 5 Un.
</a>
</li>
<li>
<a href="/ean/7798062540253">Agua Saborizada Levité Pera 1,5 Lts.</a>
</li>
<li>
<a href="/ean/7790895000430">Gaseosa Coca-Cola Sabor Original 1,5 Lts.</a>
</li>
<li>
<a href="/ean/7792200000128">Bizcochos Agridulc 9 De Oro Paq 200 Grm</a>
</li>
</ul>
</section>
<section>
<h2 class="text-lg font-bold">Random</h2>
<ul class="grid grid-cols-1 gap-4 md:grid-cols-2 lg:grid-cols-3">
{#each precios as product}
<li>
<ProductPreview {product} />
</li>
{/each}
</ul>
</section>

2
sitio/src/routes/ean/[ean]/+page.server.ts
View file

@ -1,5 +1,5 @@
import { error } from "@sveltejs/kit";
import { eq } from "drizzle-orm";
import { eq, max } from "drizzle-orm";
import type { PageServerLoad } from "./$types";
import { getDb, schema } from "$lib/server/db";
const { precios } = schema;

1
sitio/src/routes/search/+page.server.ts
View file

@ -1,3 +1,4 @@
import { error } from "@sveltejs/kit";
import { sql } from "drizzle-orm";
import type { PageServerLoad } from "./$types";
import { getDb } from "$lib/server/db";