Compare commits

...

3 commits

Author SHA1 Message Date
a1fa2796ef sitio: adaptar homepage para usar api 2024-08-04 12:55:20 -03:00
f01213aaf8 gitignore 2024-08-04 12:49:40 -03:00
b02c3f84c5 more wip 2024-08-04 12:49:37 -03:00
9 changed files with 170 additions and 166 deletions

2
.gitignore vendored
View file

@ -5,3 +5,5 @@ node_modules/
target/
.env.*
.DS_Store

View file

@ -39,9 +39,6 @@ importers:
dayjs:
specifier: ^1.11.10
version: 1.11.10
drizzle-kit:
specifier: ^0.23.0
version: 0.23.0
drizzle-orm:
specifier: ^0.32.0
version: 0.32.0(@types/better-sqlite3@7.6.9)(better-sqlite3@11.1.2)

View file

@ -6,7 +6,7 @@
{
"name": "count",
"ordinal": 0,
"type_info": "Int"
"type_info": "Integer"
}
],
"parameters": {

View file

@ -6,7 +6,7 @@
{
"name": "count",
"ordinal": 0,
"type_info": "Int"
"type_info": "Integer"
}
],
"parameters": {

134
rust/Cargo.lock generated
View file

@ -35,7 +35,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011"
dependencies = [
"cfg-if",
"getrandom 0.2.15",
"once_cell",
"version_check",
"zerocopy",
@ -163,7 +162,7 @@ checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.71",
"syn",
]
[[package]]
@ -386,10 +385,10 @@ version = "4.5.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2bac35c6dafb060fd4d275d9a4ffae97917c13a6327903a8be2153cd964f7085"
dependencies = [
"heck 0.5.0",
"heck",
"proc-macro2",
"quote",
"syn 2.0.71",
"syn",
]
[[package]]
@ -404,6 +403,15 @@ version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422"
[[package]]
name = "concurrent-queue"
version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973"
dependencies = [
"crossbeam-utils",
]
[[package]]
name = "const-oid"
version = "0.9.6"
@ -552,9 +560,14 @@ dependencies = [
[[package]]
name = "event-listener"
version = "2.5.3"
version = "5.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0"
checksum = "6032be9bd27023a771701cc49f9f053c751055f71efb2e0ae5c15809093675ba"
dependencies = [
"concurrent-queue",
"parking",
"pin-project-lite",
]
[[package]]
name = "fastrand"
@ -665,7 +678,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.71",
"syn",
]
[[package]]
@ -748,22 +761,13 @@ dependencies = [
[[package]]
name = "hashlink"
version = "0.8.4"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7"
checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af"
dependencies = [
"hashbrown",
]
[[package]]
name = "heck"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
dependencies = [
"unicode-segmentation",
]
[[package]]
name = "heck"
version = "0.5.0"
@ -1033,9 +1037,9 @@ checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058"
[[package]]
name = "libsqlite3-sys"
version = "0.27.0"
version = "0.28.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf4e226dcd58b4be396f7bd3c20da8fdee2911400705297ba7d2d7cc2c30f716"
checksum = "0c10584274047cb335c23d3e61bcef8e323adae7c5c8c760540f73610177fc3f"
dependencies = [
"cc",
"pkg-config",
@ -1234,6 +1238,12 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39"
[[package]]
name = "parking"
version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae"
[[package]]
name = "parking_lot"
version = "0.11.2"
@ -1320,7 +1330,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.71",
"syn",
]
[[package]]
@ -1794,7 +1804,7 @@ checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.71",
"syn",
]
[[package]]
@ -1900,6 +1910,9 @@ name = "smallvec"
version = "1.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67"
dependencies = [
"serde",
]
[[package]]
name = "socket2"
@ -1942,9 +1955,9 @@ dependencies = [
[[package]]
name = "sqlx"
version = "0.7.4"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c9a2ccff1a000a5a59cd33da541d9f2fdcd9e6e8229cc200565942bff36d0aaa"
checksum = "27144619c6e5802f1380337a209d2ac1c431002dd74c6e60aebff3c506dc4f0c"
dependencies = [
"sqlx-core",
"sqlx-macros",
@ -1955,11 +1968,10 @@ dependencies = [
[[package]]
name = "sqlx-core"
version = "0.7.4"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "24ba59a9342a3d9bab6c56c118be528b27c9b60e490080e9711a04dccac83ef6"
checksum = "a999083c1af5b5d6c071d34a708a19ba3e02106ad82ef7bbd69f5e48266b613b"
dependencies = [
"ahash",
"atoi",
"byteorder",
"bytes",
@ -1973,6 +1985,7 @@ dependencies = [
"futures-intrusive",
"futures-io",
"futures-util",
"hashbrown",
"hashlink",
"hex",
"indexmap",
@ -1995,26 +2008,26 @@ dependencies = [
[[package]]
name = "sqlx-macros"
version = "0.7.4"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4ea40e2345eb2faa9e1e5e326db8c34711317d2b5e08d0d5741619048a803127"
checksum = "a23217eb7d86c584b8cbe0337b9eacf12ab76fe7673c513141ec42565698bb88"
dependencies = [
"proc-macro2",
"quote",
"sqlx-core",
"sqlx-macros-core",
"syn 1.0.109",
"syn",
]
[[package]]
name = "sqlx-macros-core"
version = "0.7.4"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5833ef53aaa16d860e92123292f1f6a3d53c34ba8b1969f152ef1a7bb803f3c8"
checksum = "1a099220ae541c5db479c6424bdf1b200987934033c2584f79a0e1693601e776"
dependencies = [
"dotenvy",
"either",
"heck 0.4.1",
"heck",
"hex",
"once_cell",
"proc-macro2",
@ -2026,7 +2039,7 @@ dependencies = [
"sqlx-mysql",
"sqlx-postgres",
"sqlx-sqlite",
"syn 1.0.109",
"syn",
"tempfile",
"tokio",
"url",
@ -2034,12 +2047,12 @@ dependencies = [
[[package]]
name = "sqlx-mysql"
version = "0.7.4"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ed31390216d20e538e447a7a9b959e06ed9fc51c37b514b46eb758016ecd418"
checksum = "5afe4c38a9b417b6a9a5eeffe7235d0a106716495536e7727d1c7f4b1ff3eba6"
dependencies = [
"atoi",
"base64 0.21.7",
"base64 0.22.1",
"bitflags 2.6.0",
"byteorder",
"bytes",
@ -2077,12 +2090,12 @@ dependencies = [
[[package]]
name = "sqlx-postgres"
version = "0.7.4"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c824eb80b894f926f89a0b9da0c7f435d27cdd35b8c655b114e58223918577e"
checksum = "b1dbb157e65f10dbe01f729339c06d239120221c9ad9fa0ba8408c4cc18ecf21"
dependencies = [
"atoi",
"base64 0.21.7",
"base64 0.22.1",
"bitflags 2.6.0",
"byteorder",
"chrono",
@ -2116,9 +2129,9 @@ dependencies = [
[[package]]
name = "sqlx-sqlite"
version = "0.7.4"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b244ef0a8414da0bed4bb1910426e890b19e5e9bccc27ada6b797d05c55ae0aa"
checksum = "9b2cdd83c008a622d94499c0006d8ee5f821f36c89b7d625c900e5dc30b5c5ee"
dependencies = [
"atoi",
"chrono",
@ -2132,10 +2145,10 @@ dependencies = [
"log",
"percent-encoding",
"serde",
"serde_urlencoded",
"sqlx-core",
"tracing",
"url",
"urlencoding",
]
[[package]]
@ -2161,17 +2174,6 @@ version = "2.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
[[package]]
name = "syn"
version = "1.0.109"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
dependencies = [
"proc-macro2",
"quote",
"unicode-ident",
]
[[package]]
name = "syn"
version = "2.0.71"
@ -2224,7 +2226,7 @@ checksum = "d20468752b09f49e909e55a5d338caa8bedf615594e9d80bc4c565d30faf798c"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.71",
"syn",
]
[[package]]
@ -2285,7 +2287,7 @@ checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.71",
"syn",
]
[[package]]
@ -2383,7 +2385,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.71",
"syn",
]
[[package]]
@ -2464,12 +2466,6 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e4259d9d4425d9f0661581b804cb85fe66a4c631cadd8f490d1c13a35d5d9291"
[[package]]
name = "unicode-segmentation"
version = "1.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202"
[[package]]
name = "unicode_categories"
version = "0.1.1"
@ -2493,12 +2489,6 @@ dependencies = [
"percent-encoding",
]
[[package]]
name = "urlencoding"
version = "2.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da"
[[package]]
name = "utf8-width"
version = "0.1.7"
@ -2577,7 +2567,7 @@ dependencies = [
"once_cell",
"proc-macro2",
"quote",
"syn 2.0.71",
"syn",
"wasm-bindgen-shared",
]
@ -2611,7 +2601,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.71",
"syn",
"wasm-bindgen-backend",
"wasm-bindgen-shared",
]
@ -2863,7 +2853,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.71",
"syn",
]
[[package]]

View file

@ -12,7 +12,7 @@ base64 = "0.21.7"
chrono = "0.4"
clap = { version = "4.4.15", features = ["derive"] }
cron = "0.12.0"
sqlx = { version = "0.7", features = [ "runtime-tokio", "sqlite", "chrono" ] }
sqlx = { version = "0.8", features = [ "runtime-tokio", "sqlite", "chrono", "json" ] }
futures = "0.3.30"
html-escape = "0.2.13"
itertools = "0.12.0"

View file

@ -1,4 +1,4 @@
use axum::{extract::State, http::StatusCode, response::IntoResponse, routing::get, Router};
use axum::{extract::State, http::StatusCode, response::IntoResponse, routing::get, Json, Router};
use clap::ValueEnum;
use futures::future::join_all;
use itertools::Itertools;
@ -96,34 +96,76 @@ async fn healthcheck(State(pool): State<SqlitePool>) -> impl IntoResponse {
}
#[derive(Serialize)]
struct CategoryWithProducts {}
struct CategoryWithProducts {
category: String,
products: Vec<Product>,
}
#[derive(Serialize)]
struct Product {
ean: String,
name: Option<String>,
image_url: Option<String>,
}
async fn get_best_selling(State(pool): State<SqlitePool>) -> impl IntoResponse {
let categories = sqlx::query!(
"SELECT fetched_at, category, eans_json FROM db_best_selling
GROUP BY category
HAVING MAX(fetched_at)",
#[derive(sqlx::FromRow, Debug)]
struct ProductWithCategory {
category: String,
ean: String,
name: Option<String>,
image_url: Option<String>,
}
let products_with_category = sqlx::query_as::<_, ProductWithCategory>(
"with latest_best_selling as (
select category, eans_json
from db_best_selling
group by category
having max(fetched_at)
),
extracted_eans as (
select latest_best_selling.category, json.value as ean
from latest_best_selling, json_each(latest_best_selling.eans_json) json
)
select extracted_eans.category, extracted_eans.ean, precios.image_url, name
from extracted_eans
join precios
on extracted_eans.ean = precios.ean
where
precios.fetched_at = (
SELECT MAX(fetched_at)
FROM precios
WHERE ean = extracted_eans.ean
)",
)
.fetch_all(&pool)
.await
.unwrap();
categories.iter().map(|category| {
let eans =
serde_json::de::from_str::<Vec<String>>(&category.eans_json.clone().unwrap()).unwrap();
let products = sqlx::query!(
"SELECT ean, name, image_url FROM precios
WHERE ean in (?)
GROUP BY ean
HAVING MAX(fetched_at)",
eans,
)
.fetch_all(&pool)
.await
.unwrap();
});
let categories = products_with_category
.iter()
.map(|p| p.category.clone())
.unique()
.collect_vec();
todo!()
let categories_with_products = categories
.into_iter()
.map(|c| CategoryWithProducts {
category: c.clone(),
products: products_with_category
.iter()
.filter(|p| p.category == c)
.map(|p| Product {
ean: p.ean.clone(),
image_url: p.image_url.clone(),
name: p.name.clone(),
})
.collect_vec(),
})
.collect_vec();
Json(categories_with_products)
}
#[tokio::main]
@ -131,7 +173,7 @@ async fn main() {
tracing_subscriber::fmt::init();
let pool = SqlitePoolOptions::new()
.max_connections(1)
.max_connections(10)
.connect_with(
SqliteConnectOptions::from_str(&format!(
"sqlite://{}",
@ -139,12 +181,26 @@ async fn main() {
))
.unwrap()
.journal_mode(sqlx::sqlite::SqliteJournalMode::Wal)
.busy_timeout(Duration::from_secs(15))
.synchronous(sqlx::sqlite::SqliteSynchronous::Normal)
.busy_timeout(Duration::from_secs(30))
.optimize_on_close(true, None),
)
.await
.expect("can't connect to database");
sqlx::query("pragma temp_store = memory;")
.execute(&pool)
.await
.unwrap();
sqlx::query("pragma mmap_size = 30000000000;")
.execute(&pool)
.await
.unwrap();
sqlx::query("pragma page_size = 4096;")
.execute(&pool)
.await
.unwrap();
let app = Router::new()
.route("/", get(index))
.route("/api/healthcheck", get(healthcheck))

View file

@ -1,5 +1,5 @@
<script lang="ts" context="module">
export type Product = { ean: string; name: string; imageUrl: string | null };
export type Product = { ean: string; name: string; image_url: string | null };
</script>
<script lang="ts">
@ -7,9 +7,9 @@
</script>
<a href={`/ean/${product.ean}`} class="flex gap-2">
{#if product.imageUrl}
{#if product.image_url}
<img
src={product.imageUrl}
src={product.image_url}
alt={product.name}
class="max-h-48"
loading="lazy"

View file

@ -1,68 +1,27 @@
import type { PageServerLoad } from "./$types";
import { getDb, schema } from "$lib/server/db";
const { precios, bestSelling } = schema;
import { max, sql } from "drizzle-orm";
import z from "zod";
import type { Product } from "$lib/ProductPreview.svelte";
type Data = {
category: string;
products: Product[];
}[];
let cache: Promise<{ key: Date; data: Data }> = doQuery();
async function doQuery() {
const db = await getDb();
const categories = await db
.select({
fetchedAt: bestSelling.fetchedAt,
category: bestSelling.category,
eansJson: bestSelling.eansJson,
})
.from(bestSelling)
.groupBy(bestSelling.category)
.having(max(bestSelling.fetchedAt));
const categoriesWithProducts = await Promise.all(
categories.map(async (category) => {
const eans = z.array(z.string()).parse(JSON.parse(category.eansJson));
const products = await db
.select({
ean: precios.ean,
name: precios.name,
imageUrl: precios.imageUrl,
})
.from(precios)
.where(sql`${precios.ean} in ${eans}`)
.groupBy(precios.ean)
.having(max(precios.fetchedAt));
return {
category: category.category,
products: eans
.map((ean) => products.find((p) => p.ean === ean))
.filter((x): x is Product => !!x && !!x.name),
};
async function getBestSelling() {
const res = await fetch("http://localhost:8000/api/0/best-selling-products");
const json = await res.json();
return z
.array(
z.object({
category: z.string(),
products: z.array(
z.object({
ean: z.string(),
name: z.string().nullable(),
image_url: z.string().nullable(),
}),
);
return { key: new Date(), data: categoriesWithProducts };
),
}),
)
.parse(json);
}
console.log("setting up interval");
setInterval(
async () => {
const c = await doQuery();
cache = Promise.resolve(c);
},
4 * 60 * 60 * 1000,
);
export const load: PageServerLoad = async ({
params,
}): Promise<{ data: Data }> => {
return { data: (await cache).data };
export const load: PageServerLoad = async ({ params }) => {
return {
data: await getBestSelling(),
};
};