Compare commits

...

7 commits

Author SHA1 Message Date
d685ef8f75 bonk 2024-09-16 11:34:59 -03:00
85cb9a27bf arreglar index 2024-09-16 11:28:35 -03:00
8f573150c4 measure queries 2024-09-16 11:25:01 -03:00
d5ad504dc6 oops 2024-09-16 11:12:17 -03:00
f9c262b24b install ca-cert 2024-09-16 11:11:58 -03:00
6df4af108c sentry_auth_token 2024-09-16 11:09:06 -03:00
8a1ce924fe ci: forzar ip 2024-09-16 10:55:48 -03:00
6 changed files with 52 additions and 9 deletions

View file

@ -28,6 +28,9 @@ jobs:
B2_BUCKET_KEY_ID: ${{ secrets.B2_BUCKET_KEY_ID }} B2_BUCKET_KEY_ID: ${{ secrets.B2_BUCKET_KEY_ID }}
B2_BUCKET_KEY: ${{ secrets.B2_BUCKET_KEY }} B2_BUCKET_KEY: ${{ secrets.B2_BUCKET_KEY }}
run: | run: |
# usar un servidor especifico porque parece que a veces
# bloquean el acceso desde afuera del país
sudo echo "190.2.53.185 datos.produccion.gob.ar" | sudo tee -a /etc/hosts
cd sepa cd sepa
bun install --frozen-lockfile bun install --frozen-lockfile
bun archiver.ts bun archiver.ts

View file

@ -1,6 +1,7 @@
# use the official Bun image # use the official Bun image
# see all versions at https://hub.docker.com/r/oven/bun/tags # see all versions at https://hub.docker.com/r/oven/bun/tags
FROM oven/bun:1.1 as base FROM oven/bun:1.1 as base
RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates && rm -rf /var/lib/apt/lists/*
WORKDIR /usr/src/app WORKDIR /usr/src/app
@ -28,10 +29,12 @@ COPY db/schema.ts sitio2/src/lib/server/db/schema.ts
RUN cd sitio2 && bun install --frozen-lockfile RUN cd sitio2 && bun install --frozen-lockfile
ARG DATABASE_URL ARG DATABASE_URL
ARG SENTRY_AUTH_TOKEN
# [optional] tests & build # [optional] tests & build
ENV NODE_ENV=production ENV NODE_ENV=production
ENV DATABASE_URL=$DATABASE_URL ENV DATABASE_URL=$DATABASE_URL
ENV SENTRY_AUTH_TOKEN=$SENTRY_AUTH_TOKEN
# RUN bun test # RUN bun test
RUN cd sitio2 && env DATABASE_URL="postgres://user:password@host:5432/db-name" bun run build RUN cd sitio2 && env DATABASE_URL="postgres://user:password@host:5432/db-name" bun run build

Binary file not shown.

View file

@ -1,13 +1,29 @@
import { sql } from '$lib/server/db'; import { sql } from '$lib/server/db';
import type { PageServerLoad } from './$types'; import type { PageServerLoad } from './$types';
import * as Sentry from '@sentry/sveltekit';
export const load: PageServerLoad = async ({ setHeaders }) => { export const load: PageServerLoad = async ({ setHeaders }) => {
// https://www.cybertec-postgresql.com/en/postgresql-count-made-fast/ // https://www.cybertec-postgresql.com/en/postgresql-count-made-fast/
const count = await sql` const q = sql`
SELECT reltuples::bigint SELECT reltuples::bigint
FROM pg_catalog.pg_class FROM pg_catalog.pg_class
WHERE relname = 'precios'; WHERE relname = 'precios';
`; `;
// https://github.com/getsentry/sentry-javascript/discussions/8117#discussioncomment-7623605
const count = await Sentry.startSpan(
{
op: 'db.query',
name: ` SELECT reltuples::bigint
FROM pg_catalog.pg_class
WHERE relname = 'precios';`,
data: { 'db.system': 'postgresql' }
// these properties are important if you want to utilize Queries Performance
// read more: https://docs.sentry.io/product/performance/queries/#span-eligibility
// eslint-disable-next-line @typescript-eslint/no-explicit-any
} as any,
async () => await q
);
setHeaders({ setHeaders({
'Cache-Control': 'public, max-age=600' 'Cache-Control': 'public, max-age=600'

View file

@ -3,9 +3,10 @@ import type { PageServerLoad } from './$types';
import { datasets, precios, sucursales } from '$lib/server/db/schema'; import { datasets, precios, sucursales } from '$lib/server/db/schema';
import { and, eq, sql } from 'drizzle-orm'; import { and, eq, sql } from 'drizzle-orm';
import { error } from '@sveltejs/kit'; import { error } from '@sveltejs/kit';
import * as Sentry from '@sentry/sveltekit';
export const load: PageServerLoad = async ({ params, setHeaders }) => { export const load: PageServerLoad = async ({ params, setHeaders }) => {
const id = BigInt(params.id); const id = BigInt(params.id);
const preciosRes = await db const preciosQuery = db
.select({ .select({
id_comercio: precios.id_comercio, id_comercio: precios.id_comercio,
id_bandera: precios.id_bandera, id_bandera: precios.id_bandera,
@ -47,6 +48,15 @@ ORDER BY d1.id_comercio)
) )
) )
.leftJoin(datasets, eq(datasets.id, precios.id_dataset)); .leftJoin(datasets, eq(datasets.id, precios.id_dataset));
const preciosRes = await Sentry.startSpan(
{
op: 'db.query',
name: preciosQuery.toSQL().sql,
data: { 'db.system': 'postgresql' }
// eslint-disable-next-line @typescript-eslint/no-explicit-any
} as any,
() => preciosQuery
);
setHeaders({ setHeaders({
'Cache-Control': 'public, max-age=600' 'Cache-Control': 'public, max-age=600'

View file

@ -1,6 +1,7 @@
import { db } from '$lib/server/db'; import { db } from '$lib/server/db';
import { sql } from 'drizzle-orm'; import { sql } from 'drizzle-orm';
import type { PageServerLoad } from './$types'; import type { PageServerLoad } from './$types';
import * as Sentry from '@sentry/sveltekit';
export const load: PageServerLoad = async ({ params, setHeaders }) => { export const load: PageServerLoad = async ({ params, setHeaders }) => {
// const latestDatasetsSq = db.$with('latest_datasets').as( // const latestDatasetsSq = db.$with('latest_datasets').as(
@ -23,12 +24,7 @@ export const load: PageServerLoad = async ({ params, setHeaders }) => {
.replaceAll(/ó/giu, 'o') .replaceAll(/ó/giu, 'o')
.replaceAll(/ú/giu, 'u') .replaceAll(/ú/giu, 'u')
.replaceAll(/ñ/giu, 'n'); .replaceAll(/ñ/giu, 'n');
const productos = await db.execute<{ const productosQuery = sql`
id_producto: string;
productos_descripcion: string;
productos_marca: string | null;
in_datasets_count: number;
}>(sql`
SELECT id_producto, productos_descripcion, productos_marca, SELECT id_producto, productos_descripcion, productos_marca,
(WITH latest_datasets AS ( (WITH latest_datasets AS (
SELECT d1.id SELECT d1.id
@ -46,7 +42,22 @@ WHERE p.id_producto = index.id_producto) as in_datasets_count
WHERE productos_descripcion ILIKE ${`%${query}%`} WHERE productos_descripcion ILIKE ${`%${query}%`}
ORDER BY in_datasets_count desc ORDER BY in_datasets_count desc
LIMIT 100 LIMIT 100
`); `;
const productos = await Sentry.startSpan(
{
op: 'db.query',
name: productosQuery,
data: { 'db.system': 'postgresql' }
// eslint-disable-next-line @typescript-eslint/no-explicit-any
} as any,
() =>
db.execute<{
id_producto: string;
productos_descripcion: string;
productos_marca: string | null;
in_datasets_count: number;
}>(productosQuery)
);
const collapsedProductos = productos.reduce( const collapsedProductos = productos.reduce(
(acc, producto) => { (acc, producto) => {
const existingProduct = acc.find((p) => p.id_producto === producto.id_producto); const existingProduct = acc.find((p) => p.id_producto === producto.id_producto);