From 4e7cbc55ad04dadb1d4b6ccf7d7f838bbebe292d Mon Sep 17 00:00:00 2001 From: Nulo Date: Thu, 20 Apr 2023 12:58:13 -0300 Subject: [PATCH] vamaaa --- .gitignore | 5 + compress.js | 74 +++++++++++++ package.json | 22 ++++ pnpm-lock.yaml | 288 +++++++++++++++++++++++++++++++++++++++++++++++++ readme.md | 24 +++++ server.js | 45 ++++++++ tsconfig.json | 13 +++ 7 files changed, 471 insertions(+) create mode 100644 .gitignore create mode 100644 compress.js create mode 100644 package.json create mode 100644 pnpm-lock.yaml create mode 100644 readme.md create mode 100644 server.js create mode 100644 tsconfig.json diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..13dcb4c --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +node_modules/ +input*/ +*.sqlite3 +*.sqlite3-shm +*.sqlite3-wal diff --git a/compress.js b/compress.js new file mode 100644 index 0000000..261e779 --- /dev/null +++ b/compress.js @@ -0,0 +1,74 @@ +import load from "better-sqlite3"; +import { createReadStream } from "node:fs"; +import { readFile, readdir, rm, stat } from "node:fs/promises"; +import { extname, join } from "node:path"; +import { Writable } from "node:stream"; +import { pipeline } from "node:stream/promises"; +import { promisify } from "node:util"; +// import * as brotli from "brotli"; +import { brotliCompress, constants, createBrotliCompress } from "node:zlib"; +const compress = promisify(brotliCompress); + +const name = process.argv[2] || "./archive.sqlite3"; +await rm(name, { force: true }); +const db = load(name); +db.pragma("journal_mode = WAL"); +db.exec("create table files(path string, content blob, compressed bool)"); +db.exec("create unique index path on files(path)"); + +const insertStmt = db.prepare( + "insert into files(path, content, compressed) values(?, ?, ?)" +); + +/** + * @param {string} parent + * @param {string} strip + */ +async function recurse(parent, strip) { + const dir = await readdir(parent, { withFileTypes: true }); + await Promise.all( + dir.map(async (entry) => { + const realPath = join(parent, entry.name); + if (entry.name === "pack.js") { + debugger; + } + if (entry.isFile()) { + if (!realPath.startsWith(strip)) throw new Error("wtf"); + const virtualPath = realPath.slice(strip.length); + const file = createReadStream(realPath); + const ext = extname(entry.name).toLowerCase(); + if ([".html", ".css", ".js", ".svg", ".json"].includes(ext)) { + try { + const brotli = createBrotliCompress({ + params: { + [constants.BROTLI_PARAM_MODE]: constants.BROTLI_MODE_TEXT, + [constants.BROTLI_PARAM_QUALITY]: constants.BROTLI_MIN_QUALITY, + [constants.BROTLI_PARAM_SIZE_HINT]: (await stat(realPath)).size, + }, + }); + pipeline(file, brotli); + + const buffers = []; + for await (const data of brotli) { + buffers.push(data); + } + const finalBuffer = Buffer.concat(buffers); + + insertStmt.run(virtualPath, finalBuffer, 1); + } catch (error) { + console.error(error); + } + } else if ([".gz", ".br"].includes(ext)) { + // nada + } else { + file.close(); + insertStmt.run(virtualPath, await readFile(realPath), 0); + } + } else { + await recurse(realPath, strip); + } + }) + ); +} + +await recurse("input/", "input"); diff --git a/package.json b/package.json new file mode 100644 index 0000000..e9c3fa6 --- /dev/null +++ b/package.json @@ -0,0 +1,22 @@ +{ + "name": "tofufirme", + "type": "module", + "version": "1.0.0", + "description": "", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "keywords": [], + "author": "", + "license": "ISC", + "dependencies": { + "better-sqlite3": "^8.3.0", + "brotli": "^1.3.3" + }, + "devDependencies": { + "@types/better-sqlite3": "^7.6.4", + "@types/brotli": "^1.3.1", + "@types/node": "^18.15.12" + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml new file mode 100644 index 0000000..6fca26f --- /dev/null +++ b/pnpm-lock.yaml @@ -0,0 +1,288 @@ +lockfileVersion: '6.0' + +dependencies: + better-sqlite3: + specifier: ^8.3.0 + version: 8.3.0 + brotli: + specifier: ^1.3.3 + version: 1.3.3 + +devDependencies: + '@types/better-sqlite3': + specifier: ^7.6.4 + version: 7.6.4 + '@types/brotli': + specifier: ^1.3.1 + version: 1.3.1 + '@types/node': + specifier: ^18.15.12 + version: 18.15.12 + +packages: + + /@types/better-sqlite3@7.6.4: + resolution: {integrity: sha512-dzrRZCYPXIXfSR1/surNbJ/grU3scTaygS0OMzjlGf71i9sc2fGyHPXXiXmEvNIoE0cGwsanEFMVJxPXmco9Eg==} + dependencies: + '@types/node': 18.15.12 + dev: true + + /@types/brotli@1.3.1: + resolution: {integrity: sha512-mGwX0BBQqmpHoX8+b8Oez0X+ZEYnl2gbDL2n0HxYT4imqhTChhj1AAgAKVWNZSuPvXGZXqVoOtBS0071tN6Tkw==} + dependencies: + '@types/node': 18.15.12 + dev: true + + /@types/node@18.15.12: + resolution: {integrity: sha512-Wha1UwsB3CYdqUm2PPzh/1gujGCNtWVUYF0mB00fJFoR4gTyWTDPjSm+zBF787Ahw8vSGgBja90MkgFwvB86Dg==} + dev: true + + /base64-js@1.5.1: + resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==} + dev: false + + /better-sqlite3@8.3.0: + resolution: {integrity: sha512-JTmvBZL/JLTc+3Msbvq6gK6elbU9/wVMqiudplHrVJpr7sVMR9KJrNhZAbW+RhXKlpMcuEhYkdcHa3TXKNXQ1w==} + requiresBuild: true + dependencies: + bindings: 1.5.0 + prebuild-install: 7.1.1 + dev: false + + /bindings@1.5.0: + resolution: {integrity: sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==} + dependencies: + file-uri-to-path: 1.0.0 + dev: false + + /bl@4.1.0: + resolution: {integrity: sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==} + dependencies: + buffer: 5.7.1 + inherits: 2.0.4 + readable-stream: 3.6.2 + dev: false + + /brotli@1.3.3: + resolution: {integrity: sha512-oTKjJdShmDuGW94SyyaoQvAjf30dZaHnjJ8uAF+u2/vGJkJbJPJAT1gDiOJP5v1Zb6f9KEyW/1HpuaWIXtGHPg==} + dependencies: + base64-js: 1.5.1 + dev: false + + /buffer@5.7.1: + resolution: {integrity: sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==} + dependencies: + base64-js: 1.5.1 + ieee754: 1.2.1 + dev: false + + /chownr@1.1.4: + resolution: {integrity: sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==} + dev: false + + /decompress-response@6.0.0: + resolution: {integrity: sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==} + engines: {node: '>=10'} + dependencies: + mimic-response: 3.1.0 + dev: false + + /deep-extend@0.6.0: + resolution: {integrity: sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==} + engines: {node: '>=4.0.0'} + dev: false + + /detect-libc@2.0.1: + resolution: {integrity: sha512-463v3ZeIrcWtdgIg6vI6XUncguvr2TnGl4SzDXinkt9mSLpBJKXT3mW6xT3VQdDN11+WVs29pgvivTc4Lp8v+w==} + engines: {node: '>=8'} + dev: false + + /end-of-stream@1.4.4: + resolution: {integrity: sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==} + dependencies: + once: 1.4.0 + dev: false + + /expand-template@2.0.3: + resolution: {integrity: sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==} + engines: {node: '>=6'} + dev: false + + /file-uri-to-path@1.0.0: + resolution: {integrity: sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==} + dev: false + + /fs-constants@1.0.0: + resolution: {integrity: sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==} + dev: false + + /github-from-package@0.0.0: + resolution: {integrity: sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==} + dev: false + + /ieee754@1.2.1: + resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==} + dev: false + + /inherits@2.0.4: + resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==} + dev: false + + /ini@1.3.8: + resolution: {integrity: sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==} + dev: false + + /lru-cache@6.0.0: + resolution: {integrity: sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==} + engines: {node: '>=10'} + dependencies: + yallist: 4.0.0 + dev: false + + /mimic-response@3.1.0: + resolution: {integrity: sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==} + engines: {node: '>=10'} + dev: false + + /minimist@1.2.8: + resolution: {integrity: sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==} + dev: false + + /mkdirp-classic@0.5.3: + resolution: {integrity: sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==} + dev: false + + /napi-build-utils@1.0.2: + resolution: {integrity: sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==} + dev: false + + /node-abi@3.40.0: + resolution: {integrity: sha512-zNy02qivjjRosswoYmPi8hIKJRr8MpQyeKT6qlcq/OnOgA3Rhoae+IYOqsM9V5+JnHWmxKnWOT2GxvtqdtOCXA==} + engines: {node: '>=10'} + dependencies: + semver: 7.5.0 + dev: false + + /once@1.4.0: + resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} + dependencies: + wrappy: 1.0.2 + dev: false + + /prebuild-install@7.1.1: + resolution: {integrity: sha512-jAXscXWMcCK8GgCoHOfIr0ODh5ai8mj63L2nWrjuAgXE6tDyYGnx4/8o/rCgU+B4JSyZBKbeZqzhtwtC3ovxjw==} + engines: {node: '>=10'} + hasBin: true + dependencies: + detect-libc: 2.0.1 + expand-template: 2.0.3 + github-from-package: 0.0.0 + minimist: 1.2.8 + mkdirp-classic: 0.5.3 + napi-build-utils: 1.0.2 + node-abi: 3.40.0 + pump: 3.0.0 + rc: 1.2.8 + simple-get: 4.0.1 + tar-fs: 2.1.1 + tunnel-agent: 0.6.0 + dev: false + + /pump@3.0.0: + resolution: {integrity: sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==} + dependencies: + end-of-stream: 1.4.4 + once: 1.4.0 + dev: false + + /rc@1.2.8: + resolution: {integrity: sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==} + hasBin: true + dependencies: + deep-extend: 0.6.0 + ini: 1.3.8 + minimist: 1.2.8 + strip-json-comments: 2.0.1 + dev: false + + /readable-stream@3.6.2: + resolution: {integrity: sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==} + engines: {node: '>= 6'} + dependencies: + inherits: 2.0.4 + string_decoder: 1.3.0 + util-deprecate: 1.0.2 + dev: false + + /safe-buffer@5.2.1: + resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==} + dev: false + + /semver@7.5.0: + resolution: {integrity: sha512-+XC0AD/R7Q2mPSRuy2Id0+CGTZ98+8f+KvwirxOKIEyid+XSx6HbC63p+O4IndTHuX5Z+JxQ0TghCkO5Cg/2HA==} + engines: {node: '>=10'} + hasBin: true + dependencies: + lru-cache: 6.0.0 + dev: false + + /simple-concat@1.0.1: + resolution: {integrity: sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==} + dev: false + + /simple-get@4.0.1: + resolution: {integrity: sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==} + dependencies: + decompress-response: 6.0.0 + once: 1.4.0 + simple-concat: 1.0.1 + dev: false + + /string_decoder@1.3.0: + resolution: {integrity: sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==} + dependencies: + safe-buffer: 5.2.1 + dev: false + + /strip-json-comments@2.0.1: + resolution: {integrity: sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==} + engines: {node: '>=0.10.0'} + dev: false + + /tar-fs@2.1.1: + resolution: {integrity: sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==} + dependencies: + chownr: 1.1.4 + mkdirp-classic: 0.5.3 + pump: 3.0.0 + tar-stream: 2.2.0 + dev: false + + /tar-stream@2.2.0: + resolution: {integrity: sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==} + engines: {node: '>=6'} + dependencies: + bl: 4.1.0 + end-of-stream: 1.4.4 + fs-constants: 1.0.0 + inherits: 2.0.4 + readable-stream: 3.6.2 + dev: false + + /tunnel-agent@0.6.0: + resolution: {integrity: sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==} + dependencies: + safe-buffer: 5.2.1 + dev: false + + /util-deprecate@1.0.2: + resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} + dev: false + + /wrappy@1.0.2: + resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} + dev: false + + /yallist@4.0.0: + resolution: {integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==} + dev: false diff --git a/readme.md b/readme.md new file mode 100644 index 0000000..ec40bbf --- /dev/null +++ b/readme.md @@ -0,0 +1,24 @@ +# tofufirme [![justforfunnoreally.dev badge](https://img.shields.io/badge/justforfunnoreally-dev-9ff)](https://justforfunnoreally.dev) + +![un tofu firme](https://labodeguitadelovalledor.cl/wp-content/uploads/2020/10/img_como_hacer_tofu_casero_32813_600_square-480x480.jpg) + +un _proof of concept_ de un coso que guarda un sitio estático en una BD de sqlite3 con los archivos comprimidos con brotli, y otro coso que sirve el coso. + +## ¿por qué? + +porque la mayoría de los navegadores soportan brotli, y quizás no tiene tanto sentido guardar la versión descomprimida y la versión comprimida en gzip cuando la mayoría de los pedidos solo van a pedir la versión de brotli. + +entonces si solo guardamos en brotli, podemos servir eso y _descomprimir en vivo_ si lx usuarix no soporta brotli (que, creo que la descompresión es más rápida que la compresión). + +## ¿funciona? + +este PoC funciona. [un sitio](https://keywords.sutty.nl) pesado en texto/HTML pero liviano en otros recursos pesa 79MB vs. 352MB (incluyendo .gz y .br). + +## a ver... + +```sh +pnpm install +# el sitio tiene que estar en input/ +node compress.js output.sqlite3 +node server.js output.sqlite3 +``` diff --git a/server.js b/server.js new file mode 100644 index 0000000..9392448 --- /dev/null +++ b/server.js @@ -0,0 +1,45 @@ +import { createServer } from "http"; +import load from "better-sqlite3"; +import * as brotli from "brotli"; + +const name = process.argv[2] || "./archive.sqlite3"; +const db = load(name); +const getStmt = db.prepare( + "select content, compressed from files where path = ?" +); + +const port = 8080; +createServer((req, res) => { + let url = req.url; + if (!url) throw new Error("wtf"); + { + const indexQ = url.indexOf("?"); + if (indexQ !== -1) url = url.slice(0, indexQ); + } + if (url.endsWith("/")) url += "index.html"; + url = decodeURI(url); + + // prettier-ignore + const r = /** @type {{content:Buffer, compressed:boolean}} */(getStmt.get(url)); + if (!r) { + res.writeHead(404).end("not found"); + return; + } + + if (r.compressed) { + let encodings = req.headers["accept-encoding"]; + if (encodings instanceof Array) encodings = encodings[0]; + if (encodings) encodings = encodings.split(", "); + if (encodings?.includes("br")) { + res + .writeHead(200, { + "content-encoding": "br", + }) + .end(r.content); + } else { + res.writeHead(200).end(brotli.decompress(r.content)); + } + } else { + res.writeHead(200).end(r.content); + } +}).listen(port, () => console.debug(`Listening in ${port}`)); diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..08b3bc9 --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,13 @@ +{ + "compilerOptions": { + "target": "ESNext", + "useDefineForClassFields": true, + "module": "ESNext", + "moduleResolution": "nodenext", + "resolveJsonModule": true, + "allowJs": true, + "checkJs": true, + "strict": true, + "isolatedModules": true + } +}