Nulo/tofufirme
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

vamaaa

Browse source
This commit is contained in:
Cat /dev/Nulo 2023-04-20 12:58:13 -03:00
commit 4e7cbc55ad
7 changed files with 471 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
.gitignore vendored Normal file
View file

@ -0,0 +1,5 @@
node_modules/
input*/
*.sqlite3
*.sqlite3-shm
*.sqlite3-wal

74
compress.js Normal file
View file

@ -0,0 +1,74 @@
import load from "better-sqlite3";
import { createReadStream } from "node:fs";
import { readFile, readdir, rm, stat } from "node:fs/promises";
import { extname, join } from "node:path";
import { Writable } from "node:stream";
import { pipeline } from "node:stream/promises";
import { promisify } from "node:util";
// import * as brotli from "brotli";
import { brotliCompress, constants, createBrotliCompress } from "node:zlib";
const compress = promisify(brotliCompress);
const name = process.argv[2] || "./archive.sqlite3";
await rm(name, { force: true });
const db = load(name);
db.pragma("journal_mode = WAL");
db.exec("create table files(path string, content blob, compressed bool)");
db.exec("create unique index path on files(path)");
const insertStmt = db.prepare(
"insert into files(path, content, compressed) values(?, ?, ?)"
);
/**
* @param {string} parent
* @param {string} strip
*/
async function recurse(parent, strip) {
const dir = await readdir(parent, { withFileTypes: true });
await Promise.all(
dir.map(async (entry) => {
const realPath = join(parent, entry.name);
if (entry.name === "pack.js") {
debugger;
}
if (entry.isFile()) {
if (!realPath.startsWith(strip)) throw new Error("wtf");
const virtualPath = realPath.slice(strip.length);
const file = createReadStream(realPath);
const ext = extname(entry.name).toLowerCase();
if ([".html", ".css", ".js", ".svg", ".json"].includes(ext)) {
try {
const brotli = createBrotliCompress({
params: {
[constants.BROTLI_PARAM_MODE]: constants.BROTLI_MODE_TEXT,
[constants.BROTLI_PARAM_QUALITY]: constants.BROTLI_MIN_QUALITY,
[constants.BROTLI_PARAM_SIZE_HINT]: (await stat(realPath)).size,
},
});
pipeline(file, brotli);
const buffers = [];
for await (const data of brotli) {
buffers.push(data);
}
const finalBuffer = Buffer.concat(buffers);
insertStmt.run(virtualPath, finalBuffer, 1);
} catch (error) {
console.error(error);
}
} else if ([".gz", ".br"].includes(ext)) {
// nada
} else {
file.close();
insertStmt.run(virtualPath, await readFile(realPath), 0);
}
} else {
await recurse(realPath, strip);
}
})
);
}
await recurse("input/", "input");

22
package.json Normal file
View file

@ -0,0 +1,22 @@
{
"name": "tofufirme",
"type": "module",
"version": "1.0.0",
"description": "",
"main": "index.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
"keywords": [],
"author": "",
"license": "ISC",
"dependencies": {
"better-sqlite3": "^8.3.0",
"brotli": "^1.3.3"
},
"devDependencies": {
"@types/better-sqlite3": "^7.6.4",
"@types/brotli": "^1.3.1",
"@types/node": "^18.15.12"
}
}

288
pnpm-lock.yaml Normal file
View file

@ -0,0 +1,288 @@
lockfileVersion: '6.0'
dependencies:
better-sqlite3:
specifier: ^8.3.0
version: 8.3.0
brotli:
specifier: ^1.3.3
version: 1.3.3
devDependencies:
'@types/better-sqlite3':
specifier: ^7.6.4
version: 7.6.4
'@types/brotli':
specifier: ^1.3.1
version: 1.3.1
'@types/node':
specifier: ^18.15.12
version: 18.15.12
packages:
/@types/better-sqlite3@7.6.4:
resolution: {integrity: sha512-dzrRZCYPXIXfSR1/surNbJ/grU3scTaygS0OMzjlGf71i9sc2fGyHPXXiXmEvNIoE0cGwsanEFMVJxPXmco9Eg==}
dependencies:
'@types/node': 18.15.12
dev: true
/@types/brotli@1.3.1:
resolution: {integrity: sha512-mGwX0BBQqmpHoX8+b8Oez0X+ZEYnl2gbDL2n0HxYT4imqhTChhj1AAgAKVWNZSuPvXGZXqVoOtBS0071tN6Tkw==}
dependencies:
'@types/node': 18.15.12
dev: true
/@types/node@18.15.12:
resolution: {integrity: sha512-Wha1UwsB3CYdqUm2PPzh/1gujGCNtWVUYF0mB00fJFoR4gTyWTDPjSm+zBF787Ahw8vSGgBja90MkgFwvB86Dg==}
dev: true
/base64-js@1.5.1:
resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==}
dev: false
/better-sqlite3@8.3.0:
resolution: {integrity: sha512-JTmvBZL/JLTc+3Msbvq6gK6elbU9/wVMqiudplHrVJpr7sVMR9KJrNhZAbW+RhXKlpMcuEhYkdcHa3TXKNXQ1w==}
requiresBuild: true
dependencies:
bindings: 1.5.0
prebuild-install: 7.1.1
dev: false
/bindings@1.5.0:
resolution: {integrity: sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==}
dependencies:
file-uri-to-path: 1.0.0
dev: false
/bl@4.1.0:
resolution: {integrity: sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==}
dependencies:
buffer: 5.7.1
inherits: 2.0.4
readable-stream: 3.6.2
dev: false
/brotli@1.3.3:
resolution: {integrity: sha512-oTKjJdShmDuGW94SyyaoQvAjf30dZaHnjJ8uAF+u2/vGJkJbJPJAT1gDiOJP5v1Zb6f9KEyW/1HpuaWIXtGHPg==}
dependencies:
base64-js: 1.5.1
dev: false
/buffer@5.7.1:
resolution: {integrity: sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==}
dependencies:
base64-js: 1.5.1
ieee754: 1.2.1
dev: false
/chownr@1.1.4:
resolution: {integrity: sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==}
dev: false
/decompress-response@6.0.0:
resolution: {integrity: sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==}
engines: {node: '>=10'}
dependencies:
mimic-response: 3.1.0
dev: false
/deep-extend@0.6.0:
resolution: {integrity: sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==}
engines: {node: '>=4.0.0'}
dev: false
/detect-libc@2.0.1:
resolution: {integrity: sha512-463v3ZeIrcWtdgIg6vI6XUncguvr2TnGl4SzDXinkt9mSLpBJKXT3mW6xT3VQdDN11+WVs29pgvivTc4Lp8v+w==}
engines: {node: '>=8'}
dev: false
/end-of-stream@1.4.4:
resolution: {integrity: sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==}
dependencies:
once: 1.4.0
dev: false
/expand-template@2.0.3:
resolution: {integrity: sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==}
engines: {node: '>=6'}
dev: false
/file-uri-to-path@1.0.0:
resolution: {integrity: sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==}
dev: false
/fs-constants@1.0.0:
resolution: {integrity: sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==}
dev: false
/github-from-package@0.0.0:
resolution: {integrity: sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==}
dev: false
/ieee754@1.2.1:
resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==}
dev: false
/inherits@2.0.4:
resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==}
dev: false
/ini@1.3.8:
resolution: {integrity: sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==}
dev: false
/lru-cache@6.0.0:
resolution: {integrity: sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==}
engines: {node: '>=10'}
dependencies:
yallist: 4.0.0
dev: false
/mimic-response@3.1.0:
resolution: {integrity: sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==}
engines: {node: '>=10'}
dev: false
/minimist@1.2.8:
resolution: {integrity: sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==}
dev: false
/mkdirp-classic@0.5.3:
resolution: {integrity: sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==}
dev: false
/napi-build-utils@1.0.2:
resolution: {integrity: sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==}
dev: false
/node-abi@3.40.0:
resolution: {integrity: sha512-zNy02qivjjRosswoYmPi8hIKJRr8MpQyeKT6qlcq/OnOgA3Rhoae+IYOqsM9V5+JnHWmxKnWOT2GxvtqdtOCXA==}
engines: {node: '>=10'}
dependencies:
semver: 7.5.0
dev: false
/once@1.4.0:
resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==}
dependencies:
wrappy: 1.0.2
dev: false
/prebuild-install@7.1.1:
resolution: {integrity: sha512-jAXscXWMcCK8GgCoHOfIr0ODh5ai8mj63L2nWrjuAgXE6tDyYGnx4/8o/rCgU+B4JSyZBKbeZqzhtwtC3ovxjw==}
engines: {node: '>=10'}
hasBin: true
dependencies:
detect-libc: 2.0.1
expand-template: 2.0.3
github-from-package: 0.0.0
minimist: 1.2.8
mkdirp-classic: 0.5.3
napi-build-utils: 1.0.2
node-abi: 3.40.0
pump: 3.0.0
rc: 1.2.8
simple-get: 4.0.1
tar-fs: 2.1.1
tunnel-agent: 0.6.0
dev: false
/pump@3.0.0:
resolution: {integrity: sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==}
dependencies:
end-of-stream: 1.4.4
once: 1.4.0
dev: false
/rc@1.2.8:
resolution: {integrity: sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==}
hasBin: true
dependencies:
deep-extend: 0.6.0
ini: 1.3.8
minimist: 1.2.8
strip-json-comments: 2.0.1
dev: false
/readable-stream@3.6.2:
resolution: {integrity: sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==}
engines: {node: '>= 6'}
dependencies:
inherits: 2.0.4
string_decoder: 1.3.0
util-deprecate: 1.0.2
dev: false
/safe-buffer@5.2.1:
resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==}
dev: false
/semver@7.5.0:
resolution: {integrity: sha512-+XC0AD/R7Q2mPSRuy2Id0+CGTZ98+8f+KvwirxOKIEyid+XSx6HbC63p+O4IndTHuX5Z+JxQ0TghCkO5Cg/2HA==}
engines: {node: '>=10'}
hasBin: true
dependencies:
lru-cache: 6.0.0
dev: false
/simple-concat@1.0.1:
resolution: {integrity: sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==}
dev: false
/simple-get@4.0.1:
resolution: {integrity: sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==}
dependencies:
decompress-response: 6.0.0
once: 1.4.0
simple-concat: 1.0.1
dev: false
/string_decoder@1.3.0:
resolution: {integrity: sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==}
dependencies:
safe-buffer: 5.2.1
dev: false
/strip-json-comments@2.0.1:
resolution: {integrity: sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==}
engines: {node: '>=0.10.0'}
dev: false
/tar-fs@2.1.1:
resolution: {integrity: sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==}
dependencies:
chownr: 1.1.4
mkdirp-classic: 0.5.3
pump: 3.0.0
tar-stream: 2.2.0
dev: false
/tar-stream@2.2.0:
resolution: {integrity: sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==}
engines: {node: '>=6'}
dependencies:
bl: 4.1.0
end-of-stream: 1.4.4
fs-constants: 1.0.0
inherits: 2.0.4
readable-stream: 3.6.2
dev: false
/tunnel-agent@0.6.0:
resolution: {integrity: sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==}
dependencies:
safe-buffer: 5.2.1
dev: false
/util-deprecate@1.0.2:
resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==}
dev: false
/wrappy@1.0.2:
resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==}
dev: false
/yallist@4.0.0:
resolution: {integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==}
dev: false

24
readme.md Normal file
View file

@ -0,0 +1,24 @@
# tofufirme [![justforfunnoreally.dev badge](https://img.shields.io/badge/justforfunnoreally-dev-9ff)](https://justforfunnoreally.dev)
![un tofu firme](https://labodeguitadelovalledor.cl/wp-content/uploads/2020/10/img_como_hacer_tofu_casero_32813_600_square-480x480.jpg)
un _proof of concept_ de un coso que guarda un sitio estático en una BD de sqlite3 con los archivos comprimidos con brotli, y otro coso que sirve el coso.
## ¿por qué?
porque la mayoría de los navegadores soportan brotli, y quizás no tiene tanto sentido guardar la versión descomprimida y la versión comprimida en gzip cuando la mayoría de los pedidos solo van a pedir la versión de brotli.
entonces si solo guardamos en brotli, podemos servir eso y _descomprimir en vivo_ si lx usuarix no soporta brotli (que, creo que la descompresión es más rápida que la compresión).
## ¿funciona?
este PoC funciona. [un sitio](https://keywords.sutty.nl) pesado en texto/HTML pero liviano en otros recursos pesa 79MB vs. 352MB (incluyendo .gz y .br).
## a ver...
```sh
pnpm install
# el sitio tiene que estar en input/
node compress.js output.sqlite3
node server.js output.sqlite3
```

45
server.js Normal file
View file

@ -0,0 +1,45 @@
import { createServer } from "http";
import load from "better-sqlite3";
import * as brotli from "brotli";
const name = process.argv[2] || "./archive.sqlite3";
const db = load(name);
const getStmt = db.prepare(
"select content, compressed from files where path = ?"
);
const port = 8080;
createServer((req, res) => {
let url = req.url;
if (!url) throw new Error("wtf");
{
const indexQ = url.indexOf("?");
if (indexQ !== -1) url = url.slice(0, indexQ);
}
if (url.endsWith("/")) url += "index.html";
url = decodeURI(url);
// prettier-ignore
const r = /** @type {{content:Buffer, compressed:boolean}} */(getStmt.get(url));
if (!r) {
res.writeHead(404).end("not found");
return;
}
if (r.compressed) {
let encodings = req.headers["accept-encoding"];
if (encodings instanceof Array) encodings = encodings[0];
if (encodings) encodings = encodings.split(", ");
if (encodings?.includes("br")) {
res
.writeHead(200, {
"content-encoding": "br",
})
.end(r.content);
} else {
res.writeHead(200).end(brotli.decompress(r.content));
}
} else {
res.writeHead(200).end(r.content);
}
}).listen(port, () => console.debug(`Listening in ${port}`));

13
tsconfig.json Normal file
View file

@ -0,0 +1,13 @@
{
"compilerOptions": {
"target": "ESNext",
"useDefineForClassFields": true,
"module": "ESNext",
"moduleResolution": "nodenext",
"resolveJsonModule": true,
"allowJs": true,
"checkJs": true,
"strict": true,
"isolatedModules": true
}
}