From ce19fa4ec9e3edcca9887558687652e12b104784 Mon Sep 17 00:00:00 2001 From: Nulo Date: Mon, 19 Jun 2023 15:52:50 -0300 Subject: [PATCH] tar stream 3.1 bug repro --- .gitignore | 4 + js/.gitignore | 2 + js/container-baby.ts | 198 ++++++++++++++++++++++++++ js/fetch-hack.d.ts | 29 ++++ js/index.js | 143 +++++++++++++++++++ js/package.json | 26 ++++ js/pnpm-lock.yaml | 327 +++++++++++++++++++++++++++++++++++++++++++ js/tar2squashfs.js | 71 ++++++++++ js/tsconfig.json | 11 ++ 9 files changed, 811 insertions(+) create mode 100644 js/.gitignore create mode 100644 js/container-baby.ts create mode 100644 js/fetch-hack.d.ts create mode 100644 js/index.js create mode 100644 js/package.json create mode 100644 js/pnpm-lock.yaml create mode 100644 js/tar2squashfs.js create mode 100644 js/tsconfig.json diff --git a/.gitignore b/.gitignore index e2d40fa..68bb3c3 100644 --- a/.gitignore +++ b/.gitignore @@ -2,5 +2,9 @@ rootfs.ext4 rootfs.qcow2 fireactions *.ext4 +*.squashfs +*.config.json vmlinux.bin node_modules/ + +*.log diff --git a/js/.gitignore b/js/.gitignore new file mode 100644 index 0000000..9bec8f9 --- /dev/null +++ b/js/.gitignore @@ -0,0 +1,2 @@ +# compiled typescript +container-baby.js diff --git a/js/container-baby.ts b/js/container-baby.ts new file mode 100644 index 0000000..92df0aa --- /dev/null +++ b/js/container-baby.ts @@ -0,0 +1,198 @@ +import { execFile as _execFile } from "node:child_process"; +import { mkdtemp, open, rm } from "node:fs/promises"; +import { join } from "node:path"; +import { Writable } from "node:stream"; +import { promisify } from "node:util"; +import { tar2squashfs } from "./tar2squashfs.js"; +const execFile = promisify(_execFile); + +type RegistryName = string; +type RegistrySecret = string; +let tokenCache = new Map>(); + +const tmp = await mkdtemp("sexy-chambelan-"); +try { + console.debug(tmp); + + const image = "gitea.nulo.in/nulo/zulip-checkin-cyborg"; + const tag = "latest"; + const arch = "amd64"; + + const manifest = await getContainerManifest(image, tag); + if ( + !manifest.layers.every( + (layer) => + layer.mediaType === "application/vnd.oci.image.layer.v1.tar" || + layer.mediaType === "application/vnd.oci.image.layer.v1.tar+gzip" + ) + ) { + throw new Error("Unsupported layer"); + } + console.debug(manifest); + const layers = await Promise.all( + manifest.layers.map(async (layer) => { + const path = join(tmp, layer.digest); + await downloadBlob(image, layer.digest, path); + return { layer, path }; + }) + ); + const squashfs = `${image.replaceAll("/", "%")}.squashfs`; + await rm(squashfs, { force: true }); + await tar2squashfs( + layers.map(async ({ path }) => { + const f = await open(path); + return f.createReadStream(); + }), + squashfs + ); + await downloadBlob( + image, + manifest.config.digest, + `${image.replaceAll("/", "%")}.config.json` + ); + process.exit(1); +} finally { + await rm(tmp, { recursive: true }); +} + +async function _getToken(registryUrl: string): Promise { + const res = await fetch(`${registryUrl}/token`); + const json = await res.json(); + if ( + !json || + typeof json !== "object" || + !("token" in json) || + typeof json.token !== "string" + ) + throw new Error("Unexpected"); + return json.token; +} + +async function call(registryName: string, path: string, init: RequestInit) { + const registryUrl = `https://${registryName}/v2`; + let tokenPromise = tokenCache.get(registryName); + if (!tokenPromise) { + tokenPromise = _getToken(registryUrl); + tokenCache.set(registryName, tokenPromise); + } + const token = await tokenPromise; + console.debug(path); + const res = await fetch(`${registryUrl}${path}`, { + ...init, + headers: { + ...init.headers, + Authorization: `Bearer ${token}`, + }, + }); + if (!res.ok) throw new Error(res.statusText); + return res; +} + +type ManifestPointer = { + mediaType: "application/vnd.oci.image.manifest.v1+json"; + digest: string; + size: number; + platform: { + architecture: "amd64" | string; + os: string; + }; + annotations?: { [k: string]: string }; +}; + +// https://github.com/opencontainers/image-spec/blob/v1.0.1/image-index.md +type ManifestIndex = { + mediaType: "application/vnd.oci.image.index.v1+json"; + schemaVersion: 2; + manifests: ManifestPointer[]; +}; + +function getImageParts(image: string): [string, string] { + const parts = image.split("/"); + const registryName = parts[0]; + const imgPath = parts.slice(1).join("/"); + return [registryName, imgPath]; +} + +async function getContainerManifest(image: string, tag: string) { + const index = await getManifestIndex(image, tag); + // badly behaved registries will return whatever they want + if ( + (index.mediaType as string) === "application/vnd.oci.image.manifest.v1+json" + ) { + return index as unknown as Manifest; + } + const arch = "amd64"; + const ptr = chooseManifest(index, arch); + if (!ptr) + throw new Error(`Image ${image}:${tag} doesn't exist for arch ${arch}`); + const manifest = await getManifest(image, ptr); + return manifest; +} + +async function getManifestIndex( + image: string, + tag: string +): Promise { + const [registryName, imgPath] = getImageParts(image); + + const res = await call(registryName, `/${imgPath}/manifests/${tag}`, { + headers: { + Accept: "application/vnd.oci.image.index.v1+json", + }, + }); + const json = await res.json(); + return json as ManifestIndex; +} + +function chooseManifest(list: ManifestIndex, arch: string) { + console.debug(list); + return list.manifests.find( + (m) => m.platform.architecture === arch && m.platform.os === "linux" + ); +} + +async function getBlob(image: string, digest: string): Promise { + const [registryName, imgPath] = getImageParts(image); + return await call(registryName, `/${imgPath}/blobs/${digest}`, {}); +} +async function jsonBlob(image: string, digest: string): Promise { + const res = await getBlob(image, digest); + return (await res.json()) as T; +} +async function downloadBlob( + image: string, + digest: string, + outputPath: string +): Promise { + const res = await getBlob(image, digest); + const file = await open(outputPath, "w"); + await res.body!.pipeTo(Writable.toWeb(file.createWriteStream())); + return; +} + +// https://github.com/opencontainers/image-spec/blob/v1.0.1/descriptor.md#properties +type Descriptor = { + digest: string; + size: number; + urls?: string[]; + annotations?: { [k: string]: string }; +}; + +// https://github.com/opencontainers/image-spec/blob/v1.0.1/manifest.md +type Manifest = { + mediaType: "application/vnd.oci.image.manifest.v1+json"; + config: { + mediaType: "application/vnd.oci.image.config.v1+json"; + } & Descriptor; + layers: ({ + mediaType: + | "application/vnd.oci.image.layer.v1.tar" + | "application/vnd.oci.image.layer.v1.tar+gzip" + | "application/vnd.oci.image.layer.nondistributable.v1.tar" + | "application/vnd.oci.image.layer.nondistributable.v1.tar+gzip"; + } & Descriptor)[]; +}; + +async function getManifest(image: string, ptr: ManifestPointer) { + return await jsonBlob(image, ptr.digest); +} diff --git a/js/fetch-hack.d.ts b/js/fetch-hack.d.ts new file mode 100644 index 0000000..7a56e95 --- /dev/null +++ b/js/fetch-hack.d.ts @@ -0,0 +1,29 @@ +// https://stackoverflow.com/a/75676044 + +import * as undici_types from "undici"; + +declare global { + export const { + fetch, + FormData, + Headers, + Request, + Response, + }: typeof import("undici"); + + type FormData = undici_types.FormData; + type Headers = undici_types.Headers; + type HeadersInit = undici_types.HeadersInit; + type BodyInit = undici_types.BodyInit; + type Request = undici_types.Request; + type RequestInit = undici_types.RequestInit; + type RequestInfo = undici_types.RequestInfo; + type RequestMode = undici_types.RequestMode; + type RequestRedirect = undici_types.RequestRedirect; + type RequestCredentials = undici_types.RequestCredentials; + type RequestDestination = undici_types.RequestDestination; + type ReferrerPolicy = undici_types.ReferrerPolicy; + type Response = undici_types.Response; + type ResponseInit = undici_types.ResponseInit; + type ResponseType = undici_types.ResponseType; +} diff --git a/js/index.js b/js/index.js new file mode 100644 index 0000000..46b2939 --- /dev/null +++ b/js/index.js @@ -0,0 +1,143 @@ +import { delay } from "nanodelay"; +import { nanoid } from "nanoid"; +import { ChildProcess, execFile, spawn } from "node:child_process"; +import { writeFile } from "node:fs/promises"; +import { createServer, request as _request, IncomingMessage } from "node:http"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; + +const server = createServer(listener); + +/** + * @param {import("node:http").IncomingMessage} req + * @param {import("node:http").ServerResponse} res + */ +async function listener(req, res) { + const url = getUrl(req); + if (url.pathname == "/run") { + if (req.method == "POST") { + await runVm(); + } else res.writeHead(405).end("wrong method"); + } else res.writeHead(404).end("not found"); +} + +async function runVm() { + try { + await FirecrackerInstance.run(); + } catch (err) { + console.error(err); + } +} + +class FirecrackerInstance { + proc; + socketPath; + constructor() { + const vmid = nanoid(); + this.socketPath = join(tmpdir(), `firecracker-${vmid}.sock`); + console.debug(this.socketPath); + // TODO: jailer + this.proc = spawn( + "firecracker", + [ + ...["--api-sock", this.socketPath], + ...["--level", "Debug"], + ...["--log-path", "/dev/stderr"], + "--show-level", + "--show-log-origin", + ], + { + stdio: "inherit", + } + ); + } + static async run() { + const self = new FirecrackerInstance(); + + await self.request("PUT", "/boot-source", { + kernel_image_path: "../vmlinux.bin", + boot_args: "console=ttyS0 reboot=k panic=1 pci=off", + }); + await self.request("PUT", "/drives/rootfs", { + drive_id: "rootfs", + path_on_host: "../rootfs.ext4", + is_root_device: true, + is_read_only: false, + }); + + // API requests are handled asynchronously, it is important the configuration is + // set, before `InstanceStart`. + await delay(15); + + await self.request("PUT", "/actions", { + action_type: "InstanceStart", + }); + } + + /** + * @param {string} method + * @param {string} path + * @param {object} body + */ + async request(method, path, body) { + const jsonBody = JSON.stringify(body); + const { response, body: respBody } = await request( + { + method, + path, + socketPath: this.socketPath, + headers: { + "Content-Type": "application/json", + "Content-Length": jsonBody.length, + }, + }, + jsonBody + ); + // @ts-ignore + if (response.statusCode > 299) + throw new Error( + `Status code: ${response.statusCode} / Body: ${respBody}`, + { + cause: `${method} ${path} ${JSON.stringify(body, null, 4)}`, + } + ); + } +} + +/** + * @param {import("http").IncomingMessage} req + */ +function getUrl(req) { + // prettier-ignore + const urlString = /** @type {string} */(req.url); + const url = new URL(urlString, `http://${req.headers.host}`); + return url; +} + +/** + * @param {string | import("url").URL | import("http").RequestOptions} opts + * @param {string} [body] + * @returns {Promise<{ response: import("node:http").IncomingMessage, body: string }>} + */ +function request(opts, body) { + return new Promise((resolve, reject) => { + const req = _request(opts, (res) => { + let respBody = ""; + res.on("data", (data) => { + respBody += data; + }); + res.on("end", () => { + resolve({ response: res, body: respBody }); + }); + }); + req.on("error", reject); + if (body) + req.write(body, (error) => { + if (error) reject(error); + }); + req.end(); + }); +} + +server.listen("8080"); diff --git a/js/package.json b/js/package.json new file mode 100644 index 0000000..0bb3377 --- /dev/null +++ b/js/package.json @@ -0,0 +1,26 @@ +{ + "name": "js", + "type": "module", + "version": "1.0.0", + "description": "", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "keywords": [], + "author": "", + "license": "ISC", + "devDependencies": { + "@tsconfig/node16": "^1.0.4", + "@types/gunzip-maybe": "^1.4.0", + "@types/node": "^20.2.5", + "@types/tar-stream": "^2.2.2", + "undici": "^5.22.1" + }, + "dependencies": { + "gunzip-maybe": "^1.4.2", + "nanodelay": "^2.0.2", + "nanoid": "^4.0.2", + "tar-stream": "^3.1.2" + } +} diff --git a/js/pnpm-lock.yaml b/js/pnpm-lock.yaml new file mode 100644 index 0000000..6f6c695 --- /dev/null +++ b/js/pnpm-lock.yaml @@ -0,0 +1,327 @@ +lockfileVersion: '6.1' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +dependencies: + gunzip-maybe: + specifier: ^1.4.2 + version: 1.4.2 + nanodelay: + specifier: ^2.0.2 + version: 2.0.2 + nanoid: + specifier: ^4.0.2 + version: 4.0.2 + tar: + specifier: ^6.1.15 + version: 6.1.15 + tar-stream: + specifier: ^3.1.2 + version: 3.1.2 + +devDependencies: + '@tsconfig/node16': + specifier: ^1.0.4 + version: 1.0.4 + '@types/gunzip-maybe': + specifier: ^1.4.0 + version: 1.4.0 + '@types/node': + specifier: ^20.2.5 + version: 20.2.5 + '@types/tar': + specifier: ^6.1.5 + version: 6.1.5 + '@types/tar-stream': + specifier: ^2.2.2 + version: 2.2.2 + undici: + specifier: ^5.22.1 + version: 5.22.1 + +packages: + + /@tsconfig/node16@1.0.4: + resolution: {integrity: sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==} + dev: true + + /@types/gunzip-maybe@1.4.0: + resolution: {integrity: sha512-dFP9GrYAR9KhsjTkWJ8q8Gsfql75YIKcg9DuQOj/IrlPzR7W+1zX+cclw1McV82UXAQ+Lpufvgk3e9bC8+HzgA==} + dependencies: + '@types/node': 20.2.5 + dev: true + + /@types/node@20.2.5: + resolution: {integrity: sha512-JJulVEQXmiY9Px5axXHeYGLSjhkZEnD+MDPDGbCbIAbMslkKwmygtZFy1X6s/075Yo94sf8GuSlFfPzysQrWZQ==} + dev: true + + /@types/tar-stream@2.2.2: + resolution: {integrity: sha512-1AX+Yt3icFuU6kxwmPakaiGrJUwG44MpuiqPg4dSolRFk6jmvs4b3IbUol9wKDLIgU76gevn3EwE8y/DkSJCZQ==} + dependencies: + '@types/node': 20.2.5 + dev: true + + /@types/tar@6.1.5: + resolution: {integrity: sha512-qm2I/RlZij5RofuY7vohTpYNaYcrSQlN2MyjucQc7ZweDwaEWkdN/EeNh6e9zjK6uEm6PwjdMXkcj05BxZdX1Q==} + dependencies: + '@types/node': 20.2.5 + minipass: 4.2.8 + dev: true + + /b4a@1.6.4: + resolution: {integrity: sha512-fpWrvyVHEKyeEvbKZTVOeZF3VSKKWtJxFIxX/jaVPf+cLbGUSitjb49pHLqPV2BUNNZ0LcoeEGfE/YCpyDYHIw==} + dev: false + + /browserify-zlib@0.1.4: + resolution: {integrity: sha512-19OEpq7vWgsH6WkvkBJQDFvJS1uPcbFOQ4v9CU839dO+ZZXUZO6XpE6hNCqvlIIj+4fZvRiJ6DsAQ382GwiyTQ==} + dependencies: + pako: 0.2.9 + dev: false + + /buffer-from@1.1.2: + resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==} + dev: false + + /busboy@1.6.0: + resolution: {integrity: sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==} + engines: {node: '>=10.16.0'} + dependencies: + streamsearch: 1.1.0 + dev: true + + /chownr@2.0.0: + resolution: {integrity: sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==} + engines: {node: '>=10'} + dev: false + + /core-util-is@1.0.3: + resolution: {integrity: sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==} + dev: false + + /duplexify@3.7.1: + resolution: {integrity: sha512-07z8uv2wMyS51kKhD1KsdXJg5WQ6t93RneqRxUHnskXVtlYYkLqM0gqStQZ3pj073g687jPCHrqNfCzawLYh5g==} + dependencies: + end-of-stream: 1.4.4 + inherits: 2.0.4 + readable-stream: 2.3.8 + stream-shift: 1.0.1 + dev: false + + /end-of-stream@1.4.4: + resolution: {integrity: sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==} + dependencies: + once: 1.4.0 + dev: false + + /fast-fifo@1.2.0: + resolution: {integrity: sha512-NcvQXt7Cky1cNau15FWy64IjuO8X0JijhTBBrJj1YlxlDfRkJXNaK9RFUjwpfDPzMdv7wB38jr53l9tkNLxnWg==} + dev: false + + /fs-minipass@2.1.0: + resolution: {integrity: sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==} + engines: {node: '>= 8'} + dependencies: + minipass: 3.3.6 + dev: false + + /gunzip-maybe@1.4.2: + resolution: {integrity: sha512-4haO1M4mLO91PW57BMsDFf75UmwoRX0GkdD+Faw+Lr+r/OZrOCS0pIBwOL1xCKQqnQzbNFGgK2V2CpBUPeFNTw==} + hasBin: true + dependencies: + browserify-zlib: 0.1.4 + is-deflate: 1.0.0 + is-gzip: 1.0.0 + peek-stream: 1.1.3 + pumpify: 1.5.1 + through2: 2.0.5 + dev: false + + /inherits@2.0.4: + resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==} + dev: false + + /is-deflate@1.0.0: + resolution: {integrity: sha512-YDoFpuZWu1VRXlsnlYMzKyVRITXj7Ej/V9gXQ2/pAe7X1J7M/RNOqaIYi6qUn+B7nGyB9pDXrv02dsB58d2ZAQ==} + dev: false + + /is-gzip@1.0.0: + resolution: {integrity: sha512-rcfALRIb1YewtnksfRIHGcIY93QnK8BIQ/2c9yDYcG/Y6+vRoJuTWBmmSEbyLLYtXm7q35pHOHbZFQBaLrhlWQ==} + engines: {node: '>=0.10.0'} + dev: false + + /isarray@1.0.0: + resolution: {integrity: sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==} + dev: false + + /minipass@3.3.6: + resolution: {integrity: sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==} + engines: {node: '>=8'} + dependencies: + yallist: 4.0.0 + dev: false + + /minipass@4.2.8: + resolution: {integrity: sha512-fNzuVyifolSLFL4NzpF+wEF4qrgqaaKX0haXPQEdQ7NKAN+WecoKMHV09YcuL/DHxrUsYQOK3MiuDf7Ip2OXfQ==} + engines: {node: '>=8'} + dev: true + + /minipass@5.0.0: + resolution: {integrity: sha512-3FnjYuehv9k6ovOEbyOswadCDPX1piCfhV8ncmYtHOjuPwylVWsghTLo7rabjC3Rx5xD4HDx8Wm1xnMF7S5qFQ==} + engines: {node: '>=8'} + dev: false + + /minizlib@2.1.2: + resolution: {integrity: sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==} + engines: {node: '>= 8'} + dependencies: + minipass: 3.3.6 + yallist: 4.0.0 + dev: false + + /mkdirp@1.0.4: + resolution: {integrity: sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==} + engines: {node: '>=10'} + hasBin: true + dev: false + + /nanodelay@2.0.2: + resolution: {integrity: sha512-6AS5aCSXsjoxq2Jr9CdaAeT60yoYDOTp6po9ziqeOeY6vf6uTEHYSqWql6EFILrM3fEfXgkZ4KqE9L0rTm/wlA==} + engines: {node: ^12.0.0 || ^14.0.0 || >=16.0.0} + dev: false + + /nanoid@4.0.2: + resolution: {integrity: sha512-7ZtY5KTCNheRGfEFxnedV5zFiORN1+Y1N6zvPTnHQd8ENUvfaDBeuJDZb2bN/oXwXxu3qkTXDzy57W5vAmDTBw==} + engines: {node: ^14 || ^16 || >=18} + hasBin: true + dev: false + + /once@1.4.0: + resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} + dependencies: + wrappy: 1.0.2 + dev: false + + /pako@0.2.9: + resolution: {integrity: sha512-NUcwaKxUxWrZLpDG+z/xZaCgQITkA/Dv4V/T6bw7VON6l1Xz/VnrBqrYjZQ12TamKHzITTfOEIYUj48y2KXImA==} + dev: false + + /peek-stream@1.1.3: + resolution: {integrity: sha512-FhJ+YbOSBb9/rIl2ZeE/QHEsWn7PqNYt8ARAY3kIgNGOk13g9FGyIY6JIl/xB/3TFRVoTv5as0l11weORrTekA==} + dependencies: + buffer-from: 1.1.2 + duplexify: 3.7.1 + through2: 2.0.5 + dev: false + + /process-nextick-args@2.0.1: + resolution: {integrity: sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==} + dev: false + + /pump@2.0.1: + resolution: {integrity: sha512-ruPMNRkN3MHP1cWJc9OWr+T/xDP0jhXYCLfJcBuX54hhfIBnaQmAUMfDcG4DM5UMWByBbJY69QSphm3jtDKIkA==} + dependencies: + end-of-stream: 1.4.4 + once: 1.4.0 + dev: false + + /pumpify@1.5.1: + resolution: {integrity: sha512-oClZI37HvuUJJxSKKrC17bZ9Cu0ZYhEAGPsPUy9KlMUmv9dKX2o77RUmq7f3XjIxbwyGwYzbzQ1L2Ks8sIradQ==} + dependencies: + duplexify: 3.7.1 + inherits: 2.0.4 + pump: 2.0.1 + dev: false + + /queue-tick@1.0.1: + resolution: {integrity: sha512-kJt5qhMxoszgU/62PLP1CJytzd2NKetjSRnyuj31fDd3Rlcz3fzlFdFLD1SItunPwyqEOkca6GbV612BWfaBag==} + dev: false + + /readable-stream@2.3.8: + resolution: {integrity: sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==} + dependencies: + core-util-is: 1.0.3 + inherits: 2.0.4 + isarray: 1.0.0 + process-nextick-args: 2.0.1 + safe-buffer: 5.1.2 + string_decoder: 1.1.1 + util-deprecate: 1.0.2 + dev: false + + /safe-buffer@5.1.2: + resolution: {integrity: sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==} + dev: false + + /stream-shift@1.0.1: + resolution: {integrity: sha512-AiisoFqQ0vbGcZgQPY1cdP2I76glaVA/RauYR4G4thNFgkTqr90yXTo4LYX60Jl+sIlPNHHdGSwo01AvbKUSVQ==} + dev: false + + /streamsearch@1.1.0: + resolution: {integrity: sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==} + engines: {node: '>=10.0.0'} + dev: true + + /streamx@2.15.0: + resolution: {integrity: sha512-HcxY6ncGjjklGs1xsP1aR71INYcsXFJet5CU1CHqihQ2J5nOsbd4OjgjHO42w/4QNv9gZb3BueV+Vxok5pLEXg==} + dependencies: + fast-fifo: 1.2.0 + queue-tick: 1.0.1 + dev: false + + /string_decoder@1.1.1: + resolution: {integrity: sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==} + dependencies: + safe-buffer: 5.1.2 + dev: false + + /tar-stream@3.1.2: + resolution: {integrity: sha512-rEZHMKQop/sTykFtONCcOxyyLA+9hriHJlECxfh4z+Nfew87XGprDLp9RYFCd7yU+z3ePXfHlPbZrzgSLvc16A==} + dependencies: + b4a: 1.6.4 + streamx: 2.15.0 + dev: false + + /tar@6.1.15: + resolution: {integrity: sha512-/zKt9UyngnxIT/EAGYuxaMYgOIJiP81ab9ZfkILq4oNLPFX50qyYmu7jRj9qeXoxmJHjGlbH0+cm2uy1WCs10A==} + engines: {node: '>=10'} + dependencies: + chownr: 2.0.0 + fs-minipass: 2.1.0 + minipass: 5.0.0 + minizlib: 2.1.2 + mkdirp: 1.0.4 + yallist: 4.0.0 + dev: false + + /through2@2.0.5: + resolution: {integrity: sha512-/mrRod8xqpA+IHSLyGCQ2s8SPHiCDEeQJSep1jqLYeEUClOFG2Qsh+4FU6G9VeqpZnGW/Su8LQGc4YKni5rYSQ==} + dependencies: + readable-stream: 2.3.8 + xtend: 4.0.2 + dev: false + + /undici@5.22.1: + resolution: {integrity: sha512-Ji2IJhFXZY0x/0tVBXeQwgPlLWw13GVzpsWPQ3rV50IFMMof2I55PZZxtm4P6iNq+L5znYN9nSTAq0ZyE6lSJw==} + engines: {node: '>=14.0'} + dependencies: + busboy: 1.6.0 + dev: true + + /util-deprecate@1.0.2: + resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} + dev: false + + /wrappy@1.0.2: + resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} + dev: false + + /xtend@4.0.2: + resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==} + engines: {node: '>=0.4'} + dev: false + + /yallist@4.0.0: + resolution: {integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==} + dev: false diff --git a/js/tar2squashfs.js b/js/tar2squashfs.js new file mode 100644 index 0000000..dbfa2d1 --- /dev/null +++ b/js/tar2squashfs.js @@ -0,0 +1,71 @@ +// TODO: sandboxear este proceso (¿seccomp? ¿minijail?) +// TODO: sandboxear mkquashfs +// TODO: fijarse como hace firecracker-containerd +// TODO: fijarse como hace [ignite](https://github.com/weaveworks/ignite) + +import gunzip from "gunzip-maybe"; +import { spawn } from "node:child_process"; +import { Readable } from "node:stream"; +import { extract, pack } from "tar-stream"; + +/** + * Takes many tar streams and converts them to a squashfs image. + * + * ## Why? + * + * We need a way to download OCI images (composed of layers that are tarballs) to something + * that can be accessed inside a VM. I wanted to not need to copy the image each time a VM + * is made. So, having a local HTTP cache and having the agent inside the VM download it into + * a temporary filesystem would copy it. This is bad for the life of an SSD, slow for an HDD, + * and too much to save into RAM for each VM. + * + * Instead, we download the images before booting the VM and package the layers up into a + * squashfs image that can be mounted inside the VM. This way, we reuse downloaded images + * efficiently. + * + * @param streams {Promise[]} + * @param output {string} + */ +export async function tar2squashfs(streams, output) { + const child = spawn( + "mksquashfs", + [ + "-", + output, + "-tar", + ...["-comp", "zstd"], + ...["-Xcompression-level", "3"], + ], + { + stdio: ["pipe", "inherit", "inherit"], + } + ); + + const p = pack(); + p.pipe(child.stdin); + + for (const streamP of streams) { + const stream = await streamP; + const ex = extract(); + + ex.on("entry", (header, stream, next) => { + stream.pipe(p.entry(header, next)); + }); + + stream.pipe(gunzip()).pipe(ex); + + await new Promise((resolve) => + ex.on("finish", () => { + resolve(void 0); + }) + ); + } + console.debug("finalizing"); + p.finalize(); + + await new Promise((resolve, reject) => + child.on("close", (code) => { + code === 0 ? resolve(void 0) : reject(code); + }) + ); +} diff --git a/js/tsconfig.json b/js/tsconfig.json new file mode 100644 index 0000000..7144da2 --- /dev/null +++ b/js/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "@tsconfig/node16/tsconfig.json", + "compilerOptions": { + "allowJs": true, + "checkJs": true, + "lib": ["es2023"], + "module": "Node16", + "target": "es2022", + "noEmit": true + } +}