From 6d483c3e8bd3ad704049aceff7c3cf6fc4d345d5 Mon Sep 17 00:00:00 2001 From: Nulo Date: Mon, 19 Jun 2023 18:59:44 -0300 Subject: [PATCH] download container images --- .gitignore | 4 + js/.gitignore | 5 + js/container-baby.ts | 253 +++++++++++++++++++++++++++++++++++ js/fetch-hack.d.ts | 29 ++++ js/index.js | 143 ++++++++++++++++++++ js/package.json | 26 ++++ js/pnpm-lock.yaml | 311 +++++++++++++++++++++++++++++++++++++++++++ js/tar2squashfs.js | 73 ++++++++++ js/tsconfig.json | 11 ++ 9 files changed, 855 insertions(+) create mode 100644 js/.gitignore create mode 100644 js/container-baby.ts create mode 100644 js/fetch-hack.d.ts create mode 100644 js/index.js create mode 100644 js/package.json create mode 100644 js/pnpm-lock.yaml create mode 100644 js/tar2squashfs.js create mode 100644 js/tsconfig.json diff --git a/.gitignore b/.gitignore index e2d40fa..68bb3c3 100644 --- a/.gitignore +++ b/.gitignore @@ -2,5 +2,9 @@ rootfs.ext4 rootfs.qcow2 fireactions *.ext4 +*.squashfs +*.config.json vmlinux.bin node_modules/ + +*.log diff --git a/js/.gitignore b/js/.gitignore new file mode 100644 index 0000000..4c2386f --- /dev/null +++ b/js/.gitignore @@ -0,0 +1,5 @@ +# compiled typescript +container-baby.js + +# cache, will move later +cache/ diff --git a/js/container-baby.ts b/js/container-baby.ts new file mode 100644 index 0000000..85af051 --- /dev/null +++ b/js/container-baby.ts @@ -0,0 +1,253 @@ +import { execFile as _execFile } from "node:child_process"; +import { access, mkdir, rm } from "node:fs/promises"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { tar2squashfs } from "./tar2squashfs.js"; +import { subtle } from "node:crypto"; +const execFile = promisify(_execFile); + +type RegistryName = string; +type RegistrySecret = string; +// const downloadBlob = memoizeDownloader(_downloadBlob); +const getToken = memoizeDownloader(_getToken); +let squashfsDownloads = new Map>(); + +const tmpDir = "cache/"; +{ + const image = "gitea.nulo.in/nulo/zulip-checkin-cyborg"; + const tag = "latest"; + + await mkdir(tmpDir, { recursive: true }); + await downloadContainer(image, tag); +} + +async function downloadContainer(image: string, tag: string) { + const manifest = await getContainerManifest(image, tag); + + // sanity check + if ( + !manifest.layers.every( + (layer) => + layer.mediaType === "application/vnd.oci.image.layer.v1.tar" || + layer.mediaType === "application/vnd.oci.image.layer.v1.tar+gzip" + ) + ) { + throw new Error("Unsupported layer"); + } + + console.debug(manifest); + + await saveSquashfs(image, manifest); + const configPath = await jsonBlob(image, manifest.config.digest); +} + +// https://stackoverflow.com/a/67600346 +async function hashData(data: string, algorithm = "SHA-512"): Promise { + const ec = new TextEncoder(); + const digest: ArrayBuffer = await subtle.digest(algorithm, ec.encode(data)); + const hashArray = Array.from(new Uint8Array(digest)); + const hash = hashArray + .map((item) => item.toString(16).padStart(2, "0")) + .join(""); + return hash; +} + +async function saveSquashfs( + image: string, + manifest: Manifest +): Promise { + const manifestDigest = await hashData(JSON.stringify(manifest)); + const key = `${image.replaceAll("/", "%")}#${manifestDigest}`; + + let p = squashfsDownloads.get(key); + if (!p) { + p = (async () => { + const output = join(tmpDir, key); + try { + await access(output); + // ya está cacheado + } catch { + const layerStreams = manifest.layers.map(async (layer) => { + const res = await getBlob(image, layer.digest); + return res.body!; + }); + await tar2squashfs(layerStreams, output); + } + return output; + })(); + squashfsDownloads.set(key, p); + } + + return p; +} + +async function _getToken(registryUrl: string): Promise { + const res = await fetch(`${registryUrl}/token`); + const json = await res.json(); + if ( + !json || + typeof json !== "object" || + !("token" in json) || + typeof json.token !== "string" + ) + throw new Error("Unexpected"); + return json.token; +} + +async function call(registryName: string, path: string, init: RequestInit) { + const registryUrl = `https://${registryName}/v2`; + const token = await getToken(registryUrl); + console.debug(path); + const res = await fetch(`${registryUrl}${path}`, { + ...init, + headers: { + ...init.headers, + Authorization: `Bearer ${token}`, + }, + }); + if (!res.ok) throw new Error(res.statusText); + return res; +} + +type ManifestPointer = { + mediaType: "application/vnd.oci.image.manifest.v1+json"; + digest: string; + size: number; + platform: { + architecture: "amd64" | string; + os: string; + }; + annotations?: { [k: string]: string }; +}; + +// https://github.com/opencontainers/image-spec/blob/v1.0.1/image-index.md +type ManifestIndex = { + mediaType: "application/vnd.oci.image.index.v1+json"; + schemaVersion: 2; + manifests: ManifestPointer[]; +}; + +function getImageParts(image: string): [string, string] { + const parts = image.split("/"); + const registryName = parts[0]; + const imgPath = parts.slice(1).join("/"); + return [registryName, imgPath]; +} + +async function getContainerManifest(image: string, tag: string) { + const index = await getManifestIndex(image, tag); + // badly behaved registries will return whatever they want + if ( + (index.mediaType as string) === "application/vnd.oci.image.manifest.v1+json" + ) { + return index as unknown as Manifest; + } + const arch = "amd64"; + const ptr = chooseManifest(index, arch); + if (!ptr) + throw new Error(`Image ${image}:${tag} doesn't exist for arch ${arch}`); + const manifest = await getManifest(image, ptr); + return manifest; +} + +async function getManifestIndex( + image: string, + tag: string +): Promise { + const [registryName, imgPath] = getImageParts(image); + + const res = await call(registryName, `/${imgPath}/manifests/${tag}`, { + headers: { + Accept: "application/vnd.oci.image.index.v1+json", + }, + }); + const json = await res.json(); + return json as ManifestIndex; +} + +function chooseManifest(list: ManifestIndex, arch: string) { + console.debug(list); + return list.manifests.find( + (m) => m.platform.architecture === arch && m.platform.os === "linux" + ); +} + +async function getBlob(image: string, digest: string): Promise { + const [registryName, imgPath] = getImageParts(image); + return await call(registryName, `/${imgPath}/blobs/${digest}`, {}); +} +async function jsonBlob(image: string, digest: string): Promise { + const res = await getBlob(image, digest); + return (await res.json()) as T; +} + +// // We can't just trust the digest as some evil image or registry can lie with +// // their own digests, using one of another image but inserting malware in it. +// // If we just cache according to the digest without verifying digests, this +// // attack is possible. +// function blobKey(image: string, digest: string): string { +// return `${image.replaceAll("/", "%")}#${digest}`; +// } + +function memoizeDownloader( + downloader: (id: string) => Promise +): (id: string) => Promise { + let map = new Map>(); + + return (id) => { + let p = map.get(id); + if (!p) { + p = downloader(id); + map.set(id, p); + } + return p; + }; +} + +// async function _downloadBlob(key: string) { +// let [image, digest] = key.split("#"); +// image = image.replaceAll("%", "/"); +// const path = join(tmpDir, key); +// const res = await getBlob(image, digest); +// try { +// await access(path); +// // cacheado, actualizar mtime y devolver +// utimes(path, new Date(), new Date()); +// return path; +// } catch (error) {} + +// const tempKey = `.${key}.downloading.${nanoid()}`; +// const tempFile = await open(join(tmpDir, tempKey), "wx"); +// await res.body!.pipeTo(Writable.toWeb(tempFile.createWriteStream())); +// await tempFile.close(); + +// await rename(tempKey, path); +// return path; +// } + +// https://github.com/opencontainers/image-spec/blob/v1.0.1/descriptor.md#properties +type Descriptor = { + digest: string; + size: number; + urls?: string[]; + annotations?: { [k: string]: string }; +}; + +// https://github.com/opencontainers/image-spec/blob/v1.0.1/manifest.md +type Manifest = { + mediaType: "application/vnd.oci.image.manifest.v1+json"; + config: { + mediaType: "application/vnd.oci.image.config.v1+json"; + } & Descriptor; + layers: ({ + mediaType: + | "application/vnd.oci.image.layer.v1.tar" + | "application/vnd.oci.image.layer.v1.tar+gzip" + | "application/vnd.oci.image.layer.nondistributable.v1.tar" + | "application/vnd.oci.image.layer.nondistributable.v1.tar+gzip"; + } & Descriptor)[]; +}; + +async function getManifest(image: string, ptr: ManifestPointer) { + return await jsonBlob(image, ptr.digest); +} diff --git a/js/fetch-hack.d.ts b/js/fetch-hack.d.ts new file mode 100644 index 0000000..7a56e95 --- /dev/null +++ b/js/fetch-hack.d.ts @@ -0,0 +1,29 @@ +// https://stackoverflow.com/a/75676044 + +import * as undici_types from "undici"; + +declare global { + export const { + fetch, + FormData, + Headers, + Request, + Response, + }: typeof import("undici"); + + type FormData = undici_types.FormData; + type Headers = undici_types.Headers; + type HeadersInit = undici_types.HeadersInit; + type BodyInit = undici_types.BodyInit; + type Request = undici_types.Request; + type RequestInit = undici_types.RequestInit; + type RequestInfo = undici_types.RequestInfo; + type RequestMode = undici_types.RequestMode; + type RequestRedirect = undici_types.RequestRedirect; + type RequestCredentials = undici_types.RequestCredentials; + type RequestDestination = undici_types.RequestDestination; + type ReferrerPolicy = undici_types.ReferrerPolicy; + type Response = undici_types.Response; + type ResponseInit = undici_types.ResponseInit; + type ResponseType = undici_types.ResponseType; +} diff --git a/js/index.js b/js/index.js new file mode 100644 index 0000000..46b2939 --- /dev/null +++ b/js/index.js @@ -0,0 +1,143 @@ +import { delay } from "nanodelay"; +import { nanoid } from "nanoid"; +import { ChildProcess, execFile, spawn } from "node:child_process"; +import { writeFile } from "node:fs/promises"; +import { createServer, request as _request, IncomingMessage } from "node:http"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; + +const server = createServer(listener); + +/** + * @param {import("node:http").IncomingMessage} req + * @param {import("node:http").ServerResponse} res + */ +async function listener(req, res) { + const url = getUrl(req); + if (url.pathname == "/run") { + if (req.method == "POST") { + await runVm(); + } else res.writeHead(405).end("wrong method"); + } else res.writeHead(404).end("not found"); +} + +async function runVm() { + try { + await FirecrackerInstance.run(); + } catch (err) { + console.error(err); + } +} + +class FirecrackerInstance { + proc; + socketPath; + constructor() { + const vmid = nanoid(); + this.socketPath = join(tmpdir(), `firecracker-${vmid}.sock`); + console.debug(this.socketPath); + // TODO: jailer + this.proc = spawn( + "firecracker", + [ + ...["--api-sock", this.socketPath], + ...["--level", "Debug"], + ...["--log-path", "/dev/stderr"], + "--show-level", + "--show-log-origin", + ], + { + stdio: "inherit", + } + ); + } + static async run() { + const self = new FirecrackerInstance(); + + await self.request("PUT", "/boot-source", { + kernel_image_path: "../vmlinux.bin", + boot_args: "console=ttyS0 reboot=k panic=1 pci=off", + }); + await self.request("PUT", "/drives/rootfs", { + drive_id: "rootfs", + path_on_host: "../rootfs.ext4", + is_root_device: true, + is_read_only: false, + }); + + // API requests are handled asynchronously, it is important the configuration is + // set, before `InstanceStart`. + await delay(15); + + await self.request("PUT", "/actions", { + action_type: "InstanceStart", + }); + } + + /** + * @param {string} method + * @param {string} path + * @param {object} body + */ + async request(method, path, body) { + const jsonBody = JSON.stringify(body); + const { response, body: respBody } = await request( + { + method, + path, + socketPath: this.socketPath, + headers: { + "Content-Type": "application/json", + "Content-Length": jsonBody.length, + }, + }, + jsonBody + ); + // @ts-ignore + if (response.statusCode > 299) + throw new Error( + `Status code: ${response.statusCode} / Body: ${respBody}`, + { + cause: `${method} ${path} ${JSON.stringify(body, null, 4)}`, + } + ); + } +} + +/** + * @param {import("http").IncomingMessage} req + */ +function getUrl(req) { + // prettier-ignore + const urlString = /** @type {string} */(req.url); + const url = new URL(urlString, `http://${req.headers.host}`); + return url; +} + +/** + * @param {string | import("url").URL | import("http").RequestOptions} opts + * @param {string} [body] + * @returns {Promise<{ response: import("node:http").IncomingMessage, body: string }>} + */ +function request(opts, body) { + return new Promise((resolve, reject) => { + const req = _request(opts, (res) => { + let respBody = ""; + res.on("data", (data) => { + respBody += data; + }); + res.on("end", () => { + resolve({ response: res, body: respBody }); + }); + }); + req.on("error", reject); + if (body) + req.write(body, (error) => { + if (error) reject(error); + }); + req.end(); + }); +} + +server.listen("8080"); diff --git a/js/package.json b/js/package.json new file mode 100644 index 0000000..19b7133 --- /dev/null +++ b/js/package.json @@ -0,0 +1,26 @@ +{ + "name": "js", + "type": "module", + "version": "1.0.0", + "description": "", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "keywords": [], + "author": "", + "license": "ISC", + "devDependencies": { + "@tsconfig/node16": "^1.0.4", + "@types/gunzip-maybe": "^1.4.0", + "@types/node": "^20.2.5", + "@types/tar-stream": "^2.2.2", + "undici": "^5.22.1" + }, + "dependencies": { + "gunzip-maybe": "^1.4.2", + "nanodelay": "^2.0.2", + "nanoid": "^4.0.2", + "tar-stream": "^3.0.0" + } +} diff --git a/js/pnpm-lock.yaml b/js/pnpm-lock.yaml new file mode 100644 index 0000000..3a21b22 --- /dev/null +++ b/js/pnpm-lock.yaml @@ -0,0 +1,311 @@ +lockfileVersion: '6.1' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +dependencies: + gunzip-maybe: + specifier: ^1.4.2 + version: 1.4.2 + nanodelay: + specifier: ^2.0.2 + version: 2.0.2 + nanoid: + specifier: ^4.0.2 + version: 4.0.2 + tar-stream: + specifier: ^3.0.0 + version: 3.0.0 + +devDependencies: + '@tsconfig/node16': + specifier: ^1.0.4 + version: 1.0.4 + '@types/gunzip-maybe': + specifier: ^1.4.0 + version: 1.4.0 + '@types/node': + specifier: ^20.2.5 + version: 20.2.5 + '@types/tar-stream': + specifier: ^2.2.2 + version: 2.2.2 + undici: + specifier: ^5.22.1 + version: 5.22.1 + +packages: + + /@tsconfig/node16@1.0.4: + resolution: {integrity: sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==} + dev: true + + /@types/gunzip-maybe@1.4.0: + resolution: {integrity: sha512-dFP9GrYAR9KhsjTkWJ8q8Gsfql75YIKcg9DuQOj/IrlPzR7W+1zX+cclw1McV82UXAQ+Lpufvgk3e9bC8+HzgA==} + dependencies: + '@types/node': 20.2.5 + dev: true + + /@types/node@20.2.5: + resolution: {integrity: sha512-JJulVEQXmiY9Px5axXHeYGLSjhkZEnD+MDPDGbCbIAbMslkKwmygtZFy1X6s/075Yo94sf8GuSlFfPzysQrWZQ==} + dev: true + + /@types/tar-stream@2.2.2: + resolution: {integrity: sha512-1AX+Yt3icFuU6kxwmPakaiGrJUwG44MpuiqPg4dSolRFk6jmvs4b3IbUol9wKDLIgU76gevn3EwE8y/DkSJCZQ==} + dependencies: + '@types/node': 20.2.5 + dev: true + + /abort-controller@3.0.0: + resolution: {integrity: sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==} + engines: {node: '>=6.5'} + dependencies: + event-target-shim: 5.0.1 + dev: false + + /b4a@1.6.4: + resolution: {integrity: sha512-fpWrvyVHEKyeEvbKZTVOeZF3VSKKWtJxFIxX/jaVPf+cLbGUSitjb49pHLqPV2BUNNZ0LcoeEGfE/YCpyDYHIw==} + dev: false + + /base64-js@1.5.1: + resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==} + dev: false + + /bl@6.0.2: + resolution: {integrity: sha512-/ivXMGCGDI0EB4JI4zCqppp79j03vUgZz/zakw7TworE2NVjIuPxpL1Ti0InSsarKqFG5NLFreCBcCCSjtrTQw==} + dependencies: + buffer: 6.0.3 + inherits: 2.0.4 + readable-stream: 4.4.0 + dev: false + + /browserify-zlib@0.1.4: + resolution: {integrity: sha512-19OEpq7vWgsH6WkvkBJQDFvJS1uPcbFOQ4v9CU839dO+ZZXUZO6XpE6hNCqvlIIj+4fZvRiJ6DsAQ382GwiyTQ==} + dependencies: + pako: 0.2.9 + dev: false + + /buffer-from@1.1.2: + resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==} + dev: false + + /buffer@6.0.3: + resolution: {integrity: sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==} + dependencies: + base64-js: 1.5.1 + ieee754: 1.2.1 + dev: false + + /busboy@1.6.0: + resolution: {integrity: sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==} + engines: {node: '>=10.16.0'} + dependencies: + streamsearch: 1.1.0 + dev: true + + /core-util-is@1.0.3: + resolution: {integrity: sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==} + dev: false + + /duplexify@3.7.1: + resolution: {integrity: sha512-07z8uv2wMyS51kKhD1KsdXJg5WQ6t93RneqRxUHnskXVtlYYkLqM0gqStQZ3pj073g687jPCHrqNfCzawLYh5g==} + dependencies: + end-of-stream: 1.4.4 + inherits: 2.0.4 + readable-stream: 2.3.8 + stream-shift: 1.0.1 + dev: false + + /end-of-stream@1.4.4: + resolution: {integrity: sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==} + dependencies: + once: 1.4.0 + dev: false + + /event-target-shim@5.0.1: + resolution: {integrity: sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==} + engines: {node: '>=6'} + dev: false + + /events@3.3.0: + resolution: {integrity: sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==} + engines: {node: '>=0.8.x'} + dev: false + + /fast-fifo@1.2.0: + resolution: {integrity: sha512-NcvQXt7Cky1cNau15FWy64IjuO8X0JijhTBBrJj1YlxlDfRkJXNaK9RFUjwpfDPzMdv7wB38jr53l9tkNLxnWg==} + dev: false + + /gunzip-maybe@1.4.2: + resolution: {integrity: sha512-4haO1M4mLO91PW57BMsDFf75UmwoRX0GkdD+Faw+Lr+r/OZrOCS0pIBwOL1xCKQqnQzbNFGgK2V2CpBUPeFNTw==} + hasBin: true + dependencies: + browserify-zlib: 0.1.4 + is-deflate: 1.0.0 + is-gzip: 1.0.0 + peek-stream: 1.1.3 + pumpify: 1.5.1 + through2: 2.0.5 + dev: false + + /ieee754@1.2.1: + resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==} + dev: false + + /inherits@2.0.4: + resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==} + dev: false + + /is-deflate@1.0.0: + resolution: {integrity: sha512-YDoFpuZWu1VRXlsnlYMzKyVRITXj7Ej/V9gXQ2/pAe7X1J7M/RNOqaIYi6qUn+B7nGyB9pDXrv02dsB58d2ZAQ==} + dev: false + + /is-gzip@1.0.0: + resolution: {integrity: sha512-rcfALRIb1YewtnksfRIHGcIY93QnK8BIQ/2c9yDYcG/Y6+vRoJuTWBmmSEbyLLYtXm7q35pHOHbZFQBaLrhlWQ==} + engines: {node: '>=0.10.0'} + dev: false + + /isarray@1.0.0: + resolution: {integrity: sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==} + dev: false + + /nanodelay@2.0.2: + resolution: {integrity: sha512-6AS5aCSXsjoxq2Jr9CdaAeT60yoYDOTp6po9ziqeOeY6vf6uTEHYSqWql6EFILrM3fEfXgkZ4KqE9L0rTm/wlA==} + engines: {node: ^12.0.0 || ^14.0.0 || >=16.0.0} + dev: false + + /nanoid@4.0.2: + resolution: {integrity: sha512-7ZtY5KTCNheRGfEFxnedV5zFiORN1+Y1N6zvPTnHQd8ENUvfaDBeuJDZb2bN/oXwXxu3qkTXDzy57W5vAmDTBw==} + engines: {node: ^14 || ^16 || >=18} + hasBin: true + dev: false + + /once@1.4.0: + resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} + dependencies: + wrappy: 1.0.2 + dev: false + + /pako@0.2.9: + resolution: {integrity: sha512-NUcwaKxUxWrZLpDG+z/xZaCgQITkA/Dv4V/T6bw7VON6l1Xz/VnrBqrYjZQ12TamKHzITTfOEIYUj48y2KXImA==} + dev: false + + /peek-stream@1.1.3: + resolution: {integrity: sha512-FhJ+YbOSBb9/rIl2ZeE/QHEsWn7PqNYt8ARAY3kIgNGOk13g9FGyIY6JIl/xB/3TFRVoTv5as0l11weORrTekA==} + dependencies: + buffer-from: 1.1.2 + duplexify: 3.7.1 + through2: 2.0.5 + dev: false + + /process-nextick-args@2.0.1: + resolution: {integrity: sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==} + dev: false + + /process@0.11.10: + resolution: {integrity: sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==} + engines: {node: '>= 0.6.0'} + dev: false + + /pump@2.0.1: + resolution: {integrity: sha512-ruPMNRkN3MHP1cWJc9OWr+T/xDP0jhXYCLfJcBuX54hhfIBnaQmAUMfDcG4DM5UMWByBbJY69QSphm3jtDKIkA==} + dependencies: + end-of-stream: 1.4.4 + once: 1.4.0 + dev: false + + /pumpify@1.5.1: + resolution: {integrity: sha512-oClZI37HvuUJJxSKKrC17bZ9Cu0ZYhEAGPsPUy9KlMUmv9dKX2o77RUmq7f3XjIxbwyGwYzbzQ1L2Ks8sIradQ==} + dependencies: + duplexify: 3.7.1 + inherits: 2.0.4 + pump: 2.0.1 + dev: false + + /queue-tick@1.0.1: + resolution: {integrity: sha512-kJt5qhMxoszgU/62PLP1CJytzd2NKetjSRnyuj31fDd3Rlcz3fzlFdFLD1SItunPwyqEOkca6GbV612BWfaBag==} + dev: false + + /readable-stream@2.3.8: + resolution: {integrity: sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==} + dependencies: + core-util-is: 1.0.3 + inherits: 2.0.4 + isarray: 1.0.0 + process-nextick-args: 2.0.1 + safe-buffer: 5.1.2 + string_decoder: 1.1.1 + util-deprecate: 1.0.2 + dev: false + + /readable-stream@4.4.0: + resolution: {integrity: sha512-kDMOq0qLtxV9f/SQv522h8cxZBqNZXuXNyjyezmfAAuribMyVXziljpQ/uQhfE1XLg2/TLTW2DsnoE4VAi/krg==} + engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + dependencies: + abort-controller: 3.0.0 + buffer: 6.0.3 + events: 3.3.0 + process: 0.11.10 + dev: false + + /safe-buffer@5.1.2: + resolution: {integrity: sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==} + dev: false + + /stream-shift@1.0.1: + resolution: {integrity: sha512-AiisoFqQ0vbGcZgQPY1cdP2I76glaVA/RauYR4G4thNFgkTqr90yXTo4LYX60Jl+sIlPNHHdGSwo01AvbKUSVQ==} + dev: false + + /streamsearch@1.1.0: + resolution: {integrity: sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==} + engines: {node: '>=10.0.0'} + dev: true + + /streamx@2.15.0: + resolution: {integrity: sha512-HcxY6ncGjjklGs1xsP1aR71INYcsXFJet5CU1CHqihQ2J5nOsbd4OjgjHO42w/4QNv9gZb3BueV+Vxok5pLEXg==} + dependencies: + fast-fifo: 1.2.0 + queue-tick: 1.0.1 + dev: false + + /string_decoder@1.1.1: + resolution: {integrity: sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==} + dependencies: + safe-buffer: 5.1.2 + dev: false + + /tar-stream@3.0.0: + resolution: {integrity: sha512-O6OfUKBbQOqAhh6owTWmA730J/yZCYcpmZ1DBj2YX51ZQrt7d7NgzrR+CnO9wP6nt/viWZW2XeXLavX3/ZEbEg==} + dependencies: + b4a: 1.6.4 + bl: 6.0.2 + streamx: 2.15.0 + dev: false + + /through2@2.0.5: + resolution: {integrity: sha512-/mrRod8xqpA+IHSLyGCQ2s8SPHiCDEeQJSep1jqLYeEUClOFG2Qsh+4FU6G9VeqpZnGW/Su8LQGc4YKni5rYSQ==} + dependencies: + readable-stream: 2.3.8 + xtend: 4.0.2 + dev: false + + /undici@5.22.1: + resolution: {integrity: sha512-Ji2IJhFXZY0x/0tVBXeQwgPlLWw13GVzpsWPQ3rV50IFMMof2I55PZZxtm4P6iNq+L5znYN9nSTAq0ZyE6lSJw==} + engines: {node: '>=14.0'} + dependencies: + busboy: 1.6.0 + dev: true + + /util-deprecate@1.0.2: + resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} + dev: false + + /wrappy@1.0.2: + resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} + dev: false + + /xtend@4.0.2: + resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==} + engines: {node: '>=0.4'} + dev: false diff --git a/js/tar2squashfs.js b/js/tar2squashfs.js new file mode 100644 index 0000000..e6be542 --- /dev/null +++ b/js/tar2squashfs.js @@ -0,0 +1,73 @@ +// TODO: sandboxear este proceso (¿seccomp? ¿minijail?) +// TODO: sandboxear mkquashfs +// TODO: fijarse como hace firecracker-containerd +// TODO: fijarse como hace [ignite](https://github.com/weaveworks/ignite) + +import gunzip from "gunzip-maybe"; +import { spawn } from "node:child_process"; +import { Duplex, Readable, Writable } from "node:stream"; +import { pipeline } from "node:stream/promises"; +import { ReadableStream } from "node:stream/web"; +import { extract, pack } from "tar-stream"; + +/** + * Takes many tar streams and converts them to a squashfs image. + * + * ## Why? + * + * We need a way to download OCI images (composed of layers that are tarballs) to something + * that can be accessed inside a VM. I wanted to not need to copy the image each time a VM + * is made. So, having a local HTTP cache and having the agent inside the VM download it into + * a temporary filesystem would copy it. This is bad for the life of an SSD, slow for an HDD, + * and too much to save into RAM for each VM. + * + * Instead, we download the images before booting the VM and package the layers up into a + * squashfs image that can be mounted inside the VM. This way, we reuse downloaded images + * efficiently. + * + * @param streams {Promise[]} + * @param output {string} + */ +export async function tar2squashfs(streams, output) { + const child = spawn( + "mksquashfs", + [ + "-", + output, + "-tar", + ...["-comp", "zstd"], + ...["-Xcompression-level", "3"], + ], + { + stdio: ["pipe", "inherit", "inherit"], + } + ); + + const p = pack(); + p.pipe(child.stdin); + + for (const streamP of streams) { + const stream = await streamP; + const ex = extract(); + + ex.on("entry", (header, stream, next) => { + stream.pipe(p.entry(header, next)); + }); + + stream.pipeThrough(Duplex.toWeb(gunzip())).pipeTo(Writable.toWeb(ex)); + + await new Promise((resolve) => + ex.on("finish", () => { + resolve(void 0); + }) + ); + } + console.debug("finalizing"); + p.finalize(); + + await new Promise((resolve, reject) => + child.on("close", (code) => { + code === 0 ? resolve(void 0) : reject(code); + }) + ); +} diff --git a/js/tsconfig.json b/js/tsconfig.json new file mode 100644 index 0000000..7144da2 --- /dev/null +++ b/js/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "@tsconfig/node16/tsconfig.json", + "compilerOptions": { + "allowJs": true, + "checkJs": true, + "lib": ["es2023"], + "module": "Node16", + "target": "es2022", + "noEmit": true + } +}