fireactions/server/container-baby.ts

import { execFile as _execFile } from "node:child_process";
import { access, mkdir, open, rename, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { tar2squashfs } from "./tar2squashfs.js";
import { subtle } from "node:crypto";
import { Writable } from "node:stream";

type RegistryName = string;
type RegistrySecret = string;
const getToken = memoizeDownloader(_getToken);
let squashfsDownloads = new Map<string, Promise<string>>();

const cacheDir = "cache/";
// {
//   const image = "gitea.nulo.in/nulo/zulip-checkin-cyborg";
//   const tag = "latest";

//   await downloadImage(image, tag);
// }

export const CONFIG_PATH_IN_IMAGE = "/.fireactions-image-config.json";

export function parseImageRef(ref: string): { image: string; tag: string } {
  const [image, tag] = ref.split(":");
  if (!image || !tag) throw new Error("Invalid image ref " + ref);
  return { image, tag };
}

export async function downloadImage(image: string, tag: string) {
  await mkdir(cacheDir, { recursive: true });
  const manifest = await getManifest(image, tag);

  // sanity check
  if (
    !manifest.layers.every(
      (layer) =>
        layer.mediaType === "application/vnd.oci.image.layer.v1.tar" ||
        layer.mediaType === "application/vnd.oci.image.layer.v1.tar+gzip"
    )
  ) {
    throw new Error("Unsupported layer");
  }

  console.debug(manifest);

  const config = await jsonBlob(image, manifest.config.digest);
  const squashfsFile = await saveSquashfs(
    image,
    manifest,
    JSON.stringify(config, null, 2)
  );

  return { squashfsFile };
}

// https://stackoverflow.com/a/67600346
async function hashData(data: string, algorithm = "SHA-512"): Promise<string> {
  const ec = new TextEncoder();
  const digest: ArrayBuffer = await subtle.digest(algorithm, ec.encode(data));
  const hashArray = Array.from(new Uint8Array(digest));
  const hash = hashArray
    .map((item) => item.toString(16).padStart(2, "0"))
    .join("");
  return hash;
}

async function saveSquashfs(
  image: string,
  manifest: Manifest,
  configJson: string
): Promise<string> {
  const manifestDigest = await hashData(JSON.stringify(manifest));
  const key = `${image.replaceAll("/", "%")}#${manifestDigest}.squashfs`;

  let p = squashfsDownloads.get(key);
  if (!p) {
    p = (async () => {
      const output = join(cacheDir, key);
      const progressFile = output + ".downloading";
      try {
        await access(output);
        // ya está cacheado
      } catch {
        if (process.env.DEBUG_WRITE_BLOBS)
          await Promise.all(
            manifest.layers.map(async (layer) => {
              const res = await getBlob(image, layer.digest);
              const file = await open(layer.digest, "w");
              await res.body!.pipeTo(Writable.toWeb(file.createWriteStream()));
            })
          );
        const layerStreams = manifest.layers.map(async (layer) => {
          const res = await getBlob(image, layer.digest);
          return res.body!;
        });
        await tar2squashfs(layerStreams, progressFile, [
          {
            content: configJson,
            headers: {
              name: CONFIG_PATH_IN_IMAGE,
              uid: 0,
              gid: 0,
              mode: 0o400,
            },
          },
        ]);
        await rename(progressFile, output);
      }
      return output;
    })();
    squashfsDownloads.set(key, p);
  }

  return p;
}

async function _getToken(registryUrl: string): Promise<RegistrySecret> {
  const res = await fetch(`${registryUrl}/token`);
  const json = await res.json();
  if (
    !json ||
    typeof json !== "object" ||
    !("token" in json) ||
    typeof json.token !== "string"
  )
    throw new Error("Unexpected");
  return json.token;
}

async function call(registryName: string, path: string, init: RequestInit) {
  const registryUrl = `https://${registryName}/v2`;
  const token = await getToken(registryUrl);
  console.debug(path);
  const res = await fetch(`${registryUrl}${path}`, {
    ...init,
    headers: {
      ...init.headers,
      Authorization: `Bearer ${token}`,
    },
  });
  if (!res.ok) throw new Error(res.statusText);
  return res;
}

// https://github.com/opencontainers/image-spec/blob/v1.0.1/image-index.md
type ManifestIndex = {
  schemaVersion: 2;
  mediaType: "application/vnd.oci.image.index.v1+json";
  manifests: ManifestPointer[];
};
type DockerManifestV2 = {
  schemaVersion: 2;
  mediaType: "application/vnd.docker.distribution.manifest.v2+json";
  config: Descriptor;
  layers: DockerManifestV2Layer[];
};

type ManifestPointer = {
  mediaType: "application/vnd.oci.image.manifest.v1+json";
  digest: string;
  size: number;
  platform: {
    architecture: "amd64" | string;
    os: string;
  };
  annotations?: { [k: string]: string };
};
type DockerManifestV2Layer = {
  mediaType: "application/vnd.docker.image.rootfs.diff.tar.gzip";
} & Descriptor;

function getImageParts(image: string): [string, string] {
  const parts = image.split("/");
  const registryName = parts[0];
  const imgPath = parts.slice(1).join("/");
  return [registryName, imgPath];
}

async function getManifest(image: string, tag: string): Promise<Manifest> {
  const [registryName, imgPath] = getImageParts(image);
  const res = await call(registryName, `/${imgPath}/manifests/${tag}`, {
    headers: {
      Accept: "application/vnd.oci.image.index.v1+json",
    },
  });
  const json = (await res.json()) as object;
  if (!("mediaType" in json)) throw new Error(`Received invalid manifest`);

  if (
    json.mediaType === "application/vnd.docker.distribution.manifest.v2+json"
  ) {
    const manifest = json as DockerManifestV2;
    return {
      mediaType: "application/vnd.oci.image.manifest.v1+json",
      config: {
        ...manifest.config,
        mediaType: "application/vnd.oci.image.config.v1+json",
      },
      layers: manifest.layers.map((l) => ({
        ...l,
        mediaType: "application/vnd.oci.image.layer.v1.tar+gzip",
      })),
    };
  } else if (json.mediaType === "application/vnd.oci.image.manifest.v1+json") {
    return json as Manifest;
  } else if (json.mediaType === "application/vnd.oci.image.index.v1+json") {
    const index = json as ManifestIndex;
    // XXX: hardcoded amd64
    const arch = "amd64";
    const manifestPtr = chooseManifest(index, arch);
    if (!manifestPtr) throw new Error(`No manifest for arch ${arch}`);

    const manifest = await jsonBlob<Manifest>(image, manifestPtr.digest);
    return manifest;
  } else throw new Error(`Received invalid manifest`);
}

function chooseManifest(list: ManifestIndex, arch: string) {
  console.debug(list);
  return list.manifests.find(
    (m) => m.platform.architecture === arch && m.platform.os === "linux"
  );
}

async function getBlob(image: string, digest: string): Promise<Response> {
  const [registryName, imgPath] = getImageParts(image);
  return await call(registryName, `/${imgPath}/blobs/${digest}`, {});
}
async function jsonBlob<T>(image: string, digest: string): Promise<T> {
  const res = await getBlob(image, digest);
  return (await res.json()) as T;
}

function memoizeDownloader<T>(
  downloader: (id: string) => Promise<T>
): (id: string) => Promise<T> {
  let map = new Map<string, Promise<T>>();

  return (id) => {
    let p = map.get(id);
    if (!p) {
      p = downloader(id);
      map.set(id, p);
    }
    return p;
  };
}

// https://github.com/opencontainers/image-spec/blob/v1.0.1/descriptor.md#properties
type Descriptor = {
  digest: string;
  size: number;
  urls?: string[];
  annotations?: { [k: string]: string };
};

// https://github.com/opencontainers/image-spec/blob/v1.0.1/manifest.md
type Manifest = {
  mediaType: "application/vnd.oci.image.manifest.v1+json";
  config: {
    mediaType: "application/vnd.oci.image.config.v1+json";
  } & Descriptor;
  layers: ({
    mediaType:
      | "application/vnd.oci.image.layer.v1.tar"
      | "application/vnd.oci.image.layer.v1.tar+gzip"
      | "application/vnd.oci.image.layer.nondistributable.v1.tar"
      | "application/vnd.oci.image.layer.nondistributable.v1.tar+gzip";
  } & Descriptor)[];
};