Compare commits

...

7 commits

15 changed files with 195 additions and 154 deletions

View file

@ -1,19 +1,19 @@
import { mkdir, opendir } from "node:fs/promises";
import {
chmod,
chown,
copyFile,
mkdir,
opendir,
symlink,
writeFile,
} from "node:fs/promises";
import path from "node:path";
import { cwd } from "node:process";
import { Fstab } from "./fstab.js";
import { execFile } from "./helpers/better-api.js";
import { PasswdEntry, sudoReadPasswd } from "./helpers/passwd.js";
import {
sudoChmod,
sudoChown,
sudoCopy,
sudoMkdirP,
sudoSymlink,
sudoWriteExecutable,
sudoWriteFile,
} from "./helpers/sudo.js";
import { execFile, exists } from "./helpers/better-api.js";
import { PasswdEntry, readPasswd } from "./helpers/passwd.js";
import { logDebug } from "./helpers/logger.js";
import assert from "node:assert";
export class Alpine {
dir: string;
@ -21,9 +21,10 @@ export class Alpine {
this.dir = dir;
}
fstab: Fstab = new Fstab(this);
packages: string[] = [];
async mkdirP(dir: string): Promise<void> {
await sudoMkdirP(path.join(this.dir, dir));
await mkdir(this.path(dir), { recursive: true });
}
async writeFile(
filePath: string,
@ -32,22 +33,22 @@ export class Alpine {
): Promise<void> {
const p = path.join(this.dir, filePath);
await this.mkdirP(path.dirname(filePath));
await sudoWriteFile(p, content);
await writeFile(p, content);
if (permissions) {
await sudoChown(p, `${permissions.uid}:${permissions.gid}`);
await sudoChmod(p, "600");
await chown(p, permissions.uid, permissions.gid);
await chmod(p, 0o600);
}
}
async writeExecutable(filePath: string, content: string): Promise<void> {
await this.writeFile(filePath, content);
await sudoChmod(path.join(this.dir, filePath), "700");
await chmod(this.path(filePath), 0o700);
}
async assertExists(path: string) {
assert(await exists(this.path(path)));
}
path(p: string): string {
return path.join(this.dir, p);
}
sudoWriteExecutable(filePath: string, content: string): Promise<void> {
return sudoWriteExecutable(this.path(filePath), content);
}
private getRelativeSymlink(
target: string,
filePath: string
@ -63,16 +64,29 @@ export class Alpine {
}
async symlink(_target: string, _filePath: string): Promise<void> {
const { target, filePath } = this.getRelativeSymlink(_target, _filePath);
await sudoSymlink(target, filePath);
await symlink(target, filePath);
}
readPasswd(): Promise<PasswdEntry[]> {
return sudoReadPasswd(this.path("/etc/passwd"));
return readPasswd(this.path("/etc/passwd"));
}
async userAdd(user: string): Promise<PasswdEntry> {
await execFile("sudo", [
"useradd",
async userAdd(
user: string,
{
system,
homeDir,
createHome,
}: { system: boolean; homeDir?: string; createHome: boolean } = {
system: false,
homeDir: "",
createHome: true,
}
): Promise<PasswdEntry> {
await execFile("useradd", [
"--user-group",
...(system ? ["--system"] : []),
...(homeDir ? ["--home-dir", homeDir] : []),
...(createHome ? ["--create-home"] : []),
"--root",
this.dir,
user,
@ -85,11 +99,14 @@ export class Alpine {
return entry;
}
async addPackages(packages: string[]): Promise<void> {
async actuallyInstallPackages(): Promise<void> {
await this.installPackages(this.packages);
}
async installPackages(packages: string[]): Promise<void> {
logDebug(
"addPackages",
await execFile("sudo", [
"apk",
"installPackages",
await execFile("apk", [
"add",
"--clean-protected",
"--root",
@ -98,6 +115,9 @@ export class Alpine {
])
);
}
async addPackages(packages: string[]): Promise<void> {
this.packages = this.packages.concat(packages);
}
static async makeWorld({
dir,
@ -107,27 +127,27 @@ export class Alpine {
packages?: string[];
}): Promise<Alpine> {
const apkDir = path.join(dir, "/etc/apk");
await sudoMkdirP(apkDir);
await mkdir(apkDir, { recursive: true });
// hack
{
const cacheDir = path.join(cwd(), "cache");
await mkdir("cache", { recursive: true });
await sudoSymlink(cacheDir, path.join(apkDir, "cache"));
await symlink(cacheDir, path.join(apkDir, "cache"));
}
{
const apkKeysDir = path.join(apkDir, "keys");
const keysSrcDir = "alpine/keys";
await sudoMkdirP(apkKeysDir);
await mkdir(apkKeysDir, { recursive: true });
for await (const { name } of await opendir(keysSrcDir))
await sudoCopy(
await copyFile(
path.join(keysSrcDir, name),
path.join(apkKeysDir, name)
);
}
await sudoWriteFile(
await writeFile(
path.join(apkDir, "repositories"),
[
"https://dl-cdn.alpinelinux.org/alpine/v3.17/main",
@ -136,8 +156,7 @@ export class Alpine {
);
logDebug(
"makeWorld",
await execFile("sudo", [
"apk",
await execFile("apk", [
"add",
"--initdb",
"--clean-protected",

View file

@ -1,5 +1,4 @@
import { Alpine } from "./alpine.js";
import { sudoChmod, sudoMkdirP, sudoWriteFile } from "./helpers/sudo.js";
export class Fstab {
private alpine: Alpine;
@ -17,20 +16,19 @@ export class Fstab {
.map(([key, val]) => `,${key}=${val}`)
.join("");
await this.addMount(`tmpfs ${path} tmpfs defaults,noexec,nosuid${add} 0 0`);
await sudoMkdirP(this.alpine.path(path));
await this.alpine.mkdirP(path);
}
// Writes fstab to disk.
// Intended for internal use only, use addMount or addTmpfs instead
async write() {
const path = this.alpine.path("/etc/fstab");
await sudoWriteFile(
path,
await this.alpine.writeFile(
"/etc/fstab",
this.mounts.join("\n") +
// Busybox mount no entiende la última línea sino
"\n"
"\n",
{ uid: 0, gid: 0 }
);
await sudoChmod(path, "600");
}
}
interface TmpfsOptions {

View file

@ -3,7 +3,7 @@ import {
execFile as execFileCallback,
spawn as spawnCallback,
} from "node:child_process";
import { access } from "node:fs/promises";
import { access, stat } from "node:fs/promises";
export const execFile = promisify(execFileCallback);
export const spawn = promisify(spawnCallback);
@ -16,3 +16,12 @@ export async function canAccess(path: string): Promise<boolean> {
return false;
}
}
export async function exists(path: string): Promise<boolean> {
try {
await stat(path);
return true;
} catch {
return false;
}
}

View file

@ -1,4 +1,4 @@
import { sudoReadFile } from "./sudo.js";
import { readFile } from "node:fs/promises";
export interface PasswdEntry {
name: string;
@ -31,7 +31,7 @@ export function parsePasswd(content: string): PasswdEntry[] {
return entry;
});
}
export async function sudoReadPasswd(path: string): Promise<PasswdEntry[]> {
const content = await sudoReadFile(path);
export async function readPasswd(path: string): Promise<PasswdEntry[]> {
const content = await readFile(path, "utf-8");
return parsePasswd(content);
}

View file

@ -6,7 +6,6 @@ import { Alpine } from "./alpine.js";
import { generateForgejoSecretsFile } from "./services/forgejo/secrets.js";
import { generateGrafanaSecretsFile } from "./services/grafana/secrets.js";
import { execFile } from "./helpers/better-api.js";
import { sudoChown, sudoChownToRunningUser } from "./helpers/sudo.js";
import { setupKernel } from "./kernel.js";
import { runQemu } from "./qemu.js";
import { Runit } from "./runit/index.js";
@ -23,6 +22,13 @@ if (process.argv[2] === "generate-secrets") {
exit(0);
}
async function timed<T>(fn: () => Promise<T>): Promise<T> {
console.time(fn.toString());
const r = await fn();
console.timeEnd(fn.toString());
return r;
}
{
console.time("Building");
@ -32,9 +38,8 @@ if (process.argv[2] === "generate-secrets") {
await mkdir(kernelDir, { recursive: true });
const rootfsDir = await mkdtemp(path.join(tmpdir(), "define-alpine-"));
await sudoChown(rootfsDir, "root:root");
console.debug(rootfsDir);
const alpine = await Alpine.makeWorld({ dir: rootfsDir });
const alpine = await timed(() => Alpine.makeWorld({ dir: rootfsDir }));
await alpine.addPackages(["helix", "htop", "iproute2-ss", "socat"]);
await alpine.writeFile(
@ -44,70 +49,33 @@ socat tcp-listen:80,reuseaddr,fork tcp:localhost:3050 &
`,
{ uid: 0, gid: 0 }
);
await installFluentBit(alpine);
const runit = await Runit.setup(alpine);
await setupDhcpcd(alpine, runit);
await setupNtpsec(alpine, runit);
await setupForgejo(alpine, runit);
await setupLoki(alpine, runit);
await setupGrafana(alpine, runit);
const kernel = await setupKernel(alpine, kernelDir);
await timed(() => installFluentBit(alpine));
const runit = await timed(() => Runit.setup(alpine));
await timed(() => setupDhcpcd(alpine, runit));
await timed(() => setupNtpsec(alpine, runit));
await timed(() => setupForgejo(alpine, runit));
await timed(() => setupLoki(alpine, runit));
await timed(() => setupGrafana(alpine, runit));
const kernel = await timed(() => setupKernel(alpine, kernelDir));
await timed(() => alpine.actuallyInstallPackages());
const squashfs = path.join(artifactsDir, "image.squashfs");
await execFile("sudo", [
"mksquashfs",
alpine.dir,
squashfs,
"-root-mode",
"755",
"-comp",
"zstd",
"-Xcompression-level",
"3",
"-noappend",
"-quiet",
]);
await sudoChownToRunningUser(squashfs);
await timed(() =>
execFile("mksquashfs", [
alpine.dir,
squashfs,
"-root-mode",
"755",
"-comp",
"zstd",
"-Xcompression-level",
"1",
"-noappend",
"-quiet",
])
);
console.timeEnd("Building");
runQemu(squashfs, kernel, { graphic: true });
// await makeService({
// parentDir: rootfsDir,
// name: "grafana",
// packages: ["grafana"],
// setup: async (dir) => {},
// initScript: async (dir) => {},
// });
// try {
// await spawn("sudo", ["chroot", rootfsDir], { stdio: "inherit" });
// } catch {}
// runQemu(squashfs, kernel, { graphic: true });
}
// interface Service {}
// async function makeService({
// parentDir,
// name,
// packages,
// setup,
// initScript: _initScript,
// }: {
// parentDir: string;
// name: string;
// packages?: string[];
// setup: (dir: string) => Promise<void>;
// initScript: (dir: string) => Promise<string>;
// }) {
// const rootsDir = path.join(parentDir, "/nulo/roots/");
// await mkdir(rootsDir, { recursive: true });
// const alpine = await Alpine.makeWorld({
// dir: path.join(rootsDir, name),
// packages,
// });
// await setup(alpine.dir);
// // const initScript = await _initScript(rootfsDir);
// }

View file

@ -1,9 +1,7 @@
import { constants } from "node:fs";
import { copyFile } from "node:fs/promises";
import { copyFile, readFile, rm, writeFile } from "node:fs/promises";
import path from "node:path";
import { Alpine } from "./alpine.js";
import { canAccess } from "./helpers/better-api.js";
import { sudoChownToRunningUser, sudoCopy, sudoRm } from "./helpers/sudo.js";
import { execFile } from "./helpers/better-api.js";
export type Kind = "lts" | "virt";
export type Kernel = {
@ -63,10 +61,36 @@ default=lts
await alpine.writeFile(
"/etc/mkinitfs/mkinitfs.conf",
`features="${features.join(" ")}"`
`features="${features.join(" ")}"
disable_trigger=yes`
);
await alpine.addPackages([`linux-${kind}`]);
await alpine.installPackages([`linux-${kind}`, "zstd"]);
// patch mkinitfs to use faster zstd
const mkinitfs = alpine.path("/sbin/mkinitfs");
const newMkinitfs = (await readFile(mkinitfs, "utf-8")).replace(
'comp="zstd -19"',
'comp="zstd -3"'
);
await writeFile(mkinitfs, newMkinitfs);
const kernelRelease = (
await readFile(
alpine.path(`/usr/share/kernel/${kind}/kernel.release`),
"utf-8"
)
).trim();
// run mkinitfs manually as trigger was diabled
await execFile("chroot", [
alpine.dir,
"mkinitfs",
"-C",
"zstd",
"-o",
`/boot/initramfs-${kind}`,
kernelRelease,
]);
const initramfs = path.join(alpine.dir, `/boot/initramfs-${kind}`);
const vmlinuz = path.join(alpine.dir, `/boot/vmlinuz-${kind}`);
@ -77,11 +101,9 @@ default=lts
vmlinuz: path.join(output, "vmlinuz"),
};
await sudoCopy(initramfs, kernel.initramfs);
await sudoCopy(vmlinuz, kernel.vmlinuz);
await sudoChownToRunningUser(kernel.initramfs);
await sudoChownToRunningUser(kernel.vmlinuz);
await sudoRm(initramfs);
await sudoRm(vmlinuz);
await copyFile(initramfs, kernel.initramfs);
await copyFile(vmlinuz, kernel.vmlinuz);
await rm(initramfs);
await rm(vmlinuz);
return kernel;
}

10
npmrun Executable file
View file

@ -0,0 +1,10 @@
#!/bin/sh
script="$1"
if test -z "$script"; then
echo "No script set"
exit 1
fi
run="$(jq -r ".scripts[\"${script}\"]" < package.json)"
echo "> $run"
PATH="./node_modules/.bin:$PATH" sh -c "$run"

View file

@ -6,8 +6,10 @@
"main": "index.js",
"scripts": {
"build": "esbuild --log-level=warning --target=node18 --platform=node --sourcemap --outdir=build-javascript --format=esm --bundle index.ts",
"run": "pnpm build && node --enable-source-maps build-javascript/index.js",
"//test": "pnpm build && node --enable-source-maps build-javascript/**/*.test.js",
"build:qemu": "esbuild --log-level=warning --target=node18 --platform=node --sourcemap --outdir=build-javascript --format=esm --bundle qemu.ts",
"build-image": "./npmrun build && doas node --enable-source-maps build-javascript/index.js",
"run": "./npmrun build-image && ./npmrun build:qemu && node --enable-source-maps build-javascript/qemu.js",
"//test": "./npmrun build && node --enable-source-maps build-javascript/**/*.test.js",
"tsc:check": "tsc --noEmit"
},
"keywords": [],
@ -17,8 +19,5 @@
"@types/node": "^18.11.18",
"esbuild": "^0.17.0",
"typescript": "^4.9.4"
},
"dependencies": {
"nanoid": "^4.0.1"
}
}

16
qemu.ts
View file

@ -2,16 +2,19 @@ import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";
import { execFile } from "./helpers/better-api.js";
import { Kernel } from "./kernel.js";
import { sudoChownToRunningUser } from "./helpers/sudo.js";
export async function runQemu(
squashfs: string,
kernel: Kernel,
kernel: { initramfs: string; vmlinuz: string },
{ graphic, noShutdown }: { graphic?: boolean; noShutdown?: boolean } = {
graphic: true,
noShutdown: false,
}
) {
await sudoChownToRunningUser(kernel.initramfs);
await sudoChownToRunningUser(kernel.vmlinuz);
await sudoChownToRunningUser(squashfs);
const tmp = await mkdtemp(path.join(tmpdir(), "define-alpine-qemu-"));
try {
const disk = path.join(tmp, "tmp.ext4");
@ -59,3 +62,12 @@ export async function runQemu(
await rm(tmp, { recursive: true, force: true });
}
}
await runQemu(
"artifacts/image.squashfs",
{
initramfs: "artifacts/kernel/initramfs",
vmlinuz: "artifacts/kernel/vmlinuz",
},
{ graphic: true }
);

View file

@ -1,4 +1,4 @@
import { readFile } from "node:fs/promises";
import { readFile, rmdir } from "node:fs/promises";
import path from "node:path";
import { Alpine } from "../alpine.js";
@ -60,12 +60,12 @@ export class Runit {
);
// https://wiki.gentoo.org/wiki/Runit#Reboot_and_shutdown
await alpine.sudoWriteExecutable(
await alpine.writeExecutable(
"/usr/local/sbin/rpoweroff",
`#!/bin/sh
runit-init 0`
);
await alpine.sudoWriteExecutable(
await alpine.writeExecutable(
"/usr/local/sbin/rreboot",
`#!/bin/sh
runit-init 6`
@ -100,10 +100,10 @@ exec chpst -P getty 38400 ttyS0 linux`
logScript?: string
): Promise<void> {
const runScriptPath = path.join("/etc/sv/", name, "/run");
await this.alpine.sudoWriteExecutable(runScriptPath, script);
await this.alpine.writeExecutable(runScriptPath, script);
if (logScript) {
const logScriptPath = path.join("/etc/sv/", name, "/log/run");
await this.alpine.sudoWriteExecutable(logScriptPath, logScript);
await this.alpine.writeExecutable(logScriptPath, logScript);
await this.alpine.symlink(
`/run/runit/supervise.${name}.log`,
path.join("/etc/sv/", name, "/log/supervise")

View file

@ -1,14 +1,13 @@
import { buildForgejo } from "./build.js";
import { Alpine } from "../../alpine.js";
import { Runit } from "../../runit/index.js";
import { join } from "node:path";
import { loadForgejoSecretsFile } from "./secrets.js";
import { sudoCopy } from "../../helpers/sudo.js";
import { FluentBitParser, runitLokiLogger } from "../../software/fluentbit.js";
import { copyFile } from "node:fs/promises";
export async function setupForgejo(alpine: Alpine, runit: Runit) {
const bin = await buildForgejo();
await sudoCopy(bin, join(alpine.dir, "/usr/local/bin/forgejo"));
await copyFile(bin, alpine.path("/usr/local/bin/forgejo"));
await alpine.addPackages(["tzdata", "git"]);
const entry = await alpine.userAdd("_forgejo");

View file

@ -6,6 +6,7 @@ import { FluentBitParser, runitLokiLogger } from "../../software/fluentbit.js";
import { loadGrafanaSecretsFile } from "./secrets.js";
const provisioningDir = "/etc/grafana/provisioning/";
const grafanaHome = "/var/lib/grafana";
// TODO: grafana-image-renderer?
// /etc/conf.d/grafana
@ -23,12 +24,14 @@ export async function setupGrafana(
): Promise<void> {
await alpine.addPackages(["grafana"]);
const passwd = await alpine.readPasswd();
const user = passwd.find((e) => e.name === "grafana");
assert(!!user, "no existe el usuario grafana");
const user = await alpine.userAdd("grafana", {
system: true,
homeDir: grafanaHome,
createHome: false,
});
// TODO: data
await alpine.fstab.addTmpfs("/var/lib/grafana", {
await alpine.fstab.addTmpfs(grafanaHome, {
uid: user.uid,
gid: user.gid,
mode: "700",
@ -49,7 +52,7 @@ datasources:
user
);
await alpine.sudoWriteExecutable(
await alpine.writeExecutable(
"/usr/local/sbin/nulo-grafana-cli",
`#!/bin/sh
cd /
@ -59,7 +62,7 @@ exec chpst -u grafana:grafana grafana-cli --homepath /usr/share/grafana --config
await runit.addService(
"grafana",
`#!/bin/sh
export GRAFANA_HOME=/var/lib/grafana
export GRAFANA_HOME='${grafanaHome}'
cd "$GRAFANA_HOME"
@ -80,7 +83,7 @@ async function genConfig(): Promise<string> {
#################################### Paths ####################################
[paths]
data = /var/lib/grafana
data = ${grafanaHome}
# Temporary files in \`data\` directory older than given duration will be removed
;temp_data_lifetime = 24h
@ -88,7 +91,7 @@ data = /var/lib/grafana
# Directory where grafana can store logs
;logs = /var/log/grafana
plugins = /var/lib/grafana/plugins
plugins = ${grafanaHome}/plugins
# folder that contains provisioning config files that grafana will apply on startup and while running.
provisioning = ${provisioningDir}

View file

@ -1,12 +1,11 @@
import { join } from "node:path";
import { copyFile } from "node:fs/promises";
import { Alpine } from "../../alpine.js";
import { sudoCopy } from "../../helpers/sudo.js";
import { Runit } from "../../runit/index.js";
import { buildLoki } from "./build.js";
export async function setupLoki(alpine: Alpine, runit: Runit): Promise<void> {
const bin = await buildLoki();
await sudoCopy(bin, join(alpine.dir, "/usr/local/bin/loki"));
await copyFile(bin, alpine.path("/usr/local/bin/loki"));
const user = await alpine.userAdd("loki");

View file

@ -1,5 +1,4 @@
import { Alpine } from "../alpine.js";
import { sudoWriteFile } from "../helpers/sudo.js";
import { Runit } from "../runit/index.js";
import { FluentBitParser, runitLokiLogger } from "../software/fluentbit.js";
@ -10,8 +9,8 @@ export async function setupNtpsec(alpine: Alpine, runit: Runit) {
// file:///usr/share/doc/ntpsec/quick.html
// file:///usr/share/doc/ntpsec/NTS-QuickStart.html
// XXX: revisar driftfile, creo que tiene que poder escribir pero está readonly
await sudoWriteFile(
alpine.path("/etc/ntp.conf"),
await alpine.writeFile(
"/etc/ntp.conf",
`
driftfile /var/lib/ntp/ntp.drift

View file

@ -1,7 +1,7 @@
import { constants, copyFile } from "node:fs/promises";
import { join } from "node:path";
import { Alpine } from "../alpine.js";
import { buildRepro, reproRun } from "../helpers/repro-run.js";
import { sudoCopy } from "../helpers/sudo.js";
const parsersPath = "/etc/fluent-bit/parsers.conf";
@ -9,7 +9,11 @@ export async function installFluentBit(alpine: Alpine): Promise<void> {
const bin = await buildFluentBit();
await saveParsers(alpine);
await alpine.addPackages(["musl-fts", "yaml"]);
await sudoCopy(bin, join(alpine.dir, "/usr/local/bin/fluent-bit"));
await copyFile(
bin,
alpine.path("/usr/local/bin/fluent-bit"),
constants.COPYFILE_FICLONE
);
}
// ## Script generators