From a33377d32949446b1a0b70468416391192ec9dc2 Mon Sep 17 00:00:00 2001 From: Nulo Date: Sat, 18 Feb 2023 13:10:21 -0300 Subject: [PATCH] =?UTF-8?q?fluentbit=20b=C3=A1sico=20(forgejo)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- index.ts | 2 + runit/index.ts | 19 +++----- services/forgejo/index.ts | 5 +- software/fluentbit.ts | 100 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 112 insertions(+), 14 deletions(-) create mode 100644 software/fluentbit.ts diff --git a/index.ts b/index.ts index 3d54d0f..7a01cac 100644 --- a/index.ts +++ b/index.ts @@ -10,6 +10,7 @@ import { sudoChown, sudoChownToRunningUser } from "./helpers/sudo.js"; import { setupKernel } from "./kernel.js"; import { runQemu } from "./qemu.js"; import { Runit } from "./runit/index.js"; +import { installFluentBit } from "./software/fluentbit"; import { setupForgejo } from "./services/forgejo/index.js"; import { setupDhcpcd } from "./services/dhcpcd.js"; import { setupNtpsec } from "./services/ntpsec.js"; @@ -36,6 +37,7 @@ if (process.argv[2] === "generate-secrets") { const alpine = await Alpine.makeWorld({ dir: rootfsDir }); await alpine.addPackages(["helix", "htop", "iproute2-ss", "socat"]); + await installFluentBit(alpine); const runit = await Runit.setup(alpine); await setupDhcpcd(alpine, runit); await setupNtpsec(alpine, runit); diff --git a/runit/index.ts b/runit/index.ts index 3848c41..8c5f1cc 100644 --- a/runit/index.ts +++ b/runit/index.ts @@ -78,20 +78,17 @@ runit-init 6` await runit.addService( "getty-tty1", `#!/bin/sh -exec chpst -P getty 38400 tty1 linux`, - false +exec chpst -P getty 38400 tty1 linux` ); await runit.addService( "getty-tty2", `#!/bin/sh -exec chpst -P getty 38400 tty2 linux`, - false +exec chpst -P getty 38400 tty2 linux` ); await runit.addService( "getty-ttyS0", `#!/bin/sh -exec chpst -P getty 38400 ttyS0 linux`, - false +exec chpst -P getty 38400 ttyS0 linux` ); return runit; @@ -100,17 +97,13 @@ exec chpst -P getty 38400 ttyS0 linux`, async addService( name: string, script: string, - log: boolean = true + logScript?: string ): Promise { const runScriptPath = path.join("/etc/sv/", name, "/run"); await this.alpine.sudoWriteExecutable(runScriptPath, script); - if (log) { + if (logScript) { const logScriptPath = path.join("/etc/sv/", name, "/log/run"); - await this.alpine.sudoWriteExecutable( - logScriptPath, - `#!/bin/sh -exec logger -p daemon.info -t '${name}'` - ); + await this.alpine.sudoWriteExecutable(logScriptPath, logScript); await this.alpine.symlink( `/run/runit/supervise.${name}.log`, path.join("/etc/sv/", name, "/log/supervise") diff --git a/services/forgejo/index.ts b/services/forgejo/index.ts index 970d606..e8c1d16 100644 --- a/services/forgejo/index.ts +++ b/services/forgejo/index.ts @@ -4,6 +4,7 @@ import { Runit } from "../../runit/index.js"; import { join } from "node:path"; import { loadForgejoSecretsFile } from "./secrets.js"; import { sudoCopy } from "../../helpers/sudo.js"; +import { FluentBitParser, runitLokiLogger } from "../../software/fluentbit.js"; export async function setupForgejo(alpine: Alpine, runit: Runit) { const bin = await buildForgejo(); @@ -163,6 +164,8 @@ export FORGEJO_WORK_DIR="$HOME" cd "$HOME" exec chpst -u $USER:$USER /usr/local/bin/forgejo web --config /etc/forgejo.conf 2>&1 -` +`, + // TODO: usar mejor parser + runitLokiLogger(FluentBitParser.Raw, "forgejo") ); } diff --git a/software/fluentbit.ts b/software/fluentbit.ts new file mode 100644 index 0000000..80d82f2 --- /dev/null +++ b/software/fluentbit.ts @@ -0,0 +1,100 @@ +import { join } from "node:path"; +import { Alpine } from "../alpine.js"; +import { buildRepro, reproRun } from "../helpers/repro-run.js"; +import { sudoCopy } from "../helpers/sudo.js"; + +const parsersPath = "/etc/fluent-bit/parsers.conf"; + +export async function installFluentBit(alpine: Alpine): Promise { + const bin = await buildFluentBit(); + await saveParsers(alpine); + await alpine.addPackages(["musl-fts", "yaml"]); + await sudoCopy(bin, join(alpine.dir, "/usr/local/bin/fluent-bit")); +} + +// ## Script generators + +// Returns a logScript to be used with runit for logging to Loki +export function runitLokiLogger(parser: FluentBitParser, name: string): string { + return `#!/bin/sh +exec chpst -u nobody:nobody /usr/local/bin/fluent-bit \ + --parser='${parsersPath}' \ + --input=stdin \ + --prop=parser='${parser}' \ + --output=loki \ + --prop=labels='job=fluentbit,stream=${name}' +`; +} + +// ## Parsers + +export enum FluentBitParser { + Json = "json", + // Raw toma todo lo que haya en una línea y lo guarda en `message`. No recomendado. + Raw = "raw", +} + +async function saveParsers(alpine: Alpine): Promise { + // https://github.com/fluent/fluent-bit/blob/master/conf/parsers.conf + await alpine.writeFile( + parsersPath, + ` +[PARSER] + name raw + format regex + regex ^(?.*?)$ +` + ); +} + +// ## Build binary + +function buildFluentBit(): Promise { + const version = "v2.0.9"; + return buildRepro( + "fluentbit", + version, + `#!/bin/sh -e +runprint() { + echo "==> $@" + "$@" +} + +runprint apk add --quiet \ + make gcc g++ patch musl-dev openssl-dev linux-headers \ + bison cmake flex musl-fts-dev gtest-dev yaml-dev zlib-dev + +wget https://github.com/fluent/fluent-bit/archive/refs/tags/${version}.tar.gz -O- | tar zx +cd fluent-bit-* + +patch --strip=1 <<'EOF' +--- a/lib/chunkio/src/CMakeLists.txt ++++ b/lib/chunkio/src/CMakeLists.txt +@@ -12,6 +12,7 @@ + ) + + set(libs cio-crc32) ++set(libs \${libs} fts) + + if(\${CMAKE_SYSTEM_NAME} MATCHES "Windows") + set(src +EOF + +runprint cmake -B build \ + -DFLB_CORO_STACK_SIZE=24576 \ + . +runprint make -C build + +ls build/bin/ + +mv build/bin/fluent-bit /fluent-bit +`, + (dir) => + reproRun({ + cwd: dir, + command: "/src/build", + cache: [], + }), + (dir) => join(dir, "rootfs/fluent-bit") + ); +}