grafana: loggear a loki

alpine.ts

@@ -1,11 +1,4 @@
-import {
-  chmod,
-  copyFile,
-  mkdir,
-  opendir,
-  symlink,
-  writeFile,
-} from "node:fs/promises";
+import { mkdir, opendir } from "node:fs/promises";
 import path from "node:path";
 import { cwd } from "node:process";
 import { Fstab } from "./fstab.js";
@@ -37,13 +30,13 @@ export class Alpine {
     content: string,
     permissions?: { uid: number; gid: number }
   ): Promise<void> {
+    const p = path.join(this.dir, filePath);
     await this.mkdirP(path.dirname(filePath));
-    await sudoWriteFile(path.join(this.dir, filePath), content);
-    if (permissions)
-      await sudoChown(
-        path.join(this.dir, filePath),
-        `${permissions.uid}:${permissions.gid}`
-      );
+    await sudoWriteFile(p, content);
+    if (permissions) {
+      await sudoChown(p, `${permissions.uid}:${permissions.gid}`);
+      await sudoChmod(p, "600");
+    }
   }
   async writeExecutable(filePath: string, content: string): Promise<void> {
     await this.writeFile(filePath, content);

helpers/repro-run.ts

@@ -1,4 +1,5 @@
-import { Writable } from "node:stream";
+import { chmod, mkdir, readFile, writeFile } from "node:fs/promises";
+import { join } from "node:path";
 import { execFile } from "./better-api.js";
 
 export async function reproRun(opts: {
@@ -6,7 +7,7 @@ export async function reproRun(opts: {
   // cache/ and rootfs/
   cwd: string;
   command: string;
-  cache: string[];
+  cache?: string[];
 }): Promise<void> {
   const run = execFile("repro-run", { cwd: opts.cwd });
   if (!run.child.stdin) throw false;
@@ -21,3 +22,29 @@ export async function reproRun(opts: {
   run.child.stderr?.pipe(process.stderr);
   await run;
 }
+
+export async function buildRepro<T>(
+  cacheName: string,
+  cacheKey: string,
+  buildScript: string,
+  run: (cwd: string) => Promise<void>,
+  getArtifacts: (cwd: string) => T
+): Promise<T> {
+  const dir = join("cache", cacheName);
+  await mkdir(dir, { recursive: true });
+  const cacheKeyFile = join(dir, "version");
+  const output = getArtifacts(dir);
+  try {
+    if ((await readFile(cacheKeyFile, "utf-8")) === cacheKey) return output;
+  } catch {}
+
+  {
+    const buildScriptPath = join(dir, "build");
+    await writeFile(buildScriptPath, buildScript);
+    await chmod(buildScriptPath, 0o700);
+  }
+  await run(dir);
+  await writeFile(cacheKeyFile, cacheKey);
+
+  return output;
+}

index.ts

@@ -10,10 +10,12 @@ import { sudoChown, sudoChownToRunningUser } from "./helpers/sudo.js";
 import { setupKernel } from "./kernel.js";
 import { runQemu } from "./qemu.js";
 import { Runit } from "./runit/index.js";
+import { installFluentBit } from "./software/fluentbit";
 import { setupForgejo } from "./services/forgejo/index.js";
 import { setupDhcpcd } from "./services/dhcpcd.js";
 import { setupNtpsec } from "./services/ntpsec.js";
 import { setupGrafana } from "./services/grafana/index.js";
+import { setupLoki } from "./services/loki/index.js";
 
 if (process.argv[2] === "generate-secrets") {
   await generateForgejoSecretsFile();
@@ -35,10 +37,19 @@ if (process.argv[2] === "generate-secrets") {
   const alpine = await Alpine.makeWorld({ dir: rootfsDir });
 
   await alpine.addPackages(["helix", "htop", "iproute2-ss", "socat"]);
+  await alpine.writeFile(
+    "/root/.ash_history",
+    `
+socat tcp-listen:80,reuseaddr,fork tcp:localhost:3050 &
+`,
+    { uid: 0, gid: 0 }
+  );
+  await installFluentBit(alpine);
   const runit = await Runit.setup(alpine);
   await setupDhcpcd(alpine, runit);
   await setupNtpsec(alpine, runit);
   await setupForgejo(alpine, runit);
+  await setupLoki(alpine, runit);
   await setupGrafana(alpine, runit);
   const kernel = await setupKernel(alpine, kernelDir);
 

runit/index.ts

@@ -78,20 +78,17 @@ runit-init 6`
     await runit.addService(
       "getty-tty1",
       `#!/bin/sh
-exec chpst -P getty 38400 tty1 linux`,
-      false
+exec chpst -P getty 38400 tty1 linux`
     );
     await runit.addService(
       "getty-tty2",
       `#!/bin/sh
-exec chpst -P getty 38400 tty2 linux`,
-      false
+exec chpst -P getty 38400 tty2 linux`
     );
     await runit.addService(
       "getty-ttyS0",
       `#!/bin/sh
-exec chpst -P getty 38400 ttyS0 linux`,
-      false
+exec chpst -P getty 38400 ttyS0 linux`
     );
 
     return runit;
@@ -100,17 +97,13 @@ exec chpst -P getty 38400 ttyS0 linux`,
   async addService(
     name: string,
     script: string,
-    log: boolean = true
+    logScript?: string
   ): Promise<void> {
     const runScriptPath = path.join("/etc/sv/", name, "/run");
     await this.alpine.sudoWriteExecutable(runScriptPath, script);
-    if (log) {
+    if (logScript) {
       const logScriptPath = path.join("/etc/sv/", name, "/log/run");
-      await this.alpine.sudoWriteExecutable(
-        logScriptPath,
-        `#!/bin/sh
-exec logger -p daemon.info -t '${name}'`
-      );
+      await this.alpine.sudoWriteExecutable(logScriptPath, logScript);
       await this.alpine.symlink(
         `/run/runit/supervise.${name}.log`,
         path.join("/etc/sv/", name, "/log/supervise")

services/forgejo/build.ts

@@ -1,24 +1,14 @@
 import { chmod, mkdir, readFile, writeFile } from "node:fs/promises";
 import { join } from "node:path";
-import { reproRun } from "../../helpers/repro-run.js";
+import { buildRepro, reproRun } from "../../helpers/repro-run.js";
 
 const FORGEJO_VERSION = "v1.18.3-0";
 
 // returns path to statically compiled binary
 export async function buildForgejo(): Promise<string> {
-  const dir = "cache/forgejo";
-  await mkdir(dir, { recursive: true });
-  const versionFile = join(dir, "version");
-  const output = join(dir, "rootfs/forgejo");
-  try {
-    if ((await readFile(versionFile, "utf-8")) === FORGEJO_VERSION)
-      return output;
-  } catch {}
-
-  {
-    const buildScript = join(dir, "build");
-    await writeFile(
-      buildScript,
+  return await buildRepro(
+    "forgejo",
+    FORGEJO_VERSION,
     `#!/bin/sh -e
 runprint() {
   echo "==> $@"
@@ -33,11 +23,9 @@ cd forgejo
 
 runprint env GOOS=linux GOARCH=amd64 LDFLAGS="-linkmode external -extldflags '-static' $LDFLAGS" TAGS="bindata sqlite sqlite_unlock_notify" make build
 mv gitea /forgejo
-`
-    );
-    await chmod(buildScript, 0o700);
-  }
-  await reproRun({
+`,
+    (dir) =>
+      reproRun({
         cwd: dir,
         command: "/src/build",
         cache: [
@@ -45,8 +33,7 @@ mv gitea /forgejo
           "/home/repro/go",
           "/home/repro/.npm",
         ],
-  });
-  await writeFile(versionFile, FORGEJO_VERSION);
-
-  return output;
+      }),
+    (dir) => join(dir, "rootfs/forgejo")
+  );
 }

services/forgejo/index.ts

@@ -1,10 +1,10 @@
 import { buildForgejo } from "./build.js";
 import { Alpine } from "../../alpine.js";
 import { Runit } from "../../runit/index.js";
-import { writeFile } from "node:fs/promises";
 import { join } from "node:path";
 import { loadForgejoSecretsFile } from "./secrets.js";
-import { sudoChown, sudoCopy, sudoWriteFile } from "../../helpers/sudo.js";
+import { sudoCopy } from "../../helpers/sudo.js";
+import { FluentBitParser, runitLokiLogger } from "../../software/fluentbit.js";
 
 export async function setupForgejo(alpine: Alpine, runit: Runit) {
   const bin = await buildForgejo();
@@ -20,9 +20,8 @@ export async function setupForgejo(alpine: Alpine, runit: Runit) {
   });
 
   const secrets = await loadForgejoSecretsFile();
-  const configPath = join(alpine.dir, "/etc/forgejo.conf");
-  await sudoWriteFile(
-    configPath,
+  await alpine.writeFile(
+    "/etc/forgejo.conf",
     `
 ; see https://docs.gitea.io/en-us/config-cheat-sheet/ for additional documentation.
 
@@ -145,9 +144,8 @@ ALLOWED_HOST_LIST=external,loopback
 REPO_INDEXER_ENABLED=true
 REPO_INDEXER_EXCLUDE=**.mp4,**.jpg
     `,
-    { mode: 0o600 }
+    entry
   );
-  await sudoChown(configPath, `${entry.uid}:${entry.gid}`);
   await runit.addService(
     "forgejo",
     `#!/bin/sh
@@ -166,6 +164,7 @@ export FORGEJO_WORK_DIR="$HOME"
 cd "$HOME"
 
 exec chpst -u $USER:$USER /usr/local/bin/forgejo web --config /etc/forgejo.conf 2>&1
-`
+`,
+    runitLokiLogger(FluentBitParser.Forgejo, "forgejo")
   );
 }

services/grafana/index.ts

@@ -1,8 +1,12 @@
 import assert from "node:assert";
+import { join } from "node:path";
 import { Alpine } from "../../alpine.js";
 import { Runit } from "../../runit/index.js";
+import { FluentBitParser, runitLokiLogger } from "../../software/fluentbit.js";
 import { loadGrafanaSecretsFile } from "./secrets.js";
 
+const provisioningDir = "/etc/grafana/provisioning/";
+
 // TODO: grafana-image-renderer?
 // /etc/conf.d/grafana
 // # To enable image rendering run
@@ -31,6 +35,20 @@ export async function setupGrafana(
   });
   await alpine.writeFile("/etc/grafana.ini", await genConfig(), user);
 
+  await alpine.writeFile(
+    join(provisioningDir, "datasources/loki.yaml"),
+    `
+apiVersion: 1
+
+datasources:
+  - name: Loki
+    type: loki
+    access: proxy
+    url: http://localhost:3100
+`,
+    user
+  );
+
   await alpine.sudoWriteExecutable(
     "/usr/local/sbin/nulo-grafana-cli",
     `#!/bin/sh
@@ -45,17 +63,9 @@ export GRAFANA_HOME=/var/lib/grafana
 
 cd "$GRAFANA_HOME"
 
-install -o grafana -m755 -d \
-		$GRAFANA_HOME/provisioning \
-		$GRAFANA_HOME/provisioning/dashboards \
-		$GRAFANA_HOME/provisioning/datasources \
-		$GRAFANA_HOME/provisioning/notifiers \
-		$GRAFANA_HOME/provisioning/alerting \
-		$GRAFANA_HOME/provisioning/plugins
-
-exec chpst -u grafana:grafana grafana-server -config /etc/grafana.ini -homepath /usr/share/grafana \
-cfg:paths.provisioning="$GRAFANA_HOME"/provisioning
-`
+exec chpst -u grafana:grafana grafana-server -config /etc/grafana.ini -homepath /usr/share/grafana
+`,
+    runitLokiLogger(FluentBitParser.Logfmt, "grafana")
   );
 }
 
@@ -81,7 +91,7 @@ data = /var/lib/grafana
 plugins = /var/lib/grafana/plugins
 
 # folder that contains provisioning config files that grafana will apply on startup and while running.
-;provisioning = conf/provisioning
+provisioning = ${provisioningDir}
 
 #################################### Server ####################################
 [server]

services/loki/build.ts

@@ -0,0 +1,39 @@
+import { join } from "node:path";
+import { buildRepro, reproRun } from "../../helpers/repro-run.js";
+
+const LOKI_VERSION = "v2.7.3";
+
+// returns path to statically compiled binary
+export function buildLoki(): Promise<string> {
+  return buildRepro(
+    "loki",
+    LOKI_VERSION,
+    `#!/bin/sh -e
+runprint() {
+  echo "==> $@"
+  "$@"
+}
+
+runprint apk add --quiet git go make libc-dev bash
+
+git config --global advice.detachedHead false
+# TODO: cachear clon de repo
+runprint git clone https://github.com/grafana/loki --branch '${LOKI_VERSION}' --depth 1 --single-branch
+cd loki
+
+runprint make -j1 GOMOD=readonly logcli loki
+mv cmd/loki/loki /loki
+`,
+    (dir) =>
+      reproRun({
+        cwd: dir,
+        command: "/src/build",
+        cache: [
+          "/home/repro/.cache/go-build",
+          "/home/repro/go",
+          "/home/repro/.npm",
+        ],
+      }),
+    (dir) => join(dir, "rootfs/loki")
+  );
+}

services/loki/index.ts

@@ -0,0 +1,81 @@
+import { join } from "node:path";
+import { Alpine } from "../../alpine.js";
+import { sudoCopy } from "../../helpers/sudo.js";
+import { Runit } from "../../runit/index.js";
+import { buildLoki } from "./build.js";
+
+export async function setupLoki(alpine: Alpine, runit: Runit): Promise<void> {
+  const bin = await buildLoki();
+  await sudoCopy(bin, join(alpine.dir, "/usr/local/bin/loki"));
+
+  const user = await alpine.userAdd("loki");
+
+  // TODO: data
+  // await alpine.fstab.addTmpfs("/var/lib/grafana", {
+  //   uid: user.uid,
+  //   gid: user.gid,
+  //   mode: "700",
+  // });
+  const configPath = "/etc/loki/loki-local-config.yaml";
+  await alpine.writeFile(
+    configPath,
+    `
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+  http_listen_address: 127.0.0.1
+  # grpc_listen_port: 9096
+  grpc_listen_address: 127.0.0.1
+
+common:
+  instance_addr: 127.0.0.1
+  path_prefix: /tmp/loki
+  storage:
+    filesystem:
+      chunks_directory: /tmp/loki/chunks
+      rules_directory: /tmp/loki/rules
+  replication_factor: 1
+  ring:
+    kvstore:
+      store: inmemory
+
+query_range:
+  results_cache:
+    cache:
+      embedded_cache:
+        enabled: true
+        max_size_mb: 100
+
+schema_config:
+  configs:
+    - from: 2020-10-24
+      store: boltdb-shipper
+      object_store: filesystem
+      schema: v11
+      index:
+        prefix: index_
+        period: 24h
+
+# ruler:
+#   alertmanager_url: http://localhost:9093
+
+analytics:
+  reporting_enabled: false`,
+    user
+  );
+
+  //   await alpine.sudoWriteExecutable(
+  //     "/usr/local/sbin/nulo-grafana-cli",
+  //     `#!/bin/sh
+  // cd /
+  // exec chpst -u grafana:grafana grafana-cli --homepath /usr/share/grafana --config /etc/grafana.ini "$@"`
+  //   );
+
+  await runit.addService(
+    "loki",
+    `#!/bin/sh
+exec chpst -u ${user.name}:${user.name} /usr/local/bin/loki -config.file='${configPath}'
+`
+  );
+}

services/ntpsec.ts

@@ -1,6 +1,7 @@
 import { Alpine } from "../alpine.js";
 import { sudoWriteFile } from "../helpers/sudo.js";
 import { Runit } from "../runit/index.js";
+import { FluentBitParser, runitLokiLogger } from "../software/fluentbit.js";
 
 export async function setupNtpsec(alpine: Alpine, runit: Runit) {
   await alpine.addPackages(["ntpsec"]);
@@ -48,6 +49,7 @@ server gps.ntp.br nts iburst
     "ntpsec",
     `#!/bin/sh
 exec ntpd --nice --nofork --panicgate
-`
+`,
+    runitLokiLogger(FluentBitParser.Ntpsec, "ntpsec")
   );
 }

software/fluentbit.ts

@@ -0,0 +1,119 @@
+import { join } from "node:path";
+import { Alpine } from "../alpine.js";
+import { buildRepro, reproRun } from "../helpers/repro-run.js";
+import { sudoCopy } from "../helpers/sudo.js";
+
+const parsersPath = "/etc/fluent-bit/parsers.conf";
+
+export async function installFluentBit(alpine: Alpine): Promise<void> {
+  const bin = await buildFluentBit();
+  await saveParsers(alpine);
+  await alpine.addPackages(["musl-fts", "yaml"]);
+  await sudoCopy(bin, join(alpine.dir, "/usr/local/bin/fluent-bit"));
+}
+
+// ## Script generators
+
+// Returns a logScript to be used with runit for logging to Loki
+export function runitLokiLogger(parser: FluentBitParser, name: string): string {
+  return `#!/bin/sh
+exec chpst -u nobody:nobody /usr/local/bin/fluent-bit \
+  --parser='${parsersPath}' \
+  --input=stdin \
+    --prop=parser='${parser}' \
+  --output=loki \
+    --prop=labels='job=fluentbit,stream=${name}'
+`;
+}
+
+// ## Parsers
+
+export enum FluentBitParser {
+  Json = "json",
+  Logfmt = "logfmt",
+  // Raw toma todo lo que haya en una línea y lo guarda en `message`. No recomendado.
+  Raw = "raw",
+
+  Forgejo = "forgejo",
+  Ntpsec = "ntpsec",
+}
+
+async function saveParsers(alpine: Alpine): Promise<void> {
+  // https://github.com/fluent/fluent-bit/blob/master/conf/parsers.conf
+  await alpine.writeFile(
+    parsersPath,
+    // https://rubular.com/
+    `
+[PARSER]
+    name logfmt
+    format logfmt
+
+[PARSER]
+    name raw
+    format regex
+    regex ^(?<message>.*?)$
+
+[PARSER]
+    name forgejo
+    format regex
+    regex ^((?<time>\\d{4}\\/\\d{2}\\/\\d{2} \\d{2}:\\d{2}:\\d{2}?) )?((?<trace>.+:.+:.+\\(\\)?) \\[(?<level>.?)\\] )?(?<message>.*?)$
+
+[PARSER]
+    name ntpsec
+    format regex
+    regex ^(?<time>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}) (?<process>.+\\[\\d+\\]): (?<component>\\w+): (?<message>.+)$
+`
+  );
+}
+
+// ## Build binary
+
+function buildFluentBit(): Promise<string> {
+  const version = "v2.0.9";
+  return buildRepro(
+    "fluentbit",
+    version,
+    `#!/bin/sh -e
+runprint() {
+  echo "==> $@"
+  "$@"
+}
+
+runprint apk add --quiet \
+    make gcc g++ patch musl-dev openssl-dev linux-headers \
+    bison cmake flex musl-fts-dev gtest-dev yaml-dev zlib-dev
+
+wget https://github.com/fluent/fluent-bit/archive/refs/tags/${version}.tar.gz -O- | tar zx
+cd fluent-bit-*
+
+patch --strip=1 <<'EOF'
+--- a/lib/chunkio/src/CMakeLists.txt
++++ b/lib/chunkio/src/CMakeLists.txt
+@@ -12,6 +12,7 @@
+   )
+ 
+ set(libs cio-crc32)
++set(libs \${libs} fts)
+ 
+ if(\${CMAKE_SYSTEM_NAME} MATCHES "Windows")
+   set(src
+EOF
+    
+runprint cmake -B build \
+	-DFLB_CORO_STACK_SIZE=24576 \
+	.
+runprint make -C build
+
+ls build/bin/
+
+mv build/bin/fluent-bit /fluent-bit
+`,
+    (dir) =>
+      reproRun({
+        cwd: dir,
+        command: "/src/build",
+        cache: [],
+      }),
+    (dir) => join(dir, "rootfs/fluent-bit")
+  );
+}