fix: make Ops gateway resilient to Synology reboot

docker-compose.synology.yml

@@ -1,6 +1,7 @@
 services:
   postgres:
     image: postgres:17-alpine
+    restart: unless-stopped
     environment:
       POSTGRES_DB: ${POSTGRES_DB:-nodedc_agent_gateway}
       POSTGRES_USER: ${POSTGRES_USER:-nodedc_agent_gateway}
@@ -17,6 +18,7 @@ services:
     build:
       context: .
     init: true
+    restart: unless-stopped
     environment:
       NODE_ENV: ${NODE_ENV:-production}
       HOST: 0.0.0.0
@@ -31,6 +33,8 @@ services:
       NODEDC_LAUNCHER_INTERNAL_URL: ${NODEDC_LAUNCHER_INTERNAL_URL:-http://172.22.0.222:18080}
       NODEDC_TASKER_INTERNAL_URL: ${NODEDC_TASKER_INTERNAL_URL:-http://172.22.0.222:18090}
       NODEDC_ENGINE_INTERNAL_URL: ${NODEDC_ENGINE_INTERNAL_URL:-http://172.22.0.222:3001}
+      NODEDC_ONTOLOGY_CORE_URL: ${NODEDC_ONTOLOGY_CORE_URL:-http://172.22.0.222:18104}
+      NODEDC_ONTOLOGY_CORE_ACCESS_TOKEN: ${NODEDC_ONTOLOGY_CORE_ACCESS_TOKEN:-}
       NODEDC_INTERNAL_ACCESS_TOKEN: ${NODEDC_INTERNAL_ACCESS_TOKEN}
     depends_on:
       postgres:

scripts/build-ops-agents-reboot-resilience-artifact.mjs

@@ -0,0 +1,145 @@
+#!/usr/bin/env node
+
+import { createHash } from "node:crypto";
+import { spawnSync } from "node:child_process";
+import { cp, lstat, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { dirname, join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const scriptDir = dirname(fileURLToPath(import.meta.url));
+const sourceRoot = resolve(scriptDir, "..");
+const artifactRoot = resolve(
+  process.env.NODEDC_DEPLOY_ARTIFACT_DIR || resolve(sourceRoot, "deploy-artifacts"),
+);
+const artifactId = "ops-agents-reboot-resilience-20260809-001";
+const component = "ops-agents";
+const files = [
+  "docker-compose.synology.yml",
+  "src/db/wait-for-database.test.ts",
+  "src/db/wait-for-database.ts",
+  "src/scripts/migrate.ts",
+];
+const predecessors = {
+  "docker-compose.synology.yml": "e78d37368c3d255f257086773ba908e44aae2e475661b80ee17f98c3d55d178f",
+  "src/scripts/migrate.ts": "8396a42deded31f1fe2381f772838a33b05bfd389075ebd34a5f546593f08769",
+};
+const newPaths = [
+  "src/db/wait-for-database.test.ts",
+  "src/db/wait-for-database.ts",
+];
+
+await mkdir(artifactRoot, { recursive: true });
+const artifact = join(artifactRoot, `nodedc-${artifactId}.tgz`);
+const predecessorPath = join(artifactRoot, `nodedc-${artifactId}.predecessor.sha256`);
+const newPathsPath = join(artifactRoot, `nodedc-${artifactId}.new-paths`);
+await Promise.all([artifact, predecessorPath, newPathsPath].map(assertFresh));
+
+const stage = await mkdtemp(join(tmpdir(), "nodedc-ops-agents-reboot-resilience-"));
+const payload = join(stage, "payload");
+
+try {
+  await mkdir(payload, { recursive: true });
+  for (const relativePath of files) {
+    validateRelativePath(relativePath);
+    const source = join(sourceRoot, relativePath);
+    const info = await lstat(source);
+    if (!info.isFile() || info.isSymbolicLink()) {
+      throw new Error(`invalid_source_file:${relativePath}`);
+    }
+    await mkdir(dirname(join(payload, relativePath)), { recursive: true });
+    await cp(source, join(payload, relativePath), { force: false });
+  }
+
+  await writeFile(
+    join(stage, "manifest.env"),
+    `id=${artifactId}\ncomponent=${component}\ntype=app-overlay\n`,
+    "utf8",
+  );
+  await writeFile(join(stage, "files.txt"), `${files.join("\n")}\n`, "utf8");
+  run("python3", ["-c", canonicalTarScript(), artifact, stage]);
+
+  const productionRoot = "/volume1/docker/nodedc-platform/ops-agents";
+  await writeFile(
+    predecessorPath,
+    `${Object.entries(predecessors)
+      .map(([relativePath, digest]) => `${digest}  ${productionRoot}/${relativePath}`)
+      .join("\n")}\n`,
+    "utf8",
+  );
+  await writeFile(
+    newPathsPath,
+    `${newPaths.map((relativePath) => `${productionRoot}/${relativePath}`).join("\n")}\n`,
+    "utf8",
+  );
+
+  const artifactBytes = await readFile(artifact);
+  console.log(JSON.stringify({
+    ok: true,
+    artifactId,
+    component,
+    artifact,
+    artifactSha256: sha256(artifactBytes),
+    files,
+    predecessors,
+    newPaths,
+    expectedService: "agent-gateway",
+    expectedHealthcheck: "http://172.22.0.222:18190/readyz",
+    runnerChanged: false,
+    databaseVolumeSelected: false,
+  }, null, 2));
+} finally {
+  await rm(stage, { recursive: true, force: true });
+}
+
+async function assertFresh(path) {
+  try {
+    await lstat(path);
+  } catch (error) {
+    if (error?.code === "ENOENT") return;
+    throw error;
+  }
+  throw new Error(`output_already_exists:${path}`);
+}
+
+function validateRelativePath(value) {
+  if (
+    typeof value !== "string"
+    || !value
+    || value.startsWith("/")
+    || value.split("/").some((part) => !part || part === "." || part === "..")
+  ) {
+    throw new Error(`unsafe_relative_path:${value}`);
+  }
+}
+
+function canonicalTarScript() {
+  return [
+    "import gzip,io,pathlib,sys,tarfile",
+    "root=pathlib.Path(sys.argv[2])",
+    "with open(sys.argv[1],'wb') as out:",
+    " with gzip.GzipFile(filename='',mode='wb',fileobj=out,compresslevel=9,mtime=0) as gz:",
+    "  with tarfile.open(fileobj=gz,mode='w',format=tarfile.PAX_FORMAT) as tar:",
+    "   for top in ('manifest.env','files.txt','payload'):",
+    "    p=root/top; paths=[p]+(sorted(p.rglob('*')) if p.is_dir() else [])",
+    "    for x in paths:",
+    "     info=tar.gettarinfo(str(x),arcname=x.relative_to(root).as_posix())",
+    "     info.uid=info.gid=0; info.uname=info.gname='root'; info.mtime=0; info.mode=0o755 if info.isdir() else 0o644",
+    "     with (open(x,'rb') if info.isfile() else io.BytesIO()) as src: tar.addfile(info,src if info.isfile() else None)",
+  ].join("\n");
+}
+
+function run(command, args) {
+  const result = spawnSync(command, args, {
+    encoding: "utf8",
+    maxBuffer: 128 * 1024 * 1024,
+    stdio: ["ignore", "pipe", "pipe"],
+  });
+  if (result.status !== 0) {
+    throw new Error(`${command}_failed:${result.stderr || result.stdout}`);
+  }
+}
+
+function sha256(bytes) {
+  return createHash("sha256").update(bytes).digest("hex");
+}

scripts/build-ops-agents-reboot-resilience-artifact.test.mjs

@@ -0,0 +1,89 @@
+import assert from "node:assert/strict";
+import { createHash } from "node:crypto";
+import { spawnSync } from "node:child_process";
+import { mkdtemp, readFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { dirname, join } from "node:path";
+import test from "node:test";
+import { fileURLToPath } from "node:url";
+
+const scriptDir = dirname(fileURLToPath(import.meta.url));
+const builder = join(scriptDir, "build-ops-agents-reboot-resilience-artifact.mjs");
+const basename = "nodedc-ops-agents-reboot-resilience-20260809-001";
+
+test("builds a deterministic, minimal Ops Agents application overlay", async () => {
+  const firstRoot = await mkdtemp(join(tmpdir(), "ops-agents-artifact-first-"));
+  const secondRoot = await mkdtemp(join(tmpdir(), "ops-agents-artifact-second-"));
+
+  try {
+    const first = runBuilder(firstRoot);
+    const second = runBuilder(secondRoot);
+    const firstArtifact = join(firstRoot, `${basename}.tgz`);
+    const secondArtifact = join(secondRoot, `${basename}.tgz`);
+
+    assert.equal(first.artifactSha256, second.artifactSha256);
+    assert.equal(first.artifactSha256, sha256(await readFile(firstArtifact)));
+    assert.deepEqual(first.files, [
+      "docker-compose.synology.yml",
+      "src/db/wait-for-database.test.ts",
+      "src/db/wait-for-database.ts",
+      "src/scripts/migrate.ts",
+    ]);
+    assert.equal(first.runnerChanged, false);
+    assert.equal(first.databaseVolumeSelected, false);
+
+    const members = new Set(runTar(["-tzf", firstArtifact]).trim().split("\n"));
+    assert.deepEqual(members, new Set([
+      "manifest.env",
+      "files.txt",
+      "payload/",
+      "payload/docker-compose.synology.yml",
+      "payload/src/",
+      "payload/src/db/",
+      "payload/src/db/wait-for-database.test.ts",
+      "payload/src/db/wait-for-database.ts",
+      "payload/src/scripts/",
+      "payload/src/scripts/migrate.ts",
+    ]));
+    assert.equal([...members].some((member) => member.includes("._")), false);
+
+    assert.equal(
+      runTar(["-xOzf", firstArtifact, "manifest.env"]),
+      "id=ops-agents-reboot-resilience-20260809-001\ncomponent=ops-agents\ntype=app-overlay\n",
+    );
+    assert.equal(
+      runTar(["-xOzf", firstArtifact, "files.txt"]),
+      `${first.files.join("\n")}\n`,
+    );
+
+    const predecessors = await readFile(join(firstRoot, `${basename}.predecessor.sha256`), "utf8");
+    assert.match(predecessors, /^e78d37368c3d255f257086773ba908e44aae2e475661b80ee17f98c3d55d178f  .*docker-compose\.synology\.yml$/m);
+    assert.match(predecessors, /^8396a42deded31f1fe2381f772838a33b05bfd389075ebd34a5f546593f08769  .*src\/scripts\/migrate\.ts$/m);
+
+    const newPaths = await readFile(join(firstRoot, `${basename}.new-paths`), "utf8");
+    assert.match(newPaths, /src\/db\/wait-for-database\.ts/);
+    assert.match(newPaths, /src\/db\/wait-for-database\.test\.ts/);
+  } finally {
+    await rm(firstRoot, { recursive: true, force: true });
+    await rm(secondRoot, { recursive: true, force: true });
+  }
+});
+
+function runBuilder(outputRoot) {
+  const result = spawnSync(process.execPath, [builder], {
+    encoding: "utf8",
+    env: { ...process.env, NODEDC_DEPLOY_ARTIFACT_DIR: outputRoot },
+  });
+  assert.equal(result.status, 0, result.stderr || result.stdout);
+  return JSON.parse(result.stdout);
+}
+
+function runTar(args) {
+  const result = spawnSync("tar", args, { encoding: "utf8" });
+  assert.equal(result.status, 0, result.stderr || result.stdout);
+  return result.stdout;
+}
+
+function sha256(bytes) {
+  return createHash("sha256").update(bytes).digest("hex");
+}

src/db/wait-for-database.test.ts

@@ -0,0 +1,74 @@
+import assert from "node:assert/strict";
+import { readFile } from "node:fs/promises";
+import test from "node:test";
+
+import {
+  DEFAULT_DATABASE_STARTUP_MAX_ATTEMPTS,
+  waitForDatabase,
+} from "./wait-for-database.js";
+
+test("waits through a DNS outage longer than the legacy 30-second window", async () => {
+  let attempts = 0;
+  let sleeps = 0;
+
+  await waitForDatabase(
+    {
+      async query() {
+        attempts += 1;
+        if (attempts <= 35) {
+          const error = new Error("getaddrinfo ENOTFOUND postgres");
+          Object.assign(error, { code: "ENOTFOUND", hostname: "postgres" });
+          throw error;
+        }
+      },
+    },
+    {
+      sleep: async () => {
+        sleeps += 1;
+      },
+    },
+  );
+
+  assert.equal(DEFAULT_DATABASE_STARTUP_MAX_ATTEMPTS, 300);
+  assert.equal(attempts, 36);
+  assert.equal(sleeps, 35);
+});
+
+test("keeps the startup wait bounded and rethrows the last database error", async () => {
+  const terminalError = new Error("database unavailable");
+  let attempts = 0;
+  let sleeps = 0;
+
+  await assert.rejects(
+    waitForDatabase(
+      {
+        async query() {
+          attempts += 1;
+          throw terminalError;
+        },
+      },
+      {
+        maxAttempts: 3,
+        retryDelayMs: 0,
+        sleep: async () => {
+          sleeps += 1;
+        },
+      },
+    ),
+    terminalError,
+  );
+
+  assert.equal(attempts, 3);
+  assert.equal(sleeps, 2);
+});
+
+test("Synology compose restarts both gateway services and preserves live Ontology wiring", async () => {
+  const compose = await readFile(new URL("../../docker-compose.synology.yml", import.meta.url), "utf8");
+  const [postgresBlock, gatewayAndVolumes] = compose.split("\n  agent-gateway:");
+  const [gatewayBlock] = gatewayAndVolumes.split("\nvolumes:");
+
+  assert.match(postgresBlock, /\n    restart: unless-stopped\n/);
+  assert.match(gatewayBlock, /\n    restart: unless-stopped\n/);
+  assert.match(gatewayBlock, /NODEDC_ONTOLOGY_CORE_URL:/);
+  assert.match(gatewayBlock, /NODEDC_ONTOLOGY_CORE_ACCESS_TOKEN:/);
+});

src/db/wait-for-database.ts

@@ -0,0 +1,40 @@
+export interface DatabaseProbe {
+  query(sql: string): Promise<unknown>;
+}
+
+export interface WaitForDatabaseOptions {
+  maxAttempts?: number;
+  retryDelayMs?: number;
+  sleep?: (delayMs: number) => Promise<void>;
+}
+
+export const DEFAULT_DATABASE_STARTUP_MAX_ATTEMPTS = 300;
+export const DEFAULT_DATABASE_STARTUP_RETRY_DELAY_MS = 1_000;
+
+export async function waitForDatabase(
+  database: DatabaseProbe,
+  options: WaitForDatabaseOptions = {},
+): Promise<void> {
+  const maxAttempts = options.maxAttempts ?? DEFAULT_DATABASE_STARTUP_MAX_ATTEMPTS;
+  const retryDelayMs = options.retryDelayMs ?? DEFAULT_DATABASE_STARTUP_RETRY_DELAY_MS;
+  const sleep = options.sleep ?? ((delayMs) => new Promise((resolve) => setTimeout(resolve, delayMs)));
+
+  if (!Number.isInteger(maxAttempts) || maxAttempts < 1) {
+    throw new RangeError("maxAttempts must be a positive integer");
+  }
+  if (!Number.isFinite(retryDelayMs) || retryDelayMs < 0) {
+    throw new RangeError("retryDelayMs must be a non-negative number");
+  }
+
+  for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
+    try {
+      await database.query("SELECT 1");
+      return;
+    } catch (error) {
+      if (attempt === maxAttempts) {
+        throw error;
+      }
+      await sleep(retryDelayMs);
+    }
+  }
+}

src/scripts/migrate.ts

@@ -2,6 +2,7 @@ import { Pool } from "pg";
 
 import { loadConfig } from "../config.js";
 import { runMigrations } from "../db/migrations.js";
+import { waitForDatabase } from "../db/wait-for-database.js";
 
 const config = loadConfig();
 
@@ -22,20 +23,3 @@ try {
 } finally {
   await pool.end();
 }
-
-async function waitForDatabase(pool: Pool): Promise<void> {
-  const maxAttempts = 30;
-
-  for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
-    try {
-      await pool.query("SELECT 1");
-      return;
-    } catch (error) {
-      if (attempt === maxAttempts) {
-        throw error;
-      }
-
-      await new Promise((resolve) => setTimeout(resolve, 1000));
-    }
-  }
-}