diff --git a/docker-compose.synology.yml b/docker-compose.synology.yml index 04f9f1b..2d16e24 100644 --- a/docker-compose.synology.yml +++ b/docker-compose.synology.yml @@ -1,6 +1,7 @@ services: postgres: image: postgres:17-alpine + restart: unless-stopped environment: POSTGRES_DB: ${POSTGRES_DB:-nodedc_agent_gateway} POSTGRES_USER: ${POSTGRES_USER:-nodedc_agent_gateway} @@ -17,6 +18,7 @@ services: build: context: . init: true + restart: unless-stopped environment: NODE_ENV: ${NODE_ENV:-production} HOST: 0.0.0.0 @@ -31,6 +33,8 @@ services: NODEDC_LAUNCHER_INTERNAL_URL: ${NODEDC_LAUNCHER_INTERNAL_URL:-http://172.22.0.222:18080} NODEDC_TASKER_INTERNAL_URL: ${NODEDC_TASKER_INTERNAL_URL:-http://172.22.0.222:18090} NODEDC_ENGINE_INTERNAL_URL: ${NODEDC_ENGINE_INTERNAL_URL:-http://172.22.0.222:3001} + NODEDC_ONTOLOGY_CORE_URL: ${NODEDC_ONTOLOGY_CORE_URL:-http://172.22.0.222:18104} + NODEDC_ONTOLOGY_CORE_ACCESS_TOKEN: ${NODEDC_ONTOLOGY_CORE_ACCESS_TOKEN:-} NODEDC_INTERNAL_ACCESS_TOKEN: ${NODEDC_INTERNAL_ACCESS_TOKEN} depends_on: postgres: diff --git a/scripts/build-ops-agents-reboot-resilience-artifact.mjs b/scripts/build-ops-agents-reboot-resilience-artifact.mjs new file mode 100644 index 0000000..27b58a3 --- /dev/null +++ b/scripts/build-ops-agents-reboot-resilience-artifact.mjs @@ -0,0 +1,145 @@ +#!/usr/bin/env node + +import { createHash } from "node:crypto"; +import { spawnSync } from "node:child_process"; +import { cp, lstat, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const scriptDir = dirname(fileURLToPath(import.meta.url)); +const sourceRoot = resolve(scriptDir, ".."); +const artifactRoot = resolve( + process.env.NODEDC_DEPLOY_ARTIFACT_DIR || resolve(sourceRoot, "deploy-artifacts"), +); +const artifactId = "ops-agents-reboot-resilience-20260809-001"; +const component = "ops-agents"; +const files = [ + "docker-compose.synology.yml", + "src/db/wait-for-database.test.ts", + "src/db/wait-for-database.ts", + "src/scripts/migrate.ts", +]; +const predecessors = { + "docker-compose.synology.yml": "e78d37368c3d255f257086773ba908e44aae2e475661b80ee17f98c3d55d178f", + "src/scripts/migrate.ts": "8396a42deded31f1fe2381f772838a33b05bfd389075ebd34a5f546593f08769", +}; +const newPaths = [ + "src/db/wait-for-database.test.ts", + "src/db/wait-for-database.ts", +]; + +await mkdir(artifactRoot, { recursive: true }); +const artifact = join(artifactRoot, `nodedc-${artifactId}.tgz`); +const predecessorPath = join(artifactRoot, `nodedc-${artifactId}.predecessor.sha256`); +const newPathsPath = join(artifactRoot, `nodedc-${artifactId}.new-paths`); +await Promise.all([artifact, predecessorPath, newPathsPath].map(assertFresh)); + +const stage = await mkdtemp(join(tmpdir(), "nodedc-ops-agents-reboot-resilience-")); +const payload = join(stage, "payload"); + +try { + await mkdir(payload, { recursive: true }); + for (const relativePath of files) { + validateRelativePath(relativePath); + const source = join(sourceRoot, relativePath); + const info = await lstat(source); + if (!info.isFile() || info.isSymbolicLink()) { + throw new Error(`invalid_source_file:${relativePath}`); + } + await mkdir(dirname(join(payload, relativePath)), { recursive: true }); + await cp(source, join(payload, relativePath), { force: false }); + } + + await writeFile( + join(stage, "manifest.env"), + `id=${artifactId}\ncomponent=${component}\ntype=app-overlay\n`, + "utf8", + ); + await writeFile(join(stage, "files.txt"), `${files.join("\n")}\n`, "utf8"); + run("python3", ["-c", canonicalTarScript(), artifact, stage]); + + const productionRoot = "/volume1/docker/nodedc-platform/ops-agents"; + await writeFile( + predecessorPath, + `${Object.entries(predecessors) + .map(([relativePath, digest]) => `${digest} ${productionRoot}/${relativePath}`) + .join("\n")}\n`, + "utf8", + ); + await writeFile( + newPathsPath, + `${newPaths.map((relativePath) => `${productionRoot}/${relativePath}`).join("\n")}\n`, + "utf8", + ); + + const artifactBytes = await readFile(artifact); + console.log(JSON.stringify({ + ok: true, + artifactId, + component, + artifact, + artifactSha256: sha256(artifactBytes), + files, + predecessors, + newPaths, + expectedService: "agent-gateway", + expectedHealthcheck: "http://172.22.0.222:18190/readyz", + runnerChanged: false, + databaseVolumeSelected: false, + }, null, 2)); +} finally { + await rm(stage, { recursive: true, force: true }); +} + +async function assertFresh(path) { + try { + await lstat(path); + } catch (error) { + if (error?.code === "ENOENT") return; + throw error; + } + throw new Error(`output_already_exists:${path}`); +} + +function validateRelativePath(value) { + if ( + typeof value !== "string" + || !value + || value.startsWith("/") + || value.split("/").some((part) => !part || part === "." || part === "..") + ) { + throw new Error(`unsafe_relative_path:${value}`); + } +} + +function canonicalTarScript() { + return [ + "import gzip,io,pathlib,sys,tarfile", + "root=pathlib.Path(sys.argv[2])", + "with open(sys.argv[1],'wb') as out:", + " with gzip.GzipFile(filename='',mode='wb',fileobj=out,compresslevel=9,mtime=0) as gz:", + " with tarfile.open(fileobj=gz,mode='w',format=tarfile.PAX_FORMAT) as tar:", + " for top in ('manifest.env','files.txt','payload'):", + " p=root/top; paths=[p]+(sorted(p.rglob('*')) if p.is_dir() else [])", + " for x in paths:", + " info=tar.gettarinfo(str(x),arcname=x.relative_to(root).as_posix())", + " info.uid=info.gid=0; info.uname=info.gname='root'; info.mtime=0; info.mode=0o755 if info.isdir() else 0o644", + " with (open(x,'rb') if info.isfile() else io.BytesIO()) as src: tar.addfile(info,src if info.isfile() else None)", + ].join("\n"); +} + +function run(command, args) { + const result = spawnSync(command, args, { + encoding: "utf8", + maxBuffer: 128 * 1024 * 1024, + stdio: ["ignore", "pipe", "pipe"], + }); + if (result.status !== 0) { + throw new Error(`${command}_failed:${result.stderr || result.stdout}`); + } +} + +function sha256(bytes) { + return createHash("sha256").update(bytes).digest("hex"); +} diff --git a/scripts/build-ops-agents-reboot-resilience-artifact.test.mjs b/scripts/build-ops-agents-reboot-resilience-artifact.test.mjs new file mode 100644 index 0000000..684d70a --- /dev/null +++ b/scripts/build-ops-agents-reboot-resilience-artifact.test.mjs @@ -0,0 +1,89 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { spawnSync } from "node:child_process"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; + +const scriptDir = dirname(fileURLToPath(import.meta.url)); +const builder = join(scriptDir, "build-ops-agents-reboot-resilience-artifact.mjs"); +const basename = "nodedc-ops-agents-reboot-resilience-20260809-001"; + +test("builds a deterministic, minimal Ops Agents application overlay", async () => { + const firstRoot = await mkdtemp(join(tmpdir(), "ops-agents-artifact-first-")); + const secondRoot = await mkdtemp(join(tmpdir(), "ops-agents-artifact-second-")); + + try { + const first = runBuilder(firstRoot); + const second = runBuilder(secondRoot); + const firstArtifact = join(firstRoot, `${basename}.tgz`); + const secondArtifact = join(secondRoot, `${basename}.tgz`); + + assert.equal(first.artifactSha256, second.artifactSha256); + assert.equal(first.artifactSha256, sha256(await readFile(firstArtifact))); + assert.deepEqual(first.files, [ + "docker-compose.synology.yml", + "src/db/wait-for-database.test.ts", + "src/db/wait-for-database.ts", + "src/scripts/migrate.ts", + ]); + assert.equal(first.runnerChanged, false); + assert.equal(first.databaseVolumeSelected, false); + + const members = new Set(runTar(["-tzf", firstArtifact]).trim().split("\n")); + assert.deepEqual(members, new Set([ + "manifest.env", + "files.txt", + "payload/", + "payload/docker-compose.synology.yml", + "payload/src/", + "payload/src/db/", + "payload/src/db/wait-for-database.test.ts", + "payload/src/db/wait-for-database.ts", + "payload/src/scripts/", + "payload/src/scripts/migrate.ts", + ])); + assert.equal([...members].some((member) => member.includes("._")), false); + + assert.equal( + runTar(["-xOzf", firstArtifact, "manifest.env"]), + "id=ops-agents-reboot-resilience-20260809-001\ncomponent=ops-agents\ntype=app-overlay\n", + ); + assert.equal( + runTar(["-xOzf", firstArtifact, "files.txt"]), + `${first.files.join("\n")}\n`, + ); + + const predecessors = await readFile(join(firstRoot, `${basename}.predecessor.sha256`), "utf8"); + assert.match(predecessors, /^e78d37368c3d255f257086773ba908e44aae2e475661b80ee17f98c3d55d178f .*docker-compose\.synology\.yml$/m); + assert.match(predecessors, /^8396a42deded31f1fe2381f772838a33b05bfd389075ebd34a5f546593f08769 .*src\/scripts\/migrate\.ts$/m); + + const newPaths = await readFile(join(firstRoot, `${basename}.new-paths`), "utf8"); + assert.match(newPaths, /src\/db\/wait-for-database\.ts/); + assert.match(newPaths, /src\/db\/wait-for-database\.test\.ts/); + } finally { + await rm(firstRoot, { recursive: true, force: true }); + await rm(secondRoot, { recursive: true, force: true }); + } +}); + +function runBuilder(outputRoot) { + const result = spawnSync(process.execPath, [builder], { + encoding: "utf8", + env: { ...process.env, NODEDC_DEPLOY_ARTIFACT_DIR: outputRoot }, + }); + assert.equal(result.status, 0, result.stderr || result.stdout); + return JSON.parse(result.stdout); +} + +function runTar(args) { + const result = spawnSync("tar", args, { encoding: "utf8" }); + assert.equal(result.status, 0, result.stderr || result.stdout); + return result.stdout; +} + +function sha256(bytes) { + return createHash("sha256").update(bytes).digest("hex"); +} diff --git a/src/db/wait-for-database.test.ts b/src/db/wait-for-database.test.ts new file mode 100644 index 0000000..2ec2618 --- /dev/null +++ b/src/db/wait-for-database.test.ts @@ -0,0 +1,74 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import test from "node:test"; + +import { + DEFAULT_DATABASE_STARTUP_MAX_ATTEMPTS, + waitForDatabase, +} from "./wait-for-database.js"; + +test("waits through a DNS outage longer than the legacy 30-second window", async () => { + let attempts = 0; + let sleeps = 0; + + await waitForDatabase( + { + async query() { + attempts += 1; + if (attempts <= 35) { + const error = new Error("getaddrinfo ENOTFOUND postgres"); + Object.assign(error, { code: "ENOTFOUND", hostname: "postgres" }); + throw error; + } + }, + }, + { + sleep: async () => { + sleeps += 1; + }, + }, + ); + + assert.equal(DEFAULT_DATABASE_STARTUP_MAX_ATTEMPTS, 300); + assert.equal(attempts, 36); + assert.equal(sleeps, 35); +}); + +test("keeps the startup wait bounded and rethrows the last database error", async () => { + const terminalError = new Error("database unavailable"); + let attempts = 0; + let sleeps = 0; + + await assert.rejects( + waitForDatabase( + { + async query() { + attempts += 1; + throw terminalError; + }, + }, + { + maxAttempts: 3, + retryDelayMs: 0, + sleep: async () => { + sleeps += 1; + }, + }, + ), + terminalError, + ); + + assert.equal(attempts, 3); + assert.equal(sleeps, 2); +}); + +test("Synology compose restarts both gateway services and preserves live Ontology wiring", async () => { + const compose = await readFile(new URL("../../docker-compose.synology.yml", import.meta.url), "utf8"); + const [postgresBlock, gatewayAndVolumes] = compose.split("\n agent-gateway:"); + const [gatewayBlock] = gatewayAndVolumes.split("\nvolumes:"); + + assert.match(postgresBlock, /\n restart: unless-stopped\n/); + assert.match(gatewayBlock, /\n restart: unless-stopped\n/); + assert.match(gatewayBlock, /NODEDC_ONTOLOGY_CORE_URL:/); + assert.match(gatewayBlock, /NODEDC_ONTOLOGY_CORE_ACCESS_TOKEN:/); +}); diff --git a/src/db/wait-for-database.ts b/src/db/wait-for-database.ts new file mode 100644 index 0000000..cb40621 --- /dev/null +++ b/src/db/wait-for-database.ts @@ -0,0 +1,40 @@ +export interface DatabaseProbe { + query(sql: string): Promise; +} + +export interface WaitForDatabaseOptions { + maxAttempts?: number; + retryDelayMs?: number; + sleep?: (delayMs: number) => Promise; +} + +export const DEFAULT_DATABASE_STARTUP_MAX_ATTEMPTS = 300; +export const DEFAULT_DATABASE_STARTUP_RETRY_DELAY_MS = 1_000; + +export async function waitForDatabase( + database: DatabaseProbe, + options: WaitForDatabaseOptions = {}, +): Promise { + const maxAttempts = options.maxAttempts ?? DEFAULT_DATABASE_STARTUP_MAX_ATTEMPTS; + const retryDelayMs = options.retryDelayMs ?? DEFAULT_DATABASE_STARTUP_RETRY_DELAY_MS; + const sleep = options.sleep ?? ((delayMs) => new Promise((resolve) => setTimeout(resolve, delayMs))); + + if (!Number.isInteger(maxAttempts) || maxAttempts < 1) { + throw new RangeError("maxAttempts must be a positive integer"); + } + if (!Number.isFinite(retryDelayMs) || retryDelayMs < 0) { + throw new RangeError("retryDelayMs must be a non-negative number"); + } + + for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { + try { + await database.query("SELECT 1"); + return; + } catch (error) { + if (attempt === maxAttempts) { + throw error; + } + await sleep(retryDelayMs); + } + } +} diff --git a/src/scripts/migrate.ts b/src/scripts/migrate.ts index 905edd5..d317156 100644 --- a/src/scripts/migrate.ts +++ b/src/scripts/migrate.ts @@ -2,6 +2,7 @@ import { Pool } from "pg"; import { loadConfig } from "../config.js"; import { runMigrations } from "../db/migrations.js"; +import { waitForDatabase } from "../db/wait-for-database.js"; const config = loadConfig(); @@ -22,20 +23,3 @@ try { } finally { await pool.end(); } - -async function waitForDatabase(pool: Pool): Promise { - const maxAttempts = 30; - - for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { - try { - await pool.query("SELECT 1"); - return; - } catch (error) { - if (attempt === maxAttempts) { - throw error; - } - - await new Promise((resolve) => setTimeout(resolve, 1000)); - } - } -}