fix: make Ops gateway resilient to Synology reboot
This commit is contained in:
DCCONSTRUCTIONS
@@ -1,6 +1,7 @@
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:17-alpine
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
POSTGRES_DB: ${POSTGRES_DB:-nodedc_agent_gateway}
|
||||
POSTGRES_USER: ${POSTGRES_USER:-nodedc_agent_gateway}
|
||||
@@ -17,6 +18,7 @@ services:
|
||||
build:
|
||||
context: .
|
||||
init: true
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
NODE_ENV: ${NODE_ENV:-production}
|
||||
HOST: 0.0.0.0
|
||||
@@ -31,6 +33,8 @@ services:
|
||||
NODEDC_LAUNCHER_INTERNAL_URL: ${NODEDC_LAUNCHER_INTERNAL_URL:-http://172.22.0.222:18080}
|
||||
NODEDC_TASKER_INTERNAL_URL: ${NODEDC_TASKER_INTERNAL_URL:-http://172.22.0.222:18090}
|
||||
NODEDC_ENGINE_INTERNAL_URL: ${NODEDC_ENGINE_INTERNAL_URL:-http://172.22.0.222:3001}
|
||||
NODEDC_ONTOLOGY_CORE_URL: ${NODEDC_ONTOLOGY_CORE_URL:-http://172.22.0.222:18104}
|
||||
NODEDC_ONTOLOGY_CORE_ACCESS_TOKEN: ${NODEDC_ONTOLOGY_CORE_ACCESS_TOKEN:-}
|
||||
NODEDC_INTERNAL_ACCESS_TOKEN: ${NODEDC_INTERNAL_ACCESS_TOKEN}
|
||||
depends_on:
|
||||
postgres:
|
||||
|
||||
@@ -0,0 +1,145 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { createHash } from "node:crypto";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { cp, lstat, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const scriptDir = dirname(fileURLToPath(import.meta.url));
|
||||
const sourceRoot = resolve(scriptDir, "..");
|
||||
const artifactRoot = resolve(
|
||||
process.env.NODEDC_DEPLOY_ARTIFACT_DIR || resolve(sourceRoot, "deploy-artifacts"),
|
||||
);
|
||||
const artifactId = "ops-agents-reboot-resilience-20260809-001";
|
||||
const component = "ops-agents";
|
||||
const files = [
|
||||
"docker-compose.synology.yml",
|
||||
"src/db/wait-for-database.test.ts",
|
||||
"src/db/wait-for-database.ts",
|
||||
"src/scripts/migrate.ts",
|
||||
];
|
||||
const predecessors = {
|
||||
"docker-compose.synology.yml": "e78d37368c3d255f257086773ba908e44aae2e475661b80ee17f98c3d55d178f",
|
||||
"src/scripts/migrate.ts": "8396a42deded31f1fe2381f772838a33b05bfd389075ebd34a5f546593f08769",
|
||||
};
|
||||
const newPaths = [
|
||||
"src/db/wait-for-database.test.ts",
|
||||
"src/db/wait-for-database.ts",
|
||||
];
|
||||
|
||||
await mkdir(artifactRoot, { recursive: true });
|
||||
const artifact = join(artifactRoot, `nodedc-${artifactId}.tgz`);
|
||||
const predecessorPath = join(artifactRoot, `nodedc-${artifactId}.predecessor.sha256`);
|
||||
const newPathsPath = join(artifactRoot, `nodedc-${artifactId}.new-paths`);
|
||||
await Promise.all([artifact, predecessorPath, newPathsPath].map(assertFresh));
|
||||
|
||||
const stage = await mkdtemp(join(tmpdir(), "nodedc-ops-agents-reboot-resilience-"));
|
||||
const payload = join(stage, "payload");
|
||||
|
||||
try {
|
||||
await mkdir(payload, { recursive: true });
|
||||
for (const relativePath of files) {
|
||||
validateRelativePath(relativePath);
|
||||
const source = join(sourceRoot, relativePath);
|
||||
const info = await lstat(source);
|
||||
if (!info.isFile() || info.isSymbolicLink()) {
|
||||
throw new Error(`invalid_source_file:${relativePath}`);
|
||||
}
|
||||
await mkdir(dirname(join(payload, relativePath)), { recursive: true });
|
||||
await cp(source, join(payload, relativePath), { force: false });
|
||||
}
|
||||
|
||||
await writeFile(
|
||||
join(stage, "manifest.env"),
|
||||
`id=${artifactId}\ncomponent=${component}\ntype=app-overlay\n`,
|
||||
"utf8",
|
||||
);
|
||||
await writeFile(join(stage, "files.txt"), `${files.join("\n")}\n`, "utf8");
|
||||
run("python3", ["-c", canonicalTarScript(), artifact, stage]);
|
||||
|
||||
const productionRoot = "/volume1/docker/nodedc-platform/ops-agents";
|
||||
await writeFile(
|
||||
predecessorPath,
|
||||
`${Object.entries(predecessors)
|
||||
.map(([relativePath, digest]) => `${digest} ${productionRoot}/${relativePath}`)
|
||||
.join("\n")}\n`,
|
||||
"utf8",
|
||||
);
|
||||
await writeFile(
|
||||
newPathsPath,
|
||||
`${newPaths.map((relativePath) => `${productionRoot}/${relativePath}`).join("\n")}\n`,
|
||||
"utf8",
|
||||
);
|
||||
|
||||
const artifactBytes = await readFile(artifact);
|
||||
console.log(JSON.stringify({
|
||||
ok: true,
|
||||
artifactId,
|
||||
component,
|
||||
artifact,
|
||||
artifactSha256: sha256(artifactBytes),
|
||||
files,
|
||||
predecessors,
|
||||
newPaths,
|
||||
expectedService: "agent-gateway",
|
||||
expectedHealthcheck: "http://172.22.0.222:18190/readyz",
|
||||
runnerChanged: false,
|
||||
databaseVolumeSelected: false,
|
||||
}, null, 2));
|
||||
} finally {
|
||||
await rm(stage, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
async function assertFresh(path) {
|
||||
try {
|
||||
await lstat(path);
|
||||
} catch (error) {
|
||||
if (error?.code === "ENOENT") return;
|
||||
throw error;
|
||||
}
|
||||
throw new Error(`output_already_exists:${path}`);
|
||||
}
|
||||
|
||||
function validateRelativePath(value) {
|
||||
if (
|
||||
typeof value !== "string"
|
||||
|| !value
|
||||
|| value.startsWith("/")
|
||||
|| value.split("/").some((part) => !part || part === "." || part === "..")
|
||||
) {
|
||||
throw new Error(`unsafe_relative_path:${value}`);
|
||||
}
|
||||
}
|
||||
|
||||
function canonicalTarScript() {
|
||||
return [
|
||||
"import gzip,io,pathlib,sys,tarfile",
|
||||
"root=pathlib.Path(sys.argv[2])",
|
||||
"with open(sys.argv[1],'wb') as out:",
|
||||
" with gzip.GzipFile(filename='',mode='wb',fileobj=out,compresslevel=9,mtime=0) as gz:",
|
||||
" with tarfile.open(fileobj=gz,mode='w',format=tarfile.PAX_FORMAT) as tar:",
|
||||
" for top in ('manifest.env','files.txt','payload'):",
|
||||
" p=root/top; paths=[p]+(sorted(p.rglob('*')) if p.is_dir() else [])",
|
||||
" for x in paths:",
|
||||
" info=tar.gettarinfo(str(x),arcname=x.relative_to(root).as_posix())",
|
||||
" info.uid=info.gid=0; info.uname=info.gname='root'; info.mtime=0; info.mode=0o755 if info.isdir() else 0o644",
|
||||
" with (open(x,'rb') if info.isfile() else io.BytesIO()) as src: tar.addfile(info,src if info.isfile() else None)",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function run(command, args) {
|
||||
const result = spawnSync(command, args, {
|
||||
encoding: "utf8",
|
||||
maxBuffer: 128 * 1024 * 1024,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
if (result.status !== 0) {
|
||||
throw new Error(`${command}_failed:${result.stderr || result.stdout}`);
|
||||
}
|
||||
}
|
||||
|
||||
function sha256(bytes) {
|
||||
return createHash("sha256").update(bytes).digest("hex");
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { createHash } from "node:crypto";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { mkdtemp, readFile, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join } from "node:path";
|
||||
import test from "node:test";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const scriptDir = dirname(fileURLToPath(import.meta.url));
|
||||
const builder = join(scriptDir, "build-ops-agents-reboot-resilience-artifact.mjs");
|
||||
const basename = "nodedc-ops-agents-reboot-resilience-20260809-001";
|
||||
|
||||
test("builds a deterministic, minimal Ops Agents application overlay", async () => {
|
||||
const firstRoot = await mkdtemp(join(tmpdir(), "ops-agents-artifact-first-"));
|
||||
const secondRoot = await mkdtemp(join(tmpdir(), "ops-agents-artifact-second-"));
|
||||
|
||||
try {
|
||||
const first = runBuilder(firstRoot);
|
||||
const second = runBuilder(secondRoot);
|
||||
const firstArtifact = join(firstRoot, `${basename}.tgz`);
|
||||
const secondArtifact = join(secondRoot, `${basename}.tgz`);
|
||||
|
||||
assert.equal(first.artifactSha256, second.artifactSha256);
|
||||
assert.equal(first.artifactSha256, sha256(await readFile(firstArtifact)));
|
||||
assert.deepEqual(first.files, [
|
||||
"docker-compose.synology.yml",
|
||||
"src/db/wait-for-database.test.ts",
|
||||
"src/db/wait-for-database.ts",
|
||||
"src/scripts/migrate.ts",
|
||||
]);
|
||||
assert.equal(first.runnerChanged, false);
|
||||
assert.equal(first.databaseVolumeSelected, false);
|
||||
|
||||
const members = new Set(runTar(["-tzf", firstArtifact]).trim().split("\n"));
|
||||
assert.deepEqual(members, new Set([
|
||||
"manifest.env",
|
||||
"files.txt",
|
||||
"payload/",
|
||||
"payload/docker-compose.synology.yml",
|
||||
"payload/src/",
|
||||
"payload/src/db/",
|
||||
"payload/src/db/wait-for-database.test.ts",
|
||||
"payload/src/db/wait-for-database.ts",
|
||||
"payload/src/scripts/",
|
||||
"payload/src/scripts/migrate.ts",
|
||||
]));
|
||||
assert.equal([...members].some((member) => member.includes("._")), false);
|
||||
|
||||
assert.equal(
|
||||
runTar(["-xOzf", firstArtifact, "manifest.env"]),
|
||||
"id=ops-agents-reboot-resilience-20260809-001\ncomponent=ops-agents\ntype=app-overlay\n",
|
||||
);
|
||||
assert.equal(
|
||||
runTar(["-xOzf", firstArtifact, "files.txt"]),
|
||||
`${first.files.join("\n")}\n`,
|
||||
);
|
||||
|
||||
const predecessors = await readFile(join(firstRoot, `${basename}.predecessor.sha256`), "utf8");
|
||||
assert.match(predecessors, /^e78d37368c3d255f257086773ba908e44aae2e475661b80ee17f98c3d55d178f .*docker-compose\.synology\.yml$/m);
|
||||
assert.match(predecessors, /^8396a42deded31f1fe2381f772838a33b05bfd389075ebd34a5f546593f08769 .*src\/scripts\/migrate\.ts$/m);
|
||||
|
||||
const newPaths = await readFile(join(firstRoot, `${basename}.new-paths`), "utf8");
|
||||
assert.match(newPaths, /src\/db\/wait-for-database\.ts/);
|
||||
assert.match(newPaths, /src\/db\/wait-for-database\.test\.ts/);
|
||||
} finally {
|
||||
await rm(firstRoot, { recursive: true, force: true });
|
||||
await rm(secondRoot, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
function runBuilder(outputRoot) {
|
||||
const result = spawnSync(process.execPath, [builder], {
|
||||
encoding: "utf8",
|
||||
env: { ...process.env, NODEDC_DEPLOY_ARTIFACT_DIR: outputRoot },
|
||||
});
|
||||
assert.equal(result.status, 0, result.stderr || result.stdout);
|
||||
return JSON.parse(result.stdout);
|
||||
}
|
||||
|
||||
function runTar(args) {
|
||||
const result = spawnSync("tar", args, { encoding: "utf8" });
|
||||
assert.equal(result.status, 0, result.stderr || result.stdout);
|
||||
return result.stdout;
|
||||
}
|
||||
|
||||
function sha256(bytes) {
|
||||
return createHash("sha256").update(bytes).digest("hex");
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import test from "node:test";
|
||||
|
||||
import {
|
||||
DEFAULT_DATABASE_STARTUP_MAX_ATTEMPTS,
|
||||
waitForDatabase,
|
||||
} from "./wait-for-database.js";
|
||||
|
||||
test("waits through a DNS outage longer than the legacy 30-second window", async () => {
|
||||
let attempts = 0;
|
||||
let sleeps = 0;
|
||||
|
||||
await waitForDatabase(
|
||||
{
|
||||
async query() {
|
||||
attempts += 1;
|
||||
if (attempts <= 35) {
|
||||
const error = new Error("getaddrinfo ENOTFOUND postgres");
|
||||
Object.assign(error, { code: "ENOTFOUND", hostname: "postgres" });
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
sleep: async () => {
|
||||
sleeps += 1;
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
assert.equal(DEFAULT_DATABASE_STARTUP_MAX_ATTEMPTS, 300);
|
||||
assert.equal(attempts, 36);
|
||||
assert.equal(sleeps, 35);
|
||||
});
|
||||
|
||||
test("keeps the startup wait bounded and rethrows the last database error", async () => {
|
||||
const terminalError = new Error("database unavailable");
|
||||
let attempts = 0;
|
||||
let sleeps = 0;
|
||||
|
||||
await assert.rejects(
|
||||
waitForDatabase(
|
||||
{
|
||||
async query() {
|
||||
attempts += 1;
|
||||
throw terminalError;
|
||||
},
|
||||
},
|
||||
{
|
||||
maxAttempts: 3,
|
||||
retryDelayMs: 0,
|
||||
sleep: async () => {
|
||||
sleeps += 1;
|
||||
},
|
||||
},
|
||||
),
|
||||
terminalError,
|
||||
);
|
||||
|
||||
assert.equal(attempts, 3);
|
||||
assert.equal(sleeps, 2);
|
||||
});
|
||||
|
||||
test("Synology compose restarts both gateway services and preserves live Ontology wiring", async () => {
|
||||
const compose = await readFile(new URL("../../docker-compose.synology.yml", import.meta.url), "utf8");
|
||||
const [postgresBlock, gatewayAndVolumes] = compose.split("\n agent-gateway:");
|
||||
const [gatewayBlock] = gatewayAndVolumes.split("\nvolumes:");
|
||||
|
||||
assert.match(postgresBlock, /\n restart: unless-stopped\n/);
|
||||
assert.match(gatewayBlock, /\n restart: unless-stopped\n/);
|
||||
assert.match(gatewayBlock, /NODEDC_ONTOLOGY_CORE_URL:/);
|
||||
assert.match(gatewayBlock, /NODEDC_ONTOLOGY_CORE_ACCESS_TOKEN:/);
|
||||
});
|
||||
@@ -0,0 +1,40 @@
|
||||
export interface DatabaseProbe {
|
||||
query(sql: string): Promise<unknown>;
|
||||
}
|
||||
|
||||
export interface WaitForDatabaseOptions {
|
||||
maxAttempts?: number;
|
||||
retryDelayMs?: number;
|
||||
sleep?: (delayMs: number) => Promise<void>;
|
||||
}
|
||||
|
||||
export const DEFAULT_DATABASE_STARTUP_MAX_ATTEMPTS = 300;
|
||||
export const DEFAULT_DATABASE_STARTUP_RETRY_DELAY_MS = 1_000;
|
||||
|
||||
export async function waitForDatabase(
|
||||
database: DatabaseProbe,
|
||||
options: WaitForDatabaseOptions = {},
|
||||
): Promise<void> {
|
||||
const maxAttempts = options.maxAttempts ?? DEFAULT_DATABASE_STARTUP_MAX_ATTEMPTS;
|
||||
const retryDelayMs = options.retryDelayMs ?? DEFAULT_DATABASE_STARTUP_RETRY_DELAY_MS;
|
||||
const sleep = options.sleep ?? ((delayMs) => new Promise((resolve) => setTimeout(resolve, delayMs)));
|
||||
|
||||
if (!Number.isInteger(maxAttempts) || maxAttempts < 1) {
|
||||
throw new RangeError("maxAttempts must be a positive integer");
|
||||
}
|
||||
if (!Number.isFinite(retryDelayMs) || retryDelayMs < 0) {
|
||||
throw new RangeError("retryDelayMs must be a non-negative number");
|
||||
}
|
||||
|
||||
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
|
||||
try {
|
||||
await database.query("SELECT 1");
|
||||
return;
|
||||
} catch (error) {
|
||||
if (attempt === maxAttempts) {
|
||||
throw error;
|
||||
}
|
||||
await sleep(retryDelayMs);
|
||||
}
|
||||
}
|
||||
}
|
||||
+1
-17
@@ -2,6 +2,7 @@ import { Pool } from "pg";
|
||||
|
||||
import { loadConfig } from "../config.js";
|
||||
import { runMigrations } from "../db/migrations.js";
|
||||
import { waitForDatabase } from "../db/wait-for-database.js";
|
||||
|
||||
const config = loadConfig();
|
||||
|
||||
@@ -22,20 +23,3 @@ try {
|
||||
} finally {
|
||||
await pool.end();
|
||||
}
|
||||
|
||||
async function waitForDatabase(pool: Pool): Promise<void> {
|
||||
const maxAttempts = 30;
|
||||
|
||||
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
|
||||
try {
|
||||
await pool.query("SELECT 1");
|
||||
return;
|
||||
} catch (error) {
|
||||
if (attempt === maxAttempts) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, 1000));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user