Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
49813d7839 |
@@ -435,11 +435,6 @@ The check is source-aware: if an apply rolls back to the prior source, rollback
|
||||
acceptance uses that prior health contract instead of falsely requiring a
|
||||
feature which the restored generation does not contain.
|
||||
|
||||
Module Foundry activation and rollback wait for the Compose container health
|
||||
barrier before strict image/container inventory acceptance. The HTTP health
|
||||
endpoint may become ready while Docker still reports `starting`; that timing
|
||||
window is not treated as an application or rollback failure.
|
||||
|
||||
The Foundry ↔ Map Gateway signing key is not an application or `.env` setting.
|
||||
On the first relevant `platform` or `module-foundry` apply, the root-owned
|
||||
runner creates `/volume1/docker/nodedc-platform/secrets/map-gateway-admin-secret`
|
||||
|
||||
@@ -1,304 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { createHash } from "node:crypto";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import {
|
||||
cp,
|
||||
lstat,
|
||||
mkdir,
|
||||
mkdtemp,
|
||||
readdir,
|
||||
readFile,
|
||||
rm,
|
||||
writeFile,
|
||||
} from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join, relative, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const scriptDir = dirname(fileURLToPath(import.meta.url));
|
||||
const declaredFoundryRoot = process.env.NODEDC_FOUNDRY_SOURCE_ROOT;
|
||||
if (!declaredFoundryRoot) {
|
||||
throw new Error("NODEDC_FOUNDRY_SOURCE_ROOT_is_required");
|
||||
}
|
||||
const foundryRoot = resolve(declaredFoundryRoot);
|
||||
const expectedSourceCommit = "58800d957632320fa6717ec3fcff972023528759";
|
||||
const artifactDir = resolve(
|
||||
process.env.NODEDC_DEPLOY_ARTIFACT_DIR
|
||||
|| resolve(scriptDir, "../deploy-artifacts"),
|
||||
);
|
||||
const [
|
||||
patchId = "module-foundry-map-runtime-recovery-20260809-003",
|
||||
...extra
|
||||
] = process.argv.slice(2);
|
||||
|
||||
if (
|
||||
extra.length
|
||||
|| !/^module-foundry-map-runtime-recovery-\d{8}-\d{3}$/.test(patchId)
|
||||
) {
|
||||
throw new Error(
|
||||
"usage: build-module-foundry-map-runtime-recovery-artifact.mjs "
|
||||
+ "[module-foundry-map-runtime-recovery-YYYYMMDD-NNN]",
|
||||
);
|
||||
}
|
||||
|
||||
const files = Object.freeze([
|
||||
"package.json",
|
||||
"package-lock.json",
|
||||
"apps/catalog/src",
|
||||
"packages",
|
||||
"registry",
|
||||
"runtime-seed/page-layouts/map.json",
|
||||
"scripts",
|
||||
"server/catalog-server.mjs",
|
||||
"server/foundry-mcp.mjs",
|
||||
"server/map-grid-persistence.test.mjs",
|
||||
]);
|
||||
const ignoredBasenames = new Set([
|
||||
".DS_Store",
|
||||
".git",
|
||||
"node_modules",
|
||||
"runtime-data",
|
||||
"dist",
|
||||
]);
|
||||
const artifact = join(artifactDir, `nodedc-${patchId}.tgz`);
|
||||
const checksum = `${artifact}.sha256`;
|
||||
const stage = await mkdtemp(
|
||||
join(tmpdir(), "nodedc-foundry-map-runtime-recovery-"),
|
||||
);
|
||||
|
||||
await assertFresh(artifact);
|
||||
await assertExactSource();
|
||||
await assertRecoveryBoundary();
|
||||
|
||||
try {
|
||||
for (const sourceRelative of files) {
|
||||
await copySafe(
|
||||
resolve(foundryRoot, sourceRelative),
|
||||
join(stage, "payload", sourceRelative),
|
||||
);
|
||||
}
|
||||
await writeFile(
|
||||
join(stage, "manifest.env"),
|
||||
`id=${patchId}\ncomponent=module-foundry\ntype=app-overlay\n`,
|
||||
"utf8",
|
||||
);
|
||||
await writeFile(join(stage, "files.txt"), `${files.join("\n")}\n`, "utf8");
|
||||
await mkdir(artifactDir, { recursive: true });
|
||||
run("python3", ["-c", canonicalTarScript(), artifact, stage]);
|
||||
const sha256 = digest(await readFile(artifact));
|
||||
await writeFile(
|
||||
checksum,
|
||||
`${sha256} ${artifact.split("/").at(-1)}\n`,
|
||||
"utf8",
|
||||
);
|
||||
console.log(JSON.stringify({
|
||||
ok: true,
|
||||
patchId,
|
||||
artifact,
|
||||
checksum,
|
||||
sha256,
|
||||
sourceCommit: expectedSourceCommit,
|
||||
services: ["nodedc-module-foundry"],
|
||||
transition: "reconcile-partial-map-source-and-activate-runtime",
|
||||
files,
|
||||
}, null, 2));
|
||||
} finally {
|
||||
await rm(stage, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
async function assertExactSource() {
|
||||
const declared = process.env.NODEDC_FOUNDRY_SOURCE_COMMIT;
|
||||
if (declared !== expectedSourceCommit) {
|
||||
throw new Error("foundry_source_commit_mismatch");
|
||||
}
|
||||
const dotGit = join(foundryRoot, ".git");
|
||||
try {
|
||||
await lstat(dotGit);
|
||||
} catch (error) {
|
||||
if (error?.code === "ENOENT") return;
|
||||
throw error;
|
||||
}
|
||||
const tracked = spawnSync(
|
||||
"git",
|
||||
["-C", foundryRoot, "diff", "--quiet", "HEAD", "--", ...files],
|
||||
{ encoding: "utf8" },
|
||||
);
|
||||
if (tracked.status !== 0) {
|
||||
throw new Error("foundry_selected_source_is_dirty");
|
||||
}
|
||||
const untracked = git(foundryRoot, [
|
||||
"ls-files",
|
||||
"--others",
|
||||
"--exclude-standard",
|
||||
"--",
|
||||
...files,
|
||||
]);
|
||||
if (untracked) {
|
||||
throw new Error("foundry_selected_source_has_untracked_files");
|
||||
}
|
||||
}
|
||||
|
||||
async function assertRecoveryBoundary() {
|
||||
const packageJson = await readFile(join(foundryRoot, "package.json"), "utf8");
|
||||
const packageLock = await readFile(
|
||||
join(foundryRoot, "package-lock.json"),
|
||||
"utf8",
|
||||
);
|
||||
const preview = await readFile(
|
||||
join(foundryRoot, "apps/catalog/src/MapFixturePreview.tsx"),
|
||||
"utf8",
|
||||
);
|
||||
const renderer = await readFile(
|
||||
join(foundryRoot, "apps/catalog/src/CesiumMapRenderer.tsx"),
|
||||
"utf8",
|
||||
);
|
||||
const policy = await readFile(
|
||||
join(foundryRoot, "apps/catalog/src/mapGridPolicy.mjs"),
|
||||
"utf8",
|
||||
);
|
||||
const sector = await readFile(
|
||||
join(foundryRoot, "apps/catalog/src/mapSectorGrid.mjs"),
|
||||
"utf8",
|
||||
);
|
||||
const server = await readFile(
|
||||
join(foundryRoot, "server/catalog-server.mjs"),
|
||||
"utf8",
|
||||
);
|
||||
const registry = await readFile(
|
||||
join(foundryRoot, "registry/registry.json"),
|
||||
"utf8",
|
||||
);
|
||||
|
||||
for (const marker of [
|
||||
"@nodedc/map-cesium-react",
|
||||
"npm run build --workspace @nodedc/map-cesium-react",
|
||||
]) {
|
||||
if (!packageJson.includes(marker)) {
|
||||
throw new Error(`map_package_boundary_missing:${marker}`);
|
||||
}
|
||||
}
|
||||
if (!packageLock.includes('"packages/map-cesium-react"')) {
|
||||
throw new Error("map_package_lock_boundary_missing");
|
||||
}
|
||||
for (const marker of [
|
||||
"const sectorSpatialEntities = useMemo",
|
||||
"runtimeBindings={[...sectorScopedPrimaryRuntimeBindings, ...referenceRuntimeBindings]}",
|
||||
'label="Скрыть объекты за сектором"',
|
||||
]) {
|
||||
if (!preview.includes(marker)) {
|
||||
throw new Error(`map_workspace_boundary_missing:${marker}`);
|
||||
}
|
||||
}
|
||||
for (const marker of [
|
||||
"class GridLayerController",
|
||||
"gridLegacyMode",
|
||||
"viewer.flyTo(entity",
|
||||
]) {
|
||||
if (!renderer.includes(marker)) {
|
||||
throw new Error(`map_renderer_boundary_missing:${marker}`);
|
||||
}
|
||||
}
|
||||
for (const marker of [
|
||||
"gridLodProfiles",
|
||||
"GRID_LOD_HYSTERESIS_RATIO",
|
||||
]) {
|
||||
if (!policy.includes(marker)) {
|
||||
throw new Error(`map_policy_boundary_missing:${marker}`);
|
||||
}
|
||||
}
|
||||
for (const marker of [
|
||||
"export function geodeticToLocalGridPlane",
|
||||
"export function localSectorAtGeodetic",
|
||||
]) {
|
||||
if (!sector.includes(marker)) {
|
||||
throw new Error(`map_sector_boundary_missing:${marker}`);
|
||||
}
|
||||
}
|
||||
for (const marker of [
|
||||
"gridLodProfiles",
|
||||
"validateGridLodProfiles",
|
||||
]) {
|
||||
if (!server.includes(marker)) {
|
||||
throw new Error(`map_server_boundary_missing:${marker}`);
|
||||
}
|
||||
}
|
||||
if (!registry.includes('"@nodedc/map-cesium-react"')) {
|
||||
throw new Error("map_registry_boundary_missing");
|
||||
}
|
||||
}
|
||||
|
||||
async function copySafe(source, destination) {
|
||||
const sourceStat = await lstat(source);
|
||||
if (sourceStat.isSymbolicLink()) {
|
||||
throw new Error(
|
||||
`source_symlink_rejected:${relative(foundryRoot, source)}`,
|
||||
);
|
||||
}
|
||||
if (sourceStat.isFile()) {
|
||||
await mkdir(dirname(destination), { recursive: true });
|
||||
await cp(source, destination, { force: true, verbatimSymlinks: true });
|
||||
return;
|
||||
}
|
||||
if (!sourceStat.isDirectory()) {
|
||||
throw new Error(`source_type_rejected:${relative(foundryRoot, source)}`);
|
||||
}
|
||||
await mkdir(destination, { recursive: true });
|
||||
for (const entry of await readdir(source, { withFileTypes: true })) {
|
||||
if (
|
||||
ignoredBasenames.has(entry.name)
|
||||
|| entry.name.startsWith(".env")
|
||||
|| entry.name.endsWith(".tsbuildinfo")
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
const childSource = join(source, entry.name);
|
||||
const childDestination = join(destination, entry.name);
|
||||
if (entry.isSymbolicLink()) {
|
||||
throw new Error(
|
||||
`source_symlink_rejected:${relative(foundryRoot, childSource)}`,
|
||||
);
|
||||
}
|
||||
await copySafe(childSource, childDestination);
|
||||
}
|
||||
}
|
||||
|
||||
async function assertFresh(path) {
|
||||
try {
|
||||
await lstat(path);
|
||||
} catch (error) {
|
||||
if (error?.code === "ENOENT") return;
|
||||
throw error;
|
||||
}
|
||||
throw new Error("artifact_already_exists");
|
||||
}
|
||||
|
||||
function canonicalTarScript() {
|
||||
return [
|
||||
"import gzip,io,pathlib,sys,tarfile",
|
||||
"root=pathlib.Path(sys.argv[2])",
|
||||
"with open(sys.argv[1],'wb') as out:",
|
||||
" with gzip.GzipFile(filename='',mode='wb',fileobj=out,compresslevel=9,mtime=0) as gz:",
|
||||
" with tarfile.open(fileobj=gz,mode='w',format=tarfile.PAX_FORMAT) as tar:",
|
||||
" for top in ('manifest.env','files.txt','payload'):",
|
||||
" p=root/top; paths=[p]+(sorted(p.rglob('*')) if p.is_dir() else [])",
|
||||
" for x in paths:",
|
||||
" info=tar.gettarinfo(str(x),arcname=x.relative_to(root).as_posix())",
|
||||
" info.uid=info.gid=0; info.uname=info.gname='root'; info.mtime=0; info.mode=0o755 if info.isdir() else 0o644",
|
||||
" with (open(x,'rb') if info.isfile() else io.BytesIO()) as src: tar.addfile(info,src if info.isfile() else None)",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function digest(value) {
|
||||
return createHash("sha256").update(value).digest("hex");
|
||||
}
|
||||
|
||||
function run(command, args) {
|
||||
const result = spawnSync(command, args, {
|
||||
encoding: "utf8",
|
||||
maxBuffer: 128 * 1024 * 1024,
|
||||
});
|
||||
if (result.status !== 0) {
|
||||
throw new Error(`${command}_failed:${result.stderr || result.stdout}`);
|
||||
}
|
||||
}
|
||||
@@ -35,80 +35,6 @@ STATE_FILE = STATE_DIR / "applied.jsonl"
|
||||
FAILED_STATE_FILE = STATE_DIR / "failed.jsonl"
|
||||
LOCK_DIR = STATE_DIR / "deploy.lock"
|
||||
DOCKER = Path("/usr/local/bin/docker")
|
||||
MODULE_FOUNDRY_SERVICE = "nodedc-module-foundry"
|
||||
MODULE_FOUNDRY_RUNTIME_BEFORE_FILE = "module-foundry-runtime-before.json"
|
||||
MODULE_FOUNDRY_BUILD_ATTEMPTS = 2
|
||||
MODULE_FOUNDRY_MAP_RECOVERY_PATCH_ID = (
|
||||
"module-foundry-map-runtime-recovery-20260809-003"
|
||||
)
|
||||
MODULE_FOUNDRY_MAP_RECOVERY_ENTRIES = (
|
||||
"package.json",
|
||||
"package-lock.json",
|
||||
"apps/catalog/src",
|
||||
"packages",
|
||||
"registry",
|
||||
"runtime-seed/page-layouts/map.json",
|
||||
"scripts",
|
||||
"server/catalog-server.mjs",
|
||||
"server/foundry-mcp.mjs",
|
||||
"server/map-grid-persistence.test.mjs",
|
||||
)
|
||||
MODULE_FOUNDRY_MAP_RECOVERY_MISSING_PREDECESSOR_ENTRIES = (
|
||||
"server/map-grid-persistence.test.mjs",
|
||||
)
|
||||
MODULE_FOUNDRY_MAP_FAILED_INSTALLED_ENTRIES = (
|
||||
"apps/catalog/src/MapFixturePreview.tsx",
|
||||
"apps/catalog/src/mapPresentationProfile.ts",
|
||||
"apps/catalog/src/mapSectorGrid.d.mts",
|
||||
"apps/catalog/src/mapSectorGrid.mjs",
|
||||
"apps/catalog/src/styles.css",
|
||||
"scripts/map-object-layers.test.mjs",
|
||||
"scripts/map-presentation-filters.test.mjs",
|
||||
"scripts/map-sector-grid.test.mjs",
|
||||
)
|
||||
MODULE_FOUNDRY_MAP_RECOVERY_INSTALLED_TREE_SHA256 = (
|
||||
"8513127fb3b875dc3326c0305441723cda6c405525a68cc11030a6f645738d2b"
|
||||
)
|
||||
MODULE_FOUNDRY_MAP_RECOVERY_CANDIDATE_TREE_SHA256 = (
|
||||
"b89392e56fbaf6a8450ab67fcf263199eadba4a74ef7b70b4d8d685a57c2edc3"
|
||||
)
|
||||
MODULE_FOUNDRY_MAP_FAILED_ENTRIES = MODULE_FOUNDRY_MAP_RECOVERY_ENTRIES
|
||||
MODULE_FOUNDRY_MAP_FAILED_PATCH_ID = (
|
||||
"module-foundry-map-runtime-recovery-20260809-002"
|
||||
)
|
||||
MODULE_FOUNDRY_MAP_FAILED_ARTIFACT = (
|
||||
"nodedc-module-foundry-map-runtime-recovery-20260809-002.tgz."
|
||||
"20260809-092439"
|
||||
)
|
||||
MODULE_FOUNDRY_MAP_FAILED_ARTIFACT_SHA256 = (
|
||||
"bcaffc5dc6098a6c16f8f9362e6e6a89f8eb2745b898bf1964db8ba1f921f603"
|
||||
)
|
||||
MODULE_FOUNDRY_MAP_FAILED_BACKUP_ID = (
|
||||
"module-foundry-module-foundry-map-runtime-recovery-20260809-002-"
|
||||
"20260809-092439"
|
||||
)
|
||||
MODULE_FOUNDRY_MAP_FAILED_MESSAGE = "Module Foundry runtime inventory mismatch"
|
||||
MODULE_FOUNDRY_MAP_FAILED_ROLLBACK_STATUS = "failed:DeployError"
|
||||
MODULE_FOUNDRY_MAP_FAILED_BACKUP_SHA256 = {
|
||||
"existing-files.txt": (
|
||||
"eae18c8687a8b163f5b7bcf362c44c0be53a107c08f7c8799ee4dc6af9fb4e16"
|
||||
),
|
||||
"files.txt": (
|
||||
"cde91865f65a8d79fee66b3872a8067caa0c988a4a61f75e3dac2904ff814b10"
|
||||
),
|
||||
"manifest.env": (
|
||||
"cefce93cb613d771a4814af100f9cda7c2c020143268913905a233324edd96d9"
|
||||
),
|
||||
"missing-files.txt": (
|
||||
"bec2dcad43b34b2f040ad08414117e0654238a1bc051b37257f1b52f288ce689"
|
||||
),
|
||||
"module-foundry-runtime-before.json": (
|
||||
"bf2ec7678af00349c15d6b8dd9d4b0a43e16f427ed92c5adc842c59fab07e4fe"
|
||||
),
|
||||
"source-before.tgz": (
|
||||
"4d34654d32fcd8b328bf83d0f9f5f121fed094fe7bc2e9b1e4b322ccd77da8aa"
|
||||
),
|
||||
}
|
||||
MAP_GATEWAY_SECRET_DIR = Path("/volume1/docker/nodedc-platform/secrets")
|
||||
MAP_GATEWAY_SECRET_FILE = MAP_GATEWAY_SECRET_DIR / "map-gateway-admin-secret"
|
||||
MAP_EGRESS_PROXY_SECRET_FILE = MAP_GATEWAY_SECRET_DIR / "map-egress-proxy-token"
|
||||
@@ -15035,19 +14961,11 @@ def plan_artifact(artifact):
|
||||
provider_catalog_preflight = None
|
||||
device_plane_postgres_preflight = None
|
||||
device_plane_foundation_recovery_preflight = None
|
||||
module_foundry_map_recovery_preflight = None
|
||||
with tempfile.TemporaryDirectory(prefix="plan-", dir=TMP_DIR) as tmp:
|
||||
manifest, entries, payload_dir = load_artifact(artifact, Path(tmp))
|
||||
reject_terminal_engine_l2_failed_artifact(manifest, sha)
|
||||
reject_terminal_device_plane_foundation_artifact(manifest, sha)
|
||||
reject_terminal_device_plane_backhaul_artifact(manifest, sha)
|
||||
module_foundry_map_recovery_preflight = (
|
||||
validate_module_foundry_map_recovery_evidence(
|
||||
manifest,
|
||||
entries,
|
||||
payload_dir,
|
||||
)
|
||||
)
|
||||
transition_descriptor = None
|
||||
transition_preflight = None
|
||||
if is_engine_n8n_transition(manifest["component"], entries):
|
||||
@@ -16602,45 +16520,6 @@ def plan_artifact(artifact):
|
||||
print(f"runtime_grants=runner-managed:{FOUNDRY_BINDING_GRANTS_DIR}")
|
||||
print(f"runtime_private_key=runner-managed:{FOUNDRY_EDP_MANAGED_PROVISIONER_PRIVATE_KEY_FILE}")
|
||||
print(f"runtime_public_trust=runner-managed:{FOUNDRY_EDP_MANAGED_PROVISIONER_PUBLIC_KEY_FILE}")
|
||||
print(
|
||||
"module_foundry_build="
|
||||
f"separate-before-recreate:attempts={MODULE_FOUNDRY_BUILD_ATTEMPTS}"
|
||||
)
|
||||
print("module_foundry_runtime_pull=never")
|
||||
print("module_foundry_rollback=source+image+runtime")
|
||||
if module_foundry_map_recovery_preflight is not None:
|
||||
print(
|
||||
"module_foundry_transition="
|
||||
f"{module_foundry_map_recovery_preflight['mode']}"
|
||||
)
|
||||
print(
|
||||
"module_foundry_failed_predecessor="
|
||||
f"{module_foundry_map_recovery_preflight['failed_patch_id']}"
|
||||
)
|
||||
print(
|
||||
"module_foundry_failed_predecessor_sha256="
|
||||
f"{module_foundry_map_recovery_preflight['failed_artifact_sha256']}"
|
||||
)
|
||||
print(
|
||||
"module_foundry_failed_backup="
|
||||
f"{module_foundry_map_recovery_preflight['failed_backup_id']}"
|
||||
)
|
||||
print(
|
||||
"module_foundry_installed_tree_sha256="
|
||||
f"{module_foundry_map_recovery_preflight['installed_tree_sha256']}"
|
||||
)
|
||||
print(
|
||||
"module_foundry_candidate_tree_sha256="
|
||||
f"{module_foundry_map_recovery_preflight['candidate_tree_sha256']}"
|
||||
)
|
||||
print(
|
||||
"module_foundry_predecessor_container="
|
||||
f"{module_foundry_map_recovery_preflight['runtime_container_id']}"
|
||||
)
|
||||
print(
|
||||
"module_foundry_predecessor_image="
|
||||
f"{module_foundry_map_recovery_preflight['runtime_image_id']}"
|
||||
)
|
||||
if component == "device-plane":
|
||||
print(f"runtime_secret=runner-managed:{DEVICE_PLANE_POSTGRES_PASSWORD_FILE}")
|
||||
print(f"runtime_secret=runner-managed:{DEVICE_PLANE_GATEWAY_CORE_TOKEN_FILE}")
|
||||
@@ -17362,303 +17241,6 @@ def rollback_engine_apply(root, backup_dir, entries, current_stamp, runtime_star
|
||||
return f"source+runtime-restored:{restored_count}"
|
||||
|
||||
|
||||
def is_module_foundry_map_recovery_slice(manifest, entries):
|
||||
return (
|
||||
manifest.get("id") == MODULE_FOUNDRY_MAP_RECOVERY_PATCH_ID
|
||||
and manifest.get("component") == "module-foundry"
|
||||
and manifest.get("type") == "app-overlay"
|
||||
and tuple(entries or ()) == MODULE_FOUNDRY_MAP_RECOVERY_ENTRIES
|
||||
)
|
||||
|
||||
|
||||
def exact_file_map_sha256(files):
|
||||
return hashlib.sha256(
|
||||
json.dumps(
|
||||
files,
|
||||
ensure_ascii=True,
|
||||
separators=(",", ":"),
|
||||
sort_keys=True,
|
||||
).encode("utf-8")
|
||||
).hexdigest()
|
||||
|
||||
|
||||
def validate_module_foundry_map_recovery_evidence(
|
||||
manifest,
|
||||
entries,
|
||||
payload_dir,
|
||||
):
|
||||
if not is_module_foundry_map_recovery_slice(manifest, entries):
|
||||
return None
|
||||
|
||||
backup_dir = BACKUPS_DIR / MODULE_FOUNDRY_MAP_FAILED_BACKUP_ID
|
||||
try:
|
||||
backup_stat = backup_dir.lstat()
|
||||
except FileNotFoundError:
|
||||
die("Module Foundry map recovery backup is missing")
|
||||
if stat.S_ISLNK(backup_stat.st_mode) or not stat.S_ISDIR(
|
||||
backup_stat.st_mode
|
||||
):
|
||||
die("Module Foundry map recovery backup is unsafe")
|
||||
if {child.name for child in backup_dir.iterdir()} != set(
|
||||
MODULE_FOUNDRY_MAP_FAILED_BACKUP_SHA256
|
||||
):
|
||||
die("Module Foundry map recovery backup file set mismatch")
|
||||
for name, expected_sha256 in (
|
||||
MODULE_FOUNDRY_MAP_FAILED_BACKUP_SHA256.items()
|
||||
):
|
||||
path = backup_dir / name
|
||||
path_stat = path.lstat()
|
||||
if (
|
||||
stat.S_ISLNK(path_stat.st_mode)
|
||||
or not stat.S_ISREG(path_stat.st_mode)
|
||||
or sha256_file(path) != expected_sha256
|
||||
):
|
||||
die(
|
||||
"Module Foundry map recovery backup drift detected: "
|
||||
f"{name}"
|
||||
)
|
||||
|
||||
failed_artifact = FAILED_DIR / MODULE_FOUNDRY_MAP_FAILED_ARTIFACT
|
||||
try:
|
||||
failed_stat = failed_artifact.lstat()
|
||||
except FileNotFoundError:
|
||||
die("Module Foundry map failed artifact is missing")
|
||||
if (
|
||||
stat.S_ISLNK(failed_stat.st_mode)
|
||||
or not stat.S_ISREG(failed_stat.st_mode)
|
||||
or sha256_file(failed_artifact)
|
||||
!= MODULE_FOUNDRY_MAP_FAILED_ARTIFACT_SHA256
|
||||
):
|
||||
die("Module Foundry map failed artifact evidence mismatch")
|
||||
|
||||
try:
|
||||
state_stat = FAILED_STATE_FILE.lstat()
|
||||
state_lines = FAILED_STATE_FILE.read_text(
|
||||
encoding="utf-8"
|
||||
).splitlines()
|
||||
except (FileNotFoundError, OSError, UnicodeDecodeError):
|
||||
die("Module Foundry map failed journal is unreadable")
|
||||
if stat.S_ISLNK(state_stat.st_mode) or not stat.S_ISREG(
|
||||
state_stat.st_mode
|
||||
):
|
||||
die("Module Foundry map failed journal is unsafe")
|
||||
records = []
|
||||
for line in state_lines:
|
||||
try:
|
||||
value = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
die("Module Foundry map failed journal contains invalid JSON")
|
||||
if (
|
||||
isinstance(value, dict)
|
||||
and value.get("id") == MODULE_FOUNDRY_MAP_FAILED_PATCH_ID
|
||||
):
|
||||
records.append(value)
|
||||
if len(records) != 1:
|
||||
die("Module Foundry map failed journal evidence count mismatch")
|
||||
record = records[0]
|
||||
if (
|
||||
record.get("artifact") != MODULE_FOUNDRY_MAP_FAILED_ARTIFACT
|
||||
or record.get("backup_id")
|
||||
!= MODULE_FOUNDRY_MAP_FAILED_BACKUP_ID
|
||||
or record.get("component") != "module-foundry"
|
||||
or record.get("sha256")
|
||||
!= MODULE_FOUNDRY_MAP_FAILED_ARTIFACT_SHA256
|
||||
or record.get("started_apply") is not True
|
||||
or record.get("rollback_status")
|
||||
!= MODULE_FOUNDRY_MAP_FAILED_ROLLBACK_STATUS
|
||||
or record.get("status") != "failed"
|
||||
or record.get("message") != MODULE_FOUNDRY_MAP_FAILED_MESSAGE
|
||||
):
|
||||
die("Module Foundry map failed journal evidence mismatch")
|
||||
|
||||
with tempfile.TemporaryDirectory(
|
||||
prefix="module-foundry-map-failed-",
|
||||
dir=TMP_DIR,
|
||||
) as directory:
|
||||
failed_work = Path(directory)
|
||||
safe_extract(failed_artifact, failed_work)
|
||||
failed_manifest = parse_manifest(failed_work / "manifest.env")
|
||||
failed_entries = parse_files_list(failed_work / "files.txt")
|
||||
if (
|
||||
failed_manifest.get("id")
|
||||
!= MODULE_FOUNDRY_MAP_FAILED_PATCH_ID
|
||||
or failed_manifest.get("component") != "module-foundry"
|
||||
or failed_manifest.get("type") != "app-overlay"
|
||||
or tuple(failed_entries)
|
||||
!= MODULE_FOUNDRY_MAP_FAILED_ENTRIES
|
||||
):
|
||||
die("Module Foundry map failed artifact contract mismatch")
|
||||
failed_payload_sha256 = collect_exact_files(
|
||||
failed_work / "payload",
|
||||
failed_entries,
|
||||
"Module Foundry failed map payload",
|
||||
)
|
||||
|
||||
candidate_payload_sha256 = collect_exact_files(
|
||||
payload_dir,
|
||||
entries,
|
||||
"Module Foundry map recovery payload",
|
||||
)
|
||||
if (
|
||||
exact_file_map_sha256(candidate_payload_sha256)
|
||||
!= MODULE_FOUNDRY_MAP_RECOVERY_CANDIDATE_TREE_SHA256
|
||||
):
|
||||
die("Module Foundry map recovery candidate drift detected")
|
||||
if {
|
||||
rel: candidate_payload_sha256.get(rel)
|
||||
for rel in failed_payload_sha256
|
||||
} != failed_payload_sha256:
|
||||
die("Module Foundry failed map payload changed unexpectedly")
|
||||
|
||||
installed_root = component_root("module-foundry")
|
||||
for rel in MODULE_FOUNDRY_MAP_RECOVERY_MISSING_PREDECESSOR_ENTRIES:
|
||||
path = installed_root / rel
|
||||
if path.exists() or path.is_symlink():
|
||||
die(
|
||||
"Module Foundry map recovery missing predecessor drift: "
|
||||
f"{rel}"
|
||||
)
|
||||
installed_entries = tuple(
|
||||
rel
|
||||
for rel in entries
|
||||
if rel not in MODULE_FOUNDRY_MAP_RECOVERY_MISSING_PREDECESSOR_ENTRIES
|
||||
)
|
||||
installed_payload_sha256 = collect_exact_files(
|
||||
installed_root,
|
||||
installed_entries,
|
||||
"Module Foundry partial map predecessor",
|
||||
)
|
||||
if (
|
||||
exact_file_map_sha256(installed_payload_sha256)
|
||||
!= MODULE_FOUNDRY_MAP_RECOVERY_INSTALLED_TREE_SHA256
|
||||
):
|
||||
die("Module Foundry partial map predecessor drift detected")
|
||||
installed_failed_payload_sha256 = {
|
||||
rel: digest
|
||||
for rel, digest in failed_payload_sha256.items()
|
||||
if any(
|
||||
rel == installed or rel.startswith(f"{installed}/")
|
||||
for installed in MODULE_FOUNDRY_MAP_FAILED_INSTALLED_ENTRIES
|
||||
)
|
||||
}
|
||||
if any(
|
||||
not any(
|
||||
rel == installed or rel.startswith(f"{installed}/")
|
||||
for rel in installed_failed_payload_sha256
|
||||
)
|
||||
for installed in MODULE_FOUNDRY_MAP_FAILED_INSTALLED_ENTRIES
|
||||
):
|
||||
die("Module Foundry failed installed overlap evidence mismatch")
|
||||
if {
|
||||
rel: installed_payload_sha256.get(rel)
|
||||
for rel in installed_failed_payload_sha256
|
||||
} != installed_failed_payload_sha256:
|
||||
die("Module Foundry installed failed map payload drift detected")
|
||||
|
||||
runtime = module_foundry_runtime_inventory()
|
||||
return {
|
||||
"mode": "failed-map-overlay-source+runtime-reconciliation",
|
||||
"failed_patch_id": MODULE_FOUNDRY_MAP_FAILED_PATCH_ID,
|
||||
"failed_artifact_sha256": (
|
||||
MODULE_FOUNDRY_MAP_FAILED_ARTIFACT_SHA256
|
||||
),
|
||||
"failed_backup_id": MODULE_FOUNDRY_MAP_FAILED_BACKUP_ID,
|
||||
"installed_tree_sha256": (
|
||||
MODULE_FOUNDRY_MAP_RECOVERY_INSTALLED_TREE_SHA256
|
||||
),
|
||||
"candidate_tree_sha256": (
|
||||
MODULE_FOUNDRY_MAP_RECOVERY_CANDIDATE_TREE_SHA256
|
||||
),
|
||||
"runtime_container_id": runtime["containerId"],
|
||||
"runtime_image_id": runtime["imageId"],
|
||||
}
|
||||
|
||||
|
||||
def read_module_foundry_runtime_before(backup_dir):
|
||||
runtime = read_strict_json(
|
||||
backup_dir / MODULE_FOUNDRY_RUNTIME_BEFORE_FILE,
|
||||
"Module Foundry pre-apply runtime inventory",
|
||||
max_bytes=16 * 1024,
|
||||
)
|
||||
if (
|
||||
runtime.get("schemaVersion")
|
||||
!= "nodedc.module-foundry.runtime-inventory.v1"
|
||||
or runtime.get("composeProject") != "nodedc-module-foundry"
|
||||
or runtime.get("service") != MODULE_FOUNDRY_SERVICE
|
||||
or not re.fullmatch(
|
||||
r"[a-f0-9]{12,64}",
|
||||
str(runtime.get("containerId", "")),
|
||||
)
|
||||
or not re.fullmatch(
|
||||
r"sha256:[a-f0-9]{64}",
|
||||
str(runtime.get("imageId", "")),
|
||||
)
|
||||
or not isinstance(runtime.get("imageRef"), str)
|
||||
or not re.fullmatch(
|
||||
r"[A-Za-z0-9][A-Za-z0-9._/:@-]{0,255}",
|
||||
runtime["imageRef"],
|
||||
)
|
||||
or runtime["imageRef"].startswith("sha256:")
|
||||
or "@" in runtime["imageRef"]
|
||||
):
|
||||
die("Module Foundry pre-apply runtime inventory mismatch")
|
||||
return runtime
|
||||
|
||||
|
||||
def rollback_module_foundry_apply(
|
||||
root,
|
||||
backup_dir,
|
||||
entries,
|
||||
current_stamp,
|
||||
applied_services,
|
||||
):
|
||||
if tuple(applied_services or ()) != (MODULE_FOUNDRY_SERVICE,):
|
||||
die("Module Foundry rollback service set mismatch")
|
||||
runtime_before = read_module_foundry_runtime_before(backup_dir)
|
||||
restored_count = restore_platform_overlay(
|
||||
root,
|
||||
backup_dir,
|
||||
entries,
|
||||
current_stamp,
|
||||
)
|
||||
current = module_foundry_runtime_inventory(
|
||||
required=False,
|
||||
require_healthy=False,
|
||||
)
|
||||
if (
|
||||
current is not None
|
||||
and current["containerId"] == runtime_before["containerId"]
|
||||
and current["imageId"] == runtime_before["imageId"]
|
||||
):
|
||||
run_healthchecks("module-foundry", entries, applied_services)
|
||||
return f"source-restored-runtime-unchanged:{restored_count}"
|
||||
|
||||
subprocess.run(
|
||||
[
|
||||
str(DOCKER),
|
||||
"image",
|
||||
"tag",
|
||||
runtime_before["imageId"],
|
||||
runtime_before["imageRef"],
|
||||
],
|
||||
check=True,
|
||||
)
|
||||
if (
|
||||
inspect_local_image(
|
||||
runtime_before["imageRef"],
|
||||
"Module Foundry rollback image",
|
||||
)
|
||||
!= runtime_before["imageId"]
|
||||
):
|
||||
die("Module Foundry rollback image tag mismatch")
|
||||
run_module_foundry_compose_up(applied_services)
|
||||
run_healthchecks("module-foundry", entries, applied_services)
|
||||
restored = module_foundry_runtime_inventory()
|
||||
if restored["imageId"] != runtime_before["imageId"]:
|
||||
die("Module Foundry rollback runtime image mismatch")
|
||||
return f"source+runtime-restored:{restored_count}"
|
||||
|
||||
|
||||
def validate_engine_l2_closed_loop_stable_source(root):
|
||||
stable_entries = tuple(ENGINE_L2_CLOSED_LOOP_STABLE_SHA256)
|
||||
actual = collect_exact_files(
|
||||
@@ -18146,67 +17728,6 @@ def run_compose(component, services, entries=None):
|
||||
)
|
||||
|
||||
|
||||
def run_module_foundry_compose_up(services):
|
||||
if tuple(services or ()) != (MODULE_FOUNDRY_SERVICE,):
|
||||
die("Module Foundry Compose service set mismatch")
|
||||
compose_root = component_compose_root("module-foundry")
|
||||
cmd = [
|
||||
*compose_base_cmd("module-foundry"),
|
||||
"up",
|
||||
"-d",
|
||||
"--force-recreate",
|
||||
"--pull",
|
||||
"never",
|
||||
"--no-deps",
|
||||
*services,
|
||||
]
|
||||
try:
|
||||
subprocess.run(cmd, cwd=str(compose_root), check=True)
|
||||
except subprocess.CalledProcessError:
|
||||
subprocess.run(
|
||||
[
|
||||
*compose_base_cmd("module-foundry"),
|
||||
"logs",
|
||||
"--no-color",
|
||||
"--tail=180",
|
||||
*services,
|
||||
],
|
||||
cwd=str(compose_root),
|
||||
check=False,
|
||||
)
|
||||
raise
|
||||
subprocess.run(
|
||||
[*compose_base_cmd("module-foundry"), "ps"],
|
||||
cwd=str(compose_root),
|
||||
check=True,
|
||||
)
|
||||
|
||||
|
||||
def run_module_foundry_compose(services):
|
||||
if tuple(services or ()) != (MODULE_FOUNDRY_SERVICE,):
|
||||
die("Module Foundry build service set mismatch")
|
||||
compose_root = component_compose_root("module-foundry")
|
||||
build_cmd = [
|
||||
*compose_base_cmd("module-foundry"),
|
||||
"build",
|
||||
*services,
|
||||
]
|
||||
for attempt in range(1, MODULE_FOUNDRY_BUILD_ATTEMPTS + 1):
|
||||
try:
|
||||
subprocess.run(build_cmd, cwd=str(compose_root), check=True)
|
||||
break
|
||||
except subprocess.CalledProcessError:
|
||||
if attempt >= MODULE_FOUNDRY_BUILD_ATTEMPTS:
|
||||
raise
|
||||
print(
|
||||
"module-foundry-build-retry="
|
||||
f"{attempt}/{MODULE_FOUNDRY_BUILD_ATTEMPTS - 1}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
time.sleep(2)
|
||||
run_module_foundry_compose_up(services)
|
||||
|
||||
|
||||
def run_engine_node_intelligence_compose(services, entries):
|
||||
if not is_engine_node_intelligence_transition("engine", entries):
|
||||
die("Engine node-intelligence Compose called for unrelated artifact")
|
||||
@@ -18503,8 +18024,6 @@ def run_component_runtime(component, entries, services):
|
||||
prepare_component_runtime(component, entries)
|
||||
if is_engine_node_intelligence_transition(component, entries):
|
||||
run_engine_node_intelligence_compose(services, entries)
|
||||
elif component == "module-foundry":
|
||||
run_module_foundry_compose(services)
|
||||
else:
|
||||
run_compose(component, services, entries)
|
||||
|
||||
@@ -18904,99 +18423,6 @@ def compose_service_container_id(component, service):
|
||||
return container_ids[0]
|
||||
|
||||
|
||||
def module_foundry_runtime_inventory(required=True, require_healthy=True):
|
||||
result = subprocess.run(
|
||||
[
|
||||
*compose_base_cmd("module-foundry"),
|
||||
"ps",
|
||||
"-q",
|
||||
MODULE_FOUNDRY_SERVICE,
|
||||
],
|
||||
cwd=str(component_compose_root("module-foundry")),
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
container_ids = [
|
||||
line.strip()
|
||||
for line in result.stdout.splitlines()
|
||||
if line.strip()
|
||||
]
|
||||
if (
|
||||
result.returncode != 0
|
||||
or len(container_ids) > 1
|
||||
or any(
|
||||
not re.fullmatch(r"[a-f0-9]{12,64}", value)
|
||||
for value in container_ids
|
||||
)
|
||||
):
|
||||
die("Module Foundry runtime topology is unproven")
|
||||
if not container_ids:
|
||||
if required:
|
||||
die("Module Foundry runtime is missing")
|
||||
return None
|
||||
container_id = container_ids[0]
|
||||
containers = docker_json(
|
||||
["container", "inspect", container_id],
|
||||
"Module Foundry runtime container inspect",
|
||||
)
|
||||
if (
|
||||
not isinstance(containers, list)
|
||||
or len(containers) != 1
|
||||
or not isinstance(containers[0], dict)
|
||||
):
|
||||
die("Module Foundry runtime container inspect shape mismatch")
|
||||
container = containers[0]
|
||||
config = container.get("Config") or {}
|
||||
labels = config.get("Labels") or {}
|
||||
state = container.get("State") or {}
|
||||
image_id = container.get("Image")
|
||||
image_ref = config.get("Image")
|
||||
health = (state.get("Health") or {}).get("Status")
|
||||
if (
|
||||
labels.get("com.docker.compose.project")
|
||||
!= "nodedc-module-foundry"
|
||||
or labels.get("com.docker.compose.service")
|
||||
!= MODULE_FOUNDRY_SERVICE
|
||||
or not re.fullmatch(r"sha256:[a-f0-9]{64}", str(image_id or ""))
|
||||
or not isinstance(image_ref, str)
|
||||
or not re.fullmatch(
|
||||
r"[A-Za-z0-9][A-Za-z0-9._/:@-]{0,255}",
|
||||
image_ref,
|
||||
)
|
||||
or image_ref.startswith("sha256:")
|
||||
or "@" in image_ref
|
||||
or (
|
||||
require_healthy
|
||||
and (
|
||||
state.get("Running") is not True
|
||||
or state.get("Status") != "running"
|
||||
or health not in (None, "healthy")
|
||||
)
|
||||
)
|
||||
):
|
||||
die("Module Foundry runtime inventory mismatch")
|
||||
return {
|
||||
"schemaVersion": "nodedc.module-foundry.runtime-inventory.v1",
|
||||
"composeProject": "nodedc-module-foundry",
|
||||
"service": MODULE_FOUNDRY_SERVICE,
|
||||
"containerId": container_id,
|
||||
"imageId": image_id,
|
||||
"imageRef": image_ref,
|
||||
"health": health or "not-configured",
|
||||
}
|
||||
|
||||
|
||||
def validate_module_foundry_candidate_runtime(runtime_before):
|
||||
current = module_foundry_runtime_inventory()
|
||||
if (
|
||||
current["containerId"] == runtime_before["containerId"]
|
||||
or current["imageId"] == runtime_before["imageId"]
|
||||
):
|
||||
die("Module Foundry candidate generation was not activated")
|
||||
return current
|
||||
|
||||
|
||||
def healthcheck_compose_service(component, service):
|
||||
healthcheck_container(compose_service_container_id(component, service))
|
||||
|
||||
@@ -19506,17 +18932,6 @@ def run_healthchecks(component, entries=None, services=None):
|
||||
# successful health probe. Wait for the Compose health barrier before
|
||||
# the strict immutable-runtime preflight, both on apply and retry.
|
||||
healthcheck_compose_service("engine", "nodedc-backend")
|
||||
if component == "module-foundry":
|
||||
if tuple(services or ()) != (MODULE_FOUNDRY_SERVICE,):
|
||||
die("Module Foundry healthcheck service set mismatch")
|
||||
# The HTTP endpoint can answer before Docker publishes the first
|
||||
# successful health probe. Inventory acceptance is intentionally
|
||||
# strict, so wait for the Compose health barrier on both activation
|
||||
# and rollback before inspecting the immutable runtime identity.
|
||||
healthcheck_compose_service(
|
||||
"module-foundry",
|
||||
MODULE_FOUNDRY_SERVICE,
|
||||
)
|
||||
for url in component_healthchecks(component, entries, services):
|
||||
healthcheck_url(url)
|
||||
if (
|
||||
@@ -20140,8 +19555,6 @@ def apply_artifact(artifact):
|
||||
apply_started = False
|
||||
engine_backend_recreated = False
|
||||
engine_backend_initial_mode = None
|
||||
module_foundry_runtime_before = None
|
||||
module_foundry_map_recovery_preflight = None
|
||||
runtime_started = False
|
||||
current_stamp = stamp()
|
||||
|
||||
@@ -20220,13 +19633,6 @@ def apply_artifact(artifact):
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
else:
|
||||
die(f"component payload root not found: {root}")
|
||||
module_foundry_map_recovery_preflight = (
|
||||
validate_module_foundry_map_recovery_evidence(
|
||||
manifest,
|
||||
entries,
|
||||
payload_dir,
|
||||
)
|
||||
)
|
||||
if is_engine_data_product_publish_grant_slice(component, entries):
|
||||
preflight_engine_data_product_publish_grant_predecessor(payload_dir)
|
||||
publish_backend_preflight = preflight_engine_credential_backend_runtime()
|
||||
@@ -20506,27 +19912,6 @@ def apply_artifact(artifact):
|
||||
DEVICE_PLANE_RUNTIME_SERVICES
|
||||
)
|
||||
)
|
||||
if component == "module-foundry":
|
||||
module_foundry_runtime_before = (
|
||||
module_foundry_runtime_inventory()
|
||||
)
|
||||
if (
|
||||
module_foundry_map_recovery_preflight is not None
|
||||
and (
|
||||
module_foundry_runtime_before["containerId"]
|
||||
!= module_foundry_map_recovery_preflight[
|
||||
"runtime_container_id"
|
||||
]
|
||||
or module_foundry_runtime_before["imageId"]
|
||||
!= module_foundry_map_recovery_preflight[
|
||||
"runtime_image_id"
|
||||
]
|
||||
)
|
||||
):
|
||||
die(
|
||||
"Module Foundry map predecessor runtime "
|
||||
"changed during preflight"
|
||||
)
|
||||
|
||||
if is_engine_n8n_transition(component, entries):
|
||||
transition_descriptor = read_engine_n8n_transition_descriptor(
|
||||
@@ -20553,23 +19938,6 @@ def apply_artifact(artifact):
|
||||
|
||||
include_nginx_html = component == "engine" and component_publish_dist(component, entries)
|
||||
create_backup(root, backup_dir, entries, include_nginx_html)
|
||||
if component == "module-foundry":
|
||||
if module_foundry_runtime_before is None:
|
||||
die("Module Foundry pre-apply runtime inventory is missing")
|
||||
module_foundry_runtime_path = (
|
||||
backup_dir / MODULE_FOUNDRY_RUNTIME_BEFORE_FILE
|
||||
)
|
||||
module_foundry_runtime_path.write_text(
|
||||
json.dumps(
|
||||
module_foundry_runtime_before,
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
sort_keys=True,
|
||||
)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
module_foundry_runtime_path.chmod(0o600)
|
||||
if component == "device-plane":
|
||||
if device_plane_runtime_before is None:
|
||||
die("Device Plane pre-apply runtime inventory is missing")
|
||||
@@ -20675,12 +20043,6 @@ def apply_artifact(artifact):
|
||||
):
|
||||
die("Engine L2 closed-loop app generation was not recreated")
|
||||
run_healthchecks(component, entries, services)
|
||||
if component == "module-foundry":
|
||||
if module_foundry_runtime_before is None:
|
||||
die("Module Foundry candidate predecessor is missing")
|
||||
validate_module_foundry_candidate_runtime(
|
||||
module_foundry_runtime_before
|
||||
)
|
||||
if is_device_plane_b2_discovery_ingress_slice(
|
||||
component,
|
||||
entries,
|
||||
@@ -20887,35 +20249,6 @@ def apply_artifact(artifact):
|
||||
except Exception as rollback_exc:
|
||||
rollback_status = f"failed:{type(rollback_exc).__name__}"
|
||||
print("engine-automatic-rollback=failed", file=sys.stderr)
|
||||
elif (
|
||||
component == "module-foundry"
|
||||
and entries is not None
|
||||
and services is not None
|
||||
):
|
||||
try:
|
||||
restored_state = rollback_module_foundry_apply(
|
||||
root,
|
||||
backup_dir,
|
||||
entries,
|
||||
current_stamp,
|
||||
services,
|
||||
)
|
||||
rollback_status = (
|
||||
f"ok:module-foundry-overlay:{restored_state}"
|
||||
)
|
||||
print(
|
||||
"module-foundry-automatic-rollback="
|
||||
f"{rollback_status}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
except Exception as rollback_exc:
|
||||
rollback_status = (
|
||||
f"failed:{type(rollback_exc).__name__}"
|
||||
)
|
||||
print(
|
||||
"module-foundry-automatic-rollback=failed",
|
||||
file=sys.stderr,
|
||||
)
|
||||
elif (
|
||||
component == "platform"
|
||||
and entries is not None
|
||||
|
||||
@@ -1,174 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import hashlib
|
||||
import importlib.machinery
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import tarfile
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path, PurePosixPath
|
||||
|
||||
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
PLATFORM_ROOT = SCRIPT_DIR.parent.parent
|
||||
WORKSPACE_ROOT = PLATFORM_ROOT.parent.parent
|
||||
FOUNDRY_ROOT = WORKSPACE_ROOT / "NODEDC_DESIGN_GUIDELINE"
|
||||
BUILDER = (
|
||||
SCRIPT_DIR / "build-module-foundry-map-runtime-recovery-artifact.mjs"
|
||||
)
|
||||
RUNNER_PATH = SCRIPT_DIR / "nodedc-deploy"
|
||||
SOURCE_COMMIT = "58800d957632320fa6717ec3fcff972023528759"
|
||||
PATCH_ID = "module-foundry-map-runtime-recovery-20260809-999"
|
||||
EXPECTED_FILES = (
|
||||
"package.json",
|
||||
"package-lock.json",
|
||||
"apps/catalog/src",
|
||||
"packages",
|
||||
"registry",
|
||||
"runtime-seed/page-layouts/map.json",
|
||||
"scripts",
|
||||
"server/catalog-server.mjs",
|
||||
"server/foundry-mcp.mjs",
|
||||
"server/map-grid-persistence.test.mjs",
|
||||
)
|
||||
|
||||
|
||||
def load_runner():
|
||||
loader = importlib.machinery.SourceFileLoader(
|
||||
"nodedc_module_foundry_map_runtime_recovery_artifact",
|
||||
str(RUNNER_PATH),
|
||||
)
|
||||
spec = importlib.util.spec_from_loader(loader.name, loader)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
RUNNER = load_runner()
|
||||
|
||||
|
||||
class ModuleFoundryMapRuntimeRecoveryArtifactTest(unittest.TestCase):
|
||||
def export_source(self, root):
|
||||
archive_path = root / "source.tar"
|
||||
source_root = root / "source"
|
||||
source_root.mkdir()
|
||||
subprocess.run(
|
||||
[
|
||||
"git",
|
||||
"-C",
|
||||
str(FOUNDRY_ROOT),
|
||||
"archive",
|
||||
"--format=tar",
|
||||
f"--output={archive_path}",
|
||||
SOURCE_COMMIT,
|
||||
],
|
||||
check=True,
|
||||
)
|
||||
with tarfile.open(archive_path, "r:") as archive:
|
||||
for member in archive.getmembers():
|
||||
path = PurePosixPath(member.name)
|
||||
self.assertFalse(path.is_absolute())
|
||||
self.assertNotIn("..", path.parts)
|
||||
archive.extractall(source_root, filter="data")
|
||||
return source_root
|
||||
|
||||
def build(self, source_root, artifact_dir):
|
||||
environment = os.environ.copy()
|
||||
environment["NODEDC_FOUNDRY_SOURCE_ROOT"] = str(source_root)
|
||||
environment["NODEDC_FOUNDRY_SOURCE_COMMIT"] = SOURCE_COMMIT
|
||||
environment["NODEDC_DEPLOY_ARTIFACT_DIR"] = str(artifact_dir)
|
||||
completed = subprocess.run(
|
||||
["node", str(BUILDER), PATCH_ID],
|
||||
cwd=PLATFORM_ROOT,
|
||||
env=environment,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
return json.loads(completed.stdout)
|
||||
|
||||
def test_builder_is_deterministic_coherent_and_runner_compatible(self):
|
||||
with tempfile.TemporaryDirectory(
|
||||
prefix="nodedc-foundry-map-runtime-recovery-"
|
||||
) as directory:
|
||||
root = Path(directory)
|
||||
source_root = self.export_source(root)
|
||||
first = self.build(source_root, root / "first")
|
||||
second = self.build(source_root, root / "second")
|
||||
first_artifact = Path(first["artifact"])
|
||||
second_artifact = Path(second["artifact"])
|
||||
|
||||
self.assertEqual(first_artifact.read_bytes(), second_artifact.read_bytes())
|
||||
self.assertEqual(
|
||||
first["sha256"],
|
||||
hashlib.sha256(first_artifact.read_bytes()).hexdigest(),
|
||||
)
|
||||
self.assertEqual(first["sourceCommit"], SOURCE_COMMIT)
|
||||
self.assertEqual(tuple(first["files"]), EXPECTED_FILES)
|
||||
self.assertEqual(first["services"], ["nodedc-module-foundry"])
|
||||
|
||||
extracted = root / "loaded"
|
||||
extracted.mkdir()
|
||||
manifest, entries, payload = RUNNER.load_artifact(
|
||||
first_artifact,
|
||||
extracted,
|
||||
)
|
||||
self.assertEqual(manifest["id"], PATCH_ID)
|
||||
self.assertEqual(manifest["component"], "module-foundry")
|
||||
self.assertEqual(tuple(entries), EXPECTED_FILES)
|
||||
self.assertEqual(
|
||||
RUNNER.component_services("module-foundry", entries),
|
||||
("nodedc-module-foundry",),
|
||||
)
|
||||
|
||||
preview = (
|
||||
payload / "apps/catalog/src/MapFixturePreview.tsx"
|
||||
).read_text(encoding="utf-8")
|
||||
renderer = (
|
||||
payload / "apps/catalog/src/CesiumMapRenderer.tsx"
|
||||
).read_text(encoding="utf-8")
|
||||
package_lock = (payload / "package-lock.json").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
self.assertIn("const sectorSpatialEntities = useMemo", preview)
|
||||
self.assertIn("class GridLayerController", renderer)
|
||||
self.assertIn('"packages/map-cesium-react"', package_lock)
|
||||
self.assertTrue(
|
||||
(payload / "packages/map-cesium-react/src/index.ts").is_file()
|
||||
)
|
||||
|
||||
with tarfile.open(first_artifact, "r:gz") as archive:
|
||||
members = archive.getmembers()
|
||||
names = [member.name for member in members]
|
||||
regular_payloads = [
|
||||
archive.extractfile(member).read()
|
||||
for member in members
|
||||
if member.isfile()
|
||||
]
|
||||
self.assertFalse(
|
||||
any(Path(name).name.startswith("._") for name in names)
|
||||
)
|
||||
self.assertFalse(
|
||||
any(
|
||||
"/.git/" in name
|
||||
or "/node_modules/" in name
|
||||
or "/runtime-data/" in name
|
||||
or "/dist/" in name
|
||||
for name in names
|
||||
)
|
||||
)
|
||||
self.assertNotIn(
|
||||
"payload/scripts/spark-governance-contract.test.mjs",
|
||||
names,
|
||||
)
|
||||
self.assertNotIn(
|
||||
b"-----BEGIN PRIVATE KEY-----",
|
||||
b"\n".join(regular_payloads),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main(verbosity=2)
|
||||
@@ -1,519 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import hashlib
|
||||
import importlib.machinery
|
||||
import importlib.util
|
||||
import json
|
||||
import subprocess
|
||||
import tarfile
|
||||
import tempfile
|
||||
import unittest
|
||||
from contextlib import ExitStack
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
RUNNER_PATH = SCRIPT_DIR / "nodedc-deploy"
|
||||
|
||||
|
||||
def load_runner():
|
||||
loader = importlib.machinery.SourceFileLoader(
|
||||
"nodedc_module_foundry_runtime_recovery",
|
||||
str(RUNNER_PATH),
|
||||
)
|
||||
spec = importlib.util.spec_from_loader(loader.name, loader)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
RUNNER = load_runner()
|
||||
|
||||
|
||||
def runtime(container_char="a", image_char="b"):
|
||||
return {
|
||||
"schemaVersion": "nodedc.module-foundry.runtime-inventory.v1",
|
||||
"composeProject": "nodedc-module-foundry",
|
||||
"service": RUNNER.MODULE_FOUNDRY_SERVICE,
|
||||
"containerId": container_char * 64,
|
||||
"imageId": f"sha256:{image_char * 64}",
|
||||
"imageRef": "nodedc-module-foundry-nodedc-module-foundry:latest",
|
||||
"health": "healthy",
|
||||
}
|
||||
|
||||
|
||||
class ModuleFoundryRuntimeRecoveryTest(unittest.TestCase):
|
||||
def test_healthchecks_wait_for_compose_health_before_http(self):
|
||||
events = []
|
||||
with (
|
||||
mock.patch.object(
|
||||
RUNNER,
|
||||
"healthcheck_compose_service",
|
||||
side_effect=lambda component, service: events.append(
|
||||
("compose", component, service)
|
||||
),
|
||||
),
|
||||
mock.patch.object(
|
||||
RUNNER,
|
||||
"component_healthchecks",
|
||||
return_value=("http://foundry/healthz",),
|
||||
),
|
||||
mock.patch.object(
|
||||
RUNNER,
|
||||
"healthcheck_url",
|
||||
side_effect=lambda check: events.append(("http", check)),
|
||||
),
|
||||
):
|
||||
RUNNER.run_healthchecks(
|
||||
"module-foundry",
|
||||
("package.json",),
|
||||
(RUNNER.MODULE_FOUNDRY_SERVICE,),
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
events,
|
||||
[
|
||||
(
|
||||
"compose",
|
||||
"module-foundry",
|
||||
RUNNER.MODULE_FOUNDRY_SERVICE,
|
||||
),
|
||||
("http", "http://foundry/healthz"),
|
||||
],
|
||||
)
|
||||
|
||||
def test_build_retries_before_any_runtime_recreate(self):
|
||||
first_failure = subprocess.CalledProcessError(17, ["docker", "compose"])
|
||||
completed = subprocess.CompletedProcess([], 0)
|
||||
with (
|
||||
mock.patch.object(
|
||||
RUNNER,
|
||||
"component_compose_root",
|
||||
return_value=Path("/compose"),
|
||||
),
|
||||
mock.patch.object(
|
||||
RUNNER,
|
||||
"compose_base_cmd",
|
||||
return_value=["docker", "compose"],
|
||||
),
|
||||
mock.patch.object(
|
||||
RUNNER.subprocess,
|
||||
"run",
|
||||
side_effect=[first_failure, completed],
|
||||
) as run,
|
||||
mock.patch.object(RUNNER.time, "sleep") as sleep,
|
||||
mock.patch.object(RUNNER, "run_module_foundry_compose_up") as up,
|
||||
):
|
||||
RUNNER.run_module_foundry_compose(
|
||||
(RUNNER.MODULE_FOUNDRY_SERVICE,)
|
||||
)
|
||||
|
||||
expected = [
|
||||
"docker",
|
||||
"compose",
|
||||
"build",
|
||||
RUNNER.MODULE_FOUNDRY_SERVICE,
|
||||
]
|
||||
self.assertEqual(run.call_count, 2)
|
||||
self.assertEqual(run.call_args_list[0].args[0], expected)
|
||||
self.assertEqual(run.call_args_list[1].args[0], expected)
|
||||
sleep.assert_called_once_with(2)
|
||||
up.assert_called_once_with((RUNNER.MODULE_FOUNDRY_SERVICE,))
|
||||
|
||||
def test_runtime_recreate_uses_built_image_without_build_or_pull(self):
|
||||
completed = subprocess.CompletedProcess([], 0)
|
||||
with (
|
||||
mock.patch.object(
|
||||
RUNNER,
|
||||
"component_compose_root",
|
||||
return_value=Path("/compose"),
|
||||
),
|
||||
mock.patch.object(
|
||||
RUNNER,
|
||||
"compose_base_cmd",
|
||||
return_value=["docker", "compose"],
|
||||
),
|
||||
mock.patch.object(
|
||||
RUNNER.subprocess,
|
||||
"run",
|
||||
return_value=completed,
|
||||
) as run,
|
||||
):
|
||||
RUNNER.run_module_foundry_compose_up(
|
||||
(RUNNER.MODULE_FOUNDRY_SERVICE,)
|
||||
)
|
||||
|
||||
up = run.call_args_list[0].args[0]
|
||||
self.assertEqual(
|
||||
up,
|
||||
[
|
||||
"docker",
|
||||
"compose",
|
||||
"up",
|
||||
"-d",
|
||||
"--force-recreate",
|
||||
"--pull",
|
||||
"never",
|
||||
"--no-deps",
|
||||
RUNNER.MODULE_FOUNDRY_SERVICE,
|
||||
],
|
||||
)
|
||||
self.assertNotIn("--build", up)
|
||||
self.assertEqual(run.call_args_list[1].args[0], ["docker", "compose", "ps"])
|
||||
|
||||
def test_rollback_restores_source_and_keeps_unchanged_runtime(self):
|
||||
with tempfile.TemporaryDirectory(
|
||||
prefix="module-foundry-rollback-unchanged-"
|
||||
) as directory:
|
||||
work = Path(directory)
|
||||
root = work / "source"
|
||||
backup = work / "backup"
|
||||
tmp = work / "tmp"
|
||||
root.mkdir()
|
||||
backup.mkdir()
|
||||
tmp.mkdir()
|
||||
entries = ("old.txt", "new.txt")
|
||||
(root / "old.txt").write_text("old\n", encoding="utf-8")
|
||||
RUNNER.create_backup(root, backup, entries, False)
|
||||
(root / "old.txt").write_text("candidate\n", encoding="utf-8")
|
||||
(root / "new.txt").write_text("candidate\n", encoding="utf-8")
|
||||
before = runtime()
|
||||
(backup / RUNNER.MODULE_FOUNDRY_RUNTIME_BEFORE_FILE).write_text(
|
||||
json.dumps(before),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
with (
|
||||
mock.patch.object(RUNNER, "TMP_DIR", tmp),
|
||||
mock.patch.object(
|
||||
RUNNER,
|
||||
"module_foundry_runtime_inventory",
|
||||
return_value=before,
|
||||
),
|
||||
mock.patch.object(RUNNER, "run_healthchecks") as health,
|
||||
mock.patch.object(RUNNER.subprocess, "run") as process,
|
||||
):
|
||||
result = RUNNER.rollback_module_foundry_apply(
|
||||
root,
|
||||
backup,
|
||||
entries,
|
||||
"20260809-000000",
|
||||
(RUNNER.MODULE_FOUNDRY_SERVICE,),
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
result,
|
||||
"source-restored-runtime-unchanged:2",
|
||||
)
|
||||
self.assertEqual(
|
||||
(root / "old.txt").read_text(encoding="utf-8"),
|
||||
"old\n",
|
||||
)
|
||||
self.assertFalse((root / "new.txt").exists())
|
||||
process.assert_not_called()
|
||||
health.assert_called_once()
|
||||
|
||||
def test_rollback_retags_and_recreates_changed_runtime(self):
|
||||
with tempfile.TemporaryDirectory(
|
||||
prefix="module-foundry-rollback-changed-"
|
||||
) as directory:
|
||||
work = Path(directory)
|
||||
root = work / "source"
|
||||
backup = work / "backup"
|
||||
tmp = work / "tmp"
|
||||
root.mkdir()
|
||||
backup.mkdir()
|
||||
tmp.mkdir()
|
||||
entries = ("old.txt",)
|
||||
(root / "old.txt").write_text("old\n", encoding="utf-8")
|
||||
RUNNER.create_backup(root, backup, entries, False)
|
||||
(root / "old.txt").write_text("candidate\n", encoding="utf-8")
|
||||
before = runtime()
|
||||
candidate = runtime("c", "d")
|
||||
(backup / RUNNER.MODULE_FOUNDRY_RUNTIME_BEFORE_FILE).write_text(
|
||||
json.dumps(before),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
with (
|
||||
mock.patch.object(RUNNER, "TMP_DIR", tmp),
|
||||
mock.patch.object(
|
||||
RUNNER,
|
||||
"module_foundry_runtime_inventory",
|
||||
side_effect=[candidate, before],
|
||||
) as inventory,
|
||||
mock.patch.object(
|
||||
RUNNER,
|
||||
"inspect_local_image",
|
||||
return_value=before["imageId"],
|
||||
),
|
||||
mock.patch.object(RUNNER, "run_module_foundry_compose_up") as up,
|
||||
mock.patch.object(RUNNER, "run_healthchecks") as health,
|
||||
mock.patch.object(RUNNER.subprocess, "run") as process,
|
||||
):
|
||||
result = RUNNER.rollback_module_foundry_apply(
|
||||
root,
|
||||
backup,
|
||||
entries,
|
||||
"20260809-000000",
|
||||
(RUNNER.MODULE_FOUNDRY_SERVICE,),
|
||||
)
|
||||
|
||||
self.assertEqual(result, "source+runtime-restored:1")
|
||||
self.assertEqual(
|
||||
inventory.call_args_list[0].kwargs,
|
||||
{"required": False, "require_healthy": False},
|
||||
)
|
||||
process.assert_called_once_with(
|
||||
[
|
||||
str(RUNNER.DOCKER),
|
||||
"image",
|
||||
"tag",
|
||||
before["imageId"],
|
||||
before["imageRef"],
|
||||
],
|
||||
check=True,
|
||||
)
|
||||
up.assert_called_once_with((RUNNER.MODULE_FOUNDRY_SERVICE,))
|
||||
health.assert_called_once()
|
||||
|
||||
def test_candidate_must_replace_both_container_and_image(self):
|
||||
before = runtime()
|
||||
for candidate in (
|
||||
runtime("a", "c"),
|
||||
runtime("c", "b"),
|
||||
):
|
||||
with mock.patch.object(
|
||||
RUNNER,
|
||||
"module_foundry_runtime_inventory",
|
||||
return_value=candidate,
|
||||
):
|
||||
with self.assertRaises(RUNNER.DeployError):
|
||||
RUNNER.validate_module_foundry_candidate_runtime(before)
|
||||
|
||||
def test_preapply_runtime_rejects_digest_only_image_reference(self):
|
||||
with tempfile.TemporaryDirectory(
|
||||
prefix="module-foundry-runtime-reference-"
|
||||
) as directory:
|
||||
backup = Path(directory)
|
||||
before = runtime()
|
||||
before["imageRef"] = before["imageId"]
|
||||
(backup / RUNNER.MODULE_FOUNDRY_RUNTIME_BEFORE_FILE).write_text(
|
||||
json.dumps(before),
|
||||
encoding="utf-8",
|
||||
)
|
||||
with self.assertRaises(RUNNER.DeployError):
|
||||
RUNNER.read_module_foundry_runtime_before(backup)
|
||||
|
||||
def test_recovery_accepts_only_exact_failed_partial_predecessor(self):
|
||||
with tempfile.TemporaryDirectory(
|
||||
prefix="module-foundry-map-evidence-"
|
||||
) as directory:
|
||||
work = Path(directory)
|
||||
backups = work / "backups"
|
||||
failed = work / "failed"
|
||||
state = work / "state"
|
||||
tmp = work / "tmp"
|
||||
installed = work / "installed"
|
||||
candidate = work / "candidate"
|
||||
for path in (backups, failed, state, tmp, installed, candidate):
|
||||
path.mkdir()
|
||||
|
||||
recovery_entries = (
|
||||
"apps/catalog/src",
|
||||
"scripts",
|
||||
"server/map-grid-persistence.test.mjs",
|
||||
)
|
||||
missing_entries = ("server/map-grid-persistence.test.mjs",)
|
||||
failed_entries = (
|
||||
"apps/catalog/src/MapFixturePreview.tsx",
|
||||
"scripts/map-sector-grid.test.mjs",
|
||||
"server/map-grid-persistence.test.mjs",
|
||||
)
|
||||
candidate_files = {
|
||||
"apps/catalog/src/MapFixturePreview.tsx": "sector-current\n",
|
||||
"apps/catalog/src/CesiumMapRenderer.tsx": "renderer-next\n",
|
||||
"scripts/map-sector-grid.test.mjs": "sector-test-current\n",
|
||||
"scripts/map-grid-lod.test.mjs": "grid-test-next\n",
|
||||
"server/map-grid-persistence.test.mjs": "server-test-next\n",
|
||||
}
|
||||
installed_files = {
|
||||
"apps/catalog/src/MapFixturePreview.tsx": "sector-current\n",
|
||||
"apps/catalog/src/CesiumMapRenderer.tsx": "renderer-old\n",
|
||||
"scripts/map-sector-grid.test.mjs": "sector-test-current\n",
|
||||
"scripts/map-grid-lod.test.mjs": "grid-test-old\n",
|
||||
}
|
||||
for root, values in (
|
||||
(candidate, candidate_files),
|
||||
(installed, installed_files),
|
||||
):
|
||||
for rel, value in values.items():
|
||||
path = root / rel
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(value, encoding="utf-8")
|
||||
|
||||
failed_patch_id = "module-foundry-map-failed-test"
|
||||
failed_artifact_name = "module-foundry-map-failed-test.tgz.1"
|
||||
failed_backup_id = "module-foundry-map-failed-test-backup"
|
||||
recovery_patch_id = "module-foundry-map-recovery-test"
|
||||
failed_message = "test build failed"
|
||||
|
||||
backup = backups / failed_backup_id
|
||||
backup.mkdir()
|
||||
backup_names = {
|
||||
"existing-files.txt",
|
||||
"files.txt",
|
||||
"manifest.env",
|
||||
"missing-files.txt",
|
||||
"module-foundry-runtime-before.json",
|
||||
"source-before.tgz",
|
||||
}
|
||||
for name in backup_names:
|
||||
(backup / name).write_text(f"{name}\n", encoding="utf-8")
|
||||
backup_hashes = {
|
||||
name: hashlib.sha256((backup / name).read_bytes()).hexdigest()
|
||||
for name in backup_names
|
||||
}
|
||||
|
||||
archive_source = work / "archive-source"
|
||||
(archive_source / "payload").mkdir(parents=True)
|
||||
(archive_source / "manifest.env").write_text(
|
||||
"id="
|
||||
f"{failed_patch_id}\n"
|
||||
"component=module-foundry\n"
|
||||
"type=app-overlay\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
(archive_source / "files.txt").write_text(
|
||||
"\n".join(failed_entries) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
for rel in failed_entries:
|
||||
source = candidate / rel
|
||||
target = archive_source / "payload" / rel
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_bytes(source.read_bytes())
|
||||
failed_artifact = failed / failed_artifact_name
|
||||
with tarfile.open(failed_artifact, "w:gz") as archive:
|
||||
archive.add(
|
||||
archive_source / "manifest.env",
|
||||
arcname="manifest.env",
|
||||
)
|
||||
archive.add(
|
||||
archive_source / "files.txt",
|
||||
arcname="files.txt",
|
||||
)
|
||||
archive.add(archive_source / "payload", arcname="payload")
|
||||
failed_sha256 = hashlib.sha256(
|
||||
failed_artifact.read_bytes()
|
||||
).hexdigest()
|
||||
|
||||
journal = state / "failed.jsonl"
|
||||
journal.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"artifact": failed_artifact_name,
|
||||
"backup_id": failed_backup_id,
|
||||
"component": "module-foundry",
|
||||
"id": failed_patch_id,
|
||||
"message": failed_message,
|
||||
"rollback_status": "failed:DeployError",
|
||||
"sha256": failed_sha256,
|
||||
"started_apply": True,
|
||||
"status": "failed",
|
||||
}
|
||||
)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
manifest = {
|
||||
"id": recovery_patch_id,
|
||||
"component": "module-foundry",
|
||||
"type": "app-overlay",
|
||||
}
|
||||
candidate_tree = RUNNER.exact_file_map_sha256(
|
||||
RUNNER.collect_exact_files(
|
||||
candidate,
|
||||
recovery_entries,
|
||||
"candidate",
|
||||
)
|
||||
)
|
||||
installed_tree = RUNNER.exact_file_map_sha256(
|
||||
RUNNER.collect_exact_files(
|
||||
installed,
|
||||
recovery_entries[:-1],
|
||||
"installed",
|
||||
)
|
||||
)
|
||||
|
||||
patches = {
|
||||
"BACKUPS_DIR": backups,
|
||||
"FAILED_DIR": failed,
|
||||
"FAILED_STATE_FILE": journal,
|
||||
"TMP_DIR": tmp,
|
||||
"MODULE_FOUNDRY_MAP_RECOVERY_PATCH_ID": recovery_patch_id,
|
||||
"MODULE_FOUNDRY_MAP_RECOVERY_ENTRIES": recovery_entries,
|
||||
"MODULE_FOUNDRY_MAP_RECOVERY_MISSING_PREDECESSOR_ENTRIES": (
|
||||
missing_entries
|
||||
),
|
||||
"MODULE_FOUNDRY_MAP_FAILED_INSTALLED_ENTRIES": (
|
||||
failed_entries[:-1]
|
||||
),
|
||||
"MODULE_FOUNDRY_MAP_RECOVERY_INSTALLED_TREE_SHA256": (
|
||||
installed_tree
|
||||
),
|
||||
"MODULE_FOUNDRY_MAP_RECOVERY_CANDIDATE_TREE_SHA256": (
|
||||
candidate_tree
|
||||
),
|
||||
"MODULE_FOUNDRY_MAP_FAILED_ENTRIES": failed_entries,
|
||||
"MODULE_FOUNDRY_MAP_FAILED_PATCH_ID": failed_patch_id,
|
||||
"MODULE_FOUNDRY_MAP_FAILED_ARTIFACT": failed_artifact_name,
|
||||
"MODULE_FOUNDRY_MAP_FAILED_ARTIFACT_SHA256": failed_sha256,
|
||||
"MODULE_FOUNDRY_MAP_FAILED_BACKUP_ID": failed_backup_id,
|
||||
"MODULE_FOUNDRY_MAP_FAILED_MESSAGE": failed_message,
|
||||
"MODULE_FOUNDRY_MAP_FAILED_ROLLBACK_STATUS": (
|
||||
"failed:DeployError"
|
||||
),
|
||||
"MODULE_FOUNDRY_MAP_FAILED_BACKUP_SHA256": backup_hashes,
|
||||
}
|
||||
with ExitStack() as stack:
|
||||
for name, value in patches.items():
|
||||
stack.enter_context(mock.patch.object(RUNNER, name, value))
|
||||
stack.enter_context(
|
||||
mock.patch.object(
|
||||
RUNNER,
|
||||
"component_root",
|
||||
return_value=installed,
|
||||
)
|
||||
)
|
||||
stack.enter_context(
|
||||
mock.patch.object(
|
||||
RUNNER,
|
||||
"module_foundry_runtime_inventory",
|
||||
return_value=runtime(),
|
||||
)
|
||||
)
|
||||
result = RUNNER.validate_module_foundry_map_recovery_evidence(
|
||||
manifest,
|
||||
recovery_entries,
|
||||
candidate,
|
||||
)
|
||||
self.assertEqual(
|
||||
result["mode"],
|
||||
"failed-map-overlay-source+runtime-reconciliation",
|
||||
)
|
||||
(installed / failed_entries[0]).write_text(
|
||||
"drift\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
with self.assertRaises(RUNNER.DeployError):
|
||||
RUNNER.validate_module_foundry_map_recovery_evidence(
|
||||
manifest,
|
||||
recovery_entries,
|
||||
candidate,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main(verbosity=2)
|
||||
@@ -32,7 +32,7 @@ GET /api/map/reference-sources/v1/profiles/transport-stations.v1/search?q=<exact
|
||||
Reference-station route возвращает `nodedc.map-reference.snapshot/v1` и только
|
||||
два онтологических типа: `map.station` и `map.terminal`. Категории v1 —
|
||||
`metro`, `railway_station`, `railway_terminal`. Проекция allowlist-ит имена,
|
||||
operator/network, `uic_ref`, wheelchair и геометрию; raw OSM tags, upstream
|
||||
operator/network, локализованные/альтернативные имена, `uic_ref`, wheelchair и геометрию; raw OSM tags, upstream
|
||||
endpoint и provider credentials в snapshot отсутствуют.
|
||||
|
||||
Seed проверяется на старте и покрывает Москву без сетевого запроса. Для bbox за
|
||||
@@ -48,9 +48,11 @@ upstream не повреждает уже сохранённые cells; `complet
|
||||
upstream endpoint или payload.
|
||||
|
||||
Поиск вне уже загруженного viewport вызывается только явным Enter пользователя.
|
||||
Gateway сначала ищет в seed и persistent search index, затем при miss выполняет
|
||||
точный OSM name lookup. Upstream selector использует глобально индексированные
|
||||
`name`/`name:ru`, а найденные элементы повторно проходят fail-closed проверку
|
||||
Gateway при старте восстанавливает индекс из seed, persistent search index и
|
||||
всех сохранённых spatial cells. Локальное совпадение возвращается сразу и не
|
||||
ждёт Overpass; только полный miss выполняет точный OSM name lookup. Upstream
|
||||
selector использует глобально индексированные `name`, `name:ru`, `name:en`,
|
||||
`official_name` и `alt_name`, а найденные элементы повторно проходят fail-closed проверку
|
||||
`railway=station|halt`. Это не autocomplete и не глобальный regex scan.
|
||||
Нормализованный результат атомарно пополняет persistent search index; после
|
||||
перелёта обычный bbox-контур загружает полную spatial cell. Одинаковые запросы
|
||||
@@ -68,7 +70,7 @@ search counters.
|
||||
|
||||
`/api/map/ion/assets/:assetId/endpoint` разрешает только `CESIUM_ION_ASSET_ALLOWLIST`. Ответ не содержит credential: Browser использует публичный provider URL вместе с Cesium `DefaultProxy`, направляющим resource requests в `/api/map/cache`. Gateway валидирует scope URL, удаляет любые credential query parameters из browser request и добавляет соответствующий server-side credential только перед обращением к provider.
|
||||
|
||||
Ion endpoint metadata с asset credential сохраняется только в private persistent storage с файловыми правами `0600`. Это нужно для cold start и offline: Browser по-прежнему получает только sanitised URL, а Gateway отвечает из ранее записанного cache. В metadata никогда не записывается master token; NAS backup этого каталога считается service-sensitive.
|
||||
Ion endpoint metadata с asset credential сохраняется только в private persistent storage с файловыми правами `0600`. Это нужно для cold start и offline: Browser по-прежнему получает только sanitised URL, а Gateway отвечает из ранее записанного cache. Если asset credential истёк, а upstream/VPN недоступен, Gateway всё равно возвращает сохранённый публичный URL без credential: cache lookup выполняется раньше credential injection, поэтому warm `layer.json`, `tileset.json` и дочерние объекты остаются доступны, а настоящий miss по-прежнему fails closed. В metadata никогда не записывается master token; NAS backup этого каталога считается service-sensitive.
|
||||
|
||||
## Cache modes
|
||||
|
||||
@@ -80,6 +82,13 @@ Ion endpoint metadata с asset credential сохраняется только в
|
||||
|
||||
Первый cold miss начинает стримиться клиенту сразу после provider headers и одновременно записывается в private temp-файл. Только полностью полученный object публикуется через atomic rename и durable `index.json`; partial/error никогда не становится cache entry. Одинаковые параллельные misses делят один upstream download, а отмена одного browser request не прерывает server-owned fill для других инстансов. Warm hit не меняет index. Параллельные новые объекты объединяются в минимальное число полных index snapshots вместо одного O(N) rewrite на каждый tile.
|
||||
|
||||
Новые index entries получают безопасные `resourceKind`/`assetId` без URL и
|
||||
credentials. `/healthz` агрегирует число и размер imagery, terrain, buildings,
|
||||
metadata и legacy 3D/binary объектов. Старые hash-only entries классифицируются
|
||||
только по надёжному content type; `application/octet-stream` честно остаётся
|
||||
`3d-binary`, потому что ретроспективно приписать его конкретному Ion asset без
|
||||
сохранённого URL невозможно.
|
||||
|
||||
`offline` дополнительно требует, чтобы hostname был явно указан в `MAP_GATEWAY_OFFLINE_PROVIDER_ALLOWLIST`. По умолчанию список пуст: Gateway не превращает Cesium ion или public OSM tile service в offline distribution. Это осознанная provider policy, а не техническая ошибка cache.
|
||||
|
||||
Range requests пока transparently проксируются и не записываются как partial cache object. Полные GET responses кэшируются. Это безопасная начальная граница для terrain/3D Tiles; отдельным следующим шагом добавляется range-aware chunk cache после замеров реальных GOS/3D Tiles assets.
|
||||
|
||||
@@ -63,12 +63,13 @@ function normalizeFeature(feature, generatedAt, index) {
|
||||
const identity = nativeId.match(/^(node|way|relation)\/([1-9]\d*)$/);
|
||||
if (!identity) throw new Error(`station_seed_feature_${index}_identity_invalid`);
|
||||
const attributes = compact({
|
||||
name: firstString(properties.name, properties["name:ru"], properties.official_name, properties.loc_name),
|
||||
name: firstString(properties["name:ru"], properties.name, properties.official_name, properties.loc_name),
|
||||
category,
|
||||
network: optionalString(properties.network),
|
||||
operator: optionalString(properties.operator),
|
||||
official_name: optionalString(properties.official_name),
|
||||
local_name: optionalString(properties.loc_name),
|
||||
alternate_names: alternateNames(properties),
|
||||
uic_ref: optionalString(properties.uic_ref),
|
||||
wheelchair: optionalString(properties.wheelchair),
|
||||
});
|
||||
@@ -101,6 +102,28 @@ function optionalString(value) {
|
||||
return normalized && normalized.length <= 256 ? normalized : undefined;
|
||||
}
|
||||
|
||||
function alternateNames(properties) {
|
||||
const primary = firstString(properties["name:ru"], properties.name, properties.official_name, properties.loc_name);
|
||||
const unique = new Map();
|
||||
for (const value of [
|
||||
properties.name,
|
||||
properties["name:ru"],
|
||||
properties["name:en"],
|
||||
properties.official_name,
|
||||
properties.loc_name,
|
||||
properties.short_name,
|
||||
properties.old_name,
|
||||
...String(properties.alt_name || "").split(";"),
|
||||
]) {
|
||||
const normalized = optionalString(value);
|
||||
if (!normalized || normalized.toLocaleLowerCase("ru") === primary?.toLocaleLowerCase("ru")) continue;
|
||||
const key = normalized.toLocaleLowerCase("ru");
|
||||
if (!unique.has(key)) unique.set(key, normalized);
|
||||
}
|
||||
const values = [...unique.values()].slice(0, 16);
|
||||
return values.length ? values : undefined;
|
||||
}
|
||||
|
||||
function compact(value) {
|
||||
return Object.fromEntries(Object.entries(value).filter(([, item]) => item !== undefined));
|
||||
}
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { createHash } from "node:crypto";
|
||||
import { once } from "node:events";
|
||||
import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises";
|
||||
import { createServer } from "node:net";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { dirname, join } from "node:path";
|
||||
import { spawn } from "node:child_process";
|
||||
|
||||
const root = await mkdtemp(join(tmpdir(), "nodedc-map-gateway-credential-smoke-"));
|
||||
@@ -13,7 +14,27 @@ let child;
|
||||
|
||||
try {
|
||||
await mkdir(join(cacheDir, "ion-endpoints"), { recursive: true });
|
||||
await writeFile(join(cacheDir, "index.json"), `${JSON.stringify({ version: 1, entries: {} })}\n`);
|
||||
const buildingsUrl = "https://assets.ion.cesium.com/us-east-1/asset_depot/96188/OpenStreetMap/CWT/2025-04-01/tileset.json?v=expired";
|
||||
const buildingsKey = createHash("sha256").update(buildingsUrl).digest("hex");
|
||||
const buildingsRelativePath = join("objects", buildingsKey.slice(0, 2), `${buildingsKey}.bin`);
|
||||
const buildingsBody = JSON.stringify({ asset: { version: "1.1" }, root: { geometricError: 0 } });
|
||||
await mkdir(dirname(join(cacheDir, buildingsRelativePath)), { recursive: true });
|
||||
await writeFile(join(cacheDir, buildingsRelativePath), buildingsBody);
|
||||
await writeFile(join(cacheDir, "index.json"), `${JSON.stringify({
|
||||
version: 1,
|
||||
entries: {
|
||||
[buildingsKey]: {
|
||||
key: buildingsKey,
|
||||
file: buildingsRelativePath,
|
||||
bytes: Buffer.byteLength(buildingsBody),
|
||||
contentType: "application/json",
|
||||
etag: null,
|
||||
savedAt: Date.now(),
|
||||
lastAccessAt: Date.now(),
|
||||
expiresAt: Date.now() - 1,
|
||||
},
|
||||
},
|
||||
})}\n`);
|
||||
await writeFile(join(cacheDir, "ion-endpoints", "1.json"), `${JSON.stringify({
|
||||
assetId: "1",
|
||||
type: "TERRAIN",
|
||||
@@ -38,7 +59,7 @@ try {
|
||||
await writeFile(join(cacheDir, "ion-endpoints", "96188.json"), `${JSON.stringify({
|
||||
assetId: "96188",
|
||||
type: "3DTILES",
|
||||
url: "https://assets.ion.cesium.com/us-east-1/asset_depot/96188/OpenStreetMap/CWT/2025-04-01/tileset.json?v=expired",
|
||||
url: buildingsUrl,
|
||||
accessToken: expiredBuildingsToken,
|
||||
attributions: [],
|
||||
savedAt: Date.now(),
|
||||
@@ -48,11 +69,13 @@ try {
|
||||
cwd: new URL("..", import.meta.url),
|
||||
env: {
|
||||
...process.env,
|
||||
NODE_ENV: "test",
|
||||
PORT: String(port),
|
||||
MAP_CACHE_DIR: cacheDir,
|
||||
MAP_GATEWAY_ALLOW_ANONYMOUS: "true",
|
||||
MAP_CACHE_MODE: "readwrite",
|
||||
CESIUM_ION_TOKEN: "",
|
||||
CESIUM_ION_TOKEN: "configured-master-token",
|
||||
CESIUM_ION_API_BASE_URL: `http://127.0.0.1:${port}/`,
|
||||
CESIUM_ION_ASSET_ALLOWLIST: "1,2,96188",
|
||||
MAP_GATEWAY_UPSTREAM_ALLOWLIST: "assets.ion.cesium.com,dev.virtualearth.net",
|
||||
},
|
||||
@@ -73,10 +96,16 @@ try {
|
||||
}
|
||||
const expired = await fetch(`http://127.0.0.1:${port}/api/map/ion/assets/96188/endpoint`);
|
||||
const expiredRaw = await expired.text();
|
||||
assert.equal(expired.status, 503);
|
||||
assert.equal(expired.status, 200);
|
||||
assert.equal(expiredRaw.includes(expiredBuildingsToken), false);
|
||||
assert.equal(expiredRaw.includes("accessToken"), false);
|
||||
console.log("ok: endpoint responses contain no credentials and a known-expired Ion JWT is never served");
|
||||
assert.equal(JSON.parse(expiredRaw).cache, "ion-endpoint-stale-upstream-error");
|
||||
|
||||
const cachedRoot = await fetch(`http://127.0.0.1:${port}/api/map/cache?url=${encodeURIComponent(`${buildingsUrl}&nodedc_client_revision=2`)}`);
|
||||
assert.equal(cachedRoot.status, 200);
|
||||
assert.equal(cachedRoot.headers.get("x-nodedc-map-cache"), "live-cache-stale");
|
||||
assert.deepEqual(await cachedRoot.json(), JSON.parse(buildingsBody));
|
||||
console.log("ok: endpoint responses contain no credentials and expired metadata still unlocks warm cache during upstream outage");
|
||||
} finally {
|
||||
if (child && !child.killed) {
|
||||
child.kill("SIGTERM");
|
||||
|
||||
@@ -40,6 +40,10 @@ export async function createReferenceStationSource({
|
||||
const recentSearches = new Map();
|
||||
const indexedFacts = new Map(seed.facts.map((fact) => [fact.sourceId, fact]));
|
||||
for (const fact of await readSearchIndex(searchIndexFile)) indexedFacts.set(fact.sourceId, fact);
|
||||
// Cells are the durable record of everything the user has already viewed.
|
||||
// Rehydrate them on restart so a warm viewport remains searchable without
|
||||
// waiting for Overpass or revisiting the same coordinates first.
|
||||
for (const fact of await readCachedCellFacts(cellRoot)) indexedFacts.set(fact.sourceId, fact);
|
||||
const pendingFetches = [];
|
||||
let activeFetches = 0;
|
||||
let upstreamRequests = 0;
|
||||
@@ -51,6 +55,16 @@ export async function createReferenceStationSource({
|
||||
let searchFailures = 0;
|
||||
let upstreamStartQueue = Promise.resolve();
|
||||
let nextUpstreamStartAt = 0;
|
||||
let searchIndexWriteQueue = Promise.resolve();
|
||||
|
||||
function persistIndexedFacts() {
|
||||
// Serialize complete snapshots. Concurrent viewport-cell fills may finish
|
||||
// in either order; taking the values inside the queued task prevents a
|
||||
// later write from dropping facts indexed by an earlier completion.
|
||||
const task = searchIndexWriteQueue.then(() => writeSearchIndex(searchIndexFile, [...indexedFacts.values()]));
|
||||
searchIndexWriteQueue = task.catch(() => undefined);
|
||||
return task;
|
||||
}
|
||||
|
||||
async function snapshot({ bbox } = {}) {
|
||||
const normalizedBbox = bbox ? normalizeBbox(bbox) : null;
|
||||
@@ -109,8 +123,11 @@ export async function createReferenceStationSource({
|
||||
const normalizedLimit = Math.max(1, Math.min(MAX_SEARCH_RESULTS, Number(limit) || 12));
|
||||
const localFacts = searchIndexedFacts(indexedFacts.values(), normalizedQuery, normalizedLimit);
|
||||
let upstreamFacts = [];
|
||||
let complete = !fetchEnabled;
|
||||
if (fetchEnabled && normalizedQuery.length >= 3) {
|
||||
// Cache/seed hits are terminal. The previous implementation still waited
|
||||
// for a global Overpass request and turned an instant local result into a
|
||||
// 30-second error whenever the public service was degraded.
|
||||
let complete = localFacts.length > 0 || !fetchEnabled;
|
||||
if (fetchEnabled && localFacts.length === 0 && normalizedQuery.length >= 3) {
|
||||
const cachedSearch = recentSearches.get(normalizedQuery);
|
||||
if (cachedSearch && Date.now() - cachedSearch.storedAt < SEARCH_CACHE_TTL_MS) {
|
||||
upstreamFacts = cachedSearch.facts;
|
||||
@@ -123,7 +140,7 @@ export async function createReferenceStationSource({
|
||||
try {
|
||||
const facts = await fetchSearch(endpoint, displayQuery, normalizedLimit, normalizedTimeoutMs, fetchImpl);
|
||||
indexFacts(facts);
|
||||
await writeSearchIndex(searchIndexFile, [...indexedFacts.values()]);
|
||||
await persistIndexedFacts();
|
||||
recentSearches.set(normalizedQuery, { storedAt: Date.now(), facts });
|
||||
lastRefreshAt = new Date().toISOString();
|
||||
return { facts, complete: true };
|
||||
@@ -179,6 +196,7 @@ export async function createReferenceStationSource({
|
||||
try {
|
||||
const document = await fetchCell(endpoint, cell, normalizedTimeoutMs, fetchImpl);
|
||||
await writeCell(cellRoot, cell.key, document);
|
||||
indexFacts(document.facts);
|
||||
lastRefreshAt = document.generatedAt;
|
||||
return document;
|
||||
} catch (error) {
|
||||
@@ -239,6 +257,7 @@ export async function createReferenceStationSource({
|
||||
fetchEnabled,
|
||||
cellDegrees: normalizedCellDegrees,
|
||||
cachedCellCount,
|
||||
indexedFactCount: indexedFacts.size,
|
||||
upstreamRequests,
|
||||
upstreamFailures,
|
||||
searchRequests,
|
||||
@@ -283,10 +302,16 @@ function validateFact(value) {
|
||||
|| !Number.isFinite(coordinates[1]) || coordinates[1] < -90 || coordinates[1] > 90) {
|
||||
throw sourceError("reference_station_geometry_invalid");
|
||||
}
|
||||
const allowedAttributes = new Set(["name", "category", "network", "operator", "official_name", "local_name", "uic_ref", "wheelchair"]);
|
||||
const allowedAttributes = new Set(["name", "category", "network", "operator", "official_name", "local_name", "alternate_names", "uic_ref", "wheelchair"]);
|
||||
if (Object.keys(value.attributes).some((key) => !allowedAttributes.has(key))) {
|
||||
throw sourceError("reference_station_attribute_not_allowed");
|
||||
}
|
||||
if (value.attributes.alternate_names !== undefined
|
||||
&& (!Array.isArray(value.attributes.alternate_names)
|
||||
|| value.attributes.alternate_names.length > 16
|
||||
|| value.attributes.alternate_names.some((name) => !optionalString(name)))) {
|
||||
throw sourceError("reference_station_alternate_names_invalid");
|
||||
}
|
||||
if (value.semanticType === "map.terminal" !== (value.attributes.category === "railway_terminal")) {
|
||||
throw sourceError("reference_station_semantic_type_mismatch");
|
||||
}
|
||||
@@ -342,7 +367,7 @@ async function fetchSearch(endpoint, query, limit, timeoutMs, fetchImpl) {
|
||||
// whole planet by railway first is several orders of magnitude slower and
|
||||
// routinely times out; station semantics are therefore verified locally
|
||||
// by the same fail-closed normalizer used by viewport cells.
|
||||
const overpassQuery = `[out:json][timeout:25];(nwr["name"="${expression}"];nwr["name:ru"="${expression}"];);out center tags ${resultLimit};`;
|
||||
const overpassQuery = `[out:json][timeout:25];(nwr["name"="${expression}"];nwr["name:ru"="${expression}"];nwr["name:en"="${expression}"];nwr["official_name"="${expression}"];nwr["alt_name"="${expression}"];);out center tags ${resultLimit};`;
|
||||
const response = await fetchImpl(endpoint, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
@@ -388,6 +413,16 @@ function normalizeOverpassElement(value, observedAt) {
|
||||
const name = firstString(value.tags["name:ru"], value.tags.name, value.tags.official_name, value.tags.loc_name);
|
||||
if (!name) throw sourceError("reference_station_upstream_name_required");
|
||||
const category = classifyStation(value.tags, name);
|
||||
const alternateNames = uniqueStrings(
|
||||
value.tags.name,
|
||||
value.tags["name:ru"],
|
||||
value.tags["name:en"],
|
||||
value.tags.official_name,
|
||||
value.tags.loc_name,
|
||||
value.tags.short_name,
|
||||
value.tags.old_name,
|
||||
...String(value.tags.alt_name || "").split(";"),
|
||||
).filter((candidate) => normalizeSearchValue(candidate) !== normalizeSearchValue(name)).slice(0, 16);
|
||||
return validateFact({
|
||||
sourceId: `osm.${value.type}.${value.id}`,
|
||||
semanticType: category === "railway_terminal" ? "map.terminal" : "map.station",
|
||||
@@ -400,6 +435,7 @@ function normalizeOverpassElement(value, observedAt) {
|
||||
operator: optionalString(value.tags.operator),
|
||||
official_name: optionalString(value.tags.official_name),
|
||||
local_name: optionalString(value.tags.loc_name),
|
||||
alternate_names: alternateNames.length ? alternateNames : undefined,
|
||||
uic_ref: optionalString(value.tags.uic_ref),
|
||||
wheelchair: optionalString(value.tags.wheelchair),
|
||||
}),
|
||||
@@ -484,6 +520,7 @@ function searchIndexedFacts(values, query, limit) {
|
||||
fact.attributes.name,
|
||||
fact.attributes.official_name,
|
||||
fact.attributes.local_name,
|
||||
...(fact.attributes.alternate_names ?? []),
|
||||
fact.attributes.uic_ref,
|
||||
], query);
|
||||
return rank === null ? [] : [{ fact, rank }];
|
||||
@@ -562,6 +599,24 @@ async function readSearchIndex(file) {
|
||||
}
|
||||
}
|
||||
|
||||
async function readCachedCellFacts(root) {
|
||||
let names;
|
||||
try {
|
||||
names = (await readdir(root)).filter((name) => name.endsWith(".json")).sort();
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
const facts = new Map();
|
||||
for (const name of names) {
|
||||
const cell = await readCell(root, name.slice(0, -5));
|
||||
for (const fact of cell?.facts ?? []) {
|
||||
facts.set(fact.sourceId, fact);
|
||||
if (facts.size >= MAX_FACTS) return [...facts.values()];
|
||||
}
|
||||
}
|
||||
return [...facts.values()];
|
||||
}
|
||||
|
||||
async function writeSearchIndex(file, facts) {
|
||||
const normalizedFacts = deduplicateFacts(facts).slice(-MAX_FACTS);
|
||||
const document = {
|
||||
@@ -618,6 +673,17 @@ function optionalString(value) {
|
||||
return normalized && normalized.length <= 256 ? normalized : undefined;
|
||||
}
|
||||
|
||||
function uniqueStrings(...values) {
|
||||
const unique = new Map();
|
||||
for (const value of values) {
|
||||
const normalized = optionalString(value);
|
||||
if (!normalized) continue;
|
||||
const key = normalizeSearchValue(normalized);
|
||||
if (!unique.has(key)) unique.set(key, normalized);
|
||||
}
|
||||
return [...unique.values()];
|
||||
}
|
||||
|
||||
function compact(value) {
|
||||
return Object.fromEntries(Object.entries(value).filter(([, item]) => item !== undefined));
|
||||
}
|
||||
|
||||
@@ -114,6 +114,7 @@ const server = createServer(async (request, response) => {
|
||||
referenceSources: {
|
||||
transportStations: await referenceStationSource.status(),
|
||||
},
|
||||
providerCache: await ionProviderCacheStatus(),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -592,15 +593,18 @@ async function serveIonEndpoint(request, response, assetId) {
|
||||
const cachedState = cached ? ionEndpointCacheState(cached) : null;
|
||||
if (config.mode === "offline") {
|
||||
if (!cached) return writeJson(response, 504, { ok: false, error: "cesium_ion_offline_endpoint_miss" });
|
||||
if (!cachedState.usable) return writeJson(response, 504, { ok: false, error: "cesium_ion_offline_endpoint_expired" });
|
||||
if (!isOfflineProviderAllowed(ionEndpointUrl(cached))) return writeJson(response, 409, { ok: false, error: "map_provider_offline_not_permitted" });
|
||||
rememberIonEndpoint(cached);
|
||||
return writeJson(response, 200, { ok: true, ...publicIonEndpoint(cached), cache: "offline-endpoint-hit" });
|
||||
// The browser receives only a sanitised provider URL. An expired private
|
||||
// asset credential is irrelevant for a cache-only request because cache
|
||||
// lookup happens before credential injection and an offline miss never
|
||||
// reaches upstream.
|
||||
return writeJson(response, 200, { ok: true, ...publicIonEndpoint(cached), cache: "offline-public-endpoint-hit" });
|
||||
}
|
||||
if (!activeCesiumIonToken) {
|
||||
if (cached && cachedState.usable) {
|
||||
if (cached) {
|
||||
rememberIonEndpoint(cached);
|
||||
return writeJson(response, 200, { ok: true, ...publicIonEndpoint(cached), cache: "cached-endpoint-no-master-token" });
|
||||
return writeJson(response, 200, { ok: true, ...publicIonEndpoint(cached), cache: "cached-public-endpoint-no-master-token" });
|
||||
}
|
||||
return writeJson(response, 503, { ok: false, error: "cesium_ion_not_configured" });
|
||||
}
|
||||
@@ -616,8 +620,20 @@ async function serveIonEndpoint(request, response, assetId) {
|
||||
});
|
||||
}
|
||||
|
||||
const next = await refreshIonEndpoint(assetId, ionReferer);
|
||||
return writeJson(response, 200, { ok: true, ...publicIonEndpoint(next), cache: "ion-endpoint-online" });
|
||||
try {
|
||||
const next = await refreshIonEndpoint(assetId, ionReferer);
|
||||
return writeJson(response, 200, { ok: true, ...publicIonEndpoint(next), cache: "ion-endpoint-online" });
|
||||
} catch (error) {
|
||||
if (!cached) throw error;
|
||||
// Endpoint credentials may expire while the AMD/VPN route is offline.
|
||||
// Returning the cached *public* URL lets Cesium request layer.json or
|
||||
// tileset.json and consume every warm object. A real miss still reaches
|
||||
// injectGatewayCredentials(), attempts refresh and fails closed.
|
||||
rememberIonEndpoint(cached);
|
||||
const errorCode = safeDiagnosticCode(error?.message || "cesium_ion_endpoint_refresh_failed");
|
||||
console.warn(JSON.stringify({ event: "map_ion_endpoint_public_fallback", assetId, error: errorCode }));
|
||||
return writeJson(response, 200, { ok: true, ...publicIonEndpoint(cached), cache: "ion-endpoint-stale-upstream-error" });
|
||||
}
|
||||
}
|
||||
|
||||
function scheduleIonEndpointRefresh(assetId, ionReferer = "") {
|
||||
@@ -882,11 +898,11 @@ async function prepareStreamingCacheFill(target, cacheKey, store, ionReferer = "
|
||||
}
|
||||
|
||||
const [clientBody, cacheBody] = upstream.body.tee();
|
||||
const cacheCommit = commitCacheBody(cacheBody, upstream.headers, cacheKey, store);
|
||||
const cacheCommit = commitCacheBody(cacheBody, upstream.headers, cacheKey, store, cacheResourceIdentity(target));
|
||||
return { status: upstream.status, headers: upstream.headers, clientBody, cacheCommit };
|
||||
}
|
||||
|
||||
async function commitCacheBody(cacheBody, upstreamHeaders, cacheKey, store) {
|
||||
async function commitCacheBody(cacheBody, upstreamHeaders, cacheKey, store, resourceIdentity = null) {
|
||||
const relativePath = join("objects", cacheKey.slice(0, 2), `${cacheKey}.bin`);
|
||||
const filePath = join(store.dir, relativePath);
|
||||
const tempPath = `${filePath}.${process.pid}.${Date.now()}.tmp`;
|
||||
@@ -916,6 +932,8 @@ async function commitCacheBody(cacheBody, upstreamHeaders, cacheKey, store) {
|
||||
savedAt: now,
|
||||
lastAccessAt: now,
|
||||
expiresAt: now + responseTtl(upstreamHeaders.get("cache-control")),
|
||||
...(resourceIdentity?.resourceKind ? { resourceKind: resourceIdentity.resourceKind } : {}),
|
||||
...(resourceIdentity?.assetId ? { assetId: resourceIdentity.assetId } : {}),
|
||||
};
|
||||
store.index.entries[cacheKey] = entry;
|
||||
await releaseCacheCapacity(store, bytes);
|
||||
@@ -1317,6 +1335,14 @@ function isCacheCapacityError(error) {
|
||||
async function cacheStats(store) {
|
||||
const entries = Object.values(store.index.entries);
|
||||
const bytes = cacheBytes(store);
|
||||
const byResourceKind = {};
|
||||
for (const entry of entries) {
|
||||
const resourceKind = cacheEntryResourceKind(entry);
|
||||
const aggregate = byResourceKind[resourceKind] ?? { entries: 0, bytes: 0 };
|
||||
aggregate.entries += 1;
|
||||
aggregate.bytes += Number(entry?.bytes || 0);
|
||||
byResourceKind[resourceKind] = aggregate;
|
||||
}
|
||||
return {
|
||||
name: store.name,
|
||||
mode: store.mutable ? config.mode : "readonly",
|
||||
@@ -1326,9 +1352,65 @@ async function cacheStats(store) {
|
||||
maxBytes: store.mutable ? config.maxCacheBytes : null,
|
||||
atCapacity: store.mutable ? bytes >= config.maxCacheBytes : false,
|
||||
persistent: true,
|
||||
byResourceKind,
|
||||
};
|
||||
}
|
||||
|
||||
function cacheResourceIdentity(rawTarget) {
|
||||
let target;
|
||||
try {
|
||||
target = stripCredentialQueryParameters(new URL(String(rawTarget)));
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (/(^|\.)virtualearth\.net$/i.test(target.hostname) || target.hostname === "tile.openstreetmap.org") {
|
||||
return { resourceKind: "imagery", assetId: target.hostname === "tile.openstreetmap.org" ? "osm" : "2" };
|
||||
}
|
||||
const matched = [...activeIonEndpoints.values()]
|
||||
.filter((endpoint) => endpoint.externalType !== "BING" && endpoint.url)
|
||||
.map((endpoint) => ({ endpoint, scopeLength: endpointResourceScopeLength(target, endpoint.url) }))
|
||||
.filter(({ scopeLength }) => scopeLength >= 0)
|
||||
.sort((left, right) => right.scopeLength - left.scopeLength)[0]?.endpoint;
|
||||
if (!matched) return null;
|
||||
const assetId = String(matched.assetId);
|
||||
const resourceKind = assetId === "1" ? "terrain" : assetId === "96188" ? "buildings" : "ion-other";
|
||||
return { resourceKind, assetId };
|
||||
}
|
||||
|
||||
function cacheEntryResourceKind(entry) {
|
||||
if (["imagery", "terrain", "buildings", "ion-other"].includes(entry?.resourceKind)) return entry.resourceKind;
|
||||
const contentType = String(entry?.contentType || "").toLowerCase();
|
||||
if (contentType.startsWith("image/")) return "imagery";
|
||||
if (contentType.includes("quantized-mesh")) return "terrain";
|
||||
// Old runtime entries predate safe provider provenance. With no stored URL
|
||||
// an octet-stream cannot honestly be attributed to one asset, so expose it
|
||||
// as legacy 3D/binary rather than pretending every object is an OSM house.
|
||||
if (contentType.includes("application/octet-stream")) return "3d-binary";
|
||||
if (contentType.includes("json") || contentType.startsWith("text/")) return "metadata";
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
async function ionProviderCacheStatus() {
|
||||
return Promise.all(canonicalCesiumAssetIds.map(async (assetId) => {
|
||||
const endpoint = await readIonEndpointCache(assetId);
|
||||
if (!endpoint) return { assetId: Number(assetId), cachedEndpoint: false, credentialUsable: false, entryPointCached: false };
|
||||
const candidateUrls = assetId === "1"
|
||||
? [new URL("layer.json", endpoint.url).toString()]
|
||||
: assetId === "96188" ? [endpoint.url] : [];
|
||||
const entryPointCached = candidateUrls.some((url) => {
|
||||
const key = createHash("sha256").update(canonicalCacheUrl(new URL(url))).digest("hex");
|
||||
return Boolean(liveCache.index.entries[key]);
|
||||
});
|
||||
return {
|
||||
assetId: Number(assetId),
|
||||
type: endpoint.type,
|
||||
cachedEndpoint: true,
|
||||
credentialUsable: ionEndpointCacheState(endpoint).usable,
|
||||
entryPointCached,
|
||||
};
|
||||
}));
|
||||
}
|
||||
|
||||
function shouldUseCesiumEgress(target) {
|
||||
if (!config.mapEgress.url) return false;
|
||||
try {
|
||||
|
||||
@@ -81,6 +81,61 @@ test("bbox fetch uses one cached spatial cell and never exposes upstream payload
|
||||
assert.equal(cached.schemaVersion, "nodedc.map-reference-cell/v1");
|
||||
});
|
||||
|
||||
test("seed and viewed cells resolve locally without waiting for global search", async () => {
|
||||
const cacheDir = await mkdtemp(join(tmpdir(), "nodedc-reference-stations-"));
|
||||
let calls = 0;
|
||||
const source = await createReferenceStationSource({
|
||||
seedFile,
|
||||
cacheDir,
|
||||
cellDegrees: 1,
|
||||
fetchImpl: async () => {
|
||||
calls += 1;
|
||||
return new Response(JSON.stringify({
|
||||
elements: [{
|
||||
type: "node",
|
||||
id: 987654399,
|
||||
lat: 59.93,
|
||||
lon: 30.31,
|
||||
tags: {
|
||||
railway: "station",
|
||||
station: "subway",
|
||||
name: "Petrogradskaya",
|
||||
"name:ru": "Петроградская",
|
||||
"name:en": "Petrogradskaya station",
|
||||
},
|
||||
}],
|
||||
}), { status: 200, headers: { "content-type": "application/json" } });
|
||||
},
|
||||
});
|
||||
|
||||
const seedSnapshot = await source.snapshot();
|
||||
const seedName = seedSnapshot.facts[0].attributes.name;
|
||||
const seedSearch = await source.search({ query: seedName, limit: 8 });
|
||||
assert.equal(seedSearch.complete, true);
|
||||
assert.ok(seedSearch.facts.length > 0);
|
||||
assert.equal(calls, 0);
|
||||
|
||||
await source.snapshot({ bbox: [30, 59, 31, 60] });
|
||||
const viewedSearch = await source.search({ query: "Petrogradskaya station", limit: 8 });
|
||||
assert.equal(viewedSearch.complete, true);
|
||||
assert.equal(viewedSearch.facts[0].sourceId, "osm.node.987654399");
|
||||
assert.equal(calls, 1);
|
||||
|
||||
let restartedCalls = 0;
|
||||
const restarted = await createReferenceStationSource({
|
||||
seedFile,
|
||||
cacheDir,
|
||||
fetchImpl: async () => {
|
||||
restartedCalls += 1;
|
||||
throw new Error("warm viewed cell must not call upstream");
|
||||
},
|
||||
});
|
||||
const restartedSearch = await restarted.search({ query: "Petrogradskaya station", limit: 8 });
|
||||
assert.equal(restartedSearch.complete, true);
|
||||
assert.equal(restartedSearch.facts[0].sourceId, "osm.node.987654399");
|
||||
assert.equal(restartedCalls, 0);
|
||||
});
|
||||
|
||||
test("viewport cells use bounded concurrency and deduplicate inflight fetches", async () => {
|
||||
const cacheDir = await mkdtemp(join(tmpdir(), "nodedc-reference-stations-"));
|
||||
let active = 0;
|
||||
|
||||
Reference in New Issue
Block a user