From 4b24c2e4570c26e2af3f91bc418cdfe9c10d57c6 Mon Sep 17 00:00:00 2001 From: Codex Date: Sun, 9 Aug 2026 09:05:09 +0300 Subject: [PATCH] FIX - NAS DEPLOY: reconcile Foundry map runtime generation --- ...-foundry-map-runtime-recovery-artifact.mjs | 304 ++++++++++++++++++ infra/deploy-runner/nodedc-deploy | 252 ++++++++++----- ...e_foundry_map_runtime_recovery_artifact.py | 174 ++++++++++ .../test_module_foundry_runtime_recovery.py | 175 +++++++--- 4 files changed, 768 insertions(+), 137 deletions(-) create mode 100644 infra/deploy-runner/build-module-foundry-map-runtime-recovery-artifact.mjs create mode 100644 infra/deploy-runner/test_module_foundry_map_runtime_recovery_artifact.py diff --git a/infra/deploy-runner/build-module-foundry-map-runtime-recovery-artifact.mjs b/infra/deploy-runner/build-module-foundry-map-runtime-recovery-artifact.mjs new file mode 100644 index 0000000..01fcafd --- /dev/null +++ b/infra/deploy-runner/build-module-foundry-map-runtime-recovery-artifact.mjs @@ -0,0 +1,304 @@ +#!/usr/bin/env node + +import { createHash } from "node:crypto"; +import { spawnSync } from "node:child_process"; +import { + cp, + lstat, + mkdir, + mkdtemp, + readdir, + readFile, + rm, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join, relative, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const scriptDir = dirname(fileURLToPath(import.meta.url)); +const declaredFoundryRoot = process.env.NODEDC_FOUNDRY_SOURCE_ROOT; +if (!declaredFoundryRoot) { + throw new Error("NODEDC_FOUNDRY_SOURCE_ROOT_is_required"); +} +const foundryRoot = resolve(declaredFoundryRoot); +const expectedSourceCommit = "58800d957632320fa6717ec3fcff972023528759"; +const artifactDir = resolve( + process.env.NODEDC_DEPLOY_ARTIFACT_DIR + || resolve(scriptDir, "../deploy-artifacts"), +); +const [ + patchId = "module-foundry-map-runtime-recovery-20260809-002", + ...extra +] = process.argv.slice(2); + +if ( + extra.length + || !/^module-foundry-map-runtime-recovery-\d{8}-\d{3}$/.test(patchId) +) { + throw new Error( + "usage: build-module-foundry-map-runtime-recovery-artifact.mjs " + + "[module-foundry-map-runtime-recovery-YYYYMMDD-NNN]", + ); +} + +const files = Object.freeze([ + "package.json", + "package-lock.json", + "apps/catalog/src", + "packages", + "registry", + "runtime-seed/page-layouts/map.json", + "scripts", + "server/catalog-server.mjs", + "server/foundry-mcp.mjs", + "server/map-grid-persistence.test.mjs", +]); +const ignoredBasenames = new Set([ + ".DS_Store", + ".git", + "node_modules", + "runtime-data", + "dist", +]); +const artifact = join(artifactDir, `nodedc-${patchId}.tgz`); +const checksum = `${artifact}.sha256`; +const stage = await mkdtemp( + join(tmpdir(), "nodedc-foundry-map-runtime-recovery-"), +); + +await assertFresh(artifact); +await assertExactSource(); +await assertRecoveryBoundary(); + +try { + for (const sourceRelative of files) { + await copySafe( + resolve(foundryRoot, sourceRelative), + join(stage, "payload", sourceRelative), + ); + } + await writeFile( + join(stage, "manifest.env"), + `id=${patchId}\ncomponent=module-foundry\ntype=app-overlay\n`, + "utf8", + ); + await writeFile(join(stage, "files.txt"), `${files.join("\n")}\n`, "utf8"); + await mkdir(artifactDir, { recursive: true }); + run("python3", ["-c", canonicalTarScript(), artifact, stage]); + const sha256 = digest(await readFile(artifact)); + await writeFile( + checksum, + `${sha256} ${artifact.split("/").at(-1)}\n`, + "utf8", + ); + console.log(JSON.stringify({ + ok: true, + patchId, + artifact, + checksum, + sha256, + sourceCommit: expectedSourceCommit, + services: ["nodedc-module-foundry"], + transition: "reconcile-partial-map-source-and-activate-runtime", + files, + }, null, 2)); +} finally { + await rm(stage, { recursive: true, force: true }); +} + +async function assertExactSource() { + const declared = process.env.NODEDC_FOUNDRY_SOURCE_COMMIT; + if (declared !== expectedSourceCommit) { + throw new Error("foundry_source_commit_mismatch"); + } + const dotGit = join(foundryRoot, ".git"); + try { + await lstat(dotGit); + } catch (error) { + if (error?.code === "ENOENT") return; + throw error; + } + const tracked = spawnSync( + "git", + ["-C", foundryRoot, "diff", "--quiet", "HEAD", "--", ...files], + { encoding: "utf8" }, + ); + if (tracked.status !== 0) { + throw new Error("foundry_selected_source_is_dirty"); + } + const untracked = git(foundryRoot, [ + "ls-files", + "--others", + "--exclude-standard", + "--", + ...files, + ]); + if (untracked) { + throw new Error("foundry_selected_source_has_untracked_files"); + } +} + +async function assertRecoveryBoundary() { + const packageJson = await readFile(join(foundryRoot, "package.json"), "utf8"); + const packageLock = await readFile( + join(foundryRoot, "package-lock.json"), + "utf8", + ); + const preview = await readFile( + join(foundryRoot, "apps/catalog/src/MapFixturePreview.tsx"), + "utf8", + ); + const renderer = await readFile( + join(foundryRoot, "apps/catalog/src/CesiumMapRenderer.tsx"), + "utf8", + ); + const policy = await readFile( + join(foundryRoot, "apps/catalog/src/mapGridPolicy.mjs"), + "utf8", + ); + const sector = await readFile( + join(foundryRoot, "apps/catalog/src/mapSectorGrid.mjs"), + "utf8", + ); + const server = await readFile( + join(foundryRoot, "server/catalog-server.mjs"), + "utf8", + ); + const registry = await readFile( + join(foundryRoot, "registry/registry.json"), + "utf8", + ); + + for (const marker of [ + "@nodedc/map-cesium-react", + "npm run build --workspace @nodedc/map-cesium-react", + ]) { + if (!packageJson.includes(marker)) { + throw new Error(`map_package_boundary_missing:${marker}`); + } + } + if (!packageLock.includes('"packages/map-cesium-react"')) { + throw new Error("map_package_lock_boundary_missing"); + } + for (const marker of [ + "const sectorSpatialEntities = useMemo", + "runtimeBindings={[...sectorScopedPrimaryRuntimeBindings, ...referenceRuntimeBindings]}", + 'label="Скрыть объекты за сектором"', + ]) { + if (!preview.includes(marker)) { + throw new Error(`map_workspace_boundary_missing:${marker}`); + } + } + for (const marker of [ + "class GridLayerController", + "gridLegacyMode", + "viewer.flyTo(entity", + ]) { + if (!renderer.includes(marker)) { + throw new Error(`map_renderer_boundary_missing:${marker}`); + } + } + for (const marker of [ + "gridLodProfiles", + "GRID_LOD_HYSTERESIS_RATIO", + ]) { + if (!policy.includes(marker)) { + throw new Error(`map_policy_boundary_missing:${marker}`); + } + } + for (const marker of [ + "export function geodeticToLocalGridPlane", + "export function localSectorAtGeodetic", + ]) { + if (!sector.includes(marker)) { + throw new Error(`map_sector_boundary_missing:${marker}`); + } + } + for (const marker of [ + "gridLodProfiles", + "validateGridLodProfiles", + ]) { + if (!server.includes(marker)) { + throw new Error(`map_server_boundary_missing:${marker}`); + } + } + if (!registry.includes('"@nodedc/map-cesium-react"')) { + throw new Error("map_registry_boundary_missing"); + } +} + +async function copySafe(source, destination) { + const sourceStat = await lstat(source); + if (sourceStat.isSymbolicLink()) { + throw new Error( + `source_symlink_rejected:${relative(foundryRoot, source)}`, + ); + } + if (sourceStat.isFile()) { + await mkdir(dirname(destination), { recursive: true }); + await cp(source, destination, { force: true, verbatimSymlinks: true }); + return; + } + if (!sourceStat.isDirectory()) { + throw new Error(`source_type_rejected:${relative(foundryRoot, source)}`); + } + await mkdir(destination, { recursive: true }); + for (const entry of await readdir(source, { withFileTypes: true })) { + if ( + ignoredBasenames.has(entry.name) + || entry.name.startsWith(".env") + || entry.name.endsWith(".tsbuildinfo") + ) { + continue; + } + const childSource = join(source, entry.name); + const childDestination = join(destination, entry.name); + if (entry.isSymbolicLink()) { + throw new Error( + `source_symlink_rejected:${relative(foundryRoot, childSource)}`, + ); + } + await copySafe(childSource, childDestination); + } +} + +async function assertFresh(path) { + try { + await lstat(path); + } catch (error) { + if (error?.code === "ENOENT") return; + throw error; + } + throw new Error("artifact_already_exists"); +} + +function canonicalTarScript() { + return [ + "import gzip,io,pathlib,sys,tarfile", + "root=pathlib.Path(sys.argv[2])", + "with open(sys.argv[1],'wb') as out:", + " with gzip.GzipFile(filename='',mode='wb',fileobj=out,compresslevel=9,mtime=0) as gz:", + " with tarfile.open(fileobj=gz,mode='w',format=tarfile.PAX_FORMAT) as tar:", + " for top in ('manifest.env','files.txt','payload'):", + " p=root/top; paths=[p]+(sorted(p.rglob('*')) if p.is_dir() else [])", + " for x in paths:", + " info=tar.gettarinfo(str(x),arcname=x.relative_to(root).as_posix())", + " info.uid=info.gid=0; info.uname=info.gname='root'; info.mtime=0; info.mode=0o755 if info.isdir() else 0o644", + " with (open(x,'rb') if info.isfile() else io.BytesIO()) as src: tar.addfile(info,src if info.isfile() else None)", + ].join("\n"); +} + +function digest(value) { + return createHash("sha256").update(value).digest("hex"); +} + +function run(command, args) { + const result = spawnSync(command, args, { + encoding: "utf8", + maxBuffer: 128 * 1024 * 1024, + }); + if (result.status !== 0) { + throw new Error(`${command}_failed:${result.stderr || result.stdout}`); + } +} diff --git a/infra/deploy-runner/nodedc-deploy b/infra/deploy-runner/nodedc-deploy index 4ffe3e1..2ff2403 100755 --- a/infra/deploy-runner/nodedc-deploy +++ b/infra/deploy-runner/nodedc-deploy @@ -38,10 +38,31 @@ DOCKER = Path("/usr/local/bin/docker") MODULE_FOUNDRY_SERVICE = "nodedc-module-foundry" MODULE_FOUNDRY_RUNTIME_BEFORE_FILE = "module-foundry-runtime-before.json" MODULE_FOUNDRY_BUILD_ATTEMPTS = 2 -MODULE_FOUNDRY_SECTOR_RECOVERY_PATCH_ID = ( - "module-foundry-map-sector-workspace-20260809-001" +MODULE_FOUNDRY_MAP_RECOVERY_PATCH_ID = ( + "module-foundry-map-runtime-recovery-20260809-002" ) -MODULE_FOUNDRY_SECTOR_RECOVERY_ENTRIES = ( +MODULE_FOUNDRY_MAP_RECOVERY_ENTRIES = ( + "package.json", + "package-lock.json", + "apps/catalog/src", + "packages", + "registry", + "runtime-seed/page-layouts/map.json", + "scripts", + "server/catalog-server.mjs", + "server/foundry-mcp.mjs", + "server/map-grid-persistence.test.mjs", +) +MODULE_FOUNDRY_MAP_RECOVERY_MISSING_PREDECESSOR_ENTRIES = ( + "server/map-grid-persistence.test.mjs", +) +MODULE_FOUNDRY_MAP_RECOVERY_INSTALLED_TREE_SHA256 = ( + "8513127fb3b875dc3326c0305441723cda6c405525a68cc11030a6f645738d2b" +) +MODULE_FOUNDRY_MAP_RECOVERY_CANDIDATE_TREE_SHA256 = ( + "b89392e56fbaf6a8450ab67fcf263199eadba4a74ef7b70b4d8d685a57c2edc3" +) +MODULE_FOUNDRY_SECTOR_FAILED_ENTRIES = ( "apps/catalog/src/MapFixturePreview.tsx", "apps/catalog/src/mapPresentationProfile.ts", "apps/catalog/src/mapSectorGrid.d.mts", @@ -51,21 +72,21 @@ MODULE_FOUNDRY_SECTOR_RECOVERY_ENTRIES = ( "scripts/map-presentation-filters.test.mjs", "scripts/map-sector-grid.test.mjs", ) -MODULE_FOUNDRY_SECTOR_FAILED_PATCH_ID = ( - "module-foundry-map-sector-workspace-20260808-001" +MODULE_FOUNDRY_MAP_FAILED_PATCH_ID = ( + "module-foundry-map-sector-workspace-20260809-001" ) -MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT = ( - "nodedc-module-foundry-map-sector-workspace-20260808-001.tgz." - "20260809-014720" +MODULE_FOUNDRY_MAP_FAILED_ARTIFACT = ( + "nodedc-module-foundry-map-sector-workspace-20260809-001.tgz." + "20260809-022707" ) -MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT_SHA256 = ( - "0b45002ffcd3270d56add85fc206cd34614ed10edfb57024688347cbe7e5e740" +MODULE_FOUNDRY_MAP_FAILED_ARTIFACT_SHA256 = ( + "96f67297efe6033f545e797dc4e341983b2ce9ca6ebf6ff48a00f6ee8b8ea23b" ) -MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_ID = ( - "module-foundry-module-foundry-map-sector-workspace-20260808-001-" - "20260809-014720" +MODULE_FOUNDRY_MAP_FAILED_BACKUP_ID = ( + "module-foundry-module-foundry-map-sector-workspace-20260809-001-" + "20260809-022707" ) -MODULE_FOUNDRY_SECTOR_FAILED_MESSAGE = ( +MODULE_FOUNDRY_MAP_FAILED_MESSAGE = ( "Command '['/usr/local/bin/docker', 'compose', '-p', " "'nodedc-module-foundry', '--env-file', " "'/volume1/docker/nodedc-platform/module-foundry/source/.env', '-f', " @@ -74,21 +95,21 @@ MODULE_FOUNDRY_SECTOR_FAILED_MESSAGE = ( "'--build', '--no-deps', 'nodedc-module-foundry']' returned non-zero " "exit status 17." ) -MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_SHA256 = { +MODULE_FOUNDRY_MAP_FAILED_BACKUP_SHA256 = { "existing-files.txt": ( - "57273169e590f1de2de46fb201aff5164af2171cecb832a090669ab5cc93bb1c" + "9273d70c9d1fd9a12963f1f66d2994ec7b7c216ebbae27e254931b8fc2a07ea3" ), "files.txt": ( "9273d70c9d1fd9a12963f1f66d2994ec7b7c216ebbae27e254931b8fc2a07ea3" ), "manifest.env": ( - "cf6a5263095bb56e3bb3543bcf5ab23b1893c1d23ceea1a7445e7b76924c2aa2" + "fd1ea43c173d89e358731415e528220aa72b1a7c27b6ee8ba408292d8ddf41a2" ), "missing-files.txt": ( - "1cbc6098562af19a9fb2abefc0e41f7a1178a436bb82d58a9a4f3e989c19f38f" + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" ), "source-before.tgz": ( - "bf7e011bcf2c932e952917dc0f7ba48b84bde0b833c7116714bcc618932d5562" + "e3f14d55d5eaff456a9001fb4a1234b5b584afb93cf4d0c929b30312bd7b5668" ), } MAP_GATEWAY_SECRET_DIR = Path("/volume1/docker/nodedc-platform/secrets") @@ -15017,14 +15038,14 @@ def plan_artifact(artifact): provider_catalog_preflight = None device_plane_postgres_preflight = None device_plane_foundation_recovery_preflight = None - module_foundry_sector_recovery_preflight = None + module_foundry_map_recovery_preflight = None with tempfile.TemporaryDirectory(prefix="plan-", dir=TMP_DIR) as tmp: manifest, entries, payload_dir = load_artifact(artifact, Path(tmp)) reject_terminal_engine_l2_failed_artifact(manifest, sha) reject_terminal_device_plane_foundation_artifact(manifest, sha) reject_terminal_device_plane_backhaul_artifact(manifest, sha) - module_foundry_sector_recovery_preflight = ( - validate_module_foundry_sector_recovery_evidence( + module_foundry_map_recovery_preflight = ( + validate_module_foundry_map_recovery_evidence( manifest, entries, payload_dir, @@ -16590,30 +16611,38 @@ def plan_artifact(artifact): ) print("module_foundry_runtime_pull=never") print("module_foundry_rollback=source+image+runtime") - if module_foundry_sector_recovery_preflight is not None: + if module_foundry_map_recovery_preflight is not None: print( "module_foundry_transition=" - f"{module_foundry_sector_recovery_preflight['mode']}" + f"{module_foundry_map_recovery_preflight['mode']}" ) print( "module_foundry_failed_predecessor=" - f"{module_foundry_sector_recovery_preflight['failed_patch_id']}" + f"{module_foundry_map_recovery_preflight['failed_patch_id']}" ) print( "module_foundry_failed_predecessor_sha256=" - f"{module_foundry_sector_recovery_preflight['failed_artifact_sha256']}" + f"{module_foundry_map_recovery_preflight['failed_artifact_sha256']}" ) print( "module_foundry_failed_backup=" - f"{module_foundry_sector_recovery_preflight['failed_backup_id']}" + f"{module_foundry_map_recovery_preflight['failed_backup_id']}" + ) + print( + "module_foundry_installed_tree_sha256=" + f"{module_foundry_map_recovery_preflight['installed_tree_sha256']}" + ) + print( + "module_foundry_candidate_tree_sha256=" + f"{module_foundry_map_recovery_preflight['candidate_tree_sha256']}" ) print( "module_foundry_predecessor_container=" - f"{module_foundry_sector_recovery_preflight['runtime_container_id']}" + f"{module_foundry_map_recovery_preflight['runtime_container_id']}" ) print( "module_foundry_predecessor_image=" - f"{module_foundry_sector_recovery_preflight['runtime_image_id']}" + f"{module_foundry_map_recovery_preflight['runtime_image_id']}" ) if component == "device-plane": print(f"runtime_secret=runner-managed:{DEVICE_PLANE_POSTGRES_PASSWORD_FILE}") @@ -17336,38 +17365,49 @@ def rollback_engine_apply(root, backup_dir, entries, current_stamp, runtime_star return f"source+runtime-restored:{restored_count}" -def is_module_foundry_sector_recovery_slice(manifest, entries): +def is_module_foundry_map_recovery_slice(manifest, entries): return ( - manifest.get("id") == MODULE_FOUNDRY_SECTOR_RECOVERY_PATCH_ID + manifest.get("id") == MODULE_FOUNDRY_MAP_RECOVERY_PATCH_ID and manifest.get("component") == "module-foundry" and manifest.get("type") == "app-overlay" - and tuple(entries or ()) == MODULE_FOUNDRY_SECTOR_RECOVERY_ENTRIES + and tuple(entries or ()) == MODULE_FOUNDRY_MAP_RECOVERY_ENTRIES ) -def validate_module_foundry_sector_recovery_evidence( +def exact_file_map_sha256(files): + return hashlib.sha256( + json.dumps( + files, + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + ).hexdigest() + + +def validate_module_foundry_map_recovery_evidence( manifest, entries, payload_dir, ): - if not is_module_foundry_sector_recovery_slice(manifest, entries): + if not is_module_foundry_map_recovery_slice(manifest, entries): return None - backup_dir = BACKUPS_DIR / MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_ID + backup_dir = BACKUPS_DIR / MODULE_FOUNDRY_MAP_FAILED_BACKUP_ID try: backup_stat = backup_dir.lstat() except FileNotFoundError: - die("Module Foundry sector recovery backup is missing") + die("Module Foundry map recovery backup is missing") if stat.S_ISLNK(backup_stat.st_mode) or not stat.S_ISDIR( backup_stat.st_mode ): - die("Module Foundry sector recovery backup is unsafe") + die("Module Foundry map recovery backup is unsafe") if {child.name for child in backup_dir.iterdir()} != set( - MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_SHA256 + MODULE_FOUNDRY_MAP_FAILED_BACKUP_SHA256 ): - die("Module Foundry sector recovery backup file set mismatch") + die("Module Foundry map recovery backup file set mismatch") for name, expected_sha256 in ( - MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_SHA256.items() + MODULE_FOUNDRY_MAP_FAILED_BACKUP_SHA256.items() ): path = backup_dir / name path_stat = path.lstat() @@ -17377,22 +17417,22 @@ def validate_module_foundry_sector_recovery_evidence( or sha256_file(path) != expected_sha256 ): die( - "Module Foundry sector recovery backup drift detected: " + "Module Foundry map recovery backup drift detected: " f"{name}" ) - failed_artifact = FAILED_DIR / MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT + failed_artifact = FAILED_DIR / MODULE_FOUNDRY_MAP_FAILED_ARTIFACT try: failed_stat = failed_artifact.lstat() except FileNotFoundError: - die("Module Foundry sector failed artifact is missing") + die("Module Foundry map failed artifact is missing") if ( stat.S_ISLNK(failed_stat.st_mode) or not stat.S_ISREG(failed_stat.st_mode) or sha256_file(failed_artifact) - != MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT_SHA256 + != MODULE_FOUNDRY_MAP_FAILED_ARTIFACT_SHA256 ): - die("Module Foundry sector failed artifact evidence mismatch") + die("Module Foundry map failed artifact evidence mismatch") try: state_stat = FAILED_STATE_FILE.lstat() @@ -17400,41 +17440,41 @@ def validate_module_foundry_sector_recovery_evidence( encoding="utf-8" ).splitlines() except (FileNotFoundError, OSError, UnicodeDecodeError): - die("Module Foundry sector failed journal is unreadable") + die("Module Foundry map failed journal is unreadable") if stat.S_ISLNK(state_stat.st_mode) or not stat.S_ISREG( state_stat.st_mode ): - die("Module Foundry sector failed journal is unsafe") + die("Module Foundry map failed journal is unsafe") records = [] for line in state_lines: try: value = json.loads(line) except json.JSONDecodeError: - die("Module Foundry sector failed journal contains invalid JSON") + die("Module Foundry map failed journal contains invalid JSON") if ( isinstance(value, dict) - and value.get("id") == MODULE_FOUNDRY_SECTOR_FAILED_PATCH_ID + and value.get("id") == MODULE_FOUNDRY_MAP_FAILED_PATCH_ID ): records.append(value) if len(records) != 1: - die("Module Foundry sector failed journal evidence count mismatch") + die("Module Foundry map failed journal evidence count mismatch") record = records[0] if ( - record.get("artifact") != MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT + record.get("artifact") != MODULE_FOUNDRY_MAP_FAILED_ARTIFACT or record.get("backup_id") - != MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_ID + != MODULE_FOUNDRY_MAP_FAILED_BACKUP_ID or record.get("component") != "module-foundry" or record.get("sha256") - != MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT_SHA256 + != MODULE_FOUNDRY_MAP_FAILED_ARTIFACT_SHA256 or record.get("started_apply") is not True or record.get("rollback_status") != "not-required" or record.get("status") != "failed" - or record.get("message") != MODULE_FOUNDRY_SECTOR_FAILED_MESSAGE + or record.get("message") != MODULE_FOUNDRY_MAP_FAILED_MESSAGE ): - die("Module Foundry sector failed journal evidence mismatch") + die("Module Foundry map failed journal evidence mismatch") with tempfile.TemporaryDirectory( - prefix="module-foundry-sector-failed-", + prefix="module-foundry-map-failed-", dir=TMP_DIR, ) as directory: failed_work = Path(directory) @@ -17443,42 +17483,78 @@ def validate_module_foundry_sector_recovery_evidence( failed_entries = parse_files_list(failed_work / "files.txt") if ( failed_manifest.get("id") - != MODULE_FOUNDRY_SECTOR_FAILED_PATCH_ID + != MODULE_FOUNDRY_MAP_FAILED_PATCH_ID or failed_manifest.get("component") != "module-foundry" or failed_manifest.get("type") != "app-overlay" or tuple(failed_entries) - != MODULE_FOUNDRY_SECTOR_RECOVERY_ENTRIES + != MODULE_FOUNDRY_SECTOR_FAILED_ENTRIES ): - die("Module Foundry sector failed artifact contract mismatch") + die("Module Foundry map failed artifact contract mismatch") failed_payload_sha256 = collect_exact_files( failed_work / "payload", failed_entries, - "Module Foundry failed sector payload", + "Module Foundry failed map payload", ) candidate_payload_sha256 = collect_exact_files( payload_dir, entries, - "Module Foundry sector recovery payload", + "Module Foundry map recovery payload", + ) + if ( + exact_file_map_sha256(candidate_payload_sha256) + != MODULE_FOUNDRY_MAP_RECOVERY_CANDIDATE_TREE_SHA256 + ): + die("Module Foundry map recovery candidate drift detected") + if { + rel: candidate_payload_sha256.get(rel) + for rel in failed_payload_sha256 + } != failed_payload_sha256: + die("Module Foundry failed map payload changed unexpectedly") + + installed_root = component_root("module-foundry") + for rel in MODULE_FOUNDRY_MAP_RECOVERY_MISSING_PREDECESSOR_ENTRIES: + path = installed_root / rel + if path.exists() or path.is_symlink(): + die( + "Module Foundry map recovery missing predecessor drift: " + f"{rel}" + ) + installed_entries = tuple( + rel + for rel in entries + if rel not in MODULE_FOUNDRY_MAP_RECOVERY_MISSING_PREDECESSOR_ENTRIES ) - if candidate_payload_sha256 != failed_payload_sha256: - die("Module Foundry sector recovery payload changed unexpectedly") installed_payload_sha256 = collect_exact_files( - component_root("module-foundry"), - entries, - "Module Foundry partial sector predecessor", + installed_root, + installed_entries, + "Module Foundry partial map predecessor", ) - if installed_payload_sha256 != candidate_payload_sha256: - die("Module Foundry partial sector predecessor drift detected") + if ( + exact_file_map_sha256(installed_payload_sha256) + != MODULE_FOUNDRY_MAP_RECOVERY_INSTALLED_TREE_SHA256 + ): + die("Module Foundry partial map predecessor drift detected") + if { + rel: installed_payload_sha256.get(rel) + for rel in failed_payload_sha256 + } != failed_payload_sha256: + die("Module Foundry installed failed map payload drift detected") runtime = module_foundry_runtime_inventory() return { - "mode": "failed-overlay-runtime-reconciliation", - "failed_patch_id": MODULE_FOUNDRY_SECTOR_FAILED_PATCH_ID, + "mode": "failed-map-overlay-source+runtime-reconciliation", + "failed_patch_id": MODULE_FOUNDRY_MAP_FAILED_PATCH_ID, "failed_artifact_sha256": ( - MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT_SHA256 + MODULE_FOUNDRY_MAP_FAILED_ARTIFACT_SHA256 + ), + "failed_backup_id": MODULE_FOUNDRY_MAP_FAILED_BACKUP_ID, + "installed_tree_sha256": ( + MODULE_FOUNDRY_MAP_RECOVERY_INSTALLED_TREE_SHA256 + ), + "candidate_tree_sha256": ( + MODULE_FOUNDRY_MAP_RECOVERY_CANDIDATE_TREE_SHA256 ), - "failed_backup_id": MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_ID, "runtime_container_id": runtime["containerId"], "runtime_image_id": runtime["imageId"], } @@ -17531,7 +17607,10 @@ def rollback_module_foundry_apply( entries, current_stamp, ) - current = module_foundry_runtime_inventory(required=False) + current = module_foundry_runtime_inventory( + required=False, + require_healthy=False, + ) if ( current is not None and current["containerId"] == runtime_before["containerId"] @@ -18811,7 +18890,7 @@ def compose_service_container_id(component, service): return container_ids[0] -def module_foundry_runtime_inventory(required=True): +def module_foundry_runtime_inventory(required=True, require_healthy=True): result = subprocess.run( [ *compose_base_cmd("module-foundry"), @@ -18873,9 +18952,14 @@ def module_foundry_runtime_inventory(required=True): ) or image_ref.startswith("sha256:") or "@" in image_ref - or state.get("Running") is not True - or state.get("Status") != "running" - or health not in (None, "healthy") + or ( + require_healthy + and ( + state.get("Running") is not True + or state.get("Status") != "running" + or health not in (None, "healthy") + ) + ) ): die("Module Foundry runtime inventory mismatch") return { @@ -20032,7 +20116,7 @@ def apply_artifact(artifact): engine_backend_recreated = False engine_backend_initial_mode = None module_foundry_runtime_before = None - module_foundry_sector_recovery_preflight = None + module_foundry_map_recovery_preflight = None runtime_started = False current_stamp = stamp() @@ -20111,8 +20195,8 @@ def apply_artifact(artifact): root.mkdir(parents=True, exist_ok=True) else: die(f"component payload root not found: {root}") - module_foundry_sector_recovery_preflight = ( - validate_module_foundry_sector_recovery_evidence( + module_foundry_map_recovery_preflight = ( + validate_module_foundry_map_recovery_evidence( manifest, entries, payload_dir, @@ -20402,20 +20486,20 @@ def apply_artifact(artifact): module_foundry_runtime_inventory() ) if ( - module_foundry_sector_recovery_preflight is not None + module_foundry_map_recovery_preflight is not None and ( module_foundry_runtime_before["containerId"] - != module_foundry_sector_recovery_preflight[ + != module_foundry_map_recovery_preflight[ "runtime_container_id" ] or module_foundry_runtime_before["imageId"] - != module_foundry_sector_recovery_preflight[ + != module_foundry_map_recovery_preflight[ "runtime_image_id" ] ) ): die( - "Module Foundry sector predecessor runtime " + "Module Foundry map predecessor runtime " "changed during preflight" ) diff --git a/infra/deploy-runner/test_module_foundry_map_runtime_recovery_artifact.py b/infra/deploy-runner/test_module_foundry_map_runtime_recovery_artifact.py new file mode 100644 index 0000000..8de3a2f --- /dev/null +++ b/infra/deploy-runner/test_module_foundry_map_runtime_recovery_artifact.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 + +import hashlib +import importlib.machinery +import importlib.util +import json +import os +import subprocess +import tarfile +import tempfile +import unittest +from pathlib import Path, PurePosixPath + + +SCRIPT_DIR = Path(__file__).resolve().parent +PLATFORM_ROOT = SCRIPT_DIR.parent.parent +WORKSPACE_ROOT = PLATFORM_ROOT.parent.parent +FOUNDRY_ROOT = WORKSPACE_ROOT / "NODEDC_DESIGN_GUIDELINE" +BUILDER = ( + SCRIPT_DIR / "build-module-foundry-map-runtime-recovery-artifact.mjs" +) +RUNNER_PATH = SCRIPT_DIR / "nodedc-deploy" +SOURCE_COMMIT = "58800d957632320fa6717ec3fcff972023528759" +PATCH_ID = "module-foundry-map-runtime-recovery-20260809-999" +EXPECTED_FILES = ( + "package.json", + "package-lock.json", + "apps/catalog/src", + "packages", + "registry", + "runtime-seed/page-layouts/map.json", + "scripts", + "server/catalog-server.mjs", + "server/foundry-mcp.mjs", + "server/map-grid-persistence.test.mjs", +) + + +def load_runner(): + loader = importlib.machinery.SourceFileLoader( + "nodedc_module_foundry_map_runtime_recovery_artifact", + str(RUNNER_PATH), + ) + spec = importlib.util.spec_from_loader(loader.name, loader) + module = importlib.util.module_from_spec(spec) + loader.exec_module(module) + return module + + +RUNNER = load_runner() + + +class ModuleFoundryMapRuntimeRecoveryArtifactTest(unittest.TestCase): + def export_source(self, root): + archive_path = root / "source.tar" + source_root = root / "source" + source_root.mkdir() + subprocess.run( + [ + "git", + "-C", + str(FOUNDRY_ROOT), + "archive", + "--format=tar", + f"--output={archive_path}", + SOURCE_COMMIT, + ], + check=True, + ) + with tarfile.open(archive_path, "r:") as archive: + for member in archive.getmembers(): + path = PurePosixPath(member.name) + self.assertFalse(path.is_absolute()) + self.assertNotIn("..", path.parts) + archive.extractall(source_root, filter="data") + return source_root + + def build(self, source_root, artifact_dir): + environment = os.environ.copy() + environment["NODEDC_FOUNDRY_SOURCE_ROOT"] = str(source_root) + environment["NODEDC_FOUNDRY_SOURCE_COMMIT"] = SOURCE_COMMIT + environment["NODEDC_DEPLOY_ARTIFACT_DIR"] = str(artifact_dir) + completed = subprocess.run( + ["node", str(BUILDER), PATCH_ID], + cwd=PLATFORM_ROOT, + env=environment, + check=True, + capture_output=True, + text=True, + ) + return json.loads(completed.stdout) + + def test_builder_is_deterministic_coherent_and_runner_compatible(self): + with tempfile.TemporaryDirectory( + prefix="nodedc-foundry-map-runtime-recovery-" + ) as directory: + root = Path(directory) + source_root = self.export_source(root) + first = self.build(source_root, root / "first") + second = self.build(source_root, root / "second") + first_artifact = Path(first["artifact"]) + second_artifact = Path(second["artifact"]) + + self.assertEqual(first_artifact.read_bytes(), second_artifact.read_bytes()) + self.assertEqual( + first["sha256"], + hashlib.sha256(first_artifact.read_bytes()).hexdigest(), + ) + self.assertEqual(first["sourceCommit"], SOURCE_COMMIT) + self.assertEqual(tuple(first["files"]), EXPECTED_FILES) + self.assertEqual(first["services"], ["nodedc-module-foundry"]) + + extracted = root / "loaded" + extracted.mkdir() + manifest, entries, payload = RUNNER.load_artifact( + first_artifact, + extracted, + ) + self.assertEqual(manifest["id"], PATCH_ID) + self.assertEqual(manifest["component"], "module-foundry") + self.assertEqual(tuple(entries), EXPECTED_FILES) + self.assertEqual( + RUNNER.component_services("module-foundry", entries), + ("nodedc-module-foundry",), + ) + + preview = ( + payload / "apps/catalog/src/MapFixturePreview.tsx" + ).read_text(encoding="utf-8") + renderer = ( + payload / "apps/catalog/src/CesiumMapRenderer.tsx" + ).read_text(encoding="utf-8") + package_lock = (payload / "package-lock.json").read_text( + encoding="utf-8" + ) + self.assertIn("const sectorSpatialEntities = useMemo", preview) + self.assertIn("class GridLayerController", renderer) + self.assertIn('"packages/map-cesium-react"', package_lock) + self.assertTrue( + (payload / "packages/map-cesium-react/src/index.ts").is_file() + ) + + with tarfile.open(first_artifact, "r:gz") as archive: + members = archive.getmembers() + names = [member.name for member in members] + regular_payloads = [ + archive.extractfile(member).read() + for member in members + if member.isfile() + ] + self.assertFalse( + any(Path(name).name.startswith("._") for name in names) + ) + self.assertFalse( + any( + "/.git/" in name + or "/node_modules/" in name + or "/runtime-data/" in name + or "/dist/" in name + for name in names + ) + ) + self.assertNotIn( + "payload/scripts/spark-governance-contract.test.mjs", + names, + ) + self.assertNotIn( + b"-----BEGIN PRIVATE KEY-----", + b"\n".join(regular_payloads), + ) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/infra/deploy-runner/test_module_foundry_runtime_recovery.py b/infra/deploy-runner/test_module_foundry_runtime_recovery.py index 21341d7..9a4da79 100644 --- a/infra/deploy-runner/test_module_foundry_runtime_recovery.py +++ b/infra/deploy-runner/test_module_foundry_runtime_recovery.py @@ -8,6 +8,7 @@ import subprocess import tarfile import tempfile import unittest +from contextlib import ExitStack from pathlib import Path from unittest import mock @@ -202,7 +203,7 @@ class ModuleFoundryRuntimeRecoveryTest(unittest.TestCase): RUNNER, "module_foundry_runtime_inventory", side_effect=[candidate, before], - ), + ) as inventory, mock.patch.object( RUNNER, "inspect_local_image", @@ -221,6 +222,10 @@ class ModuleFoundryRuntimeRecoveryTest(unittest.TestCase): ) self.assertEqual(result, "source+runtime-restored:1") + self.assertEqual( + inventory.call_args_list[0].kwargs, + {"required": False, "require_healthy": False}, + ) process.assert_called_once_with( [ str(RUNNER.DOCKER), @@ -264,7 +269,7 @@ class ModuleFoundryRuntimeRecoveryTest(unittest.TestCase): def test_recovery_accepts_only_exact_failed_partial_predecessor(self): with tempfile.TemporaryDirectory( - prefix="module-foundry-sector-evidence-" + prefix="module-foundry-map-evidence-" ) as directory: work = Path(directory) backups = work / "backups" @@ -276,43 +281,79 @@ class ModuleFoundryRuntimeRecoveryTest(unittest.TestCase): for path in (backups, failed, state, tmp, installed, candidate): path.mkdir() - entries = RUNNER.MODULE_FOUNDRY_SECTOR_RECOVERY_ENTRIES - for index, rel in enumerate(entries): - for root in (installed, candidate): + recovery_entries = ( + "apps/catalog/src", + "scripts", + "server/map-grid-persistence.test.mjs", + ) + missing_entries = ("server/map-grid-persistence.test.mjs",) + failed_entries = ( + "apps/catalog/src/MapFixturePreview.tsx", + "scripts/map-sector-grid.test.mjs", + ) + candidate_files = { + "apps/catalog/src/MapFixturePreview.tsx": "sector-current\n", + "apps/catalog/src/CesiumMapRenderer.tsx": "renderer-next\n", + "scripts/map-sector-grid.test.mjs": "sector-test-current\n", + "scripts/map-grid-lod.test.mjs": "grid-test-next\n", + "server/map-grid-persistence.test.mjs": "server-test-next\n", + } + installed_files = { + "apps/catalog/src/MapFixturePreview.tsx": "sector-current\n", + "apps/catalog/src/CesiumMapRenderer.tsx": "renderer-old\n", + "scripts/map-sector-grid.test.mjs": "sector-test-current\n", + "scripts/map-grid-lod.test.mjs": "grid-test-old\n", + } + for root, values in ( + (candidate, candidate_files), + (installed, installed_files), + ): + for rel, value in values.items(): path = root / rel path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(f"payload-{index}\n", encoding="utf-8") + path.write_text(value, encoding="utf-8") - backup = backups / RUNNER.MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_ID + failed_patch_id = "module-foundry-map-failed-test" + failed_artifact_name = "module-foundry-map-failed-test.tgz.1" + failed_backup_id = "module-foundry-map-failed-test-backup" + recovery_patch_id = "module-foundry-map-recovery-test" + failed_message = "test build failed" + + backup = backups / failed_backup_id backup.mkdir() - for name in RUNNER.MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_SHA256: + backup_names = { + "existing-files.txt", + "files.txt", + "manifest.env", + "missing-files.txt", + "source-before.tgz", + } + for name in backup_names: (backup / name).write_text(f"{name}\n", encoding="utf-8") backup_hashes = { name: hashlib.sha256((backup / name).read_bytes()).hexdigest() - for name in RUNNER.MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_SHA256 + for name in backup_names } archive_source = work / "archive-source" (archive_source / "payload").mkdir(parents=True) (archive_source / "manifest.env").write_text( "id=" - f"{RUNNER.MODULE_FOUNDRY_SECTOR_FAILED_PATCH_ID}\n" + f"{failed_patch_id}\n" "component=module-foundry\n" "type=app-overlay\n", encoding="utf-8", ) (archive_source / "files.txt").write_text( - "\n".join(entries) + "\n", + "\n".join(failed_entries) + "\n", encoding="utf-8", ) - for rel in entries: + for rel in failed_entries: source = candidate / rel target = archive_source / "payload" / rel target.parent.mkdir(parents=True, exist_ok=True) target.write_bytes(source.read_bytes()) - failed_artifact = ( - failed / RUNNER.MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT - ) + failed_artifact = failed / failed_artifact_name with tarfile.open(failed_artifact, "w:gz") as archive: archive.add( archive_source / "manifest.env", @@ -331,11 +372,11 @@ class ModuleFoundryRuntimeRecoveryTest(unittest.TestCase): journal.write_text( json.dumps( { - "artifact": RUNNER.MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT, - "backup_id": RUNNER.MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_ID, + "artifact": failed_artifact_name, + "backup_id": failed_backup_id, "component": "module-foundry", - "id": RUNNER.MODULE_FOUNDRY_SECTOR_FAILED_PATCH_ID, - "message": RUNNER.MODULE_FOUNDRY_SECTOR_FAILED_MESSAGE, + "id": failed_patch_id, + "message": failed_message, "rollback_status": "not-required", "sha256": failed_sha256, "started_apply": True, @@ -346,55 +387,83 @@ class ModuleFoundryRuntimeRecoveryTest(unittest.TestCase): encoding="utf-8", ) manifest = { - "id": RUNNER.MODULE_FOUNDRY_SECTOR_RECOVERY_PATCH_ID, + "id": recovery_patch_id, "component": "module-foundry", "type": "app-overlay", } - - patches = ( - mock.patch.object(RUNNER, "BACKUPS_DIR", backups), - mock.patch.object(RUNNER, "FAILED_DIR", failed), - mock.patch.object(RUNNER, "FAILED_STATE_FILE", journal), - mock.patch.object(RUNNER, "TMP_DIR", tmp), - mock.patch.object( - RUNNER, - "MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT_SHA256", - failed_sha256, - ), - mock.patch.object( - RUNNER, - "MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_SHA256", - backup_hashes, - ), - mock.patch.object( - RUNNER, - "component_root", - return_value=installed, - ), - mock.patch.object( - RUNNER, - "module_foundry_runtime_inventory", - return_value=runtime(), - ), + candidate_tree = RUNNER.exact_file_map_sha256( + RUNNER.collect_exact_files( + candidate, + recovery_entries, + "candidate", + ) ) - with patches[0], patches[1], patches[2], patches[3], patches[4], patches[5], patches[6], patches[7]: - result = RUNNER.validate_module_foundry_sector_recovery_evidence( + installed_tree = RUNNER.exact_file_map_sha256( + RUNNER.collect_exact_files( + installed, + recovery_entries[:-1], + "installed", + ) + ) + + patches = { + "BACKUPS_DIR": backups, + "FAILED_DIR": failed, + "FAILED_STATE_FILE": journal, + "TMP_DIR": tmp, + "MODULE_FOUNDRY_MAP_RECOVERY_PATCH_ID": recovery_patch_id, + "MODULE_FOUNDRY_MAP_RECOVERY_ENTRIES": recovery_entries, + "MODULE_FOUNDRY_MAP_RECOVERY_MISSING_PREDECESSOR_ENTRIES": ( + missing_entries + ), + "MODULE_FOUNDRY_MAP_RECOVERY_INSTALLED_TREE_SHA256": ( + installed_tree + ), + "MODULE_FOUNDRY_MAP_RECOVERY_CANDIDATE_TREE_SHA256": ( + candidate_tree + ), + "MODULE_FOUNDRY_SECTOR_FAILED_ENTRIES": failed_entries, + "MODULE_FOUNDRY_MAP_FAILED_PATCH_ID": failed_patch_id, + "MODULE_FOUNDRY_MAP_FAILED_ARTIFACT": failed_artifact_name, + "MODULE_FOUNDRY_MAP_FAILED_ARTIFACT_SHA256": failed_sha256, + "MODULE_FOUNDRY_MAP_FAILED_BACKUP_ID": failed_backup_id, + "MODULE_FOUNDRY_MAP_FAILED_MESSAGE": failed_message, + "MODULE_FOUNDRY_MAP_FAILED_BACKUP_SHA256": backup_hashes, + } + with ExitStack() as stack: + for name, value in patches.items(): + stack.enter_context(mock.patch.object(RUNNER, name, value)) + stack.enter_context( + mock.patch.object( + RUNNER, + "component_root", + return_value=installed, + ) + ) + stack.enter_context( + mock.patch.object( + RUNNER, + "module_foundry_runtime_inventory", + return_value=runtime(), + ) + ) + result = RUNNER.validate_module_foundry_map_recovery_evidence( manifest, - entries, + recovery_entries, candidate, ) self.assertEqual( result["mode"], - "failed-overlay-runtime-reconciliation", + "failed-map-overlay-source+runtime-reconciliation", ) - (installed / entries[0]).write_text( + (installed / failed_entries[0]).write_text( "drift\n", encoding="utf-8", ) with self.assertRaises(RUNNER.DeployError): - RUNNER.validate_module_foundry_sector_recovery_evidence( + RUNNER.validate_module_foundry_map_recovery_evidence( manifest, - entries, + recovery_entries, candidate, )