SILVER/NODEDC_PLATFORM
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

FIX - NAS DEPLOY: reconcile Foundry map runtime generation

Browse Source
This commit is contained in:
Codex
2026-08-09 09:05:09 +03:00
parent 0565cd2642
commit 4b24c2e457
4 changed files with 768 additions and 137 deletions
Download Patch File Download Diff File Expand all files Collapse all files
infra/deploy-runner/nodedc-deploy
+168 -84
View File
@@ -38,10 +38,31 @@ DOCKER = Path("/usr/local/bin/docker")
MODULE_FOUNDRY_SERVICE = "nodedc-module-foundry"
MODULE_FOUNDRY_RUNTIME_BEFORE_FILE = "module-foundry-runtime-before.json"
MODULE_FOUNDRY_BUILD_ATTEMPTS = 2
MODULE_FOUNDRY_SECTOR_RECOVERY_PATCH_ID = (
"module-foundry-map-sector-workspace-20260809-001"
MODULE_FOUNDRY_MAP_RECOVERY_PATCH_ID = (
"module-foundry-map-runtime-recovery-20260809-002"
)
MODULE_FOUNDRY_SECTOR_RECOVERY_ENTRIES = (
MODULE_FOUNDRY_MAP_RECOVERY_ENTRIES = (
"package.json",
"package-lock.json",
"apps/catalog/src",
"packages",
"registry",
"runtime-seed/page-layouts/map.json",
"scripts",
"server/catalog-server.mjs",
"server/foundry-mcp.mjs",
"server/map-grid-persistence.test.mjs",
)
MODULE_FOUNDRY_MAP_RECOVERY_MISSING_PREDECESSOR_ENTRIES = (
"server/map-grid-persistence.test.mjs",
)
MODULE_FOUNDRY_MAP_RECOVERY_INSTALLED_TREE_SHA256 = (
"8513127fb3b875dc3326c0305441723cda6c405525a68cc11030a6f645738d2b"
)
MODULE_FOUNDRY_MAP_RECOVERY_CANDIDATE_TREE_SHA256 = (
"b89392e56fbaf6a8450ab67fcf263199eadba4a74ef7b70b4d8d685a57c2edc3"
)
MODULE_FOUNDRY_SECTOR_FAILED_ENTRIES = (
"apps/catalog/src/MapFixturePreview.tsx",
"apps/catalog/src/mapPresentationProfile.ts",
"apps/catalog/src/mapSectorGrid.d.mts",
@@ -51,21 +72,21 @@ MODULE_FOUNDRY_SECTOR_RECOVERY_ENTRIES = (
"scripts/map-presentation-filters.test.mjs",
"scripts/map-sector-grid.test.mjs",
)
MODULE_FOUNDRY_SECTOR_FAILED_PATCH_ID = (
"module-foundry-map-sector-workspace-20260808-001"
MODULE_FOUNDRY_MAP_FAILED_PATCH_ID = (
"module-foundry-map-sector-workspace-20260809-001"
)
MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT = (
"nodedc-module-foundry-map-sector-workspace-20260808-001.tgz."
"20260809-014720"
MODULE_FOUNDRY_MAP_FAILED_ARTIFACT = (
"nodedc-module-foundry-map-sector-workspace-20260809-001.tgz."
"20260809-022707"
)
MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT_SHA256 = (
"0b45002ffcd3270d56add85fc206cd34614ed10edfb57024688347cbe7e5e740"
MODULE_FOUNDRY_MAP_FAILED_ARTIFACT_SHA256 = (
"96f67297efe6033f545e797dc4e341983b2ce9ca6ebf6ff48a00f6ee8b8ea23b"
)
MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_ID = (
"module-foundry-module-foundry-map-sector-workspace-20260808-001-"
"20260809-014720"
MODULE_FOUNDRY_MAP_FAILED_BACKUP_ID = (
"module-foundry-module-foundry-map-sector-workspace-20260809-001-"
"20260809-022707"
)
MODULE_FOUNDRY_SECTOR_FAILED_MESSAGE = (
MODULE_FOUNDRY_MAP_FAILED_MESSAGE = (
"Command '['/usr/local/bin/docker', 'compose', '-p', "
"'nodedc-module-foundry', '--env-file', "
"'/volume1/docker/nodedc-platform/module-foundry/source/.env', '-f', "
@@ -74,21 +95,21 @@ MODULE_FOUNDRY_SECTOR_FAILED_MESSAGE = (
"'--build', '--no-deps', 'nodedc-module-foundry']' returned non-zero "
"exit status 17."
)
MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_SHA256 = {
MODULE_FOUNDRY_MAP_FAILED_BACKUP_SHA256 = {
"existing-files.txt": (
"57273169e590f1de2de46fb201aff5164af2171cecb832a090669ab5cc93bb1c"
"9273d70c9d1fd9a12963f1f66d2994ec7b7c216ebbae27e254931b8fc2a07ea3"
),
"files.txt": (
"9273d70c9d1fd9a12963f1f66d2994ec7b7c216ebbae27e254931b8fc2a07ea3"
),
"manifest.env": (
"cf6a5263095bb56e3bb3543bcf5ab23b1893c1d23ceea1a7445e7b76924c2aa2"
"fd1ea43c173d89e358731415e528220aa72b1a7c27b6ee8ba408292d8ddf41a2"
),
"missing-files.txt": (
"1cbc6098562af19a9fb2abefc0e41f7a1178a436bb82d58a9a4f3e989c19f38f"
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
),
"source-before.tgz": (
"bf7e011bcf2c932e952917dc0f7ba48b84bde0b833c7116714bcc618932d5562"
"e3f14d55d5eaff456a9001fb4a1234b5b584afb93cf4d0c929b30312bd7b5668"
),
}
MAP_GATEWAY_SECRET_DIR = Path("/volume1/docker/nodedc-platform/secrets")
@@ -15017,14 +15038,14 @@ def plan_artifact(artifact):
provider_catalog_preflight = None
device_plane_postgres_preflight = None
device_plane_foundation_recovery_preflight = None
module_foundry_sector_recovery_preflight = None
module_foundry_map_recovery_preflight = None
with tempfile.TemporaryDirectory(prefix="plan-", dir=TMP_DIR) as tmp:
manifest, entries, payload_dir = load_artifact(artifact, Path(tmp))
reject_terminal_engine_l2_failed_artifact(manifest, sha)
reject_terminal_device_plane_foundation_artifact(manifest, sha)
reject_terminal_device_plane_backhaul_artifact(manifest, sha)
module_foundry_sector_recovery_preflight = (
validate_module_foundry_sector_recovery_evidence(
module_foundry_map_recovery_preflight = (
validate_module_foundry_map_recovery_evidence(
manifest,
entries,
payload_dir,
@@ -16590,30 +16611,38 @@ def plan_artifact(artifact):
)
print("module_foundry_runtime_pull=never")
print("module_foundry_rollback=source+image+runtime")
if module_foundry_sector_recovery_preflight is not None:
if module_foundry_map_recovery_preflight is not None:
print(
"module_foundry_transition="
f"{module_foundry_sector_recovery_preflight['mode']}"
f"{module_foundry_map_recovery_preflight['mode']}"
)
print(
"module_foundry_failed_predecessor="
f"{module_foundry_sector_recovery_preflight['failed_patch_id']}"
f"{module_foundry_map_recovery_preflight['failed_patch_id']}"
)
print(
"module_foundry_failed_predecessor_sha256="
f"{module_foundry_sector_recovery_preflight['failed_artifact_sha256']}"
f"{module_foundry_map_recovery_preflight['failed_artifact_sha256']}"
)
print(
"module_foundry_failed_backup="
f"{module_foundry_sector_recovery_preflight['failed_backup_id']}"
f"{module_foundry_map_recovery_preflight['failed_backup_id']}"
)
print(
"module_foundry_installed_tree_sha256="
f"{module_foundry_map_recovery_preflight['installed_tree_sha256']}"
)
print(
"module_foundry_candidate_tree_sha256="
f"{module_foundry_map_recovery_preflight['candidate_tree_sha256']}"
)
print(
"module_foundry_predecessor_container="
f"{module_foundry_sector_recovery_preflight['runtime_container_id']}"
f"{module_foundry_map_recovery_preflight['runtime_container_id']}"
)
print(
"module_foundry_predecessor_image="
f"{module_foundry_sector_recovery_preflight['runtime_image_id']}"
f"{module_foundry_map_recovery_preflight['runtime_image_id']}"
)
if component == "device-plane":
print(f"runtime_secret=runner-managed:{DEVICE_PLANE_POSTGRES_PASSWORD_FILE}")
@@ -17336,38 +17365,49 @@ def rollback_engine_apply(root, backup_dir, entries, current_stamp, runtime_star
return f"source+runtime-restored:{restored_count}"
def is_module_foundry_sector_recovery_slice(manifest, entries):
def is_module_foundry_map_recovery_slice(manifest, entries):
return (
manifest.get("id") == MODULE_FOUNDRY_SECTOR_RECOVERY_PATCH_ID
manifest.get("id") == MODULE_FOUNDRY_MAP_RECOVERY_PATCH_ID
and manifest.get("component") == "module-foundry"
and manifest.get("type") == "app-overlay"
and tuple(entries or ()) == MODULE_FOUNDRY_SECTOR_RECOVERY_ENTRIES
and tuple(entries or ()) == MODULE_FOUNDRY_MAP_RECOVERY_ENTRIES
)
def validate_module_foundry_sector_recovery_evidence(
def exact_file_map_sha256(files):
return hashlib.sha256(
json.dumps(
files,
ensure_ascii=True,
separators=(",", ":"),
sort_keys=True,
).encode("utf-8")
).hexdigest()
def validate_module_foundry_map_recovery_evidence(
manifest,
entries,
payload_dir,
):
if not is_module_foundry_sector_recovery_slice(manifest, entries):
if not is_module_foundry_map_recovery_slice(manifest, entries):
return None
backup_dir = BACKUPS_DIR / MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_ID
backup_dir = BACKUPS_DIR / MODULE_FOUNDRY_MAP_FAILED_BACKUP_ID
try:
backup_stat = backup_dir.lstat()
except FileNotFoundError:
die("Module Foundry sector recovery backup is missing")
die("Module Foundry map recovery backup is missing")
if stat.S_ISLNK(backup_stat.st_mode) or not stat.S_ISDIR(
backup_stat.st_mode
):
die("Module Foundry sector recovery backup is unsafe")
die("Module Foundry map recovery backup is unsafe")
if {child.name for child in backup_dir.iterdir()} != set(
MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_SHA256
MODULE_FOUNDRY_MAP_FAILED_BACKUP_SHA256
):
die("Module Foundry sector recovery backup file set mismatch")
die("Module Foundry map recovery backup file set mismatch")
for name, expected_sha256 in (
MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_SHA256.items()
MODULE_FOUNDRY_MAP_FAILED_BACKUP_SHA256.items()
):
path = backup_dir / name
path_stat = path.lstat()
@@ -17377,22 +17417,22 @@ def validate_module_foundry_sector_recovery_evidence(
or sha256_file(path) != expected_sha256
):
die(
"Module Foundry sector recovery backup drift detected: "
"Module Foundry map recovery backup drift detected: "
f"{name}"
)
failed_artifact = FAILED_DIR / MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT
failed_artifact = FAILED_DIR / MODULE_FOUNDRY_MAP_FAILED_ARTIFACT
try:
failed_stat = failed_artifact.lstat()
except FileNotFoundError:
die("Module Foundry sector failed artifact is missing")
die("Module Foundry map failed artifact is missing")
if (
stat.S_ISLNK(failed_stat.st_mode)
or not stat.S_ISREG(failed_stat.st_mode)
or sha256_file(failed_artifact)
!= MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT_SHA256
!= MODULE_FOUNDRY_MAP_FAILED_ARTIFACT_SHA256
):
die("Module Foundry sector failed artifact evidence mismatch")
die("Module Foundry map failed artifact evidence mismatch")
try:
state_stat = FAILED_STATE_FILE.lstat()
@@ -17400,41 +17440,41 @@ def validate_module_foundry_sector_recovery_evidence(
encoding="utf-8"
).splitlines()
except (FileNotFoundError, OSError, UnicodeDecodeError):
die("Module Foundry sector failed journal is unreadable")
die("Module Foundry map failed journal is unreadable")
if stat.S_ISLNK(state_stat.st_mode) or not stat.S_ISREG(
state_stat.st_mode
):
die("Module Foundry sector failed journal is unsafe")
die("Module Foundry map failed journal is unsafe")
records = []
for line in state_lines:
try:
value = json.loads(line)
except json.JSONDecodeError:
die("Module Foundry sector failed journal contains invalid JSON")
die("Module Foundry map failed journal contains invalid JSON")
if (
isinstance(value, dict)
and value.get("id") == MODULE_FOUNDRY_SECTOR_FAILED_PATCH_ID
and value.get("id") == MODULE_FOUNDRY_MAP_FAILED_PATCH_ID
):
records.append(value)
if len(records) != 1:
die("Module Foundry sector failed journal evidence count mismatch")
die("Module Foundry map failed journal evidence count mismatch")
record = records[0]
if (
record.get("artifact") != MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT
record.get("artifact") != MODULE_FOUNDRY_MAP_FAILED_ARTIFACT
or record.get("backup_id")
!= MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_ID
!= MODULE_FOUNDRY_MAP_FAILED_BACKUP_ID
or record.get("component") != "module-foundry"
or record.get("sha256")
!= MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT_SHA256
!= MODULE_FOUNDRY_MAP_FAILED_ARTIFACT_SHA256
or record.get("started_apply") is not True
or record.get("rollback_status") != "not-required"
or record.get("status") != "failed"
or record.get("message") != MODULE_FOUNDRY_SECTOR_FAILED_MESSAGE
or record.get("message") != MODULE_FOUNDRY_MAP_FAILED_MESSAGE
):
die("Module Foundry sector failed journal evidence mismatch")
die("Module Foundry map failed journal evidence mismatch")
with tempfile.TemporaryDirectory(
prefix="module-foundry-sector-failed-",
prefix="module-foundry-map-failed-",
dir=TMP_DIR,
) as directory:
failed_work = Path(directory)
@@ -17443,42 +17483,78 @@ def validate_module_foundry_sector_recovery_evidence(
failed_entries = parse_files_list(failed_work / "files.txt")
if (
failed_manifest.get("id")
!= MODULE_FOUNDRY_SECTOR_FAILED_PATCH_ID
!= MODULE_FOUNDRY_MAP_FAILED_PATCH_ID
or failed_manifest.get("component") != "module-foundry"
or failed_manifest.get("type") != "app-overlay"
or tuple(failed_entries)
!= MODULE_FOUNDRY_SECTOR_RECOVERY_ENTRIES
!= MODULE_FOUNDRY_SECTOR_FAILED_ENTRIES
):
die("Module Foundry sector failed artifact contract mismatch")
die("Module Foundry map failed artifact contract mismatch")
failed_payload_sha256 = collect_exact_files(
failed_work / "payload",
failed_entries,
"Module Foundry failed sector payload",
"Module Foundry failed map payload",
)
candidate_payload_sha256 = collect_exact_files(
payload_dir,
entries,
"Module Foundry sector recovery payload",
"Module Foundry map recovery payload",
)
if (
exact_file_map_sha256(candidate_payload_sha256)
!= MODULE_FOUNDRY_MAP_RECOVERY_CANDIDATE_TREE_SHA256
):
die("Module Foundry map recovery candidate drift detected")
if {
rel: candidate_payload_sha256.get(rel)
for rel in failed_payload_sha256
} != failed_payload_sha256:
die("Module Foundry failed map payload changed unexpectedly")
installed_root = component_root("module-foundry")
for rel in MODULE_FOUNDRY_MAP_RECOVERY_MISSING_PREDECESSOR_ENTRIES:
path = installed_root / rel
if path.exists() or path.is_symlink():
die(
"Module Foundry map recovery missing predecessor drift: "
f"{rel}"
)
installed_entries = tuple(
rel
for rel in entries
if rel not in MODULE_FOUNDRY_MAP_RECOVERY_MISSING_PREDECESSOR_ENTRIES
)
if candidate_payload_sha256 != failed_payload_sha256:
die("Module Foundry sector recovery payload changed unexpectedly")
installed_payload_sha256 = collect_exact_files(
component_root("module-foundry"),
entries,
"Module Foundry partial sector predecessor",
installed_root,
installed_entries,
"Module Foundry partial map predecessor",
)
if installed_payload_sha256 != candidate_payload_sha256:
die("Module Foundry partial sector predecessor drift detected")
if (
exact_file_map_sha256(installed_payload_sha256)
!= MODULE_FOUNDRY_MAP_RECOVERY_INSTALLED_TREE_SHA256
):
die("Module Foundry partial map predecessor drift detected")
if {
rel: installed_payload_sha256.get(rel)
for rel in failed_payload_sha256
} != failed_payload_sha256:
die("Module Foundry installed failed map payload drift detected")
runtime = module_foundry_runtime_inventory()
return {
"mode": "failed-overlay-runtime-reconciliation",
"failed_patch_id": MODULE_FOUNDRY_SECTOR_FAILED_PATCH_ID,
"mode": "failed-map-overlay-source+runtime-reconciliation",
"failed_patch_id": MODULE_FOUNDRY_MAP_FAILED_PATCH_ID,
"failed_artifact_sha256": (
MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT_SHA256
MODULE_FOUNDRY_MAP_FAILED_ARTIFACT_SHA256
),
"failed_backup_id": MODULE_FOUNDRY_MAP_FAILED_BACKUP_ID,
"installed_tree_sha256": (
MODULE_FOUNDRY_MAP_RECOVERY_INSTALLED_TREE_SHA256
),
"candidate_tree_sha256": (
MODULE_FOUNDRY_MAP_RECOVERY_CANDIDATE_TREE_SHA256
),
"failed_backup_id": MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_ID,
"runtime_container_id": runtime["containerId"],
"runtime_image_id": runtime["imageId"],
}
@@ -17531,7 +17607,10 @@ def rollback_module_foundry_apply(
entries,
current_stamp,
)
current = module_foundry_runtime_inventory(required=False)
current = module_foundry_runtime_inventory(
required=False,
require_healthy=False,
)
if (
current is not None
and current["containerId"] == runtime_before["containerId"]
@@ -18811,7 +18890,7 @@ def compose_service_container_id(component, service):
return container_ids[0]
def module_foundry_runtime_inventory(required=True):
def module_foundry_runtime_inventory(required=True, require_healthy=True):
result = subprocess.run(
[
*compose_base_cmd("module-foundry"),
@@ -18873,9 +18952,14 @@ def module_foundry_runtime_inventory(required=True):
)
or image_ref.startswith("sha256:")
or "@" in image_ref
or state.get("Running") is not True
or state.get("Status") != "running"
or health not in (None, "healthy")
or (
require_healthy
and (
state.get("Running") is not True
or state.get("Status") != "running"
or health not in (None, "healthy")
)
)
):
die("Module Foundry runtime inventory mismatch")
return {
@@ -20032,7 +20116,7 @@ def apply_artifact(artifact):
engine_backend_recreated = False
engine_backend_initial_mode = None
module_foundry_runtime_before = None
module_foundry_sector_recovery_preflight = None
module_foundry_map_recovery_preflight = None
runtime_started = False
current_stamp = stamp()
@@ -20111,8 +20195,8 @@ def apply_artifact(artifact):
root.mkdir(parents=True, exist_ok=True)
else:
die(f"component payload root not found: {root}")
module_foundry_sector_recovery_preflight = (
validate_module_foundry_sector_recovery_evidence(
module_foundry_map_recovery_preflight = (
validate_module_foundry_map_recovery_evidence(
manifest,
entries,
payload_dir,
@@ -20402,20 +20486,20 @@ def apply_artifact(artifact):
module_foundry_runtime_inventory()
)
if (
module_foundry_sector_recovery_preflight is not None
module_foundry_map_recovery_preflight is not None
and (
module_foundry_runtime_before["containerId"]
!= module_foundry_sector_recovery_preflight[
!= module_foundry_map_recovery_preflight[
"runtime_container_id"
]
or module_foundry_runtime_before["imageId"]
!= module_foundry_sector_recovery_preflight[
!= module_foundry_map_recovery_preflight[
"runtime_image_id"
]
)
):
die(
"Module Foundry sector predecessor runtime "
"Module Foundry map predecessor runtime "
"changed during preflight"
)