FIX - NAS DEPLOY: reconcile Foundry map runtime generation
This commit is contained in:
@@ -0,0 +1,304 @@
|
|||||||
|
#!/usr/bin/env node
|
||||||
|
|
||||||
|
import { createHash } from "node:crypto";
|
||||||
|
import { spawnSync } from "node:child_process";
|
||||||
|
import {
|
||||||
|
cp,
|
||||||
|
lstat,
|
||||||
|
mkdir,
|
||||||
|
mkdtemp,
|
||||||
|
readdir,
|
||||||
|
readFile,
|
||||||
|
rm,
|
||||||
|
writeFile,
|
||||||
|
} from "node:fs/promises";
|
||||||
|
import { tmpdir } from "node:os";
|
||||||
|
import { dirname, join, relative, resolve } from "node:path";
|
||||||
|
import { fileURLToPath } from "node:url";
|
||||||
|
|
||||||
|
const scriptDir = dirname(fileURLToPath(import.meta.url));
|
||||||
|
const declaredFoundryRoot = process.env.NODEDC_FOUNDRY_SOURCE_ROOT;
|
||||||
|
if (!declaredFoundryRoot) {
|
||||||
|
throw new Error("NODEDC_FOUNDRY_SOURCE_ROOT_is_required");
|
||||||
|
}
|
||||||
|
const foundryRoot = resolve(declaredFoundryRoot);
|
||||||
|
const expectedSourceCommit = "58800d957632320fa6717ec3fcff972023528759";
|
||||||
|
const artifactDir = resolve(
|
||||||
|
process.env.NODEDC_DEPLOY_ARTIFACT_DIR
|
||||||
|
|| resolve(scriptDir, "../deploy-artifacts"),
|
||||||
|
);
|
||||||
|
const [
|
||||||
|
patchId = "module-foundry-map-runtime-recovery-20260809-002",
|
||||||
|
...extra
|
||||||
|
] = process.argv.slice(2);
|
||||||
|
|
||||||
|
if (
|
||||||
|
extra.length
|
||||||
|
|| !/^module-foundry-map-runtime-recovery-\d{8}-\d{3}$/.test(patchId)
|
||||||
|
) {
|
||||||
|
throw new Error(
|
||||||
|
"usage: build-module-foundry-map-runtime-recovery-artifact.mjs "
|
||||||
|
+ "[module-foundry-map-runtime-recovery-YYYYMMDD-NNN]",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const files = Object.freeze([
|
||||||
|
"package.json",
|
||||||
|
"package-lock.json",
|
||||||
|
"apps/catalog/src",
|
||||||
|
"packages",
|
||||||
|
"registry",
|
||||||
|
"runtime-seed/page-layouts/map.json",
|
||||||
|
"scripts",
|
||||||
|
"server/catalog-server.mjs",
|
||||||
|
"server/foundry-mcp.mjs",
|
||||||
|
"server/map-grid-persistence.test.mjs",
|
||||||
|
]);
|
||||||
|
const ignoredBasenames = new Set([
|
||||||
|
".DS_Store",
|
||||||
|
".git",
|
||||||
|
"node_modules",
|
||||||
|
"runtime-data",
|
||||||
|
"dist",
|
||||||
|
]);
|
||||||
|
const artifact = join(artifactDir, `nodedc-${patchId}.tgz`);
|
||||||
|
const checksum = `${artifact}.sha256`;
|
||||||
|
const stage = await mkdtemp(
|
||||||
|
join(tmpdir(), "nodedc-foundry-map-runtime-recovery-"),
|
||||||
|
);
|
||||||
|
|
||||||
|
await assertFresh(artifact);
|
||||||
|
await assertExactSource();
|
||||||
|
await assertRecoveryBoundary();
|
||||||
|
|
||||||
|
try {
|
||||||
|
for (const sourceRelative of files) {
|
||||||
|
await copySafe(
|
||||||
|
resolve(foundryRoot, sourceRelative),
|
||||||
|
join(stage, "payload", sourceRelative),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
await writeFile(
|
||||||
|
join(stage, "manifest.env"),
|
||||||
|
`id=${patchId}\ncomponent=module-foundry\ntype=app-overlay\n`,
|
||||||
|
"utf8",
|
||||||
|
);
|
||||||
|
await writeFile(join(stage, "files.txt"), `${files.join("\n")}\n`, "utf8");
|
||||||
|
await mkdir(artifactDir, { recursive: true });
|
||||||
|
run("python3", ["-c", canonicalTarScript(), artifact, stage]);
|
||||||
|
const sha256 = digest(await readFile(artifact));
|
||||||
|
await writeFile(
|
||||||
|
checksum,
|
||||||
|
`${sha256} ${artifact.split("/").at(-1)}\n`,
|
||||||
|
"utf8",
|
||||||
|
);
|
||||||
|
console.log(JSON.stringify({
|
||||||
|
ok: true,
|
||||||
|
patchId,
|
||||||
|
artifact,
|
||||||
|
checksum,
|
||||||
|
sha256,
|
||||||
|
sourceCommit: expectedSourceCommit,
|
||||||
|
services: ["nodedc-module-foundry"],
|
||||||
|
transition: "reconcile-partial-map-source-and-activate-runtime",
|
||||||
|
files,
|
||||||
|
}, null, 2));
|
||||||
|
} finally {
|
||||||
|
await rm(stage, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
|
||||||
|
async function assertExactSource() {
|
||||||
|
const declared = process.env.NODEDC_FOUNDRY_SOURCE_COMMIT;
|
||||||
|
if (declared !== expectedSourceCommit) {
|
||||||
|
throw new Error("foundry_source_commit_mismatch");
|
||||||
|
}
|
||||||
|
const dotGit = join(foundryRoot, ".git");
|
||||||
|
try {
|
||||||
|
await lstat(dotGit);
|
||||||
|
} catch (error) {
|
||||||
|
if (error?.code === "ENOENT") return;
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
const tracked = spawnSync(
|
||||||
|
"git",
|
||||||
|
["-C", foundryRoot, "diff", "--quiet", "HEAD", "--", ...files],
|
||||||
|
{ encoding: "utf8" },
|
||||||
|
);
|
||||||
|
if (tracked.status !== 0) {
|
||||||
|
throw new Error("foundry_selected_source_is_dirty");
|
||||||
|
}
|
||||||
|
const untracked = git(foundryRoot, [
|
||||||
|
"ls-files",
|
||||||
|
"--others",
|
||||||
|
"--exclude-standard",
|
||||||
|
"--",
|
||||||
|
...files,
|
||||||
|
]);
|
||||||
|
if (untracked) {
|
||||||
|
throw new Error("foundry_selected_source_has_untracked_files");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function assertRecoveryBoundary() {
|
||||||
|
const packageJson = await readFile(join(foundryRoot, "package.json"), "utf8");
|
||||||
|
const packageLock = await readFile(
|
||||||
|
join(foundryRoot, "package-lock.json"),
|
||||||
|
"utf8",
|
||||||
|
);
|
||||||
|
const preview = await readFile(
|
||||||
|
join(foundryRoot, "apps/catalog/src/MapFixturePreview.tsx"),
|
||||||
|
"utf8",
|
||||||
|
);
|
||||||
|
const renderer = await readFile(
|
||||||
|
join(foundryRoot, "apps/catalog/src/CesiumMapRenderer.tsx"),
|
||||||
|
"utf8",
|
||||||
|
);
|
||||||
|
const policy = await readFile(
|
||||||
|
join(foundryRoot, "apps/catalog/src/mapGridPolicy.mjs"),
|
||||||
|
"utf8",
|
||||||
|
);
|
||||||
|
const sector = await readFile(
|
||||||
|
join(foundryRoot, "apps/catalog/src/mapSectorGrid.mjs"),
|
||||||
|
"utf8",
|
||||||
|
);
|
||||||
|
const server = await readFile(
|
||||||
|
join(foundryRoot, "server/catalog-server.mjs"),
|
||||||
|
"utf8",
|
||||||
|
);
|
||||||
|
const registry = await readFile(
|
||||||
|
join(foundryRoot, "registry/registry.json"),
|
||||||
|
"utf8",
|
||||||
|
);
|
||||||
|
|
||||||
|
for (const marker of [
|
||||||
|
"@nodedc/map-cesium-react",
|
||||||
|
"npm run build --workspace @nodedc/map-cesium-react",
|
||||||
|
]) {
|
||||||
|
if (!packageJson.includes(marker)) {
|
||||||
|
throw new Error(`map_package_boundary_missing:${marker}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!packageLock.includes('"packages/map-cesium-react"')) {
|
||||||
|
throw new Error("map_package_lock_boundary_missing");
|
||||||
|
}
|
||||||
|
for (const marker of [
|
||||||
|
"const sectorSpatialEntities = useMemo",
|
||||||
|
"runtimeBindings={[...sectorScopedPrimaryRuntimeBindings, ...referenceRuntimeBindings]}",
|
||||||
|
'label="Скрыть объекты за сектором"',
|
||||||
|
]) {
|
||||||
|
if (!preview.includes(marker)) {
|
||||||
|
throw new Error(`map_workspace_boundary_missing:${marker}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for (const marker of [
|
||||||
|
"class GridLayerController",
|
||||||
|
"gridLegacyMode",
|
||||||
|
"viewer.flyTo(entity",
|
||||||
|
]) {
|
||||||
|
if (!renderer.includes(marker)) {
|
||||||
|
throw new Error(`map_renderer_boundary_missing:${marker}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for (const marker of [
|
||||||
|
"gridLodProfiles",
|
||||||
|
"GRID_LOD_HYSTERESIS_RATIO",
|
||||||
|
]) {
|
||||||
|
if (!policy.includes(marker)) {
|
||||||
|
throw new Error(`map_policy_boundary_missing:${marker}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for (const marker of [
|
||||||
|
"export function geodeticToLocalGridPlane",
|
||||||
|
"export function localSectorAtGeodetic",
|
||||||
|
]) {
|
||||||
|
if (!sector.includes(marker)) {
|
||||||
|
throw new Error(`map_sector_boundary_missing:${marker}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for (const marker of [
|
||||||
|
"gridLodProfiles",
|
||||||
|
"validateGridLodProfiles",
|
||||||
|
]) {
|
||||||
|
if (!server.includes(marker)) {
|
||||||
|
throw new Error(`map_server_boundary_missing:${marker}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!registry.includes('"@nodedc/map-cesium-react"')) {
|
||||||
|
throw new Error("map_registry_boundary_missing");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function copySafe(source, destination) {
|
||||||
|
const sourceStat = await lstat(source);
|
||||||
|
if (sourceStat.isSymbolicLink()) {
|
||||||
|
throw new Error(
|
||||||
|
`source_symlink_rejected:${relative(foundryRoot, source)}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (sourceStat.isFile()) {
|
||||||
|
await mkdir(dirname(destination), { recursive: true });
|
||||||
|
await cp(source, destination, { force: true, verbatimSymlinks: true });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!sourceStat.isDirectory()) {
|
||||||
|
throw new Error(`source_type_rejected:${relative(foundryRoot, source)}`);
|
||||||
|
}
|
||||||
|
await mkdir(destination, { recursive: true });
|
||||||
|
for (const entry of await readdir(source, { withFileTypes: true })) {
|
||||||
|
if (
|
||||||
|
ignoredBasenames.has(entry.name)
|
||||||
|
|| entry.name.startsWith(".env")
|
||||||
|
|| entry.name.endsWith(".tsbuildinfo")
|
||||||
|
) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const childSource = join(source, entry.name);
|
||||||
|
const childDestination = join(destination, entry.name);
|
||||||
|
if (entry.isSymbolicLink()) {
|
||||||
|
throw new Error(
|
||||||
|
`source_symlink_rejected:${relative(foundryRoot, childSource)}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
await copySafe(childSource, childDestination);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function assertFresh(path) {
|
||||||
|
try {
|
||||||
|
await lstat(path);
|
||||||
|
} catch (error) {
|
||||||
|
if (error?.code === "ENOENT") return;
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
throw new Error("artifact_already_exists");
|
||||||
|
}
|
||||||
|
|
||||||
|
function canonicalTarScript() {
|
||||||
|
return [
|
||||||
|
"import gzip,io,pathlib,sys,tarfile",
|
||||||
|
"root=pathlib.Path(sys.argv[2])",
|
||||||
|
"with open(sys.argv[1],'wb') as out:",
|
||||||
|
" with gzip.GzipFile(filename='',mode='wb',fileobj=out,compresslevel=9,mtime=0) as gz:",
|
||||||
|
" with tarfile.open(fileobj=gz,mode='w',format=tarfile.PAX_FORMAT) as tar:",
|
||||||
|
" for top in ('manifest.env','files.txt','payload'):",
|
||||||
|
" p=root/top; paths=[p]+(sorted(p.rglob('*')) if p.is_dir() else [])",
|
||||||
|
" for x in paths:",
|
||||||
|
" info=tar.gettarinfo(str(x),arcname=x.relative_to(root).as_posix())",
|
||||||
|
" info.uid=info.gid=0; info.uname=info.gname='root'; info.mtime=0; info.mode=0o755 if info.isdir() else 0o644",
|
||||||
|
" with (open(x,'rb') if info.isfile() else io.BytesIO()) as src: tar.addfile(info,src if info.isfile() else None)",
|
||||||
|
].join("\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
function digest(value) {
|
||||||
|
return createHash("sha256").update(value).digest("hex");
|
||||||
|
}
|
||||||
|
|
||||||
|
function run(command, args) {
|
||||||
|
const result = spawnSync(command, args, {
|
||||||
|
encoding: "utf8",
|
||||||
|
maxBuffer: 128 * 1024 * 1024,
|
||||||
|
});
|
||||||
|
if (result.status !== 0) {
|
||||||
|
throw new Error(`${command}_failed:${result.stderr || result.stdout}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -38,10 +38,31 @@ DOCKER = Path("/usr/local/bin/docker")
|
|||||||
MODULE_FOUNDRY_SERVICE = "nodedc-module-foundry"
|
MODULE_FOUNDRY_SERVICE = "nodedc-module-foundry"
|
||||||
MODULE_FOUNDRY_RUNTIME_BEFORE_FILE = "module-foundry-runtime-before.json"
|
MODULE_FOUNDRY_RUNTIME_BEFORE_FILE = "module-foundry-runtime-before.json"
|
||||||
MODULE_FOUNDRY_BUILD_ATTEMPTS = 2
|
MODULE_FOUNDRY_BUILD_ATTEMPTS = 2
|
||||||
MODULE_FOUNDRY_SECTOR_RECOVERY_PATCH_ID = (
|
MODULE_FOUNDRY_MAP_RECOVERY_PATCH_ID = (
|
||||||
"module-foundry-map-sector-workspace-20260809-001"
|
"module-foundry-map-runtime-recovery-20260809-002"
|
||||||
)
|
)
|
||||||
MODULE_FOUNDRY_SECTOR_RECOVERY_ENTRIES = (
|
MODULE_FOUNDRY_MAP_RECOVERY_ENTRIES = (
|
||||||
|
"package.json",
|
||||||
|
"package-lock.json",
|
||||||
|
"apps/catalog/src",
|
||||||
|
"packages",
|
||||||
|
"registry",
|
||||||
|
"runtime-seed/page-layouts/map.json",
|
||||||
|
"scripts",
|
||||||
|
"server/catalog-server.mjs",
|
||||||
|
"server/foundry-mcp.mjs",
|
||||||
|
"server/map-grid-persistence.test.mjs",
|
||||||
|
)
|
||||||
|
MODULE_FOUNDRY_MAP_RECOVERY_MISSING_PREDECESSOR_ENTRIES = (
|
||||||
|
"server/map-grid-persistence.test.mjs",
|
||||||
|
)
|
||||||
|
MODULE_FOUNDRY_MAP_RECOVERY_INSTALLED_TREE_SHA256 = (
|
||||||
|
"8513127fb3b875dc3326c0305441723cda6c405525a68cc11030a6f645738d2b"
|
||||||
|
)
|
||||||
|
MODULE_FOUNDRY_MAP_RECOVERY_CANDIDATE_TREE_SHA256 = (
|
||||||
|
"b89392e56fbaf6a8450ab67fcf263199eadba4a74ef7b70b4d8d685a57c2edc3"
|
||||||
|
)
|
||||||
|
MODULE_FOUNDRY_SECTOR_FAILED_ENTRIES = (
|
||||||
"apps/catalog/src/MapFixturePreview.tsx",
|
"apps/catalog/src/MapFixturePreview.tsx",
|
||||||
"apps/catalog/src/mapPresentationProfile.ts",
|
"apps/catalog/src/mapPresentationProfile.ts",
|
||||||
"apps/catalog/src/mapSectorGrid.d.mts",
|
"apps/catalog/src/mapSectorGrid.d.mts",
|
||||||
@@ -51,21 +72,21 @@ MODULE_FOUNDRY_SECTOR_RECOVERY_ENTRIES = (
|
|||||||
"scripts/map-presentation-filters.test.mjs",
|
"scripts/map-presentation-filters.test.mjs",
|
||||||
"scripts/map-sector-grid.test.mjs",
|
"scripts/map-sector-grid.test.mjs",
|
||||||
)
|
)
|
||||||
MODULE_FOUNDRY_SECTOR_FAILED_PATCH_ID = (
|
MODULE_FOUNDRY_MAP_FAILED_PATCH_ID = (
|
||||||
"module-foundry-map-sector-workspace-20260808-001"
|
"module-foundry-map-sector-workspace-20260809-001"
|
||||||
)
|
)
|
||||||
MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT = (
|
MODULE_FOUNDRY_MAP_FAILED_ARTIFACT = (
|
||||||
"nodedc-module-foundry-map-sector-workspace-20260808-001.tgz."
|
"nodedc-module-foundry-map-sector-workspace-20260809-001.tgz."
|
||||||
"20260809-014720"
|
"20260809-022707"
|
||||||
)
|
)
|
||||||
MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT_SHA256 = (
|
MODULE_FOUNDRY_MAP_FAILED_ARTIFACT_SHA256 = (
|
||||||
"0b45002ffcd3270d56add85fc206cd34614ed10edfb57024688347cbe7e5e740"
|
"96f67297efe6033f545e797dc4e341983b2ce9ca6ebf6ff48a00f6ee8b8ea23b"
|
||||||
)
|
)
|
||||||
MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_ID = (
|
MODULE_FOUNDRY_MAP_FAILED_BACKUP_ID = (
|
||||||
"module-foundry-module-foundry-map-sector-workspace-20260808-001-"
|
"module-foundry-module-foundry-map-sector-workspace-20260809-001-"
|
||||||
"20260809-014720"
|
"20260809-022707"
|
||||||
)
|
)
|
||||||
MODULE_FOUNDRY_SECTOR_FAILED_MESSAGE = (
|
MODULE_FOUNDRY_MAP_FAILED_MESSAGE = (
|
||||||
"Command '['/usr/local/bin/docker', 'compose', '-p', "
|
"Command '['/usr/local/bin/docker', 'compose', '-p', "
|
||||||
"'nodedc-module-foundry', '--env-file', "
|
"'nodedc-module-foundry', '--env-file', "
|
||||||
"'/volume1/docker/nodedc-platform/module-foundry/source/.env', '-f', "
|
"'/volume1/docker/nodedc-platform/module-foundry/source/.env', '-f', "
|
||||||
@@ -74,21 +95,21 @@ MODULE_FOUNDRY_SECTOR_FAILED_MESSAGE = (
|
|||||||
"'--build', '--no-deps', 'nodedc-module-foundry']' returned non-zero "
|
"'--build', '--no-deps', 'nodedc-module-foundry']' returned non-zero "
|
||||||
"exit status 17."
|
"exit status 17."
|
||||||
)
|
)
|
||||||
MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_SHA256 = {
|
MODULE_FOUNDRY_MAP_FAILED_BACKUP_SHA256 = {
|
||||||
"existing-files.txt": (
|
"existing-files.txt": (
|
||||||
"57273169e590f1de2de46fb201aff5164af2171cecb832a090669ab5cc93bb1c"
|
"9273d70c9d1fd9a12963f1f66d2994ec7b7c216ebbae27e254931b8fc2a07ea3"
|
||||||
),
|
),
|
||||||
"files.txt": (
|
"files.txt": (
|
||||||
"9273d70c9d1fd9a12963f1f66d2994ec7b7c216ebbae27e254931b8fc2a07ea3"
|
"9273d70c9d1fd9a12963f1f66d2994ec7b7c216ebbae27e254931b8fc2a07ea3"
|
||||||
),
|
),
|
||||||
"manifest.env": (
|
"manifest.env": (
|
||||||
"cf6a5263095bb56e3bb3543bcf5ab23b1893c1d23ceea1a7445e7b76924c2aa2"
|
"fd1ea43c173d89e358731415e528220aa72b1a7c27b6ee8ba408292d8ddf41a2"
|
||||||
),
|
),
|
||||||
"missing-files.txt": (
|
"missing-files.txt": (
|
||||||
"1cbc6098562af19a9fb2abefc0e41f7a1178a436bb82d58a9a4f3e989c19f38f"
|
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
|
||||||
),
|
),
|
||||||
"source-before.tgz": (
|
"source-before.tgz": (
|
||||||
"bf7e011bcf2c932e952917dc0f7ba48b84bde0b833c7116714bcc618932d5562"
|
"e3f14d55d5eaff456a9001fb4a1234b5b584afb93cf4d0c929b30312bd7b5668"
|
||||||
),
|
),
|
||||||
}
|
}
|
||||||
MAP_GATEWAY_SECRET_DIR = Path("/volume1/docker/nodedc-platform/secrets")
|
MAP_GATEWAY_SECRET_DIR = Path("/volume1/docker/nodedc-platform/secrets")
|
||||||
@@ -15017,14 +15038,14 @@ def plan_artifact(artifact):
|
|||||||
provider_catalog_preflight = None
|
provider_catalog_preflight = None
|
||||||
device_plane_postgres_preflight = None
|
device_plane_postgres_preflight = None
|
||||||
device_plane_foundation_recovery_preflight = None
|
device_plane_foundation_recovery_preflight = None
|
||||||
module_foundry_sector_recovery_preflight = None
|
module_foundry_map_recovery_preflight = None
|
||||||
with tempfile.TemporaryDirectory(prefix="plan-", dir=TMP_DIR) as tmp:
|
with tempfile.TemporaryDirectory(prefix="plan-", dir=TMP_DIR) as tmp:
|
||||||
manifest, entries, payload_dir = load_artifact(artifact, Path(tmp))
|
manifest, entries, payload_dir = load_artifact(artifact, Path(tmp))
|
||||||
reject_terminal_engine_l2_failed_artifact(manifest, sha)
|
reject_terminal_engine_l2_failed_artifact(manifest, sha)
|
||||||
reject_terminal_device_plane_foundation_artifact(manifest, sha)
|
reject_terminal_device_plane_foundation_artifact(manifest, sha)
|
||||||
reject_terminal_device_plane_backhaul_artifact(manifest, sha)
|
reject_terminal_device_plane_backhaul_artifact(manifest, sha)
|
||||||
module_foundry_sector_recovery_preflight = (
|
module_foundry_map_recovery_preflight = (
|
||||||
validate_module_foundry_sector_recovery_evidence(
|
validate_module_foundry_map_recovery_evidence(
|
||||||
manifest,
|
manifest,
|
||||||
entries,
|
entries,
|
||||||
payload_dir,
|
payload_dir,
|
||||||
@@ -16590,30 +16611,38 @@ def plan_artifact(artifact):
|
|||||||
)
|
)
|
||||||
print("module_foundry_runtime_pull=never")
|
print("module_foundry_runtime_pull=never")
|
||||||
print("module_foundry_rollback=source+image+runtime")
|
print("module_foundry_rollback=source+image+runtime")
|
||||||
if module_foundry_sector_recovery_preflight is not None:
|
if module_foundry_map_recovery_preflight is not None:
|
||||||
print(
|
print(
|
||||||
"module_foundry_transition="
|
"module_foundry_transition="
|
||||||
f"{module_foundry_sector_recovery_preflight['mode']}"
|
f"{module_foundry_map_recovery_preflight['mode']}"
|
||||||
)
|
)
|
||||||
print(
|
print(
|
||||||
"module_foundry_failed_predecessor="
|
"module_foundry_failed_predecessor="
|
||||||
f"{module_foundry_sector_recovery_preflight['failed_patch_id']}"
|
f"{module_foundry_map_recovery_preflight['failed_patch_id']}"
|
||||||
)
|
)
|
||||||
print(
|
print(
|
||||||
"module_foundry_failed_predecessor_sha256="
|
"module_foundry_failed_predecessor_sha256="
|
||||||
f"{module_foundry_sector_recovery_preflight['failed_artifact_sha256']}"
|
f"{module_foundry_map_recovery_preflight['failed_artifact_sha256']}"
|
||||||
)
|
)
|
||||||
print(
|
print(
|
||||||
"module_foundry_failed_backup="
|
"module_foundry_failed_backup="
|
||||||
f"{module_foundry_sector_recovery_preflight['failed_backup_id']}"
|
f"{module_foundry_map_recovery_preflight['failed_backup_id']}"
|
||||||
|
)
|
||||||
|
print(
|
||||||
|
"module_foundry_installed_tree_sha256="
|
||||||
|
f"{module_foundry_map_recovery_preflight['installed_tree_sha256']}"
|
||||||
|
)
|
||||||
|
print(
|
||||||
|
"module_foundry_candidate_tree_sha256="
|
||||||
|
f"{module_foundry_map_recovery_preflight['candidate_tree_sha256']}"
|
||||||
)
|
)
|
||||||
print(
|
print(
|
||||||
"module_foundry_predecessor_container="
|
"module_foundry_predecessor_container="
|
||||||
f"{module_foundry_sector_recovery_preflight['runtime_container_id']}"
|
f"{module_foundry_map_recovery_preflight['runtime_container_id']}"
|
||||||
)
|
)
|
||||||
print(
|
print(
|
||||||
"module_foundry_predecessor_image="
|
"module_foundry_predecessor_image="
|
||||||
f"{module_foundry_sector_recovery_preflight['runtime_image_id']}"
|
f"{module_foundry_map_recovery_preflight['runtime_image_id']}"
|
||||||
)
|
)
|
||||||
if component == "device-plane":
|
if component == "device-plane":
|
||||||
print(f"runtime_secret=runner-managed:{DEVICE_PLANE_POSTGRES_PASSWORD_FILE}")
|
print(f"runtime_secret=runner-managed:{DEVICE_PLANE_POSTGRES_PASSWORD_FILE}")
|
||||||
@@ -17336,38 +17365,49 @@ def rollback_engine_apply(root, backup_dir, entries, current_stamp, runtime_star
|
|||||||
return f"source+runtime-restored:{restored_count}"
|
return f"source+runtime-restored:{restored_count}"
|
||||||
|
|
||||||
|
|
||||||
def is_module_foundry_sector_recovery_slice(manifest, entries):
|
def is_module_foundry_map_recovery_slice(manifest, entries):
|
||||||
return (
|
return (
|
||||||
manifest.get("id") == MODULE_FOUNDRY_SECTOR_RECOVERY_PATCH_ID
|
manifest.get("id") == MODULE_FOUNDRY_MAP_RECOVERY_PATCH_ID
|
||||||
and manifest.get("component") == "module-foundry"
|
and manifest.get("component") == "module-foundry"
|
||||||
and manifest.get("type") == "app-overlay"
|
and manifest.get("type") == "app-overlay"
|
||||||
and tuple(entries or ()) == MODULE_FOUNDRY_SECTOR_RECOVERY_ENTRIES
|
and tuple(entries or ()) == MODULE_FOUNDRY_MAP_RECOVERY_ENTRIES
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def validate_module_foundry_sector_recovery_evidence(
|
def exact_file_map_sha256(files):
|
||||||
|
return hashlib.sha256(
|
||||||
|
json.dumps(
|
||||||
|
files,
|
||||||
|
ensure_ascii=True,
|
||||||
|
separators=(",", ":"),
|
||||||
|
sort_keys=True,
|
||||||
|
).encode("utf-8")
|
||||||
|
).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def validate_module_foundry_map_recovery_evidence(
|
||||||
manifest,
|
manifest,
|
||||||
entries,
|
entries,
|
||||||
payload_dir,
|
payload_dir,
|
||||||
):
|
):
|
||||||
if not is_module_foundry_sector_recovery_slice(manifest, entries):
|
if not is_module_foundry_map_recovery_slice(manifest, entries):
|
||||||
return None
|
return None
|
||||||
|
|
||||||
backup_dir = BACKUPS_DIR / MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_ID
|
backup_dir = BACKUPS_DIR / MODULE_FOUNDRY_MAP_FAILED_BACKUP_ID
|
||||||
try:
|
try:
|
||||||
backup_stat = backup_dir.lstat()
|
backup_stat = backup_dir.lstat()
|
||||||
except FileNotFoundError:
|
except FileNotFoundError:
|
||||||
die("Module Foundry sector recovery backup is missing")
|
die("Module Foundry map recovery backup is missing")
|
||||||
if stat.S_ISLNK(backup_stat.st_mode) or not stat.S_ISDIR(
|
if stat.S_ISLNK(backup_stat.st_mode) or not stat.S_ISDIR(
|
||||||
backup_stat.st_mode
|
backup_stat.st_mode
|
||||||
):
|
):
|
||||||
die("Module Foundry sector recovery backup is unsafe")
|
die("Module Foundry map recovery backup is unsafe")
|
||||||
if {child.name for child in backup_dir.iterdir()} != set(
|
if {child.name for child in backup_dir.iterdir()} != set(
|
||||||
MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_SHA256
|
MODULE_FOUNDRY_MAP_FAILED_BACKUP_SHA256
|
||||||
):
|
):
|
||||||
die("Module Foundry sector recovery backup file set mismatch")
|
die("Module Foundry map recovery backup file set mismatch")
|
||||||
for name, expected_sha256 in (
|
for name, expected_sha256 in (
|
||||||
MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_SHA256.items()
|
MODULE_FOUNDRY_MAP_FAILED_BACKUP_SHA256.items()
|
||||||
):
|
):
|
||||||
path = backup_dir / name
|
path = backup_dir / name
|
||||||
path_stat = path.lstat()
|
path_stat = path.lstat()
|
||||||
@@ -17377,22 +17417,22 @@ def validate_module_foundry_sector_recovery_evidence(
|
|||||||
or sha256_file(path) != expected_sha256
|
or sha256_file(path) != expected_sha256
|
||||||
):
|
):
|
||||||
die(
|
die(
|
||||||
"Module Foundry sector recovery backup drift detected: "
|
"Module Foundry map recovery backup drift detected: "
|
||||||
f"{name}"
|
f"{name}"
|
||||||
)
|
)
|
||||||
|
|
||||||
failed_artifact = FAILED_DIR / MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT
|
failed_artifact = FAILED_DIR / MODULE_FOUNDRY_MAP_FAILED_ARTIFACT
|
||||||
try:
|
try:
|
||||||
failed_stat = failed_artifact.lstat()
|
failed_stat = failed_artifact.lstat()
|
||||||
except FileNotFoundError:
|
except FileNotFoundError:
|
||||||
die("Module Foundry sector failed artifact is missing")
|
die("Module Foundry map failed artifact is missing")
|
||||||
if (
|
if (
|
||||||
stat.S_ISLNK(failed_stat.st_mode)
|
stat.S_ISLNK(failed_stat.st_mode)
|
||||||
or not stat.S_ISREG(failed_stat.st_mode)
|
or not stat.S_ISREG(failed_stat.st_mode)
|
||||||
or sha256_file(failed_artifact)
|
or sha256_file(failed_artifact)
|
||||||
!= MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT_SHA256
|
!= MODULE_FOUNDRY_MAP_FAILED_ARTIFACT_SHA256
|
||||||
):
|
):
|
||||||
die("Module Foundry sector failed artifact evidence mismatch")
|
die("Module Foundry map failed artifact evidence mismatch")
|
||||||
|
|
||||||
try:
|
try:
|
||||||
state_stat = FAILED_STATE_FILE.lstat()
|
state_stat = FAILED_STATE_FILE.lstat()
|
||||||
@@ -17400,41 +17440,41 @@ def validate_module_foundry_sector_recovery_evidence(
|
|||||||
encoding="utf-8"
|
encoding="utf-8"
|
||||||
).splitlines()
|
).splitlines()
|
||||||
except (FileNotFoundError, OSError, UnicodeDecodeError):
|
except (FileNotFoundError, OSError, UnicodeDecodeError):
|
||||||
die("Module Foundry sector failed journal is unreadable")
|
die("Module Foundry map failed journal is unreadable")
|
||||||
if stat.S_ISLNK(state_stat.st_mode) or not stat.S_ISREG(
|
if stat.S_ISLNK(state_stat.st_mode) or not stat.S_ISREG(
|
||||||
state_stat.st_mode
|
state_stat.st_mode
|
||||||
):
|
):
|
||||||
die("Module Foundry sector failed journal is unsafe")
|
die("Module Foundry map failed journal is unsafe")
|
||||||
records = []
|
records = []
|
||||||
for line in state_lines:
|
for line in state_lines:
|
||||||
try:
|
try:
|
||||||
value = json.loads(line)
|
value = json.loads(line)
|
||||||
except json.JSONDecodeError:
|
except json.JSONDecodeError:
|
||||||
die("Module Foundry sector failed journal contains invalid JSON")
|
die("Module Foundry map failed journal contains invalid JSON")
|
||||||
if (
|
if (
|
||||||
isinstance(value, dict)
|
isinstance(value, dict)
|
||||||
and value.get("id") == MODULE_FOUNDRY_SECTOR_FAILED_PATCH_ID
|
and value.get("id") == MODULE_FOUNDRY_MAP_FAILED_PATCH_ID
|
||||||
):
|
):
|
||||||
records.append(value)
|
records.append(value)
|
||||||
if len(records) != 1:
|
if len(records) != 1:
|
||||||
die("Module Foundry sector failed journal evidence count mismatch")
|
die("Module Foundry map failed journal evidence count mismatch")
|
||||||
record = records[0]
|
record = records[0]
|
||||||
if (
|
if (
|
||||||
record.get("artifact") != MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT
|
record.get("artifact") != MODULE_FOUNDRY_MAP_FAILED_ARTIFACT
|
||||||
or record.get("backup_id")
|
or record.get("backup_id")
|
||||||
!= MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_ID
|
!= MODULE_FOUNDRY_MAP_FAILED_BACKUP_ID
|
||||||
or record.get("component") != "module-foundry"
|
or record.get("component") != "module-foundry"
|
||||||
or record.get("sha256")
|
or record.get("sha256")
|
||||||
!= MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT_SHA256
|
!= MODULE_FOUNDRY_MAP_FAILED_ARTIFACT_SHA256
|
||||||
or record.get("started_apply") is not True
|
or record.get("started_apply") is not True
|
||||||
or record.get("rollback_status") != "not-required"
|
or record.get("rollback_status") != "not-required"
|
||||||
or record.get("status") != "failed"
|
or record.get("status") != "failed"
|
||||||
or record.get("message") != MODULE_FOUNDRY_SECTOR_FAILED_MESSAGE
|
or record.get("message") != MODULE_FOUNDRY_MAP_FAILED_MESSAGE
|
||||||
):
|
):
|
||||||
die("Module Foundry sector failed journal evidence mismatch")
|
die("Module Foundry map failed journal evidence mismatch")
|
||||||
|
|
||||||
with tempfile.TemporaryDirectory(
|
with tempfile.TemporaryDirectory(
|
||||||
prefix="module-foundry-sector-failed-",
|
prefix="module-foundry-map-failed-",
|
||||||
dir=TMP_DIR,
|
dir=TMP_DIR,
|
||||||
) as directory:
|
) as directory:
|
||||||
failed_work = Path(directory)
|
failed_work = Path(directory)
|
||||||
@@ -17443,42 +17483,78 @@ def validate_module_foundry_sector_recovery_evidence(
|
|||||||
failed_entries = parse_files_list(failed_work / "files.txt")
|
failed_entries = parse_files_list(failed_work / "files.txt")
|
||||||
if (
|
if (
|
||||||
failed_manifest.get("id")
|
failed_manifest.get("id")
|
||||||
!= MODULE_FOUNDRY_SECTOR_FAILED_PATCH_ID
|
!= MODULE_FOUNDRY_MAP_FAILED_PATCH_ID
|
||||||
or failed_manifest.get("component") != "module-foundry"
|
or failed_manifest.get("component") != "module-foundry"
|
||||||
or failed_manifest.get("type") != "app-overlay"
|
or failed_manifest.get("type") != "app-overlay"
|
||||||
or tuple(failed_entries)
|
or tuple(failed_entries)
|
||||||
!= MODULE_FOUNDRY_SECTOR_RECOVERY_ENTRIES
|
!= MODULE_FOUNDRY_SECTOR_FAILED_ENTRIES
|
||||||
):
|
):
|
||||||
die("Module Foundry sector failed artifact contract mismatch")
|
die("Module Foundry map failed artifact contract mismatch")
|
||||||
failed_payload_sha256 = collect_exact_files(
|
failed_payload_sha256 = collect_exact_files(
|
||||||
failed_work / "payload",
|
failed_work / "payload",
|
||||||
failed_entries,
|
failed_entries,
|
||||||
"Module Foundry failed sector payload",
|
"Module Foundry failed map payload",
|
||||||
)
|
)
|
||||||
|
|
||||||
candidate_payload_sha256 = collect_exact_files(
|
candidate_payload_sha256 = collect_exact_files(
|
||||||
payload_dir,
|
payload_dir,
|
||||||
entries,
|
entries,
|
||||||
"Module Foundry sector recovery payload",
|
"Module Foundry map recovery payload",
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
exact_file_map_sha256(candidate_payload_sha256)
|
||||||
|
!= MODULE_FOUNDRY_MAP_RECOVERY_CANDIDATE_TREE_SHA256
|
||||||
|
):
|
||||||
|
die("Module Foundry map recovery candidate drift detected")
|
||||||
|
if {
|
||||||
|
rel: candidate_payload_sha256.get(rel)
|
||||||
|
for rel in failed_payload_sha256
|
||||||
|
} != failed_payload_sha256:
|
||||||
|
die("Module Foundry failed map payload changed unexpectedly")
|
||||||
|
|
||||||
|
installed_root = component_root("module-foundry")
|
||||||
|
for rel in MODULE_FOUNDRY_MAP_RECOVERY_MISSING_PREDECESSOR_ENTRIES:
|
||||||
|
path = installed_root / rel
|
||||||
|
if path.exists() or path.is_symlink():
|
||||||
|
die(
|
||||||
|
"Module Foundry map recovery missing predecessor drift: "
|
||||||
|
f"{rel}"
|
||||||
|
)
|
||||||
|
installed_entries = tuple(
|
||||||
|
rel
|
||||||
|
for rel in entries
|
||||||
|
if rel not in MODULE_FOUNDRY_MAP_RECOVERY_MISSING_PREDECESSOR_ENTRIES
|
||||||
)
|
)
|
||||||
if candidate_payload_sha256 != failed_payload_sha256:
|
|
||||||
die("Module Foundry sector recovery payload changed unexpectedly")
|
|
||||||
installed_payload_sha256 = collect_exact_files(
|
installed_payload_sha256 = collect_exact_files(
|
||||||
component_root("module-foundry"),
|
installed_root,
|
||||||
entries,
|
installed_entries,
|
||||||
"Module Foundry partial sector predecessor",
|
"Module Foundry partial map predecessor",
|
||||||
)
|
)
|
||||||
if installed_payload_sha256 != candidate_payload_sha256:
|
if (
|
||||||
die("Module Foundry partial sector predecessor drift detected")
|
exact_file_map_sha256(installed_payload_sha256)
|
||||||
|
!= MODULE_FOUNDRY_MAP_RECOVERY_INSTALLED_TREE_SHA256
|
||||||
|
):
|
||||||
|
die("Module Foundry partial map predecessor drift detected")
|
||||||
|
if {
|
||||||
|
rel: installed_payload_sha256.get(rel)
|
||||||
|
for rel in failed_payload_sha256
|
||||||
|
} != failed_payload_sha256:
|
||||||
|
die("Module Foundry installed failed map payload drift detected")
|
||||||
|
|
||||||
runtime = module_foundry_runtime_inventory()
|
runtime = module_foundry_runtime_inventory()
|
||||||
return {
|
return {
|
||||||
"mode": "failed-overlay-runtime-reconciliation",
|
"mode": "failed-map-overlay-source+runtime-reconciliation",
|
||||||
"failed_patch_id": MODULE_FOUNDRY_SECTOR_FAILED_PATCH_ID,
|
"failed_patch_id": MODULE_FOUNDRY_MAP_FAILED_PATCH_ID,
|
||||||
"failed_artifact_sha256": (
|
"failed_artifact_sha256": (
|
||||||
MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT_SHA256
|
MODULE_FOUNDRY_MAP_FAILED_ARTIFACT_SHA256
|
||||||
|
),
|
||||||
|
"failed_backup_id": MODULE_FOUNDRY_MAP_FAILED_BACKUP_ID,
|
||||||
|
"installed_tree_sha256": (
|
||||||
|
MODULE_FOUNDRY_MAP_RECOVERY_INSTALLED_TREE_SHA256
|
||||||
|
),
|
||||||
|
"candidate_tree_sha256": (
|
||||||
|
MODULE_FOUNDRY_MAP_RECOVERY_CANDIDATE_TREE_SHA256
|
||||||
),
|
),
|
||||||
"failed_backup_id": MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_ID,
|
|
||||||
"runtime_container_id": runtime["containerId"],
|
"runtime_container_id": runtime["containerId"],
|
||||||
"runtime_image_id": runtime["imageId"],
|
"runtime_image_id": runtime["imageId"],
|
||||||
}
|
}
|
||||||
@@ -17531,7 +17607,10 @@ def rollback_module_foundry_apply(
|
|||||||
entries,
|
entries,
|
||||||
current_stamp,
|
current_stamp,
|
||||||
)
|
)
|
||||||
current = module_foundry_runtime_inventory(required=False)
|
current = module_foundry_runtime_inventory(
|
||||||
|
required=False,
|
||||||
|
require_healthy=False,
|
||||||
|
)
|
||||||
if (
|
if (
|
||||||
current is not None
|
current is not None
|
||||||
and current["containerId"] == runtime_before["containerId"]
|
and current["containerId"] == runtime_before["containerId"]
|
||||||
@@ -18811,7 +18890,7 @@ def compose_service_container_id(component, service):
|
|||||||
return container_ids[0]
|
return container_ids[0]
|
||||||
|
|
||||||
|
|
||||||
def module_foundry_runtime_inventory(required=True):
|
def module_foundry_runtime_inventory(required=True, require_healthy=True):
|
||||||
result = subprocess.run(
|
result = subprocess.run(
|
||||||
[
|
[
|
||||||
*compose_base_cmd("module-foundry"),
|
*compose_base_cmd("module-foundry"),
|
||||||
@@ -18873,9 +18952,14 @@ def module_foundry_runtime_inventory(required=True):
|
|||||||
)
|
)
|
||||||
or image_ref.startswith("sha256:")
|
or image_ref.startswith("sha256:")
|
||||||
or "@" in image_ref
|
or "@" in image_ref
|
||||||
or state.get("Running") is not True
|
or (
|
||||||
or state.get("Status") != "running"
|
require_healthy
|
||||||
or health not in (None, "healthy")
|
and (
|
||||||
|
state.get("Running") is not True
|
||||||
|
or state.get("Status") != "running"
|
||||||
|
or health not in (None, "healthy")
|
||||||
|
)
|
||||||
|
)
|
||||||
):
|
):
|
||||||
die("Module Foundry runtime inventory mismatch")
|
die("Module Foundry runtime inventory mismatch")
|
||||||
return {
|
return {
|
||||||
@@ -20032,7 +20116,7 @@ def apply_artifact(artifact):
|
|||||||
engine_backend_recreated = False
|
engine_backend_recreated = False
|
||||||
engine_backend_initial_mode = None
|
engine_backend_initial_mode = None
|
||||||
module_foundry_runtime_before = None
|
module_foundry_runtime_before = None
|
||||||
module_foundry_sector_recovery_preflight = None
|
module_foundry_map_recovery_preflight = None
|
||||||
runtime_started = False
|
runtime_started = False
|
||||||
current_stamp = stamp()
|
current_stamp = stamp()
|
||||||
|
|
||||||
@@ -20111,8 +20195,8 @@ def apply_artifact(artifact):
|
|||||||
root.mkdir(parents=True, exist_ok=True)
|
root.mkdir(parents=True, exist_ok=True)
|
||||||
else:
|
else:
|
||||||
die(f"component payload root not found: {root}")
|
die(f"component payload root not found: {root}")
|
||||||
module_foundry_sector_recovery_preflight = (
|
module_foundry_map_recovery_preflight = (
|
||||||
validate_module_foundry_sector_recovery_evidence(
|
validate_module_foundry_map_recovery_evidence(
|
||||||
manifest,
|
manifest,
|
||||||
entries,
|
entries,
|
||||||
payload_dir,
|
payload_dir,
|
||||||
@@ -20402,20 +20486,20 @@ def apply_artifact(artifact):
|
|||||||
module_foundry_runtime_inventory()
|
module_foundry_runtime_inventory()
|
||||||
)
|
)
|
||||||
if (
|
if (
|
||||||
module_foundry_sector_recovery_preflight is not None
|
module_foundry_map_recovery_preflight is not None
|
||||||
and (
|
and (
|
||||||
module_foundry_runtime_before["containerId"]
|
module_foundry_runtime_before["containerId"]
|
||||||
!= module_foundry_sector_recovery_preflight[
|
!= module_foundry_map_recovery_preflight[
|
||||||
"runtime_container_id"
|
"runtime_container_id"
|
||||||
]
|
]
|
||||||
or module_foundry_runtime_before["imageId"]
|
or module_foundry_runtime_before["imageId"]
|
||||||
!= module_foundry_sector_recovery_preflight[
|
!= module_foundry_map_recovery_preflight[
|
||||||
"runtime_image_id"
|
"runtime_image_id"
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
):
|
):
|
||||||
die(
|
die(
|
||||||
"Module Foundry sector predecessor runtime "
|
"Module Foundry map predecessor runtime "
|
||||||
"changed during preflight"
|
"changed during preflight"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,174 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import importlib.machinery
|
||||||
|
import importlib.util
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
import tarfile
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from pathlib import Path, PurePosixPath
|
||||||
|
|
||||||
|
|
||||||
|
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||||
|
PLATFORM_ROOT = SCRIPT_DIR.parent.parent
|
||||||
|
WORKSPACE_ROOT = PLATFORM_ROOT.parent.parent
|
||||||
|
FOUNDRY_ROOT = WORKSPACE_ROOT / "NODEDC_DESIGN_GUIDELINE"
|
||||||
|
BUILDER = (
|
||||||
|
SCRIPT_DIR / "build-module-foundry-map-runtime-recovery-artifact.mjs"
|
||||||
|
)
|
||||||
|
RUNNER_PATH = SCRIPT_DIR / "nodedc-deploy"
|
||||||
|
SOURCE_COMMIT = "58800d957632320fa6717ec3fcff972023528759"
|
||||||
|
PATCH_ID = "module-foundry-map-runtime-recovery-20260809-999"
|
||||||
|
EXPECTED_FILES = (
|
||||||
|
"package.json",
|
||||||
|
"package-lock.json",
|
||||||
|
"apps/catalog/src",
|
||||||
|
"packages",
|
||||||
|
"registry",
|
||||||
|
"runtime-seed/page-layouts/map.json",
|
||||||
|
"scripts",
|
||||||
|
"server/catalog-server.mjs",
|
||||||
|
"server/foundry-mcp.mjs",
|
||||||
|
"server/map-grid-persistence.test.mjs",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def load_runner():
|
||||||
|
loader = importlib.machinery.SourceFileLoader(
|
||||||
|
"nodedc_module_foundry_map_runtime_recovery_artifact",
|
||||||
|
str(RUNNER_PATH),
|
||||||
|
)
|
||||||
|
spec = importlib.util.spec_from_loader(loader.name, loader)
|
||||||
|
module = importlib.util.module_from_spec(spec)
|
||||||
|
loader.exec_module(module)
|
||||||
|
return module
|
||||||
|
|
||||||
|
|
||||||
|
RUNNER = load_runner()
|
||||||
|
|
||||||
|
|
||||||
|
class ModuleFoundryMapRuntimeRecoveryArtifactTest(unittest.TestCase):
|
||||||
|
def export_source(self, root):
|
||||||
|
archive_path = root / "source.tar"
|
||||||
|
source_root = root / "source"
|
||||||
|
source_root.mkdir()
|
||||||
|
subprocess.run(
|
||||||
|
[
|
||||||
|
"git",
|
||||||
|
"-C",
|
||||||
|
str(FOUNDRY_ROOT),
|
||||||
|
"archive",
|
||||||
|
"--format=tar",
|
||||||
|
f"--output={archive_path}",
|
||||||
|
SOURCE_COMMIT,
|
||||||
|
],
|
||||||
|
check=True,
|
||||||
|
)
|
||||||
|
with tarfile.open(archive_path, "r:") as archive:
|
||||||
|
for member in archive.getmembers():
|
||||||
|
path = PurePosixPath(member.name)
|
||||||
|
self.assertFalse(path.is_absolute())
|
||||||
|
self.assertNotIn("..", path.parts)
|
||||||
|
archive.extractall(source_root, filter="data")
|
||||||
|
return source_root
|
||||||
|
|
||||||
|
def build(self, source_root, artifact_dir):
|
||||||
|
environment = os.environ.copy()
|
||||||
|
environment["NODEDC_FOUNDRY_SOURCE_ROOT"] = str(source_root)
|
||||||
|
environment["NODEDC_FOUNDRY_SOURCE_COMMIT"] = SOURCE_COMMIT
|
||||||
|
environment["NODEDC_DEPLOY_ARTIFACT_DIR"] = str(artifact_dir)
|
||||||
|
completed = subprocess.run(
|
||||||
|
["node", str(BUILDER), PATCH_ID],
|
||||||
|
cwd=PLATFORM_ROOT,
|
||||||
|
env=environment,
|
||||||
|
check=True,
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
)
|
||||||
|
return json.loads(completed.stdout)
|
||||||
|
|
||||||
|
def test_builder_is_deterministic_coherent_and_runner_compatible(self):
|
||||||
|
with tempfile.TemporaryDirectory(
|
||||||
|
prefix="nodedc-foundry-map-runtime-recovery-"
|
||||||
|
) as directory:
|
||||||
|
root = Path(directory)
|
||||||
|
source_root = self.export_source(root)
|
||||||
|
first = self.build(source_root, root / "first")
|
||||||
|
second = self.build(source_root, root / "second")
|
||||||
|
first_artifact = Path(first["artifact"])
|
||||||
|
second_artifact = Path(second["artifact"])
|
||||||
|
|
||||||
|
self.assertEqual(first_artifact.read_bytes(), second_artifact.read_bytes())
|
||||||
|
self.assertEqual(
|
||||||
|
first["sha256"],
|
||||||
|
hashlib.sha256(first_artifact.read_bytes()).hexdigest(),
|
||||||
|
)
|
||||||
|
self.assertEqual(first["sourceCommit"], SOURCE_COMMIT)
|
||||||
|
self.assertEqual(tuple(first["files"]), EXPECTED_FILES)
|
||||||
|
self.assertEqual(first["services"], ["nodedc-module-foundry"])
|
||||||
|
|
||||||
|
extracted = root / "loaded"
|
||||||
|
extracted.mkdir()
|
||||||
|
manifest, entries, payload = RUNNER.load_artifact(
|
||||||
|
first_artifact,
|
||||||
|
extracted,
|
||||||
|
)
|
||||||
|
self.assertEqual(manifest["id"], PATCH_ID)
|
||||||
|
self.assertEqual(manifest["component"], "module-foundry")
|
||||||
|
self.assertEqual(tuple(entries), EXPECTED_FILES)
|
||||||
|
self.assertEqual(
|
||||||
|
RUNNER.component_services("module-foundry", entries),
|
||||||
|
("nodedc-module-foundry",),
|
||||||
|
)
|
||||||
|
|
||||||
|
preview = (
|
||||||
|
payload / "apps/catalog/src/MapFixturePreview.tsx"
|
||||||
|
).read_text(encoding="utf-8")
|
||||||
|
renderer = (
|
||||||
|
payload / "apps/catalog/src/CesiumMapRenderer.tsx"
|
||||||
|
).read_text(encoding="utf-8")
|
||||||
|
package_lock = (payload / "package-lock.json").read_text(
|
||||||
|
encoding="utf-8"
|
||||||
|
)
|
||||||
|
self.assertIn("const sectorSpatialEntities = useMemo", preview)
|
||||||
|
self.assertIn("class GridLayerController", renderer)
|
||||||
|
self.assertIn('"packages/map-cesium-react"', package_lock)
|
||||||
|
self.assertTrue(
|
||||||
|
(payload / "packages/map-cesium-react/src/index.ts").is_file()
|
||||||
|
)
|
||||||
|
|
||||||
|
with tarfile.open(first_artifact, "r:gz") as archive:
|
||||||
|
members = archive.getmembers()
|
||||||
|
names = [member.name for member in members]
|
||||||
|
regular_payloads = [
|
||||||
|
archive.extractfile(member).read()
|
||||||
|
for member in members
|
||||||
|
if member.isfile()
|
||||||
|
]
|
||||||
|
self.assertFalse(
|
||||||
|
any(Path(name).name.startswith("._") for name in names)
|
||||||
|
)
|
||||||
|
self.assertFalse(
|
||||||
|
any(
|
||||||
|
"/.git/" in name
|
||||||
|
or "/node_modules/" in name
|
||||||
|
or "/runtime-data/" in name
|
||||||
|
or "/dist/" in name
|
||||||
|
for name in names
|
||||||
|
)
|
||||||
|
)
|
||||||
|
self.assertNotIn(
|
||||||
|
"payload/scripts/spark-governance-contract.test.mjs",
|
||||||
|
names,
|
||||||
|
)
|
||||||
|
self.assertNotIn(
|
||||||
|
b"-----BEGIN PRIVATE KEY-----",
|
||||||
|
b"\n".join(regular_payloads),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main(verbosity=2)
|
||||||
@@ -8,6 +8,7 @@ import subprocess
|
|||||||
import tarfile
|
import tarfile
|
||||||
import tempfile
|
import tempfile
|
||||||
import unittest
|
import unittest
|
||||||
|
from contextlib import ExitStack
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from unittest import mock
|
from unittest import mock
|
||||||
|
|
||||||
@@ -202,7 +203,7 @@ class ModuleFoundryRuntimeRecoveryTest(unittest.TestCase):
|
|||||||
RUNNER,
|
RUNNER,
|
||||||
"module_foundry_runtime_inventory",
|
"module_foundry_runtime_inventory",
|
||||||
side_effect=[candidate, before],
|
side_effect=[candidate, before],
|
||||||
),
|
) as inventory,
|
||||||
mock.patch.object(
|
mock.patch.object(
|
||||||
RUNNER,
|
RUNNER,
|
||||||
"inspect_local_image",
|
"inspect_local_image",
|
||||||
@@ -221,6 +222,10 @@ class ModuleFoundryRuntimeRecoveryTest(unittest.TestCase):
|
|||||||
)
|
)
|
||||||
|
|
||||||
self.assertEqual(result, "source+runtime-restored:1")
|
self.assertEqual(result, "source+runtime-restored:1")
|
||||||
|
self.assertEqual(
|
||||||
|
inventory.call_args_list[0].kwargs,
|
||||||
|
{"required": False, "require_healthy": False},
|
||||||
|
)
|
||||||
process.assert_called_once_with(
|
process.assert_called_once_with(
|
||||||
[
|
[
|
||||||
str(RUNNER.DOCKER),
|
str(RUNNER.DOCKER),
|
||||||
@@ -264,7 +269,7 @@ class ModuleFoundryRuntimeRecoveryTest(unittest.TestCase):
|
|||||||
|
|
||||||
def test_recovery_accepts_only_exact_failed_partial_predecessor(self):
|
def test_recovery_accepts_only_exact_failed_partial_predecessor(self):
|
||||||
with tempfile.TemporaryDirectory(
|
with tempfile.TemporaryDirectory(
|
||||||
prefix="module-foundry-sector-evidence-"
|
prefix="module-foundry-map-evidence-"
|
||||||
) as directory:
|
) as directory:
|
||||||
work = Path(directory)
|
work = Path(directory)
|
||||||
backups = work / "backups"
|
backups = work / "backups"
|
||||||
@@ -276,43 +281,79 @@ class ModuleFoundryRuntimeRecoveryTest(unittest.TestCase):
|
|||||||
for path in (backups, failed, state, tmp, installed, candidate):
|
for path in (backups, failed, state, tmp, installed, candidate):
|
||||||
path.mkdir()
|
path.mkdir()
|
||||||
|
|
||||||
entries = RUNNER.MODULE_FOUNDRY_SECTOR_RECOVERY_ENTRIES
|
recovery_entries = (
|
||||||
for index, rel in enumerate(entries):
|
"apps/catalog/src",
|
||||||
for root in (installed, candidate):
|
"scripts",
|
||||||
|
"server/map-grid-persistence.test.mjs",
|
||||||
|
)
|
||||||
|
missing_entries = ("server/map-grid-persistence.test.mjs",)
|
||||||
|
failed_entries = (
|
||||||
|
"apps/catalog/src/MapFixturePreview.tsx",
|
||||||
|
"scripts/map-sector-grid.test.mjs",
|
||||||
|
)
|
||||||
|
candidate_files = {
|
||||||
|
"apps/catalog/src/MapFixturePreview.tsx": "sector-current\n",
|
||||||
|
"apps/catalog/src/CesiumMapRenderer.tsx": "renderer-next\n",
|
||||||
|
"scripts/map-sector-grid.test.mjs": "sector-test-current\n",
|
||||||
|
"scripts/map-grid-lod.test.mjs": "grid-test-next\n",
|
||||||
|
"server/map-grid-persistence.test.mjs": "server-test-next\n",
|
||||||
|
}
|
||||||
|
installed_files = {
|
||||||
|
"apps/catalog/src/MapFixturePreview.tsx": "sector-current\n",
|
||||||
|
"apps/catalog/src/CesiumMapRenderer.tsx": "renderer-old\n",
|
||||||
|
"scripts/map-sector-grid.test.mjs": "sector-test-current\n",
|
||||||
|
"scripts/map-grid-lod.test.mjs": "grid-test-old\n",
|
||||||
|
}
|
||||||
|
for root, values in (
|
||||||
|
(candidate, candidate_files),
|
||||||
|
(installed, installed_files),
|
||||||
|
):
|
||||||
|
for rel, value in values.items():
|
||||||
path = root / rel
|
path = root / rel
|
||||||
path.parent.mkdir(parents=True, exist_ok=True)
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
path.write_text(f"payload-{index}\n", encoding="utf-8")
|
path.write_text(value, encoding="utf-8")
|
||||||
|
|
||||||
backup = backups / RUNNER.MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_ID
|
failed_patch_id = "module-foundry-map-failed-test"
|
||||||
|
failed_artifact_name = "module-foundry-map-failed-test.tgz.1"
|
||||||
|
failed_backup_id = "module-foundry-map-failed-test-backup"
|
||||||
|
recovery_patch_id = "module-foundry-map-recovery-test"
|
||||||
|
failed_message = "test build failed"
|
||||||
|
|
||||||
|
backup = backups / failed_backup_id
|
||||||
backup.mkdir()
|
backup.mkdir()
|
||||||
for name in RUNNER.MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_SHA256:
|
backup_names = {
|
||||||
|
"existing-files.txt",
|
||||||
|
"files.txt",
|
||||||
|
"manifest.env",
|
||||||
|
"missing-files.txt",
|
||||||
|
"source-before.tgz",
|
||||||
|
}
|
||||||
|
for name in backup_names:
|
||||||
(backup / name).write_text(f"{name}\n", encoding="utf-8")
|
(backup / name).write_text(f"{name}\n", encoding="utf-8")
|
||||||
backup_hashes = {
|
backup_hashes = {
|
||||||
name: hashlib.sha256((backup / name).read_bytes()).hexdigest()
|
name: hashlib.sha256((backup / name).read_bytes()).hexdigest()
|
||||||
for name in RUNNER.MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_SHA256
|
for name in backup_names
|
||||||
}
|
}
|
||||||
|
|
||||||
archive_source = work / "archive-source"
|
archive_source = work / "archive-source"
|
||||||
(archive_source / "payload").mkdir(parents=True)
|
(archive_source / "payload").mkdir(parents=True)
|
||||||
(archive_source / "manifest.env").write_text(
|
(archive_source / "manifest.env").write_text(
|
||||||
"id="
|
"id="
|
||||||
f"{RUNNER.MODULE_FOUNDRY_SECTOR_FAILED_PATCH_ID}\n"
|
f"{failed_patch_id}\n"
|
||||||
"component=module-foundry\n"
|
"component=module-foundry\n"
|
||||||
"type=app-overlay\n",
|
"type=app-overlay\n",
|
||||||
encoding="utf-8",
|
encoding="utf-8",
|
||||||
)
|
)
|
||||||
(archive_source / "files.txt").write_text(
|
(archive_source / "files.txt").write_text(
|
||||||
"\n".join(entries) + "\n",
|
"\n".join(failed_entries) + "\n",
|
||||||
encoding="utf-8",
|
encoding="utf-8",
|
||||||
)
|
)
|
||||||
for rel in entries:
|
for rel in failed_entries:
|
||||||
source = candidate / rel
|
source = candidate / rel
|
||||||
target = archive_source / "payload" / rel
|
target = archive_source / "payload" / rel
|
||||||
target.parent.mkdir(parents=True, exist_ok=True)
|
target.parent.mkdir(parents=True, exist_ok=True)
|
||||||
target.write_bytes(source.read_bytes())
|
target.write_bytes(source.read_bytes())
|
||||||
failed_artifact = (
|
failed_artifact = failed / failed_artifact_name
|
||||||
failed / RUNNER.MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT
|
|
||||||
)
|
|
||||||
with tarfile.open(failed_artifact, "w:gz") as archive:
|
with tarfile.open(failed_artifact, "w:gz") as archive:
|
||||||
archive.add(
|
archive.add(
|
||||||
archive_source / "manifest.env",
|
archive_source / "manifest.env",
|
||||||
@@ -331,11 +372,11 @@ class ModuleFoundryRuntimeRecoveryTest(unittest.TestCase):
|
|||||||
journal.write_text(
|
journal.write_text(
|
||||||
json.dumps(
|
json.dumps(
|
||||||
{
|
{
|
||||||
"artifact": RUNNER.MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT,
|
"artifact": failed_artifact_name,
|
||||||
"backup_id": RUNNER.MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_ID,
|
"backup_id": failed_backup_id,
|
||||||
"component": "module-foundry",
|
"component": "module-foundry",
|
||||||
"id": RUNNER.MODULE_FOUNDRY_SECTOR_FAILED_PATCH_ID,
|
"id": failed_patch_id,
|
||||||
"message": RUNNER.MODULE_FOUNDRY_SECTOR_FAILED_MESSAGE,
|
"message": failed_message,
|
||||||
"rollback_status": "not-required",
|
"rollback_status": "not-required",
|
||||||
"sha256": failed_sha256,
|
"sha256": failed_sha256,
|
||||||
"started_apply": True,
|
"started_apply": True,
|
||||||
@@ -346,55 +387,83 @@ class ModuleFoundryRuntimeRecoveryTest(unittest.TestCase):
|
|||||||
encoding="utf-8",
|
encoding="utf-8",
|
||||||
)
|
)
|
||||||
manifest = {
|
manifest = {
|
||||||
"id": RUNNER.MODULE_FOUNDRY_SECTOR_RECOVERY_PATCH_ID,
|
"id": recovery_patch_id,
|
||||||
"component": "module-foundry",
|
"component": "module-foundry",
|
||||||
"type": "app-overlay",
|
"type": "app-overlay",
|
||||||
}
|
}
|
||||||
|
candidate_tree = RUNNER.exact_file_map_sha256(
|
||||||
patches = (
|
RUNNER.collect_exact_files(
|
||||||
mock.patch.object(RUNNER, "BACKUPS_DIR", backups),
|
candidate,
|
||||||
mock.patch.object(RUNNER, "FAILED_DIR", failed),
|
recovery_entries,
|
||||||
mock.patch.object(RUNNER, "FAILED_STATE_FILE", journal),
|
"candidate",
|
||||||
mock.patch.object(RUNNER, "TMP_DIR", tmp),
|
)
|
||||||
mock.patch.object(
|
|
||||||
RUNNER,
|
|
||||||
"MODULE_FOUNDRY_SECTOR_FAILED_ARTIFACT_SHA256",
|
|
||||||
failed_sha256,
|
|
||||||
),
|
|
||||||
mock.patch.object(
|
|
||||||
RUNNER,
|
|
||||||
"MODULE_FOUNDRY_SECTOR_FAILED_BACKUP_SHA256",
|
|
||||||
backup_hashes,
|
|
||||||
),
|
|
||||||
mock.patch.object(
|
|
||||||
RUNNER,
|
|
||||||
"component_root",
|
|
||||||
return_value=installed,
|
|
||||||
),
|
|
||||||
mock.patch.object(
|
|
||||||
RUNNER,
|
|
||||||
"module_foundry_runtime_inventory",
|
|
||||||
return_value=runtime(),
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
with patches[0], patches[1], patches[2], patches[3], patches[4], patches[5], patches[6], patches[7]:
|
installed_tree = RUNNER.exact_file_map_sha256(
|
||||||
result = RUNNER.validate_module_foundry_sector_recovery_evidence(
|
RUNNER.collect_exact_files(
|
||||||
|
installed,
|
||||||
|
recovery_entries[:-1],
|
||||||
|
"installed",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
patches = {
|
||||||
|
"BACKUPS_DIR": backups,
|
||||||
|
"FAILED_DIR": failed,
|
||||||
|
"FAILED_STATE_FILE": journal,
|
||||||
|
"TMP_DIR": tmp,
|
||||||
|
"MODULE_FOUNDRY_MAP_RECOVERY_PATCH_ID": recovery_patch_id,
|
||||||
|
"MODULE_FOUNDRY_MAP_RECOVERY_ENTRIES": recovery_entries,
|
||||||
|
"MODULE_FOUNDRY_MAP_RECOVERY_MISSING_PREDECESSOR_ENTRIES": (
|
||||||
|
missing_entries
|
||||||
|
),
|
||||||
|
"MODULE_FOUNDRY_MAP_RECOVERY_INSTALLED_TREE_SHA256": (
|
||||||
|
installed_tree
|
||||||
|
),
|
||||||
|
"MODULE_FOUNDRY_MAP_RECOVERY_CANDIDATE_TREE_SHA256": (
|
||||||
|
candidate_tree
|
||||||
|
),
|
||||||
|
"MODULE_FOUNDRY_SECTOR_FAILED_ENTRIES": failed_entries,
|
||||||
|
"MODULE_FOUNDRY_MAP_FAILED_PATCH_ID": failed_patch_id,
|
||||||
|
"MODULE_FOUNDRY_MAP_FAILED_ARTIFACT": failed_artifact_name,
|
||||||
|
"MODULE_FOUNDRY_MAP_FAILED_ARTIFACT_SHA256": failed_sha256,
|
||||||
|
"MODULE_FOUNDRY_MAP_FAILED_BACKUP_ID": failed_backup_id,
|
||||||
|
"MODULE_FOUNDRY_MAP_FAILED_MESSAGE": failed_message,
|
||||||
|
"MODULE_FOUNDRY_MAP_FAILED_BACKUP_SHA256": backup_hashes,
|
||||||
|
}
|
||||||
|
with ExitStack() as stack:
|
||||||
|
for name, value in patches.items():
|
||||||
|
stack.enter_context(mock.patch.object(RUNNER, name, value))
|
||||||
|
stack.enter_context(
|
||||||
|
mock.patch.object(
|
||||||
|
RUNNER,
|
||||||
|
"component_root",
|
||||||
|
return_value=installed,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
stack.enter_context(
|
||||||
|
mock.patch.object(
|
||||||
|
RUNNER,
|
||||||
|
"module_foundry_runtime_inventory",
|
||||||
|
return_value=runtime(),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
result = RUNNER.validate_module_foundry_map_recovery_evidence(
|
||||||
manifest,
|
manifest,
|
||||||
entries,
|
recovery_entries,
|
||||||
candidate,
|
candidate,
|
||||||
)
|
)
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
result["mode"],
|
result["mode"],
|
||||||
"failed-overlay-runtime-reconciliation",
|
"failed-map-overlay-source+runtime-reconciliation",
|
||||||
)
|
)
|
||||||
(installed / entries[0]).write_text(
|
(installed / failed_entries[0]).write_text(
|
||||||
"drift\n",
|
"drift\n",
|
||||||
encoding="utf-8",
|
encoding="utf-8",
|
||||||
)
|
)
|
||||||
with self.assertRaises(RUNNER.DeployError):
|
with self.assertRaises(RUNNER.DeployError):
|
||||||
RUNNER.validate_module_foundry_sector_recovery_evidence(
|
RUNNER.validate_module_foundry_map_recovery_evidence(
|
||||||
manifest,
|
manifest,
|
||||||
entries,
|
recovery_entries,
|
||||||
candidate,
|
candidate,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user