From 3d3c0eced2eed621659879da25ab5cd907669a5b Mon Sep 17 00:00:00 2001 From: DCCONSTRUCTIONS Date: Sun, 23 Aug 2026 20:19:13 +0300 Subject: [PATCH] isolate M4.7 worker shadow runtime --- ...E_4_OBJECT_CENTRIC_RECORDED_REALTIME_CV.md | 13 +- scripts/Invoke-M47CanonicalGraphShadow.ps1 | 260 +++++++++++++----- .../build_m47_worker_graph_shadow_artifact.py | 21 +- .../test_m47_worker_graph_shadow_artifact.py | 22 +- 4 files changed, 228 insertions(+), 88 deletions(-) diff --git a/docs/22_MILESTONE_4_OBJECT_CENTRIC_RECORDED_REALTIME_CV.md b/docs/22_MILESTONE_4_OBJECT_CENTRIC_RECORDED_REALTIME_CV.md index 2c5efa7..6139dfd 100644 --- a/docs/22_MILESTONE_4_OBJECT_CENTRIC_RECORDED_REALTIME_CV.md +++ b/docs/22_MILESTONE_4_OBJECT_CENTRIC_RECORDED_REALTIME_CV.md @@ -1172,7 +1172,7 @@ terminal, report and manifest ledgers; acceptance requires exactly `4,489` delivered frames with zero failed, stale, superseded, rejected or unavailable terminal outcomes. -The new Worker package uses transition `m47-canonical-graph-shadow-v1`. It is a +The first Worker package used transition `m47-canonical-graph-shadow-v1`. It is a separate deterministic artifact and does not relabel the accepted historical `m4-detector-shadow-v1` wheel. Its PowerShell runner verifies release files, inputs, dependency trees, disk reserve, pinned Worker/Triton predecessor and @@ -1182,6 +1182,17 @@ the run and re-verifies the predecessor. Provider readiness and graph readiness are emitted separately. The production builder refuses a dirty worktree so an artifact cannot claim a Git revision which does not contain its wheel. +The live audit on 2026-08-23 found both pinned historical Mission Core +containers present with their exact identities but stopped. M4.7 therefore +uses the additive transition `m47-canonical-graph-isolated-shadow-v1`: it does +not start, stop or replace either historical container. The runner creates a +private, no-public-port Triton candidate from the same digest and a separate +one-shot graph candidate, binds the accepted model/evidence read-only, and +removes both candidates on every exit path. It records and re-verifies the +identity and running state of the historical E15 worker and Triton before +accepting the run. This makes Worker 006 usable without touching the stabilized +K1/Zarya connection path. + Local contract, graph, result-sealing, artifact and historical-rollback tests pass. This is implementation evidence only. It does not claim that Worker 006 has the pinned local-surface input, that preflight has passed, that the 4,489 diff --git a/scripts/Invoke-M47CanonicalGraphShadow.ps1 b/scripts/Invoke-M47CanonicalGraphShadow.ps1 index 91910d0..4bef1b3 100644 --- a/scripts/Invoke-M47CanonicalGraphShadow.ps1 +++ b/scripts/Invoke-M47CanonicalGraphShadow.ps1 @@ -143,11 +143,11 @@ if ($env:COMPUTERNAME -cne "DESKTOP-OPJ8J04") { $release = Resolve-DDirectory $ReleaseRoot "M4.7 release root" $false $payload = Resolve-DDirectory (Join-Path $release "payload") "M4.7 payload" $false $artifact = Assert-FileSha256 $ArtifactPath $ExpectedArtifactSha256 "M4.7 release artifact" -$descriptorPath = Join-Path $payload "mission-core-worker-m47-graph-shadow-v2.json" +$descriptorPath = Join-Path $payload "mission-core-worker-m47-graph-shadow-v3.json" $descriptor = Get-Content -LiteralPath $descriptorPath -Raw | ConvertFrom-Json if ( - $descriptor.schema_version -cne "nodedc.mission-core-worker.shadow-release/v2" -or - $descriptor.transition -cne "m47-canonical-graph-shadow-v1" -or + $descriptor.schema_version -cne "nodedc.mission-core-worker.shadow-release/v3" -or + $descriptor.transition -cne "m47-canonical-graph-isolated-shadow-v1" -or $descriptor.component -cne "mission-core-worker" -or $descriptor.host.node -cne $env:COMPUTERNAME -or $descriptor.host.worker_id -cne "worker-006" -or @@ -215,31 +215,52 @@ foreach ($dependency in $descriptor.dependencies) { $imageRef = [string]$descriptor.container.image_ref & docker image inspect $imageRef *> $null Assert-LastExitCode "Pinned M4.7 image inspection" -$predecessor = $descriptor.predecessor.durable_worker -$tritonExpected = $descriptor.predecessor.triton -$durable = Assert-ContainerIdentity ( - $predecessor.name -) $predecessor.container_id $predecessor.image_id $false -$triton = Assert-ContainerIdentity ( - $tritonExpected.name -) $tritonExpected.container_id $tritonExpected.image_id $true + +function Get-PreservedContainerSnapshot([object]$Expected, [string]$Label) { + $container = Get-ContainerIdentity $Expected.name + if ( + $container.Id -cne $Expected.container_id -or + $container.Image -cne $Expected.image_id + ) { + throw "$Label identity changed" + } + return [pscustomobject]@{ + Id = [string]$container.Id + Image = [string]$container.Image + Running = [bool]$container.State.Running + } +} + +function Assert-PreservedContainerSnapshot( + [object]$Expected, + [object]$Before, + [string]$Label +) { + $after = Get-PreservedContainerSnapshot $Expected $Label + if ($after.Running -ne $Before.Running) { + throw "$Label running state changed during isolated shadow" + } + return $after +} + +$durableExpected = $descriptor.predecessor.durable_worker +$historicalTritonExpected = $descriptor.predecessor.triton +$durableBefore = Get-PreservedContainerSnapshot $durableExpected "Historical durable worker" +$historicalTritonBefore = Get-PreservedContainerSnapshot ( + $historicalTritonExpected +) "Historical Triton" +$modelRepository = Resolve-DDirectory ( + [string]$descriptor.container.model_repository_host_path +) "M4.7 model repository" $false $output = Resolve-DDirectory $OutputRoot "M4.7 output root" $true $freeBefore = Assert-FreeSpace "preflight" -if ($PreflightOnly) { - Write-Output ("PATCH_ID={0}" -f $descriptor.patch_id) - Write-Output ("ARTIFACT_SHA256={0}" -f $ExpectedArtifactSha256) - Write-Output ("DURABLE_WORKER_ID={0}" -f $durable.Id) - Write-Output ("TRITON_CONTAINER_ID={0}" -f $triton.Id) - Write-Output "PROVIDER_READINESS=accepted" - Write-Output "GRAPH_READINESS=not-run" - Write-Output "PREFLIGHT=accepted" - return -} - -$candidateName = "ndc-mission-core-m47-graph-shadow" -if (& docker ps -a --format "{{.Names}}" --filter "name=^/$candidateName$") { - throw "M4.7 candidate container already exists" +$candidateName = [string]$descriptor.container.name +$tritonCandidateName = [string]$descriptor.container.triton_name +foreach ($name in @($candidateName, $tritonCandidateName)) { + if (& docker ps -a --format "{{.Names}}" --filter "name=^/$name$") { + throw "M4.7 isolated candidate $name already exists" + } } $scratch = Join-Path $output (".runtime-{0}" -f $descriptor.patch_id) if (Test-Path -LiteralPath $scratch) { @@ -251,10 +272,40 @@ $runtimeIdentityPath = Join-Path $scratch "runtime-identity.json" $dockerPayload = Convert-ToDockerPath $payload $dockerOutput = Convert-ToDockerPath $output $dockerScratch = Convert-ToDockerPath $scratch -$dockerArguments = @( +$dockerModelRepository = Convert-ToDockerPath $modelRepository +$tritonArguments = @( + "create", + "--name", $tritonCandidateName, + "--read-only", + "--security-opt", "no-new-privileges:true", + "--cap-drop", "ALL", + "--pids-limit", "512", + "--shm-size", "1g", + "--gpus", "all", + "--tmpfs", "/tmp:rw,noexec,nosuid,size=2g", + "--health-cmd", "curl --fail --silent http://127.0.0.1:8000/v2/health/ready", + "--health-interval", "5s", + "--health-timeout", "3s", + "--health-start-period", "20s", + "--health-retries", "24", + "-v", ("{0}:/models:ro" -f $dockerModelRepository), + $imageRef, + "tritonserver", + "--model-repository=/models", + "--model-control-mode=explicit", + "--load-model=yolox_s", + "--disable-auto-complete-config", + "--strict-readiness=true", + "--exit-on-error=true", + "--allow-http=true", + "--allow-grpc=false", + "--allow-metrics=false" +) + +$graphArguments = @( "create", "--name", $candidateName, - "--network", ("container:{0}" -f $tritonExpected.name), + "--network", ("container:{0}" -f $tritonCandidateName), "--read-only", "--security-opt", "no-new-privileges:true", "--cap-drop", "ALL", @@ -268,7 +319,7 @@ $dockerArguments = @( ) foreach ($entry in $descriptor.inputs.PSObject.Properties) { if ($null -ne $entry.Value.container_path) { - $dockerArguments += @( + $graphArguments += @( "-v", ("{0}:{1}:ro" -f ( Convert-ToDockerPath $entry.Value.host_path ), $entry.Value.container_path) @@ -276,13 +327,13 @@ foreach ($entry in $descriptor.inputs.PSObject.Properties) { } } foreach ($dependency in $descriptor.dependencies) { - $dockerArguments += @( + $graphArguments += @( "-v", ("{0}:{1}:ro" -f ( Convert-ToDockerPath $dependency.host_path ), $dependency.container_path) ) } -$dockerArguments += @( +$graphArguments += @( "--entrypoint", "python3", $imageRef, "-m", "k1link.perception.reference_graph_cli", @@ -305,72 +356,131 @@ $dockerArguments += @( "--output-root", "/output" ) -$candidateCreated = $false +$tritonCreated = $false +$graphCreated = $false +$providerAccepted = $false +$graphAccepted = $false $runFailure = $null try { - $candidateId = (& docker @dockerArguments).Trim() - Assert-LastExitCode "M4.7 candidate creation" - if ($candidateId -notmatch "^[a-f0-9]{64}$") { - throw "M4.7 candidate id is invalid" + $tritonCandidateId = (& docker @tritonArguments).Trim() + Assert-LastExitCode "M4.7 isolated Triton creation" + if ($tritonCandidateId -notmatch "^[a-f0-9]{64}$") { + throw "M4.7 isolated Triton id is invalid" + } + $tritonCreated = $true + & docker start $tritonCandidateName *> $null + Assert-LastExitCode "M4.7 isolated Triton start" + $tritonCandidate = $null + foreach ($attempt in 1..120) { + $tritonCandidate = Get-ContainerIdentity $tritonCandidateName + if (-not $tritonCandidate.State.Running) { + throw "M4.7 isolated Triton stopped before readiness" + } + if ($tritonCandidate.State.Health.Status -ceq "healthy") { + break + } + if ($attempt -eq 120) { + throw "M4.7 isolated Triton readiness timed out" + } + Start-Sleep -Seconds 2 } - $candidateCreated = $true - $candidate = Get-ContainerIdentity $candidateName if ( - $candidate.Id -cne $candidateId -or - $candidate.Image -cne $descriptor.container.image_id -or - $candidate.HostConfig.NetworkMode -cne ("container:{0}" -f $triton.Id) -or - -not $candidate.HostConfig.ReadonlyRootfs + $tritonCandidate.Id -cne $tritonCandidateId -or + $tritonCandidate.Image -cne $descriptor.container.image_id -or + -not $tritonCandidate.HostConfig.ReadonlyRootfs -or + $null -ne $tritonCandidate.HostConfig.PortBindings ) { - throw "M4.7 candidate isolation contract changed" + throw "M4.7 isolated Triton contract changed" } - $runtimeIdentity = [ordered]@{ - schema_version = "missioncore.reference-graph-runtime-identity/v1" - worker_id = "worker-006" - worker_node = $env:COMPUTERNAME - worker_container_id = $candidate.Id - worker_image_id = $candidate.Image - triton_container_id = $triton.Id - triton_image_id = $triton.Image - artifact_sha256 = $ExpectedArtifactSha256 - code_revision = $descriptor.code_revision - graph_id = $descriptor.readiness.graph.graph_id - source_mount_read_only = $true - model_service_reused = $true - public_worker_port_added = $false - commands_enabled = $false - actuation_allowed = $false + $providerAccepted = $true + + if (-not $PreflightOnly) { + $candidateId = (& docker @graphArguments).Trim() + Assert-LastExitCode "M4.7 graph candidate creation" + if ($candidateId -notmatch "^[a-f0-9]{64}$") { + throw "M4.7 graph candidate id is invalid" + } + $graphCreated = $true + $candidate = Get-ContainerIdentity $candidateName + if ( + $candidate.Id -cne $candidateId -or + $candidate.Image -cne $descriptor.container.image_id -or + $candidate.HostConfig.NetworkMode -cne ("container:{0}" -f $tritonCandidate.Id) -or + -not $candidate.HostConfig.ReadonlyRootfs -or + $null -ne $candidate.HostConfig.PortBindings + ) { + throw "M4.7 graph candidate isolation contract changed" + } + $runtimeIdentity = [ordered]@{ + schema_version = "missioncore.reference-graph-runtime-identity/v2" + worker_id = "worker-006" + worker_node = $env:COMPUTERNAME + worker_container_id = $candidate.Id + worker_image_id = $candidate.Image + isolated_triton_container_id = $tritonCandidate.Id + isolated_triton_image_id = $tritonCandidate.Image + historical_worker_container_id = $durableBefore.Id + historical_worker_running = $durableBefore.Running + historical_triton_container_id = $historicalTritonBefore.Id + historical_triton_running = $historicalTritonBefore.Running + artifact_sha256 = $ExpectedArtifactSha256 + code_revision = $descriptor.code_revision + graph_id = $descriptor.readiness.graph.graph_id + source_mount_read_only = $true + isolated_model_service = $true + public_worker_port_added = $false + commands_enabled = $false + actuation_allowed = $false + } + Write-Utf8NoBom $runtimeIdentityPath ($runtimeIdentity | ConvertTo-Json -Depth 4) + & docker start --attach $candidateName + Assert-LastExitCode "M4.7 canonical graph isolated shadow" + $graphAccepted = $true } - Write-Utf8NoBom $runtimeIdentityPath ($runtimeIdentity | ConvertTo-Json -Depth 4) - Write-Output ("PATCH_ID={0}" -f $descriptor.patch_id) - Write-Output ("ARTIFACT_SHA256={0}" -f $ExpectedArtifactSha256) - Write-Output ("CANDIDATE_CONTAINER_ID={0}" -f $candidate.Id) - Write-Output "PROVIDER_READINESS=accepted" - & docker start --attach $candidateName - Assert-LastExitCode "M4.7 canonical graph shadow" - Write-Output "GRAPH_READINESS=accepted" } catch { $runFailure = $_ } finally { - if ($candidateCreated) { + if ($graphCreated) { & docker rm --force $candidateName *> $null if ($LASTEXITCODE -ne 0 -and $null -eq $runFailure) { - $runFailure = "M4.7 candidate cleanup failed" + $runFailure = "M4.7 graph candidate cleanup failed" + } + } + if ($tritonCreated) { + & docker rm --force $tritonCandidateName *> $null + if ($LASTEXITCODE -ne 0 -and $null -eq $runFailure) { + $runFailure = "M4.7 isolated Triton cleanup failed" } } Remove-Item -LiteralPath $scratch -Force -Recurse -ErrorAction SilentlyContinue } -$null = Assert-ContainerIdentity $predecessor.name $predecessor.container_id ( - $predecessor.image_id -) $false -$null = Assert-ContainerIdentity $tritonExpected.name $tritonExpected.container_id ( - $tritonExpected.image_id -) $true +$durableAfter = Assert-PreservedContainerSnapshot ( + $durableExpected +) $durableBefore "Historical durable worker" +$historicalTritonAfter = Assert-PreservedContainerSnapshot ( + $historicalTritonExpected +) $historicalTritonBefore "Historical Triton" $freeAfter = Assert-FreeSpace "completed" +Write-Output ("PATCH_ID={0}" -f $descriptor.patch_id) +Write-Output ("ARTIFACT_SHA256={0}" -f $ExpectedArtifactSha256) Write-Output ("DISK_FREE_BYTES_BEFORE={0}" -f $freeBefore) Write-Output ("DISK_FREE_BYTES_AFTER={0}" -f $freeAfter) +Write-Output ("HISTORICAL_DURABLE_WORKER_RUNNING={0}" -f $durableAfter.Running) +Write-Output ("HISTORICAL_TRITON_RUNNING={0}" -f $historicalTritonAfter.Running) Write-Output "DURABLE_WORKER_ACTION=none" -Write-Output "TRITON_ACTION=none" +Write-Output "HISTORICAL_TRITON_ACTION=none" +Write-Output "ISOLATED_TRITON_ACTION=removed" if ($null -ne $runFailure) { throw $runFailure } +if (-not $providerAccepted) { + throw "M4.7 provider readiness was not accepted" +} +Write-Output "PROVIDER_READINESS=accepted" +if ($PreflightOnly) { + Write-Output "GRAPH_READINESS=not-run" + Write-Output "PREFLIGHT=accepted" +} elseif ($graphAccepted) { + Write-Output "GRAPH_READINESS=accepted" +} diff --git a/scripts/build_m47_worker_graph_shadow_artifact.py b/scripts/build_m47_worker_graph_shadow_artifact.py index d6f8cba..a929dbb 100644 --- a/scripts/build_m47_worker_graph_shadow_artifact.py +++ b/scripts/build_m47_worker_graph_shadow_artifact.py @@ -19,7 +19,7 @@ REPOSITORY_ROOT = Path(__file__).resolve().parents[1] BASE_TEMPLATE = REPOSITORY_ROOT / "config/deployment/mission-core-worker-shadow-v1.template.json" RUNNER = REPOSITORY_ROOT / "scripts/Invoke-M47CanonicalGraphShadow.ps1" WHEEL_NAME = "nodedc_mission_core-0.1.0-py3-none-any.whl" -DESCRIPTOR_NAME = "mission-core-worker-m47-graph-shadow-v2.json" +DESCRIPTOR_NAME = "mission-core-worker-m47-graph-shadow-v3.json" PATCH_ID = re.compile(r"^[A-Za-z0-9._-]{1,96}$") EXPECTED_BASE_TEMPLATE_SHA256 = ( "319e7ac7f14e5911ad44234c9ec918c73e11a3406724d5cee3e2ef64bb036e0c" @@ -101,16 +101,18 @@ def render_descriptor( descriptor = json.loads(BASE_TEMPLATE.read_text("utf-8")) descriptor.update( { - "schema_version": "nodedc.mission-core-worker.shadow-release/v2", + "schema_version": "nodedc.mission-core-worker.shadow-release/v3", "patch_id": patch_id, "code_revision": revision, - "transition": "m47-canonical-graph-shadow-v1", + "transition": "m47-canonical-graph-isolated-shadow-v1", "artifact_type": "shadow-release", } ) descriptor["container"].update( { "name": "ndc-mission-core-m47-graph-shadow", + "triton_name": "ndc-mission-core-m47-triton-shadow", + "model_repository_host_path": "D:\\NDC_MISSIONCORE\\runtime\\models", "python_path": ( f"/release/{WHEEL_NAME}:/opt/media:/opt/opencv:/opt/pillow" ), @@ -179,10 +181,19 @@ def render_descriptor( "actuation_allowed": False, }, } + descriptor["rollback"] = { + "durable_worker_action": "none", + "historical_triton_action": "none", + "preserve_failed_evidence": True, + "remove_candidate_graph_container": True, + "remove_candidate_triton_container": True, + "remove_unaccepted_release": True, + } if ( descriptor["predecessor"]["durable_worker"]["name"] != "ndc-mission-core-perception-worker" - or descriptor["rollback"]["durable_worker_action"] != "none" + or descriptor["predecessor"]["triton"]["name"] + != "ndc-mission-core-triton" or descriptor["boundary"]["external_deploy_registry"] is not False ): raise ArtifactBuildError("M4.7 predecessor or deployment boundary changed") @@ -278,7 +289,7 @@ def build_artifact( "code_revision": selected_revision, "wheel_sha256": wheel_sha256, "payload_files": payload_files, - "transition": "m47-canonical-graph-shadow-v1", + "transition": "m47-canonical-graph-isolated-shadow-v1", } diff --git a/tests/test_m47_worker_graph_shadow_artifact.py b/tests/test_m47_worker_graph_shadow_artifact.py index 7236b03..d90f3d0 100644 --- a/tests/test_m47_worker_graph_shadow_artifact.py +++ b/tests/test_m47_worker_graph_shadow_artifact.py @@ -39,7 +39,7 @@ def test_m47_worker_artifact_is_deterministic_and_self_contained(tmp_path: Path) assert first_bytes == second_bytes assert first["sha256"] == _sha256(first_bytes) - assert first["transition"] == "m47-canonical-graph-shadow-v1" + assert first["transition"] == "m47-canonical-graph-isolated-shadow-v1" with tarfile.open(first["artifact"], "r:gz") as archive: regular = _regular_files(archive) payload_names = sorted( @@ -57,7 +57,7 @@ def test_m47_worker_artifact_is_deterministic_and_self_contained(tmp_path: Path) ).read_bytes() -def test_m47_descriptor_preserves_predecessor_and_separates_readiness() -> None: +def test_m47_descriptor_preserves_nonparticipants_and_separates_readiness() -> None: descriptor = json.loads( BUILDER.render_descriptor( "mission-core-m47-graph-shadow-unit-002", @@ -66,10 +66,16 @@ def test_m47_descriptor_preserves_predecessor_and_separates_readiness() -> None: ) ) - assert descriptor["schema_version"] == "nodedc.mission-core-worker.shadow-release/v2" - assert descriptor["transition"] == "m47-canonical-graph-shadow-v1" + assert descriptor["schema_version"] == "nodedc.mission-core-worker.shadow-release/v3" + assert descriptor["transition"] == "m47-canonical-graph-isolated-shadow-v1" assert descriptor["boundary"]["external_deploy_registry"] is False assert descriptor["container"]["public_ports"] is False + assert descriptor["container"]["triton_name"] == ( + "ndc-mission-core-m47-triton-shadow" + ) + assert descriptor["container"]["model_repository_host_path"] == ( + "D:\\NDC_MISSIONCORE\\runtime\\models" + ) assert descriptor["inputs"]["local_surface"]["sha256"] == ( "f57eb2485b6cef47f2a97a2d9ff1aa9fd9265fe1eb69cd5852d12f39e13b8bc6" ) @@ -84,10 +90,11 @@ def test_m47_descriptor_preserves_predecessor_and_separates_readiness() -> None: ) assert descriptor["rollback"] == { "durable_worker_action": "none", + "historical_triton_action": "none", "preserve_failed_evidence": True, - "remove_candidate_container": True, + "remove_candidate_graph_container": True, + "remove_candidate_triton_container": True, "remove_unaccepted_release": True, - "triton_action": "none", } assert descriptor["acceptance"] == { "run_mode": "lossless-replay", @@ -123,4 +130,5 @@ def test_m47_runner_calls_only_the_canonical_graph_entrypoint() -> None: assert 'Write-Output "PROVIDER_READINESS=accepted"' in runner assert 'Write-Output "GRAPH_READINESS=accepted"' in runner assert 'Write-Output "DURABLE_WORKER_ACTION=none"' in runner - assert 'Write-Output "TRITON_ACTION=none"' in runner + assert 'Write-Output "HISTORICAL_TRITON_ACTION=none"' in runner + assert 'Write-Output "ISOLATED_TRITON_ACTION=removed"' in runner