SILVER/NODEDC_MISSION_CORE
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

isolate M4.7 worker shadow runtime

Browse Source
This commit is contained in:
DCCONSTRUCTIONS
2026-08-23 20:19:13 +03:00
parent d50d3bb3d3
commit 3d3c0eced2
4 changed files with 228 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/22_MILESTONE_4_OBJECT_CENTRIC_RECORDED_REALTIME_CV.md
+12 -1
View File
@@ -1172,7 +1172,7 @@ terminal, report and manifest ledgers; acceptance requires exactly `4,489`
delivered frames with zero failed, stale, superseded, rejected or unavailable
terminal outcomes.
The new Worker package uses transition `m47-canonical-graph-shadow-v1`. It is a
The first Worker package used transition `m47-canonical-graph-shadow-v1`. It is a
separate deterministic artifact and does not relabel the accepted historical
`m4-detector-shadow-v1` wheel. Its PowerShell runner verifies release files,
inputs, dependency trees, disk reserve, pinned Worker/Triton predecessor and
@@ -1182,6 +1182,17 @@ the run and re-verifies the predecessor. Provider readiness and graph readiness
are emitted separately. The production builder refuses a dirty worktree so an
artifact cannot claim a Git revision which does not contain its wheel.
The live audit on 2026-08-23 found both pinned historical Mission Core
containers present with their exact identities but stopped. M4.7 therefore
uses the additive transition `m47-canonical-graph-isolated-shadow-v1`: it does
not start, stop or replace either historical container. The runner creates a
private, no-public-port Triton candidate from the same digest and a separate
one-shot graph candidate, binds the accepted model/evidence read-only, and
removes both candidates on every exit path. It records and re-verifies the
identity and running state of the historical E15 worker and Triton before
accepting the run. This makes Worker 006 usable without touching the stabilized
K1/Zarya connection path.
Local contract, graph, result-sealing, artifact and historical-rollback tests
pass. This is implementation evidence only. It does not claim that Worker 006
has the pinned local-surface input, that preflight has passed, that the 4,489
scripts/Invoke-M47CanonicalGraphShadow.ps1
+185 -75
View File
@@ -143,11 +143,11 @@ if ($env:COMPUTERNAME -cne "DESKTOP-OPJ8J04") {
$release = Resolve-DDirectory $ReleaseRoot "M4.7 release root" $false
$payload = Resolve-DDirectory (Join-Path $release "payload") "M4.7 payload" $false
$artifact = Assert-FileSha256 $ArtifactPath $ExpectedArtifactSha256 "M4.7 release artifact"
$descriptorPath = Join-Path $payload "mission-core-worker-m47-graph-shadow-v2.json"
$descriptorPath = Join-Path $payload "mission-core-worker-m47-graph-shadow-v3.json"
$descriptor = Get-Content -LiteralPath $descriptorPath -Raw | ConvertFrom-Json
if (
$descriptor.schema_version -cne "nodedc.mission-core-worker.shadow-release/v2" -or
$descriptor.transition -cne "m47-canonical-graph-shadow-v1" -or
$descriptor.schema_version -cne "nodedc.mission-core-worker.shadow-release/v3" -or
$descriptor.transition -cne "m47-canonical-graph-isolated-shadow-v1" -or
$descriptor.component -cne "mission-core-worker" -or
$descriptor.host.node -cne $env:COMPUTERNAME -or
$descriptor.host.worker_id -cne "worker-006" -or
@@ -215,31 +215,52 @@ foreach ($dependency in $descriptor.dependencies) {
$imageRef = [string]$descriptor.container.image_ref
& docker image inspect $imageRef *> $null
Assert-LastExitCode "Pinned M4.7 image inspection"
$predecessor = $descriptor.predecessor.durable_worker
$tritonExpected = $descriptor.predecessor.triton
$durable = Assert-ContainerIdentity (
$predecessor.name
) $predecessor.container_id $predecessor.image_id $false
$triton = Assert-ContainerIdentity (
$tritonExpected.name
) $tritonExpected.container_id $tritonExpected.image_id $true
function Get-PreservedContainerSnapshot([object]$Expected, [string]$Label) {
$container = Get-ContainerIdentity $Expected.name
if (
$container.Id -cne $Expected.container_id -or
$container.Image -cne $Expected.image_id
) {
throw "$Label identity changed"
}
return [pscustomobject]@{
Id = [string]$container.Id
Image = [string]$container.Image
Running = [bool]$container.State.Running
}
}
function Assert-PreservedContainerSnapshot(
[object]$Expected,
[object]$Before,
[string]$Label
) {
$after = Get-PreservedContainerSnapshot $Expected $Label
if ($after.Running -ne $Before.Running) {
throw "$Label running state changed during isolated shadow"
}
return $after
}
$durableExpected = $descriptor.predecessor.durable_worker
$historicalTritonExpected = $descriptor.predecessor.triton
$durableBefore = Get-PreservedContainerSnapshot $durableExpected "Historical durable worker"
$historicalTritonBefore = Get-PreservedContainerSnapshot (
$historicalTritonExpected
) "Historical Triton"
$modelRepository = Resolve-DDirectory (
[string]$descriptor.container.model_repository_host_path
) "M4.7 model repository" $false
$output = Resolve-DDirectory $OutputRoot "M4.7 output root" $true
$freeBefore = Assert-FreeSpace "preflight"
if ($PreflightOnly) {
Write-Output ("PATCH_ID={0}" -f $descriptor.patch_id)
Write-Output ("ARTIFACT_SHA256={0}" -f $ExpectedArtifactSha256)
Write-Output ("DURABLE_WORKER_ID={0}" -f $durable.Id)
Write-Output ("TRITON_CONTAINER_ID={0}" -f $triton.Id)
Write-Output "PROVIDER_READINESS=accepted"
Write-Output "GRAPH_READINESS=not-run"
Write-Output "PREFLIGHT=accepted"
return
}
$candidateName = "ndc-mission-core-m47-graph-shadow"
if (& docker ps -a --format "{{.Names}}" --filter "name=^/$candidateName$") {
throw "M4.7 candidate container already exists"
$candidateName = [string]$descriptor.container.name
$tritonCandidateName = [string]$descriptor.container.triton_name
foreach ($name in @($candidateName, $tritonCandidateName)) {
if (& docker ps -a --format "{{.Names}}" --filter "name=^/$name$") {
throw "M4.7 isolated candidate $name already exists"
}
}
$scratch = Join-Path $output (".runtime-{0}" -f $descriptor.patch_id)
if (Test-Path -LiteralPath $scratch) {
@@ -251,10 +272,40 @@ $runtimeIdentityPath = Join-Path $scratch "runtime-identity.json"
$dockerPayload = Convert-ToDockerPath $payload
$dockerOutput = Convert-ToDockerPath $output
$dockerScratch = Convert-ToDockerPath $scratch
$dockerArguments = @(
$dockerModelRepository = Convert-ToDockerPath $modelRepository
$tritonArguments = @(
"create",
"--name", $tritonCandidateName,
"--read-only",
"--security-opt", "no-new-privileges:true",
"--cap-drop", "ALL",
"--pids-limit", "512",
"--shm-size", "1g",
"--gpus", "all",
"--tmpfs", "/tmp:rw,noexec,nosuid,size=2g",
"--health-cmd", "curl --fail --silent http://127.0.0.1:8000/v2/health/ready",
"--health-interval", "5s",
"--health-timeout", "3s",
"--health-start-period", "20s",
"--health-retries", "24",
"-v", ("{0}:/models:ro" -f $dockerModelRepository),
$imageRef,
"tritonserver",
"--model-repository=/models",
"--model-control-mode=explicit",
"--load-model=yolox_s",
"--disable-auto-complete-config",
"--strict-readiness=true",
"--exit-on-error=true",
"--allow-http=true",
"--allow-grpc=false",
"--allow-metrics=false"
)
$graphArguments = @(
"create",
"--name", $candidateName,
"--network", ("container:{0}" -f $tritonExpected.name),
"--network", ("container:{0}" -f $tritonCandidateName),
"--read-only",
"--security-opt", "no-new-privileges:true",
"--cap-drop", "ALL",
@@ -268,7 +319,7 @@ $dockerArguments = @(
)
foreach ($entry in $descriptor.inputs.PSObject.Properties) {
if ($null -ne $entry.Value.container_path) {
$dockerArguments += @(
$graphArguments += @(
"-v", ("{0}:{1}:ro" -f (
Convert-ToDockerPath $entry.Value.host_path
), $entry.Value.container_path)
@@ -276,13 +327,13 @@ foreach ($entry in $descriptor.inputs.PSObject.Properties) {
}
}
foreach ($dependency in $descriptor.dependencies) {
$dockerArguments += @(
$graphArguments += @(
"-v", ("{0}:{1}:ro" -f (
Convert-ToDockerPath $dependency.host_path
), $dependency.container_path)
)
}
$dockerArguments += @(
$graphArguments += @(
"--entrypoint", "python3",
$imageRef,
"-m", "k1link.perception.reference_graph_cli",
@@ -305,72 +356,131 @@ $dockerArguments += @(
"--output-root", "/output"
)
$candidateCreated = $false
$tritonCreated = $false
$graphCreated = $false
$providerAccepted = $false
$graphAccepted = $false
$runFailure = $null
try {
$candidateId = (& docker @dockerArguments).Trim()
Assert-LastExitCode "M4.7 candidate creation"
if ($candidateId -notmatch "^[a-f0-9]{64}$") {
throw "M4.7 candidate id is invalid"
$tritonCandidateId = (& docker @tritonArguments).Trim()
Assert-LastExitCode "M4.7 isolated Triton creation"
if ($tritonCandidateId -notmatch "^[a-f0-9]{64}$") {
throw "M4.7 isolated Triton id is invalid"
}
$tritonCreated = $true
& docker start $tritonCandidateName *> $null
Assert-LastExitCode "M4.7 isolated Triton start"
$tritonCandidate = $null
foreach ($attempt in 1..120) {
$tritonCandidate = Get-ContainerIdentity $tritonCandidateName
if (-not $tritonCandidate.State.Running) {
throw "M4.7 isolated Triton stopped before readiness"
}
if ($tritonCandidate.State.Health.Status -ceq "healthy") {
break
}
if ($attempt -eq 120) {
throw "M4.7 isolated Triton readiness timed out"
}
Start-Sleep -Seconds 2
}
$candidateCreated = $true
$candidate = Get-ContainerIdentity $candidateName
if (
$candidate.Id -cne $candidateId -or
$candidate.Image -cne $descriptor.container.image_id -or
$candidate.HostConfig.NetworkMode -cne ("container:{0}" -f $triton.Id) -or
-not $candidate.HostConfig.ReadonlyRootfs
$tritonCandidate.Id -cne $tritonCandidateId -or
$tritonCandidate.Image -cne $descriptor.container.image_id -or
-not $tritonCandidate.HostConfig.ReadonlyRootfs -or
$null -ne $tritonCandidate.HostConfig.PortBindings
) {
throw "M4.7 candidate isolation contract changed"
throw "M4.7 isolated Triton contract changed"
}
$runtimeIdentity = [ordered]@{
schema_version = "missioncore.reference-graph-runtime-identity/v1"
worker_id = "worker-006"
worker_node = $env:COMPUTERNAME
worker_container_id = $candidate.Id
worker_image_id = $candidate.Image
triton_container_id = $triton.Id
triton_image_id = $triton.Image
artifact_sha256 = $ExpectedArtifactSha256
code_revision = $descriptor.code_revision
graph_id = $descriptor.readiness.graph.graph_id
source_mount_read_only = $true
model_service_reused = $true
public_worker_port_added = $false
commands_enabled = $false
actuation_allowed = $false
$providerAccepted = $true
if (-not $PreflightOnly) {
$candidateId = (& docker @graphArguments).Trim()
Assert-LastExitCode "M4.7 graph candidate creation"
if ($candidateId -notmatch "^[a-f0-9]{64}$") {
throw "M4.7 graph candidate id is invalid"
}
$graphCreated = $true
$candidate = Get-ContainerIdentity $candidateName
if (
$candidate.Id -cne $candidateId -or
$candidate.Image -cne $descriptor.container.image_id -or
$candidate.HostConfig.NetworkMode -cne ("container:{0}" -f $tritonCandidate.Id) -or
-not $candidate.HostConfig.ReadonlyRootfs -or
$null -ne $candidate.HostConfig.PortBindings
) {
throw "M4.7 graph candidate isolation contract changed"
}
$runtimeIdentity = [ordered]@{
schema_version = "missioncore.reference-graph-runtime-identity/v2"
worker_id = "worker-006"
worker_node = $env:COMPUTERNAME
worker_container_id = $candidate.Id
worker_image_id = $candidate.Image
isolated_triton_container_id = $tritonCandidate.Id
isolated_triton_image_id = $tritonCandidate.Image
historical_worker_container_id = $durableBefore.Id
historical_worker_running = $durableBefore.Running
historical_triton_container_id = $historicalTritonBefore.Id
historical_triton_running = $historicalTritonBefore.Running
artifact_sha256 = $ExpectedArtifactSha256
code_revision = $descriptor.code_revision
graph_id = $descriptor.readiness.graph.graph_id
source_mount_read_only = $true
isolated_model_service = $true
public_worker_port_added = $false
commands_enabled = $false
actuation_allowed = $false
}
Write-Utf8NoBom $runtimeIdentityPath ($runtimeIdentity | ConvertTo-Json -Depth 4)
& docker start --attach $candidateName
Assert-LastExitCode "M4.7 canonical graph isolated shadow"
$graphAccepted = $true
}
Write-Utf8NoBom $runtimeIdentityPath ($runtimeIdentity | ConvertTo-Json -Depth 4)
Write-Output ("PATCH_ID={0}" -f $descriptor.patch_id)
Write-Output ("ARTIFACT_SHA256={0}" -f $ExpectedArtifactSha256)
Write-Output ("CANDIDATE_CONTAINER_ID={0}" -f $candidate.Id)
Write-Output "PROVIDER_READINESS=accepted"
& docker start --attach $candidateName
Assert-LastExitCode "M4.7 canonical graph shadow"
Write-Output "GRAPH_READINESS=accepted"
} catch {
$runFailure = $_
} finally {
if ($candidateCreated) {
if ($graphCreated) {
& docker rm --force $candidateName *> $null
if ($LASTEXITCODE -ne 0 -and $null -eq $runFailure) {
$runFailure = "M4.7 candidate cleanup failed"
$runFailure = "M4.7 graph candidate cleanup failed"
}
}
if ($tritonCreated) {
& docker rm --force $tritonCandidateName *> $null
if ($LASTEXITCODE -ne 0 -and $null -eq $runFailure) {
$runFailure = "M4.7 isolated Triton cleanup failed"
}
}
Remove-Item -LiteralPath $scratch -Force -Recurse -ErrorAction SilentlyContinue
}
$null = Assert-ContainerIdentity $predecessor.name $predecessor.container_id (
$predecessor.image_id
) $false
$null = Assert-ContainerIdentity $tritonExpected.name $tritonExpected.container_id (
$tritonExpected.image_id
) $true
$durableAfter = Assert-PreservedContainerSnapshot (
$durableExpected
) $durableBefore "Historical durable worker"
$historicalTritonAfter = Assert-PreservedContainerSnapshot (
$historicalTritonExpected
) $historicalTritonBefore "Historical Triton"
$freeAfter = Assert-FreeSpace "completed"
Write-Output ("PATCH_ID={0}" -f $descriptor.patch_id)
Write-Output ("ARTIFACT_SHA256={0}" -f $ExpectedArtifactSha256)
Write-Output ("DISK_FREE_BYTES_BEFORE={0}" -f $freeBefore)
Write-Output ("DISK_FREE_BYTES_AFTER={0}" -f $freeAfter)
Write-Output ("HISTORICAL_DURABLE_WORKER_RUNNING={0}" -f $durableAfter.Running)
Write-Output ("HISTORICAL_TRITON_RUNNING={0}" -f $historicalTritonAfter.Running)
Write-Output "DURABLE_WORKER_ACTION=none"
Write-Output "TRITON_ACTION=none"
Write-Output "HISTORICAL_TRITON_ACTION=none"
Write-Output "ISOLATED_TRITON_ACTION=removed"
if ($null -ne $runFailure) {
throw $runFailure
}
if (-not $providerAccepted) {
throw "M4.7 provider readiness was not accepted"
}
Write-Output "PROVIDER_READINESS=accepted"
if ($PreflightOnly) {
Write-Output "GRAPH_READINESS=not-run"
Write-Output "PREFLIGHT=accepted"
} elseif ($graphAccepted) {
Write-Output "GRAPH_READINESS=accepted"
}
scripts/build_m47_worker_graph_shadow_artifact.py
+16 -5
View File
@@ -19,7 +19,7 @@ REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
BASE_TEMPLATE = REPOSITORY_ROOT / "config/deployment/mission-core-worker-shadow-v1.template.json"
RUNNER = REPOSITORY_ROOT / "scripts/Invoke-M47CanonicalGraphShadow.ps1"
WHEEL_NAME = "nodedc_mission_core-0.1.0-py3-none-any.whl"
DESCRIPTOR_NAME = "mission-core-worker-m47-graph-shadow-v2.json"
DESCRIPTOR_NAME = "mission-core-worker-m47-graph-shadow-v3.json"
PATCH_ID = re.compile(r"^[A-Za-z0-9._-]{1,96}$")
EXPECTED_BASE_TEMPLATE_SHA256 = (
"319e7ac7f14e5911ad44234c9ec918c73e11a3406724d5cee3e2ef64bb036e0c"
@@ -101,16 +101,18 @@ def render_descriptor(
descriptor = json.loads(BASE_TEMPLATE.read_text("utf-8"))
descriptor.update(
{
"schema_version": "nodedc.mission-core-worker.shadow-release/v2",
"schema_version": "nodedc.mission-core-worker.shadow-release/v3",
"patch_id": patch_id,
"code_revision": revision,
"transition": "m47-canonical-graph-shadow-v1",
"transition": "m47-canonical-graph-isolated-shadow-v1",
"artifact_type": "shadow-release",
}
)
descriptor["container"].update(
{
"name": "ndc-mission-core-m47-graph-shadow",
"triton_name": "ndc-mission-core-m47-triton-shadow",
"model_repository_host_path": "D:\\NDC_MISSIONCORE\\runtime\\models",
"python_path": (
f"/release/{WHEEL_NAME}:/opt/media:/opt/opencv:/opt/pillow"
),
@@ -179,10 +181,19 @@ def render_descriptor(
"actuation_allowed": False,
},
}
descriptor["rollback"] = {
"durable_worker_action": "none",
"historical_triton_action": "none",
"preserve_failed_evidence": True,
"remove_candidate_graph_container": True,
"remove_candidate_triton_container": True,
"remove_unaccepted_release": True,
}
if (
descriptor["predecessor"]["durable_worker"]["name"]
!= "ndc-mission-core-perception-worker"
or descriptor["rollback"]["durable_worker_action"] != "none"
or descriptor["predecessor"]["triton"]["name"]
!= "ndc-mission-core-triton"
or descriptor["boundary"]["external_deploy_registry"] is not False
):
raise ArtifactBuildError("M4.7 predecessor or deployment boundary changed")
@@ -278,7 +289,7 @@ def build_artifact(
"code_revision": selected_revision,
"wheel_sha256": wheel_sha256,
"payload_files": payload_files,
"transition": "m47-canonical-graph-shadow-v1",
"transition": "m47-canonical-graph-isolated-shadow-v1",
}
tests/test_m47_worker_graph_shadow_artifact.py
+15 -7
View File
@@ -39,7 +39,7 @@ def test_m47_worker_artifact_is_deterministic_and_self_contained(tmp_path: Path)
assert first_bytes == second_bytes
assert first["sha256"] == _sha256(first_bytes)
assert first["transition"] == "m47-canonical-graph-shadow-v1"
assert first["transition"] == "m47-canonical-graph-isolated-shadow-v1"
with tarfile.open(first["artifact"], "r:gz") as archive:
regular = _regular_files(archive)
payload_names = sorted(
@@ -57,7 +57,7 @@ def test_m47_worker_artifact_is_deterministic_and_self_contained(tmp_path: Path)
).read_bytes()
def test_m47_descriptor_preserves_predecessor_and_separates_readiness() -> None:
def test_m47_descriptor_preserves_nonparticipants_and_separates_readiness() -> None:
descriptor = json.loads(
BUILDER.render_descriptor(
"mission-core-m47-graph-shadow-unit-002",
@@ -66,10 +66,16 @@ def test_m47_descriptor_preserves_predecessor_and_separates_readiness() -> None:
)
)
assert descriptor["schema_version"] == "nodedc.mission-core-worker.shadow-release/v2"
assert descriptor["transition"] == "m47-canonical-graph-shadow-v1"
assert descriptor["schema_version"] == "nodedc.mission-core-worker.shadow-release/v3"
assert descriptor["transition"] == "m47-canonical-graph-isolated-shadow-v1"
assert descriptor["boundary"]["external_deploy_registry"] is False
assert descriptor["container"]["public_ports"] is False
assert descriptor["container"]["triton_name"] == (
"ndc-mission-core-m47-triton-shadow"
)
assert descriptor["container"]["model_repository_host_path"] == (
"D:\\NDC_MISSIONCORE\\runtime\\models"
)
assert descriptor["inputs"]["local_surface"]["sha256"] == (
"f57eb2485b6cef47f2a97a2d9ff1aa9fd9265fe1eb69cd5852d12f39e13b8bc6"
)
@@ -84,10 +90,11 @@ def test_m47_descriptor_preserves_predecessor_and_separates_readiness() -> None:
)
assert descriptor["rollback"] == {
"durable_worker_action": "none",
"historical_triton_action": "none",
"preserve_failed_evidence": True,
"remove_candidate_container": True,
"remove_candidate_graph_container": True,
"remove_candidate_triton_container": True,
"remove_unaccepted_release": True,
"triton_action": "none",
}
assert descriptor["acceptance"] == {
"run_mode": "lossless-replay",
@@ -123,4 +130,5 @@ def test_m47_runner_calls_only_the_canonical_graph_entrypoint() -> None:
assert 'Write-Output "PROVIDER_READINESS=accepted"' in runner
assert 'Write-Output "GRAPH_READINESS=accepted"' in runner
assert 'Write-Output "DURABLE_WORKER_ACTION=none"' in runner
assert 'Write-Output "TRITON_ACTION=none"' in runner
assert 'Write-Output "HISTORICAL_TRITON_ACTION=none"' in runner
assert 'Write-Output "ISOLATED_TRITON_ACTION=removed"' in runner