isolate M4.7 worker shadow runtime
This commit is contained in:
DCCONSTRUCTIONS
@@ -143,11 +143,11 @@ if ($env:COMPUTERNAME -cne "DESKTOP-OPJ8J04") {
|
||||
$release = Resolve-DDirectory $ReleaseRoot "M4.7 release root" $false
|
||||
$payload = Resolve-DDirectory (Join-Path $release "payload") "M4.7 payload" $false
|
||||
$artifact = Assert-FileSha256 $ArtifactPath $ExpectedArtifactSha256 "M4.7 release artifact"
|
||||
$descriptorPath = Join-Path $payload "mission-core-worker-m47-graph-shadow-v2.json"
|
||||
$descriptorPath = Join-Path $payload "mission-core-worker-m47-graph-shadow-v3.json"
|
||||
$descriptor = Get-Content -LiteralPath $descriptorPath -Raw | ConvertFrom-Json
|
||||
if (
|
||||
$descriptor.schema_version -cne "nodedc.mission-core-worker.shadow-release/v2" -or
|
||||
$descriptor.transition -cne "m47-canonical-graph-shadow-v1" -or
|
||||
$descriptor.schema_version -cne "nodedc.mission-core-worker.shadow-release/v3" -or
|
||||
$descriptor.transition -cne "m47-canonical-graph-isolated-shadow-v1" -or
|
||||
$descriptor.component -cne "mission-core-worker" -or
|
||||
$descriptor.host.node -cne $env:COMPUTERNAME -or
|
||||
$descriptor.host.worker_id -cne "worker-006" -or
|
||||
@@ -215,31 +215,52 @@ foreach ($dependency in $descriptor.dependencies) {
|
||||
$imageRef = [string]$descriptor.container.image_ref
|
||||
& docker image inspect $imageRef *> $null
|
||||
Assert-LastExitCode "Pinned M4.7 image inspection"
|
||||
$predecessor = $descriptor.predecessor.durable_worker
|
||||
$tritonExpected = $descriptor.predecessor.triton
|
||||
$durable = Assert-ContainerIdentity (
|
||||
$predecessor.name
|
||||
) $predecessor.container_id $predecessor.image_id $false
|
||||
$triton = Assert-ContainerIdentity (
|
||||
$tritonExpected.name
|
||||
) $tritonExpected.container_id $tritonExpected.image_id $true
|
||||
|
||||
function Get-PreservedContainerSnapshot([object]$Expected, [string]$Label) {
|
||||
$container = Get-ContainerIdentity $Expected.name
|
||||
if (
|
||||
$container.Id -cne $Expected.container_id -or
|
||||
$container.Image -cne $Expected.image_id
|
||||
) {
|
||||
throw "$Label identity changed"
|
||||
}
|
||||
return [pscustomobject]@{
|
||||
Id = [string]$container.Id
|
||||
Image = [string]$container.Image
|
||||
Running = [bool]$container.State.Running
|
||||
}
|
||||
}
|
||||
|
||||
function Assert-PreservedContainerSnapshot(
|
||||
[object]$Expected,
|
||||
[object]$Before,
|
||||
[string]$Label
|
||||
) {
|
||||
$after = Get-PreservedContainerSnapshot $Expected $Label
|
||||
if ($after.Running -ne $Before.Running) {
|
||||
throw "$Label running state changed during isolated shadow"
|
||||
}
|
||||
return $after
|
||||
}
|
||||
|
||||
$durableExpected = $descriptor.predecessor.durable_worker
|
||||
$historicalTritonExpected = $descriptor.predecessor.triton
|
||||
$durableBefore = Get-PreservedContainerSnapshot $durableExpected "Historical durable worker"
|
||||
$historicalTritonBefore = Get-PreservedContainerSnapshot (
|
||||
$historicalTritonExpected
|
||||
) "Historical Triton"
|
||||
$modelRepository = Resolve-DDirectory (
|
||||
[string]$descriptor.container.model_repository_host_path
|
||||
) "M4.7 model repository" $false
|
||||
$output = Resolve-DDirectory $OutputRoot "M4.7 output root" $true
|
||||
$freeBefore = Assert-FreeSpace "preflight"
|
||||
|
||||
if ($PreflightOnly) {
|
||||
Write-Output ("PATCH_ID={0}" -f $descriptor.patch_id)
|
||||
Write-Output ("ARTIFACT_SHA256={0}" -f $ExpectedArtifactSha256)
|
||||
Write-Output ("DURABLE_WORKER_ID={0}" -f $durable.Id)
|
||||
Write-Output ("TRITON_CONTAINER_ID={0}" -f $triton.Id)
|
||||
Write-Output "PROVIDER_READINESS=accepted"
|
||||
Write-Output "GRAPH_READINESS=not-run"
|
||||
Write-Output "PREFLIGHT=accepted"
|
||||
return
|
||||
}
|
||||
|
||||
$candidateName = "ndc-mission-core-m47-graph-shadow"
|
||||
if (& docker ps -a --format "{{.Names}}" --filter "name=^/$candidateName$") {
|
||||
throw "M4.7 candidate container already exists"
|
||||
$candidateName = [string]$descriptor.container.name
|
||||
$tritonCandidateName = [string]$descriptor.container.triton_name
|
||||
foreach ($name in @($candidateName, $tritonCandidateName)) {
|
||||
if (& docker ps -a --format "{{.Names}}" --filter "name=^/$name$") {
|
||||
throw "M4.7 isolated candidate $name already exists"
|
||||
}
|
||||
}
|
||||
$scratch = Join-Path $output (".runtime-{0}" -f $descriptor.patch_id)
|
||||
if (Test-Path -LiteralPath $scratch) {
|
||||
@@ -251,10 +272,40 @@ $runtimeIdentityPath = Join-Path $scratch "runtime-identity.json"
|
||||
$dockerPayload = Convert-ToDockerPath $payload
|
||||
$dockerOutput = Convert-ToDockerPath $output
|
||||
$dockerScratch = Convert-ToDockerPath $scratch
|
||||
$dockerArguments = @(
|
||||
$dockerModelRepository = Convert-ToDockerPath $modelRepository
|
||||
$tritonArguments = @(
|
||||
"create",
|
||||
"--name", $tritonCandidateName,
|
||||
"--read-only",
|
||||
"--security-opt", "no-new-privileges:true",
|
||||
"--cap-drop", "ALL",
|
||||
"--pids-limit", "512",
|
||||
"--shm-size", "1g",
|
||||
"--gpus", "all",
|
||||
"--tmpfs", "/tmp:rw,noexec,nosuid,size=2g",
|
||||
"--health-cmd", "curl --fail --silent http://127.0.0.1:8000/v2/health/ready",
|
||||
"--health-interval", "5s",
|
||||
"--health-timeout", "3s",
|
||||
"--health-start-period", "20s",
|
||||
"--health-retries", "24",
|
||||
"-v", ("{0}:/models:ro" -f $dockerModelRepository),
|
||||
$imageRef,
|
||||
"tritonserver",
|
||||
"--model-repository=/models",
|
||||
"--model-control-mode=explicit",
|
||||
"--load-model=yolox_s",
|
||||
"--disable-auto-complete-config",
|
||||
"--strict-readiness=true",
|
||||
"--exit-on-error=true",
|
||||
"--allow-http=true",
|
||||
"--allow-grpc=false",
|
||||
"--allow-metrics=false"
|
||||
)
|
||||
|
||||
$graphArguments = @(
|
||||
"create",
|
||||
"--name", $candidateName,
|
||||
"--network", ("container:{0}" -f $tritonExpected.name),
|
||||
"--network", ("container:{0}" -f $tritonCandidateName),
|
||||
"--read-only",
|
||||
"--security-opt", "no-new-privileges:true",
|
||||
"--cap-drop", "ALL",
|
||||
@@ -268,7 +319,7 @@ $dockerArguments = @(
|
||||
)
|
||||
foreach ($entry in $descriptor.inputs.PSObject.Properties) {
|
||||
if ($null -ne $entry.Value.container_path) {
|
||||
$dockerArguments += @(
|
||||
$graphArguments += @(
|
||||
"-v", ("{0}:{1}:ro" -f (
|
||||
Convert-ToDockerPath $entry.Value.host_path
|
||||
), $entry.Value.container_path)
|
||||
@@ -276,13 +327,13 @@ foreach ($entry in $descriptor.inputs.PSObject.Properties) {
|
||||
}
|
||||
}
|
||||
foreach ($dependency in $descriptor.dependencies) {
|
||||
$dockerArguments += @(
|
||||
$graphArguments += @(
|
||||
"-v", ("{0}:{1}:ro" -f (
|
||||
Convert-ToDockerPath $dependency.host_path
|
||||
), $dependency.container_path)
|
||||
)
|
||||
}
|
||||
$dockerArguments += @(
|
||||
$graphArguments += @(
|
||||
"--entrypoint", "python3",
|
||||
$imageRef,
|
||||
"-m", "k1link.perception.reference_graph_cli",
|
||||
@@ -305,72 +356,131 @@ $dockerArguments += @(
|
||||
"--output-root", "/output"
|
||||
)
|
||||
|
||||
$candidateCreated = $false
|
||||
$tritonCreated = $false
|
||||
$graphCreated = $false
|
||||
$providerAccepted = $false
|
||||
$graphAccepted = $false
|
||||
$runFailure = $null
|
||||
try {
|
||||
$candidateId = (& docker @dockerArguments).Trim()
|
||||
Assert-LastExitCode "M4.7 candidate creation"
|
||||
if ($candidateId -notmatch "^[a-f0-9]{64}$") {
|
||||
throw "M4.7 candidate id is invalid"
|
||||
$tritonCandidateId = (& docker @tritonArguments).Trim()
|
||||
Assert-LastExitCode "M4.7 isolated Triton creation"
|
||||
if ($tritonCandidateId -notmatch "^[a-f0-9]{64}$") {
|
||||
throw "M4.7 isolated Triton id is invalid"
|
||||
}
|
||||
$tritonCreated = $true
|
||||
& docker start $tritonCandidateName *> $null
|
||||
Assert-LastExitCode "M4.7 isolated Triton start"
|
||||
$tritonCandidate = $null
|
||||
foreach ($attempt in 1..120) {
|
||||
$tritonCandidate = Get-ContainerIdentity $tritonCandidateName
|
||||
if (-not $tritonCandidate.State.Running) {
|
||||
throw "M4.7 isolated Triton stopped before readiness"
|
||||
}
|
||||
if ($tritonCandidate.State.Health.Status -ceq "healthy") {
|
||||
break
|
||||
}
|
||||
if ($attempt -eq 120) {
|
||||
throw "M4.7 isolated Triton readiness timed out"
|
||||
}
|
||||
Start-Sleep -Seconds 2
|
||||
}
|
||||
$candidateCreated = $true
|
||||
$candidate = Get-ContainerIdentity $candidateName
|
||||
if (
|
||||
$candidate.Id -cne $candidateId -or
|
||||
$candidate.Image -cne $descriptor.container.image_id -or
|
||||
$candidate.HostConfig.NetworkMode -cne ("container:{0}" -f $triton.Id) -or
|
||||
-not $candidate.HostConfig.ReadonlyRootfs
|
||||
$tritonCandidate.Id -cne $tritonCandidateId -or
|
||||
$tritonCandidate.Image -cne $descriptor.container.image_id -or
|
||||
-not $tritonCandidate.HostConfig.ReadonlyRootfs -or
|
||||
$null -ne $tritonCandidate.HostConfig.PortBindings
|
||||
) {
|
||||
throw "M4.7 candidate isolation contract changed"
|
||||
throw "M4.7 isolated Triton contract changed"
|
||||
}
|
||||
$runtimeIdentity = [ordered]@{
|
||||
schema_version = "missioncore.reference-graph-runtime-identity/v1"
|
||||
worker_id = "worker-006"
|
||||
worker_node = $env:COMPUTERNAME
|
||||
worker_container_id = $candidate.Id
|
||||
worker_image_id = $candidate.Image
|
||||
triton_container_id = $triton.Id
|
||||
triton_image_id = $triton.Image
|
||||
artifact_sha256 = $ExpectedArtifactSha256
|
||||
code_revision = $descriptor.code_revision
|
||||
graph_id = $descriptor.readiness.graph.graph_id
|
||||
source_mount_read_only = $true
|
||||
model_service_reused = $true
|
||||
public_worker_port_added = $false
|
||||
commands_enabled = $false
|
||||
actuation_allowed = $false
|
||||
$providerAccepted = $true
|
||||
|
||||
if (-not $PreflightOnly) {
|
||||
$candidateId = (& docker @graphArguments).Trim()
|
||||
Assert-LastExitCode "M4.7 graph candidate creation"
|
||||
if ($candidateId -notmatch "^[a-f0-9]{64}$") {
|
||||
throw "M4.7 graph candidate id is invalid"
|
||||
}
|
||||
$graphCreated = $true
|
||||
$candidate = Get-ContainerIdentity $candidateName
|
||||
if (
|
||||
$candidate.Id -cne $candidateId -or
|
||||
$candidate.Image -cne $descriptor.container.image_id -or
|
||||
$candidate.HostConfig.NetworkMode -cne ("container:{0}" -f $tritonCandidate.Id) -or
|
||||
-not $candidate.HostConfig.ReadonlyRootfs -or
|
||||
$null -ne $candidate.HostConfig.PortBindings
|
||||
) {
|
||||
throw "M4.7 graph candidate isolation contract changed"
|
||||
}
|
||||
$runtimeIdentity = [ordered]@{
|
||||
schema_version = "missioncore.reference-graph-runtime-identity/v2"
|
||||
worker_id = "worker-006"
|
||||
worker_node = $env:COMPUTERNAME
|
||||
worker_container_id = $candidate.Id
|
||||
worker_image_id = $candidate.Image
|
||||
isolated_triton_container_id = $tritonCandidate.Id
|
||||
isolated_triton_image_id = $tritonCandidate.Image
|
||||
historical_worker_container_id = $durableBefore.Id
|
||||
historical_worker_running = $durableBefore.Running
|
||||
historical_triton_container_id = $historicalTritonBefore.Id
|
||||
historical_triton_running = $historicalTritonBefore.Running
|
||||
artifact_sha256 = $ExpectedArtifactSha256
|
||||
code_revision = $descriptor.code_revision
|
||||
graph_id = $descriptor.readiness.graph.graph_id
|
||||
source_mount_read_only = $true
|
||||
isolated_model_service = $true
|
||||
public_worker_port_added = $false
|
||||
commands_enabled = $false
|
||||
actuation_allowed = $false
|
||||
}
|
||||
Write-Utf8NoBom $runtimeIdentityPath ($runtimeIdentity | ConvertTo-Json -Depth 4)
|
||||
& docker start --attach $candidateName
|
||||
Assert-LastExitCode "M4.7 canonical graph isolated shadow"
|
||||
$graphAccepted = $true
|
||||
}
|
||||
Write-Utf8NoBom $runtimeIdentityPath ($runtimeIdentity | ConvertTo-Json -Depth 4)
|
||||
Write-Output ("PATCH_ID={0}" -f $descriptor.patch_id)
|
||||
Write-Output ("ARTIFACT_SHA256={0}" -f $ExpectedArtifactSha256)
|
||||
Write-Output ("CANDIDATE_CONTAINER_ID={0}" -f $candidate.Id)
|
||||
Write-Output "PROVIDER_READINESS=accepted"
|
||||
& docker start --attach $candidateName
|
||||
Assert-LastExitCode "M4.7 canonical graph shadow"
|
||||
Write-Output "GRAPH_READINESS=accepted"
|
||||
} catch {
|
||||
$runFailure = $_
|
||||
} finally {
|
||||
if ($candidateCreated) {
|
||||
if ($graphCreated) {
|
||||
& docker rm --force $candidateName *> $null
|
||||
if ($LASTEXITCODE -ne 0 -and $null -eq $runFailure) {
|
||||
$runFailure = "M4.7 candidate cleanup failed"
|
||||
$runFailure = "M4.7 graph candidate cleanup failed"
|
||||
}
|
||||
}
|
||||
if ($tritonCreated) {
|
||||
& docker rm --force $tritonCandidateName *> $null
|
||||
if ($LASTEXITCODE -ne 0 -and $null -eq $runFailure) {
|
||||
$runFailure = "M4.7 isolated Triton cleanup failed"
|
||||
}
|
||||
}
|
||||
Remove-Item -LiteralPath $scratch -Force -Recurse -ErrorAction SilentlyContinue
|
||||
}
|
||||
|
||||
$null = Assert-ContainerIdentity $predecessor.name $predecessor.container_id (
|
||||
$predecessor.image_id
|
||||
) $false
|
||||
$null = Assert-ContainerIdentity $tritonExpected.name $tritonExpected.container_id (
|
||||
$tritonExpected.image_id
|
||||
) $true
|
||||
$durableAfter = Assert-PreservedContainerSnapshot (
|
||||
$durableExpected
|
||||
) $durableBefore "Historical durable worker"
|
||||
$historicalTritonAfter = Assert-PreservedContainerSnapshot (
|
||||
$historicalTritonExpected
|
||||
) $historicalTritonBefore "Historical Triton"
|
||||
$freeAfter = Assert-FreeSpace "completed"
|
||||
Write-Output ("PATCH_ID={0}" -f $descriptor.patch_id)
|
||||
Write-Output ("ARTIFACT_SHA256={0}" -f $ExpectedArtifactSha256)
|
||||
Write-Output ("DISK_FREE_BYTES_BEFORE={0}" -f $freeBefore)
|
||||
Write-Output ("DISK_FREE_BYTES_AFTER={0}" -f $freeAfter)
|
||||
Write-Output ("HISTORICAL_DURABLE_WORKER_RUNNING={0}" -f $durableAfter.Running)
|
||||
Write-Output ("HISTORICAL_TRITON_RUNNING={0}" -f $historicalTritonAfter.Running)
|
||||
Write-Output "DURABLE_WORKER_ACTION=none"
|
||||
Write-Output "TRITON_ACTION=none"
|
||||
Write-Output "HISTORICAL_TRITON_ACTION=none"
|
||||
Write-Output "ISOLATED_TRITON_ACTION=removed"
|
||||
if ($null -ne $runFailure) {
|
||||
throw $runFailure
|
||||
}
|
||||
if (-not $providerAccepted) {
|
||||
throw "M4.7 provider readiness was not accepted"
|
||||
}
|
||||
Write-Output "PROVIDER_READINESS=accepted"
|
||||
if ($PreflightOnly) {
|
||||
Write-Output "GRAPH_READINESS=not-run"
|
||||
Write-Output "PREFLIGHT=accepted"
|
||||
} elseif ($graphAccepted) {
|
||||
Write-Output "GRAPH_READINESS=accepted"
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user