feat(perception): stage TRAVEL qualification
This commit is contained in:
@@ -0,0 +1,120 @@
|
|||||||
|
[CmdletBinding()]
|
||||||
|
param(
|
||||||
|
[Parameter(Mandatory = $true)]
|
||||||
|
[string]$BuildContext,
|
||||||
|
[Parameter(Mandatory = $true)]
|
||||||
|
[ValidatePattern("^[A-Za-z0-9._-]{1,96}$")]
|
||||||
|
[string]$RunId,
|
||||||
|
[string]$OutputRoot = "D:\NDC_MISSIONCORE\runtime\results\m49-t3-travel"
|
||||||
|
)
|
||||||
|
|
||||||
|
$ErrorActionPreference = "Stop"
|
||||||
|
$ProgressPreference = "SilentlyContinue"
|
||||||
|
|
||||||
|
function Assert-LastExitCode([string]$Operation) {
|
||||||
|
if ($LASTEXITCODE -ne 0) { throw "$Operation failed with exit code $LASTEXITCODE" }
|
||||||
|
}
|
||||||
|
|
||||||
|
function Resolve-DDirectory([string]$Path, [string]$Label, [bool]$Create) {
|
||||||
|
if ($Create -and -not (Test-Path -LiteralPath $Path)) {
|
||||||
|
$null = New-Item -ItemType Directory -Path $Path
|
||||||
|
}
|
||||||
|
$item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force
|
||||||
|
if (
|
||||||
|
-not $item.PSIsContainer -or
|
||||||
|
($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -or
|
||||||
|
[IO.Path]::GetPathRoot($item.FullName).TrimEnd("\") -ine "D:"
|
||||||
|
) {
|
||||||
|
throw "$Label must be a real D: directory"
|
||||||
|
}
|
||||||
|
return $item.FullName
|
||||||
|
}
|
||||||
|
|
||||||
|
function Get-Container([string]$Name) {
|
||||||
|
$rows = @((& docker inspect $Name) | ConvertFrom-Json)
|
||||||
|
Assert-LastExitCode "Docker inspection for $Name"
|
||||||
|
if ($rows.Count -ne 1) { throw "Container identity for $Name is not unique" }
|
||||||
|
return $rows[0]
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($env:COMPUTERNAME -cne "DESKTOP-OPJ8J04") {
|
||||||
|
throw "M49 T3 is pinned to Worker 006"
|
||||||
|
}
|
||||||
|
|
||||||
|
$context = Resolve-DDirectory $BuildContext "M49 T3 build context" $false
|
||||||
|
$output = Resolve-DDirectory $OutputRoot "M49 T3 output root" $true
|
||||||
|
$runOutput = Join-Path $output $RunId
|
||||||
|
if (Test-Path -LiteralPath $runOutput) { throw "M49 T3 output already exists" }
|
||||||
|
$null = New-Item -ItemType Directory -Path $runOutput
|
||||||
|
$runOutput = Resolve-DDirectory $runOutput "M49 T3 run output" $false
|
||||||
|
$dockerConfig = Join-Path $runOutput "docker-config"
|
||||||
|
$null = New-Item -ItemType Directory -Path $dockerConfig
|
||||||
|
'{"auths":{}}' | Set-Content -LiteralPath (Join-Path $dockerConfig "config.json") -Encoding ascii
|
||||||
|
$env:DOCKER_CONFIG = $dockerConfig
|
||||||
|
|
||||||
|
$os = Get-CimInstance Win32_OperatingSystem
|
||||||
|
$freeMemoryGiB = [double]$os.FreePhysicalMemory / 1MB
|
||||||
|
if ($freeMemoryGiB -lt 16.0) {
|
||||||
|
throw ("M49 T3 requires 16 GiB free memory; observed {0:N2} GiB" -f $freeMemoryGiB)
|
||||||
|
}
|
||||||
|
|
||||||
|
$canonicalTriton = Get-Container "ndc-mission-core-triton"
|
||||||
|
if (-not $canonicalTriton.State.Running -or $canonicalTriton.State.Health.Status -cne "healthy") {
|
||||||
|
throw "Canonical Mission Core Triton must remain healthy during M49 T3"
|
||||||
|
}
|
||||||
|
|
||||||
|
$containerName = "ndc-mission-core-m49-t3-travel-$RunId"
|
||||||
|
if (& docker ps -a --format "{{.Names}}" --filter "name=^/$containerName$") {
|
||||||
|
throw "M49 T3 container name already exists"
|
||||||
|
}
|
||||||
|
|
||||||
|
$imageTag = "ndc/mission-core-m49-t3-travel:20260826"
|
||||||
|
$buildStarted = [DateTimeOffset]::UtcNow
|
||||||
|
& docker build --pull=false --tag $imageTag $context
|
||||||
|
Assert-LastExitCode "M49 T3 image build"
|
||||||
|
$buildCompleted = [DateTimeOffset]::UtcNow
|
||||||
|
|
||||||
|
$runStarted = [DateTimeOffset]::UtcNow
|
||||||
|
try {
|
||||||
|
& docker run --rm --name $containerName --cpus 16 --memory 24g `
|
||||||
|
--volume ((($runOutput -replace "\\", "/")) + ":/evidence") `
|
||||||
|
$imageTag
|
||||||
|
Assert-LastExitCode "M49 T3 upstream qualification"
|
||||||
|
} finally {
|
||||||
|
if (& docker ps -a --format "{{.Names}}" --filter "name=^/$containerName$") {
|
||||||
|
& docker rm --force $containerName *> $null
|
||||||
|
}
|
||||||
|
}
|
||||||
|
$runCompleted = [DateTimeOffset]::UtcNow
|
||||||
|
|
||||||
|
$image = @((& docker image inspect $imageTag) | ConvertFrom-Json)[0]
|
||||||
|
Assert-LastExitCode "M49 T3 image inspection"
|
||||||
|
$resultPath = Join-Path $runOutput "result.json"
|
||||||
|
if (-not (Test-Path -LiteralPath $resultPath -PathType Leaf)) {
|
||||||
|
throw "M49 T3 result.json is missing"
|
||||||
|
}
|
||||||
|
$result = Get-Content -LiteralPath $resultPath -Raw | ConvertFrom-Json
|
||||||
|
if ($result.status -cne "passed") { throw "M49 T3 qualification did not pass" }
|
||||||
|
|
||||||
|
$summary = [ordered]@{
|
||||||
|
schema_version = "missioncore.m49-t3-worker-summary/v1"
|
||||||
|
worker_id = "worker-006"
|
||||||
|
run_id = $RunId
|
||||||
|
image_tag = $imageTag
|
||||||
|
image_id = [string]$image.Id
|
||||||
|
image_size_bytes = [long]$image.Size
|
||||||
|
build_started_utc = $buildStarted.ToString("o")
|
||||||
|
build_wall_seconds = [math]::Round(($buildCompleted - $buildStarted).TotalSeconds, 6)
|
||||||
|
qualification_wall_seconds = [math]::Round(($runCompleted - $runStarted).TotalSeconds, 6)
|
||||||
|
free_memory_gib_before = [math]::Round($freeMemoryGiB, 6)
|
||||||
|
canonical_triton_id = [string]$canonicalTriton.Id
|
||||||
|
canonical_triton_health = [string]$canonicalTriton.State.Health.Status
|
||||||
|
candidate_accepted = $true
|
||||||
|
ravnoves00_quality_accepted = $false
|
||||||
|
realtime_accepted = $false
|
||||||
|
navigation_or_actuation_allowed = $false
|
||||||
|
}
|
||||||
|
$summary | ConvertTo-Json -Depth 3 | Set-Content -LiteralPath (
|
||||||
|
Join-Path $runOutput "worker-summary.json"
|
||||||
|
) -Encoding utf8
|
||||||
|
$summary | ConvertTo-Json -Depth 3
|
||||||
@@ -0,0 +1,62 @@
|
|||||||
|
[CmdletBinding()]
|
||||||
|
param(
|
||||||
|
[Parameter(Mandatory = $true)]
|
||||||
|
[string]$ReleaseRoot,
|
||||||
|
[Parameter(Mandatory = $true)]
|
||||||
|
[ValidatePattern("^[A-Za-z0-9._-]{1,96}$")]
|
||||||
|
[string]$RunId
|
||||||
|
)
|
||||||
|
|
||||||
|
$ErrorActionPreference = "Stop"
|
||||||
|
$taskName = "MissionCore-M49T3Travel"
|
||||||
|
$release = (Resolve-Path -LiteralPath $ReleaseRoot).Path
|
||||||
|
$payload = Join-Path $release "payload"
|
||||||
|
$runner = Join-Path $payload "Invoke-M49T3TravelQualification.ps1"
|
||||||
|
if (-not (Test-Path -LiteralPath $runner -PathType Leaf)) {
|
||||||
|
throw "M49 T3 runner is missing"
|
||||||
|
}
|
||||||
|
|
||||||
|
$existing = Get-ScheduledTask -TaskName $taskName -ErrorAction SilentlyContinue
|
||||||
|
if ($existing -and $existing.State -eq "Running") {
|
||||||
|
throw "$taskName is already running"
|
||||||
|
}
|
||||||
|
|
||||||
|
$powerShell = "$env:SystemRoot\System32\WindowsPowerShell\v1.0\powershell.exe"
|
||||||
|
$arguments = @(
|
||||||
|
"-NoLogo", "-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass",
|
||||||
|
"-File", "`"$runner`"",
|
||||||
|
"-BuildContext", "`"$payload`"",
|
||||||
|
"-RunId", "`"$RunId`""
|
||||||
|
) -join " "
|
||||||
|
$userId = [System.Security.Principal.WindowsIdentity]::GetCurrent().Name
|
||||||
|
$action = New-ScheduledTaskAction `
|
||||||
|
-Execute $powerShell `
|
||||||
|
-Argument $arguments `
|
||||||
|
-WorkingDirectory $payload
|
||||||
|
$principal = New-ScheduledTaskPrincipal `
|
||||||
|
-UserId $userId `
|
||||||
|
-LogonType Interactive `
|
||||||
|
-RunLevel Limited
|
||||||
|
$trigger = New-ScheduledTaskTrigger -Once -At ((Get-Date).AddMinutes(30))
|
||||||
|
$settings = New-ScheduledTaskSettingsSet `
|
||||||
|
-AllowStartIfOnBatteries `
|
||||||
|
-DontStopIfGoingOnBatteries `
|
||||||
|
-StartWhenAvailable `
|
||||||
|
-ExecutionTimeLimit ([TimeSpan]::FromHours(3))
|
||||||
|
|
||||||
|
Register-ScheduledTask `
|
||||||
|
-TaskName $taskName `
|
||||||
|
-Action $action `
|
||||||
|
-Principal $principal `
|
||||||
|
-Trigger $trigger `
|
||||||
|
-Settings $settings `
|
||||||
|
-Description "One-shot M49 T3 pinned TRAVEL upstream qualification." `
|
||||||
|
-Force | Out-Null
|
||||||
|
Start-ScheduledTask -TaskName $taskName
|
||||||
|
|
||||||
|
[pscustomobject]@{
|
||||||
|
task_name = $taskName
|
||||||
|
run_id = $RunId
|
||||||
|
release_root = $release
|
||||||
|
state = (Get-ScheduledTask -TaskName $taskName).State.ToString()
|
||||||
|
} | ConvertTo-Json -Compress
|
||||||
@@ -0,0 +1,72 @@
|
|||||||
|
FROM ros:jazzy-ros-base@sha256:2589a8fba5257307857890173c069852c2abf913a0be7970f172478baecb09e4
|
||||||
|
|
||||||
|
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
|
||||||
|
|
||||||
|
ARG TRAVEL_REVISION=95dc2fbd66a343efd9060c45a5711b6307a950a4
|
||||||
|
ARG KITTI_FIXTURE_SHA256=bf272996d5b6d25cc5589e1089137cb20a98b63bd4823a7fea5631b359f6d68c
|
||||||
|
ARG KITTI_GOLD_SHA256=8aaacaa57d17a8a2f043c3ea0a0c134d36a3c50da2385f2364ca4eff53ea9b9f
|
||||||
|
|
||||||
|
ENV DEBIAN_FRONTEND=noninteractive
|
||||||
|
|
||||||
|
RUN apt-get update \
|
||||||
|
&& apt-get install -y --no-install-recommends \
|
||||||
|
build-essential \
|
||||||
|
ca-certificates \
|
||||||
|
cmake \
|
||||||
|
curl \
|
||||||
|
git \
|
||||||
|
libboost-filesystem-dev \
|
||||||
|
libboost-system-dev \
|
||||||
|
libeigen3-dev \
|
||||||
|
libpcl-dev \
|
||||||
|
mpi-default-dev \
|
||||||
|
python3-colcon-common-extensions \
|
||||||
|
ros-jazzy-pcl-conversions \
|
||||||
|
ros-jazzy-rclcpp \
|
||||||
|
ros-jazzy-sensor-msgs \
|
||||||
|
ros-jazzy-std-msgs \
|
||||||
|
time \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
RUN mkdir -p /opt/travel/src/TRAVEL \
|
||||||
|
&& git -C /opt/travel/src/TRAVEL init \
|
||||||
|
&& git -C /opt/travel/src/TRAVEL remote add origin https://github.com/url-kaist/TRAVEL.git \
|
||||||
|
&& git -C /opt/travel/src/TRAVEL fetch --depth 1 origin "${TRAVEL_REVISION}" \
|
||||||
|
&& git -C /opt/travel/src/TRAVEL checkout --detach FETCH_HEAD \
|
||||||
|
&& test "$(git -C /opt/travel/src/TRAVEL rev-parse HEAD)" = "${TRAVEL_REVISION}" \
|
||||||
|
&& test -z "$(git -C /opt/travel/src/TRAVEL status --porcelain)"
|
||||||
|
|
||||||
|
RUN mkdir -p /opt/travel/fixture/00/velodyne \
|
||||||
|
&& curl -L --fail --retry 3 \
|
||||||
|
-o /opt/travel/fixture/00/velodyne/000000.bin \
|
||||||
|
https://github.com/url-kaist/TRAVEL/releases/download/test-data-v1/kitti00_000000.bin \
|
||||||
|
&& test "$(stat -c%s /opt/travel/fixture/00/velodyne/000000.bin)" = "1994688" \
|
||||||
|
&& echo "${KITTI_FIXTURE_SHA256} /opt/travel/fixture/00/velodyne/000000.bin" | sha256sum -c - \
|
||||||
|
&& echo "${KITTI_GOLD_SHA256} /opt/travel/src/TRAVEL/cpp/tests/data/kitti00_000000_gold.bin" | sha256sum -c -
|
||||||
|
|
||||||
|
RUN cmake \
|
||||||
|
-S /opt/travel/src/TRAVEL/cpp/travel \
|
||||||
|
-B /opt/travel/core-build \
|
||||||
|
-DTRAVEL_BUILD_EXAMPLES=ON \
|
||||||
|
-DTRAVEL_BUILD_TESTS=ON \
|
||||||
|
-DCMAKE_BUILD_TYPE=Release \
|
||||||
|
&& cmake --build /opt/travel/core-build \
|
||||||
|
--target regression_kitti run_travel_kitti \
|
||||||
|
--parallel 16
|
||||||
|
|
||||||
|
RUN set +u \
|
||||||
|
&& source /opt/ros/jazzy/setup.bash \
|
||||||
|
&& set -u \
|
||||||
|
&& colcon build \
|
||||||
|
--base-paths /opt/travel/src/TRAVEL \
|
||||||
|
--build-base /opt/travel/ros-build \
|
||||||
|
--install-base /opt/travel/ros-install \
|
||||||
|
--merge-install \
|
||||||
|
--packages-select travel_ros \
|
||||||
|
--cmake-args -DCMAKE_BUILD_TYPE=Release \
|
||||||
|
&& test -x /opt/travel/ros-install/lib/travel_ros/travel_node
|
||||||
|
|
||||||
|
COPY qualify.sh /usr/local/bin/m49-t3-travel-qualify
|
||||||
|
RUN chmod 0755 /usr/local/bin/m49-t3-travel-qualify
|
||||||
|
|
||||||
|
ENTRYPOINT ["/usr/local/bin/m49-t3-travel-qualify"]
|
||||||
@@ -0,0 +1,100 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
readonly EVIDENCE_ROOT=/evidence
|
||||||
|
readonly RESULT_PATH="${EVIDENCE_ROOT}/result.json"
|
||||||
|
readonly LOG_PATH="${EVIDENCE_ROOT}/qualification.log"
|
||||||
|
readonly RESOURCE_PATH="${EVIDENCE_ROOT}/resource.txt"
|
||||||
|
readonly STARTED_UTC="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||||
|
readonly STARTED_NS="$(date +%s%N)"
|
||||||
|
|
||||||
|
mkdir -p "${EVIDENCE_ROOT}"
|
||||||
|
if [[ -e "${RESULT_PATH}" || -e "${LOG_PATH}" || -e "${RESOURCE_PATH}" ]]; then
|
||||||
|
echo "evidence output already exists" >&2
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
|
||||||
|
status=failed
|
||||||
|
failure_stage=bootstrap
|
||||||
|
|
||||||
|
finalize() {
|
||||||
|
local exit_code=$?
|
||||||
|
local completed_ns
|
||||||
|
local elapsed_ms
|
||||||
|
completed_ns="$(date +%s%N)"
|
||||||
|
elapsed_ms="$(( (completed_ns - STARTED_NS) / 1000000 ))"
|
||||||
|
python3 - "${RESULT_PATH}" "${status}" "${failure_stage}" "${exit_code}" \
|
||||||
|
"${STARTED_UTC}" "${elapsed_ms}" <<'PY'
|
||||||
|
import json
|
||||||
|
import pathlib
|
||||||
|
import sys
|
||||||
|
|
||||||
|
target, status, stage, exit_code, started_utc, elapsed_ms = sys.argv[1:]
|
||||||
|
document = {
|
||||||
|
"schema_version": "missioncore.m49-t3-travel-upstream-qualification/v1",
|
||||||
|
"status": status,
|
||||||
|
"failure_stage": None if status == "passed" else stage,
|
||||||
|
"exit_code": int(exit_code),
|
||||||
|
"started_utc": started_utc,
|
||||||
|
"elapsed_ms": int(elapsed_ms),
|
||||||
|
"authority": {
|
||||||
|
"candidate_build_qualified": status == "passed",
|
||||||
|
"ravnoves00_quality_accepted": False,
|
||||||
|
"realtime_accepted": False,
|
||||||
|
"navigation_or_actuation_allowed": False,
|
||||||
|
},
|
||||||
|
"revisions": {
|
||||||
|
"travel": "95dc2fbd66a343efd9060c45a5711b6307a950a4",
|
||||||
|
},
|
||||||
|
"fixture": {
|
||||||
|
"bytes": 1994688,
|
||||||
|
"sha256": "bf272996d5b6d25cc5589e1089137cb20a98b63bd4823a7fea5631b359f6d68c",
|
||||||
|
"gold_sha256": "8aaacaa57d17a8a2f043c3ea0a0c134d36a3c50da2385f2364ca4eff53ea9b9f",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
pathlib.Path(target).write_text(
|
||||||
|
json.dumps(document, indent=2, sort_keys=True) + "\n", encoding="utf-8"
|
||||||
|
)
|
||||||
|
PY
|
||||||
|
}
|
||||||
|
trap finalize EXIT
|
||||||
|
|
||||||
|
exec > >(tee "${LOG_PATH}") 2>&1
|
||||||
|
|
||||||
|
failure_stage=source-integrity
|
||||||
|
test "$(git -C /opt/travel/src/TRAVEL rev-parse HEAD)" = \
|
||||||
|
"95dc2fbd66a343efd9060c45a5711b6307a950a4"
|
||||||
|
test -z "$(git -C /opt/travel/src/TRAVEL status --porcelain)"
|
||||||
|
test "$(stat -c%s /opt/travel/fixture/00/velodyne/000000.bin)" = "1994688"
|
||||||
|
echo "bf272996d5b6d25cc5589e1089137cb20a98b63bd4823a7fea5631b359f6d68c /opt/travel/fixture/00/velodyne/000000.bin" | sha256sum -c -
|
||||||
|
echo "8aaacaa57d17a8a2f043c3ea0a0c134d36a3c50da2385f2364ca4eff53ea9b9f /opt/travel/src/TRAVEL/cpp/tests/data/kitti00_000000_gold.bin" | sha256sum -c -
|
||||||
|
|
||||||
|
failure_stage=cpp-kitti-regression
|
||||||
|
/usr/bin/time -v -o "${RESOURCE_PATH}" \
|
||||||
|
/opt/travel/core-build/tests/regression_kitti \
|
||||||
|
/opt/travel/fixture/00/velodyne/000000.bin \
|
||||||
|
/tmp/travel-run-dump.bin \
|
||||||
|
/opt/travel/src/TRAVEL/cpp/tests/data/kitti00_000000_gold.bin
|
||||||
|
|
||||||
|
failure_stage=cpp-example-smoke
|
||||||
|
mkdir -p /tmp/travel-example
|
||||||
|
/opt/travel/core-build/examples/run_travel_kitti \
|
||||||
|
/opt/travel/fixture/00 0 /tmp/travel-example
|
||||||
|
for output in \
|
||||||
|
/tmp/travel-example/0_ground.bin \
|
||||||
|
/tmp/travel-example/0_nonground.bin \
|
||||||
|
/tmp/travel-example/0_labeled.bin; do
|
||||||
|
test -s "${output}"
|
||||||
|
done
|
||||||
|
|
||||||
|
failure_stage=ros2-discovery
|
||||||
|
set +u
|
||||||
|
source /opt/ros/jazzy/setup.bash
|
||||||
|
source /opt/travel/ros-install/setup.bash
|
||||||
|
set -u
|
||||||
|
test -x /opt/travel/ros-install/lib/travel_ros/travel_node
|
||||||
|
test "$(ros2 pkg prefix travel_ros)" = "/opt/travel/ros-install"
|
||||||
|
ros2 pkg executables travel_ros | grep -F "travel_ros travel_node"
|
||||||
|
|
||||||
|
failure_stage=complete
|
||||||
|
status=passed
|
||||||
@@ -0,0 +1,172 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Build the deterministic M49 T3 TRAVEL Worker 006 qualification release."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import gzip
|
||||||
|
import hashlib
|
||||||
|
import io
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
import subprocess
|
||||||
|
import tarfile
|
||||||
|
import tempfile
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
SOURCES = (
|
||||||
|
Path("experiments/perception/worker/m49_t3_travel/Dockerfile"),
|
||||||
|
Path("experiments/perception/worker/m49_t3_travel/qualify.sh"),
|
||||||
|
Path("experiments/perception/worker/Invoke-M49T3TravelQualification.ps1"),
|
||||||
|
Path(
|
||||||
|
"experiments/perception/worker/"
|
||||||
|
"Invoke-M49T3TravelQualificationAsInteractiveUser.ps1"
|
||||||
|
),
|
||||||
|
Path("config/perception/m49-traversability-candidate-manifest-v1.json"),
|
||||||
|
Path("config/perception/m49-camera-lidar-traversability-v1.json"),
|
||||||
|
)
|
||||||
|
PATCH_ID = re.compile(r"^[A-Za-z0-9._-]{1,96}$")
|
||||||
|
|
||||||
|
|
||||||
|
class ArtifactBuildError(RuntimeError):
|
||||||
|
"""The qualification artifact cannot be built from the declared source."""
|
||||||
|
|
||||||
|
|
||||||
|
def sha256_file(path: Path) -> str:
|
||||||
|
digest = hashlib.sha256()
|
||||||
|
with path.open("rb") as stream:
|
||||||
|
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
||||||
|
digest.update(chunk)
|
||||||
|
return digest.hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def git_revision() -> str:
|
||||||
|
result = subprocess.run(
|
||||||
|
["git", "rev-parse", "HEAD"],
|
||||||
|
cwd=REPOSITORY_ROOT,
|
||||||
|
check=True,
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
)
|
||||||
|
revision = result.stdout.strip()
|
||||||
|
if re.fullmatch(r"[a-f0-9]{40}", revision) is None:
|
||||||
|
raise ArtifactBuildError("Git revision is not a full SHA-1")
|
||||||
|
return revision
|
||||||
|
|
||||||
|
|
||||||
|
def tar_info(path: Path, arcname: str) -> tarfile.TarInfo:
|
||||||
|
info = tarfile.TarInfo(arcname)
|
||||||
|
info.uid = info.gid = 0
|
||||||
|
info.uname = info.gname = "root"
|
||||||
|
info.mtime = 0
|
||||||
|
if path.is_dir():
|
||||||
|
info.type = tarfile.DIRTYPE
|
||||||
|
info.mode = 0o755
|
||||||
|
else:
|
||||||
|
info.type = tarfile.REGTYPE
|
||||||
|
info.mode = 0o755 if path.suffix in {".sh", ".ps1"} else 0o644
|
||||||
|
info.size = path.stat().st_size
|
||||||
|
return info
|
||||||
|
|
||||||
|
|
||||||
|
def write_archive(stage: Path, target: Path) -> None:
|
||||||
|
members = [stage / "manifest.env", stage / "files.txt", stage / "payload"]
|
||||||
|
members.extend(sorted((stage / "payload").rglob("*")))
|
||||||
|
target.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
with (
|
||||||
|
target.open("wb") as raw,
|
||||||
|
gzip.GzipFile(filename="", mode="wb", fileobj=raw, mtime=0) as compressed,
|
||||||
|
tarfile.open(fileobj=compressed, mode="w", format=tarfile.PAX_FORMAT) as archive,
|
||||||
|
):
|
||||||
|
for path in members:
|
||||||
|
info = tar_info(path, path.relative_to(stage).as_posix())
|
||||||
|
if path.is_file():
|
||||||
|
with path.open("rb") as stream:
|
||||||
|
archive.addfile(info, stream)
|
||||||
|
else:
|
||||||
|
archive.addfile(info, io.BytesIO())
|
||||||
|
|
||||||
|
|
||||||
|
def build(
|
||||||
|
patch_id: str,
|
||||||
|
output_directory: Path,
|
||||||
|
*,
|
||||||
|
revision: str | None = None,
|
||||||
|
) -> dict[str, object]:
|
||||||
|
if PATCH_ID.fullmatch(patch_id) is None:
|
||||||
|
raise ArtifactBuildError("patch id is invalid")
|
||||||
|
sources = tuple(REPOSITORY_ROOT / source for source in SOURCES)
|
||||||
|
if any(path.is_symlink() or not path.is_file() for path in sources):
|
||||||
|
raise ArtifactBuildError("release input is not a regular file")
|
||||||
|
selected_revision = revision or git_revision()
|
||||||
|
if re.fullmatch(r"[a-f0-9]{40}", selected_revision) is None:
|
||||||
|
raise ArtifactBuildError("artifact revision is invalid")
|
||||||
|
with tempfile.TemporaryDirectory(prefix="mission-core-m49-t3-") as directory:
|
||||||
|
stage = Path(directory)
|
||||||
|
payload = stage / "payload"
|
||||||
|
payload.mkdir()
|
||||||
|
files: dict[str, dict[str, object]] = {}
|
||||||
|
for source in sources:
|
||||||
|
destination = payload / source.name
|
||||||
|
destination.write_bytes(source.read_bytes())
|
||||||
|
files[destination.name] = {
|
||||||
|
"bytes": destination.stat().st_size,
|
||||||
|
"sha256": sha256_file(destination),
|
||||||
|
}
|
||||||
|
release = {
|
||||||
|
"schema_version": "missioncore.m49-t3-worker-release/v1",
|
||||||
|
"patch_id": patch_id,
|
||||||
|
"code_revision": selected_revision,
|
||||||
|
"worker_id": "worker-006",
|
||||||
|
"candidate_id": "travel",
|
||||||
|
"license": "GPL-3.0-or-later",
|
||||||
|
"authority": {
|
||||||
|
"ravnoves00_quality_accepted": False,
|
||||||
|
"navigation_or_actuation_allowed": False,
|
||||||
|
},
|
||||||
|
"files": files,
|
||||||
|
}
|
||||||
|
release_path = payload / "release.json"
|
||||||
|
release_path.write_text(
|
||||||
|
json.dumps(release, indent=2, sort_keys=True) + "\n", encoding="utf-8"
|
||||||
|
)
|
||||||
|
payload_names = sorted((*files, release_path.name))
|
||||||
|
(stage / "manifest.env").write_text(
|
||||||
|
f"id={patch_id}\ncomponent=mission-core-worker\ntype=qualification-release\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
(stage / "files.txt").write_text(
|
||||||
|
"\n".join(payload_names) + "\n", encoding="utf-8"
|
||||||
|
)
|
||||||
|
target = output_directory.resolve() / f"nodedc-{patch_id}.tgz"
|
||||||
|
write_archive(stage, target)
|
||||||
|
return {
|
||||||
|
"ok": True,
|
||||||
|
"artifact": str(target),
|
||||||
|
"sha256": sha256_file(target),
|
||||||
|
"patch_id": patch_id,
|
||||||
|
"code_revision": selected_revision,
|
||||||
|
"payload_files": payload_names,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument("patch_id")
|
||||||
|
parser.add_argument(
|
||||||
|
"--output-directory",
|
||||||
|
type=Path,
|
||||||
|
default=REPOSITORY_ROOT / ".runtime/worker-artifacts",
|
||||||
|
)
|
||||||
|
arguments = parser.parse_args()
|
||||||
|
try:
|
||||||
|
result = build(arguments.patch_id, arguments.output_directory)
|
||||||
|
except (ArtifactBuildError, OSError, subprocess.SubprocessError) as exc:
|
||||||
|
parser.error(str(exc))
|
||||||
|
print(json.dumps(result, indent=2, sort_keys=True))
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -0,0 +1,61 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import importlib.util
|
||||||
|
import json
|
||||||
|
import tarfile
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
BUILDER_PATH = REPOSITORY_ROOT / "scripts/build_m49_t3_travel_worker_artifact.py"
|
||||||
|
SPEC = importlib.util.spec_from_file_location("m49_t3_travel_builder", BUILDER_PATH)
|
||||||
|
assert SPEC is not None and SPEC.loader is not None
|
||||||
|
BUILDER = importlib.util.module_from_spec(SPEC)
|
||||||
|
SPEC.loader.exec_module(BUILDER)
|
||||||
|
|
||||||
|
|
||||||
|
def _sha256(value: bytes) -> str:
|
||||||
|
return hashlib.sha256(value).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def _regular_files(archive: tarfile.TarFile) -> dict[str, bytes]:
|
||||||
|
result: dict[str, bytes] = {}
|
||||||
|
for member in archive.getmembers():
|
||||||
|
if not member.isfile():
|
||||||
|
continue
|
||||||
|
stream = archive.extractfile(member)
|
||||||
|
assert stream is not None
|
||||||
|
result[member.name] = stream.read()
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def test_m49_t3_worker_artifact_is_deterministic_and_bounded(tmp_path: Path) -> None:
|
||||||
|
patch_id = "mission-core-m49-t3-travel-unit-001"
|
||||||
|
revision = "a" * 40
|
||||||
|
first = BUILDER.build(patch_id, tmp_path / "first", revision=revision)
|
||||||
|
second = BUILDER.build(patch_id, tmp_path / "second", revision=revision)
|
||||||
|
|
||||||
|
first_bytes = Path(first["artifact"]).read_bytes()
|
||||||
|
assert first_bytes == Path(second["artifact"]).read_bytes()
|
||||||
|
assert first["sha256"] == _sha256(first_bytes)
|
||||||
|
with tarfile.open(first["artifact"], "r:gz") as archive:
|
||||||
|
members = archive.getmembers()
|
||||||
|
regular = _regular_files(archive)
|
||||||
|
assert all(not member.issym() and not member.islnk() for member in members)
|
||||||
|
assert set(regular) == {
|
||||||
|
"manifest.env",
|
||||||
|
"files.txt",
|
||||||
|
*(f"payload/{name}" for name in first["payload_files"]),
|
||||||
|
}
|
||||||
|
assert regular["files.txt"].decode().splitlines() == first["payload_files"]
|
||||||
|
release = json.loads(regular["payload/release.json"])
|
||||||
|
assert release["code_revision"] == revision
|
||||||
|
assert release["candidate_id"] == "travel"
|
||||||
|
assert release["license"] == "GPL-3.0-or-later"
|
||||||
|
assert release["authority"] == {
|
||||||
|
"navigation_or_actuation_allowed": False,
|
||||||
|
"ravnoves00_quality_accepted": False,
|
||||||
|
}
|
||||||
|
serialized = json.dumps(release).lower()
|
||||||
|
assert "password=" not in serialized
|
||||||
|
assert "private key" not in serialized
|
||||||
Reference in New Issue
Block a user