From aed59030b4fa755aed3362d638efb65c40437c14 Mon Sep 17 00:00:00 2001 From: DCCONSTRUCTIONS Date: Wed, 26 Aug 2026 18:27:39 +0300 Subject: [PATCH] feat(perception): stage GSeg3D qualification --- .../m49-camera-lidar-traversability-v1.json | 59 +++++++ ...-traversability-candidate-manifest-v1.json | 6 + ...LITY_AND_STATIC_OBSTACLE_STACK_DECISION.md | 46 ++++- ..._T0_TRAVERSABILITY_PREFLIGHT_2026-08-26.md | 23 +++ .../Invoke-M49T1GSeg3DQualification.ps1 | 116 ++++++++++++ .../worker/m49_t1_gseg3d/Dockerfile | 65 +++++++ .../worker/m49_t1_gseg3d/qualify.sh | 94 ++++++++++ .../build_m49_t1_gseg3d_worker_artifact.py | 167 ++++++++++++++++++ tests/test_m49_t1_gseg3d_worker_artifact.py | 65 +++++++ 9 files changed, 639 insertions(+), 2 deletions(-) create mode 100644 config/perception/m49-camera-lidar-traversability-v1.json create mode 100644 experiments/perception/worker/Invoke-M49T1GSeg3DQualification.ps1 create mode 100644 experiments/perception/worker/m49_t1_gseg3d/Dockerfile create mode 100644 experiments/perception/worker/m49_t1_gseg3d/qualify.sh create mode 100644 scripts/build_m49_t1_gseg3d_worker_artifact.py create mode 100644 tests/test_m49_t1_gseg3d_worker_artifact.py diff --git a/config/perception/m49-camera-lidar-traversability-v1.json b/config/perception/m49-camera-lidar-traversability-v1.json new file mode 100644 index 0000000..8795284 --- /dev/null +++ b/config/perception/m49-camera-lidar-traversability-v1.json @@ -0,0 +1,59 @@ +{ + "schema_version": "missioncore.camera-lidar-traversability/v1", + "roles": { + "camera_semantics": "material-and-risk-meaning-primary", + "lidar": "metric-geometry-and-positive-support-primary", + "arbitration": "preserve-contradiction-never-unilateral-clear" + }, + "camera_contract": { + "raster": "native-800x600-fisheye", + "rectification_allowed": false, + "new_model_allowed": false, + "vegetation_classes": [ + "grass_low_vegetation", + "tree_woody_vegetation" + ] + }, + "states": [ + { + "id": "RIGID_OR_UNKNOWN_OBSTACLE", + "required_evidence": "stable-metric-nonground-without-vegetation-support", + "cost": "lethal-or-unknown" + }, + { + "id": "VEGETATION_UNKNOWN", + "required_evidence": "camera-vegetation-without-positive-ground-support", + "cost": "unknown-never-free" + }, + { + "id": "VEGETATION_POTENTIALLY_TRAVERSABLE", + "required_evidence": "camera-low-vegetation-plus-positive-ground-without-concentrated-rigid-return", + "cost": "high-cost-terrain" + }, + { + "id": "VEGETATION_WITH_RIGID_GEOMETRY", + "required_evidence": "camera-vegetation-plus-concentrated-persistent-vertical-geometry", + "cost": "lethal" + }, + { + "id": "SUPPORTED_GROUND", + "required_evidence": "positive-ground-support-without-blocking-geometry", + "cost": "candidate-traversable" + } + ], + "invariants": { + "camera_can_clear_positive_rigid_geometry": false, + "lidar_nonground_alone_means_rigid_object": false, + "missing_lidar_support_means_free": false, + "vegetation_is_binary_free_or_lethal": false, + "physical_navigation_authority": false + }, + "deferred_robot_profile": [ + "footprint", + "ground_clearance", + "traction", + "maximum_slope", + "admissible_vegetation_height", + "vegetation_contact_allowed" + ] +} diff --git a/config/perception/m49-traversability-candidate-manifest-v1.json b/config/perception/m49-traversability-candidate-manifest-v1.json index 33c7302..2fc5ef1 100644 --- a/config/perception/m49-traversability-candidate-manifest-v1.json +++ b/config/perception/m49-traversability-candidate-manifest-v1.json @@ -15,6 +15,12 @@ "candidate_id": "gseg3d-ground-consistency", "role": "primary", "components": [ + { + "repository": "https://github.com/dfki-ric/ground_segmentation.git", + "revision": "116f510485ea5b4707e2b476958d34a5a067f647", + "observed_ref": "refs/heads/master", + "license": "BSD-3-Clause" + }, { "repository": "https://github.com/dfki-ric/ground_segmentation_ros2.git", "revision": "5c5ba6f5ca0d682db2d59b2ad2f6a317ee938b95", diff --git a/docs/23_TRAVERSABILITY_AND_STATIC_OBSTACLE_STACK_DECISION.md b/docs/23_TRAVERSABILITY_AND_STATIC_OBSTACLE_STACK_DECISION.md index f1b786a..2082e95 100644 --- a/docs/23_TRAVERSABILITY_AND_STATIC_OBSTACLE_STACK_DECISION.md +++ b/docs/23_TRAVERSABILITY_AND_STATIC_OBSTACLE_STACK_DECISION.md @@ -33,8 +33,13 @@ The sequential evaluation order is: RF-DETR remains the semantic risk provider for people, children, animals, cars, trucks, motorcycles and bicycles. It does not decide the presence or -shape of an unknown static obstacle. Native `800x600` fisheye input remains -unchanged. +shape of an unknown static obstacle. The admitted camera semantic layer owns +surface/material hints such as low vegetation and woody vegetation. Native +`800x600` fisheye input remains unchanged. + +There is no single globally primary sensor. The camera is primary for semantic +meaning; LiDAR is primary for metric geometry and positive support. Neither is +allowed to erase contradictory evidence from the other. ## Required product representation @@ -55,12 +60,42 @@ robot-footprint collision cost + traversability +----> camera projection (diagnostic visualization only) camera RF-DETR ----> dynamic semantic risk hints ----------------+ +camera semantics --> material / vegetation hint -----------------+ ``` The canonical safety output is a costmap/elevation/occupancy product. Camera boxes may be derived for operator review, but a box must never be the source of occupied cells, clearance, passage width or a control action. +## Camera/LiDAR arbitration and vegetation + +Tall grass demonstrates why a binary obstacle mask is insufficient. LiDAR may +correctly report many non-ground returns while being unable to tell whether +they came from compressible grass, a rigid trunk hidden inside the grass, or a +drop-off with no positive support. Camera semantics can identify vegetation, +but a camera label cannot prove that the ground below is load-bearing or free +of a hidden rigid object. + +Mission Core therefore preserves at least these distinct states: + +| Evidence | Traversability state | +| --- | --- | +| Stable metric non-ground evidence without vegetation support | `RIGID_OR_UNKNOWN_OBSTACLE`; lethal or unknown according to support confidence | +| Camera vegetation plus diffuse LiDAR returns, but no positive ground support below | `VEGETATION_UNKNOWN`; never silently free | +| Camera low vegetation plus positive local ground support and no concentrated rigid return | `VEGETATION_POTENTIALLY_TRAVERSABLE`; high-cost terrain, not a wall | +| Camera vegetation plus concentrated persistent vertical geometry | `VEGETATION_WITH_RIGID_GEOMETRY`; lethal | +| Positive ground support without blocking geometry | `SUPPORTED_GROUND`; candidate traversable terrain | + +`VEGETATION_POTENTIALLY_TRAVERSABLE` is a representation decision, not an +authorization to drive through it. The later robot profile must still declare +ground clearance, footprint, traction, admissible slope, vegetation height and +whether contact with vegetation is allowed. Until that profile exists, the +planner may prefer a supported hard path and retain vegetation as a higher-cost +or unknown alternative. + +The executable evidence contract is +[`config/perception/m49-camera-lidar-traversability-v1.json`](../config/perception/m49-camera-lidar-traversability-v1.json). + ## Candidate A — GSeg3D and Ground Consistency ### GSeg3D @@ -78,6 +113,11 @@ publishes separate ground and obstacle point clouds and provides: Mission Core evaluates it as the first replacement for the current local height connected-component heuristic. +The upstream GSeg3D documentation explicitly lists recall degradation in dense +vegetation as a remaining challenge and semantic-aware refinement as future +work. Candidate A can therefore qualify the metric ground/non-ground seam, but +cannot alone close the vegetation product gate. + ### Nav2 Ground Consistency [DFKI Ground Consistency](https://github.com/dfki-ric/nav2_ground_consistency_costmap_plugin) @@ -246,6 +286,8 @@ failure. - Review the ten mandatory anchors and compute full-run stability/resource metrics. - Do not draw system camera boxes until the costmap result is sealed. +- Replay the existing camera semantic vegetation evidence as an independent + arbitration input; do not let it clear unsupported or concentrated geometry. ### T3 — Candidate B challenger diff --git a/experiments/perception/M49_T0_TRAVERSABILITY_PREFLIGHT_2026-08-26.md b/experiments/perception/M49_T0_TRAVERSABILITY_PREFLIGHT_2026-08-26.md index 0b4e6f7..efdb7b3 100644 --- a/experiments/perception/M49_T0_TRAVERSABILITY_PREFLIGHT_2026-08-26.md +++ b/experiments/perception/M49_T0_TRAVERSABILITY_PREFLIGHT_2026-08-26.md @@ -61,6 +61,7 @@ This T0 check does not reopen or alter that evidence. | Order | Candidate | Exact revision | Admission role | | ---: | --- | --- | --- | +| 1 | DFKI GSeg3D core | `116f510485ea5b4707e2b476958d34a5a067f647` | Pinned core dependency | | 1 | DFKI GSeg3D | `5c5ba6f5ca0d682db2d59b2ad2f6a317ee938b95` | Primary terrain-relative ground/non-ground provider | | 1 | DFKI Nav2 Ground Consistency | `41cec620efba6c370dccfc59a6ec1134775ff48a` | Primary temporal costmap evidence layer | | 2 | TRAVEL | `95dc2fbd66a343efd9060c45a5711b6307a950a4` | Class-free challenger | @@ -86,6 +87,28 @@ Candidate A may start only when all of the following are true: 4. canonical Mission Core Triton is healthy and is not stopped or replaced; 5. the candidate uses a new `ndc-` isolated build/runtime contour. +## Resource incident localized before T1 + +The apparent continued Worker pressure was not caused by an active GAUSS job. +A PowerShell process from an earlier Mission Core diagnostic had been orphaned +after loading and recursively serializing the complete nested M4.8R3 +`result.json`. At discovery it held approximately `74,912.6 MiB` working set +and `193,275.8 MiB` private memory. The pressure killed +`ndc-mission-core-triton` with exit code `137`. + +The orphaned diagnostic process was terminated at +`2026-08-26T15:17:39Z`. Free physical memory subsequently recovered to +`83,374,248 KiB`. The existing canonical Triton container was started without +replacement or rebuild and returned to `healthy` at +`2026-08-26T15:18:10Z`. + +Future Worker inspection must not pass a complete nested replay result through +PowerShell `ConvertFrom-Json` followed by deep `ConvertTo-Json`. Diagnostics +must select bounded scalar fields before serialization and must have a wall +timeout plus process cleanup. A separate compute-runtime change is required to +give canonical Triton a declared restart policy; this report does not mutate +the Worker Compose source. + ## Next command boundary The next authorized action is T1 Candidate A upstream qualification: build the diff --git a/experiments/perception/worker/Invoke-M49T1GSeg3DQualification.ps1 b/experiments/perception/worker/Invoke-M49T1GSeg3DQualification.ps1 new file mode 100644 index 0000000..dad0738 --- /dev/null +++ b/experiments/perception/worker/Invoke-M49T1GSeg3DQualification.ps1 @@ -0,0 +1,116 @@ +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] + [string]$BuildContext, + [Parameter(Mandatory = $true)] + [ValidatePattern("^[A-Za-z0-9._-]{1,96}$")] + [string]$RunId, + [string]$OutputRoot = "D:\NDC_MISSIONCORE\runtime\results\m49-t1-gseg3d" +) + +$ErrorActionPreference = "Stop" +$ProgressPreference = "SilentlyContinue" + +function Assert-LastExitCode([string]$Operation) { + if ($LASTEXITCODE -ne 0) { throw "$Operation failed with exit code $LASTEXITCODE" } +} + +function Resolve-DDirectory([string]$Path, [string]$Label, [bool]$Create) { + if ($Create -and -not (Test-Path -LiteralPath $Path)) { + $null = New-Item -ItemType Directory -Path $Path + } + $item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force + if ( + -not $item.PSIsContainer -or + ($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -or + [IO.Path]::GetPathRoot($item.FullName).TrimEnd("\") -ine "D:" + ) { + throw "$Label must be a real D: directory" + } + return $item.FullName +} + +function Get-Container([string]$Name) { + $rows = @((& docker inspect $Name) | ConvertFrom-Json) + Assert-LastExitCode "Docker inspection for $Name" + if ($rows.Count -ne 1) { throw "Container identity for $Name is not unique" } + return $rows[0] +} + +if ($env:COMPUTERNAME -cne "DESKTOP-OPJ8J04") { + throw "M49 T1 is pinned to Worker 006" +} + +$context = Resolve-DDirectory $BuildContext "M49 T1 build context" $false +$output = Resolve-DDirectory $OutputRoot "M49 T1 output root" $true +$runOutput = Join-Path $output $RunId +if (Test-Path -LiteralPath $runOutput) { throw "M49 T1 output already exists" } +$null = New-Item -ItemType Directory -Path $runOutput +$runOutput = Resolve-DDirectory $runOutput "M49 T1 run output" $false + +$os = Get-CimInstance Win32_OperatingSystem +$freeMemoryGiB = [double]$os.FreePhysicalMemory / 1MB +if ($freeMemoryGiB -lt 16.0) { + throw ("M49 T1 requires 16 GiB free memory; observed {0:N2} GiB" -f $freeMemoryGiB) +} + +$canonicalTriton = Get-Container "ndc-mission-core-triton" +if (-not $canonicalTriton.State.Running -or $canonicalTriton.State.Health.Status -cne "healthy") { + throw "Canonical Mission Core Triton must remain healthy during M49 T1" +} + +$containerName = "ndc-mission-core-m49-t1-gseg3d-$RunId" +if (& docker ps -a --format "{{.Names}}" --filter "name=^/$containerName$") { + throw "M49 T1 container name already exists" +} + +$imageTag = "ndc/mission-core-m49-t1-gseg3d:20260826" +$buildStarted = [DateTimeOffset]::UtcNow +& docker build --pull=false --tag $imageTag $context +Assert-LastExitCode "M49 T1 image build" +$buildCompleted = [DateTimeOffset]::UtcNow + +$runStarted = [DateTimeOffset]::UtcNow +try { + & docker run --rm --name $containerName --cpus 16 --memory 24g ` + --volume ((($runOutput -replace "\\", "/")) + ":/evidence") ` + $imageTag + Assert-LastExitCode "M49 T1 upstream qualification" +} finally { + if (& docker ps -a --format "{{.Names}}" --filter "name=^/$containerName$") { + & docker rm --force $containerName *> $null + } +} +$runCompleted = [DateTimeOffset]::UtcNow + +$image = @((& docker image inspect $imageTag) | ConvertFrom-Json)[0] +Assert-LastExitCode "M49 T1 image inspection" +$resultPath = Join-Path $runOutput "result.json" +if (-not (Test-Path -LiteralPath $resultPath -PathType Leaf)) { + throw "M49 T1 result.json is missing" +} +$result = Get-Content -LiteralPath $resultPath -Raw | ConvertFrom-Json +if ($result.status -cne "passed") { throw "M49 T1 qualification did not pass" } + +$summary = [ordered]@{ + schema_version = "missioncore.m49-t1-worker-summary/v1" + worker_id = "worker-006" + run_id = $RunId + image_tag = $imageTag + image_id = [string]$image.Id + image_size_bytes = [long]$image.Size + build_started_utc = $buildStarted.ToString("o") + build_wall_seconds = [math]::Round(($buildCompleted - $buildStarted).TotalSeconds, 6) + qualification_wall_seconds = [math]::Round(($runCompleted - $runStarted).TotalSeconds, 6) + free_memory_gib_before = [math]::Round($freeMemoryGiB, 6) + canonical_triton_id = [string]$canonicalTriton.Id + canonical_triton_health = [string]$canonicalTriton.State.Health.Status + candidate_accepted = $true + ravnoves00_quality_accepted = $false + realtime_accepted = $false + navigation_or_actuation_allowed = $false +} +$summary | ConvertTo-Json -Depth 3 | Set-Content -LiteralPath ( + Join-Path $runOutput "worker-summary.json" +) -Encoding utf8 +$summary | ConvertTo-Json -Depth 3 diff --git a/experiments/perception/worker/m49_t1_gseg3d/Dockerfile b/experiments/perception/worker/m49_t1_gseg3d/Dockerfile new file mode 100644 index 0000000..f126127 --- /dev/null +++ b/experiments/perception/worker/m49_t1_gseg3d/Dockerfile @@ -0,0 +1,65 @@ +FROM ros:jazzy-ros-base@sha256:2589a8fba5257307857890173c069852c2abf913a0be7970f172478baecb09e4 + +ARG GSEG3D_CORE_REVISION=116f510485ea5b4707e2b476958d34a5a067f647 +ARG GSEG3D_ROS2_REVISION=5c5ba6f5ca0d682db2d59b2ad2f6a317ee938b95 +ARG GROUND_CONSISTENCY_REVISION=41cec620efba6c370dccfc59a6ec1134775ff48a +ARG BUILD_JOBS=12 + +ENV DEBIAN_FRONTEND=noninteractive +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +RUN apt-get update \ + && apt-get install --yes --no-install-recommends \ + build-essential \ + cmake \ + git \ + jq \ + libeigen3-dev \ + libgtest-dev \ + libnanoflann-dev \ + libpcl-dev \ + ninja-build \ + ros-jazzy-message-filters \ + ros-jazzy-nav2-costmap-2d \ + ros-jazzy-pcl-conversions \ + ros-jazzy-rclcpp-lifecycle \ + ros-jazzy-tf2-eigen \ + ros-jazzy-tf2-geometry-msgs \ + time \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /opt/m49/src + +RUN git clone --filter=blob:none https://github.com/dfki-ric/ground_segmentation.git \ + && git -C ground_segmentation checkout --detach "${GSEG3D_CORE_REVISION}" \ + && git clone --filter=blob:none https://github.com/dfki-ric/ground_segmentation_ros2.git \ + && git -C ground_segmentation_ros2 checkout --detach "${GSEG3D_ROS2_REVISION}" \ + && git clone --filter=blob:none https://github.com/dfki-ric/nav2_ground_consistency_costmap_plugin.git \ + && git -C nav2_ground_consistency_costmap_plugin checkout --detach "${GROUND_CONSISTENCY_REVISION}" + +RUN cmake -S /opt/m49/src/ground_segmentation \ + -B /opt/m49/core-build \ + -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/opt/m49/core-install \ + && cmake --build /opt/m49/core-build --parallel "${BUILD_JOBS}" \ + && ctest --test-dir /opt/m49/core-build --output-on-failure \ + && cmake --install /opt/m49/core-build + +RUN source /opt/ros/jazzy/setup.bash \ + && export CMAKE_PREFIX_PATH="/opt/m49/core-install:${CMAKE_PREFIX_PATH:-}" \ + && colcon build \ + --base-paths \ + /opt/m49/src/ground_segmentation_ros2 \ + /opt/m49/src/nav2_ground_consistency_costmap_plugin \ + --build-base /opt/m49/ros-build \ + --install-base /opt/m49/ros-install \ + --merge-install \ + --cmake-args -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=ON \ + --executor sequential \ + --event-handlers console_direct+ + +COPY qualify.sh /usr/local/bin/m49-gseg3d-qualify +RUN chmod 0755 /usr/local/bin/m49-gseg3d-qualify + +ENTRYPOINT ["/usr/local/bin/m49-gseg3d-qualify"] diff --git a/experiments/perception/worker/m49_t1_gseg3d/qualify.sh b/experiments/perception/worker/m49_t1_gseg3d/qualify.sh new file mode 100644 index 0000000..8019a4f --- /dev/null +++ b/experiments/perception/worker/m49_t1_gseg3d/qualify.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash +set -euo pipefail + +readonly EVIDENCE_ROOT=/evidence +readonly RESULT_PATH="${EVIDENCE_ROOT}/result.json" +readonly LOG_PATH="${EVIDENCE_ROOT}/qualification.log" +readonly RESOURCE_PATH="${EVIDENCE_ROOT}/resource.txt" +readonly STARTED_UTC="$(date -u +%Y-%m-%dT%H:%M:%SZ)" +readonly STARTED_NS="$(date +%s%N)" + +mkdir -p "${EVIDENCE_ROOT}" +if [[ -e "${RESULT_PATH}" || -e "${LOG_PATH}" || -e "${RESOURCE_PATH}" ]]; then + echo "evidence output already exists" >&2 + exit 2 +fi + +status=failed +failure_stage=bootstrap + +finalize() { + local exit_code=$? + local completed_ns + local elapsed_ms + completed_ns="$(date +%s%N)" + elapsed_ms="$(( (completed_ns - STARTED_NS) / 1000000 ))" + python3 - "${RESULT_PATH}" "${status}" "${failure_stage}" "${exit_code}" \ + "${STARTED_UTC}" "${elapsed_ms}" <<'PY' +import json +import pathlib +import sys + +target, status, stage, exit_code, started_utc, elapsed_ms = sys.argv[1:] +document = { + "schema_version": "missioncore.m49-t1-gseg3d-upstream-qualification/v1", + "status": status, + "failure_stage": None if status == "passed" else stage, + "exit_code": int(exit_code), + "started_utc": started_utc, + "elapsed_ms": int(elapsed_ms), + "authority": { + "candidate_build_qualified": status == "passed", + "ravnoves00_quality_accepted": False, + "realtime_accepted": False, + "navigation_or_actuation_allowed": False, + }, + "revisions": { + "ground_segmentation": "116f510485ea5b4707e2b476958d34a5a067f647", + "ground_segmentation_ros2": "5c5ba6f5ca0d682db2d59b2ad2f6a317ee938b95", + "ground_consistency": "41cec620efba6c370dccfc59a6ec1134775ff48a", + }, +} +pathlib.Path(target).write_text( + json.dumps(document, indent=2, sort_keys=True) + "\n", encoding="utf-8" +) +PY +} +trap finalize EXIT + +exec > >(tee "${LOG_PATH}") 2>&1 + +failure_stage=source-integrity +test "$(git -C /opt/m49/src/ground_segmentation rev-parse HEAD)" = \ + "116f510485ea5b4707e2b476958d34a5a067f647" +test "$(git -C /opt/m49/src/ground_segmentation_ros2 rev-parse HEAD)" = \ + "5c5ba6f5ca0d682db2d59b2ad2f6a317ee938b95" +test "$(git -C /opt/m49/src/nav2_ground_consistency_costmap_plugin rev-parse HEAD)" = \ + "41cec620efba6c370dccfc59a6ec1134775ff48a" +test -z "$(git -C /opt/m49/src/ground_segmentation status --porcelain)" +test -z "$(git -C /opt/m49/src/ground_segmentation_ros2 status --porcelain)" +test -z "$(git -C /opt/m49/src/nav2_ground_consistency_costmap_plugin status --porcelain)" + +failure_stage=core-upstream-tests +/usr/bin/time -v -o "${RESOURCE_PATH}" \ + ctest --test-dir /opt/m49/core-build --output-on-failure + +failure_stage=ros2-plugin-tests +source /opt/ros/jazzy/setup.bash +source /opt/m49/ros-install/setup.bash +colcon test \ + --build-base /opt/m49/ros-build \ + --install-base /opt/m49/ros-install \ + --test-result-base /opt/m49/test-results \ + --packages-select nav2_ground_consistency_costmap_plugin \ + --event-handlers console_direct+ +colcon test-result --test-result-base /opt/m49/test-results --verbose + +failure_stage=ros2-discovery +ros2 pkg prefix ground_segmentation_ros2 +ros2 pkg prefix nav2_ground_consistency_costmap_plugin +ros2 pkg executables ground_segmentation_ros2 | \ + grep -F "ground_segmentation_ros2_node" + +failure_stage=complete +status=passed diff --git a/scripts/build_m49_t1_gseg3d_worker_artifact.py b/scripts/build_m49_t1_gseg3d_worker_artifact.py new file mode 100644 index 0000000..907dd5a --- /dev/null +++ b/scripts/build_m49_t1_gseg3d_worker_artifact.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +"""Build the deterministic M49 T1 GSeg3D Worker 006 qualification release.""" + +from __future__ import annotations + +import argparse +import gzip +import hashlib +import io +import json +import re +import subprocess +import tarfile +import tempfile +from pathlib import Path + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +SOURCES = ( + Path("experiments/perception/worker/m49_t1_gseg3d/Dockerfile"), + Path("experiments/perception/worker/m49_t1_gseg3d/qualify.sh"), + Path("experiments/perception/worker/Invoke-M49T1GSeg3DQualification.ps1"), + Path("config/perception/m49-traversability-candidate-manifest-v1.json"), + Path("config/perception/m49-camera-lidar-traversability-v1.json"), +) +PATCH_ID = re.compile(r"^[A-Za-z0-9._-]{1,96}$") + + +class ArtifactBuildError(RuntimeError): + """The qualification artifact cannot be built from the declared source.""" + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def git_revision() -> str: + result = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=REPOSITORY_ROOT, + check=True, + capture_output=True, + text=True, + ) + revision = result.stdout.strip() + if re.fullmatch(r"[a-f0-9]{40}", revision) is None: + raise ArtifactBuildError("Git revision is not a full SHA-1") + return revision + + +def tar_info(path: Path, arcname: str) -> tarfile.TarInfo: + info = tarfile.TarInfo(arcname) + info.uid = info.gid = 0 + info.uname = info.gname = "root" + info.mtime = 0 + if path.is_dir(): + info.type = tarfile.DIRTYPE + info.mode = 0o755 + else: + info.type = tarfile.REGTYPE + info.mode = 0o755 if path.suffix in {".sh", ".ps1"} else 0o644 + info.size = path.stat().st_size + return info + + +def write_archive(stage: Path, target: Path) -> None: + members = [stage / "manifest.env", stage / "files.txt", stage / "payload"] + members.extend(sorted((stage / "payload").rglob("*"))) + target.parent.mkdir(parents=True, exist_ok=True) + with ( + target.open("wb") as raw, + gzip.GzipFile(filename="", mode="wb", fileobj=raw, mtime=0) as compressed, + tarfile.open(fileobj=compressed, mode="w", format=tarfile.PAX_FORMAT) as archive, + ): + for path in members: + info = tar_info(path, path.relative_to(stage).as_posix()) + if path.is_file(): + with path.open("rb") as stream: + archive.addfile(info, stream) + else: + archive.addfile(info, io.BytesIO()) + + +def build( + patch_id: str, + output_directory: Path, + *, + revision: str | None = None, +) -> dict[str, object]: + if PATCH_ID.fullmatch(patch_id) is None: + raise ArtifactBuildError("patch id is invalid") + sources = tuple(REPOSITORY_ROOT / source for source in SOURCES) + if any(path.is_symlink() or not path.is_file() for path in sources): + raise ArtifactBuildError("release input is not a regular file") + selected_revision = revision or git_revision() + if re.fullmatch(r"[a-f0-9]{40}", selected_revision) is None: + raise ArtifactBuildError("artifact revision is invalid") + with tempfile.TemporaryDirectory(prefix="mission-core-m49-t1-") as directory: + stage = Path(directory) + payload = stage / "payload" + payload.mkdir() + files: dict[str, dict[str, object]] = {} + for source in sources: + destination = payload / source.name + destination.write_bytes(source.read_bytes()) + files[destination.name] = { + "bytes": destination.stat().st_size, + "sha256": sha256_file(destination), + } + release = { + "schema_version": "missioncore.m49-t1-worker-release/v1", + "patch_id": patch_id, + "code_revision": selected_revision, + "worker_id": "worker-006", + "candidate_id": "gseg3d-ground-consistency", + "authority": { + "ravnoves00_quality_accepted": False, + "navigation_or_actuation_allowed": False, + }, + "files": files, + } + release_path = payload / "release.json" + release_path.write_text( + json.dumps(release, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + payload_names = sorted((*files, release_path.name)) + (stage / "manifest.env").write_text( + f"id={patch_id}\ncomponent=mission-core-worker\ntype=qualification-release\n", + encoding="utf-8", + ) + (stage / "files.txt").write_text( + "\n".join(payload_names) + "\n", encoding="utf-8" + ) + target = output_directory.resolve() / f"nodedc-{patch_id}.tgz" + write_archive(stage, target) + return { + "ok": True, + "artifact": str(target), + "sha256": sha256_file(target), + "patch_id": patch_id, + "code_revision": selected_revision, + "payload_files": payload_names, + } + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("patch_id") + parser.add_argument( + "--output-directory", + type=Path, + default=REPOSITORY_ROOT / ".runtime/worker-artifacts", + ) + arguments = parser.parse_args() + try: + result = build(arguments.patch_id, arguments.output_directory) + except (ArtifactBuildError, OSError, subprocess.SubprocessError) as exc: + parser.error(str(exc)) + print(json.dumps(result, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_m49_t1_gseg3d_worker_artifact.py b/tests/test_m49_t1_gseg3d_worker_artifact.py new file mode 100644 index 0000000..6c6159a --- /dev/null +++ b/tests/test_m49_t1_gseg3d_worker_artifact.py @@ -0,0 +1,65 @@ +from __future__ import annotations + +import hashlib +import importlib.util +import json +import tarfile +from pathlib import Path + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +BUILDER_PATH = REPOSITORY_ROOT / "scripts/build_m49_t1_gseg3d_worker_artifact.py" +SPEC = importlib.util.spec_from_file_location("m49_t1_gseg3d_builder", BUILDER_PATH) +assert SPEC is not None and SPEC.loader is not None +BUILDER = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(BUILDER) + + +def _sha256(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def _regular_files(archive: tarfile.TarFile) -> dict[str, bytes]: + result: dict[str, bytes] = {} + for member in archive.getmembers(): + if not member.isfile(): + continue + stream = archive.extractfile(member) + assert stream is not None + result[member.name] = stream.read() + return result + + +def test_m49_t1_worker_artifact_is_deterministic_and_bounded(tmp_path: Path) -> None: + patch_id = "mission-core-m49-t1-gseg3d-unit-001" + revision = "a" * 40 + first = BUILDER.build(patch_id, tmp_path / "first", revision=revision) + second = BUILDER.build(patch_id, tmp_path / "second", revision=revision) + + first_bytes = Path(first["artifact"]).read_bytes() + assert first_bytes == Path(second["artifact"]).read_bytes() + assert first["sha256"] == _sha256(first_bytes) + with tarfile.open(first["artifact"], "r:gz") as archive: + members = archive.getmembers() + regular = _regular_files(archive) + assert all(not member.issym() and not member.islnk() for member in members) + assert set(regular) == { + "manifest.env", + "files.txt", + *(f"payload/{name}" for name in first["payload_files"]), + } + assert regular["files.txt"].decode().splitlines() == first["payload_files"] + assert regular["manifest.env"].decode().splitlines() == [ + f"id={patch_id}", + "component=mission-core-worker", + "type=qualification-release", + ] + release = json.loads(regular["payload/release.json"]) + assert release["code_revision"] == revision + assert release["candidate_id"] == "gseg3d-ground-consistency" + assert release["authority"] == { + "navigation_or_actuation_allowed": False, + "ravnoves00_quality_accepted": False, + } + serialized = json.dumps(release).lower() + assert "password=" not in serialized + assert "private key" not in serialized