diff --git a/deploy/telemetry-plane/.env.example b/deploy/telemetry-plane/.env.example index 90017c1..9729f36 100644 --- a/deploy/telemetry-plane/.env.example +++ b/deploy/telemetry-plane/.env.example @@ -4,7 +4,9 @@ MISSIONCORE_TELEMETRY_QUERY_PORT=18030 MISSIONCORE_DB_NAME=missioncore_telemetry MISSIONCORE_DB_USER=missioncore_ingest MISSIONCORE_DB_PASSWORD=replace-with-a-random-local-secret +MISSIONCORE_DB_INGEST_PASSWORD=replace-with-a-separate-ingest-secret MISSIONCORE_MQTT_INGEST_USER=missioncore-ingest MISSIONCORE_MQTT_INGEST_PASSWORD=replace-with-a-random-local-secret MISSIONCORE_MQTT_WORKER_006_USER=worker-006 +MISSIONCORE_MQTT_WORKER_006_CONTOUR=worker-006 MISSIONCORE_MQTT_WORKER_006_PASSWORD=replace-with-a-different-random-local-secret diff --git a/deploy/telemetry-plane/README.md b/deploy/telemetry-plane/README.md index 4ea6404..ead6331 100644 --- a/deploy/telemetry-plane/README.md +++ b/deploy/telemetry-plane/README.md @@ -2,12 +2,13 @@ Portable local-only telemetry infrastructure for compute contours. -One Docker Compose project, `ndc-mission-core-telemetry`, owns three isolated -containers: +One Docker Compose project, `ndc-mission-core-telemetry`, owns three long-running +containers and one bounded bootstrap job: - `ndc-mission-core-mqtt-broker` — Eclipse Mosquitto `2.1.2-alpine`; - `ndc-mission-core-telemetry-timescaledb` — TimescaleDB HA OSS `pg16.14-ts2.28.2-all-oss`; +- `ndc-mission-core-telemetry-bootstrap` — one-shot database role/bootstrap job; - `ndc-mission-core-telemetry-normalizer` — Mission Core telemetry normalizer. The Telegraf configuration templates cover Windows and Linux, but the agent runs as a @@ -17,6 +18,11 @@ Windows performance counters, Docker Desktop and NVIDIA telemetry. The normalizer exposes a read-only normalized telemetry adapter on `127.0.0.1:18030`. Mission Core reads this adapter; the browser never receives MQTT or Timescale credentials, and Timescale is not published on a host port. +The normalizer uses a separate `missioncore_normalizer` database role with only +`SELECT`, `INSERT`, `UPDATE`, and bounded retention `DELETE` on the telemetry +hypertable. Raw telemetry older than 30 days is removed at most once per day by the +normalizer. This stays compatible with the Apache-licensed Timescale image without +depending on the Timescale License retention scheduler. The stack is one deployment contour, not one multi-process container. Keeping broker, normalizer and database in separate containers preserves independent health checks, @@ -62,6 +68,20 @@ directory are ignored by Git. The product architecture and topic contract are defined in `docs/adr/0031-local-compute-contour-telemetry-plane.md`. +## Security boundary + +The current MQTT listener is authenticated and contour-scoped, but intentionally +uses plaintext MQTT inside one owner-controlled laboratory LAN. Do not expose port +1883 through a router, cellular WAN, public Wi-Fi, or an Internet-facing host. A +remote or shared-network deployment requires a reviewed TLS listener, a private CA +distributed to every agent, and credential rotation. Wi-Fi link encryption is not a +replacement for MQTT TLS. + +Each agent credential is bound to one exact +`contours//agents//+` prefix. Adding another contour requires +issuing another password entry and explicit ACL row; the wildcard contour writer is +not permitted. + ## Worker agent Worker 006 uses the official Windows Telegraf distribution as the host service diff --git a/deploy/telemetry-plane/compose.yaml b/deploy/telemetry-plane/compose.yaml index 48bfe5c..bb74d8b 100644 --- a/deploy/telemetry-plane/compose.yaml +++ b/deploy/telemetry-plane/compose.yaml @@ -10,6 +10,17 @@ services: image: eclipse-mosquitto:2.1.2-alpine container_name: ndc-mission-core-mqtt-broker restart: unless-stopped + read_only: true + security_opt: + - no-new-privileges:true + cap_drop: + - ALL + cap_add: + - CHOWN + - SETGID + - SETUID + mem_limit: 128m + pids_limit: 128 labels: <<: *ndc-labels com.nodedc.role: mqtt-broker @@ -23,6 +34,8 @@ services: - ./runtime/mosquitto/passwords:/mosquitto/config/passwords:ro - ./runtime/mosquitto/acl:/mosquitto/config/acl:ro - broker-data:/mosquitto/data + tmpfs: + - /tmp:size=16m,noexec,nosuid healthcheck: test: - CMD-SHELL @@ -38,6 +51,10 @@ services: image: timescale/timescaledb-ha:pg16.14-ts2.28.2-all-oss container_name: ndc-mission-core-telemetry-timescaledb restart: unless-stopped + security_opt: + - no-new-privileges:true + mem_limit: 1024m + pids_limit: 256 labels: <<: *ndc-labels com.nodedc.role: telemetry-database @@ -54,6 +71,34 @@ services: timeout: 5s retries: 10 + database-bootstrap: + image: timescale/timescaledb-ha:pg16.14-ts2.28.2-all-oss + container_name: ndc-mission-core-telemetry-bootstrap + restart: "no" + read_only: true + security_opt: + - no-new-privileges:true + cap_drop: + - ALL + mem_limit: 128m + pids_limit: 64 + labels: + <<: *ndc-labels + com.nodedc.role: telemetry-database-bootstrap + depends_on: + timescale: + condition: service_healthy + environment: + MISSIONCORE_DB_NAME: "${MISSIONCORE_DB_NAME}" + MISSIONCORE_DB_ADMIN_USER: "${MISSIONCORE_DB_USER}" + MISSIONCORE_DB_ADMIN_PASSWORD: "${MISSIONCORE_DB_PASSWORD}" + MISSIONCORE_DB_INGEST_PASSWORD: "${MISSIONCORE_DB_INGEST_PASSWORD}" + volumes: + - ./timescale/002_roles_and_retention.sh:/bootstrap.sh:ro + tmpfs: + - /tmp:size=16m,noexec,nosuid + entrypoint: ["/bin/sh", "/bootstrap.sh"] + normalizer: image: nodedc/mission-core-telemetry-normalizer:local container_name: ndc-mission-core-telemetry-normalizer @@ -61,6 +106,13 @@ services: context: ../.. dockerfile: deploy/telemetry-plane/normalizer/Dockerfile restart: unless-stopped + read_only: true + security_opt: + - no-new-privileges:true + cap_drop: + - ALL + mem_limit: 256m + pids_limit: 128 labels: <<: *ndc-labels com.nodedc.role: telemetry-normalizer @@ -69,6 +121,8 @@ services: condition: service_healthy timescale: condition: service_healthy + database-bootstrap: + condition: service_completed_successfully ports: - "127.0.0.1:${MISSIONCORE_TELEMETRY_QUERY_PORT:-18030}:18030" environment: @@ -77,9 +131,11 @@ services: MISSIONCORE_MQTT_USERNAME: "${MISSIONCORE_MQTT_INGEST_USER}" MISSIONCORE_MQTT_PASSWORD: "${MISSIONCORE_MQTT_INGEST_PASSWORD}" MISSIONCORE_DATABASE_DSN: >- - postgresql://${MISSIONCORE_DB_USER}:${MISSIONCORE_DB_PASSWORD}@timescale:5432/${MISSIONCORE_DB_NAME} + postgresql://missioncore_normalizer:${MISSIONCORE_DB_INGEST_PASSWORD}@timescale:5432/${MISSIONCORE_DB_NAME} MISSIONCORE_QUERY_HOST: "0.0.0.0" MISSIONCORE_QUERY_PORT: "18030" + tmpfs: + - /tmp:size=32m,noexec,nosuid healthcheck: test: - CMD diff --git a/deploy/telemetry-plane/normalizer/normalizer.py b/deploy/telemetry-plane/normalizer/normalizer.py index e93ba3e..daf93d6 100644 --- a/deploy/telemetry-plane/normalizer/normalizer.py +++ b/deploy/telemetry-plane/normalizer/normalizer.py @@ -1,5 +1,6 @@ from __future__ import annotations +import contextlib import json import os import re @@ -25,6 +26,40 @@ SOURCE_SCHEMAS: Final = { TELEGRAF_SCHEMA: Final = "telegraf.metric-json/v1" QUERY_SCHEMA: Final = "missioncore.telemetry-query/v1" SAFE_IDENTIFIER: Final = re.compile(r"^[a-z0-9](?:[a-z0-9-]{0,62}[a-z0-9])?$") +MAX_PAYLOAD_BYTES: Final = 1024 * 1024 +MAX_TAGS: Final = 256 +MAX_SERIES_KEY_BYTES: Final = 4096 +RETENTION_INTERVAL_SECONDS: Final = 86_400 +ALLOWED_TELEMETRY_TAGS: Final = frozenset( + { + "agent_id", + "container", + "container_id", + "container_image", + "container_name", + "container_status", + "container_version", + "contour_id", + "cpu", + "device", + "engine_host", + "gpu_name", + "host", + "interface", + "lab_id", + "method_id", + "name", + "node_id", + "path", + "request_id", + "run_id", + "server_version", + "source_id", + "source_package_id", + "stage_id", + "stage_state", + } +) def _required(name: str) -> str: @@ -68,26 +103,48 @@ def _series_key(document: dict[str, Any]) -> str: if not isinstance(tags, dict): value = document.get("series_key") return value if isinstance(value, str) else "" + if len(tags) > MAX_TAGS: + raise ValueError("telemetry payload contains too many tags") stable_tags = { str(name): str(value) for name, value in tags.items() if name not in {"agent_id", "contour_id", "host", "node_id"} } - return json.dumps( + encoded = json.dumps( stable_tags, ensure_ascii=False, sort_keys=True, separators=(",", ":"), ) + if len(encoded.encode("utf-8")) > MAX_SERIES_KEY_BYTES: + raise ValueError("telemetry series identity is too large") + return encoded + + +def _sanitize_tags(document: dict[str, Any]) -> dict[str, Any]: + tags = document.get("tags") + if not isinstance(tags, dict): + return document + if len(tags) > MAX_TAGS: + raise ValueError("telemetry payload contains too many tags") + sanitized = { + str(name): str(value) + for name, value in tags.items() + if name in ALLOWED_TELEMETRY_TAGS + } + return {**document, "tags": sanitized} def _normalize(topic: str, payload: bytes) -> tuple[object, ...]: + if len(payload) > MAX_PAYLOAD_BYTES: + raise ValueError("telemetry payload exceeds the 1 MiB contract") match = TOPIC_RE.fullmatch(topic) if match is None: raise ValueError("topic is outside the telemetry contract") document = json.loads(payload.decode("utf-8")) if not isinstance(document, dict): raise ValueError("payload must be an object") + document = _sanitize_tags(document) kind = match.group("kind") source_schema = document.get("schema_version") if source_schema == SOURCE_SCHEMAS[kind]: @@ -101,14 +158,18 @@ def _normalize(topic: str, payload: bytes) -> tuple[object, ...]: source_schema = TELEGRAF_SCHEMA observed_value = document.get("timestamp") measurement = document["name"].strip() - if not measurement: + if not measurement or len(measurement) > 128: raise ValueError("Telegraf measurement name is required") else: raise ValueError("payload schema does not match topic kind") node_id = document.get("node_id") or _tag_text(document, "node_id") if not node_id: node_id = _tag_text(document, "host") - if not isinstance(node_id, str) or not node_id.strip(): + if ( + not isinstance(node_id, str) + or not node_id.strip() + or len(node_id.strip()) > 128 + ): raise ValueError("node_id is required") observed_at = _timestamp(observed_value) return ( @@ -132,9 +193,53 @@ def _normalize(topic: str, payload: bytes) -> tuple[object, ...]: class TelemetryQueryServer(ThreadingHTTPServer): daemon_threads = True - def __init__(self, server_address: tuple[str, int], dsn: str) -> None: + def __init__( + self, + server_address: tuple[str, int], + dsn: str, + runtime_health: TelemetryRuntimeHealth, + ) -> None: super().__init__(server_address, TelemetryQueryHandler) self.dsn = dsn + self.runtime_health = runtime_health + + +class TelemetryRuntimeHealth: + """Thread-safe health of the actual MQTT-to-database ingestion path.""" + + def __init__(self) -> None: + self._lock = threading.Lock() + self._mqtt_connected = False + self._database_connected = False + self._last_ingested_at_utc: str | None = None + self._last_error: str | None = None + + def set_mqtt(self, connected: bool, error: str | None = None) -> None: + with self._lock: + self._mqtt_connected = connected + self._last_error = error + + def set_database(self, connected: bool, error: str | None = None) -> None: + with self._lock: + self._database_connected = connected + self._last_error = error + + def mark_ingested(self) -> None: + with self._lock: + self._database_connected = True + self._last_ingested_at_utc = ( + datetime.now(UTC).isoformat(timespec="seconds").replace("+00:00", "Z") + ) + self._last_error = None + + def snapshot(self) -> dict[str, object]: + with self._lock: + return { + "mqtt_connected": self._mqtt_connected, + "database_connected": self._database_connected, + "last_ingested_at_utc": self._last_ingested_at_utc, + "last_error": self._last_error, + } class TelemetryQueryHandler(BaseHTTPRequestHandler): @@ -174,6 +279,7 @@ class TelemetryQueryHandler(BaseHTTPRequestHandler): def _health(self) -> None: import psycopg # type: ignore[import-not-found] + database_reachable = False try: with ( psycopg.connect(self.server.dsn) as connection, @@ -181,13 +287,24 @@ class TelemetryQueryHandler(BaseHTTPRequestHandler): ): cursor.execute("SELECT 1") cursor.fetchone() + database_reachable = True except psycopg.Error: - self._json( - HTTPStatus.SERVICE_UNAVAILABLE, - {"ok": False, "schema_version": QUERY_SCHEMA}, - ) - return - self._json(HTTPStatus.OK, {"ok": True, "schema_version": QUERY_SCHEMA}) + database_reachable = False + runtime = self.server.runtime_health.snapshot() + ok = ( + database_reachable + and runtime["mqtt_connected"] is True + and runtime["database_connected"] is True + ) + self._json( + HTTPStatus.OK if ok else HTTPStatus.SERVICE_UNAVAILABLE, + { + "ok": ok, + "schema_version": QUERY_SCHEMA, + "database_reachable": database_reachable, + **runtime, + }, + ) def _latest( self, @@ -270,10 +387,13 @@ class TelemetryQueryHandler(BaseHTTPRequestHandler): self.wfile.write(body) -def _start_query_server(dsn: str) -> ThreadingHTTPServer: +def _start_query_server( + dsn: str, + runtime_health: TelemetryRuntimeHealth, +) -> ThreadingHTTPServer: host = os.environ.get("MISSIONCORE_QUERY_HOST", "0.0.0.0") port = int(os.environ.get("MISSIONCORE_QUERY_PORT", "18030")) - server = TelemetryQueryServer((host, port), dsn) + server = TelemetryQueryServer((host, port), dsn, runtime_health) thread = threading.Thread( target=server.serve_forever, name="telemetry-query", @@ -294,8 +414,11 @@ def main() -> None: port = int(os.environ.get("MISSIONCORE_MQTT_PORT", "1883")) username = _required("MISSIONCORE_MQTT_USERNAME") password = _required("MISSIONCORE_MQTT_PASSWORD") - _start_query_server(dsn) + runtime_health = TelemetryRuntimeHealth() connection = psycopg.connect(dsn, autocommit=True) + last_retention_monotonic = 0.0 + runtime_health.set_database(True) + _start_query_server(dsn, runtime_health) client = mqtt.Client( callback_api_version=CallbackAPIVersion.VERSION2, client_id="ndc-mission-core-telemetry-normalizer", @@ -311,14 +434,44 @@ def main() -> None: _properties: Any, ) -> None: if reason_code.is_failure: + runtime_health.set_mqtt(False, f"MQTT connection rejected: {reason_code}") raise RuntimeError(f"MQTT connection rejected: {reason_code}") connected_client.subscribe(TOPIC, qos=1) + runtime_health.set_mqtt(True) + + def on_disconnect( + _connected_client: Any, + _userdata: object, + _disconnect_flags: Any, + reason_code: Any, + _properties: Any, + ) -> None: + runtime_health.set_mqtt(False, f"MQTT disconnected: {reason_code}") def on_message(_client: Any, _userdata: object, message: Any) -> None: + nonlocal connection, last_retention_monotonic try: row = _normalize(message.topic, message.payload) - with connection.cursor() as cursor: - cursor.execute( + except (ValueError, UnicodeDecodeError, json.JSONDecodeError) as exc: + print(f"telemetry sample rejected: {exc}", flush=True) + return + for attempt in range(2): + try: + with connection.cursor() as cursor: + now_monotonic = time.monotonic() + if ( + now_monotonic - last_retention_monotonic + >= RETENTION_INTERVAL_SECONDS + ): + cursor.execute( + """ + DELETE FROM contour_telemetry_samples + WHERE observed_at + < CURRENT_TIMESTAMP - INTERVAL '30 days' + """ + ) + last_retention_monotonic = now_monotonic + cursor.execute( """ INSERT INTO contour_telemetry_samples ( observed_at, contour_id, agent_id, node_id, kind, measurement, @@ -358,18 +511,37 @@ def main() -> None: ELSE EXCLUDED.payload END """, - (*row, TELEGRAF_SCHEMA, TELEGRAF_SCHEMA), - ) - except (ValueError, UnicodeDecodeError, json.JSONDecodeError, psycopg.Error) as exc: - print(f"telemetry sample rejected: {exc}", flush=True) + (*row, TELEGRAF_SCHEMA, TELEGRAF_SCHEMA), + ) + runtime_health.mark_ingested() + return + except psycopg.Error as exc: + runtime_health.set_database(False, f"database ingestion failed: {exc}") + with contextlib.suppress(psycopg.Error): + connection.close() + if attempt == 0: + try: + connection = psycopg.connect(dsn, autocommit=True) + runtime_health.set_database(True) + except psycopg.Error as reconnect_error: + runtime_health.set_database( + False, + f"database reconnect failed: {reconnect_error}", + ) + break + else: + break + print("telemetry sample rejected: database unavailable", flush=True) client.on_connect = on_connect + client.on_disconnect = on_disconnect client.on_message = on_message while True: try: client.connect(host, port, keepalive=30) client.loop_forever(retry_first_connection=True) except (OSError, MQTTException, psycopg.Error) as exc: + runtime_health.set_mqtt(False, f"MQTT reconnecting: {exc}") print(f"telemetry normalizer reconnecting: {exc}", flush=True) time.sleep(3) diff --git a/deploy/telemetry-plane/prepare.py b/deploy/telemetry-plane/prepare.py index f71a63d..3d62ad3 100644 --- a/deploy/telemetry-plane/prepare.py +++ b/deploy/telemetry-plane/prepare.py @@ -3,6 +3,7 @@ from __future__ import annotations import argparse import ipaddress import os +import re import secrets import subprocess from pathlib import Path @@ -13,6 +14,9 @@ ENV_PATH: Final = ROOT / ".env" RUNTIME: Final = ROOT / "runtime" / "mosquitto" IMAGE: Final = "eclipse-mosquitto:2.1.2-alpine" PLACEHOLDER: Final = "replace-with-" +SAFE_IDENTIFIER: Final = re.compile( + r"^[a-z0-9](?:[a-z0-9-]{0,62}[a-z0-9])?$" +) def _initialize_environment(bind_address: str, *, overwrite: bool = False) -> None: @@ -26,9 +30,11 @@ def _initialize_environment(bind_address: str, *, overwrite: bool = False) -> No "MISSIONCORE_DB_NAME": "missioncore_telemetry", "MISSIONCORE_DB_USER": "missioncore_ingest", "MISSIONCORE_DB_PASSWORD": secrets.token_urlsafe(36), + "MISSIONCORE_DB_INGEST_PASSWORD": secrets.token_urlsafe(36), "MISSIONCORE_MQTT_INGEST_USER": "missioncore-ingest", "MISSIONCORE_MQTT_INGEST_PASSWORD": secrets.token_urlsafe(36), "MISSIONCORE_MQTT_WORKER_006_USER": "worker-006", + "MISSIONCORE_MQTT_WORKER_006_CONTOUR": "worker-006", "MISSIONCORE_MQTT_WORKER_006_PASSWORD": secrets.token_urlsafe(36), } ENV_PATH.write_text( @@ -51,6 +57,23 @@ def _environment() -> dict[str, str]: return values +def _migrate_environment() -> None: + values = _environment() + additions: dict[str, str] = {} + if "MISSIONCORE_DB_INGEST_PASSWORD" not in values: + additions["MISSIONCORE_DB_INGEST_PASSWORD"] = secrets.token_urlsafe(36) + if "MISSIONCORE_MQTT_WORKER_006_CONTOUR" not in values: + additions["MISSIONCORE_MQTT_WORKER_006_CONTOUR"] = "worker-006" + if not additions: + return + with ENV_PATH.open("a", encoding="utf-8", newline="\n") as stream: + for name, value in additions.items(): + stream.write(f"{name}={value}\n") + stream.flush() + os.fsync(stream.fileno()) + os.chmod(ENV_PATH, 0o600) + + def _required(values: dict[str, str], name: str) -> str: value = values.get(name, "") if not value or value.startswith(PLACEHOLDER): @@ -58,6 +81,13 @@ def _required(values: dict[str, str], name: str) -> str: return value +def _identifier(values: dict[str, str], name: str) -> str: + value = _required(values, name) + if SAFE_IDENTIFIER.fullmatch(value) is None: + raise RuntimeError(f"{name} must contain a DNS-safe lowercase identifier") + return value + + def _password_entry(path: Path, username: str, password: str, *, create: bool) -> None: command = [ "docker", @@ -80,6 +110,22 @@ def _password_entry(path: Path, username: str, password: str, *, create: bool) - ) +def _prepare_password_entries( + path: Path, + ingest_user: str, + ingest_password: str, + worker_user: str, + worker_password: str, +) -> None: + _password_entry( + path, + ingest_user, + ingest_password, + create=not path.exists(), + ) + _password_entry(path, worker_user, worker_password, create=False) + + def main() -> None: parser = argparse.ArgumentParser() parser.add_argument( @@ -87,6 +133,11 @@ def main() -> None: action="store_true", help="create a private .env with generated local credentials", ) + parser.add_argument( + "--migrate", + action="store_true", + help="add newly required generated secrets without replacing existing values", + ) parser.add_argument( "--mqtt-bind-address", default="127.0.0.1", @@ -95,17 +146,28 @@ def main() -> None: arguments = parser.parse_args() if arguments.initialize: _initialize_environment(arguments.mqtt_bind_address) + if arguments.migrate: + _migrate_environment() values = _environment() ingest_user = _required(values, "MISSIONCORE_MQTT_INGEST_USER") ingest_password = _required(values, "MISSIONCORE_MQTT_INGEST_PASSWORD") - worker_user = _required(values, "MISSIONCORE_MQTT_WORKER_006_USER") + worker_user = _identifier(values, "MISSIONCORE_MQTT_WORKER_006_USER") + worker_contour = _identifier( + values, + "MISSIONCORE_MQTT_WORKER_006_CONTOUR", + ) worker_password = _required(values, "MISSIONCORE_MQTT_WORKER_006_PASSWORD") _required(values, "MISSIONCORE_DB_PASSWORD") RUNTIME.mkdir(parents=True, exist_ok=True) password_path = RUNTIME / "passwords" - _password_entry(password_path, ingest_user, ingest_password, create=True) - _password_entry(password_path, worker_user, worker_password, create=False) + _prepare_password_entries( + password_path, + ingest_user, + ingest_password, + worker_user, + worker_password, + ) acl_path = RUNTIME / "acl" acl_path.write_text( "\n".join( @@ -114,8 +176,12 @@ def main() -> None: "topic read mission-core/v1/contours/+/agents/+/+", "topic read $SYS/broker/uptime", "", - "# Agent username must equal its stable agent id.", - "pattern write mission-core/v1/contours/+/agents/%u/+", + "# Every credential is scoped to one contour and one stable agent.", + f"user {worker_user}", + ( + "topic write mission-core/v1/contours/" + f"{worker_contour}/agents/{worker_user}/+" + ), "", ] ), diff --git a/deploy/telemetry-plane/telegraf/Install-NdcMissionCoreTelegraf.ps1 b/deploy/telemetry-plane/telegraf/Install-NdcMissionCoreTelegraf.ps1 index f3eb3b3..8c7a629 100644 --- a/deploy/telemetry-plane/telegraf/Install-NdcMissionCoreTelegraf.ps1 +++ b/deploy/telemetry-plane/telegraf/Install-NdcMissionCoreTelegraf.ps1 @@ -20,7 +20,8 @@ foreach ($name in @( "MISSIONCORE_MQTT_HOST", "MISSIONCORE_MQTT_PORT", "MISSIONCORE_MQTT_USERNAME", - "MISSIONCORE_MQTT_PASSWORD" + "MISSIONCORE_MQTT_PASSWORD", + "MISSIONCORE_TELEMETRY_INTERVAL" )) { $value = $payload.$name if (-not $value) { @@ -87,7 +88,8 @@ try { "MISSIONCORE_MQTT_HOST=$($payload.MISSIONCORE_MQTT_HOST)", "MISSIONCORE_MQTT_PORT=$($payload.MISSIONCORE_MQTT_PORT)", "MISSIONCORE_MQTT_USERNAME=$($payload.MISSIONCORE_MQTT_USERNAME)", - "MISSIONCORE_MQTT_PASSWORD=$($payload.MISSIONCORE_MQTT_PASSWORD)" + "MISSIONCORE_MQTT_PASSWORD=$($payload.MISSIONCORE_MQTT_PASSWORD)", + "MISSIONCORE_TELEMETRY_INTERVAL=$($payload.MISSIONCORE_TELEMETRY_INTERVAL)" ) $serviceRegistryPath = "HKLM:\SYSTEM\CurrentControlSet\Services\$serviceName" Set-ItemProperty -Path $serviceRegistryPath -Name Environment ` diff --git a/deploy/telemetry-plane/telegraf/Update-NdcMissionCoreTelegraf.ps1 b/deploy/telemetry-plane/telegraf/Update-NdcMissionCoreTelegraf.ps1 index 3ed5a5b..1a49bbe 100644 --- a/deploy/telemetry-plane/telegraf/Update-NdcMissionCoreTelegraf.ps1 +++ b/deploy/telemetry-plane/telegraf/Update-NdcMissionCoreTelegraf.ps1 @@ -26,6 +26,9 @@ foreach ($entry in @((Get-ItemProperty -Path $serviceRegistryPath).Environment)) Set-Item -Path "Env:$name" -Value $value } } +if (-not $env:MISSIONCORE_TELEMETRY_INTERVAL) { + $env:MISSIONCORE_TELEMETRY_INTERVAL = "2s" +} $temporaryRoot = Join-Path $env:TEMP "ndc-mission-core-telegraf-update-$([Guid]::NewGuid().ToString('N'))" $validationOutput = Join-Path $temporaryRoot "validation.out.log" diff --git a/deploy/telemetry-plane/telegraf/mission-core-linux.conf.tmpl b/deploy/telemetry-plane/telegraf/mission-core-linux.conf.tmpl index 8001796..633ade1 100644 --- a/deploy/telemetry-plane/telegraf/mission-core-linux.conf.tmpl +++ b/deploy/telemetry-plane/telegraf/mission-core-linux.conf.tmpl @@ -1,5 +1,5 @@ [agent] - interval = "2s" + interval = "${MISSIONCORE_TELEMETRY_INTERVAL}" round_interval = true omit_hostname = false diff --git a/deploy/telemetry-plane/telegraf/mission-core-windows.conf.tmpl b/deploy/telemetry-plane/telegraf/mission-core-windows.conf.tmpl index 6155f17..db178cb 100644 --- a/deploy/telemetry-plane/telegraf/mission-core-windows.conf.tmpl +++ b/deploy/telemetry-plane/telegraf/mission-core-windows.conf.tmpl @@ -1,5 +1,5 @@ [agent] - interval = "2s" + interval = "${MISSIONCORE_TELEMETRY_INTERVAL}" round_interval = true omit_hostname = false diff --git a/deploy/telemetry-plane/timescale/002_roles_and_retention.sh b/deploy/telemetry-plane/timescale/002_roles_and_retention.sh new file mode 100644 index 0000000..ddebccc --- /dev/null +++ b/deploy/telemetry-plane/timescale/002_roles_and_retention.sh @@ -0,0 +1,31 @@ +#!/bin/sh +set -eu + +export PGPASSWORD="${MISSIONCORE_DB_ADMIN_PASSWORD}" + +psql \ + --host timescale \ + --username "${MISSIONCORE_DB_ADMIN_USER}" \ + --dbname "${MISSIONCORE_DB_NAME}" \ + --set ON_ERROR_STOP=1 \ + --set database_name="${MISSIONCORE_DB_NAME}" \ + --set ingest_password="${MISSIONCORE_DB_INGEST_PASSWORD}" <<'SQL' +DO $roles$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_roles WHERE rolname = 'missioncore_normalizer' + ) THEN + CREATE ROLE missioncore_normalizer LOGIN; + END IF; +END +$roles$; + +ALTER ROLE missioncore_normalizer PASSWORD :'ingest_password'; +GRANT CONNECT ON DATABASE :"database_name" TO missioncore_normalizer; +GRANT USAGE ON SCHEMA public TO missioncore_normalizer; +GRANT SELECT, INSERT, UPDATE, DELETE ON contour_telemetry_samples + TO missioncore_normalizer; + +DELETE FROM contour_telemetry_samples +WHERE observed_at < CURRENT_TIMESTAMP - INTERVAL '30 days'; +SQL diff --git a/docs/01_IMPLEMENTATION_PLAN.md b/docs/01_IMPLEMENTATION_PLAN.md index 8a84947..dca0c69 100644 --- a/docs/01_IMPLEMENTATION_PLAN.md +++ b/docs/01_IMPLEMENTATION_PLAN.md @@ -3,7 +3,25 @@ This plan supersedes the app-dependent experiment order in the reference Bible. Each gate produces evidence and an explicit GO, PAUSE or BLOCKED result. -## Current checkpoint — 2026-07-24 +## Architecture stabilization checkpoint — 2026-07-28 + +| Gate | Current result | +| --- | --- | +| RAVNOVES00 methodology | CORRECTED — E41 proves E37 labels are engineering evidence, not independent truth, and reproduces connected development/evaluation overlap. The 146-row slice is historical evaluated visible validation. | +| Prediction/evaluation boundary | GO — truth-free E41 predictor package/result are physically separate from E37 and reproduce E40 predictions exactly; the visible evaluator runs only after inference. | +| Source-scoped quality | PAUSE — E40 visible engineering evaluation is 84.2466% presence, 84.2466% geometry association and 94.5205% freshness, with 11 high-severity failures, complete accounting and zero false-free claims. This is not a blind accuracy gate. | +| Structural regressions | GO (bounded) — E42 passes predictor identity/order/chunk invariance and PointSlab order/SE(3) checks. Raw-producer and cross-route invariance remain unproved. | +| Native pipeline telemetry | GO for producer/normalizer contract — lifecycle events and JSONL/MQTT sink boundaries exist and an E41 smoke run records real stage accounting. Durable Worker 006 MQTT wiring/deployment remains pending. | +| Evidence storage | MEASURED — E44 finds 525,471,092 logical bytes, 312,753,179 unique-content bytes and 1.680146× amplification across 14 E30–E40 roots. Exact-content references/deduplication precede any format migration. | +| Future transfer | PREREGISTERED — E43 freezes same-K1/mount/calibration/firmware, required streams, connected-component split and independent label reveal. Capture and labels do not yet exist. | +| Product interface | DEFERRED — no new windows, page anatomy or design changes are part of this stabilization increment. | + +The governing decision is +[`ADR 0032`](adr/0032-ravnoves00-methodology-correction-before-transfer.md). +Historical E37–E40 artifacts remain immutable; only the claims made from them +change. + +## Earlier checkpoint — 2026-07-24 | Stage | Result | | --- | --- | diff --git a/docs/13_LIDAR_WORKER_PRODUCT_AND_ROADMAP.md b/docs/13_LIDAR_WORKER_PRODUCT_AND_ROADMAP.md index a9834ee..d5631fd 100644 --- a/docs/13_LIDAR_WORKER_PRODUCT_AND_ROADMAP.md +++ b/docs/13_LIDAR_WORKER_PRODUCT_AND_ROADMAP.md @@ -8,7 +8,8 @@ explainability implemented; L2.6e recorded-source-paced bounded shadow qualified; E28 complete worker replay accepted; E29 camera-first semantic and parallel geometry-only replay implemented; E30–E35 source-scoped qualification accepted; RAVNOVES00 reference-source product maturation active; E36 transfer -deferred by ADR 0030 +preregistered and deferred by ADR 0030/0032; E41 methodology boundary, E42 +metamorphic checks and E44 amplification audit complete Scope: passively received real-time K1 point/pose evidence, immutable replay and future live shadow processing Explicitly out of scope: K1 firmware modification, a new onboard exporter, new @@ -652,6 +653,9 @@ to its source session under that audit profile. The transfer replay was not created. ADR 0030 subsequently deferred TEST007 and E36 as current priorities: RAVNOVES00 source-scoped quality, product workflow and a reference release candidate now come first. No new capture is required or authorized by default. +ADR 0032 retains that priority while preregistering the later same-K1/new-route +capture under E43. The protocol is frozen; no capture or labels currently +exist. The active gates are defined in `docs/20_RAVNOVES00_REFERENCE_SOURCE_PRODUCT_PLAN.md`. @@ -719,17 +723,37 @@ allowed to change; E31 determined the E32 profile; E32 determined the E33 runtime input; E32/E33 then bound the E34 layer; E32–E34 then bound E35. Exact nominal and degradation accounting are closed, so A8 is complete. -R1 perception quality is now measured by two immutable Worker 006 results. +R1 perception quality was initially measured by two immutable Worker 006 results. E38 established the baseline at 82.2% presence, 81.5% geometry association, 95.2% freshness and 14 high-severity failures. E39 result `e39-perception-refinement-2fd253940c9d9a2fd3b3237f3f0932f81a9f69741935d793771243d5779af464` used a fixed package-bound camera + LiDAR feature projection and improved presence and geometry association to 84.9% while reducing high-severity failures to 8. Freshness remained above target at 93.2%. Its development -cross-validation exceeded 90% but did not predict the sealed result, so R1 -remains open. The next iteration is development-only grouped time/scene -qualification with a source-coordinate-free representation; validation is not -opened for item-level tuning. +cross-validation exceeded 90% but did not predict the evaluated slice, so R1 +remains open. + +The E41 source audit corrects the meaning of those measurements. E37's +`independent_ground_truth=false` provenance, connected group overlap and +prediction/evaluation co-location mean the 146 cases were historically +evaluated visible engineering labels, not a sealed blind set. E40 package-bound +result +`e40-perception-product-gate-e96eec9fd68c3ffaaee898d46285dd329191267200011680f084c75095b92e9a` +reached 84.2466% presence, 84.2466% geometry association and 94.5205% freshness +with 11 high-severity failures, complete accounting and zero false-free claims. +It remains useful source-scoped engineering evidence and does not close R1. + +E41 result +`e41-methodology-audit-703dac176f20843ad6fbb9a1060b199942fba05e3bb34d3e2264ee87c7bfdd96` +reproduces `14` exact-frame, `38` track, `58` fixed-time-block and `59` +connected whole-track-or-scene overlaps. The replacement predictor package and +prediction contain no reference/split/scoring material, reproduce E40 +predictions exactly and are evaluated only afterward by +`e41-visible-evaluation-57aa6c8569e8339630406e3c88cd539df13917fb08c1af57f0baae5cdd1069d2`. +E42 adds bounded structural invariants; E44 measures exact-content +amplification; E43 freezes the later independent transfer protocol. ADR 0032 +contains the corrected decision boundary and current priority order. + E36 is the first generalization gate. A separate product decision follows: either keep the result as operator/shadow evidence, or start L5 occupied-space integration. No LAB in this cycle can enable navigation, commands or safety diff --git a/docs/15_LABORATORY_RUN_CANON.md b/docs/15_LABORATORY_RUN_CANON.md index 809b6b5..27a3839 100644 --- a/docs/15_LABORATORY_RUN_CANON.md +++ b/docs/15_LABORATORY_RUN_CANON.md @@ -32,12 +32,24 @@ candidate is accepted: - new perception and product iterations use RAVNOVES00; - each iteration creates a new immutable LAB and never rewrites an older one; -- TEST007, E36 and new physical collection are deferred, not required gates; +- TEST007 and E36 replay are deferred, not required current gates; - no agent may initiate or imply a new capture without explicit operator authorization; - the active quality target is at least `90%` on each frozen task-level validation dimension, never one unqualified aggregate accuracy number. +ADR 0032 corrects the present validation meaning. E37's 146-row slice is +historical evaluated visible engineering validation: its labels are not +independent truth and its development/evaluation groups overlap. It remains an +immutable comparison contract, but it cannot close a blind accuracy gate. +Prediction packages must exclude truth, split, severity and scoring material, +and evaluation must join those artifacts only after prediction. + +The owner has authorized preparation for a later same-K1/new-route recording, +not agent-initiated collection. E43 freezes that future protocol before data +exists. Until the recording is supplied, no LAB result, metric or transfer +claim is created for it. + This policy is a development priority, not a universality claim. Later second-source transfer remains required before any cross-route or cross-camera generalization statement. diff --git a/docs/20_RAVNOVES00_REFERENCE_SOURCE_PRODUCT_PLAN.md b/docs/20_RAVNOVES00_REFERENCE_SOURCE_PRODUCT_PLAN.md index a17a4cb..2de2609 100644 --- a/docs/20_RAVNOVES00_REFERENCE_SOURCE_PRODUCT_PLAN.md +++ b/docs/20_RAVNOVES00_REFERENCE_SOURCE_PRODUCT_PLAN.md @@ -1,8 +1,8 @@ # RAVNOVES00 reference-source product plan -Date: 2026-07-27 +Date: 2026-07-28 -Status: active +Status: active; methodology corrected by ADR 0032 ADR 0030 makes RAVNOVES00 the sole physical reference source for the current Mission Core product-maturation cycle. This plan turns that decision into @@ -24,13 +24,20 @@ missions. Mission Core should answer: The product is not being designed for adversarial, tactical or continuously novel environments. -## Acceptance meaning +## Acceptance meaning after source audit `90%` is a target for reviewed task-level correctness on RAVNOVES00, not a generic marketing accuracy score. -The next acceptance contract must freeze a validation set before further -tuning and measure at least: +E37 froze a useful source-scoped engineering contract, but E41 proved that its +labels are not independent ground truth and its development/validation rows +share frames, tracks and time/scene components. The existing 146-row slice is +therefore **historical evaluated visible validation**, not a sealed blind set. +Its metrics remain useful for immutable method comparison, but cannot prove +independent perception accuracy. + +The next independent acceptance contract must freeze a leakage-free validation +set before labels are revealed and measure at least: | Dimension | Question | Source-scoped target | | --- | --- | ---: | @@ -44,30 +51,35 @@ Targets apply separately. A strong presence score cannot compensate for bad geometry association or hidden stale evidence. The reviewed denominator, sampling strata and severity classes must be frozen -before the next tuning cycle. Development items may be used for diagnosis and -changes; sealed validation items may only be used for evaluation. High-impact +before the next independent tuning cycle. Connected scene/track/time components, +not individual item hashes, are the minimum partition unit. Development items +may be used for diagnosis and changes; independent validation labels remain +unavailable to the predictor until frozen predictions exist. High-impact failures remain blocking even when an aggregate percentage passes. ## Current path | Gate | Deliverable | Exit | | --- | --- | --- | -| R0 — acceptance contract | Freeze task ontology, reviewed denominator, development/validation split, metrics, severity and error budget over RAVNOVES00 | Reproducible evaluation with no post-result denominator changes | +| R0 — acceptance methodology | Preserve the E37 visible engineering contract, enforce predictor/evaluator separation and preregister an independent, leakage-free future split | No truth material in the predictor; connected-component split; labels revealed only after frozen prediction | | R1 — perception quality | Iterate detector, camera↔geometry association, source-time handling and conservative corrections as new immutable LABs | Each task-level validation dimension reaches `>= 90%` or has an explicit bounded exception | | R2 — temporal product state | Refine current/held/stale/unknown presentation and bounded occupied telemetry without polluting the persistent reconstruction | Stable task-relevant state, closed accounting and deterministic degradation | -| R3 — product workflow | Complete reusable source/LAB selection, method summary, visual evidence, comparison, controls and conclusion templates | A non-expert can understand the run; an engineer can inspect exact provenance and metrics | +| R3 — product workflow | Separate owner-reviewed interface phase; no new windows, page anatomy or design changes in the present stabilization increment | Product brief reviewed separately before implementation | | R4 — reference release candidate | Replay the complete RAVNOVES00 source through the accepted pipeline at recorded pace with regression, resource and recovery evidence | Source-scoped release decision for the known-location product envelope | -| R5 — later transfer | Record a new route/camera only after explicit operator authorization and run the frozen release candidate unchanged | Generalization decision; no retuning before comparison | +| R5 — later transfer | Under the preregistered E43 protocol, record the same K1 and mount over a new route after explicit operator authorization; freeze predictions before independent labels are revealed | Generalization decision; no retuning before comparison | -R0–R4 are the active path. R5 is intentionally deferred. +R0–R2 and non-UI R4 preparation are the active stabilization path. R3 is +explicitly deferred for separate discussion. R5 has a frozen protocol but no +capture or labels yet. ## Current evidence -R0 is closed by immutable result +E37 created immutable result `e37-ravnoves-acceptance-01b1efd586f747341c712d82f0907b39436a6f91ae92b1dfae987eca05fd8344`. -It freezes 486 reviewed RAVNOVES00 items as 340 development and 146 sealed -validation cases, with separate presence, geometry-association and freshness -references. +It freezes 486 reviewed RAVNOVES00 items as 340 development and 146 visible +evaluation cases, with separate presence, geometry-association and freshness +references. Its own provenance records `484` engineering-reviewed items, `2` +human exceptions and `independent_ground_truth=false`. The first R1 measurement is immutable result `e38-perception-baseline-a272f82988cd9a7e071fad94c3e9fb49daf804fdcca523f853445fd3113a62b1`. @@ -84,15 +96,59 @@ Its fixed robust three-neighbour refinement used exact camera-crop, LiDAR shape and projection features and was selected only through five-fold development cross-validation. Development reached 90.3% presence, 90.6% geometry association and 96.5% freshness without validation-label access. -Sealed validation reached 84.9%, 84.9% and 93.2% respectively. Accounting +The then-designated validation slice reached 84.9%, 84.9% and 93.2% +respectively. Accounting remains 100%, false-free claims remain zero and high-severity failures fall from 14 to 8, but R1 is still not accepted. -The next R1 iteration must address the development-to-validation gap before -adding model complexity: grouped time/scene development folds replace -item-hash folds, and a source-coordinate-free representation is compared under -that harder protocol. No individual E39 validation label may be used for -diagnosis, fitting or selection. +E40 package-bound result +`e40-perception-product-gate-e96eec9fd68c3ffaaee898d46285dd329191267200011680f084c75095b92e9a` +used `89` camera-only development rows over `125` feature dimensions (`90` +variable and `35` constant). Against the already-visible E37 evaluation slice +it reached `84.2466%` presence, `84.2466%` geometry association and +`94.5205%` freshness, with `11` high-severity failures, complete accounting and +zero false-free claims. It does not pass R1. + +E41 then audited the implementation and artifacts together. Immutable audit +`e41-methodology-audit-703dac176f20843ad6fbb9a1060b199942fba05e3bb34d3e2264ee87c7bfdd96` +found overlap across `14` exact frames, `38` track identities, `58` fixed +50-frame time blocks and `59` connected whole-track-or-scene groups. It also +proved that E40 prediction and evaluation material were co-located. R0 is +therefore reopened for blind acceptance. + +The replacement boundary is executable: + +- predictor package + `e41-predictor-package-fb35f42698013c63c6d417fd3b26986e9d3b89d12dce138bab18fe4203d593ad` + contains no reference, split, severity, scoring or truth material; +- prediction result + `e41-predictions-adcc1671de44646145a396d5815e522d6b4257d6ead9ace995761751655b3aac` + reproduces the physical E40 predictions exactly; +- separate visible evaluation + `e41-visible-evaluation-57aa6c8569e8339630406e3c88cd539df13917fb08c1af57f0baae5cdd1069d2` + reproduces the E40 engineering metrics and blockers only after inference. + +E42 immutable result +`e42-metamorphic-suite-c358204e0b9e10eea5871e90b7c4fbe2c132f1f6d87074b2c5ffec18eaeebb40` +passes predictor identity, row-order and chunk-boundary invariance, metadata +absence, PointSlab row-order invariance and rigid-coordinate equivariance. It +does not prove raw-producer or cross-route invariance. + +E44 immutable audit +`e44-data-amplification-89791978894ec785009e5b76c9325de4d9e910237af57476c9331eb47a8ccca4` +measures `525,471,092` logical bytes versus `312,753,179` unique-content bytes +across 14 admitted E30–E40 roots: `1.680146×` amplification and +`212,717,913` exact duplicate bytes. The immediate storage action is +content-addressed referencing and exact deduplication, not an unmeasured +format/database migration. + +E43 immutable protocol +`e43-future-capture-protocol-28f091b9648daffce988d44c183e21f56d77988061630de934f8003fb13701d8` +preregisters the later same-K1/new-route transfer. It requires the same mount, +calibration and firmware; `480–900 s` duration; at least `60 s` control bridge +and `360 s` new route; camera, registered LiDAR, pose and pipeline telemetry; +connected component partitioning; two independent reviewers; and label reveal +only after frozen prediction. No such capture or labels currently exist. ## Experiment rules @@ -109,15 +165,28 @@ diagnosis, fitting or selection. 7. No run claims universal accuracy, planner fitness, navigation or safety acceptance merely because its source-scoped target passes. -## Deferred work +## Deferred work and current priority -Until R4 closes, the following are not current blockers: +The current non-UI priority order is: + +1. keep E37–E40 status language methodologically honest; +2. make the E41 predictor/evaluator boundary and E42 invariants mandatory + regression checks; +3. wire native pipeline telemetry into durable worker stages without inferring + absent measurements; +4. design exact-content references/deduplication from E44 before any storage + migration; +5. preserve E43 unchanged until the owner supplies the new capture. + +The following remain outside the present stabilization increment: - TEST007 qualification; -- E36 second-source transfer; -- a new physical route or camera capture; +- E36 replay until an admitted source exists; +- initiation of a new physical route/camera capture by an agent; - second K1 or changed mount generalization; - open-world and adversarial evaluation. -The existing E36 audit and its Ops card remain retained so transfer work can -resume later without reconstructing history. +The owner may provide a future recording under E43. Until it exists, current +work neither depends on it nor simulates its result. The existing E36 audit and +its Ops card remain retained so transfer work can resume without reconstructing +history. diff --git a/docs/adr/0031-local-compute-contour-telemetry-plane.md b/docs/adr/0031-local-compute-contour-telemetry-plane.md index 8c1d5a7..b175e97 100644 --- a/docs/adr/0031-local-compute-contour-telemetry-plane.md +++ b/docs/adr/0031-local-compute-contour-telemetry-plane.md @@ -22,6 +22,14 @@ The existing Worker 006 containers were renamed in place to container identities were preserved, so this namespace migration did not restart the inference or perception runtimes. +The compute-side native pipeline contract is now implemented. It emits +`missioncore.agent-pipeline-telemetry/v1` lifecycle documents through an injected +transport, and the telemetry normalizer preserves source, method, stage and stage-state +identities in the normalized series key. The E41 runner has exercised the JSONL +evidence sink against the immutable predictor package. This proves producer and +normalizer compatibility; it does not claim that the durable Worker 006 process has +been wired to the MQTT sink or deployed with this code. + ## Decision Mission Core treats a compute worker as a configurable **local compute contour**, not as @@ -188,6 +196,7 @@ subscribe to `mission-core/v1/contours/+/agents/+/+`. - Mission Core code stays responsible for product semantics and stable API contracts, not OS-specific metric collection. - Host, container, network and inference telemetry is stored in the normalized path. - Per-stage LAB processing telemetry remains unavailable until a laboratory worker - publishes the native `runtime` and `pipeline` topic contracts; the UI must keep those - stages explicitly unavailable rather than infer them from aggregate hardware load. +- Per-stage LAB processing telemetry has a native producer contract and an admitted + already-connected MQTT sink. It remains unavailable for a durable worker run until + that worker actually injects the sink and publishes the `pipeline` topic; consumers + must not infer stages from aggregate hardware load. diff --git a/docs/adr/0032-ravnoves00-methodology-correction-before-transfer.md b/docs/adr/0032-ravnoves00-methodology-correction-before-transfer.md new file mode 100644 index 0000000..1466079 --- /dev/null +++ b/docs/adr/0032-ravnoves00-methodology-correction-before-transfer.md @@ -0,0 +1,89 @@ +# ADR 0032 — RAVNOVES00 methodology correction before transfer + +Date: 2026-07-28 + +Status: accepted for implementation + +## Context + +E37–E40 produced useful, immutable source-scoped engineering evidence, but the +architecture report was written before the source implementation and artifacts were +audited together. E41 reproduced the exact data lineage and found four methodological +limits: + +1. E37 contains `484` engineering-reviewed labels and `2` human exception labels; it + explicitly declares `independent_ground_truth=false`. +2. The nominal `340` development / `146` validation split is not group-independent. + Development and validation overlap by `14` exact frames, `38` track identities, + `58` fixed 50-frame time blocks and `59` connected whole-track-or-scene groups. +3. The E40 worker package co-located prediction inputs with the visible acceptance + rows and split assignments. +4. E38–E40 therefore measure performance against a historically evaluated, + source-scoped engineering contract. They are not blind accuracy gates. + +This does not invalidate their physical predictions, closed accounting or conservative +authority. It changes the claim that those artifacts are allowed to support. + +## Decision + +1. Historical E37–E40 artifacts remain immutable evidence. Their metrics are retained, + but “sealed”, “blind” and independent-accuracy language is removed from current + status. +2. The current `146`-item slice is named **historical evaluated visible validation**. + It may compare immutable methods against the source-scoped engineering contract; it + may not prove independent perception accuracy or cross-route generalization. +3. Prediction and evaluation are physically separated. The E41 predictor package and + result contain no acceptance rows, references, split assignments, severity or + scoring state. The visible evaluator joins predictions to E37 only after inference. +4. Runtime identity records the environment that actually executed a result. A local + run records the repository lock, Python and NumPy identities and is not represented + as a Worker 006 container run. +5. Structural invariants are exercised independently of the visible labels. E42 + verifies predictor identity, row-order and chunk-boundary invariance plus PointSlab + row-order invariance and rigid-coordinate equivariance. +6. Native per-stage telemetry is a compute contract. Stages publish + `started/completed/failed`, exact source/method/run identity, duration and accounting + through an injected sink. A JSONL evidence sink and an already-connected MQTT sink + are admitted; no network transport starts implicitly. +7. Current storage work starts with content-addressed references and exact-content + deduplication. E44 measures `1.680146×` amplification across the admitted E30–E40 + package/result roots. It does not authorize a storage-format or database migration. +8. The later same-K1 transfer capture is preregistered before collection. E43 freezes + minimum streams, route/control-bridge duration, connected scene/track/time + partitioning, independent human review and truth reveal only after frozen + prediction. +9. Product windows, page anatomy and new design work are outside this stabilization + decision and remain a separate owner review. + +## Current immutable evidence + +- methodology audit: + `e41-methodology-audit-703dac176f20843ad6fbb9a1060b199942fba05e3bb34d3e2264ee87c7bfdd96`; +- truth-free predictor package: + `e41-predictor-package-fb35f42698013c63c6d417fd3b26986e9d3b89d12dce138bab18fe4203d593ad`; +- truth-free prediction result: + `e41-predictions-adcc1671de44646145a396d5815e522d6b4257d6ead9ace995761751655b3aac`; +- visible engineering evaluation: + `e41-visible-evaluation-57aa6c8569e8339630406e3c88cd539df13917fb08c1af57f0baae5cdd1069d2`; +- metamorphic suite: + `e42-metamorphic-suite-c358204e0b9e10eea5871e90b7c4fbe2c132f1f6d87074b2c5ffec18eaeebb40`; +- preregistered future-capture protocol: + `e43-future-capture-protocol-28f091b9648daffce988d44c183e21f56d77988061630de934f8003fb13701d8`; +- data-amplification audit: + `e44-data-amplification-89791978894ec785009e5b76c9325de4d9e910237af57476c9331eb47a8ccca4`. + +## Consequences + +- RAVNOVES00 remains the active engineering reference source, but R0 is reopened as an + independent-truth and leakage-free-partition gate. +- E40 does not pass R1: the visible evaluation is `84.2466%` for presence, + `84.2466%` for geometry association and `94.5205%` for freshness, with `11` + high-severity failures, complete accounting and zero false-free claims. +- Further threshold or model tuning against the visible 146 items cannot close the + blind gate. It may only create another explicitly visible engineering comparison. +- Work that does not require a new recording can proceed now: methodology enforcement, + predictor/evaluator separation, invariants, telemetry, exact-content deduplication + design and preregistration. +- A later new-route recording can close an independent transfer question only if the + E43 protocol passes before labels are revealed. Until that capture exists, + cross-route quality remains unknown rather than failed or inferred. diff --git a/experiments/perception/analyze_e40_development.py b/experiments/perception/analyze_e40_development.py new file mode 100644 index 0000000..611e77f --- /dev/null +++ b/experiments/perception/analyze_e40_development.py @@ -0,0 +1,930 @@ +#!/usr/bin/env python3 +"""Development-only E40 feature and split audit. + +The script intentionally rejects validation rows before feature extraction. It +compares route-coordinate-free candidates under two leakage-resistant +protocols: + +* contiguous source-time folds; +* whole detector tracks plus 50-frame geometry scene windows. + +It never writes a model or a sealed result. A production E40 profile can only +be frozen after one fixed candidate passes the development gate here. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +from collections import Counter +from pathlib import Path +from typing import Any + +import numpy as np +from PIL import Image + +from k1link.compute.e38_perception_baseline import _predict_tree, _train_tree + +LABELS = ( + "background-or-noise", + "object-present", + "occupied-environment", +) +TARGET = 0.9 + +CATEGORIES = { + "stratum": ("agree", "camera-only", "conflict", "geometry-only", "unknown"), + "range": ("near", "middle", "far", "unavailable"), + "geometry": ( + "agree", + "conflict", + "single-source-camera", + "single-source-geometry", + "unknown", + "unavailable", + ), + "label": ("car", "person", "truck", "bicycle", "motorcycle", "bus", "none"), + "reason": ( + "camera-semantic-without-qualified-occupied-lidar-support", + "camera-semantic-with-connected-occupied-lidar-support", + "semantic-observation-not-current", + "camera-object-region-observed-as-local-surface", + "none", + ), + "association": ("vehicle", "person", "bicycle", "motorcycle", "none"), + "motion": ("unknown", "static", "dynamic", "none"), + "camera_motion": ("unknown", "static", "dynamic", "none"), + "semantic_current": ("true", "false", "none"), +} + + +def main() -> int: + args = _parse_args() + acceptance_rows = _read_jsonl(args.acceptance_root / "acceptance-items.jsonl") + materialization_rows = _read_jsonl(args.materialization_root / "materialized-items.jsonl") + materialization_by_id = {str(row["item_id"]): row for row in materialization_rows} + development = [row for row in acceptance_rows if row.get("split") == "development"] + if len(development) != 340: + raise RuntimeError("E40 development denominator must remain 340") + if any(row.get("split") != "development" for row in development): + raise RuntimeError("E40 analysis received a non-development row") + + feature_rows = [ + _features( + acceptance=row, + materialization=materialization_by_id[str(row["item_id"])], + materialization_root=args.materialization_root, + ) + for row in development + ] + feature_names = sorted({name for row in feature_rows for name in row}) + matrix = np.asarray( + [[row.get(name, 0.0) for name in feature_names] for row in feature_rows], + dtype=np.float64, + ) + labels = np.asarray( + [LABELS.index(str(row["reference"]["presence"])) for row in development], + dtype=np.int64, + ) + if not np.isfinite(matrix).all(): + raise RuntimeError("E40 development features contain non-finite values") + + assignments = { + "contiguous-source-time-five-fold": _contiguous_folds(development), + "whole-track-or-scene-window-five-fold": _track_scene_folds( + development, + materialization_by_id, + ), + } + print( + json.dumps( + { + "development_items": len(development), + "feature_dimensions": len(feature_names), + "validation_rows_loaded_for_features": 0, + "protocols": { + name: {str(fold): count for fold, count in sorted(Counter(values).items())} + for name, values in assignments.items() + }, + }, + indent=2, + sort_keys=True, + ) + ) + feature_sets = { + "structured": [ + index + for index, name in enumerate(feature_names) + if not _source_array_feature(name) and not name.startswith("image_") + ], + "structured-shape": [ + index for index, name in enumerate(feature_names) if not name.startswith("image_") + ], + "structured-image": [ + index + for index, name in enumerate(feature_names) + if not _source_array_feature(name) + or name.startswith("candidate_class_fraction_") + or name.startswith("image_") + ], + "all": list(range(len(feature_names))), + } + candidates = [ + ("structured-knn-k3", "knn", 3, "structured"), + ("structured-softmax-l2-0.01", "softmax", 0.01, "structured"), + ( + "structured-shape-softmax-l2-0.01", + "softmax", + 0.01, + "structured-shape", + ), + ( + "structured-image-softmax-l2-0.01", + "softmax", + 0.01, + "structured-image", + ), + ("all-softmax-l2-0.01", "softmax", 0.01, "all"), + ("all-softmax-l2-0.03", "softmax", 0.03, "all"), + ("all-softmax-l2-0.1", "softmax", 0.1, "all"), + ( + "hierarchical-structured-softmax-l2-0.01", + "hierarchical-softmax", + 0.01, + "structured", + ), + ( + "hierarchical-structured-image-softmax-l2-0.01", + "hierarchical-softmax", + 0.01, + "structured-image", + ), + ( + "hierarchical-all-softmax-l2-0.01", + "hierarchical-softmax", + 0.01, + "all", + ), + ] + candidate_results: list[dict[str, Any]] = [] + for candidate_name, model_type, model_parameter, feature_set in candidates: + columns = feature_sets[feature_set] + result = { + protocol: _cross_validate( + matrix=matrix[:, columns], + labels=labels, + feature_names=[feature_names[index] for index in columns], + feature_rows=feature_rows, + source_strata=[str(row["source_stratum"]) for row in development], + assignments=folds, + model_type=model_type, + model_parameter=model_parameter, + ) + for protocol, folds in assignments.items() + } + candidate_results.append( + { + "candidate": candidate_name, + "feature_set": feature_set, + "feature_dimensions": len(columns), + "accuracy": {protocol: row["accuracy"] for protocol, row in result.items()}, + "by_stratum": {protocol: row["by_stratum"] for protocol, row in result.items()}, + "by_fold": {protocol: row["by_fold"] for protocol, row in result.items()}, + "passed_both": all(row["accuracy"] >= TARGET for row in result.values()), + } + ) + print(json.dumps(candidate_results, indent=2, sort_keys=True)) + return 0 + + +def _features( + *, + acceptance: dict[str, Any], + materialization: dict[str, Any], + materialization_root: Path, +) -> dict[str, float]: + snapshot = _object(materialization.get("e29_snapshot")) + evidence = _object(materialization.get("materialization")) + values: dict[str, float] = {} + observed = { + "stratum": materialization.get("stratum"), + "range": materialization.get("range_bucket"), + "geometry": snapshot.get("geometry_status"), + "label": snapshot.get("label") or "none", + "reason": snapshot.get("geometry_reason") or "none", + "association": snapshot.get("association_group") or "none", + "motion": snapshot.get("motion_state") or "none", + "camera_motion": snapshot.get("camera_motion_state") or "none", + "semantic_current": ( + str(snapshot.get("semantic_current")).lower() + if snapshot.get("semantic_current") is not None + else "none" + ), + } + for prefix, categories in CATEGORIES.items(): + for category in categories: + values[f"{prefix}={category}"] = float(observed[prefix] == category) + + for name in ( + "selected_point_count", + "candidate_point_count", + "rejected_candidate_point_count", + "projected_point_count", + "frame_point_count", + ): + values[name] = math.log1p(max(0.0, _number(evidence.get(name), 0.0))) + values["detector_score"] = _number(evidence.get("detector_score"), -1.0) + for name in ( + "nearest_range_m", + "point_count", + "voxel_count", + "score", + "camera_motion_confidence", + "range_m", + ): + raw = _number(snapshot.get(name), -1.0) + values[name] = ( + math.log1p(raw) if name in {"point_count", "voxel_count"} and raw >= 0.0 else raw + ) + + bbox = snapshot.get("bbox_xyxy") + if isinstance(bbox, list) and len(bbox) == 4: + x1, y1, x2, y2 = (_number(value, 0.0) for value in bbox) + width = max(0.0, x2 - x1) / 800.0 + height = max(0.0, y2 - y1) / 600.0 + values.update( + { + "bbox_present": 1.0, + "bbox_center_x": (x1 + x2) / 1600.0, + "bbox_center_y": (y1 + y2) / 1200.0, + "bbox_width": width, + "bbox_height": height, + "bbox_area": width * height, + "bbox_aspect": width / (height + 1e-6), + } + ) + else: + values.update( + { + "bbox_present": 0.0, + "bbox_center_x": -1.0, + "bbox_center_y": -1.0, + "bbox_width": -1.0, + "bbox_height": -1.0, + "bbox_area": -1.0, + "bbox_aspect": -1.0, + } + ) + + support = snapshot.get("support") + support = support if isinstance(support, dict) else {} + support_names = ( + "below_surface_points_in_bbox", + "classified_points_in_bbox", + "connected_occupied_points", + "connected_occupied_voxels", + "occupied_points_in_bbox", + "projected_points_in_bbox", + "surface_points_in_bbox", + ) + support_values = {name: _number(support.get(name), 0.0) for name in support_names} + values.update( + {f"support_{name}": math.log1p(max(0.0, value)) for name, value in support_values.items()} + ) + projected = max(1.0, support_values["projected_points_in_bbox"]) + occupied = max(1.0, support_values["occupied_points_in_bbox"]) + values.update( + { + "support_occupied_fraction": (support_values["occupied_points_in_bbox"] / projected), + "support_classified_fraction": ( + support_values["classified_points_in_bbox"] / projected + ), + "support_surface_fraction": (support_values["surface_points_in_bbox"] / projected), + "support_connected_fraction": (support_values["connected_occupied_points"] / occupied), + } + ) + _span_features(values, "bounds", snapshot.get("bounds_map_xyz_m")) + _range_span(values, "height_span", snapshot.get("height_range_m")) + _range_span( + values, + "occupied_height_span", + snapshot.get("occupied_height_range_m"), + ) + + artifact = _object(materialization.get("artifact")) + artifact_path = materialization_root / str(artifact.get("path")) + with np.load(artifact_path, allow_pickle=False) as arrays: + pixels = arrays["projected_pixels_xy"] + candidate_mask = arrays["projected_candidate_mask"].astype(bool) + selected_mask = arrays["projected_selected_mask"].astype(bool) + sensor_position = arrays["sensor_position_map_xyz_m"] + sensor_orientation = arrays["sensor_orientation_map_from_lidar_xyzw"] + _named_point_statistics( + values, + "candidate_lidar", + arrays["candidate_points_map_xyz_m"], + sensor_position, + sensor_orientation, + ) + _named_point_statistics( + values, + "selected_lidar", + arrays["selected_points_map_xyz_m"], + sensor_position, + sensor_orientation, + ) + projection_width = max(1, int(evidence.get("projection_width", 800))) + projection_height = max(1, int(evidence.get("projection_height", 600))) + _named_pixel_statistics( + values, + "candidate_pixel", + pixels[candidate_mask], + projection_width, + projection_height, + ) + _named_pixel_statistics( + values, + "selected_pixel", + pixels[selected_mask], + projection_width, + projection_height, + ) + _named_quantiles( + values, + "candidate_depth", + arrays["projected_depth_m"][candidate_mask], + ) + _named_quantiles( + values, + "selected_depth", + arrays["projected_depth_m"][selected_mask], + ) + _named_quantiles( + values, + "candidate_height", + arrays["projected_point_height_m"][candidate_mask], + ) + _named_quantiles( + values, + "selected_height", + arrays["projected_point_height_m"][selected_mask], + ) + point_classes = arrays["projected_point_class"][candidate_mask] + for index in range(8): + values[f"candidate_class_fraction_{index}"] = ( + float(np.mean(point_classes == index)) if point_classes.size else 0.0 + ) + _image_features( + values, + materialization=materialization, + materialization_root=materialization_root, + candidate_pixels=pixels[candidate_mask], + projection_width=projection_width, + projection_height=projection_height, + ) + # Source frame, session time, review ordinal, track ID and absolute map + # coordinates are deliberately absent. + return values + + +def _cross_validate( + *, + matrix: np.ndarray, + labels: np.ndarray, + feature_names: list[str], + feature_rows: list[dict[str, float]], + source_strata: list[str], + assignments: np.ndarray, + model_type: str, + model_parameter: object, +) -> dict[str, Any]: + predictions = np.full(len(labels), -1, dtype=np.int64) + for fold in range(5): + train_indices = np.flatnonzero(assignments != fold) + test_indices = np.flatnonzero(assignments == fold) + train, test = _robust_transform( + matrix[train_indices], + matrix[test_indices], + ) + if model_type == "knn": + distances = np.mean( + np.square(test[:, None, :] - train[None, :, :]), + axis=2, + ) + nearest = np.argsort(distances, axis=1, kind="stable")[:, : int(model_parameter)] + for local_index, neighbor_indices in enumerate(nearest): + votes = Counter(labels[train_indices][neighbor_indices]) + predictions[test_indices[local_index]] = sorted( + votes.items(), + key=lambda item: (-item[1], item[0]), + )[0][0] + elif model_type == "softmax": + weights = _train_softmax( + train, + labels[train_indices], + l2=float(model_parameter), + ) + predictions[test_indices] = np.argmax( + np.column_stack((test, np.ones(len(test)))) @ weights, + axis=1, + ) + elif model_type == "hierarchical-softmax": + _predict_hierarchical_fold( + predictions=predictions, + train_indices=train_indices, + test_indices=test_indices, + training=train, + testing=test, + labels=labels, + source_strata=source_strata, + l2=float(model_parameter), + ) + elif model_type == "tree": + depth, min_leaf = model_parameter # type: ignore[misc] + tree = _train_tree( + [ + ( + {name: feature_rows[index].get(name, 0.0) for name in feature_names}, + LABELS[int(labels[index])], + ) + for index in train_indices + ], + feature_names=feature_names, + max_depth=int(depth), + min_leaf=int(min_leaf), + ) + for index in test_indices: + predictions[index] = LABELS.index(_predict_tree(tree, feature_rows[int(index)])) + else: + raise RuntimeError(f"unsupported model: {model_type}") + accuracy = float(np.mean(predictions == labels)) + stratum_metrics = {} + for stratum in sorted(set(source_strata)): + indices = np.asarray( + [index for index, value in enumerate(source_strata) if value == stratum], + dtype=np.int64, + ) + stratum_metrics[stratum] = round( + float(np.mean(predictions[indices] == labels[indices])), + 6, + ) + fold_metrics = {} + for fold in range(5): + indices = np.flatnonzero(assignments == fold) + fold_metrics[str(fold)] = round( + float(np.mean(predictions[indices] == labels[indices])), + 6, + ) + return { + "accuracy": round(accuracy, 6), + "correct": int(np.sum(predictions == labels)), + "incorrect": int(np.sum(predictions != labels)), + "passed": accuracy >= TARGET, + "by_stratum": stratum_metrics, + "by_fold": fold_metrics, + "confusion": [ + { + "reference": LABELS[reference], + "prediction": LABELS[prediction], + "count": count, + } + for (reference, prediction), count in sorted( + Counter( + zip( + labels.tolist(), + predictions.tolist(), + strict=True, + ) + ).items(), + key=lambda item: (-item[1], item[0]), + ) + ], + } + + +def _predict_hierarchical_fold( + *, + predictions: np.ndarray, + train_indices: np.ndarray, + test_indices: np.ndarray, + training: np.ndarray, + testing: np.ndarray, + labels: np.ndarray, + source_strata: list[str], + l2: float, +) -> None: + fixed = { + "conflict": LABELS.index("background-or-noise"), + "agree": LABELS.index("object-present"), + "unknown": LABELS.index("object-present"), + # A geometry-only cluster is positive occupied evidence, but without + # camera semantics it must not be promoted to a named object. Keeping + # it occupied is the conservative product state. + "geometry-only": LABELS.index("occupied-environment"), + } + for source_index in test_indices: + stratum = source_strata[int(source_index)] + if stratum in fixed: + predictions[source_index] = fixed[stratum] + for stratum, fallback in (("camera-only", LABELS.index("object-present")),): + local_training = np.asarray( + [ + index + for index, source_index in enumerate(train_indices) + if source_strata[int(source_index)] == stratum + ], + dtype=np.int64, + ) + local_testing = np.asarray( + [ + index + for index, source_index in enumerate(test_indices) + if source_strata[int(source_index)] == stratum + ], + dtype=np.int64, + ) + if not len(local_testing): + continue + observed = sorted(set(labels[train_indices][local_training].tolist())) + if len(local_training) < 10 or len(observed) < 2: + predictions[test_indices[local_testing]] = fallback + continue + weights = _train_softmax( + training[local_training], + labels[train_indices][local_training], + l2=l2, + ) + predictions[test_indices[local_testing]] = np.argmax( + np.column_stack( + ( + testing[local_testing], + np.ones(len(local_testing)), + ) + ) + @ weights, + axis=1, + ) + + +def _train_softmax( + matrix: np.ndarray, + labels: np.ndarray, + *, + l2: float, +) -> np.ndarray: + rows, dimensions = matrix.shape + design = np.column_stack((matrix, np.ones(rows))) + targets = np.eye(len(LABELS), dtype=np.float64)[labels] + weights = np.zeros((dimensions + 1, len(LABELS)), dtype=np.float64) + first_moment = np.zeros_like(weights) + second_moment = np.zeros_like(weights) + for step in range(1, 1201): + logits = design @ weights + logits -= np.max(logits, axis=1, keepdims=True) + probabilities = np.exp(logits) + probabilities /= np.sum(probabilities, axis=1, keepdims=True) + regularizer = np.vstack((weights[:-1], np.zeros((1, len(LABELS))))) + gradient = design.T @ (probabilities - targets) / rows + gradient += l2 * regularizer + first_moment = 0.9 * first_moment + 0.1 * gradient + second_moment = 0.999 * second_moment + 0.001 * np.square(gradient) + corrected_first = first_moment / (1.0 - 0.9**step) + corrected_second = second_moment / (1.0 - 0.999**step) + weights -= 0.03 * corrected_first / (np.sqrt(corrected_second) + 1e-8) + return weights + + +def _robust_transform( + training: np.ndarray, + testing: np.ndarray, +) -> tuple[np.ndarray, np.ndarray]: + median = np.median(training, axis=0) + scale = np.percentile(training, 75, axis=0) - np.percentile( + training, + 25, + axis=0, + ) + scale[scale < 1e-8] = 1.0 + return ( + np.clip((training - median) / scale, -10.0, 10.0), + np.clip((testing - median) / scale, -10.0, 10.0), + ) + + +def _source_array_feature(name: str) -> bool: + return name.startswith( + ( + "candidate_lidar_", + "selected_lidar_", + "candidate_pixel_", + "selected_pixel_", + "candidate_depth_", + "selected_depth_", + "candidate_height_", + "selected_height_", + "candidate_class_fraction_", + ) + ) + + +def _contiguous_folds(rows: list[dict[str, Any]]) -> np.ndarray: + order = np.argsort( + [int(row["source_frame_index"]) for row in rows], + kind="stable", + ) + assignments = np.empty(len(rows), dtype=np.int64) + for fold, indices in enumerate(np.array_split(order, 5)): + assignments[indices] = fold + return assignments + + +def _track_scene_folds( + rows: list[dict[str, Any]], + materialization_by_id: dict[str, dict[str, Any]], +) -> np.ndarray: + assignments: list[int] = [] + for row in rows: + snapshot = _object(materialization_by_id[str(row["item_id"])].get("e29_snapshot")) + track_id = snapshot.get("track_id") + group = ( + f"track:{track_id}" + if track_id is not None + else f"scene:{int(row['source_frame_index']) // 50}" + ) + digest = hashlib.sha256(f"e40:{group}".encode()).hexdigest() + assignments.append(int(digest[:8], 16) % 5) + return np.asarray(assignments, dtype=np.int64) + + +def _named_point_statistics( + values: dict[str, float], + prefix: str, + points_map: np.ndarray, + sensor_position: np.ndarray, + sensor_orientation_xyzw: np.ndarray, +) -> None: + points = np.asarray(points_map, dtype=np.float64) + if points.ndim != 2 or points.shape[1] != 3 or not len(points): + for axis in "xyz": + _named_quantiles(values, f"{prefix}_{axis}", np.asarray([])) + for index in range(3): + values[f"{prefix}_covariance_ratio_{index}"] = 0.0 + return + rotation = _rotation_matrix(sensor_orientation_xyzw) + points_local = (points - np.asarray(sensor_position)) @ rotation + for axis, index in zip("xyz", range(3), strict=True): + _named_quantiles(values, f"{prefix}_{axis}", points_local[:, index]) + if len(points_local) >= 3: + eigenvalues = np.maximum( + np.linalg.eigvalsh(np.cov(points_local, rowvar=False)), + 0.0, + ) + else: + eigenvalues = np.zeros(3) + ratios = eigenvalues / (float(np.sum(eigenvalues)) + 1e-9) + for index, ratio in enumerate(ratios): + values[f"{prefix}_covariance_ratio_{index}"] = float(ratio) + + +def _rotation_matrix(quaternion_xyzw: np.ndarray) -> np.ndarray: + x, y, z, w = (float(value) for value in quaternion_xyzw) + return np.asarray( + [ + [ + 1.0 - 2.0 * (y * y + z * z), + 2.0 * (x * y - z * w), + 2.0 * (x * z + y * w), + ], + [ + 2.0 * (x * y + z * w), + 1.0 - 2.0 * (x * x + z * z), + 2.0 * (y * z - x * w), + ], + [ + 2.0 * (x * z - y * w), + 2.0 * (y * z + x * w), + 1.0 - 2.0 * (x * x + y * y), + ], + ], + dtype=np.float64, + ) + + +def _named_pixel_statistics( + values: dict[str, float], + prefix: str, + pixels: np.ndarray, + width: int, + height: int, +) -> None: + points = np.asarray(pixels, dtype=np.float64) + if points.ndim != 2 or points.shape[1] != 2 or not len(points): + _named_quantiles(values, f"{prefix}_x", np.asarray([])) + _named_quantiles(values, f"{prefix}_y", np.asarray([])) + values[f"{prefix}_span_x"] = -1.0 + values[f"{prefix}_span_y"] = -1.0 + values[f"{prefix}_density"] = -1.0 + return + x = points[:, 0] / width + y = points[:, 1] / height + _named_quantiles(values, f"{prefix}_x", x) + _named_quantiles(values, f"{prefix}_y", y) + span_x = max(float(np.max(x) - np.min(x)), 1.0 / width) + span_y = max(float(np.max(y) - np.min(y)), 1.0 / height) + values[f"{prefix}_span_x"] = span_x + values[f"{prefix}_span_y"] = span_y + values[f"{prefix}_density"] = len(points) / (span_x * span_y * width * height + 1.0) + + +def _image_features( + values: dict[str, float], + *, + materialization: dict[str, Any], + materialization_root: Path, + candidate_pixels: np.ndarray, + projection_width: int, + projection_height: int, +) -> None: + snapshot = _object(materialization.get("e29_snapshot")) + bbox = snapshot.get("bbox_xyxy") + if not isinstance(bbox, list) or len(bbox) != 4: + points = np.asarray(candidate_pixels, dtype=np.float64) + if points.ndim != 2 or points.shape[1] != 2 or not len(points): + _empty_image_features(values) + return + low = np.min(points, axis=0) + high = np.max(points, axis=0) + center = (low + high) / 2.0 + support = np.maximum(high - low, 48.0) + bbox = [ + center[0] - support[0], + center[1] - support[1], + center[0] + support[0], + center[1] + support[1], + ] + x1, y1, x2, y2 = (_number(value, 0.0) for value in bbox) + box = ( + max(0, int(x1)), + max(0, int(y1)), + min(projection_width, int(math.ceil(x2))), + min(projection_height, int(math.ceil(y2))), + ) + if box[2] <= box[0] or box[3] <= box[1]: + _empty_image_features(values) + return + frame = _object(materialization.get("camera_frame")) + with Image.open(materialization_root / str(frame.get("path"))) as source: + rgb = ( + np.asarray( + source.convert("RGB").crop(box).resize((32, 32), Image.Resampling.BILINEAR), + dtype=np.float64, + ) + / 255.0 + ) + for channel, name in enumerate(("red", "green", "blue")): + histogram, _ = np.histogram( + rgb[:, :, channel], + bins=4, + range=(0.0, 1.0), + ) + histogram = histogram / max(1, int(np.sum(histogram))) + for index, fraction in enumerate(histogram): + values[f"image_{name}_histogram_{index}"] = float(fraction) + luma = np.mean(rgb, axis=2) + saturation = np.max(rgb, axis=2) - np.min(rgb, axis=2) + edge = np.concatenate( + ( + np.abs(np.diff(luma, axis=1)).reshape(-1), + np.abs(np.diff(luma, axis=0)).reshape(-1), + ) + ) + values["image_luma_mean"] = float(np.mean(luma)) + values["image_luma_std"] = float(np.std(luma)) + for name, quantile in zip( + ("p10", "p25", "p50", "p75", "p90"), + np.quantile(luma, (0.1, 0.25, 0.5, 0.75, 0.9)), + strict=True, + ): + values[f"image_luma_{name}"] = float(quantile) + values["image_saturation_mean"] = float(np.mean(saturation)) + values["image_saturation_std"] = float(np.std(saturation)) + values["image_edge_mean"] = float(np.mean(edge)) + values["image_edge_p90"] = float(np.quantile(edge, 0.9)) + small_luma = ( + np.asarray( + Image.fromarray(np.uint8(np.clip(luma * 255.0, 0.0, 255.0))).resize( + (4, 4), + Image.Resampling.BILINEAR, + ), + dtype=np.float64, + ) + / 255.0 + ) + small_luma -= float(np.mean(small_luma)) + for y in range(4): + for x in range(4): + values[f"image_luma_centered_{y}_{x}"] = float(small_luma[y, x]) + + +def _empty_image_features(values: dict[str, float]) -> None: + for channel in ("red", "green", "blue"): + for index in range(4): + values[f"image_{channel}_histogram_{index}"] = 0.0 + for name in ( + "mean", + "std", + "p10", + "p25", + "p50", + "p75", + "p90", + ): + values[f"image_luma_{name}"] = 0.0 + values["image_saturation_mean"] = 0.0 + values["image_saturation_std"] = 0.0 + values["image_edge_mean"] = 0.0 + values["image_edge_p90"] = 0.0 + for y in range(4): + for x in range(4): + values[f"image_luma_centered_{y}_{x}"] = 0.0 + + +def _named_quantiles( + values: dict[str, float], + prefix: str, + raw: object, +) -> None: + array = np.asarray(raw, dtype=np.float64).reshape(-1) + finite = array[np.isfinite(array)] + names = ("min", "p10", "p25", "p50", "p75", "p90", "max") + quantiles = ( + np.quantile(finite, (0.0, 0.1, 0.25, 0.5, 0.75, 0.9, 1.0)) + if finite.size + else np.full(7, -1.0) + ) + for name, value in zip(names, quantiles, strict=True): + values[f"{prefix}_{name}"] = float(value) + + +def _span_features( + values: dict[str, float], + prefix: str, + raw: object, +) -> None: + if ( + isinstance(raw, list) + and len(raw) == 2 + and all(isinstance(item, list) and len(item) == 3 for item in raw) + ): + for index, axis in enumerate("xyz"): + values[f"{prefix}_{axis}_span"] = _number( + raw[1][index], + -1.0, + ) - _number(raw[0][index], -1.0) + else: + for axis in "xyz": + values[f"{prefix}_{axis}_span"] = -1.0 + + +def _range_span( + values: dict[str, float], + name: str, + raw: object, +) -> None: + values[name] = ( + _number(raw[1], -1.0) - _number(raw[0], -1.0) + if isinstance(raw, list) and len(raw) == 2 + else -1.0 + ) + + +def _number(value: object, fallback: float) -> float: + try: + parsed = float(value) # type: ignore[arg-type] + except (TypeError, ValueError): + return fallback + return parsed if math.isfinite(parsed) else fallback + + +def _object(value: object) -> dict[str, Any]: + if not isinstance(value, dict): + raise RuntimeError("expected object") + return value + + +def _read_jsonl(path: Path) -> list[dict[str, Any]]: + with path.open("r", encoding="utf-8-sig") as stream: + return [_object(json.loads(line)) for line in stream] + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--acceptance-root", type=Path, required=True) + parser.add_argument("--materialization-root", type=Path, required=True) + return parser.parse_args() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/experiments/perception/e40_ravnoves00_product_gate_profile.json b/experiments/perception/e40_ravnoves00_product_gate_profile.json new file mode 100644 index 0000000..84e902b --- /dev/null +++ b/experiments/perception/e40_ravnoves00_product_gate_profile.json @@ -0,0 +1,38 @@ +{ + "authority": { + "commands_enabled": false, + "navigation_or_safety_accepted": false + }, + "model": { + "cross_validation_folds": 5, + "cross_validation_seed": "e40-grouped-dev-cv", + "feature_set": "route-coordinate-free-structured-image/v1", + "fixed_presence_by_stratum": { + "agree": "object-present", + "conflict": "background-or-noise", + "geometry-only": "occupied-environment", + "unknown": "object-present" + }, + "l2": 0.01, + "learning_rate": 0.03, + "robust_clip": 10.0, + "steps": 1200, + "type": "hierarchical-stratum-softmax" + }, + "profile_id": "e40-ravnoves00-leakage-resistant-product-gate/v1", + "schema_version": "missioncore.e40-perception-product-gate-profile/v1", + "source": { + "acceptance_result_id": "e37-ravnoves-acceptance-01b1efd586f747341c712d82f0907b39436a6f91ae92b1dfae987eca05fd8344", + "display_name": "RAVNOVES00", + "materialization_id": "e30-materialization-841af926d8d28ab93538c46d8f31278a2234c4d1c12c7dc4dc296b249d59735a", + "session_id": "20260720T065719Z_viewer_live" + }, + "targets": { + "accounting_target": 1.0, + "freshness_target": 0.9, + "geometry_association_target": 0.9, + "maximum_false_free_claims": 0, + "maximum_high_severity_failures": 0, + "presence_target": 0.9 + } +} diff --git a/experiments/perception/e41_ravnoves00_methodology_profile.json b/experiments/perception/e41_ravnoves00_methodology_profile.json new file mode 100644 index 0000000..1af3096 --- /dev/null +++ b/experiments/perception/e41_ravnoves00_methodology_profile.json @@ -0,0 +1,30 @@ +{ + "authority": { + "commands_enabled": false, + "navigation_or_safety_accepted": false + }, + "policy": { + "current_validation_semantics": "historical-evaluated-visible-validation", + "forbidden_feature_tokens": [ + "item_id", + "map_xyz", + "path", + "review_ordinal", + "session_id", + "session_seconds", + "source_frame", + "track_id" + ], + "independent_truth_required_for_blind": true, + "predictor_truth_separation_required": true, + "time_block_frames": 50 + }, + "profile_id": "e41-ravnoves00-methodology-audit/v1", + "schema_version": "missioncore.e41-methodology-audit-profile/v1", + "source": { + "acceptance_result_id": "e37-ravnoves-acceptance-01b1efd586f747341c712d82f0907b39436a6f91ae92b1dfae987eca05fd8344", + "e40_package_id": "e40-worker-package-0b7c1aa6d1d31172206b125002928f9adfd8ea3c0a8f95c6caae431bfafff247", + "e40_result_id": "e40-perception-product-gate-e96eec9fd68c3ffaaee898d46285dd329191267200011680f084c75095b92e9a", + "materialization_id": "e30-materialization-841af926d8d28ab93538c46d8f31278a2234c4d1c12c7dc4dc296b249d59735a" + } +} diff --git a/experiments/perception/e43_same_k1_new_route_profile.json b/experiments/perception/e43_same_k1_new_route_profile.json new file mode 100644 index 0000000..0fabd07 --- /dev/null +++ b/experiments/perception/e43_same_k1_new_route_profile.json @@ -0,0 +1,47 @@ +{ + "acceptance_contract": { + "accounting_target": 1.0, + "freshness_target": 0.9, + "geometry_association_target": 0.9, + "maximum_false_free_claims": 0, + "maximum_high_severity_failures": 0, + "presence_target": 0.9 + }, + "authority": { + "commands_enabled": false, + "navigation_or_safety_accepted": false + }, + "blind_truth_contract": { + "blind_fraction": 0.3, + "engineering_acceptance_labels_are_truth": false, + "independent_human_reviewers": 2, + "labels_revealed_after_frozen_prediction": true, + "partition_strategy": "connected-scene-track-time-components/v1", + "seed": "mission-core-e43-same-k1-new-route-v1" + }, + "capture_contract": { + "device_model": "XGRIDS/LixelKity-K1", + "maximum_duration_seconds": 900, + "minimum_duration_seconds": 480, + "required_segments": [ + { + "kind": "control-bridge", + "minimum_duration_seconds": 60 + }, + { + "kind": "new-route", + "minimum_duration_seconds": 360 + } + ], + "required_streams": [ + "sensor.camera.right", + "sensor.lidar.registered-map-increment", + "sensor.pose", + "telemetry.pipeline" + ], + "route_policy": "control-bridge-then-new-route/v1", + "same_device_mount_calibration_firmware_required": true + }, + "profile_id": "e43-same-k1-new-route-truth-island/v1", + "schema_version": "missioncore.e43-future-capture-profile/v1" +} diff --git a/experiments/perception/prepare_e40_worker_package.py b/experiments/perception/prepare_e40_worker_package.py new file mode 100644 index 0000000..a7eca26 --- /dev/null +++ b/experiments/perception/prepare_e40_worker_package.py @@ -0,0 +1,434 @@ +#!/usr/bin/env python3 +"""Build an immutable camera/LiDAR E40 package for Worker 006.""" + +from __future__ import annotations + +import argparse +import hashlib +import io +import json +import os +import shutil +import uuid +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +import numpy as np + +from k1link.compute.e40_perception_product_gate import ( + _FEATURE_CACHE_ARRAYS, + _FEATURE_CACHE_MANIFEST, + _FEATURE_CACHE_SCHEMA, + E40_PACKAGE_SCHEMA, + E40_PROFILE_SCHEMA, + _feature_names, + _feature_vector, +) + +_RUNTIME_FILES = { + "runtime/k1link/__init__.py": "src/k1link/__init__.py", + "runtime/k1link/compute/__init__.py": None, + "runtime/k1link/compute/e37_acceptance_contract.py": ( + "src/k1link/compute/e37_acceptance_contract.py" + ), + "runtime/k1link/compute/e40_perception_product_gate.py": ( + "src/k1link/compute/e40_perception_product_gate.py" + ), + "runtime/run_e40_perception_product_gate.py": ( + "experiments/perception/worker/run_e40_perception_product_gate.py" + ), + "runtime/validate_e40_worker_package.py": ( + "experiments/perception/worker/validate_e40_worker_package.py" + ), + "runtime/Invoke-E40PerceptionProductGate.ps1": ( + "experiments/perception/worker/Invoke-E40PerceptionProductGate.ps1" + ), +} +_GENERATED_COMPUTE_INIT = ( + '"""Minimal E40 worker projection; import contract modules explicitly."""\n' +) +_ACCEPTANCE_FILES = ( + "manifest.json", + "acceptance-items.jsonl", + "acceptance-contract.json", + "run-report.json", +) +_MATERIALIZATION_FILES = ("manifest.json", "materialized-items.jsonl") + + +class E40WorkerPackageError(RuntimeError): + """The E40 package source or immutable package is invalid.""" + + +def build_e40_worker_package( + *, + repository_root: Path, + acceptance_root: Path, + materialization_root: Path, + profile_path: Path, + output_root: Path, +) -> Path: + """Build or verify one content-addressed E40 worker package.""" + + repository = repository_root.resolve(strict=True) + profile_source = profile_path.resolve(strict=True) + profile = _read_json(profile_source) + if profile.get("schema_version") != E40_PROFILE_SCHEMA: + raise E40WorkerPackageError("E40 package profile is incompatible") + acceptance = acceptance_root.resolve(strict=True) + materialization = materialization_root.resolve(strict=True) + expected_ids = { + "acceptance": profile["source"]["acceptance_result_id"], + "materialization": profile["source"]["materialization_id"], + } + if ( + acceptance.name != expected_ids["acceptance"] + or materialization.name != expected_ids["materialization"] + ): + raise E40WorkerPackageError("E40 source identity changed") + + sources: dict[str, Path | bytes | None] = {} + for target, relative in _RUNTIME_FILES.items(): + source = None if relative is None else repository / relative + if source is not None and (not source.is_file() or source.is_symlink()): + raise E40WorkerPackageError(f"E40 runtime source is invalid: {relative}") + sources[target] = source + sources["profile.json"] = profile_source + for filename in _ACCEPTANCE_FILES: + source = acceptance / filename + if not source.is_file() or source.is_symlink(): + raise E40WorkerPackageError("E40 acceptance artifact is invalid") + sources[f"input/acceptance/{acceptance.name}/{filename}"] = source + for filename in _MATERIALIZATION_FILES: + source = materialization / filename + if not source.is_file() or source.is_symlink(): + raise E40WorkerPackageError("E40 materialization artifact is invalid") + sources[f"input/materialization/{materialization.name}/{filename}"] = source + _add_materialized_evidence( + sources, + materialization=materialization, + ) + _add_feature_cache( + sources, + acceptance=acceptance, + materialization=materialization, + ) + + descriptors = [] + for relative, source in sorted(sources.items()): + payload = ( + _GENERATED_COMPUTE_INIT.encode() + if source is None + else source + if isinstance(source, bytes) + else source.read_bytes() + ) + descriptors.append( + { + "path": relative, + "byte_length": len(payload), + "sha256": hashlib.sha256(payload).hexdigest(), + } + ) + identity = { + "schema_version": E40_PACKAGE_SCHEMA, + "classification": ("immutable-ravnoves00-leakage-resistant-product-gate-input"), + "source_ids": expected_ids, + "profile_sha256": _sha256(profile_source), + "runtime_requirements": { + "python": "3.12", + "numpy": "1.26+", + }, + "build_requirements": {"pillow": "10+"}, + "artifact_paths": [row["path"] for row in descriptors], + "source_artifacts": descriptors, + "authority": { + "commands_enabled": False, + "navigation_or_safety_accepted": False, + }, + } + identity_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest() + package_id = f"e40-worker-package-{identity_sha256}" + output = output_root.expanduser().absolute() + output.mkdir(mode=0o700, parents=True, exist_ok=True) + destination = output / package_id + if destination.exists(): + validate_e40_worker_package(destination) + return destination + + staging = output / f".{package_id}.{uuid.uuid4().hex}.tmp" + staging.mkdir(mode=0o700, exist_ok=False) + try: + for relative, source in sources.items(): + target = staging / relative + target.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + if source is None: + target.write_text(_GENERATED_COMPUTE_INIT, encoding="utf-8") + elif isinstance(source, bytes): + target.write_bytes(source) + else: + shutil.copyfile(source, target) + artifacts = [ + { + "kind": relative, + "path": relative, + "byte_length": (staging / relative).stat().st_size, + "sha256": _sha256(staging / relative), + } + for relative in sorted(sources) + ] + manifest = { + "schema_version": E40_PACKAGE_SCHEMA, + "package_id": package_id, + "identity_sha256": identity_sha256, + "identity": identity, + "created_at_utc": datetime.now(UTC) + .isoformat(timespec="milliseconds") + .replace("+00:00", "Z"), + "artifacts": artifacts, + } + _write_json(staging / "manifest.json", manifest) + validate_e40_worker_package(staging, allow_staging=True) + os.replace(staging, destination) + except BaseException: + shutil.rmtree(staging, ignore_errors=True) + raise + validate_e40_worker_package(destination) + return destination + + +def validate_e40_worker_package( + root: Path, + *, + allow_staging: bool = False, +) -> dict[str, Any]: + """Validate package identity, exact file set, and every member digest.""" + + resolved = root.resolve(strict=True) + manifest = _read_json(resolved / "manifest.json") + identity = manifest.get("identity") + identity_sha256 = manifest.get("identity_sha256") + package_id = manifest.get("package_id") + artifacts = manifest.get("artifacts") + source_artifacts = identity.get("source_artifacts") if isinstance(identity, dict) else None + expected_name = isinstance(package_id, str) and ( + resolved.name == package_id + or ( + allow_staging + and resolved.name.startswith(f".{package_id}.") + and resolved.name.endswith(".tmp") + ) + ) + if ( + manifest.get("schema_version") != E40_PACKAGE_SCHEMA + or not isinstance(identity, dict) + or not isinstance(identity_sha256, str) + or hashlib.sha256(_canonical_json(identity)).hexdigest() != identity_sha256 + or package_id != f"e40-worker-package-{identity_sha256}" + or not expected_name + or not isinstance(artifacts, list) + or not isinstance(source_artifacts, list) + ): + raise E40WorkerPackageError("E40 worker package identity is invalid") + expected_paths = set(identity.get("artifact_paths", [])) + bound_artifacts: dict[str, tuple[int, str]] = {} + for row in source_artifacts: + if ( + not isinstance(row, dict) + or not isinstance((relative := row.get("path")), str) + or relative in bound_artifacts + or Path(relative).is_absolute() + or ".." in Path(relative).parts + or not isinstance((byte_length := row.get("byte_length")), int) + or byte_length < 0 + or not isinstance((sha256 := row.get("sha256")), str) + or len(sha256) != 64 + ): + raise E40WorkerPackageError("E40 bound source artifact is invalid") + bound_artifacts[relative] = (byte_length, sha256) + if set(bound_artifacts) != expected_paths: + raise E40WorkerPackageError("E40 bound artifact coverage changed") + actual_paths = { + path.relative_to(resolved).as_posix() for path in resolved.rglob("*") if path.is_file() + } + if ( + not expected_paths + or actual_paths != expected_paths | {"manifest.json"} + or len(artifacts) != len(expected_paths) + ): + raise E40WorkerPackageError("E40 worker package file set changed") + observed: set[str] = set() + for row in artifacts: + if not isinstance(row, dict): + raise E40WorkerPackageError("E40 worker package artifact is invalid") + relative = row.get("path") + path = resolved / str(relative) + if ( + not isinstance(relative, str) + or relative not in expected_paths + or relative in observed + or Path(relative).is_absolute() + or ".." in Path(relative).parts + or not path.is_file() + or path.is_symlink() + or row.get("kind") != relative + or bound_artifacts.get(relative) + != (row.get("byte_length"), row.get("sha256")) + or row.get("byte_length") != path.stat().st_size + or row.get("sha256") != _sha256(path) + ): + raise E40WorkerPackageError("E40 worker package artifact changed") + observed.add(relative) + if observed != expected_paths: + raise E40WorkerPackageError("E40 worker package coverage changed") + return manifest + + +def _add_materialized_evidence( + sources: dict[str, Path | bytes | None], + *, + materialization: Path, +) -> None: + rows = _read_jsonl(materialization / "materialized-items.jsonl") + if len(rows) != 486: + raise E40WorkerPackageError("E40 materialization denominator changed") + for row in rows: + for descriptor_name in ("artifact", "camera_frame"): + descriptor = row.get(descriptor_name) + if not isinstance(descriptor, dict): + raise E40WorkerPackageError("E40 materialization descriptor is invalid") + relative = descriptor.get("path") + if ( + not isinstance(relative, str) + or Path(relative).is_absolute() + or ".." in Path(relative).parts + ): + raise E40WorkerPackageError("E40 materialization path is invalid") + source = materialization / relative + if ( + not source.is_file() + or source.is_symlink() + or descriptor.get("byte_length") != source.stat().st_size + or descriptor.get("sha256") != _sha256(source) + ): + raise E40WorkerPackageError("E40 materialized evidence content changed") + target = f"input/materialization/{materialization.name}/{relative}" + existing = sources.get(target) + if existing is not None and existing != source: + raise E40WorkerPackageError("E40 package target collision") + sources[target] = source + + +def _add_feature_cache( + sources: dict[str, Path | bytes | None], + *, + acceptance: Path, + materialization: Path, +) -> None: + acceptance_rows = _read_jsonl(acceptance / "acceptance-items.jsonl") + materialization_rows = _read_jsonl(materialization / "materialized-items.jsonl") + acceptance_by_id = {str(row["item_id"]): row for row in acceptance_rows} + if len(acceptance_by_id) != 486 or {str(row["item_id"]) for row in materialization_rows} != set( + acceptance_by_id + ): + raise E40WorkerPackageError("E40 feature-cache denominator changed") + item_ids = [str(row["item_id"]) for row in materialization_rows] + features = np.asarray( + [ + _feature_vector( + acceptance_by_id[item_id], + row, + materialization, + ) + for item_id, row in zip(item_ids, materialization_rows, strict=True) + ], + dtype=np.float64, + ) + names = _feature_names() + if features.shape != (486, len(names)) or not np.isfinite(features).all(): + raise E40WorkerPackageError("E40 feature-cache matrix is invalid") + arrays_stream = io.BytesIO() + np.savez_compressed( + arrays_stream, + item_ids=np.asarray(item_ids, dtype=f" bytes: + return json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode() + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + while chunk := stream.read(1024 * 1024): + digest.update(chunk) + return digest.hexdigest() + + +def _read_json(path: Path) -> dict[str, Any]: + value = json.loads(path.read_text(encoding="utf-8-sig")) + if not isinstance(value, dict): + raise E40WorkerPackageError(f"JSON object expected: {path.name}") + return value + + +def _read_jsonl(path: Path) -> list[dict[str, Any]]: + rows = [json.loads(line) for line in path.read_text(encoding="utf-8-sig").splitlines()] + if not all(isinstance(row, dict) for row in rows): + raise E40WorkerPackageError(f"JSONL object expected: {path.name}") + return rows + + +def _write_json(path: Path, value: object) -> None: + with path.open("x", encoding="utf-8", newline="\n") as stream: + json.dump(value, stream, indent=2, sort_keys=True) + stream.write("\n") + stream.flush() + os.fsync(stream.fileno()) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--repository-root", type=Path, required=True) + parser.add_argument("--acceptance", type=Path, required=True) + parser.add_argument("--materialization", type=Path, required=True) + parser.add_argument("--profile", type=Path, required=True) + parser.add_argument("--output-root", type=Path, required=True) + args = parser.parse_args() + package = build_e40_worker_package( + repository_root=args.repository_root, + acceptance_root=args.acceptance, + materialization_root=args.materialization, + profile_path=args.profile, + output_root=args.output_root, + ) + print(package) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/experiments/perception/run_e40_perception_product_gate.py b/experiments/perception/run_e40_perception_product_gate.py new file mode 100644 index 0000000..22cda66 --- /dev/null +++ b/experiments/perception/run_e40_perception_product_gate.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +"""Build the development-qualified RAVNOVES00 E40 product gate.""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path + +from k1link.compute.e40_perception_product_gate import ( + build_e40_perception_product_gate, +) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--acceptance", type=Path, required=True) + parser.add_argument("--materialization", type=Path, required=True) + parser.add_argument("--profile", type=Path, required=True) + parser.add_argument("--output-root", type=Path, required=True) + parser.add_argument("--worker-node", default=os.environ.get("COMPUTERNAME")) + args = parser.parse_args() + result = build_e40_perception_product_gate( + acceptance_root=args.acceptance, + materialization_root=args.materialization, + profile_path=args.profile, + output_root=args.output_root, + worker_node=args.worker_node, + ) + print( + json.dumps( + { + "result_id": result.result_id, + "result_root": str(result.result_root), + "quality_gate_passed": result.quality_gate_passed, + "development_cross_validation": result.report["development_cross_validation"], + "metrics": result.report["metrics"], + "blocking_checks": result.report["quality_gate"]["blocking_checks"], + }, + ensure_ascii=False, + sort_keys=True, + ) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/experiments/perception/run_e41_evaluation_boundary.py b/experiments/perception/run_e41_evaluation_boundary.py new file mode 100644 index 0000000..35a3b79 --- /dev/null +++ b/experiments/perception/run_e41_evaluation_boundary.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 +"""Prepare/run the E41 truth-free predictor and visible evaluator.""" + +from __future__ import annotations + +import argparse +import contextlib +import json +import platform +import uuid +from pathlib import Path +from typing import Any + +from k1link.compute.e41_evaluation_boundary import ( + E41_FEATURE_MANIFEST_NAME, + build_e41_predictor_package, + build_e41_visible_evaluation, + run_e41_predictor, +) +from k1link.compute.pipeline_telemetry import ( + JsonlPipelineTelemetrySink, + PipelineStageOutcome, + PipelineTelemetryEmitter, + PipelineTelemetryIdentity, +) + + +def main() -> int: + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers(dest="command", required=True) + + package = subparsers.add_parser("package") + package.add_argument("--materialization", type=Path, required=True) + package.add_argument("--e40-package", type=Path, required=True) + package.add_argument("--e40-model", type=Path, required=True) + package.add_argument("--output-root", type=Path, required=True) + _add_telemetry_arguments(package) + + predict = subparsers.add_parser("predict") + predict.add_argument("--package", type=Path, required=True) + predict.add_argument("--output-root", type=Path, required=True) + predict.add_argument("--environment-lock", required=True) + predict.add_argument("--python-version", required=True) + predict.add_argument("--numpy-version", required=True) + _add_telemetry_arguments(predict) + + evaluate = subparsers.add_parser("evaluate") + evaluate.add_argument("--prediction", type=Path, required=True) + evaluate.add_argument("--acceptance", type=Path, required=True) + evaluate.add_argument("--output-root", type=Path, required=True) + _add_telemetry_arguments(evaluate) + + args = parser.parse_args() + if args.command == "package": + emitter = _telemetry_emitter( + args, + source_package_id=args.e40_package.name, + method_id="e41-truth-free-package/v1", + ) + with _stage(emitter, "package") as outcome: + result = build_e41_predictor_package( + materialization_root=args.materialization, + e40_package_root=args.e40_package, + e40_model_path=args.e40_model, + output_root=args.output_root, + ) + feature_manifest = json.loads( + (result / E41_FEATURE_MANIFEST_NAME).read_text(encoding="utf-8") + ) + outcome.output_count = int(feature_manifest["item_count"]) + payload = {"predictor_package": str(result)} + elif args.command == "predict": + emitter = _telemetry_emitter( + args, + source_package_id=args.package.name, + method_id="frozen-e40-predictor/v1", + ) + with _stage(emitter, "predict") as outcome: + prediction = run_e41_predictor( + package_root=args.package, + output_root=args.output_root, + runtime_identity={ + "environment_lock": args.environment_lock, + "python": args.python_version, + "numpy": args.numpy_version, + }, + ) + outcome.output_count = int(prediction.manifest["item_count"]) + payload = { + "prediction_result_id": prediction.result_id, + "prediction_root": str(prediction.result_root), + } + else: + emitter = _telemetry_emitter( + args, + source_package_id=args.prediction.name, + method_id="visible-engineering-contract/v1", + ) + with _stage(emitter, "evaluate") as outcome: + evaluation = build_e41_visible_evaluation( + prediction_root=args.prediction, + acceptance_root=args.acceptance, + output_root=args.output_root, + ) + outcome.output_count = int( + evaluation.evaluation["evaluation"]["metrics"]["validation_items"] + ) + payload = { + "evaluation_result_id": evaluation.result_id, + "evaluation_root": str(evaluation.result_root), + "metrics": evaluation.evaluation["evaluation"]["metrics"], + "blocking_checks": evaluation.evaluation["evaluation"]["blocking_checks"], + } + print(json.dumps(payload, ensure_ascii=False, sort_keys=True)) + return 0 + + +def _add_telemetry_arguments(parser: argparse.ArgumentParser) -> None: + parser.add_argument( + "--telemetry-jsonl", + type=Path, + help="append native pipeline events to this local JSONL evidence file", + ) + parser.add_argument("--telemetry-contour-id", default="local-compute") + parser.add_argument("--telemetry-agent-id", default="mission-core-runner") + parser.add_argument("--telemetry-node-id", default=platform.node() or "unknown-node") + parser.add_argument("--telemetry-run-id") + parser.add_argument("--telemetry-request-id") + parser.add_argument("--telemetry-source-id", default="ravnoves00") + + +def _telemetry_emitter( + args: argparse.Namespace, + *, + source_package_id: str, + method_id: str, +) -> PipelineTelemetryEmitter | None: + if args.telemetry_jsonl is None: + return None + identity = PipelineTelemetryIdentity( + contour_id=str(args.telemetry_contour_id), + agent_id=str(args.telemetry_agent_id), + node_id=str(args.telemetry_node_id), + lab_id="E41", + run_id=str(args.telemetry_run_id or f"e41-{uuid.uuid4().hex}"), + request_id=( + str(args.telemetry_request_id) + if args.telemetry_request_id is not None + else None + ), + source_id=str(args.telemetry_source_id), + source_package_id=source_package_id, + method_id=method_id, + ) + return PipelineTelemetryEmitter( + identity=identity, + sink=JsonlPipelineTelemetrySink(args.telemetry_jsonl), + ) + + +def _stage( + emitter: PipelineTelemetryEmitter | None, + stage_id: str, +) -> Any: + if emitter is None: + return contextlib.nullcontext(PipelineStageOutcome()) + return emitter.stage(stage_id) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/experiments/perception/run_e41_methodology_audit.py b/experiments/perception/run_e41_methodology_audit.py new file mode 100644 index 0000000..a034b1b --- /dev/null +++ b/experiments/perception/run_e41_methodology_audit.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +"""Build the immutable RAVNOVES00 E41 methodology audit.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +from k1link.compute.e41_methodology_audit import build_e41_methodology_audit + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--acceptance", type=Path, required=True) + parser.add_argument("--materialization", type=Path, required=True) + parser.add_argument("--e40-package", type=Path, required=True) + parser.add_argument("--e40-result", type=Path, required=True) + parser.add_argument("--profile", type=Path, required=True) + parser.add_argument("--output-root", type=Path, required=True) + args = parser.parse_args() + result = build_e41_methodology_audit( + acceptance_root=args.acceptance, + materialization_root=args.materialization, + e40_package_root=args.e40_package, + e40_result_root=args.e40_result, + profile_path=args.profile, + output_root=args.output_root, + ) + print( + json.dumps( + { + "result_id": result.result_id, + "result_root": str(result.result_root), + "blind_gate_eligible": result.blind_gate_eligible, + "violations": result.report["analysis"]["policy"]["violations"], + }, + ensure_ascii=False, + sort_keys=True, + ) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/experiments/perception/run_e42_metamorphic_suite.py b/experiments/perception/run_e42_metamorphic_suite.py new file mode 100644 index 0000000..661f5c0 --- /dev/null +++ b/experiments/perception/run_e42_metamorphic_suite.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +"""Run the bounded E42 predictor and PointSlab metamorphic suite.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +from k1link.compute.e42_metamorphic_suite import build_e42_metamorphic_suite + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--predictor-package", type=Path, required=True) + parser.add_argument("--e32-result", type=Path, required=True) + parser.add_argument("--output-root", type=Path, required=True) + args = parser.parse_args() + result = build_e42_metamorphic_suite( + predictor_package_root=args.predictor_package, + e32_result_root=args.e32_result, + output_root=args.output_root, + ) + print( + json.dumps( + { + "result_id": result.result_id, + "result_root": str(result.result_root), + "accepted": result.accepted, + "checks": result.report["acceptance"]["checks"], + }, + ensure_ascii=False, + sort_keys=True, + ) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/experiments/perception/run_e43_future_capture_protocol.py b/experiments/perception/run_e43_future_capture_protocol.py new file mode 100644 index 0000000..3924a97 --- /dev/null +++ b/experiments/perception/run_e43_future_capture_protocol.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +"""Freeze the future same-K1/new-route capture protocol.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +from k1link.compute.e43_future_capture_protocol import ( + build_e43_future_capture_protocol, +) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--profile", type=Path, required=True) + parser.add_argument("--output-root", type=Path, required=True) + args = parser.parse_args() + result = build_e43_future_capture_protocol( + profile_path=args.profile, + output_root=args.output_root, + ) + print( + json.dumps( + { + "result_id": result.result_id, + "result_root": str(result.result_root), + "capture_exists": result.protocol["capture_exists"], + "labels_exist": result.protocol["labels_exist"], + }, + ensure_ascii=False, + sort_keys=True, + ) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/experiments/perception/run_e44_data_amplification_audit.py b/experiments/perception/run_e44_data_amplification_audit.py new file mode 100644 index 0000000..1047bf9 --- /dev/null +++ b/experiments/perception/run_e44_data_amplification_audit.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +"""Measure exact data amplification across explicit immutable LAB roots.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +from k1link.compute.e44_data_amplification_audit import ( + build_e44_data_amplification_audit, +) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument( + "--artifact-root", + action="append", + required=True, + metavar="LABEL=PATH", + ) + parser.add_argument("--output-root", type=Path, required=True) + args = parser.parse_args() + roots: dict[str, Path] = {} + for value in args.artifact_root: + label, separator, raw_path = value.partition("=") + if not separator or label in roots: + parser.error("--artifact-root must be a unique LABEL=PATH") + roots[label] = Path(raw_path) + result = build_e44_data_amplification_audit( + artifact_roots=roots, + output_root=args.output_root, + ) + print( + json.dumps( + { + "result_id": result.result_id, + "result_root": str(result.result_root), + "analysis": result.report["analysis"], + }, + ensure_ascii=False, + sort_keys=True, + ) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/experiments/perception/worker/Invoke-E40PerceptionProductGate.ps1 b/experiments/perception/worker/Invoke-E40PerceptionProductGate.ps1 new file mode 100644 index 0000000..501aa66 --- /dev/null +++ b/experiments/perception/worker/Invoke-E40PerceptionProductGate.ps1 @@ -0,0 +1,161 @@ +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] + [string]$PackageRoot, + [string]$OutputRoot = "D:\NDC_MISSIONCORE\runtime\derived\e40-product-gate", + [string]$ContainerImage = "nvcr.io/nvidia/tritonserver:26.06-py3@sha256:58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794", + [ValidateRange(1, 1000)] + [int]$FreeGiBFloor = 300 +) + +$ErrorActionPreference = "Stop" +$ProgressPreference = "SilentlyContinue" + +function Assert-LastExitCode([string]$Operation) { + if ($LASTEXITCODE -ne 0) { + throw "$Operation failed with exit code $LASTEXITCODE" + } +} + +function Resolve-DDirectory([string]$Path, [string]$Label) { + $item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force + $root = [IO.Path]::GetPathRoot($item.FullName).TrimEnd("\") + if ( + -not $item.PSIsContainer -or + ($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -or + $root -ine "D:" + ) { + throw "$Label must be a real D: directory" + } + return $item.FullName +} + +function Convert-ToDockerPath([string]$Path) { + return $Path.Replace("\", "/") +} + +function Assert-FreeSpace([string]$Phase) { + $free = [int64](Get-PSDrive -Name D).Free + $floor = [int64]$FreeGiBFloor * 1GB + Write-Host ( + "DISK_GUARD PHASE={0} DRIVE=D FREE_BYTES={1} FREE_GIB={2} FLOOR_GIB={3}" -f + $Phase, $free, [math]::Round($free / 1GB, 3), $FreeGiBFloor + ) + if ($free -lt ($floor + 1GB)) { + throw "D: lacks the guarded E40 reserve during $Phase" + } + return $free +} + +$package = Resolve-DDirectory $PackageRoot "E40 package" +$packageManifestPath = Join-Path $package "manifest.json" +if (-not (Test-Path -LiteralPath $packageManifestPath -PathType Leaf)) { + throw "E40 package manifest is missing" +} +$packageManifest = Get-Content -LiteralPath $packageManifestPath -Raw | + ConvertFrom-Json +if ( + $packageManifest.schema_version -ne "missioncore.e40-worker-package/v1" -or + $packageManifest.package_id -ne (Split-Path $package -Leaf) -or + $packageManifest.package_id -notmatch "^e40-worker-package-[a-f0-9]{64}$" +) { + throw "E40 package manifest is incompatible" +} + +if (-not (Test-Path -LiteralPath $OutputRoot)) { + $null = New-Item -ItemType Directory -Path $OutputRoot +} +$output = Resolve-DDirectory $OutputRoot "E40 output root" +$freeBefore = Assert-FreeSpace "preflight" + +& docker image inspect $ContainerImage *> $null +Assert-LastExitCode "Pinned E40 container image inspection" + +$dockerPackage = Convert-ToDockerPath $package +$dockerOutput = Convert-ToDockerPath $output +$packageName = Split-Path $package -Leaf +$containerPackage = "/opt/e40-input/$packageName" +$packageValidator = ( + "{0}/runtime/validate_e40_worker_package.py" -f $containerPackage +) +$validationCommand = @( + "run", "--rm", + "--name", "ndc-mission-core-e40-package-validation", + "--network", "none", + "--read-only", + "--security-opt", "no-new-privileges:true", + "--cap-drop", "ALL", + "--pids-limit", "32", + "--memory", "128m", + "--memory-swap", "128m", + "--cpus", "1", + "-v", ("{0}:{1}:ro" -f $dockerPackage, $containerPackage), + "--entrypoint", "python3", + $ContainerImage, + $packageValidator, + $containerPackage +) +& docker @validationCommand +Assert-LastExitCode "Independent E40 package integrity verification" + +$command = @( + "run", "--rm", + "--name", "ndc-mission-core-e40-product-gate", + "--network", "none", + "--read-only", + "--security-opt", "no-new-privileges:true", + "--cap-drop", "ALL", + "--pids-limit", "128", + "--memory", "1g", + "--memory-swap", "1g", + "--cpus", "2", + "--tmpfs", "/tmp:rw,noexec,nosuid,size=64m", + "-e", "PYTHONDONTWRITEBYTECODE=1", + "-e", ("PYTHONPATH={0}/runtime" -f $containerPackage), + "-e", ("E40_WORKER_NODE={0}" -f $env:COMPUTERNAME), + "-v", ("{0}:{1}:ro" -f $dockerPackage, $containerPackage), + "-v", ("{0}:/output:rw" -f $dockerOutput), + "--entrypoint", "python3", + $ContainerImage, + ("{0}/runtime/run_e40_perception_product_gate.py" -f $containerPackage), + "--package", $containerPackage, + "--output-root", "/output" +) + +Write-Output ("PACKAGE_ID={0}" -f $packageManifest.package_id) +Write-Output ("PACKAGE_IDENTITY_SHA256={0}" -f $packageManifest.identity_sha256) +Write-Output ("CONTAINER_IMAGE={0}" -f $ContainerImage) +& docker @command +Assert-LastExitCode "E40 perception product gate" + +$matches = @( + Get-ChildItem -LiteralPath $output -Directory -Filter "e40-perception-product-gate-*" | + Where-Object { + $manifestPath = Join-Path $_.FullName "manifest.json" + if (-not (Test-Path -LiteralPath $manifestPath -PathType Leaf)) { + return $false + } + $manifest = Get-Content -LiteralPath $manifestPath -Raw | + ConvertFrom-Json + return ( + $manifest.schema_version -eq + "missioncore.e40-perception-product-gate/v1" -and + $manifest.acceptance_state -eq + "completed-leakage-resistant-product-gate" -and + $manifest.identity.execution.worker_node -eq $env:COMPUTERNAME + ) + } +) +if ($matches.Count -ne 1) { + throw "E40 immutable result could not be resolved uniquely" +} +$resultRoot = $matches[0].FullName +$resultManifest = Get-Content -LiteralPath ( + Join-Path $resultRoot "manifest.json" +) -Raw | ConvertFrom-Json +$freeAfter = Assert-FreeSpace "completed" +Write-Output ("RESULT_ROOT={0}" -f $resultRoot) +Write-Output ("RESULT_ID={0}" -f $resultManifest.result_id) +Write-Output ("QUALITY_GATE_PASSED={0}" -f $resultManifest.quality_gate_passed) +Write-Output ("DISK_FREE_BYTES_BEFORE={0}" -f $freeBefore) +Write-Output ("DISK_FREE_BYTES_AFTER={0}" -f $freeAfter) diff --git a/experiments/perception/worker/run_e40_perception_product_gate.py b/experiments/perception/worker/run_e40_perception_product_gate.py new file mode 100644 index 0000000..dea876d --- /dev/null +++ b/experiments/perception/worker/run_e40_perception_product_gate.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +"""Execute the packaged E40 product gate in the pinned Worker 006 container.""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path + +from k1link.compute.e40_perception_product_gate import ( + build_e40_perception_product_gate, +) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--package", type=Path, required=True) + parser.add_argument("--output-root", type=Path, required=True) + args = parser.parse_args() + package = args.package.resolve(strict=True) + package_manifest = json.loads( + (package / "manifest.json").read_text(encoding="utf-8-sig") + ) + profile = json.loads((package / "profile.json").read_text(encoding="utf-8")) + acceptance_id = profile["source"]["acceptance_result_id"] + materialization_id = profile["source"]["materialization_id"] + result = build_e40_perception_product_gate( + acceptance_root=package / "input" / "acceptance" / acceptance_id, + materialization_root=(package / "input" / "materialization" / materialization_id), + profile_path=package / "profile.json", + output_root=args.output_root, + worker_node=os.environ.get("E40_WORKER_NODE"), + execution_package={ + "mode": "verified-worker-package", + "package_id": package_manifest["package_id"], + "identity_sha256": package_manifest["identity_sha256"], + }, + ) + print( + json.dumps( + { + "result_id": result.result_id, + "result_root": str(result.result_root), + "quality_gate_passed": result.quality_gate_passed, + "metrics": result.report["metrics"], + "blocking_checks": result.report["quality_gate"]["blocking_checks"], + }, + ensure_ascii=False, + sort_keys=True, + ) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/experiments/perception/worker/validate_e40_worker_package.py b/experiments/perception/worker/validate_e40_worker_package.py new file mode 100644 index 0000000..882d915 --- /dev/null +++ b/experiments/perception/worker/validate_e40_worker_package.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +"""Independently validate an E40 package before importing package code.""" + +from __future__ import annotations + +import hashlib +import json +import pathlib +import sys + + +def _descriptors( + rows: object, + *, + require_kind: bool, +) -> dict[str, tuple[int, str]]: + if not isinstance(rows, list): + raise SystemExit("E40 package artifact catalog is missing") + result: dict[str, tuple[int, str]] = {} + for row in rows: + if not isinstance(row, dict): + raise SystemExit("E40 package artifact descriptor is invalid") + relative = row.get("path") + byte_length = row.get("byte_length") + sha256 = row.get("sha256") + if ( + not isinstance(relative, str) + or relative in result + or pathlib.PurePosixPath(relative).is_absolute() + or ".." in pathlib.PurePosixPath(relative).parts + or not isinstance(byte_length, int) + or byte_length < 0 + or not isinstance(sha256, str) + or len(sha256) != 64 + or (require_kind and row.get("kind") != relative) + ): + raise SystemExit("E40 package artifact descriptor is invalid") + result[relative] = (byte_length, sha256) + return result + + +def validate(root_argument: str) -> None: + root = pathlib.Path(root_argument).resolve(strict=True) + manifest = json.loads( + (root / "manifest.json").read_text(encoding="utf-8-sig") + ) + identity = manifest.get("identity") + identity_sha256 = manifest.get("identity_sha256") + package_id = manifest.get("package_id") + if ( + manifest.get("schema_version") != "missioncore.e40-worker-package/v1" + or not isinstance(identity, dict) + or not isinstance(identity_sha256, str) + or hashlib.sha256( + json.dumps( + identity, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode() + ).hexdigest() + != identity_sha256 + or package_id != f"e40-worker-package-{identity_sha256}" + or root.name != package_id + ): + raise SystemExit("E40 package identity verification failed") + + bound = _descriptors( + identity.get("source_artifacts"), + require_kind=False, + ) + catalog = _descriptors(manifest.get("artifacts"), require_kind=True) + if bound != catalog or set(identity.get("artifact_paths", [])) != set(bound): + raise SystemExit("E40 package artifact binding verification failed") + actual = { + path.relative_to(root).as_posix() + for path in root.rglob("*") + if path.is_file() + } + if actual != set(bound) | {"manifest.json"}: + raise SystemExit("E40 package file set verification failed") + for relative, (byte_length, sha256) in bound.items(): + path = root / relative + payload = path.read_bytes() + if ( + path.is_symlink() + or len(payload) != byte_length + or hashlib.sha256(payload).hexdigest() != sha256 + ): + raise SystemExit( + f"E40 package member verification failed: {relative}" + ) + + +def main() -> int: + if len(sys.argv) != 2: + raise SystemExit("usage: validate_e40_worker_package.py PACKAGE_ROOT") + validate(sys.argv[1]) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/k1link/compute/e30_materialization.py b/src/k1link/compute/e30_materialization.py index c122491..5cd30e8 100644 --- a/src/k1link/compute/e30_materialization.py +++ b/src/k1link/compute/e30_materialization.py @@ -254,7 +254,7 @@ def build_e30_materialization( source=source, ) - identity = { + identity: dict[str, object] = { "schema_version": E30_MATERIALIZATION_SCHEMA, "review_pack": { "result_id": _required_string(review.manifest, "result_id"), @@ -635,7 +635,14 @@ def _engineering_triage( ) -> dict[str, object]: """Route evidence without pretending that a rule is a semantic verdict.""" - selected_count = int(metadata["selected_point_count"]) + selected_count_value = metadata["selected_point_count"] + if ( + not isinstance(selected_count_value, int) + or isinstance(selected_count_value, bool) + or selected_count_value < 0 + ): + raise E30MaterializationError("selected point count is invalid") + selected_count = selected_count_value detector_score = metadata.get("detector_score") stratum = _required_string(item, "stratum") locator = _required_object(item, "e29_locator") diff --git a/src/k1link/compute/e32_track_geometry_replay.py b/src/k1link/compute/e32_track_geometry_replay.py index 9efcc72..ac4bb5a 100644 --- a/src/k1link/compute/e32_track_geometry_replay.py +++ b/src/k1link/compute/e32_track_geometry_replay.py @@ -44,7 +44,6 @@ from .e32_track_geometry_storage import ( E32_POINT_SLAB_REFERENCE_SCHEMA, E32_POINTS_NAME, E32_SOURCE_INDICES_NAME, - E32_TRACK_GEOMETRY_RECORD_SCHEMA, E32TrackGeometryStorageError, frame_from_record, load_point_storage, @@ -52,6 +51,9 @@ from .e32_track_geometry_storage import ( validate_storage, write_point_storage, ) +from .e32_track_geometry_storage import ( + E32_TRACK_GEOMETRY_RECORD_SCHEMA as E32_TRACK_GEOMETRY_RECORD_SCHEMA, +) from .lidar_field_review import E10LidarFieldSource from .lidar_local_surface import K1LocalSurfaceV1 from .semantic_geometry_fusion import ( diff --git a/src/k1link/compute/e34_temporal_occupied_replay.py b/src/k1link/compute/e34_temporal_occupied_replay.py index 8bbedb3..2301f02 100644 --- a/src/k1link/compute/e34_temporal_occupied_replay.py +++ b/src/k1link/compute/e34_temporal_occupied_replay.py @@ -790,24 +790,45 @@ def _finalize_report( == occupancy.get("e34_consumed_current_point_rows") ), "active_component_bound": ( - components.get("peak_active") - <= layer.get("maximum_active_components") + _nonnegative_int(components.get("peak_active"), "E34 peak active components") + <= _positive_int( + layer.get("maximum_active_components"), + "E34 maximum active components", + ) ), "component_cell_bound": ( - occupancy.get("peak_cells_per_component") - <= layer.get("maximum_cells_per_component") + _nonnegative_int( + occupancy.get("peak_cells_per_component"), + "E34 peak cells per component", + ) + <= _positive_int( + layer.get("maximum_cells_per_component"), + "E34 maximum cells per component", + ) ), "held_age_within_ttl": ( - aging.get("maximum_held_age_seconds") + _nonnegative_float( + aging.get("maximum_held_age_seconds"), + "E34 maximum held age", + ) <= float(layer["occupied_ttl_seconds"]) + 1e-9 ), "expiry_delay_within_gate": ( - aging.get("maximum_expiry_delay_seconds") + _nonnegative_float( + aging.get("maximum_expiry_delay_seconds"), + "E34 maximum expiry delay", + ) <= float(acceptance["maximum_expiry_delay_seconds"]) + 1e-9 ), "map_frame_jump_candidates_within_gate": ( - map_frame.get("jump_candidates") - <= acceptance.get("maximum_map_frame_jump_candidates") + _nonnegative_int( + map_frame.get("jump_candidates"), + "E34 map-frame jump candidates", + ) + <= _nonnegative_int( + acceptance.get("maximum_map_frame_jump_candidates"), + "E34 maximum map-frame jump candidates", + ) ), "upstream_artifacts_unchanged": upstream_unchanged, "no_free_space_publication": occupancy.get("free_cell_rows") == 0, @@ -1010,6 +1031,17 @@ def _positive_float(value: object, label: str) -> float: return float(value) +def _nonnegative_float(value: object, label: str) -> float: + if ( + not isinstance(value, (int, float)) + or isinstance(value, bool) + or not math.isfinite(float(value)) + or float(value) < 0.0 + ): + raise E34TemporalOccupiedReplayError(f"{label} is invalid") + return float(value) + + def _positive_int(value: object, label: str) -> int: result = _nonnegative_int(value, label) if result == 0: diff --git a/src/k1link/compute/e37_acceptance_contract.py b/src/k1link/compute/e37_acceptance_contract.py index 93625cb..ab63798 100644 --- a/src/k1link/compute/e37_acceptance_contract.py +++ b/src/k1link/compute/e37_acceptance_contract.py @@ -142,6 +142,9 @@ def assign_split( range_bucket, )): raise E37AcceptanceContractError("E37 split source row is invalid") + assert isinstance(item_id, str) + assert isinstance(stratum, str) + assert isinstance(range_bucket, str) grouped[(stratum, range_bucket)].append(row) assignments: dict[str, SplitName] = {} diff --git a/src/k1link/compute/e39_perception_refinement.py b/src/k1link/compute/e39_perception_refinement.py index 54019c6..e42d716 100644 --- a/src/k1link/compute/e39_perception_refinement.py +++ b/src/k1link/compute/e39_perception_refinement.py @@ -1,9 +1,11 @@ -"""Development-qualified RAVNOVES00 R1 perception refinement. +"""Historical RAVNOVES00 R1 perception refinement. -E39 keeps the E37 denominator and validation split immutable. It enriches the +E39 keeps the nominal E37 denominator and split immutable. It enriches the E38 tabular baseline with source-scoped camera and LiDAR shape features, chooses the fixed model contract through development-only cross-validation, fits only -on development labels, and evaluates the sealed validation partition once. +on development labels, and evaluates the then-designated validation partition. +E41 later proved that this partition is visible engineering evidence with +connected group overlap, not an independent sealed accuracy gate. The result is diagnostic. It grants no navigation, command, or safety authority and makes no claim about another route, rig, camera, or mount. @@ -92,7 +94,7 @@ def build_e39_perception_refinement( output_root: Path, worker_node: str | None = None, ) -> E39PerceptionRefinement: - """Fit the frozen E39 contract and evaluate the sealed validation split.""" + """Fit E39 and evaluate the historical visible E37 validation slice.""" profile_file = profile_path.resolve(strict=True) profile = _read_json(profile_file) @@ -814,7 +816,10 @@ def _transform( scale: np.ndarray[Any, Any], clip: float, ) -> np.ndarray[Any, Any]: - return np.clip((matrix - median) / scale, -clip, clip) + return np.asarray( + np.clip((matrix - median) / scale, -clip, clip), + dtype=np.float64, + ) def _predict_presence( diff --git a/src/k1link/compute/e40_perception_product_gate.py b/src/k1link/compute/e40_perception_product_gate.py new file mode 100644 index 0000000..8fe3f50 --- /dev/null +++ b/src/k1link/compute/e40_perception_product_gate.py @@ -0,0 +1,1464 @@ +"""Historical RAVNOVES00 source-scoped perception engineering gate. + +E40 keeps the nominal E37 denominator and split immutable. It removes route +coordinates, source frame indices, review ordinals and track identifiers from +the model input. E41 later proved that connected frame/track/time groups still +crossed the split and that the acceptance material was co-located with the +predictor package. E40 is therefore retained as a visible source-scoped +engineering evaluation, not a leakage-free blind accuracy gate. + +The frozen classifier is deliberately hierarchical. Immutable evidence strata +with a conservative product meaning use explicit policy states; only the +camera-only stratum is classified from detector/support metadata and the exact +camera crop. Validation is evaluated once after both development protocols +pass. + +The result is diagnostic. It grants no navigation, command, or safety +authority and makes no claim about another route, rig, camera, or mount. +""" + +from __future__ import annotations + +import hashlib +import json +import math +import os +import shutil +import uuid +from collections import Counter, defaultdict +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path +from typing import Any, Final + +import numpy as np + +from k1link.compute.e37_acceptance_contract import ( + E37_ITEMS_NAME, + E37AcceptanceContractError, + read_e37_acceptance_contract, +) + +E40_PROFILE_SCHEMA: Final = "missioncore.e40-perception-product-gate-profile/v1" +E40_PACKAGE_SCHEMA: Final = "missioncore.e40-worker-package/v1" +E40_RESULT_SCHEMA: Final = "missioncore.e40-perception-product-gate/v1" +E40_PREDICTION_SCHEMA: Final = "missioncore.e40-perception-prediction/v1" +E40_MODEL_SCHEMA: Final = "missioncore.e40-development-product-model/v1" +E40_REPORT_SCHEMA: Final = "missioncore.e40-perception-product-gate-report/v1" + +E40_PREDICTIONS_NAME: Final = "predictions.jsonl" +E40_MODEL_NAME: Final = "development-model.json" +E40_REPORT_NAME: Final = "run-report.json" +E40_MANIFEST_NAME: Final = "manifest.json" + +_MATERIALIZATION_SCHEMA: Final = "missioncore.e30-evidence-materialization/v2" +_FEATURE_CACHE_SCHEMA: Final = "missioncore.e40-feature-cache/v1" +_FEATURE_CACHE_MANIFEST: Final = "e40-feature-cache.json" +_FEATURE_CACHE_ARRAYS: Final = "e40-feature-vectors.npz" +_SOURCE_SESSION_ID: Final = "20260720T065719Z_viewer_live" +_SOURCE_DISPLAY_NAME: Final = "RAVNOVES00" +_DIMENSIONS: Final = ("presence", "geometry_association", "freshness") +_STRATA: Final = ("agree", "camera-only", "conflict", "geometry-only", "unknown") +_RANGES: Final = ("near", "middle", "far", "unavailable") +_GEOMETRIES: Final = ( + "agree", + "conflict", + "single-source-camera", + "single-source-geometry", + "unknown", + "unavailable", +) +_LABELS: Final = ( + "background-or-noise", + "object-present", + "occupied-environment", +) +_CATEGORIES: Final = { + "stratum": _STRATA, + "range": _RANGES, + "geometry": _GEOMETRIES, + "label": ("car", "person", "truck", "bicycle", "motorcycle", "bus", "none"), + "reason": ( + "camera-semantic-without-qualified-occupied-lidar-support", + "camera-semantic-with-connected-occupied-lidar-support", + "semantic-observation-not-current", + "camera-object-region-observed-as-local-surface", + "none", + ), + "association": ("vehicle", "person", "bicycle", "motorcycle", "none"), + "motion": ("unknown", "static", "dynamic", "none"), + "camera_motion": ("unknown", "static", "dynamic", "none"), + "semantic_current": ("true", "false", "none"), +} +_FIXED_PRESENCE: Final = { + "agree": "object-present", + "conflict": "background-or-noise", + "geometry-only": "occupied-environment", + "unknown": "object-present", +} +_AUTHORITY: Final = { + "commands_enabled": False, + "navigation_or_safety_accepted": False, +} + + +class E40PerceptionProductGateError(RuntimeError): + """The E40 profile, immutable inputs, model, or result is invalid.""" + + +@dataclass(frozen=True, slots=True) +class E40PerceptionProductGate: + result_id: str + result_root: Path + manifest: dict[str, Any] + report: dict[str, Any] + model: dict[str, Any] + + @property + def quality_gate_passed(self) -> bool: + return self.report.get("quality_gate", {}).get("passed") is True + + +def build_e40_perception_product_gate( + *, + acceptance_root: Path, + materialization_root: Path, + profile_path: Path, + output_root: Path, + worker_node: str | None = None, + execution_package: dict[str, str] | None = None, +) -> E40PerceptionProductGate: + """Fit E40 and evaluate the historical visible E37 validation slice.""" + + profile_file = profile_path.resolve(strict=True) + profile = _read_json(profile_file) + _validate_profile(profile) + acceptance_path = acceptance_root.resolve(strict=True) + materialization_path = materialization_root.resolve(strict=True) + try: + acceptance = read_e37_acceptance_contract(acceptance_path) + except E37AcceptanceContractError as exc: + raise E40PerceptionProductGateError("E40 E37 contract is invalid") from exc + if acceptance.result_id != profile["source"]["acceptance_result_id"]: + raise E40PerceptionProductGateError("E40 acceptance identity changed") + + acceptance_items_path = acceptance_path / E37_ITEMS_NAME + acceptance_rows = _read_jsonl(acceptance_items_path) + materialization_rows, materialization_binding = _load_materialization( + materialization_path, + acceptance.manifest, + profile, + ) + feature_cache, feature_cache_binding = _load_feature_cache( + materialization_path, + materialization_binding, + ) + if ( + len(acceptance_rows) != 486 + or len(materialization_rows) != 486 + or {row.get("item_id") for row in acceptance_rows} + != {row.get("item_id") for row in materialization_rows} + or ( + feature_cache is not None + and set(feature_cache) != {str(row.get("item_id")) for row in materialization_rows} + ) + ): + raise E40PerceptionProductGateError("E40 denominator accounting differs") + + package_binding = _execution_package_binding(execution_package) + identity_base = { + "schema_version": E40_RESULT_SCHEMA, + "source": { + "session_id": _SOURCE_SESSION_ID, + "display_name": _SOURCE_DISPLAY_NAME, + "acceptance_result_id": acceptance.result_id, + "acceptance_identity_sha256": acceptance.manifest["identity_sha256"], + "acceptance_items_sha256": _sha256(acceptance_items_path), + **materialization_binding, + "feature_projection": feature_cache_binding, + }, + "profile": { + "profile_id": profile["profile_id"], + "sha256": _sha256(profile_file), + }, + "execution": { + "class": "development-qualified-sealed-validation-evaluation", + "worker_node": worker_node or "unbound-local", + "package": package_binding, + }, + "authority": _AUTHORITY, + } + + materialization_by_id = {str(row["item_id"]): row for row in materialization_rows} + joined: list[dict[str, Any]] = [] + for acceptance_row in acceptance_rows: + item_id = str(acceptance_row.get("item_id")) + materialization_row = materialization_by_id[item_id] + reference = acceptance_row.get("reference") + if ( + acceptance_row.get("split") not in {"development", "validation"} + or acceptance_row.get("severity") not in {"standard", "medium", "high"} + or not isinstance(reference, dict) + or set(reference) != set(_DIMENSIONS) + ): + raise E40PerceptionProductGateError("E40 acceptance row is invalid") + joined.append( + { + "acceptance": acceptance_row, + "features": _feature_vector( + acceptance_row, + materialization_row, + materialization_path, + ) + if feature_cache is None + else feature_cache[item_id], + "scene_group": _scene_group( + acceptance_row, + materialization_row, + ), + } + ) + + feature_names = _feature_names() + if any(len(row["features"]) != len(feature_names) for row in joined): + raise E40PerceptionProductGateError("E40 feature contract changed") + development = [row for row in joined if row["acceptance"]["split"] == "development"] + validation = [row for row in joined if row["acceptance"]["split"] == "validation"] + if len(development) != 340 or len(validation) != 146: + raise E40PerceptionProductGateError("E40 frozen split changed") + + model_profile = profile["model"] + clip = float(model_profile["robust_clip"]) + l2 = float(model_profile["l2"]) + camera_development = [ + row for row in development if row["acceptance"]["source_stratum"] == "camera-only" + ] + if len(camera_development) != 89: + raise E40PerceptionProductGateError("E40 camera-only development denominator changed") + camera_matrix = np.asarray( + [row["features"] for row in camera_development], + dtype=np.float64, + ) + median, scale = _fit_robust_scaler(camera_matrix) + transformed_camera = _transform( + camera_matrix, + median=median, + scale=scale, + clip=clip, + ) + camera_labels = np.asarray( + [ + _LABELS.index(str(row["acceptance"]["reference"]["presence"])) + for row in camera_development + ], + dtype=np.int64, + ) + weights = _train_softmax( + transformed_camera, + camera_labels, + l2=l2, + steps=int(model_profile["steps"]), + learning_rate=float(model_profile["learning_rate"]), + ) + cross_validation = _development_cross_validation( + development, + clip=clip, + seed=str(model_profile["cross_validation_seed"]), + folds=int(model_profile["cross_validation_folds"]), + l2=l2, + steps=int(model_profile["steps"]), + learning_rate=float(model_profile["learning_rate"]), + targets=profile["targets"], + ) + if not cross_validation["passed"]: + raise E40PerceptionProductGateError( + "E40 frozen profile did not pass development qualification" + ) + + predictions: list[dict[str, Any]] = [] + for row in joined: + stratum = str(row["acceptance"]["source_stratum"]) + presence, confidence = _predict_product_presence( + stratum=stratum, + features=np.asarray(row["features"], dtype=np.float64), + median=median, + scale=scale, + weights=weights, + clip=clip, + ) + predicted = _project_dimensions(stratum, presence) + acceptance_row = row["acceptance"] + predictions.append( + { + "schema_version": E40_PREDICTION_SCHEMA, + "sequence": acceptance_row["sequence"], + "item_id": acceptance_row["item_id"], + "review_key": acceptance_row["review_key"], + "source_frame_index": acceptance_row["source_frame_index"], + "source_stratum": stratum, + "severity": acceptance_row["severity"], + "split": acceptance_row["split"], + "prediction": predicted, + "presence_confidence": round(confidence, 6), + "reference": acceptance_row["reference"], + "scored": acceptance_row["split"] == "validation", + "authority": _AUTHORITY, + } + ) + + validation_predictions = [row for row in predictions if row["split"] == "validation"] + dimension_metrics = { + dimension: _dimension_metrics( + validation_predictions, + dimension=dimension, + target=float(profile["targets"][f"{dimension}_target"]), + ) + for dimension in _DIMENSIONS + } + high_severity_failures = sum( + row["severity"] == "high" + and any( + row["prediction"][dimension] != row["reference"][dimension] for dimension in _DIMENSIONS + ) + for row in validation_predictions + ) + accounting_fraction = len(validation_predictions) / 146 + false_free_claims = sum( + value == "free" for row in predictions for value in row["prediction"].values() + ) + gate_checks = { + "development_cv_targets_reached": cross_validation["passed"], + "presence_target_reached": dimension_metrics["presence"]["passed"], + "geometry_association_target_reached": dimension_metrics["geometry_association"]["passed"], + "freshness_target_reached": dimension_metrics["freshness"]["passed"], + "validation_accounting_complete": math.isclose(accounting_fraction, 1.0), + "false_free_claims_zero": false_free_claims == 0, + "high_severity_failures_zero": high_severity_failures == 0, + "authority_remains_diagnostic": True, + } + quality_gate_passed = all(gate_checks.values()) + model_document = { + "schema_version": E40_MODEL_SCHEMA, + "profile_id": profile["profile_id"], + "feature_set": model_profile["feature_set"], + "feature_names": feature_names, + "training_split": "development", + "training_items": len(development), + "camera_only_training_items": len(camera_development), + "validation_labels_used_for_training": False, + "robust_clip": clip, + "classifier": { + "type": "deterministic-softmax", + "l2": l2, + "steps": int(model_profile["steps"]), + "learning_rate": float(model_profile["learning_rate"]), + "labels": list(_LABELS), + "weights": [_rounded_vector(row) for row in np.asarray(weights, dtype=np.float64)], + "scaler": { + "median": _rounded_vector(median), + "scale": _rounded_vector(scale), + }, + }, + "fixed_presence_by_stratum": _FIXED_PRESENCE, + "dimension_projection": "source-stratum-plus-presence/v1", + "development_cross_validation": cross_validation, + "authority": _AUTHORITY, + } + report = { + "schema_version": E40_REPORT_SCHEMA, + "status": "measured-leakage-resistant-product-gate", + "source_session_id": _SOURCE_SESSION_ID, + "source_display_name": _SOURCE_DISPLAY_NAME, + "profile_id": profile["profile_id"], + "execution": identity_base["execution"], + "method": { + "summary": ( + "conservative stratum policy plus a robust-scaled softmax " + "classifier for camera-only cases using detector/support " + "metadata and exact camera crops" + ), + "selection": ( + "the fixed profile passed both contiguous source-time and " + "whole-track/scene-window five-fold development protocols" + ), + "dimension_projection": ( + "geometry association and freshness are derived from the " + "predicted presence state plus the immutable source stratum" + ), + }, + "development_cross_validation": cross_validation, + "metrics": { + "development_items": len(development), + "validation_items": len(validation_predictions), + "terminal_outcomes": len(validation_predictions), + "accounting_fraction": round(accounting_fraction, 6), + "false_free_claims": false_free_claims, + "high_severity_failures": high_severity_failures, + "dimensions": dimension_metrics, + }, + "quality_gate": { + "passed": quality_gate_passed, + "checks": gate_checks, + "blocking_checks": [name for name, passed in gate_checks.items() if not passed], + }, + "decision": { + "product_gate_measured": True, + "accepted_for_ravnoves00_product_track": quality_gate_passed, + "next_gate": ( + "R2 temporal product state and live rover replay" + if quality_gate_passed + else "remaining sealed dimension and high-severity errors" + ), + }, + "limitations": [ + ( + "the model is qualified only for the RAVNOVES00 source domain; " + "it contains no frame, review, track or absolute map coordinate" + ), + ( + "labels are an engineering-reviewed substrate rather than " + "independent physical ground truth" + ), + ( + "validation labels are evaluation-only and were not used for " + "feature selection, model fitting or development qualification" + ), + ( + "the result does not prove another route, camera, rig, mount, " + "vegetation environment, or rover installation" + ), + "navigation, command and safety authority remain false", + ], + "authority": _AUTHORITY, + } + content_identity = _result_content_identity( + predictions=predictions, + model=model_document, + report=report, + ) + identity = {**identity_base, "result_content": content_identity} + identity_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest() + result_id = f"e40-perception-product-gate-{identity_sha256}" + destination = output_root.expanduser().absolute() / result_id + if destination.exists(): + return read_e40_perception_product_gate(destination) + report = { + **report, + "result_id": result_id, + "identity_sha256": identity_sha256, + } + + destination.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + staging = destination.parent / f".{result_id}.{uuid.uuid4().hex}.tmp" + staging.mkdir(mode=0o700, exist_ok=False) + try: + _write_jsonl(staging / E40_PREDICTIONS_NAME, predictions) + _write_json(staging / E40_MODEL_NAME, model_document) + _write_json(staging / E40_REPORT_NAME, report) + artifacts = [ + _artifact(staging / E40_PREDICTIONS_NAME, "sealed-evaluation"), + _artifact(staging / E40_MODEL_NAME, "development-qualified-model"), + _artifact(staging / E40_REPORT_NAME, "quality-report"), + ] + manifest = { + "schema_version": E40_RESULT_SCHEMA, + "result_id": result_id, + "identity_sha256": identity_sha256, + "identity": identity, + "created_at_utc": _utc_now(), + "acceptance_state": "completed-leakage-resistant-product-gate", + "quality_gate_passed": quality_gate_passed, + "artifacts": artifacts, + } + _write_json(staging / E40_MANIFEST_NAME, manifest) + os.replace(staging, destination) + except BaseException: + shutil.rmtree(staging, ignore_errors=True) + raise + return read_e40_perception_product_gate(destination) + + +def read_e40_perception_product_gate(root: Path) -> E40PerceptionProductGate: + """Validate and read one immutable E40 result.""" + + resolved = root.resolve(strict=True) + manifest = _read_json(resolved / E40_MANIFEST_NAME) + identity = manifest.get("identity") + identity_sha256 = manifest.get("identity_sha256") + if ( + manifest.get("schema_version") != E40_RESULT_SCHEMA + or not isinstance(identity, dict) + or not isinstance(identity_sha256, str) + or hashlib.sha256(_canonical_json(identity)).hexdigest() != identity_sha256 + or manifest.get("result_id") != f"e40-perception-product-gate-{identity_sha256}" + or resolved.name != manifest.get("result_id") + or manifest.get("acceptance_state") != "completed-leakage-resistant-product-gate" + or identity.get("authority") != _AUTHORITY + ): + raise E40PerceptionProductGateError("E40 result identity is invalid") + expected = { + E40_PREDICTIONS_NAME: "sealed-evaluation", + E40_MODEL_NAME: "development-qualified-model", + E40_REPORT_NAME: "quality-report", + } + artifacts = manifest.get("artifacts") + if not isinstance(artifacts, list) or len(artifacts) != len(expected): + raise E40PerceptionProductGateError("E40 artifact catalog is invalid") + for row in artifacts: + if not isinstance(row, dict): + raise E40PerceptionProductGateError("E40 artifact descriptor is invalid") + name = row.get("path") + path = resolved / str(name) + if ( + name not in expected + or row.get("role") != expected[name] + or not path.is_file() + or path.is_symlink() + or row.get("byte_length") != path.stat().st_size + or row.get("sha256") != _sha256(path) + ): + raise E40PerceptionProductGateError("E40 artifact content changed") + report = _read_json(resolved / E40_REPORT_NAME) + model = _read_json(resolved / E40_MODEL_NAME) + predictions = _read_jsonl(resolved / E40_PREDICTIONS_NAME) + report_payload = dict(report) + report_payload.pop("result_id", None) + report_payload.pop("identity_sha256", None) + if ( + report.get("schema_version") != E40_REPORT_SCHEMA + or report.get("result_id") != resolved.name + or report.get("identity_sha256") != identity_sha256 + or report.get("status") != "measured-leakage-resistant-product-gate" + or model.get("schema_version") != E40_MODEL_SCHEMA + or model.get("validation_labels_used_for_training") is not False + or manifest.get("quality_gate_passed") is not report.get("quality_gate", {}).get("passed") + or identity.get("result_content") + != _result_content_identity( + predictions=predictions, + model=model, + report=report_payload, + ) + ): + raise E40PerceptionProductGateError("E40 report or model is invalid") + return E40PerceptionProductGate( + result_id=resolved.name, + result_root=resolved, + manifest=manifest, + report=report, + model=model, + ) + + +def _load_materialization( + root: Path, + acceptance_manifest: dict[str, Any], + profile: dict[str, Any], +) -> tuple[list[dict[str, Any]], dict[str, Any]]: + manifest_path = root / "manifest.json" + index_path = root / "materialized-items.jsonl" + manifest = _read_json(manifest_path) + binding = acceptance_manifest.get("identity", {}).get("reviewed_substrate", {}) + if ( + manifest.get("schema_version") != _MATERIALIZATION_SCHEMA + or root.name != profile["source"]["materialization_id"] + or manifest.get("result_id") != root.name + or root.name != binding.get("materialization_id") + or manifest.get("identity_sha256") != binding.get("materialization_identity_sha256") + or _sha256(manifest_path) != binding.get("materialization_manifest_sha256") + or not index_path.is_file() + or index_path.is_symlink() + or _sha256(index_path) != binding.get("materialization_index_sha256") + ): + raise E40PerceptionProductGateError("E40 materialization identity changed") + rows = _read_jsonl(index_path) + return rows, { + "materialization_id": root.name, + "materialization_identity_sha256": manifest["identity_sha256"], + "materialization_index_sha256": _sha256(index_path), + } + + +def _feature_names() -> list[str]: + names: list[str] = [] + for prefix, categories in _CATEGORIES.items(): + names += [f"{prefix}={value}" for value in categories] + names += [ + "detector_score", + "selected_point_count", + "candidate_point_count", + "rejected_candidate_point_count", + "projected_point_count", + "frame_point_count", + "nearest_range_m", + "point_count", + "voxel_count", + "score", + "camera_motion_confidence", + "range_m", + "bbox_present", + "bbox_center_x", + "bbox_center_y", + "bbox_width", + "bbox_height", + "bbox_area", + "bbox_aspect", + ] + support_names = ( + "below_surface_points_in_bbox", + "classified_points_in_bbox", + "connected_occupied_points", + "connected_occupied_voxels", + "occupied_points_in_bbox", + "projected_points_in_bbox", + "surface_points_in_bbox", + ) + names += [f"support_{name}" for name in support_names] + names += [ + "support_occupied_fraction", + "support_classified_fraction", + "support_surface_fraction", + "support_connected_fraction", + "bounds_x_span", + "bounds_y_span", + "bounds_z_span", + "height_span", + "occupied_height_span", + ] + names += [f"candidate_class_fraction_{index}" for index in range(8)] + names += [ + f"image_{channel}_histogram_{index}" + for channel in ("red", "green", "blue") + for index in range(4) + ] + names += [ + "image_luma_mean", + "image_luma_std", + "image_luma_p10", + "image_luma_p25", + "image_luma_p50", + "image_luma_p75", + "image_luma_p90", + "image_saturation_mean", + "image_saturation_std", + "image_edge_mean", + "image_edge_p90", + ] + names += [f"image_luma_centered_{y}_{x}" for y in range(4) for x in range(4)] + return sorted(names) + + +def _load_feature_cache( + materialization_root: Path, + materialization_binding: dict[str, Any], +) -> tuple[dict[str, list[float]] | None, dict[str, Any]]: + manifest_path = materialization_root / _FEATURE_CACHE_MANIFEST + arrays_path = materialization_root / _FEATURE_CACHE_ARRAYS + if not manifest_path.exists() and not arrays_path.exists(): + return None, {"mode": "runtime-source-extraction"} + if ( + not manifest_path.is_file() + or manifest_path.is_symlink() + or not arrays_path.is_file() + or arrays_path.is_symlink() + ): + raise E40PerceptionProductGateError("E40 feature cache is incomplete") + manifest = _read_json(manifest_path) + names = _feature_names() + names_sha256 = hashlib.sha256(_canonical_json(names)).hexdigest() + if ( + manifest.get("schema_version") != _FEATURE_CACHE_SCHEMA + or manifest.get("materialization_id") != materialization_root.name + or manifest.get("materialization_index_sha256") + != materialization_binding["materialization_index_sha256"] + or manifest.get("feature_names_sha256") != names_sha256 + or manifest.get("items") != 486 + or manifest.get("dimensions") != len(names) + or manifest.get("arrays_path") != _FEATURE_CACHE_ARRAYS + or manifest.get("arrays_byte_length") != arrays_path.stat().st_size + or manifest.get("arrays_sha256") != _sha256(arrays_path) + ): + raise E40PerceptionProductGateError("E40 feature cache binding changed") + with np.load(arrays_path, allow_pickle=False) as arrays: + item_ids = arrays["item_ids"] + features = arrays["features"] + if ( + item_ids.shape != (486,) + or features.shape != (486, len(names)) + or not np.issubdtype(item_ids.dtype, np.str_) + or not np.isfinite(features).all() + or len(set(str(value) for value in item_ids)) != 486 + ): + raise E40PerceptionProductGateError("E40 feature cache arrays are invalid") + cache = { + str(item_id): [float(value) for value in vector] + for item_id, vector in zip(item_ids, features, strict=True) + } + return cache, { + "mode": "package-bound-feature-cache", + "schema_version": _FEATURE_CACHE_SCHEMA, + "manifest_sha256": _sha256(manifest_path), + "arrays_sha256": _sha256(arrays_path), + "feature_names_sha256": names_sha256, + } + + +def _feature_vector( + acceptance: dict[str, Any], + materialization: dict[str, Any], + materialization_root: Path, +) -> list[float]: + del acceptance + snapshot = _object(materialization.get("e29_snapshot"), "E40 snapshot") + evidence = _object( + materialization.get("materialization"), + "E40 materialization evidence", + ) + values: dict[str, float] = {} + observed = { + "stratum": materialization.get("stratum"), + "range": materialization.get("range_bucket"), + "geometry": snapshot.get("geometry_status"), + "label": snapshot.get("label") or "none", + "reason": snapshot.get("geometry_reason") or "none", + "association": snapshot.get("association_group") or "none", + "motion": snapshot.get("motion_state") or "none", + "camera_motion": snapshot.get("camera_motion_state") or "none", + "semantic_current": ( + str(snapshot.get("semantic_current")).lower() + if snapshot.get("semantic_current") is not None + else "none" + ), + } + for prefix, categories in _CATEGORIES.items(): + for category in categories: + values[f"{prefix}={category}"] = float(observed[prefix] == category) + for name in ( + "selected_point_count", + "candidate_point_count", + "rejected_candidate_point_count", + "projected_point_count", + "frame_point_count", + ): + values[name] = math.log1p(max(0.0, _number_or(evidence.get(name), 0.0))) + values["detector_score"] = _number_or( + evidence.get("detector_score"), + -1.0, + ) + for name in ( + "nearest_range_m", + "point_count", + "voxel_count", + "score", + "camera_motion_confidence", + "range_m", + ): + raw = _number_or(snapshot.get(name), -1.0) + values[name] = ( + math.log1p(raw) if name in {"point_count", "voxel_count"} and raw >= 0.0 else raw + ) + _bbox_features(values, snapshot.get("bbox_xyxy")) + _support_features(values, snapshot.get("support")) + _bounds_features(values, snapshot.get("bounds_map_xyz_m")) + values["height_span"] = _range_span(snapshot.get("height_range_m")) + values["occupied_height_span"] = _range_span(snapshot.get("occupied_height_range_m")) + + artifact = _object(materialization.get("artifact"), "E40 NPZ artifact") + artifact_path = materialization_root / str(artifact.get("path")) + if not artifact_path.is_file() or artifact_path.is_symlink(): + raise E40PerceptionProductGateError("E40 NPZ artifact is unavailable") + with np.load(artifact_path, allow_pickle=False) as arrays: + projected_pixels = arrays["projected_pixels_xy"] + candidate_mask = arrays["projected_candidate_mask"].astype(bool) + width = max(1, int(evidence.get("projection_width", 800))) + height = max(1, int(evidence.get("projection_height", 600))) + point_classes = arrays["projected_point_class"][candidate_mask] + for index in range(8): + values[f"candidate_class_fraction_{index}"] = ( + float(np.mean(point_classes == index)) if point_classes.size else 0.0 + ) + _camera_crop_features( + values, + materialization, + materialization_root, + projected_pixels[candidate_mask], + width=width, + height=height, + ) + names = _feature_names() + vector = [values.get(name, 0.0) for name in names] + if not np.isfinite(np.asarray(vector, dtype=np.float64)).all(): + raise E40PerceptionProductGateError("E40 produced a non-finite feature") + return vector + + +def _bbox_features(values: dict[str, float], raw: object) -> None: + if isinstance(raw, list) and len(raw) == 4: + x1, y1, x2, y2 = (_number_or(value, 0.0) for value in raw) + width = max(0.0, x2 - x1) / 800.0 + height = max(0.0, y2 - y1) / 600.0 + values.update( + { + "bbox_present": 1.0, + "bbox_center_x": (x1 + x2) / 1600.0, + "bbox_center_y": (y1 + y2) / 1200.0, + "bbox_width": width, + "bbox_height": height, + "bbox_area": width * height, + "bbox_aspect": width / (height + 1e-6), + } + ) + else: + values.update( + { + "bbox_present": 0.0, + "bbox_center_x": -1.0, + "bbox_center_y": -1.0, + "bbox_width": -1.0, + "bbox_height": -1.0, + "bbox_area": -1.0, + "bbox_aspect": -1.0, + } + ) + + +def _support_features(values: dict[str, float], raw: object) -> None: + support = raw if isinstance(raw, dict) else {} + names = ( + "below_surface_points_in_bbox", + "classified_points_in_bbox", + "connected_occupied_points", + "connected_occupied_voxels", + "occupied_points_in_bbox", + "projected_points_in_bbox", + "surface_points_in_bbox", + ) + observed = {name: _number_or(support.get(name), 0.0) for name in names} + values.update( + {f"support_{name}": math.log1p(max(0.0, value)) for name, value in observed.items()} + ) + projected = max(1.0, observed["projected_points_in_bbox"]) + occupied = max(1.0, observed["occupied_points_in_bbox"]) + values.update( + { + "support_occupied_fraction": (observed["occupied_points_in_bbox"] / projected), + "support_classified_fraction": (observed["classified_points_in_bbox"] / projected), + "support_surface_fraction": (observed["surface_points_in_bbox"] / projected), + "support_connected_fraction": (observed["connected_occupied_points"] / occupied), + } + ) + + +def _bounds_features(values: dict[str, float], raw: object) -> None: + if ( + isinstance(raw, list) + and len(raw) == 2 + and all(isinstance(item, list) and len(item) == 3 for item in raw) + ): + for index, axis in enumerate("xyz"): + values[f"bounds_{axis}_span"] = _number_or( + raw[1][index], + -1.0, + ) - _number_or(raw[0][index], -1.0) + return + for axis in "xyz": + values[f"bounds_{axis}_span"] = -1.0 + + +def _range_span(raw: object) -> float: + return ( + _number_or(raw[1], -1.0) - _number_or(raw[0], -1.0) + if isinstance(raw, list) and len(raw) == 2 + else -1.0 + ) + + +def _camera_crop_features( + features: dict[str, float], + materialization: dict[str, Any], + materialization_root: Path, + candidate_pixels: np.ndarray[Any, Any], + *, + width: int, + height: int, +) -> None: + from PIL import Image + + snapshot = _object(materialization.get("e29_snapshot"), "E40 snapshot") + bbox = snapshot.get("bbox_xyxy") + values = np.asarray(candidate_pixels, dtype=np.float64) + if not isinstance(bbox, list) or len(bbox) != 4: + if values.ndim != 2 or values.shape[1] != 2 or not len(values): + _empty_camera_features(features) + return + low = np.min(values, axis=0) + high = np.max(values, axis=0) + center = (low + high) / 2.0 + support = np.maximum(high - low, 48.0) + bbox = [ + center[0] - support[0], + center[1] - support[1], + center[0] + support[0], + center[1] + support[1], + ] + frame = _object(materialization.get("camera_frame"), "E40 camera frame") + frame_path = materialization_root / str(frame.get("path")) + if not frame_path.is_file() or frame_path.is_symlink(): + raise E40PerceptionProductGateError("E40 camera frame is unavailable") + x1, y1, x2, y2 = (_number_or(value, 0.0) for value in bbox) + box = ( + max(0, int(x1)), + max(0, int(y1)), + min(width, int(math.ceil(x2))), + min(height, int(math.ceil(y2))), + ) + if box[2] <= box[0] or box[3] <= box[1]: + _empty_camera_features(features) + return + with Image.open(frame_path) as source: + rgb = ( + np.asarray( + source.convert("RGB") + .crop(box) + .resize((32, 32), resample=Image.Resampling.BILINEAR), + dtype=np.float64, + ) + / 255.0 + ) + for channel, name in enumerate(("red", "green", "blue")): + histogram_counts, _ = np.histogram( + rgb[:, :, channel], + bins=4, + range=(0.0, 1.0), + ) + histogram = histogram_counts.astype(np.float64) / max( + 1, + int(np.sum(histogram_counts)), + ) + for index, fraction in enumerate(histogram): + features[f"image_{name}_histogram_{index}"] = float(fraction) + luma = np.mean(rgb, axis=2) + saturation = np.max(rgb, axis=2) - np.min(rgb, axis=2) + edge = np.concatenate( + ( + np.abs(np.diff(luma, axis=1)).reshape(-1), + np.abs(np.diff(luma, axis=0)).reshape(-1), + ) + ) + features["image_luma_mean"] = float(np.mean(luma)) + features["image_luma_std"] = float(np.std(luma)) + for name, quantile in zip( + ("p10", "p25", "p50", "p75", "p90"), + np.quantile(luma, (0.1, 0.25, 0.5, 0.75, 0.9)), + strict=True, + ): + features[f"image_luma_{name}"] = float(quantile) + features["image_saturation_mean"] = float(np.mean(saturation)) + features["image_saturation_std"] = float(np.std(saturation)) + features["image_edge_mean"] = float(np.mean(edge)) + features["image_edge_p90"] = float(np.quantile(edge, 0.9)) + small_luma = ( + np.asarray( + Image.fromarray( + np.clip(luma * 255.0, 0.0, 255.0).astype(np.uint8) + ).resize( + (4, 4), + Image.Resampling.BILINEAR, + ), + dtype=np.float64, + ) + / 255.0 + ) + small_luma -= float(np.mean(small_luma)) + for y in range(4): + for x in range(4): + features[f"image_luma_centered_{y}_{x}"] = float(small_luma[y, x]) + + +def _empty_camera_features(features: dict[str, float]) -> None: + for channel in ("red", "green", "blue"): + for index in range(4): + features[f"image_{channel}_histogram_{index}"] = 0.0 + for name in ("mean", "std", "p10", "p25", "p50", "p75", "p90"): + features[f"image_luma_{name}"] = 0.0 + features["image_saturation_mean"] = 0.0 + features["image_saturation_std"] = 0.0 + features["image_edge_mean"] = 0.0 + features["image_edge_p90"] = 0.0 + for y in range(4): + for x in range(4): + features[f"image_luma_centered_{y}_{x}"] = 0.0 + + +def _fit_robust_scaler(matrix: np.ndarray[Any, Any]) -> tuple[np.ndarray, np.ndarray]: + median = np.median(matrix, axis=0) + scale = np.percentile(matrix, 75, axis=0) - np.percentile(matrix, 25, axis=0) + scale[scale < 1e-8] = 1.0 + return median, scale + + +def _transform( + matrix: np.ndarray[Any, Any], + *, + median: np.ndarray[Any, Any], + scale: np.ndarray[Any, Any], + clip: float, +) -> np.ndarray[Any, Any]: + return np.asarray( + np.clip((matrix - median) / scale, -clip, clip), + dtype=np.float64, + ) + + +def _train_softmax( + matrix: np.ndarray[Any, Any], + labels: np.ndarray[Any, Any], + *, + l2: float, + steps: int, + learning_rate: float, +) -> np.ndarray[Any, Any]: + rows, dimensions = matrix.shape + design = np.column_stack((matrix, np.ones(rows))) + targets = np.eye(len(_LABELS), dtype=np.float64)[labels] + weights = np.zeros((dimensions + 1, len(_LABELS)), dtype=np.float64) + first_moment = np.zeros_like(weights) + second_moment = np.zeros_like(weights) + for step in range(1, steps + 1): + logits = design @ weights + logits -= np.max(logits, axis=1, keepdims=True) + probabilities = np.exp(logits) + probabilities /= np.sum(probabilities, axis=1, keepdims=True) + regularizer = np.vstack((weights[:-1], np.zeros((1, len(_LABELS))))) + gradient = design.T @ (probabilities - targets) / rows + gradient += l2 * regularizer + first_moment = 0.9 * first_moment + 0.1 * gradient + second_moment = 0.999 * second_moment + 0.001 * np.square(gradient) + corrected_first = first_moment / (1.0 - 0.9**step) + corrected_second = second_moment / (1.0 - 0.999**step) + weights -= learning_rate * corrected_first / (np.sqrt(corrected_second) + 1e-8) + return weights + + +def _predict_product_presence( + *, + stratum: str, + features: np.ndarray[Any, Any], + median: np.ndarray[Any, Any], + scale: np.ndarray[Any, Any], + weights: np.ndarray[Any, Any], + clip: float, +) -> tuple[str, float]: + fixed = _FIXED_PRESENCE.get(stratum) + if fixed is not None: + return fixed, 1.0 + if stratum != "camera-only": + raise E40PerceptionProductGateError("E40 stratum is unsupported") + transformed = _transform( + np.asarray([features], dtype=np.float64), + median=median, + scale=scale, + clip=clip, + ) + logits = np.column_stack((transformed, np.ones(1))) @ weights + logits -= np.max(logits, axis=1, keepdims=True) + probabilities = np.exp(logits) + probabilities /= np.sum(probabilities, axis=1, keepdims=True) + index = int(np.argmax(probabilities[0])) + return _LABELS[index], float(probabilities[0, index]) + + +def _project_dimensions(stratum: str, presence: str) -> dict[str, str]: + if presence == "background-or-noise" or stratum == "conflict": + geometry = "rejected-nonobject" + elif presence == "unknown": + geometry = "unknown" + elif stratum == "camera-only": + geometry = "insufficient-support" + elif stratum == "unknown": + geometry = "unknown" + elif stratum == "geometry-only": + geometry = "object-associated" if presence == "object-present" else "independent-occupied" + else: + geometry = "object-associated" + + if stratum == "camera-only": + freshness = "current" if presence == "background-or-noise" else "unavailable" + elif stratum == "unknown": + freshness = "current" if presence == "background-or-noise" else "stale" + else: + freshness = "current" + return { + "presence": presence, + "geometry_association": geometry, + "freshness": freshness, + } + + +def _development_cross_validation( + development: list[dict[str, Any]], + *, + clip: float, + seed: str, + folds: int, + l2: float, + steps: int, + learning_rate: float, + targets: dict[str, Any], +) -> dict[str, Any]: + matrix = np.asarray( + [row["features"] for row in development], + dtype=np.float64, + ) + order = np.argsort( + [int(row["acceptance"]["source_frame_index"]) for row in development], + kind="stable", + ) + contiguous = np.empty(len(development), dtype=np.int64) + for fold, indices in enumerate(np.array_split(order, folds)): + contiguous[indices] = fold + grouped = np.asarray( + [ + int( + hashlib.sha256(f"{seed}:{row['scene_group']}".encode()).hexdigest()[:8], + 16, + ) + % folds + for row in development + ], + dtype=np.int64, + ) + protocols = { + "contiguous-source-time-five-fold": contiguous, + "whole-track-or-scene-window-five-fold": grouped, + } + results = { + name: _cross_validate_protocol( + development=development, + matrix=matrix, + assignments=assignments, + clip=clip, + folds=folds, + l2=l2, + steps=steps, + learning_rate=learning_rate, + targets=targets, + ) + for name, assignments in protocols.items() + } + return { + "strategy": "dual-leakage-resistant-development-five-fold", + "seed": seed, + "folds": folds, + "items": len(development), + "validation_labels_used": False, + "protocols": results, + "passed": all(row["passed"] for row in results.values()), + } + + +def _cross_validate_protocol( + *, + development: list[dict[str, Any]], + matrix: np.ndarray[Any, Any], + assignments: np.ndarray[Any, Any], + clip: float, + folds: int, + l2: float, + steps: int, + learning_rate: float, + targets: dict[str, Any], +) -> dict[str, Any]: + predictions: list[dict[str, str] | None] = [None] * len(development) + for fold in range(folds): + train_indices = np.flatnonzero(assignments != fold) + test_indices = np.flatnonzero(assignments == fold) + camera_train = np.asarray( + [ + index + for index in train_indices + if development[int(index)]["acceptance"]["source_stratum"] == "camera-only" + ], + dtype=np.int64, + ) + if len(camera_train) < 20: + raise E40PerceptionProductGateError( + "E40 development fold lacks camera-only training rows" + ) + median, scale = _fit_robust_scaler(matrix[camera_train]) + train_matrix = _transform( + matrix[camera_train], + median=median, + scale=scale, + clip=clip, + ) + train_labels = np.asarray( + [ + _LABELS.index(str(development[int(index)]["acceptance"]["reference"]["presence"])) + for index in camera_train + ], + dtype=np.int64, + ) + weights = _train_softmax( + train_matrix, + train_labels, + l2=l2, + steps=steps, + learning_rate=learning_rate, + ) + for source_index in test_indices: + stratum = str(development[int(source_index)]["acceptance"]["source_stratum"]) + presence, _ = _predict_product_presence( + stratum=stratum, + features=matrix[int(source_index)], + median=median, + scale=scale, + weights=weights, + clip=clip, + ) + predictions[int(source_index)] = _project_dimensions( + stratum, + presence, + ) + if any(value is None for value in predictions): + raise E40PerceptionProductGateError("E40 development CV is incomplete") + dimensions = {} + for dimension in _DIMENSIONS: + correct = sum( + prediction is not None + and prediction[dimension] == row["acceptance"]["reference"][dimension] + for prediction, row in zip(predictions, development, strict=True) + ) + accuracy = correct / len(development) + target = float(targets[f"{dimension}_target"]) + dimensions[dimension] = { + "correct": correct, + "incorrect": len(development) - correct, + "total": len(development), + "accuracy": round(accuracy, 6), + "target": target, + "passed": accuracy >= target, + } + return { + "items": len(development), + "fold_sizes": {str(fold): int(np.sum(assignments == fold)) for fold in range(folds)}, + "dimensions": dimensions, + "passed": all(row["passed"] for row in dimensions.values()), + } + + +def _scene_group( + acceptance: dict[str, Any], + materialization: dict[str, Any], +) -> str: + snapshot = _object(materialization.get("e29_snapshot"), "E40 snapshot") + track_id = snapshot.get("track_id") + if track_id is not None: + return f"track:{track_id}" + return f"scene:{int(acceptance['source_frame_index']) // 50}" + + +def _dimension_metrics( + rows: list[dict[str, Any]], + *, + dimension: str, + target: float, +) -> dict[str, Any]: + confusion: Counter[tuple[str, str]] = Counter() + stratum_counts: dict[str, Counter[str]] = defaultdict(Counter) + correct = 0 + for row in rows: + reference = str(row["reference"][dimension]) + prediction = str(row["prediction"][dimension]) + confusion[(reference, prediction)] += 1 + matched = reference == prediction + correct += matched + stratum_counts[str(row["source_stratum"])]["correct" if matched else "incorrect"] += 1 + total = len(rows) + accuracy = correct / total if total else 0.0 + return { + "correct": correct, + "incorrect": total - correct, + "total": total, + "accuracy": round(accuracy, 6), + "target": target, + "passed": accuracy >= target, + "confusion": [ + { + "reference": reference, + "prediction": prediction, + "count": count, + } + for (reference, prediction), count in sorted( + confusion.items(), + key=lambda item: (-item[1], item[0]), + ) + ], + "by_stratum": { + stratum: { + "correct": counts["correct"], + "incorrect": counts["incorrect"], + "total": sum(counts.values()), + "accuracy": round(counts["correct"] / sum(counts.values()), 6), + } + for stratum, counts in sorted(stratum_counts.items()) + }, + } + + +def _validate_profile(profile: dict[str, Any]) -> None: + source = _object(profile.get("source"), "E40 source") + model = _object(profile.get("model"), "E40 model") + targets = _object(profile.get("targets"), "E40 targets") + if ( + profile.get("schema_version") != E40_PROFILE_SCHEMA + or profile.get("profile_id") != "e40-ravnoves00-leakage-resistant-product-gate/v1" + or source.get("session_id") != _SOURCE_SESSION_ID + or source.get("display_name") != _SOURCE_DISPLAY_NAME + or not isinstance(source.get("acceptance_result_id"), str) + or not str(source["acceptance_result_id"]).startswith("e37-ravnoves-acceptance-") + or not isinstance(source.get("materialization_id"), str) + or not str(source["materialization_id"]).startswith("e30-materialization-") + or model.get("type") != "hierarchical-stratum-softmax" + or model.get("feature_set") != "route-coordinate-free-structured-image/v1" + or model.get("fixed_presence_by_stratum") != _FIXED_PRESENCE + or model.get("robust_clip") != 10.0 + or model.get("l2") != 0.01 + or model.get("steps") != 1200 + or model.get("learning_rate") != 0.03 + or model.get("cross_validation_folds") != 5 + or not isinstance(model.get("cross_validation_seed"), str) + or not model["cross_validation_seed"] + or any(targets.get(f"{name}_target") != 0.9 for name in _DIMENSIONS) + or targets.get("accounting_target") != 1.0 + or targets.get("maximum_false_free_claims") != 0 + or targets.get("maximum_high_severity_failures") != 0 + or profile.get("authority") != _AUTHORITY + ): + raise E40PerceptionProductGateError("E40 profile contract changed") + + +def _number_or(value: object, fallback: float) -> float: + try: + parsed = float(value) # type: ignore[arg-type] + except (TypeError, ValueError): + return fallback + return parsed if math.isfinite(parsed) else fallback + + +def _rounded_vector(values: object) -> list[float]: + return [round(float(value), 12) for value in np.asarray(values, dtype=np.float64).reshape(-1)] + + +def _artifact(path: Path, role: str) -> dict[str, Any]: + return { + "role": role, + "path": path.name, + "byte_length": path.stat().st_size, + "sha256": _sha256(path), + } + + +def _execution_package_binding(value: dict[str, str] | None) -> dict[str, str]: + if value is None: + return { + "mode": "unpackaged-local", + "runtime_sha256": _sha256(Path(__file__).resolve(strict=True)), + } + if ( + set(value) != {"mode", "package_id", "identity_sha256"} + or value.get("mode") != "verified-worker-package" + or not isinstance((package_id := value.get("package_id")), str) + or not package_id.startswith("e40-worker-package-") + or not isinstance((identity_sha256 := value.get("identity_sha256")), str) + or package_id != f"e40-worker-package-{identity_sha256}" + ): + raise E40PerceptionProductGateError("E40 execution package binding is invalid") + return dict(value) + + +def _result_content_identity( + *, + predictions: list[dict[str, Any]], + model: dict[str, Any], + report: dict[str, Any], +) -> dict[str, str]: + prediction_payload = b"".join( + _canonical_json(row) + b"\n" for row in predictions + ) + return { + "predictions_sha256": hashlib.sha256(prediction_payload).hexdigest(), + "model_sha256": hashlib.sha256(_canonical_json(model)).hexdigest(), + "report_payload_sha256": hashlib.sha256(_canonical_json(report)).hexdigest(), + } + + +def _object(value: object, label: str) -> dict[str, Any]: + if not isinstance(value, dict): + raise E40PerceptionProductGateError(f"{label} must be an object") + return value + + +def _read_json(path: Path) -> dict[str, Any]: + try: + value = json.loads(path.read_text(encoding="utf-8-sig")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: + raise E40PerceptionProductGateError(f"invalid JSON: {path.name}") from exc + return _object(value, path.name) + + +def _read_jsonl(path: Path) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + try: + with path.open("r", encoding="utf-8-sig") as stream: + for line in stream: + rows.append(_object(json.loads(line), path.name)) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: + raise E40PerceptionProductGateError(f"invalid JSONL: {path.name}") from exc + return rows + + +def _write_json(path: Path, value: object) -> None: + with path.open("x", encoding="utf-8", newline="\n") as stream: + json.dump(value, stream, ensure_ascii=False, indent=2, sort_keys=True) + stream.write("\n") + stream.flush() + os.fsync(stream.fileno()) + + +def _write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None: + with path.open("x", encoding="utf-8", newline="\n") as stream: + for row in rows: + stream.write( + json.dumps( + row, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ) + ) + stream.write("\n") + stream.flush() + os.fsync(stream.fileno()) + + +def _canonical_json(value: object) -> bytes: + return json.dumps( + value, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode() + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + while chunk := stream.read(1024 * 1024): + digest.update(chunk) + return digest.hexdigest() + + +def _utc_now() -> str: + return datetime.now(UTC).isoformat(timespec="milliseconds").replace("+00:00", "Z") diff --git a/src/k1link/compute/e41_evaluation_boundary.py b/src/k1link/compute/e41_evaluation_boundary.py new file mode 100644 index 0000000..c0c11f7 --- /dev/null +++ b/src/k1link/compute/e41_evaluation_boundary.py @@ -0,0 +1,908 @@ +"""Physically separate frozen E40 prediction from visible contract evaluation. + +The predictor package contains a trained model, a feature matrix and stripped +item metadata. It contains no acceptance rows, split assignments, reference +labels, severity or scoring state. The predictor therefore cannot inspect +truth. A separate evaluator joins the immutable prediction artifact to E37 +after inference and reports source-scoped engineering-contract conformance. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import shutil +import uuid +from collections import Counter, defaultdict +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path +from typing import Any, Final + +import numpy as np + +from k1link.compute.e37_acceptance_contract import ( + E37_ITEMS_NAME, + read_e37_acceptance_contract, +) +from k1link.compute.e40_perception_product_gate import ( + _LABELS, + _predict_product_presence, + _project_dimensions, +) +from k1link.compute.e41_methodology_audit import ( + _load_feature_cache, + _validate_e40_package, +) + +E41_PREDICTOR_PACKAGE_SCHEMA: Final = "missioncore.e41-predictor-package/v1" +E41_PREDICTION_RESULT_SCHEMA: Final = "missioncore.e41-prediction-result/v1" +E41_PREDICTION_ROW_SCHEMA: Final = "missioncore.e41-prediction-row/v1" +E41_VISIBLE_EVALUATION_SCHEMA: Final = "missioncore.e41-visible-evaluation/v1" + +E41_PACKAGE_MANIFEST_NAME: Final = "manifest.json" +E41_MODEL_NAME: Final = "model.json" +E41_ITEMS_NAME: Final = "items.jsonl" +E41_FEATURE_MANIFEST_NAME: Final = "feature-cache.json" +E41_FEATURE_ARRAYS_NAME: Final = "feature-vectors.npz" +E41_PREDICTIONS_NAME: Final = "predictions.jsonl" +E41_EVALUATION_NAME: Final = "evaluation.json" + +_AUTHORITY: Final = { + "commands_enabled": False, + "navigation_or_safety_accepted": False, +} +_STRATA: Final = {"agree", "camera-only", "conflict", "geometry-only", "unknown"} +_DIMENSIONS: Final = ("presence", "geometry_association", "freshness") +_PREDICTION_FORBIDDEN_KEYS: Final = frozenset( + { + "acceptance", + "reference", + "scored", + "severity", + "split", + "truth", + } +) + + +class E41EvaluationBoundaryError(RuntimeError): + """An E41 predictor package, prediction, or visible evaluation is invalid.""" + + +@dataclass(frozen=True, slots=True) +class E41PredictionResult: + result_id: str + result_root: Path + manifest: dict[str, Any] + + +@dataclass(frozen=True, slots=True) +class E41VisibleEvaluation: + result_id: str + result_root: Path + manifest: dict[str, Any] + evaluation: dict[str, Any] + + +def build_e41_predictor_package( + *, + materialization_root: Path, + e40_package_root: Path, + e40_model_path: Path, + output_root: Path, +) -> Path: + """Build a content-addressed predictor-only package with no truth material.""" + + materialization = materialization_root.resolve(strict=True) + package_root = e40_package_root.resolve(strict=True) + e40_package, package_artifacts = _validate_e40_package(package_root) + model_source = e40_model_path.resolve(strict=True) + model = _read_json(model_source) + materialization_rows = _read_jsonl(materialization / "materialized-items.jsonl") + materialization_by_id = _unique_by_item_id(materialization_rows, "materialization") + feature_root = ( + package_root / "input" / "materialization" / materialization.name + ) + item_ids, feature_names, feature_matrix, feature_binding = _load_feature_cache( + feature_root, + package_artifacts=package_artifacts, + ) + _validate_frozen_model(model, feature_names) + items = [] + for sequence, item_id in enumerate(item_ids): + row = materialization_by_id.get(item_id) + if row is None: + raise E41EvaluationBoundaryError("E41 predictor item denominator changed") + stratum = row.get("stratum") + if stratum not in _STRATA: + raise E41EvaluationBoundaryError("E41 predictor item stratum is invalid") + items.append( + { + "schema_version": "missioncore.e41-predictor-item/v1", + "sequence": sequence, + "item_id": item_id, + "source_stratum": stratum, + } + ) + _assert_truth_free(items, "E41 predictor item metadata") + + arrays_payload = _npz_payload(item_ids, feature_matrix) + feature_manifest = { + "schema_version": "missioncore.e41-predictor-feature-cache/v1", + "item_count": len(item_ids), + "dimensions": len(feature_names), + "feature_names": feature_names, + "feature_names_sha256": hashlib.sha256(_canonical_json(feature_names)).hexdigest(), + "arrays_path": E41_FEATURE_ARRAYS_NAME, + "arrays_byte_length": len(arrays_payload), + "arrays_sha256": hashlib.sha256(arrays_payload).hexdigest(), + } + items_payload = _jsonl_payload(items) + model_payload = _json_payload(model) + feature_manifest_payload = _json_payload(feature_manifest) + sources = { + E41_MODEL_NAME: model_payload, + E41_ITEMS_NAME: items_payload, + E41_FEATURE_MANIFEST_NAME: feature_manifest_payload, + E41_FEATURE_ARRAYS_NAME: arrays_payload, + } + source_artifacts = [ + { + "path": name, + "byte_length": len(payload), + "sha256": hashlib.sha256(payload).hexdigest(), + } + for name, payload in sorted(sources.items()) + ] + identity = { + "schema_version": E41_PREDICTOR_PACKAGE_SCHEMA, + "classification": "truth-free-frozen-e40-predictor-input", + "source": { + "materialization_id": materialization.name, + "materialization_index_sha256": _sha256( + materialization / "materialized-items.jsonl" + ), + "e40_worker_package_id": e40_package["package_id"], + "e40_worker_package_identity_sha256": e40_package["identity_sha256"], + "e40_model_sha256": _sha256(model_source), + "feature_cache": feature_binding, + }, + "runtime_contract": { + "python": "3.12", + "numpy_api": "numpy-1.26-or-newer", + "determinism": "frozen-weights-no-randomness", + }, + "source_artifacts": source_artifacts, + "truth_material_included": False, + "producer_sha256": _sha256(Path(__file__).resolve(strict=True)), + "authority": _AUTHORITY, + } + identity_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest() + package_id = f"e41-predictor-package-{identity_sha256}" + destination = output_root.expanduser().absolute() / package_id + if destination.exists(): + validate_e41_predictor_package(destination) + return destination + destination.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + staging = destination.parent / f".{package_id}.{uuid.uuid4().hex}.tmp" + staging.mkdir(mode=0o700, exist_ok=False) + try: + for name, payload in sources.items(): + _write_bytes(staging / name, payload) + manifest = { + "schema_version": E41_PREDICTOR_PACKAGE_SCHEMA, + "package_id": package_id, + "identity_sha256": identity_sha256, + "identity": identity, + "created_at_utc": _utc_now(), + "artifacts": [ + { + **row, + "role": "predictor-only-input", + } + for row in source_artifacts + ], + } + _write_json(staging / E41_PACKAGE_MANIFEST_NAME, manifest) + validate_e41_predictor_package(staging, allow_staging=True) + os.replace(staging, destination) + except BaseException: + shutil.rmtree(staging, ignore_errors=True) + raise + validate_e41_predictor_package(destination) + return destination + + +def validate_e41_predictor_package( + root: Path, + *, + allow_staging: bool = False, +) -> dict[str, Any]: + """Validate exact package identity, artifacts, and the truth-free boundary.""" + + resolved = root.resolve(strict=True) + manifest = _read_json(resolved / E41_PACKAGE_MANIFEST_NAME) + identity = _object(manifest.get("identity"), "E41 predictor identity") + identity_sha256 = manifest.get("identity_sha256") + package_id = manifest.get("package_id") + expected_name = isinstance(package_id, str) and ( + resolved.name == package_id + or ( + allow_staging + and resolved.name.startswith(f".{package_id}.") + and resolved.name.endswith(".tmp") + ) + ) + if ( + manifest.get("schema_version") != E41_PREDICTOR_PACKAGE_SCHEMA + or not isinstance(identity_sha256, str) + or hashlib.sha256(_canonical_json(identity)).hexdigest() != identity_sha256 + or package_id != f"e41-predictor-package-{identity_sha256}" + or not expected_name + or identity.get("classification") != "truth-free-frozen-e40-predictor-input" + or identity.get("truth_material_included") is not False + or identity.get("authority") != _AUTHORITY + ): + raise E41EvaluationBoundaryError("E41 predictor package identity is invalid") + expected_rows = identity.get("source_artifacts") + artifacts = manifest.get("artifacts") + if not isinstance(expected_rows, list) or not isinstance(artifacts, list): + raise E41EvaluationBoundaryError("E41 predictor package catalog is invalid") + expected: dict[str, tuple[int, str]] = {} + for row in expected_rows: + if ( + not isinstance(row, dict) + or not isinstance((name := row.get("path")), str) + or name in expected + or Path(name).is_absolute() + or ".." in Path(name).parts + or not isinstance((length := row.get("byte_length")), int) + or not isinstance((sha256 := row.get("sha256")), str) + ): + raise E41EvaluationBoundaryError("E41 predictor source artifact is invalid") + expected[name] = (length, sha256) + actual_files = { + path.relative_to(resolved).as_posix() + for path in resolved.rglob("*") + if path.is_file() + } + if actual_files != set(expected) | {E41_PACKAGE_MANIFEST_NAME}: + raise E41EvaluationBoundaryError("E41 predictor package file set changed") + observed: set[str] = set() + for row in artifacts: + if not isinstance(row, dict): + raise E41EvaluationBoundaryError("E41 predictor artifact is invalid") + name = row.get("path") + path = resolved / str(name) + if ( + not isinstance(name, str) + or name in observed + or row.get("role") != "predictor-only-input" + or expected.get(name) != (row.get("byte_length"), row.get("sha256")) + or not path.is_file() + or path.is_symlink() + or row.get("byte_length") != path.stat().st_size + or row.get("sha256") != _sha256(path) + ): + raise E41EvaluationBoundaryError("E41 predictor artifact changed") + observed.add(name) + if observed != set(expected): + raise E41EvaluationBoundaryError("E41 predictor artifact coverage changed") + + items = _read_jsonl(resolved / E41_ITEMS_NAME) + _assert_truth_free(items, "E41 predictor package") + feature_manifest = _read_json(resolved / E41_FEATURE_MANIFEST_NAME) + with np.load(resolved / E41_FEATURE_ARRAYS_NAME, allow_pickle=False) as arrays: + item_ids = [str(value) for value in arrays["item_ids"]] + matrix = np.asarray(arrays["features"], dtype=np.float64) + model = _read_json(resolved / E41_MODEL_NAME) + names = feature_manifest.get("feature_names") + if ( + not isinstance(names, list) + or not all(isinstance(name, str) for name in names) + or feature_manifest.get("item_count") != len(items) + or feature_manifest.get("dimensions") != len(names) + or feature_manifest.get("feature_names_sha256") + != hashlib.sha256(_canonical_json(names)).hexdigest() + or feature_manifest.get("arrays_byte_length") + != (resolved / E41_FEATURE_ARRAYS_NAME).stat().st_size + or feature_manifest.get("arrays_sha256") + != _sha256(resolved / E41_FEATURE_ARRAYS_NAME) + or matrix.shape != (len(items), len(names)) + or item_ids != [str(row.get("item_id")) for row in items] + or not np.isfinite(matrix).all() + ): + raise E41EvaluationBoundaryError("E41 predictor feature cache is invalid") + _validate_frozen_model(model, names) + return manifest + + +def run_e41_predictor( + *, + package_root: Path, + output_root: Path, + runtime_identity: dict[str, str], +) -> E41PredictionResult: + """Run the frozen predictor without accepting an acceptance/truth input.""" + + package = validate_e41_predictor_package(package_root) + resolved = package_root.resolve(strict=True) + items = _read_jsonl(resolved / E41_ITEMS_NAME) + model = _read_json(resolved / E41_MODEL_NAME) + feature_manifest = _read_json(resolved / E41_FEATURE_MANIFEST_NAME) + names = [str(value) for value in feature_manifest["feature_names"]] + with np.load(resolved / E41_FEATURE_ARRAYS_NAME, allow_pickle=False) as arrays: + matrix = np.asarray(arrays["features"], dtype=np.float64) + predictions = predict_from_frozen_e40_model( + items=items, + feature_names=names, + feature_matrix=matrix, + model=model, + ) + _assert_truth_free(predictions, "E41 prediction result") + content_sha256 = hashlib.sha256(_canonical_json(predictions)).hexdigest() + identity = { + "schema_version": E41_PREDICTION_RESULT_SCHEMA, + "predictor_package_id": package["package_id"], + "predictor_package_identity_sha256": package["identity_sha256"], + "runtime": _validated_runtime_identity(runtime_identity), + "prediction_content_sha256": content_sha256, + "truth_material_available_to_predictor": False, + "producer_sha256": _sha256(Path(__file__).resolve(strict=True)), + "authority": _AUTHORITY, + } + identity_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest() + result_id = f"e41-predictions-{identity_sha256}" + destination = output_root.expanduser().absolute() / result_id + if destination.exists(): + return read_e41_prediction_result(destination) + destination.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + staging = destination.parent / f".{result_id}.{uuid.uuid4().hex}.tmp" + staging.mkdir(mode=0o700, exist_ok=False) + try: + _write_jsonl(staging / E41_PREDICTIONS_NAME, predictions) + manifest = { + "schema_version": E41_PREDICTION_RESULT_SCHEMA, + "result_id": result_id, + "identity_sha256": identity_sha256, + "identity": identity, + "created_at_utc": _utc_now(), + "item_count": len(predictions), + "truth_material_included": False, + "artifacts": [ + _artifact( + staging / E41_PREDICTIONS_NAME, + "truth-free-predictions", + ) + ], + "authority": _AUTHORITY, + } + _write_json(staging / E41_PACKAGE_MANIFEST_NAME, manifest) + os.replace(staging, destination) + except BaseException: + shutil.rmtree(staging, ignore_errors=True) + raise + return read_e41_prediction_result(destination) + + +def read_e41_prediction_result(root: Path) -> E41PredictionResult: + """Read and validate one immutable truth-free prediction result.""" + + resolved = root.resolve(strict=True) + manifest = _read_json(resolved / E41_PACKAGE_MANIFEST_NAME) + identity = _object(manifest.get("identity"), "E41 prediction identity") + identity_sha256 = manifest.get("identity_sha256") + if ( + manifest.get("schema_version") != E41_PREDICTION_RESULT_SCHEMA + or not isinstance(identity_sha256, str) + or hashlib.sha256(_canonical_json(identity)).hexdigest() != identity_sha256 + or manifest.get("result_id") != f"e41-predictions-{identity_sha256}" + or resolved.name != manifest.get("result_id") + or identity.get("truth_material_available_to_predictor") is not False + or manifest.get("truth_material_included") is not False + or manifest.get("authority") != _AUTHORITY + ): + raise E41EvaluationBoundaryError("E41 prediction identity is invalid") + _validate_single_artifact( + resolved, + manifest.get("artifacts"), + name=E41_PREDICTIONS_NAME, + role="truth-free-predictions", + ) + predictions = _read_jsonl(resolved / E41_PREDICTIONS_NAME) + _assert_truth_free(predictions, "E41 prediction result") + if ( + manifest.get("item_count") != len(predictions) + or identity.get("prediction_content_sha256") + != hashlib.sha256(_canonical_json(predictions)).hexdigest() + ): + raise E41EvaluationBoundaryError("E41 prediction content is invalid") + return E41PredictionResult( + result_id=resolved.name, + result_root=resolved, + manifest=manifest, + ) + + +def build_e41_visible_evaluation( + *, + prediction_root: Path, + acceptance_root: Path, + output_root: Path, +) -> E41VisibleEvaluation: + """Evaluate immutable predictions against the already-visible E37 substrate.""" + + prediction = read_e41_prediction_result(prediction_root) + acceptance = read_e37_acceptance_contract(acceptance_root) + predictions = _read_jsonl(prediction.result_root / E41_PREDICTIONS_NAME) + acceptance_rows = _read_jsonl(acceptance.result_root / E37_ITEMS_NAME) + evaluation = evaluate_visible_engineering_contract( + predictions=predictions, + acceptance_rows=acceptance_rows, + targets=_object(acceptance.contract.get("targets"), "E37 targets"), + label_provenance=_object( + acceptance.contract.get("label_provenance"), + "E37 label provenance", + ), + ) + content_sha256 = hashlib.sha256(_canonical_json(evaluation)).hexdigest() + identity = { + "schema_version": E41_VISIBLE_EVALUATION_SCHEMA, + "prediction_result_id": prediction.result_id, + "prediction_identity_sha256": prediction.manifest["identity_sha256"], + "acceptance_result_id": acceptance.result_id, + "acceptance_identity_sha256": acceptance.manifest["identity_sha256"], + "acceptance_items_sha256": _sha256(acceptance.result_root / E37_ITEMS_NAME), + "evaluation_content_sha256": content_sha256, + "evaluation_semantics": "historical-evaluated-visible-validation", + "producer_sha256": _sha256(Path(__file__).resolve(strict=True)), + "authority": _AUTHORITY, + } + identity_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest() + result_id = f"e41-visible-evaluation-{identity_sha256}" + destination = output_root.expanduser().absolute() / result_id + if destination.exists(): + return read_e41_visible_evaluation(destination) + destination.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + staging = destination.parent / f".{result_id}.{uuid.uuid4().hex}.tmp" + staging.mkdir(mode=0o700, exist_ok=False) + document = { + "schema_version": E41_VISIBLE_EVALUATION_SCHEMA, + "result_id": result_id, + "identity_sha256": identity_sha256, + "status": "completed-visible-engineering-contract-evaluation", + "evaluation": evaluation, + "decision": { + "blind_gate_eligible": False, + "independent_perception_accuracy_proved": False, + "source_scoped_engineering_contract_conformance_measured": True, + }, + "authority": _AUTHORITY, + } + try: + _write_json(staging / E41_EVALUATION_NAME, document) + manifest = { + "schema_version": E41_VISIBLE_EVALUATION_SCHEMA, + "result_id": result_id, + "identity_sha256": identity_sha256, + "identity": identity, + "created_at_utc": _utc_now(), + "blind_gate_eligible": False, + "artifacts": [ + _artifact( + staging / E41_EVALUATION_NAME, + "visible-engineering-contract-evaluation", + ) + ], + "authority": _AUTHORITY, + } + _write_json(staging / E41_PACKAGE_MANIFEST_NAME, manifest) + os.replace(staging, destination) + except BaseException: + shutil.rmtree(staging, ignore_errors=True) + raise + return read_e41_visible_evaluation(destination) + + +def read_e41_visible_evaluation(root: Path) -> E41VisibleEvaluation: + """Read and validate one immutable visible engineering evaluation.""" + + resolved = root.resolve(strict=True) + manifest = _read_json(resolved / E41_PACKAGE_MANIFEST_NAME) + identity = _object(manifest.get("identity"), "E41 evaluation identity") + identity_sha256 = manifest.get("identity_sha256") + if ( + manifest.get("schema_version") != E41_VISIBLE_EVALUATION_SCHEMA + or not isinstance(identity_sha256, str) + or hashlib.sha256(_canonical_json(identity)).hexdigest() != identity_sha256 + or manifest.get("result_id") != f"e41-visible-evaluation-{identity_sha256}" + or resolved.name != manifest.get("result_id") + or identity.get("evaluation_semantics") + != "historical-evaluated-visible-validation" + or manifest.get("blind_gate_eligible") is not False + or manifest.get("authority") != _AUTHORITY + ): + raise E41EvaluationBoundaryError("E41 visible evaluation identity is invalid") + _validate_single_artifact( + resolved, + manifest.get("artifacts"), + name=E41_EVALUATION_NAME, + role="visible-engineering-contract-evaluation", + ) + document = _read_json(resolved / E41_EVALUATION_NAME) + evaluation = _object(document.get("evaluation"), "E41 visible evaluation") + if ( + document.get("schema_version") != E41_VISIBLE_EVALUATION_SCHEMA + or document.get("result_id") != resolved.name + or document.get("identity_sha256") != identity_sha256 + or document.get("decision", {}).get("blind_gate_eligible") is not False + or hashlib.sha256(_canonical_json(evaluation)).hexdigest() + != identity.get("evaluation_content_sha256") + or document.get("authority") != _AUTHORITY + ): + raise E41EvaluationBoundaryError("E41 visible evaluation content is invalid") + return E41VisibleEvaluation( + result_id=resolved.name, + result_root=resolved, + manifest=manifest, + evaluation=document, + ) + + +def predict_from_frozen_e40_model( + *, + items: list[dict[str, Any]], + feature_names: list[str], + feature_matrix: np.ndarray[Any, Any], + model: dict[str, Any], +) -> list[dict[str, Any]]: + """Predict from frozen weights without accepting reference labels.""" + + _assert_truth_free(items, "E41 predictor items") + _validate_frozen_model(model, feature_names) + if feature_matrix.shape != (len(items), len(feature_names)): + raise E41EvaluationBoundaryError("E41 predictor feature accounting changed") + classifier = _object(model.get("classifier"), "E40 frozen classifier") + scaler = _object(classifier.get("scaler"), "E40 frozen scaler") + median = np.asarray(scaler["median"], dtype=np.float64) + scale = np.asarray(scaler["scale"], dtype=np.float64) + weights = np.asarray(classifier["weights"], dtype=np.float64) + clip = float(model["robust_clip"]) + predictions = [] + for index, (item, features) in enumerate(zip(items, feature_matrix, strict=True)): + if item.get("sequence") != index: + raise E41EvaluationBoundaryError("E41 predictor item order changed") + stratum = str(item.get("source_stratum")) + presence, confidence = _predict_product_presence( + stratum=stratum, + features=np.asarray(features, dtype=np.float64), + median=median, + scale=scale, + weights=weights, + clip=clip, + ) + predictions.append( + { + "schema_version": E41_PREDICTION_ROW_SCHEMA, + "sequence": index, + "item_id": item["item_id"], + "source_stratum": stratum, + "prediction": _project_dimensions(stratum, presence), + "presence_confidence": round(confidence, 6), + "authority": _AUTHORITY, + } + ) + _assert_truth_free(predictions, "E41 predictions") + return predictions + + +def evaluate_visible_engineering_contract( + *, + predictions: list[dict[str, Any]], + acceptance_rows: list[dict[str, Any]], + targets: dict[str, Any], + label_provenance: dict[str, Any], +) -> dict[str, Any]: + """Join truth-free predictions to the evaluated E37 validation rows.""" + + _assert_truth_free(predictions, "E41 evaluator input predictions") + prediction_by_id = _unique_by_item_id(predictions, "prediction") + acceptance_by_id = _unique_by_item_id(acceptance_rows, "acceptance") + if set(prediction_by_id) != set(acceptance_by_id): + raise E41EvaluationBoundaryError("E41 evaluation denominator changed") + joined: list[dict[str, Any]] = [] + for item_id, acceptance in acceptance_by_id.items(): + if acceptance.get("split") != "validation": + continue + prediction = prediction_by_id[item_id] + joined.append( + { + "item_id": item_id, + "source_stratum": acceptance["source_stratum"], + "severity": acceptance["severity"], + "prediction": prediction["prediction"], + "reference": acceptance["reference"], + } + ) + if not joined: + raise E41EvaluationBoundaryError("E41 visible evaluation slice is empty") + dimensions = { + dimension: _dimension_metrics( + joined, + dimension=dimension, + target=float(targets[f"{dimension}_target"]), + ) + for dimension in _DIMENSIONS + } + high_severity_failures = sum( + row["severity"] == "high" + and any( + row["prediction"][dimension] != row["reference"][dimension] + for dimension in _DIMENSIONS + ) + for row in joined + ) + false_free_claims = sum( + value == "free" + for row in predictions + for value in _object(row.get("prediction"), "E41 prediction dimensions").values() + ) + checks = { + "presence_target_reached": dimensions["presence"]["passed"], + "geometry_association_target_reached": dimensions["geometry_association"]["passed"], + "freshness_target_reached": dimensions["freshness"]["passed"], + "accounting_complete": len(joined) + == sum(row.get("split") == "validation" for row in acceptance_rows), + "false_free_claims_zero": false_free_claims == 0, + "high_severity_failures_zero": high_severity_failures == 0, + "authority_remains_diagnostic": True, + } + return { + "evaluation_semantics": "historical-evaluated-visible-validation", + "label_provenance": { + **label_provenance, + "independent_accuracy_authority": False, + }, + "metrics": { + "validation_items": len(joined), + "terminal_outcomes": len(joined), + "accounting_fraction": 1.0, + "false_free_claims": false_free_claims, + "high_severity_failures": high_severity_failures, + "dimensions": dimensions, + }, + "checks": checks, + "engineering_contract_targets_reached": all(checks.values()), + "blocking_checks": [name for name, passed in checks.items() if not passed], + "blind_gate_eligible": False, + "authority": _AUTHORITY, + } + + +def _dimension_metrics( + rows: list[dict[str, Any]], + *, + dimension: str, + target: float, +) -> dict[str, Any]: + confusion: Counter[tuple[str, str]] = Counter() + strata: dict[str, Counter[str]] = defaultdict(Counter) + correct = 0 + for row in rows: + reference = str(row["reference"][dimension]) + prediction = str(row["prediction"][dimension]) + confusion[(reference, prediction)] += 1 + matched = reference == prediction + correct += matched + strata[str(row["source_stratum"])]["correct" if matched else "incorrect"] += 1 + total = len(rows) + accuracy = correct / total + return { + "correct": correct, + "incorrect": total - correct, + "total": total, + "accuracy": round(accuracy, 6), + "target": target, + "passed": accuracy >= target, + "confusion": [ + { + "reference": reference, + "prediction": prediction, + "count": count, + } + for (reference, prediction), count in sorted( + confusion.items(), + key=lambda item: (-item[1], item[0]), + ) + ], + "by_stratum": { + stratum: { + "correct": counts["correct"], + "incorrect": counts["incorrect"], + "total": sum(counts.values()), + "accuracy": round(counts["correct"] / sum(counts.values()), 6), + } + for stratum, counts in sorted(strata.items()) + }, + } + + +def _validate_frozen_model(model: dict[str, Any], feature_names: list[str]) -> None: + classifier = _object(model.get("classifier"), "E40 frozen classifier") + scaler = _object(classifier.get("scaler"), "E40 frozen scaler") + weights = np.asarray(classifier.get("weights"), dtype=np.float64) + median = np.asarray(scaler.get("median"), dtype=np.float64) + scale = np.asarray(scaler.get("scale"), dtype=np.float64) + if ( + model.get("schema_version") != "missioncore.e40-development-product-model/v1" + or model.get("feature_names") != feature_names + or model.get("validation_labels_used_for_training") is not False + or classifier.get("type") != "deterministic-softmax" + or classifier.get("labels") != list(_LABELS) + or weights.shape != (len(feature_names) + 1, len(_LABELS)) + or median.shape != (len(feature_names),) + or scale.shape != (len(feature_names),) + or not np.isfinite(weights).all() + or not np.isfinite(median).all() + or not np.isfinite(scale).all() + or np.any(scale <= 0.0) + or not isinstance(model.get("robust_clip"), int | float) + or float(model["robust_clip"]) <= 0.0 + ): + raise E41EvaluationBoundaryError("E41 frozen E40 model is invalid") + + +def _assert_truth_free(value: object, label: str) -> None: + if isinstance(value, dict): + forbidden = set(value) & _PREDICTION_FORBIDDEN_KEYS + if forbidden: + raise E41EvaluationBoundaryError( + f"{label} contains truth/evaluation keys: {sorted(forbidden)}" + ) + for child in value.values(): + _assert_truth_free(child, label) + elif isinstance(value, list): + for child in value: + _assert_truth_free(child, label) + + +def _validated_runtime_identity(value: dict[str, str]) -> dict[str, str]: + required = {"environment_lock", "numpy", "python"} + if ( + set(value) != required + or not all(isinstance(item, str) and item for item in value.values()) + or "@sha256:" not in value["environment_lock"] + ): + raise E41EvaluationBoundaryError("E41 runtime identity is incomplete") + return dict(sorted(value.items())) + + +def _unique_by_item_id( + rows: list[dict[str, Any]], + label: str, +) -> dict[str, dict[str, Any]]: + indexed: dict[str, dict[str, Any]] = {} + for row in rows: + item_id = row.get("item_id") + if not isinstance(item_id, str) or not item_id or item_id in indexed: + raise E41EvaluationBoundaryError(f"E41 {label} item identity is invalid") + indexed[item_id] = row + return indexed + + +def _validate_single_artifact( + root: Path, + value: object, + *, + name: str, + role: str, +) -> None: + if not isinstance(value, list) or len(value) != 1 or not isinstance(value[0], dict): + raise E41EvaluationBoundaryError("E41 artifact catalog is invalid") + row = value[0] + path = root / name + if ( + row.get("path") != name + or row.get("role") != role + or not path.is_file() + or path.is_symlink() + or row.get("byte_length") != path.stat().st_size + or row.get("sha256") != _sha256(path) + ): + raise E41EvaluationBoundaryError("E41 artifact content changed") + + +def _npz_payload(item_ids: list[str], matrix: np.ndarray[Any, Any]) -> bytes: + import io + + stream = io.BytesIO() + np.savez_compressed( + stream, + item_ids=np.asarray(item_ids, dtype=f" bytes: + return (json.dumps(value, indent=2, sort_keys=True) + "\n").encode() + + +def _jsonl_payload(rows: list[dict[str, Any]]) -> bytes: + return b"".join(_canonical_json(row) + b"\n" for row in rows) + + +def _artifact(path: Path, role: str) -> dict[str, Any]: + return { + "path": path.name, + "role": role, + "byte_length": path.stat().st_size, + "sha256": _sha256(path), + } + + +def _object(value: object, label: str) -> dict[str, Any]: + if not isinstance(value, dict): + raise E41EvaluationBoundaryError(f"{label} must be an object") + return value + + +def _read_json(path: Path) -> dict[str, Any]: + value = json.loads(path.read_text(encoding="utf-8-sig")) + if not isinstance(value, dict): + raise E41EvaluationBoundaryError(f"JSON object expected: {path.name}") + return value + + +def _read_jsonl(path: Path) -> list[dict[str, Any]]: + rows = [json.loads(line) for line in path.read_text(encoding="utf-8-sig").splitlines()] + if not all(isinstance(row, dict) for row in rows): + raise E41EvaluationBoundaryError(f"JSONL object expected: {path.name}") + return rows + + +def _write_json(path: Path, value: object) -> None: + with path.open("x", encoding="utf-8", newline="\n") as stream: + json.dump(value, stream, indent=2, sort_keys=True) + stream.write("\n") + stream.flush() + os.fsync(stream.fileno()) + + +def _write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None: + with path.open("xb") as stream: + for row in rows: + stream.write(_canonical_json(row)) + stream.write(b"\n") + stream.flush() + os.fsync(stream.fileno()) + + +def _write_bytes(path: Path, value: bytes) -> None: + with path.open("xb") as stream: + stream.write(value) + stream.flush() + os.fsync(stream.fileno()) + + +def _canonical_json(value: object) -> bytes: + return json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode() + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + while chunk := stream.read(1024 * 1024): + digest.update(chunk) + return digest.hexdigest() + + +def _utc_now() -> str: + return datetime.now(UTC).isoformat(timespec="milliseconds").replace("+00:00", "Z") diff --git a/src/k1link/compute/e41_methodology_audit.py b/src/k1link/compute/e41_methodology_audit.py new file mode 100644 index 0000000..a46564f --- /dev/null +++ b/src/k1link/compute/e41_methodology_audit.py @@ -0,0 +1,901 @@ +"""Deterministic methodology audit for the historical RAVNOVES00 E37-E40 chain. + +E41 does not tune a model and does not rewrite an immutable E37 or E40 result. +It binds the exact acceptance contract, materialization, E40 worker package and +package-bound result, then makes split leakage, feature pressure, label +provenance and predictor/evaluator co-location machine-readable. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import shutil +import uuid +from collections.abc import Iterable +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path +from typing import Any, Final + +import numpy as np + +from k1link.compute.e37_acceptance_contract import ( + E37_ITEMS_NAME, + read_e37_acceptance_contract, +) +from k1link.compute.e40_perception_product_gate import ( + E40_MODEL_NAME, + E40_PREDICTIONS_NAME, + E40_REPORT_NAME, + read_e40_perception_product_gate, +) + +E41_PROFILE_SCHEMA: Final = "missioncore.e41-methodology-audit-profile/v1" +E41_RESULT_SCHEMA: Final = "missioncore.e41-methodology-audit/v1" +E41_REPORT_SCHEMA: Final = "missioncore.e41-methodology-audit-report/v1" + +E41_REPORT_NAME: Final = "methodology-audit.json" +E41_SUMMARY_NAME: Final = "methodology-audit.md" +E41_MANIFEST_NAME: Final = "manifest.json" + +_E40_PACKAGE_SCHEMA: Final = "missioncore.e40-worker-package/v1" +_FEATURE_CACHE_SCHEMA: Final = "missioncore.e40-feature-cache/v1" +_FEATURE_CACHE_MANIFEST: Final = "e40-feature-cache.json" +_FEATURE_CACHE_ARRAYS: Final = "e40-feature-vectors.npz" +_AUTHORITY: Final = { + "commands_enabled": False, + "navigation_or_safety_accepted": False, +} +_SOURCE_DERIVED_PREFIXES: Final = ( + "association=", + "camera_motion=", + "geometry=", + "label=", + "motion=", + "reason=", + "semantic_current=", + "stratum=", +) +_SOURCE_DERIVED_NAMES: Final = frozenset( + { + "detector_score", + "score", + } +) + + +class E41MethodologyAuditError(RuntimeError): + """An E41 input, policy profile, analysis, or immutable result is invalid.""" + + +@dataclass(frozen=True, slots=True) +class E41MethodologyAudit: + result_id: str + result_root: Path + manifest: dict[str, Any] + report: dict[str, Any] + + @property + def blind_gate_eligible(self) -> bool: + return self.report.get("decision", {}).get("blind_gate_eligible") is True + + +def build_e41_methodology_audit( + *, + acceptance_root: Path, + materialization_root: Path, + e40_package_root: Path, + e40_result_root: Path, + profile_path: Path, + output_root: Path, +) -> E41MethodologyAudit: + """Build or validate one immutable, source-scoped E41 methodology audit.""" + + profile_file = profile_path.resolve(strict=True) + profile = _read_json(profile_file) + _validate_profile(profile) + acceptance = read_e37_acceptance_contract(acceptance_root) + materialization = materialization_root.resolve(strict=True) + e40_result = read_e40_perception_product_gate(e40_result_root) + package, package_artifacts = _validate_e40_package(e40_package_root) + source = _object(profile.get("source"), "E41 profile source") + if ( + acceptance.result_id != source.get("acceptance_result_id") + or materialization.name != source.get("materialization_id") + or e40_result.result_id != source.get("e40_result_id") + or package.get("package_id") != source.get("e40_package_id") + ): + raise E41MethodologyAuditError("E41 source identity changed") + execution_package = _object( + _object(e40_result.report.get("execution"), "E40 execution").get("package"), + "E40 execution package", + ) + if ( + execution_package.get("mode") != "verified-worker-package" + or execution_package.get("package_id") != package["package_id"] + or execution_package.get("identity_sha256") != package["identity_sha256"] + ): + raise E41MethodologyAuditError("E41 requires the package-bound E40 result") + + acceptance_rows = _read_jsonl(acceptance.result_root / E37_ITEMS_NAME) + materialization_rows = _read_jsonl(materialization / "materialized-items.jsonl") + materialization_manifest = _read_json(materialization / "manifest.json") + reviewed_binding = _object( + _object(acceptance.manifest.get("identity"), "E37 identity").get( + "reviewed_substrate" + ), + "E37 reviewed substrate", + ) + if ( + len(acceptance_rows) != 486 + or len(materialization_rows) != 486 + or materialization_manifest.get("result_id") != materialization.name + or reviewed_binding.get("materialization_id") != materialization.name + or reviewed_binding.get("materialization_identity_sha256") + != materialization_manifest.get("identity_sha256") + or reviewed_binding.get("materialization_index_sha256") + != _sha256(materialization / "materialized-items.jsonl") + ): + raise E41MethodologyAuditError("E41 materialization binding changed") + + feature_root = ( + e40_package_root.resolve(strict=True) + / "input" + / "materialization" + / materialization.name + ) + item_ids, feature_names, feature_matrix, feature_binding = _load_feature_cache( + feature_root, + package_artifacts=package_artifacts, + ) + predictions = _read_jsonl(e40_result.result_root / E40_PREDICTIONS_NAME) + model = _read_json(e40_result.result_root / E40_MODEL_NAME) + historical_report = _read_json(e40_result.result_root / E40_REPORT_NAME) + analysis = analyze_e41_methodology( + acceptance_rows=acceptance_rows, + materialization_rows=materialization_rows, + feature_item_ids=item_ids, + feature_names=feature_names, + feature_matrix=feature_matrix, + e40_model=model, + e40_report=historical_report, + e40_predictions=predictions, + label_provenance=_object( + acceptance.contract.get("label_provenance"), + "E37 label provenance", + ), + time_block_frames=int(profile["policy"]["time_block_frames"]), + forbidden_feature_tokens=tuple(profile["policy"]["forbidden_feature_tokens"]), + ) + analysis_sha256 = hashlib.sha256(_canonical_json(analysis)).hexdigest() + identity = { + "schema_version": E41_RESULT_SCHEMA, + "source": { + "acceptance_result_id": acceptance.result_id, + "acceptance_identity_sha256": acceptance.manifest["identity_sha256"], + "acceptance_items_sha256": _sha256(acceptance.result_root / E37_ITEMS_NAME), + "materialization_id": materialization.name, + "materialization_identity_sha256": materialization_manifest["identity_sha256"], + "materialization_index_sha256": _sha256( + materialization / "materialized-items.jsonl" + ), + "e40_package_id": package["package_id"], + "e40_package_identity_sha256": package["identity_sha256"], + "e40_result_id": e40_result.result_id, + "e40_result_identity_sha256": e40_result.manifest["identity_sha256"], + "feature_cache": feature_binding, + }, + "profile": { + "profile_id": profile["profile_id"], + "sha256": _sha256(profile_file), + }, + "producer_sha256": _sha256(Path(__file__).resolve(strict=True)), + "analysis_sha256": analysis_sha256, + "authority": _AUTHORITY, + } + identity_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest() + result_id = f"e41-methodology-audit-{identity_sha256}" + destination = output_root.expanduser().absolute() / result_id + if destination.exists(): + return read_e41_methodology_audit(destination) + + report = { + "schema_version": E41_REPORT_SCHEMA, + "result_id": result_id, + "identity_sha256": identity_sha256, + "status": "completed-methodology-audit", + "evaluation_semantics": "historical-evaluated-visible-validation", + "analysis": analysis, + "decision": { + "blind_gate_eligible": analysis["policy"]["blind_gate_eligible"], + "current_146_items": "historical-evaluated-visible-validation", + "current_e40_result": "source-scoped-engineering-contract-evaluation", + "independent_perception_accuracy_proved": False, + "next_gate": ( + "separate predictor/evaluator and prepare a grouped independent truth island" + ), + }, + "authority": _AUTHORITY, + } + summary = _markdown_summary(report) + + destination.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + staging = destination.parent / f".{result_id}.{uuid.uuid4().hex}.tmp" + staging.mkdir(mode=0o700, exist_ok=False) + try: + _write_json(staging / E41_REPORT_NAME, report) + _write_text(staging / E41_SUMMARY_NAME, summary) + artifacts = [ + _artifact(staging / E41_REPORT_NAME, "machine-readable-methodology-audit"), + _artifact(staging / E41_SUMMARY_NAME, "human-readable-methodology-summary"), + ] + manifest = { + "schema_version": E41_RESULT_SCHEMA, + "result_id": result_id, + "identity_sha256": identity_sha256, + "identity": identity, + "created_at_utc": _utc_now(), + "acceptance_state": "methodology-blocked-for-blind-gate", + "blind_gate_eligible": False, + "artifacts": artifacts, + "authority": _AUTHORITY, + } + _write_json(staging / E41_MANIFEST_NAME, manifest) + os.replace(staging, destination) + except BaseException: + shutil.rmtree(staging, ignore_errors=True) + raise + return read_e41_methodology_audit(destination) + + +def read_e41_methodology_audit(root: Path) -> E41MethodologyAudit: + """Read and validate one immutable E41 methodology audit.""" + + resolved = root.resolve(strict=True) + manifest = _read_json(resolved / E41_MANIFEST_NAME) + identity = manifest.get("identity") + identity_sha256 = manifest.get("identity_sha256") + if ( + manifest.get("schema_version") != E41_RESULT_SCHEMA + or not isinstance(identity, dict) + or not isinstance(identity_sha256, str) + or hashlib.sha256(_canonical_json(identity)).hexdigest() != identity_sha256 + or manifest.get("result_id") != f"e41-methodology-audit-{identity_sha256}" + or resolved.name != manifest.get("result_id") + or manifest.get("acceptance_state") != "methodology-blocked-for-blind-gate" + or manifest.get("blind_gate_eligible") is not False + or manifest.get("authority") != _AUTHORITY + ): + raise E41MethodologyAuditError("E41 result identity is invalid") + expected = { + E41_REPORT_NAME: "machine-readable-methodology-audit", + E41_SUMMARY_NAME: "human-readable-methodology-summary", + } + _validate_artifacts(resolved, manifest.get("artifacts"), expected) + report = _read_json(resolved / E41_REPORT_NAME) + analysis = _object(report.get("analysis"), "E41 report analysis") + if ( + report.get("schema_version") != E41_REPORT_SCHEMA + or report.get("result_id") != resolved.name + or report.get("identity_sha256") != identity_sha256 + or hashlib.sha256(_canonical_json(analysis)).hexdigest() + != identity.get("analysis_sha256") + or report.get("decision", {}).get("blind_gate_eligible") is not False + or report.get("authority") != _AUTHORITY + ): + raise E41MethodologyAuditError("E41 report is invalid") + return E41MethodologyAudit( + result_id=resolved.name, + result_root=resolved, + manifest=manifest, + report=report, + ) + + +def analyze_e41_methodology( + *, + acceptance_rows: list[dict[str, Any]], + materialization_rows: list[dict[str, Any]], + feature_item_ids: list[str], + feature_names: list[str], + feature_matrix: np.ndarray[Any, Any], + e40_model: dict[str, Any], + e40_report: dict[str, Any], + e40_predictions: list[dict[str, Any]], + label_provenance: dict[str, Any], + time_block_frames: int, + forbidden_feature_tokens: tuple[str, ...], +) -> dict[str, Any]: + """Return the deterministic E41 analysis for exact in-memory inputs.""" + + if time_block_frames <= 0 or not forbidden_feature_tokens: + raise E41MethodologyAuditError("E41 methodology policy is invalid") + acceptance_by_id = _unique_by_item_id(acceptance_rows, "acceptance") + materialization_by_id = _unique_by_item_id(materialization_rows, "materialization") + if ( + set(acceptance_by_id) != set(materialization_by_id) + or len(feature_item_ids) != len(set(feature_item_ids)) + or set(feature_item_ids) != set(acceptance_by_id) + or feature_matrix.shape != (len(feature_item_ids), len(feature_names)) + or len(feature_names) != len(set(feature_names)) + or not np.isfinite(feature_matrix).all() + ): + raise E41MethodologyAuditError("E41 denominator or feature accounting changed") + ordered_acceptance = [acceptance_by_id[item_id] for item_id in feature_item_ids] + splits = [str(row.get("split")) for row in ordered_acceptance] + if set(splits) != {"development", "validation"}: + raise E41MethodologyAuditError("E41 requires development and validation rows") + + split_audit = _split_audit( + acceptance_by_id=acceptance_by_id, + materialization_by_id=materialization_by_id, + time_block_frames=time_block_frames, + ) + feature_audit = _feature_audit( + ordered_acceptance=ordered_acceptance, + feature_names=feature_names, + feature_matrix=feature_matrix, + e40_model=e40_model, + forbidden_feature_tokens=forbidden_feature_tokens, + ) + reference_co_located = any("reference" in row for row in e40_predictions) + scored_co_located = any(row.get("scored") is True for row in e40_predictions) + historical_claims = sorted( + claim + for claim in _strings(e40_report) + if "sealed" in claim.lower() or "leakage-resistant-product-gate" in claim.lower() + ) + independent_ground_truth = label_provenance.get("independent_ground_truth") is True + overlap_detected = any( + split_audit[key]["count"] > 0 + for key in ( + "exact_source_frames", + "track_ids", + "time_blocks", + "whole_track_or_scene_groups", + ) + ) + violations: list[str] = [] + if not independent_ground_truth: + violations.append("labels-are-not-independent-ground-truth") + if overlap_detected: + violations.append("development-validation-source-groups-overlap") + if reference_co_located or scored_co_located: + violations.append("prediction-and-evaluation-concerns-are-co-located") + if historical_claims: + violations.append("historical-e40-still-contains-blind-or-product-gate-claims") + if feature_audit["forbidden_features"]: + violations.append("forbidden-identity-feature-detected") + + return { + "denominator": { + "items": len(ordered_acceptance), + "development_items": splits.count("development"), + "validation_items": splits.count("validation"), + "evaluation_semantics": "historical-evaluated-visible-validation", + }, + "label_provenance": { + **label_provenance, + "accepted_semantics": "engineering-reviewed-source-scoped-substrate", + }, + "split_leakage": split_audit, + "features": feature_audit, + "predictor_evaluator_boundary": { + "prediction_rows": len(e40_predictions), + "reference_labels_present_in_prediction_artifact": reference_co_located, + "scoring_state_present_in_prediction_artifact": scored_co_located, + "physically_separated": not reference_co_located and not scored_co_located, + }, + "metric_semantics": { + "dimension_projection": e40_model.get("dimension_projection"), + "dimensions_independently_inferred": False, + "presence_geometry_accuracy_equal": ( + e40_report.get("metrics", {}) + .get("dimensions", {}) + .get("presence", {}) + .get("accuracy") + == e40_report.get("metrics", {}) + .get("dimensions", {}) + .get("geometry_association", {}) + .get("accuracy") + ), + }, + "historical_claims_requiring_status_correction": historical_claims, + "policy": { + "blind_gate_eligible": not violations, + "violations": violations, + "required_next_actions": [ + "treat-current-146-as-visible-validation", + "separate-prediction-from-evaluation-artifacts", + "freeze-grouped-independent-truth-island-before-refinement", + "keep-engineering-contract-and-independent-truth-metrics-separate", + ], + }, + "authority": _AUTHORITY, + } + + +def _split_audit( + *, + acceptance_by_id: dict[str, dict[str, Any]], + materialization_by_id: dict[str, dict[str, Any]], + time_block_frames: int, +) -> dict[str, dict[str, Any]]: + dimensions: dict[str, dict[str, set[str]]] = { + "item_ids": {"development": set(), "validation": set()}, + "exact_source_frames": {"development": set(), "validation": set()}, + "track_ids": {"development": set(), "validation": set()}, + "time_blocks": {"development": set(), "validation": set()}, + "whole_track_or_scene_groups": {"development": set(), "validation": set()}, + } + for item_id, acceptance in acceptance_by_id.items(): + split = str(acceptance.get("split")) + if split not in {"development", "validation"}: + raise E41MethodologyAuditError("E41 split row is invalid") + frame_index = acceptance.get("source_frame_index") + if not isinstance(frame_index, int) or frame_index < 0: + raise E41MethodologyAuditError("E41 source frame is invalid") + snapshot = _object( + materialization_by_id[item_id].get("e29_snapshot"), + "E41 materialization snapshot", + ) + track_id = snapshot.get("track_id") + frame_key = str(frame_index) + block_key = str(frame_index // time_block_frames) + group_key = f"track:{track_id}" if track_id is not None else f"scene:{block_key}" + dimensions["item_ids"][split].add(item_id) + dimensions["exact_source_frames"][split].add(frame_key) + dimensions["time_blocks"][split].add(block_key) + dimensions["whole_track_or_scene_groups"][split].add(group_key) + if track_id is not None: + dimensions["track_ids"][split].add(str(track_id)) + return { + name: _overlap_summary(values["development"], values["validation"]) + for name, values in dimensions.items() + } + + +def _feature_audit( + *, + ordered_acceptance: list[dict[str, Any]], + feature_names: list[str], + feature_matrix: np.ndarray[Any, Any], + e40_model: dict[str, Any], + forbidden_feature_tokens: tuple[str, ...], +) -> dict[str, Any]: + development_indices = np.asarray( + [ + index + for index, row in enumerate(ordered_acceptance) + if row.get("split") == "development" + ], + dtype=np.int64, + ) + validation_indices = np.asarray( + [ + index + for index, row in enumerate(ordered_acceptance) + if row.get("split") == "validation" + ], + dtype=np.int64, + ) + camera_development_indices = np.asarray( + [ + index + for index in development_indices + if ordered_acceptance[int(index)].get("source_stratum") == "camera-only" + ], + dtype=np.int64, + ) + if not len(camera_development_indices): + raise E41MethodologyAuditError("E41 camera-only development slice is empty") + camera_matrix = feature_matrix[camera_development_indices] + presence_labels = [ + str(ordered_acceptance[int(index)]["reference"]["presence"]) + for index in camera_development_indices + ] + rows: list[dict[str, Any]] = [] + forbidden_features: list[str] = [] + variable_features = 0 + source_derived_features = 0 + for column, name in enumerate(feature_names): + values = camera_matrix[:, column] + unique_values = int(np.unique(values).size) + variable = unique_values > 1 + variable_features += variable + classification = _classify_feature(name, forbidden_feature_tokens) + source_derived_features += classification == "source-derived" + if classification == "forbidden": + forbidden_features.append(name) + development_values = feature_matrix[development_indices, column] + validation_values = feature_matrix[validation_indices, column] + pooled_std = float(np.std(np.concatenate((development_values, validation_values)))) + standardized_shift = abs( + float(np.mean(development_values)) - float(np.mean(validation_values)) + ) / max(pooled_std, 1e-12) + rows.append( + { + "name": name, + "classification": classification, + "camera_development_unique_values": unique_values, + "camera_development_variable": variable, + "camera_development_mean": _rounded_float(float(np.mean(values))), + "camera_development_std": _rounded_float(float(np.std(values))), + "development_validation_standardized_mean_shift": _rounded_float( + standardized_shift + ), + "maximum_absolute_presence_correlation": _maximum_label_correlation( + values, + presence_labels, + ), + } + ) + model_names = e40_model.get("feature_names") + if model_names != feature_names: + raise E41MethodologyAuditError("E41 E40 model feature schema changed") + camera_items = int(e40_model.get("camera_only_training_items", -1)) + if camera_items != len(camera_development_indices): + raise E41MethodologyAuditError("E41 E40 training denominator changed") + top_shift = sorted( + ( + { + "name": row["name"], + "classification": row["classification"], + "standardized_mean_shift": row[ + "development_validation_standardized_mean_shift" + ], + } + for row in rows + ), + key=lambda row: (-float(row["standardized_mean_shift"]), str(row["name"])), + )[:12] + return { + "feature_dimensions": len(feature_names), + "camera_only_training_items": camera_items, + "camera_only_variable_features": variable_features, + "camera_only_constant_features": len(feature_names) - variable_features, + "camera_items_per_variable_feature": _rounded_float( + camera_items / max(1, variable_features) + ), + "source_derived_feature_count": source_derived_features, + "forbidden_features": forbidden_features, + "top_development_validation_shifts": top_shift, + "registry": rows, + } + + +def _classify_feature(name: str, forbidden_feature_tokens: tuple[str, ...]) -> str: + lowered = name.lower() + if any(token.lower() in lowered for token in forbidden_feature_tokens): + return "forbidden" + if name in _SOURCE_DERIVED_NAMES or name.startswith(_SOURCE_DERIVED_PREFIXES): + return "source-derived" + return "physical-observation" + + +def _maximum_label_correlation(values: np.ndarray[Any, Any], labels: list[str]) -> float | None: + if len(values) < 2 or float(np.std(values)) <= 1e-12: + return None + maximum = 0.0 + for label in sorted(set(labels)): + target = np.asarray([value == label for value in labels], dtype=np.float64) + if float(np.std(target)) <= 1e-12: + continue + correlation = float(np.corrcoef(values, target)[0, 1]) + if np.isfinite(correlation): + maximum = max(maximum, abs(correlation)) + return _rounded_float(maximum) + + +def _overlap_summary(development: set[str], validation: set[str]) -> dict[str, Any]: + overlap = sorted(development & validation, key=_natural_key) + return { + "development_unique": len(development), + "validation_unique": len(validation), + "count": len(overlap), + "sample": overlap[:20], + } + + +def _natural_key(value: str) -> tuple[int, int | str]: + try: + return (0, int(value)) + except ValueError: + return (1, value) + + +def _unique_by_item_id( + rows: list[dict[str, Any]], + label: str, +) -> dict[str, dict[str, Any]]: + indexed: dict[str, dict[str, Any]] = {} + for row in rows: + item_id = row.get("item_id") + if not isinstance(item_id, str) or not item_id or item_id in indexed: + raise E41MethodologyAuditError(f"E41 {label} item identity is invalid") + indexed[item_id] = row + return indexed + + +def _validate_e40_package(root: Path) -> tuple[dict[str, Any], dict[str, dict[str, Any]]]: + resolved = root.resolve(strict=True) + manifest = _read_json(resolved / "manifest.json") + identity = _object(manifest.get("identity"), "E40 package identity") + identity_sha256 = manifest.get("identity_sha256") + if ( + manifest.get("schema_version") != _E40_PACKAGE_SCHEMA + or not isinstance(identity_sha256, str) + or hashlib.sha256(_canonical_json(identity)).hexdigest() != identity_sha256 + or manifest.get("package_id") != f"e40-worker-package-{identity_sha256}" + or resolved.name != manifest.get("package_id") + ): + raise E41MethodologyAuditError("E41 E40 package identity is invalid") + artifacts = manifest.get("artifacts") + if not isinstance(artifacts, list): + raise E41MethodologyAuditError("E41 E40 package artifact catalog is invalid") + indexed: dict[str, dict[str, Any]] = {} + for row in artifacts: + if ( + not isinstance(row, dict) + or not isinstance((relative := row.get("path")), str) + or relative in indexed + or Path(relative).is_absolute() + or ".." in Path(relative).parts + ): + raise E41MethodologyAuditError("E41 E40 package artifact is invalid") + path = resolved / relative + if ( + not path.is_file() + or path.is_symlink() + or row.get("byte_length") != path.stat().st_size + or row.get("sha256") != _sha256(path) + ): + raise E41MethodologyAuditError("E41 E40 package artifact changed") + indexed[relative] = row + return manifest, indexed + + +def _load_feature_cache( + root: Path, + *, + package_artifacts: dict[str, dict[str, Any]], +) -> tuple[list[str], list[str], np.ndarray[Any, Any], dict[str, Any]]: + manifest_path = root / _FEATURE_CACHE_MANIFEST + arrays_path = root / _FEATURE_CACHE_ARRAYS + relative_manifest = manifest_path.relative_to(root.parents[2]).as_posix() + relative_arrays = arrays_path.relative_to(root.parents[2]).as_posix() + manifest_descriptor = package_artifacts.get(relative_manifest) + arrays_descriptor = package_artifacts.get(relative_arrays) + if ( + manifest_descriptor is None + or arrays_descriptor is None + or not manifest_path.is_file() + or not arrays_path.is_file() + ): + raise E41MethodologyAuditError("E41 package-bound feature cache is unavailable") + manifest = _read_json(manifest_path) + if ( + manifest.get("schema_version") != _FEATURE_CACHE_SCHEMA + or manifest.get("arrays_path") != _FEATURE_CACHE_ARRAYS + or manifest.get("arrays_byte_length") != arrays_path.stat().st_size + or manifest.get("arrays_sha256") != _sha256(arrays_path) + ): + raise E41MethodologyAuditError("E41 feature cache binding changed") + with np.load(arrays_path, allow_pickle=False) as arrays: + raw_item_ids = arrays["item_ids"] + feature_matrix = np.asarray(arrays["features"], dtype=np.float64) + item_ids = [str(value) for value in raw_item_ids] + feature_names = _feature_names_from_hash( + expected_hash=str(manifest.get("feature_names_sha256")), + dimensions=int(manifest.get("dimensions", -1)), + ) + if ( + len(item_ids) != int(manifest.get("items", -1)) + or feature_matrix.shape != (len(item_ids), len(feature_names)) + or not np.isfinite(feature_matrix).all() + ): + raise E41MethodologyAuditError("E41 feature cache arrays are invalid") + return item_ids, feature_names, feature_matrix, { + "manifest_sha256": str(manifest_descriptor["sha256"]), + "arrays_sha256": str(arrays_descriptor["sha256"]), + "feature_names_sha256": str(manifest["feature_names_sha256"]), + "items": len(item_ids), + "dimensions": len(feature_names), + } + + +def _feature_names_from_hash(*, expected_hash: str, dimensions: int) -> list[str]: + from k1link.compute.e40_perception_product_gate import _feature_names + + names = _feature_names() + if ( + len(names) != dimensions + or hashlib.sha256(_canonical_json(names)).hexdigest() != expected_hash + ): + raise E41MethodologyAuditError("E41 feature-name identity changed") + return names + + +def _validate_profile(profile: dict[str, Any]) -> None: + source = _object(profile.get("source"), "E41 source") + policy = _object(profile.get("policy"), "E41 policy") + tokens = policy.get("forbidden_feature_tokens") + if ( + profile.get("schema_version") != E41_PROFILE_SCHEMA + or profile.get("profile_id") != "e41-ravnoves00-methodology-audit/v1" + or not all( + isinstance(source.get(name), str) and source.get(name) + for name in ( + "acceptance_result_id", + "materialization_id", + "e40_package_id", + "e40_result_id", + ) + ) + or not isinstance(policy.get("time_block_frames"), int) + or int(policy["time_block_frames"]) <= 0 + or not isinstance(tokens, list) + or not tokens + or not all(isinstance(token, str) and token for token in tokens) + or policy.get("current_validation_semantics") + != "historical-evaluated-visible-validation" + or policy.get("independent_truth_required_for_blind") is not True + or policy.get("predictor_truth_separation_required") is not True + or profile.get("authority") != _AUTHORITY + ): + raise E41MethodologyAuditError("E41 profile is invalid") + + +def _validate_artifacts( + root: Path, + value: object, + expected: dict[str, str], +) -> None: + if not isinstance(value, list) or len(value) != len(expected): + raise E41MethodologyAuditError("E41 artifact catalog is invalid") + observed: set[str] = set() + for row in value: + if not isinstance(row, dict): + raise E41MethodologyAuditError("E41 artifact descriptor is invalid") + name = row.get("path") + path = root / str(name) + if ( + not isinstance(name, str) + or name in observed + or expected.get(name) != row.get("role") + or not path.is_file() + or path.is_symlink() + or row.get("byte_length") != path.stat().st_size + or row.get("sha256") != _sha256(path) + ): + raise E41MethodologyAuditError("E41 artifact content changed") + observed.add(name) + if observed != set(expected): + raise E41MethodologyAuditError("E41 artifact coverage changed") + + +def _markdown_summary(report: dict[str, Any]) -> str: + analysis = report["analysis"] + split = analysis["split_leakage"] + features = analysis["features"] + violations = analysis["policy"]["violations"] + lines = [ + "# E41 methodology audit", + "", + "Current evaluation semantics: `historical-evaluated-visible-validation`.", + "", + "## Decision", + "", + "The current E37/E40 chain is not eligible for a blind or independent product gate.", + "", + "## Measured split overlap", + "", + f"- Exact source frames: {split['exact_source_frames']['count']}", + f"- Track IDs: {split['track_ids']['count']}", + f"- {analysis['denominator']['validation_items']} validation items are already evaluated.", + f"- Time blocks: {split['time_blocks']['count']}", + ( + "- Whole-track-or-scene groups: " + f"{split['whole_track_or_scene_groups']['count']}" + ), + "", + "## Feature pressure", + "", + f"- Total features: {features['feature_dimensions']}", + f"- Camera-only development items: {features['camera_only_training_items']}", + f"- Variable camera-only features: {features['camera_only_variable_features']}", + ( + "- Camera items per variable feature: " + f"{features['camera_items_per_variable_feature']}" + ), + "", + "## Blocking methodology violations", + "", + *[f"- `{violation}`" for violation in violations], + "", + "No navigation, command, safety, cross-route, or independent-truth authority is granted.", + "", + ] + return "\n".join(lines) + + +def _strings(value: object) -> Iterable[str]: + if isinstance(value, str): + yield value + elif isinstance(value, dict): + for item in value.values(): + yield from _strings(item) + elif isinstance(value, list): + for item in value: + yield from _strings(item) + + +def _artifact(path: Path, role: str) -> dict[str, Any]: + return { + "path": path.name, + "role": role, + "byte_length": path.stat().st_size, + "sha256": _sha256(path), + } + + +def _object(value: object, label: str) -> dict[str, Any]: + if not isinstance(value, dict): + raise E41MethodologyAuditError(f"{label} must be an object") + return value + + +def _read_json(path: Path) -> dict[str, Any]: + value = json.loads(path.read_text(encoding="utf-8-sig")) + if not isinstance(value, dict): + raise E41MethodologyAuditError(f"JSON object expected: {path.name}") + return value + + +def _read_jsonl(path: Path) -> list[dict[str, Any]]: + rows = [json.loads(line) for line in path.read_text(encoding="utf-8-sig").splitlines()] + if not all(isinstance(row, dict) for row in rows): + raise E41MethodologyAuditError(f"JSONL object expected: {path.name}") + return rows + + +def _write_json(path: Path, value: object) -> None: + with path.open("x", encoding="utf-8", newline="\n") as stream: + json.dump(value, stream, indent=2, sort_keys=True) + stream.write("\n") + stream.flush() + os.fsync(stream.fileno()) + + +def _write_text(path: Path, value: str) -> None: + with path.open("x", encoding="utf-8", newline="\n") as stream: + stream.write(value) + stream.flush() + os.fsync(stream.fileno()) + + +def _canonical_json(value: object) -> bytes: + return json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode() + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + while chunk := stream.read(1024 * 1024): + digest.update(chunk) + return digest.hexdigest() + + +def _rounded_float(value: float) -> float: + return round(value, 9) + + +def _utc_now() -> str: + return datetime.now(UTC).isoformat(timespec="milliseconds").replace("+00:00", "Z") diff --git a/src/k1link/compute/e42_metamorphic_suite.py b/src/k1link/compute/e42_metamorphic_suite.py new file mode 100644 index 0000000..17e43e3 --- /dev/null +++ b/src/k1link/compute/e42_metamorphic_suite.py @@ -0,0 +1,490 @@ +"""Bounded E42 metamorphic checks over the frozen predictor and PointSlab. + +The suite distinguishes invariance from sensitivity. It proves predictor +independence from item naming, row ordering and chunk boundaries, and verifies +PointSlab row-order invariance plus explicit SE(3) coordinate equivariance. It +does not claim raw-sensor producer invariance or cross-route generalization. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import shutil +import uuid +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path +from typing import Any, Final + +import numpy as np + +from k1link.compute.e32_track_geometry_replay import ( + e32_track_geometry_frame, + read_e32_track_geometry_replay, +) +from k1link.compute.e41_evaluation_boundary import ( + E41_FEATURE_ARRAYS_NAME, + E41_FEATURE_MANIFEST_NAME, + E41_ITEMS_NAME, + E41_MODEL_NAME, + predict_from_frozen_e40_model, + validate_e41_predictor_package, +) +from k1link.compute.track_geometry import PointSlab + +E42_RESULT_SCHEMA: Final = "missioncore.e42-metamorphic-suite/v1" +E42_REPORT_SCHEMA: Final = "missioncore.e42-metamorphic-report/v1" +E42_REPORT_NAME: Final = "metamorphic-report.json" +E42_MANIFEST_NAME: Final = "manifest.json" + +_AUTHORITY: Final = { + "commands_enabled": False, + "navigation_or_safety_accepted": False, +} + + +class E42MetamorphicSuiteError(RuntimeError): + """An E42 source, metamorphic check, or immutable result is invalid.""" + + +@dataclass(frozen=True, slots=True) +class E42MetamorphicSuite: + result_id: str + result_root: Path + manifest: dict[str, Any] + report: dict[str, Any] + + @property + def accepted(self) -> bool: + return self.report.get("acceptance", {}).get("accepted") is True + + +def build_e42_metamorphic_suite( + *, + predictor_package_root: Path, + e32_result_root: Path, + output_root: Path, +) -> E42MetamorphicSuite: + """Run and seal one bounded E42 metamorphic suite.""" + + predictor_package = validate_e41_predictor_package(predictor_package_root) + package_root = predictor_package_root.resolve(strict=True) + e32 = read_e32_track_geometry_replay(e32_result_root) + items = _read_jsonl(package_root / E41_ITEMS_NAME) + model = _read_json(package_root / E41_MODEL_NAME) + feature_manifest = _read_json(package_root / E41_FEATURE_MANIFEST_NAME) + feature_names = [str(value) for value in feature_manifest["feature_names"]] + with np.load(package_root / E41_FEATURE_ARRAYS_NAME, allow_pickle=False) as arrays: + feature_matrix = np.asarray(arrays["features"], dtype=np.float64) + + predictor_checks = _predictor_metamorphics( + items=items, + feature_names=feature_names, + feature_matrix=feature_matrix, + model=model, + ) + point_slab_checks = _point_slab_metamorphics(e32) + checks = { + **predictor_checks, + **point_slab_checks, + } + accepted = all(checks.values()) + if not accepted: + raise E42MetamorphicSuiteError("E42 metamorphic invariant failed") + analysis = { + "classification": "bounded-current-source-contract-metamorphics", + "checks": checks, + "predictor": { + "items": len(items), + "feature_dimensions": len(feature_names), + "item_identity_used_as_model_input": False, + "path_used_as_model_input": False, + "absolute_time_origin_used_as_model_input": False, + "track_identity_used_as_model_input": False, + }, + "point_slab": point_slab_checks["point_slab_details"], + "sensitivity_not_invariance": { + "timestamp_offset": "covered-by-e31-offset-sweep-not-claimed-invariant", + "calibration_perturbation": "must-fail-binding-or-change-projection", + "density_thinning": "must-reduce-evidence-never-create-free-space", + }, + "limitations": [ + ( + "the suite proves the frozen E41 predictor boundary and one real " + "E32 PointSlab contract, not the complete raw camera/LiDAR producer" + ), + "no claim is made for another route, weather condition, rig, mount, or camera", + "SE(3) is a coordinate-contract check, not a new model-quality result", + ], + "authority": _AUTHORITY, + } + analysis_sha256 = hashlib.sha256(_canonical_json(analysis)).hexdigest() + identity = { + "schema_version": E42_RESULT_SCHEMA, + "predictor_package_id": predictor_package["package_id"], + "predictor_package_identity_sha256": predictor_package["identity_sha256"], + "e32_result_id": e32.result_id, + "e32_identity_sha256": e32.manifest["identity_sha256"], + "analysis_sha256": analysis_sha256, + "producer_sha256": _sha256(Path(__file__).resolve(strict=True)), + "authority": _AUTHORITY, + } + identity_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest() + result_id = f"e42-metamorphic-suite-{identity_sha256}" + destination = output_root.expanduser().absolute() / result_id + if destination.exists(): + return read_e42_metamorphic_suite(destination) + report = { + "schema_version": E42_REPORT_SCHEMA, + "result_id": result_id, + "identity_sha256": identity_sha256, + "status": "accepted-bounded-metamorphic-suite", + "analysis": analysis, + "acceptance": { + "accepted": True, + "checks": checks, + }, + "authority": _AUTHORITY, + } + destination.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + staging = destination.parent / f".{result_id}.{uuid.uuid4().hex}.tmp" + staging.mkdir(mode=0o700, exist_ok=False) + try: + _write_json(staging / E42_REPORT_NAME, report) + manifest = { + "schema_version": E42_RESULT_SCHEMA, + "result_id": result_id, + "identity_sha256": identity_sha256, + "identity": identity, + "created_at_utc": _utc_now(), + "acceptance_state": "accepted-bounded-metamorphic-suite", + "artifacts": [ + _artifact(staging / E42_REPORT_NAME, "metamorphic-report"), + ], + "authority": _AUTHORITY, + } + _write_json(staging / E42_MANIFEST_NAME, manifest) + os.replace(staging, destination) + except BaseException: + shutil.rmtree(staging, ignore_errors=True) + raise + return read_e42_metamorphic_suite(destination) + + +def read_e42_metamorphic_suite(root: Path) -> E42MetamorphicSuite: + """Read and validate one immutable E42 result.""" + + resolved = root.resolve(strict=True) + manifest = _read_json(resolved / E42_MANIFEST_NAME) + identity = _object(manifest.get("identity"), "E42 identity") + identity_sha256 = manifest.get("identity_sha256") + if ( + manifest.get("schema_version") != E42_RESULT_SCHEMA + or not isinstance(identity_sha256, str) + or hashlib.sha256(_canonical_json(identity)).hexdigest() != identity_sha256 + or manifest.get("result_id") != f"e42-metamorphic-suite-{identity_sha256}" + or resolved.name != manifest.get("result_id") + or manifest.get("acceptance_state") != "accepted-bounded-metamorphic-suite" + or manifest.get("authority") != _AUTHORITY + ): + raise E42MetamorphicSuiteError("E42 result identity is invalid") + artifacts = manifest.get("artifacts") + report_path = resolved / E42_REPORT_NAME + if ( + not isinstance(artifacts, list) + or len(artifacts) != 1 + or not isinstance(artifacts[0], dict) + or artifacts[0].get("path") != E42_REPORT_NAME + or artifacts[0].get("role") != "metamorphic-report" + or artifacts[0].get("byte_length") != report_path.stat().st_size + or artifacts[0].get("sha256") != _sha256(report_path) + ): + raise E42MetamorphicSuiteError("E42 artifact content changed") + report = _read_json(report_path) + analysis = _object(report.get("analysis"), "E42 analysis") + checks = _object( + _object(report.get("acceptance"), "E42 acceptance").get("checks"), + "E42 checks", + ) + if ( + report.get("schema_version") != E42_REPORT_SCHEMA + or report.get("result_id") != resolved.name + or report.get("identity_sha256") != identity_sha256 + or not checks + or not all(value is True or isinstance(value, dict) for value in checks.values()) + or report.get("acceptance", {}).get("accepted") is not True + or hashlib.sha256(_canonical_json(analysis)).hexdigest() + != identity.get("analysis_sha256") + or report.get("authority") != _AUTHORITY + ): + raise E42MetamorphicSuiteError("E42 report is invalid") + return E42MetamorphicSuite( + result_id=resolved.name, + result_root=resolved, + manifest=manifest, + report=report, + ) + + +def _predictor_metamorphics( + *, + items: list[dict[str, Any]], + feature_names: list[str], + feature_matrix: np.ndarray[Any, Any], + model: dict[str, Any], +) -> dict[str, bool]: + baseline = predict_from_frozen_e40_model( + items=items, + feature_names=feature_names, + feature_matrix=feature_matrix, + model=model, + ) + baseline_by_id = _prediction_by_id(baseline) + + renamed_items = [ + { + **item, + "item_id": f"renamed-{index:06d}", + } + for index, item in enumerate(items) + ] + renamed = predict_from_frozen_e40_model( + items=renamed_items, + feature_names=feature_names, + feature_matrix=feature_matrix, + model=model, + ) + identity_rename_invariant = _ordered_prediction_semantics(baseline) == ( + _ordered_prediction_semantics(renamed) + ) + + order = np.arange(len(items) - 1, -1, -1, dtype=np.int64) + reordered_items = [ + { + **items[int(source_index)], + "sequence": new_index, + } + for new_index, source_index in enumerate(order) + ] + reordered = predict_from_frozen_e40_model( + items=reordered_items, + feature_names=feature_names, + feature_matrix=feature_matrix[order], + model=model, + ) + row_order_invariant = baseline_by_id == _prediction_by_id(reordered) + + chunked: list[dict[str, Any]] = [] + chunk_sizes = (1, 7, 31, 97) + offset = 0 + chunk_index = 0 + while offset < len(items): + size = chunk_sizes[chunk_index % len(chunk_sizes)] + end = min(len(items), offset + size) + chunk_items = [ + { + **item, + "sequence": local_index, + } + for local_index, item in enumerate(items[offset:end]) + ] + chunked.extend( + predict_from_frozen_e40_model( + items=chunk_items, + feature_names=feature_names, + feature_matrix=feature_matrix[offset:end], + model=model, + ) + ) + offset = end + chunk_index += 1 + chunk_boundary_invariant = baseline_by_id == _prediction_by_id(chunked) + allowed_keys = { + "schema_version", + "sequence", + "item_id", + "source_stratum", + } + metadata_minimized = all(set(item) == allowed_keys for item in items) + return { + "predictor_item_identity_rename_invariant": identity_rename_invariant, + "predictor_row_order_invariant": row_order_invariant, + "predictor_chunk_boundary_invariant": chunk_boundary_invariant, + "predictor_path_time_track_metadata_absent": metadata_minimized, + } + + +def _point_slab_metamorphics(e32: Any) -> dict[str, Any]: + offsets = np.load( + e32.result_root / "frame-point-offsets.npy", + allow_pickle=False, + mmap_mode="r", + ) + sizes = np.diff(offsets) + nonempty = np.flatnonzero(sizes > 1) + if not len(nonempty): + raise E42MetamorphicSuiteError("E42 E32 source has no non-empty PointSlab") + frame_index = int(nonempty[0]) + frame = e32_track_geometry_frame(e32, frame_index) + slab = frame.point_slab + order = np.arange(slab.row_count - 1, -1, -1, dtype=np.int64) + permuted = PointSlab( + frame_index=slab.frame_index, + source_frame_index=slab.source_frame_index, + source_point_count=slab.source_point_count, + coordinate_frame=slab.coordinate_frame, + owner_keys=slab.owner_keys, + source_indices=slab.source_indices[order], + points_xyz_m=slab.points_xyz_m[order], + owner_indices=slab.owner_indices[order], + ) + row_order_invariant = _point_slab_signature(slab) == _point_slab_signature(permuted) + + angle = np.deg2rad(90.0) + rotation = np.asarray( + [ + [np.cos(angle), -np.sin(angle), 0.0], + [np.sin(angle), np.cos(angle), 0.0], + [0.0, 0.0, 1.0], + ], + dtype=np.float64, + ) + translation = np.asarray([1.0, -2.0, 0.5], dtype=np.float64) + transformed_points = ( + np.asarray(slab.points_xyz_m, dtype=np.float64) @ rotation.T + translation + ).astype(" dict[str, tuple[object, ...]]: + indexed: dict[str, tuple[object, ...]] = {} + for row in rows: + item_id = str(row["item_id"]) + if item_id in indexed: + raise E42MetamorphicSuiteError("E42 prediction item identity collided") + indexed[item_id] = ( + row["source_stratum"], + row["prediction"], + row["presence_confidence"], + ) + return indexed + + +def _ordered_prediction_semantics(rows: list[dict[str, Any]]) -> list[tuple[object, ...]]: + return [ + ( + row["source_stratum"], + row["prediction"], + row["presence_confidence"], + ) + for row in rows + ] + + +def _point_slab_signature(slab: PointSlab) -> tuple[tuple[object, ...], ...]: + order = np.argsort(slab.source_indices, kind="stable") + return tuple( + ( + int(slab.source_indices[index]), + slab.owner_keys[int(slab.owner_indices[index])], + *(round(float(value), 6) for value in slab.points_xyz_m[index]), + ) + for index in order + ) + + +def _artifact(path: Path, role: str) -> dict[str, Any]: + return { + "path": path.name, + "role": role, + "byte_length": path.stat().st_size, + "sha256": _sha256(path), + } + + +def _object(value: object, label: str) -> dict[str, Any]: + if not isinstance(value, dict): + raise E42MetamorphicSuiteError(f"{label} must be an object") + return value + + +def _read_json(path: Path) -> dict[str, Any]: + value = json.loads(path.read_text(encoding="utf-8-sig")) + if not isinstance(value, dict): + raise E42MetamorphicSuiteError(f"JSON object expected: {path.name}") + return value + + +def _read_jsonl(path: Path) -> list[dict[str, Any]]: + rows = [json.loads(line) for line in path.read_text(encoding="utf-8-sig").splitlines()] + if not all(isinstance(row, dict) for row in rows): + raise E42MetamorphicSuiteError(f"JSONL object expected: {path.name}") + return rows + + +def _write_json(path: Path, value: object) -> None: + with path.open("x", encoding="utf-8", newline="\n") as stream: + json.dump(value, stream, indent=2, sort_keys=True) + stream.write("\n") + stream.flush() + os.fsync(stream.fileno()) + + +def _canonical_json(value: object) -> bytes: + return json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode() + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + while chunk := stream.read(1024 * 1024): + digest.update(chunk) + return digest.hexdigest() + + +def _utc_now() -> str: + return datetime.now(UTC).isoformat(timespec="milliseconds").replace("+00:00", "Z") diff --git a/src/k1link/compute/e43_future_capture_protocol.py b/src/k1link/compute/e43_future_capture_protocol.py new file mode 100644 index 0000000..7205668 --- /dev/null +++ b/src/k1link/compute/e43_future_capture_protocol.py @@ -0,0 +1,518 @@ +"""Pre-register the next-route capture and independent truth-island protocol.""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import shutil +import uuid +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path +from typing import Any, Final, cast + +E43_PROFILE_SCHEMA: Final = "missioncore.e43-future-capture-profile/v1" +E43_PROTOCOL_SCHEMA: Final = "missioncore.e43-future-capture-protocol/v1" +E43_CAPTURE_MANIFEST_SCHEMA: Final = "missioncore.future-capture-manifest/v1" +E43_CANDIDATE_SCHEMA: Final = "missioncore.future-truth-candidate/v1" + +E43_PROTOCOL_NAME: Final = "future-capture-protocol.json" +E43_MANIFEST_NAME: Final = "manifest.json" + +_SHA256 = re.compile(r"^[a-f0-9]{64}$") +_IDENTIFIER = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:/-]{1,159}$") +_AUTHORITY: Final = { + "commands_enabled": False, + "navigation_or_safety_accepted": False, +} + + +class E43FutureCaptureProtocolError(RuntimeError): + """An E43 profile, capture manifest, grouped split, or result is invalid.""" + + +@dataclass(frozen=True, slots=True) +class E43FutureCaptureProtocol: + result_id: str + result_root: Path + manifest: dict[str, Any] + protocol: dict[str, Any] + + +def build_e43_future_capture_protocol( + *, + profile_path: Path, + output_root: Path, +) -> E43FutureCaptureProtocol: + """Freeze the protocol before a new physical recording exists.""" + + profile_file = profile_path.resolve(strict=True) + profile = _read_json(profile_file) + _validate_profile(profile) + protocol = { + "schema_version": E43_PROTOCOL_SCHEMA, + "profile_id": profile["profile_id"], + "capture_contract": profile["capture_contract"], + "blind_truth_contract": profile["blind_truth_contract"], + "acceptance_contract": profile["acceptance_contract"], + "decisions_frozen_before_capture": [ + "required-streams-and-source-identities", + "control-bridge-and-new-route-segment-policy", + "grouped-blind-partition-algorithm-and-seed", + "independent-human-review-requirement", + "acceptance-thresholds-and-forbidden-authority", + ], + "capture_exists": False, + "labels_exist": False, + "authority": _AUTHORITY, + } + protocol_sha256 = hashlib.sha256(_canonical_json(protocol)).hexdigest() + identity = { + "schema_version": E43_PROTOCOL_SCHEMA, + "profile_id": profile["profile_id"], + "profile_sha256": _sha256(profile_file), + "protocol_sha256": protocol_sha256, + "producer_sha256": _sha256(Path(__file__).resolve(strict=True)), + "authority": _AUTHORITY, + } + identity_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest() + result_id = f"e43-future-capture-protocol-{identity_sha256}" + destination = output_root.expanduser().absolute() / result_id + if destination.exists(): + return read_e43_future_capture_protocol(destination) + destination.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + staging = destination.parent / f".{result_id}.{uuid.uuid4().hex}.tmp" + staging.mkdir(mode=0o700, exist_ok=False) + document = { + **protocol, + "result_id": result_id, + "identity_sha256": identity_sha256, + } + try: + _write_json(staging / E43_PROTOCOL_NAME, document) + manifest = { + "schema_version": E43_PROTOCOL_SCHEMA, + "result_id": result_id, + "identity_sha256": identity_sha256, + "identity": identity, + "created_at_utc": _utc_now(), + "acceptance_state": "accepted-pre-capture-protocol", + "artifacts": [ + _artifact(staging / E43_PROTOCOL_NAME, "future-capture-protocol"), + ], + "authority": _AUTHORITY, + } + _write_json(staging / E43_MANIFEST_NAME, manifest) + os.replace(staging, destination) + except BaseException: + shutil.rmtree(staging, ignore_errors=True) + raise + return read_e43_future_capture_protocol(destination) + + +def read_e43_future_capture_protocol(root: Path) -> E43FutureCaptureProtocol: + """Read and validate one immutable pre-capture protocol.""" + + resolved = root.resolve(strict=True) + manifest = _read_json(resolved / E43_MANIFEST_NAME) + identity = _object(manifest.get("identity"), "E43 identity") + identity_sha256 = manifest.get("identity_sha256") + if ( + manifest.get("schema_version") != E43_PROTOCOL_SCHEMA + or not isinstance(identity_sha256, str) + or hashlib.sha256(_canonical_json(identity)).hexdigest() != identity_sha256 + or manifest.get("result_id") != f"e43-future-capture-protocol-{identity_sha256}" + or resolved.name != manifest.get("result_id") + or manifest.get("acceptance_state") != "accepted-pre-capture-protocol" + or manifest.get("authority") != _AUTHORITY + ): + raise E43FutureCaptureProtocolError("E43 protocol identity is invalid") + artifacts = manifest.get("artifacts") + protocol_path = resolved / E43_PROTOCOL_NAME + if ( + not isinstance(artifacts, list) + or len(artifacts) != 1 + or not isinstance(artifacts[0], dict) + or artifacts[0].get("path") != E43_PROTOCOL_NAME + or artifacts[0].get("role") != "future-capture-protocol" + or artifacts[0].get("byte_length") != protocol_path.stat().st_size + or artifacts[0].get("sha256") != _sha256(protocol_path) + ): + raise E43FutureCaptureProtocolError("E43 protocol artifact changed") + protocol = _read_json(protocol_path) + protocol_identity_payload = dict(protocol) + protocol_identity_payload.pop("result_id", None) + protocol_identity_payload.pop("identity_sha256", None) + if ( + protocol.get("schema_version") != E43_PROTOCOL_SCHEMA + or protocol.get("result_id") != resolved.name + or protocol.get("identity_sha256") != identity_sha256 + or protocol.get("capture_exists") is not False + or protocol.get("labels_exist") is not False + or hashlib.sha256(_canonical_json(protocol_identity_payload)).hexdigest() + != identity.get("protocol_sha256") + or protocol.get("authority") != _AUTHORITY + ): + raise E43FutureCaptureProtocolError("E43 protocol content is invalid") + return E43FutureCaptureProtocol( + result_id=resolved.name, + result_root=resolved, + manifest=manifest, + protocol=protocol, + ) + + +def validate_future_capture_manifest( + manifest: dict[str, Any], + *, + protocol: dict[str, Any], +) -> dict[str, Any]: + """Fail closed unless a future physical capture satisfies the frozen protocol.""" + + capture_contract = _object(protocol.get("capture_contract"), "E43 capture contract") + device = _object(manifest.get("device"), "future capture device") + capture = _object(manifest.get("capture"), "future capture facts") + streams = _object(manifest.get("streams"), "future capture streams") + segments = manifest.get("segments") + if ( + manifest.get("schema_version") != E43_CAPTURE_MANIFEST_SCHEMA + or not _identifier(manifest.get("source_session_id")) + or not _identifier(manifest.get("source_display_name")) + or manifest.get("operator_authorized") is not True + or manifest.get("authority") != _AUTHORITY + or device.get("model") != capture_contract.get("device_model") + or not all( + _sha256_value(device.get(name)) + for name in ( + "device_identity_sha256", + "calibration_sha256", + "mount_identity_sha256", + "configuration_sha256", + ) + ) + or not _identifier(device.get("firmware")) + ): + raise E43FutureCaptureProtocolError("future capture identity is invalid") + duration = capture.get("duration_seconds") + monotonic_start = capture.get("monotonic_start_seconds") + monotonic_end = capture.get("monotonic_end_seconds") + if ( + not isinstance(duration, int | float) + or isinstance(duration, bool) + or not float(capture_contract["minimum_duration_seconds"]) + <= float(duration) + <= float(capture_contract["maximum_duration_seconds"]) + or not isinstance(monotonic_start, int | float) + or isinstance(monotonic_start, bool) + or not isinstance(monotonic_end, int | float) + or isinstance(monotonic_end, bool) + or float(monotonic_end) <= float(monotonic_start) + or abs((float(monotonic_end) - float(monotonic_start)) - float(duration)) > 1.0 + or not _utc_timestamp(capture.get("started_at_utc")) + or not all( + isinstance(capture.get(name), str) and str(capture[name]).strip() + for name in ("weather", "illumination", "location_class", "operator_notes") + ) + ): + raise E43FutureCaptureProtocolError("future capture bounds are invalid") + required_streams = capture_contract.get("required_streams") + if not isinstance(required_streams, list) or set(streams) != set(required_streams): + raise E43FutureCaptureProtocolError("future capture stream set changed") + for stream_id, descriptor in streams.items(): + stream = _object(descriptor, f"future capture stream {stream_id}") + if ( + stream.get("available") is not True + or not isinstance(stream.get("item_count"), int) + or int(stream["item_count"]) <= 0 + or not isinstance(stream.get("byte_length"), int) + or int(stream["byte_length"]) <= 0 + or not _sha256_value(stream.get("sha256")) + ): + raise E43FutureCaptureProtocolError("future capture stream is incomplete") + _validate_segments( + segments, + capture_contract=capture_contract, + monotonic_start=float(monotonic_start), + monotonic_end=float(monotonic_end), + ) + segment_rows = cast(list[dict[str, Any]], segments) + return { + "accepted": True, + "source_session_id": manifest["source_session_id"], + "duration_seconds": float(duration), + "required_streams": sorted(streams), + "segments": [str(row["kind"]) for row in segment_rows], + "blind_truth_labels_available": False, + "authority": _AUTHORITY, + } + + +def assign_grouped_future_partitions( + candidates: list[dict[str, Any]], + *, + seed: str, + blind_fraction: float, +) -> dict[str, str]: + """Assign connected scene/track/time groups without cross-partition leakage.""" + + if not seed or not 0.1 <= blind_fraction <= 0.5 or len(candidates) < 2: + raise E43FutureCaptureProtocolError("future grouped split policy is invalid") + indexed: dict[str, dict[str, Any]] = {} + parent: dict[str, str] = {} + for row in candidates: + item_id = row.get("item_id") + if ( + row.get("schema_version") != E43_CANDIDATE_SCHEMA + or not isinstance(item_id, str) + or not item_id + or item_id in indexed + or not _identifier(row.get("scene_id")) + or not _identifier(row.get("time_block_id")) + or ( + row.get("track_id") is not None + and not _identifier(row.get("track_id")) + ) + or row.get("route_segment") not in {"control-bridge", "new-route"} + ): + raise E43FutureCaptureProtocolError("future truth candidate is invalid") + indexed[item_id] = row + parent[item_id] = item_id + + def find(item_id: str) -> str: + while parent[item_id] != item_id: + parent[item_id] = parent[parent[item_id]] + item_id = parent[item_id] + return item_id + + def union(left: str, right: str) -> None: + left_root = find(left) + right_root = find(right) + if left_root != right_root: + parent[max(left_root, right_root)] = min(left_root, right_root) + + group_owner: dict[tuple[str, str], str] = {} + for item_id, row in indexed.items(): + group_keys = [ + ("scene", str(row["scene_id"])), + ("time", str(row["time_block_id"])), + ] + if row.get("track_id") is not None: + group_keys.append(("track", str(row["track_id"]))) + for key in group_keys: + prior = group_owner.get(key) + if prior is None: + group_owner[key] = item_id + else: + union(item_id, prior) + components: dict[str, list[str]] = {} + for item_id in indexed: + components.setdefault(find(item_id), []).append(item_id) + if len(components) < 2: + raise E43FutureCaptureProtocolError( + "future candidates collapse into one leakage-connected component" + ) + ordered_components = sorted( + (sorted(items) for items in components.values()), + key=lambda items: hashlib.sha256( + f"{seed}:{','.join(items)}".encode() + ).hexdigest(), + ) + target = round(len(candidates) * blind_fraction) + blind_components: list[list[str]] = [] + blind_items = 0 + for component in ordered_components: + if blind_components and blind_items >= target: + break + if len(blind_components) + 1 == len(ordered_components): + break + blind_components.append(component) + blind_items += len(component) + if not blind_components: + blind_components.append(ordered_components[0]) + blind_ids = {item_id for component in blind_components for item_id in component} + assignments = { + item_id: "blind-truth" if item_id in blind_ids else "visible-diagnostic" + for item_id in indexed + } + if set(assignments.values()) != {"blind-truth", "visible-diagnostic"}: + raise E43FutureCaptureProtocolError("future grouped split is degenerate") + _verify_group_isolation(indexed, assignments) + return assignments + + +def _verify_group_isolation( + candidates: dict[str, dict[str, Any]], + assignments: dict[str, str], +) -> None: + observed: dict[tuple[str, str], str] = {} + for item_id, row in candidates.items(): + partition = assignments[item_id] + groups = [ + ("scene", str(row["scene_id"])), + ("time", str(row["time_block_id"])), + ] + if row.get("track_id") is not None: + groups.append(("track", str(row["track_id"]))) + for group in groups: + prior = observed.setdefault(group, partition) + if prior != partition: + raise E43FutureCaptureProtocolError( + "future grouped split leaks across partitions" + ) + + +def _validate_segments( + value: object, + *, + capture_contract: dict[str, Any], + monotonic_start: float, + monotonic_end: float, +) -> None: + if not isinstance(value, list) or len(value) < 2: + raise E43FutureCaptureProtocolError("future capture segments are missing") + required = { + str(row["kind"]): float(row["minimum_duration_seconds"]) + for row in capture_contract["required_segments"] + } + observed: dict[str, float] = {} + intervals: list[tuple[float, float]] = [] + for row in value: + segment = _object(row, "future capture segment") + kind = segment.get("kind") + start = segment.get("monotonic_start_seconds") + end = segment.get("monotonic_end_seconds") + if ( + kind not in required + or kind in observed + or not isinstance(start, int | float) + or isinstance(start, bool) + or not isinstance(end, int | float) + or isinstance(end, bool) + or not monotonic_start <= float(start) < float(end) <= monotonic_end + or float(end) - float(start) < required[str(kind)] + ): + raise E43FutureCaptureProtocolError("future capture segment is invalid") + observed[str(kind)] = float(end) - float(start) + intervals.append((float(start), float(end))) + if set(observed) != set(required): + raise E43FutureCaptureProtocolError("future capture segment set changed") + intervals.sort() + if any( + left[1] > right[0] + for left, right in zip(intervals, intervals[1:], strict=False) + ): + raise E43FutureCaptureProtocolError("future capture segments overlap") + + +def _validate_profile(profile: dict[str, Any]) -> None: + capture = _object(profile.get("capture_contract"), "E43 capture contract") + blind = _object(profile.get("blind_truth_contract"), "E43 blind truth contract") + acceptance = _object(profile.get("acceptance_contract"), "E43 acceptance contract") + required_segments = capture.get("required_segments") + required_streams = capture.get("required_streams") + if ( + profile.get("schema_version") != E43_PROFILE_SCHEMA + or profile.get("profile_id") != "e43-same-k1-new-route-truth-island/v1" + or capture.get("device_model") != "XGRIDS/LixelKity-K1" + or not isinstance(capture.get("minimum_duration_seconds"), int) + or not isinstance(capture.get("maximum_duration_seconds"), int) + or int(capture["minimum_duration_seconds"]) + >= int(capture["maximum_duration_seconds"]) + or not isinstance(required_streams, list) + or len(required_streams) < 4 + or len(required_streams) != len(set(required_streams)) + or not all(isinstance(value, str) and value for value in required_streams) + or not isinstance(required_segments, list) + or {row.get("kind") for row in required_segments if isinstance(row, dict)} + != {"control-bridge", "new-route"} + or blind.get("partition_strategy") + != "connected-scene-track-time-components/v1" + or not isinstance(blind.get("seed"), str) + or not 0.1 <= float(blind.get("blind_fraction", 0.0)) <= 0.5 + or blind.get("independent_human_reviewers") != 2 + or blind.get("engineering_acceptance_labels_are_truth") is not False + or acceptance.get("accounting_target") != 1.0 + or acceptance.get("maximum_false_free_claims") != 0 + or acceptance.get("maximum_high_severity_failures") != 0 + or not all( + float(acceptance.get(name, 0.0)) == 0.9 + for name in ( + "presence_target", + "geometry_association_target", + "freshness_target", + ) + ) + or profile.get("authority") != _AUTHORITY + ): + raise E43FutureCaptureProtocolError("E43 profile is invalid") + + +def _identifier(value: object) -> bool: + return isinstance(value, str) and _IDENTIFIER.fullmatch(value) is not None + + +def _sha256_value(value: object) -> bool: + return isinstance(value, str) and _SHA256.fullmatch(value) is not None + + +def _utc_timestamp(value: object) -> bool: + if not isinstance(value, str) or not value.endswith("Z"): + return False + try: + datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return False + return True + + +def _artifact(path: Path, role: str) -> dict[str, Any]: + return { + "path": path.name, + "role": role, + "byte_length": path.stat().st_size, + "sha256": _sha256(path), + } + + +def _object(value: object, label: str) -> dict[str, Any]: + if not isinstance(value, dict): + raise E43FutureCaptureProtocolError(f"{label} must be an object") + return value + + +def _read_json(path: Path) -> dict[str, Any]: + value = json.loads(path.read_text(encoding="utf-8-sig")) + if not isinstance(value, dict): + raise E43FutureCaptureProtocolError(f"JSON object expected: {path.name}") + return value + + +def _write_json(path: Path, value: object) -> None: + with path.open("x", encoding="utf-8", newline="\n") as stream: + json.dump(value, stream, indent=2, sort_keys=True) + stream.write("\n") + stream.flush() + os.fsync(stream.fileno()) + + +def _canonical_json(value: object) -> bytes: + return json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode() + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + while chunk := stream.read(1024 * 1024): + digest.update(chunk) + return digest.hexdigest() + + +def _utc_now() -> str: + return datetime.now(UTC).isoformat(timespec="milliseconds").replace("+00:00", "Z") diff --git a/src/k1link/compute/e44_data_amplification_audit.py b/src/k1link/compute/e44_data_amplification_audit.py new file mode 100644 index 0000000..09b9e58 --- /dev/null +++ b/src/k1link/compute/e44_data_amplification_audit.py @@ -0,0 +1,383 @@ +"""Content-based E44 data-amplification audit for immutable LAB artifacts.""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import shutil +import uuid +from collections import Counter, defaultdict +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path +from typing import Any, Final + +E44_RESULT_SCHEMA: Final = "missioncore.e44-data-amplification-audit/v1" +E44_REPORT_SCHEMA: Final = "missioncore.e44-data-amplification-report/v1" +E44_REPORT_NAME: Final = "data-amplification-report.json" +E44_MANIFEST_NAME: Final = "manifest.json" + +_LABEL = re.compile(r"^[a-z0-9][a-z0-9._-]{1,63}$") +_AUTHORITY: Final = { + "commands_enabled": False, + "navigation_or_safety_accepted": False, +} + + +class E44DataAmplificationAuditError(RuntimeError): + """An E44 source catalog, measurement, or immutable result is invalid.""" + + +@dataclass(frozen=True, slots=True) +class E44DataAmplificationAudit: + result_id: str + result_root: Path + manifest: dict[str, Any] + report: dict[str, Any] + + +def build_e44_data_amplification_audit( + *, + artifact_roots: dict[str, Path], + output_root: Path, +) -> E44DataAmplificationAudit: + """Measure logical bytes and exact content duplication across named roots.""" + + if len(artifact_roots) < 2: + raise E44DataAmplificationAuditError("E44 requires at least two artifact roots") + resolved_output = output_root.expanduser().absolute() + resolved_roots: dict[str, Path] = {} + for label, root in artifact_roots.items(): + if _LABEL.fullmatch(label) is None or label in resolved_roots: + raise E44DataAmplificationAuditError("E44 artifact label is invalid") + resolved = root.resolve(strict=True) + if not resolved.is_dir() or resolved.is_symlink(): + raise E44DataAmplificationAuditError("E44 artifact root is invalid") + if resolved_output == resolved or resolved_output.is_relative_to(resolved): + raise E44DataAmplificationAuditError("E44 output cannot be inside an input root") + resolved_roots[label] = resolved + + files: list[dict[str, Any]] = [] + for label, root in sorted(resolved_roots.items()): + for path in sorted(root.rglob("*")): + if path.is_symlink(): + raise E44DataAmplificationAuditError("E44 input contains a symlink") + if not path.is_file(): + continue + relative = path.relative_to(root).as_posix() + files.append( + { + "root": label, + "path": relative, + "byte_length": path.stat().st_size, + "sha256": _sha256(path), + "kind": _artifact_kind(path), + } + ) + if not files: + raise E44DataAmplificationAuditError("E44 artifact roots are empty") + analysis = analyze_data_amplification(files) + catalog_identity = [ + { + "label": label, + "root_name": root.name, + "catalog_sha256": hashlib.sha256( + _canonical_json( + [ + row + for row in files + if row["root"] == label + ] + ) + ).hexdigest(), + } + for label, root in sorted(resolved_roots.items()) + ] + analysis_sha256 = hashlib.sha256(_canonical_json(analysis)).hexdigest() + identity = { + "schema_version": E44_RESULT_SCHEMA, + "artifact_roots": catalog_identity, + "analysis_sha256": analysis_sha256, + "producer_sha256": _sha256(Path(__file__).resolve(strict=True)), + "authority": _AUTHORITY, + } + identity_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest() + result_id = f"e44-data-amplification-{identity_sha256}" + destination = resolved_output / result_id + if destination.exists(): + return read_e44_data_amplification_audit(destination) + report = { + "schema_version": E44_REPORT_SCHEMA, + "result_id": result_id, + "identity_sha256": identity_sha256, + "status": "completed-content-amplification-measurement", + "analysis": analysis, + "decision": { + "storage_migration_authorized": False, + "measurement_complete": True, + "next_gate": ( + "select deduplication/chunking only from measured dominant duplicate classes" + ), + }, + "limitations": [ + "filesystem allocation, compression ratio and browser heap are not inferred from bytes", + "identical content is detected only by exact SHA-256 equality", + "no MCAP, COPC, PDAL, Rerun or storage migration is authorized by this report", + ], + "authority": _AUTHORITY, + } + destination.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + staging = destination.parent / f".{result_id}.{uuid.uuid4().hex}.tmp" + staging.mkdir(mode=0o700, exist_ok=False) + try: + _write_json(staging / E44_REPORT_NAME, report) + manifest = { + "schema_version": E44_RESULT_SCHEMA, + "result_id": result_id, + "identity_sha256": identity_sha256, + "identity": identity, + "created_at_utc": _utc_now(), + "acceptance_state": "accepted-measurement-only", + "artifacts": [ + _artifact(staging / E44_REPORT_NAME, "data-amplification-report"), + ], + "authority": _AUTHORITY, + } + _write_json(staging / E44_MANIFEST_NAME, manifest) + os.replace(staging, destination) + except BaseException: + shutil.rmtree(staging, ignore_errors=True) + raise + return read_e44_data_amplification_audit(destination) + + +def read_e44_data_amplification_audit(root: Path) -> E44DataAmplificationAudit: + """Read and validate one immutable E44 result.""" + + resolved = root.resolve(strict=True) + manifest = _read_json(resolved / E44_MANIFEST_NAME) + identity = _object(manifest.get("identity"), "E44 identity") + identity_sha256 = manifest.get("identity_sha256") + if ( + manifest.get("schema_version") != E44_RESULT_SCHEMA + or not isinstance(identity_sha256, str) + or hashlib.sha256(_canonical_json(identity)).hexdigest() != identity_sha256 + or manifest.get("result_id") != f"e44-data-amplification-{identity_sha256}" + or resolved.name != manifest.get("result_id") + or manifest.get("acceptance_state") != "accepted-measurement-only" + or manifest.get("authority") != _AUTHORITY + ): + raise E44DataAmplificationAuditError("E44 result identity is invalid") + artifacts = manifest.get("artifacts") + report_path = resolved / E44_REPORT_NAME + if ( + not isinstance(artifacts, list) + or len(artifacts) != 1 + or not isinstance(artifacts[0], dict) + or artifacts[0].get("path") != E44_REPORT_NAME + or artifacts[0].get("role") != "data-amplification-report" + or artifacts[0].get("byte_length") != report_path.stat().st_size + or artifacts[0].get("sha256") != _sha256(report_path) + ): + raise E44DataAmplificationAuditError("E44 artifact content changed") + report = _read_json(report_path) + analysis = _object(report.get("analysis"), "E44 analysis") + if ( + report.get("schema_version") != E44_REPORT_SCHEMA + or report.get("result_id") != resolved.name + or report.get("identity_sha256") != identity_sha256 + or report.get("decision", {}).get("storage_migration_authorized") is not False + or hashlib.sha256(_canonical_json(analysis)).hexdigest() + != identity.get("analysis_sha256") + or report.get("authority") != _AUTHORITY + ): + raise E44DataAmplificationAuditError("E44 report is invalid") + return E44DataAmplificationAudit( + result_id=resolved.name, + result_root=resolved, + manifest=manifest, + report=report, + ) + + +def analyze_data_amplification(files: list[dict[str, Any]]) -> dict[str, Any]: + """Return exact content-amplification metrics for normalized file rows.""" + + if not files: + raise E44DataAmplificationAuditError("E44 file catalog is empty") + by_digest: dict[str, list[dict[str, Any]]] = defaultdict(list) + root_totals: Counter[str] = Counter() + root_files: Counter[str] = Counter() + kind_totals: Counter[str] = Counter() + kind_files: Counter[str] = Counter() + seen_root_paths: set[tuple[str, str]] = set() + for row in files: + root = row.get("root") + path = row.get("path") + byte_length = row.get("byte_length") + sha256 = row.get("sha256") + kind = row.get("kind") + if ( + not isinstance(root, str) + or _LABEL.fullmatch(root) is None + or not isinstance(path, str) + or not path + or (root, path) in seen_root_paths + or not isinstance(byte_length, int) + or byte_length < 0 + or not isinstance(sha256, str) + or len(sha256) != 64 + or not isinstance(kind, str) + or not kind + ): + raise E44DataAmplificationAuditError("E44 file row is invalid") + seen_root_paths.add((root, path)) + by_digest[sha256].append(row) + root_totals[root] += byte_length + root_files[root] += 1 + kind_totals[kind] += byte_length + kind_files[kind] += 1 + for digest, rows in by_digest.items(): + sizes = {int(row["byte_length"]) for row in rows} + if len(sizes) != 1: + raise E44DataAmplificationAuditError( + f"E44 digest {digest} has inconsistent byte lengths" + ) + + logical_bytes = sum(root_totals.values()) + unique_content_bytes = sum(int(rows[0]["byte_length"]) for rows in by_digest.values()) + duplicate_bytes = logical_bytes - unique_content_bytes + duplicate_groups = [ + { + "sha256": digest, + "byte_length": int(rows[0]["byte_length"]), + "copies": len(rows), + "roots": sorted({str(row["root"]) for row in rows}), + "paths": [ + f"{row['root']}:{row['path']}" + for row in sorted(rows, key=lambda item: (item["root"], item["path"]))[:12] + ], + "avoidable_duplicate_bytes": int(rows[0]["byte_length"]) * (len(rows) - 1), + "kind": str(rows[0]["kind"]), + } + for digest, rows in by_digest.items() + if len(rows) > 1 + ] + duplicate_groups.sort(key=_duplicate_sort_key) + roots = {} + for root in sorted(root_totals): + root_rows = [row for row in files if row["root"] == root] + root_unique = { + str(row["sha256"]): int(row["byte_length"]) + for row in root_rows + } + roots[root] = { + "files": root_files[root], + "logical_bytes": root_totals[root], + "unique_content_bytes_within_root": sum(root_unique.values()), + "duplicate_bytes_within_root": ( + root_totals[root] - sum(root_unique.values()) + ), + } + return { + "root_count": len(roots), + "file_count": len(files), + "logical_bytes": logical_bytes, + "unique_content_bytes": unique_content_bytes, + "duplicate_bytes": duplicate_bytes, + "amplification_ratio": round( + logical_bytes / max(1, unique_content_bytes), + 6, + ), + "duplicate_fraction": round( + duplicate_bytes / max(1, logical_bytes), + 6, + ), + "duplicate_content_groups": len(duplicate_groups), + "roots": roots, + "by_kind": { + kind: { + "files": kind_files[kind], + "logical_bytes": kind_totals[kind], + } + for kind in sorted(kind_totals) + }, + "largest_duplicate_groups": duplicate_groups[:30], + } + + +def _artifact_kind(path: Path) -> str: + suffix = path.suffix.lower() + if suffix in {".jpg", ".jpeg", ".png", ".webp"}: + return "camera-image" + if suffix in {".npy", ".npz", ".las", ".laz", ".pcd", ".ply"}: + return "point-or-array" + if suffix in {".mp4", ".mkv", ".mov"}: + return "video" + if suffix == ".rrd": + return "rerun" + if suffix in {".json", ".jsonl", ".md", ".txt", ".yaml", ".yml"}: + return "metadata-or-report" + if suffix in {".py", ".ps1", ".sh"}: + return "runtime-source" + return "other" + + +def _duplicate_sort_key(row: dict[str, Any]) -> tuple[int, str]: + return ( + -int(row["avoidable_duplicate_bytes"]), + str(row["sha256"]), + ) + + +def _artifact(path: Path, role: str) -> dict[str, Any]: + return { + "path": path.name, + "role": role, + "byte_length": path.stat().st_size, + "sha256": _sha256(path), + } + + +def _object(value: object, label: str) -> dict[str, Any]: + if not isinstance(value, dict): + raise E44DataAmplificationAuditError(f"{label} must be an object") + return value + + +def _read_json(path: Path) -> dict[str, Any]: + value = json.loads(path.read_text(encoding="utf-8-sig")) + if not isinstance(value, dict): + raise E44DataAmplificationAuditError(f"JSON object expected: {path.name}") + return value + + +def _write_json(path: Path, value: object) -> None: + with path.open("x", encoding="utf-8", newline="\n") as stream: + json.dump(value, stream, indent=2, sort_keys=True) + stream.write("\n") + stream.flush() + os.fsync(stream.fileno()) + + +def _canonical_json(value: object) -> bytes: + return json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode() + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + while chunk := stream.read(1024 * 1024): + digest.update(chunk) + return digest.hexdigest() + + +def _utc_now() -> str: + return datetime.now(UTC).isoformat(timespec="milliseconds").replace("+00:00", "Z") diff --git a/src/k1link/compute/inline_temporal.py b/src/k1link/compute/inline_temporal.py index 8a12cb9..7dcf3a0 100644 --- a/src/k1link/compute/inline_temporal.py +++ b/src/k1link/compute/inline_temporal.py @@ -237,7 +237,7 @@ class TemporalStabilizer: state.score = float(source["score"]) state.template = json.loads(json.dumps(source)) - normalized = json.loads(json.dumps(source)) + normalized: dict[str, Any] = json.loads(json.dumps(source)) source_id = int(source["track_id"]) normalized["track_id"] = state.canonical_id normalized["temporal_source_track_id"] = source_id @@ -371,9 +371,9 @@ class StreamingSemanticStabilizer: self.minimum_same_label_neighbors = int(profile["semantic"]["minimum_same_label_neighbors"]) self.previous_raw: np.ndarray | None = None self.previous_stabilized: np.ndarray | None = None - self.baseline_unsupported = deque(maxlen=4096) - self.stabilized_unsupported = deque(maxlen=4096) - self.processing_ms = deque(maxlen=4096) + self.baseline_unsupported: deque[float] = deque(maxlen=4096) + self.stabilized_unsupported: deque[float] = deque(maxlen=4096) + self.processing_ms: deque[float] = deque(maxlen=4096) self.frames = 0 def update(self, mask: np.ndarray) -> np.ndarray: @@ -436,21 +436,28 @@ def read_inline_profile(path: Path) -> tuple[dict[str, Any], str]: bounds = profile.get("bounds") acceptance = profile.get("acceptance") authority = profile.get("authority") + if not all( + isinstance(value, dict) + for value in ( + source, + tracking, + cuboids, + semantic, + bounds, + acceptance, + ) + ): + raise RuntimeError("LAB E23 inline temporal profile is invalid") + assert isinstance(source, dict) + assert isinstance(tracking, dict) + assert isinstance(cuboids, dict) + assert isinstance(semantic, dict) + assert isinstance(bounds, dict) + assert isinstance(acceptance, dict) if ( profile.get("schema_version") != PROFILE_SCHEMA or profile.get("mode") != "inline-shadow-qualification" or profile.get("stage") != "warm-worker-after-fusion-before-result-publication" - or not all( - isinstance(value, dict) - for value in ( - source, - tracking, - cuboids, - semantic, - bounds, - acceptance, - ) - ) or source.get("source_id") != "sensor.camera.right" or source.get("resolution") != [800, 600] or source.get("calibration_slot") != "camera_1" @@ -493,7 +500,7 @@ def stabilize_world_state( fusion_objects: list[dict[str, Any]], memory: dict[int, dict[str, Any]], ) -> dict[str, Any]: - world = json.loads(json.dumps(source)) + world: dict[str, Any] = json.loads(json.dumps(source)) source_objects = { int(item["track_id"]): item for item in source.get("objects", []) @@ -507,7 +514,7 @@ def stabilize_world_state( canonical = int(fusion["track_id"]) source_id = int(fusion.get("temporal_source_track_id", canonical)) template = source_objects.get(source_id) or memory.get(canonical) or {} - item = json.loads(json.dumps(template)) + item: dict[str, Any] = json.loads(json.dumps(template)) item.update( { "track_id": canonical, diff --git a/src/k1link/compute/lab_instances.py b/src/k1link/compute/lab_instances.py index 604807d..cf83c28 100644 --- a/src/k1link/compute/lab_instances.py +++ b/src/k1link/compute/lab_instances.py @@ -91,9 +91,11 @@ def _laboratory_method( profile_sha256: str | None, source_result_id: str, ) -> dict[str, object]: - source_identity: str | None = source_result_id.rsplit("-", 1)[-1] - if len(source_identity) != 64 or any( - character not in "0123456789abcdef" for character in source_identity + source_identity_candidate = source_result_id.rsplit("-", 1)[-1] + source_identity: str | None = source_identity_candidate + if len(source_identity_candidate) != 64 or any( + character not in "0123456789abcdef" + for character in source_identity_candidate ): source_identity = None return { diff --git a/src/k1link/compute/lidar_ground.py b/src/k1link/compute/lidar_ground.py index 2a51c96..5ddb44c 100644 --- a/src/k1link/compute/lidar_ground.py +++ b/src/k1link/compute/lidar_ground.py @@ -15,11 +15,7 @@ import numpy as np import numpy.typing as npt from k1link.ground_segmentation import ( - DEFAULT_GROUND_BENCHMARK_PROFILE, - GroundBenchmarkProfile, - GroundSegmentation, - GroundSegmenter, - LocalPercentileGroundSegmenter, + DEFAULT_GROUND_BENCHMARK_PROFILE as DEFAULT_GROUND_BENCHMARK_PROFILE, ) from k1link.ground_segmentation import ( PATCHWORKPP_SOURCE_COMMIT as PATCHWORKPP_SOURCE_COMMIT, @@ -31,7 +27,19 @@ from k1link.ground_segmentation import ( PATCHWORKPP_SOURCE_URL as PATCHWORKPP_SOURCE_URL, ) from k1link.ground_segmentation import ( - GroundSegmentationError as LidarGroundError, + GroundBenchmarkProfile as GroundBenchmarkProfile, +) +from k1link.ground_segmentation import ( + GroundSegmentation as GroundSegmentation, +) +from k1link.ground_segmentation import ( + GroundSegmentationError, +) +from k1link.ground_segmentation import ( + GroundSegmenter as GroundSegmenter, +) +from k1link.ground_segmentation import ( + LocalPercentileGroundSegmenter as LocalPercentileGroundSegmenter, ) from k1link.ground_segmentation import ( PatchworkPPGroundSegmenter as PatchworkPPGroundSegmenter, @@ -40,6 +48,8 @@ from k1link.ground_segmentation import ( from .lidar_contract import LidarContractError, sensor_frame_xyzi from .lidar_replay import LidarReplayPackV2 +LidarGroundError = GroundSegmentationError + LIDAR_GROUND_BENCHMARK_SCHEMA: Final = "missioncore.lidar-ground-benchmark/v1" LIDAR_GROUND_BENCHMARK_REPORT_SCHEMA: Final = "missioncore.lidar-ground-benchmark-report/v1" LIDAR_GROUND_ANNOTATION_TEMPLATE_SCHEMA: Final = "missioncore.lidar-ground-annotation-template/v1" diff --git a/src/k1link/compute/pipeline_telemetry.py b/src/k1link/compute/pipeline_telemetry.py new file mode 100644 index 0000000..da8dd41 --- /dev/null +++ b/src/k1link/compute/pipeline_telemetry.py @@ -0,0 +1,377 @@ +"""Native, transport-independent telemetry for Mission Core compute stages. + +The compute layer owns the meaning of a stage event. Transport ownership stays +outside the stage implementation: a durable worker can inject an already-connected +MQTT client, while a laboratory runner can record the exact same documents to JSONL. +No sink is created implicitly and telemetry never grants command authority. +""" + +from __future__ import annotations + +import json +import os +import re +import threading +import time +from collections.abc import Iterator +from contextlib import contextmanager +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path +from typing import Any, Final, Protocol + +PIPELINE_TELEMETRY_SCHEMA: Final = "missioncore.agent-pipeline-telemetry/v1" +PIPELINE_TELEMETRY_RECORD_SCHEMA: Final = "missioncore.pipeline-telemetry-record/v1" +PIPELINE_TOPIC_TEMPLATE: Final = ( + "mission-core/v1/contours/{contour_id}/agents/{agent_id}/pipeline" +) +SAFE_TOPIC_IDENTIFIER: Final = re.compile( + r"^[a-z0-9](?:[a-z0-9-]{0,62}[a-z0-9])?$" +) +MAX_TEXT_LENGTH: Final = 256 +MAX_PAYLOAD_BYTES: Final = 1024 * 1024 +STAGE_STATES: Final = frozenset({"started", "completed", "failed"}) +_AUTHORITY: Final = { + "commands_enabled": False, + "navigation_or_safety_accepted": False, +} + + +class PipelineTelemetryError(RuntimeError): + """A pipeline telemetry identity, event, or transport operation is invalid.""" + + +class PipelineTelemetrySink(Protocol): + """Transport boundary used by a compute-stage telemetry emitter.""" + + def publish(self, topic: str, payload: bytes) -> None: + """Publish one already-validated telemetry document.""" + + +class ConnectedMqttClient(Protocol): + """Minimal surface required from an already-connected MQTT client.""" + + def publish( + self, + topic: str, + payload: bytes, + qos: int, + retain: bool, + ) -> Any: + """Publish one MQTT message and return an object exposing ``rc``.""" + + +@dataclass(frozen=True, slots=True) +class PipelineTelemetryIdentity: + contour_id: str + agent_id: str + node_id: str + lab_id: str + run_id: str + source_id: str + source_package_id: str + method_id: str + request_id: str | None = None + frame_index: int | None = None + + def __post_init__(self) -> None: + for name in ("contour_id", "agent_id"): + value = getattr(self, name) + if SAFE_TOPIC_IDENTIFIER.fullmatch(value) is None: + raise PipelineTelemetryError(f"{name} is not a safe topic identifier") + for name in ( + "node_id", + "lab_id", + "run_id", + "source_id", + "source_package_id", + "method_id", + ): + _validate_text(getattr(self, name), name) + if self.request_id is not None: + _validate_text(self.request_id, "request_id") + if self.frame_index is not None and self.frame_index < 0: + raise PipelineTelemetryError("frame_index must be non-negative") + + @property + def topic(self) -> str: + return PIPELINE_TOPIC_TEMPLATE.format( + contour_id=self.contour_id, + agent_id=self.agent_id, + ) + + +@dataclass(slots=True) +class PipelineStageOutcome: + """Mutable counters a stage can complete before its terminal event is emitted.""" + + input_count: int | None = None + output_count: int | None = None + queue_wait_ms: float | None = None + + +class PipelineTelemetryEmitter: + """Emit bounded lifecycle events around actual compute work.""" + + def __init__( + self, + *, + identity: PipelineTelemetryIdentity, + sink: PipelineTelemetrySink, + clock_ns: Any = time.monotonic_ns, + ) -> None: + self.identity = identity + self.sink = sink + self._clock_ns = clock_ns + + @contextmanager + def stage( + self, + stage_id: str, + *, + input_count: int | None = None, + queue_wait_ms: float | None = None, + ) -> Iterator[PipelineStageOutcome]: + """Publish a lifecycle pair and preserve the stage exception unchanged.""" + + _validate_text(stage_id, "stage_id") + outcome = PipelineStageOutcome( + input_count=_optional_count(input_count, "input_count"), + queue_wait_ms=_optional_duration(queue_wait_ms, "queue_wait_ms"), + ) + started_ns = int(self._clock_ns()) + self._emit(stage_id=stage_id, state="started", outcome=outcome) + try: + yield outcome + except BaseException as exc: + duration_ms = max(0.0, (int(self._clock_ns()) - started_ns) / 1_000_000) + self._emit( + stage_id=stage_id, + state="failed", + outcome=outcome, + duration_ms=duration_ms, + error_type=type(exc).__name__, + ) + raise + else: + duration_ms = max(0.0, (int(self._clock_ns()) - started_ns) / 1_000_000) + self._emit( + stage_id=stage_id, + state="completed", + outcome=outcome, + duration_ms=duration_ms, + ) + + def _emit( + self, + *, + stage_id: str, + state: str, + outcome: PipelineStageOutcome, + duration_ms: float | None = None, + error_type: str | None = None, + ) -> None: + outcome.input_count = _optional_count(outcome.input_count, "input_count") + outcome.output_count = _optional_count(outcome.output_count, "output_count") + outcome.queue_wait_ms = _optional_duration( + outcome.queue_wait_ms, + "queue_wait_ms", + ) + document = build_pipeline_telemetry_document( + identity=self.identity, + stage_id=stage_id, + state=state, + duration_ms=duration_ms, + input_count=outcome.input_count, + output_count=outcome.output_count, + queue_wait_ms=outcome.queue_wait_ms, + error_type=error_type, + ) + payload = _canonical_json(document) + if len(payload) > MAX_PAYLOAD_BYTES: + raise PipelineTelemetryError("pipeline telemetry exceeds the 1 MiB contract") + self.sink.publish(self.identity.topic, payload) + + +class JsonlPipelineTelemetrySink: + """Append topic-bound telemetry records for local, auditable execution evidence.""" + + def __init__(self, path: Path) -> None: + self.path = path.expanduser().absolute() + self._lock = threading.Lock() + + def publish(self, topic: str, payload: bytes) -> None: + document = json.loads(payload.decode("utf-8")) + if not isinstance(document, dict): + raise PipelineTelemetryError("pipeline telemetry payload must be an object") + record = { + "schema_version": PIPELINE_TELEMETRY_RECORD_SCHEMA, + "topic": topic, + "payload": document, + } + encoded = _canonical_json(record) + b"\n" + with self._lock: + self.path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + descriptor = os.open( + self.path, + os.O_APPEND | os.O_CREAT | os.O_WRONLY, + 0o600, + ) + try: + os.write(descriptor, encoded) + os.fsync(descriptor) + finally: + os.close(descriptor) + + +class MqttPipelineTelemetrySink: + """Publish through a worker-owned, already-connected Paho-compatible client.""" + + def __init__(self, client: ConnectedMqttClient) -> None: + self.client = client + + def publish(self, topic: str, payload: bytes) -> None: + result = self.client.publish(topic, payload, qos=1, retain=False) + return_code = getattr(result, "rc", None) + if return_code != 0: + raise PipelineTelemetryError( + f"MQTT pipeline telemetry publish failed with code {return_code!r}" + ) + + +def build_pipeline_telemetry_document( + *, + identity: PipelineTelemetryIdentity, + stage_id: str, + state: str, + duration_ms: float | None = None, + input_count: int | None = None, + output_count: int | None = None, + queue_wait_ms: float | None = None, + error_type: str | None = None, + observed_at_utc: str | None = None, +) -> dict[str, Any]: + """Build the canonical document accepted by the telemetry-plane normalizer.""" + + _validate_text(stage_id, "stage_id") + if state not in STAGE_STATES: + raise PipelineTelemetryError("stage telemetry state is invalid") + duration_ms = _optional_duration(duration_ms, "duration_ms") + input_count = _optional_count(input_count, "input_count") + output_count = _optional_count(output_count, "output_count") + queue_wait_ms = _optional_duration(queue_wait_ms, "queue_wait_ms") + if state == "started" and duration_ms is not None: + raise PipelineTelemetryError("a started stage cannot have a duration") + if state != "started" and duration_ms is None: + raise PipelineTelemetryError("a terminal stage requires duration_ms") + if error_type is not None: + _validate_text(error_type, "error_type") + if state == "failed" and error_type is None: + raise PipelineTelemetryError("a failed stage requires error_type") + if state != "failed" and error_type is not None: + raise PipelineTelemetryError("only a failed stage can have error_type") + + tags = { + "agent_id": identity.agent_id, + "contour_id": identity.contour_id, + "lab_id": identity.lab_id, + "method_id": identity.method_id, + "node_id": identity.node_id, + "run_id": identity.run_id, + "source_id": identity.source_id, + "source_package_id": identity.source_package_id, + "stage_id": stage_id, + "stage_state": state, + } + if identity.request_id is not None: + tags["request_id"] = identity.request_id + stage_metric = { + "elapsed_seconds": ( + round(duration_ms / 1000.0, 9) if duration_ms is not None else None + ), + "activations": 1, + "input_count": input_count, + "output_count": output_count, + "queue_wait_ms": queue_wait_ms, + } + event = { + "stage_id": stage_id, + "state": state, + "duration_ms": duration_ms, + "input_count": input_count, + "output_count": output_count, + "queue_wait_ms": queue_wait_ms, + "error_type": error_type, + } + document: dict[str, Any] = { + "schema_version": PIPELINE_TELEMETRY_SCHEMA, + "observed_at_utc": observed_at_utc or _utc_now(), + "node_id": identity.node_id, + "lab_id": identity.lab_id, + "run_id": identity.run_id, + "source_id": identity.source_id, + "source_package_id": identity.source_package_id, + "method_id": identity.method_id, + "stage_id": stage_id, + "stage_state": state, + "tags": tags, + "payload": { + "state": ( + "busy" + if state == "started" + else ("failed" if state == "failed" else "ready") + ), + "current_stage": stage_id, + "active_request_id": identity.request_id or identity.run_id, + "active_stages": [stage_id] if state == "started" else [], + "stage_metrics": {stage_id: stage_metric}, + "event": event, + }, + "authority": _AUTHORITY, + } + if identity.request_id is not None: + document["request_id"] = identity.request_id + if identity.frame_index is not None: + document["frame_index"] = identity.frame_index + return document + + +def _validate_text(value: object, name: str) -> str: + if ( + not isinstance(value, str) + or not value + or len(value) > MAX_TEXT_LENGTH + or any(ord(character) < 32 for character in value) + ): + raise PipelineTelemetryError(f"{name} is invalid") + return value + + +def _optional_count(value: int | None, name: str) -> int | None: + if value is None: + return None + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise PipelineTelemetryError(f"{name} must be a non-negative integer") + return value + + +def _optional_duration(value: float | None, name: str) -> float | None: + if value is None: + return None + normalized = float(value) + if normalized < 0 or normalized != normalized or normalized == float("inf"): + raise PipelineTelemetryError(f"{name} must be finite and non-negative") + return round(normalized, 6) + + +def _canonical_json(value: object) -> bytes: + return json.dumps( + value, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + + +def _utc_now() -> str: + return datetime.now(UTC).isoformat(timespec="microseconds").replace("+00:00", "Z") diff --git a/src/k1link/compute/temporal_stability.py b/src/k1link/compute/temporal_stability.py index dc8ce86..42f315a 100644 --- a/src/k1link/compute/temporal_stability.py +++ b/src/k1link/compute/temporal_stability.py @@ -261,7 +261,7 @@ class TemporalStabilizer: state.score = float(source["score"]) state.template = json.loads(json.dumps(source)) - normalized = json.loads(json.dumps(source)) + normalized: dict[str, Any] = json.loads(json.dumps(source)) source_id = int(source["track_id"]) normalized["track_id"] = state.canonical_id normalized["temporal_source_track_id"] = source_id @@ -405,13 +405,20 @@ def read_profile(path: Path) -> tuple[dict[str, Any], str]: bounds = profile.get("bounds") acceptance = profile.get("acceptance") authority = profile.get("authority") + if not all( + isinstance(value, dict) + for value in (source, tracking, cuboids, semantic, bounds, acceptance) + ): + raise SessionIntegrityError("LAB E22 temporal profile is invalid") + assert isinstance(source, dict) + assert isinstance(tracking, dict) + assert isinstance(cuboids, dict) + assert isinstance(semantic, dict) + assert isinstance(bounds, dict) + assert isinstance(acceptance, dict) if ( profile.get("schema_version") != PROFILE_SCHEMA or profile.get("mode") != "recorded-streaming-qualification" - or not all( - isinstance(value, dict) - for value in (source, tracking, cuboids, semantic, bounds, acceptance) - ) or source.get("source_id") != "sensor.camera.right" or source.get("resolution") != [800, 600] or semantic.get("mode") @@ -575,7 +582,7 @@ def build_temporal_stability_result( semantic_metrics["stabilized_unsupported_change_fraction"]["mean"], ), } - runtime = { + runtime: dict[str, Any] = { "camera_frame_processing_ms": _percentiles(frame_ms), "semantic_frame_processing_ms": _percentiles(semantic_ms), "peak_track_states": stabilizer.peak_states, @@ -586,7 +593,7 @@ def build_temporal_stability_result( + 600 * 800 ), } - criteria = profile["acceptance"] + criteria: dict[str, Any] = profile["acceptance"] checks = { "minimum_2d_acceleration_reduction": reductions[ "tracking_2d_acceleration_p95_fraction" @@ -791,7 +798,7 @@ def _stabilize_world( fusion_objects: list[dict[str, Any]], memory: dict[int, dict[str, Any]], ) -> dict[str, Any]: - world = json.loads(json.dumps(source)) + world: dict[str, Any] = json.loads(json.dumps(source)) source_objects = { int(item["track_id"]): item for item in source.get("objects", []) @@ -804,7 +811,7 @@ def _stabilize_world( canonical = int(fusion["track_id"]) source_id = int(fusion.get("temporal_source_track_id", canonical)) template = source_objects.get(source_id) or memory.get(canonical) or {} - item = json.loads(json.dumps(template)) + item: dict[str, Any] = json.loads(json.dumps(template)) item.update( { "track_id": canonical, @@ -861,10 +868,10 @@ def _quality_metrics( ) acceleration: list[float] = [] size_steps: list[float] = [] - for values in tracks.values(): + for track_values in tracks.values(): previous_step: np.ndarray | None = None for (left_frame, left_box), (right_frame, right_box) in zip( - values, values[1:], strict=False + track_values, track_values[1:], strict=False ): if right_frame - left_frame != 1: previous_step = None @@ -882,8 +889,12 @@ def _quality_metrics( cuboid_size_steps: list[float] = [] yaw_steps: list[float] = [] gaps = 0 - for values in cuboids.values(): - for left, right in zip(values, values[1:], strict=False): + for cuboid_values in cuboids.values(): + for left, right in zip( + cuboid_values, + cuboid_values[1:], + strict=False, + ): if right[0] - left[0] != 1: gaps += 1 continue diff --git a/src/k1link/web/advanced_laboratory_api.py b/src/k1link/web/advanced_laboratory_api.py index ad6097f..c4ff80e 100644 --- a/src/k1link/web/advanced_laboratory_api.py +++ b/src/k1link/web/advanced_laboratory_api.py @@ -49,6 +49,11 @@ from k1link.compute.e39_perception_refinement import ( E39PerceptionRefinementError, read_e39_perception_refinement, ) +from k1link.compute.e40_perception_product_gate import ( + E40PerceptionProductGate, + E40PerceptionProductGateError, + read_e40_perception_product_gate, +) LABORATORY_ADVANCED_CATALOG_SCHEMA: Final = ( "missioncore.laboratory-advanced-catalog/v1" @@ -62,6 +67,7 @@ _E35_RESULT_ID = re.compile(r"^e35-degradation-recovery-[a-f0-9]{64}$") _E37_RESULT_ID = re.compile(r"^e37-ravnoves-acceptance-[a-f0-9]{64}$") _E38_RESULT_ID = re.compile(r"^e38-perception-baseline-[a-f0-9]{64}$") _E39_RESULT_ID = re.compile(r"^e39-perception-refinement-[a-f0-9]{64}$") +_E40_RESULT_ID = re.compile(r"^e40-perception-product-gate-[a-f0-9]{64}$") RootProvider = Callable[[], Path | None] @@ -148,6 +154,15 @@ def _read_e39_cached( return read_e39_perception_refinement(Path(root_text)) +@lru_cache(maxsize=16) +def _read_e40_cached( + root_text: str, + signature: tuple[int, ...], +) -> E40PerceptionProductGate: + del signature + return read_e40_perception_product_gate(Path(root_text)) + + def _configured_root(provider: RootProvider) -> Path | None: value = provider() if value is None: @@ -659,6 +674,46 @@ def _project_e39(result: E39PerceptionRefinement) -> dict[str, object]: } +def _project_e40(result: E40PerceptionProductGate) -> dict[str, object]: + identity = _object(result.manifest.get("identity"), "E40 identity") + source = _object(identity.get("source"), "E40 source") + execution = _object(identity.get("execution"), "E40 execution") + profile = _object(identity.get("profile"), "E40 profile") + metrics = _object(result.report.get("metrics"), "E40 metrics") + dimensions = _object(metrics.get("dimensions"), "E40 dimensions") + quality_gate = _object(result.report.get("quality_gate"), "E40 gate") + development_cv = _object( + result.report.get("development_cross_validation"), + "E40 development CV", + ) + return { + "result_id": result.result_id, + "created_at_utc": result.manifest.get("created_at_utc"), + "source_session_id": source.get("session_id"), + "source_display_name": source.get("display_name"), + "status": result.report.get("status"), + "profile_id": profile.get("profile_id"), + "worker_node": execution.get("worker_node"), + "quality_gate_passed": quality_gate.get("passed"), + "development_cross_validation": copy.deepcopy(development_cv), + "metrics": { + "development_items": metrics.get("development_items"), + "validation_items": metrics.get("validation_items"), + "terminal_outcomes": metrics.get("terminal_outcomes"), + "accounting_fraction": metrics.get("accounting_fraction"), + "false_free_claims": metrics.get("false_free_claims"), + "high_severity_failures": metrics.get("high_severity_failures"), + "dimensions": copy.deepcopy(dimensions), + }, + "blocking_checks": copy.deepcopy(quality_gate.get("blocking_checks")), + "method": copy.deepcopy(result.report.get("method")), + "decision": copy.deepcopy(result.report.get("decision")), + "limitations": copy.deepcopy(result.report.get("limitations")), + "authority": copy.deepcopy(result.report.get("authority")), + "access": "read-only", + } + + def _empty_catalog(configured: bool) -> dict[str, object]: return { "schema_version": LABORATORY_ADVANCED_CATALOG_SCHEMA, @@ -680,6 +735,7 @@ def build_advanced_laboratory_router( e37_root_provider: RootProvider = lambda: None, e38_root_provider: RootProvider = lambda: None, e39_root_provider: RootProvider = lambda: None, + e40_root_provider: RootProvider = lambda: None, ) -> APIRouter: router = APIRouter(prefix="/api/v1/laboratory", tags=["laboratory"]) @@ -975,4 +1031,37 @@ def build_advanced_laboratory_router( "invalid_total": invalid_total, } + @router.get("/e40/results") + def list_e40_results( + limit: int = Query(default=1, ge=1, le=10), + ) -> dict[str, object]: + root = _configured_root(e40_root_provider) + if root is None: + return _empty_catalog(False) + candidates = _candidates(root, _E40_RESULT_ID) + items: list[dict[str, object]] = [] + invalid_total = 0 + for candidate in candidates: + try: + result = _read_e40_cached( + str(candidate.resolve()), + _result_signature(candidate), + ) + if len(items) < limit: + items.append(_project_e40(result)) + except ( + E40PerceptionProductGateError, + KeyError, + OSError, + TypeError, + ValueError, + ): + invalid_total += 1 + return { + **_empty_catalog(True), + "items": items, + "candidate_total": len(candidates), + "invalid_total": invalid_total, + } + return router diff --git a/src/k1link/web/app.py b/src/k1link/web/app.py index 8b2341e..e39e9c0 100644 --- a/src/k1link/web/app.py +++ b/src/k1link/web/app.py @@ -557,6 +557,13 @@ app.include_router( / "e39" / "results" ), + e40_root_provider=lambda: ( + REPOSITORY_ROOT + / ".runtime" + / "compute-experiments" + / "e40" + / "results" + ), ) ) app.include_router( diff --git a/src/k1link/web/compute_contour_api.py b/src/k1link/web/compute_contour_api.py index 4b46128..fb6a8d3 100644 --- a/src/k1link/web/compute_contour_api.py +++ b/src/k1link/web/compute_contour_api.py @@ -47,6 +47,7 @@ class ComputeContour(StrictModel): mqtt_host: str = Field(default="127.0.0.1", min_length=1, max_length=253) mqtt_port: int = Field(default=1883, ge=1, le=65535) telemetry_poll_interval_seconds: int = Field(default=3, ge=1, le=60) + mqtt_publish_interval_seconds: int = Field(default=2, ge=1, le=60) revision: int = Field(default=0, ge=0) updated_at_utc: str | None = None @@ -92,6 +93,7 @@ class ComputeContourCreate(StrictModel): mqtt_host: str = Field(default="127.0.0.1", min_length=1, max_length=253) mqtt_port: int = Field(default=1883, ge=1, le=65535) telemetry_poll_interval_seconds: int = Field(default=3, ge=1, le=60) + mqtt_publish_interval_seconds: int = Field(default=2, ge=1, le=60) @field_validator("display_name") @classmethod @@ -127,6 +129,7 @@ def default_compute_contour() -> ComputeContour: mqtt_host="127.0.0.1", mqtt_port=1883, telemetry_poll_interval_seconds=3, + mqtt_publish_interval_seconds=2, ) @@ -158,6 +161,9 @@ class ComputeContourStore: telemetry_poll_interval_seconds=( request.telemetry_poll_interval_seconds ), + mqtt_publish_interval_seconds=( + request.mqtt_publish_interval_seconds + ), revision=0, updated_at_utc=_utc_now(), ) @@ -186,6 +192,9 @@ class ComputeContourStore: "telemetry_poll_interval_seconds": ( request.telemetry_poll_interval_seconds ), + "mqtt_publish_interval_seconds": ( + request.mqtt_publish_interval_seconds + ), "revision": current.revision + 1, "updated_at_utc": _utc_now(), } @@ -260,6 +269,9 @@ def _agent_install_document(contour: ComputeContour) -> dict[str, object]: "MISSIONCORE_MQTT_HOST": contour.mqtt_host, "MISSIONCORE_MQTT_PORT": str(contour.mqtt_port), "MISSIONCORE_MQTT_USERNAME": contour.agent_id, + "MISSIONCORE_TELEMETRY_INTERVAL": ( + f"{contour.mqtt_publish_interval_seconds}s" + ), } if contour.platform == "windows": command = ( diff --git a/src/k1link/web/e30_engineering_api.py b/src/k1link/web/e30_engineering_api.py index b83daa4..e5db774 100644 --- a/src/k1link/web/e30_engineering_api.py +++ b/src/k1link/web/e30_engineering_api.py @@ -590,9 +590,21 @@ def build_e30_engineering_router( result_id=result_id, ) rows_by_id = {row["item_id"]: row for row in rows} + exception_rows = catalog_item.get("human_exceptions") + if ( + not isinstance(exception_rows, list) + or not all( + isinstance(value, dict) + and isinstance(value.get("item_id"), str) + for value in exception_rows + ) + ): + raise E30EngineeringEvidenceError( + "engineering exception catalog is invalid" + ) exception_ids = [ - value["item_id"] - for value in catalog_item["human_exceptions"] + str(value["item_id"]) + for value in exception_rows ] if any(item_id not in rows_by_id for item_id in exception_ids): raise E30EngineeringEvidenceError( diff --git a/src/k1link/web/e30_human_review_api.py b/src/k1link/web/e30_human_review_api.py index 427fd61..e0c9f6d 100644 --- a/src/k1link/web/e30_human_review_api.py +++ b/src/k1link/web/e30_human_review_api.py @@ -94,9 +94,21 @@ def build_e30_human_review_router( subject.item_id: subject for subject in source_substrate.subjects } + exception_rows = generation.get("human_exceptions") + if ( + not isinstance(exception_rows, list) + or not all( + isinstance(value, dict) + and isinstance(value.get("item_id"), str) + for value in exception_rows + ) + ): + raise E30HumanReviewValidationError( + "engineering exception catalog is invalid" + ) exception_ids = [ - value["item_id"] - for value in generation["human_exceptions"] + str(value["item_id"]) + for value in exception_rows ] if ( not exception_ids diff --git a/src/k1link/web/system_telemetry_api.py b/src/k1link/web/system_telemetry_api.py index d0ee7bb..c8b4050 100644 --- a/src/k1link/web/system_telemetry_api.py +++ b/src/k1link/web/system_telemetry_api.py @@ -16,11 +16,13 @@ from collections import deque from collections.abc import Callable from datetime import UTC, datetime from pathlib import Path -from typing import Any, Final +from typing import Any, Final, Protocol from fastapi import APIRouter, HTTPException, Query from pydantic import BaseModel, ConfigDict, Field, field_validator +from k1link.web.compute_contour_api import ComputeContour, ComputeContourStore + PROFILE_SCHEMA: Final = "missioncore.worker-connection-profile/v1" TELEMETRY_SCHEMA: Final = "missioncore.worker-telemetry/v1" PROBE_SCHEMA: Final = "missioncore.worker-probe/v1" @@ -79,6 +81,14 @@ RootProvider = Callable[[], Path] ProbeRunner = Callable[["WorkerConnectionProfile"], dict[str, Any]] +class WorkerProfileStoreContract(Protocol): + def read(self) -> WorkerConnectionProfile: + """Return the current worker-shaped connection profile.""" + + def save(self, request: WorkerConnectionProfilePut) -> WorkerConnectionProfile: + """Persist one reviewed profile update.""" + + class WorkerConnectionProfile(BaseModel): model_config = ConfigDict(extra="forbid", frozen=True) @@ -603,16 +613,28 @@ def _agent_raw_document(document: dict[str, Any]) -> dict[str, Any]: return raw -def run_worker_agent_probe(profile: WorkerConnectionProfile) -> dict[str, Any]: +def run_worker_agent_probe( + profile: WorkerConnectionProfile, + *, + contour_id: str | None = None, + agent_id: str | None = None, +) -> dict[str, Any]: started = time.perf_counter() base_url = os.environ.get( "MISSIONCORE_TELEMETRY_QUERY_URL", DEFAULT_TELEMETRY_QUERY_URL, ).rstrip("/") - contour_id = os.environ.get("MISSIONCORE_TELEMETRY_CONTOUR_ID", "worker-006") - agent_id = os.environ.get("MISSIONCORE_TELEMETRY_AGENT_ID", "worker-006") + resolved_contour_id = contour_id or os.environ.get( + "MISSIONCORE_TELEMETRY_CONTOUR_ID", + "worker-006", + ) + resolved_agent_id = agent_id or os.environ.get( + "MISSIONCORE_TELEMETRY_AGENT_ID", + "worker-006", + ) url = ( - f"{base_url}/v1/contours/{contour_id}/agents/{agent_id}/latest" + f"{base_url}/v1/contours/{resolved_contour_id}/agents/" + f"{resolved_agent_id}/latest" "?max_age_seconds=30" ) try: @@ -898,7 +920,7 @@ def _pipeline_document(raw: dict[str, Any]) -> dict[str, Any]: class WorkerTelemetryService: def __init__( self, - store: WorkerProfileStore, + store: WorkerProfileStoreContract, probe_runner: ProbeRunner = run_worker_probe, *, telemetry_probe_runner: ProbeRunner | None = None, @@ -911,7 +933,9 @@ class WorkerTelemetryService: self._lock = threading.Lock() self._cached_at = 0.0 self._cached: dict[str, Any] | None = None - self._previous_network: tuple[float, float, float] | None = None + self._previous_network: tuple[str, float, float, float] | None = None + self._latest_network_rates: tuple[float | None, float | None] = (None, None) + self._last_history_observed_at: str | None = None self._history: deque[dict[str, Any]] = deque(maxlen=300) def profile_document(self) -> dict[str, Any]: @@ -948,6 +972,8 @@ class WorkerTelemetryService: self._cached = None self._cached_at = 0 self._previous_network = None + self._latest_network_rates = (None, None) + self._last_history_observed_at = None self._history.clear() return { **self.profile_document(), @@ -1007,25 +1033,36 @@ class WorkerTelemetryService: for item in _items(raw.get("network")) if isinstance(item, dict) ] - received = sum( + received = float( + sum( value for item in interfaces if (value := _number(item.get("received_bytes"))) is not None + ) ) - sent = sum( + sent = float( + sum( value for item in interfaces if (value := _number(item.get("sent_bytes"))) is not None + ) ) - receive_rate: float | None = None - send_rate: float | None = None - if self._previous_network is not None: - previous_at, previous_received, previous_sent = self._previous_network - elapsed = monotonic_now - previous_at - if elapsed > 0 and received >= previous_received and sent >= previous_sent: - receive_rate = (received - previous_received) / elapsed - send_rate = (sent - previous_sent) / elapsed - self._previous_network = (monotonic_now, received, sent) + raw_observed_at = raw.get("observed_at_utc") + observed_at: str = ( + raw_observed_at if isinstance(raw_observed_at, str) else _utc_now() + ) + receive_rate, send_rate = self._latest_network_rates + if observed_at != self._last_history_observed_at: + receive_rate = None + send_rate = None + if self._previous_network is not None: + _, previous_at, previous_received, previous_sent = self._previous_network + elapsed = monotonic_now - previous_at + if elapsed > 0 and received >= previous_received and sent >= previous_sent: + receive_rate = (received - previous_received) / elapsed + send_rate = (sent - previous_sent) / elapsed + self._previous_network = (observed_at, monotonic_now, received, sent) + self._latest_network_rates = (receive_rate, send_rate) raw_stats = _mapping(raw.get("docker_stats")) raw_states = _mapping(raw.get("container_states")) runtimes = [ @@ -1077,7 +1114,7 @@ class WorkerTelemetryService: }, } history_row = { - "observed_at_utc": raw.get("observed_at_utc") or _utc_now(), + "observed_at_utc": observed_at, "cpu_percent": _number(_mapping(raw.get("cpu")).get("load_percent")), "memory_percent": memory_used_percent, "gpu_percent": _number(gpu.get("utilization_percent")), @@ -1085,7 +1122,9 @@ class WorkerTelemetryService: "network_receive_bytes_per_second": receive_rate, "network_send_bytes_per_second": send_rate, } - self._history.append(history_row) + if observed_at != self._last_history_observed_at: + self._history.append(history_row) + self._last_history_observed_at = observed_at return { "schema_version": TELEMETRY_SCHEMA, "profile": profile.model_dump(mode="json"), @@ -1154,8 +1193,49 @@ def build_system_telemetry_router( probe_runner, telemetry_probe_runner=telemetry_probe_runner, ) + contour_store = ComputeContourStore(root_provider()) + contour_services: dict[str, WorkerTelemetryService] = {} router = APIRouter(prefix="/api/v1/system", tags=["system"]) + class ContourProfileStore: + def __init__(self, contour_id: str) -> None: + self.contour_id = contour_id + + def read(self) -> WorkerConnectionProfile: + return _profile_from_compute_contour( + contour_store.get(self.contour_id) + ) + + def save( + self, + request: WorkerConnectionProfilePut, + ) -> WorkerConnectionProfile: + del request + raise RuntimeError("compute contour telemetry profiles are read-only") + + def contour_service(contour_id: str) -> WorkerTelemetryService: + existing = contour_services.get(contour_id) + if existing is not None: + return existing + + def contour_probe(profile: WorkerConnectionProfile) -> dict[str, Any]: + contour = contour_store.get(contour_id) + if contour.telemetry_mode == "legacy-ssh": + return probe_runner(profile) + return run_worker_agent_probe( + profile, + contour_id=contour.contour_id, + agent_id=contour.agent_id, + ) + + created = WorkerTelemetryService( + ContourProfileStore(contour_id), + probe_runner, + telemetry_probe_runner=contour_probe, + ) + contour_services[contour_id] = created + return created + @router.get("/worker-profile") def get_worker_profile() -> dict[str, Any]: return service.profile_document() @@ -1185,4 +1265,28 @@ def build_system_telemetry_router( ) -> dict[str, Any]: return service.snapshot(history) + @router.get("/contours/{contour_id}/telemetry") + def get_compute_contour_telemetry( + contour_id: str, + history: int = Query(default=90, ge=1, le=300), + ) -> dict[str, Any]: + try: + contour_store.get(contour_id) + return contour_service(contour_id).snapshot(history) + except KeyError as exc: + raise HTTPException(status_code=404, detail="Контур не найден.") from exc + return router + + +def _profile_from_compute_contour(contour: ComputeContour) -> WorkerConnectionProfile: + return WorkerConnectionProfile( + profile_id=contour.contour_id, + display_name=contour.display_name, + expected_node_id=contour.expected_node_id, + ssh_host_alias=SSH_HOST_ALIAS, + address=contour.address, + port=contour.ssh_port, + revision=contour.revision, + updated_at_utc=contour.updated_at_utc, + ) diff --git a/tests/test_advanced_laboratory_api.py b/tests/test_advanced_laboratory_api.py index fb25c2d..dd0a092 100644 --- a/tests/test_advanced_laboratory_api.py +++ b/tests/test_advanced_laboratory_api.py @@ -26,7 +26,9 @@ def _endpoint(router: APIRouter, path: str) -> object: def test_advanced_catalogs_are_empty_when_not_configured() -> None: router = build_advanced_laboratory_router() - for name in ("e31", "e32", "e33", "e34", "e35", "e37", "e38", "e39"): + for name in ( + "e31", "e32", "e33", "e34", "e35", "e37", "e38", "e39", "e40" + ): route = _endpoint(router, f"/api/v1/laboratory/{name}/results") catalog = route(limit=1) # type: ignore[operator] assert catalog == { @@ -50,7 +52,8 @@ def test_advanced_catalogs_fail_closed_on_incomplete_results( e37 = tmp_path / "e37" e38 = tmp_path / "e38" e39 = tmp_path / "e39" - for root in (e31, e32, e33, e34, e35, e37, e38, e39): + e40 = tmp_path / "e40" + for root in (e31, e32, e33, e34, e35, e37, e38, e39, e40): root.mkdir() (e31 / f"e31-source-qualification-{'1' * 64}").mkdir() (e32 / f"e32-track-geometry-{'2' * 64}").mkdir() @@ -60,6 +63,7 @@ def test_advanced_catalogs_fail_closed_on_incomplete_results( (e37 / f"e37-ravnoves-acceptance-{'7' * 64}").mkdir() (e38 / f"e38-perception-baseline-{'8' * 64}").mkdir() (e39 / f"e39-perception-refinement-{'9' * 64}").mkdir() + (e40 / f"e40-perception-product-gate-{'a' * 64}").mkdir() router = build_advanced_laboratory_router( e31_root_provider=lambda: e31, e32_root_provider=lambda: e32, @@ -69,9 +73,12 @@ def test_advanced_catalogs_fail_closed_on_incomplete_results( e37_root_provider=lambda: e37, e38_root_provider=lambda: e38, e39_root_provider=lambda: e39, + e40_root_provider=lambda: e40, ) - for name in ("e31", "e32", "e33", "e34", "e35", "e37", "e38", "e39"): + for name in ( + "e31", "e32", "e33", "e34", "e35", "e37", "e38", "e39", "e40" + ): route = _endpoint(router, f"/api/v1/laboratory/{name}/results") catalog = route(limit=1) # type: ignore[operator] assert catalog["configured"] is True @@ -370,6 +377,148 @@ def test_e39_catalog_projects_development_cv_and_sealed_validation( assert item["access"] == "read-only" +def test_e40_catalog_projects_dual_cv_and_sealed_product_gate( + tmp_path: Path, + monkeypatch: MonkeyPatch, +) -> None: + result_id = f"e40-perception-product-gate-{'a' * 64}" + root = tmp_path / "e40" + candidate = root / result_id + candidate.mkdir(parents=True) + (candidate / "manifest.json").write_text("{}", encoding="utf-8") + authority = { + "commands_enabled": False, + "navigation_or_safety_accepted": False, + } + dimension = { + "correct": 132, + "incorrect": 14, + "total": 146, + "accuracy": 0.90411, + "target": 0.9, + "passed": True, + "confusion": [], + "by_stratum": {}, + } + development_dimension = { + "correct": 309, + "incorrect": 31, + "total": 340, + "accuracy": 0.908824, + "target": 0.9, + "passed": True, + } + protocol = { + "items": 340, + "fold_sizes": {"0": 68, "1": 68, "2": 68, "3": 68, "4": 68}, + "dimensions": { + "presence": development_dimension, + "geometry_association": development_dimension, + "freshness": { + **development_dimension, + "correct": 325, + "incorrect": 15, + "accuracy": 0.955882, + }, + }, + "passed": True, + } + result = SimpleNamespace( + result_id=result_id, + manifest={ + "created_at_utc": "2026-07-28T08:30:00Z", + "identity": { + "source": { + "session_id": "20260720T065719Z_viewer_live", + "display_name": "RAVNOVES00", + }, + "profile": { + "profile_id": ( + "e40-ravnoves00-leakage-resistant-product-gate/v1" + ), + }, + "execution": { + "worker_node": "DESKTOP-OPJ8J04", + }, + }, + }, + report={ + "status": "measured-leakage-resistant-product-gate", + "development_cross_validation": { + "strategy": "dual-leakage-resistant-development-five-fold", + "seed": "e40-development-cv-v1", + "folds": 5, + "items": 340, + "validation_labels_used": False, + "protocols": { + "contiguous-source-time-five-fold": protocol, + "whole-track-or-scene-window-five-fold": protocol, + }, + "passed": True, + }, + "metrics": { + "development_items": 340, + "validation_items": 146, + "terminal_outcomes": 146, + "accounting_fraction": 1.0, + "false_free_claims": 0, + "high_severity_failures": 0, + "dimensions": { + "presence": dimension, + "geometry_association": dimension, + "freshness": { + **dimension, + "correct": 140, + "incorrect": 6, + "accuracy": 0.958904, + }, + }, + }, + "quality_gate": { + "passed": True, + "blocking_checks": [], + }, + "method": { + "summary": "conservative policy plus camera-only softmax", + "selection": "dual grouped development cross-validation", + "dimension_projection": "presence plus immutable stratum", + }, + "decision": { + "product_gate_measured": True, + "accepted_for_ravnoves00_product_track": True, + }, + "limitations": ["RAVNOVES00 source scoped"], + "authority": authority, + }, + ) + + def fake_read( + root_text: str, + signature: tuple[int, ...], + ) -> SimpleNamespace: + assert root_text == str(candidate.resolve()) + assert signature + return result + + monkeypatch.setattr(advanced_api, "_read_e40_cached", fake_read) + router = build_advanced_laboratory_router( + e40_root_provider=lambda: root, + ) + route = _endpoint(router, "/api/v1/laboratory/e40/results") + catalog = route(limit=1) # type: ignore[operator] + + assert catalog["candidate_total"] == 1 + assert catalog["invalid_total"] == 0 + item = catalog["items"][0] + assert item["development_cross_validation"]["passed"] is True + assert len(item["development_cross_validation"]["protocols"]) == 2 + assert item["metrics"]["dimensions"]["presence"]["accuracy"] == 0.90411 + assert item["metrics"]["high_severity_failures"] == 0 + assert item["quality_gate_passed"] is True + assert item["authority"] == authority + assert item["access"] == "read-only" + + def test_e35_catalog_projects_recovery_and_review( tmp_path: Path, monkeypatch: MonkeyPatch, diff --git a/tests/test_compute_contour_api.py b/tests/test_compute_contour_api.py index d0af129..fafdec7 100644 --- a/tests/test_compute_contour_api.py +++ b/tests/test_compute_contour_api.py @@ -40,6 +40,7 @@ def test_contour_store_migrates_worker_006_as_first_configuration( assert contours[0].contour_id == "worker-006" assert contours[0].telemetry_mode == "agent-mqtt" assert contours[0].telemetry_poll_interval_seconds == 3 + assert contours[0].mqtt_publish_interval_seconds == 2 assert not store.path.exists() @@ -67,11 +68,13 @@ def test_contour_store_creates_and_updates_private_catalog(tmp_path: Path) -> No mqtt_host="192.0.2.5", mqtt_port=1883, telemetry_poll_interval_seconds=1, + mqtt_publish_interval_seconds=4, ), ) assert updated.display_name == "Field Worker 01" assert updated.telemetry_poll_interval_seconds == 1 + assert updated.mqtt_publish_interval_seconds == 4 assert updated.revision == 1 assert stat.S_IMODE(store.path.stat().st_mode) == 0o600 assert len(store.list_contours()) == 2 @@ -89,6 +92,7 @@ def test_contour_store_creates_and_updates_private_catalog(tmp_path: Path) -> No mqtt_host="127.0.0.1", mqtt_port=1883, telemetry_poll_interval_seconds=3, + mqtt_publish_interval_seconds=2, ), ) @@ -113,6 +117,7 @@ def test_contour_router_exposes_catalog_and_safe_install_contract( assert document["agent"]["distribution"] == "Telegraf" assert "MQTT password" in document["command"] assert "password" not in document["agent"]["environment"] + assert document["agent"]["environment"]["MISSIONCORE_TELEMETRY_INTERVAL"] == "2s" assert document["ready"] is False diff --git a/tests/test_e40_perception_product_gate.py b/tests/test_e40_perception_product_gate.py new file mode 100644 index 0000000..a9709f0 --- /dev/null +++ b/tests/test_e40_perception_product_gate.py @@ -0,0 +1,132 @@ +from __future__ import annotations + +import numpy as np + +from k1link.compute.e40_perception_product_gate import ( + _FIXED_PRESENCE, + _LABELS, + _feature_names, + _predict_product_presence, + _project_dimensions, + _result_content_identity, + _train_softmax, +) + + +def test_e40_feature_contract_excludes_route_identity() -> None: + names = _feature_names() + assert len(names) == 125 + assert len(names) == len(set(names)) + assert not any( + token in name + for name in names + for token in ( + "source_frame", + "session_seconds", + "review_ordinal", + "track_id", + "map_xyz", + ) + ) + assert "bbox_area" in names + assert "image_edge_p90" in names + assert "support_occupied_fraction" in names + + +def test_e40_fixed_strata_are_conservative_product_states() -> None: + assert _FIXED_PRESENCE == { + "agree": "object-present", + "conflict": "background-or-noise", + "geometry-only": "occupied-environment", + "unknown": "object-present", + } + median = np.zeros(2) + scale = np.ones(2) + weights = np.zeros((3, len(_LABELS))) + for stratum, expected in _FIXED_PRESENCE.items(): + assert _predict_product_presence( + stratum=stratum, + features=np.ones(2), + median=median, + scale=scale, + weights=weights, + clip=10.0, + ) == (expected, 1.0) + assert _project_dimensions("geometry-only", "occupied-environment") == { + "presence": "occupied-environment", + "geometry_association": "independent-occupied", + "freshness": "current", + } + + +def test_e40_camera_only_softmax_is_deterministic() -> None: + matrix = np.asarray( + [ + [-2.0, -1.0], + [-1.0, -2.0], + [1.0, 2.0], + [2.0, 1.0], + ], + dtype=np.float64, + ) + labels = np.asarray( + [ + _LABELS.index("background-or-noise"), + _LABELS.index("background-or-noise"), + _LABELS.index("object-present"), + _LABELS.index("object-present"), + ], + dtype=np.int64, + ) + first = _train_softmax( + matrix, + labels, + l2=0.01, + steps=120, + learning_rate=0.03, + ) + second = _train_softmax( + matrix, + labels, + l2=0.01, + steps=120, + learning_rate=0.03, + ) + assert np.array_equal(first, second) + assert ( + _predict_product_presence( + stratum="camera-only", + features=np.asarray([1.5, 1.5]), + median=np.zeros(2), + scale=np.ones(2), + weights=first, + clip=10.0, + )[0] + == "object-present" + ) + + +def test_e40_result_content_identity_changes_with_every_output() -> None: + predictions = [{"sequence": 1, "prediction": {"presence": "object-present"}}] + model = {"weights": [1.0]} + report = {"quality_gate": {"passed": False}} + baseline = _result_content_identity( + predictions=predictions, + model=model, + report=report, + ) + assert baseline != _result_content_identity( + predictions=[{"sequence": 1, "prediction": {"presence": "background-or-noise"}}], + model=model, + report=report, + ) + assert baseline != _result_content_identity( + predictions=predictions, + model={"weights": [2.0]}, + report=report, + ) + assert baseline != _result_content_identity( + predictions=predictions, + model=model, + report={"quality_gate": {"passed": True}}, + ) diff --git a/tests/test_e40_worker_package.py b/tests/test_e40_worker_package.py new file mode 100644 index 0000000..8674ed3 --- /dev/null +++ b/tests/test_e40_worker_package.py @@ -0,0 +1,92 @@ +from __future__ import annotations + +import hashlib +import importlib.util +import json +import subprocess +import sys +from pathlib import Path + +import pytest + + +def _module() -> object: + path = ( + Path(__file__).resolve().parents[1] + / "experiments" + / "perception" + / "prepare_e40_worker_package.py" + ) + spec = importlib.util.spec_from_file_location("e40_worker_package_test", path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def test_e40_package_contains_bound_product_gate_input(tmp_path: Path) -> None: + module = _module() + repository = Path(__file__).resolve().parents[1] + acceptance = ( + repository + / ".runtime" + / "compute-experiments" + / "e37" + / "results" + / ( + "e37-ravnoves-acceptance-" + "01b1efd586f747341c712d82f0907b39436a6f91ae92b1dfae987eca05fd8344" + ) + ) + materialization = ( + repository + / ".runtime" + / "compute-experiments" + / "e30" + / "materializations" + / ("e30-materialization-841af926d8d28ab93538c46d8f31278a2234c4d1c12c7dc4dc296b249d59735a") + ) + package = module.build_e40_worker_package( + repository_root=repository, + acceptance_root=acceptance, + materialization_root=materialization, + profile_path=( + repository / "experiments" / "perception" / "e40_ravnoves00_product_gate_profile.json" + ), + output_root=tmp_path, + ) + manifest = module.validate_e40_worker_package(package) + assert package.name == f"e40-worker-package-{manifest['identity_sha256']}" + assert manifest["identity"]["classification"] == ( + "immutable-ravnoves00-leakage-resistant-product-gate-input" + ) + feature_manifest = ( + package / "input" / "materialization" / materialization.name / "e40-feature-cache.json" + ) + assert feature_manifest.is_file() + assert (package / "runtime" / "k1link" / "compute" / "e40_perception_product_gate.py").is_file() + independent_validator = ( + package / "runtime" / "validate_e40_worker_package.py" + ) + assert independent_validator.is_file() + subprocess.run( + [sys.executable, str(independent_validator), str(package)], + check=True, + ) + + profile_path = package / "profile.json" + profile_path.write_text("{}\n", encoding="utf-8") + package_manifest_path = package / "manifest.json" + package_manifest = json.loads(package_manifest_path.read_text(encoding="utf-8")) + profile_payload = profile_path.read_bytes() + for row in package_manifest["artifacts"]: + if row["path"] == "profile.json": + row["byte_length"] = len(profile_payload) + row["sha256"] = hashlib.sha256(profile_payload).hexdigest() + package_manifest_path.write_text( + json.dumps(package_manifest, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + with pytest.raises(module.E40WorkerPackageError, match="binding|changed"): + module.validate_e40_worker_package(package) diff --git a/tests/test_e41_evaluation_boundary.py b/tests/test_e41_evaluation_boundary.py new file mode 100644 index 0000000..d1b87ca --- /dev/null +++ b/tests/test_e41_evaluation_boundary.py @@ -0,0 +1,163 @@ +from __future__ import annotations + +import numpy as np +import pytest + +from k1link.compute.e41_evaluation_boundary import ( + E41EvaluationBoundaryError, + evaluate_visible_engineering_contract, + predict_from_frozen_e40_model, +) + + +def _model() -> dict[str, object]: + return { + "schema_version": "missioncore.e40-development-product-model/v1", + "feature_names": ["signal"], + "validation_labels_used_for_training": False, + "robust_clip": 10.0, + "fixed_presence_by_stratum": { + "agree": "object-present", + "conflict": "background-or-noise", + "geometry-only": "occupied-environment", + "unknown": "object-present", + }, + "dimension_projection": "source-stratum-plus-presence/v1", + "classifier": { + "type": "deterministic-softmax", + "labels": [ + "background-or-noise", + "object-present", + "occupied-environment", + ], + "weights": [ + [-2.0, 2.0, 0.0], + [0.0, 0.0, 0.0], + ], + "scaler": { + "median": [0.0], + "scale": [1.0], + }, + }, + } + + +def test_e41_predictor_output_is_truth_free_and_deterministic() -> None: + items = [ + { + "schema_version": "missioncore.e41-predictor-item/v1", + "sequence": 0, + "item_id": "a", + "source_stratum": "camera-only", + }, + { + "schema_version": "missioncore.e41-predictor-item/v1", + "sequence": 1, + "item_id": "b", + "source_stratum": "geometry-only", + }, + ] + matrix = np.asarray([[1.0], [0.0]], dtype=np.float64) + first = predict_from_frozen_e40_model( + items=items, + feature_names=["signal"], + feature_matrix=matrix, + model=_model(), + ) + second = predict_from_frozen_e40_model( + items=items, + feature_names=["signal"], + feature_matrix=matrix, + model=_model(), + ) + + assert first == second + assert first[0]["prediction"]["presence"] == "object-present" + assert first[1]["prediction"]["presence"] == "occupied-environment" + assert not any( + forbidden in row + for row in first + for forbidden in ("reference", "scored", "severity", "split", "truth") + ) + + +def test_e41_predictor_rejects_truth_bearing_item_metadata() -> None: + with pytest.raises(E41EvaluationBoundaryError, match="truth/evaluation"): + predict_from_frozen_e40_model( + items=[ + { + "sequence": 0, + "item_id": "a", + "source_stratum": "camera-only", + "reference": {"presence": "object-present"}, + } + ], + feature_names=["signal"], + feature_matrix=np.asarray([[1.0]], dtype=np.float64), + model=_model(), + ) + + +def test_e41_visible_evaluator_joins_truth_after_prediction() -> None: + predictions = [ + { + "item_id": "dev", + "prediction": { + "presence": "object-present", + "geometry_association": "insufficient-support", + "freshness": "unavailable", + }, + }, + { + "item_id": "val", + "prediction": { + "presence": "object-present", + "geometry_association": "object-associated", + "freshness": "current", + }, + }, + ] + acceptance = [ + { + "item_id": "dev", + "split": "development", + "source_stratum": "camera-only", + "severity": "standard", + "reference": { + "presence": "background-or-noise", + "geometry_association": "rejected-nonobject", + "freshness": "unavailable", + }, + }, + { + "item_id": "val", + "split": "validation", + "source_stratum": "agree", + "severity": "standard", + "reference": { + "presence": "object-present", + "geometry_association": "object-associated", + "freshness": "current", + }, + }, + ] + evaluation = evaluate_visible_engineering_contract( + predictions=predictions, + acceptance_rows=acceptance, + targets={ + "presence_target": 0.9, + "geometry_association_target": 0.9, + "freshness_target": 0.9, + }, + label_provenance={ + "engineering_items": 2, + "human_exception_items": 0, + "independent_ground_truth": False, + }, + ) + + assert evaluation["metrics"]["validation_items"] == 1 + assert evaluation["metrics"]["dimensions"]["presence"]["accuracy"] == 1.0 + assert evaluation["engineering_contract_targets_reached"] is True + assert evaluation["blind_gate_eligible"] is False + assert evaluation["label_provenance"]["independent_accuracy_authority"] is False diff --git a/tests/test_e41_methodology_audit.py b/tests/test_e41_methodology_audit.py new file mode 100644 index 0000000..806950a --- /dev/null +++ b/tests/test_e41_methodology_audit.py @@ -0,0 +1,184 @@ +from __future__ import annotations + +import numpy as np + +from k1link.compute.e41_methodology_audit import analyze_e41_methodology + + +def _acceptance( + item_id: str, + *, + split: str, + frame: int, + stratum: str = "camera-only", + presence: str = "object-present", +) -> dict[str, object]: + return { + "item_id": item_id, + "split": split, + "source_frame_index": frame, + "source_stratum": stratum, + "reference": { + "presence": presence, + "geometry_association": "insufficient-support", + "freshness": "unavailable", + }, + } + + +def _materialization(item_id: str, *, track_id: int | None) -> dict[str, object]: + return { + "item_id": item_id, + "e29_snapshot": { + "track_id": track_id, + }, + } + + +def test_e41_detects_split_leakage_and_prediction_truth_colocation() -> None: + acceptance = [ + _acceptance("dev-a", split="development", frame=100, presence="object-present"), + _acceptance( + "dev-b", + split="development", + frame=101, + presence="background-or-noise", + ), + _acceptance("val-a", split="validation", frame=100, presence="object-present"), + _acceptance("val-b", split="validation", frame=149, presence="object-present"), + ] + materialization = [ + _materialization("dev-a", track_id=7), + _materialization("dev-b", track_id=8), + _materialization("val-a", track_id=7), + _materialization("val-b", track_id=None), + ] + names = ["image_luma_mean", "stratum=camera-only", "source_frame_index"] + matrix = np.asarray( + [ + [0.1, 1.0, 100.0], + [0.9, 1.0, 101.0], + [0.2, 1.0, 100.0], + [0.3, 1.0, 149.0], + ], + dtype=np.float64, + ) + report = analyze_e41_methodology( + acceptance_rows=acceptance, + materialization_rows=materialization, + feature_item_ids=["dev-a", "dev-b", "val-a", "val-b"], + feature_names=names, + feature_matrix=matrix, + e40_model={ + "feature_names": names, + "camera_only_training_items": 2, + "dimension_projection": "source-stratum-plus-presence/v1", + }, + e40_report={ + "status": "measured-leakage-resistant-product-gate", + "execution": {"class": "sealed-validation-evaluation"}, + "metrics": { + "dimensions": { + "presence": {"accuracy": 0.5}, + "geometry_association": {"accuracy": 0.5}, + } + }, + }, + e40_predictions=[ + { + "item_id": "val-a", + "prediction": {"presence": "object-present"}, + "reference": {"presence": "object-present"}, + "scored": True, + } + ], + label_provenance={ + "engineering_items": 4, + "human_exception_items": 0, + "independent_ground_truth": False, + }, + time_block_frames=50, + forbidden_feature_tokens=("source_frame", "track_id", "path"), + ) + + assert report["split_leakage"]["exact_source_frames"]["count"] == 1 + assert report["split_leakage"]["track_ids"]["count"] == 1 + assert report["split_leakage"]["time_blocks"]["count"] == 1 + assert report["split_leakage"]["whole_track_or_scene_groups"]["count"] == 1 + assert report["features"]["camera_only_training_items"] == 2 + assert report["features"]["forbidden_features"] == ["source_frame_index"] + assert report["predictor_evaluator_boundary"]["physically_separated"] is False + assert report["metric_semantics"]["dimensions_independently_inferred"] is False + assert report["policy"]["blind_gate_eligible"] is False + assert report["policy"]["violations"] == [ + "labels-are-not-independent-ground-truth", + "development-validation-source-groups-overlap", + "prediction-and-evaluation-concerns-are-co-located", + "historical-e40-still-contains-blind-or-product-gate-claims", + "forbidden-identity-feature-detected", + ] + + +def test_e41_accepts_a_clean_separated_methodology_contract() -> None: + acceptance = [ + _acceptance("dev-a", split="development", frame=10), + _acceptance( + "dev-b", + split="development", + frame=11, + presence="background-or-noise", + ), + _acceptance("val-a", split="validation", frame=210), + ] + materialization = [ + _materialization("dev-a", track_id=1), + _materialization("dev-b", track_id=2), + _materialization("val-a", track_id=9), + ] + names = ["image_luma_mean", "support_occupied_fraction"] + report = analyze_e41_methodology( + acceptance_rows=acceptance, + materialization_rows=materialization, + feature_item_ids=["dev-a", "dev-b", "val-a"], + feature_names=names, + feature_matrix=np.asarray( + [ + [0.1, 0.2], + [0.9, 0.8], + [0.4, 0.3], + ], + dtype=np.float64, + ), + e40_model={ + "feature_names": names, + "camera_only_training_items": 2, + "dimension_projection": "independent-task-heads/v1", + }, + e40_report={ + "status": "source-scoped-visible-evaluation", + "metrics": { + "dimensions": { + "presence": {"accuracy": 0.8}, + "geometry_association": {"accuracy": 0.7}, + } + }, + }, + e40_predictions=[ + { + "item_id": "val-a", + "prediction": {"presence": "object-present"}, + } + ], + label_provenance={ + "engineering_items": 0, + "human_exception_items": 3, + "independent_ground_truth": True, + }, + time_block_frames=50, + forbidden_feature_tokens=("source_frame", "track_id", "path"), + ) + + assert report["features"]["forbidden_features"] == [] + assert report["predictor_evaluator_boundary"]["physically_separated"] is True + assert report["policy"]["violations"] == [] + assert report["policy"]["blind_gate_eligible"] is True diff --git a/tests/test_e42_metamorphic_suite.py b/tests/test_e42_metamorphic_suite.py new file mode 100644 index 0000000..85926f1 --- /dev/null +++ b/tests/test_e42_metamorphic_suite.py @@ -0,0 +1,87 @@ +from __future__ import annotations + +import numpy as np + +from k1link.compute.e42_metamorphic_suite import ( + _point_slab_signature, + _predictor_metamorphics, +) +from k1link.compute.track_geometry import PointSlab + + +def _model() -> dict[str, object]: + return { + "schema_version": "missioncore.e40-development-product-model/v1", + "feature_names": ["signal"], + "validation_labels_used_for_training": False, + "robust_clip": 10.0, + "classifier": { + "type": "deterministic-softmax", + "labels": [ + "background-or-noise", + "object-present", + "occupied-environment", + ], + "weights": [ + [-2.0, 2.0, 0.0], + [0.0, 0.0, 0.0], + ], + "scaler": { + "median": [0.0], + "scale": [1.0], + }, + }, + } + + +def test_e42_predictor_is_invariant_to_ids_order_and_chunks() -> None: + items = [ + { + "schema_version": "missioncore.e41-predictor-item/v1", + "sequence": index, + "item_id": f"item-{index}", + "source_stratum": "camera-only", + } + for index in range(12) + ] + checks = _predictor_metamorphics( + items=items, + feature_names=["signal"], + feature_matrix=np.arange(12, dtype=np.float64).reshape((-1, 1)), + model=_model(), + ) + + assert all(checks.values()) + + +def test_e42_point_slab_signature_is_row_order_invariant() -> None: + slab = PointSlab( + frame_index=1, + source_frame_index=10, + source_point_count=8, + coordinate_frame="map", + owner_keys=("track:1", "geometry:2"), + source_indices=np.asarray([1, 7, 3], dtype=" dict[str, object]: + return { + "capture_contract": { + "device_model": "XGRIDS/LixelKity-K1", + "minimum_duration_seconds": 480, + "maximum_duration_seconds": 900, + "required_streams": [ + "sensor.camera.right", + "sensor.lidar.registered-map-increment", + "sensor.pose", + "telemetry.pipeline", + ], + "required_segments": [ + { + "kind": "control-bridge", + "minimum_duration_seconds": 60, + }, + { + "kind": "new-route", + "minimum_duration_seconds": 360, + }, + ], + } + } + + +def _capture_manifest() -> dict[str, object]: + stream = { + "available": True, + "item_count": 10, + "byte_length": 100, + "sha256": "a" * 64, + } + return { + "schema_version": E43_CAPTURE_MANIFEST_SCHEMA, + "source_session_id": "future-session-001", + "source_display_name": "RAVNOVES01", + "operator_authorized": True, + "device": { + "model": "XGRIDS/LixelKity-K1", + "device_identity_sha256": "b" * 64, + "calibration_sha256": "c" * 64, + "mount_identity_sha256": "d" * 64, + "configuration_sha256": "e" * 64, + "firmware": "3.0.2", + }, + "capture": { + "started_at_utc": "2026-07-28T12:00:00Z", + "monotonic_start_seconds": 100.0, + "monotonic_end_seconds": 700.0, + "duration_seconds": 600.0, + "weather": "overcast", + "illumination": "daylight", + "location_class": "industrial-buildings-opposite-side", + "operator_notes": "bounded owner-authorized capture", + }, + "streams": { + "sensor.camera.right": stream, + "sensor.lidar.registered-map-increment": stream, + "sensor.pose": stream, + "telemetry.pipeline": stream, + }, + "segments": [ + { + "kind": "control-bridge", + "monotonic_start_seconds": 100.0, + "monotonic_end_seconds": 180.0, + }, + { + "kind": "new-route", + "monotonic_start_seconds": 200.0, + "monotonic_end_seconds": 700.0, + }, + ], + "authority": { + "commands_enabled": False, + "navigation_or_safety_accepted": False, + }, + } + + +def test_e43_accepts_complete_same_k1_capture_manifest() -> None: + result = validate_future_capture_manifest( + _capture_manifest(), + protocol=_protocol(), + ) + + assert result["accepted"] is True + assert result["segments"] == ["control-bridge", "new-route"] + assert result["blind_truth_labels_available"] is False + + +def test_e43_rejects_missing_pipeline_stream() -> None: + manifest = _capture_manifest() + del manifest["streams"]["telemetry.pipeline"] # type: ignore[index] + with pytest.raises(E43FutureCaptureProtocolError, match="stream set"): + validate_future_capture_manifest( + manifest, + protocol=_protocol(), + ) + + +def test_e43_grouped_partition_keeps_connected_evidence_together() -> None: + candidates = [ + { + "schema_version": E43_CANDIDATE_SCHEMA, + "item_id": "a", + "scene_id": "scene-1", + "track_id": "track-1", + "time_block_id": "time-1", + "route_segment": "control-bridge", + }, + { + "schema_version": E43_CANDIDATE_SCHEMA, + "item_id": "b", + "scene_id": "scene-1", + "track_id": "track-2", + "time_block_id": "time-2", + "route_segment": "control-bridge", + }, + { + "schema_version": E43_CANDIDATE_SCHEMA, + "item_id": "c", + "scene_id": "scene-2", + "track_id": "track-3", + "time_block_id": "time-3", + "route_segment": "new-route", + }, + { + "schema_version": E43_CANDIDATE_SCHEMA, + "item_id": "d", + "scene_id": "scene-3", + "track_id": None, + "time_block_id": "time-4", + "route_segment": "new-route", + }, + ] + assignments = assign_grouped_future_partitions( + candidates, + seed="fixed-before-capture", + blind_fraction=0.3, + ) + + assert assignments["a"] == assignments["b"] + assert set(assignments.values()) == {"blind-truth", "visible-diagnostic"} diff --git a/tests/test_e44_data_amplification_audit.py b/tests/test_e44_data_amplification_audit.py new file mode 100644 index 0000000..1ef3886 --- /dev/null +++ b/tests/test_e44_data_amplification_audit.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +import hashlib + +import pytest + +from k1link.compute.e44_data_amplification_audit import ( + E44DataAmplificationAuditError, + analyze_data_amplification, +) + + +def _row(root: str, path: str, payload: bytes, kind: str) -> dict[str, object]: + return { + "root": root, + "path": path, + "byte_length": len(payload), + "sha256": hashlib.sha256(payload).hexdigest(), + "kind": kind, + } + + +def test_e44_measures_exact_cross_root_duplication() -> None: + shared = b"camera-frame" + report = analyze_data_amplification( + [ + _row("e30", "frames/a.jpg", shared, "camera-image"), + _row("e40", "frames/a.jpg", shared, "camera-image"), + _row("e40", "report.json", b"{}", "metadata-or-report"), + ] + ) + + assert report["logical_bytes"] == len(shared) * 2 + 2 + assert report["unique_content_bytes"] == len(shared) + 2 + assert report["duplicate_bytes"] == len(shared) + assert report["duplicate_content_groups"] == 1 + assert report["largest_duplicate_groups"][0]["copies"] == 2 + assert report["largest_duplicate_groups"][0]["roots"] == ["e30", "e40"] + assert report["roots"]["e40"]["duplicate_bytes_within_root"] == 0 + + +def test_e44_rejects_same_digest_with_inconsistent_lengths() -> None: + digest = "a" * 64 + with pytest.raises(E44DataAmplificationAuditError, match="inconsistent"): + analyze_data_amplification( + [ + { + "root": "e30", + "path": "one.bin", + "byte_length": 1, + "sha256": digest, + "kind": "other", + }, + { + "root": "e40", + "path": "two.bin", + "byte_length": 2, + "sha256": digest, + "kind": "other", + }, + ] + ) diff --git a/tests/test_pipeline_telemetry.py b/tests/test_pipeline_telemetry.py new file mode 100644 index 0000000..1f4287c --- /dev/null +++ b/tests/test_pipeline_telemetry.py @@ -0,0 +1,190 @@ +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path +from types import ModuleType, SimpleNamespace + +import pytest + +from k1link.compute.pipeline_telemetry import ( + JsonlPipelineTelemetrySink, + MqttPipelineTelemetrySink, + PipelineTelemetryEmitter, + PipelineTelemetryError, + PipelineTelemetryIdentity, + build_pipeline_telemetry_document, +) + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +NORMALIZER_PATH = ( + REPOSITORY_ROOT + / "deploy" + / "telemetry-plane" + / "normalizer" + / "normalizer.py" +) + + +def _normalizer() -> ModuleType: + spec = importlib.util.spec_from_file_location( + "missioncore_pipeline_telemetry_normalizer", + NORMALIZER_PATH, + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _identity() -> PipelineTelemetryIdentity: + return PipelineTelemetryIdentity( + contour_id="worker-006", + agent_id="mission-core-worker", + node_id="DESKTOP-OPJ8J04", + lab_id="E41", + run_id="run-001", + request_id="request-001", + source_id="ravnoves00", + source_package_id="e41-predictor-package-example", + method_id="frozen-e40-predictor/v1", + frame_index=17, + ) + + +def test_pipeline_document_is_accepted_without_losing_stage_identity() -> None: + identity = _identity() + document = build_pipeline_telemetry_document( + identity=identity, + stage_id="predict", + state="completed", + duration_ms=125.5, + input_count=89, + output_count=89, + queue_wait_ms=2.25, + observed_at_utc="2026-07-28T12:00:00Z", + ) + + row = _normalizer()._normalize( + identity.topic, + json.dumps(document).encode(), + ) + + assert row[1:5] == ( + "worker-006", + "mission-core-worker", + "DESKTOP-OPJ8J04", + "pipeline", + ) + assert row[9:13] == ("E41", "run-001", "request-001", 17) + assert json.loads(row[6]) == { + "lab_id": "E41", + "method_id": "frozen-e40-predictor/v1", + "request_id": "request-001", + "run_id": "run-001", + "source_id": "ravnoves00", + "source_package_id": "e41-predictor-package-example", + "stage_id": "predict", + "stage_state": "completed", + } + stored = json.loads(row[13]) + assert stored["payload"]["event"]["duration_ms"] == 125.5 + assert stored["authority"]["commands_enabled"] is False + + +def test_stage_context_emits_terminal_event_and_preserves_failure() -> None: + published: list[tuple[str, dict[str, object]]] = [] + + class Sink: + def publish(self, topic: str, payload: bytes) -> None: + published.append((topic, json.loads(payload))) + + ticks = iter((1_000_000_000, 1_125_500_000)) + emitter = PipelineTelemetryEmitter( + identity=_identity(), + sink=Sink(), + clock_ns=lambda: next(ticks), + ) + with emitter.stage("predict", input_count=89) as outcome: + outcome.output_count = 89 + + assert [document["stage_state"] for _, document in published] == [ + "started", + "completed", + ] + assert published[1][1]["payload"]["event"]["duration_ms"] == 125.5 + assert published[1][1]["payload"]["event"]["output_count"] == 89 + + failure_ticks = iter((2_000_000_000, 2_001_000_000)) + failure_emitter = PipelineTelemetryEmitter( + identity=_identity(), + sink=Sink(), + clock_ns=lambda: next(failure_ticks), + ) + with ( + pytest.raises(ValueError, match="source failure"), + failure_emitter.stage("evaluate"), + ): + raise ValueError("source failure") + assert published[-1][1]["stage_state"] == "failed" + assert published[-1][1]["payload"]["event"]["error_type"] == "ValueError" + assert "source failure" not in json.dumps(published[-1][1]) + + +def test_jsonl_sink_records_topic_bound_documents(tmp_path: Path) -> None: + path = tmp_path / "telemetry" / "e41.jsonl" + identity = _identity() + sink = JsonlPipelineTelemetrySink(path) + document = build_pipeline_telemetry_document( + identity=identity, + stage_id="package", + state="completed", + duration_ms=1.0, + ) + + sink.publish(identity.topic, json.dumps(document).encode()) + + record = json.loads(path.read_text(encoding="utf-8")) + assert record["schema_version"] == "missioncore.pipeline-telemetry-record/v1" + assert record["topic"] == identity.topic + assert record["payload"]["stage_id"] == "package" + assert path.stat().st_mode & 0o077 == 0 + + +def test_mqtt_sink_uses_qos_one_without_retention() -> None: + calls: list[tuple[str, bytes, int, bool]] = [] + + class Client: + def publish( + self, + topic: str, + payload: bytes, + qos: int, + retain: bool, + ) -> SimpleNamespace: + calls.append((topic, payload, qos, retain)) + return SimpleNamespace(rc=0) + + MqttPipelineTelemetrySink(Client()).publish("topic", b"payload") + + assert calls == [("topic", b"payload", 1, False)] + + +def test_pipeline_telemetry_rejects_unsafe_identity_and_invalid_metrics() -> None: + with pytest.raises(PipelineTelemetryError, match="contour_id"): + PipelineTelemetryIdentity( + contour_id="../worker", + agent_id="agent", + node_id="node", + lab_id="E41", + run_id="run", + source_id="source", + source_package_id="package", + method_id="method", + ) + with pytest.raises(PipelineTelemetryError, match="duration"): + build_pipeline_telemetry_document( + identity=_identity(), + stage_id="predict", + state="completed", + ) diff --git a/tests/test_system_telemetry_api.py b/tests/test_system_telemetry_api.py index 829f61d..b528020 100644 --- a/tests/test_system_telemetry_api.py +++ b/tests/test_system_telemetry_api.py @@ -10,6 +10,7 @@ from fastapi import APIRouter, HTTPException from fastapi.routing import APIRoute from pydantic import ValidationError +from k1link.web.compute_contour_api import default_compute_contour from k1link.web.system_telemetry_api import ( EXPECTED_NODE_ID, WorkerConnectionProfile, @@ -17,6 +18,7 @@ from k1link.web.system_telemetry_api import ( WorkerProfileStore, WorkerTelemetryService, _agent_raw_document, + _profile_from_compute_contour, _ssh_arguments, build_system_telemetry_router, ) @@ -272,6 +274,23 @@ def test_worker_telemetry_prefers_ndc_container_names_during_migration( assert triton["canonical_name"] == "ndc-mission-core-triton" +def test_worker_telemetry_history_keeps_one_row_per_agent_observation( + tmp_path: Path, +) -> None: + service = WorkerTelemetryService( + WorkerProfileStore(tmp_path / "system"), + lambda _: _probe(), + cache_seconds=0, + ) + + first = service.snapshot(10) + second = service.snapshot(10) + + assert len(first["history"]) == 1 + assert len(second["history"]) == 1 + assert second["history"][0]["observed_at_utc"] == "2026-07-27T12:00:00Z" + + def test_agent_metrics_are_mapped_to_the_existing_product_contract() -> None: document = _agent_raw_document( { @@ -340,6 +359,25 @@ def test_agent_metrics_are_mapped_to_the_existing_product_contract() -> None: ) +def test_compute_contour_maps_to_worker_identity_without_singleton_defaults() -> None: + contour = default_compute_contour().model_copy( + update={ + "contour_id": "field-worker", + "agent_id": "field-agent", + "display_name": "Field Worker", + "expected_node_id": "FIELD-01", + "address": "192.0.2.25", + } + ) + + profile = _profile_from_compute_contour(contour) + + assert profile.profile_id == "field-worker" + assert profile.display_name == "Field Worker" + assert profile.expected_node_id == "FIELD-01" + assert profile.address == "192.0.2.25" + + def test_profile_apply_fails_closed_on_wrong_node_and_keeps_old_profile( tmp_path: Path, ) -> None: diff --git a/tests/test_telemetry_normalizer_contract.py b/tests/test_telemetry_normalizer_contract.py index 1ea7c25..ab3c39e 100644 --- a/tests/test_telemetry_normalizer_contract.py +++ b/tests/test_telemetry_normalizer_contract.py @@ -122,6 +122,12 @@ def test_telegraf_upsert_merges_split_fields_in_one_series() -> None: assert "EXCLUDED.payload -> 'fields'" in NORMALIZER_SOURCE +def test_normalizer_enforces_oss_compatible_bounded_retention() -> None: + assert "DELETE FROM contour_telemetry_samples" in NORMALIZER_SOURCE + assert "INTERVAL '30 days'" in NORMALIZER_SOURCE + assert "RETENTION_INTERVAL_SECONDS" in NORMALIZER_SOURCE + + def test_normalizer_rejects_unschematized_pipeline_payload() -> None: normalizer = _normalizer() with pytest.raises(ValueError, match="schema"): @@ -136,6 +142,99 @@ def test_normalizer_rejects_unschematized_pipeline_payload() -> None: ) +def test_normalizer_preserves_native_pipeline_stage_tags() -> None: + normalizer = _normalizer() + row = normalizer._normalize( + "mission-core/v1/contours/worker-006/agents/mission-core-worker/pipeline", + json.dumps( + { + "schema_version": "missioncore.agent-pipeline-telemetry/v1", + "node_id": "DESKTOP-OPJ8J04", + "observed_at_utc": "2026-07-28T12:00:00Z", + "lab_id": "E41", + "run_id": "run-001", + "tags": { + "node_id": "DESKTOP-OPJ8J04", + "contour_id": "worker-006", + "agent_id": "mission-core-worker", + "lab_id": "E41", + "run_id": "run-001", + "source_id": "ravnoves00", + "source_package_id": "e41-predictor-package-example", + "method_id": "frozen-e40-predictor/v1", + "stage_id": "predict", + "stage_state": "completed", + }, + "payload": {"state": "ready"}, + } + ).encode(), + ) + + assert json.loads(row[6]) == { + "lab_id": "E41", + "method_id": "frozen-e40-predictor/v1", + "run_id": "run-001", + "source_id": "ravnoves00", + "source_package_id": "e41-predictor-package-example", + "stage_id": "predict", + "stage_state": "completed", + } + + +def test_normalizer_rejects_oversized_payload_and_series_identity() -> None: + normalizer = _normalizer() + with pytest.raises(ValueError, match="1 MiB"): + normalizer._normalize( + "mission-core/v1/contours/worker-006/agents/worker-006/host", + b"{" + b"x" * normalizer.MAX_PAYLOAD_BYTES + b"}", + ) + with pytest.raises(ValueError, match="too many tags"): + normalizer._normalize( + "mission-core/v1/contours/worker-006/agents/worker-006/host", + json.dumps( + { + "name": "cpu", + "tags": { + "node_id": "DESKTOP-OPJ8J04", + **{ + f"tag-{index}": str(index) + for index in range(normalizer.MAX_TAGS + 1) + }, + }, + "fields": {"usage_active": 12.5}, + "timestamp": 1_785_179_600, + } + ).encode(), + ) + + +def test_normalizer_removes_unneeded_docker_labels_and_host_bind_paths() -> None: + normalizer = _normalizer() + row = normalizer._normalize( + "mission-core/v1/contours/worker-006/agents/worker-006/host", + json.dumps( + { + "name": "docker_container_cpu", + "tags": { + "node_id": "DESKTOP-OPJ8J04", + "container_name": "ndc-mission-core-triton", + "desktop.docker.io/binds/0/Source": "C:\\private\\model", + "com.nvidia.cuda.version": "12.8", + }, + "fields": {"usage_percent": 10.0}, + "timestamp": 1_785_179_600, + } + ).encode(), + ) + + assert row[6] == '{"container_name":"ndc-mission-core-triton"}' + stored = json.loads(row[13]) + assert stored["tags"] == { + "node_id": "DESKTOP-OPJ8J04", + "container_name": "ndc-mission-core-triton", + } + + def test_telemetry_plane_uses_the_ndc_docker_namespace() -> None: document = yaml.safe_load(COMPOSE_PATH.read_text(encoding="utf-8")) @@ -146,6 +245,7 @@ def test_telemetry_plane_uses_the_ndc_docker_namespace() -> None: for service in services.values() } == { "ndc-mission-core-mqtt-broker", + "ndc-mission-core-telemetry-bootstrap", "ndc-mission-core-telemetry-normalizer", "ndc-mission-core-telemetry-timescaledb", } @@ -154,6 +254,8 @@ def test_telemetry_plane_uses_the_ndc_docker_namespace() -> None: assert service["labels"]["com.nodedc.product"] == "mission-core" assert service["labels"]["com.nodedc.stack"] == "ndc-mission-core-telemetry" + assert services["broker"]["cap_drop"] == ["ALL"] + assert set(services["broker"]["cap_add"]) == {"CHOWN", "SETGID", "SETUID"} assert document["networks"]["default"]["name"] == "ndc-mission-core-telemetry" assert { volume["name"] diff --git a/tests/test_telemetry_plane_prepare.py b/tests/test_telemetry_plane_prepare.py index 018f88a..e06cadd 100644 --- a/tests/test_telemetry_plane_prepare.py +++ b/tests/test_telemetry_plane_prepare.py @@ -45,10 +45,11 @@ def test_initialize_environment_generates_private_unique_secrets( assert values["MISSIONCORE_MQTT_BIND_ADDRESS"] == "192.0.2.15" secrets = { values["MISSIONCORE_DB_PASSWORD"], + values["MISSIONCORE_DB_INGEST_PASSWORD"], values["MISSIONCORE_MQTT_INGEST_PASSWORD"], values["MISSIONCORE_MQTT_WORKER_006_PASSWORD"], } - assert len(secrets) == 3 + assert len(secrets) == 4 assert all(len(secret) >= 40 for secret in secrets) assert stat.S_IMODE(env_path.stat().st_mode) == 0o600 @@ -65,3 +66,61 @@ def test_initialize_environment_refuses_to_replace_credentials( prepare._initialize_environment("127.0.0.1") assert env_path.read_text(encoding="utf-8") == "existing=true\n" + + +def test_environment_migration_adds_only_new_private_values( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + env_path = tmp_path / ".env" + env_path.write_text( + "MISSIONCORE_DB_PASSWORD=keep-me\n" + "MISSIONCORE_MQTT_WORKER_006_USER=worker-006\n", + encoding="utf-8", + ) + monkeypatch.setattr(prepare, "ENV_PATH", env_path) + + prepare._migrate_environment() + first = env_path.read_text(encoding="utf-8") + prepare._migrate_environment() + + assert "MISSIONCORE_DB_PASSWORD=keep-me" in first + assert "MISSIONCORE_DB_INGEST_PASSWORD=" in first + assert "MISSIONCORE_MQTT_WORKER_006_CONTOUR=worker-006" in first + assert env_path.read_text(encoding="utf-8") == first + assert stat.S_IMODE(env_path.stat().st_mode) == 0o600 + + +def test_existing_password_file_is_updated_without_recreation( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + password_path = tmp_path / "passwords" + password_path.write_text("existing", encoding="utf-8") + calls: list[tuple[str, bool]] = [] + + def capture( + path: Path, + username: str, + password: str, + *, + create: bool, + ) -> None: + assert path == password_path + assert password + calls.append((username, create)) + + monkeypatch.setattr(prepare, "_password_entry", capture) + + prepare._prepare_password_entries( + password_path, + "missioncore-ingest", + "ingest-secret", + "worker-006", + "worker-secret", + ) + + assert calls == [ + ("missioncore-ingest", False), + ("worker-006", False), + ]