SILVER/NODEDC_MISSION_CORE
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat(perception): stabilize pre-capture methodology

Browse Source
This commit is contained in:
DCCONSTRUCTIONS
2026-07-28 17:47:06 +03:00
parent 1f20e0d7d9
commit d729abab31
65 changed files with 9698 additions and 152 deletions
Download Patch File Download Diff File Expand all files Collapse all files
deploy/telemetry-plane/.env.example
+2
View File
@@ -4,7 +4,9 @@ MISSIONCORE_TELEMETRY_QUERY_PORT=18030
MISSIONCORE_DB_NAME=missioncore_telemetry
MISSIONCORE_DB_USER=missioncore_ingest
MISSIONCORE_DB_PASSWORD=replace-with-a-random-local-secret
MISSIONCORE_DB_INGEST_PASSWORD=replace-with-a-separate-ingest-secret
MISSIONCORE_MQTT_INGEST_USER=missioncore-ingest
MISSIONCORE_MQTT_INGEST_PASSWORD=replace-with-a-random-local-secret
MISSIONCORE_MQTT_WORKER_006_USER=worker-006
MISSIONCORE_MQTT_WORKER_006_CONTOUR=worker-006
MISSIONCORE_MQTT_WORKER_006_PASSWORD=replace-with-a-different-random-local-secret
deploy/telemetry-plane/README.md
+22 -2
View File
@@ -2,12 +2,13 @@
Portable local-only telemetry infrastructure for compute contours.
One Docker Compose project, `ndc-mission-core-telemetry`, owns three isolated
containers:
One Docker Compose project, `ndc-mission-core-telemetry`, owns three long-running
containers and one bounded bootstrap job:
- `ndc-mission-core-mqtt-broker` — Eclipse Mosquitto `2.1.2-alpine`;
- `ndc-mission-core-telemetry-timescaledb` — TimescaleDB HA OSS
`pg16.14-ts2.28.2-all-oss`;
- `ndc-mission-core-telemetry-bootstrap` — one-shot database role/bootstrap job;
- `ndc-mission-core-telemetry-normalizer` — Mission Core telemetry normalizer.
The Telegraf configuration templates cover Windows and Linux, but the agent runs as a
@@ -17,6 +18,11 @@ Windows performance counters, Docker Desktop and NVIDIA telemetry.
The normalizer exposes a read-only normalized telemetry adapter on
`127.0.0.1:18030`. Mission Core reads this adapter; the browser never receives MQTT or
Timescale credentials, and Timescale is not published on a host port.
The normalizer uses a separate `missioncore_normalizer` database role with only
`SELECT`, `INSERT`, `UPDATE`, and bounded retention `DELETE` on the telemetry
hypertable. Raw telemetry older than 30 days is removed at most once per day by the
normalizer. This stays compatible with the Apache-licensed Timescale image without
depending on the Timescale License retention scheduler.
The stack is one deployment contour, not one multi-process container. Keeping broker,
normalizer and database in separate containers preserves independent health checks,
@@ -62,6 +68,20 @@ directory are ignored by Git.
The product architecture and topic contract are defined in
`docs/adr/0031-local-compute-contour-telemetry-plane.md`.
## Security boundary
The current MQTT listener is authenticated and contour-scoped, but intentionally
uses plaintext MQTT inside one owner-controlled laboratory LAN. Do not expose port
1883 through a router, cellular WAN, public Wi-Fi, or an Internet-facing host. A
remote or shared-network deployment requires a reviewed TLS listener, a private CA
distributed to every agent, and credential rotation. Wi-Fi link encryption is not a
replacement for MQTT TLS.
Each agent credential is bound to one exact
`contours/<contour-id>/agents/<agent-id>/+` prefix. Adding another contour requires
issuing another password entry and explicit ACL row; the wildcard contour writer is
not permitted.
## Worker agent
Worker 006 uses the official Windows Telegraf distribution as the host service
deploy/telemetry-plane/compose.yaml
+57 -1
View File
@@ -10,6 +10,17 @@ services:
image: eclipse-mosquitto:2.1.2-alpine
container_name: ndc-mission-core-mqtt-broker
restart: unless-stopped
read_only: true
security_opt:
- no-new-privileges:true
cap_drop:
- ALL
cap_add:
- CHOWN
- SETGID
- SETUID
mem_limit: 128m
pids_limit: 128
labels:
<<: *ndc-labels
com.nodedc.role: mqtt-broker
@@ -23,6 +34,8 @@ services:
- ./runtime/mosquitto/passwords:/mosquitto/config/passwords:ro
- ./runtime/mosquitto/acl:/mosquitto/config/acl:ro
- broker-data:/mosquitto/data
tmpfs:
- /tmp:size=16m,noexec,nosuid
healthcheck:
test:
- CMD-SHELL
@@ -38,6 +51,10 @@ services:
image: timescale/timescaledb-ha:pg16.14-ts2.28.2-all-oss
container_name: ndc-mission-core-telemetry-timescaledb
restart: unless-stopped
security_opt:
- no-new-privileges:true
mem_limit: 1024m
pids_limit: 256
labels:
<<: *ndc-labels
com.nodedc.role: telemetry-database
@@ -54,6 +71,34 @@ services:
timeout: 5s
retries: 10
database-bootstrap:
image: timescale/timescaledb-ha:pg16.14-ts2.28.2-all-oss
container_name: ndc-mission-core-telemetry-bootstrap
restart: "no"
read_only: true
security_opt:
- no-new-privileges:true
cap_drop:
- ALL
mem_limit: 128m
pids_limit: 64
labels:
<<: *ndc-labels
com.nodedc.role: telemetry-database-bootstrap
depends_on:
timescale:
condition: service_healthy
environment:
MISSIONCORE_DB_NAME: "${MISSIONCORE_DB_NAME}"
MISSIONCORE_DB_ADMIN_USER: "${MISSIONCORE_DB_USER}"
MISSIONCORE_DB_ADMIN_PASSWORD: "${MISSIONCORE_DB_PASSWORD}"
MISSIONCORE_DB_INGEST_PASSWORD: "${MISSIONCORE_DB_INGEST_PASSWORD}"
volumes:
- ./timescale/002_roles_and_retention.sh:/bootstrap.sh:ro
tmpfs:
- /tmp:size=16m,noexec,nosuid
entrypoint: ["/bin/sh", "/bootstrap.sh"]
normalizer:
image: nodedc/mission-core-telemetry-normalizer:local
container_name: ndc-mission-core-telemetry-normalizer
@@ -61,6 +106,13 @@ services:
context: ../..
dockerfile: deploy/telemetry-plane/normalizer/Dockerfile
restart: unless-stopped
read_only: true
security_opt:
- no-new-privileges:true
cap_drop:
- ALL
mem_limit: 256m
pids_limit: 128
labels:
<<: *ndc-labels
com.nodedc.role: telemetry-normalizer
@@ -69,6 +121,8 @@ services:
condition: service_healthy
timescale:
condition: service_healthy
database-bootstrap:
condition: service_completed_successfully
ports:
- "127.0.0.1:${MISSIONCORE_TELEMETRY_QUERY_PORT:-18030}:18030"
environment:
@@ -77,9 +131,11 @@ services:
MISSIONCORE_MQTT_USERNAME: "${MISSIONCORE_MQTT_INGEST_USER}"
MISSIONCORE_MQTT_PASSWORD: "${MISSIONCORE_MQTT_INGEST_PASSWORD}"
MISSIONCORE_DATABASE_DSN: >-
postgresql://${MISSIONCORE_DB_USER}:${MISSIONCORE_DB_PASSWORD}@timescale:5432/${MISSIONCORE_DB_NAME}
postgresql://missioncore_normalizer:${MISSIONCORE_DB_INGEST_PASSWORD}@timescale:5432/${MISSIONCORE_DB_NAME}
MISSIONCORE_QUERY_HOST: "0.0.0.0"
MISSIONCORE_QUERY_PORT: "18030"
tmpfs:
- /tmp:size=32m,noexec,nosuid
healthcheck:
test:
- CMD
deploy/telemetry-plane/normalizer/normalizer.py
+191 -19
View File
@@ -1,5 +1,6 @@
from __future__ import annotations
import contextlib
import json
import os
import re
@@ -25,6 +26,40 @@ SOURCE_SCHEMAS: Final = {
TELEGRAF_SCHEMA: Final = "telegraf.metric-json/v1"
QUERY_SCHEMA: Final = "missioncore.telemetry-query/v1"
SAFE_IDENTIFIER: Final = re.compile(r"^[a-z0-9](?:[a-z0-9-]{0,62}[a-z0-9])?$")
MAX_PAYLOAD_BYTES: Final = 1024 * 1024
MAX_TAGS: Final = 256
MAX_SERIES_KEY_BYTES: Final = 4096
RETENTION_INTERVAL_SECONDS: Final = 86_400
ALLOWED_TELEMETRY_TAGS: Final = frozenset(
{
"agent_id",
"container",
"container_id",
"container_image",
"container_name",
"container_status",
"container_version",
"contour_id",
"cpu",
"device",
"engine_host",
"gpu_name",
"host",
"interface",
"lab_id",
"method_id",
"name",
"node_id",
"path",
"request_id",
"run_id",
"server_version",
"source_id",
"source_package_id",
"stage_id",
"stage_state",
}
)
def _required(name: str) -> str:
@@ -68,26 +103,48 @@ def _series_key(document: dict[str, Any]) -> str:
if not isinstance(tags, dict):
value = document.get("series_key")
return value if isinstance(value, str) else ""
if len(tags) > MAX_TAGS:
raise ValueError("telemetry payload contains too many tags")
stable_tags = {
str(name): str(value)
for name, value in tags.items()
if name not in {"agent_id", "contour_id", "host", "node_id"}
}
return json.dumps(
encoded = json.dumps(
stable_tags,
ensure_ascii=False,
sort_keys=True,
separators=(",", ":"),
)
if len(encoded.encode("utf-8")) > MAX_SERIES_KEY_BYTES:
raise ValueError("telemetry series identity is too large")
return encoded
def _sanitize_tags(document: dict[str, Any]) -> dict[str, Any]:
tags = document.get("tags")
if not isinstance(tags, dict):
return document
if len(tags) > MAX_TAGS:
raise ValueError("telemetry payload contains too many tags")
sanitized = {
str(name): str(value)
for name, value in tags.items()
if name in ALLOWED_TELEMETRY_TAGS
}
return {**document, "tags": sanitized}
def _normalize(topic: str, payload: bytes) -> tuple[object, ...]:
if len(payload) > MAX_PAYLOAD_BYTES:
raise ValueError("telemetry payload exceeds the 1 MiB contract")
match = TOPIC_RE.fullmatch(topic)
if match is None:
raise ValueError("topic is outside the telemetry contract")
document = json.loads(payload.decode("utf-8"))
if not isinstance(document, dict):
raise ValueError("payload must be an object")
document = _sanitize_tags(document)
kind = match.group("kind")
source_schema = document.get("schema_version")
if source_schema == SOURCE_SCHEMAS[kind]:
@@ -101,14 +158,18 @@ def _normalize(topic: str, payload: bytes) -> tuple[object, ...]:
source_schema = TELEGRAF_SCHEMA
observed_value = document.get("timestamp")
measurement = document["name"].strip()
if not measurement:
if not measurement or len(measurement) > 128:
raise ValueError("Telegraf measurement name is required")
else:
raise ValueError("payload schema does not match topic kind")
node_id = document.get("node_id") or _tag_text(document, "node_id")
if not node_id:
node_id = _tag_text(document, "host")
if not isinstance(node_id, str) or not node_id.strip():
if (
not isinstance(node_id, str)
or not node_id.strip()
or len(node_id.strip()) > 128
):
raise ValueError("node_id is required")
observed_at = _timestamp(observed_value)
return (
@@ -132,9 +193,53 @@ def _normalize(topic: str, payload: bytes) -> tuple[object, ...]:
class TelemetryQueryServer(ThreadingHTTPServer):
daemon_threads = True
def __init__(self, server_address: tuple[str, int], dsn: str) -> None:
def __init__(
self,
server_address: tuple[str, int],
dsn: str,
runtime_health: TelemetryRuntimeHealth,
) -> None:
super().__init__(server_address, TelemetryQueryHandler)
self.dsn = dsn
self.runtime_health = runtime_health
class TelemetryRuntimeHealth:
"""Thread-safe health of the actual MQTT-to-database ingestion path."""
def __init__(self) -> None:
self._lock = threading.Lock()
self._mqtt_connected = False
self._database_connected = False
self._last_ingested_at_utc: str | None = None
self._last_error: str | None = None
def set_mqtt(self, connected: bool, error: str | None = None) -> None:
with self._lock:
self._mqtt_connected = connected
self._last_error = error
def set_database(self, connected: bool, error: str | None = None) -> None:
with self._lock:
self._database_connected = connected
self._last_error = error
def mark_ingested(self) -> None:
with self._lock:
self._database_connected = True
self._last_ingested_at_utc = (
datetime.now(UTC).isoformat(timespec="seconds").replace("+00:00", "Z")
)
self._last_error = None
def snapshot(self) -> dict[str, object]:
with self._lock:
return {
"mqtt_connected": self._mqtt_connected,
"database_connected": self._database_connected,
"last_ingested_at_utc": self._last_ingested_at_utc,
"last_error": self._last_error,
}
class TelemetryQueryHandler(BaseHTTPRequestHandler):
@@ -174,6 +279,7 @@ class TelemetryQueryHandler(BaseHTTPRequestHandler):
def _health(self) -> None:
import psycopg # type: ignore[import-not-found]
database_reachable = False
try:
with (
psycopg.connect(self.server.dsn) as connection,
@@ -181,13 +287,24 @@ class TelemetryQueryHandler(BaseHTTPRequestHandler):
):
cursor.execute("SELECT 1")
cursor.fetchone()
database_reachable = True
except psycopg.Error:
self._json(
HTTPStatus.SERVICE_UNAVAILABLE,
{"ok": False, "schema_version": QUERY_SCHEMA},
)
return
self._json(HTTPStatus.OK, {"ok": True, "schema_version": QUERY_SCHEMA})
database_reachable = False
runtime = self.server.runtime_health.snapshot()
ok = (
database_reachable
and runtime["mqtt_connected"] is True
and runtime["database_connected"] is True
)
self._json(
HTTPStatus.OK if ok else HTTPStatus.SERVICE_UNAVAILABLE,
{
"ok": ok,
"schema_version": QUERY_SCHEMA,
"database_reachable": database_reachable,
**runtime,
},
)
def _latest(
self,
@@ -270,10 +387,13 @@ class TelemetryQueryHandler(BaseHTTPRequestHandler):
self.wfile.write(body)
def _start_query_server(dsn: str) -> ThreadingHTTPServer:
def _start_query_server(
dsn: str,
runtime_health: TelemetryRuntimeHealth,
) -> ThreadingHTTPServer:
host = os.environ.get("MISSIONCORE_QUERY_HOST", "0.0.0.0")
port = int(os.environ.get("MISSIONCORE_QUERY_PORT", "18030"))
server = TelemetryQueryServer((host, port), dsn)
server = TelemetryQueryServer((host, port), dsn, runtime_health)
thread = threading.Thread(
target=server.serve_forever,
name="telemetry-query",
@@ -294,8 +414,11 @@ def main() -> None:
port = int(os.environ.get("MISSIONCORE_MQTT_PORT", "1883"))
username = _required("MISSIONCORE_MQTT_USERNAME")
password = _required("MISSIONCORE_MQTT_PASSWORD")
_start_query_server(dsn)
runtime_health = TelemetryRuntimeHealth()
connection = psycopg.connect(dsn, autocommit=True)
last_retention_monotonic = 0.0
runtime_health.set_database(True)
_start_query_server(dsn, runtime_health)
client = mqtt.Client(
callback_api_version=CallbackAPIVersion.VERSION2,
client_id="ndc-mission-core-telemetry-normalizer",
@@ -311,14 +434,44 @@ def main() -> None:
_properties: Any,
) -> None:
if reason_code.is_failure:
runtime_health.set_mqtt(False, f"MQTT connection rejected: {reason_code}")
raise RuntimeError(f"MQTT connection rejected: {reason_code}")
connected_client.subscribe(TOPIC, qos=1)
runtime_health.set_mqtt(True)
def on_disconnect(
_connected_client: Any,
_userdata: object,
_disconnect_flags: Any,
reason_code: Any,
_properties: Any,
) -> None:
runtime_health.set_mqtt(False, f"MQTT disconnected: {reason_code}")
def on_message(_client: Any, _userdata: object, message: Any) -> None:
nonlocal connection, last_retention_monotonic
try:
row = _normalize(message.topic, message.payload)
with connection.cursor() as cursor:
cursor.execute(
except (ValueError, UnicodeDecodeError, json.JSONDecodeError) as exc:
print(f"telemetry sample rejected: {exc}", flush=True)
return
for attempt in range(2):
try:
with connection.cursor() as cursor:
now_monotonic = time.monotonic()
if (
now_monotonic - last_retention_monotonic
>= RETENTION_INTERVAL_SECONDS
):
cursor.execute(
"""
DELETE FROM contour_telemetry_samples
WHERE observed_at
< CURRENT_TIMESTAMP - INTERVAL '30 days'
"""
)
last_retention_monotonic = now_monotonic
cursor.execute(
"""
INSERT INTO contour_telemetry_samples (
observed_at, contour_id, agent_id, node_id, kind, measurement,
@@ -358,18 +511,37 @@ def main() -> None:
ELSE EXCLUDED.payload
END
""",
(*row, TELEGRAF_SCHEMA, TELEGRAF_SCHEMA),
)
except (ValueError, UnicodeDecodeError, json.JSONDecodeError, psycopg.Error) as exc:
print(f"telemetry sample rejected: {exc}", flush=True)
(*row, TELEGRAF_SCHEMA, TELEGRAF_SCHEMA),
)
runtime_health.mark_ingested()
return
except psycopg.Error as exc:
runtime_health.set_database(False, f"database ingestion failed: {exc}")
with contextlib.suppress(psycopg.Error):
connection.close()
if attempt == 0:
try:
connection = psycopg.connect(dsn, autocommit=True)
runtime_health.set_database(True)
except psycopg.Error as reconnect_error:
runtime_health.set_database(
False,
f"database reconnect failed: {reconnect_error}",
)
break
else:
break
print("telemetry sample rejected: database unavailable", flush=True)
client.on_connect = on_connect
client.on_disconnect = on_disconnect
client.on_message = on_message
while True:
try:
client.connect(host, port, keepalive=30)
client.loop_forever(retry_first_connection=True)
except (OSError, MQTTException, psycopg.Error) as exc:
runtime_health.set_mqtt(False, f"MQTT reconnecting: {exc}")
print(f"telemetry normalizer reconnecting: {exc}", flush=True)
time.sleep(3)
deploy/telemetry-plane/prepare.py
+71 -5
View File
@@ -3,6 +3,7 @@ from __future__ import annotations
import argparse
import ipaddress
import os
import re
import secrets
import subprocess
from pathlib import Path
@@ -13,6 +14,9 @@ ENV_PATH: Final = ROOT / ".env"
RUNTIME: Final = ROOT / "runtime" / "mosquitto"
IMAGE: Final = "eclipse-mosquitto:2.1.2-alpine"
PLACEHOLDER: Final = "replace-with-"
SAFE_IDENTIFIER: Final = re.compile(
r"^[a-z0-9](?:[a-z0-9-]{0,62}[a-z0-9])?$"
)
def _initialize_environment(bind_address: str, *, overwrite: bool = False) -> None:
@@ -26,9 +30,11 @@ def _initialize_environment(bind_address: str, *, overwrite: bool = False) -> No
"MISSIONCORE_DB_NAME": "missioncore_telemetry",
"MISSIONCORE_DB_USER": "missioncore_ingest",
"MISSIONCORE_DB_PASSWORD": secrets.token_urlsafe(36),
"MISSIONCORE_DB_INGEST_PASSWORD": secrets.token_urlsafe(36),
"MISSIONCORE_MQTT_INGEST_USER": "missioncore-ingest",
"MISSIONCORE_MQTT_INGEST_PASSWORD": secrets.token_urlsafe(36),
"MISSIONCORE_MQTT_WORKER_006_USER": "worker-006",
"MISSIONCORE_MQTT_WORKER_006_CONTOUR": "worker-006",
"MISSIONCORE_MQTT_WORKER_006_PASSWORD": secrets.token_urlsafe(36),
}
ENV_PATH.write_text(
@@ -51,6 +57,23 @@ def _environment() -> dict[str, str]:
return values
def _migrate_environment() -> None:
values = _environment()
additions: dict[str, str] = {}
if "MISSIONCORE_DB_INGEST_PASSWORD" not in values:
additions["MISSIONCORE_DB_INGEST_PASSWORD"] = secrets.token_urlsafe(36)
if "MISSIONCORE_MQTT_WORKER_006_CONTOUR" not in values:
additions["MISSIONCORE_MQTT_WORKER_006_CONTOUR"] = "worker-006"
if not additions:
return
with ENV_PATH.open("a", encoding="utf-8", newline="\n") as stream:
for name, value in additions.items():
stream.write(f"{name}={value}\n")
stream.flush()
os.fsync(stream.fileno())
os.chmod(ENV_PATH, 0o600)
def _required(values: dict[str, str], name: str) -> str:
value = values.get(name, "")
if not value or value.startswith(PLACEHOLDER):
@@ -58,6 +81,13 @@ def _required(values: dict[str, str], name: str) -> str:
return value
def _identifier(values: dict[str, str], name: str) -> str:
value = _required(values, name)
if SAFE_IDENTIFIER.fullmatch(value) is None:
raise RuntimeError(f"{name} must contain a DNS-safe lowercase identifier")
return value
def _password_entry(path: Path, username: str, password: str, *, create: bool) -> None:
command = [
"docker",
@@ -80,6 +110,22 @@ def _password_entry(path: Path, username: str, password: str, *, create: bool) -
)
def _prepare_password_entries(
path: Path,
ingest_user: str,
ingest_password: str,
worker_user: str,
worker_password: str,
) -> None:
_password_entry(
path,
ingest_user,
ingest_password,
create=not path.exists(),
)
_password_entry(path, worker_user, worker_password, create=False)
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument(
@@ -87,6 +133,11 @@ def main() -> None:
action="store_true",
help="create a private .env with generated local credentials",
)
parser.add_argument(
"--migrate",
action="store_true",
help="add newly required generated secrets without replacing existing values",
)
parser.add_argument(
"--mqtt-bind-address",
default="127.0.0.1",
@@ -95,17 +146,28 @@ def main() -> None:
arguments = parser.parse_args()
if arguments.initialize:
_initialize_environment(arguments.mqtt_bind_address)
if arguments.migrate:
_migrate_environment()
values = _environment()
ingest_user = _required(values, "MISSIONCORE_MQTT_INGEST_USER")
ingest_password = _required(values, "MISSIONCORE_MQTT_INGEST_PASSWORD")
worker_user = _required(values, "MISSIONCORE_MQTT_WORKER_006_USER")
worker_user = _identifier(values, "MISSIONCORE_MQTT_WORKER_006_USER")
worker_contour = _identifier(
values,
"MISSIONCORE_MQTT_WORKER_006_CONTOUR",
)
worker_password = _required(values, "MISSIONCORE_MQTT_WORKER_006_PASSWORD")
_required(values, "MISSIONCORE_DB_PASSWORD")
RUNTIME.mkdir(parents=True, exist_ok=True)
password_path = RUNTIME / "passwords"
_password_entry(password_path, ingest_user, ingest_password, create=True)
_password_entry(password_path, worker_user, worker_password, create=False)
_prepare_password_entries(
password_path,
ingest_user,
ingest_password,
worker_user,
worker_password,
)
acl_path = RUNTIME / "acl"
acl_path.write_text(
"\n".join(
@@ -114,8 +176,12 @@ def main() -> None:
"topic read mission-core/v1/contours/+/agents/+/+",
"topic read $SYS/broker/uptime",
"",
"# Agent username must equal its stable agent id.",
"pattern write mission-core/v1/contours/+/agents/%u/+",
"# Every credential is scoped to one contour and one stable agent.",
f"user {worker_user}",
(
"topic write mission-core/v1/contours/"
f"{worker_contour}/agents/{worker_user}/+"
),
"",
]
),
deploy/telemetry-plane/telegraf/Install-NdcMissionCoreTelegraf.ps1
+4 -2
View File
@@ -20,7 +20,8 @@ foreach ($name in @(
"MISSIONCORE_MQTT_HOST",
"MISSIONCORE_MQTT_PORT",
"MISSIONCORE_MQTT_USERNAME",
"MISSIONCORE_MQTT_PASSWORD"
"MISSIONCORE_MQTT_PASSWORD",
"MISSIONCORE_TELEMETRY_INTERVAL"
)) {
$value = $payload.$name
if (-not $value) {
@@ -87,7 +88,8 @@ try {
"MISSIONCORE_MQTT_HOST=$($payload.MISSIONCORE_MQTT_HOST)",
"MISSIONCORE_MQTT_PORT=$($payload.MISSIONCORE_MQTT_PORT)",
"MISSIONCORE_MQTT_USERNAME=$($payload.MISSIONCORE_MQTT_USERNAME)",
"MISSIONCORE_MQTT_PASSWORD=$($payload.MISSIONCORE_MQTT_PASSWORD)"
"MISSIONCORE_MQTT_PASSWORD=$($payload.MISSIONCORE_MQTT_PASSWORD)",
"MISSIONCORE_TELEMETRY_INTERVAL=$($payload.MISSIONCORE_TELEMETRY_INTERVAL)"
)
$serviceRegistryPath = "HKLM:\SYSTEM\CurrentControlSet\Services\$serviceName"
Set-ItemProperty -Path $serviceRegistryPath -Name Environment `
deploy/telemetry-plane/telegraf/Update-NdcMissionCoreTelegraf.ps1
+3
View File
@@ -26,6 +26,9 @@ foreach ($entry in @((Get-ItemProperty -Path $serviceRegistryPath).Environment))
Set-Item -Path "Env:$name" -Value $value
}
}
if (-not $env:MISSIONCORE_TELEMETRY_INTERVAL) {
$env:MISSIONCORE_TELEMETRY_INTERVAL = "2s"
}
$temporaryRoot = Join-Path $env:TEMP "ndc-mission-core-telegraf-update-$([Guid]::NewGuid().ToString('N'))"
$validationOutput = Join-Path $temporaryRoot "validation.out.log"
deploy/telemetry-plane/telegraf/mission-core-linux.conf.tmpl
+1 -1
View File
@@ -1,5 +1,5 @@
[agent]
interval = "2s"
interval = "${MISSIONCORE_TELEMETRY_INTERVAL}"
round_interval = true
omit_hostname = false
deploy/telemetry-plane/telegraf/mission-core-windows.conf.tmpl
+1 -1
View File
@@ -1,5 +1,5 @@
[agent]
interval = "2s"
interval = "${MISSIONCORE_TELEMETRY_INTERVAL}"
round_interval = true
omit_hostname = false
deploy/telemetry-plane/timescale/002_roles_and_retention.sh
+31
View File
@@ -0,0 +1,31 @@
#!/bin/sh
set -eu
export PGPASSWORD="${MISSIONCORE_DB_ADMIN_PASSWORD}"
psql \
--host timescale \
--username "${MISSIONCORE_DB_ADMIN_USER}" \
--dbname "${MISSIONCORE_DB_NAME}" \
--set ON_ERROR_STOP=1 \
--set database_name="${MISSIONCORE_DB_NAME}" \
--set ingest_password="${MISSIONCORE_DB_INGEST_PASSWORD}" <<'SQL'
DO $roles$
BEGIN
IF NOT EXISTS (
SELECT 1 FROM pg_roles WHERE rolname = 'missioncore_normalizer'
) THEN
CREATE ROLE missioncore_normalizer LOGIN;
END IF;
END
$roles$;
ALTER ROLE missioncore_normalizer PASSWORD :'ingest_password';
GRANT CONNECT ON DATABASE :"database_name" TO missioncore_normalizer;
GRANT USAGE ON SCHEMA public TO missioncore_normalizer;
GRANT SELECT, INSERT, UPDATE, DELETE ON contour_telemetry_samples
TO missioncore_normalizer;
DELETE FROM contour_telemetry_samples
WHERE observed_at < CURRENT_TIMESTAMP - INTERVAL '30 days';
SQL
docs/01_IMPLEMENTATION_PLAN.md
+19 -1
View File
@@ -3,7 +3,25 @@
This plan supersedes the app-dependent experiment order in the reference Bible.
Each gate produces evidence and an explicit GO, PAUSE or BLOCKED result.
## Current checkpoint — 2026-07-24
## Architecture stabilization checkpoint — 2026-07-28
| Gate | Current result |
| --- | --- |
| RAVNOVES00 methodology | CORRECTED — E41 proves E37 labels are engineering evidence, not independent truth, and reproduces connected development/evaluation overlap. The 146-row slice is historical evaluated visible validation. |
| Prediction/evaluation boundary | GO — truth-free E41 predictor package/result are physically separate from E37 and reproduce E40 predictions exactly; the visible evaluator runs only after inference. |
| Source-scoped quality | PAUSE — E40 visible engineering evaluation is 84.2466% presence, 84.2466% geometry association and 94.5205% freshness, with 11 high-severity failures, complete accounting and zero false-free claims. This is not a blind accuracy gate. |
| Structural regressions | GO (bounded) — E42 passes predictor identity/order/chunk invariance and PointSlab order/SE(3) checks. Raw-producer and cross-route invariance remain unproved. |
| Native pipeline telemetry | GO for producer/normalizer contract — lifecycle events and JSONL/MQTT sink boundaries exist and an E41 smoke run records real stage accounting. Durable Worker 006 MQTT wiring/deployment remains pending. |
| Evidence storage | MEASURED — E44 finds 525,471,092 logical bytes, 312,753,179 unique-content bytes and 1.680146× amplification across 14 E30–E40 roots. Exact-content references/deduplication precede any format migration. |
| Future transfer | PREREGISTERED — E43 freezes same-K1/mount/calibration/firmware, required streams, connected-component split and independent label reveal. Capture and labels do not yet exist. |
| Product interface | DEFERRED — no new windows, page anatomy or design changes are part of this stabilization increment. |
The governing decision is
[`ADR 0032`](adr/0032-ravnoves00-methodology-correction-before-transfer.md).
Historical E37–E40 artifacts remain immutable; only the claims made from them
change.
## Earlier checkpoint — 2026-07-24
| Stage | Result |
| --- | --- |
docs/13_LIDAR_WORKER_PRODUCT_AND_ROADMAP.md
+30 -6
View File
@@ -8,7 +8,8 @@ explainability implemented; L2.6e recorded-source-paced bounded shadow
qualified; E28 complete worker replay accepted; E29 camera-first semantic and
parallel geometry-only replay implemented; E30–E35 source-scoped qualification
accepted; RAVNOVES00 reference-source product maturation active; E36 transfer
deferred by ADR 0030
preregistered and deferred by ADR 0030/0032; E41 methodology boundary, E42
metamorphic checks and E44 amplification audit complete
Scope: passively received real-time K1 point/pose evidence, immutable replay and
future live shadow processing
Explicitly out of scope: K1 firmware modification, a new onboard exporter, new
@@ -652,6 +653,9 @@ to its source session under that audit profile. The transfer replay was not
created. ADR 0030 subsequently deferred TEST007 and E36 as current priorities:
RAVNOVES00 source-scoped quality, product workflow and a reference release
candidate now come first. No new capture is required or authorized by default.
ADR 0032 retains that priority while preregistering the later same-K1/new-route
capture under E43. The protocol is frozen; no capture or labels currently
exist.
The active gates are defined in
`docs/20_RAVNOVES00_REFERENCE_SOURCE_PRODUCT_PLAN.md`.
@@ -719,17 +723,37 @@ allowed to change; E31 determined the E32 profile; E32 determined the E33
runtime input; E32/E33 then bound the E34 layer; E32–E34 then bound E35.
Exact nominal and degradation accounting are closed, so A8 is complete.
R1 perception quality is now measured by two immutable Worker 006 results.
R1 perception quality was initially measured by two immutable Worker 006 results.
E38 established the baseline at 82.2% presence, 81.5% geometry association,
95.2% freshness and 14 high-severity failures. E39 result
`e39-perception-refinement-2fd253940c9d9a2fd3b3237f3f0932f81a9f69741935d793771243d5779af464`
used a fixed package-bound camera + LiDAR feature projection and improved
presence and geometry association to 84.9% while reducing high-severity
failures to 8. Freshness remained above target at 93.2%. Its development
cross-validation exceeded 90% but did not predict the sealed result, so R1
remains open. The next iteration is development-only grouped time/scene
qualification with a source-coordinate-free representation; validation is not
opened for item-level tuning.
cross-validation exceeded 90% but did not predict the evaluated slice, so R1
remains open.
The E41 source audit corrects the meaning of those measurements. E37's
`independent_ground_truth=false` provenance, connected group overlap and
prediction/evaluation co-location mean the 146 cases were historically
evaluated visible engineering labels, not a sealed blind set. E40 package-bound
result
`e40-perception-product-gate-e96eec9fd68c3ffaaee898d46285dd329191267200011680f084c75095b92e9a`
reached 84.2466% presence, 84.2466% geometry association and 94.5205% freshness
with 11 high-severity failures, complete accounting and zero false-free claims.
It remains useful source-scoped engineering evidence and does not close R1.
E41 result
`e41-methodology-audit-703dac176f20843ad6fbb9a1060b199942fba05e3bb34d3e2264ee87c7bfdd96`
reproduces `14` exact-frame, `38` track, `58` fixed-time-block and `59`
connected whole-track-or-scene overlaps. The replacement predictor package and
prediction contain no reference/split/scoring material, reproduce E40
predictions exactly and are evaluated only afterward by
`e41-visible-evaluation-57aa6c8569e8339630406e3c88cd539df13917fb08c1af57f0baae5cdd1069d2`.
E42 adds bounded structural invariants; E44 measures exact-content
amplification; E43 freezes the later independent transfer protocol. ADR 0032
contains the corrected decision boundary and current priority order.
E36 is the first generalization gate. A separate product decision follows:
either keep the result as operator/shadow evidence, or start L5 occupied-space
integration. No LAB in this cycle can enable navigation, commands or safety
docs/15_LABORATORY_RUN_CANON.md
+13 -1
View File
@@ -32,12 +32,24 @@ candidate is accepted:
- new perception and product iterations use RAVNOVES00;
- each iteration creates a new immutable LAB and never rewrites an older one;
- TEST007, E36 and new physical collection are deferred, not required gates;
- TEST007 and E36 replay are deferred, not required current gates;
- no agent may initiate or imply a new capture without explicit operator
authorization;
- the active quality target is at least `90%` on each frozen task-level
validation dimension, never one unqualified aggregate accuracy number.
ADR 0032 corrects the present validation meaning. E37's 146-row slice is
historical evaluated visible engineering validation: its labels are not
independent truth and its development/evaluation groups overlap. It remains an
immutable comparison contract, but it cannot close a blind accuracy gate.
Prediction packages must exclude truth, split, severity and scoring material,
and evaluation must join those artifacts only after prediction.
The owner has authorized preparation for a later same-K1/new-route recording,
not agent-initiated collection. E43 freezes that future protocol before data
exists. Until the recording is supplied, no LAB result, metric or transfer
claim is created for it.
This policy is a development priority, not a universality claim. Later
second-source transfer remains required before any cross-route or cross-camera
generalization statement.
docs/20_RAVNOVES00_REFERENCE_SOURCE_PRODUCT_PLAN.md
+96 -27
View File
@@ -1,8 +1,8 @@
# RAVNOVES00 reference-source product plan
Date: 2026-07-27
Date: 2026-07-28
Status: active
Status: active; methodology corrected by ADR 0032
ADR 0030 makes RAVNOVES00 the sole physical reference source for the current
Mission Core product-maturation cycle. This plan turns that decision into
@@ -24,13 +24,20 @@ missions. Mission Core should answer:
The product is not being designed for adversarial, tactical or continuously
novel environments.
## Acceptance meaning
## Acceptance meaning after source audit
`90%` is a target for reviewed task-level correctness on RAVNOVES00, not a
generic marketing accuracy score.
The next acceptance contract must freeze a validation set before further
tuning and measure at least:
E37 froze a useful source-scoped engineering contract, but E41 proved that its
labels are not independent ground truth and its development/validation rows
share frames, tracks and time/scene components. The existing 146-row slice is
therefore **historical evaluated visible validation**, not a sealed blind set.
Its metrics remain useful for immutable method comparison, but cannot prove
independent perception accuracy.
The next independent acceptance contract must freeze a leakage-free validation
set before labels are revealed and measure at least:
| Dimension | Question | Source-scoped target |
| --- | --- | ---: |
@@ -44,30 +51,35 @@ Targets apply separately. A strong presence score cannot compensate for bad
geometry association or hidden stale evidence.
The reviewed denominator, sampling strata and severity classes must be frozen
before the next tuning cycle. Development items may be used for diagnosis and
changes; sealed validation items may only be used for evaluation. High-impact
before the next independent tuning cycle. Connected scene/track/time components,
not individual item hashes, are the minimum partition unit. Development items
may be used for diagnosis and changes; independent validation labels remain
unavailable to the predictor until frozen predictions exist. High-impact
failures remain blocking even when an aggregate percentage passes.
## Current path
| Gate | Deliverable | Exit |
| --- | --- | --- |
| R0 — acceptance contract | Freeze task ontology, reviewed denominator, development/validation split, metrics, severity and error budget over RAVNOVES00 | Reproducible evaluation with no post-result denominator changes |
| R0 — acceptance methodology | Preserve the E37 visible engineering contract, enforce predictor/evaluator separation and preregister an independent, leakage-free future split | No truth material in the predictor; connected-component split; labels revealed only after frozen prediction |
| R1 — perception quality | Iterate detector, camera↔geometry association, source-time handling and conservative corrections as new immutable LABs | Each task-level validation dimension reaches `>= 90%` or has an explicit bounded exception |
| R2 — temporal product state | Refine current/held/stale/unknown presentation and bounded occupied telemetry without polluting the persistent reconstruction | Stable task-relevant state, closed accounting and deterministic degradation |
| R3 — product workflow | Complete reusable source/LAB selection, method summary, visual evidence, comparison, controls and conclusion templates | A non-expert can understand the run; an engineer can inspect exact provenance and metrics |
| R3 — product workflow | Separate owner-reviewed interface phase; no new windows, page anatomy or design changes in the present stabilization increment | Product brief reviewed separately before implementation |
| R4 — reference release candidate | Replay the complete RAVNOVES00 source through the accepted pipeline at recorded pace with regression, resource and recovery evidence | Source-scoped release decision for the known-location product envelope |
| R5 — later transfer | Record a new route/camera only after explicit operator authorization and run the frozen release candidate unchanged | Generalization decision; no retuning before comparison |
| R5 — later transfer | Under the preregistered E43 protocol, record the same K1 and mount over a new route after explicit operator authorization; freeze predictions before independent labels are revealed | Generalization decision; no retuning before comparison |
R0–R4 are the active path. R5 is intentionally deferred.
R0–R2 and non-UI R4 preparation are the active stabilization path. R3 is
explicitly deferred for separate discussion. R5 has a frozen protocol but no
capture or labels yet.
## Current evidence
R0 is closed by immutable result
E37 created immutable result
`e37-ravnoves-acceptance-01b1efd586f747341c712d82f0907b39436a6f91ae92b1dfae987eca05fd8344`.
It freezes 486 reviewed RAVNOVES00 items as 340 development and 146 sealed
validation cases, with separate presence, geometry-association and freshness
references.
It freezes 486 reviewed RAVNOVES00 items as 340 development and 146 visible
evaluation cases, with separate presence, geometry-association and freshness
references. Its own provenance records `484` engineering-reviewed items, `2`
human exceptions and `independent_ground_truth=false`.
The first R1 measurement is immutable result
`e38-perception-baseline-a272f82988cd9a7e071fad94c3e9fb49daf804fdcca523f853445fd3113a62b1`.
@@ -84,15 +96,59 @@ Its fixed robust three-neighbour refinement used exact camera-crop, LiDAR
shape and projection features and was selected only through five-fold
development cross-validation. Development reached 90.3% presence, 90.6%
geometry association and 96.5% freshness without validation-label access.
Sealed validation reached 84.9%, 84.9% and 93.2% respectively. Accounting
The then-designated validation slice reached 84.9%, 84.9% and 93.2%
respectively. Accounting
remains 100%, false-free claims remain zero and high-severity failures fall
from 14 to 8, but R1 is still not accepted.
The next R1 iteration must address the development-to-validation gap before
adding model complexity: grouped time/scene development folds replace
item-hash folds, and a source-coordinate-free representation is compared under
that harder protocol. No individual E39 validation label may be used for
diagnosis, fitting or selection.
E40 package-bound result
`e40-perception-product-gate-e96eec9fd68c3ffaaee898d46285dd329191267200011680f084c75095b92e9a`
used `89` camera-only development rows over `125` feature dimensions (`90`
variable and `35` constant). Against the already-visible E37 evaluation slice
it reached `84.2466%` presence, `84.2466%` geometry association and
`94.5205%` freshness, with `11` high-severity failures, complete accounting and
zero false-free claims. It does not pass R1.
E41 then audited the implementation and artifacts together. Immutable audit
`e41-methodology-audit-703dac176f20843ad6fbb9a1060b199942fba05e3bb34d3e2264ee87c7bfdd96`
found overlap across `14` exact frames, `38` track identities, `58` fixed
50-frame time blocks and `59` connected whole-track-or-scene groups. It also
proved that E40 prediction and evaluation material were co-located. R0 is
therefore reopened for blind acceptance.
The replacement boundary is executable:
- predictor package
`e41-predictor-package-fb35f42698013c63c6d417fd3b26986e9d3b89d12dce138bab18fe4203d593ad`
contains no reference, split, severity, scoring or truth material;
- prediction result
`e41-predictions-adcc1671de44646145a396d5815e522d6b4257d6ead9ace995761751655b3aac`
reproduces the physical E40 predictions exactly;
- separate visible evaluation
`e41-visible-evaluation-57aa6c8569e8339630406e3c88cd539df13917fb08c1af57f0baae5cdd1069d2`
reproduces the E40 engineering metrics and blockers only after inference.
E42 immutable result
`e42-metamorphic-suite-c358204e0b9e10eea5871e90b7c4fbe2c132f1f6d87074b2c5ffec18eaeebb40`
passes predictor identity, row-order and chunk-boundary invariance, metadata
absence, PointSlab row-order invariance and rigid-coordinate equivariance. It
does not prove raw-producer or cross-route invariance.
E44 immutable audit
`e44-data-amplification-89791978894ec785009e5b76c9325de4d9e910237af57476c9331eb47a8ccca4`
measures `525,471,092` logical bytes versus `312,753,179` unique-content bytes
across 14 admitted E30–E40 roots: `1.680146×` amplification and
`212,717,913` exact duplicate bytes. The immediate storage action is
content-addressed referencing and exact deduplication, not an unmeasured
format/database migration.
E43 immutable protocol
`e43-future-capture-protocol-28f091b9648daffce988d44c183e21f56d77988061630de934f8003fb13701d8`
preregisters the later same-K1/new-route transfer. It requires the same mount,
calibration and firmware; `480–900 s` duration; at least `60 s` control bridge
and `360 s` new route; camera, registered LiDAR, pose and pipeline telemetry;
connected component partitioning; two independent reviewers; and label reveal
only after frozen prediction. No such capture or labels currently exist.
## Experiment rules
@@ -109,15 +165,28 @@ diagnosis, fitting or selection.
7. No run claims universal accuracy, planner fitness, navigation or safety
acceptance merely because its source-scoped target passes.
## Deferred work
## Deferred work and current priority
Until R4 closes, the following are not current blockers:
The current non-UI priority order is:
1. keep E37–E40 status language methodologically honest;
2. make the E41 predictor/evaluator boundary and E42 invariants mandatory
regression checks;
3. wire native pipeline telemetry into durable worker stages without inferring
absent measurements;
4. design exact-content references/deduplication from E44 before any storage
migration;
5. preserve E43 unchanged until the owner supplies the new capture.
The following remain outside the present stabilization increment:
- TEST007 qualification;
- E36 second-source transfer;
- a new physical route or camera capture;
- E36 replay until an admitted source exists;
- initiation of a new physical route/camera capture by an agent;
- second K1 or changed mount generalization;
- open-world and adversarial evaluation.
The existing E36 audit and its Ops card remain retained so transfer work can
resume later without reconstructing history.
The owner may provide a future recording under E43. Until it exists, current
work neither depends on it nor simulates its result. The existing E36 audit and
its Ops card remain retained so transfer work can resume without reconstructing
history.
docs/adr/0031-local-compute-contour-telemetry-plane.md
+12 -3
View File
@@ -22,6 +22,14 @@ The existing Worker 006 containers were renamed in place to
container identities were preserved, so this namespace migration did not restart the
inference or perception runtimes.
The compute-side native pipeline contract is now implemented. It emits
`missioncore.agent-pipeline-telemetry/v1` lifecycle documents through an injected
transport, and the telemetry normalizer preserves source, method, stage and stage-state
identities in the normalized series key. The E41 runner has exercised the JSONL
evidence sink against the immutable predictor package. This proves producer and
normalizer compatibility; it does not claim that the durable Worker 006 process has
been wired to the MQTT sink or deployed with this code.
## Decision
Mission Core treats a compute worker as a configurable **local compute contour**, not as
@@ -188,6 +196,7 @@ subscribe to `mission-core/v1/contours/+/agents/+/+`.
- Mission Core code stays responsible for product semantics and stable API contracts,
not OS-specific metric collection.
- Host, container, network and inference telemetry is stored in the normalized path.
Per-stage LAB processing telemetry remains unavailable until a laboratory worker
publishes the native `runtime` and `pipeline` topic contracts; the UI must keep those
stages explicitly unavailable rather than infer them from aggregate hardware load.
- Per-stage LAB processing telemetry has a native producer contract and an admitted
already-connected MQTT sink. It remains unavailable for a durable worker run until
that worker actually injects the sink and publishes the `pipeline` topic; consumers
must not infer stages from aggregate hardware load.
docs/adr/0032-ravnoves00-methodology-correction-before-transfer.md
+89
View File
@@ -0,0 +1,89 @@
# ADR 0032 — RAVNOVES00 methodology correction before transfer
Date: 2026-07-28
Status: accepted for implementation
## Context
E37–E40 produced useful, immutable source-scoped engineering evidence, but the
architecture report was written before the source implementation and artifacts were
audited together. E41 reproduced the exact data lineage and found four methodological
limits:
1. E37 contains `484` engineering-reviewed labels and `2` human exception labels; it
explicitly declares `independent_ground_truth=false`.
2. The nominal `340` development / `146` validation split is not group-independent.
Development and validation overlap by `14` exact frames, `38` track identities,
`58` fixed 50-frame time blocks and `59` connected whole-track-or-scene groups.
3. The E40 worker package co-located prediction inputs with the visible acceptance
rows and split assignments.
4. E38–E40 therefore measure performance against a historically evaluated,
source-scoped engineering contract. They are not blind accuracy gates.
This does not invalidate their physical predictions, closed accounting or conservative
authority. It changes the claim that those artifacts are allowed to support.
## Decision
1. Historical E37–E40 artifacts remain immutable evidence. Their metrics are retained,
but “sealed”, “blind” and independent-accuracy language is removed from current
status.
2. The current `146`-item slice is named **historical evaluated visible validation**.
It may compare immutable methods against the source-scoped engineering contract; it
may not prove independent perception accuracy or cross-route generalization.
3. Prediction and evaluation are physically separated. The E41 predictor package and
result contain no acceptance rows, references, split assignments, severity or
scoring state. The visible evaluator joins predictions to E37 only after inference.
4. Runtime identity records the environment that actually executed a result. A local
run records the repository lock, Python and NumPy identities and is not represented
as a Worker 006 container run.
5. Structural invariants are exercised independently of the visible labels. E42
verifies predictor identity, row-order and chunk-boundary invariance plus PointSlab
row-order invariance and rigid-coordinate equivariance.
6. Native per-stage telemetry is a compute contract. Stages publish
`started/completed/failed`, exact source/method/run identity, duration and accounting
through an injected sink. A JSONL evidence sink and an already-connected MQTT sink
are admitted; no network transport starts implicitly.
7. Current storage work starts with content-addressed references and exact-content
deduplication. E44 measures `1.680146×` amplification across the admitted E30–E40
package/result roots. It does not authorize a storage-format or database migration.
8. The later same-K1 transfer capture is preregistered before collection. E43 freezes
minimum streams, route/control-bridge duration, connected scene/track/time
partitioning, independent human review and truth reveal only after frozen
prediction.
9. Product windows, page anatomy and new design work are outside this stabilization
decision and remain a separate owner review.
## Current immutable evidence
- methodology audit:
`e41-methodology-audit-703dac176f20843ad6fbb9a1060b199942fba05e3bb34d3e2264ee87c7bfdd96`;
- truth-free predictor package:
`e41-predictor-package-fb35f42698013c63c6d417fd3b26986e9d3b89d12dce138bab18fe4203d593ad`;
- truth-free prediction result:
`e41-predictions-adcc1671de44646145a396d5815e522d6b4257d6ead9ace995761751655b3aac`;
- visible engineering evaluation:
`e41-visible-evaluation-57aa6c8569e8339630406e3c88cd539df13917fb08c1af57f0baae5cdd1069d2`;
- metamorphic suite:
`e42-metamorphic-suite-c358204e0b9e10eea5871e90b7c4fbe2c132f1f6d87074b2c5ffec18eaeebb40`;
- preregistered future-capture protocol:
`e43-future-capture-protocol-28f091b9648daffce988d44c183e21f56d77988061630de934f8003fb13701d8`;
- data-amplification audit:
`e44-data-amplification-89791978894ec785009e5b76c9325de4d9e910237af57476c9331eb47a8ccca4`.
## Consequences
- RAVNOVES00 remains the active engineering reference source, but R0 is reopened as an
independent-truth and leakage-free-partition gate.
- E40 does not pass R1: the visible evaluation is `84.2466%` for presence,
`84.2466%` for geometry association and `94.5205%` for freshness, with `11`
high-severity failures, complete accounting and zero false-free claims.
- Further threshold or model tuning against the visible 146 items cannot close the
blind gate. It may only create another explicitly visible engineering comparison.
- Work that does not require a new recording can proceed now: methodology enforcement,
predictor/evaluator separation, invariants, telemetry, exact-content deduplication
design and preregistration.
- A later new-route recording can close an independent transfer question only if the
E43 protocol passes before labels are revealed. Until that capture exists,
cross-route quality remains unknown rather than failed or inferred.
experiments/perception/analyze_e40_development.py
+930
View File
@@ -0,0 +1,930 @@
#!/usr/bin/env python3
"""Development-only E40 feature and split audit.
The script intentionally rejects validation rows before feature extraction. It
compares route-coordinate-free candidates under two leakage-resistant
protocols:
* contiguous source-time folds;
* whole detector tracks plus 50-frame geometry scene windows.
It never writes a model or a sealed result. A production E40 profile can only
be frozen after one fixed candidate passes the development gate here.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import math
from collections import Counter
from pathlib import Path
from typing import Any
import numpy as np
from PIL import Image
from k1link.compute.e38_perception_baseline import _predict_tree, _train_tree
LABELS = (
"background-or-noise",
"object-present",
"occupied-environment",
)
TARGET = 0.9
CATEGORIES = {
"stratum": ("agree", "camera-only", "conflict", "geometry-only", "unknown"),
"range": ("near", "middle", "far", "unavailable"),
"geometry": (
"agree",
"conflict",
"single-source-camera",
"single-source-geometry",
"unknown",
"unavailable",
),
"label": ("car", "person", "truck", "bicycle", "motorcycle", "bus", "none"),
"reason": (
"camera-semantic-without-qualified-occupied-lidar-support",
"camera-semantic-with-connected-occupied-lidar-support",
"semantic-observation-not-current",
"camera-object-region-observed-as-local-surface",
"none",
),
"association": ("vehicle", "person", "bicycle", "motorcycle", "none"),
"motion": ("unknown", "static", "dynamic", "none"),
"camera_motion": ("unknown", "static", "dynamic", "none"),
"semantic_current": ("true", "false", "none"),
}
def main() -> int:
args = _parse_args()
acceptance_rows = _read_jsonl(args.acceptance_root / "acceptance-items.jsonl")
materialization_rows = _read_jsonl(args.materialization_root / "materialized-items.jsonl")
materialization_by_id = {str(row["item_id"]): row for row in materialization_rows}
development = [row for row in acceptance_rows if row.get("split") == "development"]
if len(development) != 340:
raise RuntimeError("E40 development denominator must remain 340")
if any(row.get("split") != "development" for row in development):
raise RuntimeError("E40 analysis received a non-development row")
feature_rows = [
_features(
acceptance=row,
materialization=materialization_by_id[str(row["item_id"])],
materialization_root=args.materialization_root,
)
for row in development
]
feature_names = sorted({name for row in feature_rows for name in row})
matrix = np.asarray(
[[row.get(name, 0.0) for name in feature_names] for row in feature_rows],
dtype=np.float64,
)
labels = np.asarray(
[LABELS.index(str(row["reference"]["presence"])) for row in development],
dtype=np.int64,
)
if not np.isfinite(matrix).all():
raise RuntimeError("E40 development features contain non-finite values")
assignments = {
"contiguous-source-time-five-fold": _contiguous_folds(development),
"whole-track-or-scene-window-five-fold": _track_scene_folds(
development,
materialization_by_id,
),
}
print(
json.dumps(
{
"development_items": len(development),
"feature_dimensions": len(feature_names),
"validation_rows_loaded_for_features": 0,
"protocols": {
name: {str(fold): count for fold, count in sorted(Counter(values).items())}
for name, values in assignments.items()
},
},
indent=2,
sort_keys=True,
)
)
feature_sets = {
"structured": [
index
for index, name in enumerate(feature_names)
if not _source_array_feature(name) and not name.startswith("image_")
],
"structured-shape": [
index for index, name in enumerate(feature_names) if not name.startswith("image_")
],
"structured-image": [
index
for index, name in enumerate(feature_names)
if not _source_array_feature(name)
or name.startswith("candidate_class_fraction_")
or name.startswith("image_")
],
"all": list(range(len(feature_names))),
}
candidates = [
("structured-knn-k3", "knn", 3, "structured"),
("structured-softmax-l2-0.01", "softmax", 0.01, "structured"),
(
"structured-shape-softmax-l2-0.01",
"softmax",
0.01,
"structured-shape",
),
(
"structured-image-softmax-l2-0.01",
"softmax",
0.01,
"structured-image",
),
("all-softmax-l2-0.01", "softmax", 0.01, "all"),
("all-softmax-l2-0.03", "softmax", 0.03, "all"),
("all-softmax-l2-0.1", "softmax", 0.1, "all"),
(
"hierarchical-structured-softmax-l2-0.01",
"hierarchical-softmax",
0.01,
"structured",
),
(
"hierarchical-structured-image-softmax-l2-0.01",
"hierarchical-softmax",
0.01,
"structured-image",
),
(
"hierarchical-all-softmax-l2-0.01",
"hierarchical-softmax",
0.01,
"all",
),
]
candidate_results: list[dict[str, Any]] = []
for candidate_name, model_type, model_parameter, feature_set in candidates:
columns = feature_sets[feature_set]
result = {
protocol: _cross_validate(
matrix=matrix[:, columns],
labels=labels,
feature_names=[feature_names[index] for index in columns],
feature_rows=feature_rows,
source_strata=[str(row["source_stratum"]) for row in development],
assignments=folds,
model_type=model_type,
model_parameter=model_parameter,
)
for protocol, folds in assignments.items()
}
candidate_results.append(
{
"candidate": candidate_name,
"feature_set": feature_set,
"feature_dimensions": len(columns),
"accuracy": {protocol: row["accuracy"] for protocol, row in result.items()},
"by_stratum": {protocol: row["by_stratum"] for protocol, row in result.items()},
"by_fold": {protocol: row["by_fold"] for protocol, row in result.items()},
"passed_both": all(row["accuracy"] >= TARGET for row in result.values()),
}
)
print(json.dumps(candidate_results, indent=2, sort_keys=True))
return 0
def _features(
*,
acceptance: dict[str, Any],
materialization: dict[str, Any],
materialization_root: Path,
) -> dict[str, float]:
snapshot = _object(materialization.get("e29_snapshot"))
evidence = _object(materialization.get("materialization"))
values: dict[str, float] = {}
observed = {
"stratum": materialization.get("stratum"),
"range": materialization.get("range_bucket"),
"geometry": snapshot.get("geometry_status"),
"label": snapshot.get("label") or "none",
"reason": snapshot.get("geometry_reason") or "none",
"association": snapshot.get("association_group") or "none",
"motion": snapshot.get("motion_state") or "none",
"camera_motion": snapshot.get("camera_motion_state") or "none",
"semantic_current": (
str(snapshot.get("semantic_current")).lower()
if snapshot.get("semantic_current") is not None
else "none"
),
}
for prefix, categories in CATEGORIES.items():
for category in categories:
values[f"{prefix}={category}"] = float(observed[prefix] == category)
for name in (
"selected_point_count",
"candidate_point_count",
"rejected_candidate_point_count",
"projected_point_count",
"frame_point_count",
):
values[name] = math.log1p(max(0.0, _number(evidence.get(name), 0.0)))
values["detector_score"] = _number(evidence.get("detector_score"), -1.0)
for name in (
"nearest_range_m",
"point_count",
"voxel_count",
"score",
"camera_motion_confidence",
"range_m",
):
raw = _number(snapshot.get(name), -1.0)
values[name] = (
math.log1p(raw) if name in {"point_count", "voxel_count"} and raw >= 0.0 else raw
)
bbox = snapshot.get("bbox_xyxy")
if isinstance(bbox, list) and len(bbox) == 4:
x1, y1, x2, y2 = (_number(value, 0.0) for value in bbox)
width = max(0.0, x2 - x1) / 800.0
height = max(0.0, y2 - y1) / 600.0
values.update(
{
"bbox_present": 1.0,
"bbox_center_x": (x1 + x2) / 1600.0,
"bbox_center_y": (y1 + y2) / 1200.0,
"bbox_width": width,
"bbox_height": height,
"bbox_area": width * height,
"bbox_aspect": width / (height + 1e-6),
}
)
else:
values.update(
{
"bbox_present": 0.0,
"bbox_center_x": -1.0,
"bbox_center_y": -1.0,
"bbox_width": -1.0,
"bbox_height": -1.0,
"bbox_area": -1.0,
"bbox_aspect": -1.0,
}
)
support = snapshot.get("support")
support = support if isinstance(support, dict) else {}
support_names = (
"below_surface_points_in_bbox",
"classified_points_in_bbox",
"connected_occupied_points",
"connected_occupied_voxels",
"occupied_points_in_bbox",
"projected_points_in_bbox",
"surface_points_in_bbox",
)
support_values = {name: _number(support.get(name), 0.0) for name in support_names}
values.update(
{f"support_{name}": math.log1p(max(0.0, value)) for name, value in support_values.items()}
)
projected = max(1.0, support_values["projected_points_in_bbox"])
occupied = max(1.0, support_values["occupied_points_in_bbox"])
values.update(
{
"support_occupied_fraction": (support_values["occupied_points_in_bbox"] / projected),
"support_classified_fraction": (
support_values["classified_points_in_bbox"] / projected
),
"support_surface_fraction": (support_values["surface_points_in_bbox"] / projected),
"support_connected_fraction": (support_values["connected_occupied_points"] / occupied),
}
)
_span_features(values, "bounds", snapshot.get("bounds_map_xyz_m"))
_range_span(values, "height_span", snapshot.get("height_range_m"))
_range_span(
values,
"occupied_height_span",
snapshot.get("occupied_height_range_m"),
)
artifact = _object(materialization.get("artifact"))
artifact_path = materialization_root / str(artifact.get("path"))
with np.load(artifact_path, allow_pickle=False) as arrays:
pixels = arrays["projected_pixels_xy"]
candidate_mask = arrays["projected_candidate_mask"].astype(bool)
selected_mask = arrays["projected_selected_mask"].astype(bool)
sensor_position = arrays["sensor_position_map_xyz_m"]
sensor_orientation = arrays["sensor_orientation_map_from_lidar_xyzw"]
_named_point_statistics(
values,
"candidate_lidar",
arrays["candidate_points_map_xyz_m"],
sensor_position,
sensor_orientation,
)
_named_point_statistics(
values,
"selected_lidar",
arrays["selected_points_map_xyz_m"],
sensor_position,
sensor_orientation,
)
projection_width = max(1, int(evidence.get("projection_width", 800)))
projection_height = max(1, int(evidence.get("projection_height", 600)))
_named_pixel_statistics(
values,
"candidate_pixel",
pixels[candidate_mask],
projection_width,
projection_height,
)
_named_pixel_statistics(
values,
"selected_pixel",
pixels[selected_mask],
projection_width,
projection_height,
)
_named_quantiles(
values,
"candidate_depth",
arrays["projected_depth_m"][candidate_mask],
)
_named_quantiles(
values,
"selected_depth",
arrays["projected_depth_m"][selected_mask],
)
_named_quantiles(
values,
"candidate_height",
arrays["projected_point_height_m"][candidate_mask],
)
_named_quantiles(
values,
"selected_height",
arrays["projected_point_height_m"][selected_mask],
)
point_classes = arrays["projected_point_class"][candidate_mask]
for index in range(8):
values[f"candidate_class_fraction_{index}"] = (
float(np.mean(point_classes == index)) if point_classes.size else 0.0
)
_image_features(
values,
materialization=materialization,
materialization_root=materialization_root,
candidate_pixels=pixels[candidate_mask],
projection_width=projection_width,
projection_height=projection_height,
)
# Source frame, session time, review ordinal, track ID and absolute map
# coordinates are deliberately absent.
return values
def _cross_validate(
*,
matrix: np.ndarray,
labels: np.ndarray,
feature_names: list[str],
feature_rows: list[dict[str, float]],
source_strata: list[str],
assignments: np.ndarray,
model_type: str,
model_parameter: object,
) -> dict[str, Any]:
predictions = np.full(len(labels), -1, dtype=np.int64)
for fold in range(5):
train_indices = np.flatnonzero(assignments != fold)
test_indices = np.flatnonzero(assignments == fold)
train, test = _robust_transform(
matrix[train_indices],
matrix[test_indices],
)
if model_type == "knn":
distances = np.mean(
np.square(test[:, None, :] - train[None, :, :]),
axis=2,
)
nearest = np.argsort(distances, axis=1, kind="stable")[:, : int(model_parameter)]
for local_index, neighbor_indices in enumerate(nearest):
votes = Counter(labels[train_indices][neighbor_indices])
predictions[test_indices[local_index]] = sorted(
votes.items(),
key=lambda item: (-item[1], item[0]),
)[0][0]
elif model_type == "softmax":
weights = _train_softmax(
train,
labels[train_indices],
l2=float(model_parameter),
)
predictions[test_indices] = np.argmax(
np.column_stack((test, np.ones(len(test)))) @ weights,
axis=1,
)
elif model_type == "hierarchical-softmax":
_predict_hierarchical_fold(
predictions=predictions,
train_indices=train_indices,
test_indices=test_indices,
training=train,
testing=test,
labels=labels,
source_strata=source_strata,
l2=float(model_parameter),
)
elif model_type == "tree":
depth, min_leaf = model_parameter # type: ignore[misc]
tree = _train_tree(
[
(
{name: feature_rows[index].get(name, 0.0) for name in feature_names},
LABELS[int(labels[index])],
)
for index in train_indices
],
feature_names=feature_names,
max_depth=int(depth),
min_leaf=int(min_leaf),
)
for index in test_indices:
predictions[index] = LABELS.index(_predict_tree(tree, feature_rows[int(index)]))
else:
raise RuntimeError(f"unsupported model: {model_type}")
accuracy = float(np.mean(predictions == labels))
stratum_metrics = {}
for stratum in sorted(set(source_strata)):
indices = np.asarray(
[index for index, value in enumerate(source_strata) if value == stratum],
dtype=np.int64,
)
stratum_metrics[stratum] = round(
float(np.mean(predictions[indices] == labels[indices])),
6,
)
fold_metrics = {}
for fold in range(5):
indices = np.flatnonzero(assignments == fold)
fold_metrics[str(fold)] = round(
float(np.mean(predictions[indices] == labels[indices])),
6,
)
return {
"accuracy": round(accuracy, 6),
"correct": int(np.sum(predictions == labels)),
"incorrect": int(np.sum(predictions != labels)),
"passed": accuracy >= TARGET,
"by_stratum": stratum_metrics,
"by_fold": fold_metrics,
"confusion": [
{
"reference": LABELS[reference],
"prediction": LABELS[prediction],
"count": count,
}
for (reference, prediction), count in sorted(
Counter(
zip(
labels.tolist(),
predictions.tolist(),
strict=True,
)
).items(),
key=lambda item: (-item[1], item[0]),
)
],
}
def _predict_hierarchical_fold(
*,
predictions: np.ndarray,
train_indices: np.ndarray,
test_indices: np.ndarray,
training: np.ndarray,
testing: np.ndarray,
labels: np.ndarray,
source_strata: list[str],
l2: float,
) -> None:
fixed = {
"conflict": LABELS.index("background-or-noise"),
"agree": LABELS.index("object-present"),
"unknown": LABELS.index("object-present"),
# A geometry-only cluster is positive occupied evidence, but without
# camera semantics it must not be promoted to a named object. Keeping
# it occupied is the conservative product state.
"geometry-only": LABELS.index("occupied-environment"),
}
for source_index in test_indices:
stratum = source_strata[int(source_index)]
if stratum in fixed:
predictions[source_index] = fixed[stratum]
for stratum, fallback in (("camera-only", LABELS.index("object-present")),):
local_training = np.asarray(
[
index
for index, source_index in enumerate(train_indices)
if source_strata[int(source_index)] == stratum
],
dtype=np.int64,
)
local_testing = np.asarray(
[
index
for index, source_index in enumerate(test_indices)
if source_strata[int(source_index)] == stratum
],
dtype=np.int64,
)
if not len(local_testing):
continue
observed = sorted(set(labels[train_indices][local_training].tolist()))
if len(local_training) < 10 or len(observed) < 2:
predictions[test_indices[local_testing]] = fallback
continue
weights = _train_softmax(
training[local_training],
labels[train_indices][local_training],
l2=l2,
)
predictions[test_indices[local_testing]] = np.argmax(
np.column_stack(
(
testing[local_testing],
np.ones(len(local_testing)),
)
)
@ weights,
axis=1,
)
def _train_softmax(
matrix: np.ndarray,
labels: np.ndarray,
*,
l2: float,
) -> np.ndarray:
rows, dimensions = matrix.shape
design = np.column_stack((matrix, np.ones(rows)))
targets = np.eye(len(LABELS), dtype=np.float64)[labels]
weights = np.zeros((dimensions + 1, len(LABELS)), dtype=np.float64)
first_moment = np.zeros_like(weights)
second_moment = np.zeros_like(weights)
for step in range(1, 1201):
logits = design @ weights
logits -= np.max(logits, axis=1, keepdims=True)
probabilities = np.exp(logits)
probabilities /= np.sum(probabilities, axis=1, keepdims=True)
regularizer = np.vstack((weights[:-1], np.zeros((1, len(LABELS)))))
gradient = design.T @ (probabilities - targets) / rows
gradient += l2 * regularizer
first_moment = 0.9 * first_moment + 0.1 * gradient
second_moment = 0.999 * second_moment + 0.001 * np.square(gradient)
corrected_first = first_moment / (1.0 - 0.9**step)
corrected_second = second_moment / (1.0 - 0.999**step)
weights -= 0.03 * corrected_first / (np.sqrt(corrected_second) + 1e-8)
return weights
def _robust_transform(
training: np.ndarray,
testing: np.ndarray,
) -> tuple[np.ndarray, np.ndarray]:
median = np.median(training, axis=0)
scale = np.percentile(training, 75, axis=0) - np.percentile(
training,
25,
axis=0,
)
scale[scale < 1e-8] = 1.0
return (
np.clip((training - median) / scale, -10.0, 10.0),
np.clip((testing - median) / scale, -10.0, 10.0),
)
def _source_array_feature(name: str) -> bool:
return name.startswith(
(
"candidate_lidar_",
"selected_lidar_",
"candidate_pixel_",
"selected_pixel_",
"candidate_depth_",
"selected_depth_",
"candidate_height_",
"selected_height_",
"candidate_class_fraction_",
)
)
def _contiguous_folds(rows: list[dict[str, Any]]) -> np.ndarray:
order = np.argsort(
[int(row["source_frame_index"]) for row in rows],
kind="stable",
)
assignments = np.empty(len(rows), dtype=np.int64)
for fold, indices in enumerate(np.array_split(order, 5)):
assignments[indices] = fold
return assignments
def _track_scene_folds(
rows: list[dict[str, Any]],
materialization_by_id: dict[str, dict[str, Any]],
) -> np.ndarray:
assignments: list[int] = []
for row in rows:
snapshot = _object(materialization_by_id[str(row["item_id"])].get("e29_snapshot"))
track_id = snapshot.get("track_id")
group = (
f"track:{track_id}"
if track_id is not None
else f"scene:{int(row['source_frame_index']) // 50}"
)
digest = hashlib.sha256(f"e40:{group}".encode()).hexdigest()
assignments.append(int(digest[:8], 16) % 5)
return np.asarray(assignments, dtype=np.int64)
def _named_point_statistics(
values: dict[str, float],
prefix: str,
points_map: np.ndarray,
sensor_position: np.ndarray,
sensor_orientation_xyzw: np.ndarray,
) -> None:
points = np.asarray(points_map, dtype=np.float64)
if points.ndim != 2 or points.shape[1] != 3 or not len(points):
for axis in "xyz":
_named_quantiles(values, f"{prefix}_{axis}", np.asarray([]))
for index in range(3):
values[f"{prefix}_covariance_ratio_{index}"] = 0.0
return
rotation = _rotation_matrix(sensor_orientation_xyzw)
points_local = (points - np.asarray(sensor_position)) @ rotation
for axis, index in zip("xyz", range(3), strict=True):
_named_quantiles(values, f"{prefix}_{axis}", points_local[:, index])
if len(points_local) >= 3:
eigenvalues = np.maximum(
np.linalg.eigvalsh(np.cov(points_local, rowvar=False)),
0.0,
)
else:
eigenvalues = np.zeros(3)
ratios = eigenvalues / (float(np.sum(eigenvalues)) + 1e-9)
for index, ratio in enumerate(ratios):
values[f"{prefix}_covariance_ratio_{index}"] = float(ratio)
def _rotation_matrix(quaternion_xyzw: np.ndarray) -> np.ndarray:
x, y, z, w = (float(value) for value in quaternion_xyzw)
return np.asarray(
[
[
1.0 - 2.0 * (y * y + z * z),
2.0 * (x * y - z * w),
2.0 * (x * z + y * w),
],
[
2.0 * (x * y + z * w),
1.0 - 2.0 * (x * x + z * z),
2.0 * (y * z - x * w),
],
[
2.0 * (x * z - y * w),
2.0 * (y * z + x * w),
1.0 - 2.0 * (x * x + y * y),
],
],
dtype=np.float64,
)
def _named_pixel_statistics(
values: dict[str, float],
prefix: str,
pixels: np.ndarray,
width: int,
height: int,
) -> None:
points = np.asarray(pixels, dtype=np.float64)
if points.ndim != 2 or points.shape[1] != 2 or not len(points):
_named_quantiles(values, f"{prefix}_x", np.asarray([]))
_named_quantiles(values, f"{prefix}_y", np.asarray([]))
values[f"{prefix}_span_x"] = -1.0
values[f"{prefix}_span_y"] = -1.0
values[f"{prefix}_density"] = -1.0
return
x = points[:, 0] / width
y = points[:, 1] / height
_named_quantiles(values, f"{prefix}_x", x)
_named_quantiles(values, f"{prefix}_y", y)
span_x = max(float(np.max(x) - np.min(x)), 1.0 / width)
span_y = max(float(np.max(y) - np.min(y)), 1.0 / height)
values[f"{prefix}_span_x"] = span_x
values[f"{prefix}_span_y"] = span_y
values[f"{prefix}_density"] = len(points) / (span_x * span_y * width * height + 1.0)
def _image_features(
values: dict[str, float],
*,
materialization: dict[str, Any],
materialization_root: Path,
candidate_pixels: np.ndarray,
projection_width: int,
projection_height: int,
) -> None:
snapshot = _object(materialization.get("e29_snapshot"))
bbox = snapshot.get("bbox_xyxy")
if not isinstance(bbox, list) or len(bbox) != 4:
points = np.asarray(candidate_pixels, dtype=np.float64)
if points.ndim != 2 or points.shape[1] != 2 or not len(points):
_empty_image_features(values)
return
low = np.min(points, axis=0)
high = np.max(points, axis=0)
center = (low + high) / 2.0
support = np.maximum(high - low, 48.0)
bbox = [
center[0] - support[0],
center[1] - support[1],
center[0] + support[0],
center[1] + support[1],
]
x1, y1, x2, y2 = (_number(value, 0.0) for value in bbox)
box = (
max(0, int(x1)),
max(0, int(y1)),
min(projection_width, int(math.ceil(x2))),
min(projection_height, int(math.ceil(y2))),
)
if box[2] <= box[0] or box[3] <= box[1]:
_empty_image_features(values)
return
frame = _object(materialization.get("camera_frame"))
with Image.open(materialization_root / str(frame.get("path"))) as source:
rgb = (
np.asarray(
source.convert("RGB").crop(box).resize((32, 32), Image.Resampling.BILINEAR),
dtype=np.float64,
)
/ 255.0
)
for channel, name in enumerate(("red", "green", "blue")):
histogram, _ = np.histogram(
rgb[:, :, channel],
bins=4,
range=(0.0, 1.0),
)
histogram = histogram / max(1, int(np.sum(histogram)))
for index, fraction in enumerate(histogram):
values[f"image_{name}_histogram_{index}"] = float(fraction)
luma = np.mean(rgb, axis=2)
saturation = np.max(rgb, axis=2) - np.min(rgb, axis=2)
edge = np.concatenate(
(
np.abs(np.diff(luma, axis=1)).reshape(-1),
np.abs(np.diff(luma, axis=0)).reshape(-1),
)
)
values["image_luma_mean"] = float(np.mean(luma))
values["image_luma_std"] = float(np.std(luma))
for name, quantile in zip(
("p10", "p25", "p50", "p75", "p90"),
np.quantile(luma, (0.1, 0.25, 0.5, 0.75, 0.9)),
strict=True,
):
values[f"image_luma_{name}"] = float(quantile)
values["image_saturation_mean"] = float(np.mean(saturation))
values["image_saturation_std"] = float(np.std(saturation))
values["image_edge_mean"] = float(np.mean(edge))
values["image_edge_p90"] = float(np.quantile(edge, 0.9))
small_luma = (
np.asarray(
Image.fromarray(np.uint8(np.clip(luma * 255.0, 0.0, 255.0))).resize(
(4, 4),
Image.Resampling.BILINEAR,
),
dtype=np.float64,
)
/ 255.0
)
small_luma -= float(np.mean(small_luma))
for y in range(4):
for x in range(4):
values[f"image_luma_centered_{y}_{x}"] = float(small_luma[y, x])
def _empty_image_features(values: dict[str, float]) -> None:
for channel in ("red", "green", "blue"):
for index in range(4):
values[f"image_{channel}_histogram_{index}"] = 0.0
for name in (
"mean",
"std",
"p10",
"p25",
"p50",
"p75",
"p90",
):
values[f"image_luma_{name}"] = 0.0
values["image_saturation_mean"] = 0.0
values["image_saturation_std"] = 0.0
values["image_edge_mean"] = 0.0
values["image_edge_p90"] = 0.0
for y in range(4):
for x in range(4):
values[f"image_luma_centered_{y}_{x}"] = 0.0
def _named_quantiles(
values: dict[str, float],
prefix: str,
raw: object,
) -> None:
array = np.asarray(raw, dtype=np.float64).reshape(-1)
finite = array[np.isfinite(array)]
names = ("min", "p10", "p25", "p50", "p75", "p90", "max")
quantiles = (
np.quantile(finite, (0.0, 0.1, 0.25, 0.5, 0.75, 0.9, 1.0))
if finite.size
else np.full(7, -1.0)
)
for name, value in zip(names, quantiles, strict=True):
values[f"{prefix}_{name}"] = float(value)
def _span_features(
values: dict[str, float],
prefix: str,
raw: object,
) -> None:
if (
isinstance(raw, list)
and len(raw) == 2
and all(isinstance(item, list) and len(item) == 3 for item in raw)
):
for index, axis in enumerate("xyz"):
values[f"{prefix}_{axis}_span"] = _number(
raw[1][index],
-1.0,
) - _number(raw[0][index], -1.0)
else:
for axis in "xyz":
values[f"{prefix}_{axis}_span"] = -1.0
def _range_span(
values: dict[str, float],
name: str,
raw: object,
) -> None:
values[name] = (
_number(raw[1], -1.0) - _number(raw[0], -1.0)
if isinstance(raw, list) and len(raw) == 2
else -1.0
)
def _number(value: object, fallback: float) -> float:
try:
parsed = float(value) # type: ignore[arg-type]
except (TypeError, ValueError):
return fallback
return parsed if math.isfinite(parsed) else fallback
def _object(value: object) -> dict[str, Any]:
if not isinstance(value, dict):
raise RuntimeError("expected object")
return value
def _read_jsonl(path: Path) -> list[dict[str, Any]]:
with path.open("r", encoding="utf-8-sig") as stream:
return [_object(json.loads(line)) for line in stream]
def _parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser()
parser.add_argument("--acceptance-root", type=Path, required=True)
parser.add_argument("--materialization-root", type=Path, required=True)
return parser.parse_args()
if __name__ == "__main__":
raise SystemExit(main())
experiments/perception/e40_ravnoves00_product_gate_profile.json
+38
View File
@@ -0,0 +1,38 @@
{
"authority": {
"commands_enabled": false,
"navigation_or_safety_accepted": false
},
"model": {
"cross_validation_folds": 5,
"cross_validation_seed": "e40-grouped-dev-cv",
"feature_set": "route-coordinate-free-structured-image/v1",
"fixed_presence_by_stratum": {
"agree": "object-present",
"conflict": "background-or-noise",
"geometry-only": "occupied-environment",
"unknown": "object-present"
},
"l2": 0.01,
"learning_rate": 0.03,
"robust_clip": 10.0,
"steps": 1200,
"type": "hierarchical-stratum-softmax"
},
"profile_id": "e40-ravnoves00-leakage-resistant-product-gate/v1",
"schema_version": "missioncore.e40-perception-product-gate-profile/v1",
"source": {
"acceptance_result_id": "e37-ravnoves-acceptance-01b1efd586f747341c712d82f0907b39436a6f91ae92b1dfae987eca05fd8344",
"display_name": "RAVNOVES00",
"materialization_id": "e30-materialization-841af926d8d28ab93538c46d8f31278a2234c4d1c12c7dc4dc296b249d59735a",
"session_id": "20260720T065719Z_viewer_live"
},
"targets": {
"accounting_target": 1.0,
"freshness_target": 0.9,
"geometry_association_target": 0.9,
"maximum_false_free_claims": 0,
"maximum_high_severity_failures": 0,
"presence_target": 0.9
}
}
experiments/perception/e41_ravnoves00_methodology_profile.json
+30
View File
@@ -0,0 +1,30 @@
{
"authority": {
"commands_enabled": false,
"navigation_or_safety_accepted": false
},
"policy": {
"current_validation_semantics": "historical-evaluated-visible-validation",
"forbidden_feature_tokens": [
"item_id",
"map_xyz",
"path",
"review_ordinal",
"session_id",
"session_seconds",
"source_frame",
"track_id"
],
"independent_truth_required_for_blind": true,
"predictor_truth_separation_required": true,
"time_block_frames": 50
},
"profile_id": "e41-ravnoves00-methodology-audit/v1",
"schema_version": "missioncore.e41-methodology-audit-profile/v1",
"source": {
"acceptance_result_id": "e37-ravnoves-acceptance-01b1efd586f747341c712d82f0907b39436a6f91ae92b1dfae987eca05fd8344",
"e40_package_id": "e40-worker-package-0b7c1aa6d1d31172206b125002928f9adfd8ea3c0a8f95c6caae431bfafff247",
"e40_result_id": "e40-perception-product-gate-e96eec9fd68c3ffaaee898d46285dd329191267200011680f084c75095b92e9a",
"materialization_id": "e30-materialization-841af926d8d28ab93538c46d8f31278a2234c4d1c12c7dc4dc296b249d59735a"
}
}
experiments/perception/e43_same_k1_new_route_profile.json
+47
View File
@@ -0,0 +1,47 @@
{
"acceptance_contract": {
"accounting_target": 1.0,
"freshness_target": 0.9,
"geometry_association_target": 0.9,
"maximum_false_free_claims": 0,
"maximum_high_severity_failures": 0,
"presence_target": 0.9
},
"authority": {
"commands_enabled": false,
"navigation_or_safety_accepted": false
},
"blind_truth_contract": {
"blind_fraction": 0.3,
"engineering_acceptance_labels_are_truth": false,
"independent_human_reviewers": 2,
"labels_revealed_after_frozen_prediction": true,
"partition_strategy": "connected-scene-track-time-components/v1",
"seed": "mission-core-e43-same-k1-new-route-v1"
},
"capture_contract": {
"device_model": "XGRIDS/LixelKity-K1",
"maximum_duration_seconds": 900,
"minimum_duration_seconds": 480,
"required_segments": [
{
"kind": "control-bridge",
"minimum_duration_seconds": 60
},
{
"kind": "new-route",
"minimum_duration_seconds": 360
}
],
"required_streams": [
"sensor.camera.right",
"sensor.lidar.registered-map-increment",
"sensor.pose",
"telemetry.pipeline"
],
"route_policy": "control-bridge-then-new-route/v1",
"same_device_mount_calibration_firmware_required": true
},
"profile_id": "e43-same-k1-new-route-truth-island/v1",
"schema_version": "missioncore.e43-future-capture-profile/v1"
}
experiments/perception/prepare_e40_worker_package.py
+434
View File
@@ -0,0 +1,434 @@
#!/usr/bin/env python3
"""Build an immutable camera/LiDAR E40 package for Worker 006."""
from __future__ import annotations
import argparse
import hashlib
import io
import json
import os
import shutil
import uuid
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
import numpy as np
from k1link.compute.e40_perception_product_gate import (
_FEATURE_CACHE_ARRAYS,
_FEATURE_CACHE_MANIFEST,
_FEATURE_CACHE_SCHEMA,
E40_PACKAGE_SCHEMA,
E40_PROFILE_SCHEMA,
_feature_names,
_feature_vector,
)
_RUNTIME_FILES = {
"runtime/k1link/__init__.py": "src/k1link/__init__.py",
"runtime/k1link/compute/__init__.py": None,
"runtime/k1link/compute/e37_acceptance_contract.py": (
"src/k1link/compute/e37_acceptance_contract.py"
),
"runtime/k1link/compute/e40_perception_product_gate.py": (
"src/k1link/compute/e40_perception_product_gate.py"
),
"runtime/run_e40_perception_product_gate.py": (
"experiments/perception/worker/run_e40_perception_product_gate.py"
),
"runtime/validate_e40_worker_package.py": (
"experiments/perception/worker/validate_e40_worker_package.py"
),
"runtime/Invoke-E40PerceptionProductGate.ps1": (
"experiments/perception/worker/Invoke-E40PerceptionProductGate.ps1"
),
}
_GENERATED_COMPUTE_INIT = (
'"""Minimal E40 worker projection; import contract modules explicitly."""\n'
)
_ACCEPTANCE_FILES = (
"manifest.json",
"acceptance-items.jsonl",
"acceptance-contract.json",
"run-report.json",
)
_MATERIALIZATION_FILES = ("manifest.json", "materialized-items.jsonl")
class E40WorkerPackageError(RuntimeError):
"""The E40 package source or immutable package is invalid."""
def build_e40_worker_package(
*,
repository_root: Path,
acceptance_root: Path,
materialization_root: Path,
profile_path: Path,
output_root: Path,
) -> Path:
"""Build or verify one content-addressed E40 worker package."""
repository = repository_root.resolve(strict=True)
profile_source = profile_path.resolve(strict=True)
profile = _read_json(profile_source)
if profile.get("schema_version") != E40_PROFILE_SCHEMA:
raise E40WorkerPackageError("E40 package profile is incompatible")
acceptance = acceptance_root.resolve(strict=True)
materialization = materialization_root.resolve(strict=True)
expected_ids = {
"acceptance": profile["source"]["acceptance_result_id"],
"materialization": profile["source"]["materialization_id"],
}
if (
acceptance.name != expected_ids["acceptance"]
or materialization.name != expected_ids["materialization"]
):
raise E40WorkerPackageError("E40 source identity changed")
sources: dict[str, Path | bytes | None] = {}
for target, relative in _RUNTIME_FILES.items():
source = None if relative is None else repository / relative
if source is not None and (not source.is_file() or source.is_symlink()):
raise E40WorkerPackageError(f"E40 runtime source is invalid: {relative}")
sources[target] = source
sources["profile.json"] = profile_source
for filename in _ACCEPTANCE_FILES:
source = acceptance / filename
if not source.is_file() or source.is_symlink():
raise E40WorkerPackageError("E40 acceptance artifact is invalid")
sources[f"input/acceptance/{acceptance.name}/{filename}"] = source
for filename in _MATERIALIZATION_FILES:
source = materialization / filename
if not source.is_file() or source.is_symlink():
raise E40WorkerPackageError("E40 materialization artifact is invalid")
sources[f"input/materialization/{materialization.name}/{filename}"] = source
_add_materialized_evidence(
sources,
materialization=materialization,
)
_add_feature_cache(
sources,
acceptance=acceptance,
materialization=materialization,
)
descriptors = []
for relative, source in sorted(sources.items()):
payload = (
_GENERATED_COMPUTE_INIT.encode()
if source is None
else source
if isinstance(source, bytes)
else source.read_bytes()
)
descriptors.append(
{
"path": relative,
"byte_length": len(payload),
"sha256": hashlib.sha256(payload).hexdigest(),
}
)
identity = {
"schema_version": E40_PACKAGE_SCHEMA,
"classification": ("immutable-ravnoves00-leakage-resistant-product-gate-input"),
"source_ids": expected_ids,
"profile_sha256": _sha256(profile_source),
"runtime_requirements": {
"python": "3.12",
"numpy": "1.26+",
},
"build_requirements": {"pillow": "10+"},
"artifact_paths": [row["path"] for row in descriptors],
"source_artifacts": descriptors,
"authority": {
"commands_enabled": False,
"navigation_or_safety_accepted": False,
},
}
identity_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest()
package_id = f"e40-worker-package-{identity_sha256}"
output = output_root.expanduser().absolute()
output.mkdir(mode=0o700, parents=True, exist_ok=True)
destination = output / package_id
if destination.exists():
validate_e40_worker_package(destination)
return destination
staging = output / f".{package_id}.{uuid.uuid4().hex}.tmp"
staging.mkdir(mode=0o700, exist_ok=False)
try:
for relative, source in sources.items():
target = staging / relative
target.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
if source is None:
target.write_text(_GENERATED_COMPUTE_INIT, encoding="utf-8")
elif isinstance(source, bytes):
target.write_bytes(source)
else:
shutil.copyfile(source, target)
artifacts = [
{
"kind": relative,
"path": relative,
"byte_length": (staging / relative).stat().st_size,
"sha256": _sha256(staging / relative),
}
for relative in sorted(sources)
]
manifest = {
"schema_version": E40_PACKAGE_SCHEMA,
"package_id": package_id,
"identity_sha256": identity_sha256,
"identity": identity,
"created_at_utc": datetime.now(UTC)
.isoformat(timespec="milliseconds")
.replace("+00:00", "Z"),
"artifacts": artifacts,
}
_write_json(staging / "manifest.json", manifest)
validate_e40_worker_package(staging, allow_staging=True)
os.replace(staging, destination)
except BaseException:
shutil.rmtree(staging, ignore_errors=True)
raise
validate_e40_worker_package(destination)
return destination
def validate_e40_worker_package(
root: Path,
*,
allow_staging: bool = False,
) -> dict[str, Any]:
"""Validate package identity, exact file set, and every member digest."""
resolved = root.resolve(strict=True)
manifest = _read_json(resolved / "manifest.json")
identity = manifest.get("identity")
identity_sha256 = manifest.get("identity_sha256")
package_id = manifest.get("package_id")
artifacts = manifest.get("artifacts")
source_artifacts = identity.get("source_artifacts") if isinstance(identity, dict) else None
expected_name = isinstance(package_id, str) and (
resolved.name == package_id
or (
allow_staging
and resolved.name.startswith(f".{package_id}.")
and resolved.name.endswith(".tmp")
)
)
if (
manifest.get("schema_version") != E40_PACKAGE_SCHEMA
or not isinstance(identity, dict)
or not isinstance(identity_sha256, str)
or hashlib.sha256(_canonical_json(identity)).hexdigest() != identity_sha256
or package_id != f"e40-worker-package-{identity_sha256}"
or not expected_name
or not isinstance(artifacts, list)
or not isinstance(source_artifacts, list)
):
raise E40WorkerPackageError("E40 worker package identity is invalid")
expected_paths = set(identity.get("artifact_paths", []))
bound_artifacts: dict[str, tuple[int, str]] = {}
for row in source_artifacts:
if (
not isinstance(row, dict)
or not isinstance((relative := row.get("path")), str)
or relative in bound_artifacts
or Path(relative).is_absolute()
or ".." in Path(relative).parts
or not isinstance((byte_length := row.get("byte_length")), int)
or byte_length < 0
or not isinstance((sha256 := row.get("sha256")), str)
or len(sha256) != 64
):
raise E40WorkerPackageError("E40 bound source artifact is invalid")
bound_artifacts[relative] = (byte_length, sha256)
if set(bound_artifacts) != expected_paths:
raise E40WorkerPackageError("E40 bound artifact coverage changed")
actual_paths = {
path.relative_to(resolved).as_posix() for path in resolved.rglob("*") if path.is_file()
}
if (
not expected_paths
or actual_paths != expected_paths | {"manifest.json"}
or len(artifacts) != len(expected_paths)
):
raise E40WorkerPackageError("E40 worker package file set changed")
observed: set[str] = set()
for row in artifacts:
if not isinstance(row, dict):
raise E40WorkerPackageError("E40 worker package artifact is invalid")
relative = row.get("path")
path = resolved / str(relative)
if (
not isinstance(relative, str)
or relative not in expected_paths
or relative in observed
or Path(relative).is_absolute()
or ".." in Path(relative).parts
or not path.is_file()
or path.is_symlink()
or row.get("kind") != relative
or bound_artifacts.get(relative)
!= (row.get("byte_length"), row.get("sha256"))
or row.get("byte_length") != path.stat().st_size
or row.get("sha256") != _sha256(path)
):
raise E40WorkerPackageError("E40 worker package artifact changed")
observed.add(relative)
if observed != expected_paths:
raise E40WorkerPackageError("E40 worker package coverage changed")
return manifest
def _add_materialized_evidence(
sources: dict[str, Path | bytes | None],
*,
materialization: Path,
) -> None:
rows = _read_jsonl(materialization / "materialized-items.jsonl")
if len(rows) != 486:
raise E40WorkerPackageError("E40 materialization denominator changed")
for row in rows:
for descriptor_name in ("artifact", "camera_frame"):
descriptor = row.get(descriptor_name)
if not isinstance(descriptor, dict):
raise E40WorkerPackageError("E40 materialization descriptor is invalid")
relative = descriptor.get("path")
if (
not isinstance(relative, str)
or Path(relative).is_absolute()
or ".." in Path(relative).parts
):
raise E40WorkerPackageError("E40 materialization path is invalid")
source = materialization / relative
if (
not source.is_file()
or source.is_symlink()
or descriptor.get("byte_length") != source.stat().st_size
or descriptor.get("sha256") != _sha256(source)
):
raise E40WorkerPackageError("E40 materialized evidence content changed")
target = f"input/materialization/{materialization.name}/{relative}"
existing = sources.get(target)
if existing is not None and existing != source:
raise E40WorkerPackageError("E40 package target collision")
sources[target] = source
def _add_feature_cache(
sources: dict[str, Path | bytes | None],
*,
acceptance: Path,
materialization: Path,
) -> None:
acceptance_rows = _read_jsonl(acceptance / "acceptance-items.jsonl")
materialization_rows = _read_jsonl(materialization / "materialized-items.jsonl")
acceptance_by_id = {str(row["item_id"]): row for row in acceptance_rows}
if len(acceptance_by_id) != 486 or {str(row["item_id"]) for row in materialization_rows} != set(
acceptance_by_id
):
raise E40WorkerPackageError("E40 feature-cache denominator changed")
item_ids = [str(row["item_id"]) for row in materialization_rows]
features = np.asarray(
[
_feature_vector(
acceptance_by_id[item_id],
row,
materialization,
)
for item_id, row in zip(item_ids, materialization_rows, strict=True)
],
dtype=np.float64,
)
names = _feature_names()
if features.shape != (486, len(names)) or not np.isfinite(features).all():
raise E40WorkerPackageError("E40 feature-cache matrix is invalid")
arrays_stream = io.BytesIO()
np.savez_compressed(
arrays_stream,
item_ids=np.asarray(item_ids, dtype=f"<U{max(map(len, item_ids))}"),
features=features,
)
arrays_payload = arrays_stream.getvalue()
manifest = {
"schema_version": _FEATURE_CACHE_SCHEMA,
"materialization_id": materialization.name,
"materialization_index_sha256": _sha256(materialization / "materialized-items.jsonl"),
"feature_names_sha256": hashlib.sha256(_canonical_json(names)).hexdigest(),
"items": 486,
"dimensions": len(names),
"arrays_path": _FEATURE_CACHE_ARRAYS,
"arrays_byte_length": len(arrays_payload),
"arrays_sha256": hashlib.sha256(arrays_payload).hexdigest(),
}
manifest_payload = (json.dumps(manifest, indent=2, sort_keys=True) + "\n").encode()
prefix = f"input/materialization/{materialization.name}"
sources[f"{prefix}/{_FEATURE_CACHE_ARRAYS}"] = arrays_payload
sources[f"{prefix}/{_FEATURE_CACHE_MANIFEST}"] = manifest_payload
def _canonical_json(value: object) -> bytes:
return json.dumps(
value,
sort_keys=True,
separators=(",", ":"),
allow_nan=False,
).encode()
def _sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as stream:
while chunk := stream.read(1024 * 1024):
digest.update(chunk)
return digest.hexdigest()
def _read_json(path: Path) -> dict[str, Any]:
value = json.loads(path.read_text(encoding="utf-8-sig"))
if not isinstance(value, dict):
raise E40WorkerPackageError(f"JSON object expected: {path.name}")
return value
def _read_jsonl(path: Path) -> list[dict[str, Any]]:
rows = [json.loads(line) for line in path.read_text(encoding="utf-8-sig").splitlines()]
if not all(isinstance(row, dict) for row in rows):
raise E40WorkerPackageError(f"JSONL object expected: {path.name}")
return rows
def _write_json(path: Path, value: object) -> None:
with path.open("x", encoding="utf-8", newline="\n") as stream:
json.dump(value, stream, indent=2, sort_keys=True)
stream.write("\n")
stream.flush()
os.fsync(stream.fileno())
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--repository-root", type=Path, required=True)
parser.add_argument("--acceptance", type=Path, required=True)
parser.add_argument("--materialization", type=Path, required=True)
parser.add_argument("--profile", type=Path, required=True)
parser.add_argument("--output-root", type=Path, required=True)
args = parser.parse_args()
package = build_e40_worker_package(
repository_root=args.repository_root,
acceptance_root=args.acceptance,
materialization_root=args.materialization,
profile_path=args.profile,
output_root=args.output_root,
)
print(package)
return 0
if __name__ == "__main__":
raise SystemExit(main())
experiments/perception/run_e40_perception_product_gate.py
+49
View File
@@ -0,0 +1,49 @@
#!/usr/bin/env python3
"""Build the development-qualified RAVNOVES00 E40 product gate."""
from __future__ import annotations
import argparse
import json
import os
from pathlib import Path
from k1link.compute.e40_perception_product_gate import (
build_e40_perception_product_gate,
)
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--acceptance", type=Path, required=True)
parser.add_argument("--materialization", type=Path, required=True)
parser.add_argument("--profile", type=Path, required=True)
parser.add_argument("--output-root", type=Path, required=True)
parser.add_argument("--worker-node", default=os.environ.get("COMPUTERNAME"))
args = parser.parse_args()
result = build_e40_perception_product_gate(
acceptance_root=args.acceptance,
materialization_root=args.materialization,
profile_path=args.profile,
output_root=args.output_root,
worker_node=args.worker_node,
)
print(
json.dumps(
{
"result_id": result.result_id,
"result_root": str(result.result_root),
"quality_gate_passed": result.quality_gate_passed,
"development_cross_validation": result.report["development_cross_validation"],
"metrics": result.report["metrics"],
"blocking_checks": result.report["quality_gate"]["blocking_checks"],
},
ensure_ascii=False,
sort_keys=True,
)
)
return 0
if __name__ == "__main__":
raise SystemExit(main())
experiments/perception/run_e41_evaluation_boundary.py
+171
View File
@@ -0,0 +1,171 @@
#!/usr/bin/env python3
"""Prepare/run the E41 truth-free predictor and visible evaluator."""
from __future__ import annotations
import argparse
import contextlib
import json
import platform
import uuid
from pathlib import Path
from typing import Any
from k1link.compute.e41_evaluation_boundary import (
E41_FEATURE_MANIFEST_NAME,
build_e41_predictor_package,
build_e41_visible_evaluation,
run_e41_predictor,
)
from k1link.compute.pipeline_telemetry import (
JsonlPipelineTelemetrySink,
PipelineStageOutcome,
PipelineTelemetryEmitter,
PipelineTelemetryIdentity,
)
def main() -> int:
parser = argparse.ArgumentParser()
subparsers = parser.add_subparsers(dest="command", required=True)
package = subparsers.add_parser("package")
package.add_argument("--materialization", type=Path, required=True)
package.add_argument("--e40-package", type=Path, required=True)
package.add_argument("--e40-model", type=Path, required=True)
package.add_argument("--output-root", type=Path, required=True)
_add_telemetry_arguments(package)
predict = subparsers.add_parser("predict")
predict.add_argument("--package", type=Path, required=True)
predict.add_argument("--output-root", type=Path, required=True)
predict.add_argument("--environment-lock", required=True)
predict.add_argument("--python-version", required=True)
predict.add_argument("--numpy-version", required=True)
_add_telemetry_arguments(predict)
evaluate = subparsers.add_parser("evaluate")
evaluate.add_argument("--prediction", type=Path, required=True)
evaluate.add_argument("--acceptance", type=Path, required=True)
evaluate.add_argument("--output-root", type=Path, required=True)
_add_telemetry_arguments(evaluate)
args = parser.parse_args()
if args.command == "package":
emitter = _telemetry_emitter(
args,
source_package_id=args.e40_package.name,
method_id="e41-truth-free-package/v1",
)
with _stage(emitter, "package") as outcome:
result = build_e41_predictor_package(
materialization_root=args.materialization,
e40_package_root=args.e40_package,
e40_model_path=args.e40_model,
output_root=args.output_root,
)
feature_manifest = json.loads(
(result / E41_FEATURE_MANIFEST_NAME).read_text(encoding="utf-8")
)
outcome.output_count = int(feature_manifest["item_count"])
payload = {"predictor_package": str(result)}
elif args.command == "predict":
emitter = _telemetry_emitter(
args,
source_package_id=args.package.name,
method_id="frozen-e40-predictor/v1",
)
with _stage(emitter, "predict") as outcome:
prediction = run_e41_predictor(
package_root=args.package,
output_root=args.output_root,
runtime_identity={
"environment_lock": args.environment_lock,
"python": args.python_version,
"numpy": args.numpy_version,
},
)
outcome.output_count = int(prediction.manifest["item_count"])
payload = {
"prediction_result_id": prediction.result_id,
"prediction_root": str(prediction.result_root),
}
else:
emitter = _telemetry_emitter(
args,
source_package_id=args.prediction.name,
method_id="visible-engineering-contract/v1",
)
with _stage(emitter, "evaluate") as outcome:
evaluation = build_e41_visible_evaluation(
prediction_root=args.prediction,
acceptance_root=args.acceptance,
output_root=args.output_root,
)
outcome.output_count = int(
evaluation.evaluation["evaluation"]["metrics"]["validation_items"]
)
payload = {
"evaluation_result_id": evaluation.result_id,
"evaluation_root": str(evaluation.result_root),
"metrics": evaluation.evaluation["evaluation"]["metrics"],
"blocking_checks": evaluation.evaluation["evaluation"]["blocking_checks"],
}
print(json.dumps(payload, ensure_ascii=False, sort_keys=True))
return 0
def _add_telemetry_arguments(parser: argparse.ArgumentParser) -> None:
parser.add_argument(
"--telemetry-jsonl",
type=Path,
help="append native pipeline events to this local JSONL evidence file",
)
parser.add_argument("--telemetry-contour-id", default="local-compute")
parser.add_argument("--telemetry-agent-id", default="mission-core-runner")
parser.add_argument("--telemetry-node-id", default=platform.node() or "unknown-node")
parser.add_argument("--telemetry-run-id")
parser.add_argument("--telemetry-request-id")
parser.add_argument("--telemetry-source-id", default="ravnoves00")
def _telemetry_emitter(
args: argparse.Namespace,
*,
source_package_id: str,
method_id: str,
) -> PipelineTelemetryEmitter | None:
if args.telemetry_jsonl is None:
return None
identity = PipelineTelemetryIdentity(
contour_id=str(args.telemetry_contour_id),
agent_id=str(args.telemetry_agent_id),
node_id=str(args.telemetry_node_id),
lab_id="E41",
run_id=str(args.telemetry_run_id or f"e41-{uuid.uuid4().hex}"),
request_id=(
str(args.telemetry_request_id)
if args.telemetry_request_id is not None
else None
),
source_id=str(args.telemetry_source_id),
source_package_id=source_package_id,
method_id=method_id,
)
return PipelineTelemetryEmitter(
identity=identity,
sink=JsonlPipelineTelemetrySink(args.telemetry_jsonl),
)
def _stage(
emitter: PipelineTelemetryEmitter | None,
stage_id: str,
) -> Any:
if emitter is None:
return contextlib.nullcontext(PipelineStageOutcome())
return emitter.stage(stage_id)
if __name__ == "__main__":
raise SystemExit(main())
experiments/perception/run_e41_methodology_audit.py
+46
View File
@@ -0,0 +1,46 @@
#!/usr/bin/env python3
"""Build the immutable RAVNOVES00 E41 methodology audit."""
from __future__ import annotations
import argparse
import json
from pathlib import Path
from k1link.compute.e41_methodology_audit import build_e41_methodology_audit
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--acceptance", type=Path, required=True)
parser.add_argument("--materialization", type=Path, required=True)
parser.add_argument("--e40-package", type=Path, required=True)
parser.add_argument("--e40-result", type=Path, required=True)
parser.add_argument("--profile", type=Path, required=True)
parser.add_argument("--output-root", type=Path, required=True)
args = parser.parse_args()
result = build_e41_methodology_audit(
acceptance_root=args.acceptance,
materialization_root=args.materialization,
e40_package_root=args.e40_package,
e40_result_root=args.e40_result,
profile_path=args.profile,
output_root=args.output_root,
)
print(
json.dumps(
{
"result_id": result.result_id,
"result_root": str(result.result_root),
"blind_gate_eligible": result.blind_gate_eligible,
"violations": result.report["analysis"]["policy"]["violations"],
},
ensure_ascii=False,
sort_keys=True,
)
)
return 0
if __name__ == "__main__":
raise SystemExit(main())
experiments/perception/run_e42_metamorphic_suite.py
+40
View File
@@ -0,0 +1,40 @@
#!/usr/bin/env python3
"""Run the bounded E42 predictor and PointSlab metamorphic suite."""
from __future__ import annotations
import argparse
import json
from pathlib import Path
from k1link.compute.e42_metamorphic_suite import build_e42_metamorphic_suite
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--predictor-package", type=Path, required=True)
parser.add_argument("--e32-result", type=Path, required=True)
parser.add_argument("--output-root", type=Path, required=True)
args = parser.parse_args()
result = build_e42_metamorphic_suite(
predictor_package_root=args.predictor_package,
e32_result_root=args.e32_result,
output_root=args.output_root,
)
print(
json.dumps(
{
"result_id": result.result_id,
"result_root": str(result.result_root),
"accepted": result.accepted,
"checks": result.report["acceptance"]["checks"],
},
ensure_ascii=False,
sort_keys=True,
)
)
return 0
if __name__ == "__main__":
raise SystemExit(main())
experiments/perception/run_e43_future_capture_protocol.py
+40
View File
@@ -0,0 +1,40 @@
#!/usr/bin/env python3
"""Freeze the future same-K1/new-route capture protocol."""
from __future__ import annotations
import argparse
import json
from pathlib import Path
from k1link.compute.e43_future_capture_protocol import (
build_e43_future_capture_protocol,
)
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--profile", type=Path, required=True)
parser.add_argument("--output-root", type=Path, required=True)
args = parser.parse_args()
result = build_e43_future_capture_protocol(
profile_path=args.profile,
output_root=args.output_root,
)
print(
json.dumps(
{
"result_id": result.result_id,
"result_root": str(result.result_root),
"capture_exists": result.protocol["capture_exists"],
"labels_exist": result.protocol["labels_exist"],
},
ensure_ascii=False,
sort_keys=True,
)
)
return 0
if __name__ == "__main__":
raise SystemExit(main())
experiments/perception/run_e44_data_amplification_audit.py
+50
View File
@@ -0,0 +1,50 @@
#!/usr/bin/env python3
"""Measure exact data amplification across explicit immutable LAB roots."""
from __future__ import annotations
import argparse
import json
from pathlib import Path
from k1link.compute.e44_data_amplification_audit import (
build_e44_data_amplification_audit,
)
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument(
"--artifact-root",
action="append",
required=True,
metavar="LABEL=PATH",
)
parser.add_argument("--output-root", type=Path, required=True)
args = parser.parse_args()
roots: dict[str, Path] = {}
for value in args.artifact_root:
label, separator, raw_path = value.partition("=")
if not separator or label in roots:
parser.error("--artifact-root must be a unique LABEL=PATH")
roots[label] = Path(raw_path)
result = build_e44_data_amplification_audit(
artifact_roots=roots,
output_root=args.output_root,
)
print(
json.dumps(
{
"result_id": result.result_id,
"result_root": str(result.result_root),
"analysis": result.report["analysis"],
},
ensure_ascii=False,
sort_keys=True,
)
)
return 0
if __name__ == "__main__":
raise SystemExit(main())
experiments/perception/worker/Invoke-E40PerceptionProductGate.ps1
+161
View File
@@ -0,0 +1,161 @@
[CmdletBinding()]
param(
[Parameter(Mandatory = $true)]
[string]$PackageRoot,
[string]$OutputRoot = "D:\NDC_MISSIONCORE\runtime\derived\e40-product-gate",
[string]$ContainerImage = "nvcr.io/nvidia/tritonserver:26.06-py3@sha256:58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794",
[ValidateRange(1, 1000)]
[int]$FreeGiBFloor = 300
)
$ErrorActionPreference = "Stop"
$ProgressPreference = "SilentlyContinue"
function Assert-LastExitCode([string]$Operation) {
if ($LASTEXITCODE -ne 0) {
throw "$Operation failed with exit code $LASTEXITCODE"
}
}
function Resolve-DDirectory([string]$Path, [string]$Label) {
$item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force
$root = [IO.Path]::GetPathRoot($item.FullName).TrimEnd("\")
if (
-not $item.PSIsContainer -or
($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -or
$root -ine "D:"
) {
throw "$Label must be a real D: directory"
}
return $item.FullName
}
function Convert-ToDockerPath([string]$Path) {
return $Path.Replace("\", "/")
}
function Assert-FreeSpace([string]$Phase) {
$free = [int64](Get-PSDrive -Name D).Free
$floor = [int64]$FreeGiBFloor * 1GB
Write-Host (
"DISK_GUARD PHASE={0} DRIVE=D FREE_BYTES={1} FREE_GIB={2} FLOOR_GIB={3}" -f
$Phase, $free, [math]::Round($free / 1GB, 3), $FreeGiBFloor
)
if ($free -lt ($floor + 1GB)) {
throw "D: lacks the guarded E40 reserve during $Phase"
}
return $free
}
$package = Resolve-DDirectory $PackageRoot "E40 package"
$packageManifestPath = Join-Path $package "manifest.json"
if (-not (Test-Path -LiteralPath $packageManifestPath -PathType Leaf)) {
throw "E40 package manifest is missing"
}
$packageManifest = Get-Content -LiteralPath $packageManifestPath -Raw |
ConvertFrom-Json
if (
$packageManifest.schema_version -ne "missioncore.e40-worker-package/v1" -or
$packageManifest.package_id -ne (Split-Path $package -Leaf) -or
$packageManifest.package_id -notmatch "^e40-worker-package-[a-f0-9]{64}$"
) {
throw "E40 package manifest is incompatible"
}
if (-not (Test-Path -LiteralPath $OutputRoot)) {
$null = New-Item -ItemType Directory -Path $OutputRoot
}
$output = Resolve-DDirectory $OutputRoot "E40 output root"
$freeBefore = Assert-FreeSpace "preflight"
& docker image inspect $ContainerImage *> $null
Assert-LastExitCode "Pinned E40 container image inspection"
$dockerPackage = Convert-ToDockerPath $package
$dockerOutput = Convert-ToDockerPath $output
$packageName = Split-Path $package -Leaf
$containerPackage = "/opt/e40-input/$packageName"
$packageValidator = (
"{0}/runtime/validate_e40_worker_package.py" -f $containerPackage
)
$validationCommand = @(
"run", "--rm",
"--name", "ndc-mission-core-e40-package-validation",
"--network", "none",
"--read-only",
"--security-opt", "no-new-privileges:true",
"--cap-drop", "ALL",
"--pids-limit", "32",
"--memory", "128m",
"--memory-swap", "128m",
"--cpus", "1",
"-v", ("{0}:{1}:ro" -f $dockerPackage, $containerPackage),
"--entrypoint", "python3",
$ContainerImage,
$packageValidator,
$containerPackage
)
& docker @validationCommand
Assert-LastExitCode "Independent E40 package integrity verification"
$command = @(
"run", "--rm",
"--name", "ndc-mission-core-e40-product-gate",
"--network", "none",
"--read-only",
"--security-opt", "no-new-privileges:true",
"--cap-drop", "ALL",
"--pids-limit", "128",
"--memory", "1g",
"--memory-swap", "1g",
"--cpus", "2",
"--tmpfs", "/tmp:rw,noexec,nosuid,size=64m",
"-e", "PYTHONDONTWRITEBYTECODE=1",
"-e", ("PYTHONPATH={0}/runtime" -f $containerPackage),
"-e", ("E40_WORKER_NODE={0}" -f $env:COMPUTERNAME),
"-v", ("{0}:{1}:ro" -f $dockerPackage, $containerPackage),
"-v", ("{0}:/output:rw" -f $dockerOutput),
"--entrypoint", "python3",
$ContainerImage,
("{0}/runtime/run_e40_perception_product_gate.py" -f $containerPackage),
"--package", $containerPackage,
"--output-root", "/output"
)
Write-Output ("PACKAGE_ID={0}" -f $packageManifest.package_id)
Write-Output ("PACKAGE_IDENTITY_SHA256={0}" -f $packageManifest.identity_sha256)
Write-Output ("CONTAINER_IMAGE={0}" -f $ContainerImage)
& docker @command
Assert-LastExitCode "E40 perception product gate"
$matches = @(
Get-ChildItem -LiteralPath $output -Directory -Filter "e40-perception-product-gate-*" |
Where-Object {
$manifestPath = Join-Path $_.FullName "manifest.json"
if (-not (Test-Path -LiteralPath $manifestPath -PathType Leaf)) {
return $false
}
$manifest = Get-Content -LiteralPath $manifestPath -Raw |
ConvertFrom-Json
return (
$manifest.schema_version -eq
"missioncore.e40-perception-product-gate/v1" -and
$manifest.acceptance_state -eq
"completed-leakage-resistant-product-gate" -and
$manifest.identity.execution.worker_node -eq $env:COMPUTERNAME
)
}
)
if ($matches.Count -ne 1) {
throw "E40 immutable result could not be resolved uniquely"
}
$resultRoot = $matches[0].FullName
$resultManifest = Get-Content -LiteralPath (
Join-Path $resultRoot "manifest.json"
) -Raw | ConvertFrom-Json
$freeAfter = Assert-FreeSpace "completed"
Write-Output ("RESULT_ROOT={0}" -f $resultRoot)
Write-Output ("RESULT_ID={0}" -f $resultManifest.result_id)
Write-Output ("QUALITY_GATE_PASSED={0}" -f $resultManifest.quality_gate_passed)
Write-Output ("DISK_FREE_BYTES_BEFORE={0}" -f $freeBefore)
Write-Output ("DISK_FREE_BYTES_AFTER={0}" -f $freeAfter)
experiments/perception/worker/run_e40_perception_product_gate.py
+57
View File
@@ -0,0 +1,57 @@
#!/usr/bin/env python3
"""Execute the packaged E40 product gate in the pinned Worker 006 container."""
from __future__ import annotations
import argparse
import json
import os
from pathlib import Path
from k1link.compute.e40_perception_product_gate import (
build_e40_perception_product_gate,
)
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--package", type=Path, required=True)
parser.add_argument("--output-root", type=Path, required=True)
args = parser.parse_args()
package = args.package.resolve(strict=True)
package_manifest = json.loads(
(package / "manifest.json").read_text(encoding="utf-8-sig")
)
profile = json.loads((package / "profile.json").read_text(encoding="utf-8"))
acceptance_id = profile["source"]["acceptance_result_id"]
materialization_id = profile["source"]["materialization_id"]
result = build_e40_perception_product_gate(
acceptance_root=package / "input" / "acceptance" / acceptance_id,
materialization_root=(package / "input" / "materialization" / materialization_id),
profile_path=package / "profile.json",
output_root=args.output_root,
worker_node=os.environ.get("E40_WORKER_NODE"),
execution_package={
"mode": "verified-worker-package",
"package_id": package_manifest["package_id"],
"identity_sha256": package_manifest["identity_sha256"],
},
)
print(
json.dumps(
{
"result_id": result.result_id,
"result_root": str(result.result_root),
"quality_gate_passed": result.quality_gate_passed,
"metrics": result.report["metrics"],
"blocking_checks": result.report["quality_gate"]["blocking_checks"],
},
ensure_ascii=False,
sort_keys=True,
)
)
return 0
if __name__ == "__main__":
raise SystemExit(main())
experiments/perception/worker/validate_e40_worker_package.py
+103
View File
@@ -0,0 +1,103 @@
#!/usr/bin/env python3
"""Independently validate an E40 package before importing package code."""
from __future__ import annotations
import hashlib
import json
import pathlib
import sys
def _descriptors(
rows: object,
*,
require_kind: bool,
) -> dict[str, tuple[int, str]]:
if not isinstance(rows, list):
raise SystemExit("E40 package artifact catalog is missing")
result: dict[str, tuple[int, str]] = {}
for row in rows:
if not isinstance(row, dict):
raise SystemExit("E40 package artifact descriptor is invalid")
relative = row.get("path")
byte_length = row.get("byte_length")
sha256 = row.get("sha256")
if (
not isinstance(relative, str)
or relative in result
or pathlib.PurePosixPath(relative).is_absolute()
or ".." in pathlib.PurePosixPath(relative).parts
or not isinstance(byte_length, int)
or byte_length < 0
or not isinstance(sha256, str)
or len(sha256) != 64
or (require_kind and row.get("kind") != relative)
):
raise SystemExit("E40 package artifact descriptor is invalid")
result[relative] = (byte_length, sha256)
return result
def validate(root_argument: str) -> None:
root = pathlib.Path(root_argument).resolve(strict=True)
manifest = json.loads(
(root / "manifest.json").read_text(encoding="utf-8-sig")
)
identity = manifest.get("identity")
identity_sha256 = manifest.get("identity_sha256")
package_id = manifest.get("package_id")
if (
manifest.get("schema_version") != "missioncore.e40-worker-package/v1"
or not isinstance(identity, dict)
or not isinstance(identity_sha256, str)
or hashlib.sha256(
json.dumps(
identity,
sort_keys=True,
separators=(",", ":"),
allow_nan=False,
).encode()
).hexdigest()
!= identity_sha256
or package_id != f"e40-worker-package-{identity_sha256}"
or root.name != package_id
):
raise SystemExit("E40 package identity verification failed")
bound = _descriptors(
identity.get("source_artifacts"),
require_kind=False,
)
catalog = _descriptors(manifest.get("artifacts"), require_kind=True)
if bound != catalog or set(identity.get("artifact_paths", [])) != set(bound):
raise SystemExit("E40 package artifact binding verification failed")
actual = {
path.relative_to(root).as_posix()
for path in root.rglob("*")
if path.is_file()
}
if actual != set(bound) | {"manifest.json"}:
raise SystemExit("E40 package file set verification failed")
for relative, (byte_length, sha256) in bound.items():
path = root / relative
payload = path.read_bytes()
if (
path.is_symlink()
or len(payload) != byte_length
or hashlib.sha256(payload).hexdigest() != sha256
):
raise SystemExit(
f"E40 package member verification failed: {relative}"
)
def main() -> int:
if len(sys.argv) != 2:
raise SystemExit("usage: validate_e40_worker_package.py PACKAGE_ROOT")
validate(sys.argv[1])
return 0
if __name__ == "__main__":
raise SystemExit(main())
src/k1link/compute/e30_materialization.py
+9 -2
View File
@@ -254,7 +254,7 @@ def build_e30_materialization(
source=source,
)
identity = {
identity: dict[str, object] = {
"schema_version": E30_MATERIALIZATION_SCHEMA,
"review_pack": {
"result_id": _required_string(review.manifest, "result_id"),
@@ -635,7 +635,14 @@ def _engineering_triage(
) -> dict[str, object]:
"""Route evidence without pretending that a rule is a semantic verdict."""
selected_count = int(metadata["selected_point_count"])
selected_count_value = metadata["selected_point_count"]
if (
not isinstance(selected_count_value, int)
or isinstance(selected_count_value, bool)
or selected_count_value < 0
):
raise E30MaterializationError("selected point count is invalid")
selected_count = selected_count_value
detector_score = metadata.get("detector_score")
stratum = _required_string(item, "stratum")
locator = _required_object(item, "e29_locator")
src/k1link/compute/e32_track_geometry_replay.py
+3 -1
View File
@@ -44,7 +44,6 @@ from .e32_track_geometry_storage import (
E32_POINT_SLAB_REFERENCE_SCHEMA,
E32_POINTS_NAME,
E32_SOURCE_INDICES_NAME,
E32_TRACK_GEOMETRY_RECORD_SCHEMA,
E32TrackGeometryStorageError,
frame_from_record,
load_point_storage,
@@ -52,6 +51,9 @@ from .e32_track_geometry_storage import (
validate_storage,
write_point_storage,
)
from .e32_track_geometry_storage import (
E32_TRACK_GEOMETRY_RECORD_SCHEMA as E32_TRACK_GEOMETRY_RECORD_SCHEMA,
)
from .lidar_field_review import E10LidarFieldSource
from .lidar_local_surface import K1LocalSurfaceV1
from .semantic_geometry_fusion import (
src/k1link/compute/e34_temporal_occupied_replay.py
+40 -8
View File
@@ -790,24 +790,45 @@ def _finalize_report(
== occupancy.get("e34_consumed_current_point_rows")
),
"active_component_bound": (
components.get("peak_active")
<= layer.get("maximum_active_components")
_nonnegative_int(components.get("peak_active"), "E34 peak active components")
<= _positive_int(
layer.get("maximum_active_components"),
"E34 maximum active components",
)
),
"component_cell_bound": (
occupancy.get("peak_cells_per_component")
<= layer.get("maximum_cells_per_component")
_nonnegative_int(
occupancy.get("peak_cells_per_component"),
"E34 peak cells per component",
)
<= _positive_int(
layer.get("maximum_cells_per_component"),
"E34 maximum cells per component",
)
),
"held_age_within_ttl": (
aging.get("maximum_held_age_seconds")
_nonnegative_float(
aging.get("maximum_held_age_seconds"),
"E34 maximum held age",
)
<= float(layer["occupied_ttl_seconds"]) + 1e-9
),
"expiry_delay_within_gate": (
aging.get("maximum_expiry_delay_seconds")
_nonnegative_float(
aging.get("maximum_expiry_delay_seconds"),
"E34 maximum expiry delay",
)
<= float(acceptance["maximum_expiry_delay_seconds"]) + 1e-9
),
"map_frame_jump_candidates_within_gate": (
map_frame.get("jump_candidates")
<= acceptance.get("maximum_map_frame_jump_candidates")
_nonnegative_int(
map_frame.get("jump_candidates"),
"E34 map-frame jump candidates",
)
<= _nonnegative_int(
acceptance.get("maximum_map_frame_jump_candidates"),
"E34 maximum map-frame jump candidates",
)
),
"upstream_artifacts_unchanged": upstream_unchanged,
"no_free_space_publication": occupancy.get("free_cell_rows") == 0,
@@ -1010,6 +1031,17 @@ def _positive_float(value: object, label: str) -> float:
return float(value)
def _nonnegative_float(value: object, label: str) -> float:
if (
not isinstance(value, (int, float))
or isinstance(value, bool)
or not math.isfinite(float(value))
or float(value) < 0.0
):
raise E34TemporalOccupiedReplayError(f"{label} is invalid")
return float(value)
def _positive_int(value: object, label: str) -> int:
result = _nonnegative_int(value, label)
if result == 0:
src/k1link/compute/e37_acceptance_contract.py
+3
View File
@@ -142,6 +142,9 @@ def assign_split(
range_bucket,
)):
raise E37AcceptanceContractError("E37 split source row is invalid")
assert isinstance(item_id, str)
assert isinstance(stratum, str)
assert isinstance(range_bucket, str)
grouped[(stratum, range_bucket)].append(row)
assignments: dict[str, SplitName] = {}
src/k1link/compute/e39_perception_refinement.py
+10 -5
View File
@@ -1,9 +1,11 @@
"""Development-qualified RAVNOVES00 R1 perception refinement.
"""Historical RAVNOVES00 R1 perception refinement.
E39 keeps the E37 denominator and validation split immutable. It enriches the
E39 keeps the nominal E37 denominator and split immutable. It enriches the
E38 tabular baseline with source-scoped camera and LiDAR shape features, chooses
the fixed model contract through development-only cross-validation, fits only
on development labels, and evaluates the sealed validation partition once.
on development labels, and evaluates the then-designated validation partition.
E41 later proved that this partition is visible engineering evidence with
connected group overlap, not an independent sealed accuracy gate.
The result is diagnostic. It grants no navigation, command, or safety
authority and makes no claim about another route, rig, camera, or mount.
@@ -92,7 +94,7 @@ def build_e39_perception_refinement(
output_root: Path,
worker_node: str | None = None,
) -> E39PerceptionRefinement:
"""Fit the frozen E39 contract and evaluate the sealed validation split."""
"""Fit E39 and evaluate the historical visible E37 validation slice."""
profile_file = profile_path.resolve(strict=True)
profile = _read_json(profile_file)
@@ -814,7 +816,10 @@ def _transform(
scale: np.ndarray[Any, Any],
clip: float,
) -> np.ndarray[Any, Any]:
return np.clip((matrix - median) / scale, -clip, clip)
return np.asarray(
np.clip((matrix - median) / scale, -clip, clip),
dtype=np.float64,
)
def _predict_presence(
src/k1link/compute/e40_perception_product_gate.py
+1464
View File
File diff suppressed because it is too large Load Diff
src/k1link/compute/e41_evaluation_boundary.py
+908
View File
@@ -0,0 +1,908 @@
"""Physically separate frozen E40 prediction from visible contract evaluation.
The predictor package contains a trained model, a feature matrix and stripped
item metadata. It contains no acceptance rows, split assignments, reference
labels, severity or scoring state. The predictor therefore cannot inspect
truth. A separate evaluator joins the immutable prediction artifact to E37
after inference and reports source-scoped engineering-contract conformance.
"""
from __future__ import annotations
import hashlib
import json
import os
import shutil
import uuid
from collections import Counter, defaultdict
from dataclasses import dataclass
from datetime import UTC, datetime
from pathlib import Path
from typing import Any, Final
import numpy as np
from k1link.compute.e37_acceptance_contract import (
E37_ITEMS_NAME,
read_e37_acceptance_contract,
)
from k1link.compute.e40_perception_product_gate import (
_LABELS,
_predict_product_presence,
_project_dimensions,
)
from k1link.compute.e41_methodology_audit import (
_load_feature_cache,
_validate_e40_package,
)
E41_PREDICTOR_PACKAGE_SCHEMA: Final = "missioncore.e41-predictor-package/v1"
E41_PREDICTION_RESULT_SCHEMA: Final = "missioncore.e41-prediction-result/v1"
E41_PREDICTION_ROW_SCHEMA: Final = "missioncore.e41-prediction-row/v1"
E41_VISIBLE_EVALUATION_SCHEMA: Final = "missioncore.e41-visible-evaluation/v1"
E41_PACKAGE_MANIFEST_NAME: Final = "manifest.json"
E41_MODEL_NAME: Final = "model.json"
E41_ITEMS_NAME: Final = "items.jsonl"
E41_FEATURE_MANIFEST_NAME: Final = "feature-cache.json"
E41_FEATURE_ARRAYS_NAME: Final = "feature-vectors.npz"
E41_PREDICTIONS_NAME: Final = "predictions.jsonl"
E41_EVALUATION_NAME: Final = "evaluation.json"
_AUTHORITY: Final = {
"commands_enabled": False,
"navigation_or_safety_accepted": False,
}
_STRATA: Final = {"agree", "camera-only", "conflict", "geometry-only", "unknown"}
_DIMENSIONS: Final = ("presence", "geometry_association", "freshness")
_PREDICTION_FORBIDDEN_KEYS: Final = frozenset(
{
"acceptance",
"reference",
"scored",
"severity",
"split",
"truth",
}
)
class E41EvaluationBoundaryError(RuntimeError):
"""An E41 predictor package, prediction, or visible evaluation is invalid."""
@dataclass(frozen=True, slots=True)
class E41PredictionResult:
result_id: str
result_root: Path
manifest: dict[str, Any]
@dataclass(frozen=True, slots=True)
class E41VisibleEvaluation:
result_id: str
result_root: Path
manifest: dict[str, Any]
evaluation: dict[str, Any]
def build_e41_predictor_package(
*,
materialization_root: Path,
e40_package_root: Path,
e40_model_path: Path,
output_root: Path,
) -> Path:
"""Build a content-addressed predictor-only package with no truth material."""
materialization = materialization_root.resolve(strict=True)
package_root = e40_package_root.resolve(strict=True)
e40_package, package_artifacts = _validate_e40_package(package_root)
model_source = e40_model_path.resolve(strict=True)
model = _read_json(model_source)
materialization_rows = _read_jsonl(materialization / "materialized-items.jsonl")
materialization_by_id = _unique_by_item_id(materialization_rows, "materialization")
feature_root = (
package_root / "input" / "materialization" / materialization.name
)
item_ids, feature_names, feature_matrix, feature_binding = _load_feature_cache(
feature_root,
package_artifacts=package_artifacts,
)
_validate_frozen_model(model, feature_names)
items = []
for sequence, item_id in enumerate(item_ids):
row = materialization_by_id.get(item_id)
if row is None:
raise E41EvaluationBoundaryError("E41 predictor item denominator changed")
stratum = row.get("stratum")
if stratum not in _STRATA:
raise E41EvaluationBoundaryError("E41 predictor item stratum is invalid")
items.append(
{
"schema_version": "missioncore.e41-predictor-item/v1",
"sequence": sequence,
"item_id": item_id,
"source_stratum": stratum,
}
)
_assert_truth_free(items, "E41 predictor item metadata")
arrays_payload = _npz_payload(item_ids, feature_matrix)
feature_manifest = {
"schema_version": "missioncore.e41-predictor-feature-cache/v1",
"item_count": len(item_ids),
"dimensions": len(feature_names),
"feature_names": feature_names,
"feature_names_sha256": hashlib.sha256(_canonical_json(feature_names)).hexdigest(),
"arrays_path": E41_FEATURE_ARRAYS_NAME,
"arrays_byte_length": len(arrays_payload),
"arrays_sha256": hashlib.sha256(arrays_payload).hexdigest(),
}
items_payload = _jsonl_payload(items)
model_payload = _json_payload(model)
feature_manifest_payload = _json_payload(feature_manifest)
sources = {
E41_MODEL_NAME: model_payload,
E41_ITEMS_NAME: items_payload,
E41_FEATURE_MANIFEST_NAME: feature_manifest_payload,
E41_FEATURE_ARRAYS_NAME: arrays_payload,
}
source_artifacts = [
{
"path": name,
"byte_length": len(payload),
"sha256": hashlib.sha256(payload).hexdigest(),
}
for name, payload in sorted(sources.items())
]
identity = {
"schema_version": E41_PREDICTOR_PACKAGE_SCHEMA,
"classification": "truth-free-frozen-e40-predictor-input",
"source": {
"materialization_id": materialization.name,
"materialization_index_sha256": _sha256(
materialization / "materialized-items.jsonl"
),
"e40_worker_package_id": e40_package["package_id"],
"e40_worker_package_identity_sha256": e40_package["identity_sha256"],
"e40_model_sha256": _sha256(model_source),
"feature_cache": feature_binding,
},
"runtime_contract": {
"python": "3.12",
"numpy_api": "numpy-1.26-or-newer",
"determinism": "frozen-weights-no-randomness",
},
"source_artifacts": source_artifacts,
"truth_material_included": False,
"producer_sha256": _sha256(Path(__file__).resolve(strict=True)),
"authority": _AUTHORITY,
}
identity_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest()
package_id = f"e41-predictor-package-{identity_sha256}"
destination = output_root.expanduser().absolute() / package_id
if destination.exists():
validate_e41_predictor_package(destination)
return destination
destination.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
staging = destination.parent / f".{package_id}.{uuid.uuid4().hex}.tmp"
staging.mkdir(mode=0o700, exist_ok=False)
try:
for name, payload in sources.items():
_write_bytes(staging / name, payload)
manifest = {
"schema_version": E41_PREDICTOR_PACKAGE_SCHEMA,
"package_id": package_id,
"identity_sha256": identity_sha256,
"identity": identity,
"created_at_utc": _utc_now(),
"artifacts": [
{
**row,
"role": "predictor-only-input",
}
for row in source_artifacts
],
}
_write_json(staging / E41_PACKAGE_MANIFEST_NAME, manifest)
validate_e41_predictor_package(staging, allow_staging=True)
os.replace(staging, destination)
except BaseException:
shutil.rmtree(staging, ignore_errors=True)
raise
validate_e41_predictor_package(destination)
return destination
def validate_e41_predictor_package(
root: Path,
*,
allow_staging: bool = False,
) -> dict[str, Any]:
"""Validate exact package identity, artifacts, and the truth-free boundary."""
resolved = root.resolve(strict=True)
manifest = _read_json(resolved / E41_PACKAGE_MANIFEST_NAME)
identity = _object(manifest.get("identity"), "E41 predictor identity")
identity_sha256 = manifest.get("identity_sha256")
package_id = manifest.get("package_id")
expected_name = isinstance(package_id, str) and (
resolved.name == package_id
or (
allow_staging
and resolved.name.startswith(f".{package_id}.")
and resolved.name.endswith(".tmp")
)
)
if (
manifest.get("schema_version") != E41_PREDICTOR_PACKAGE_SCHEMA
or not isinstance(identity_sha256, str)
or hashlib.sha256(_canonical_json(identity)).hexdigest() != identity_sha256
or package_id != f"e41-predictor-package-{identity_sha256}"
or not expected_name
or identity.get("classification") != "truth-free-frozen-e40-predictor-input"
or identity.get("truth_material_included") is not False
or identity.get("authority") != _AUTHORITY
):
raise E41EvaluationBoundaryError("E41 predictor package identity is invalid")
expected_rows = identity.get("source_artifacts")
artifacts = manifest.get("artifacts")
if not isinstance(expected_rows, list) or not isinstance(artifacts, list):
raise E41EvaluationBoundaryError("E41 predictor package catalog is invalid")
expected: dict[str, tuple[int, str]] = {}
for row in expected_rows:
if (
not isinstance(row, dict)
or not isinstance((name := row.get("path")), str)
or name in expected
or Path(name).is_absolute()
or ".." in Path(name).parts
or not isinstance((length := row.get("byte_length")), int)
or not isinstance((sha256 := row.get("sha256")), str)
):
raise E41EvaluationBoundaryError("E41 predictor source artifact is invalid")
expected[name] = (length, sha256)
actual_files = {
path.relative_to(resolved).as_posix()
for path in resolved.rglob("*")
if path.is_file()
}
if actual_files != set(expected) | {E41_PACKAGE_MANIFEST_NAME}:
raise E41EvaluationBoundaryError("E41 predictor package file set changed")
observed: set[str] = set()
for row in artifacts:
if not isinstance(row, dict):
raise E41EvaluationBoundaryError("E41 predictor artifact is invalid")
name = row.get("path")
path = resolved / str(name)
if (
not isinstance(name, str)
or name in observed
or row.get("role") != "predictor-only-input"
or expected.get(name) != (row.get("byte_length"), row.get("sha256"))
or not path.is_file()
or path.is_symlink()
or row.get("byte_length") != path.stat().st_size
or row.get("sha256") != _sha256(path)
):
raise E41EvaluationBoundaryError("E41 predictor artifact changed")
observed.add(name)
if observed != set(expected):
raise E41EvaluationBoundaryError("E41 predictor artifact coverage changed")
items = _read_jsonl(resolved / E41_ITEMS_NAME)
_assert_truth_free(items, "E41 predictor package")
feature_manifest = _read_json(resolved / E41_FEATURE_MANIFEST_NAME)
with np.load(resolved / E41_FEATURE_ARRAYS_NAME, allow_pickle=False) as arrays:
item_ids = [str(value) for value in arrays["item_ids"]]
matrix = np.asarray(arrays["features"], dtype=np.float64)
model = _read_json(resolved / E41_MODEL_NAME)
names = feature_manifest.get("feature_names")
if (
not isinstance(names, list)
or not all(isinstance(name, str) for name in names)
or feature_manifest.get("item_count") != len(items)
or feature_manifest.get("dimensions") != len(names)
or feature_manifest.get("feature_names_sha256")
!= hashlib.sha256(_canonical_json(names)).hexdigest()
or feature_manifest.get("arrays_byte_length")
!= (resolved / E41_FEATURE_ARRAYS_NAME).stat().st_size
or feature_manifest.get("arrays_sha256")
!= _sha256(resolved / E41_FEATURE_ARRAYS_NAME)
or matrix.shape != (len(items), len(names))
or item_ids != [str(row.get("item_id")) for row in items]
or not np.isfinite(matrix).all()
):
raise E41EvaluationBoundaryError("E41 predictor feature cache is invalid")
_validate_frozen_model(model, names)
return manifest
def run_e41_predictor(
*,
package_root: Path,
output_root: Path,
runtime_identity: dict[str, str],
) -> E41PredictionResult:
"""Run the frozen predictor without accepting an acceptance/truth input."""
package = validate_e41_predictor_package(package_root)
resolved = package_root.resolve(strict=True)
items = _read_jsonl(resolved / E41_ITEMS_NAME)
model = _read_json(resolved / E41_MODEL_NAME)
feature_manifest = _read_json(resolved / E41_FEATURE_MANIFEST_NAME)
names = [str(value) for value in feature_manifest["feature_names"]]
with np.load(resolved / E41_FEATURE_ARRAYS_NAME, allow_pickle=False) as arrays:
matrix = np.asarray(arrays["features"], dtype=np.float64)
predictions = predict_from_frozen_e40_model(
items=items,
feature_names=names,
feature_matrix=matrix,
model=model,
)
_assert_truth_free(predictions, "E41 prediction result")
content_sha256 = hashlib.sha256(_canonical_json(predictions)).hexdigest()
identity = {
"schema_version": E41_PREDICTION_RESULT_SCHEMA,
"predictor_package_id": package["package_id"],
"predictor_package_identity_sha256": package["identity_sha256"],
"runtime": _validated_runtime_identity(runtime_identity),
"prediction_content_sha256": content_sha256,
"truth_material_available_to_predictor": False,
"producer_sha256": _sha256(Path(__file__).resolve(strict=True)),
"authority": _AUTHORITY,
}
identity_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest()
result_id = f"e41-predictions-{identity_sha256}"
destination = output_root.expanduser().absolute() / result_id
if destination.exists():
return read_e41_prediction_result(destination)
destination.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
staging = destination.parent / f".{result_id}.{uuid.uuid4().hex}.tmp"
staging.mkdir(mode=0o700, exist_ok=False)
try:
_write_jsonl(staging / E41_PREDICTIONS_NAME, predictions)
manifest = {
"schema_version": E41_PREDICTION_RESULT_SCHEMA,
"result_id": result_id,
"identity_sha256": identity_sha256,
"identity": identity,
"created_at_utc": _utc_now(),
"item_count": len(predictions),
"truth_material_included": False,
"artifacts": [
_artifact(
staging / E41_PREDICTIONS_NAME,
"truth-free-predictions",
)
],
"authority": _AUTHORITY,
}
_write_json(staging / E41_PACKAGE_MANIFEST_NAME, manifest)
os.replace(staging, destination)
except BaseException:
shutil.rmtree(staging, ignore_errors=True)
raise
return read_e41_prediction_result(destination)
def read_e41_prediction_result(root: Path) -> E41PredictionResult:
"""Read and validate one immutable truth-free prediction result."""
resolved = root.resolve(strict=True)
manifest = _read_json(resolved / E41_PACKAGE_MANIFEST_NAME)
identity = _object(manifest.get("identity"), "E41 prediction identity")
identity_sha256 = manifest.get("identity_sha256")
if (
manifest.get("schema_version") != E41_PREDICTION_RESULT_SCHEMA
or not isinstance(identity_sha256, str)
or hashlib.sha256(_canonical_json(identity)).hexdigest() != identity_sha256
or manifest.get("result_id") != f"e41-predictions-{identity_sha256}"
or resolved.name != manifest.get("result_id")
or identity.get("truth_material_available_to_predictor") is not False
or manifest.get("truth_material_included") is not False
or manifest.get("authority") != _AUTHORITY
):
raise E41EvaluationBoundaryError("E41 prediction identity is invalid")
_validate_single_artifact(
resolved,
manifest.get("artifacts"),
name=E41_PREDICTIONS_NAME,
role="truth-free-predictions",
)
predictions = _read_jsonl(resolved / E41_PREDICTIONS_NAME)
_assert_truth_free(predictions, "E41 prediction result")
if (
manifest.get("item_count") != len(predictions)
or identity.get("prediction_content_sha256")
!= hashlib.sha256(_canonical_json(predictions)).hexdigest()
):
raise E41EvaluationBoundaryError("E41 prediction content is invalid")
return E41PredictionResult(
result_id=resolved.name,
result_root=resolved,
manifest=manifest,
)
def build_e41_visible_evaluation(
*,
prediction_root: Path,
acceptance_root: Path,
output_root: Path,
) -> E41VisibleEvaluation:
"""Evaluate immutable predictions against the already-visible E37 substrate."""
prediction = read_e41_prediction_result(prediction_root)
acceptance = read_e37_acceptance_contract(acceptance_root)
predictions = _read_jsonl(prediction.result_root / E41_PREDICTIONS_NAME)
acceptance_rows = _read_jsonl(acceptance.result_root / E37_ITEMS_NAME)
evaluation = evaluate_visible_engineering_contract(
predictions=predictions,
acceptance_rows=acceptance_rows,
targets=_object(acceptance.contract.get("targets"), "E37 targets"),
label_provenance=_object(
acceptance.contract.get("label_provenance"),
"E37 label provenance",
),
)
content_sha256 = hashlib.sha256(_canonical_json(evaluation)).hexdigest()
identity = {
"schema_version": E41_VISIBLE_EVALUATION_SCHEMA,
"prediction_result_id": prediction.result_id,
"prediction_identity_sha256": prediction.manifest["identity_sha256"],
"acceptance_result_id": acceptance.result_id,
"acceptance_identity_sha256": acceptance.manifest["identity_sha256"],
"acceptance_items_sha256": _sha256(acceptance.result_root / E37_ITEMS_NAME),
"evaluation_content_sha256": content_sha256,
"evaluation_semantics": "historical-evaluated-visible-validation",
"producer_sha256": _sha256(Path(__file__).resolve(strict=True)),
"authority": _AUTHORITY,
}
identity_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest()
result_id = f"e41-visible-evaluation-{identity_sha256}"
destination = output_root.expanduser().absolute() / result_id
if destination.exists():
return read_e41_visible_evaluation(destination)
destination.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
staging = destination.parent / f".{result_id}.{uuid.uuid4().hex}.tmp"
staging.mkdir(mode=0o700, exist_ok=False)
document = {
"schema_version": E41_VISIBLE_EVALUATION_SCHEMA,
"result_id": result_id,
"identity_sha256": identity_sha256,
"status": "completed-visible-engineering-contract-evaluation",
"evaluation": evaluation,
"decision": {
"blind_gate_eligible": False,
"independent_perception_accuracy_proved": False,
"source_scoped_engineering_contract_conformance_measured": True,
},
"authority": _AUTHORITY,
}
try:
_write_json(staging / E41_EVALUATION_NAME, document)
manifest = {
"schema_version": E41_VISIBLE_EVALUATION_SCHEMA,
"result_id": result_id,
"identity_sha256": identity_sha256,
"identity": identity,
"created_at_utc": _utc_now(),
"blind_gate_eligible": False,
"artifacts": [
_artifact(
staging / E41_EVALUATION_NAME,
"visible-engineering-contract-evaluation",
)
],
"authority": _AUTHORITY,
}
_write_json(staging / E41_PACKAGE_MANIFEST_NAME, manifest)
os.replace(staging, destination)
except BaseException:
shutil.rmtree(staging, ignore_errors=True)
raise
return read_e41_visible_evaluation(destination)
def read_e41_visible_evaluation(root: Path) -> E41VisibleEvaluation:
"""Read and validate one immutable visible engineering evaluation."""
resolved = root.resolve(strict=True)
manifest = _read_json(resolved / E41_PACKAGE_MANIFEST_NAME)
identity = _object(manifest.get("identity"), "E41 evaluation identity")
identity_sha256 = manifest.get("identity_sha256")
if (
manifest.get("schema_version") != E41_VISIBLE_EVALUATION_SCHEMA
or not isinstance(identity_sha256, str)
or hashlib.sha256(_canonical_json(identity)).hexdigest() != identity_sha256
or manifest.get("result_id") != f"e41-visible-evaluation-{identity_sha256}"
or resolved.name != manifest.get("result_id")
or identity.get("evaluation_semantics")
!= "historical-evaluated-visible-validation"
or manifest.get("blind_gate_eligible") is not False
or manifest.get("authority") != _AUTHORITY
):
raise E41EvaluationBoundaryError("E41 visible evaluation identity is invalid")
_validate_single_artifact(
resolved,
manifest.get("artifacts"),
name=E41_EVALUATION_NAME,
role="visible-engineering-contract-evaluation",
)
document = _read_json(resolved / E41_EVALUATION_NAME)
evaluation = _object(document.get("evaluation"), "E41 visible evaluation")
if (
document.get("schema_version") != E41_VISIBLE_EVALUATION_SCHEMA
or document.get("result_id") != resolved.name
or document.get("identity_sha256") != identity_sha256
or document.get("decision", {}).get("blind_gate_eligible") is not False
or hashlib.sha256(_canonical_json(evaluation)).hexdigest()
!= identity.get("evaluation_content_sha256")
or document.get("authority") != _AUTHORITY
):
raise E41EvaluationBoundaryError("E41 visible evaluation content is invalid")
return E41VisibleEvaluation(
result_id=resolved.name,
result_root=resolved,
manifest=manifest,
evaluation=document,
)
def predict_from_frozen_e40_model(
*,
items: list[dict[str, Any]],
feature_names: list[str],
feature_matrix: np.ndarray[Any, Any],
model: dict[str, Any],
) -> list[dict[str, Any]]:
"""Predict from frozen weights without accepting reference labels."""
_assert_truth_free(items, "E41 predictor items")
_validate_frozen_model(model, feature_names)
if feature_matrix.shape != (len(items), len(feature_names)):
raise E41EvaluationBoundaryError("E41 predictor feature accounting changed")
classifier = _object(model.get("classifier"), "E40 frozen classifier")
scaler = _object(classifier.get("scaler"), "E40 frozen scaler")
median = np.asarray(scaler["median"], dtype=np.float64)
scale = np.asarray(scaler["scale"], dtype=np.float64)
weights = np.asarray(classifier["weights"], dtype=np.float64)
clip = float(model["robust_clip"])
predictions = []
for index, (item, features) in enumerate(zip(items, feature_matrix, strict=True)):
if item.get("sequence") != index:
raise E41EvaluationBoundaryError("E41 predictor item order changed")
stratum = str(item.get("source_stratum"))
presence, confidence = _predict_product_presence(
stratum=stratum,
features=np.asarray(features, dtype=np.float64),
median=median,
scale=scale,
weights=weights,
clip=clip,
)
predictions.append(
{
"schema_version": E41_PREDICTION_ROW_SCHEMA,
"sequence": index,
"item_id": item["item_id"],
"source_stratum": stratum,
"prediction": _project_dimensions(stratum, presence),
"presence_confidence": round(confidence, 6),
"authority": _AUTHORITY,
}
)
_assert_truth_free(predictions, "E41 predictions")
return predictions
def evaluate_visible_engineering_contract(
*,
predictions: list[dict[str, Any]],
acceptance_rows: list[dict[str, Any]],
targets: dict[str, Any],
label_provenance: dict[str, Any],
) -> dict[str, Any]:
"""Join truth-free predictions to the evaluated E37 validation rows."""
_assert_truth_free(predictions, "E41 evaluator input predictions")
prediction_by_id = _unique_by_item_id(predictions, "prediction")
acceptance_by_id = _unique_by_item_id(acceptance_rows, "acceptance")
if set(prediction_by_id) != set(acceptance_by_id):
raise E41EvaluationBoundaryError("E41 evaluation denominator changed")
joined: list[dict[str, Any]] = []
for item_id, acceptance in acceptance_by_id.items():
if acceptance.get("split") != "validation":
continue
prediction = prediction_by_id[item_id]
joined.append(
{
"item_id": item_id,
"source_stratum": acceptance["source_stratum"],
"severity": acceptance["severity"],
"prediction": prediction["prediction"],
"reference": acceptance["reference"],
}
)
if not joined:
raise E41EvaluationBoundaryError("E41 visible evaluation slice is empty")
dimensions = {
dimension: _dimension_metrics(
joined,
dimension=dimension,
target=float(targets[f"{dimension}_target"]),
)
for dimension in _DIMENSIONS
}
high_severity_failures = sum(
row["severity"] == "high"
and any(
row["prediction"][dimension] != row["reference"][dimension]
for dimension in _DIMENSIONS
)
for row in joined
)
false_free_claims = sum(
value == "free"
for row in predictions
for value in _object(row.get("prediction"), "E41 prediction dimensions").values()
)
checks = {
"presence_target_reached": dimensions["presence"]["passed"],
"geometry_association_target_reached": dimensions["geometry_association"]["passed"],
"freshness_target_reached": dimensions["freshness"]["passed"],
"accounting_complete": len(joined)
== sum(row.get("split") == "validation" for row in acceptance_rows),
"false_free_claims_zero": false_free_claims == 0,
"high_severity_failures_zero": high_severity_failures == 0,
"authority_remains_diagnostic": True,
}
return {
"evaluation_semantics": "historical-evaluated-visible-validation",
"label_provenance": {
**label_provenance,
"independent_accuracy_authority": False,
},
"metrics": {
"validation_items": len(joined),
"terminal_outcomes": len(joined),
"accounting_fraction": 1.0,
"false_free_claims": false_free_claims,
"high_severity_failures": high_severity_failures,
"dimensions": dimensions,
},
"checks": checks,
"engineering_contract_targets_reached": all(checks.values()),
"blocking_checks": [name for name, passed in checks.items() if not passed],
"blind_gate_eligible": False,
"authority": _AUTHORITY,
}
def _dimension_metrics(
rows: list[dict[str, Any]],
*,
dimension: str,
target: float,
) -> dict[str, Any]:
confusion: Counter[tuple[str, str]] = Counter()
strata: dict[str, Counter[str]] = defaultdict(Counter)
correct = 0
for row in rows:
reference = str(row["reference"][dimension])
prediction = str(row["prediction"][dimension])
confusion[(reference, prediction)] += 1
matched = reference == prediction
correct += matched
strata[str(row["source_stratum"])]["correct" if matched else "incorrect"] += 1
total = len(rows)
accuracy = correct / total
return {
"correct": correct,
"incorrect": total - correct,
"total": total,
"accuracy": round(accuracy, 6),
"target": target,
"passed": accuracy >= target,
"confusion": [
{
"reference": reference,
"prediction": prediction,
"count": count,
}
for (reference, prediction), count in sorted(
confusion.items(),
key=lambda item: (-item[1], item[0]),
)
],
"by_stratum": {
stratum: {
"correct": counts["correct"],
"incorrect": counts["incorrect"],
"total": sum(counts.values()),
"accuracy": round(counts["correct"] / sum(counts.values()), 6),
}
for stratum, counts in sorted(strata.items())
},
}
def _validate_frozen_model(model: dict[str, Any], feature_names: list[str]) -> None:
classifier = _object(model.get("classifier"), "E40 frozen classifier")
scaler = _object(classifier.get("scaler"), "E40 frozen scaler")
weights = np.asarray(classifier.get("weights"), dtype=np.float64)
median = np.asarray(scaler.get("median"), dtype=np.float64)
scale = np.asarray(scaler.get("scale"), dtype=np.float64)
if (
model.get("schema_version") != "missioncore.e40-development-product-model/v1"
or model.get("feature_names") != feature_names
or model.get("validation_labels_used_for_training") is not False
or classifier.get("type") != "deterministic-softmax"
or classifier.get("labels") != list(_LABELS)
or weights.shape != (len(feature_names) + 1, len(_LABELS))
or median.shape != (len(feature_names),)
or scale.shape != (len(feature_names),)
or not np.isfinite(weights).all()
or not np.isfinite(median).all()
or not np.isfinite(scale).all()
or np.any(scale <= 0.0)
or not isinstance(model.get("robust_clip"), int | float)
or float(model["robust_clip"]) <= 0.0
):
raise E41EvaluationBoundaryError("E41 frozen E40 model is invalid")
def _assert_truth_free(value: object, label: str) -> None:
if isinstance(value, dict):
forbidden = set(value) & _PREDICTION_FORBIDDEN_KEYS
if forbidden:
raise E41EvaluationBoundaryError(
f"{label} contains truth/evaluation keys: {sorted(forbidden)}"
)
for child in value.values():
_assert_truth_free(child, label)
elif isinstance(value, list):
for child in value:
_assert_truth_free(child, label)
def _validated_runtime_identity(value: dict[str, str]) -> dict[str, str]:
required = {"environment_lock", "numpy", "python"}
if (
set(value) != required
or not all(isinstance(item, str) and item for item in value.values())
or "@sha256:" not in value["environment_lock"]
):
raise E41EvaluationBoundaryError("E41 runtime identity is incomplete")
return dict(sorted(value.items()))
def _unique_by_item_id(
rows: list[dict[str, Any]],
label: str,
) -> dict[str, dict[str, Any]]:
indexed: dict[str, dict[str, Any]] = {}
for row in rows:
item_id = row.get("item_id")
if not isinstance(item_id, str) or not item_id or item_id in indexed:
raise E41EvaluationBoundaryError(f"E41 {label} item identity is invalid")
indexed[item_id] = row
return indexed
def _validate_single_artifact(
root: Path,
value: object,
*,
name: str,
role: str,
) -> None:
if not isinstance(value, list) or len(value) != 1 or not isinstance(value[0], dict):
raise E41EvaluationBoundaryError("E41 artifact catalog is invalid")
row = value[0]
path = root / name
if (
row.get("path") != name
or row.get("role") != role
or not path.is_file()
or path.is_symlink()
or row.get("byte_length") != path.stat().st_size
or row.get("sha256") != _sha256(path)
):
raise E41EvaluationBoundaryError("E41 artifact content changed")
def _npz_payload(item_ids: list[str], matrix: np.ndarray[Any, Any]) -> bytes:
import io
stream = io.BytesIO()
np.savez_compressed(
stream,
item_ids=np.asarray(item_ids, dtype=f"<U{max(map(len, item_ids))}"),
features=np.asarray(matrix, dtype=np.float64),
)
return stream.getvalue()
def _json_payload(value: object) -> bytes:
return (json.dumps(value, indent=2, sort_keys=True) + "\n").encode()
def _jsonl_payload(rows: list[dict[str, Any]]) -> bytes:
return b"".join(_canonical_json(row) + b"\n" for row in rows)
def _artifact(path: Path, role: str) -> dict[str, Any]:
return {
"path": path.name,
"role": role,
"byte_length": path.stat().st_size,
"sha256": _sha256(path),
}
def _object(value: object, label: str) -> dict[str, Any]:
if not isinstance(value, dict):
raise E41EvaluationBoundaryError(f"{label} must be an object")
return value
def _read_json(path: Path) -> dict[str, Any]:
value = json.loads(path.read_text(encoding="utf-8-sig"))
if not isinstance(value, dict):
raise E41EvaluationBoundaryError(f"JSON object expected: {path.name}")
return value
def _read_jsonl(path: Path) -> list[dict[str, Any]]:
rows = [json.loads(line) for line in path.read_text(encoding="utf-8-sig").splitlines()]
if not all(isinstance(row, dict) for row in rows):
raise E41EvaluationBoundaryError(f"JSONL object expected: {path.name}")
return rows
def _write_json(path: Path, value: object) -> None:
with path.open("x", encoding="utf-8", newline="\n") as stream:
json.dump(value, stream, indent=2, sort_keys=True)
stream.write("\n")
stream.flush()
os.fsync(stream.fileno())
def _write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None:
with path.open("xb") as stream:
for row in rows:
stream.write(_canonical_json(row))
stream.write(b"\n")
stream.flush()
os.fsync(stream.fileno())
def _write_bytes(path: Path, value: bytes) -> None:
with path.open("xb") as stream:
stream.write(value)
stream.flush()
os.fsync(stream.fileno())
def _canonical_json(value: object) -> bytes:
return json.dumps(
value,
sort_keys=True,
separators=(",", ":"),
allow_nan=False,
).encode()
def _sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as stream:
while chunk := stream.read(1024 * 1024):
digest.update(chunk)
return digest.hexdigest()
def _utc_now() -> str:
return datetime.now(UTC).isoformat(timespec="milliseconds").replace("+00:00", "Z")
src/k1link/compute/e41_methodology_audit.py
+901
View File
@@ -0,0 +1,901 @@
"""Deterministic methodology audit for the historical RAVNOVES00 E37-E40 chain.
E41 does not tune a model and does not rewrite an immutable E37 or E40 result.
It binds the exact acceptance contract, materialization, E40 worker package and
package-bound result, then makes split leakage, feature pressure, label
provenance and predictor/evaluator co-location machine-readable.
"""
from __future__ import annotations
import hashlib
import json
import os
import shutil
import uuid
from collections.abc import Iterable
from dataclasses import dataclass
from datetime import UTC, datetime
from pathlib import Path
from typing import Any, Final
import numpy as np
from k1link.compute.e37_acceptance_contract import (
E37_ITEMS_NAME,
read_e37_acceptance_contract,
)
from k1link.compute.e40_perception_product_gate import (
E40_MODEL_NAME,
E40_PREDICTIONS_NAME,
E40_REPORT_NAME,
read_e40_perception_product_gate,
)
E41_PROFILE_SCHEMA: Final = "missioncore.e41-methodology-audit-profile/v1"
E41_RESULT_SCHEMA: Final = "missioncore.e41-methodology-audit/v1"
E41_REPORT_SCHEMA: Final = "missioncore.e41-methodology-audit-report/v1"
E41_REPORT_NAME: Final = "methodology-audit.json"
E41_SUMMARY_NAME: Final = "methodology-audit.md"
E41_MANIFEST_NAME: Final = "manifest.json"
_E40_PACKAGE_SCHEMA: Final = "missioncore.e40-worker-package/v1"
_FEATURE_CACHE_SCHEMA: Final = "missioncore.e40-feature-cache/v1"
_FEATURE_CACHE_MANIFEST: Final = "e40-feature-cache.json"
_FEATURE_CACHE_ARRAYS: Final = "e40-feature-vectors.npz"
_AUTHORITY: Final = {
"commands_enabled": False,
"navigation_or_safety_accepted": False,
}
_SOURCE_DERIVED_PREFIXES: Final = (
"association=",
"camera_motion=",
"geometry=",
"label=",
"motion=",
"reason=",
"semantic_current=",
"stratum=",
)
_SOURCE_DERIVED_NAMES: Final = frozenset(
{
"detector_score",
"score",
}
)
class E41MethodologyAuditError(RuntimeError):
"""An E41 input, policy profile, analysis, or immutable result is invalid."""
@dataclass(frozen=True, slots=True)
class E41MethodologyAudit:
result_id: str
result_root: Path
manifest: dict[str, Any]
report: dict[str, Any]
@property
def blind_gate_eligible(self) -> bool:
return self.report.get("decision", {}).get("blind_gate_eligible") is True
def build_e41_methodology_audit(
*,
acceptance_root: Path,
materialization_root: Path,
e40_package_root: Path,
e40_result_root: Path,
profile_path: Path,
output_root: Path,
) -> E41MethodologyAudit:
"""Build or validate one immutable, source-scoped E41 methodology audit."""
profile_file = profile_path.resolve(strict=True)
profile = _read_json(profile_file)
_validate_profile(profile)
acceptance = read_e37_acceptance_contract(acceptance_root)
materialization = materialization_root.resolve(strict=True)
e40_result = read_e40_perception_product_gate(e40_result_root)
package, package_artifacts = _validate_e40_package(e40_package_root)
source = _object(profile.get("source"), "E41 profile source")
if (
acceptance.result_id != source.get("acceptance_result_id")
or materialization.name != source.get("materialization_id")
or e40_result.result_id != source.get("e40_result_id")
or package.get("package_id") != source.get("e40_package_id")
):
raise E41MethodologyAuditError("E41 source identity changed")
execution_package = _object(
_object(e40_result.report.get("execution"), "E40 execution").get("package"),
"E40 execution package",
)
if (
execution_package.get("mode") != "verified-worker-package"
or execution_package.get("package_id") != package["package_id"]
or execution_package.get("identity_sha256") != package["identity_sha256"]
):
raise E41MethodologyAuditError("E41 requires the package-bound E40 result")
acceptance_rows = _read_jsonl(acceptance.result_root / E37_ITEMS_NAME)
materialization_rows = _read_jsonl(materialization / "materialized-items.jsonl")
materialization_manifest = _read_json(materialization / "manifest.json")
reviewed_binding = _object(
_object(acceptance.manifest.get("identity"), "E37 identity").get(
"reviewed_substrate"
),
"E37 reviewed substrate",
)
if (
len(acceptance_rows) != 486
or len(materialization_rows) != 486
or materialization_manifest.get("result_id") != materialization.name
or reviewed_binding.get("materialization_id") != materialization.name
or reviewed_binding.get("materialization_identity_sha256")
!= materialization_manifest.get("identity_sha256")
or reviewed_binding.get("materialization_index_sha256")
!= _sha256(materialization / "materialized-items.jsonl")
):
raise E41MethodologyAuditError("E41 materialization binding changed")
feature_root = (
e40_package_root.resolve(strict=True)
/ "input"
/ "materialization"
/ materialization.name
)
item_ids, feature_names, feature_matrix, feature_binding = _load_feature_cache(
feature_root,
package_artifacts=package_artifacts,
)
predictions = _read_jsonl(e40_result.result_root / E40_PREDICTIONS_NAME)
model = _read_json(e40_result.result_root / E40_MODEL_NAME)
historical_report = _read_json(e40_result.result_root / E40_REPORT_NAME)
analysis = analyze_e41_methodology(
acceptance_rows=acceptance_rows,
materialization_rows=materialization_rows,
feature_item_ids=item_ids,
feature_names=feature_names,
feature_matrix=feature_matrix,
e40_model=model,
e40_report=historical_report,
e40_predictions=predictions,
label_provenance=_object(
acceptance.contract.get("label_provenance"),
"E37 label provenance",
),
time_block_frames=int(profile["policy"]["time_block_frames"]),
forbidden_feature_tokens=tuple(profile["policy"]["forbidden_feature_tokens"]),
)
analysis_sha256 = hashlib.sha256(_canonical_json(analysis)).hexdigest()
identity = {
"schema_version": E41_RESULT_SCHEMA,
"source": {
"acceptance_result_id": acceptance.result_id,
"acceptance_identity_sha256": acceptance.manifest["identity_sha256"],
"acceptance_items_sha256": _sha256(acceptance.result_root / E37_ITEMS_NAME),
"materialization_id": materialization.name,
"materialization_identity_sha256": materialization_manifest["identity_sha256"],
"materialization_index_sha256": _sha256(
materialization / "materialized-items.jsonl"
),
"e40_package_id": package["package_id"],
"e40_package_identity_sha256": package["identity_sha256"],
"e40_result_id": e40_result.result_id,
"e40_result_identity_sha256": e40_result.manifest["identity_sha256"],
"feature_cache": feature_binding,
},
"profile": {
"profile_id": profile["profile_id"],
"sha256": _sha256(profile_file),
},
"producer_sha256": _sha256(Path(__file__).resolve(strict=True)),
"analysis_sha256": analysis_sha256,
"authority": _AUTHORITY,
}
identity_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest()
result_id = f"e41-methodology-audit-{identity_sha256}"
destination = output_root.expanduser().absolute() / result_id
if destination.exists():
return read_e41_methodology_audit(destination)
report = {
"schema_version": E41_REPORT_SCHEMA,
"result_id": result_id,
"identity_sha256": identity_sha256,
"status": "completed-methodology-audit",
"evaluation_semantics": "historical-evaluated-visible-validation",
"analysis": analysis,
"decision": {
"blind_gate_eligible": analysis["policy"]["blind_gate_eligible"],
"current_146_items": "historical-evaluated-visible-validation",
"current_e40_result": "source-scoped-engineering-contract-evaluation",
"independent_perception_accuracy_proved": False,
"next_gate": (
"separate predictor/evaluator and prepare a grouped independent truth island"
),
},
"authority": _AUTHORITY,
}
summary = _markdown_summary(report)
destination.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
staging = destination.parent / f".{result_id}.{uuid.uuid4().hex}.tmp"
staging.mkdir(mode=0o700, exist_ok=False)
try:
_write_json(staging / E41_REPORT_NAME, report)
_write_text(staging / E41_SUMMARY_NAME, summary)
artifacts = [
_artifact(staging / E41_REPORT_NAME, "machine-readable-methodology-audit"),
_artifact(staging / E41_SUMMARY_NAME, "human-readable-methodology-summary"),
]
manifest = {
"schema_version": E41_RESULT_SCHEMA,
"result_id": result_id,
"identity_sha256": identity_sha256,
"identity": identity,
"created_at_utc": _utc_now(),
"acceptance_state": "methodology-blocked-for-blind-gate",
"blind_gate_eligible": False,
"artifacts": artifacts,
"authority": _AUTHORITY,
}
_write_json(staging / E41_MANIFEST_NAME, manifest)
os.replace(staging, destination)
except BaseException:
shutil.rmtree(staging, ignore_errors=True)
raise
return read_e41_methodology_audit(destination)
def read_e41_methodology_audit(root: Path) -> E41MethodologyAudit:
"""Read and validate one immutable E41 methodology audit."""
resolved = root.resolve(strict=True)
manifest = _read_json(resolved / E41_MANIFEST_NAME)
identity = manifest.get("identity")
identity_sha256 = manifest.get("identity_sha256")
if (
manifest.get("schema_version") != E41_RESULT_SCHEMA
or not isinstance(identity, dict)
or not isinstance(identity_sha256, str)
or hashlib.sha256(_canonical_json(identity)).hexdigest() != identity_sha256
or manifest.get("result_id") != f"e41-methodology-audit-{identity_sha256}"
or resolved.name != manifest.get("result_id")
or manifest.get("acceptance_state") != "methodology-blocked-for-blind-gate"
or manifest.get("blind_gate_eligible") is not False
or manifest.get("authority") != _AUTHORITY
):
raise E41MethodologyAuditError("E41 result identity is invalid")
expected = {
E41_REPORT_NAME: "machine-readable-methodology-audit",
E41_SUMMARY_NAME: "human-readable-methodology-summary",
}
_validate_artifacts(resolved, manifest.get("artifacts"), expected)
report = _read_json(resolved / E41_REPORT_NAME)
analysis = _object(report.get("analysis"), "E41 report analysis")
if (
report.get("schema_version") != E41_REPORT_SCHEMA
or report.get("result_id") != resolved.name
or report.get("identity_sha256") != identity_sha256
or hashlib.sha256(_canonical_json(analysis)).hexdigest()
!= identity.get("analysis_sha256")
or report.get("decision", {}).get("blind_gate_eligible") is not False
or report.get("authority") != _AUTHORITY
):
raise E41MethodologyAuditError("E41 report is invalid")
return E41MethodologyAudit(
result_id=resolved.name,
result_root=resolved,
manifest=manifest,
report=report,
)
def analyze_e41_methodology(
*,
acceptance_rows: list[dict[str, Any]],
materialization_rows: list[dict[str, Any]],
feature_item_ids: list[str],
feature_names: list[str],
feature_matrix: np.ndarray[Any, Any],
e40_model: dict[str, Any],
e40_report: dict[str, Any],
e40_predictions: list[dict[str, Any]],
label_provenance: dict[str, Any],
time_block_frames: int,
forbidden_feature_tokens: tuple[str, ...],
) -> dict[str, Any]:
"""Return the deterministic E41 analysis for exact in-memory inputs."""
if time_block_frames <= 0 or not forbidden_feature_tokens:
raise E41MethodologyAuditError("E41 methodology policy is invalid")
acceptance_by_id = _unique_by_item_id(acceptance_rows, "acceptance")
materialization_by_id = _unique_by_item_id(materialization_rows, "materialization")
if (
set(acceptance_by_id) != set(materialization_by_id)
or len(feature_item_ids) != len(set(feature_item_ids))
or set(feature_item_ids) != set(acceptance_by_id)
or feature_matrix.shape != (len(feature_item_ids), len(feature_names))
or len(feature_names) != len(set(feature_names))
or not np.isfinite(feature_matrix).all()
):
raise E41MethodologyAuditError("E41 denominator or feature accounting changed")
ordered_acceptance = [acceptance_by_id[item_id] for item_id in feature_item_ids]
splits = [str(row.get("split")) for row in ordered_acceptance]
if set(splits) != {"development", "validation"}:
raise E41MethodologyAuditError("E41 requires development and validation rows")
split_audit = _split_audit(
acceptance_by_id=acceptance_by_id,
materialization_by_id=materialization_by_id,
time_block_frames=time_block_frames,
)
feature_audit = _feature_audit(
ordered_acceptance=ordered_acceptance,
feature_names=feature_names,
feature_matrix=feature_matrix,
e40_model=e40_model,
forbidden_feature_tokens=forbidden_feature_tokens,
)
reference_co_located = any("reference" in row for row in e40_predictions)
scored_co_located = any(row.get("scored") is True for row in e40_predictions)
historical_claims = sorted(
claim
for claim in _strings(e40_report)
if "sealed" in claim.lower() or "leakage-resistant-product-gate" in claim.lower()
)
independent_ground_truth = label_provenance.get("independent_ground_truth") is True
overlap_detected = any(
split_audit[key]["count"] > 0
for key in (
"exact_source_frames",
"track_ids",
"time_blocks",
"whole_track_or_scene_groups",
)
)
violations: list[str] = []
if not independent_ground_truth:
violations.append("labels-are-not-independent-ground-truth")
if overlap_detected:
violations.append("development-validation-source-groups-overlap")
if reference_co_located or scored_co_located:
violations.append("prediction-and-evaluation-concerns-are-co-located")
if historical_claims:
violations.append("historical-e40-still-contains-blind-or-product-gate-claims")
if feature_audit["forbidden_features"]:
violations.append("forbidden-identity-feature-detected")
return {
"denominator": {
"items": len(ordered_acceptance),
"development_items": splits.count("development"),
"validation_items": splits.count("validation"),
"evaluation_semantics": "historical-evaluated-visible-validation",
},
"label_provenance": {
**label_provenance,
"accepted_semantics": "engineering-reviewed-source-scoped-substrate",
},
"split_leakage": split_audit,
"features": feature_audit,
"predictor_evaluator_boundary": {
"prediction_rows": len(e40_predictions),
"reference_labels_present_in_prediction_artifact": reference_co_located,
"scoring_state_present_in_prediction_artifact": scored_co_located,
"physically_separated": not reference_co_located and not scored_co_located,
},
"metric_semantics": {
"dimension_projection": e40_model.get("dimension_projection"),
"dimensions_independently_inferred": False,
"presence_geometry_accuracy_equal": (
e40_report.get("metrics", {})
.get("dimensions", {})
.get("presence", {})
.get("accuracy")
== e40_report.get("metrics", {})
.get("dimensions", {})
.get("geometry_association", {})
.get("accuracy")
),
},
"historical_claims_requiring_status_correction": historical_claims,
"policy": {
"blind_gate_eligible": not violations,
"violations": violations,
"required_next_actions": [
"treat-current-146-as-visible-validation",
"separate-prediction-from-evaluation-artifacts",
"freeze-grouped-independent-truth-island-before-refinement",
"keep-engineering-contract-and-independent-truth-metrics-separate",
],
},
"authority": _AUTHORITY,
}
def _split_audit(
*,
acceptance_by_id: dict[str, dict[str, Any]],
materialization_by_id: dict[str, dict[str, Any]],
time_block_frames: int,
) -> dict[str, dict[str, Any]]:
dimensions: dict[str, dict[str, set[str]]] = {
"item_ids": {"development": set(), "validation": set()},
"exact_source_frames": {"development": set(), "validation": set()},
"track_ids": {"development": set(), "validation": set()},
"time_blocks": {"development": set(), "validation": set()},
"whole_track_or_scene_groups": {"development": set(), "validation": set()},
}
for item_id, acceptance in acceptance_by_id.items():
split = str(acceptance.get("split"))
if split not in {"development", "validation"}:
raise E41MethodologyAuditError("E41 split row is invalid")
frame_index = acceptance.get("source_frame_index")
if not isinstance(frame_index, int) or frame_index < 0:
raise E41MethodologyAuditError("E41 source frame is invalid")
snapshot = _object(
materialization_by_id[item_id].get("e29_snapshot"),
"E41 materialization snapshot",
)
track_id = snapshot.get("track_id")
frame_key = str(frame_index)
block_key = str(frame_index // time_block_frames)
group_key = f"track:{track_id}" if track_id is not None else f"scene:{block_key}"
dimensions["item_ids"][split].add(item_id)
dimensions["exact_source_frames"][split].add(frame_key)
dimensions["time_blocks"][split].add(block_key)
dimensions["whole_track_or_scene_groups"][split].add(group_key)
if track_id is not None:
dimensions["track_ids"][split].add(str(track_id))
return {
name: _overlap_summary(values["development"], values["validation"])
for name, values in dimensions.items()
}
def _feature_audit(
*,
ordered_acceptance: list[dict[str, Any]],
feature_names: list[str],
feature_matrix: np.ndarray[Any, Any],
e40_model: dict[str, Any],
forbidden_feature_tokens: tuple[str, ...],
) -> dict[str, Any]:
development_indices = np.asarray(
[
index
for index, row in enumerate(ordered_acceptance)
if row.get("split") == "development"
],
dtype=np.int64,
)
validation_indices = np.asarray(
[
index
for index, row in enumerate(ordered_acceptance)
if row.get("split") == "validation"
],
dtype=np.int64,
)
camera_development_indices = np.asarray(
[
index
for index in development_indices
if ordered_acceptance[int(index)].get("source_stratum") == "camera-only"
],
dtype=np.int64,
)
if not len(camera_development_indices):
raise E41MethodologyAuditError("E41 camera-only development slice is empty")
camera_matrix = feature_matrix[camera_development_indices]
presence_labels = [
str(ordered_acceptance[int(index)]["reference"]["presence"])
for index in camera_development_indices
]
rows: list[dict[str, Any]] = []
forbidden_features: list[str] = []
variable_features = 0
source_derived_features = 0
for column, name in enumerate(feature_names):
values = camera_matrix[:, column]
unique_values = int(np.unique(values).size)
variable = unique_values > 1
variable_features += variable
classification = _classify_feature(name, forbidden_feature_tokens)
source_derived_features += classification == "source-derived"
if classification == "forbidden":
forbidden_features.append(name)
development_values = feature_matrix[development_indices, column]
validation_values = feature_matrix[validation_indices, column]
pooled_std = float(np.std(np.concatenate((development_values, validation_values))))
standardized_shift = abs(
float(np.mean(development_values)) - float(np.mean(validation_values))
) / max(pooled_std, 1e-12)
rows.append(
{
"name": name,
"classification": classification,
"camera_development_unique_values": unique_values,
"camera_development_variable": variable,
"camera_development_mean": _rounded_float(float(np.mean(values))),
"camera_development_std": _rounded_float(float(np.std(values))),
"development_validation_standardized_mean_shift": _rounded_float(
standardized_shift
),
"maximum_absolute_presence_correlation": _maximum_label_correlation(
values,
presence_labels,
),
}
)
model_names = e40_model.get("feature_names")
if model_names != feature_names:
raise E41MethodologyAuditError("E41 E40 model feature schema changed")
camera_items = int(e40_model.get("camera_only_training_items", -1))
if camera_items != len(camera_development_indices):
raise E41MethodologyAuditError("E41 E40 training denominator changed")
top_shift = sorted(
(
{
"name": row["name"],
"classification": row["classification"],
"standardized_mean_shift": row[
"development_validation_standardized_mean_shift"
],
}
for row in rows
),
key=lambda row: (-float(row["standardized_mean_shift"]), str(row["name"])),
)[:12]
return {
"feature_dimensions": len(feature_names),
"camera_only_training_items": camera_items,
"camera_only_variable_features": variable_features,
"camera_only_constant_features": len(feature_names) - variable_features,
"camera_items_per_variable_feature": _rounded_float(
camera_items / max(1, variable_features)
),
"source_derived_feature_count": source_derived_features,
"forbidden_features": forbidden_features,
"top_development_validation_shifts": top_shift,
"registry": rows,
}
def _classify_feature(name: str, forbidden_feature_tokens: tuple[str, ...]) -> str:
lowered = name.lower()
if any(token.lower() in lowered for token in forbidden_feature_tokens):
return "forbidden"
if name in _SOURCE_DERIVED_NAMES or name.startswith(_SOURCE_DERIVED_PREFIXES):
return "source-derived"
return "physical-observation"
def _maximum_label_correlation(values: np.ndarray[Any, Any], labels: list[str]) -> float | None:
if len(values) < 2 or float(np.std(values)) <= 1e-12:
return None
maximum = 0.0
for label in sorted(set(labels)):
target = np.asarray([value == label for value in labels], dtype=np.float64)
if float(np.std(target)) <= 1e-12:
continue
correlation = float(np.corrcoef(values, target)[0, 1])
if np.isfinite(correlation):
maximum = max(maximum, abs(correlation))
return _rounded_float(maximum)
def _overlap_summary(development: set[str], validation: set[str]) -> dict[str, Any]:
overlap = sorted(development & validation, key=_natural_key)
return {
"development_unique": len(development),
"validation_unique": len(validation),
"count": len(overlap),
"sample": overlap[:20],
}
def _natural_key(value: str) -> tuple[int, int | str]:
try:
return (0, int(value))
except ValueError:
return (1, value)
def _unique_by_item_id(
rows: list[dict[str, Any]],
label: str,
) -> dict[str, dict[str, Any]]:
indexed: dict[str, dict[str, Any]] = {}
for row in rows:
item_id = row.get("item_id")
if not isinstance(item_id, str) or not item_id or item_id in indexed:
raise E41MethodologyAuditError(f"E41 {label} item identity is invalid")
indexed[item_id] = row
return indexed
def _validate_e40_package(root: Path) -> tuple[dict[str, Any], dict[str, dict[str, Any]]]:
resolved = root.resolve(strict=True)
manifest = _read_json(resolved / "manifest.json")
identity = _object(manifest.get("identity"), "E40 package identity")
identity_sha256 = manifest.get("identity_sha256")
if (
manifest.get("schema_version") != _E40_PACKAGE_SCHEMA
or not isinstance(identity_sha256, str)
or hashlib.sha256(_canonical_json(identity)).hexdigest() != identity_sha256
or manifest.get("package_id") != f"e40-worker-package-{identity_sha256}"
or resolved.name != manifest.get("package_id")
):
raise E41MethodologyAuditError("E41 E40 package identity is invalid")
artifacts = manifest.get("artifacts")
if not isinstance(artifacts, list):
raise E41MethodologyAuditError("E41 E40 package artifact catalog is invalid")
indexed: dict[str, dict[str, Any]] = {}
for row in artifacts:
if (
not isinstance(row, dict)
or not isinstance((relative := row.get("path")), str)
or relative in indexed
or Path(relative).is_absolute()
or ".." in Path(relative).parts
):
raise E41MethodologyAuditError("E41 E40 package artifact is invalid")
path = resolved / relative
if (
not path.is_file()
or path.is_symlink()
or row.get("byte_length") != path.stat().st_size
or row.get("sha256") != _sha256(path)
):
raise E41MethodologyAuditError("E41 E40 package artifact changed")
indexed[relative] = row
return manifest, indexed
def _load_feature_cache(
root: Path,
*,
package_artifacts: dict[str, dict[str, Any]],
) -> tuple[list[str], list[str], np.ndarray[Any, Any], dict[str, Any]]:
manifest_path = root / _FEATURE_CACHE_MANIFEST
arrays_path = root / _FEATURE_CACHE_ARRAYS
relative_manifest = manifest_path.relative_to(root.parents[2]).as_posix()
relative_arrays = arrays_path.relative_to(root.parents[2]).as_posix()
manifest_descriptor = package_artifacts.get(relative_manifest)
arrays_descriptor = package_artifacts.get(relative_arrays)
if (
manifest_descriptor is None
or arrays_descriptor is None
or not manifest_path.is_file()
or not arrays_path.is_file()
):
raise E41MethodologyAuditError("E41 package-bound feature cache is unavailable")
manifest = _read_json(manifest_path)
if (
manifest.get("schema_version") != _FEATURE_CACHE_SCHEMA
or manifest.get("arrays_path") != _FEATURE_CACHE_ARRAYS
or manifest.get("arrays_byte_length") != arrays_path.stat().st_size
or manifest.get("arrays_sha256") != _sha256(arrays_path)
):
raise E41MethodologyAuditError("E41 feature cache binding changed")
with np.load(arrays_path, allow_pickle=False) as arrays:
raw_item_ids = arrays["item_ids"]
feature_matrix = np.asarray(arrays["features"], dtype=np.float64)
item_ids = [str(value) for value in raw_item_ids]
feature_names = _feature_names_from_hash(
expected_hash=str(manifest.get("feature_names_sha256")),
dimensions=int(manifest.get("dimensions", -1)),
)
if (
len(item_ids) != int(manifest.get("items", -1))
or feature_matrix.shape != (len(item_ids), len(feature_names))
or not np.isfinite(feature_matrix).all()
):
raise E41MethodologyAuditError("E41 feature cache arrays are invalid")
return item_ids, feature_names, feature_matrix, {
"manifest_sha256": str(manifest_descriptor["sha256"]),
"arrays_sha256": str(arrays_descriptor["sha256"]),
"feature_names_sha256": str(manifest["feature_names_sha256"]),
"items": len(item_ids),
"dimensions": len(feature_names),
}
def _feature_names_from_hash(*, expected_hash: str, dimensions: int) -> list[str]:
from k1link.compute.e40_perception_product_gate import _feature_names
names = _feature_names()
if (
len(names) != dimensions
or hashlib.sha256(_canonical_json(names)).hexdigest() != expected_hash
):
raise E41MethodologyAuditError("E41 feature-name identity changed")
return names
def _validate_profile(profile: dict[str, Any]) -> None:
source = _object(profile.get("source"), "E41 source")
policy = _object(profile.get("policy"), "E41 policy")
tokens = policy.get("forbidden_feature_tokens")
if (
profile.get("schema_version") != E41_PROFILE_SCHEMA
or profile.get("profile_id") != "e41-ravnoves00-methodology-audit/v1"
or not all(
isinstance(source.get(name), str) and source.get(name)
for name in (
"acceptance_result_id",
"materialization_id",
"e40_package_id",
"e40_result_id",
)
)
or not isinstance(policy.get("time_block_frames"), int)
or int(policy["time_block_frames"]) <= 0
or not isinstance(tokens, list)
or not tokens
or not all(isinstance(token, str) and token for token in tokens)
or policy.get("current_validation_semantics")
!= "historical-evaluated-visible-validation"
or policy.get("independent_truth_required_for_blind") is not True
or policy.get("predictor_truth_separation_required") is not True
or profile.get("authority") != _AUTHORITY
):
raise E41MethodologyAuditError("E41 profile is invalid")
def _validate_artifacts(
root: Path,
value: object,
expected: dict[str, str],
) -> None:
if not isinstance(value, list) or len(value) != len(expected):
raise E41MethodologyAuditError("E41 artifact catalog is invalid")
observed: set[str] = set()
for row in value:
if not isinstance(row, dict):
raise E41MethodologyAuditError("E41 artifact descriptor is invalid")
name = row.get("path")
path = root / str(name)
if (
not isinstance(name, str)
or name in observed
or expected.get(name) != row.get("role")
or not path.is_file()
or path.is_symlink()
or row.get("byte_length") != path.stat().st_size
or row.get("sha256") != _sha256(path)
):
raise E41MethodologyAuditError("E41 artifact content changed")
observed.add(name)
if observed != set(expected):
raise E41MethodologyAuditError("E41 artifact coverage changed")
def _markdown_summary(report: dict[str, Any]) -> str:
analysis = report["analysis"]
split = analysis["split_leakage"]
features = analysis["features"]
violations = analysis["policy"]["violations"]
lines = [
"# E41 methodology audit",
"",
"Current evaluation semantics: `historical-evaluated-visible-validation`.",
"",
"## Decision",
"",
"The current E37/E40 chain is not eligible for a blind or independent product gate.",
"",
"## Measured split overlap",
"",
f"- Exact source frames: {split['exact_source_frames']['count']}",
f"- Track IDs: {split['track_ids']['count']}",
f"- {analysis['denominator']['validation_items']} validation items are already evaluated.",
f"- Time blocks: {split['time_blocks']['count']}",
(
"- Whole-track-or-scene groups: "
f"{split['whole_track_or_scene_groups']['count']}"
),
"",
"## Feature pressure",
"",
f"- Total features: {features['feature_dimensions']}",
f"- Camera-only development items: {features['camera_only_training_items']}",
f"- Variable camera-only features: {features['camera_only_variable_features']}",
(
"- Camera items per variable feature: "
f"{features['camera_items_per_variable_feature']}"
),
"",
"## Blocking methodology violations",
"",
*[f"- `{violation}`" for violation in violations],
"",
"No navigation, command, safety, cross-route, or independent-truth authority is granted.",
"",
]
return "\n".join(lines)
def _strings(value: object) -> Iterable[str]:
if isinstance(value, str):
yield value
elif isinstance(value, dict):
for item in value.values():
yield from _strings(item)
elif isinstance(value, list):
for item in value:
yield from _strings(item)
def _artifact(path: Path, role: str) -> dict[str, Any]:
return {
"path": path.name,
"role": role,
"byte_length": path.stat().st_size,
"sha256": _sha256(path),
}
def _object(value: object, label: str) -> dict[str, Any]:
if not isinstance(value, dict):
raise E41MethodologyAuditError(f"{label} must be an object")
return value
def _read_json(path: Path) -> dict[str, Any]:
value = json.loads(path.read_text(encoding="utf-8-sig"))
if not isinstance(value, dict):
raise E41MethodologyAuditError(f"JSON object expected: {path.name}")
return value
def _read_jsonl(path: Path) -> list[dict[str, Any]]:
rows = [json.loads(line) for line in path.read_text(encoding="utf-8-sig").splitlines()]
if not all(isinstance(row, dict) for row in rows):
raise E41MethodologyAuditError(f"JSONL object expected: {path.name}")
return rows
def _write_json(path: Path, value: object) -> None:
with path.open("x", encoding="utf-8", newline="\n") as stream:
json.dump(value, stream, indent=2, sort_keys=True)
stream.write("\n")
stream.flush()
os.fsync(stream.fileno())
def _write_text(path: Path, value: str) -> None:
with path.open("x", encoding="utf-8", newline="\n") as stream:
stream.write(value)
stream.flush()
os.fsync(stream.fileno())
def _canonical_json(value: object) -> bytes:
return json.dumps(
value,
sort_keys=True,
separators=(",", ":"),
allow_nan=False,
).encode()
def _sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as stream:
while chunk := stream.read(1024 * 1024):
digest.update(chunk)
return digest.hexdigest()
def _rounded_float(value: float) -> float:
return round(value, 9)
def _utc_now() -> str:
return datetime.now(UTC).isoformat(timespec="milliseconds").replace("+00:00", "Z")
src/k1link/compute/e42_metamorphic_suite.py
+490
View File
@@ -0,0 +1,490 @@
"""Bounded E42 metamorphic checks over the frozen predictor and PointSlab.
The suite distinguishes invariance from sensitivity. It proves predictor
independence from item naming, row ordering and chunk boundaries, and verifies
PointSlab row-order invariance plus explicit SE(3) coordinate equivariance. It
does not claim raw-sensor producer invariance or cross-route generalization.
"""
from __future__ import annotations
import hashlib
import json
import os
import shutil
import uuid
from dataclasses import dataclass
from datetime import UTC, datetime
from pathlib import Path
from typing import Any, Final
import numpy as np
from k1link.compute.e32_track_geometry_replay import (
e32_track_geometry_frame,
read_e32_track_geometry_replay,
)
from k1link.compute.e41_evaluation_boundary import (
E41_FEATURE_ARRAYS_NAME,
E41_FEATURE_MANIFEST_NAME,
E41_ITEMS_NAME,
E41_MODEL_NAME,
predict_from_frozen_e40_model,
validate_e41_predictor_package,
)
from k1link.compute.track_geometry import PointSlab
E42_RESULT_SCHEMA: Final = "missioncore.e42-metamorphic-suite/v1"
E42_REPORT_SCHEMA: Final = "missioncore.e42-metamorphic-report/v1"
E42_REPORT_NAME: Final = "metamorphic-report.json"
E42_MANIFEST_NAME: Final = "manifest.json"
_AUTHORITY: Final = {
"commands_enabled": False,
"navigation_or_safety_accepted": False,
}
class E42MetamorphicSuiteError(RuntimeError):
"""An E42 source, metamorphic check, or immutable result is invalid."""
@dataclass(frozen=True, slots=True)
class E42MetamorphicSuite:
result_id: str
result_root: Path
manifest: dict[str, Any]
report: dict[str, Any]
@property
def accepted(self) -> bool:
return self.report.get("acceptance", {}).get("accepted") is True
def build_e42_metamorphic_suite(
*,
predictor_package_root: Path,
e32_result_root: Path,
output_root: Path,
) -> E42MetamorphicSuite:
"""Run and seal one bounded E42 metamorphic suite."""
predictor_package = validate_e41_predictor_package(predictor_package_root)
package_root = predictor_package_root.resolve(strict=True)
e32 = read_e32_track_geometry_replay(e32_result_root)
items = _read_jsonl(package_root / E41_ITEMS_NAME)
model = _read_json(package_root / E41_MODEL_NAME)
feature_manifest = _read_json(package_root / E41_FEATURE_MANIFEST_NAME)
feature_names = [str(value) for value in feature_manifest["feature_names"]]
with np.load(package_root / E41_FEATURE_ARRAYS_NAME, allow_pickle=False) as arrays:
feature_matrix = np.asarray(arrays["features"], dtype=np.float64)
predictor_checks = _predictor_metamorphics(
items=items,
feature_names=feature_names,
feature_matrix=feature_matrix,
model=model,
)
point_slab_checks = _point_slab_metamorphics(e32)
checks = {
**predictor_checks,
**point_slab_checks,
}
accepted = all(checks.values())
if not accepted:
raise E42MetamorphicSuiteError("E42 metamorphic invariant failed")
analysis = {
"classification": "bounded-current-source-contract-metamorphics",
"checks": checks,
"predictor": {
"items": len(items),
"feature_dimensions": len(feature_names),
"item_identity_used_as_model_input": False,
"path_used_as_model_input": False,
"absolute_time_origin_used_as_model_input": False,
"track_identity_used_as_model_input": False,
},
"point_slab": point_slab_checks["point_slab_details"],
"sensitivity_not_invariance": {
"timestamp_offset": "covered-by-e31-offset-sweep-not-claimed-invariant",
"calibration_perturbation": "must-fail-binding-or-change-projection",
"density_thinning": "must-reduce-evidence-never-create-free-space",
},
"limitations": [
(
"the suite proves the frozen E41 predictor boundary and one real "
"E32 PointSlab contract, not the complete raw camera/LiDAR producer"
),
"no claim is made for another route, weather condition, rig, mount, or camera",
"SE(3) is a coordinate-contract check, not a new model-quality result",
],
"authority": _AUTHORITY,
}
analysis_sha256 = hashlib.sha256(_canonical_json(analysis)).hexdigest()
identity = {
"schema_version": E42_RESULT_SCHEMA,
"predictor_package_id": predictor_package["package_id"],
"predictor_package_identity_sha256": predictor_package["identity_sha256"],
"e32_result_id": e32.result_id,
"e32_identity_sha256": e32.manifest["identity_sha256"],
"analysis_sha256": analysis_sha256,
"producer_sha256": _sha256(Path(__file__).resolve(strict=True)),
"authority": _AUTHORITY,
}
identity_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest()
result_id = f"e42-metamorphic-suite-{identity_sha256}"
destination = output_root.expanduser().absolute() / result_id
if destination.exists():
return read_e42_metamorphic_suite(destination)
report = {
"schema_version": E42_REPORT_SCHEMA,
"result_id": result_id,
"identity_sha256": identity_sha256,
"status": "accepted-bounded-metamorphic-suite",
"analysis": analysis,
"acceptance": {
"accepted": True,
"checks": checks,
},
"authority": _AUTHORITY,
}
destination.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
staging = destination.parent / f".{result_id}.{uuid.uuid4().hex}.tmp"
staging.mkdir(mode=0o700, exist_ok=False)
try:
_write_json(staging / E42_REPORT_NAME, report)
manifest = {
"schema_version": E42_RESULT_SCHEMA,
"result_id": result_id,
"identity_sha256": identity_sha256,
"identity": identity,
"created_at_utc": _utc_now(),
"acceptance_state": "accepted-bounded-metamorphic-suite",
"artifacts": [
_artifact(staging / E42_REPORT_NAME, "metamorphic-report"),
],
"authority": _AUTHORITY,
}
_write_json(staging / E42_MANIFEST_NAME, manifest)
os.replace(staging, destination)
except BaseException:
shutil.rmtree(staging, ignore_errors=True)
raise
return read_e42_metamorphic_suite(destination)
def read_e42_metamorphic_suite(root: Path) -> E42MetamorphicSuite:
"""Read and validate one immutable E42 result."""
resolved = root.resolve(strict=True)
manifest = _read_json(resolved / E42_MANIFEST_NAME)
identity = _object(manifest.get("identity"), "E42 identity")
identity_sha256 = manifest.get("identity_sha256")
if (
manifest.get("schema_version") != E42_RESULT_SCHEMA
or not isinstance(identity_sha256, str)
or hashlib.sha256(_canonical_json(identity)).hexdigest() != identity_sha256
or manifest.get("result_id") != f"e42-metamorphic-suite-{identity_sha256}"
or resolved.name != manifest.get("result_id")
or manifest.get("acceptance_state") != "accepted-bounded-metamorphic-suite"
or manifest.get("authority") != _AUTHORITY
):
raise E42MetamorphicSuiteError("E42 result identity is invalid")
artifacts = manifest.get("artifacts")
report_path = resolved / E42_REPORT_NAME
if (
not isinstance(artifacts, list)
or len(artifacts) != 1
or not isinstance(artifacts[0], dict)
or artifacts[0].get("path") != E42_REPORT_NAME
or artifacts[0].get("role") != "metamorphic-report"
or artifacts[0].get("byte_length") != report_path.stat().st_size
or artifacts[0].get("sha256") != _sha256(report_path)
):
raise E42MetamorphicSuiteError("E42 artifact content changed")
report = _read_json(report_path)
analysis = _object(report.get("analysis"), "E42 analysis")
checks = _object(
_object(report.get("acceptance"), "E42 acceptance").get("checks"),
"E42 checks",
)
if (
report.get("schema_version") != E42_REPORT_SCHEMA
or report.get("result_id") != resolved.name
or report.get("identity_sha256") != identity_sha256
or not checks
or not all(value is True or isinstance(value, dict) for value in checks.values())
or report.get("acceptance", {}).get("accepted") is not True
or hashlib.sha256(_canonical_json(analysis)).hexdigest()
!= identity.get("analysis_sha256")
or report.get("authority") != _AUTHORITY
):
raise E42MetamorphicSuiteError("E42 report is invalid")
return E42MetamorphicSuite(
result_id=resolved.name,
result_root=resolved,
manifest=manifest,
report=report,
)
def _predictor_metamorphics(
*,
items: list[dict[str, Any]],
feature_names: list[str],
feature_matrix: np.ndarray[Any, Any],
model: dict[str, Any],
) -> dict[str, bool]:
baseline = predict_from_frozen_e40_model(
items=items,
feature_names=feature_names,
feature_matrix=feature_matrix,
model=model,
)
baseline_by_id = _prediction_by_id(baseline)
renamed_items = [
{
**item,
"item_id": f"renamed-{index:06d}",
}
for index, item in enumerate(items)
]
renamed = predict_from_frozen_e40_model(
items=renamed_items,
feature_names=feature_names,
feature_matrix=feature_matrix,
model=model,
)
identity_rename_invariant = _ordered_prediction_semantics(baseline) == (
_ordered_prediction_semantics(renamed)
)
order = np.arange(len(items) - 1, -1, -1, dtype=np.int64)
reordered_items = [
{
**items[int(source_index)],
"sequence": new_index,
}
for new_index, source_index in enumerate(order)
]
reordered = predict_from_frozen_e40_model(
items=reordered_items,
feature_names=feature_names,
feature_matrix=feature_matrix[order],
model=model,
)
row_order_invariant = baseline_by_id == _prediction_by_id(reordered)
chunked: list[dict[str, Any]] = []
chunk_sizes = (1, 7, 31, 97)
offset = 0
chunk_index = 0
while offset < len(items):
size = chunk_sizes[chunk_index % len(chunk_sizes)]
end = min(len(items), offset + size)
chunk_items = [
{
**item,
"sequence": local_index,
}
for local_index, item in enumerate(items[offset:end])
]
chunked.extend(
predict_from_frozen_e40_model(
items=chunk_items,
feature_names=feature_names,
feature_matrix=feature_matrix[offset:end],
model=model,
)
)
offset = end
chunk_index += 1
chunk_boundary_invariant = baseline_by_id == _prediction_by_id(chunked)
allowed_keys = {
"schema_version",
"sequence",
"item_id",
"source_stratum",
}
metadata_minimized = all(set(item) == allowed_keys for item in items)
return {
"predictor_item_identity_rename_invariant": identity_rename_invariant,
"predictor_row_order_invariant": row_order_invariant,
"predictor_chunk_boundary_invariant": chunk_boundary_invariant,
"predictor_path_time_track_metadata_absent": metadata_minimized,
}
def _point_slab_metamorphics(e32: Any) -> dict[str, Any]:
offsets = np.load(
e32.result_root / "frame-point-offsets.npy",
allow_pickle=False,
mmap_mode="r",
)
sizes = np.diff(offsets)
nonempty = np.flatnonzero(sizes > 1)
if not len(nonempty):
raise E42MetamorphicSuiteError("E42 E32 source has no non-empty PointSlab")
frame_index = int(nonempty[0])
frame = e32_track_geometry_frame(e32, frame_index)
slab = frame.point_slab
order = np.arange(slab.row_count - 1, -1, -1, dtype=np.int64)
permuted = PointSlab(
frame_index=slab.frame_index,
source_frame_index=slab.source_frame_index,
source_point_count=slab.source_point_count,
coordinate_frame=slab.coordinate_frame,
owner_keys=slab.owner_keys,
source_indices=slab.source_indices[order],
points_xyz_m=slab.points_xyz_m[order],
owner_indices=slab.owner_indices[order],
)
row_order_invariant = _point_slab_signature(slab) == _point_slab_signature(permuted)
angle = np.deg2rad(90.0)
rotation = np.asarray(
[
[np.cos(angle), -np.sin(angle), 0.0],
[np.sin(angle), np.cos(angle), 0.0],
[0.0, 0.0, 1.0],
],
dtype=np.float64,
)
translation = np.asarray([1.0, -2.0, 0.5], dtype=np.float64)
transformed_points = (
np.asarray(slab.points_xyz_m, dtype=np.float64) @ rotation.T + translation
).astype("<f4")
transformed = PointSlab(
frame_index=slab.frame_index,
source_frame_index=slab.source_frame_index,
source_point_count=slab.source_point_count,
coordinate_frame=slab.coordinate_frame,
owner_keys=slab.owner_keys,
source_indices=slab.source_indices,
points_xyz_m=transformed_points,
owner_indices=slab.owner_indices,
)
expected = (
np.asarray(slab.points_xyz_m, dtype=np.float64) @ rotation.T + translation
)
se3_equivariant = bool(
np.allclose(
np.asarray(transformed.points_xyz_m, dtype=np.float64),
expected,
rtol=0.0,
atol=1e-5,
)
and np.array_equal(transformed.source_indices, slab.source_indices)
and np.array_equal(transformed.owner_indices, slab.owner_indices)
and transformed.owner_keys == slab.owner_keys
)
details = {
"frame_index": frame_index,
"row_count": slab.row_count,
"owner_count": len(slab.owner_keys),
"coordinate_frame": slab.coordinate_frame,
"row_permutation": "reverse",
"se3_rotation": "z-plus-90-degrees",
"se3_translation_m": translation.tolist(),
}
return {
"point_slab_row_order_invariant": row_order_invariant,
"point_slab_se3_coordinate_equivariant": se3_equivariant,
"point_slab_details": details,
}
def _prediction_by_id(rows: list[dict[str, Any]]) -> dict[str, tuple[object, ...]]:
indexed: dict[str, tuple[object, ...]] = {}
for row in rows:
item_id = str(row["item_id"])
if item_id in indexed:
raise E42MetamorphicSuiteError("E42 prediction item identity collided")
indexed[item_id] = (
row["source_stratum"],
row["prediction"],
row["presence_confidence"],
)
return indexed
def _ordered_prediction_semantics(rows: list[dict[str, Any]]) -> list[tuple[object, ...]]:
return [
(
row["source_stratum"],
row["prediction"],
row["presence_confidence"],
)
for row in rows
]
def _point_slab_signature(slab: PointSlab) -> tuple[tuple[object, ...], ...]:
order = np.argsort(slab.source_indices, kind="stable")
return tuple(
(
int(slab.source_indices[index]),
slab.owner_keys[int(slab.owner_indices[index])],
*(round(float(value), 6) for value in slab.points_xyz_m[index]),
)
for index in order
)
def _artifact(path: Path, role: str) -> dict[str, Any]:
return {
"path": path.name,
"role": role,
"byte_length": path.stat().st_size,
"sha256": _sha256(path),
}
def _object(value: object, label: str) -> dict[str, Any]:
if not isinstance(value, dict):
raise E42MetamorphicSuiteError(f"{label} must be an object")
return value
def _read_json(path: Path) -> dict[str, Any]:
value = json.loads(path.read_text(encoding="utf-8-sig"))
if not isinstance(value, dict):
raise E42MetamorphicSuiteError(f"JSON object expected: {path.name}")
return value
def _read_jsonl(path: Path) -> list[dict[str, Any]]:
rows = [json.loads(line) for line in path.read_text(encoding="utf-8-sig").splitlines()]
if not all(isinstance(row, dict) for row in rows):
raise E42MetamorphicSuiteError(f"JSONL object expected: {path.name}")
return rows
def _write_json(path: Path, value: object) -> None:
with path.open("x", encoding="utf-8", newline="\n") as stream:
json.dump(value, stream, indent=2, sort_keys=True)
stream.write("\n")
stream.flush()
os.fsync(stream.fileno())
def _canonical_json(value: object) -> bytes:
return json.dumps(
value,
sort_keys=True,
separators=(",", ":"),
allow_nan=False,
).encode()
def _sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as stream:
while chunk := stream.read(1024 * 1024):
digest.update(chunk)
return digest.hexdigest()
def _utc_now() -> str:
return datetime.now(UTC).isoformat(timespec="milliseconds").replace("+00:00", "Z")
src/k1link/compute/e43_future_capture_protocol.py
+518
View File
@@ -0,0 +1,518 @@
"""Pre-register the next-route capture and independent truth-island protocol."""
from __future__ import annotations
import hashlib
import json
import os
import re
import shutil
import uuid
from dataclasses import dataclass
from datetime import UTC, datetime
from pathlib import Path
from typing import Any, Final, cast
E43_PROFILE_SCHEMA: Final = "missioncore.e43-future-capture-profile/v1"
E43_PROTOCOL_SCHEMA: Final = "missioncore.e43-future-capture-protocol/v1"
E43_CAPTURE_MANIFEST_SCHEMA: Final = "missioncore.future-capture-manifest/v1"
E43_CANDIDATE_SCHEMA: Final = "missioncore.future-truth-candidate/v1"
E43_PROTOCOL_NAME: Final = "future-capture-protocol.json"
E43_MANIFEST_NAME: Final = "manifest.json"
_SHA256 = re.compile(r"^[a-f0-9]{64}$")
_IDENTIFIER = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:/-]{1,159}$")
_AUTHORITY: Final = {
"commands_enabled": False,
"navigation_or_safety_accepted": False,
}
class E43FutureCaptureProtocolError(RuntimeError):
"""An E43 profile, capture manifest, grouped split, or result is invalid."""
@dataclass(frozen=True, slots=True)
class E43FutureCaptureProtocol:
result_id: str
result_root: Path
manifest: dict[str, Any]
protocol: dict[str, Any]
def build_e43_future_capture_protocol(
*,
profile_path: Path,
output_root: Path,
) -> E43FutureCaptureProtocol:
"""Freeze the protocol before a new physical recording exists."""
profile_file = profile_path.resolve(strict=True)
profile = _read_json(profile_file)
_validate_profile(profile)
protocol = {
"schema_version": E43_PROTOCOL_SCHEMA,
"profile_id": profile["profile_id"],
"capture_contract": profile["capture_contract"],
"blind_truth_contract": profile["blind_truth_contract"],
"acceptance_contract": profile["acceptance_contract"],
"decisions_frozen_before_capture": [
"required-streams-and-source-identities",
"control-bridge-and-new-route-segment-policy",
"grouped-blind-partition-algorithm-and-seed",
"independent-human-review-requirement",
"acceptance-thresholds-and-forbidden-authority",
],
"capture_exists": False,
"labels_exist": False,
"authority": _AUTHORITY,
}
protocol_sha256 = hashlib.sha256(_canonical_json(protocol)).hexdigest()
identity = {
"schema_version": E43_PROTOCOL_SCHEMA,
"profile_id": profile["profile_id"],
"profile_sha256": _sha256(profile_file),
"protocol_sha256": protocol_sha256,
"producer_sha256": _sha256(Path(__file__).resolve(strict=True)),
"authority": _AUTHORITY,
}
identity_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest()
result_id = f"e43-future-capture-protocol-{identity_sha256}"
destination = output_root.expanduser().absolute() / result_id
if destination.exists():
return read_e43_future_capture_protocol(destination)
destination.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
staging = destination.parent / f".{result_id}.{uuid.uuid4().hex}.tmp"
staging.mkdir(mode=0o700, exist_ok=False)
document = {
**protocol,
"result_id": result_id,
"identity_sha256": identity_sha256,
}
try:
_write_json(staging / E43_PROTOCOL_NAME, document)
manifest = {
"schema_version": E43_PROTOCOL_SCHEMA,
"result_id": result_id,
"identity_sha256": identity_sha256,
"identity": identity,
"created_at_utc": _utc_now(),
"acceptance_state": "accepted-pre-capture-protocol",
"artifacts": [
_artifact(staging / E43_PROTOCOL_NAME, "future-capture-protocol"),
],
"authority": _AUTHORITY,
}
_write_json(staging / E43_MANIFEST_NAME, manifest)
os.replace(staging, destination)
except BaseException:
shutil.rmtree(staging, ignore_errors=True)
raise
return read_e43_future_capture_protocol(destination)
def read_e43_future_capture_protocol(root: Path) -> E43FutureCaptureProtocol:
"""Read and validate one immutable pre-capture protocol."""
resolved = root.resolve(strict=True)
manifest = _read_json(resolved / E43_MANIFEST_NAME)
identity = _object(manifest.get("identity"), "E43 identity")
identity_sha256 = manifest.get("identity_sha256")
if (
manifest.get("schema_version") != E43_PROTOCOL_SCHEMA
or not isinstance(identity_sha256, str)
or hashlib.sha256(_canonical_json(identity)).hexdigest() != identity_sha256
or manifest.get("result_id") != f"e43-future-capture-protocol-{identity_sha256}"
or resolved.name != manifest.get("result_id")
or manifest.get("acceptance_state") != "accepted-pre-capture-protocol"
or manifest.get("authority") != _AUTHORITY
):
raise E43FutureCaptureProtocolError("E43 protocol identity is invalid")
artifacts = manifest.get("artifacts")
protocol_path = resolved / E43_PROTOCOL_NAME
if (
not isinstance(artifacts, list)
or len(artifacts) != 1
or not isinstance(artifacts[0], dict)
or artifacts[0].get("path") != E43_PROTOCOL_NAME
or artifacts[0].get("role") != "future-capture-protocol"
or artifacts[0].get("byte_length") != protocol_path.stat().st_size
or artifacts[0].get("sha256") != _sha256(protocol_path)
):
raise E43FutureCaptureProtocolError("E43 protocol artifact changed")
protocol = _read_json(protocol_path)
protocol_identity_payload = dict(protocol)
protocol_identity_payload.pop("result_id", None)
protocol_identity_payload.pop("identity_sha256", None)
if (
protocol.get("schema_version") != E43_PROTOCOL_SCHEMA
or protocol.get("result_id") != resolved.name
or protocol.get("identity_sha256") != identity_sha256
or protocol.get("capture_exists") is not False
or protocol.get("labels_exist") is not False
or hashlib.sha256(_canonical_json(protocol_identity_payload)).hexdigest()
!= identity.get("protocol_sha256")
or protocol.get("authority") != _AUTHORITY
):
raise E43FutureCaptureProtocolError("E43 protocol content is invalid")
return E43FutureCaptureProtocol(
result_id=resolved.name,
result_root=resolved,
manifest=manifest,
protocol=protocol,
)
def validate_future_capture_manifest(
manifest: dict[str, Any],
*,
protocol: dict[str, Any],
) -> dict[str, Any]:
"""Fail closed unless a future physical capture satisfies the frozen protocol."""
capture_contract = _object(protocol.get("capture_contract"), "E43 capture contract")
device = _object(manifest.get("device"), "future capture device")
capture = _object(manifest.get("capture"), "future capture facts")
streams = _object(manifest.get("streams"), "future capture streams")
segments = manifest.get("segments")
if (
manifest.get("schema_version") != E43_CAPTURE_MANIFEST_SCHEMA
or not _identifier(manifest.get("source_session_id"))
or not _identifier(manifest.get("source_display_name"))
or manifest.get("operator_authorized") is not True
or manifest.get("authority") != _AUTHORITY
or device.get("model") != capture_contract.get("device_model")
or not all(
_sha256_value(device.get(name))
for name in (
"device_identity_sha256",
"calibration_sha256",
"mount_identity_sha256",
"configuration_sha256",
)
)
or not _identifier(device.get("firmware"))
):
raise E43FutureCaptureProtocolError("future capture identity is invalid")
duration = capture.get("duration_seconds")
monotonic_start = capture.get("monotonic_start_seconds")
monotonic_end = capture.get("monotonic_end_seconds")
if (
not isinstance(duration, int | float)
or isinstance(duration, bool)
or not float(capture_contract["minimum_duration_seconds"])
<= float(duration)
<= float(capture_contract["maximum_duration_seconds"])
or not isinstance(monotonic_start, int | float)
or isinstance(monotonic_start, bool)
or not isinstance(monotonic_end, int | float)
or isinstance(monotonic_end, bool)
or float(monotonic_end) <= float(monotonic_start)
or abs((float(monotonic_end) - float(monotonic_start)) - float(duration)) > 1.0
or not _utc_timestamp(capture.get("started_at_utc"))
or not all(
isinstance(capture.get(name), str) and str(capture[name]).strip()
for name in ("weather", "illumination", "location_class", "operator_notes")
)
):
raise E43FutureCaptureProtocolError("future capture bounds are invalid")
required_streams = capture_contract.get("required_streams")
if not isinstance(required_streams, list) or set(streams) != set(required_streams):
raise E43FutureCaptureProtocolError("future capture stream set changed")
for stream_id, descriptor in streams.items():
stream = _object(descriptor, f"future capture stream {stream_id}")
if (
stream.get("available") is not True
or not isinstance(stream.get("item_count"), int)
or int(stream["item_count"]) <= 0
or not isinstance(stream.get("byte_length"), int)
or int(stream["byte_length"]) <= 0
or not _sha256_value(stream.get("sha256"))
):
raise E43FutureCaptureProtocolError("future capture stream is incomplete")
_validate_segments(
segments,
capture_contract=capture_contract,
monotonic_start=float(monotonic_start),
monotonic_end=float(monotonic_end),
)
segment_rows = cast(list[dict[str, Any]], segments)
return {
"accepted": True,
"source_session_id": manifest["source_session_id"],
"duration_seconds": float(duration),
"required_streams": sorted(streams),
"segments": [str(row["kind"]) for row in segment_rows],
"blind_truth_labels_available": False,
"authority": _AUTHORITY,
}
def assign_grouped_future_partitions(
candidates: list[dict[str, Any]],
*,
seed: str,
blind_fraction: float,
) -> dict[str, str]:
"""Assign connected scene/track/time groups without cross-partition leakage."""
if not seed or not 0.1 <= blind_fraction <= 0.5 or len(candidates) < 2:
raise E43FutureCaptureProtocolError("future grouped split policy is invalid")
indexed: dict[str, dict[str, Any]] = {}
parent: dict[str, str] = {}
for row in candidates:
item_id = row.get("item_id")
if (
row.get("schema_version") != E43_CANDIDATE_SCHEMA
or not isinstance(item_id, str)
or not item_id
or item_id in indexed
or not _identifier(row.get("scene_id"))
or not _identifier(row.get("time_block_id"))
or (
row.get("track_id") is not None
and not _identifier(row.get("track_id"))
)
or row.get("route_segment") not in {"control-bridge", "new-route"}
):
raise E43FutureCaptureProtocolError("future truth candidate is invalid")
indexed[item_id] = row
parent[item_id] = item_id
def find(item_id: str) -> str:
while parent[item_id] != item_id:
parent[item_id] = parent[parent[item_id]]
item_id = parent[item_id]
return item_id
def union(left: str, right: str) -> None:
left_root = find(left)
right_root = find(right)
if left_root != right_root:
parent[max(left_root, right_root)] = min(left_root, right_root)
group_owner: dict[tuple[str, str], str] = {}
for item_id, row in indexed.items():
group_keys = [
("scene", str(row["scene_id"])),
("time", str(row["time_block_id"])),
]
if row.get("track_id") is not None:
group_keys.append(("track", str(row["track_id"])))
for key in group_keys:
prior = group_owner.get(key)
if prior is None:
group_owner[key] = item_id
else:
union(item_id, prior)
components: dict[str, list[str]] = {}
for item_id in indexed:
components.setdefault(find(item_id), []).append(item_id)
if len(components) < 2:
raise E43FutureCaptureProtocolError(
"future candidates collapse into one leakage-connected component"
)
ordered_components = sorted(
(sorted(items) for items in components.values()),
key=lambda items: hashlib.sha256(
f"{seed}:{','.join(items)}".encode()
).hexdigest(),
)
target = round(len(candidates) * blind_fraction)
blind_components: list[list[str]] = []
blind_items = 0
for component in ordered_components:
if blind_components and blind_items >= target:
break
if len(blind_components) + 1 == len(ordered_components):
break
blind_components.append(component)
blind_items += len(component)
if not blind_components:
blind_components.append(ordered_components[0])
blind_ids = {item_id for component in blind_components for item_id in component}
assignments = {
item_id: "blind-truth" if item_id in blind_ids else "visible-diagnostic"
for item_id in indexed
}
if set(assignments.values()) != {"blind-truth", "visible-diagnostic"}:
raise E43FutureCaptureProtocolError("future grouped split is degenerate")
_verify_group_isolation(indexed, assignments)
return assignments
def _verify_group_isolation(
candidates: dict[str, dict[str, Any]],
assignments: dict[str, str],
) -> None:
observed: dict[tuple[str, str], str] = {}
for item_id, row in candidates.items():
partition = assignments[item_id]
groups = [
("scene", str(row["scene_id"])),
("time", str(row["time_block_id"])),
]
if row.get("track_id") is not None:
groups.append(("track", str(row["track_id"])))
for group in groups:
prior = observed.setdefault(group, partition)
if prior != partition:
raise E43FutureCaptureProtocolError(
"future grouped split leaks across partitions"
)
def _validate_segments(
value: object,
*,
capture_contract: dict[str, Any],
monotonic_start: float,
monotonic_end: float,
) -> None:
if not isinstance(value, list) or len(value) < 2:
raise E43FutureCaptureProtocolError("future capture segments are missing")
required = {
str(row["kind"]): float(row["minimum_duration_seconds"])
for row in capture_contract["required_segments"]
}
observed: dict[str, float] = {}
intervals: list[tuple[float, float]] = []
for row in value:
segment = _object(row, "future capture segment")
kind = segment.get("kind")
start = segment.get("monotonic_start_seconds")
end = segment.get("monotonic_end_seconds")
if (
kind not in required
or kind in observed
or not isinstance(start, int | float)
or isinstance(start, bool)
or not isinstance(end, int | float)
or isinstance(end, bool)
or not monotonic_start <= float(start) < float(end) <= monotonic_end
or float(end) - float(start) < required[str(kind)]
):
raise E43FutureCaptureProtocolError("future capture segment is invalid")
observed[str(kind)] = float(end) - float(start)
intervals.append((float(start), float(end)))
if set(observed) != set(required):
raise E43FutureCaptureProtocolError("future capture segment set changed")
intervals.sort()
if any(
left[1] > right[0]
for left, right in zip(intervals, intervals[1:], strict=False)
):
raise E43FutureCaptureProtocolError("future capture segments overlap")
def _validate_profile(profile: dict[str, Any]) -> None:
capture = _object(profile.get("capture_contract"), "E43 capture contract")
blind = _object(profile.get("blind_truth_contract"), "E43 blind truth contract")
acceptance = _object(profile.get("acceptance_contract"), "E43 acceptance contract")
required_segments = capture.get("required_segments")
required_streams = capture.get("required_streams")
if (
profile.get("schema_version") != E43_PROFILE_SCHEMA
or profile.get("profile_id") != "e43-same-k1-new-route-truth-island/v1"
or capture.get("device_model") != "XGRIDS/LixelKity-K1"
or not isinstance(capture.get("minimum_duration_seconds"), int)
or not isinstance(capture.get("maximum_duration_seconds"), int)
or int(capture["minimum_duration_seconds"])
>= int(capture["maximum_duration_seconds"])
or not isinstance(required_streams, list)
or len(required_streams) < 4
or len(required_streams) != len(set(required_streams))
or not all(isinstance(value, str) and value for value in required_streams)
or not isinstance(required_segments, list)
or {row.get("kind") for row in required_segments if isinstance(row, dict)}
!= {"control-bridge", "new-route"}
or blind.get("partition_strategy")
!= "connected-scene-track-time-components/v1"
or not isinstance(blind.get("seed"), str)
or not 0.1 <= float(blind.get("blind_fraction", 0.0)) <= 0.5
or blind.get("independent_human_reviewers") != 2
or blind.get("engineering_acceptance_labels_are_truth") is not False
or acceptance.get("accounting_target") != 1.0
or acceptance.get("maximum_false_free_claims") != 0
or acceptance.get("maximum_high_severity_failures") != 0
or not all(
float(acceptance.get(name, 0.0)) == 0.9
for name in (
"presence_target",
"geometry_association_target",
"freshness_target",
)
)
or profile.get("authority") != _AUTHORITY
):
raise E43FutureCaptureProtocolError("E43 profile is invalid")
def _identifier(value: object) -> bool:
return isinstance(value, str) and _IDENTIFIER.fullmatch(value) is not None
def _sha256_value(value: object) -> bool:
return isinstance(value, str) and _SHA256.fullmatch(value) is not None
def _utc_timestamp(value: object) -> bool:
if not isinstance(value, str) or not value.endswith("Z"):
return False
try:
datetime.fromisoformat(value.replace("Z", "+00:00"))
except ValueError:
return False
return True
def _artifact(path: Path, role: str) -> dict[str, Any]:
return {
"path": path.name,
"role": role,
"byte_length": path.stat().st_size,
"sha256": _sha256(path),
}
def _object(value: object, label: str) -> dict[str, Any]:
if not isinstance(value, dict):
raise E43FutureCaptureProtocolError(f"{label} must be an object")
return value
def _read_json(path: Path) -> dict[str, Any]:
value = json.loads(path.read_text(encoding="utf-8-sig"))
if not isinstance(value, dict):
raise E43FutureCaptureProtocolError(f"JSON object expected: {path.name}")
return value
def _write_json(path: Path, value: object) -> None:
with path.open("x", encoding="utf-8", newline="\n") as stream:
json.dump(value, stream, indent=2, sort_keys=True)
stream.write("\n")
stream.flush()
os.fsync(stream.fileno())
def _canonical_json(value: object) -> bytes:
return json.dumps(
value,
sort_keys=True,
separators=(",", ":"),
allow_nan=False,
).encode()
def _sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as stream:
while chunk := stream.read(1024 * 1024):
digest.update(chunk)
return digest.hexdigest()
def _utc_now() -> str:
return datetime.now(UTC).isoformat(timespec="milliseconds").replace("+00:00", "Z")
src/k1link/compute/e44_data_amplification_audit.py
+383
View File
@@ -0,0 +1,383 @@
"""Content-based E44 data-amplification audit for immutable LAB artifacts."""
from __future__ import annotations
import hashlib
import json
import os
import re
import shutil
import uuid
from collections import Counter, defaultdict
from dataclasses import dataclass
from datetime import UTC, datetime
from pathlib import Path
from typing import Any, Final
E44_RESULT_SCHEMA: Final = "missioncore.e44-data-amplification-audit/v1"
E44_REPORT_SCHEMA: Final = "missioncore.e44-data-amplification-report/v1"
E44_REPORT_NAME: Final = "data-amplification-report.json"
E44_MANIFEST_NAME: Final = "manifest.json"
_LABEL = re.compile(r"^[a-z0-9][a-z0-9._-]{1,63}$")
_AUTHORITY: Final = {
"commands_enabled": False,
"navigation_or_safety_accepted": False,
}
class E44DataAmplificationAuditError(RuntimeError):
"""An E44 source catalog, measurement, or immutable result is invalid."""
@dataclass(frozen=True, slots=True)
class E44DataAmplificationAudit:
result_id: str
result_root: Path
manifest: dict[str, Any]
report: dict[str, Any]
def build_e44_data_amplification_audit(
*,
artifact_roots: dict[str, Path],
output_root: Path,
) -> E44DataAmplificationAudit:
"""Measure logical bytes and exact content duplication across named roots."""
if len(artifact_roots) < 2:
raise E44DataAmplificationAuditError("E44 requires at least two artifact roots")
resolved_output = output_root.expanduser().absolute()
resolved_roots: dict[str, Path] = {}
for label, root in artifact_roots.items():
if _LABEL.fullmatch(label) is None or label in resolved_roots:
raise E44DataAmplificationAuditError("E44 artifact label is invalid")
resolved = root.resolve(strict=True)
if not resolved.is_dir() or resolved.is_symlink():
raise E44DataAmplificationAuditError("E44 artifact root is invalid")
if resolved_output == resolved or resolved_output.is_relative_to(resolved):
raise E44DataAmplificationAuditError("E44 output cannot be inside an input root")
resolved_roots[label] = resolved
files: list[dict[str, Any]] = []
for label, root in sorted(resolved_roots.items()):
for path in sorted(root.rglob("*")):
if path.is_symlink():
raise E44DataAmplificationAuditError("E44 input contains a symlink")
if not path.is_file():
continue
relative = path.relative_to(root).as_posix()
files.append(
{
"root": label,
"path": relative,
"byte_length": path.stat().st_size,
"sha256": _sha256(path),
"kind": _artifact_kind(path),
}
)
if not files:
raise E44DataAmplificationAuditError("E44 artifact roots are empty")
analysis = analyze_data_amplification(files)
catalog_identity = [
{
"label": label,
"root_name": root.name,
"catalog_sha256": hashlib.sha256(
_canonical_json(
[
row
for row in files
if row["root"] == label
]
)
).hexdigest(),
}
for label, root in sorted(resolved_roots.items())
]
analysis_sha256 = hashlib.sha256(_canonical_json(analysis)).hexdigest()
identity = {
"schema_version": E44_RESULT_SCHEMA,
"artifact_roots": catalog_identity,
"analysis_sha256": analysis_sha256,
"producer_sha256": _sha256(Path(__file__).resolve(strict=True)),
"authority": _AUTHORITY,
}
identity_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest()
result_id = f"e44-data-amplification-{identity_sha256}"
destination = resolved_output / result_id
if destination.exists():
return read_e44_data_amplification_audit(destination)
report = {
"schema_version": E44_REPORT_SCHEMA,
"result_id": result_id,
"identity_sha256": identity_sha256,
"status": "completed-content-amplification-measurement",
"analysis": analysis,
"decision": {
"storage_migration_authorized": False,
"measurement_complete": True,
"next_gate": (
"select deduplication/chunking only from measured dominant duplicate classes"
),
},
"limitations": [
"filesystem allocation, compression ratio and browser heap are not inferred from bytes",
"identical content is detected only by exact SHA-256 equality",
"no MCAP, COPC, PDAL, Rerun or storage migration is authorized by this report",
],
"authority": _AUTHORITY,
}
destination.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
staging = destination.parent / f".{result_id}.{uuid.uuid4().hex}.tmp"
staging.mkdir(mode=0o700, exist_ok=False)
try:
_write_json(staging / E44_REPORT_NAME, report)
manifest = {
"schema_version": E44_RESULT_SCHEMA,
"result_id": result_id,
"identity_sha256": identity_sha256,
"identity": identity,
"created_at_utc": _utc_now(),
"acceptance_state": "accepted-measurement-only",
"artifacts": [
_artifact(staging / E44_REPORT_NAME, "data-amplification-report"),
],
"authority": _AUTHORITY,
}
_write_json(staging / E44_MANIFEST_NAME, manifest)
os.replace(staging, destination)
except BaseException:
shutil.rmtree(staging, ignore_errors=True)
raise
return read_e44_data_amplification_audit(destination)
def read_e44_data_amplification_audit(root: Path) -> E44DataAmplificationAudit:
"""Read and validate one immutable E44 result."""
resolved = root.resolve(strict=True)
manifest = _read_json(resolved / E44_MANIFEST_NAME)
identity = _object(manifest.get("identity"), "E44 identity")
identity_sha256 = manifest.get("identity_sha256")
if (
manifest.get("schema_version") != E44_RESULT_SCHEMA
or not isinstance(identity_sha256, str)
or hashlib.sha256(_canonical_json(identity)).hexdigest() != identity_sha256
or manifest.get("result_id") != f"e44-data-amplification-{identity_sha256}"
or resolved.name != manifest.get("result_id")
or manifest.get("acceptance_state") != "accepted-measurement-only"
or manifest.get("authority") != _AUTHORITY
):
raise E44DataAmplificationAuditError("E44 result identity is invalid")
artifacts = manifest.get("artifacts")
report_path = resolved / E44_REPORT_NAME
if (
not isinstance(artifacts, list)
or len(artifacts) != 1
or not isinstance(artifacts[0], dict)
or artifacts[0].get("path") != E44_REPORT_NAME
or artifacts[0].get("role") != "data-amplification-report"
or artifacts[0].get("byte_length") != report_path.stat().st_size
or artifacts[0].get("sha256") != _sha256(report_path)
):
raise E44DataAmplificationAuditError("E44 artifact content changed")
report = _read_json(report_path)
analysis = _object(report.get("analysis"), "E44 analysis")
if (
report.get("schema_version") != E44_REPORT_SCHEMA
or report.get("result_id") != resolved.name
or report.get("identity_sha256") != identity_sha256
or report.get("decision", {}).get("storage_migration_authorized") is not False
or hashlib.sha256(_canonical_json(analysis)).hexdigest()
!= identity.get("analysis_sha256")
or report.get("authority") != _AUTHORITY
):
raise E44DataAmplificationAuditError("E44 report is invalid")
return E44DataAmplificationAudit(
result_id=resolved.name,
result_root=resolved,
manifest=manifest,
report=report,
)
def analyze_data_amplification(files: list[dict[str, Any]]) -> dict[str, Any]:
"""Return exact content-amplification metrics for normalized file rows."""
if not files:
raise E44DataAmplificationAuditError("E44 file catalog is empty")
by_digest: dict[str, list[dict[str, Any]]] = defaultdict(list)
root_totals: Counter[str] = Counter()
root_files: Counter[str] = Counter()
kind_totals: Counter[str] = Counter()
kind_files: Counter[str] = Counter()
seen_root_paths: set[tuple[str, str]] = set()
for row in files:
root = row.get("root")
path = row.get("path")
byte_length = row.get("byte_length")
sha256 = row.get("sha256")
kind = row.get("kind")
if (
not isinstance(root, str)
or _LABEL.fullmatch(root) is None
or not isinstance(path, str)
or not path
or (root, path) in seen_root_paths
or not isinstance(byte_length, int)
or byte_length < 0
or not isinstance(sha256, str)
or len(sha256) != 64
or not isinstance(kind, str)
or not kind
):
raise E44DataAmplificationAuditError("E44 file row is invalid")
seen_root_paths.add((root, path))
by_digest[sha256].append(row)
root_totals[root] += byte_length
root_files[root] += 1
kind_totals[kind] += byte_length
kind_files[kind] += 1
for digest, rows in by_digest.items():
sizes = {int(row["byte_length"]) for row in rows}
if len(sizes) != 1:
raise E44DataAmplificationAuditError(
f"E44 digest {digest} has inconsistent byte lengths"
)
logical_bytes = sum(root_totals.values())
unique_content_bytes = sum(int(rows[0]["byte_length"]) for rows in by_digest.values())
duplicate_bytes = logical_bytes - unique_content_bytes
duplicate_groups = [
{
"sha256": digest,
"byte_length": int(rows[0]["byte_length"]),
"copies": len(rows),
"roots": sorted({str(row["root"]) for row in rows}),
"paths": [
f"{row['root']}:{row['path']}"
for row in sorted(rows, key=lambda item: (item["root"], item["path"]))[:12]
],
"avoidable_duplicate_bytes": int(rows[0]["byte_length"]) * (len(rows) - 1),
"kind": str(rows[0]["kind"]),
}
for digest, rows in by_digest.items()
if len(rows) > 1
]
duplicate_groups.sort(key=_duplicate_sort_key)
roots = {}
for root in sorted(root_totals):
root_rows = [row for row in files if row["root"] == root]
root_unique = {
str(row["sha256"]): int(row["byte_length"])
for row in root_rows
}
roots[root] = {
"files": root_files[root],
"logical_bytes": root_totals[root],
"unique_content_bytes_within_root": sum(root_unique.values()),
"duplicate_bytes_within_root": (
root_totals[root] - sum(root_unique.values())
),
}
return {
"root_count": len(roots),
"file_count": len(files),
"logical_bytes": logical_bytes,
"unique_content_bytes": unique_content_bytes,
"duplicate_bytes": duplicate_bytes,
"amplification_ratio": round(
logical_bytes / max(1, unique_content_bytes),
6,
),
"duplicate_fraction": round(
duplicate_bytes / max(1, logical_bytes),
6,
),
"duplicate_content_groups": len(duplicate_groups),
"roots": roots,
"by_kind": {
kind: {
"files": kind_files[kind],
"logical_bytes": kind_totals[kind],
}
for kind in sorted(kind_totals)
},
"largest_duplicate_groups": duplicate_groups[:30],
}
def _artifact_kind(path: Path) -> str:
suffix = path.suffix.lower()
if suffix in {".jpg", ".jpeg", ".png", ".webp"}:
return "camera-image"
if suffix in {".npy", ".npz", ".las", ".laz", ".pcd", ".ply"}:
return "point-or-array"
if suffix in {".mp4", ".mkv", ".mov"}:
return "video"
if suffix == ".rrd":
return "rerun"
if suffix in {".json", ".jsonl", ".md", ".txt", ".yaml", ".yml"}:
return "metadata-or-report"
if suffix in {".py", ".ps1", ".sh"}:
return "runtime-source"
return "other"
def _duplicate_sort_key(row: dict[str, Any]) -> tuple[int, str]:
return (
-int(row["avoidable_duplicate_bytes"]),
str(row["sha256"]),
)
def _artifact(path: Path, role: str) -> dict[str, Any]:
return {
"path": path.name,
"role": role,
"byte_length": path.stat().st_size,
"sha256": _sha256(path),
}
def _object(value: object, label: str) -> dict[str, Any]:
if not isinstance(value, dict):
raise E44DataAmplificationAuditError(f"{label} must be an object")
return value
def _read_json(path: Path) -> dict[str, Any]:
value = json.loads(path.read_text(encoding="utf-8-sig"))
if not isinstance(value, dict):
raise E44DataAmplificationAuditError(f"JSON object expected: {path.name}")
return value
def _write_json(path: Path, value: object) -> None:
with path.open("x", encoding="utf-8", newline="\n") as stream:
json.dump(value, stream, indent=2, sort_keys=True)
stream.write("\n")
stream.flush()
os.fsync(stream.fileno())
def _canonical_json(value: object) -> bytes:
return json.dumps(
value,
sort_keys=True,
separators=(",", ":"),
allow_nan=False,
).encode()
def _sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as stream:
while chunk := stream.read(1024 * 1024):
digest.update(chunk)
return digest.hexdigest()
def _utc_now() -> str:
return datetime.now(UTC).isoformat(timespec="milliseconds").replace("+00:00", "Z")
src/k1link/compute/inline_temporal.py
+24 -17
View File
@@ -237,7 +237,7 @@ class TemporalStabilizer:
state.score = float(source["score"])
state.template = json.loads(json.dumps(source))
normalized = json.loads(json.dumps(source))
normalized: dict[str, Any] = json.loads(json.dumps(source))
source_id = int(source["track_id"])
normalized["track_id"] = state.canonical_id
normalized["temporal_source_track_id"] = source_id
@@ -371,9 +371,9 @@ class StreamingSemanticStabilizer:
self.minimum_same_label_neighbors = int(profile["semantic"]["minimum_same_label_neighbors"])
self.previous_raw: np.ndarray | None = None
self.previous_stabilized: np.ndarray | None = None
self.baseline_unsupported = deque(maxlen=4096)
self.stabilized_unsupported = deque(maxlen=4096)
self.processing_ms = deque(maxlen=4096)
self.baseline_unsupported: deque[float] = deque(maxlen=4096)
self.stabilized_unsupported: deque[float] = deque(maxlen=4096)
self.processing_ms: deque[float] = deque(maxlen=4096)
self.frames = 0
def update(self, mask: np.ndarray) -> np.ndarray:
@@ -436,21 +436,28 @@ def read_inline_profile(path: Path) -> tuple[dict[str, Any], str]:
bounds = profile.get("bounds")
acceptance = profile.get("acceptance")
authority = profile.get("authority")
if not all(
isinstance(value, dict)
for value in (
source,
tracking,
cuboids,
semantic,
bounds,
acceptance,
)
):
raise RuntimeError("LAB E23 inline temporal profile is invalid")
assert isinstance(source, dict)
assert isinstance(tracking, dict)
assert isinstance(cuboids, dict)
assert isinstance(semantic, dict)
assert isinstance(bounds, dict)
assert isinstance(acceptance, dict)
if (
profile.get("schema_version") != PROFILE_SCHEMA
or profile.get("mode") != "inline-shadow-qualification"
or profile.get("stage") != "warm-worker-after-fusion-before-result-publication"
or not all(
isinstance(value, dict)
for value in (
source,
tracking,
cuboids,
semantic,
bounds,
acceptance,
)
)
or source.get("source_id") != "sensor.camera.right"
or source.get("resolution") != [800, 600]
or source.get("calibration_slot") != "camera_1"
@@ -493,7 +500,7 @@ def stabilize_world_state(
fusion_objects: list[dict[str, Any]],
memory: dict[int, dict[str, Any]],
) -> dict[str, Any]:
world = json.loads(json.dumps(source))
world: dict[str, Any] = json.loads(json.dumps(source))
source_objects = {
int(item["track_id"]): item
for item in source.get("objects", [])
@@ -507,7 +514,7 @@ def stabilize_world_state(
canonical = int(fusion["track_id"])
source_id = int(fusion.get("temporal_source_track_id", canonical))
template = source_objects.get(source_id) or memory.get(canonical) or {}
item = json.loads(json.dumps(template))
item: dict[str, Any] = json.loads(json.dumps(template))
item.update(
{
"track_id": canonical,
src/k1link/compute/lab_instances.py
+5 -3
View File
@@ -91,9 +91,11 @@ def _laboratory_method(
profile_sha256: str | None,
source_result_id: str,
) -> dict[str, object]:
source_identity: str | None = source_result_id.rsplit("-", 1)[-1]
if len(source_identity) != 64 or any(
character not in "0123456789abcdef" for character in source_identity
source_identity_candidate = source_result_id.rsplit("-", 1)[-1]
source_identity: str | None = source_identity_candidate
if len(source_identity_candidate) != 64 or any(
character not in "0123456789abcdef"
for character in source_identity_candidate
):
source_identity = None
return {
src/k1link/compute/lidar_ground.py
+16 -6
View File
@@ -15,11 +15,7 @@ import numpy as np
import numpy.typing as npt
from k1link.ground_segmentation import (
DEFAULT_GROUND_BENCHMARK_PROFILE,
GroundBenchmarkProfile,
GroundSegmentation,
GroundSegmenter,
LocalPercentileGroundSegmenter,
DEFAULT_GROUND_BENCHMARK_PROFILE as DEFAULT_GROUND_BENCHMARK_PROFILE,
)
from k1link.ground_segmentation import (
PATCHWORKPP_SOURCE_COMMIT as PATCHWORKPP_SOURCE_COMMIT,
@@ -31,7 +27,19 @@ from k1link.ground_segmentation import (
PATCHWORKPP_SOURCE_URL as PATCHWORKPP_SOURCE_URL,
)
from k1link.ground_segmentation import (
GroundSegmentationError as LidarGroundError,
GroundBenchmarkProfile as GroundBenchmarkProfile,
)
from k1link.ground_segmentation import (
GroundSegmentation as GroundSegmentation,
)
from k1link.ground_segmentation import (
GroundSegmentationError,
)
from k1link.ground_segmentation import (
GroundSegmenter as GroundSegmenter,
)
from k1link.ground_segmentation import (
LocalPercentileGroundSegmenter as LocalPercentileGroundSegmenter,
)
from k1link.ground_segmentation import (
PatchworkPPGroundSegmenter as PatchworkPPGroundSegmenter,
@@ -40,6 +48,8 @@ from k1link.ground_segmentation import (
from .lidar_contract import LidarContractError, sensor_frame_xyzi
from .lidar_replay import LidarReplayPackV2
LidarGroundError = GroundSegmentationError
LIDAR_GROUND_BENCHMARK_SCHEMA: Final = "missioncore.lidar-ground-benchmark/v1"
LIDAR_GROUND_BENCHMARK_REPORT_SCHEMA: Final = "missioncore.lidar-ground-benchmark-report/v1"
LIDAR_GROUND_ANNOTATION_TEMPLATE_SCHEMA: Final = "missioncore.lidar-ground-annotation-template/v1"
src/k1link/compute/pipeline_telemetry.py
+377
View File
@@ -0,0 +1,377 @@
"""Native, transport-independent telemetry for Mission Core compute stages.
The compute layer owns the meaning of a stage event. Transport ownership stays
outside the stage implementation: a durable worker can inject an already-connected
MQTT client, while a laboratory runner can record the exact same documents to JSONL.
No sink is created implicitly and telemetry never grants command authority.
"""
from __future__ import annotations
import json
import os
import re
import threading
import time
from collections.abc import Iterator
from contextlib import contextmanager
from dataclasses import dataclass
from datetime import UTC, datetime
from pathlib import Path
from typing import Any, Final, Protocol
PIPELINE_TELEMETRY_SCHEMA: Final = "missioncore.agent-pipeline-telemetry/v1"
PIPELINE_TELEMETRY_RECORD_SCHEMA: Final = "missioncore.pipeline-telemetry-record/v1"
PIPELINE_TOPIC_TEMPLATE: Final = (
"mission-core/v1/contours/{contour_id}/agents/{agent_id}/pipeline"
)
SAFE_TOPIC_IDENTIFIER: Final = re.compile(
r"^[a-z0-9](?:[a-z0-9-]{0,62}[a-z0-9])?$"
)
MAX_TEXT_LENGTH: Final = 256
MAX_PAYLOAD_BYTES: Final = 1024 * 1024
STAGE_STATES: Final = frozenset({"started", "completed", "failed"})
_AUTHORITY: Final = {
"commands_enabled": False,
"navigation_or_safety_accepted": False,
}
class PipelineTelemetryError(RuntimeError):
"""A pipeline telemetry identity, event, or transport operation is invalid."""
class PipelineTelemetrySink(Protocol):
"""Transport boundary used by a compute-stage telemetry emitter."""
def publish(self, topic: str, payload: bytes) -> None:
"""Publish one already-validated telemetry document."""
class ConnectedMqttClient(Protocol):
"""Minimal surface required from an already-connected MQTT client."""
def publish(
self,
topic: str,
payload: bytes,
qos: int,
retain: bool,
) -> Any:
"""Publish one MQTT message and return an object exposing ``rc``."""
@dataclass(frozen=True, slots=True)
class PipelineTelemetryIdentity:
contour_id: str
agent_id: str
node_id: str
lab_id: str
run_id: str
source_id: str
source_package_id: str
method_id: str
request_id: str | None = None
frame_index: int | None = None
def __post_init__(self) -> None:
for name in ("contour_id", "agent_id"):
value = getattr(self, name)
if SAFE_TOPIC_IDENTIFIER.fullmatch(value) is None:
raise PipelineTelemetryError(f"{name} is not a safe topic identifier")
for name in (
"node_id",
"lab_id",
"run_id",
"source_id",
"source_package_id",
"method_id",
):
_validate_text(getattr(self, name), name)
if self.request_id is not None:
_validate_text(self.request_id, "request_id")
if self.frame_index is not None and self.frame_index < 0:
raise PipelineTelemetryError("frame_index must be non-negative")
@property
def topic(self) -> str:
return PIPELINE_TOPIC_TEMPLATE.format(
contour_id=self.contour_id,
agent_id=self.agent_id,
)
@dataclass(slots=True)
class PipelineStageOutcome:
"""Mutable counters a stage can complete before its terminal event is emitted."""
input_count: int | None = None
output_count: int | None = None
queue_wait_ms: float | None = None
class PipelineTelemetryEmitter:
"""Emit bounded lifecycle events around actual compute work."""
def __init__(
self,
*,
identity: PipelineTelemetryIdentity,
sink: PipelineTelemetrySink,
clock_ns: Any = time.monotonic_ns,
) -> None:
self.identity = identity
self.sink = sink
self._clock_ns = clock_ns
@contextmanager
def stage(
self,
stage_id: str,
*,
input_count: int | None = None,
queue_wait_ms: float | None = None,
) -> Iterator[PipelineStageOutcome]:
"""Publish a lifecycle pair and preserve the stage exception unchanged."""
_validate_text(stage_id, "stage_id")
outcome = PipelineStageOutcome(
input_count=_optional_count(input_count, "input_count"),
queue_wait_ms=_optional_duration(queue_wait_ms, "queue_wait_ms"),
)
started_ns = int(self._clock_ns())
self._emit(stage_id=stage_id, state="started", outcome=outcome)
try:
yield outcome
except BaseException as exc:
duration_ms = max(0.0, (int(self._clock_ns()) - started_ns) / 1_000_000)
self._emit(
stage_id=stage_id,
state="failed",
outcome=outcome,
duration_ms=duration_ms,
error_type=type(exc).__name__,
)
raise
else:
duration_ms = max(0.0, (int(self._clock_ns()) - started_ns) / 1_000_000)
self._emit(
stage_id=stage_id,
state="completed",
outcome=outcome,
duration_ms=duration_ms,
)
def _emit(
self,
*,
stage_id: str,
state: str,
outcome: PipelineStageOutcome,
duration_ms: float | None = None,
error_type: str | None = None,
) -> None:
outcome.input_count = _optional_count(outcome.input_count, "input_count")
outcome.output_count = _optional_count(outcome.output_count, "output_count")
outcome.queue_wait_ms = _optional_duration(
outcome.queue_wait_ms,
"queue_wait_ms",
)
document = build_pipeline_telemetry_document(
identity=self.identity,
stage_id=stage_id,
state=state,
duration_ms=duration_ms,
input_count=outcome.input_count,
output_count=outcome.output_count,
queue_wait_ms=outcome.queue_wait_ms,
error_type=error_type,
)
payload = _canonical_json(document)
if len(payload) > MAX_PAYLOAD_BYTES:
raise PipelineTelemetryError("pipeline telemetry exceeds the 1 MiB contract")
self.sink.publish(self.identity.topic, payload)
class JsonlPipelineTelemetrySink:
"""Append topic-bound telemetry records for local, auditable execution evidence."""
def __init__(self, path: Path) -> None:
self.path = path.expanduser().absolute()
self._lock = threading.Lock()
def publish(self, topic: str, payload: bytes) -> None:
document = json.loads(payload.decode("utf-8"))
if not isinstance(document, dict):
raise PipelineTelemetryError("pipeline telemetry payload must be an object")
record = {
"schema_version": PIPELINE_TELEMETRY_RECORD_SCHEMA,
"topic": topic,
"payload": document,
}
encoded = _canonical_json(record) + b"\n"
with self._lock:
self.path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
descriptor = os.open(
self.path,
os.O_APPEND | os.O_CREAT | os.O_WRONLY,
0o600,
)
try:
os.write(descriptor, encoded)
os.fsync(descriptor)
finally:
os.close(descriptor)
class MqttPipelineTelemetrySink:
"""Publish through a worker-owned, already-connected Paho-compatible client."""
def __init__(self, client: ConnectedMqttClient) -> None:
self.client = client
def publish(self, topic: str, payload: bytes) -> None:
result = self.client.publish(topic, payload, qos=1, retain=False)
return_code = getattr(result, "rc", None)
if return_code != 0:
raise PipelineTelemetryError(
f"MQTT pipeline telemetry publish failed with code {return_code!r}"
)
def build_pipeline_telemetry_document(
*,
identity: PipelineTelemetryIdentity,
stage_id: str,
state: str,
duration_ms: float | None = None,
input_count: int | None = None,
output_count: int | None = None,
queue_wait_ms: float | None = None,
error_type: str | None = None,
observed_at_utc: str | None = None,
) -> dict[str, Any]:
"""Build the canonical document accepted by the telemetry-plane normalizer."""
_validate_text(stage_id, "stage_id")
if state not in STAGE_STATES:
raise PipelineTelemetryError("stage telemetry state is invalid")
duration_ms = _optional_duration(duration_ms, "duration_ms")
input_count = _optional_count(input_count, "input_count")
output_count = _optional_count(output_count, "output_count")
queue_wait_ms = _optional_duration(queue_wait_ms, "queue_wait_ms")
if state == "started" and duration_ms is not None:
raise PipelineTelemetryError("a started stage cannot have a duration")
if state != "started" and duration_ms is None:
raise PipelineTelemetryError("a terminal stage requires duration_ms")
if error_type is not None:
_validate_text(error_type, "error_type")
if state == "failed" and error_type is None:
raise PipelineTelemetryError("a failed stage requires error_type")
if state != "failed" and error_type is not None:
raise PipelineTelemetryError("only a failed stage can have error_type")
tags = {
"agent_id": identity.agent_id,
"contour_id": identity.contour_id,
"lab_id": identity.lab_id,
"method_id": identity.method_id,
"node_id": identity.node_id,
"run_id": identity.run_id,
"source_id": identity.source_id,
"source_package_id": identity.source_package_id,
"stage_id": stage_id,
"stage_state": state,
}
if identity.request_id is not None:
tags["request_id"] = identity.request_id
stage_metric = {
"elapsed_seconds": (
round(duration_ms / 1000.0, 9) if duration_ms is not None else None
),
"activations": 1,
"input_count": input_count,
"output_count": output_count,
"queue_wait_ms": queue_wait_ms,
}
event = {
"stage_id": stage_id,
"state": state,
"duration_ms": duration_ms,
"input_count": input_count,
"output_count": output_count,
"queue_wait_ms": queue_wait_ms,
"error_type": error_type,
}
document: dict[str, Any] = {
"schema_version": PIPELINE_TELEMETRY_SCHEMA,
"observed_at_utc": observed_at_utc or _utc_now(),
"node_id": identity.node_id,
"lab_id": identity.lab_id,
"run_id": identity.run_id,
"source_id": identity.source_id,
"source_package_id": identity.source_package_id,
"method_id": identity.method_id,
"stage_id": stage_id,
"stage_state": state,
"tags": tags,
"payload": {
"state": (
"busy"
if state == "started"
else ("failed" if state == "failed" else "ready")
),
"current_stage": stage_id,
"active_request_id": identity.request_id or identity.run_id,
"active_stages": [stage_id] if state == "started" else [],
"stage_metrics": {stage_id: stage_metric},
"event": event,
},
"authority": _AUTHORITY,
}
if identity.request_id is not None:
document["request_id"] = identity.request_id
if identity.frame_index is not None:
document["frame_index"] = identity.frame_index
return document
def _validate_text(value: object, name: str) -> str:
if (
not isinstance(value, str)
or not value
or len(value) > MAX_TEXT_LENGTH
or any(ord(character) < 32 for character in value)
):
raise PipelineTelemetryError(f"{name} is invalid")
return value
def _optional_count(value: int | None, name: str) -> int | None:
if value is None:
return None
if isinstance(value, bool) or not isinstance(value, int) or value < 0:
raise PipelineTelemetryError(f"{name} must be a non-negative integer")
return value
def _optional_duration(value: float | None, name: str) -> float | None:
if value is None:
return None
normalized = float(value)
if normalized < 0 or normalized != normalized or normalized == float("inf"):
raise PipelineTelemetryError(f"{name} must be finite and non-negative")
return round(normalized, 6)
def _canonical_json(value: object) -> bytes:
return json.dumps(
value,
ensure_ascii=False,
sort_keys=True,
separators=(",", ":"),
).encode("utf-8")
def _utc_now() -> str:
return datetime.now(UTC).isoformat(timespec="microseconds").replace("+00:00", "Z")
src/k1link/compute/temporal_stability.py
+24 -13
View File
@@ -261,7 +261,7 @@ class TemporalStabilizer:
state.score = float(source["score"])
state.template = json.loads(json.dumps(source))
normalized = json.loads(json.dumps(source))
normalized: dict[str, Any] = json.loads(json.dumps(source))
source_id = int(source["track_id"])
normalized["track_id"] = state.canonical_id
normalized["temporal_source_track_id"] = source_id
@@ -405,13 +405,20 @@ def read_profile(path: Path) -> tuple[dict[str, Any], str]:
bounds = profile.get("bounds")
acceptance = profile.get("acceptance")
authority = profile.get("authority")
if not all(
isinstance(value, dict)
for value in (source, tracking, cuboids, semantic, bounds, acceptance)
):
raise SessionIntegrityError("LAB E22 temporal profile is invalid")
assert isinstance(source, dict)
assert isinstance(tracking, dict)
assert isinstance(cuboids, dict)
assert isinstance(semantic, dict)
assert isinstance(bounds, dict)
assert isinstance(acceptance, dict)
if (
profile.get("schema_version") != PROFILE_SCHEMA
or profile.get("mode") != "recorded-streaming-qualification"
or not all(
isinstance(value, dict)
for value in (source, tracking, cuboids, semantic, bounds, acceptance)
)
or source.get("source_id") != "sensor.camera.right"
or source.get("resolution") != [800, 600]
or semantic.get("mode")
@@ -575,7 +582,7 @@ def build_temporal_stability_result(
semantic_metrics["stabilized_unsupported_change_fraction"]["mean"],
),
}
runtime = {
runtime: dict[str, Any] = {
"camera_frame_processing_ms": _percentiles(frame_ms),
"semantic_frame_processing_ms": _percentiles(semantic_ms),
"peak_track_states": stabilizer.peak_states,
@@ -586,7 +593,7 @@ def build_temporal_stability_result(
+ 600 * 800
),
}
criteria = profile["acceptance"]
criteria: dict[str, Any] = profile["acceptance"]
checks = {
"minimum_2d_acceleration_reduction": reductions[
"tracking_2d_acceleration_p95_fraction"
@@ -791,7 +798,7 @@ def _stabilize_world(
fusion_objects: list[dict[str, Any]],
memory: dict[int, dict[str, Any]],
) -> dict[str, Any]:
world = json.loads(json.dumps(source))
world: dict[str, Any] = json.loads(json.dumps(source))
source_objects = {
int(item["track_id"]): item
for item in source.get("objects", [])
@@ -804,7 +811,7 @@ def _stabilize_world(
canonical = int(fusion["track_id"])
source_id = int(fusion.get("temporal_source_track_id", canonical))
template = source_objects.get(source_id) or memory.get(canonical) or {}
item = json.loads(json.dumps(template))
item: dict[str, Any] = json.loads(json.dumps(template))
item.update(
{
"track_id": canonical,
@@ -861,10 +868,10 @@ def _quality_metrics(
)
acceleration: list[float] = []
size_steps: list[float] = []
for values in tracks.values():
for track_values in tracks.values():
previous_step: np.ndarray | None = None
for (left_frame, left_box), (right_frame, right_box) in zip(
values, values[1:], strict=False
track_values, track_values[1:], strict=False
):
if right_frame - left_frame != 1:
previous_step = None
@@ -882,8 +889,12 @@ def _quality_metrics(
cuboid_size_steps: list[float] = []
yaw_steps: list[float] = []
gaps = 0
for values in cuboids.values():
for left, right in zip(values, values[1:], strict=False):
for cuboid_values in cuboids.values():
for left, right in zip(
cuboid_values,
cuboid_values[1:],
strict=False,
):
if right[0] - left[0] != 1:
gaps += 1
continue
src/k1link/web/advanced_laboratory_api.py
+89
View File
@@ -49,6 +49,11 @@ from k1link.compute.e39_perception_refinement import (
E39PerceptionRefinementError,
read_e39_perception_refinement,
)
from k1link.compute.e40_perception_product_gate import (
E40PerceptionProductGate,
E40PerceptionProductGateError,
read_e40_perception_product_gate,
)
LABORATORY_ADVANCED_CATALOG_SCHEMA: Final = (
"missioncore.laboratory-advanced-catalog/v1"
@@ -62,6 +67,7 @@ _E35_RESULT_ID = re.compile(r"^e35-degradation-recovery-[a-f0-9]{64}$")
_E37_RESULT_ID = re.compile(r"^e37-ravnoves-acceptance-[a-f0-9]{64}$")
_E38_RESULT_ID = re.compile(r"^e38-perception-baseline-[a-f0-9]{64}$")
_E39_RESULT_ID = re.compile(r"^e39-perception-refinement-[a-f0-9]{64}$")
_E40_RESULT_ID = re.compile(r"^e40-perception-product-gate-[a-f0-9]{64}$")
RootProvider = Callable[[], Path | None]
@@ -148,6 +154,15 @@ def _read_e39_cached(
return read_e39_perception_refinement(Path(root_text))
@lru_cache(maxsize=16)
def _read_e40_cached(
root_text: str,
signature: tuple[int, ...],
) -> E40PerceptionProductGate:
del signature
return read_e40_perception_product_gate(Path(root_text))
def _configured_root(provider: RootProvider) -> Path | None:
value = provider()
if value is None:
@@ -659,6 +674,46 @@ def _project_e39(result: E39PerceptionRefinement) -> dict[str, object]:
}
def _project_e40(result: E40PerceptionProductGate) -> dict[str, object]:
identity = _object(result.manifest.get("identity"), "E40 identity")
source = _object(identity.get("source"), "E40 source")
execution = _object(identity.get("execution"), "E40 execution")
profile = _object(identity.get("profile"), "E40 profile")
metrics = _object(result.report.get("metrics"), "E40 metrics")
dimensions = _object(metrics.get("dimensions"), "E40 dimensions")
quality_gate = _object(result.report.get("quality_gate"), "E40 gate")
development_cv = _object(
result.report.get("development_cross_validation"),
"E40 development CV",
)
return {
"result_id": result.result_id,
"created_at_utc": result.manifest.get("created_at_utc"),
"source_session_id": source.get("session_id"),
"source_display_name": source.get("display_name"),
"status": result.report.get("status"),
"profile_id": profile.get("profile_id"),
"worker_node": execution.get("worker_node"),
"quality_gate_passed": quality_gate.get("passed"),
"development_cross_validation": copy.deepcopy(development_cv),
"metrics": {
"development_items": metrics.get("development_items"),
"validation_items": metrics.get("validation_items"),
"terminal_outcomes": metrics.get("terminal_outcomes"),
"accounting_fraction": metrics.get("accounting_fraction"),
"false_free_claims": metrics.get("false_free_claims"),
"high_severity_failures": metrics.get("high_severity_failures"),
"dimensions": copy.deepcopy(dimensions),
},
"blocking_checks": copy.deepcopy(quality_gate.get("blocking_checks")),
"method": copy.deepcopy(result.report.get("method")),
"decision": copy.deepcopy(result.report.get("decision")),
"limitations": copy.deepcopy(result.report.get("limitations")),
"authority": copy.deepcopy(result.report.get("authority")),
"access": "read-only",
}
def _empty_catalog(configured: bool) -> dict[str, object]:
return {
"schema_version": LABORATORY_ADVANCED_CATALOG_SCHEMA,
@@ -680,6 +735,7 @@ def build_advanced_laboratory_router(
e37_root_provider: RootProvider = lambda: None,
e38_root_provider: RootProvider = lambda: None,
e39_root_provider: RootProvider = lambda: None,
e40_root_provider: RootProvider = lambda: None,
) -> APIRouter:
router = APIRouter(prefix="/api/v1/laboratory", tags=["laboratory"])
@@ -975,4 +1031,37 @@ def build_advanced_laboratory_router(
"invalid_total": invalid_total,
}
@router.get("/e40/results")
def list_e40_results(
limit: int = Query(default=1, ge=1, le=10),
) -> dict[str, object]:
root = _configured_root(e40_root_provider)
if root is None:
return _empty_catalog(False)
candidates = _candidates(root, _E40_RESULT_ID)
items: list[dict[str, object]] = []
invalid_total = 0
for candidate in candidates:
try:
result = _read_e40_cached(
str(candidate.resolve()),
_result_signature(candidate),
)
if len(items) < limit:
items.append(_project_e40(result))
except (
E40PerceptionProductGateError,
KeyError,
OSError,
TypeError,
ValueError,
):
invalid_total += 1
return {
**_empty_catalog(True),
"items": items,
"candidate_total": len(candidates),
"invalid_total": invalid_total,
}
return router
src/k1link/web/app.py
+7
View File
@@ -557,6 +557,13 @@ app.include_router(
/ "e39"
/ "results"
),
e40_root_provider=lambda: (
REPOSITORY_ROOT
/ ".runtime"
/ "compute-experiments"
/ "e40"
/ "results"
),
)
)
app.include_router(
src/k1link/web/compute_contour_api.py
+12
View File
@@ -47,6 +47,7 @@ class ComputeContour(StrictModel):
mqtt_host: str = Field(default="127.0.0.1", min_length=1, max_length=253)
mqtt_port: int = Field(default=1883, ge=1, le=65535)
telemetry_poll_interval_seconds: int = Field(default=3, ge=1, le=60)
mqtt_publish_interval_seconds: int = Field(default=2, ge=1, le=60)
revision: int = Field(default=0, ge=0)
updated_at_utc: str | None = None
@@ -92,6 +93,7 @@ class ComputeContourCreate(StrictModel):
mqtt_host: str = Field(default="127.0.0.1", min_length=1, max_length=253)
mqtt_port: int = Field(default=1883, ge=1, le=65535)
telemetry_poll_interval_seconds: int = Field(default=3, ge=1, le=60)
mqtt_publish_interval_seconds: int = Field(default=2, ge=1, le=60)
@field_validator("display_name")
@classmethod
@@ -127,6 +129,7 @@ def default_compute_contour() -> ComputeContour:
mqtt_host="127.0.0.1",
mqtt_port=1883,
telemetry_poll_interval_seconds=3,
mqtt_publish_interval_seconds=2,
)
@@ -158,6 +161,9 @@ class ComputeContourStore:
telemetry_poll_interval_seconds=(
request.telemetry_poll_interval_seconds
),
mqtt_publish_interval_seconds=(
request.mqtt_publish_interval_seconds
),
revision=0,
updated_at_utc=_utc_now(),
)
@@ -186,6 +192,9 @@ class ComputeContourStore:
"telemetry_poll_interval_seconds": (
request.telemetry_poll_interval_seconds
),
"mqtt_publish_interval_seconds": (
request.mqtt_publish_interval_seconds
),
"revision": current.revision + 1,
"updated_at_utc": _utc_now(),
}
@@ -260,6 +269,9 @@ def _agent_install_document(contour: ComputeContour) -> dict[str, object]:
"MISSIONCORE_MQTT_HOST": contour.mqtt_host,
"MISSIONCORE_MQTT_PORT": str(contour.mqtt_port),
"MISSIONCORE_MQTT_USERNAME": contour.agent_id,
"MISSIONCORE_TELEMETRY_INTERVAL": (
f"{contour.mqtt_publish_interval_seconds}s"
),
}
if contour.platform == "windows":
command = (
src/k1link/web/e30_engineering_api.py
+14 -2
View File
@@ -590,9 +590,21 @@ def build_e30_engineering_router(
result_id=result_id,
)
rows_by_id = {row["item_id"]: row for row in rows}
exception_rows = catalog_item.get("human_exceptions")
if (
not isinstance(exception_rows, list)
or not all(
isinstance(value, dict)
and isinstance(value.get("item_id"), str)
for value in exception_rows
)
):
raise E30EngineeringEvidenceError(
"engineering exception catalog is invalid"
)
exception_ids = [
value["item_id"]
for value in catalog_item["human_exceptions"]
str(value["item_id"])
for value in exception_rows
]
if any(item_id not in rows_by_id for item_id in exception_ids):
raise E30EngineeringEvidenceError(
src/k1link/web/e30_human_review_api.py
+14 -2
View File
@@ -94,9 +94,21 @@ def build_e30_human_review_router(
subject.item_id: subject
for subject in source_substrate.subjects
}
exception_rows = generation.get("human_exceptions")
if (
not isinstance(exception_rows, list)
or not all(
isinstance(value, dict)
and isinstance(value.get("item_id"), str)
for value in exception_rows
)
):
raise E30HumanReviewValidationError(
"engineering exception catalog is invalid"
)
exception_ids = [
value["item_id"]
for value in generation["human_exceptions"]
str(value["item_id"])
for value in exception_rows
]
if (
not exception_ids
src/k1link/web/system_telemetry_api.py
+124 -20
View File
@@ -16,11 +16,13 @@ from collections import deque
from collections.abc import Callable
from datetime import UTC, datetime
from pathlib import Path
from typing import Any, Final
from typing import Any, Final, Protocol
from fastapi import APIRouter, HTTPException, Query
from pydantic import BaseModel, ConfigDict, Field, field_validator
from k1link.web.compute_contour_api import ComputeContour, ComputeContourStore
PROFILE_SCHEMA: Final = "missioncore.worker-connection-profile/v1"
TELEMETRY_SCHEMA: Final = "missioncore.worker-telemetry/v1"
PROBE_SCHEMA: Final = "missioncore.worker-probe/v1"
@@ -79,6 +81,14 @@ RootProvider = Callable[[], Path]
ProbeRunner = Callable[["WorkerConnectionProfile"], dict[str, Any]]
class WorkerProfileStoreContract(Protocol):
def read(self) -> WorkerConnectionProfile:
"""Return the current worker-shaped connection profile."""
def save(self, request: WorkerConnectionProfilePut) -> WorkerConnectionProfile:
"""Persist one reviewed profile update."""
class WorkerConnectionProfile(BaseModel):
model_config = ConfigDict(extra="forbid", frozen=True)
@@ -603,16 +613,28 @@ def _agent_raw_document(document: dict[str, Any]) -> dict[str, Any]:
return raw
def run_worker_agent_probe(profile: WorkerConnectionProfile) -> dict[str, Any]:
def run_worker_agent_probe(
profile: WorkerConnectionProfile,
*,
contour_id: str | None = None,
agent_id: str | None = None,
) -> dict[str, Any]:
started = time.perf_counter()
base_url = os.environ.get(
"MISSIONCORE_TELEMETRY_QUERY_URL",
DEFAULT_TELEMETRY_QUERY_URL,
).rstrip("/")
contour_id = os.environ.get("MISSIONCORE_TELEMETRY_CONTOUR_ID", "worker-006")
agent_id = os.environ.get("MISSIONCORE_TELEMETRY_AGENT_ID", "worker-006")
resolved_contour_id = contour_id or os.environ.get(
"MISSIONCORE_TELEMETRY_CONTOUR_ID",
"worker-006",
)
resolved_agent_id = agent_id or os.environ.get(
"MISSIONCORE_TELEMETRY_AGENT_ID",
"worker-006",
)
url = (
f"{base_url}/v1/contours/{contour_id}/agents/{agent_id}/latest"
f"{base_url}/v1/contours/{resolved_contour_id}/agents/"
f"{resolved_agent_id}/latest"
"?max_age_seconds=30"
)
try:
@@ -898,7 +920,7 @@ def _pipeline_document(raw: dict[str, Any]) -> dict[str, Any]:
class WorkerTelemetryService:
def __init__(
self,
store: WorkerProfileStore,
store: WorkerProfileStoreContract,
probe_runner: ProbeRunner = run_worker_probe,
*,
telemetry_probe_runner: ProbeRunner | None = None,
@@ -911,7 +933,9 @@ class WorkerTelemetryService:
self._lock = threading.Lock()
self._cached_at = 0.0
self._cached: dict[str, Any] | None = None
self._previous_network: tuple[float, float, float] | None = None
self._previous_network: tuple[str, float, float, float] | None = None
self._latest_network_rates: tuple[float | None, float | None] = (None, None)
self._last_history_observed_at: str | None = None
self._history: deque[dict[str, Any]] = deque(maxlen=300)
def profile_document(self) -> dict[str, Any]:
@@ -948,6 +972,8 @@ class WorkerTelemetryService:
self._cached = None
self._cached_at = 0
self._previous_network = None
self._latest_network_rates = (None, None)
self._last_history_observed_at = None
self._history.clear()
return {
**self.profile_document(),
@@ -1007,25 +1033,36 @@ class WorkerTelemetryService:
for item in _items(raw.get("network"))
if isinstance(item, dict)
]
received = sum(
received = float(
sum(
value
for item in interfaces
if (value := _number(item.get("received_bytes"))) is not None
)
)
sent = sum(
sent = float(
sum(
value
for item in interfaces
if (value := _number(item.get("sent_bytes"))) is not None
)
)
receive_rate: float | None = None
send_rate: float | None = None
if self._previous_network is not None:
previous_at, previous_received, previous_sent = self._previous_network
elapsed = monotonic_now - previous_at
if elapsed > 0 and received >= previous_received and sent >= previous_sent:
receive_rate = (received - previous_received) / elapsed
send_rate = (sent - previous_sent) / elapsed
self._previous_network = (monotonic_now, received, sent)
raw_observed_at = raw.get("observed_at_utc")
observed_at: str = (
raw_observed_at if isinstance(raw_observed_at, str) else _utc_now()
)
receive_rate, send_rate = self._latest_network_rates
if observed_at != self._last_history_observed_at:
receive_rate = None
send_rate = None
if self._previous_network is not None:
_, previous_at, previous_received, previous_sent = self._previous_network
elapsed = monotonic_now - previous_at
if elapsed > 0 and received >= previous_received and sent >= previous_sent:
receive_rate = (received - previous_received) / elapsed
send_rate = (sent - previous_sent) / elapsed
self._previous_network = (observed_at, monotonic_now, received, sent)
self._latest_network_rates = (receive_rate, send_rate)
raw_stats = _mapping(raw.get("docker_stats"))
raw_states = _mapping(raw.get("container_states"))
runtimes = [
@@ -1077,7 +1114,7 @@ class WorkerTelemetryService:
},
}
history_row = {
"observed_at_utc": raw.get("observed_at_utc") or _utc_now(),
"observed_at_utc": observed_at,
"cpu_percent": _number(_mapping(raw.get("cpu")).get("load_percent")),
"memory_percent": memory_used_percent,
"gpu_percent": _number(gpu.get("utilization_percent")),
@@ -1085,7 +1122,9 @@ class WorkerTelemetryService:
"network_receive_bytes_per_second": receive_rate,
"network_send_bytes_per_second": send_rate,
}
self._history.append(history_row)
if observed_at != self._last_history_observed_at:
self._history.append(history_row)
self._last_history_observed_at = observed_at
return {
"schema_version": TELEMETRY_SCHEMA,
"profile": profile.model_dump(mode="json"),
@@ -1154,8 +1193,49 @@ def build_system_telemetry_router(
probe_runner,
telemetry_probe_runner=telemetry_probe_runner,
)
contour_store = ComputeContourStore(root_provider())
contour_services: dict[str, WorkerTelemetryService] = {}
router = APIRouter(prefix="/api/v1/system", tags=["system"])
class ContourProfileStore:
def __init__(self, contour_id: str) -> None:
self.contour_id = contour_id
def read(self) -> WorkerConnectionProfile:
return _profile_from_compute_contour(
contour_store.get(self.contour_id)
)
def save(
self,
request: WorkerConnectionProfilePut,
) -> WorkerConnectionProfile:
del request
raise RuntimeError("compute contour telemetry profiles are read-only")
def contour_service(contour_id: str) -> WorkerTelemetryService:
existing = contour_services.get(contour_id)
if existing is not None:
return existing
def contour_probe(profile: WorkerConnectionProfile) -> dict[str, Any]:
contour = contour_store.get(contour_id)
if contour.telemetry_mode == "legacy-ssh":
return probe_runner(profile)
return run_worker_agent_probe(
profile,
contour_id=contour.contour_id,
agent_id=contour.agent_id,
)
created = WorkerTelemetryService(
ContourProfileStore(contour_id),
probe_runner,
telemetry_probe_runner=contour_probe,
)
contour_services[contour_id] = created
return created
@router.get("/worker-profile")
def get_worker_profile() -> dict[str, Any]:
return service.profile_document()
@@ -1185,4 +1265,28 @@ def build_system_telemetry_router(
) -> dict[str, Any]:
return service.snapshot(history)
@router.get("/contours/{contour_id}/telemetry")
def get_compute_contour_telemetry(
contour_id: str,
history: int = Query(default=90, ge=1, le=300),
) -> dict[str, Any]:
try:
contour_store.get(contour_id)
return contour_service(contour_id).snapshot(history)
except KeyError as exc:
raise HTTPException(status_code=404, detail="Контур не найден.") from exc
return router
def _profile_from_compute_contour(contour: ComputeContour) -> WorkerConnectionProfile:
return WorkerConnectionProfile(
profile_id=contour.contour_id,
display_name=contour.display_name,
expected_node_id=contour.expected_node_id,
ssh_host_alias=SSH_HOST_ALIAS,
address=contour.address,
port=contour.ssh_port,
revision=contour.revision,
updated_at_utc=contour.updated_at_utc,
)
tests/test_advanced_laboratory_api.py
+152 -3
View File
@@ -26,7 +26,9 @@ def _endpoint(router: APIRouter, path: str) -> object:
def test_advanced_catalogs_are_empty_when_not_configured() -> None:
router = build_advanced_laboratory_router()
for name in ("e31", "e32", "e33", "e34", "e35", "e37", "e38", "e39"):
for name in (
"e31", "e32", "e33", "e34", "e35", "e37", "e38", "e39", "e40"
):
route = _endpoint(router, f"/api/v1/laboratory/{name}/results")
catalog = route(limit=1) # type: ignore[operator]
assert catalog == {
@@ -50,7 +52,8 @@ def test_advanced_catalogs_fail_closed_on_incomplete_results(
e37 = tmp_path / "e37"
e38 = tmp_path / "e38"
e39 = tmp_path / "e39"
for root in (e31, e32, e33, e34, e35, e37, e38, e39):
e40 = tmp_path / "e40"
for root in (e31, e32, e33, e34, e35, e37, e38, e39, e40):
root.mkdir()
(e31 / f"e31-source-qualification-{'1' * 64}").mkdir()
(e32 / f"e32-track-geometry-{'2' * 64}").mkdir()
@@ -60,6 +63,7 @@ def test_advanced_catalogs_fail_closed_on_incomplete_results(
(e37 / f"e37-ravnoves-acceptance-{'7' * 64}").mkdir()
(e38 / f"e38-perception-baseline-{'8' * 64}").mkdir()
(e39 / f"e39-perception-refinement-{'9' * 64}").mkdir()
(e40 / f"e40-perception-product-gate-{'a' * 64}").mkdir()
router = build_advanced_laboratory_router(
e31_root_provider=lambda: e31,
e32_root_provider=lambda: e32,
@@ -69,9 +73,12 @@ def test_advanced_catalogs_fail_closed_on_incomplete_results(
e37_root_provider=lambda: e37,
e38_root_provider=lambda: e38,
e39_root_provider=lambda: e39,
e40_root_provider=lambda: e40,
)
for name in ("e31", "e32", "e33", "e34", "e35", "e37", "e38", "e39"):
for name in (
"e31", "e32", "e33", "e34", "e35", "e37", "e38", "e39", "e40"
):
route = _endpoint(router, f"/api/v1/laboratory/{name}/results")
catalog = route(limit=1) # type: ignore[operator]
assert catalog["configured"] is True
@@ -370,6 +377,148 @@ def test_e39_catalog_projects_development_cv_and_sealed_validation(
assert item["access"] == "read-only"
def test_e40_catalog_projects_dual_cv_and_sealed_product_gate(
tmp_path: Path,
monkeypatch: MonkeyPatch,
) -> None:
result_id = f"e40-perception-product-gate-{'a' * 64}"
root = tmp_path / "e40"
candidate = root / result_id
candidate.mkdir(parents=True)
(candidate / "manifest.json").write_text("{}", encoding="utf-8")
authority = {
"commands_enabled": False,
"navigation_or_safety_accepted": False,
}
dimension = {
"correct": 132,
"incorrect": 14,
"total": 146,
"accuracy": 0.90411,
"target": 0.9,
"passed": True,
"confusion": [],
"by_stratum": {},
}
development_dimension = {
"correct": 309,
"incorrect": 31,
"total": 340,
"accuracy": 0.908824,
"target": 0.9,
"passed": True,
}
protocol = {
"items": 340,
"fold_sizes": {"0": 68, "1": 68, "2": 68, "3": 68, "4": 68},
"dimensions": {
"presence": development_dimension,
"geometry_association": development_dimension,
"freshness": {
**development_dimension,
"correct": 325,
"incorrect": 15,
"accuracy": 0.955882,
},
},
"passed": True,
}
result = SimpleNamespace(
result_id=result_id,
manifest={
"created_at_utc": "2026-07-28T08:30:00Z",
"identity": {
"source": {
"session_id": "20260720T065719Z_viewer_live",
"display_name": "RAVNOVES00",
},
"profile": {
"profile_id": (
"e40-ravnoves00-leakage-resistant-product-gate/v1"
),
},
"execution": {
"worker_node": "DESKTOP-OPJ8J04",
},
},
},
report={
"status": "measured-leakage-resistant-product-gate",
"development_cross_validation": {
"strategy": "dual-leakage-resistant-development-five-fold",
"seed": "e40-development-cv-v1",
"folds": 5,
"items": 340,
"validation_labels_used": False,
"protocols": {
"contiguous-source-time-five-fold": protocol,
"whole-track-or-scene-window-five-fold": protocol,
},
"passed": True,
},
"metrics": {
"development_items": 340,
"validation_items": 146,
"terminal_outcomes": 146,
"accounting_fraction": 1.0,
"false_free_claims": 0,
"high_severity_failures": 0,
"dimensions": {
"presence": dimension,
"geometry_association": dimension,
"freshness": {
**dimension,
"correct": 140,
"incorrect": 6,
"accuracy": 0.958904,
},
},
},
"quality_gate": {
"passed": True,
"blocking_checks": [],
},
"method": {
"summary": "conservative policy plus camera-only softmax",
"selection": "dual grouped development cross-validation",
"dimension_projection": "presence plus immutable stratum",
},
"decision": {
"product_gate_measured": True,
"accepted_for_ravnoves00_product_track": True,
},
"limitations": ["RAVNOVES00 source scoped"],
"authority": authority,
},
)
def fake_read(
root_text: str,
signature: tuple[int, ...],
) -> SimpleNamespace:
assert root_text == str(candidate.resolve())
assert signature
return result
monkeypatch.setattr(advanced_api, "_read_e40_cached", fake_read)
router = build_advanced_laboratory_router(
e40_root_provider=lambda: root,
)
route = _endpoint(router, "/api/v1/laboratory/e40/results")
catalog = route(limit=1) # type: ignore[operator]
assert catalog["candidate_total"] == 1
assert catalog["invalid_total"] == 0
item = catalog["items"][0]
assert item["development_cross_validation"]["passed"] is True
assert len(item["development_cross_validation"]["protocols"]) == 2
assert item["metrics"]["dimensions"]["presence"]["accuracy"] == 0.90411
assert item["metrics"]["high_severity_failures"] == 0
assert item["quality_gate_passed"] is True
assert item["authority"] == authority
assert item["access"] == "read-only"
def test_e35_catalog_projects_recovery_and_review(
tmp_path: Path,
monkeypatch: MonkeyPatch,
tests/test_compute_contour_api.py
+5
View File
@@ -40,6 +40,7 @@ def test_contour_store_migrates_worker_006_as_first_configuration(
assert contours[0].contour_id == "worker-006"
assert contours[0].telemetry_mode == "agent-mqtt"
assert contours[0].telemetry_poll_interval_seconds == 3
assert contours[0].mqtt_publish_interval_seconds == 2
assert not store.path.exists()
@@ -67,11 +68,13 @@ def test_contour_store_creates_and_updates_private_catalog(tmp_path: Path) -> No
mqtt_host="192.0.2.5",
mqtt_port=1883,
telemetry_poll_interval_seconds=1,
mqtt_publish_interval_seconds=4,
),
)
assert updated.display_name == "Field Worker 01"
assert updated.telemetry_poll_interval_seconds == 1
assert updated.mqtt_publish_interval_seconds == 4
assert updated.revision == 1
assert stat.S_IMODE(store.path.stat().st_mode) == 0o600
assert len(store.list_contours()) == 2
@@ -89,6 +92,7 @@ def test_contour_store_creates_and_updates_private_catalog(tmp_path: Path) -> No
mqtt_host="127.0.0.1",
mqtt_port=1883,
telemetry_poll_interval_seconds=3,
mqtt_publish_interval_seconds=2,
),
)
@@ -113,6 +117,7 @@ def test_contour_router_exposes_catalog_and_safe_install_contract(
assert document["agent"]["distribution"] == "Telegraf"
assert "MQTT password" in document["command"]
assert "password" not in document["agent"]["environment"]
assert document["agent"]["environment"]["MISSIONCORE_TELEMETRY_INTERVAL"] == "2s"
assert document["ready"] is False
tests/test_e40_perception_product_gate.py
+132
View File
@@ -0,0 +1,132 @@
from __future__ import annotations
import numpy as np
from k1link.compute.e40_perception_product_gate import (
_FIXED_PRESENCE,
_LABELS,
_feature_names,
_predict_product_presence,
_project_dimensions,
_result_content_identity,
_train_softmax,
)
def test_e40_feature_contract_excludes_route_identity() -> None:
names = _feature_names()
assert len(names) == 125
assert len(names) == len(set(names))
assert not any(
token in name
for name in names
for token in (
"source_frame",
"session_seconds",
"review_ordinal",
"track_id",
"map_xyz",
)
)
assert "bbox_area" in names
assert "image_edge_p90" in names
assert "support_occupied_fraction" in names
def test_e40_fixed_strata_are_conservative_product_states() -> None:
assert _FIXED_PRESENCE == {
"agree": "object-present",
"conflict": "background-or-noise",
"geometry-only": "occupied-environment",
"unknown": "object-present",
}
median = np.zeros(2)
scale = np.ones(2)
weights = np.zeros((3, len(_LABELS)))
for stratum, expected in _FIXED_PRESENCE.items():
assert _predict_product_presence(
stratum=stratum,
features=np.ones(2),
median=median,
scale=scale,
weights=weights,
clip=10.0,
) == (expected, 1.0)
assert _project_dimensions("geometry-only", "occupied-environment") == {
"presence": "occupied-environment",
"geometry_association": "independent-occupied",
"freshness": "current",
}
def test_e40_camera_only_softmax_is_deterministic() -> None:
matrix = np.asarray(
[
[-2.0, -1.0],
[-1.0, -2.0],
[1.0, 2.0],
[2.0, 1.0],
],
dtype=np.float64,
)
labels = np.asarray(
[
_LABELS.index("background-or-noise"),
_LABELS.index("background-or-noise"),
_LABELS.index("object-present"),
_LABELS.index("object-present"),
],
dtype=np.int64,
)
first = _train_softmax(
matrix,
labels,
l2=0.01,
steps=120,
learning_rate=0.03,
)
second = _train_softmax(
matrix,
labels,
l2=0.01,
steps=120,
learning_rate=0.03,
)
assert np.array_equal(first, second)
assert (
_predict_product_presence(
stratum="camera-only",
features=np.asarray([1.5, 1.5]),
median=np.zeros(2),
scale=np.ones(2),
weights=first,
clip=10.0,
)[0]
== "object-present"
)
def test_e40_result_content_identity_changes_with_every_output() -> None:
predictions = [{"sequence": 1, "prediction": {"presence": "object-present"}}]
model = {"weights": [1.0]}
report = {"quality_gate": {"passed": False}}
baseline = _result_content_identity(
predictions=predictions,
model=model,
report=report,
)
assert baseline != _result_content_identity(
predictions=[{"sequence": 1, "prediction": {"presence": "background-or-noise"}}],
model=model,
report=report,
)
assert baseline != _result_content_identity(
predictions=predictions,
model={"weights": [2.0]},
report=report,
)
assert baseline != _result_content_identity(
predictions=predictions,
model=model,
report={"quality_gate": {"passed": True}},
)
tests/test_e40_worker_package.py
+92
View File
@@ -0,0 +1,92 @@
from __future__ import annotations
import hashlib
import importlib.util
import json
import subprocess
import sys
from pathlib import Path
import pytest
def _module() -> object:
path = (
Path(__file__).resolve().parents[1]
/ "experiments"
/ "perception"
/ "prepare_e40_worker_package.py"
)
spec = importlib.util.spec_from_file_location("e40_worker_package_test", path)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
return module
def test_e40_package_contains_bound_product_gate_input(tmp_path: Path) -> None:
module = _module()
repository = Path(__file__).resolve().parents[1]
acceptance = (
repository
/ ".runtime"
/ "compute-experiments"
/ "e37"
/ "results"
/ (
"e37-ravnoves-acceptance-"
"01b1efd586f747341c712d82f0907b39436a6f91ae92b1dfae987eca05fd8344"
)
)
materialization = (
repository
/ ".runtime"
/ "compute-experiments"
/ "e30"
/ "materializations"
/ ("e30-materialization-841af926d8d28ab93538c46d8f31278a2234c4d1c12c7dc4dc296b249d59735a")
)
package = module.build_e40_worker_package(
repository_root=repository,
acceptance_root=acceptance,
materialization_root=materialization,
profile_path=(
repository / "experiments" / "perception" / "e40_ravnoves00_product_gate_profile.json"
),
output_root=tmp_path,
)
manifest = module.validate_e40_worker_package(package)
assert package.name == f"e40-worker-package-{manifest['identity_sha256']}"
assert manifest["identity"]["classification"] == (
"immutable-ravnoves00-leakage-resistant-product-gate-input"
)
feature_manifest = (
package / "input" / "materialization" / materialization.name / "e40-feature-cache.json"
)
assert feature_manifest.is_file()
assert (package / "runtime" / "k1link" / "compute" / "e40_perception_product_gate.py").is_file()
independent_validator = (
package / "runtime" / "validate_e40_worker_package.py"
)
assert independent_validator.is_file()
subprocess.run(
[sys.executable, str(independent_validator), str(package)],
check=True,
)
profile_path = package / "profile.json"
profile_path.write_text("{}\n", encoding="utf-8")
package_manifest_path = package / "manifest.json"
package_manifest = json.loads(package_manifest_path.read_text(encoding="utf-8"))
profile_payload = profile_path.read_bytes()
for row in package_manifest["artifacts"]:
if row["path"] == "profile.json":
row["byte_length"] = len(profile_payload)
row["sha256"] = hashlib.sha256(profile_payload).hexdigest()
package_manifest_path.write_text(
json.dumps(package_manifest, indent=2, sort_keys=True) + "\n",
encoding="utf-8",
)
with pytest.raises(module.E40WorkerPackageError, match="binding|changed"):
module.validate_e40_worker_package(package)
tests/test_e41_evaluation_boundary.py
+163
View File
@@ -0,0 +1,163 @@
from __future__ import annotations
import numpy as np
import pytest
from k1link.compute.e41_evaluation_boundary import (
E41EvaluationBoundaryError,
evaluate_visible_engineering_contract,
predict_from_frozen_e40_model,
)
def _model() -> dict[str, object]:
return {
"schema_version": "missioncore.e40-development-product-model/v1",
"feature_names": ["signal"],
"validation_labels_used_for_training": False,
"robust_clip": 10.0,
"fixed_presence_by_stratum": {
"agree": "object-present",
"conflict": "background-or-noise",
"geometry-only": "occupied-environment",
"unknown": "object-present",
},
"dimension_projection": "source-stratum-plus-presence/v1",
"classifier": {
"type": "deterministic-softmax",
"labels": [
"background-or-noise",
"object-present",
"occupied-environment",
],
"weights": [
[-2.0, 2.0, 0.0],
[0.0, 0.0, 0.0],
],
"scaler": {
"median": [0.0],
"scale": [1.0],
},
},
}
def test_e41_predictor_output_is_truth_free_and_deterministic() -> None:
items = [
{
"schema_version": "missioncore.e41-predictor-item/v1",
"sequence": 0,
"item_id": "a",
"source_stratum": "camera-only",
},
{
"schema_version": "missioncore.e41-predictor-item/v1",
"sequence": 1,
"item_id": "b",
"source_stratum": "geometry-only",
},
]
matrix = np.asarray([[1.0], [0.0]], dtype=np.float64)
first = predict_from_frozen_e40_model(
items=items,
feature_names=["signal"],
feature_matrix=matrix,
model=_model(),
)
second = predict_from_frozen_e40_model(
items=items,
feature_names=["signal"],
feature_matrix=matrix,
model=_model(),
)
assert first == second
assert first[0]["prediction"]["presence"] == "object-present"
assert first[1]["prediction"]["presence"] == "occupied-environment"
assert not any(
forbidden in row
for row in first
for forbidden in ("reference", "scored", "severity", "split", "truth")
)
def test_e41_predictor_rejects_truth_bearing_item_metadata() -> None:
with pytest.raises(E41EvaluationBoundaryError, match="truth/evaluation"):
predict_from_frozen_e40_model(
items=[
{
"sequence": 0,
"item_id": "a",
"source_stratum": "camera-only",
"reference": {"presence": "object-present"},
}
],
feature_names=["signal"],
feature_matrix=np.asarray([[1.0]], dtype=np.float64),
model=_model(),
)
def test_e41_visible_evaluator_joins_truth_after_prediction() -> None:
predictions = [
{
"item_id": "dev",
"prediction": {
"presence": "object-present",
"geometry_association": "insufficient-support",
"freshness": "unavailable",
},
},
{
"item_id": "val",
"prediction": {
"presence": "object-present",
"geometry_association": "object-associated",
"freshness": "current",
},
},
]
acceptance = [
{
"item_id": "dev",
"split": "development",
"source_stratum": "camera-only",
"severity": "standard",
"reference": {
"presence": "background-or-noise",
"geometry_association": "rejected-nonobject",
"freshness": "unavailable",
},
},
{
"item_id": "val",
"split": "validation",
"source_stratum": "agree",
"severity": "standard",
"reference": {
"presence": "object-present",
"geometry_association": "object-associated",
"freshness": "current",
},
},
]
evaluation = evaluate_visible_engineering_contract(
predictions=predictions,
acceptance_rows=acceptance,
targets={
"presence_target": 0.9,
"geometry_association_target": 0.9,
"freshness_target": 0.9,
},
label_provenance={
"engineering_items": 2,
"human_exception_items": 0,
"independent_ground_truth": False,
},
)
assert evaluation["metrics"]["validation_items"] == 1
assert evaluation["metrics"]["dimensions"]["presence"]["accuracy"] == 1.0
assert evaluation["engineering_contract_targets_reached"] is True
assert evaluation["blind_gate_eligible"] is False
assert evaluation["label_provenance"]["independent_accuracy_authority"] is False
tests/test_e41_methodology_audit.py
+184
View File
@@ -0,0 +1,184 @@
from __future__ import annotations
import numpy as np
from k1link.compute.e41_methodology_audit import analyze_e41_methodology
def _acceptance(
item_id: str,
*,
split: str,
frame: int,
stratum: str = "camera-only",
presence: str = "object-present",
) -> dict[str, object]:
return {
"item_id": item_id,
"split": split,
"source_frame_index": frame,
"source_stratum": stratum,
"reference": {
"presence": presence,
"geometry_association": "insufficient-support",
"freshness": "unavailable",
},
}
def _materialization(item_id: str, *, track_id: int | None) -> dict[str, object]:
return {
"item_id": item_id,
"e29_snapshot": {
"track_id": track_id,
},
}
def test_e41_detects_split_leakage_and_prediction_truth_colocation() -> None:
acceptance = [
_acceptance("dev-a", split="development", frame=100, presence="object-present"),
_acceptance(
"dev-b",
split="development",
frame=101,
presence="background-or-noise",
),
_acceptance("val-a", split="validation", frame=100, presence="object-present"),
_acceptance("val-b", split="validation", frame=149, presence="object-present"),
]
materialization = [
_materialization("dev-a", track_id=7),
_materialization("dev-b", track_id=8),
_materialization("val-a", track_id=7),
_materialization("val-b", track_id=None),
]
names = ["image_luma_mean", "stratum=camera-only", "source_frame_index"]
matrix = np.asarray(
[
[0.1, 1.0, 100.0],
[0.9, 1.0, 101.0],
[0.2, 1.0, 100.0],
[0.3, 1.0, 149.0],
],
dtype=np.float64,
)
report = analyze_e41_methodology(
acceptance_rows=acceptance,
materialization_rows=materialization,
feature_item_ids=["dev-a", "dev-b", "val-a", "val-b"],
feature_names=names,
feature_matrix=matrix,
e40_model={
"feature_names": names,
"camera_only_training_items": 2,
"dimension_projection": "source-stratum-plus-presence/v1",
},
e40_report={
"status": "measured-leakage-resistant-product-gate",
"execution": {"class": "sealed-validation-evaluation"},
"metrics": {
"dimensions": {
"presence": {"accuracy": 0.5},
"geometry_association": {"accuracy": 0.5},
}
},
},
e40_predictions=[
{
"item_id": "val-a",
"prediction": {"presence": "object-present"},
"reference": {"presence": "object-present"},
"scored": True,
}
],
label_provenance={
"engineering_items": 4,
"human_exception_items": 0,
"independent_ground_truth": False,
},
time_block_frames=50,
forbidden_feature_tokens=("source_frame", "track_id", "path"),
)
assert report["split_leakage"]["exact_source_frames"]["count"] == 1
assert report["split_leakage"]["track_ids"]["count"] == 1
assert report["split_leakage"]["time_blocks"]["count"] == 1
assert report["split_leakage"]["whole_track_or_scene_groups"]["count"] == 1
assert report["features"]["camera_only_training_items"] == 2
assert report["features"]["forbidden_features"] == ["source_frame_index"]
assert report["predictor_evaluator_boundary"]["physically_separated"] is False
assert report["metric_semantics"]["dimensions_independently_inferred"] is False
assert report["policy"]["blind_gate_eligible"] is False
assert report["policy"]["violations"] == [
"labels-are-not-independent-ground-truth",
"development-validation-source-groups-overlap",
"prediction-and-evaluation-concerns-are-co-located",
"historical-e40-still-contains-blind-or-product-gate-claims",
"forbidden-identity-feature-detected",
]
def test_e41_accepts_a_clean_separated_methodology_contract() -> None:
acceptance = [
_acceptance("dev-a", split="development", frame=10),
_acceptance(
"dev-b",
split="development",
frame=11,
presence="background-or-noise",
),
_acceptance("val-a", split="validation", frame=210),
]
materialization = [
_materialization("dev-a", track_id=1),
_materialization("dev-b", track_id=2),
_materialization("val-a", track_id=9),
]
names = ["image_luma_mean", "support_occupied_fraction"]
report = analyze_e41_methodology(
acceptance_rows=acceptance,
materialization_rows=materialization,
feature_item_ids=["dev-a", "dev-b", "val-a"],
feature_names=names,
feature_matrix=np.asarray(
[
[0.1, 0.2],
[0.9, 0.8],
[0.4, 0.3],
],
dtype=np.float64,
),
e40_model={
"feature_names": names,
"camera_only_training_items": 2,
"dimension_projection": "independent-task-heads/v1",
},
e40_report={
"status": "source-scoped-visible-evaluation",
"metrics": {
"dimensions": {
"presence": {"accuracy": 0.8},
"geometry_association": {"accuracy": 0.7},
}
},
},
e40_predictions=[
{
"item_id": "val-a",
"prediction": {"presence": "object-present"},
}
],
label_provenance={
"engineering_items": 0,
"human_exception_items": 3,
"independent_ground_truth": True,
},
time_block_frames=50,
forbidden_feature_tokens=("source_frame", "track_id", "path"),
)
assert report["features"]["forbidden_features"] == []
assert report["predictor_evaluator_boundary"]["physically_separated"] is True
assert report["policy"]["violations"] == []
assert report["policy"]["blind_gate_eligible"] is True
tests/test_e42_metamorphic_suite.py
+87
View File
@@ -0,0 +1,87 @@
from __future__ import annotations
import numpy as np
from k1link.compute.e42_metamorphic_suite import (
_point_slab_signature,
_predictor_metamorphics,
)
from k1link.compute.track_geometry import PointSlab
def _model() -> dict[str, object]:
return {
"schema_version": "missioncore.e40-development-product-model/v1",
"feature_names": ["signal"],
"validation_labels_used_for_training": False,
"robust_clip": 10.0,
"classifier": {
"type": "deterministic-softmax",
"labels": [
"background-or-noise",
"object-present",
"occupied-environment",
],
"weights": [
[-2.0, 2.0, 0.0],
[0.0, 0.0, 0.0],
],
"scaler": {
"median": [0.0],
"scale": [1.0],
},
},
}
def test_e42_predictor_is_invariant_to_ids_order_and_chunks() -> None:
items = [
{
"schema_version": "missioncore.e41-predictor-item/v1",
"sequence": index,
"item_id": f"item-{index}",
"source_stratum": "camera-only",
}
for index in range(12)
]
checks = _predictor_metamorphics(
items=items,
feature_names=["signal"],
feature_matrix=np.arange(12, dtype=np.float64).reshape((-1, 1)),
model=_model(),
)
assert all(checks.values())
def test_e42_point_slab_signature_is_row_order_invariant() -> None:
slab = PointSlab(
frame_index=1,
source_frame_index=10,
source_point_count=8,
coordinate_frame="map",
owner_keys=("track:1", "geometry:2"),
source_indices=np.asarray([1, 7, 3], dtype="<i8"),
points_xyz_m=np.asarray(
[
[1.0, 0.0, 0.0],
[7.0, 0.0, 0.0],
[3.0, 0.0, 0.0],
],
dtype="<f4",
),
owner_indices=np.asarray([0, 1, 0], dtype="<u4"),
)
order = np.asarray([2, 0, 1], dtype=np.int64)
permuted = PointSlab(
frame_index=slab.frame_index,
source_frame_index=slab.source_frame_index,
source_point_count=slab.source_point_count,
coordinate_frame=slab.coordinate_frame,
owner_keys=slab.owner_keys,
source_indices=slab.source_indices[order],
points_xyz_m=slab.points_xyz_m[order],
owner_indices=slab.owner_indices[order],
)
assert _point_slab_signature(slab) == _point_slab_signature(permuted)
tests/test_e43_future_capture_protocol.py
+158
View File
@@ -0,0 +1,158 @@
from __future__ import annotations
import pytest
from k1link.compute.e43_future_capture_protocol import (
E43_CANDIDATE_SCHEMA,
E43_CAPTURE_MANIFEST_SCHEMA,
E43FutureCaptureProtocolError,
assign_grouped_future_partitions,
validate_future_capture_manifest,
)
def _protocol() -> dict[str, object]:
return {
"capture_contract": {
"device_model": "XGRIDS/LixelKity-K1",
"minimum_duration_seconds": 480,
"maximum_duration_seconds": 900,
"required_streams": [
"sensor.camera.right",
"sensor.lidar.registered-map-increment",
"sensor.pose",
"telemetry.pipeline",
],
"required_segments": [
{
"kind": "control-bridge",
"minimum_duration_seconds": 60,
},
{
"kind": "new-route",
"minimum_duration_seconds": 360,
},
],
}
}
def _capture_manifest() -> dict[str, object]:
stream = {
"available": True,
"item_count": 10,
"byte_length": 100,
"sha256": "a" * 64,
}
return {
"schema_version": E43_CAPTURE_MANIFEST_SCHEMA,
"source_session_id": "future-session-001",
"source_display_name": "RAVNOVES01",
"operator_authorized": True,
"device": {
"model": "XGRIDS/LixelKity-K1",
"device_identity_sha256": "b" * 64,
"calibration_sha256": "c" * 64,
"mount_identity_sha256": "d" * 64,
"configuration_sha256": "e" * 64,
"firmware": "3.0.2",
},
"capture": {
"started_at_utc": "2026-07-28T12:00:00Z",
"monotonic_start_seconds": 100.0,
"monotonic_end_seconds": 700.0,
"duration_seconds": 600.0,
"weather": "overcast",
"illumination": "daylight",
"location_class": "industrial-buildings-opposite-side",
"operator_notes": "bounded owner-authorized capture",
},
"streams": {
"sensor.camera.right": stream,
"sensor.lidar.registered-map-increment": stream,
"sensor.pose": stream,
"telemetry.pipeline": stream,
},
"segments": [
{
"kind": "control-bridge",
"monotonic_start_seconds": 100.0,
"monotonic_end_seconds": 180.0,
},
{
"kind": "new-route",
"monotonic_start_seconds": 200.0,
"monotonic_end_seconds": 700.0,
},
],
"authority": {
"commands_enabled": False,
"navigation_or_safety_accepted": False,
},
}
def test_e43_accepts_complete_same_k1_capture_manifest() -> None:
result = validate_future_capture_manifest(
_capture_manifest(),
protocol=_protocol(),
)
assert result["accepted"] is True
assert result["segments"] == ["control-bridge", "new-route"]
assert result["blind_truth_labels_available"] is False
def test_e43_rejects_missing_pipeline_stream() -> None:
manifest = _capture_manifest()
del manifest["streams"]["telemetry.pipeline"] # type: ignore[index]
with pytest.raises(E43FutureCaptureProtocolError, match="stream set"):
validate_future_capture_manifest(
manifest,
protocol=_protocol(),
)
def test_e43_grouped_partition_keeps_connected_evidence_together() -> None:
candidates = [
{
"schema_version": E43_CANDIDATE_SCHEMA,
"item_id": "a",
"scene_id": "scene-1",
"track_id": "track-1",
"time_block_id": "time-1",
"route_segment": "control-bridge",
},
{
"schema_version": E43_CANDIDATE_SCHEMA,
"item_id": "b",
"scene_id": "scene-1",
"track_id": "track-2",
"time_block_id": "time-2",
"route_segment": "control-bridge",
},
{
"schema_version": E43_CANDIDATE_SCHEMA,
"item_id": "c",
"scene_id": "scene-2",
"track_id": "track-3",
"time_block_id": "time-3",
"route_segment": "new-route",
},
{
"schema_version": E43_CANDIDATE_SCHEMA,
"item_id": "d",
"scene_id": "scene-3",
"track_id": None,
"time_block_id": "time-4",
"route_segment": "new-route",
},
]
assignments = assign_grouped_future_partitions(
candidates,
seed="fixed-before-capture",
blind_fraction=0.3,
)
assert assignments["a"] == assignments["b"]
assert set(assignments.values()) == {"blind-truth", "visible-diagnostic"}
tests/test_e44_data_amplification_audit.py
+62
View File
@@ -0,0 +1,62 @@
from __future__ import annotations
import hashlib
import pytest
from k1link.compute.e44_data_amplification_audit import (
E44DataAmplificationAuditError,
analyze_data_amplification,
)
def _row(root: str, path: str, payload: bytes, kind: str) -> dict[str, object]:
return {
"root": root,
"path": path,
"byte_length": len(payload),
"sha256": hashlib.sha256(payload).hexdigest(),
"kind": kind,
}
def test_e44_measures_exact_cross_root_duplication() -> None:
shared = b"camera-frame"
report = analyze_data_amplification(
[
_row("e30", "frames/a.jpg", shared, "camera-image"),
_row("e40", "frames/a.jpg", shared, "camera-image"),
_row("e40", "report.json", b"{}", "metadata-or-report"),
]
)
assert report["logical_bytes"] == len(shared) * 2 + 2
assert report["unique_content_bytes"] == len(shared) + 2
assert report["duplicate_bytes"] == len(shared)
assert report["duplicate_content_groups"] == 1
assert report["largest_duplicate_groups"][0]["copies"] == 2
assert report["largest_duplicate_groups"][0]["roots"] == ["e30", "e40"]
assert report["roots"]["e40"]["duplicate_bytes_within_root"] == 0
def test_e44_rejects_same_digest_with_inconsistent_lengths() -> None:
digest = "a" * 64
with pytest.raises(E44DataAmplificationAuditError, match="inconsistent"):
analyze_data_amplification(
[
{
"root": "e30",
"path": "one.bin",
"byte_length": 1,
"sha256": digest,
"kind": "other",
},
{
"root": "e40",
"path": "two.bin",
"byte_length": 2,
"sha256": digest,
"kind": "other",
},
]
)
tests/test_pipeline_telemetry.py
+190
View File
@@ -0,0 +1,190 @@
from __future__ import annotations
import importlib.util
import json
from pathlib import Path
from types import ModuleType, SimpleNamespace
import pytest
from k1link.compute.pipeline_telemetry import (
JsonlPipelineTelemetrySink,
MqttPipelineTelemetrySink,
PipelineTelemetryEmitter,
PipelineTelemetryError,
PipelineTelemetryIdentity,
build_pipeline_telemetry_document,
)
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
NORMALIZER_PATH = (
REPOSITORY_ROOT
/ "deploy"
/ "telemetry-plane"
/ "normalizer"
/ "normalizer.py"
)
def _normalizer() -> ModuleType:
spec = importlib.util.spec_from_file_location(
"missioncore_pipeline_telemetry_normalizer",
NORMALIZER_PATH,
)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def _identity() -> PipelineTelemetryIdentity:
return PipelineTelemetryIdentity(
contour_id="worker-006",
agent_id="mission-core-worker",
node_id="DESKTOP-OPJ8J04",
lab_id="E41",
run_id="run-001",
request_id="request-001",
source_id="ravnoves00",
source_package_id="e41-predictor-package-example",
method_id="frozen-e40-predictor/v1",
frame_index=17,
)
def test_pipeline_document_is_accepted_without_losing_stage_identity() -> None:
identity = _identity()
document = build_pipeline_telemetry_document(
identity=identity,
stage_id="predict",
state="completed",
duration_ms=125.5,
input_count=89,
output_count=89,
queue_wait_ms=2.25,
observed_at_utc="2026-07-28T12:00:00Z",
)
row = _normalizer()._normalize(
identity.topic,
json.dumps(document).encode(),
)
assert row[1:5] == (
"worker-006",
"mission-core-worker",
"DESKTOP-OPJ8J04",
"pipeline",
)
assert row[9:13] == ("E41", "run-001", "request-001", 17)
assert json.loads(row[6]) == {
"lab_id": "E41",
"method_id": "frozen-e40-predictor/v1",
"request_id": "request-001",
"run_id": "run-001",
"source_id": "ravnoves00",
"source_package_id": "e41-predictor-package-example",
"stage_id": "predict",
"stage_state": "completed",
}
stored = json.loads(row[13])
assert stored["payload"]["event"]["duration_ms"] == 125.5
assert stored["authority"]["commands_enabled"] is False
def test_stage_context_emits_terminal_event_and_preserves_failure() -> None:
published: list[tuple[str, dict[str, object]]] = []
class Sink:
def publish(self, topic: str, payload: bytes) -> None:
published.append((topic, json.loads(payload)))
ticks = iter((1_000_000_000, 1_125_500_000))
emitter = PipelineTelemetryEmitter(
identity=_identity(),
sink=Sink(),
clock_ns=lambda: next(ticks),
)
with emitter.stage("predict", input_count=89) as outcome:
outcome.output_count = 89
assert [document["stage_state"] for _, document in published] == [
"started",
"completed",
]
assert published[1][1]["payload"]["event"]["duration_ms"] == 125.5
assert published[1][1]["payload"]["event"]["output_count"] == 89
failure_ticks = iter((2_000_000_000, 2_001_000_000))
failure_emitter = PipelineTelemetryEmitter(
identity=_identity(),
sink=Sink(),
clock_ns=lambda: next(failure_ticks),
)
with (
pytest.raises(ValueError, match="source failure"),
failure_emitter.stage("evaluate"),
):
raise ValueError("source failure")
assert published[-1][1]["stage_state"] == "failed"
assert published[-1][1]["payload"]["event"]["error_type"] == "ValueError"
assert "source failure" not in json.dumps(published[-1][1])
def test_jsonl_sink_records_topic_bound_documents(tmp_path: Path) -> None:
path = tmp_path / "telemetry" / "e41.jsonl"
identity = _identity()
sink = JsonlPipelineTelemetrySink(path)
document = build_pipeline_telemetry_document(
identity=identity,
stage_id="package",
state="completed",
duration_ms=1.0,
)
sink.publish(identity.topic, json.dumps(document).encode())
record = json.loads(path.read_text(encoding="utf-8"))
assert record["schema_version"] == "missioncore.pipeline-telemetry-record/v1"
assert record["topic"] == identity.topic
assert record["payload"]["stage_id"] == "package"
assert path.stat().st_mode & 0o077 == 0
def test_mqtt_sink_uses_qos_one_without_retention() -> None:
calls: list[tuple[str, bytes, int, bool]] = []
class Client:
def publish(
self,
topic: str,
payload: bytes,
qos: int,
retain: bool,
) -> SimpleNamespace:
calls.append((topic, payload, qos, retain))
return SimpleNamespace(rc=0)
MqttPipelineTelemetrySink(Client()).publish("topic", b"payload")
assert calls == [("topic", b"payload", 1, False)]
def test_pipeline_telemetry_rejects_unsafe_identity_and_invalid_metrics() -> None:
with pytest.raises(PipelineTelemetryError, match="contour_id"):
PipelineTelemetryIdentity(
contour_id="../worker",
agent_id="agent",
node_id="node",
lab_id="E41",
run_id="run",
source_id="source",
source_package_id="package",
method_id="method",
)
with pytest.raises(PipelineTelemetryError, match="duration"):
build_pipeline_telemetry_document(
identity=_identity(),
stage_id="predict",
state="completed",
)
tests/test_system_telemetry_api.py
+38
View File
@@ -10,6 +10,7 @@ from fastapi import APIRouter, HTTPException
from fastapi.routing import APIRoute
from pydantic import ValidationError
from k1link.web.compute_contour_api import default_compute_contour
from k1link.web.system_telemetry_api import (
EXPECTED_NODE_ID,
WorkerConnectionProfile,
@@ -17,6 +18,7 @@ from k1link.web.system_telemetry_api import (
WorkerProfileStore,
WorkerTelemetryService,
_agent_raw_document,
_profile_from_compute_contour,
_ssh_arguments,
build_system_telemetry_router,
)
@@ -272,6 +274,23 @@ def test_worker_telemetry_prefers_ndc_container_names_during_migration(
assert triton["canonical_name"] == "ndc-mission-core-triton"
def test_worker_telemetry_history_keeps_one_row_per_agent_observation(
tmp_path: Path,
) -> None:
service = WorkerTelemetryService(
WorkerProfileStore(tmp_path / "system"),
lambda _: _probe(),
cache_seconds=0,
)
first = service.snapshot(10)
second = service.snapshot(10)
assert len(first["history"]) == 1
assert len(second["history"]) == 1
assert second["history"][0]["observed_at_utc"] == "2026-07-27T12:00:00Z"
def test_agent_metrics_are_mapped_to_the_existing_product_contract() -> None:
document = _agent_raw_document(
{
@@ -340,6 +359,25 @@ def test_agent_metrics_are_mapped_to_the_existing_product_contract() -> None:
)
def test_compute_contour_maps_to_worker_identity_without_singleton_defaults() -> None:
contour = default_compute_contour().model_copy(
update={
"contour_id": "field-worker",
"agent_id": "field-agent",
"display_name": "Field Worker",
"expected_node_id": "FIELD-01",
"address": "192.0.2.25",
}
)
profile = _profile_from_compute_contour(contour)
assert profile.profile_id == "field-worker"
assert profile.display_name == "Field Worker"
assert profile.expected_node_id == "FIELD-01"
assert profile.address == "192.0.2.25"
def test_profile_apply_fails_closed_on_wrong_node_and_keeps_old_profile(
tmp_path: Path,
) -> None:
tests/test_telemetry_normalizer_contract.py
+102
View File
@@ -122,6 +122,12 @@ def test_telegraf_upsert_merges_split_fields_in_one_series() -> None:
assert "EXCLUDED.payload -> 'fields'" in NORMALIZER_SOURCE
def test_normalizer_enforces_oss_compatible_bounded_retention() -> None:
assert "DELETE FROM contour_telemetry_samples" in NORMALIZER_SOURCE
assert "INTERVAL '30 days'" in NORMALIZER_SOURCE
assert "RETENTION_INTERVAL_SECONDS" in NORMALIZER_SOURCE
def test_normalizer_rejects_unschematized_pipeline_payload() -> None:
normalizer = _normalizer()
with pytest.raises(ValueError, match="schema"):
@@ -136,6 +142,99 @@ def test_normalizer_rejects_unschematized_pipeline_payload() -> None:
)
def test_normalizer_preserves_native_pipeline_stage_tags() -> None:
normalizer = _normalizer()
row = normalizer._normalize(
"mission-core/v1/contours/worker-006/agents/mission-core-worker/pipeline",
json.dumps(
{
"schema_version": "missioncore.agent-pipeline-telemetry/v1",
"node_id": "DESKTOP-OPJ8J04",
"observed_at_utc": "2026-07-28T12:00:00Z",
"lab_id": "E41",
"run_id": "run-001",
"tags": {
"node_id": "DESKTOP-OPJ8J04",
"contour_id": "worker-006",
"agent_id": "mission-core-worker",
"lab_id": "E41",
"run_id": "run-001",
"source_id": "ravnoves00",
"source_package_id": "e41-predictor-package-example",
"method_id": "frozen-e40-predictor/v1",
"stage_id": "predict",
"stage_state": "completed",
},
"payload": {"state": "ready"},
}
).encode(),
)
assert json.loads(row[6]) == {
"lab_id": "E41",
"method_id": "frozen-e40-predictor/v1",
"run_id": "run-001",
"source_id": "ravnoves00",
"source_package_id": "e41-predictor-package-example",
"stage_id": "predict",
"stage_state": "completed",
}
def test_normalizer_rejects_oversized_payload_and_series_identity() -> None:
normalizer = _normalizer()
with pytest.raises(ValueError, match="1 MiB"):
normalizer._normalize(
"mission-core/v1/contours/worker-006/agents/worker-006/host",
b"{" + b"x" * normalizer.MAX_PAYLOAD_BYTES + b"}",
)
with pytest.raises(ValueError, match="too many tags"):
normalizer._normalize(
"mission-core/v1/contours/worker-006/agents/worker-006/host",
json.dumps(
{
"name": "cpu",
"tags": {
"node_id": "DESKTOP-OPJ8J04",
**{
f"tag-{index}": str(index)
for index in range(normalizer.MAX_TAGS + 1)
},
},
"fields": {"usage_active": 12.5},
"timestamp": 1_785_179_600,
}
).encode(),
)
def test_normalizer_removes_unneeded_docker_labels_and_host_bind_paths() -> None:
normalizer = _normalizer()
row = normalizer._normalize(
"mission-core/v1/contours/worker-006/agents/worker-006/host",
json.dumps(
{
"name": "docker_container_cpu",
"tags": {
"node_id": "DESKTOP-OPJ8J04",
"container_name": "ndc-mission-core-triton",
"desktop.docker.io/binds/0/Source": "C:\\private\\model",
"com.nvidia.cuda.version": "12.8",
},
"fields": {"usage_percent": 10.0},
"timestamp": 1_785_179_600,
}
).encode(),
)
assert row[6] == '{"container_name":"ndc-mission-core-triton"}'
stored = json.loads(row[13])
assert stored["tags"] == {
"node_id": "DESKTOP-OPJ8J04",
"container_name": "ndc-mission-core-triton",
}
def test_telemetry_plane_uses_the_ndc_docker_namespace() -> None:
document = yaml.safe_load(COMPOSE_PATH.read_text(encoding="utf-8"))
@@ -146,6 +245,7 @@ def test_telemetry_plane_uses_the_ndc_docker_namespace() -> None:
for service in services.values()
} == {
"ndc-mission-core-mqtt-broker",
"ndc-mission-core-telemetry-bootstrap",
"ndc-mission-core-telemetry-normalizer",
"ndc-mission-core-telemetry-timescaledb",
}
@@ -154,6 +254,8 @@ def test_telemetry_plane_uses_the_ndc_docker_namespace() -> None:
assert service["labels"]["com.nodedc.product"] == "mission-core"
assert service["labels"]["com.nodedc.stack"] == "ndc-mission-core-telemetry"
assert services["broker"]["cap_drop"] == ["ALL"]
assert set(services["broker"]["cap_add"]) == {"CHOWN", "SETGID", "SETUID"}
assert document["networks"]["default"]["name"] == "ndc-mission-core-telemetry"
assert {
volume["name"]
tests/test_telemetry_plane_prepare.py
+60 -1
View File
@@ -45,10 +45,11 @@ def test_initialize_environment_generates_private_unique_secrets(
assert values["MISSIONCORE_MQTT_BIND_ADDRESS"] == "192.0.2.15"
secrets = {
values["MISSIONCORE_DB_PASSWORD"],
values["MISSIONCORE_DB_INGEST_PASSWORD"],
values["MISSIONCORE_MQTT_INGEST_PASSWORD"],
values["MISSIONCORE_MQTT_WORKER_006_PASSWORD"],
}
assert len(secrets) == 3
assert len(secrets) == 4
assert all(len(secret) >= 40 for secret in secrets)
assert stat.S_IMODE(env_path.stat().st_mode) == 0o600
@@ -65,3 +66,61 @@ def test_initialize_environment_refuses_to_replace_credentials(
prepare._initialize_environment("127.0.0.1")
assert env_path.read_text(encoding="utf-8") == "existing=true\n"
def test_environment_migration_adds_only_new_private_values(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
env_path = tmp_path / ".env"
env_path.write_text(
"MISSIONCORE_DB_PASSWORD=keep-me\n"
"MISSIONCORE_MQTT_WORKER_006_USER=worker-006\n",
encoding="utf-8",
)
monkeypatch.setattr(prepare, "ENV_PATH", env_path)
prepare._migrate_environment()
first = env_path.read_text(encoding="utf-8")
prepare._migrate_environment()
assert "MISSIONCORE_DB_PASSWORD=keep-me" in first
assert "MISSIONCORE_DB_INGEST_PASSWORD=" in first
assert "MISSIONCORE_MQTT_WORKER_006_CONTOUR=worker-006" in first
assert env_path.read_text(encoding="utf-8") == first
assert stat.S_IMODE(env_path.stat().st_mode) == 0o600
def test_existing_password_file_is_updated_without_recreation(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
password_path = tmp_path / "passwords"
password_path.write_text("existing", encoding="utf-8")
calls: list[tuple[str, bool]] = []
def capture(
path: Path,
username: str,
password: str,
*,
create: bool,
) -> None:
assert path == password_path
assert password
calls.append((username, create))
monkeypatch.setattr(prepare, "_password_entry", capture)
prepare._prepare_password_entries(
password_path,
"missioncore-ingest",
"ingest-secret",
"worker-006",
"worker-secret",
)
assert calls == [
("missioncore-ingest", False),
("worker-006", False),
]