refactor(platform): freeze laboratory and telemetry boundaries

This commit is contained in:
DCCONSTRUCTIONS
2026-08-05 12:29:06 +03:00
parent 6d0abbc569
commit 1b3e0b3406
22 changed files with 1657 additions and 148 deletions
+5 -1
View File
@@ -127,9 +127,13 @@ def test_product_registry_declares_every_advanced_evidence_source() -> None:
repository_root / "config" / "laboratories"
)
assert len(registry.definitions) == 27
assert len(registry.definitions) == 31
assert {item.work_id for item in registry.definitions} >= {
"e31-source-binding",
"e46j-raw-fisheye-realtime",
"l3-pointpillars-visual-audit",
"l31-pointpillars-ravnoves",
"l32-pointpillars-camera-review",
"l33-camera-first-detector-review",
"l34f-adjudicated-reference",
}
+193
View File
@@ -0,0 +1,193 @@
from __future__ import annotations
import hashlib
import json
from dataclasses import replace
from pathlib import Path
import pytest
from k1link.compute.pipeline_telemetry import JsonlPipelineTelemetrySink
from k1link.laboratory import (
LaboratoryAdapterResult,
LaboratoryEvidenceRegistry,
LaboratoryExecutionDefinition,
LaboratoryExecutionError,
LaboratoryExecutionRegistry,
LaboratoryRunner,
LaboratoryRunRequest,
)
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
def _registries() -> tuple[LaboratoryEvidenceRegistry, LaboratoryExecutionRegistry]:
evidence = LaboratoryEvidenceRegistry.from_directory(
REPOSITORY_ROOT / "config" / "laboratories"
)
execution = LaboratoryExecutionRegistry.from_file(
REPOSITORY_ROOT / "config" / "laboratory-execution.json",
evidence,
)
return evidence, execution
def _evidence_result(root: Path, *, work_id: str) -> LaboratoryAdapterResult:
evidence, _ = _registries()
definition = next(row for row in evidence.definitions if row.work_id == work_id)
artifact = b"canonical evidence\n"
identity = {
"schema_version": definition.result_schema_version,
"source": {"id": "fixture-source"},
"authority": {
"commands_enabled": False,
"navigation_or_safety_accepted": False,
},
}
identity_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest()
result_id = f"{definition.result_id_prefix}-{identity_sha256}"
result_root = root / result_id
result_root.mkdir(parents=True, exist_ok=True)
(result_root / "artifact.txt").write_bytes(artifact)
document = {
"schema_version": definition.result_schema_version,
"result_id": result_id,
"identity_sha256": identity_sha256,
"identity": identity,
"artifacts": [
{
"role": "fixture",
"path": "artifact.txt",
"byte_length": len(artifact),
"sha256": hashlib.sha256(artifact).hexdigest(),
}
],
}
(result_root / definition.document_name).write_bytes(_canonical_json(document) + b"\n")
return LaboratoryAdapterResult(result_root=result_root, result_id=result_id)
def _request(tmp_path: Path) -> LaboratoryRunRequest:
package = tmp_path / "input-package"
package.mkdir()
return LaboratoryRunRequest(
work_id="e33-worker-shadow",
run_id="run-001",
request_id="request-001",
contour_id="worker-006",
agent_id="worker-006",
node_id="DESKTOP-OPJ8J04",
source_id="e32-fixture",
source_package_id="e32-package-fixture",
method_id="e33-worker-shadow/v1",
inputs={"package_root": package},
output_root=tmp_path / "results",
receipt_root=tmp_path / "receipts",
)
def test_repository_registry_classifies_every_evidence_definition() -> None:
evidence, execution = _registries()
assert {row.work_id for row in execution.definitions} == {
"e33-worker-shadow",
"e35-degradation-recovery",
"e46j-raw-fisheye-realtime",
}
assert all(row.lifecycle == "canonical" for row in execution.definitions)
assert len(execution.definitions) + len(execution.legacy_work_ids) == len(
evidence.definitions
)
def test_legacy_is_read_only_and_experimental_requires_bounded_adapter() -> None:
_, execution = _registries()
with pytest.raises(LaboratoryExecutionError, match="read-only"):
execution.executable("e31-source-binding")
with pytest.raises(LaboratoryExecutionError, match="bounded-adapter"):
LaboratoryExecutionDefinition(
work_id="new-experiment",
lifecycle="experimental",
isolation="core-adapter",
adapter_id="experimental.new-experiment/v1",
input_roles=("source_root",),
source_contract="missioncore.experimental-source/v1",
provider_contract="missioncore.experimental-provider/v1",
graph_contract="missioncore.experimental-graph/v1",
run_contract="missioncore.laboratory-run/v1",
evidence_contract="missioncore.experimental-result/v1",
)
def test_runner_uses_common_telemetry_evidence_and_immutable_receipt(
tmp_path: Path,
) -> None:
evidence, execution = _registries()
request = _request(tmp_path)
def adapter(run_request: LaboratoryRunRequest) -> LaboratoryAdapterResult:
assert run_request == request
return _evidence_result(run_request.output_root, work_id=run_request.work_id)
telemetry_path = tmp_path / "pipeline.jsonl"
runner = LaboratoryRunner(
registry=execution,
evidence_registry=evidence,
sink=JsonlPipelineTelemetrySink(telemetry_path),
adapters={"canonical.e33-worker-shadow/v1": adapter},
)
first = runner.run(request)
second = runner.run(request)
assert first.receipt_id == second.receipt_id
assert first.receipt_root == second.receipt_root
assert first.receipt["result_id"] == first.result_id
assert first.receipt["authority"]["commands_enabled"] is False
receipt = json.loads((first.receipt_root / "receipt.json").read_text())
assert receipt == first.receipt
records = [json.loads(line) for line in telemetry_path.read_text().splitlines()]
states = [row["payload"]["payload"]["event"]["state"] for row in records]
assert states.count("started") == 8
assert states.count("completed") == 8
assert all(row["payload"]["lab_id"] == request.work_id for row in records)
def test_runner_rejects_undeclared_input_before_adapter(tmp_path: Path) -> None:
evidence, execution = _registries()
request = _request(tmp_path)
request = replace(
request,
inputs={
**request.inputs,
"hidden_profile": request.inputs["package_root"],
},
)
called = False
def adapter(_: LaboratoryRunRequest) -> LaboratoryAdapterResult:
nonlocal called
called = True
raise AssertionError("must not run")
runner = LaboratoryRunner(
registry=execution,
evidence_registry=evidence,
sink=JsonlPipelineTelemetrySink(tmp_path / "pipeline.jsonl"),
adapters={"canonical.e33-worker-shadow/v1": adapter},
)
with pytest.raises(LaboratoryExecutionError, match="unexpected=.*hidden_profile"):
runner.run(request)
assert called is False
def _canonical_json(value: object) -> bytes:
return json.dumps(
value,
sort_keys=True,
separators=(",", ":"),
ensure_ascii=False,
).encode("utf-8")
+36
View File
@@ -1,5 +1,6 @@
from __future__ import annotations
import hashlib
import importlib.util
import json
from pathlib import Path
@@ -8,6 +9,7 @@ from types import ModuleType, SimpleNamespace
import pytest
from k1link.compute.pipeline_telemetry import (
MAX_PAYLOAD_BYTES,
JsonlPipelineTelemetrySink,
MqttPipelineTelemetrySink,
PipelineTelemetryEmitter,
@@ -231,6 +233,40 @@ def test_jsonl_sink_records_topic_bound_documents(tmp_path: Path) -> None:
assert path.stat().st_mode & 0o077 == 0
def test_jsonl_sink_rotates_to_content_addressed_segment(tmp_path: Path) -> None:
path = tmp_path / "pipeline-telemetry.jsonl"
maximum = MAX_PAYLOAD_BYTES + 4096
existing = (b"{}\n" * (maximum // 3))[: maximum - 32]
path.write_bytes(existing)
sink = JsonlPipelineTelemetrySink(path, max_bytes=maximum, max_segments=2)
sink.publish(_identity().topic, b"{}")
segments = tuple(tmp_path.glob("pipeline-telemetry.*.jsonl"))
assert len(segments) == 1
assert segments[0].read_bytes() == existing
assert segments[0].stem.split(".")[-1] == hashlib.sha256(existing).hexdigest()
assert json.loads(path.read_text())["payload"] == {}
def test_jsonl_sink_refuses_to_delete_unacknowledged_segment_at_bound(
tmp_path: Path,
) -> None:
path = tmp_path / "pipeline-telemetry.jsonl"
maximum = MAX_PAYLOAD_BYTES + 4096
active = b"x" * maximum
path.write_bytes(active)
retained = tmp_path / f"pipeline-telemetry.{'a' * 64}.jsonl"
retained.write_bytes(b"retained")
sink = JsonlPipelineTelemetrySink(path, max_bytes=maximum, max_segments=1)
with pytest.raises(PipelineTelemetryError, match="segment bound reached"):
sink.publish(_identity().topic, b"{}")
assert path.read_bytes() == active
assert retained.read_bytes() == b"retained"
def test_mqtt_sink_uses_qos_one_without_retention() -> None:
calls: list[tuple[str, bytes, int, bool]] = []
+142 -31
View File
@@ -1,7 +1,10 @@
from __future__ import annotations
import importlib.util
import json
import stat
import sys
import zipfile
from pathlib import Path
from types import ModuleType
@@ -22,6 +25,7 @@ def _prepare_module() -> ModuleType:
assert specification is not None
assert specification.loader is not None
module = importlib.util.module_from_spec(specification)
sys.modules[specification.name] = module
specification.loader.exec_module(module)
return module
@@ -29,74 +33,122 @@ def _prepare_module() -> ModuleType:
prepare = _prepare_module()
def test_initialize_environment_generates_private_unique_secrets(
def _private_paths(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(prepare, "ENV_PATH", tmp_path / ".env")
monkeypatch.setattr(prepare, "RUNTIME_ROOT", tmp_path / "runtime")
monkeypatch.setattr(prepare, "RUNTIME", tmp_path / "runtime" / "mosquitto")
monkeypatch.setattr(
prepare,
"AGENT_REGISTRY_PATH",
tmp_path / "runtime" / "agents.json",
)
def _write_environment(path: Path) -> None:
path.write_text(
"MISSIONCORE_MQTT_BIND_ADDRESS=192.168.68.52\n"
"MISSIONCORE_MQTT_PORT=1883\n"
"MISSIONCORE_DB_PASSWORD=db-secret\n"
"MISSIONCORE_DB_INGEST_PASSWORD=db-ingest-secret\n"
"MISSIONCORE_MQTT_INGEST_USER=missioncore-ingest\n"
"MISSIONCORE_MQTT_INGEST_PASSWORD=mqtt-ingest-secret\n",
encoding="utf-8",
)
def test_initialize_environment_generates_only_plane_secrets(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
env_path = tmp_path / ".env"
monkeypatch.setattr(prepare, "ENV_PATH", env_path)
_private_paths(tmp_path, monkeypatch)
prepare._initialize_environment("192.0.2.15")
values = dict(
line.split("=", 1)
for line in env_path.read_text(encoding="utf-8").splitlines()
for line in prepare.ENV_PATH.read_text(encoding="utf-8").splitlines()
)
assert values["MISSIONCORE_MQTT_BIND_ADDRESS"] == "192.0.2.15"
secrets = {
generated = {
values["MISSIONCORE_DB_PASSWORD"],
values["MISSIONCORE_DB_INGEST_PASSWORD"],
values["MISSIONCORE_MQTT_INGEST_PASSWORD"],
values["MISSIONCORE_MQTT_WORKER_006_PASSWORD"],
}
assert len(secrets) == 4
assert all(len(secret) >= 40 for secret in secrets)
assert stat.S_IMODE(env_path.stat().st_mode) == 0o600
assert len(generated) == 3
assert all(len(secret) >= 40 for secret in generated)
assert not any("WORKER_006" in name for name in values)
assert stat.S_IMODE(prepare.ENV_PATH.stat().st_mode) == 0o600
def test_initialize_environment_refuses_to_replace_credentials(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
env_path = tmp_path / ".env"
env_path.write_text("existing=true\n", encoding="utf-8")
monkeypatch.setattr(prepare, "ENV_PATH", env_path)
_private_paths(tmp_path, monkeypatch)
prepare.ENV_PATH.write_text("existing=true\n", encoding="utf-8")
with pytest.raises(RuntimeError, match="refusing to overwrite"):
prepare._initialize_environment("127.0.0.1")
assert env_path.read_text(encoding="utf-8") == "existing=true\n"
assert prepare.ENV_PATH.read_text(encoding="utf-8") == "existing=true\n"
def test_environment_migration_adds_only_new_private_values(
def test_migration_moves_legacy_worker_to_private_registry(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
env_path = tmp_path / ".env"
env_path.write_text(
_private_paths(tmp_path, monkeypatch)
prepare.ENV_PATH.write_text(
"MISSIONCORE_DB_PASSWORD=keep-me\n"
"MISSIONCORE_MQTT_WORKER_006_USER=worker-006\n",
"MISSIONCORE_MQTT_WORKER_006_USER=worker-006\n"
"MISSIONCORE_MQTT_WORKER_006_CONTOUR=worker-006\n"
f"MISSIONCORE_MQTT_WORKER_006_PASSWORD={'l' * 40}\n",
encoding="utf-8",
)
monkeypatch.setattr(prepare, "ENV_PATH", env_path)
prepare._migrate_environment()
first = env_path.read_text(encoding="utf-8")
first_env = prepare.ENV_PATH.read_text(encoding="utf-8")
first_registry = prepare.AGENT_REGISTRY_PATH.read_bytes()
prepare._migrate_environment()
assert "MISSIONCORE_DB_PASSWORD=keep-me" in first
assert "MISSIONCORE_DB_INGEST_PASSWORD=" in first
assert "MISSIONCORE_MQTT_WORKER_006_CONTOUR=worker-006" in first
assert env_path.read_text(encoding="utf-8") == first
assert stat.S_IMODE(env_path.stat().st_mode) == 0o600
assert "MISSIONCORE_DB_PASSWORD=keep-me" in first_env
assert "MISSIONCORE_DB_INGEST_PASSWORD=" in first_env
assert prepare.ENV_PATH.read_text(encoding="utf-8") == first_env
assert prepare.AGENT_REGISTRY_PATH.read_bytes() == first_registry
assert prepare._read_agent_registry() == (
prepare.AgentCredential("worker-006", "worker-006", "l" * 40),
)
assert stat.S_IMODE(prepare.AGENT_REGISTRY_PATH.stat().st_mode) == 0o600
def test_existing_password_file_is_updated_without_recreation(
def test_enrollment_is_generic_private_and_rejects_duplicate_agent_id(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
password_path = tmp_path / "passwords"
password_path.write_text("existing", encoding="utf-8")
_private_paths(tmp_path, monkeypatch)
first = prepare._enroll_agent("compute-east", "worker-006")
second = prepare._enroll_agent("compute-west", "worker-007")
assert first.password != second.password
assert prepare._read_agent_registry() == (first, second)
assert stat.S_IMODE(prepare.AGENT_REGISTRY_PATH.stat().st_mode) == 0o600
with pytest.raises(RuntimeError, match="already enrolled"):
prepare._enroll_agent("another-contour", "worker-006")
def test_password_file_is_rebuilt_and_acl_is_exactly_scoped(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
_private_paths(tmp_path, monkeypatch)
prepare.RUNTIME.mkdir(parents=True)
password_path = prepare.RUNTIME / "passwords"
password_path.write_text("stale-user", encoding="utf-8")
agents = (
prepare.AgentCredential("compute-west", "worker-007", "w" * 40),
prepare.AgentCredential("compute-east", "worker-006", "e" * 40),
)
calls: list[tuple[str, bool]] = []
def capture(
@@ -106,9 +158,9 @@ def test_existing_password_file_is_updated_without_recreation(
*,
create: bool,
) -> None:
assert path == password_path
assert password
calls.append((username, create))
path.write_text("new-password-file", encoding="utf-8")
monkeypatch.setattr(prepare, "_password_entry", capture)
@@ -116,11 +168,70 @@ def test_existing_password_file_is_updated_without_recreation(
password_path,
"missioncore-ingest",
"ingest-secret",
"worker-006",
"worker-secret",
agents,
)
assert calls == [
("missioncore-ingest", False),
("missioncore-ingest", True),
("worker-006", False),
("worker-007", False),
]
assert password_path.read_text() == "new-password-file"
acl = prepare._acl_document("missioncore-ingest", agents)
assert "topic write mission-core/v1/contours/compute-east/agents/worker-006/+" in acl
assert "topic write mission-core/v1/contours/compute-west/agents/worker-007/+" in acl
assert "topic write mission-core/v1/contours/+/agents/+/+" not in acl
def test_payload_export_is_private_separate_and_non_overwriting(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
_private_paths(tmp_path, monkeypatch)
_write_environment(prepare.ENV_PATH)
credential = prepare._enroll_agent("compute-east", "worker-007")
output = tmp_path / "worker-007.private.json"
prepare._export_agent_payload(
agent_id="worker-007",
node_id="WORKSTATION-007",
output=output,
)
payload = json.loads(output.read_text())
assert payload["schema_version"] == prepare.AGENT_PAYLOAD_SCHEMA
assert payload["MISSIONCORE_CONTOUR_ID"] == "compute-east"
assert payload["MISSIONCORE_AGENT_ID"] == "worker-007"
assert payload["MISSIONCORE_MQTT_PASSWORD"] == credential.password
assert stat.S_IMODE(output.stat().st_mode) == 0o600
with pytest.raises(RuntimeError, match="refusing to replace"):
prepare._export_agent_payload(
agent_id="worker-007",
node_id="WORKSTATION-007",
output=output,
)
def test_windows_agent_bundle_is_deterministic_and_contains_no_credentials(
tmp_path: Path,
) -> None:
first = tmp_path / "agent-a.zip"
second = tmp_path / "agent-b.zip"
first_id = prepare._build_agent_bundle(platform_name="windows", output=first)
second_id = prepare._build_agent_bundle(platform_name="windows", output=second)
assert first_id == second_id
assert first.read_bytes() == second.read_bytes()
with zipfile.ZipFile(first) as archive:
assert set(archive.namelist()) == {
"manifest.json",
*prepare.WINDOWS_BUNDLE_FILES,
}
manifest = json.loads(archive.read("manifest.json"))
assert manifest["bundle_id"] == first_id
assert manifest["credential_embedded"] is False
archive_bytes = b"".join(archive.read(name) for name in archive.namelist())
assert b"replace-with-a-random-local-secret" not in archive_bytes
assert b"legacy-worker-secret" not in archive_bytes
assert stat.S_IMODE(first.stat().st_mode) == 0o600