From c9242480c244e2e6b066b3dc9eba472729e4a1dd Mon Sep 17 00:00:00 2001 From: DCCONSTRUCTIONS Date: Wed, 5 Aug 2026 14:51:33 +0300 Subject: [PATCH] refactor(perception): decouple replay from lab layout --- ...E_4_OBJECT_CENTRIC_RECORDED_REALTIME_CV.md | 9 ++ src/k1link/perception/__init__.py | 2 + src/k1link/perception/baseline.py | 88 +++++++++++++++---- src/k1link/perception/detector_replay_cli.py | 25 ++++-- .../perception/detector_replay_result.py | 2 + .../perception/detector_replay_validation.py | 3 + tests/test_perception_architecture.py | 56 ++++++++++++ 7 files changed, 159 insertions(+), 26 deletions(-) diff --git a/docs/22_MILESTONE_4_OBJECT_CENTRIC_RECORDED_REALTIME_CV.md b/docs/22_MILESTONE_4_OBJECT_CENTRIC_RECORDED_REALTIME_CV.md index 29dea6e..b1239e2 100644 --- a/docs/22_MILESTONE_4_OBJECT_CENTRIC_RECORDED_REALTIME_CV.md +++ b/docs/22_MILESTONE_4_OBJECT_CENTRIC_RECORDED_REALTIME_CV.md @@ -717,6 +717,11 @@ runner: - the baseline validator now freezes calibration, detector parameters, non-goals and the complete E15 rollback identity instead of merely retaining those fields in JSON; +- the deployable CLI no longer assumes a Git checkout or historical LAB layout: + baseline, camera summary/index and source-pack manifest/artifact are explicit + read-only inputs, and their exact file digests are verified before decoding; +- the result identity binds the exact baseline-profile file digest, so a + semantically similar but byte-different profile cannot reuse the receipt; - `require_m4_detector_replay_acceptance` refuses short smoke runs, another worker/node, failed frames, less than 4,489 frames or less than 10.004 FPS. @@ -730,6 +735,10 @@ Validation after adding the execution seam: 42 focused-and-related tests and the complete Python suite (`1230 passed, 1 skipped`). Scoped Ruff and strict mypy pass for the complete `src/k1link/perception` package and the frozen YOLOX primitive. +Portable-worker hardening adds two architecture checks and leaves the complete +suite at `1232 passed, 1 skipped`; scoped Ruff and strict mypy remain clean. This +hardening changes no durable Worker 006 process and makes no fresh capacity claim. + ## Implementation order The implementation sequence is intentionally strict: diff --git a/src/k1link/perception/__init__.py b/src/k1link/perception/__init__.py index f7fdca5..7d58a38 100644 --- a/src/k1link/perception/__init__.py +++ b/src/k1link/perception/__init__.py @@ -14,6 +14,7 @@ from .baseline import ( load_m4_baseline, validate_reuse_inventory, verify_m4_baseline, + verify_m4_execution_source, ) from .contracts import ( LOCAL_OBSTACLE_MAP_SCHEMA, @@ -120,6 +121,7 @@ __all__ = [ "BaselineVerification", "load_m4_baseline", "validate_reuse_inventory", + "verify_m4_execution_source", "verify_m4_baseline", "PerceptionAdapterError", "observations_from_track_geometry", diff --git a/src/k1link/perception/baseline.py b/src/k1link/perception/baseline.py index 281e4d2..1575b4f 100644 --- a/src/k1link/perception/baseline.py +++ b/src/k1link/perception/baseline.py @@ -24,15 +24,27 @@ BASELINE_RECORDED_JOB_ID: Final = "recorded-camera-602ac89026ed12978619801d" BASELINE_CAMERA_STREAM_SHA256: Final = ( "cadd1696ff000904eb78633a0a8418104b8024f178b91f3421789021ccb160e8" ) +BASELINE_CAMERA_SUMMARY_SHA256: Final = ( + "b280f40b198aad5d5335819107fb1405ad65d5695d187c61a2c027ad853a3181" +) +BASELINE_CAMERA_INDEX_SHA256: Final = ( + "e029815a60ad9fbfedb6169142c7449df2b119a51d1ce001f08806e04eb0be14" +) BASELINE_SOURCE_PACK_ID: Final = ( "e10-lidar-pack-576c994a6c814e2592dd6240ace3902a5db94843312c759a73ba0c9166157d2b" ) +BASELINE_SOURCE_PACK_MANIFEST_SHA256: Final = ( + "7e4af82eb7dfd47e242dc5207d55456cd0a5ab4c6780c584d4685ef854a73bf5" +) BASELINE_SOURCE_PACK_SHA256: Final = ( "0685d24219d8236caf8b7f1685e93f6d6b59e7fd015a768d88a92bbe8b154944" ) BASELINE_PREPROCESS_PROFILE_SHA256: Final = ( "19c17dbc23f2b1c539eb6b8214e69fa487ea3fec8db9d88768dffc714895fb88" ) +BASELINE_FILE_SHA256: Final = ( + "ea10359339e6cce31b5780a2710299771cab7cc0c1c2a2b56a1621f786b31fa8" +) _SHA256 = re.compile(r"^[a-f0-9]{64}$") _EXPECTED_EVIDENCE_ROLES: Final = { @@ -232,6 +244,8 @@ def load_m4_baseline(path: Path) -> BaselineProfile: rollback = _object(document.get("rollback"), "rollback") if rollback != _EXPECTED_ROLLBACK: raise BaselineContractError("rollback E15 identity changed") + if _file_sha256(path) != BASELINE_FILE_SHA256: + raise BaselineContractError("baseline file bytes changed") return BaselineProfile(path=path, document=document, evidence=evidence) @@ -240,7 +254,15 @@ def verify_m4_baseline(repository_root: Path, profile: BaselineProfile) -> Basel """Resolve every immutable evidence document and verify its exact digest.""" root = repository_root.resolve() - verified = list(_verify_source_artifacts(root, profile)) + source_paths = _repository_source_paths(root, profile) + verify_m4_execution_source( + profile, + camera_summary_path=source_paths[0], + camera_index_path=source_paths[1], + source_pack_manifest_path=source_paths[2], + source_pack_path=source_paths[3], + ) + verified = [str(path.relative_to(root)) for path in source_paths] for item in profile.evidence: evidence_path = (root / item.relative_path).resolve() if root not in evidence_path.parents: @@ -270,22 +292,34 @@ def verify_m4_baseline(repository_root: Path, profile: BaselineProfile) -> Basel ) -def _verify_source_artifacts(root: Path, profile: BaselineProfile) -> tuple[str, ...]: +def verify_m4_execution_source( + profile: BaselineProfile, + *, + camera_summary_path: Path, + camera_index_path: Path, + source_pack_manifest_path: Path, + source_pack_path: Path, +) -> tuple[Path, ...]: + """Verify only immutable source inputs required by a deployed replay runner.""" + source = _object(profile.document.get("source"), "source") - recorded_job_id = _string(source.get("recorded_job_id"), "recorded job id") camera_source_id = _string(source.get("camera_source_id"), "camera source id") source_pack_id = _string(source.get("source_pack_id"), "source pack id") expected_frames = _integer(source.get("frame_count"), "source frame count") - camera_root = ( - root - / ".runtime/compute-jobs" - / recorded_job_id - / "input/camera" - / camera_source_id - / "epoch-1" + source_inputs = ( + camera_summary_path, + camera_index_path, + source_pack_manifest_path, + source_pack_path, ) - summary_path = camera_root / "summary.json" - index_path = camera_root / "index.jsonl" + if any(path.is_symlink() or not path.is_file() for path in source_inputs): + raise BaselineContractError("execution source inputs must be regular non-symlink files") + paths = tuple(path.resolve(strict=True) for path in source_inputs) + summary_path, index_path, pack_manifest_path, pack_artifact_path = paths + if _file_sha256(summary_path) != BASELINE_CAMERA_SUMMARY_SHA256: + raise BaselineContractError("camera recording summary digest changed") + if _file_sha256(index_path) != BASELINE_CAMERA_INDEX_SHA256: + raise BaselineContractError("camera recording index digest changed") summary = _read_object(summary_path) if summary.get("schema_version") != "missioncore.camera-recording/v1": raise BaselineContractError("camera recording summary schema changed") @@ -300,9 +334,8 @@ def _verify_source_artifacts(root: Path, profile: BaselineProfile) -> tuple[str, if _file_sha256(index_path) != summary.get("index_sha256"): raise BaselineContractError("camera recording index digest changed") - pack_root = root / ".runtime/compute-experiments/e10/lidar-packs" / source_pack_id - pack_manifest_path = pack_root / "manifest.json" - pack_artifact_path = pack_root / "lidar-pack.npz" + if _file_sha256(pack_manifest_path) != BASELINE_SOURCE_PACK_MANIFEST_SHA256: + raise BaselineContractError("source pack manifest file digest changed") pack_manifest = _read_object(pack_manifest_path) if pack_manifest.get("schema_version") != "missioncore.e10-lidar-replay-pack/v1": raise BaselineContractError("source pack schema changed") @@ -320,9 +353,28 @@ def _verify_source_artifacts(root: Path, profile: BaselineProfile) -> tuple[str, raise BaselineContractError("source pack manifest digest changed") if _file_sha256(pack_artifact_path) != expected_pack_sha: raise BaselineContractError("source pack artifact digest changed") - return tuple( - str(path.relative_to(root)) - for path in (summary_path, index_path, pack_manifest_path, pack_artifact_path) + return paths + + +def _repository_source_paths(root: Path, profile: BaselineProfile) -> tuple[Path, ...]: + source = _object(profile.document.get("source"), "source") + recorded_job_id = _string(source.get("recorded_job_id"), "recorded job id") + camera_source_id = _string(source.get("camera_source_id"), "camera source id") + source_pack_id = _string(source.get("source_pack_id"), "source pack id") + camera_root = ( + root + / ".runtime/compute-jobs" + / recorded_job_id + / "input/camera" + / camera_source_id + / "epoch-1" + ) + pack_root = root / ".runtime/compute-experiments/e10/lidar-packs" / source_pack_id + return ( + camera_root / "summary.json", + camera_root / "index.jsonl", + pack_root / "manifest.json", + pack_root / "lidar-pack.npz", ) diff --git a/src/k1link/perception/detector_replay_cli.py b/src/k1link/perception/detector_replay_cli.py index 9a328b0..460b535 100644 --- a/src/k1link/perception/detector_replay_cli.py +++ b/src/k1link/perception/detector_replay_cli.py @@ -14,7 +14,7 @@ from k1link.compute.yolox_object_detector import ( load_valid_fov_mask, ) -from .baseline import load_m4_baseline, verify_m4_baseline +from .baseline import load_m4_baseline, verify_m4_execution_source from .detector import FrozenYoloxDetectorProvider from .detector_replay import run_detector_replay from .detector_replay_result import ( @@ -34,7 +34,11 @@ def _arguments() -> argparse.Namespace: parser = argparse.ArgumentParser( description="Run the product-neutral M4 detector replay/capacity gate." ) - parser.add_argument("--repository-root", type=Path, required=True) + parser.add_argument("--baseline", type=Path, required=True) + parser.add_argument("--camera-summary", type=Path, required=True) + parser.add_argument("--camera-index", type=Path, required=True) + parser.add_argument("--source-pack-manifest", type=Path, required=True) + parser.add_argument("--source-pack", type=Path, required=True) parser.add_argument("--video", type=Path, required=True) parser.add_argument("--valid-fov-mask", type=Path, required=True) parser.add_argument("--triton-origin", required=True, type=_loopback_triton_origin) @@ -59,16 +63,21 @@ def _loopback_triton_origin(value: str) -> str: def main() -> int: args = _arguments() - repository_root = args.repository_root.resolve(strict=True) - baseline = load_m4_baseline( - repository_root / "config/perception/m4-recorded-realtime-baseline-v1.json" + baseline = load_m4_baseline(args.baseline) + source_paths = verify_m4_execution_source( + baseline, + camera_summary_path=args.camera_summary, + camera_index_path=args.camera_index, + source_pack_manifest_path=args.source_pack_manifest, + source_pack_path=args.source_pack, ) - verify_m4_baseline(repository_root, baseline) + _, camera_index_path, _, source_pack_path = source_paths runtime_value = json.loads(args.runtime_identity.resolve(strict=True).read_text("utf-8")) runtime = DetectorRuntimeIdentity.from_dict(runtime_value) source = DecodedRecordedSource( - source=RecordedRavnoves00Source.from_repository( - repository_root, + source=RecordedRavnoves00Source( + camera_index_path=camera_index_path, + source_pack_path=source_pack_path, pacing=ReplayPacing.UNCAPPED, ), decoder=PyAvRecordedImageDecoder(args.video), diff --git a/src/k1link/perception/detector_replay_result.py b/src/k1link/perception/detector_replay_result.py index fa99072..4c3f7fb 100644 --- a/src/k1link/perception/detector_replay_result.py +++ b/src/k1link/perception/detector_replay_result.py @@ -20,6 +20,7 @@ from k1link.compute.yolox_object_detector import ( from .baseline import ( BASELINE_CAMERA_STREAM_SHA256, + BASELINE_FILE_SHA256, BASELINE_PREPROCESS_PROFILE_SHA256, BASELINE_PROFILE_ID, BASELINE_RECORDED_JOB_ID, @@ -83,6 +84,7 @@ def seal_detector_replay_result( identity = { "schema_version": DETECTOR_REPLAY_RESULT_SCHEMA, "baseline_profile_id": BASELINE_PROFILE_ID, + "baseline_profile_sha256": BASELINE_FILE_SHA256, "source": { "provider_id": RECORDED_SOURCE_PROVIDER_ID, "source_id": BASELINE_SOURCE_ID, diff --git a/src/k1link/perception/detector_replay_validation.py b/src/k1link/perception/detector_replay_validation.py index 957cca7..ab1099b 100644 --- a/src/k1link/perception/detector_replay_validation.py +++ b/src/k1link/perception/detector_replay_validation.py @@ -16,6 +16,7 @@ from k1link.compute.yolox_object_detector import ( from .baseline import ( BASELINE_CAMERA_STREAM_SHA256, + BASELINE_FILE_SHA256, BASELINE_PREPROCESS_PROFILE_SHA256, BASELINE_PROFILE_ID, BASELINE_RECORDED_JOB_ID, @@ -179,6 +180,7 @@ def _validate_identity(identity: dict[str, object]) -> None: { "schema_version", "baseline_profile_id", + "baseline_profile_sha256", "source", "detector", "runtime", @@ -196,6 +198,7 @@ def _validate_identity(identity: dict[str, object]) -> None: if ( identity.get("schema_version") != DETECTOR_REPLAY_RESULT_SCHEMA or identity.get("baseline_profile_id") != BASELINE_PROFILE_ID + or identity.get("baseline_profile_sha256") != BASELINE_FILE_SHA256 or source != { "provider_id": RECORDED_SOURCE_PROVIDER_ID, diff --git a/tests/test_perception_architecture.py b/tests/test_perception_architecture.py index 5512357..3820b10 100644 --- a/tests/test_perception_architecture.py +++ b/tests/test_perception_architecture.py @@ -12,6 +12,7 @@ from k1link.perception.baseline import ( load_m4_baseline, validate_reuse_inventory, verify_m4_baseline, + verify_m4_execution_source, ) REPOSITORY_ROOT = Path(__file__).resolve().parents[1] @@ -39,6 +40,61 @@ def test_m4_baseline_is_exact_and_every_local_evidence_digest_resolves() -> None assert len(verification.verified_paths) == 10 +def test_worker_execution_source_verification_needs_no_historical_lab_tree() -> None: + profile = load_m4_baseline(BASELINE_PATH) + camera_root = ( + REPOSITORY_ROOT + / ".runtime/compute-jobs/recorded-camera-602ac89026ed12978619801d" + / "input/camera/sensor.camera.right/epoch-1" + ) + pack_root = ( + REPOSITORY_ROOT + / ".runtime/compute-experiments/e10/lidar-packs" + / "e10-lidar-pack-576c994a6c814e2592dd6240ace3902a5db94843312c759a73ba0c9166157d2b" + ) + + verified = verify_m4_execution_source( + profile, + camera_summary_path=camera_root / "summary.json", + camera_index_path=camera_root / "index.jsonl", + source_pack_manifest_path=pack_root / "manifest.json", + source_pack_path=pack_root / "lidar-pack.npz", + ) + + assert tuple(path.name for path in verified) == ( + "summary.json", + "index.jsonl", + "manifest.json", + "lidar-pack.npz", + ) + + +def test_worker_execution_source_rejects_rewritten_manifest_bytes(tmp_path: Path) -> None: + profile = load_m4_baseline(BASELINE_PATH) + camera_root = ( + REPOSITORY_ROOT + / ".runtime/compute-jobs/recorded-camera-602ac89026ed12978619801d" + / "input/camera/sensor.camera.right/epoch-1" + ) + pack_root = ( + REPOSITORY_ROOT + / ".runtime/compute-experiments/e10/lidar-packs" + / "e10-lidar-pack-576c994a6c814e2592dd6240ace3902a5db94843312c759a73ba0c9166157d2b" + ) + rewritten_manifest = tmp_path / "manifest.json" + document = json.loads((pack_root / "manifest.json").read_text("utf-8")) + rewritten_manifest.write_text(json.dumps(document), "utf-8") + + with pytest.raises(BaselineContractError, match="manifest file digest"): + verify_m4_execution_source( + profile, + camera_summary_path=camera_root / "summary.json", + camera_index_path=camera_root / "index.jsonl", + source_pack_manifest_path=rewritten_manifest, + source_pack_path=pack_root / "lidar-pack.npz", + ) + + def test_m4_baseline_cannot_silently_select_another_source(tmp_path: Path) -> None: document = json.loads(BASELINE_PATH.read_text("utf-8")) incompatible = copy.deepcopy(document)