feat(simulation): add S1 worker lifecycle ownership
This commit is contained in:
DCCONSTRUCTIONS
@@ -0,0 +1,354 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path, PurePosixPath
|
||||
|
||||
import pytest
|
||||
|
||||
from k1link.simulation import (
|
||||
ActiveQualificationRunError,
|
||||
AuthorityProfile,
|
||||
PosixProcessSupervisor,
|
||||
ProcessSpec,
|
||||
ProcessSupervisorError,
|
||||
ProviderPin,
|
||||
QualificationRun,
|
||||
QualificationRunConflictError,
|
||||
QualificationRunStore,
|
||||
ReproducibilityTier,
|
||||
RunKind,
|
||||
RunState,
|
||||
S0WorkerGuard,
|
||||
SimulationApplicationService,
|
||||
WorkerAdmissionError,
|
||||
WorkerStartResult,
|
||||
WorkerStopResult,
|
||||
)
|
||||
|
||||
PROFILE = Path(__file__).resolve().parents[1] / "simulation/s0/qualification-profile.yaml"
|
||||
SHA_A = "a" * 64
|
||||
SHA_B = "b" * 64
|
||||
|
||||
|
||||
def _run(
|
||||
run_id: str = "run-s1b-001",
|
||||
episode_id: str = "episode-s1b-001",
|
||||
*,
|
||||
host_profile_sha256: str | None = None,
|
||||
) -> QualificationRun:
|
||||
return QualificationRun(
|
||||
run_id=run_id,
|
||||
episode_id=episode_id,
|
||||
kind=RunKind.SIMULATION_CLOSED_LOOP,
|
||||
state=RunState.ADMITTED,
|
||||
scenario_generation="stock-rover-v1",
|
||||
scenario_sha256=SHA_A,
|
||||
profile_generation="s1-ackermann-v1",
|
||||
profile_sha256=SHA_B,
|
||||
mission_core_commit="12fba7126d247f809f8d135c5f68922eb08d4683",
|
||||
providers=(
|
||||
ProviderPin("px4-autopilot", "v1.17.0", "v1.17.0"),
|
||||
ProviderPin("gazebo", "harmonic", "8.14.0"),
|
||||
),
|
||||
host_profile_id="ai-worker-px4-gazebo-v1",
|
||||
host_profile_sha256=host_profile_sha256 or ("c" * 64),
|
||||
seed=42,
|
||||
reproducibility_tier=ReproducibilityTier.R1,
|
||||
authority=AuthorityProfile(
|
||||
generation=1,
|
||||
command_ttl_max_ns=250_000_000,
|
||||
heartbeat_timeout_monotonic_ns=500_000_000,
|
||||
),
|
||||
clock_domain="gazebo:/clock",
|
||||
created_at_utc="2026-07-24T18:00:00Z",
|
||||
)
|
||||
|
||||
|
||||
class _FakeWorker:
|
||||
def __init__(self, profile_sha256: str = "c" * 64) -> None:
|
||||
self.profile_sha256 = profile_sha256
|
||||
self.calls: list[str] = []
|
||||
self.fail_start = False
|
||||
self.residue: tuple[str, ...] = ()
|
||||
|
||||
def start(self, run: QualificationRun) -> WorkerStartResult:
|
||||
self.calls.append(f"start:{run.run_id}")
|
||||
if self.fail_start:
|
||||
raise RuntimeError("synthetic start failure")
|
||||
return WorkerStartResult(("gazebo-server", "px4-sitl"), self.profile_sha256)
|
||||
|
||||
def pause(self, run: QualificationRun) -> None:
|
||||
self.calls.append(f"pause:{run.run_id}")
|
||||
|
||||
def resume(self, run: QualificationRun) -> None:
|
||||
self.calls.append(f"resume:{run.run_id}")
|
||||
|
||||
def step(self, run: QualificationRun, step_count: int) -> int:
|
||||
self.calls.append(f"step:{run.run_id}:{step_count}")
|
||||
return step_count * 2_000_000
|
||||
|
||||
def stop(self, run: QualificationRun) -> WorkerStopResult:
|
||||
self.calls.append(f"stop:{run.run_id}")
|
||||
return WorkerStopResult(("px4-sitl", "gazebo-server"), self.residue)
|
||||
|
||||
def reconcile(self, run: QualificationRun) -> WorkerStopResult:
|
||||
self.calls.append(f"reconcile:{run.run_id}")
|
||||
return WorkerStopResult(("px4-sitl", "gazebo-server"), self.residue)
|
||||
|
||||
|
||||
def _service(tmp_path: Path) -> tuple[SimulationApplicationService, _FakeWorker]:
|
||||
store = QualificationRunStore(tmp_path)
|
||||
store.create(_run())
|
||||
worker = _FakeWorker()
|
||||
return SimulationApplicationService(store, worker), worker
|
||||
|
||||
|
||||
def test_application_service_owns_idempotent_start_pause_step_resume_stop(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
service, worker = _service(tmp_path)
|
||||
running = service.start(
|
||||
"run-s1b-001",
|
||||
idempotency_key="start-001",
|
||||
observed_at_utc="2026-07-24T18:00:01Z",
|
||||
host_monotonic_ns=1,
|
||||
)
|
||||
repeated = service.start(
|
||||
"run-s1b-001",
|
||||
idempotency_key="start-001",
|
||||
observed_at_utc="2026-07-24T18:00:02Z",
|
||||
host_monotonic_ns=2,
|
||||
)
|
||||
paused = service.pause(
|
||||
running.run_id,
|
||||
observed_at_utc="2026-07-24T18:00:03Z",
|
||||
host_monotonic_ns=3,
|
||||
sim_time_ns=2_000_000,
|
||||
)
|
||||
advanced = service.step(
|
||||
running.run_id,
|
||||
step_count=2,
|
||||
observed_at_utc="2026-07-24T18:00:04Z",
|
||||
host_monotonic_ns=4,
|
||||
sim_time_ns=2_000_000,
|
||||
)
|
||||
resumed = service.resume(
|
||||
running.run_id,
|
||||
observed_at_utc="2026-07-24T18:00:05Z",
|
||||
host_monotonic_ns=5,
|
||||
sim_time_ns=6_000_000,
|
||||
)
|
||||
completed = service.stop(
|
||||
running.run_id,
|
||||
idempotency_key="stop-001",
|
||||
observed_at_utc="2026-07-24T18:00:06Z",
|
||||
host_monotonic_ns=6,
|
||||
sim_time_ns=6_000_000,
|
||||
)
|
||||
repeated_stop = service.stop(
|
||||
running.run_id,
|
||||
idempotency_key="stop-001",
|
||||
observed_at_utc="2026-07-24T18:00:07Z",
|
||||
host_monotonic_ns=7,
|
||||
sim_time_ns=6_000_000,
|
||||
)
|
||||
|
||||
assert running.state is RunState.RUNNING
|
||||
assert repeated.state is RunState.RUNNING
|
||||
assert paused.state is RunState.PAUSED
|
||||
assert advanced == 4_000_000
|
||||
assert resumed.state is RunState.RUNNING
|
||||
assert completed.state is RunState.COMPLETED
|
||||
assert repeated_stop == completed
|
||||
assert worker.calls.count("start:run-s1b-001") == 1
|
||||
assert worker.calls.count("stop:run-s1b-001") == 1
|
||||
|
||||
|
||||
def test_application_service_rejects_second_active_owner_and_rebound_keys(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
store = QualificationRunStore(tmp_path)
|
||||
store.create(_run())
|
||||
store.create(_run("run-s1b-002", "episode-s1b-002"))
|
||||
worker = _FakeWorker()
|
||||
service = SimulationApplicationService(store, worker)
|
||||
service.start(
|
||||
"run-s1b-001",
|
||||
idempotency_key="start-001",
|
||||
observed_at_utc="2026-07-24T18:00:01Z",
|
||||
host_monotonic_ns=1,
|
||||
)
|
||||
|
||||
with pytest.raises(ActiveQualificationRunError, match="owned"):
|
||||
service.start(
|
||||
"run-s1b-002",
|
||||
idempotency_key="start-002",
|
||||
observed_at_utc="2026-07-24T18:00:02Z",
|
||||
host_monotonic_ns=2,
|
||||
)
|
||||
with pytest.raises(QualificationRunConflictError, match="another start"):
|
||||
service.start(
|
||||
"run-s1b-001",
|
||||
idempotency_key="different",
|
||||
observed_at_utc="2026-07-24T18:00:02Z",
|
||||
host_monotonic_ns=2,
|
||||
)
|
||||
|
||||
|
||||
def test_start_failure_and_profile_drift_fail_terminal(tmp_path: Path) -> None:
|
||||
service, worker = _service(tmp_path)
|
||||
worker.fail_start = True
|
||||
failed = service.start(
|
||||
"run-s1b-001",
|
||||
idempotency_key="start-fail",
|
||||
observed_at_utc="2026-07-24T18:00:01Z",
|
||||
host_monotonic_ns=1,
|
||||
)
|
||||
assert failed.state is RunState.FAILED
|
||||
assert failed.terminal_reason == "orchestrator-start-failed"
|
||||
|
||||
store = QualificationRunStore(tmp_path / "drift")
|
||||
store.create(_run())
|
||||
drift_worker = _FakeWorker("d" * 64)
|
||||
drift_service = SimulationApplicationService(store, drift_worker)
|
||||
drifted = drift_service.start(
|
||||
"run-s1b-001",
|
||||
idempotency_key="start-drift",
|
||||
observed_at_utc="2026-07-24T18:00:01Z",
|
||||
host_monotonic_ns=1,
|
||||
)
|
||||
assert drifted.state is RunState.FAILED
|
||||
assert drifted.terminal_reason == "orchestrator-profile-drift-failed"
|
||||
assert "stop:run-s1b-001" in drift_worker.calls
|
||||
|
||||
|
||||
def test_reset_stops_parent_and_creates_new_identity(tmp_path: Path) -> None:
|
||||
service, _ = _service(tmp_path)
|
||||
service.start(
|
||||
"run-s1b-001",
|
||||
idempotency_key="start-001",
|
||||
observed_at_utc="2026-07-24T18:00:01Z",
|
||||
host_monotonic_ns=1,
|
||||
)
|
||||
reset = service.reset(
|
||||
"run-s1b-001",
|
||||
idempotency_key="reset-001",
|
||||
new_run_id="run-s1b-002",
|
||||
new_episode_id="episode-s1b-002",
|
||||
observed_at_utc="2026-07-24T18:00:02Z",
|
||||
host_monotonic_ns=2,
|
||||
sim_time_ns=2_000_000,
|
||||
)
|
||||
|
||||
assert service.store.load("run-s1b-001").state is RunState.ABORTED
|
||||
assert reset.state is RunState.ADMITTED
|
||||
assert reset.parent_run_id == "run-s1b-001"
|
||||
|
||||
|
||||
def test_restart_reconcile_never_resumes_and_records_residue(tmp_path: Path) -> None:
|
||||
service, worker = _service(tmp_path)
|
||||
service.start(
|
||||
"run-s1b-001",
|
||||
idempotency_key="start-001",
|
||||
observed_at_utc="2026-07-24T18:00:01Z",
|
||||
host_monotonic_ns=1,
|
||||
)
|
||||
worker.residue = ("px4-sitl",)
|
||||
|
||||
reconciled = service.reconcile(
|
||||
observed_at_utc="2026-07-24T18:01:00Z",
|
||||
host_monotonic_ns=60,
|
||||
)
|
||||
|
||||
assert reconciled[0].state is RunState.FAILED
|
||||
assert reconciled[0].terminal_reason == "orchestrator-recovery-interrupted"
|
||||
assert any(
|
||||
event.event_type == "orchestrator.recovery-residue"
|
||||
for event in service.store.list_events("run-s1b-001")
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(os.name != "posix", reason="target supervisor is POSIX-only")
|
||||
def test_process_supervisor_owns_groups_and_stops_in_declared_order(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
order_path = tmp_path / "order.log"
|
||||
child = (
|
||||
"import os,signal,time;"
|
||||
"p=os.environ['ORDER_FILE'];n=os.environ['NAME'];"
|
||||
"signal.signal(signal.SIGINT,lambda *_:(open(p,'a').write(n+'\\n'),exit(0)));"
|
||||
"time.sleep(60)"
|
||||
)
|
||||
supervisor = PosixProcessSupervisor(tmp_path / "runtime")
|
||||
records = supervisor.start(
|
||||
"run-process-001",
|
||||
(
|
||||
ProcessSpec(
|
||||
process_id="gazebo-server",
|
||||
argv=(sys.executable, "-c", child),
|
||||
start_order=1,
|
||||
shutdown_order=20,
|
||||
environment=(("ORDER_FILE", str(order_path)), ("NAME", "gazebo")),
|
||||
),
|
||||
ProcessSpec(
|
||||
process_id="px4-sitl",
|
||||
argv=(sys.executable, "-c", child),
|
||||
start_order=2,
|
||||
shutdown_order=10,
|
||||
environment=(("ORDER_FILE", str(order_path)), ("NAME", "px4")),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
assert [record.process_id for record in records] == ["gazebo-server", "px4-sitl"]
|
||||
assert all(record.pid == record.pgid for record in records)
|
||||
result = supervisor.stop()
|
||||
|
||||
assert result.residue_process_ids == ()
|
||||
assert result.stopped_process_ids == ("px4-sitl", "gazebo-server")
|
||||
assert order_path.read_text(encoding="utf-8").splitlines() == ["px4", "gazebo"]
|
||||
assert supervisor.residue() == ()
|
||||
|
||||
|
||||
@pytest.mark.skipif(os.name != "posix", reason="target supervisor is POSIX-only")
|
||||
def test_process_supervisor_rolls_back_partial_start(tmp_path: Path) -> None:
|
||||
supervisor = PosixProcessSupervisor(tmp_path / "runtime")
|
||||
with pytest.raises(ProcessSupervisorError, match="exited during startup"):
|
||||
supervisor.start(
|
||||
"run-process-001",
|
||||
(
|
||||
ProcessSpec(
|
||||
process_id="long-lived",
|
||||
argv=(sys.executable, "-c", "import time;time.sleep(60)"),
|
||||
start_order=1,
|
||||
shutdown_order=10,
|
||||
),
|
||||
ProcessSpec(
|
||||
process_id="failed",
|
||||
argv=(sys.executable, "-c", "raise SystemExit(7)"),
|
||||
start_order=2,
|
||||
shutdown_order=20,
|
||||
),
|
||||
),
|
||||
)
|
||||
assert supervisor.residue() == ()
|
||||
|
||||
|
||||
def test_s0_worker_guard_binds_exact_digest_and_d_only_run_path() -> None:
|
||||
guard = S0WorkerGuard(PROFILE)
|
||||
run = _run(host_profile_sha256=hashlib.sha256(PROFILE.read_bytes()).hexdigest())
|
||||
root = PurePosixPath("/mnt/d/NDC_MISSIONCORE/simulation/artifacts/s1/runs/run-s1b-001")
|
||||
process_root = PurePosixPath("/mnt/d/NDC_MISSIONCORE/simulation/runtime/s1/runs/run-s1b-001")
|
||||
|
||||
admission = guard.admit(run, root, process_root)
|
||||
|
||||
assert admission.profile.profile_id == "ai-worker-px4-gazebo-v1"
|
||||
assert admission.profile_sha256 == run.host_profile_sha256
|
||||
with pytest.raises(WorkerAdmissionError, match="escapes"):
|
||||
guard.admit(run, PurePosixPath("/tmp/run-s1b-001"), process_root)
|
||||
with pytest.raises(WorkerAdmissionError, match="process root"):
|
||||
guard.admit(run, root, PurePosixPath("/tmp/run-s1b-001"))
|
||||
with pytest.raises(WorkerAdmissionError, match="digest"):
|
||||
guard.admit(_run(), root, process_root)
|
||||
Reference in New Issue
Block a user