feat(simulation): add S1 worker lifecycle ownership
This commit is contained in:
DCCONSTRUCTIONS
@@ -16,6 +16,21 @@ from k1link.simulation.contracts import (
|
||||
RunState,
|
||||
SimulationContractError,
|
||||
)
|
||||
from k1link.simulation.orchestrator import (
|
||||
ActiveQualificationRunError,
|
||||
SimulationApplicationService,
|
||||
SimulationOrchestratorError,
|
||||
SimulationWorkerPort,
|
||||
WorkerStartResult,
|
||||
WorkerStopResult,
|
||||
)
|
||||
from k1link.simulation.process_supervisor import (
|
||||
OwnedProcess,
|
||||
PosixProcessSupervisor,
|
||||
ProcessSpec,
|
||||
ProcessStopResult,
|
||||
ProcessSupervisorError,
|
||||
)
|
||||
from k1link.simulation.run_store import (
|
||||
QualificationRunConflictError,
|
||||
QualificationRunIntegrityError,
|
||||
@@ -34,9 +49,17 @@ from k1link.simulation.s0 import (
|
||||
load_s0_profile,
|
||||
run_s0_doctor,
|
||||
)
|
||||
from k1link.simulation.worker import (
|
||||
LocalProcessWorkerAdapter,
|
||||
S0WorkerGuard,
|
||||
SimulationWorldControl,
|
||||
WorkerAdmission,
|
||||
WorkerAdmissionError,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"AckermannControlSetpoint",
|
||||
"ActiveQualificationRunError",
|
||||
"AuthorityProfile",
|
||||
"CheckStatus",
|
||||
"CommandAuthorityScope",
|
||||
@@ -44,6 +67,12 @@ __all__ = [
|
||||
"ControlSetpoint",
|
||||
"DifferentialControlSetpoint",
|
||||
"DoctorVerdict",
|
||||
"LocalProcessWorkerAdapter",
|
||||
"OwnedProcess",
|
||||
"PosixProcessSupervisor",
|
||||
"ProcessSpec",
|
||||
"ProcessStopResult",
|
||||
"ProcessSupervisorError",
|
||||
"ProviderPin",
|
||||
"QualificationArtifact",
|
||||
"QualificationEvent",
|
||||
@@ -58,10 +87,19 @@ __all__ = [
|
||||
"RuntimeAcceptance",
|
||||
"RunKind",
|
||||
"RunState",
|
||||
"S0WorkerGuard",
|
||||
"S0DoctorReport",
|
||||
"S0Profile",
|
||||
"S0ProfileError",
|
||||
"SimulationContractError",
|
||||
"SimulationApplicationService",
|
||||
"SimulationOrchestratorError",
|
||||
"SimulationWorkerPort",
|
||||
"SimulationWorldControl",
|
||||
"WorkerAdmission",
|
||||
"WorkerAdmissionError",
|
||||
"WorkerStartResult",
|
||||
"WorkerStopResult",
|
||||
"load_s0_profile",
|
||||
"run_s0_doctor",
|
||||
]
|
||||
|
||||
@@ -0,0 +1,544 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
from contextlib import suppress
|
||||
from dataclasses import dataclass
|
||||
from typing import Protocol
|
||||
|
||||
from k1link.simulation.contracts import QualificationEvent, QualificationRun, RunState
|
||||
from k1link.simulation.run_store import (
|
||||
ACTIVE_RECOVERY_STATES,
|
||||
QualificationRunConflictError,
|
||||
QualificationRunStore,
|
||||
QualificationRunTransitionError,
|
||||
)
|
||||
|
||||
|
||||
class SimulationOrchestratorError(RuntimeError):
|
||||
"""The server-owned S1 lifecycle could not be completed safely."""
|
||||
|
||||
|
||||
class ActiveQualificationRunError(SimulationOrchestratorError):
|
||||
"""Another qualification run already owns the worker authority."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class WorkerStartResult:
|
||||
provider_ids: tuple[str, ...]
|
||||
profile_sha256: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class WorkerStopResult:
|
||||
stopped_provider_ids: tuple[str, ...]
|
||||
residue_provider_ids: tuple[str, ...]
|
||||
|
||||
@property
|
||||
def clean(self) -> bool:
|
||||
return not self.residue_provider_ids
|
||||
|
||||
|
||||
class SimulationWorkerPort(Protocol):
|
||||
"""Transport-neutral worker boundary; no browser/PX4 shortcut exists."""
|
||||
|
||||
def start(self, run: QualificationRun) -> WorkerStartResult: ...
|
||||
|
||||
def pause(self, run: QualificationRun) -> None: ...
|
||||
|
||||
def resume(self, run: QualificationRun) -> None: ...
|
||||
|
||||
def step(self, run: QualificationRun, step_count: int) -> int: ...
|
||||
|
||||
def stop(self, run: QualificationRun) -> WorkerStopResult: ...
|
||||
|
||||
def reconcile(self, run: QualificationRun) -> WorkerStopResult: ...
|
||||
|
||||
|
||||
class SimulationApplicationService:
|
||||
"""Sole owner of one worker's persisted qualification-run lifecycle."""
|
||||
|
||||
def __init__(self, store: QualificationRunStore, worker: SimulationWorkerPort) -> None:
|
||||
self.store = store
|
||||
self.worker = worker
|
||||
self._lock = threading.RLock()
|
||||
|
||||
def start(
|
||||
self,
|
||||
run_id: str,
|
||||
*,
|
||||
idempotency_key: str,
|
||||
observed_at_utc: str,
|
||||
host_monotonic_ns: int,
|
||||
) -> QualificationRun:
|
||||
key = _idempotency_key(idempotency_key)
|
||||
with self._lock:
|
||||
run = self.store.load(run_id)
|
||||
existing = self._start_request(run_id, key)
|
||||
if existing:
|
||||
if run.state is RunState.ADMITTED:
|
||||
return self._continue_start(
|
||||
run,
|
||||
observed_at_utc=observed_at_utc,
|
||||
host_monotonic_ns=host_monotonic_ns,
|
||||
)
|
||||
return run
|
||||
if self._has_other_start_request(run_id):
|
||||
raise QualificationRunConflictError(
|
||||
"qualification run is already bound to another start request"
|
||||
)
|
||||
active = tuple(
|
||||
candidate
|
||||
for candidate in self.store.list_runs()
|
||||
if candidate.run_id != run_id and candidate.state in ACTIVE_RECOVERY_STATES
|
||||
)
|
||||
if active:
|
||||
raise ActiveQualificationRunError(
|
||||
f"worker authority is owned by {active[0].run_id}"
|
||||
)
|
||||
if run.state is not RunState.ADMITTED:
|
||||
raise QualificationRunTransitionError(
|
||||
"only an admitted qualification run can start"
|
||||
)
|
||||
event = self.store.append_event(
|
||||
run_id,
|
||||
event_type="orchestrator.start-requested",
|
||||
observed_at_utc=observed_at_utc,
|
||||
host_monotonic_ns=host_monotonic_ns,
|
||||
payload={"idempotency_key": key},
|
||||
expected_revision=run.revision,
|
||||
)
|
||||
run = self.store.load(run_id)
|
||||
if run.revision != event.sequence:
|
||||
raise QualificationRunConflictError("start request journal changed")
|
||||
return self._continue_start(
|
||||
run,
|
||||
observed_at_utc=observed_at_utc,
|
||||
host_monotonic_ns=host_monotonic_ns,
|
||||
)
|
||||
|
||||
def pause(
|
||||
self,
|
||||
run_id: str,
|
||||
*,
|
||||
observed_at_utc: str,
|
||||
host_monotonic_ns: int,
|
||||
sim_time_ns: int,
|
||||
) -> QualificationRun:
|
||||
with self._lock:
|
||||
run = self.store.load(run_id)
|
||||
if run.state is not RunState.RUNNING:
|
||||
raise QualificationRunTransitionError("only a running run can pause")
|
||||
try:
|
||||
self.worker.pause(run)
|
||||
except Exception as exc:
|
||||
return self._fail_worker_operation(
|
||||
run,
|
||||
operation="pause",
|
||||
exc=exc,
|
||||
observed_at_utc=observed_at_utc,
|
||||
host_monotonic_ns=host_monotonic_ns,
|
||||
sim_time_ns=sim_time_ns,
|
||||
)
|
||||
return self.store.transition(
|
||||
run_id,
|
||||
RunState.PAUSED,
|
||||
expected_revision=run.revision,
|
||||
observed_at_utc=observed_at_utc,
|
||||
host_monotonic_ns=host_monotonic_ns,
|
||||
sim_time_ns=sim_time_ns,
|
||||
)
|
||||
|
||||
def resume(
|
||||
self,
|
||||
run_id: str,
|
||||
*,
|
||||
observed_at_utc: str,
|
||||
host_monotonic_ns: int,
|
||||
sim_time_ns: int,
|
||||
) -> QualificationRun:
|
||||
with self._lock:
|
||||
run = self.store.load(run_id)
|
||||
if run.state is not RunState.PAUSED:
|
||||
raise QualificationRunTransitionError("only a paused run can resume")
|
||||
try:
|
||||
self.worker.resume(run)
|
||||
except Exception as exc:
|
||||
return self._fail_worker_operation(
|
||||
run,
|
||||
operation="resume",
|
||||
exc=exc,
|
||||
observed_at_utc=observed_at_utc,
|
||||
host_monotonic_ns=host_monotonic_ns,
|
||||
sim_time_ns=sim_time_ns,
|
||||
)
|
||||
return self.store.transition(
|
||||
run_id,
|
||||
RunState.RUNNING,
|
||||
expected_revision=run.revision,
|
||||
observed_at_utc=observed_at_utc,
|
||||
host_monotonic_ns=host_monotonic_ns,
|
||||
sim_time_ns=sim_time_ns,
|
||||
)
|
||||
|
||||
def step(
|
||||
self,
|
||||
run_id: str,
|
||||
*,
|
||||
step_count: int,
|
||||
observed_at_utc: str,
|
||||
host_monotonic_ns: int,
|
||||
sim_time_ns: int,
|
||||
) -> int:
|
||||
if step_count < 1:
|
||||
raise ValueError("step count must be positive")
|
||||
with self._lock:
|
||||
run = self.store.load(run_id)
|
||||
if run.state is not RunState.PAUSED:
|
||||
raise QualificationRunTransitionError("single-step requires a paused run")
|
||||
try:
|
||||
advanced_ns = self.worker.step(run, step_count)
|
||||
except Exception as exc:
|
||||
self._fail_worker_operation(
|
||||
run,
|
||||
operation="step",
|
||||
exc=exc,
|
||||
observed_at_utc=observed_at_utc,
|
||||
host_monotonic_ns=host_monotonic_ns,
|
||||
sim_time_ns=sim_time_ns,
|
||||
)
|
||||
raise SimulationOrchestratorError("worker step failed") from exc
|
||||
if advanced_ns < 1:
|
||||
self._fail_worker_operation(
|
||||
run,
|
||||
operation="step-invalid-advance",
|
||||
exc=ValueError("worker returned a non-positive clock advance"),
|
||||
observed_at_utc=observed_at_utc,
|
||||
host_monotonic_ns=host_monotonic_ns,
|
||||
sim_time_ns=sim_time_ns,
|
||||
)
|
||||
raise SimulationOrchestratorError(
|
||||
"worker step returned a non-positive clock advance"
|
||||
)
|
||||
self.store.append_event(
|
||||
run_id,
|
||||
event_type="clock.single-step",
|
||||
observed_at_utc=observed_at_utc,
|
||||
host_monotonic_ns=host_monotonic_ns,
|
||||
sim_time_ns=sim_time_ns + advanced_ns,
|
||||
payload={"step_count": step_count, "advanced_ns": advanced_ns},
|
||||
expected_revision=run.revision,
|
||||
)
|
||||
return advanced_ns
|
||||
|
||||
def stop(
|
||||
self,
|
||||
run_id: str,
|
||||
*,
|
||||
idempotency_key: str,
|
||||
observed_at_utc: str,
|
||||
host_monotonic_ns: int,
|
||||
sim_time_ns: int | None,
|
||||
terminal_state: RunState = RunState.COMPLETED,
|
||||
reason: str = "operator-stop-clean",
|
||||
) -> QualificationRun:
|
||||
key = _idempotency_key(idempotency_key)
|
||||
if terminal_state not in {RunState.COMPLETED, RunState.ABORTED}:
|
||||
raise ValueError("operator stop terminal state must be completed or aborted")
|
||||
with self._lock:
|
||||
run = self.store.load(run_id)
|
||||
existing = self._stop_request(run_id, key)
|
||||
if existing is not None and (
|
||||
existing.payload.get("terminal_state") != terminal_state.value
|
||||
or existing.payload.get("reason") != reason
|
||||
):
|
||||
raise QualificationRunConflictError(
|
||||
"idempotency key is bound to a different stop request"
|
||||
)
|
||||
if existing is not None and run.state.terminal:
|
||||
return run
|
||||
if existing is None:
|
||||
if self._has_other_stop_request(run_id):
|
||||
raise QualificationRunConflictError(
|
||||
"qualification run is already bound to another stop request"
|
||||
)
|
||||
event = self.store.append_event(
|
||||
run_id,
|
||||
event_type="orchestrator.stop-requested",
|
||||
observed_at_utc=observed_at_utc,
|
||||
host_monotonic_ns=host_monotonic_ns,
|
||||
sim_time_ns=sim_time_ns,
|
||||
payload={
|
||||
"idempotency_key": key,
|
||||
"terminal_state": terminal_state.value,
|
||||
"reason": reason,
|
||||
},
|
||||
expected_revision=run.revision,
|
||||
)
|
||||
run = self.store.load(run_id)
|
||||
if run.revision != event.sequence:
|
||||
raise QualificationRunConflictError("stop request journal changed")
|
||||
if run.state not in {RunState.RUNNING, RunState.PAUSED, RunState.STOPPING}:
|
||||
raise QualificationRunTransitionError(
|
||||
"only a running, paused or stopping run can stop"
|
||||
)
|
||||
if run.state is not RunState.STOPPING:
|
||||
run = self.store.transition(
|
||||
run_id,
|
||||
RunState.STOPPING,
|
||||
expected_revision=run.revision,
|
||||
observed_at_utc=observed_at_utc,
|
||||
host_monotonic_ns=host_monotonic_ns,
|
||||
sim_time_ns=sim_time_ns,
|
||||
)
|
||||
try:
|
||||
outcome = self.worker.stop(run)
|
||||
except Exception as exc:
|
||||
return self._terminal_failure(
|
||||
run,
|
||||
operation="stop",
|
||||
detail=type(exc).__name__,
|
||||
observed_at_utc=observed_at_utc,
|
||||
host_monotonic_ns=host_monotonic_ns,
|
||||
sim_time_ns=sim_time_ns,
|
||||
)
|
||||
if not outcome.clean:
|
||||
return self._terminal_failure(
|
||||
run,
|
||||
operation="stop-residue",
|
||||
detail=",".join(outcome.residue_provider_ids),
|
||||
observed_at_utc=observed_at_utc,
|
||||
host_monotonic_ns=host_monotonic_ns,
|
||||
sim_time_ns=sim_time_ns,
|
||||
)
|
||||
event = self.store.append_event(
|
||||
run_id,
|
||||
event_type="orchestrator.providers-stopped",
|
||||
observed_at_utc=observed_at_utc,
|
||||
host_monotonic_ns=host_monotonic_ns,
|
||||
sim_time_ns=sim_time_ns,
|
||||
payload={"provider_ids": list(outcome.stopped_provider_ids)},
|
||||
expected_revision=run.revision,
|
||||
)
|
||||
return self.store.transition(
|
||||
run_id,
|
||||
terminal_state,
|
||||
expected_revision=event.sequence,
|
||||
observed_at_utc=observed_at_utc,
|
||||
host_monotonic_ns=host_monotonic_ns,
|
||||
sim_time_ns=sim_time_ns,
|
||||
reason=reason,
|
||||
)
|
||||
|
||||
def reset(
|
||||
self,
|
||||
run_id: str,
|
||||
*,
|
||||
idempotency_key: str,
|
||||
new_run_id: str,
|
||||
new_episode_id: str,
|
||||
observed_at_utc: str,
|
||||
host_monotonic_ns: int,
|
||||
sim_time_ns: int | None,
|
||||
) -> QualificationRun:
|
||||
with self._lock:
|
||||
key = _idempotency_key(idempotency_key)
|
||||
if len(key) > 155:
|
||||
raise ValueError("reset idempotency key must contain at most 155 characters")
|
||||
terminal = self.stop(
|
||||
run_id,
|
||||
idempotency_key=f"{key}:stop",
|
||||
observed_at_utc=observed_at_utc,
|
||||
host_monotonic_ns=host_monotonic_ns,
|
||||
sim_time_ns=sim_time_ns,
|
||||
terminal_state=RunState.ABORTED,
|
||||
reason="operator-reset",
|
||||
)
|
||||
return self.store.create_reset_episode(
|
||||
terminal.run_id,
|
||||
run_id=new_run_id,
|
||||
episode_id=new_episode_id,
|
||||
created_at_utc=observed_at_utc,
|
||||
)
|
||||
|
||||
def reconcile(
|
||||
self,
|
||||
*,
|
||||
observed_at_utc: str,
|
||||
host_monotonic_ns: int,
|
||||
) -> tuple[QualificationRun, ...]:
|
||||
reconciled: list[QualificationRun] = []
|
||||
with self._lock:
|
||||
for run in self.store.list_runs():
|
||||
if run.state not in ACTIVE_RECOVERY_STATES:
|
||||
continue
|
||||
try:
|
||||
outcome = self.worker.reconcile(run)
|
||||
residue = outcome.residue_provider_ids
|
||||
except Exception as exc:
|
||||
residue = (f"reconcile-error:{type(exc).__name__}",)
|
||||
if residue:
|
||||
event = self.store.append_event(
|
||||
run.run_id,
|
||||
event_type="orchestrator.recovery-residue",
|
||||
observed_at_utc=observed_at_utc,
|
||||
host_monotonic_ns=host_monotonic_ns,
|
||||
payload={"provider_ids": list(residue)},
|
||||
expected_revision=run.revision,
|
||||
)
|
||||
run = self.store.load(run.run_id)
|
||||
if run.revision != event.sequence:
|
||||
raise QualificationRunConflictError("recovery event journal changed")
|
||||
reconciled.append(
|
||||
self.store.reconcile_interrupted(
|
||||
run.run_id,
|
||||
observed_at_utc=observed_at_utc,
|
||||
host_monotonic_ns=host_monotonic_ns,
|
||||
)
|
||||
)
|
||||
return tuple(reconciled)
|
||||
|
||||
def _continue_start(
|
||||
self,
|
||||
run: QualificationRun,
|
||||
*,
|
||||
observed_at_utc: str,
|
||||
host_monotonic_ns: int,
|
||||
) -> QualificationRun:
|
||||
starting = self.store.transition(
|
||||
run.run_id,
|
||||
RunState.STARTING,
|
||||
expected_revision=run.revision,
|
||||
observed_at_utc=observed_at_utc,
|
||||
host_monotonic_ns=host_monotonic_ns,
|
||||
)
|
||||
try:
|
||||
outcome = self.worker.start(starting)
|
||||
except Exception as exc:
|
||||
with suppress(Exception):
|
||||
self.worker.stop(starting)
|
||||
return self._terminal_failure(
|
||||
starting,
|
||||
operation="start",
|
||||
detail=type(exc).__name__,
|
||||
observed_at_utc=observed_at_utc,
|
||||
host_monotonic_ns=host_monotonic_ns,
|
||||
sim_time_ns=0,
|
||||
)
|
||||
if outcome.profile_sha256 != starting.host_profile_sha256:
|
||||
with suppress(Exception):
|
||||
self.worker.stop(starting)
|
||||
return self._terminal_failure(
|
||||
starting,
|
||||
operation="profile-drift",
|
||||
detail=outcome.profile_sha256,
|
||||
observed_at_utc=observed_at_utc,
|
||||
host_monotonic_ns=host_monotonic_ns,
|
||||
sim_time_ns=0,
|
||||
)
|
||||
event = self.store.append_event(
|
||||
run.run_id,
|
||||
event_type="orchestrator.providers-started",
|
||||
observed_at_utc=observed_at_utc,
|
||||
host_monotonic_ns=host_monotonic_ns,
|
||||
sim_time_ns=0,
|
||||
payload={
|
||||
"provider_ids": list(outcome.provider_ids),
|
||||
"profile_sha256": outcome.profile_sha256,
|
||||
},
|
||||
expected_revision=starting.revision,
|
||||
)
|
||||
return self.store.transition(
|
||||
run.run_id,
|
||||
RunState.RUNNING,
|
||||
expected_revision=event.sequence,
|
||||
observed_at_utc=observed_at_utc,
|
||||
host_monotonic_ns=host_monotonic_ns,
|
||||
sim_time_ns=0,
|
||||
)
|
||||
|
||||
def _fail_worker_operation(
|
||||
self,
|
||||
run: QualificationRun,
|
||||
*,
|
||||
operation: str,
|
||||
exc: Exception,
|
||||
observed_at_utc: str,
|
||||
host_monotonic_ns: int,
|
||||
sim_time_ns: int | None,
|
||||
) -> QualificationRun:
|
||||
with suppress(Exception):
|
||||
self.worker.stop(run)
|
||||
return self._terminal_failure(
|
||||
run,
|
||||
operation=operation,
|
||||
detail=type(exc).__name__,
|
||||
observed_at_utc=observed_at_utc,
|
||||
host_monotonic_ns=host_monotonic_ns,
|
||||
sim_time_ns=sim_time_ns,
|
||||
)
|
||||
|
||||
def _terminal_failure(
|
||||
self,
|
||||
run: QualificationRun,
|
||||
*,
|
||||
operation: str,
|
||||
detail: str,
|
||||
observed_at_utc: str,
|
||||
host_monotonic_ns: int,
|
||||
sim_time_ns: int | None,
|
||||
) -> QualificationRun:
|
||||
event = self.store.append_event(
|
||||
run.run_id,
|
||||
event_type="orchestrator.operation-failed",
|
||||
observed_at_utc=observed_at_utc,
|
||||
host_monotonic_ns=host_monotonic_ns,
|
||||
sim_time_ns=sim_time_ns,
|
||||
payload={"operation": operation, "detail": detail[:256]},
|
||||
expected_revision=run.revision,
|
||||
)
|
||||
return self.store.transition(
|
||||
run.run_id,
|
||||
RunState.FAILED,
|
||||
expected_revision=event.sequence,
|
||||
observed_at_utc=observed_at_utc,
|
||||
host_monotonic_ns=host_monotonic_ns,
|
||||
sim_time_ns=sim_time_ns,
|
||||
reason=f"orchestrator-{operation}-failed",
|
||||
)
|
||||
|
||||
def _start_request(self, run_id: str, key: str) -> bool:
|
||||
return any(
|
||||
event.event_type == "orchestrator.start-requested"
|
||||
and event.payload.get("idempotency_key") == key
|
||||
for event in self.store.list_events(run_id)
|
||||
)
|
||||
|
||||
def _has_other_start_request(self, run_id: str) -> bool:
|
||||
return any(
|
||||
event.event_type == "orchestrator.start-requested"
|
||||
for event in self.store.list_events(run_id)
|
||||
)
|
||||
|
||||
def _stop_request(self, run_id: str, key: str) -> QualificationEvent | None:
|
||||
return next(
|
||||
(
|
||||
event
|
||||
for event in self.store.list_events(run_id)
|
||||
if event.event_type == "orchestrator.stop-requested"
|
||||
and event.payload.get("idempotency_key") == key
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
def _has_other_stop_request(self, run_id: str) -> bool:
|
||||
return any(
|
||||
event.event_type == "orchestrator.stop-requested"
|
||||
for event in self.store.list_events(run_id)
|
||||
)
|
||||
|
||||
|
||||
def _idempotency_key(value: str) -> str:
|
||||
normalized = value.strip()
|
||||
if not 1 <= len(normalized) <= 160:
|
||||
raise ValueError("idempotency key must contain 1..160 characters")
|
||||
return normalized
|
||||
@@ -0,0 +1,302 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import signal
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
from k1link.artifacts import write_json_atomic
|
||||
|
||||
PROCESS_ID_PATTERN: Final = re.compile(r"^[a-z0-9][a-z0-9-]{0,63}$")
|
||||
|
||||
|
||||
class ProcessSupervisorError(RuntimeError):
|
||||
"""A provider process graph could not be owned or stopped safely."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ProcessSpec:
|
||||
process_id: str
|
||||
argv: tuple[str, ...]
|
||||
start_order: int
|
||||
shutdown_order: int
|
||||
environment: tuple[tuple[str, str], ...] = ()
|
||||
startup_grace_seconds: float = 0.05
|
||||
interrupt_timeout_seconds: float = 2.0
|
||||
terminate_timeout_seconds: float = 1.0
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if not PROCESS_ID_PATTERN.fullmatch(self.process_id):
|
||||
raise ValueError("process id must be a safe lowercase identifier")
|
||||
if not self.argv or any(not item or "\x00" in item for item in self.argv):
|
||||
raise ValueError("process argv must contain nonempty NUL-free values")
|
||||
if self.start_order < 0 or self.shutdown_order < 0:
|
||||
raise ValueError("process order must not be negative")
|
||||
if (
|
||||
self.startup_grace_seconds < 0
|
||||
or self.interrupt_timeout_seconds < 0
|
||||
or self.terminate_timeout_seconds < 0
|
||||
):
|
||||
raise ValueError("process timeouts must not be negative")
|
||||
keys = [key for key, _ in self.environment]
|
||||
if len(keys) != len(set(keys)) or any(
|
||||
not key or "=" in key or "\x00" in key or "\x00" in value
|
||||
for key, value in self.environment
|
||||
):
|
||||
raise ValueError("process environment must contain unique safe entries")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class OwnedProcess:
|
||||
process_id: str
|
||||
pid: int
|
||||
pgid: int
|
||||
start_order: int
|
||||
shutdown_order: int
|
||||
stdout_path: Path
|
||||
stderr_path: Path
|
||||
|
||||
def to_dict(self) -> dict[str, object]:
|
||||
return {
|
||||
"process_id": self.process_id,
|
||||
"pid": self.pid,
|
||||
"pgid": self.pgid,
|
||||
"start_order": self.start_order,
|
||||
"shutdown_order": self.shutdown_order,
|
||||
"stdout_path": self.stdout_path.name,
|
||||
"stderr_path": self.stderr_path.name,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ProcessStopResult:
|
||||
stopped_process_ids: tuple[str, ...]
|
||||
residue_process_ids: tuple[str, ...]
|
||||
|
||||
|
||||
class PosixProcessSupervisor:
|
||||
"""Own provider PGIDs and stop them in explicit reverse dependency order."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
runtime_root: Path,
|
||||
*,
|
||||
base_environment: dict[str, str] | None = None,
|
||||
) -> None:
|
||||
if os.name != "posix":
|
||||
raise ProcessSupervisorError("S1 process supervisor requires a POSIX worker")
|
||||
self.runtime_root = runtime_root.expanduser().resolve()
|
||||
self.runtime_root.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
self.base_environment = dict(base_environment or {})
|
||||
self._lock = threading.RLock()
|
||||
self._run_id: str | None = None
|
||||
self._processes: dict[str, tuple[ProcessSpec, subprocess.Popen[bytes]]] = {}
|
||||
|
||||
def start(self, run_id: str, specs: tuple[ProcessSpec, ...]) -> tuple[OwnedProcess, ...]:
|
||||
if not PROCESS_ID_PATTERN.fullmatch(run_id):
|
||||
raise ProcessSupervisorError("run id must be a safe lowercase identifier")
|
||||
if not specs:
|
||||
raise ProcessSupervisorError("provider graph must not be empty")
|
||||
identifiers = [spec.process_id for spec in specs]
|
||||
if len(identifiers) != len(set(identifiers)):
|
||||
raise ProcessSupervisorError("provider process ids must be unique")
|
||||
start_orders = [spec.start_order for spec in specs]
|
||||
shutdown_orders = [spec.shutdown_order for spec in specs]
|
||||
if len(start_orders) != len(set(start_orders)):
|
||||
raise ProcessSupervisorError("provider start orders must be unique")
|
||||
if len(shutdown_orders) != len(set(shutdown_orders)):
|
||||
raise ProcessSupervisorError("provider shutdown orders must be unique")
|
||||
with self._lock:
|
||||
if self._processes:
|
||||
raise ProcessSupervisorError(
|
||||
f"process authority is already owned by {self._run_id}"
|
||||
)
|
||||
run_root = self.runtime_root / run_id
|
||||
if run_root.exists():
|
||||
raise ProcessSupervisorError("run-scoped process directory already exists")
|
||||
run_root.mkdir(mode=0o700)
|
||||
self._run_id = run_id
|
||||
try:
|
||||
for spec in sorted(specs, key=lambda item: item.start_order):
|
||||
self._start_one(run_root, spec)
|
||||
records = self.snapshot()
|
||||
self._persist_registry(run_root, records, terminal=False)
|
||||
return records
|
||||
except BaseException:
|
||||
self.stop()
|
||||
raise
|
||||
|
||||
def snapshot(self) -> tuple[OwnedProcess, ...]:
|
||||
with self._lock:
|
||||
records: list[OwnedProcess] = []
|
||||
for process_id, (spec, process) in self._processes.items():
|
||||
return_code = process.poll()
|
||||
if return_code is not None:
|
||||
raise ProcessSupervisorError(f"provider {process_id} exited with {return_code}")
|
||||
records.append(
|
||||
OwnedProcess(
|
||||
process_id=process_id,
|
||||
pid=process.pid,
|
||||
pgid=os.getpgid(process.pid),
|
||||
start_order=spec.start_order,
|
||||
shutdown_order=spec.shutdown_order,
|
||||
stdout_path=self.runtime_root
|
||||
/ str(self._run_id)
|
||||
/ f"{process_id}.stdout.log",
|
||||
stderr_path=self.runtime_root
|
||||
/ str(self._run_id)
|
||||
/ f"{process_id}.stderr.log",
|
||||
)
|
||||
)
|
||||
return tuple(sorted(records, key=lambda item: item.start_order))
|
||||
|
||||
def stop(self) -> ProcessStopResult:
|
||||
with self._lock:
|
||||
stopped: list[str] = []
|
||||
residue: list[str] = []
|
||||
remaining: dict[str, tuple[ProcessSpec, subprocess.Popen[bytes]]] = {}
|
||||
run_id = self._run_id
|
||||
for process_id, (spec, process) in sorted(
|
||||
self._processes.items(),
|
||||
key=lambda item: item[1][0].shutdown_order,
|
||||
):
|
||||
pgid = process.pid
|
||||
if self._stop_group(process, pgid, spec):
|
||||
stopped.append(process_id)
|
||||
else:
|
||||
residue.append(process_id)
|
||||
remaining[process_id] = (spec, process)
|
||||
if run_id is not None:
|
||||
run_root = self.runtime_root / run_id
|
||||
records = tuple(
|
||||
OwnedProcess(
|
||||
process_id=process_id,
|
||||
pid=process.pid,
|
||||
pgid=process.pid,
|
||||
start_order=spec.start_order,
|
||||
shutdown_order=spec.shutdown_order,
|
||||
stdout_path=run_root / f"{process_id}.stdout.log",
|
||||
stderr_path=run_root / f"{process_id}.stderr.log",
|
||||
)
|
||||
for process_id, (spec, process) in self._processes.items()
|
||||
)
|
||||
self._persist_registry(
|
||||
run_root,
|
||||
records,
|
||||
terminal=True,
|
||||
residue=tuple(residue),
|
||||
)
|
||||
self._processes = remaining
|
||||
if not remaining:
|
||||
self._run_id = None
|
||||
return ProcessStopResult(tuple(stopped), tuple(residue))
|
||||
|
||||
def residue(self) -> tuple[str, ...]:
|
||||
with self._lock:
|
||||
return tuple(
|
||||
process_id
|
||||
for process_id, (_, process) in self._processes.items()
|
||||
if _group_exists(process.pid)
|
||||
)
|
||||
|
||||
def _start_one(self, run_root: Path, spec: ProcessSpec) -> None:
|
||||
stdout_path = run_root / f"{spec.process_id}.stdout.log"
|
||||
stderr_path = run_root / f"{spec.process_id}.stderr.log"
|
||||
environment = dict(self.base_environment)
|
||||
environment.update(spec.environment)
|
||||
with stdout_path.open("xb") as stdout, stderr_path.open("xb") as stderr:
|
||||
process = subprocess.Popen(
|
||||
spec.argv,
|
||||
cwd=run_root,
|
||||
env=environment,
|
||||
stdin=subprocess.DEVNULL,
|
||||
stdout=stdout,
|
||||
stderr=stderr,
|
||||
start_new_session=True,
|
||||
close_fds=True,
|
||||
)
|
||||
self._processes[spec.process_id] = (spec, process)
|
||||
if spec.startup_grace_seconds:
|
||||
time.sleep(spec.startup_grace_seconds)
|
||||
return_code = process.poll()
|
||||
if return_code is not None:
|
||||
raise ProcessSupervisorError(
|
||||
f"provider {spec.process_id} exited during startup with {return_code}"
|
||||
)
|
||||
if os.getpgid(process.pid) != process.pid:
|
||||
raise ProcessSupervisorError(
|
||||
f"provider {spec.process_id} did not acquire a dedicated process group"
|
||||
)
|
||||
|
||||
def _stop_group(
|
||||
self,
|
||||
process: subprocess.Popen[bytes],
|
||||
pgid: int,
|
||||
spec: ProcessSpec,
|
||||
) -> bool:
|
||||
group_signal_supported = True
|
||||
for sent_signal, timeout in (
|
||||
(signal.SIGINT, spec.interrupt_timeout_seconds),
|
||||
(signal.SIGTERM, spec.terminate_timeout_seconds),
|
||||
(signal.SIGKILL, 1.0),
|
||||
):
|
||||
if process.poll() is not None and not _group_exists(pgid):
|
||||
process.wait(timeout=0)
|
||||
return True
|
||||
try:
|
||||
if group_signal_supported:
|
||||
os.killpg(pgid, sent_signal)
|
||||
else:
|
||||
process.send_signal(sent_signal)
|
||||
except ProcessLookupError:
|
||||
return True
|
||||
except PermissionError:
|
||||
# Some macOS application sandboxes deny killpg for an owned
|
||||
# child session. The reviewed WSL target retains group signals;
|
||||
# local development falls back to the group leader and does
|
||||
# not claim target residue acceptance from this path.
|
||||
group_signal_supported = False
|
||||
process.send_signal(sent_signal)
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
if process.poll() is not None:
|
||||
process.wait(timeout=0)
|
||||
return not group_signal_supported or not _group_exists(pgid)
|
||||
time.sleep(0.01)
|
||||
return process.poll() is not None and (
|
||||
not group_signal_supported or not _group_exists(pgid)
|
||||
)
|
||||
|
||||
def _persist_registry(
|
||||
self,
|
||||
run_root: Path,
|
||||
records: tuple[OwnedProcess, ...],
|
||||
*,
|
||||
terminal: bool,
|
||||
residue: tuple[str, ...] = (),
|
||||
) -> None:
|
||||
write_json_atomic(
|
||||
run_root / "process-registry.json",
|
||||
{
|
||||
"schema_version": "missioncore.simulation-process-registry/v1",
|
||||
"run_id": run_root.name,
|
||||
"terminal": terminal,
|
||||
"processes": [record.to_dict() for record in records],
|
||||
"residue_process_ids": list(residue),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _group_exists(pgid: int) -> bool:
|
||||
try:
|
||||
os.killpg(pgid, 0)
|
||||
except ProcessLookupError:
|
||||
return False
|
||||
except PermissionError:
|
||||
return True
|
||||
return True
|
||||
@@ -257,6 +257,11 @@ class QualificationRunStore:
|
||||
commands.append(command)
|
||||
return tuple(commands)
|
||||
|
||||
def list_events(self, run_id: str) -> tuple[QualificationEvent, ...]:
|
||||
path = self._existing_run_path(run_id)
|
||||
with self._lock:
|
||||
return self._read_events(path, run_id)
|
||||
|
||||
def register_artifact(
|
||||
self,
|
||||
run_id: str,
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import Protocol
|
||||
|
||||
from k1link.simulation.contracts import QualificationRun
|
||||
from k1link.simulation.orchestrator import WorkerStartResult, WorkerStopResult
|
||||
from k1link.simulation.process_supervisor import (
|
||||
PosixProcessSupervisor,
|
||||
ProcessSpec,
|
||||
)
|
||||
from k1link.simulation.s0 import S0Profile, load_s0_profile
|
||||
|
||||
|
||||
class WorkerAdmissionError(RuntimeError):
|
||||
"""The worker no longer matches the accepted S0 generation or D boundary."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class WorkerAdmission:
|
||||
profile: S0Profile
|
||||
profile_sha256: str
|
||||
artifact_run_root: PurePosixPath
|
||||
process_run_root: PurePosixPath
|
||||
|
||||
|
||||
class SimulationWorldControl(Protocol):
|
||||
def pause(self, run_id: str) -> None: ...
|
||||
|
||||
def resume(self, run_id: str) -> None: ...
|
||||
|
||||
def step(self, run_id: str, step_count: int) -> int: ...
|
||||
|
||||
|
||||
class S0WorkerGuard:
|
||||
"""Bind every S1 start to the exact accepted S0 profile and D artifact root."""
|
||||
|
||||
def __init__(self, profile_path: Path) -> None:
|
||||
self.profile_path = profile_path.expanduser().resolve()
|
||||
self.profile = load_s0_profile(self.profile_path)
|
||||
self.profile_sha256 = hashlib.sha256(self.profile_path.read_bytes()).hexdigest()
|
||||
|
||||
def admit(
|
||||
self,
|
||||
run: QualificationRun,
|
||||
artifact_run_root: PurePosixPath,
|
||||
process_run_root: PurePosixPath,
|
||||
) -> WorkerAdmission:
|
||||
if run.host_profile_id != self.profile.profile_id:
|
||||
raise WorkerAdmissionError("run host profile id does not match accepted S0")
|
||||
if run.host_profile_sha256 != self.profile_sha256:
|
||||
raise WorkerAdmissionError("run host profile digest does not match accepted S0")
|
||||
artifact_roots = tuple(
|
||||
item.wsl_path
|
||||
for item in self.profile.storage.mutable_paths
|
||||
if item.identifier == "artifacts"
|
||||
)
|
||||
if len(artifact_roots) != 1:
|
||||
raise WorkerAdmissionError("accepted S0 profile has no unique artifact root")
|
||||
runtime_roots = tuple(
|
||||
item.wsl_path
|
||||
for item in self.profile.storage.mutable_paths
|
||||
if item.identifier == "runtime"
|
||||
)
|
||||
if len(runtime_roots) != 1:
|
||||
raise WorkerAdmissionError("accepted S0 profile has no unique runtime root")
|
||||
expected_artifact_parent = artifact_roots[0] / "s1" / "runs"
|
||||
expected_runtime_parent = runtime_roots[0] / "s1" / "runs"
|
||||
if artifact_run_root != expected_artifact_parent / run.run_id:
|
||||
raise WorkerAdmissionError("S1 run root escapes the accepted D-only layout")
|
||||
if process_run_root != expected_runtime_parent / run.run_id:
|
||||
raise WorkerAdmissionError("S1 process root escapes the accepted D-only layout")
|
||||
if not all(
|
||||
component.qualification_state == "accepted" for component in self.profile.components
|
||||
):
|
||||
raise WorkerAdmissionError("S0 component generation is no longer accepted")
|
||||
return WorkerAdmission(
|
||||
profile=self.profile,
|
||||
profile_sha256=self.profile_sha256,
|
||||
artifact_run_root=artifact_run_root,
|
||||
process_run_root=process_run_root,
|
||||
)
|
||||
|
||||
|
||||
class LocalProcessWorkerAdapter:
|
||||
"""Worker-local S1B adapter; transport and PX4 mapping remain separate ports."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
guard: S0WorkerGuard,
|
||||
supervisor: PosixProcessSupervisor,
|
||||
specs: tuple[ProcessSpec, ...],
|
||||
world_control: SimulationWorldControl,
|
||||
) -> None:
|
||||
self.guard = guard
|
||||
self.supervisor = supervisor
|
||||
self.specs = specs
|
||||
self.world_control = world_control
|
||||
|
||||
def start(self, run: QualificationRun) -> WorkerStartResult:
|
||||
artifact_root = next(
|
||||
item.wsl_path
|
||||
for item in self.guard.profile.storage.mutable_paths
|
||||
if item.identifier == "artifacts"
|
||||
)
|
||||
artifact_run_root = artifact_root / "s1" / "runs" / run.run_id
|
||||
process_run_root = PurePosixPath(self.supervisor.runtime_root.as_posix()) / run.run_id
|
||||
admission = self.guard.admit(run, artifact_run_root, process_run_root)
|
||||
records = self.supervisor.start(run.run_id, self.specs)
|
||||
return WorkerStartResult(
|
||||
provider_ids=tuple(record.process_id for record in records),
|
||||
profile_sha256=admission.profile_sha256,
|
||||
)
|
||||
|
||||
def pause(self, run: QualificationRun) -> None:
|
||||
self.world_control.pause(run.run_id)
|
||||
|
||||
def resume(self, run: QualificationRun) -> None:
|
||||
self.world_control.resume(run.run_id)
|
||||
|
||||
def step(self, run: QualificationRun, step_count: int) -> int:
|
||||
return self.world_control.step(run.run_id, step_count)
|
||||
|
||||
def stop(self, run: QualificationRun) -> WorkerStopResult:
|
||||
result = self.supervisor.stop()
|
||||
return WorkerStopResult(
|
||||
stopped_provider_ids=result.stopped_process_ids,
|
||||
residue_provider_ids=result.residue_process_ids,
|
||||
)
|
||||
|
||||
def reconcile(self, run: QualificationRun) -> WorkerStopResult:
|
||||
return self.stop(run)
|
||||
Reference in New Issue
Block a user