feat(observatory): expose attempt-bound recorded progress
This commit is contained in:
@@ -47,6 +47,7 @@ from k1link.observatory.portable_worker_runtime import (
|
||||
PortableWorkerSourceStage,
|
||||
inspect_runtime_candidate,
|
||||
)
|
||||
from k1link.observatory.recorded_progress import report_recorded_progress
|
||||
from k1link.observatory.worker_agent import ObservatoryWorkerExecutorRegistration
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -467,7 +468,9 @@ class InstalledLabPackageProfileRunner:
|
||||
}
|
||||
),
|
||||
)
|
||||
for container in _topological_containers(self.package):
|
||||
containers = _topological_containers(self.package)
|
||||
report_recorded_progress("computing", 0, len(containers), "steps")
|
||||
for step_index, container in enumerate(containers):
|
||||
container_output_root = output_root
|
||||
if container.role == "step":
|
||||
container_output_root = steps_root / container.container_id
|
||||
@@ -483,6 +486,8 @@ class InstalledLabPackageProfileRunner:
|
||||
name_token=attempt_token,
|
||||
)
|
||||
)
|
||||
report_recorded_progress("computing", step_index + 1, len(containers), "steps")
|
||||
report_recorded_progress("result-assembly", unit="steps")
|
||||
return _read_result_draft(
|
||||
output_root,
|
||||
plan=plan,
|
||||
|
||||
@@ -19,7 +19,9 @@ import shutil
|
||||
import stat
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from contextlib import suppress
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Final, Protocol, cast
|
||||
@@ -36,6 +38,7 @@ from k1link.observatory.m49_portable_source import (
|
||||
materialize_m49_portable_source_from_worker_stage,
|
||||
validate_m49_portable_source_stage_binding,
|
||||
)
|
||||
from k1link.observatory.m49_timing_progress import M49TimingProgress
|
||||
from k1link.observatory.portable_result_contract import (
|
||||
OBSERVATION_ONLY_AUTHORITY,
|
||||
canonical_json,
|
||||
@@ -53,6 +56,7 @@ from k1link.observatory.portable_worker_runtime import (
|
||||
PortableWorkerSourceMaterializer,
|
||||
PortableWorkerSourceStage,
|
||||
)
|
||||
from k1link.observatory.recorded_progress import report_recorded_progress
|
||||
from k1link.observatory.worker_agent import SealedObservatoryRecordedJob
|
||||
|
||||
M49_COMPILED_RUNNER_BUILD_SCHEMA: Final = "missioncore.m49-tgs-portable-compiled-runner-build/v1"
|
||||
@@ -257,6 +261,7 @@ class M49PortableProfileRunnerAdapter:
|
||||
try:
|
||||
output = workspace / "outputs"
|
||||
timing = workspace / "timing.tsv"
|
||||
report_recorded_progress("computing", 0, stage.timeline_frame_count)
|
||||
invoker = self.invoker or _invoke_exact_runner
|
||||
invoker(
|
||||
binary=self.installation.runner_binary_path,
|
||||
@@ -267,6 +272,7 @@ class M49PortableProfileRunnerAdapter:
|
||||
workspace=workspace,
|
||||
timeout_seconds=self.installation.timeout_seconds,
|
||||
)
|
||||
report_recorded_progress("result-assembly", unit="steps")
|
||||
package = assemble_m49_portable_result(
|
||||
source_stage=stage,
|
||||
runner_output_root=output,
|
||||
@@ -335,19 +341,32 @@ def _invoke_exact_runner(
|
||||
stderr = workspace / "runner.stderr.log"
|
||||
try:
|
||||
with stdout.open("xb") as stdout_stream, stderr.open("xb") as stderr_stream:
|
||||
completed = subprocess.run(
|
||||
started = time.monotonic()
|
||||
progress = M49TimingProgress(timing)
|
||||
process = subprocess.Popen(
|
||||
[str(binary), str(sequence), str(schedule), str(output), str(timing)],
|
||||
cwd=workspace,
|
||||
env={"LANG": "C", "LC_ALL": "C", "TZ": "UTC"},
|
||||
stdin=subprocess.DEVNULL,
|
||||
stdout=stdout_stream,
|
||||
stderr=stderr_stream,
|
||||
check=False,
|
||||
timeout=timeout_seconds,
|
||||
)
|
||||
try:
|
||||
while process.poll() is None:
|
||||
report_recorded_progress("computing", progress.poll())
|
||||
remaining = timeout_seconds - (time.monotonic() - started)
|
||||
if remaining <= 0:
|
||||
raise subprocess.TimeoutExpired(str(binary), timeout_seconds)
|
||||
with suppress(subprocess.TimeoutExpired):
|
||||
process.wait(timeout=min(0.5, remaining))
|
||||
report_recorded_progress("computing", progress.poll())
|
||||
finally:
|
||||
if process.poll() is None:
|
||||
process.kill()
|
||||
process.wait()
|
||||
except (OSError, subprocess.SubprocessError) as exc:
|
||||
raise M49PortableExecutorError("portable M4.9 runner invocation failed") from exc
|
||||
if completed.returncode != 0:
|
||||
if process.returncode != 0:
|
||||
raise M49PortableExecutorError("portable M4.9 runner rejected its exact source stage")
|
||||
|
||||
|
||||
|
||||
@@ -34,6 +34,7 @@ import numpy.typing as npt
|
||||
|
||||
from k1link.compute.lidar_replay import LidarReplayPackV2, build_lidar_replay_pack_v2
|
||||
from k1link.observatory.portable_result_contract import canonical_json
|
||||
from k1link.observatory.recorded_progress import report_recorded_progress
|
||||
from k1link.observatory.source_admission import (
|
||||
PORTABLE_SOURCE_BUNDLE_SCHEMA,
|
||||
PORTABLE_SOURCE_CAPABILITY_SCHEMA,
|
||||
@@ -175,6 +176,7 @@ def materialize_m49_portable_source_from_worker_stage(
|
||||
or worker_stage.source_adapter_sha256 != job.source_adapter_sha256
|
||||
):
|
||||
raise M49PortableSourceError("Worker source stage belongs to another job")
|
||||
report_recorded_progress("source-preparation")
|
||||
root = _safe_directory(worker_stage.root, "Worker source stage")
|
||||
manifest_payload, manifest = _read_canonical_document(
|
||||
root / "materialization-manifest.json",
|
||||
@@ -487,6 +489,7 @@ def _materialize_stage(
|
||||
index_rows: list[dict[str, object]] = []
|
||||
available_slot = 0
|
||||
sequence_logical = hashlib.sha256()
|
||||
report_recorded_progress("source-preparation", 0, len(anchors))
|
||||
for anchor in anchors:
|
||||
point_index = int(
|
||||
np.searchsorted(point_times, anchor.session_seconds, side="right") - 1
|
||||
@@ -534,6 +537,9 @@ def _materialize_stage(
|
||||
f"{anchor.timeline_frame_index}\t{anchor.source_frame_index}"
|
||||
f"\t{anchor.session_seconds:.9f}\t-1\t0"
|
||||
)
|
||||
report_recorded_progress(
|
||||
"source-preparation", anchor.timeline_frame_index + 1, len(anchors),
|
||||
)
|
||||
continue
|
||||
|
||||
start_seconds = anchor.session_seconds - profile.history_seconds
|
||||
@@ -593,6 +599,9 @@ def _materialize_stage(
|
||||
f"\t{anchor.session_seconds:.9f}\t{available_slot}\t{native.shape[0]}"
|
||||
)
|
||||
available_slot += 1
|
||||
report_recorded_progress(
|
||||
"source-preparation", anchor.timeline_frame_index + 1, len(anchors),
|
||||
)
|
||||
|
||||
if available_slot < 1:
|
||||
raise M49PortableSourceError("portable K1 source has no admissible LiDAR frames")
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
"""Incremental observation of flushed TGS timing rows, not result validation."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class M49TimingProgress:
|
||||
def __init__(self, path: Path) -> None:
|
||||
self.path = path
|
||||
self.offset = 0
|
||||
self.pending = b""
|
||||
self.header = False
|
||||
self.completed = 0
|
||||
self.invalid = False
|
||||
|
||||
def poll(self) -> int:
|
||||
if self.invalid:
|
||||
return self.completed
|
||||
try:
|
||||
with self.path.open("rb") as stream:
|
||||
stream.seek(self.offset)
|
||||
data = stream.read(64 * 1024)
|
||||
self.offset += len(data)
|
||||
except OSError:
|
||||
return self.completed
|
||||
lines = (self.pending + data).split(b"\n")
|
||||
self.pending = lines.pop()
|
||||
if len(self.pending) > 2048:
|
||||
self.invalid = True
|
||||
self.pending = b""
|
||||
return self.completed
|
||||
for line in lines:
|
||||
if not self.header:
|
||||
self.header = line.startswith(b"timeline_frame_index\tsource_frame_index\t")
|
||||
if not self.header:
|
||||
self.invalid = True
|
||||
break
|
||||
continue
|
||||
fields = line.split(b"\t")
|
||||
if len(fields) != 10 or fields[0] != str(self.completed).encode():
|
||||
self.invalid = True
|
||||
break
|
||||
self.completed += 1
|
||||
return self.completed
|
||||
@@ -27,6 +27,7 @@ from k1link.observatory.portable_run_definitions import (
|
||||
PortableRunDefinitionRegistry,
|
||||
canonical_sha256,
|
||||
)
|
||||
from k1link.observatory.recorded_progress import report_recorded_progress
|
||||
from k1link.observatory.worker_agent import (
|
||||
ObservatoryWorkerExecutionResult,
|
||||
ObservatoryWorkerExecutorIdentity,
|
||||
@@ -732,6 +733,7 @@ class PortableWorkerExecutorAdapter:
|
||||
job: SealedObservatoryRecordedJob,
|
||||
) -> ObservatoryWorkerExecutionResult:
|
||||
self._verify_job(job)
|
||||
report_recorded_progress("source-transfer", unit="members")
|
||||
source = self.source_materializer.materialize(job)
|
||||
if (
|
||||
source.source_bundle_sha256 != job.source_bundle_sha256
|
||||
@@ -764,11 +766,13 @@ class PortableWorkerExecutorAdapter:
|
||||
result_contract_sha256=self.candidate.result_contract_sha256,
|
||||
phases=tuple(phase.phase_id for phase in self.candidate.phases),
|
||||
)
|
||||
report_recorded_progress("computing", unit="steps")
|
||||
draft = self.runner.run(plan, source)
|
||||
if draft.result_contract_sha256 != self.candidate.result_contract_sha256:
|
||||
raise PortableWorkerRuntimeJobRejectedError(
|
||||
"runtime result uses another result contract"
|
||||
)
|
||||
report_recorded_progress("result-transfer", unit="members")
|
||||
published = self.publisher.publish(job, draft)
|
||||
if (
|
||||
published.result_id != draft.result_id
|
||||
|
||||
@@ -32,6 +32,7 @@ from typing import Final, Literal
|
||||
from uuid import uuid4
|
||||
|
||||
from k1link.artifacts import utc_now_iso
|
||||
from k1link.observatory.recorded_progress import RecordedProgress
|
||||
|
||||
OBSERVATORY_RECORDED_JOB_SCHEMA: Final = "missioncore.observatory-recorded-job/v1"
|
||||
OBSERVATORY_RECORDED_JOB_REQUEST_SCHEMA: Final = "missioncore.observatory-recorded-job-request/v1"
|
||||
@@ -159,6 +160,12 @@ CREATE TABLE IF NOT EXISTS observatory_recorded_jobs (
|
||||
CREATE INDEX IF NOT EXISTS observatory_recorded_jobs_queue_order
|
||||
ON observatory_recorded_jobs (state, priority_rank, created_at_utc, job_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS observatory_recorded_progress (
|
||||
job_id TEXT PRIMARY KEY REFERENCES observatory_recorded_jobs(job_id),
|
||||
snapshot_json TEXT NOT NULL CHECK (length(snapshot_json) <= 2048),
|
||||
received_at_utc TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS observatory_recorded_claim_receipts (
|
||||
claim_request_id TEXT PRIMARY KEY,
|
||||
request_sha256 TEXT NOT NULL,
|
||||
@@ -1614,6 +1621,73 @@ class ObservatoryRecordedJobQueue:
|
||||
)
|
||||
return self._get_job(connection, job_id)
|
||||
|
||||
def report_progress(
|
||||
self, job_id: str, *, claim_token: str, claimant_id: str,
|
||||
progress: RecordedProgress,
|
||||
) -> None:
|
||||
"""Replace one small observation, fenced in the ownership transaction."""
|
||||
_validate_pattern(job_id, _JOB_ID, "recorded job id")
|
||||
_validate_pattern(claim_token, _TOKEN, "claim token")
|
||||
payload = progress.model_dump_json()
|
||||
with self._transaction() as connection:
|
||||
job = self._get_job(connection, job_id)
|
||||
now = self._timestamp()
|
||||
self._require_active_claim(job, claim_token, now=now)
|
||||
if (
|
||||
job.claim_generation != progress.claim_generation
|
||||
or job.active_claimant_id != claimant_id
|
||||
):
|
||||
raise ObservatoryRecordedQueueStaleClaimError("progress owner is stale")
|
||||
if job.state != "running":
|
||||
raise ObservatoryRecordedQueueConflictError("progress requires running execution")
|
||||
row = connection.execute(
|
||||
"SELECT snapshot_json FROM observatory_recorded_progress WHERE job_id = ?",
|
||||
(job_id,),
|
||||
).fetchone()
|
||||
if row is not None:
|
||||
previous = RecordedProgress.model_validate_json(row["snapshot_json"])
|
||||
if previous.claim_generation == progress.claim_generation:
|
||||
if previous == progress:
|
||||
return
|
||||
if not progress.follows(previous):
|
||||
raise ObservatoryRecordedQueueConflictError("progress snapshot regressed")
|
||||
connection.execute(
|
||||
"INSERT INTO observatory_recorded_progress VALUES (?, ?, ?) "
|
||||
"ON CONFLICT(job_id) DO UPDATE SET snapshot_json = excluded.snapshot_json, "
|
||||
"received_at_utc = excluded.received_at_utc",
|
||||
(job_id, payload, now),
|
||||
)
|
||||
|
||||
def progress(self, job_id: str) -> dict[str, object]:
|
||||
"""Read-only projection; never claims, renews, publishes or computes."""
|
||||
_validate_pattern(job_id, _JOB_ID, "recorded job id")
|
||||
with self._read_connection() as connection:
|
||||
job = self._get_job(connection, job_id)
|
||||
row = connection.execute(
|
||||
"SELECT snapshot_json, received_at_utc FROM observatory_recorded_progress "
|
||||
"WHERE job_id = ?", (job_id,),
|
||||
).fetchone()
|
||||
progress = None if row is None else RecordedProgress.model_validate_json(
|
||||
row["snapshot_json"]
|
||||
)
|
||||
if progress is not None and progress.claim_generation != job.claim_generation:
|
||||
progress = None
|
||||
return {
|
||||
"schema_version": "missioncore.observatory-recorded-progress-view/v1",
|
||||
"job_id": job.job_id,
|
||||
"source_session_id": job.source_session_id,
|
||||
"setup_id": job.setup_id,
|
||||
"definition_sha256": job.definition_sha256,
|
||||
"claim_generation": job.claim_generation,
|
||||
"state": job.state,
|
||||
"received_at_utc": None if progress is None else row["received_at_utc"],
|
||||
"age_seconds": None if progress is None else max(0.0, (
|
||||
_parse_timestamp(self._timestamp(), "clock")
|
||||
- _parse_timestamp(row["received_at_utc"], "progress receipt")
|
||||
).total_seconds()),
|
||||
"progress": None if progress is None else progress.model_dump(mode="json"),
|
||||
}
|
||||
|
||||
def authorize_claim_access(
|
||||
self,
|
||||
job_id: str,
|
||||
@@ -2816,6 +2890,7 @@ class ObservatoryRecordedJobQueue:
|
||||
|
||||
def _validate_schema(self, connection: sqlite3.Connection) -> None:
|
||||
expected = {
|
||||
"observatory_recorded_progress": 3,
|
||||
"observatory_recorded_jobs": 50,
|
||||
"observatory_recorded_claim_receipts": 6,
|
||||
"observatory_recorded_claim_grants_v3": 6,
|
||||
|
||||
@@ -0,0 +1,166 @@
|
||||
"""Bounded, attempt-scoped observation of recorded execution, never authority.
|
||||
|
||||
The execution thread owns counters; a separate bounded sender samples them.
|
||||
Progress loss cannot cancel compute or renew a claim. No frame event journal,
|
||||
estimated completion, filesystem paths, or model-dependent UI contract.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Callable, Iterator
|
||||
from contextlib import contextmanager, suppress
|
||||
from contextvars import ContextVar
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
||||
|
||||
type ProgressPhase = Literal[
|
||||
"source-transfer",
|
||||
"source-preparation",
|
||||
"computing",
|
||||
"result-assembly",
|
||||
"result-transfer",
|
||||
]
|
||||
type ProgressUnit = Literal["frames", "members", "steps"]
|
||||
|
||||
|
||||
class RecordedProgress(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", strict=True, frozen=True)
|
||||
|
||||
schema_version: Literal["missioncore.observatory-recorded-progress/v1"] = (
|
||||
"missioncore.observatory-recorded-progress/v1"
|
||||
)
|
||||
claim_generation: int = Field(ge=1, le=2**53 - 1)
|
||||
sequence: int = Field(ge=1, le=2**53 - 1)
|
||||
phase_index: int = Field(ge=0, le=2**53 - 1)
|
||||
phase: ProgressPhase
|
||||
unit: ProgressUnit
|
||||
completed: int = Field(ge=0, le=2**53 - 1)
|
||||
total: int | None = Field(default=None, ge=1, le=2**53 - 1)
|
||||
elapsed_seconds: float = Field(ge=0, allow_inf_nan=False)
|
||||
phase_elapsed_seconds: float = Field(ge=0, allow_inf_nan=False)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_counts(self) -> RecordedProgress:
|
||||
if self.total is not None and self.completed > self.total:
|
||||
raise ValueError("completed exceeds the observed total")
|
||||
if self.phase_elapsed_seconds > self.elapsed_seconds:
|
||||
raise ValueError("phase duration exceeds execution duration")
|
||||
return self
|
||||
|
||||
def follows(self, previous: RecordedProgress) -> bool:
|
||||
if self.claim_generation != previous.claim_generation:
|
||||
return False
|
||||
if self.sequence <= previous.sequence or self.phase_index < previous.phase_index:
|
||||
return False
|
||||
if self.elapsed_seconds < previous.elapsed_seconds:
|
||||
return False
|
||||
if self.phase_index != previous.phase_index:
|
||||
return True
|
||||
return (
|
||||
self.phase == previous.phase
|
||||
and self.unit == previous.unit
|
||||
and self.completed >= previous.completed
|
||||
and (previous.total is None or self.total == previous.total)
|
||||
and self.phase_elapsed_seconds >= previous.phase_elapsed_seconds
|
||||
)
|
||||
|
||||
|
||||
class RecordedProgressTracker:
|
||||
def __init__(self, generation: int, *, clock: Callable[[], float] = time.monotonic):
|
||||
self.generation = generation
|
||||
self.clock = clock
|
||||
self.started = clock()
|
||||
self.phase_started = self.started
|
||||
self.lock = threading.Lock()
|
||||
self.phase: ProgressPhase = "source-transfer"
|
||||
self.unit: ProgressUnit = "members"
|
||||
self.completed = 0
|
||||
self.total: int | None = None
|
||||
self.phase_index = 0
|
||||
self.sequence = 0
|
||||
|
||||
def update(
|
||||
self,
|
||||
phase: ProgressPhase,
|
||||
completed: int = 0,
|
||||
total: int | None = None,
|
||||
unit: ProgressUnit = "frames",
|
||||
) -> None:
|
||||
with self.lock:
|
||||
if phase != self.phase or unit != self.unit:
|
||||
self.phase_index += 1
|
||||
self.phase_started = self.clock()
|
||||
elif completed < self.completed:
|
||||
raise ValueError("progress counter moved backwards within one phase")
|
||||
elif total is None:
|
||||
total = self.total
|
||||
self.phase, self.unit = phase, unit
|
||||
self.completed, self.total = completed, total
|
||||
|
||||
def sample(self) -> RecordedProgress:
|
||||
with self.lock:
|
||||
now = self.clock()
|
||||
self.sequence += 1
|
||||
return RecordedProgress(
|
||||
claim_generation=self.generation,
|
||||
sequence=self.sequence,
|
||||
phase_index=self.phase_index,
|
||||
phase=self.phase,
|
||||
unit=self.unit,
|
||||
completed=self.completed,
|
||||
total=self.total,
|
||||
elapsed_seconds=max(0.0, now - self.started),
|
||||
phase_elapsed_seconds=max(0.0, now - self.phase_started),
|
||||
)
|
||||
|
||||
|
||||
_CURRENT: ContextVar[RecordedProgressTracker | None] = ContextVar(
|
||||
"observatory_recorded_progress",
|
||||
default=None,
|
||||
)
|
||||
|
||||
|
||||
def report_recorded_progress(
|
||||
phase: ProgressPhase,
|
||||
completed: int = 0,
|
||||
total: int | None = None,
|
||||
unit: ProgressUnit = "frames",
|
||||
) -> None:
|
||||
tracker = _CURRENT.get()
|
||||
if tracker is not None:
|
||||
tracker.update(phase, completed, total, unit)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def observe_recorded_execution(
|
||||
generation: int,
|
||||
send: Callable[[RecordedProgress], None],
|
||||
*,
|
||||
interval_seconds: float = 1.0,
|
||||
) -> Iterator[RecordedProgressTracker]:
|
||||
if not math.isfinite(interval_seconds) or not 0.01 <= interval_seconds <= 30:
|
||||
raise ValueError("progress sampling interval is invalid")
|
||||
tracker = RecordedProgressTracker(generation)
|
||||
stop = threading.Event()
|
||||
|
||||
def pump() -> None:
|
||||
while not stop.is_set():
|
||||
# Progress is secondary observation, not lease or result authority.
|
||||
with suppress(Exception):
|
||||
send(tracker.sample())
|
||||
stop.wait(interval_seconds)
|
||||
|
||||
token = _CURRENT.set(tracker)
|
||||
thread = threading.Thread(target=pump, name="observatory-progress", daemon=True)
|
||||
thread.start()
|
||||
try:
|
||||
yield tracker
|
||||
finally:
|
||||
_CURRENT.reset(token)
|
||||
stop.set()
|
||||
# The production sender reads no body and uses 2-second I/O timeouts.
|
||||
thread.join(timeout=3.0)
|
||||
@@ -18,6 +18,7 @@ import json
|
||||
import re
|
||||
import threading
|
||||
from collections.abc import Callable, Mapping
|
||||
from contextlib import nullcontext
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Annotated, Final, Literal, Protocol
|
||||
@@ -31,6 +32,7 @@ from k1link.observatory.recorded_jobs import (
|
||||
OBSERVATORY_RECORDED_JOB_SCHEMA,
|
||||
RecordedExecutorIdentity,
|
||||
)
|
||||
from k1link.observatory.recorded_progress import observe_recorded_execution
|
||||
|
||||
WORKER_006_CONTOUR_ID: Final = "worker-006"
|
||||
MAX_EXECUTOR_FAILURE_MESSAGE_LENGTH: Final = 512
|
||||
@@ -498,7 +500,18 @@ class ObservatoryWorkerAgent:
|
||||
heartbeat.start()
|
||||
|
||||
try:
|
||||
result = adapter.execute(active_job)
|
||||
send_progress = getattr(self._transport, "report_progress", None)
|
||||
observation = (
|
||||
observe_recorded_execution(
|
||||
active_job.claim_generation,
|
||||
lambda snapshot: send_progress(
|
||||
job_id=active_job.job_id, claim_token=claim.claim_token,
|
||||
progress=snapshot,
|
||||
),
|
||||
) if callable(send_progress) else nullcontext()
|
||||
)
|
||||
with observation:
|
||||
result = adapter.execute(active_job)
|
||||
if not isinstance(result, ObservatoryWorkerExecutionResult):
|
||||
raise TypeError("executor returned an unknown result contract")
|
||||
except Exception as exc:
|
||||
|
||||
@@ -47,6 +47,7 @@ from k1link.observatory.portable_worker_runtime import (
|
||||
PortableWorkerResultDraft,
|
||||
PortableWorkerSourceStage,
|
||||
)
|
||||
from k1link.observatory.recorded_progress import RecordedProgress, report_recorded_progress
|
||||
from k1link.observatory.source_admission import (
|
||||
PORTABLE_SPATIAL_REPLAY_METADATA_ARTIFACT_ID,
|
||||
PORTABLE_SPATIAL_REPLAY_METADATA_MEDIA_TYPE,
|
||||
@@ -279,6 +280,23 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
|
||||
},
|
||||
)
|
||||
|
||||
def report_progress(
|
||||
self, *, job_id: str, claim_token: str, progress: RecordedProgress,
|
||||
) -> None:
|
||||
context = self._require_cached_claim(job_id, claim_token)
|
||||
if progress.claim_generation != context.claim_generation:
|
||||
raise ObservatoryWorkerHttpError("progress generation changed")
|
||||
# No response body is needed. Bounded I/O cannot stall the execution thread.
|
||||
with self._client.stream(
|
||||
"POST", self._job_path(job_id, "progress"),
|
||||
json={
|
||||
"schema_version": "missioncore.observatory-worker-progress-request/v1",
|
||||
"claim_token": claim_token, "progress": progress.model_dump(mode="json"),
|
||||
}, timeout=httpx.Timeout(2.0),
|
||||
) as response:
|
||||
if response.status_code != 204:
|
||||
raise ObservatoryWorkerHttpError("progress observation was not accepted")
|
||||
|
||||
def succeed(
|
||||
self,
|
||||
*,
|
||||
@@ -336,6 +354,7 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
|
||||
headers=headers,
|
||||
)
|
||||
members = _source_members(manifest, job)
|
||||
report_recorded_progress("source-transfer", 0, len(members), "members")
|
||||
root = _secure_directory(
|
||||
self._work_root
|
||||
/ "sources"
|
||||
@@ -368,10 +387,16 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
|
||||
members=camera_members,
|
||||
layout=layout,
|
||||
)
|
||||
completed_members = len(camera_members) if camera_epoch_ready else 0
|
||||
report_recorded_progress("source-transfer", completed_members, len(members), "members")
|
||||
for destination, member in destinations.items():
|
||||
if camera_epoch_ready and member.kind in {"camera-init", "camera-segment"}:
|
||||
continue
|
||||
if _matches_file(destination, member.sha256, member.byte_length):
|
||||
completed_members += 1
|
||||
report_recorded_progress(
|
||||
"source-transfer", completed_members, len(members), "members",
|
||||
)
|
||||
continue
|
||||
self._download_member(
|
||||
job_id=job.job_id,
|
||||
@@ -379,6 +404,8 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
|
||||
member=member,
|
||||
destination=destination,
|
||||
)
|
||||
completed_members += 1
|
||||
report_recorded_progress("source-transfer", completed_members, len(members), "members")
|
||||
manifest_path = root / "materialization-manifest.json"
|
||||
_write_local_exact(manifest_path, canonical_json(manifest))
|
||||
return PortableWorkerSourceStage(
|
||||
@@ -440,7 +467,8 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
|
||||
or {member.role for member in upload_members} != set(artifacts)
|
||||
):
|
||||
raise ObservatoryWorkerHttpError("Worker result artifact roles are not unique")
|
||||
for member in upload_members:
|
||||
report_recorded_progress("result-transfer", 0, len(upload_members), "members")
|
||||
for member_index, member in enumerate(upload_members):
|
||||
artifact = artifacts.get(member.role)
|
||||
expected_member_id = hashlib.sha256(
|
||||
canonical_json(
|
||||
@@ -461,6 +489,9 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
|
||||
"result upload plan differs from the local manifest"
|
||||
)
|
||||
if member.uploaded:
|
||||
report_recorded_progress(
|
||||
"result-transfer", member_index + 1, len(upload_members), "members",
|
||||
)
|
||||
continue
|
||||
relative = relative_artifact_path(artifact.relative_path)
|
||||
source = _confined_local_member(draft.root, relative.parts)
|
||||
@@ -489,6 +520,9 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
|
||||
raise ObservatoryWorkerHttpError(
|
||||
"result upload acknowledgement did not seal its member"
|
||||
)
|
||||
report_recorded_progress(
|
||||
"result-transfer", member_index + 1, len(upload_members), "members",
|
||||
)
|
||||
receipt = self._required_json_request(
|
||||
"POST",
|
||||
self._job_path(
|
||||
|
||||
@@ -1151,6 +1151,17 @@ def build_observatory_router(
|
||||
"authority": dict(_OBSERVATION_ONLY_AUTHORITY),
|
||||
}
|
||||
|
||||
@router.get("/api/v1/observatory/runs/{job_id}/progress")
|
||||
def get_observatory_progress(
|
||||
job_id: str = ApiPath(pattern=r"^observatory-run-[a-f0-9]{32}$"),
|
||||
) -> dict[str, object]:
|
||||
try:
|
||||
return recorded_job_queue.progress(job_id)
|
||||
except ObservatoryRecordedQueueNotFoundError as exc:
|
||||
raise HTTPException(status_code=404, detail="Расчёт не найден.") from exc
|
||||
except (ObservatoryRecordedQueueError, ValueError) as exc:
|
||||
raise HTTPException(status_code=503, detail="Прогресс недоступен.") from exc
|
||||
|
||||
@router.get("/api/v1/observatory/runs/{job_id}")
|
||||
def get_observatory_recorded_run(
|
||||
job_id: str = ApiPath(
|
||||
|
||||
@@ -51,6 +51,7 @@ from k1link.observatory.recorded_jobs import (
|
||||
ObservatoryRecordedQueueStaleClaimError,
|
||||
RecordedExecutorIdentity,
|
||||
)
|
||||
from k1link.observatory.recorded_progress import RecordedProgress
|
||||
|
||||
OBSERVATORY_WORKER_CAPABILITY_CLAIM_REQUEST_SCHEMA: Final = (
|
||||
"missioncore.observatory-worker-claim-request/v2"
|
||||
@@ -194,6 +195,12 @@ class ObservatoryWorkerRenewRequest(_StrictWorkerRequest):
|
||||
heartbeat_sequence: int = Field(ge=1)
|
||||
|
||||
|
||||
class ObservatoryWorkerProgressRequest(_StrictWorkerRequest):
|
||||
schema_version: Literal["missioncore.observatory-worker-progress-request/v1"]
|
||||
claim_token: str = Field(pattern=_CLAIM_TOKEN_PATTERN)
|
||||
progress: RecordedProgress
|
||||
|
||||
|
||||
class ObservatoryWorkerCheckpointRequest(_StrictWorkerRequest):
|
||||
schema_version: Literal["missioncore.observatory-worker-checkpoint-request/v1"]
|
||||
claim_token: str = Field(pattern=_CLAIM_TOKEN_PATTERN)
|
||||
@@ -333,6 +340,17 @@ def build_observatory_worker_router(
|
||||
)
|
||||
).as_dict()
|
||||
|
||||
@router.post("/recorded-jobs/{job_id}/progress", status_code=204)
|
||||
def report_job_progress(
|
||||
request: ObservatoryWorkerProgressRequest,
|
||||
job_id: Annotated[str, ApiPath(pattern=_JOB_ID_PATTERN)],
|
||||
) -> Response:
|
||||
_queue_call(lambda: queue.report_progress(
|
||||
job_id, claim_token=request.claim_token,
|
||||
claimant_id=authentication.contour_id, progress=request.progress,
|
||||
))
|
||||
return Response(status_code=204)
|
||||
|
||||
@router.post("/recorded-jobs/{job_id}/checkpoint")
|
||||
def checkpoint_job(
|
||||
request: ObservatoryWorkerCheckpointRequest,
|
||||
|
||||
Reference in New Issue
Block a user