SILVER/NODEDC_MISSION_CORE
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat(perception): stabilize pre-capture methodology

Browse Source
This commit is contained in:
DCCONSTRUCTIONS
2026-07-28 17:47:06 +03:00
parent 1f20e0d7d9
commit d729abab31
65 changed files with 9698 additions and 152 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/k1link/web/advanced_laboratory_api.py
+89
View File
@@ -49,6 +49,11 @@ from k1link.compute.e39_perception_refinement import (
E39PerceptionRefinementError,
read_e39_perception_refinement,
)
from k1link.compute.e40_perception_product_gate import (
E40PerceptionProductGate,
E40PerceptionProductGateError,
read_e40_perception_product_gate,
)
LABORATORY_ADVANCED_CATALOG_SCHEMA: Final = (
"missioncore.laboratory-advanced-catalog/v1"
@@ -62,6 +67,7 @@ _E35_RESULT_ID = re.compile(r"^e35-degradation-recovery-[a-f0-9]{64}$")
_E37_RESULT_ID = re.compile(r"^e37-ravnoves-acceptance-[a-f0-9]{64}$")
_E38_RESULT_ID = re.compile(r"^e38-perception-baseline-[a-f0-9]{64}$")
_E39_RESULT_ID = re.compile(r"^e39-perception-refinement-[a-f0-9]{64}$")
_E40_RESULT_ID = re.compile(r"^e40-perception-product-gate-[a-f0-9]{64}$")
RootProvider = Callable[[], Path | None]
@@ -148,6 +154,15 @@ def _read_e39_cached(
return read_e39_perception_refinement(Path(root_text))
@lru_cache(maxsize=16)
def _read_e40_cached(
root_text: str,
signature: tuple[int, ...],
) -> E40PerceptionProductGate:
del signature
return read_e40_perception_product_gate(Path(root_text))
def _configured_root(provider: RootProvider) -> Path | None:
value = provider()
if value is None:
@@ -659,6 +674,46 @@ def _project_e39(result: E39PerceptionRefinement) -> dict[str, object]:
}
def _project_e40(result: E40PerceptionProductGate) -> dict[str, object]:
identity = _object(result.manifest.get("identity"), "E40 identity")
source = _object(identity.get("source"), "E40 source")
execution = _object(identity.get("execution"), "E40 execution")
profile = _object(identity.get("profile"), "E40 profile")
metrics = _object(result.report.get("metrics"), "E40 metrics")
dimensions = _object(metrics.get("dimensions"), "E40 dimensions")
quality_gate = _object(result.report.get("quality_gate"), "E40 gate")
development_cv = _object(
result.report.get("development_cross_validation"),
"E40 development CV",
)
return {
"result_id": result.result_id,
"created_at_utc": result.manifest.get("created_at_utc"),
"source_session_id": source.get("session_id"),
"source_display_name": source.get("display_name"),
"status": result.report.get("status"),
"profile_id": profile.get("profile_id"),
"worker_node": execution.get("worker_node"),
"quality_gate_passed": quality_gate.get("passed"),
"development_cross_validation": copy.deepcopy(development_cv),
"metrics": {
"development_items": metrics.get("development_items"),
"validation_items": metrics.get("validation_items"),
"terminal_outcomes": metrics.get("terminal_outcomes"),
"accounting_fraction": metrics.get("accounting_fraction"),
"false_free_claims": metrics.get("false_free_claims"),
"high_severity_failures": metrics.get("high_severity_failures"),
"dimensions": copy.deepcopy(dimensions),
},
"blocking_checks": copy.deepcopy(quality_gate.get("blocking_checks")),
"method": copy.deepcopy(result.report.get("method")),
"decision": copy.deepcopy(result.report.get("decision")),
"limitations": copy.deepcopy(result.report.get("limitations")),
"authority": copy.deepcopy(result.report.get("authority")),
"access": "read-only",
}
def _empty_catalog(configured: bool) -> dict[str, object]:
return {
"schema_version": LABORATORY_ADVANCED_CATALOG_SCHEMA,
@@ -680,6 +735,7 @@ def build_advanced_laboratory_router(
e37_root_provider: RootProvider = lambda: None,
e38_root_provider: RootProvider = lambda: None,
e39_root_provider: RootProvider = lambda: None,
e40_root_provider: RootProvider = lambda: None,
) -> APIRouter:
router = APIRouter(prefix="/api/v1/laboratory", tags=["laboratory"])
@@ -975,4 +1031,37 @@ def build_advanced_laboratory_router(
"invalid_total": invalid_total,
}
@router.get("/e40/results")
def list_e40_results(
limit: int = Query(default=1, ge=1, le=10),
) -> dict[str, object]:
root = _configured_root(e40_root_provider)
if root is None:
return _empty_catalog(False)
candidates = _candidates(root, _E40_RESULT_ID)
items: list[dict[str, object]] = []
invalid_total = 0
for candidate in candidates:
try:
result = _read_e40_cached(
str(candidate.resolve()),
_result_signature(candidate),
)
if len(items) < limit:
items.append(_project_e40(result))
except (
E40PerceptionProductGateError,
KeyError,
OSError,
TypeError,
ValueError,
):
invalid_total += 1
return {
**_empty_catalog(True),
"items": items,
"candidate_total": len(candidates),
"invalid_total": invalid_total,
}
return router
src/k1link/web/app.py
+7
View File
@@ -557,6 +557,13 @@ app.include_router(
/ "e39"
/ "results"
),
e40_root_provider=lambda: (
REPOSITORY_ROOT
/ ".runtime"
/ "compute-experiments"
/ "e40"
/ "results"
),
)
)
app.include_router(
src/k1link/web/compute_contour_api.py
+12
View File
@@ -47,6 +47,7 @@ class ComputeContour(StrictModel):
mqtt_host: str = Field(default="127.0.0.1", min_length=1, max_length=253)
mqtt_port: int = Field(default=1883, ge=1, le=65535)
telemetry_poll_interval_seconds: int = Field(default=3, ge=1, le=60)
mqtt_publish_interval_seconds: int = Field(default=2, ge=1, le=60)
revision: int = Field(default=0, ge=0)
updated_at_utc: str | None = None
@@ -92,6 +93,7 @@ class ComputeContourCreate(StrictModel):
mqtt_host: str = Field(default="127.0.0.1", min_length=1, max_length=253)
mqtt_port: int = Field(default=1883, ge=1, le=65535)
telemetry_poll_interval_seconds: int = Field(default=3, ge=1, le=60)
mqtt_publish_interval_seconds: int = Field(default=2, ge=1, le=60)
@field_validator("display_name")
@classmethod
@@ -127,6 +129,7 @@ def default_compute_contour() -> ComputeContour:
mqtt_host="127.0.0.1",
mqtt_port=1883,
telemetry_poll_interval_seconds=3,
mqtt_publish_interval_seconds=2,
)
@@ -158,6 +161,9 @@ class ComputeContourStore:
telemetry_poll_interval_seconds=(
request.telemetry_poll_interval_seconds
),
mqtt_publish_interval_seconds=(
request.mqtt_publish_interval_seconds
),
revision=0,
updated_at_utc=_utc_now(),
)
@@ -186,6 +192,9 @@ class ComputeContourStore:
"telemetry_poll_interval_seconds": (
request.telemetry_poll_interval_seconds
),
"mqtt_publish_interval_seconds": (
request.mqtt_publish_interval_seconds
),
"revision": current.revision + 1,
"updated_at_utc": _utc_now(),
}
@@ -260,6 +269,9 @@ def _agent_install_document(contour: ComputeContour) -> dict[str, object]:
"MISSIONCORE_MQTT_HOST": contour.mqtt_host,
"MISSIONCORE_MQTT_PORT": str(contour.mqtt_port),
"MISSIONCORE_MQTT_USERNAME": contour.agent_id,
"MISSIONCORE_TELEMETRY_INTERVAL": (
f"{contour.mqtt_publish_interval_seconds}s"
),
}
if contour.platform == "windows":
command = (
src/k1link/web/e30_engineering_api.py
+14 -2
View File
@@ -590,9 +590,21 @@ def build_e30_engineering_router(
result_id=result_id,
)
rows_by_id = {row["item_id"]: row for row in rows}
exception_rows = catalog_item.get("human_exceptions")
if (
not isinstance(exception_rows, list)
or not all(
isinstance(value, dict)
and isinstance(value.get("item_id"), str)
for value in exception_rows
)
):
raise E30EngineeringEvidenceError(
"engineering exception catalog is invalid"
)
exception_ids = [
value["item_id"]
for value in catalog_item["human_exceptions"]
str(value["item_id"])
for value in exception_rows
]
if any(item_id not in rows_by_id for item_id in exception_ids):
raise E30EngineeringEvidenceError(
src/k1link/web/e30_human_review_api.py
+14 -2
View File
@@ -94,9 +94,21 @@ def build_e30_human_review_router(
subject.item_id: subject
for subject in source_substrate.subjects
}
exception_rows = generation.get("human_exceptions")
if (
not isinstance(exception_rows, list)
or not all(
isinstance(value, dict)
and isinstance(value.get("item_id"), str)
for value in exception_rows
)
):
raise E30HumanReviewValidationError(
"engineering exception catalog is invalid"
)
exception_ids = [
value["item_id"]
for value in generation["human_exceptions"]
str(value["item_id"])
for value in exception_rows
]
if (
not exception_ids
src/k1link/web/system_telemetry_api.py
+124 -20
View File
@@ -16,11 +16,13 @@ from collections import deque
from collections.abc import Callable
from datetime import UTC, datetime
from pathlib import Path
from typing import Any, Final
from typing import Any, Final, Protocol
from fastapi import APIRouter, HTTPException, Query
from pydantic import BaseModel, ConfigDict, Field, field_validator
from k1link.web.compute_contour_api import ComputeContour, ComputeContourStore
PROFILE_SCHEMA: Final = "missioncore.worker-connection-profile/v1"
TELEMETRY_SCHEMA: Final = "missioncore.worker-telemetry/v1"
PROBE_SCHEMA: Final = "missioncore.worker-probe/v1"
@@ -79,6 +81,14 @@ RootProvider = Callable[[], Path]
ProbeRunner = Callable[["WorkerConnectionProfile"], dict[str, Any]]
class WorkerProfileStoreContract(Protocol):
def read(self) -> WorkerConnectionProfile:
"""Return the current worker-shaped connection profile."""
def save(self, request: WorkerConnectionProfilePut) -> WorkerConnectionProfile:
"""Persist one reviewed profile update."""
class WorkerConnectionProfile(BaseModel):
model_config = ConfigDict(extra="forbid", frozen=True)
@@ -603,16 +613,28 @@ def _agent_raw_document(document: dict[str, Any]) -> dict[str, Any]:
return raw
def run_worker_agent_probe(profile: WorkerConnectionProfile) -> dict[str, Any]:
def run_worker_agent_probe(
profile: WorkerConnectionProfile,
*,
contour_id: str | None = None,
agent_id: str | None = None,
) -> dict[str, Any]:
started = time.perf_counter()
base_url = os.environ.get(
"MISSIONCORE_TELEMETRY_QUERY_URL",
DEFAULT_TELEMETRY_QUERY_URL,
).rstrip("/")
contour_id = os.environ.get("MISSIONCORE_TELEMETRY_CONTOUR_ID", "worker-006")
agent_id = os.environ.get("MISSIONCORE_TELEMETRY_AGENT_ID", "worker-006")
resolved_contour_id = contour_id or os.environ.get(
"MISSIONCORE_TELEMETRY_CONTOUR_ID",
"worker-006",
)
resolved_agent_id = agent_id or os.environ.get(
"MISSIONCORE_TELEMETRY_AGENT_ID",
"worker-006",
)
url = (
f"{base_url}/v1/contours/{contour_id}/agents/{agent_id}/latest"
f"{base_url}/v1/contours/{resolved_contour_id}/agents/"
f"{resolved_agent_id}/latest"
"?max_age_seconds=30"
)
try:
@@ -898,7 +920,7 @@ def _pipeline_document(raw: dict[str, Any]) -> dict[str, Any]:
class WorkerTelemetryService:
def __init__(
self,
store: WorkerProfileStore,
store: WorkerProfileStoreContract,
probe_runner: ProbeRunner = run_worker_probe,
*,
telemetry_probe_runner: ProbeRunner | None = None,
@@ -911,7 +933,9 @@ class WorkerTelemetryService:
self._lock = threading.Lock()
self._cached_at = 0.0
self._cached: dict[str, Any] | None = None
self._previous_network: tuple[float, float, float] | None = None
self._previous_network: tuple[str, float, float, float] | None = None
self._latest_network_rates: tuple[float | None, float | None] = (None, None)
self._last_history_observed_at: str | None = None
self._history: deque[dict[str, Any]] = deque(maxlen=300)
def profile_document(self) -> dict[str, Any]:
@@ -948,6 +972,8 @@ class WorkerTelemetryService:
self._cached = None
self._cached_at = 0
self._previous_network = None
self._latest_network_rates = (None, None)
self._last_history_observed_at = None
self._history.clear()
return {
**self.profile_document(),
@@ -1007,25 +1033,36 @@ class WorkerTelemetryService:
for item in _items(raw.get("network"))
if isinstance(item, dict)
]
received = sum(
received = float(
sum(
value
for item in interfaces
if (value := _number(item.get("received_bytes"))) is not None
)
)
sent = sum(
sent = float(
sum(
value
for item in interfaces
if (value := _number(item.get("sent_bytes"))) is not None
)
)
receive_rate: float | None = None
send_rate: float | None = None
if self._previous_network is not None:
previous_at, previous_received, previous_sent = self._previous_network
elapsed = monotonic_now - previous_at
if elapsed > 0 and received >= previous_received and sent >= previous_sent:
receive_rate = (received - previous_received) / elapsed
send_rate = (sent - previous_sent) / elapsed
self._previous_network = (monotonic_now, received, sent)
raw_observed_at = raw.get("observed_at_utc")
observed_at: str = (
raw_observed_at if isinstance(raw_observed_at, str) else _utc_now()
)
receive_rate, send_rate = self._latest_network_rates
if observed_at != self._last_history_observed_at:
receive_rate = None
send_rate = None
if self._previous_network is not None:
_, previous_at, previous_received, previous_sent = self._previous_network
elapsed = monotonic_now - previous_at
if elapsed > 0 and received >= previous_received and sent >= previous_sent:
receive_rate = (received - previous_received) / elapsed
send_rate = (sent - previous_sent) / elapsed
self._previous_network = (observed_at, monotonic_now, received, sent)
self._latest_network_rates = (receive_rate, send_rate)
raw_stats = _mapping(raw.get("docker_stats"))
raw_states = _mapping(raw.get("container_states"))
runtimes = [
@@ -1077,7 +1114,7 @@ class WorkerTelemetryService:
},
}
history_row = {
"observed_at_utc": raw.get("observed_at_utc") or _utc_now(),
"observed_at_utc": observed_at,
"cpu_percent": _number(_mapping(raw.get("cpu")).get("load_percent")),
"memory_percent": memory_used_percent,
"gpu_percent": _number(gpu.get("utilization_percent")),
@@ -1085,7 +1122,9 @@ class WorkerTelemetryService:
"network_receive_bytes_per_second": receive_rate,
"network_send_bytes_per_second": send_rate,
}
self._history.append(history_row)
if observed_at != self._last_history_observed_at:
self._history.append(history_row)
self._last_history_observed_at = observed_at
return {
"schema_version": TELEMETRY_SCHEMA,
"profile": profile.model_dump(mode="json"),
@@ -1154,8 +1193,49 @@ def build_system_telemetry_router(
probe_runner,
telemetry_probe_runner=telemetry_probe_runner,
)
contour_store = ComputeContourStore(root_provider())
contour_services: dict[str, WorkerTelemetryService] = {}
router = APIRouter(prefix="/api/v1/system", tags=["system"])
class ContourProfileStore:
def __init__(self, contour_id: str) -> None:
self.contour_id = contour_id
def read(self) -> WorkerConnectionProfile:
return _profile_from_compute_contour(
contour_store.get(self.contour_id)
)
def save(
self,
request: WorkerConnectionProfilePut,
) -> WorkerConnectionProfile:
del request
raise RuntimeError("compute contour telemetry profiles are read-only")
def contour_service(contour_id: str) -> WorkerTelemetryService:
existing = contour_services.get(contour_id)
if existing is not None:
return existing
def contour_probe(profile: WorkerConnectionProfile) -> dict[str, Any]:
contour = contour_store.get(contour_id)
if contour.telemetry_mode == "legacy-ssh":
return probe_runner(profile)
return run_worker_agent_probe(
profile,
contour_id=contour.contour_id,
agent_id=contour.agent_id,
)
created = WorkerTelemetryService(
ContourProfileStore(contour_id),
probe_runner,
telemetry_probe_runner=contour_probe,
)
contour_services[contour_id] = created
return created
@router.get("/worker-profile")
def get_worker_profile() -> dict[str, Any]:
return service.profile_document()
@@ -1185,4 +1265,28 @@ def build_system_telemetry_router(
) -> dict[str, Any]:
return service.snapshot(history)
@router.get("/contours/{contour_id}/telemetry")
def get_compute_contour_telemetry(
contour_id: str,
history: int = Query(default=90, ge=1, le=300),
) -> dict[str, Any]:
try:
contour_store.get(contour_id)
return contour_service(contour_id).snapshot(history)
except KeyError as exc:
raise HTTPException(status_code=404, detail="Контур не найден.") from exc
return router
def _profile_from_compute_contour(contour: ComputeContour) -> WorkerConnectionProfile:
return WorkerConnectionProfile(
profile_id=contour.contour_id,
display_name=contour.display_name,
expected_node_id=contour.expected_node_id,
ssh_host_alias=SSH_HOST_ALIAS,
address=contour.address,
port=contour.ssh_port,
revision=contour.revision,
updated_at_utc=contour.updated_at_utc,
)