SILVER/NODEDC_MISSION_CORE
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat(worker): compose all ready observatory profiles

Browse Source
This commit is contained in:
DCCONSTRUCTIONS
2026-08-31 18:31:54 +03:00
parent 1ff527b264
commit f64fbb1fa6
4 changed files with 524 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_observatory_m49_worker_entrypoint.py
+36 -2
View File
@@ -3,6 +3,7 @@ from __future__ import annotations
import copy
import hashlib
import json
from dataclasses import replace
from pathlib import Path
from threading import Event
from typing import cast
@@ -32,7 +33,10 @@ from k1link.observatory.portable_run_definitions import (
PortableRunDefinitionRegistry,
canonical_sha256,
)
from k1link.observatory.portable_worker_runtime import PortableWorkerExecutorAdapter
from k1link.observatory.portable_worker_runtime import (
PortableWorkerExecutorAdapter,
PortableWorkerRuntimeRegistry,
)
from k1link.observatory.worker_http_transport import ObservatoryWorkerHttpGateway
from k1link.observatory.worker_service import ObservatoryWorkerServiceConfiguration
@@ -223,7 +227,7 @@ def _runtime_registry(
receipt_payload: bytes,
) -> Path:
definition = definitions.resolve_setup(service_module.M49_WORKER_SETUP_ID)
assets = [
assets: list[dict[str, object]] = [
{
"asset_id": M49_PORTABLE_COMPILED_RUNNER_ASSET_ID,
"kind": "local-file",
@@ -390,6 +394,36 @@ def test_fixed_m49_composition_uses_one_gateway_for_agent_source_and_result(
assert gateway._client.is_closed # noqa: SLF001
def test_fixed_m49_identity_allows_an_additional_ready_worker_profile(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
configuration, _receipt = _fixture(tmp_path, monkeypatch)
definitions = PortableRunDefinitionRegistry.from_file(configuration.definitions_file)
definition = definitions.resolve_setup(service_module.M49_WORKER_SETUP_ID)
runtime = PortableWorkerRuntimeRegistry.from_file(
configuration.runtime_registry_file,
definitions=definitions,
)
candidate = runtime.resolve(definition.setup_id, definition.definition_sha256)
additional = replace(
definitions.ready_recorded_definitions()[0],
setup_id="lab-v1-eomt-ddrnet-portable-v1",
definition_id="lab-v1-eomt-ddrnet-portable",
definition_sha256="7" * 64,
)
class _DefinitionsWithAdditionalReadyProfile:
def ready_recorded_definitions(self): # type: ignore[no-untyped-def]
return (*definitions.ready_recorded_definitions(), additional)
service_module._verify_ready_identity( # noqa: SLF001
cast(PortableRunDefinitionRegistry, _DefinitionsWithAdditionalReadyProfile()),
definition,
candidate,
)
def test_fixed_m49_composition_rejects_receipt_asset_drift_before_gateway(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
tests/test_observatory_worker_service.py
+252 -4
View File
@@ -1,13 +1,15 @@
from __future__ import annotations
from dataclasses import dataclass
from dataclasses import dataclass, replace
from pathlib import Path
from threading import Event
from typing import cast
import httpx
import pytest
from k1link.observatory.portable_run_definitions import PortableRunDefinitionRegistry
from k1link.observatory.portable_worker_runtime import PortableWorkerRuntimeRegistry
from k1link.observatory.recorded_jobs import RecordedRunDefinition
from k1link.observatory.worker_agent import (
ObservatoryWorkerCycleReport,
@@ -20,8 +22,11 @@ from k1link.observatory.worker_agent import (
from k1link.observatory.worker_http_transport import ObservatoryWorkerHttpError
from k1link.observatory.worker_service import (
InstalledObservatoryWorkerService,
ObservatoryWorkerExecutorBuildContext,
ObservatoryWorkerExecutorBuilderRegistration,
ObservatoryWorkerServiceConfiguration,
ObservatoryWorkerServiceError,
compose_installed_observatory_worker_service_from_builders,
load_observatory_worker_bearer_token,
require_ready_executor_coverage,
)
@@ -55,6 +60,58 @@ class _ReadyDefinitions:
def ready_recorded_definitions(self) -> tuple[RecordedRunDefinition, ...]:
return self.definitions
def resolve(self, setup_id: str, definition_sha256: str) -> RecordedRunDefinition:
for definition in self.definitions:
if (
definition.setup_id == setup_id
and definition.definition_sha256 == definition_sha256
):
return definition
raise AssertionError((setup_id, definition_sha256))
@dataclass(frozen=True)
class _RuntimeCandidate:
setup_id: str
definition_sha256: str
identity: ObservatoryWorkerExecutorIdentity
ready: bool = True
def executor_identity(self) -> ObservatoryWorkerExecutorIdentity:
return self.identity
@dataclass(frozen=True)
class _RuntimeRegistry:
candidates: tuple[_RuntimeCandidate, ...]
def resolve(self, setup_id: str, definition_sha256: str) -> _RuntimeCandidate:
for candidate in self.candidates:
if candidate.setup_id == setup_id and candidate.definition_sha256 == definition_sha256:
return candidate
raise AssertionError((setup_id, definition_sha256))
@dataclass
class _Builder:
identity: ObservatoryWorkerExecutorIdentity
contexts: list[ObservatoryWorkerExecutorBuildContext]
def __call__(
self,
context: ObservatoryWorkerExecutorBuildContext,
) -> ObservatoryWorkerExecutorRegistration:
self.contexts.append(context)
return ObservatoryWorkerExecutorRegistration(self.identity, _Executor())
class _TrackingMockTransport(httpx.MockTransport):
closed = False
def close(self) -> None:
self.closed = True
super().close()
class _Executor:
def execute(
@@ -73,6 +130,51 @@ def _identity(definition: RecordedRunDefinition) -> ObservatoryWorkerExecutorIde
)
def _two_ready_profiles() -> tuple[
PortableRunDefinitionRegistry,
PortableWorkerRuntimeRegistry,
RecordedRunDefinition,
RecordedRunDefinition,
]:
first = _definition()
second = replace(
_definition(),
setup_id="portable-lab-v2",
definition_id="portable-lab-v2-definition",
definition_sha256="7" * 64,
executor_release_id="portable-lab-v2-worker",
executor_release_sha256="8" * 64,
executor_image_sha256="9" * 64,
model_manifest_sha256="a" * 64,
resource_profile_sha256="b" * 64,
)
definitions = cast(
PortableRunDefinitionRegistry,
_ReadyDefinitions((first, second)),
)
runtime_registry = cast(
PortableWorkerRuntimeRegistry,
_RuntimeRegistry(
tuple(
_RuntimeCandidate(
definition.setup_id,
definition.definition_sha256,
_identity(definition),
)
for definition in (first, second)
)
),
)
return definitions, runtime_registry, first, second
def _private_token(tmp_path: Path) -> Path:
token = tmp_path / "worker.token"
token.write_text("worker-006-test-bearer-token-000001", encoding="ascii")
token.chmod(0o600)
return token
def _configuration(tmp_path: Path, **overrides: object) -> ObservatoryWorkerServiceConfiguration:
values: dict[str, object] = {
"base_url": "http://127.0.0.1:18080",
@@ -111,9 +213,7 @@ def test_worker_token_loader_requires_private_exact_ascii_file(tmp_path: Path) -
token.write_text("worker-006-test-bearer-token-000001", encoding="ascii")
token.chmod(0o600)
assert load_observatory_worker_bearer_token(token) == (
"worker-006-test-bearer-token-000001"
)
assert load_observatory_worker_bearer_token(token) == ("worker-006-test-bearer-token-000001")
token.chmod(0o644)
with pytest.raises(ObservatoryWorkerServiceError, match="permissions"):
@@ -156,6 +256,154 @@ def test_install_time_coverage_requires_each_ready_executor_identity() -> None:
)
def test_builder_composition_registers_all_ready_profiles_before_one_agent_claims(
tmp_path: Path,
) -> None:
definitions, runtime_registry, first, second = _two_ready_profiles()
_private_token(tmp_path)
contexts: list[ObservatoryWorkerExecutorBuildContext] = []
first_builder = _Builder(_identity(first), contexts)
second_builder = _Builder(_identity(second), contexts)
requests: list[str] = []
def handle(request: httpx.Request) -> httpx.Response:
requests.append(request.url.path)
return httpx.Response(204)
service = compose_installed_observatory_worker_service_from_builders(
configuration=_configuration(tmp_path),
definitions=definitions,
runtime_registry=runtime_registry,
builders=(
ObservatoryWorkerExecutorBuilderRegistration(
first.setup_id,
first_builder,
),
ObservatoryWorkerExecutorBuilderRegistration(
second.setup_id,
second_builder,
),
),
http_transport=httpx.MockTransport(handle),
)
assert len(service.agent._executors.registrations) == 2 # noqa: SLF001
assert len(contexts) == 2
assert all(context.source_transport is service.gateway for context in contexts)
assert all(context.result_transport is service.gateway for context in contexts)
assert service.agent.run_once().state == "empty"
assert requests == ["/api/v1/worker/observatory/recorded-jobs/claims"]
service.close()
def test_builder_composition_fails_closed_before_claim_and_closes_gateway(
tmp_path: Path,
) -> None:
definitions, runtime_registry, first, _second = _two_ready_profiles()
_private_token(tmp_path)
contexts: list[ObservatoryWorkerExecutorBuildContext] = []
requests: list[str] = []
def handle(request: httpx.Request) -> httpx.Response:
requests.append(request.url.path)
return httpx.Response(204)
transport = _TrackingMockTransport(handle)
with pytest.raises(ObservatoryWorkerServiceError, match="no installed local builder"):
compose_installed_observatory_worker_service_from_builders(
configuration=_configuration(tmp_path),
definitions=definitions,
runtime_registry=runtime_registry,
builders=(
ObservatoryWorkerExecutorBuilderRegistration(
first.setup_id,
_Builder(_identity(first), contexts),
),
),
http_transport=transport,
)
assert requests == []
assert contexts == []
assert transport.closed is True
def test_builder_composition_rejects_a_builder_for_a_non_ready_profile(
tmp_path: Path,
) -> None:
definition = _definition()
definitions = cast(
PortableRunDefinitionRegistry,
_ReadyDefinitions((definition,)),
)
runtime_registry = cast(
PortableWorkerRuntimeRegistry,
_RuntimeRegistry(
(
_RuntimeCandidate(
definition.setup_id,
definition.definition_sha256,
_identity(definition),
),
)
),
)
_private_token(tmp_path)
contexts: list[ObservatoryWorkerExecutorBuildContext] = []
with pytest.raises(ObservatoryWorkerServiceError, match="non-ready RunDefinition"):
compose_installed_observatory_worker_service_from_builders(
configuration=_configuration(tmp_path),
definitions=definitions,
runtime_registry=runtime_registry,
builders=(
ObservatoryWorkerExecutorBuilderRegistration(
definition.setup_id,
_Builder(_identity(definition), contexts),
),
ObservatoryWorkerExecutorBuilderRegistration(
"lab-v1-eomt-ddrnet-portable-v1",
_Builder(_identity(definition), contexts),
),
),
)
assert contexts == []
def test_builder_composition_rejects_identity_drift_before_claim(tmp_path: Path) -> None:
definition = _definition()
definitions = cast(
PortableRunDefinitionRegistry,
_ReadyDefinitions((definition,)),
)
runtime_registry = _RuntimeRegistry(
(
_RuntimeCandidate(
definition.setup_id,
definition.definition_sha256,
_identity(definition),
),
)
)
_private_token(tmp_path)
mismatched = replace(_identity(definition), release_sha256="f" * 64)
with pytest.raises(ObservatoryWorkerServiceError, match="another exact identity"):
compose_installed_observatory_worker_service_from_builders(
configuration=_configuration(tmp_path),
definitions=definitions,
runtime_registry=cast(PortableWorkerRuntimeRegistry, runtime_registry),
builders=(
ObservatoryWorkerExecutorBuilderRegistration(
definition.setup_id,
_Builder(mismatched, []),
),
),
)
@dataclass
class _FakeGateway:
closed: bool = False