feat(observatory): add installed package dispatch and durable publication
Checkpoint existing backend lifecycle changes. Focused verification found nine legacy fixture failures in portable LAB V1 executor/runtime tests; repair follows separately without rewriting this snapshot. ADR date retains its intentional Markdown hard break.
This commit is contained in:
DCCONSTRUCTIONS
@@ -789,12 +789,23 @@ def test_result_v2_assembler_and_exact_validator_round_trip(
|
||||
)
|
||||
plan = PortableWorkerRuntimePlan(
|
||||
job_id=sealed.job_id,
|
||||
request_sha256=sealed.request_sha256,
|
||||
identity_sha256=sealed.identity_sha256,
|
||||
submission_receipt_sha256=sealed.submission_receipt_sha256,
|
||||
claim_generation=sealed.claim_generation,
|
||||
adapter_id="m49-tgs-worker006-portable-v2",
|
||||
candidate_sha256="f" * 64,
|
||||
setup_id=sealed.setup_id,
|
||||
definition_id=sealed.definition_id,
|
||||
definition_version=sealed.definition_version,
|
||||
definition_sha256=sealed.definition_sha256,
|
||||
source_session_id=sealed.source_session_id,
|
||||
source_catalog_sha256=sealed.source_catalog_sha256,
|
||||
source_bundle_sha256=sealed.source_bundle_sha256,
|
||||
source_capability_manifest_sha256=sealed.source_capability_manifest_sha256,
|
||||
source_adapter_id=sealed.source_adapter_id,
|
||||
source_adapter_version=sealed.source_adapter_version,
|
||||
source_adapter_sha256=sealed.source_adapter_sha256,
|
||||
result_contract_sha256=definition.result_contract.contract_sha256,
|
||||
phases=M49_PORTABLE_RUNTIME_PHASES,
|
||||
)
|
||||
|
||||
@@ -0,0 +1,362 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from dataclasses import replace
|
||||
from pathlib import Path
|
||||
from typing import cast
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from k1link.observatory.installed_lab_package_runner import (
|
||||
DockerEngineInstalledLabLauncher,
|
||||
InstalledLabDockerLaunch,
|
||||
InstalledLabDockerMount,
|
||||
InstalledLabLocalAssetBinding,
|
||||
InstalledLabPackageProfileRunner,
|
||||
InstalledLabPackageRunnerError,
|
||||
)
|
||||
from k1link.observatory.installed_lab_packages import (
|
||||
INSTALLED_LAB_PACKAGE_REGISTRY_SCHEMA,
|
||||
InstalledLabContainer,
|
||||
InstalledLabPackageMount,
|
||||
seal_installed_lab_package,
|
||||
)
|
||||
from k1link.observatory.portable_result_contract import (
|
||||
PORTABLE_RESULT_PACKAGE_IDENTITY_SCHEMA,
|
||||
PortableResultArtifact,
|
||||
PortableResultPackageManifest,
|
||||
canonical_json,
|
||||
result_identity_document,
|
||||
run_definition_document,
|
||||
)
|
||||
from k1link.observatory.portable_run_definitions import (
|
||||
PortableRunDefinitionRegistry,
|
||||
canonical_sha256,
|
||||
)
|
||||
from k1link.observatory.portable_worker_runtime import (
|
||||
PortableWorkerRuntimePlan,
|
||||
PortableWorkerRuntimeRegistry,
|
||||
PortableWorkerSourceStage,
|
||||
)
|
||||
|
||||
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
|
||||
DEFINITIONS_FILE = REPOSITORY_ROOT / "config" / "observatory-portable-run-definitions.json"
|
||||
RUNTIME_FILE = REPOSITORY_ROOT / "config" / "observatory-worker-runtime-candidates.json"
|
||||
|
||||
|
||||
def _package_and_runtime():
|
||||
definitions = PortableRunDefinitionRegistry.from_file(DEFINITIONS_FILE)
|
||||
runtime = PortableWorkerRuntimeRegistry.from_file(
|
||||
RUNTIME_FILE,
|
||||
definitions=definitions,
|
||||
)
|
||||
definition = definitions.resolve_setup("m49-tgs-portable-v2")
|
||||
candidate = runtime.resolve(definition.setup_id, definition.definition_sha256)
|
||||
assert candidate.executor is not None
|
||||
mounted_asset = next(
|
||||
asset for asset in candidate.reusable_assets if asset.kind != "container-image"
|
||||
)
|
||||
writer = InstalledLabContainer(
|
||||
container_id="portable-result-writer",
|
||||
role="result-writer",
|
||||
image_sha256=candidate.executor.image_sha256,
|
||||
argv=("/missioncore/package/run", INSTALLED_LAB_PACKAGE_REGISTRY_SCHEMA),
|
||||
depends_on=(),
|
||||
mounts=(
|
||||
InstalledLabPackageMount(
|
||||
asset_id=mounted_asset.asset_id,
|
||||
target=f"/missioncore/package/assets/{mounted_asset.asset_id}",
|
||||
),
|
||||
),
|
||||
network="none",
|
||||
gpu_count=0,
|
||||
memory_bytes=8 * 1024**3,
|
||||
nano_cpus=2_000_000_000,
|
||||
pids_limit=512,
|
||||
shm_bytes=64 * 1024**2,
|
||||
tmpfs_bytes=512 * 1024**2,
|
||||
timeout_seconds=3600,
|
||||
)
|
||||
package = seal_installed_lab_package(
|
||||
package_id="m49-generic-runner-test",
|
||||
package_version=1,
|
||||
setup_id=definition.setup_id,
|
||||
definition_id=definition.definition_id,
|
||||
definition_version=definition.version,
|
||||
definition_sha256=definition.definition_sha256,
|
||||
runtime_candidate_sha256=candidate.candidate_sha256,
|
||||
source_adapter_sha256=definition.source_adapter.contract_sha256,
|
||||
result_contract_sha256=definition.result_contract.contract_sha256,
|
||||
executor_identity=candidate.executor_identity(),
|
||||
execution_mode="single-container",
|
||||
asset_ids=tuple(asset.asset_id for asset in candidate.reusable_assets),
|
||||
containers=(writer,),
|
||||
)
|
||||
return definition, candidate, package, mounted_asset.asset_id
|
||||
|
||||
|
||||
def _fixed_stack(package):
|
||||
writer_template = package.containers[0]
|
||||
step = replace(
|
||||
writer_template,
|
||||
container_id="compute-step",
|
||||
role="step",
|
||||
)
|
||||
writer = replace(writer_template, depends_on=(step.container_id,))
|
||||
return seal_installed_lab_package(
|
||||
package_id=package.package_id,
|
||||
package_version=package.package_version,
|
||||
setup_id=package.setup_id,
|
||||
definition_id=package.definition_id,
|
||||
definition_version=package.definition_version,
|
||||
definition_sha256=package.definition_sha256,
|
||||
runtime_candidate_sha256=package.runtime_candidate_sha256,
|
||||
source_adapter_sha256=package.source_adapter_sha256,
|
||||
result_contract_sha256=package.result_contract_sha256,
|
||||
executor_identity=package.executor_identity,
|
||||
execution_mode="fixed-stack",
|
||||
asset_ids=package.asset_ids,
|
||||
containers=(step, writer),
|
||||
)
|
||||
|
||||
|
||||
def _plan(definition, candidate) -> PortableWorkerRuntimePlan:
|
||||
return PortableWorkerRuntimePlan(
|
||||
job_id=f"observatory-run-{'1' * 32}",
|
||||
request_sha256="4" * 64,
|
||||
identity_sha256="5" * 64,
|
||||
submission_receipt_sha256="6" * 64,
|
||||
claim_generation=1,
|
||||
adapter_id=candidate.adapter_id,
|
||||
candidate_sha256=candidate.candidate_sha256,
|
||||
setup_id=definition.setup_id,
|
||||
definition_id=definition.definition_id,
|
||||
definition_version=definition.version,
|
||||
definition_sha256=definition.definition_sha256,
|
||||
source_session_id="source-a",
|
||||
source_catalog_sha256="7" * 64,
|
||||
source_bundle_sha256="2" * 64,
|
||||
source_capability_manifest_sha256="3" * 64,
|
||||
source_adapter_id=definition.source_adapter.adapter_id,
|
||||
source_adapter_version=definition.source_adapter.version,
|
||||
source_adapter_sha256=definition.source_adapter.contract_sha256,
|
||||
result_contract_sha256=definition.result_contract.contract_sha256,
|
||||
phases=tuple(phase.phase_id for phase in candidate.phases),
|
||||
)
|
||||
|
||||
|
||||
def _write_result(root: Path, *, plan: PortableWorkerRuntimePlan, definition) -> None:
|
||||
result_id = "portable-result-generic-runner"
|
||||
artifact_payload = canonical_json(
|
||||
{"schema_version": definition.result_contract.result_schema, "verified": True}
|
||||
)
|
||||
artifact_path = root / "artifacts" / "result.json"
|
||||
artifact_path.parent.mkdir(parents=True)
|
||||
artifact_path.write_bytes(artifact_payload)
|
||||
artifact = PortableResultArtifact(
|
||||
role="result-document",
|
||||
relative_path="artifacts/result.json",
|
||||
media_type="application/json",
|
||||
byte_length=len(artifact_payload),
|
||||
sha256=hashlib.sha256(artifact_payload).hexdigest(),
|
||||
)
|
||||
job = {
|
||||
"job_id": plan.job_id,
|
||||
"request_sha256": plan.request_sha256,
|
||||
"identity_sha256": plan.identity_sha256,
|
||||
"submission_receipt_sha256": plan.submission_receipt_sha256,
|
||||
"claim_generation": plan.claim_generation,
|
||||
}
|
||||
source = {
|
||||
"session_id": plan.source_session_id,
|
||||
"catalog_sha256": plan.source_catalog_sha256,
|
||||
"bundle_sha256": plan.source_bundle_sha256,
|
||||
"capability_manifest_sha256": plan.source_capability_manifest_sha256,
|
||||
"adapter": {
|
||||
"adapter_id": definition.source_adapter.adapter_id,
|
||||
"version": definition.source_adapter.version,
|
||||
"adapter_sha256": definition.source_adapter.contract_sha256,
|
||||
},
|
||||
}
|
||||
run_definition = run_definition_document(definition)
|
||||
result = result_identity_document(definition, result_id)
|
||||
authority = definition.authority.as_dict()
|
||||
created_at_utc = "2026-09-01T12:00:00Z"
|
||||
identity_document = {
|
||||
"schema_version": PORTABLE_RESULT_PACKAGE_IDENTITY_SCHEMA,
|
||||
"created_at_utc": created_at_utc,
|
||||
"job": job,
|
||||
"source": source,
|
||||
"run_definition": run_definition,
|
||||
"result": result,
|
||||
"authority": authority,
|
||||
"artifacts": [artifact.as_dict()],
|
||||
}
|
||||
manifest = PortableResultPackageManifest(
|
||||
identity_sha256=canonical_sha256(identity_document),
|
||||
created_at_utc=created_at_utc,
|
||||
job=job,
|
||||
source=source,
|
||||
run_definition=run_definition,
|
||||
result=result,
|
||||
authority=authority,
|
||||
artifacts=(artifact,),
|
||||
)
|
||||
(root / "manifest.json").write_bytes(manifest.canonical_bytes)
|
||||
|
||||
|
||||
def test_generic_fixed_stack_runs_topologically_and_returns_verified_package(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
definition, candidate, single, mounted_asset_id = _package_and_runtime()
|
||||
package = _fixed_stack(single)
|
||||
source_root = tmp_path / "materialized-source"
|
||||
source_root.mkdir()
|
||||
asset_path = tmp_path / "installed-asset"
|
||||
asset_path.write_bytes(b"asset")
|
||||
plan = _plan(definition, candidate)
|
||||
launches: list[InstalledLabDockerLaunch] = []
|
||||
|
||||
def launcher(launch: InstalledLabDockerLaunch) -> None:
|
||||
launches.append(launch)
|
||||
if launch.container.role == "result-writer":
|
||||
output = next(
|
||||
mount for mount in launch.mounts if mount.container_path == "/missioncore/output"
|
||||
)
|
||||
_write_result(Path(output.engine_path), plan=plan, definition=definition)
|
||||
|
||||
runner = InstalledLabPackageProfileRunner(
|
||||
package=package,
|
||||
definition=definition,
|
||||
controller_work_root=tmp_path,
|
||||
engine_work_root=str(tmp_path),
|
||||
local_assets=(
|
||||
InstalledLabLocalAssetBinding(
|
||||
asset_id=mounted_asset_id,
|
||||
controller_path=asset_path,
|
||||
engine_path=str(asset_path),
|
||||
),
|
||||
),
|
||||
launcher=launcher,
|
||||
token_factory=lambda: "abcdef0123456789",
|
||||
)
|
||||
draft = runner.run(
|
||||
plan,
|
||||
PortableWorkerSourceStage(
|
||||
root=source_root,
|
||||
source_bundle_sha256=plan.source_bundle_sha256,
|
||||
source_capability_manifest_sha256=plan.source_capability_manifest_sha256,
|
||||
source_adapter_sha256=definition.source_adapter.contract_sha256,
|
||||
),
|
||||
)
|
||||
|
||||
assert [launch.container.container_id for launch in launches] == [
|
||||
"compute-step",
|
||||
"portable-result-writer",
|
||||
]
|
||||
writer_launch = launches[-1]
|
||||
assert any(
|
||||
mount.container_path == "/missioncore/input/steps/compute-step"
|
||||
and mount.read_only
|
||||
for mount in writer_launch.mounts
|
||||
)
|
||||
assert draft.result_id == "portable-result-generic-runner"
|
||||
assert draft.result_contract_sha256 == definition.result_contract.contract_sha256
|
||||
assert (draft.root / "manifest.json").is_file()
|
||||
for launch in launches:
|
||||
writable = [mount for mount in launch.mounts if not mount.read_only]
|
||||
assert [mount.container_path for mount in writable] == ["/missioncore/output"]
|
||||
|
||||
|
||||
def test_generic_docker_launcher_uses_hardened_one_shot_contract() -> None:
|
||||
_definition, _candidate, package, _mounted_asset_id = _package_and_runtime()
|
||||
requests: list[httpx.Request] = []
|
||||
create_document: dict[str, object] = {}
|
||||
container_id = "f" * 64
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
requests.append(request)
|
||||
if request.url.path.startswith("/v1.47/images/"):
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={"Id": f"sha256:{package.containers[0].image_sha256}"},
|
||||
)
|
||||
if request.url.path == "/v1.47/containers/create":
|
||||
create_document.update(json.loads(request.content))
|
||||
return httpx.Response(201, json={"Id": container_id, "Warnings": None})
|
||||
if request.url.path.endswith("/start"):
|
||||
return httpx.Response(204)
|
||||
if request.url.path.endswith("/wait"):
|
||||
return httpx.Response(200, json={"StatusCode": 0, "Error": None})
|
||||
if request.method == "DELETE":
|
||||
return httpx.Response(204)
|
||||
raise AssertionError(f"unexpected request: {request.method} {request.url}")
|
||||
|
||||
launch = InstalledLabDockerLaunch(
|
||||
package_id=package.package_id,
|
||||
container=package.containers[0],
|
||||
mounts=(
|
||||
InstalledLabDockerMount("/engine/plan.json", "/missioncore/input/run-plan.json", True),
|
||||
InstalledLabDockerMount("/engine/source", "/missioncore/input/source", True),
|
||||
InstalledLabDockerMount("/engine/output", "/missioncore/output", False),
|
||||
InstalledLabDockerMount("/engine/asset", "/missioncore/package/assets/runner", True),
|
||||
),
|
||||
labels={
|
||||
"com.nodedc.authority": "observation-only",
|
||||
"com.nodedc.component": package.containers[0].container_id,
|
||||
"com.nodedc.definition-sha256": package.definition_sha256,
|
||||
"com.nodedc.job-id": f"observatory-run-{'1' * 32}",
|
||||
"com.nodedc.managed-by": "mission-core-worker",
|
||||
"com.nodedc.package-sha256": package.package_sha256,
|
||||
"com.nodedc.product": "mission-core",
|
||||
"com.nodedc.stack": "observatory",
|
||||
},
|
||||
name_token="0123456789abcdef",
|
||||
)
|
||||
DockerEngineInstalledLabLauncher(transport_factory=lambda: httpx.MockTransport(handler))(launch)
|
||||
|
||||
assert [request.method for request in requests] == [
|
||||
"GET",
|
||||
"POST",
|
||||
"POST",
|
||||
"POST",
|
||||
"DELETE",
|
||||
]
|
||||
assert create_document["Image"] == f"sha256:{package.containers[0].image_sha256}"
|
||||
assert create_document["NetworkDisabled"] is True
|
||||
host = cast(dict[str, object], create_document["HostConfig"])
|
||||
assert host["NetworkMode"] == "none"
|
||||
assert host["ReadonlyRootfs"] is True
|
||||
assert host["CapDrop"] == ["ALL"]
|
||||
assert host["SecurityOpt"] == ["no-new-privileges:true"]
|
||||
assert host["Privileged"] is False
|
||||
assert host["DeviceRequests"] == []
|
||||
mounts = cast(list[dict[str, object]], host["Mounts"])
|
||||
assert sum(not cast(bool, mount["ReadOnly"]) for mount in mounts) == 1
|
||||
assert next(mount for mount in mounts if not mount["ReadOnly"])["Target"] == (
|
||||
"/missioncore/output"
|
||||
)
|
||||
|
||||
|
||||
def test_generic_docker_launcher_verifies_image_inventory_without_container_calls() -> None:
|
||||
image_sha256s = ("1" * 64, "2" * 64)
|
||||
requests: list[httpx.Request] = []
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
requests.append(request)
|
||||
digest = request.url.path.split("sha256:", 1)[1].split("/", 1)[0]
|
||||
return httpx.Response(200, json={"Id": f"sha256:{digest}"})
|
||||
|
||||
DockerEngineInstalledLabLauncher(
|
||||
transport_factory=lambda: httpx.MockTransport(handler)
|
||||
).verify_images(image_sha256s)
|
||||
|
||||
assert [request.method for request in requests] == ["GET", "GET"]
|
||||
assert all("/images/sha256:" in request.url.path for request in requests)
|
||||
|
||||
with pytest.raises(InstalledLabPackageRunnerError, match="inventory"):
|
||||
DockerEngineInstalledLabLauncher(
|
||||
transport_factory=lambda: httpx.MockTransport(handler)
|
||||
).verify_images(tuple(reversed(image_sha256s)))
|
||||
@@ -0,0 +1,216 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from dataclasses import replace
|
||||
from pathlib import Path
|
||||
from typing import cast
|
||||
|
||||
import pytest
|
||||
|
||||
from k1link.observatory.installed_lab_packages import (
|
||||
INSTALLED_LAB_PACKAGE_REGISTRY_SCHEMA,
|
||||
InstalledLabContainer,
|
||||
InstalledLabPackageError,
|
||||
InstalledLabPackageMount,
|
||||
InstalledLabPackageRegistry,
|
||||
seal_installed_lab_package,
|
||||
)
|
||||
from k1link.observatory.portable_run_definitions import PortableRunDefinitionRegistry
|
||||
from k1link.observatory.portable_worker_runtime import (
|
||||
PortableWorkerResultPublisher,
|
||||
PortableWorkerRuntimeRegistry,
|
||||
PortableWorkerSourceMaterializer,
|
||||
)
|
||||
from k1link.observatory.worker_agent import (
|
||||
ObservatoryWorkerExecutionResult,
|
||||
ObservatoryWorkerExecutorRegistration,
|
||||
SealedObservatoryRecordedJob,
|
||||
)
|
||||
from k1link.observatory.worker_service import (
|
||||
ObservatoryWorkerPackageExecutorBuildContext,
|
||||
build_ready_executor_registry_from_packages,
|
||||
)
|
||||
|
||||
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
|
||||
DEFINITIONS_FILE = REPOSITORY_ROOT / "config" / "observatory-portable-run-definitions.json"
|
||||
RUNTIME_FILE = REPOSITORY_ROOT / "config" / "observatory-worker-runtime-candidates.json"
|
||||
|
||||
|
||||
def _registries() -> tuple[PortableRunDefinitionRegistry, PortableWorkerRuntimeRegistry]:
|
||||
definitions = PortableRunDefinitionRegistry.from_file(DEFINITIONS_FILE)
|
||||
runtime = PortableWorkerRuntimeRegistry.from_file(
|
||||
RUNTIME_FILE,
|
||||
definitions=definitions,
|
||||
)
|
||||
return definitions, runtime
|
||||
|
||||
|
||||
def _m49_package():
|
||||
definitions, runtime = _registries()
|
||||
definition = definitions.resolve_setup("m49-tgs-portable-v2")
|
||||
candidate = runtime.resolve(definition.setup_id, definition.definition_sha256)
|
||||
assert candidate.executor is not None
|
||||
non_image_asset = next(
|
||||
asset for asset in candidate.reusable_assets if asset.kind != "container-image"
|
||||
)
|
||||
container = InstalledLabContainer(
|
||||
container_id="portable-result-writer",
|
||||
role="result-writer",
|
||||
image_sha256=candidate.executor.image_sha256,
|
||||
argv=("/missioncore/package/run", INSTALLED_LAB_PACKAGE_REGISTRY_SCHEMA),
|
||||
depends_on=(),
|
||||
mounts=(
|
||||
InstalledLabPackageMount(
|
||||
asset_id=non_image_asset.asset_id,
|
||||
target=f"/missioncore/package/assets/{non_image_asset.asset_id}",
|
||||
),
|
||||
),
|
||||
network="none",
|
||||
gpu_count=0,
|
||||
memory_bytes=8 * 1024**3,
|
||||
nano_cpus=2_000_000_000,
|
||||
pids_limit=512,
|
||||
shm_bytes=64 * 1024**2,
|
||||
tmpfs_bytes=512 * 1024**2,
|
||||
timeout_seconds=3600,
|
||||
)
|
||||
return seal_installed_lab_package(
|
||||
package_id="m49-container-contract-test",
|
||||
package_version=1,
|
||||
setup_id=definition.setup_id,
|
||||
definition_id=definition.definition_id,
|
||||
definition_version=definition.version,
|
||||
definition_sha256=definition.definition_sha256,
|
||||
runtime_candidate_sha256=candidate.candidate_sha256,
|
||||
source_adapter_sha256=definition.source_adapter.contract_sha256,
|
||||
result_contract_sha256=definition.result_contract.contract_sha256,
|
||||
executor_identity=candidate.executor_identity(),
|
||||
execution_mode="single-container",
|
||||
asset_ids=tuple(asset.asset_id for asset in candidate.reusable_assets),
|
||||
containers=(container,),
|
||||
)
|
||||
|
||||
|
||||
def test_installed_package_binds_exact_ready_definition_and_runtime() -> None:
|
||||
definitions, runtime = _registries()
|
||||
package = _m49_package()
|
||||
definition = definitions.resolve(package.setup_id, package.definition_sha256)
|
||||
candidate = runtime.resolve(package.setup_id, package.definition_sha256)
|
||||
|
||||
package.bind(definition, candidate)
|
||||
|
||||
document = package.identity_document()
|
||||
serialized = json.dumps(document, sort_keys=True)
|
||||
assert document["container_io"] == {
|
||||
"schema_version": "missioncore.observatory-installed-lab-container-io/v2",
|
||||
"source_root": "/missioncore/input/source",
|
||||
"plan_path": "/missioncore/input/run-plan.json",
|
||||
"step_input_root": "/missioncore/input/steps",
|
||||
"result_root": "/missioncore/output",
|
||||
"work_root": "/missioncore/work",
|
||||
}
|
||||
assert "host_path" not in serialized
|
||||
assert "source_session_id" not in serialized
|
||||
|
||||
|
||||
def test_installed_package_registry_round_trips_and_rejects_host_inputs(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
definitions, runtime = _registries()
|
||||
package = _m49_package()
|
||||
document = {
|
||||
"schema_version": INSTALLED_LAB_PACKAGE_REGISTRY_SCHEMA,
|
||||
"packages": [{**package.identity_document(), "package_sha256": package.package_sha256}],
|
||||
}
|
||||
path = tmp_path / "installed-packages.json"
|
||||
path.write_text(json.dumps(document), encoding="utf-8")
|
||||
|
||||
loaded = InstalledLabPackageRegistry.from_file(
|
||||
path,
|
||||
definitions=definitions,
|
||||
runtime_registry=runtime,
|
||||
)
|
||||
assert loaded.resolve(package.setup_id, package.definition_sha256) == package
|
||||
|
||||
document["packages"][0]["containers"][0]["host_path"] = "/unsafe"
|
||||
path.write_text(json.dumps(document), encoding="utf-8")
|
||||
with pytest.raises(InstalledLabPackageError, match="forbidden"):
|
||||
InstalledLabPackageRegistry.from_file(
|
||||
path,
|
||||
definitions=definitions,
|
||||
runtime_registry=runtime,
|
||||
)
|
||||
|
||||
|
||||
def test_package_topology_and_digest_fail_closed() -> None:
|
||||
package = _m49_package()
|
||||
with pytest.raises(InstalledLabPackageError, match="digest"):
|
||||
replace(package, package_sha256="f" * 64)
|
||||
|
||||
first = replace(
|
||||
package.containers[0],
|
||||
container_id="first-service",
|
||||
role="step",
|
||||
depends_on=("portable-result-writer",),
|
||||
)
|
||||
writer = replace(
|
||||
package.containers[0],
|
||||
depends_on=("first-service",),
|
||||
)
|
||||
with pytest.raises(InstalledLabPackageError, match="cycle"):
|
||||
seal_installed_lab_package(
|
||||
package_id=package.package_id,
|
||||
package_version=package.package_version,
|
||||
setup_id=package.setup_id,
|
||||
definition_id=package.definition_id,
|
||||
definition_version=package.definition_version,
|
||||
definition_sha256=package.definition_sha256,
|
||||
runtime_candidate_sha256=package.runtime_candidate_sha256,
|
||||
source_adapter_sha256=package.source_adapter_sha256,
|
||||
result_contract_sha256=package.result_contract_sha256,
|
||||
executor_identity=package.executor_identity,
|
||||
execution_mode="fixed-stack",
|
||||
asset_ids=package.asset_ids,
|
||||
containers=(first, writer),
|
||||
)
|
||||
|
||||
with pytest.raises(InstalledLabPackageError, match="unsafe"):
|
||||
InstalledLabPackageMount(
|
||||
asset_id=package.asset_ids[0],
|
||||
target="../../host",
|
||||
)
|
||||
|
||||
|
||||
def test_one_generic_factory_builds_installed_ready_subset(tmp_path: Path) -> None:
|
||||
definitions, runtime = _registries()
|
||||
package = _m49_package()
|
||||
contexts: list[ObservatoryWorkerPackageExecutorBuildContext] = []
|
||||
|
||||
class Executor:
|
||||
def execute(
|
||||
self,
|
||||
job: SealedObservatoryRecordedJob,
|
||||
) -> ObservatoryWorkerExecutionResult:
|
||||
raise AssertionError(job)
|
||||
|
||||
def factory(
|
||||
context: ObservatoryWorkerPackageExecutorBuildContext,
|
||||
) -> ObservatoryWorkerExecutorRegistration:
|
||||
contexts.append(context)
|
||||
return ObservatoryWorkerExecutorRegistration(
|
||||
context.package.executor_identity,
|
||||
Executor(),
|
||||
)
|
||||
|
||||
executors = build_ready_executor_registry_from_packages(
|
||||
definitions=definitions,
|
||||
runtime_registry=runtime,
|
||||
packages=InstalledLabPackageRegistry((package,)),
|
||||
executor_factory=factory,
|
||||
source_transport=cast(PortableWorkerSourceMaterializer, object()),
|
||||
result_transport=cast(PortableWorkerResultPublisher, object()),
|
||||
work_root=tmp_path,
|
||||
)
|
||||
|
||||
assert executors.supported_identities == (package.executor_identity,)
|
||||
assert [context.package.package_id for context in contexts] == [package.package_id]
|
||||
@@ -0,0 +1,63 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import socket
|
||||
import socketserver
|
||||
import threading
|
||||
|
||||
import pytest
|
||||
|
||||
import k1link.observatory.installed_lab_worker_container_main as container_main
|
||||
from k1link.observatory.worker_container_proxy import (
|
||||
FixedObservatoryContainerLoopbackProxy,
|
||||
)
|
||||
|
||||
|
||||
class _EchoHandler(socketserver.BaseRequestHandler):
|
||||
def handle(self) -> None:
|
||||
payload = self.request.recv(1024)
|
||||
self.request.sendall(payload)
|
||||
|
||||
|
||||
def test_generic_proxy_bridges_loopback_to_fixed_upstream() -> None:
|
||||
upstream = socketserver.ThreadingTCPServer(("127.0.0.1", 0), _EchoHandler)
|
||||
upstream_thread = threading.Thread(target=upstream.serve_forever, daemon=True)
|
||||
upstream_thread.start()
|
||||
try:
|
||||
upstream_port = upstream.server_address[1]
|
||||
assert isinstance(upstream_port, int)
|
||||
with FixedObservatoryContainerLoopbackProxy(
|
||||
listen_port=0,
|
||||
upstream_host="127.0.0.1",
|
||||
upstream_port=upstream_port,
|
||||
) as proxy, socket.create_connection(("127.0.0.1", proxy.listen_port)) as client:
|
||||
client.sendall(b"generic-observatory-proxy")
|
||||
client.shutdown(socket.SHUT_WR)
|
||||
assert client.recv(1024) == b"generic-observatory-proxy"
|
||||
finally:
|
||||
upstream.shutdown()
|
||||
upstream.server_close()
|
||||
upstream_thread.join(timeout=5.0)
|
||||
|
||||
|
||||
def test_generic_entrypoint_owns_proxy_around_package_worker(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
lifecycle: list[str] = []
|
||||
|
||||
class _Proxy:
|
||||
def __enter__(self) -> _Proxy:
|
||||
lifecycle.append("proxy-started")
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args: object) -> None:
|
||||
lifecycle.append("proxy-stopped")
|
||||
|
||||
def worker(arguments: object) -> int:
|
||||
lifecycle.append(f"worker:{arguments!r}")
|
||||
return 23
|
||||
|
||||
monkeypatch.setattr(container_main, "FixedObservatoryContainerLoopbackProxy", _Proxy)
|
||||
monkeypatch.setattr(container_main.installed_lab_worker_service, "main", worker)
|
||||
|
||||
assert container_main.main(("--once",)) == 23
|
||||
assert lifecycle == ["proxy-started", "worker:('--once',)", "proxy-stopped"]
|
||||
@@ -0,0 +1,131 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from k1link.observatory.installed_lab_worker_service import (
|
||||
INSTALLED_LAB_ASSET_BINDINGS_SCHEMA,
|
||||
InstalledLabWorkerAssetBindings,
|
||||
InstalledLabWorkerCompositionError,
|
||||
InstalledLabWorkerEntrypointConfiguration,
|
||||
InstalledLabWorkerValidationConfiguration,
|
||||
)
|
||||
|
||||
|
||||
def _binding_document() -> dict[str, object]:
|
||||
return {
|
||||
"schema_version": INSTALLED_LAB_ASSET_BINDINGS_SCHEMA,
|
||||
"engine_work_root": "D:\\NDC_MISSIONCORE\\runtime\\worker\\work",
|
||||
"assets": [
|
||||
{
|
||||
"asset_id": "eomt-image",
|
||||
"controller_path": None,
|
||||
"engine_path": None,
|
||||
"image_sha256": "1" * 64,
|
||||
},
|
||||
{
|
||||
"asset_id": "ddrnet-checkpoint",
|
||||
"controller_path": "/runtime/assets/ddrnet-checkpoint.pth",
|
||||
"engine_path": (
|
||||
"D:\\NDC_MISSIONCORE\\runtime\\assets\\ddrnet-checkpoint.pth"
|
||||
),
|
||||
"image_sha256": None,
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def test_asset_binding_file_loads_only_reviewed_paths_or_image_identities(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
path = tmp_path / "bindings.json"
|
||||
path.write_text(json.dumps(_binding_document()), encoding="utf-8")
|
||||
|
||||
bindings = InstalledLabWorkerAssetBindings.from_file(path)
|
||||
|
||||
assert bindings.engine_work_root.endswith("\\work")
|
||||
assert tuple(item.asset_id for item in bindings.assets) == (
|
||||
"ddrnet-checkpoint",
|
||||
"eomt-image",
|
||||
)
|
||||
assert bindings.assets[0].controller_path == Path(
|
||||
"/runtime/assets/ddrnet-checkpoint.pth"
|
||||
)
|
||||
assert bindings.assets[1].image_sha256 == "1" * 64
|
||||
|
||||
|
||||
def test_asset_binding_file_rejects_ambiguous_locator(tmp_path: Path) -> None:
|
||||
document = _binding_document()
|
||||
assets = document["assets"]
|
||||
assert isinstance(assets, list)
|
||||
first = assets[0]
|
||||
assert isinstance(first, dict)
|
||||
first["controller_path"] = "/runtime/image"
|
||||
first["engine_path"] = "D:\\runtime\\image"
|
||||
path = tmp_path / "bindings.json"
|
||||
path.write_text(json.dumps(document), encoding="utf-8")
|
||||
|
||||
with pytest.raises(InstalledLabWorkerCompositionError, match="ambiguous"):
|
||||
InstalledLabWorkerAssetBindings.from_file(path)
|
||||
|
||||
|
||||
def test_entrypoint_environment_contains_paths_but_no_executable_selector(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
environment = {
|
||||
"MISSIONCORE_OBSERVATORY_WORKER_TOKEN_FILE": str(tmp_path / "worker.token"),
|
||||
"MISSIONCORE_OBSERVATORY_WORKER_WORK_ROOT": str(tmp_path / "work"),
|
||||
"MISSIONCORE_OBSERVATORY_WORKER_DEFINITIONS_FILE": str(tmp_path / "definitions.json"),
|
||||
"MISSIONCORE_OBSERVATORY_WORKER_RUNTIME_REGISTRY_FILE": str(
|
||||
tmp_path / "runtime.json"
|
||||
),
|
||||
"MISSIONCORE_OBSERVATORY_WORKER_PACKAGE_REGISTRY_FILE": str(
|
||||
tmp_path / "packages.json"
|
||||
),
|
||||
"MISSIONCORE_OBSERVATORY_WORKER_PACKAGE_ASSET_BINDINGS_FILE": str(
|
||||
tmp_path / "bindings.json"
|
||||
),
|
||||
}
|
||||
|
||||
configuration = InstalledLabWorkerEntrypointConfiguration.from_environment(environment)
|
||||
|
||||
assert configuration.package_registry_file == tmp_path / "packages.json"
|
||||
assert configuration.asset_bindings_file == tmp_path / "bindings.json"
|
||||
assert not any("COMMAND" in key or "MODULE" in key for key in environment)
|
||||
|
||||
|
||||
def test_validation_environment_requires_no_token_or_backend_url(tmp_path: Path) -> None:
|
||||
environment = {
|
||||
"MISSIONCORE_OBSERVATORY_WORKER_WORK_ROOT": str(tmp_path / "work"),
|
||||
"MISSIONCORE_OBSERVATORY_WORKER_DEFINITIONS_FILE": str(tmp_path / "definitions.json"),
|
||||
"MISSIONCORE_OBSERVATORY_WORKER_RUNTIME_REGISTRY_FILE": str(
|
||||
tmp_path / "runtime.json"
|
||||
),
|
||||
"MISSIONCORE_OBSERVATORY_WORKER_PACKAGE_REGISTRY_FILE": str(
|
||||
tmp_path / "packages.json"
|
||||
),
|
||||
"MISSIONCORE_OBSERVATORY_WORKER_PACKAGE_ASSET_BINDINGS_FILE": str(
|
||||
tmp_path / "bindings.json"
|
||||
),
|
||||
}
|
||||
|
||||
configuration = InstalledLabWorkerValidationConfiguration.from_environment(environment)
|
||||
|
||||
assert configuration.work_root == tmp_path / "work"
|
||||
assert not any("TOKEN" in key or "BASE_URL" in key for key in environment)
|
||||
|
||||
|
||||
def test_asset_binding_file_rejects_unresolved_engine_path(tmp_path: Path) -> None:
|
||||
document = _binding_document()
|
||||
assets = document["assets"]
|
||||
assert isinstance(assets, list)
|
||||
file_asset = assets[1]
|
||||
assert isinstance(file_asset, dict)
|
||||
file_asset["engine_path"] = "D:\\runtime\\lab$release\\asset"
|
||||
path = tmp_path / "bindings.json"
|
||||
path.write_text(json.dumps(document), encoding="utf-8")
|
||||
|
||||
with pytest.raises(InstalledLabWorkerCompositionError, match="locator is invalid"):
|
||||
InstalledLabWorkerAssetBindings.from_file(path)
|
||||
@@ -544,6 +544,19 @@ def test_entrypoint_environment_requires_all_absolute_fixed_files(tmp_path: Path
|
||||
assert configuration.lab_v1_release_candidate_file == (
|
||||
service_module._FIXED_LAB_V1_RELEASE_CANDIDATE_FILE # noqa: SLF001
|
||||
)
|
||||
m49_only = service_module.M49WorkerEntrypointConfiguration.from_environment(
|
||||
{
|
||||
key: value
|
||||
for key, value in environment.items()
|
||||
if key
|
||||
not in {
|
||||
service_module.LAB_V1_WORKER_INSTALLATION_RECEIPT_FILE_ENV,
|
||||
service_module.LAB_V1_WORKER_RELEASE_CANDIDATE_FILE_ENV,
|
||||
}
|
||||
}
|
||||
)
|
||||
assert m49_only.lab_v1_installation_receipt_file is None
|
||||
assert m49_only.lab_v1_release_candidate_file is None
|
||||
environment[service_module.LAB_V1_WORKER_RELEASE_CANDIDATE_FILE_ENV] = str(
|
||||
tmp_path / "lab-v1-release.json"
|
||||
)
|
||||
|
||||
@@ -126,7 +126,7 @@ def _source_documents(definition: PortableRunDefinition) -> tuple[bytes, bytes]:
|
||||
"spatial_replay": {},
|
||||
"camera": {
|
||||
"artifact_id": "camera-recording",
|
||||
"public_source_id": "sensor.camera.right",
|
||||
"public_source_id": "recorded.camera.right",
|
||||
"generation_sha256": CAMERA_GENERATION_SHA256,
|
||||
"synchronization": "host-arrival-best-effort",
|
||||
"epoch": {
|
||||
@@ -198,22 +198,22 @@ def _camera_job(tmp_path: Path) -> CameraComputeJob:
|
||||
root.mkdir()
|
||||
files = [
|
||||
{
|
||||
"path": "input/camera/sensor.camera.right/epoch-1/summary.json",
|
||||
"path": "input/camera/recorded.camera.right/epoch-1/summary.json",
|
||||
"byte_length": 2,
|
||||
"sha256": "1" * 64,
|
||||
},
|
||||
{
|
||||
"path": "input/camera/sensor.camera.right/epoch-1/index.jsonl",
|
||||
"path": "input/camera/recorded.camera.right/epoch-1/index.jsonl",
|
||||
"byte_length": 3,
|
||||
"sha256": "2" * 64,
|
||||
},
|
||||
{
|
||||
"path": "input/camera/sensor.camera.right/epoch-1/init.mp4",
|
||||
"path": "input/camera/recorded.camera.right/epoch-1/init.mp4",
|
||||
"byte_length": 4,
|
||||
"sha256": CAMERA_INIT_SHA256,
|
||||
},
|
||||
{
|
||||
"path": "input/camera/sensor.camera.right/epoch-1/segments/1.m4s",
|
||||
"path": "input/camera/recorded.camera.right/epoch-1/segments/1.m4s",
|
||||
"byte_length": 7,
|
||||
"sha256": CAMERA_SEGMENT_SHA256,
|
||||
},
|
||||
@@ -221,7 +221,7 @@ def _camera_job(tmp_path: Path) -> CameraComputeJob:
|
||||
input_document = {
|
||||
"kind": "canonical-camera-epoch",
|
||||
"session_id": SOURCE_SESSION_ID,
|
||||
"source_id": "sensor.camera.right",
|
||||
"source_id": "recorded.camera.right",
|
||||
"codec_epoch": 1,
|
||||
"synchronization": "host-arrival-best-effort",
|
||||
"media_type": "video/mp4; codecs=\"avc1.641028\"",
|
||||
@@ -254,7 +254,7 @@ def _camera_job(tmp_path: Path) -> CameraComputeJob:
|
||||
job_root=root,
|
||||
manifest_path=root / "job.json",
|
||||
session_id=SOURCE_SESSION_ID,
|
||||
source_id="sensor.camera.right",
|
||||
source_id="recorded.camera.right",
|
||||
codec_epoch=1,
|
||||
input_sha256=input_sha256,
|
||||
input_byte_length=16,
|
||||
@@ -477,7 +477,7 @@ def _component_outputs(
|
||||
"job_id": plan.source_input.camera_job_id,
|
||||
"input_sha256": plan.source_input.camera_input_sha256,
|
||||
"session_id": SOURCE_SESSION_ID,
|
||||
"source_id": "sensor.camera.right",
|
||||
"source_id": "recorded.camera.right",
|
||||
"codec_epoch": 1,
|
||||
"timestamp_basis": "session-time-seconds",
|
||||
"timeline_start_seconds": 1.0,
|
||||
@@ -857,7 +857,7 @@ def _release_for_definition(
|
||||
)
|
||||
|
||||
|
||||
def test_release_candidate_matches_repository_but_stays_honestly_blocked(
|
||||
def test_legacy_release_candidate_reports_repository_drift_and_stays_blocked(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
release = PortableLabV1ReleaseCandidate.from_file(
|
||||
@@ -870,7 +870,16 @@ def test_release_candidate_matches_repository_but_stays_honestly_blocked(
|
||||
repository_assets = {
|
||||
asset.asset_id for asset in release.assets if asset.repository_path is not None
|
||||
}
|
||||
assert set(inspection.matched_assets) == repository_assets
|
||||
mismatched_assets = {
|
||||
blocker.removeprefix("asset-").removesuffix("-mismatched")
|
||||
for blocker in inspection.blockers
|
||||
if blocker.startswith("asset-") and blocker.endswith("-mismatched")
|
||||
}
|
||||
assert set(inspection.matched_assets) | mismatched_assets == repository_assets
|
||||
assert {
|
||||
"lab-v1-portable-contracts",
|
||||
"portable-worker-runtime",
|
||||
} <= mismatched_assets
|
||||
assert inspection.ready is False
|
||||
assert "executor-image-unsealed" in inspection.blockers
|
||||
assert "commit-bound-source-unavailable" in inspection.blockers
|
||||
@@ -1132,7 +1141,7 @@ def test_shared_worker_stage_materializes_a_version_bound_camera_job(
|
||||
materialized.camera_job_root
|
||||
/ "input"
|
||||
/ "camera"
|
||||
/ "sensor.camera.right"
|
||||
/ "recorded.camera.right"
|
||||
/ "epoch-1"
|
||||
/ "init.mp4"
|
||||
).read_bytes() == init_payload
|
||||
@@ -1140,7 +1149,7 @@ def test_shared_worker_stage_materializes_a_version_bound_camera_job(
|
||||
materialized.camera_job_root
|
||||
/ "input"
|
||||
/ "camera"
|
||||
/ "sensor.camera.right"
|
||||
/ "recorded.camera.right"
|
||||
/ "epoch-1"
|
||||
/ "segments"
|
||||
/ "1.m4s"
|
||||
@@ -1273,14 +1282,25 @@ def test_profile_runner_sequences_exact_components_and_packages_sealed_job(
|
||||
)
|
||||
runtime_plan = PortableWorkerRuntimePlan(
|
||||
job_id=sealed.job_id,
|
||||
request_sha256=sealed.request_sha256,
|
||||
identity_sha256=sealed.identity_sha256,
|
||||
submission_receipt_sha256=sealed.submission_receipt_sha256,
|
||||
claim_generation=sealed.claim_generation,
|
||||
adapter_id="lab-v1-eomt-ddrnet-worker006-v1",
|
||||
candidate_sha256="f" * 64,
|
||||
setup_id=sealed.setup_id,
|
||||
definition_id=sealed.definition_id,
|
||||
definition_version=sealed.definition_version,
|
||||
definition_sha256=sealed.definition_sha256,
|
||||
source_session_id=sealed.source_session_id,
|
||||
source_catalog_sha256=sealed.source_catalog_sha256,
|
||||
source_bundle_sha256=sealed.source_bundle_sha256,
|
||||
source_capability_manifest_sha256=(
|
||||
sealed.source_capability_manifest_sha256
|
||||
),
|
||||
source_adapter_id=sealed.source_adapter_id,
|
||||
source_adapter_version=sealed.source_adapter_version,
|
||||
source_adapter_sha256=sealed.source_adapter_sha256,
|
||||
result_contract_sha256=definition.result_contract.contract_sha256,
|
||||
phases=PORTABLE_LAB_V1_RUNTIME_PHASES,
|
||||
)
|
||||
|
||||
@@ -115,9 +115,8 @@ class _Launcher:
|
||||
|
||||
|
||||
def _ready_definition() -> PortableRunDefinition:
|
||||
base = PortableRunDefinitionRegistry.from_file(DEFINITIONS_PATH).resolve(
|
||||
base = PortableRunDefinitionRegistry.from_file(DEFINITIONS_PATH).resolve_setup(
|
||||
service_module.PORTABLE_LAB_V1_SETUP_ID,
|
||||
"3692d41cec3949f348a36eb60a501fb2cd483fed1645679b0ec58061a2fc6dc2",
|
||||
)
|
||||
executor = PortableExecutorAvailability(
|
||||
contour_id=base.executor.contour_id,
|
||||
|
||||
@@ -52,7 +52,30 @@ INIT_SHA256 = "e2279963e16d84c91d68e7dbb1f7efed840533387dfeb844b7398bff45fbde38"
|
||||
|
||||
|
||||
def _blocked_registry() -> PortableRunDefinitionRegistry:
|
||||
return PortableRunDefinitionRegistry.from_file(REGISTRY_PATH)
|
||||
production = PortableRunDefinitionRegistry.from_file(REGISTRY_PATH)
|
||||
ready = production.resolve_setup("lab-v1-eomt-ddrnet-portable-v1")
|
||||
executor = PortableExecutorAvailability(
|
||||
contour_id=ready.executor.contour_id,
|
||||
state="not-installed",
|
||||
release_id=None,
|
||||
release_sha256=None,
|
||||
image_sha256=None,
|
||||
reason_code="executor-not-installed",
|
||||
reason="Executor release is not sealed or installed on Worker 006.",
|
||||
)
|
||||
identity = ready.identity_document()
|
||||
identity["executor"] = executor.identity_document()
|
||||
blocked = replace(
|
||||
ready,
|
||||
executor=executor,
|
||||
definition_sha256=canonical_sha256(identity),
|
||||
)
|
||||
return PortableRunDefinitionRegistry(
|
||||
(
|
||||
blocked,
|
||||
production.resolve_setup("m49-tgs-portable-v2"),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _ready_registry() -> PortableRunDefinitionRegistry:
|
||||
|
||||
@@ -29,6 +29,7 @@ from k1link.observatory.portable_result_publisher import (
|
||||
PortableObservatoryResultPublisher,
|
||||
resolve_published_portable_calculation_profile,
|
||||
)
|
||||
from k1link.observatory.portable_result_view import PortableResultViewService
|
||||
from k1link.observatory.portable_run_definitions import (
|
||||
PortableRunDefinition,
|
||||
PortableRunDefinitionRegistry,
|
||||
@@ -372,7 +373,18 @@ def test_verified_package_publishes_immutable_binding_and_profile_provenance(
|
||||
assert first.binding.session_id == RESULT_ID
|
||||
assert first.binding.source_session_id == SOURCE_SESSION_ID
|
||||
assert first.binding.config_sha256 == definition.definition_sha256
|
||||
assert first.binding.replay_capability is None
|
||||
assert first.binding.replay_capability is not None
|
||||
assert first.binding.replay_capability.as_dict() == {
|
||||
"schema_version": "missioncore.observation-lab-replay-capability/v2",
|
||||
"kind": "portable-result-review",
|
||||
"viewer_profile": "portable-result",
|
||||
"timeline": "result-defined",
|
||||
"activation": "explicit",
|
||||
"commands_enabled": False,
|
||||
}
|
||||
assert first.binding.provenance["replay_capability"] == (
|
||||
first.binding.replay_capability.as_dict()
|
||||
)
|
||||
assert first.binding.provenance["calculation_profile"] == {
|
||||
"schema_version": OBSERVATORY_CALCULATION_PROFILE_SCHEMA,
|
||||
"setup_id": definition.setup_id,
|
||||
@@ -388,6 +400,15 @@ def test_verified_package_publishes_immutable_binding_and_profile_provenance(
|
||||
assert store.get_lab_instance(RESULT_ID) == first.binding
|
||||
assert store.get_session(SOURCE_SESSION_ID).summary.lab is None
|
||||
|
||||
view = PortableResultViewService(
|
||||
sessions=store,
|
||||
artifacts=CentralArtifactStore(tmp_path / "central-artifacts"),
|
||||
).read(RESULT_ID)
|
||||
assert view["result_id"] == RESULT_ID
|
||||
assert view["source_session_id"] == SOURCE_SESSION_ID
|
||||
assert view["definition_sha256"] == definition.definition_sha256
|
||||
assert view["viewer_capability"] == first.binding.replay_capability.as_dict()
|
||||
|
||||
summary = store.get_session(RESULT_ID).summary
|
||||
assert summary.display_name == (
|
||||
"Portable result source · полный маршрут и воспроизведение"
|
||||
|
||||
@@ -19,7 +19,7 @@ from k1link.observatory.portable_run_definitions import (
|
||||
|
||||
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
|
||||
REGISTRY_PATH = REPOSITORY_ROOT / "config" / "observatory-portable-run-definitions.json"
|
||||
DEFINITION_SHA256 = "3692d41cec3949f348a36eb60a501fb2cd483fed1645679b0ec58061a2fc6dc2"
|
||||
DEFINITION_SHA256 = "269d71a24b4e63cff54e01273f9d4b35fc6cdd72bc6fadec206169ae0777e6ac"
|
||||
MODEL_MANIFEST_SHA256 = "3fd2d43af73bd73f89d9ffae95d8770cfdeb46033ec967509124fac6ae4afe56"
|
||||
M49_DEFINITION_SHA256 = "f56d6321bd794ccdfb7d2e3b05d044b11f616ffb81ee29517386cc253046d4eb"
|
||||
M49_MODEL_MANIFEST_SHA256 = "489a43448f720a9b5c7993dc8279d167b77191a586f0d87b6d38b81cf728e2f1"
|
||||
@@ -47,6 +47,26 @@ def _first(document: dict[str, object]) -> dict[str, object]:
|
||||
return first
|
||||
|
||||
|
||||
def _blocked_lab_definition():
|
||||
ready = _registry().resolve_setup("lab-v1-eomt-ddrnet-portable-v1")
|
||||
executor = PortableExecutorAvailability(
|
||||
contour_id=ready.executor.contour_id,
|
||||
state="not-installed",
|
||||
release_id=None,
|
||||
release_sha256=None,
|
||||
image_sha256=None,
|
||||
reason_code="executor-not-installed",
|
||||
reason="Executor release is not sealed or installed on Worker 006.",
|
||||
)
|
||||
identity = ready.identity_document()
|
||||
identity["executor"] = executor.identity_document()
|
||||
return replace(
|
||||
ready,
|
||||
executor=executor,
|
||||
definition_sha256=canonical_sha256(identity),
|
||||
)
|
||||
|
||||
|
||||
def test_production_definition_is_source_independent_and_requirements_are_immutable() -> None:
|
||||
definition = _registry().definitions[0]
|
||||
requirements = definition.source_requirements
|
||||
@@ -279,32 +299,43 @@ def test_duplicate_definition_and_incomplete_ready_executor_are_rejected(
|
||||
executor = first["executor"]
|
||||
assert isinstance(executor, dict)
|
||||
executor["state"] = "ready"
|
||||
executor["release_id"] = None
|
||||
executor["release_sha256"] = None
|
||||
executor["image_sha256"] = None
|
||||
executor["reason_code"] = None
|
||||
executor["reason"] = None
|
||||
with pytest.raises(PortableRunDefinitionRegistryError, match="release identity"):
|
||||
PortableRunDefinitionRegistry.from_file(_write(tmp_path, incomplete))
|
||||
|
||||
|
||||
def test_blocked_lab_definition_does_not_hide_ready_m49_definition() -> None:
|
||||
def test_production_definitions_are_both_ready_and_convertible() -> None:
|
||||
registry = _registry()
|
||||
definition = registry.definitions[0]
|
||||
|
||||
assert definition.executor.state == "not-installed"
|
||||
assert definition.executor.release_id is None
|
||||
assert definition.executor.release_sha256 is None
|
||||
assert definition.executor.image_sha256 is None
|
||||
with pytest.raises(
|
||||
PortableRunDefinitionUnavailableError,
|
||||
match="not sealed or installed",
|
||||
):
|
||||
definition.to_recorded_run_definition()
|
||||
assert definition.executor.state == "ready"
|
||||
assert definition.executor.release_id == "lab-v1-installed-package-v1"
|
||||
assert definition.executor.release_sha256 == (
|
||||
"667858623962cd6d9849a8985b803f59e429916b5c56f76a6fc6c80c0c54526b"
|
||||
)
|
||||
assert definition.executor.image_sha256 == (
|
||||
"5ad7d95baac63af13812cb693d492add4e806a333aba8e60edb2ea1aba754373"
|
||||
)
|
||||
assert definition.to_recorded_run_definition().setup_id == definition.setup_id
|
||||
ready = registry.ready_recorded_definitions()
|
||||
assert tuple(row.setup_id for row in ready) == ("m49-tgs-portable-v2",)
|
||||
assert tuple(row.setup_id for row in ready) == (
|
||||
"lab-v1-eomt-ddrnet-portable-v1",
|
||||
"m49-tgs-portable-v2",
|
||||
)
|
||||
assert registry.to_recorded_registry().definitions == ready
|
||||
|
||||
|
||||
def test_conversion_to_recorded_definition_requires_and_preserves_sealed_identities() -> None:
|
||||
blocked = _registry().definitions[0]
|
||||
blocked = _blocked_lab_definition()
|
||||
with pytest.raises(
|
||||
PortableRunDefinitionUnavailableError,
|
||||
match="not sealed or installed",
|
||||
):
|
||||
blocked.to_recorded_run_definition()
|
||||
ready_executor = PortableExecutorAvailability(
|
||||
contour_id="worker-006",
|
||||
state="ready",
|
||||
@@ -339,30 +370,11 @@ def test_conversion_to_recorded_definition_requires_and_preserves_sealed_identit
|
||||
|
||||
def test_blocked_definition_does_not_hide_an_unrelated_ready_definition() -> None:
|
||||
registry = _registry()
|
||||
blocked_lab = registry.resolve_setup("lab-v1-eomt-ddrnet-portable-v1")
|
||||
blocked_lab = _blocked_lab_definition()
|
||||
ready_m49 = registry.resolve_setup("m49-tgs-portable-v2")
|
||||
ready_executor = PortableExecutorAvailability(
|
||||
contour_id="worker-006",
|
||||
state="ready",
|
||||
release_id="lab-v1-eomt-ddrnet-executor-v1",
|
||||
release_sha256="1" * 64,
|
||||
image_sha256="2" * 64,
|
||||
reason_code=None,
|
||||
reason=None,
|
||||
)
|
||||
identity = blocked_lab.identity_document()
|
||||
identity["executor"] = ready_executor.identity_document()
|
||||
ready_lab = replace(
|
||||
blocked_lab,
|
||||
executor=ready_executor,
|
||||
definition_sha256=canonical_sha256(identity),
|
||||
)
|
||||
mixed = PortableRunDefinitionRegistry((ready_lab, ready_m49))
|
||||
mixed = PortableRunDefinitionRegistry((blocked_lab, ready_m49))
|
||||
|
||||
expected = (
|
||||
ready_lab.to_recorded_run_definition(),
|
||||
ready_m49.to_recorded_run_definition(),
|
||||
)
|
||||
expected = (ready_m49.to_recorded_run_definition(),)
|
||||
assert mixed.ready_recorded_definitions() == expected
|
||||
assert mixed.to_recorded_registry().definitions == expected
|
||||
assert mixed.resolve_setup("m49-tgs-portable-v2") is ready_m49
|
||||
@@ -394,7 +406,7 @@ def test_production_lab_v1_model_component_and_result_identities_are_exact() ->
|
||||
"ea583966bc3409f5cf563cbf4fad05e366907e67187082eb692aff53d9f5d875"
|
||||
)
|
||||
assert components["eomt-recorded-runner-v1"].sha256 == (
|
||||
"651e8e06c3912dffb036b7fd08f2c0623f7563d8306cc7aee05db562798518f4"
|
||||
"1e64869de48d10f1531c742e6067c4c3ae2a709c5eb0d770d1fab74b4a2431ff"
|
||||
)
|
||||
assert components["k1-camera-1-calibration-v1"].sha256 == (
|
||||
"05f3ad9b38b3a4fc95388a8ec83da83c745e217709e51787b3d5aad0969f6fa9"
|
||||
|
||||
@@ -92,7 +92,7 @@ def test_portable_setup_catalog_exposes_capability_and_executor_separately() ->
|
||||
setup = document["setups"][0]
|
||||
assert setup["origin"] == "portable-definition"
|
||||
assert setup["source_compatibility"]["outcome"] == "pass"
|
||||
assert setup["executor"]["state"] == "not-installed"
|
||||
assert setup["executor"]["state"] == "ready"
|
||||
assert setup["existing_results"] == []
|
||||
assert setup["preflight"]["outcome"] == "blocked"
|
||||
assert setup["preflight"]["submission_allowed"] is False
|
||||
@@ -153,6 +153,32 @@ def _ready_lab_registry() -> PortableRunDefinitionRegistry:
|
||||
)
|
||||
|
||||
|
||||
def _blocked_lab_registry() -> PortableRunDefinitionRegistry:
|
||||
ready = PortableRunDefinitionRegistry.from_file(REGISTRY_PATH).resolve_setup(
|
||||
"lab-v1-eomt-ddrnet-portable-v1"
|
||||
)
|
||||
executor = PortableExecutorAvailability(
|
||||
contour_id="worker-006",
|
||||
state="not-installed",
|
||||
release_id=None,
|
||||
release_sha256=None,
|
||||
image_sha256=None,
|
||||
reason_code="executor-not-installed",
|
||||
reason="Immutable executor release не установлен.",
|
||||
)
|
||||
identity = ready.identity_document()
|
||||
identity["executor"] = executor.identity_document()
|
||||
return PortableRunDefinitionRegistry(
|
||||
(
|
||||
replace(
|
||||
ready,
|
||||
executor=executor,
|
||||
definition_sha256=canonical_sha256(identity),
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
class _PortableBinding:
|
||||
def __init__(
|
||||
self,
|
||||
@@ -278,7 +304,7 @@ def test_portable_api_check_sha_fences_ready_submission(tmp_path: Path) -> None:
|
||||
def test_portable_api_rejects_blocked_lab_executor_before_binding_submit(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
full_registry = PortableRunDefinitionRegistry.from_file(REGISTRY_PATH)
|
||||
full_registry = _blocked_lab_registry()
|
||||
ready_registry = _ready_lab_registry()
|
||||
queue = ObservatoryRecordedJobQueue(
|
||||
tmp_path,
|
||||
|
||||
@@ -79,15 +79,18 @@ def test_local_worker_gate_is_fail_closed_and_accepts_only_exact_one() -> None:
|
||||
)
|
||||
|
||||
|
||||
def test_validator_registry_fails_when_one_required_profile_is_absent() -> None:
|
||||
def test_validator_registry_selects_only_contracts_present_in_definitions() -> None:
|
||||
definitions = _definitions()
|
||||
only_lab_v1 = PortableRunDefinitionRegistry((definitions.definitions[0],))
|
||||
|
||||
with pytest.raises(
|
||||
PortableWorkerIntegrationError,
|
||||
match="required portable setup is unavailable",
|
||||
):
|
||||
portable_result_validator_registry(only_lab_v1)
|
||||
validators = portable_result_validator_registry(only_lab_v1)
|
||||
|
||||
assert validators.registrations == (
|
||||
PortableResultContractValidatorRegistration(
|
||||
PORTABLE_LAB_V1_RESULT_CONTRACT_SHA256,
|
||||
validate_lab_v1_result_v2,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def test_server_integration_constructs_dormant_transport_and_publisher(
|
||||
|
||||
@@ -13,6 +13,7 @@ from k1link.observatory.portable_run_definitions import (
|
||||
canonical_sha256,
|
||||
)
|
||||
from k1link.observatory.portable_worker_runtime import (
|
||||
PortableWorkerAssetRequirement,
|
||||
PortableWorkerAssetVerification,
|
||||
PortableWorkerExecutorAdapter,
|
||||
PortableWorkerExecutorSeal,
|
||||
@@ -25,6 +26,7 @@ from k1link.observatory.portable_worker_runtime import (
|
||||
PortableWorkerRuntimeRegistryError,
|
||||
PortableWorkerRuntimeUnavailableError,
|
||||
PortableWorkerSourceStage,
|
||||
_matches_tree,
|
||||
inspect_runtime_candidate,
|
||||
)
|
||||
from k1link.observatory.worker_agent import (
|
||||
@@ -49,6 +51,27 @@ def _runtime() -> PortableWorkerRuntimeRegistry:
|
||||
)
|
||||
|
||||
|
||||
def _blocked_candidate(candidate):
|
||||
phases = tuple(
|
||||
PortableWorkerRuntimePhase(phase.phase_id, "missing")
|
||||
for phase in candidate.phases
|
||||
)
|
||||
blockers = ("executor-release-unsealed",)
|
||||
identity = candidate.identity_document()
|
||||
identity["state"] = "blocked"
|
||||
identity["executor"] = None
|
||||
identity["phases"] = [phase.as_dict() for phase in phases]
|
||||
identity["blockers"] = list(blockers)
|
||||
return replace(
|
||||
candidate,
|
||||
state="blocked",
|
||||
executor=None,
|
||||
phases=phases,
|
||||
blockers=blockers,
|
||||
candidate_sha256=canonical_sha256(identity),
|
||||
)
|
||||
|
||||
|
||||
def _all_keys(value: object) -> set[str]:
|
||||
if isinstance(value, dict):
|
||||
return set(value) | {nested for child in value.values() for nested in _all_keys(child)}
|
||||
@@ -57,7 +80,7 @@ def _all_keys(value: object) -> set[str]:
|
||||
return set()
|
||||
|
||||
|
||||
def test_production_candidates_bind_exact_definitions_and_only_m49_is_ready() -> None:
|
||||
def test_production_candidates_bind_exact_definitions_and_both_are_ready() -> None:
|
||||
registry = _runtime()
|
||||
|
||||
assert {candidate.setup_id for candidate in registry.candidates} == {
|
||||
@@ -66,15 +89,16 @@ def test_production_candidates_bind_exact_definitions_and_only_m49_is_ready() ->
|
||||
}
|
||||
by_setup = {candidate.setup_id: candidate for candidate in registry.candidates}
|
||||
lab_v1 = by_setup["lab-v1-eomt-ddrnet-portable-v1"]
|
||||
assert lab_v1.ready is False
|
||||
assert lab_v1.executor is None
|
||||
assert "executor-release-unsealed" in lab_v1.blockers
|
||||
assert any(phase.state == "missing" for phase in lab_v1.phases)
|
||||
with pytest.raises(
|
||||
PortableWorkerRuntimeUnavailableError,
|
||||
match="no executor identity",
|
||||
):
|
||||
lab_v1.executor_identity()
|
||||
assert lab_v1.ready is True
|
||||
assert lab_v1.executor is not None
|
||||
assert lab_v1.blockers == ()
|
||||
assert all(phase.state == "implemented" for phase in lab_v1.phases)
|
||||
assert lab_v1.executor_identity().release_sha256 == (
|
||||
"4788005a24ae3e1664aa7aaaf728fb816243bfde8555c80a608f40d1061925b3"
|
||||
)
|
||||
assert lab_v1.executor_identity().image_sha256 == (
|
||||
"840175ccac731b16b7f2815eaebc6e4ef426c49b57daf1911064ff5b7945a8fd"
|
||||
)
|
||||
|
||||
m49 = by_setup["m49-tgs-portable-v2"]
|
||||
assert m49.ready is True
|
||||
@@ -180,15 +204,15 @@ def test_m49_portable_runner_source_release_is_content_addressed() -> None:
|
||||
assert hashlib.sha256(path.read_bytes()).hexdigest() == item["sha256"]
|
||||
|
||||
|
||||
def test_lab_candidate_verifies_reusable_repository_assets_without_claiming_executor() -> None:
|
||||
def test_ready_lab_candidate_still_requires_complete_local_asset_admission() -> None:
|
||||
candidate = _runtime().resolve(
|
||||
"lab-v1-eomt-ddrnet-portable-v1",
|
||||
"3692d41cec3949f348a36eb60a501fb2cd483fed1645679b0ec58061a2fc6dc2",
|
||||
"11460f0fa03f713b4f64c8742bc30cc31eac66321931e1edd035553f5c72974a",
|
||||
)
|
||||
bindings = {
|
||||
"ddrnet-goose-image": PortableWorkerLocalAssetBinding(
|
||||
asset_id="ddrnet-goose-image",
|
||||
image_sha256=("591cb382c099eeb05e7ec16e2371e0b2da54d2bb5c49ec0f4ac88dbf72b0f0cd"),
|
||||
"agent-image": PortableWorkerLocalAssetBinding(
|
||||
asset_id="agent-image",
|
||||
image_sha256=("840175ccac731b16b7f2815eaebc6e4ef426c49b57daf1911064ff5b7945a8fd"),
|
||||
),
|
||||
"ddrnet-goose-runner": PortableWorkerLocalAssetBinding(
|
||||
asset_id="ddrnet-goose-runner",
|
||||
@@ -201,39 +225,22 @@ def test_lab_candidate_verifies_reusable_repository_assets_without_claiming_exec
|
||||
/ "run_goose_vegetation_benchmark.py"
|
||||
),
|
||||
),
|
||||
"eomt-image": PortableWorkerLocalAssetBinding(
|
||||
asset_id="eomt-image",
|
||||
image_sha256=("58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794"),
|
||||
),
|
||||
"eomt-orchestrator": PortableWorkerLocalAssetBinding(
|
||||
asset_id="eomt-orchestrator",
|
||||
"ddrnet-portable-config": PortableWorkerLocalAssetBinding(
|
||||
asset_id="ddrnet-portable-config",
|
||||
file_path=(
|
||||
REPOSITORY_ROOT
|
||||
/ "experiments"
|
||||
/ "config"
|
||||
/ "perception"
|
||||
/ "worker"
|
||||
/ "Invoke-E4FullSessionSegmentation.ps1"
|
||||
/ "lab-v1-eomt-ddrnet-portable-v2.json"
|
||||
),
|
||||
),
|
||||
"eomt-profile": PortableWorkerLocalAssetBinding(
|
||||
asset_id="eomt-profile",
|
||||
file_path=(
|
||||
REPOSITORY_ROOT
|
||||
/ "experiments"
|
||||
/ "perception"
|
||||
/ "worker"
|
||||
/ "e3_k1_camera1_profile.json"
|
||||
),
|
||||
"ddrnet-step-image": PortableWorkerLocalAssetBinding(
|
||||
asset_id="ddrnet-step-image",
|
||||
image_sha256=("759eeac66145221b3c64f226b9ba9220363af36e4fda9353ef8169551587901c"),
|
||||
),
|
||||
"eomt-runner": PortableWorkerLocalAssetBinding(
|
||||
asset_id="eomt-runner",
|
||||
file_path=(
|
||||
REPOSITORY_ROOT
|
||||
/ "experiments"
|
||||
/ "perception"
|
||||
/ "worker"
|
||||
/ "run_e4_full_session_segmentation.py"
|
||||
),
|
||||
"eomt-step-image": PortableWorkerLocalAssetBinding(
|
||||
asset_id="eomt-step-image",
|
||||
image_sha256=("29a7e8caef51e1809f11b66498ef8bf3f9b2c5d3de40acdab2968cdd34b9623b"),
|
||||
),
|
||||
"vegetation-policy": PortableWorkerLocalAssetBinding(
|
||||
asset_id="vegetation-policy",
|
||||
@@ -258,18 +265,16 @@ def test_lab_candidate_verifies_reusable_repository_assets_without_claiming_exec
|
||||
admission = inspect_runtime_candidate(candidate, bindings)
|
||||
states = {item.asset_id: item.state for item in admission.assets}
|
||||
|
||||
assert states["ddrnet-goose-image"] == "matched"
|
||||
assert states["agent-image"] == "matched"
|
||||
assert states["ddrnet-goose-runner"] == "matched"
|
||||
assert states["eomt-image"] == "matched"
|
||||
assert states["eomt-orchestrator"] == "matched"
|
||||
assert states["eomt-profile"] == "matched"
|
||||
assert states["eomt-runner"] == "matched"
|
||||
assert states["ddrnet-portable-config"] == "missing"
|
||||
assert states["ddrnet-portable-config"] == "matched"
|
||||
assert states["ddrnet-step-image"] == "matched"
|
||||
assert states["eomt-step-image"] == "matched"
|
||||
assert states["ddrnet-checkpoint"] == "missing"
|
||||
assert states["eomt-model-weights"] == "missing"
|
||||
assert states["eomt-model-cache"] == "missing"
|
||||
assert admission.ready is False
|
||||
assert "ddrnet-component-port-uninstalled" in admission.blockers
|
||||
assert "worker-installation-receipt-unavailable" in admission.blockers
|
||||
assert "asset-ddrnet-checkpoint-missing" in admission.blockers
|
||||
assert "asset-eomt-model-cache-missing" in admission.blockers
|
||||
|
||||
|
||||
def test_local_asset_tampering_is_reported_without_execution(tmp_path: Path) -> None:
|
||||
@@ -305,6 +310,78 @@ def test_local_asset_tampering_is_reported_without_execution(tmp_path: Path) ->
|
||||
assert "asset-travel-tgs-image-mismatched" in admission.blockers
|
||||
|
||||
|
||||
def test_sealed_local_tree_uses_manifest_receipt_and_member_metadata(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
tree = tmp_path / "model-tree"
|
||||
tree.mkdir()
|
||||
(tree / "model.bin").write_bytes(b"model")
|
||||
(tree / "profile.json").write_bytes(b"{}\n")
|
||||
manifest = (
|
||||
b"model.bin\t5\t9372c470eeadd5ec5f36cb0b9adf10545c93c5132503830bf1465fe7654b117b\n"
|
||||
b"profile.json\t3\tca3d163bab055381827226140568f3bef7eaac187cebd76878e0b63e9e442356\n"
|
||||
)
|
||||
identity_sha256 = hashlib.sha256(manifest).hexdigest()
|
||||
(tree / "tree-manifest.tsv").write_bytes(manifest)
|
||||
(tree / "tree-receipt.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"schema_version": "missioncore.sealed-tree-runtime/v1",
|
||||
"asset_id": "eomt-model-tree",
|
||||
"identity_algorithm": "relative-path-tab-size-tab-file-sha256-lf/v1",
|
||||
"identity_sha256": identity_sha256,
|
||||
"file_count": 2,
|
||||
"byte_length": 8,
|
||||
"manifest_relative_path": "tree-manifest.tsv",
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
requirement = PortableWorkerAssetRequirement(
|
||||
asset_id="eomt-model-tree",
|
||||
kind="local-tree",
|
||||
sha256=identity_sha256,
|
||||
byte_length=8,
|
||||
component_id=None,
|
||||
model_release_id=None,
|
||||
model_artifact_role=None,
|
||||
)
|
||||
|
||||
assert _matches_tree(requirement, tree) is True
|
||||
|
||||
receipt = json.loads((tree / "tree-receipt.json").read_text(encoding="utf-8"))
|
||||
receipt.update(
|
||||
{
|
||||
"source_image_sha256": "1" * 64,
|
||||
"source_path": "/usr/lib/ffmpeg/7.0",
|
||||
"binaries": {
|
||||
"model-bin": {
|
||||
"relative_path": "model.bin",
|
||||
"byte_length": 5,
|
||||
"sha256": (
|
||||
"9372c470eeadd5ec5f36cb0b9adf10545c93c5132503830bf1465fe7654b117b"
|
||||
),
|
||||
}
|
||||
},
|
||||
}
|
||||
)
|
||||
(tree / "tree-receipt.json").write_text(json.dumps(receipt), encoding="utf-8")
|
||||
|
||||
assert _matches_tree(requirement, tree) is True
|
||||
|
||||
receipt["unexpected"] = "not-admitted"
|
||||
(tree / "tree-receipt.json").write_text(json.dumps(receipt), encoding="utf-8")
|
||||
|
||||
assert _matches_tree(requirement, tree) is False
|
||||
|
||||
receipt.pop("unexpected")
|
||||
(tree / "tree-receipt.json").write_text(json.dumps(receipt), encoding="utf-8")
|
||||
|
||||
(tree / "profile.json").write_bytes(b"drift")
|
||||
|
||||
assert _matches_tree(requirement, tree) is False
|
||||
|
||||
|
||||
def test_runtime_registry_rejects_digest_drift_and_executable_instructions(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
@@ -328,12 +405,23 @@ def test_runtime_registry_rejects_digest_drift_and_executable_instructions(
|
||||
def test_runtime_plan_is_identity_only_and_observation_only() -> None:
|
||||
plan = PortableWorkerRuntimePlan(
|
||||
job_id="observatory-run-" + ("a" * 32),
|
||||
request_sha256="6" * 64,
|
||||
identity_sha256="7" * 64,
|
||||
submission_receipt_sha256="8" * 64,
|
||||
claim_generation=1,
|
||||
adapter_id="m49-tgs-worker006-portable-v2",
|
||||
candidate_sha256="1" * 64,
|
||||
setup_id="m49-tgs-portable-v2",
|
||||
definition_id="m49-tgs-portable",
|
||||
definition_version=3,
|
||||
definition_sha256="2" * 64,
|
||||
source_session_id="source-a",
|
||||
source_catalog_sha256="9" * 64,
|
||||
source_bundle_sha256="3" * 64,
|
||||
source_capability_manifest_sha256="4" * 64,
|
||||
source_adapter_id="xgrids-k1-recorded-observatory-v2",
|
||||
source_adapter_version=2,
|
||||
source_adapter_sha256="a" * 64,
|
||||
result_contract_sha256="5" * 64,
|
||||
phases=("source-delivery", "portable-tgs-runner"),
|
||||
).as_dict()
|
||||
@@ -373,10 +461,10 @@ def test_ready_local_adapter_composes_only_local_ports_and_exact_job(
|
||||
definition_sha256=canonical_sha256(definition_identity),
|
||||
)
|
||||
|
||||
blocked = _runtime().resolve(
|
||||
blocked = _blocked_candidate(_runtime().resolve(
|
||||
"lab-v1-eomt-ddrnet-portable-v1",
|
||||
base_definition.definition_sha256,
|
||||
)
|
||||
))
|
||||
executor_seal = PortableWorkerExecutorSeal(
|
||||
release_id="lab-v1-portable-executor-v1",
|
||||
release_sha256="1" * 64,
|
||||
@@ -512,7 +600,10 @@ def test_ready_local_adapter_composes_only_local_ports_and_exact_job(
|
||||
result_sha256=result_sha256,
|
||||
)
|
||||
|
||||
with pytest.raises(PortableWorkerRuntimeUnavailableError):
|
||||
with pytest.raises(
|
||||
PortableWorkerRuntimeRegistryError,
|
||||
match="blocked local runtime cannot bind a ready RunDefinition",
|
||||
):
|
||||
PortableWorkerExecutorAdapter(
|
||||
candidate=blocked,
|
||||
definition=base_definition,
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import Any, cast
|
||||
|
||||
from k1link.observatory.portable_publication_reconciler import (
|
||||
PortablePublicationReconciler,
|
||||
)
|
||||
|
||||
NOW = datetime(2026, 9, 1, 12, 0, tzinfo=UTC)
|
||||
|
||||
|
||||
class _Queue:
|
||||
def __init__(self, jobs: tuple[SimpleNamespace, ...]) -> None:
|
||||
self.jobs = jobs
|
||||
self.published: list[str] = []
|
||||
self.failed: list[str] = []
|
||||
|
||||
def pending_publications(self) -> tuple[SimpleNamespace, ...]:
|
||||
return self.jobs
|
||||
|
||||
def mark_published(self, job_id: str) -> None:
|
||||
self.published.append(job_id)
|
||||
|
||||
def mark_publication_failed(self, job_id: str, *, message: str) -> None:
|
||||
assert message
|
||||
self.failed.append(job_id)
|
||||
|
||||
|
||||
class _Transport:
|
||||
def __init__(self, root: Path) -> None:
|
||||
self.root = root
|
||||
self.calls: list[str] = []
|
||||
|
||||
def package_root_for_terminal(self, job: SimpleNamespace) -> Path:
|
||||
self.calls.append(job.job_id)
|
||||
return self.root
|
||||
|
||||
|
||||
class _Publisher:
|
||||
def __init__(self) -> None:
|
||||
self.calls: list[str] = []
|
||||
|
||||
def publish(self, *, job: SimpleNamespace, package_root: Path) -> None:
|
||||
assert package_root.is_dir()
|
||||
self.calls.append(job.job_id)
|
||||
|
||||
|
||||
def test_reconciler_publishes_pending_result_without_compute(tmp_path: Path) -> None:
|
||||
package = tmp_path / "package"
|
||||
package.mkdir()
|
||||
job = _job("pending", attempts=0, updated_at="2026-09-01T11:59:00Z")
|
||||
queue = _Queue((job,))
|
||||
transport = _Transport(package)
|
||||
publisher = _Publisher()
|
||||
reconciler = PortablePublicationReconciler(
|
||||
queue=cast(Any, queue),
|
||||
artifact_transport=cast(Any, transport),
|
||||
result_publisher=cast(Any, publisher),
|
||||
clock=lambda: NOW,
|
||||
)
|
||||
|
||||
result = reconciler.run_once()
|
||||
|
||||
assert result.published == 1
|
||||
assert result.failed == 0
|
||||
assert queue.published == [job.job_id]
|
||||
assert transport.calls == [job.job_id]
|
||||
assert publisher.calls == [job.job_id]
|
||||
|
||||
|
||||
def test_reconciler_defers_backoff_and_exhausts_bounded_attempts(tmp_path: Path) -> None:
|
||||
package = tmp_path / "package"
|
||||
package.mkdir()
|
||||
deferred = _job("failed", attempts=1, updated_at="2026-09-01T11:59:50Z")
|
||||
exhausted = _job("failed", attempts=5, updated_at="2026-09-01T10:00:00Z")
|
||||
queue = _Queue((deferred, exhausted))
|
||||
transport = _Transport(package)
|
||||
publisher = _Publisher()
|
||||
reconciler = PortablePublicationReconciler(
|
||||
queue=cast(Any, queue),
|
||||
artifact_transport=cast(Any, transport),
|
||||
result_publisher=cast(Any, publisher),
|
||||
clock=lambda: NOW,
|
||||
)
|
||||
|
||||
result = reconciler.run_once()
|
||||
|
||||
assert result.deferred == 1
|
||||
assert result.exhausted == 1
|
||||
assert result.published == 0
|
||||
assert transport.calls == []
|
||||
assert publisher.calls == []
|
||||
|
||||
|
||||
def _job(
|
||||
publication_state: str,
|
||||
*,
|
||||
attempts: int,
|
||||
updated_at: str,
|
||||
) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
job_id=f"observatory-run-{attempts:032x}",
|
||||
publication_state=publication_state,
|
||||
publication_attempts=attempts,
|
||||
updated_at_utc=updated_at,
|
||||
)
|
||||
@@ -0,0 +1,87 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any, cast
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from k1link.sessions.store import SessionStore
|
||||
from k1link.web.observatory_api import build_observatory_router
|
||||
|
||||
JOB_ID = "observatory-run-0123456789abcdef0123456789abcdef"
|
||||
|
||||
|
||||
class _Job:
|
||||
state = "succeeded"
|
||||
publication_state = "failed"
|
||||
|
||||
def as_dict(self) -> dict[str, object]:
|
||||
return {
|
||||
"job_id": JOB_ID,
|
||||
"state": self.state,
|
||||
"publication": {"state": self.publication_state},
|
||||
}
|
||||
|
||||
|
||||
class _Queue:
|
||||
def __init__(self) -> None:
|
||||
self.job = _Job()
|
||||
self.published: list[str] = []
|
||||
|
||||
def get(self, job_id: str) -> _Job:
|
||||
assert job_id == JOB_ID
|
||||
return self.job
|
||||
|
||||
def mark_published(self, job_id: str) -> _Job:
|
||||
assert job_id == JOB_ID
|
||||
self.published.append(job_id)
|
||||
self.job.publication_state = "published"
|
||||
return self.job
|
||||
|
||||
|
||||
class _Transport:
|
||||
def __init__(self, package_root: Path) -> None:
|
||||
self.package_root = package_root
|
||||
self.calls: list[str] = []
|
||||
|
||||
def package_root_for_terminal(self, job: _Job) -> Path:
|
||||
assert job.state == "succeeded"
|
||||
self.calls.append(JOB_ID)
|
||||
return self.package_root
|
||||
|
||||
|
||||
class _Publisher:
|
||||
def __init__(self) -> None:
|
||||
self.calls: list[tuple[_Job, Path]] = []
|
||||
|
||||
def publish(self, *, job: _Job, package_root: Path) -> None:
|
||||
self.calls.append((job, package_root))
|
||||
|
||||
|
||||
def test_operator_retry_republishes_without_creating_compute_work(tmp_path: Path) -> None:
|
||||
queue = _Queue()
|
||||
package_root = tmp_path / "sealed-result"
|
||||
package_root.mkdir()
|
||||
transport = _Transport(package_root)
|
||||
publisher = _Publisher()
|
||||
app = FastAPI()
|
||||
app.include_router(
|
||||
build_observatory_router(
|
||||
SessionStore(tmp_path / "repository", data_dir=tmp_path / "data"),
|
||||
recorded_job_queue=cast(Any, queue),
|
||||
recorded_binding_service=cast(Any, object()),
|
||||
portable_artifact_transport=cast(Any, transport),
|
||||
portable_result_publisher=cast(Any, publisher),
|
||||
)
|
||||
)
|
||||
|
||||
response = TestClient(app).post(
|
||||
f"/api/v1/observatory/runs/{JOB_ID}/publication/retry"
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json()["publication"]["state"] == "published"
|
||||
assert transport.calls == [JOB_ID]
|
||||
assert publisher.calls == [(queue.job, package_root)]
|
||||
assert queue.published == [JOB_ID]
|
||||
@@ -21,6 +21,7 @@ from k1link.observatory.recorded_jobs import (
|
||||
ObservatoryRecordedQueueStaleClaimError,
|
||||
ObservatoryRecordedReconciliationRequest,
|
||||
ObservatoryRecordedResourceReleaseAttestation,
|
||||
RecordedExecutorIdentity,
|
||||
RecordedRunDefinition,
|
||||
RecordedRunDefinitionRegistry,
|
||||
)
|
||||
@@ -392,6 +393,129 @@ def test_claim_is_exactly_idempotent_including_empty_result(tmp_path: Path) -> N
|
||||
)
|
||||
|
||||
|
||||
def test_capability_aware_claim_skips_incompatible_queued_job(tmp_path: Path) -> None:
|
||||
first, second = _definitions().definitions
|
||||
compatible = replace(
|
||||
second,
|
||||
setup_id="compatible-portable-v1",
|
||||
definition_id="compatible-portable",
|
||||
definition_sha256="8" * 64,
|
||||
executor_release_sha256="9" * 64,
|
||||
executor_image_sha256="a" * 64,
|
||||
model_manifest_sha256="b" * 64,
|
||||
resource_profile_sha256="c" * 64,
|
||||
)
|
||||
queue = ObservatoryRecordedJobQueue(
|
||||
tmp_path,
|
||||
definitions=RecordedRunDefinitionRegistry((first, compatible)),
|
||||
clock=lambda: NOW,
|
||||
)
|
||||
incompatible_job, _ = queue.submit(_intent(idempotency_key="capability:first"), enqueue=True)
|
||||
compatible_job, _ = queue.submit(
|
||||
_intent(
|
||||
idempotency_key="capability:second",
|
||||
setup_id=compatible.setup_id,
|
||||
definition_sha256=compatible.definition_sha256,
|
||||
),
|
||||
enqueue=True,
|
||||
)
|
||||
capability = RecordedExecutorIdentity(
|
||||
release_sha256=compatible.executor_release_sha256,
|
||||
image_sha256=compatible.executor_image_sha256,
|
||||
model_manifest_sha256=compatible.model_manifest_sha256,
|
||||
resource_profile_sha256=compatible.resource_profile_sha256,
|
||||
)
|
||||
|
||||
claim = queue.claim_next(
|
||||
claimant_id="recorded-worker",
|
||||
claim_request_id="capability-aware-poll",
|
||||
supported_executor_identities=(capability,),
|
||||
)
|
||||
|
||||
assert claim is not None
|
||||
assert claim.job.job_id == compatible_job.job_id
|
||||
assert queue.get(incompatible_job.job_id).state == "queued"
|
||||
|
||||
|
||||
def test_capability_claim_identity_binds_snapshot_and_empty_snapshot_claims_nothing(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
queue = _queue(tmp_path)
|
||||
job, _ = queue.submit(_intent(), enqueue=True)
|
||||
|
||||
assert queue.claim_next(
|
||||
claimant_id="recorded-worker",
|
||||
claim_request_id="capability-empty-poll",
|
||||
supported_executor_identities=(),
|
||||
) is None
|
||||
capability = RecordedExecutorIdentity(
|
||||
release_sha256=EXECUTOR_RELEASE_SHA,
|
||||
image_sha256=EXECUTOR_IMAGE_SHA,
|
||||
model_manifest_sha256=MODEL_MANIFEST_SHA,
|
||||
resource_profile_sha256=RESOURCE_PROFILE_SHA,
|
||||
)
|
||||
with pytest.raises(ObservatoryRecordedQueueConflictError, match="claim"):
|
||||
queue.claim_next(
|
||||
claimant_id="recorded-worker",
|
||||
claim_request_id="capability-empty-poll",
|
||||
supported_executor_identities=(capability,),
|
||||
)
|
||||
legacy_claim = queue.claim_next(
|
||||
claimant_id="recorded-worker",
|
||||
claim_request_id="legacy-poll-after-empty-capability",
|
||||
)
|
||||
assert legacy_claim is not None
|
||||
assert legacy_claim.job.job_id == job.job_id
|
||||
|
||||
|
||||
def test_execution_success_uses_durable_idempotent_publication_outbox(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
queue = _queue(tmp_path)
|
||||
job, _ = queue.submit(_intent(), enqueue=True)
|
||||
claim = queue.claim_next(
|
||||
claimant_id="recorded-worker",
|
||||
claim_request_id="publication-outbox-claim",
|
||||
)
|
||||
assert claim is not None
|
||||
queue.start(job.job_id, claim_token=claim.claim_token)
|
||||
|
||||
completed = queue.complete_for_publication(
|
||||
job.job_id,
|
||||
claim_token=claim.claim_token,
|
||||
result_id="portable-result-001",
|
||||
result_sha256=RESULT_SHA,
|
||||
)
|
||||
assert completed.state == "succeeded"
|
||||
assert completed.publication_state == "pending"
|
||||
assert completed.publication_attempts == 0
|
||||
assert queue.pending_publications() == (completed,)
|
||||
|
||||
failed = queue.mark_publication_failed(
|
||||
job.job_id,
|
||||
message="catalog temporarily unavailable",
|
||||
)
|
||||
assert failed.publication_state == "failed"
|
||||
assert failed.publication_attempts == 1
|
||||
assert failed.publication_error == "catalog temporarily unavailable"
|
||||
|
||||
replay = queue.complete_for_publication(
|
||||
job.job_id,
|
||||
claim_token=claim.claim_token,
|
||||
result_id="portable-result-001",
|
||||
result_sha256=RESULT_SHA,
|
||||
)
|
||||
assert replay == failed
|
||||
|
||||
published = queue.mark_published(job.job_id)
|
||||
assert published.publication_state == "published"
|
||||
assert published.publication_attempts == 2
|
||||
assert published.publication_error is None
|
||||
assert published.published_at_utc == NOW
|
||||
assert queue.mark_published(job.job_id) == published
|
||||
assert queue.pending_publications() == ()
|
||||
|
||||
|
||||
def test_claim_lease_renews_idempotently_and_requeues_expired_unstarted_job(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
|
||||
@@ -114,10 +114,12 @@ class FakeTransport:
|
||||
*,
|
||||
claimant_id: str,
|
||||
claim_request_id: str,
|
||||
supported_executor_identities: tuple[ObservatoryWorkerExecutorIdentity, ...],
|
||||
) -> Mapping[str, object] | None:
|
||||
claim = self.queue.claim_next(
|
||||
claimant_id=claimant_id,
|
||||
claim_request_id=claim_request_id,
|
||||
supported_executor_identities=supported_executor_identities,
|
||||
)
|
||||
if claim is None:
|
||||
return None
|
||||
@@ -397,7 +399,7 @@ def test_worker_agent_leaves_empty_queue_untouched(tmp_path: Path) -> None:
|
||||
assert executor.jobs == []
|
||||
|
||||
|
||||
def test_exact_release_mismatch_fails_closed_without_start(tmp_path: Path) -> None:
|
||||
def test_exact_release_mismatch_is_not_claimed_or_failed(tmp_path: Path) -> None:
|
||||
queue = _queue(tmp_path)
|
||||
job_id = _enqueue(queue)
|
||||
transport = FakeTransport(queue)
|
||||
@@ -409,19 +411,13 @@ def test_exact_release_mismatch_fails_closed_without_start(tmp_path: Path) -> No
|
||||
identity=_identity(release_sha256="b" * 64),
|
||||
).run_once()
|
||||
|
||||
assert report.state == "failed"
|
||||
assert report.failure_code == "executor-not-allowlisted"
|
||||
assert report.state == "empty"
|
||||
assert report.failure_code is None
|
||||
assert transport.starts == []
|
||||
assert transport.successes == []
|
||||
assert transport.failures == [
|
||||
(
|
||||
job_id,
|
||||
"executor-not-allowlisted",
|
||||
"Exact executor identity is not installed on Worker 006.",
|
||||
)
|
||||
]
|
||||
assert transport.failures == []
|
||||
assert executor.jobs == []
|
||||
assert queue.get(job_id).state == "failed"
|
||||
assert queue.get(job_id).state == "queued"
|
||||
|
||||
|
||||
def _spoof_claimant(payload: dict[str, object]) -> dict[str, object]:
|
||||
@@ -525,9 +521,11 @@ class BlockingEmptyTransport:
|
||||
*,
|
||||
claimant_id: str,
|
||||
claim_request_id: str,
|
||||
supported_executor_identities: tuple[ObservatoryWorkerExecutorIdentity, ...],
|
||||
) -> Mapping[str, object] | None:
|
||||
assert claimant_id == WORKER_006_CONTOUR_ID
|
||||
assert claim_request_id == "worker-006:overlap-test"
|
||||
assert supported_executor_identities == ()
|
||||
self.entered.set()
|
||||
assert self.release.wait(timeout=2)
|
||||
return None
|
||||
|
||||
@@ -11,6 +11,7 @@ from fastapi.testclient import TestClient
|
||||
from k1link.observatory.portable_artifact_transport import (
|
||||
PortableArtifactTransportUnavailableError,
|
||||
)
|
||||
from k1link.observatory.portable_result_contract import PortableResultPublisherError
|
||||
from k1link.observatory.recorded_jobs import (
|
||||
ObservatoryRecordedJobIntent,
|
||||
ObservatoryRecordedJobQueue,
|
||||
@@ -32,7 +33,7 @@ WORKER_HEADERS = {
|
||||
"Authorization": f"Bearer {WORKER_TOKEN}",
|
||||
OBSERVATORY_WORKER_CONTOUR_HEADER: "worker-006",
|
||||
}
|
||||
CLAIM_SCHEMA = "missioncore.observatory-worker-claim-request/v1"
|
||||
CLAIM_SCHEMA = "missioncore.observatory-worker-claim-request/v2"
|
||||
START_SCHEMA = "missioncore.observatory-worker-start-request/v1"
|
||||
RENEW_SCHEMA = "missioncore.observatory-worker-renew-request/v1"
|
||||
CHECKPOINT_SCHEMA = "missioncore.observatory-worker-checkpoint-request/v1"
|
||||
@@ -61,6 +62,22 @@ def _definition() -> RecordedRunDefinition:
|
||||
)
|
||||
|
||||
|
||||
def _claim_request(claim_request_id: str) -> dict[str, object]:
|
||||
definition = _definition()
|
||||
return {
|
||||
"schema_version": CLAIM_SCHEMA,
|
||||
"claim_request_id": claim_request_id,
|
||||
"supported_executor_identities": [
|
||||
{
|
||||
"release_sha256": definition.executor_release_sha256,
|
||||
"image_sha256": definition.executor_image_sha256,
|
||||
"model_manifest_sha256": definition.model_manifest_sha256,
|
||||
"resource_profile_sha256": definition.resource_profile_sha256,
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _services(tmp_path: Path) -> tuple[TestClient, ObservatoryRecordedJobQueue]:
|
||||
definition = _definition()
|
||||
queue = ObservatoryRecordedJobQueue(
|
||||
@@ -85,6 +102,28 @@ class _ArtifactTransportWithoutCompletedPackage:
|
||||
raise PortableArtifactTransportUnavailableError("package is incomplete")
|
||||
|
||||
|
||||
class _CompletedArtifactTransport:
|
||||
def __init__(self, package_root: Path) -> None:
|
||||
self.package_root = package_root
|
||||
|
||||
def require_completed_for_success(self, **_values: object) -> Path:
|
||||
return self.package_root
|
||||
|
||||
def package_root_for_terminal(self, _job: object) -> Path:
|
||||
return self.package_root
|
||||
|
||||
|
||||
class _TransientResultPublisher:
|
||||
def __init__(self) -> None:
|
||||
self.attempts = 0
|
||||
|
||||
def publish(self, **_values: object) -> object:
|
||||
self.attempts += 1
|
||||
if self.attempts == 1:
|
||||
raise PortableResultPublisherError("catalog temporarily unavailable")
|
||||
return object()
|
||||
|
||||
|
||||
def _services_with_artifact_transport(
|
||||
tmp_path: Path,
|
||||
) -> tuple[TestClient, ObservatoryRecordedJobQueue]:
|
||||
@@ -107,6 +146,30 @@ def _services_with_artifact_transport(
|
||||
return TestClient(app), queue
|
||||
|
||||
|
||||
def _services_with_transient_publisher(
|
||||
tmp_path: Path,
|
||||
) -> tuple[TestClient, ObservatoryRecordedJobQueue, _TransientResultPublisher]:
|
||||
definition = _definition()
|
||||
queue = ObservatoryRecordedJobQueue(
|
||||
tmp_path,
|
||||
definitions=RecordedRunDefinitionRegistry((definition,)),
|
||||
)
|
||||
publisher = _TransientResultPublisher()
|
||||
app = FastAPI()
|
||||
app.include_router(
|
||||
build_observatory_worker_router(
|
||||
queue,
|
||||
authentication=ObservatoryWorkerAuthentication(
|
||||
bearer_token_sha256=WORKER_TOKEN_SHA256,
|
||||
contour_id="worker-006",
|
||||
),
|
||||
artifact_transport=_CompletedArtifactTransport(tmp_path / "package"), # type: ignore[arg-type]
|
||||
result_publisher=publisher, # type: ignore[arg-type]
|
||||
)
|
||||
)
|
||||
return TestClient(app), queue, publisher
|
||||
|
||||
|
||||
def _enqueue(
|
||||
queue: ObservatoryRecordedJobQueue,
|
||||
*,
|
||||
@@ -137,10 +200,7 @@ def _claim(
|
||||
response = client.post(
|
||||
"/api/v1/worker/observatory/recorded-jobs/claims",
|
||||
headers=WORKER_HEADERS,
|
||||
json={
|
||||
"schema_version": CLAIM_SCHEMA,
|
||||
"claim_request_id": claim_request_id,
|
||||
},
|
||||
json=_claim_request(claim_request_id),
|
||||
)
|
||||
assert response.status_code == 200
|
||||
return cast(dict[str, Any], response.json())
|
||||
@@ -151,10 +211,7 @@ def test_worker_authentication_requires_digest_and_configured_contour(
|
||||
) -> None:
|
||||
client, queue = _services(tmp_path)
|
||||
_enqueue(queue)
|
||||
request = {
|
||||
"schema_version": CLAIM_SCHEMA,
|
||||
"claim_request_id": "worker-006:auth-claim",
|
||||
}
|
||||
request = _claim_request("worker-006:auth-claim")
|
||||
|
||||
missing = client.post(
|
||||
"/api/v1/worker/observatory/recorded-jobs/claims",
|
||||
@@ -198,10 +255,7 @@ def test_claim_is_idempotent_and_request_schema_rejects_execution_inputs(
|
||||
) -> None:
|
||||
client, queue = _services(tmp_path)
|
||||
job_id = _enqueue(queue)
|
||||
request = {
|
||||
"schema_version": CLAIM_SCHEMA,
|
||||
"claim_request_id": "worker-006:stable-claim",
|
||||
}
|
||||
request = _claim_request("worker-006:stable-claim")
|
||||
|
||||
first = client.post(
|
||||
"/api/v1/worker/observatory/recorded-jobs/claims",
|
||||
@@ -235,14 +289,30 @@ def test_claim_is_idempotent_and_request_schema_rejects_execution_inputs(
|
||||
assert queue.get(job_id).state == "claimed"
|
||||
|
||||
|
||||
def test_legacy_claim_without_capability_snapshot_cannot_take_another_profile(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
client, queue = _services(tmp_path)
|
||||
job_id = _enqueue(queue)
|
||||
|
||||
response = client.post(
|
||||
"/api/v1/worker/observatory/recorded-jobs/claims",
|
||||
headers=WORKER_HEADERS,
|
||||
json={
|
||||
"schema_version": "missioncore.observatory-worker-claim-request/v1",
|
||||
"claim_request_id": "worker-006:legacy-unbounded-claim",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 422
|
||||
assert queue.get(job_id).state == "queued"
|
||||
|
||||
|
||||
def test_empty_claim_is_204_and_empty_receipt_remains_idempotent(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
client, queue = _services(tmp_path)
|
||||
request = {
|
||||
"schema_version": CLAIM_SCHEMA,
|
||||
"claim_request_id": "worker-006:empty-poll",
|
||||
}
|
||||
request = _claim_request("worker-006:empty-poll")
|
||||
|
||||
first = client.post(
|
||||
"/api/v1/worker/observatory/recorded-jobs/claims",
|
||||
@@ -258,10 +328,7 @@ def test_empty_claim_is_204_and_empty_receipt_remains_idempotent(
|
||||
fresh = client.post(
|
||||
"/api/v1/worker/observatory/recorded-jobs/claims",
|
||||
headers=WORKER_HEADERS,
|
||||
json={
|
||||
"schema_version": CLAIM_SCHEMA,
|
||||
"claim_request_id": "worker-006:fresh-poll",
|
||||
},
|
||||
json=_claim_request("worker-006:fresh-poll"),
|
||||
)
|
||||
|
||||
assert first.status_code == 204
|
||||
@@ -456,6 +523,57 @@ def test_artifact_transport_blocks_success_without_completed_package(
|
||||
assert queue.get(job_id).state == "running"
|
||||
|
||||
|
||||
def test_publication_failure_is_durable_and_retry_does_not_rerun_execution(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
client, queue, publisher = _services_with_transient_publisher(tmp_path)
|
||||
job_id = _enqueue(queue)
|
||||
claim_token = _claim(client)["claim_token"]
|
||||
client.post(
|
||||
f"/api/v1/worker/observatory/recorded-jobs/{job_id}/start",
|
||||
headers=WORKER_HEADERS,
|
||||
json={"schema_version": START_SCHEMA, "claim_token": claim_token},
|
||||
)
|
||||
|
||||
sealed = client.post(
|
||||
f"/api/v1/worker/observatory/recorded-jobs/{job_id}/succeed",
|
||||
headers=WORKER_HEADERS,
|
||||
json={
|
||||
"schema_version": SUCCEED_SCHEMA,
|
||||
"claim_token": claim_token,
|
||||
"result_id": "portable-result-publication-retry",
|
||||
"result_sha256": "b" * 64,
|
||||
},
|
||||
)
|
||||
|
||||
assert sealed.status_code == 202
|
||||
assert sealed.json()["state"] == "succeeded"
|
||||
assert sealed.json()["publication"] == {
|
||||
"state": "failed",
|
||||
"attempts": 1,
|
||||
"error": "catalog temporarily unavailable",
|
||||
"published_at_utc": None,
|
||||
}
|
||||
assert queue.get(job_id).claim_generation == 1
|
||||
|
||||
retried = client.post(
|
||||
f"/api/v1/worker/observatory/recorded-jobs/{job_id}/publication/retry",
|
||||
headers=WORKER_HEADERS,
|
||||
)
|
||||
repeated = client.post(
|
||||
f"/api/v1/worker/observatory/recorded-jobs/{job_id}/publication/retry",
|
||||
headers=WORKER_HEADERS,
|
||||
)
|
||||
|
||||
assert retried.status_code == 200
|
||||
assert retried.json()["publication"]["state"] == "published"
|
||||
assert retried.json()["publication"]["attempts"] == 2
|
||||
assert repeated.status_code == 200
|
||||
assert repeated.json()["publication"]["attempts"] == 2
|
||||
assert queue.get(job_id).claim_generation == 1
|
||||
assert publisher.attempts == 2
|
||||
|
||||
|
||||
def test_worker_can_fail_claimed_job_idempotently(tmp_path: Path) -> None:
|
||||
client, queue = _services(tmp_path)
|
||||
job_id = _enqueue(queue)
|
||||
|
||||
@@ -99,6 +99,7 @@ def _cache_claim(gateway: ObservatoryWorkerHttpGateway) -> None:
|
||||
payload = gateway.claim_next(
|
||||
claimant_id=WORKER_006_CONTOUR_ID,
|
||||
claim_request_id="worker-006:http-transport-test",
|
||||
supported_executor_identities=(),
|
||||
)
|
||||
assert payload is not None
|
||||
|
||||
|
||||
Reference in New Issue
Block a user