fix(observatory): version non-binding idle claims and preserve grant history
This commit is contained in:
DCCONSTRUCTIONS
@@ -0,0 +1,396 @@
|
||||
"""Bounded Worker006 control-agent migration; never alters a compute package.
|
||||
|
||||
Run inside an existing agent image with Docker's socket and a new /migration
|
||||
evidence directory mounted. Plan is read-only. Apply requires its exact hash.
|
||||
It commits a one-file, offline child layer, preserves the executor declarations,
|
||||
and retains stopped predecessors (restart=no) for explicit rollback. The saved
|
||||
create-body is the durable transport declaration; old package launchers must
|
||||
not replace it with the v2 parent image. No model/executor is started here.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import copy
|
||||
import hashlib
|
||||
import http.client
|
||||
import io
|
||||
import json
|
||||
import socket
|
||||
import tarfile
|
||||
import time
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from urllib.parse import quote, urlencode
|
||||
|
||||
INSTALLED_PARENT_SHA = "5ad7d95baac63af13812cb693d492add4e806a333aba8e60edb2ea1aba754373"
|
||||
M49_PARENT_SHA = "d00274a2a76e254d79b28f887b421cbc267a0483fca0e83e8c2e1c416ac8593c"
|
||||
TARGETS = {
|
||||
"ndc-observatory-installed-lab-worker-agent": INSTALLED_PARENT_SHA,
|
||||
"ndc-observatory-m49-worker-agent": M49_PARENT_SHA,
|
||||
}
|
||||
SOURCE = "/opt/nodedc/mission-core/src/k1link/observatory/worker_http_transport.py"
|
||||
BEFORE_SHA = "fb67ae174c8da66f7d04022310be663b2735fff72966883aaa5b50fdbc3e89c7"
|
||||
OLD = b"missioncore.observatory-worker-claim-request/v2"
|
||||
NEW = b"missioncore.observatory-worker-claim-request/v3"
|
||||
PARENT_LABEL = "com.nodedc.claim-transport-parent.sha256"
|
||||
|
||||
|
||||
def canonical(value: object) -> bytes:
|
||||
return json.dumps(value, sort_keys=True, separators=(",", ":")).encode()
|
||||
|
||||
|
||||
def sha(payload: bytes) -> str:
|
||||
return hashlib.sha256(payload).hexdigest()
|
||||
|
||||
|
||||
def patch_source(payload: bytes) -> bytes:
|
||||
if sha(payload) != BEFORE_SHA or payload.count(OLD) != 1:
|
||||
raise ValueError("installed transport source is not the reviewed baseline")
|
||||
result = payload.replace(OLD, NEW)
|
||||
compile(result, SOURCE, "exec")
|
||||
return result
|
||||
|
||||
|
||||
class UnixConnection(http.client.HTTPConnection):
|
||||
def connect(self) -> None:
|
||||
self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
self.sock.settimeout(45)
|
||||
self.sock.connect("/var/run/docker.sock")
|
||||
|
||||
|
||||
class Engine:
|
||||
def request(self, method: str, path: str, body=None, *, raw=False):
|
||||
connection = UnixConnection("localhost", timeout=45)
|
||||
payload = body if isinstance(body, bytes) else (None if body is None else canonical(body))
|
||||
content_type = "application/x-tar" if isinstance(body, bytes) else "application/json"
|
||||
try:
|
||||
connection.request(method, "/v1.45" + path, payload, {"Content-Type": content_type})
|
||||
response = connection.getresponse()
|
||||
data = response.read(2 * 1024 * 1024 + 1)
|
||||
if not 200 <= response.status < 300 or len(data) > 2 * 1024 * 1024:
|
||||
raise RuntimeError(f"Docker request failed: {method} {path}: {response.status}")
|
||||
return data if raw else (json.loads(data) if data else None)
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
def inspect(self, name: str):
|
||||
return self.request("GET", f"/containers/{quote(name, safe='')}/json")
|
||||
|
||||
def execute_json(self, name: str, source: str):
|
||||
execution = self.request(
|
||||
"POST",
|
||||
f"/containers/{name}/exec",
|
||||
{
|
||||
"AttachStdout": True,
|
||||
"AttachStderr": True,
|
||||
"Cmd": ["python3", "-c", source],
|
||||
"Env": ["PYTHONDONTWRITEBYTECODE=1"],
|
||||
},
|
||||
)["Id"]
|
||||
data = self.request("POST", f"/exec/{execution}/start", {}, raw=True)
|
||||
status = self.request("GET", f"/exec/{execution}/json")
|
||||
if status["Running"] or status["ExitCode"] != 0:
|
||||
raise RuntimeError("control-only readiness/probe failed; no credential output retained")
|
||||
streams = bytearray()
|
||||
while data:
|
||||
if len(data) < 8:
|
||||
raise RuntimeError("truncated Docker exec stream")
|
||||
size = int.from_bytes(data[4:8], "big")
|
||||
if data[0] != 1 or size > len(data) - 8:
|
||||
raise RuntimeError("unexpected Docker exec output")
|
||||
streams.extend(data[8 : 8 + size])
|
||||
data = data[8 + size :]
|
||||
return json.loads(streams)
|
||||
|
||||
|
||||
READINESS = """import json, os, pathlib, httpx
|
||||
headers = {"Authorization": "Bearer " + pathlib.Path(os.environ[
|
||||
"MISSIONCORE_OBSERVATORY_WORKER_TOKEN_FILE"]).read_text().strip(),
|
||||
"X-Mission-Core-Contour-Id": "worker-006"}
|
||||
with httpx.Client(base_url=os.environ["MISSIONCORE_OBSERVATORY_WORKER_BASE_URL"],
|
||||
headers=headers, timeout=10, trust_env=False) as client:
|
||||
response = client.get("/api/v1/worker/observatory/recorded-jobs/claim-readiness")
|
||||
assert response.status_code == 200
|
||||
print(json.dumps(response.json()))
|
||||
"""
|
||||
|
||||
PROBE = """import json, os, pathlib, uuid
|
||||
from k1link.observatory.worker_http_transport import ObservatoryWorkerHttpGateway
|
||||
with ObservatoryWorkerHttpGateway(
|
||||
base_url=os.environ["MISSIONCORE_OBSERVATORY_WORKER_BASE_URL"],
|
||||
bearer_token=pathlib.Path(os.environ["MISSIONCORE_OBSERVATORY_WORKER_TOKEN_FILE"]).read_text().strip(),
|
||||
work_root=pathlib.Path(os.environ["MISSIONCORE_OBSERVATORY_WORKER_WORK_ROOT"]),
|
||||
) as client:
|
||||
for index in range(8):
|
||||
assert client.claim_next(claimant_id="worker-006",
|
||||
claim_request_id="v3-control-probe-" + uuid.uuid4().hex,
|
||||
supported_executor_identities=()) is None
|
||||
print(json.dumps({"idle_claims": 8, "capabilities": [], "model_jobs": 0}))
|
||||
"""
|
||||
|
||||
|
||||
def require_idle(readiness: dict) -> None:
|
||||
if readiness.get("schema_version") != "missioncore.observatory-claim-readiness/v1":
|
||||
raise ValueError("backend v3 readiness is unavailable")
|
||||
if readiness["open_live_lease_count"] or any(
|
||||
count
|
||||
for state, count in readiness["recorded_jobs_by_state"].items()
|
||||
if state not in {"failed", "succeeded", "cancelled"}
|
||||
):
|
||||
raise ValueError("queue is not idle; migration must not interrupt work")
|
||||
if not readiness["protocols"][1]["grant_capacity_available"]:
|
||||
raise ValueError("v3 grant capacity is unavailable")
|
||||
|
||||
|
||||
def validate_target(name: str, inspection: dict) -> None:
|
||||
config, host = inspection["Config"], inspection["HostConfig"]
|
||||
if inspection["Name"] != "/" + name or inspection["Image"] != "sha256:" + TARGETS[name]:
|
||||
raise ValueError("target container or parent image changed")
|
||||
if not inspection["State"]["Running"] or not host["ReadonlyRootfs"]:
|
||||
raise ValueError("expected durable read-only agent is not running")
|
||||
if host["NetworkMode"] != "bridge" or host.get("DeviceRequests") or host.get("Privileged"):
|
||||
raise ValueError("unexpected network/GPU/privileged agent configuration")
|
||||
labels = config["Labels"]
|
||||
if (
|
||||
labels.get("com.nodedc.product") != "mission-core"
|
||||
or labels.get("com.nodedc.authority") != "observation-only"
|
||||
):
|
||||
raise ValueError("agent ownership or authority changed")
|
||||
for entry in config["Env"]:
|
||||
key = entry.split("=", 1)[0].upper()
|
||||
sensitive = any(word in key for word in ("TOKEN", "PASSWORD", "SECRET"))
|
||||
if sensitive and not key.endswith("_FILE"):
|
||||
raise ValueError("inline credential is not allowed in a transport declaration")
|
||||
|
||||
|
||||
def plan(engine: Engine) -> dict:
|
||||
targets = []
|
||||
for name, parent in TARGETS.items():
|
||||
inspection = engine.inspect(name)
|
||||
validate_target(name, inspection)
|
||||
readiness = engine.execute_json(name, READINESS)
|
||||
require_idle(readiness)
|
||||
targets.append(
|
||||
{
|
||||
"name": name,
|
||||
"container_id": inspection["Id"],
|
||||
"parent_image_sha256": parent,
|
||||
"create_body_sha256": sha(
|
||||
canonical(
|
||||
{
|
||||
"Config": inspection["Config"],
|
||||
"HostConfig": inspection["HostConfig"],
|
||||
}
|
||||
)
|
||||
),
|
||||
}
|
||||
)
|
||||
return {
|
||||
"schema_version": "missioncore.claim-transport-migration-plan/v1",
|
||||
"targets": targets,
|
||||
"before_source_sha256": BEFORE_SHA,
|
||||
"script_sha256": sha(Path(__file__).read_bytes()),
|
||||
"model_jobs": 0,
|
||||
}
|
||||
|
||||
|
||||
def save(path: Path, value: object) -> None:
|
||||
# New immutable evidence only. An existing plan/release is never overwritten.
|
||||
with path.open("xb") as output:
|
||||
output.write(canonical(value) + b"\n")
|
||||
|
||||
|
||||
def replacement_body(inspection: dict, image: dict) -> dict:
|
||||
body = copy.deepcopy(inspection["Config"])
|
||||
body["Image"] = image["image"]
|
||||
body["Labels"].update(
|
||||
{
|
||||
PARENT_LABEL: image["parent_image_sha256"],
|
||||
"com.nodedc.claim-protocol": "3",
|
||||
"com.nodedc.claim-transport-source.sha256": image["source_sha256"],
|
||||
}
|
||||
)
|
||||
body["HostConfig"] = copy.deepcopy(inspection["HostConfig"])
|
||||
return body
|
||||
|
||||
|
||||
def build_transport(engine: Engine, name: str, parent: str, script_sha: str) -> dict:
|
||||
temporary = engine.request(
|
||||
"POST",
|
||||
"/containers/create?"
|
||||
+ urlencode(
|
||||
{
|
||||
"name": name + "-claim-v3-layer",
|
||||
}
|
||||
),
|
||||
{"Image": "sha256:" + parent, "HostConfig": {"NetworkMode": "none"}},
|
||||
)["Id"]
|
||||
try:
|
||||
archive = engine.request(
|
||||
"GET", f"/containers/{temporary}/archive?" + urlencode({"path": SOURCE}), raw=True
|
||||
)
|
||||
with tarfile.open(fileobj=io.BytesIO(archive)) as source_tar:
|
||||
members = source_tar.getmembers()
|
||||
if len(members) != 1 or not members[0].isfile() or members[0].size > 512_000:
|
||||
raise ValueError("unexpected transport source archive")
|
||||
member = members[0]
|
||||
original = source_tar.extractfile(member).read()
|
||||
replacement = patch_source(original)
|
||||
member.name = Path(SOURCE).name
|
||||
member.size = len(replacement)
|
||||
member.pax_headers = {}
|
||||
data = io.BytesIO()
|
||||
with tarfile.open(fileobj=data, mode="w") as output:
|
||||
output.addfile(member, io.BytesIO(replacement))
|
||||
engine.request(
|
||||
"PUT",
|
||||
f"/containers/{temporary}/archive?"
|
||||
+ urlencode(
|
||||
{
|
||||
"path": str(Path(SOURCE).parent),
|
||||
}
|
||||
),
|
||||
data.getvalue(),
|
||||
)
|
||||
changes = engine.request("GET", f"/containers/{temporary}/changes")
|
||||
allowed = {str(path) for path in Path(SOURCE).parents} | {SOURCE}
|
||||
if not changes or any(item["Kind"] != 0 or item["Path"] not in allowed for item in changes):
|
||||
raise ValueError("transport layer changed files outside the reviewed source")
|
||||
original_image = engine.request("GET", f"/images/sha256:{parent}/json")
|
||||
config = copy.deepcopy(original_image["Config"])
|
||||
config.setdefault("Labels", {}).update(
|
||||
{
|
||||
PARENT_LABEL: parent,
|
||||
"com.nodedc.claim-protocol": "3",
|
||||
"com.nodedc.claim-transport-source.sha256": sha(replacement),
|
||||
"com.nodedc.claim-migration-script.sha256": script_sha,
|
||||
}
|
||||
)
|
||||
image = engine.request(
|
||||
"POST",
|
||||
"/commit?"
|
||||
+ urlencode(
|
||||
{
|
||||
"container": temporary,
|
||||
"repo": name + "-claim-transport",
|
||||
"tag": "v3",
|
||||
}
|
||||
),
|
||||
config,
|
||||
)["Id"]
|
||||
actual = engine.request("GET", f"/images/{image}/json")
|
||||
if actual["RootFS"]["Layers"][:-1] != original_image["RootFS"]["Layers"]:
|
||||
raise ValueError("transport image did not retain the exact parent layers")
|
||||
return {
|
||||
"image": image,
|
||||
"parent_image_sha256": parent,
|
||||
"source_sha256": sha(replacement),
|
||||
"filesystem_changes": changes,
|
||||
}
|
||||
finally:
|
||||
engine.request("DELETE", f"/containers/{temporary}")
|
||||
|
||||
|
||||
def apply(engine: Engine, expected_plan_sha: str, evidence: Path) -> dict:
|
||||
started = time.monotonic_ns()
|
||||
proposal = plan(engine)
|
||||
if sha(canonical(proposal)) != expected_plan_sha:
|
||||
raise ValueError("migration plan changed")
|
||||
evidence.mkdir(parents=False, exist_ok=False)
|
||||
save(evidence / "plan.json", proposal)
|
||||
results = []
|
||||
for target in proposal["targets"]:
|
||||
name = target["name"]
|
||||
inspection = engine.inspect(name)
|
||||
validate_target(name, inspection)
|
||||
create_sha = sha(
|
||||
canonical(
|
||||
{
|
||||
"Config": inspection["Config"],
|
||||
"HostConfig": inspection["HostConfig"],
|
||||
}
|
||||
)
|
||||
)
|
||||
if inspection["Id"] != target["container_id"] or create_sha != target["create_body_sha256"]:
|
||||
raise ValueError("agent changed after plan")
|
||||
require_idle(engine.execute_json(name, READINESS))
|
||||
image = build_transport(
|
||||
engine, name, target["parent_image_sha256"], proposal["script_sha256"]
|
||||
)
|
||||
body = replacement_body(inspection, image)
|
||||
# Reusing the old hostname is intentional; paths, resources, argv and env stay exact.
|
||||
backup = name + "-v2-rollback-" + inspection["Id"][:12]
|
||||
declaration = {
|
||||
"name": name,
|
||||
"create_body": body,
|
||||
"rollback_name": backup,
|
||||
"rollback_container_id": inspection["Id"],
|
||||
"rollback_restart_policy": inspection["HostConfig"]["RestartPolicy"],
|
||||
"transport": image,
|
||||
}
|
||||
save(evidence / (name + "-declaration.json"), declaration)
|
||||
require_idle(engine.execute_json(name, READINESS))
|
||||
engine.request("POST", f"/containers/{inspection['Id']}/stop?t=15")
|
||||
engine.request(
|
||||
"POST", f"/containers/{inspection['Id']}/update", {"RestartPolicy": {"Name": "no"}}
|
||||
)
|
||||
engine.request(
|
||||
"POST", f"/containers/{inspection['Id']}/rename?" + urlencode({"name": backup})
|
||||
)
|
||||
created = engine.request("POST", "/containers/create?" + urlencode({"name": name}), body)[
|
||||
"Id"
|
||||
]
|
||||
engine.request("POST", f"/containers/{created}/start")
|
||||
# Never automatically delete/roll back a started agent: new operator work may arrive.
|
||||
# On failure the exact predecessor and create declaration remain for reconciliation.
|
||||
time.sleep(3)
|
||||
after = engine.inspect(name)
|
||||
if not after["State"]["Running"] or after["RestartCount"] != 0:
|
||||
raise RuntimeError("replacement agent needs reconciliation; predecessor retained")
|
||||
before_probe = engine.execute_json(name, READINESS)
|
||||
require_idle(before_probe)
|
||||
probe = engine.execute_json(name, PROBE)
|
||||
after_probe = engine.execute_json(name, READINESS)
|
||||
if before_probe != after_probe:
|
||||
raise RuntimeError("queue state changed during non-binding control probe")
|
||||
result = {
|
||||
"name": name,
|
||||
"container_id": created,
|
||||
"transport": image,
|
||||
"probe": probe,
|
||||
"readiness": after_probe,
|
||||
"rollback_name": backup,
|
||||
}
|
||||
save(evidence / (name + "-acceptance.json"), result)
|
||||
results.append(result)
|
||||
receipt = {
|
||||
"schema_version": "missioncore.claim-transport-migration/v1",
|
||||
"utc": datetime.now(UTC).isoformat(),
|
||||
"monotonic_start_ns": started,
|
||||
"monotonic_end_ns": time.monotonic_ns(),
|
||||
"plan_sha256": expected_plan_sha,
|
||||
"agents": results,
|
||||
"compute_packages_changed": False,
|
||||
"model_jobs": 0,
|
||||
}
|
||||
save(evidence / "receipt.json", receipt)
|
||||
return receipt
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--apply-plan-sha256")
|
||||
parser.add_argument("--evidence", type=Path, default=Path("/migration/release"))
|
||||
arguments = parser.parse_args()
|
||||
engine = Engine()
|
||||
if arguments.apply_plan_sha256:
|
||||
result = apply(engine, arguments.apply_plan_sha256, arguments.evidence)
|
||||
else:
|
||||
result = plan(engine)
|
||||
result = {"plan": result, "plan_sha256": sha(canonical(result))}
|
||||
print(json.dumps(result, sort_keys=True))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user