SILVER/NODEDC_1C
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Закрепить целевой AGENT-прогон hot value-flow handoff

Browse Source
This commit is contained in:
dctouch
2026-05-22 23:00:37 +03:00
parent e7603a9d29
commit 50d938b8f1
14 changed files with 636 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/domain_case_loop.py
+13
View File
@@ -2343,12 +2343,25 @@ def build_scenario_step_state(
),
"mcp_discovery_route_candidate_next_action": debug.get("mcp_discovery_route_candidate_next_action"),
"mcp_discovery_response_applied": debug.get("mcp_discovery_response_applied"),
"mcp_discovery_hot_runtime_wired": debug.get("mcp_discovery_hot_runtime_wired"),
"mcp_discovery_selected_chain_id": debug.get("mcp_discovery_selected_chain_id"),
"mcp_discovery_execution_handoff_status": debug.get("mcp_discovery_execution_handoff_status"),
"mcp_discovery_execution_handoff_allowed_hot_chain": debug.get(
"mcp_discovery_execution_handoff_allowed_hot_chain"
),
"mcp_discovery_execution_handoff_can_use_guarded_response": debug.get(
"mcp_discovery_execution_handoff_can_use_guarded_response"
),
"mcp_discovery_response_candidate_status": (
debug.get("mcp_discovery_response_candidate_v1", {}).get("candidate_status")
if isinstance(debug.get("mcp_discovery_response_candidate_v1"), dict)
else None
),
"mcp_discovery_response_candidate_hot_runtime_wired": (
debug.get("mcp_discovery_response_candidate_v1", {}).get("hot_runtime_wired")
if isinstance(debug.get("mcp_discovery_response_candidate_v1"), dict)
else None
),
"mcp_discovery_response_reply_type": (
debug.get("mcp_discovery_response_candidate_v1", {}).get("reply_type")
if isinstance(debug.get("mcp_discovery_response_candidate_v1"), dict)
scripts/domain_truth_harness.py
+185
View File
@@ -28,6 +28,13 @@ TECHNICAL_QUESTION_FIELDS = (
"expected_catalog_alignment_status",
"expected_catalog_chain_top_match",
"expected_catalog_selected_matches_top",
"expected_mcp_discovery_response_applied",
"expected_mcp_discovery_selected_chain_id",
"expected_mcp_discovery_response_candidate_status",
"expected_mcp_discovery_candidate_hot_runtime_wired",
"expected_mcp_discovery_hot_runtime_wired",
"expected_mcp_discovery_execution_handoff_status",
"expected_mcp_discovery_execution_handoff_can_use_guarded_response",
"expected_route_candidate_status",
"expected_route_candidate_executable_now",
"expected_route_candidate_missing_axes",
@@ -103,6 +110,27 @@ def normalize_step_spec(index: int, raw_step: Any) -> dict[str, Any]:
str(step.get("expected_catalog_chain_top_match") or "").strip() or None
)
normalized_step["expected_catalog_selected_matches_top"] = step.get("expected_catalog_selected_matches_top")
normalized_step["expected_mcp_discovery_response_applied"] = step.get(
"expected_mcp_discovery_response_applied"
)
normalized_step["expected_mcp_discovery_selected_chain_id"] = (
str(step.get("expected_mcp_discovery_selected_chain_id") or "").strip() or None
)
normalized_step["expected_mcp_discovery_response_candidate_status"] = (
str(step.get("expected_mcp_discovery_response_candidate_status") or "").strip() or None
)
normalized_step["expected_mcp_discovery_candidate_hot_runtime_wired"] = step.get(
"expected_mcp_discovery_candidate_hot_runtime_wired"
)
normalized_step["expected_mcp_discovery_hot_runtime_wired"] = step.get(
"expected_mcp_discovery_hot_runtime_wired"
)
normalized_step["expected_mcp_discovery_execution_handoff_status"] = (
str(step.get("expected_mcp_discovery_execution_handoff_status") or "").strip() or None
)
normalized_step["expected_mcp_discovery_execution_handoff_can_use_guarded_response"] = step.get(
"expected_mcp_discovery_execution_handoff_can_use_guarded_response"
)
normalized_step["expected_route_candidate_status"] = (
str(step.get("expected_route_candidate_status") or "").strip() or None
)
@@ -486,6 +514,13 @@ def evaluate_truth_step(
capability_id = str(step_state.get("capability_id") or "").strip()
catalog_alignment_status = str(step_state.get("mcp_discovery_catalog_chain_alignment_status") or "").strip()
catalog_chain_top_match = str(step_state.get("mcp_discovery_catalog_chain_top_match") or "").strip()
mcp_discovery_selected_chain_id = str(step_state.get("mcp_discovery_selected_chain_id") or "").strip()
mcp_discovery_response_candidate_status = str(
step_state.get("mcp_discovery_response_candidate_status") or ""
).strip()
mcp_discovery_execution_handoff_status = str(
step_state.get("mcp_discovery_execution_handoff_status") or ""
).strip()
route_candidate_status = str(step_state.get("mcp_discovery_route_candidate_status") or "").strip()
limited_reason_category = str(step_state.get("limited_reason_category") or "").strip()
extracted_filters = (
@@ -569,6 +604,156 @@ def evaluate_truth_step(
expected=expected_catalog_selected_matches_top,
)
expected_mcp_discovery_response_applied = normalize_optional_bool(
resolve_nested_placeholders(
step.get("expected_mcp_discovery_response_applied"),
step_results,
bindings,
runtime_bindings,
)
)
if expected_mcp_discovery_response_applied is not None:
actual_mcp_discovery_response_applied = step_state.get("mcp_discovery_response_applied") is True
if actual_mcp_discovery_response_applied != expected_mcp_discovery_response_applied:
append_finding(
findings,
step,
"wrong_mcp_discovery_response_applied",
"MCP discovery response replacement flag does not match the expected hot handoff behavior.",
actual=actual_mcp_discovery_response_applied,
expected=expected_mcp_discovery_response_applied,
)
expected_mcp_discovery_selected_chain_id = str(
resolve_nested_placeholders(
step.get("expected_mcp_discovery_selected_chain_id"),
step_results,
bindings,
runtime_bindings,
)
or ""
).strip()
if (
expected_mcp_discovery_selected_chain_id
and mcp_discovery_selected_chain_id != expected_mcp_discovery_selected_chain_id
):
append_finding(
findings,
step,
"wrong_mcp_discovery_selected_chain_id",
"MCP discovery selected chain does not match the expected autonomy chain for this step.",
actual=mcp_discovery_selected_chain_id or None,
expected=expected_mcp_discovery_selected_chain_id,
)
expected_mcp_discovery_response_candidate_status = str(
resolve_nested_placeholders(
step.get("expected_mcp_discovery_response_candidate_status"),
step_results,
bindings,
runtime_bindings,
)
or ""
).strip()
if (
expected_mcp_discovery_response_candidate_status
and mcp_discovery_response_candidate_status != expected_mcp_discovery_response_candidate_status
):
append_finding(
findings,
step,
"wrong_mcp_discovery_response_candidate_status",
"MCP discovery response candidate status does not match the expected guarded response readiness.",
actual=mcp_discovery_response_candidate_status or None,
expected=expected_mcp_discovery_response_candidate_status,
)
expected_mcp_discovery_candidate_hot_runtime_wired = normalize_optional_bool(
resolve_nested_placeholders(
step.get("expected_mcp_discovery_candidate_hot_runtime_wired"),
step_results,
bindings,
runtime_bindings,
)
)
if expected_mcp_discovery_candidate_hot_runtime_wired is not None:
actual_candidate_hot_runtime_wired = (
step_state.get("mcp_discovery_response_candidate_hot_runtime_wired") is True
)
if actual_candidate_hot_runtime_wired != expected_mcp_discovery_candidate_hot_runtime_wired:
append_finding(
findings,
step,
"wrong_mcp_discovery_candidate_hot_runtime_wired",
"MCP discovery response candidate hot-runtime flag does not match the expected guarded handoff.",
actual=actual_candidate_hot_runtime_wired,
expected=expected_mcp_discovery_candidate_hot_runtime_wired,
)
expected_mcp_discovery_hot_runtime_wired = normalize_optional_bool(
resolve_nested_placeholders(
step.get("expected_mcp_discovery_hot_runtime_wired"),
step_results,
bindings,
runtime_bindings,
)
)
if expected_mcp_discovery_hot_runtime_wired is not None:
actual_hot_runtime_wired = step_state.get("mcp_discovery_hot_runtime_wired") is True
if actual_hot_runtime_wired != expected_mcp_discovery_hot_runtime_wired:
append_finding(
findings,
step,
"wrong_mcp_discovery_hot_runtime_wired",
"Top-level MCP discovery hot-runtime flag does not match the expected guarded handoff.",
actual=actual_hot_runtime_wired,
expected=expected_mcp_discovery_hot_runtime_wired,
)
expected_mcp_discovery_execution_handoff_status = str(
resolve_nested_placeholders(
step.get("expected_mcp_discovery_execution_handoff_status"),
step_results,
bindings,
runtime_bindings,
)
or ""
).strip()
if (
expected_mcp_discovery_execution_handoff_status
and mcp_discovery_execution_handoff_status != expected_mcp_discovery_execution_handoff_status
):
append_finding(
findings,
step,
"wrong_mcp_discovery_execution_handoff_status",
"MCP discovery execution handoff status does not match the expected guarded response status.",
actual=mcp_discovery_execution_handoff_status or None,
expected=expected_mcp_discovery_execution_handoff_status,
)
expected_mcp_discovery_execution_handoff_can_use_guarded_response = normalize_optional_bool(
resolve_nested_placeholders(
step.get("expected_mcp_discovery_execution_handoff_can_use_guarded_response"),
step_results,
bindings,
runtime_bindings,
)
)
if expected_mcp_discovery_execution_handoff_can_use_guarded_response is not None:
actual_can_use_guarded_response = (
step_state.get("mcp_discovery_execution_handoff_can_use_guarded_response") is True
)
if actual_can_use_guarded_response != expected_mcp_discovery_execution_handoff_can_use_guarded_response:
append_finding(
findings,
step,
"wrong_mcp_discovery_execution_handoff_guarded_response",
"MCP discovery execution handoff guarded-response flag does not match the expected hot path.",
actual=actual_can_use_guarded_response,
expected=expected_mcp_discovery_execution_handoff_can_use_guarded_response,
)
expected_route_candidate_status = str(
resolve_nested_placeholders(
step.get("expected_route_candidate_status"),