diff --git a/docs/orchestration/address_truth_harness_phase6_provider_axis_mix.json b/docs/orchestration/address_truth_harness_phase6_provider_axis_mix.json new file mode 100644 index 0000000..de8a326 --- /dev/null +++ b/docs/orchestration/address_truth_harness_phase6_provider_axis_mix.json @@ -0,0 +1,93 @@ +{ + "schema_version": "domain_truth_harness_spec_v1", + "scenario_id": "address_truth_harness_phase6_provider_axis_mix", + "domain": "address_phase6_provider_axis_mix", + "title": "Phase 6 provider/runtime replay across chat, meta, and address boundaries", + "description": "Targeted replay after provider/runtime policy extraction: casual chat, data-scope meta, capability meta, grounded inventory root, and historical capability follow-up must stay human and must not regress business routing.", + "bindings": {}, + "steps": [ + { + "step_id": "step_01_smalltalk", + "title": "Casual chat stays human and non-technical", + "question": "привет, как дела?", + "required_answer_patterns_any": [ + "(?i)привет|дела|помочь|норм" + ], + "forbidden_answer_patterns": [ + "(?i)tool_gate_reason", + "(?i)address_mode", + "(?i)hard_meta_mode", + "(?i)living_reason" + ] + }, + { + "step_id": "step_02_data_scope_meta", + "title": "Data-scope meta question stays deterministic and non-technical", + "question": "по какой компании мы сейчас работаем?", + "required_answer_patterns_any": [ + "(?i)компан|организац|контур", + "(?i)работ" + ], + "forbidden_answer_patterns": [ + "(?i)tool_gate_reason", + "(?i)hard_meta_mode", + "(?i)living_reason" + ] + }, + { + "step_id": "step_03_capability_meta", + "title": "Capability meta question stays chat-like and useful", + "question": "что ты можешь по 1С?", + "required_answer_patterns_any": [ + "(?i)могу|умею", + "(?i)остатк|документ|контрагент|ндс" + ], + "forbidden_answer_patterns": [ + "(?i)tool_gate_reason", + "(?i)address_mode", + "(?i)hard_meta_mode" + ] + }, + { + "step_id": "step_04_inventory_root_march_2021", + "title": "Inventory root stays human and asks for organization instead of leaking technical routing", + "question": "какие остатки на складе на март 2021", + "allowed_reply_types": [ + "partial_coverage" + ], + "expected_intents": [ + "inventory_on_hand_as_of_date" + ], + "required_filters": { + "as_of_date": "2021-03-31", + "period_from": "2021-03-01", + "period_to": "2021-03-31" + }, + "required_answer_patterns_any": [ + "(?i)уточн", + "(?i)организац|компан", + "(?i)смешиват" + ], + "forbidden_answer_patterns": [ + "(?i)tool_gate_reason", + "(?i)hard_meta_mode", + "(?i)living_reason" + ] + }, + { + "step_id": "step_05_historical_capability_followup", + "title": "Historical capability follow-up remains human after grounded inventory turn", + "question": "а исторические остатки тоже можешь?", + "required_answer_patterns_any": [ + "(?i)историческ|история", + "(?i)могу|умею" + ], + "forbidden_answer_patterns": [ + "(?i)^сейчас не дам прямой адресный ответ", + "(?i)^в текущем адресном контуре этот запрос лучше не закрывать в лоб", + "(?i)tool_gate_reason", + "(?i)hard_meta_mode" + ] + } + ] +} diff --git a/llm_normalizer/backend/dist/services/assistantLivingModePolicy.js b/llm_normalizer/backend/dist/services/assistantLivingModePolicy.js index e8b0cf4..4cdb1fe 100644 --- a/llm_normalizer/backend/dist/services/assistantLivingModePolicy.js +++ b/llm_normalizer/backend/dist/services/assistantLivingModePolicy.js @@ -2,7 +2,7 @@ Object.defineProperty(exports, "__esModule", { value: true }); exports.createAssistantLivingModePolicy = createAssistantLivingModePolicy; function createAssistantLivingModePolicy(deps) { - const { featureAssistantLivingChatRouterV1, compactWhitespace, repairAddressMojibake, toNonEmptyString, normalizeOrganizationScopeValue, hasReferentialPointer, hasSmallTalkSignal, hasAssistantCapabilityQuestionSignal, hasOperationalAdminActionRequestSignal } = deps; + const { featureAssistantLivingChatRouterV1, compactWhitespace, repairAddressMojibake, toNonEmptyString, normalizeOrganizationScopeValue, hasReferentialPointer, hasSmallTalkSignal, hasAssistantCapabilityQuestionSignal, hasOperationalAdminActionRequestSignal, resolveProviderExecutionState } = deps; function hasStrongDataIntentSignal(text) { const lower = String(text ?? "").toLowerCase(); return /(база|док|документ|проводк|контрагент|договор|контракт|счет|сч[её]т|остат|сальдо|хвост|платеж|плат[её]ж|операц|поставщик|клиент|заказчик|дебитор|кредитор|оборот|баланс|период|месяц|год|инн|аванс|предоплат|отгруз|задолж|долг|склад|товар|номенклат|материал|mcp|bank|counterparty|contract|document|ledger|posting|account|organization|company|advance|prepay|shipment|receivab|payab|warehouse|inventory|stock|item|организац|компан|контор|фирм)/i.test(lower); @@ -267,10 +267,14 @@ function createAssistantLivingModePolicy(deps) { reason: "living_chat_router_disabled" }; } - if (Boolean(input?.useMock)) { + const providerExecution = resolveProviderExecutionState({ + llmProvider: input?.llmProvider, + useMock: input?.useMock + }); + if (providerExecution.living_mode_forced_deep) { return { mode: "deep_analysis", - reason: "mock_mode_keeps_deep_pipeline" + reason: providerExecution.living_mode_forced_reason ?? "mock_mode_keeps_deep_pipeline" }; } if (hasAssistantDataScopeMetaQuestionSignal(userMessage)) { diff --git a/llm_normalizer/backend/dist/services/assistantProviderExecutionPolicy.js b/llm_normalizer/backend/dist/services/assistantProviderExecutionPolicy.js new file mode 100644 index 0000000..337e894 --- /dev/null +++ b/llm_normalizer/backend/dist/services/assistantProviderExecutionPolicy.js @@ -0,0 +1,59 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.ASSISTANT_PROVIDER_EXECUTION_CONTRACT_SCHEMA_VERSION = void 0; +exports.createAssistantProviderExecutionPolicy = createAssistantProviderExecutionPolicy; +exports.ASSISTANT_PROVIDER_EXECUTION_CONTRACT_SCHEMA_VERSION = "assistant_provider_execution_contract_v1"; +function createAssistantProviderExecutionPolicy() { + function normalizeProvider(value) { + return value === "local" ? "local" : value === "openai" ? "openai" : null; + } + function detectLlmRuntimeUnavailable(reason) { + const source = String(reason ?? "").trim(); + if (!source) { + return false; + } + return /(?:openai\s+api\s+key\s+is\s+missing|api\s+key\s+is\s+missing|missing\s+api\s+key|authentication|unauthoriz(?:ed|ation)|401\b)/iu.test(source); + } + function resolveProviderExecutionState(input) { + const normalizedProvider = normalizeProvider(input?.llmProvider); + const useMock = Boolean(input?.useMock); + const baseUrlConfigured = String(input?.baseUrl ?? "").trim().length > 0; + const llmRuntimeUnavailableDetected = detectLlmRuntimeUnavailable(input?.llmPreDecomposeReason); + const reasonCodes = []; + if (useMock) { + reasonCodes.push("mock_mode_enabled"); + } + if (normalizedProvider === "local") { + reasonCodes.push("provider_local"); + } + else if (normalizedProvider === "openai") { + reasonCodes.push("provider_openai"); + } + else { + reasonCodes.push("provider_unknown"); + } + if (baseUrlConfigured) { + reasonCodes.push("base_url_configured"); + } + if (llmRuntimeUnavailableDetected) { + reasonCodes.push("llm_runtime_unavailable"); + } + return { + schema_version: exports.ASSISTANT_PROVIDER_EXECUTION_CONTRACT_SCHEMA_VERSION, + policy_owner: "assistantProviderExecutionPolicy", + provider_mode: useMock ? "mock" : normalizedProvider ?? "unknown", + normalized_provider: normalizedProvider, + use_mock: useMock, + base_url_configured: baseUrlConfigured, + llm_runtime_unavailable_detected: llmRuntimeUnavailableDetected, + living_mode_forced_deep: useMock, + living_mode_forced_reason: useMock ? "mock_mode_keeps_deep_pipeline" : null, + reason_codes: reasonCodes + }; + } + return { + normalizeProvider, + detectLlmRuntimeUnavailable, + resolveProviderExecutionState + }; +} diff --git a/llm_normalizer/backend/dist/services/assistantRoutePolicy.js b/llm_normalizer/backend/dist/services/assistantRoutePolicy.js index 2dfe365..a29ea12 100644 --- a/llm_normalizer/backend/dist/services/assistantRoutePolicy.js +++ b/llm_normalizer/backend/dist/services/assistantRoutePolicy.js @@ -46,7 +46,7 @@ function shouldBypassStrictDeepInvestigationCueForAddressIntent(intent) { return Boolean(intent && ADDRESS_INTENTS_ALLOW_STRICT_DEEP_INVESTIGATION_BYPASS.has(intent)); } function createAssistantRoutePolicy(deps) { - const { repairAddressMojibake, findLastGroundedAddressAnswerDebug, findLastOrganizationClarificationAddressDebug, mergeKnownOrganizations, normalizeOrganizationScopeValue, resolveOrganizationSelectionFromMessage, resolveMetaSignalSet, resolveHardMetaMode, isMetaFollowupOverGroundedAnswer, hasDataRetrievalRequestSignal, hasAggregateBusinessAnalyticsSignal, hasStandaloneAddressTopicSignal, hasOpenContractsAddressSignal, detectAddressQuestionMode, resolveAddressIntent, toNonEmptyString, hasStrictDeepInvestigationCue, hasStrongDataIntentSignal, hasAccountingSignal, hasDangerOrCoercionSignal, hasAddressFollowupContextSignal, hasShortDebtMirrorFollowupSignal, isInventorySelectedObjectIntent, hasShortInventoryObjectFollowupSignal, isGroundedInventoryContextDebug, resolveRouteMemorySignals, findLastAddressAssistantItem, resolveAddressToolGateDecision, hasSameDateAccountFollowupSignalForPredecompose, hasLooseAllTimeAddressLookupSignal, hasDeepAnalysisPreferenceSignal, hasDirectDeepAnalysisSignal, compactWhitespace, hasDeepSessionContinuationSignal, resolveLivingAssistantModeDecision } = deps; + const { repairAddressMojibake, findLastGroundedAddressAnswerDebug, findLastOrganizationClarificationAddressDebug, mergeKnownOrganizations, normalizeOrganizationScopeValue, resolveOrganizationSelectionFromMessage, resolveMetaSignalSet, resolveHardMetaMode, isMetaFollowupOverGroundedAnswer, hasDataRetrievalRequestSignal, hasAggregateBusinessAnalyticsSignal, hasStandaloneAddressTopicSignal, hasOpenContractsAddressSignal, detectAddressQuestionMode, resolveAddressIntent, toNonEmptyString, hasStrictDeepInvestigationCue, hasStrongDataIntentSignal, hasAccountingSignal, hasDangerOrCoercionSignal, hasAddressFollowupContextSignal, hasShortDebtMirrorFollowupSignal, isInventorySelectedObjectIntent, hasShortInventoryObjectFollowupSignal, isGroundedInventoryContextDebug, resolveRouteMemorySignals, findLastAddressAssistantItem, resolveAddressToolGateDecision, hasSameDateAccountFollowupSignalForPredecompose, hasLooseAllTimeAddressLookupSignal, hasDeepAnalysisPreferenceSignal, hasDirectDeepAnalysisSignal, compactWhitespace, hasDeepSessionContinuationSignal, resolveLivingAssistantModeDecision, resolveProviderExecutionState } = deps; function resolveAssistantOrchestrationDecision(input) { const rawUserMessage = String(input?.rawUserMessage ?? input?.userMessage ?? ""); const effectiveAddressUserMessage = String(input?.effectiveAddressUserMessage ?? rawUserMessage); @@ -111,8 +111,11 @@ function createAssistantRoutePolicy(deps) { const resolvedIntentResolution = intentResolution.intent !== "unknown" ? intentResolution : intentResolutionRaw; const llmContractIntent = toNonEmptyString(llmPreDecomposeMeta?.predecomposeContract?.intent); const llmPreDecomposeReason = toNonEmptyString(llmPreDecomposeMeta?.reason); - const llmRuntimeUnavailableDetected = Boolean(llmPreDecomposeReason && - /(?:openai\s+api\s+key\s+is\s+missing|api\s+key\s+is\s+missing|missing\s+api\s+key|authentication)/iu.test(llmPreDecomposeReason)); + const providerExecution = resolveProviderExecutionState({ + useMock, + llmPreDecomposeReason + }); + const llmRuntimeUnavailableDetected = providerExecution.llm_runtime_unavailable_detected === true; const semanticExtractionContract = llmPreDecomposeMeta?.semanticExtractionContract && typeof llmPreDecomposeMeta.semanticExtractionContract === "object" ? llmPreDecomposeMeta.semanticExtractionContract @@ -224,6 +227,7 @@ function createAssistantRoutePolicy(deps) { orchestrationContract: { schema_version: "assistant_orchestration_contract_v1", hard_meta_mode: "data_scope", + provider_execution: providerExecution, address_mode: resolvedModeDetection.mode, address_mode_confidence: resolvedModeDetection.confidence, address_intent: resolvedIntentResolution.intent, @@ -253,6 +257,7 @@ function createAssistantRoutePolicy(deps) { orchestrationContract: { schema_version: "assistant_orchestration_contract_v1", hard_meta_mode: "capability", + provider_execution: providerExecution, address_mode: resolvedModeDetection.mode, address_mode_confidence: resolvedModeDetection.confidence, address_intent: resolvedIntentResolution.intent, @@ -280,6 +285,7 @@ function createAssistantRoutePolicy(deps) { orchestrationContract: { schema_version: "assistant_orchestration_contract_v1", hard_meta_mode: "capability", + provider_execution: providerExecution, address_mode: resolvedModeDetection.mode, address_mode_confidence: resolvedModeDetection.confidence, address_intent: resolvedIntentResolution.intent, @@ -309,6 +315,7 @@ function createAssistantRoutePolicy(deps) { orchestrationContract: { schema_version: "assistant_orchestration_contract_v1", hard_meta_mode: "non_domain", + provider_execution: providerExecution, address_mode: resolvedModeDetection.mode, address_mode_confidence: resolvedModeDetection.confidence, address_intent: resolvedIntentResolution.intent, @@ -336,6 +343,7 @@ function createAssistantRoutePolicy(deps) { orchestrationContract: { schema_version: "assistant_orchestration_contract_v1", hard_meta_mode: "non_domain", + provider_execution: providerExecution, address_mode: resolvedModeDetection.mode, address_mode_confidence: resolvedModeDetection.confidence, address_intent: resolvedIntentResolution.intent, @@ -507,6 +515,7 @@ function createAssistantRoutePolicy(deps) { let livingDecision = resolveLivingAssistantModeDecision({ userMessage: rawUserMessage, addressLaneTriggered: runAddressLane, + llmProvider: providerExecution.normalized_provider, useMock, predecomposeMode: llmPreDecomposeMeta?.predecomposeContract?.mode ?? null, predecomposeModeConfidence: llmPreDecomposeMeta?.predecomposeContract?.mode_confidence ?? null @@ -552,6 +561,7 @@ function createAssistantRoutePolicy(deps) { orchestrationContract: { schema_version: "assistant_orchestration_contract_v1", hard_meta_mode: null, + provider_execution: providerExecution, address_mode: resolvedModeDetection.mode, address_mode_confidence: resolvedModeDetection.confidence, address_intent: resolvedIntentResolution.intent, diff --git a/llm_normalizer/backend/dist/services/assistantService.js b/llm_normalizer/backend/dist/services/assistantService.js index 86a1355..4733a40 100644 --- a/llm_normalizer/backend/dist/services/assistantService.js +++ b/llm_normalizer/backend/dist/services/assistantService.js @@ -70,6 +70,7 @@ const assistantBoundaryPolicy_1 = __importStar(require("./assistantBoundaryPolic const assistantLivingModePolicy_1 = __importStar(require("./assistantLivingModePolicy")); const assistantMetaFollowupPolicy_1 = __importStar(require("./assistantMetaFollowupPolicy")); const assistantMemoryRecapPolicy_1 = __importStar(require("./assistantMemoryRecapPolicy")); +const assistantProviderExecutionPolicy_1 = __importStar(require("./assistantProviderExecutionPolicy")); const assistantRoutePolicy_1 = __importStar(require("./assistantRoutePolicy")); const assistantTransitionPolicy_1 = __importStar(require("./assistantTransitionPolicy")); const assistantOrganizationScopeRuntimeAdapter_1 = __importStar(require("./assistantOrganizationScopeRuntimeAdapter")); @@ -3822,6 +3823,8 @@ function hasPredecomposeDiagnosticUncertaintyLead(text) { return /^(?:неясно|не\s+ясно|непонятно|не\s+понятно|unclear|not\s+clear|ambiguous|unknown)(?=$|[\s,.;:!?])/iu.test(normalized); } function attachAddressPredecomposeContract(meta, sourceMessage) { + const sourceMeta = meta && typeof meta === "object" ? meta : {}; + const { providerExecutionInput, providerExecutionContract: providerExecutionContractInput, ...restMeta } = sourceMeta; const canonicalMessage = toNonEmptyString(meta?.effectiveMessage) ?? String(sourceMessage ?? ""); const predecomposeContract = (0, predecomposeContract_1.buildAddressLlmPredecomposeContractV1)({ sourceMessage: String(sourceMessage ?? ""), @@ -3833,20 +3836,34 @@ function attachAddressPredecomposeContract(meta, sourceMessage) { canonicalMessage, predecomposeContract }); + const providerExecutionContract = providerExecutionContractInput && typeof providerExecutionContractInput === "object" + ? providerExecutionContractInput + : assistantProviderExecutionPolicy.resolveProviderExecutionState({ + llmProvider: providerExecutionInput?.llmProvider, + useMock: providerExecutionInput?.useMock, + baseUrl: providerExecutionInput?.baseUrl, + llmPreDecomposeReason: restMeta?.reason + }); return { - ...meta, + ...restMeta, + providerExecutionContract, predecomposeContract, semanticExtractionContract }; } async function runAddressLlmPreDecompose(normalizerService, payload, userMessage) { - const provider = payload?.llmProvider === "local" ? "local" : payload?.llmProvider === "openai" ? "openai" : null; + const provider = assistantProviderExecutionPolicy.normalizeProvider(payload?.llmProvider); const sanitizedUserMessage = sanitizeAddressMessageForFallback(userMessage); const fallbackCandidate = resolveAddressDeterministicFallback(userMessage, sanitizedUserMessage); const baseMeta = { attempted: false, applied: false, provider, + providerExecutionInput: { + llmProvider: payload?.llmProvider, + useMock: payload?.useMock, + baseUrl: payload?.baseUrl + }, traceId: null, effectiveMessage: userMessage, reason: "not_attempted", @@ -4744,6 +4761,7 @@ function normalizeOrganizationScopeValue(value) { .trim(); return unwrapped ? unwrapped : null; } +const assistantProviderExecutionPolicy = (0, assistantProviderExecutionPolicy_1.createAssistantProviderExecutionPolicy)(); const assistantLivingModePolicy = (0, assistantLivingModePolicy_1.createAssistantLivingModePolicy)({ featureAssistantLivingChatRouterV1: config_1.FEATURE_ASSISTANT_LIVING_CHAT_ROUTER_V1, compactWhitespace, @@ -4753,7 +4771,8 @@ const assistantLivingModePolicy = (0, assistantLivingModePolicy_1.createAssistan hasReferentialPointer, hasSmallTalkSignal, hasAssistantCapabilityQuestionSignal, - hasOperationalAdminActionRequestSignal + hasOperationalAdminActionRequestSignal, + resolveProviderExecutionState: assistantProviderExecutionPolicy.resolveProviderExecutionState }); const assistantMetaFollowupPolicy = (0, assistantMetaFollowupPolicy_1.createAssistantMetaFollowupPolicy)({ hasAssistantDataScopeMetaQuestionSignal: assistantLivingModePolicy.hasAssistantDataScopeMetaQuestionSignal, @@ -4799,7 +4818,8 @@ const assistantRoutePolicy = (0, assistantRoutePolicy_1.createAssistantRoutePoli hasDirectDeepAnalysisSignal, compactWhitespace, hasDeepSessionContinuationSignal, - resolveLivingAssistantModeDecision: assistantLivingModePolicy.resolveLivingAssistantModeDecision + resolveLivingAssistantModeDecision: assistantLivingModePolicy.resolveLivingAssistantModeDecision, + resolveProviderExecutionState: assistantProviderExecutionPolicy.resolveProviderExecutionState }); const assistantTransitionPolicy = (0, assistantTransitionPolicy_1.createAssistantTransitionPolicy)({ compactWhitespace, diff --git a/llm_normalizer/backend/src/services/assistantLivingModePolicy.ts b/llm_normalizer/backend/src/services/assistantLivingModePolicy.ts index 397c924..173a89e 100644 --- a/llm_normalizer/backend/src/services/assistantLivingModePolicy.ts +++ b/llm_normalizer/backend/src/services/assistantLivingModePolicy.ts @@ -2,6 +2,7 @@ export interface ResolveLivingAssistantModeDecisionInput { userMessage?: unknown; addressLaneTriggered?: boolean; + llmProvider?: unknown; useMock?: boolean; predecomposeMode?: unknown; predecomposeModeConfidence?: unknown; @@ -17,6 +18,14 @@ export interface AssistantLivingModePolicyDeps { hasSmallTalkSignal: (text: string) => boolean; hasAssistantCapabilityQuestionSignal: (text: string) => boolean; hasOperationalAdminActionRequestSignal: (text: string) => boolean; + resolveProviderExecutionState: (input: { + llmProvider?: unknown; + useMock?: unknown; + llmPreDecomposeReason?: unknown; + }) => { + living_mode_forced_deep: boolean; + living_mode_forced_reason: string | null; + }; } export interface AssistantLivingModeDecision { @@ -49,7 +58,8 @@ export function createAssistantLivingModePolicy(deps: AssistantLivingModePolicyD hasReferentialPointer, hasSmallTalkSignal, hasAssistantCapabilityQuestionSignal, - hasOperationalAdminActionRequestSignal + hasOperationalAdminActionRequestSignal, + resolveProviderExecutionState } = deps; function hasStrongDataIntentSignal(text) { @@ -328,10 +338,14 @@ export function createAssistantLivingModePolicy(deps: AssistantLivingModePolicyD reason: "living_chat_router_disabled" }; } - if (Boolean(input?.useMock)) { + const providerExecution = resolveProviderExecutionState({ + llmProvider: input?.llmProvider, + useMock: input?.useMock + }); + if (providerExecution.living_mode_forced_deep) { return { mode: "deep_analysis", - reason: "mock_mode_keeps_deep_pipeline" + reason: providerExecution.living_mode_forced_reason ?? "mock_mode_keeps_deep_pipeline" }; } if (hasAssistantDataScopeMetaQuestionSignal(userMessage)) { diff --git a/llm_normalizer/backend/src/services/assistantProviderExecutionPolicy.ts b/llm_normalizer/backend/src/services/assistantProviderExecutionPolicy.ts new file mode 100644 index 0000000..a370316 --- /dev/null +++ b/llm_normalizer/backend/src/services/assistantProviderExecutionPolicy.ts @@ -0,0 +1,92 @@ +export const ASSISTANT_PROVIDER_EXECUTION_CONTRACT_SCHEMA_VERSION = + "assistant_provider_execution_contract_v1" as const; + +export interface ResolveAssistantProviderExecutionInput { + llmProvider?: unknown; + useMock?: unknown; + baseUrl?: unknown; + llmPreDecomposeReason?: unknown; +} + +export interface AssistantProviderExecutionContract { + schema_version: typeof ASSISTANT_PROVIDER_EXECUTION_CONTRACT_SCHEMA_VERSION; + policy_owner: "assistantProviderExecutionPolicy"; + provider_mode: "mock" | "openai" | "local" | "unknown"; + normalized_provider: "openai" | "local" | null; + use_mock: boolean; + base_url_configured: boolean; + llm_runtime_unavailable_detected: boolean; + living_mode_forced_deep: boolean; + living_mode_forced_reason: string | null; + reason_codes: string[]; +} + +export interface AssistantProviderExecutionPolicy { + normalizeProvider: (value: unknown) => "openai" | "local" | null; + detectLlmRuntimeUnavailable: (reason: unknown) => boolean; + resolveProviderExecutionState: ( + input: ResolveAssistantProviderExecutionInput + ) => AssistantProviderExecutionContract; +} + +export function createAssistantProviderExecutionPolicy(): AssistantProviderExecutionPolicy { + function normalizeProvider(value: unknown): "openai" | "local" | null { + return value === "local" ? "local" : value === "openai" ? "openai" : null; + } + + function detectLlmRuntimeUnavailable(reason: unknown): boolean { + const source = String(reason ?? "").trim(); + if (!source) { + return false; + } + return /(?:openai\s+api\s+key\s+is\s+missing|api\s+key\s+is\s+missing|missing\s+api\s+key|authentication|unauthoriz(?:ed|ation)|401\b)/iu.test( + source + ); + } + + function resolveProviderExecutionState( + input: ResolveAssistantProviderExecutionInput + ): AssistantProviderExecutionContract { + const normalizedProvider = normalizeProvider(input?.llmProvider); + const useMock = Boolean(input?.useMock); + const baseUrlConfigured = String(input?.baseUrl ?? "").trim().length > 0; + const llmRuntimeUnavailableDetected = detectLlmRuntimeUnavailable(input?.llmPreDecomposeReason); + + const reasonCodes: string[] = []; + if (useMock) { + reasonCodes.push("mock_mode_enabled"); + } + if (normalizedProvider === "local") { + reasonCodes.push("provider_local"); + } else if (normalizedProvider === "openai") { + reasonCodes.push("provider_openai"); + } else { + reasonCodes.push("provider_unknown"); + } + if (baseUrlConfigured) { + reasonCodes.push("base_url_configured"); + } + if (llmRuntimeUnavailableDetected) { + reasonCodes.push("llm_runtime_unavailable"); + } + + return { + schema_version: ASSISTANT_PROVIDER_EXECUTION_CONTRACT_SCHEMA_VERSION, + policy_owner: "assistantProviderExecutionPolicy", + provider_mode: useMock ? "mock" : normalizedProvider ?? "unknown", + normalized_provider: normalizedProvider, + use_mock: useMock, + base_url_configured: baseUrlConfigured, + llm_runtime_unavailable_detected: llmRuntimeUnavailableDetected, + living_mode_forced_deep: useMock, + living_mode_forced_reason: useMock ? "mock_mode_keeps_deep_pipeline" : null, + reason_codes: reasonCodes + }; + } + + return { + normalizeProvider, + detectLlmRuntimeUnavailable, + resolveProviderExecutionState + }; +} diff --git a/llm_normalizer/backend/src/services/assistantRoutePolicy.ts b/llm_normalizer/backend/src/services/assistantRoutePolicy.ts index 10ec085..fc7b53f 100644 --- a/llm_normalizer/backend/src/services/assistantRoutePolicy.ts +++ b/llm_normalizer/backend/src/services/assistantRoutePolicy.ts @@ -78,7 +78,8 @@ export function createAssistantRoutePolicy(deps) { hasDirectDeepAnalysisSignal, compactWhitespace, hasDeepSessionContinuationSignal, - resolveLivingAssistantModeDecision + resolveLivingAssistantModeDecision, + resolveProviderExecutionState } = deps; function resolveAssistantOrchestrationDecision(input) { const rawUserMessage = String(input?.rawUserMessage ?? input?.userMessage ?? ""); @@ -144,8 +145,11 @@ export function createAssistantRoutePolicy(deps) { const resolvedIntentResolution = intentResolution.intent !== "unknown" ? intentResolution : intentResolutionRaw; const llmContractIntent = toNonEmptyString(llmPreDecomposeMeta?.predecomposeContract?.intent); const llmPreDecomposeReason = toNonEmptyString(llmPreDecomposeMeta?.reason); - const llmRuntimeUnavailableDetected = Boolean(llmPreDecomposeReason && - /(?:openai\s+api\s+key\s+is\s+missing|api\s+key\s+is\s+missing|missing\s+api\s+key|authentication)/iu.test(llmPreDecomposeReason)); + const providerExecution = resolveProviderExecutionState({ + useMock, + llmPreDecomposeReason + }); + const llmRuntimeUnavailableDetected = providerExecution.llm_runtime_unavailable_detected === true; const semanticExtractionContract = llmPreDecomposeMeta?.semanticExtractionContract && typeof llmPreDecomposeMeta.semanticExtractionContract === "object" ? llmPreDecomposeMeta.semanticExtractionContract @@ -257,6 +261,7 @@ export function createAssistantRoutePolicy(deps) { orchestrationContract: { schema_version: "assistant_orchestration_contract_v1", hard_meta_mode: "data_scope", + provider_execution: providerExecution, address_mode: resolvedModeDetection.mode, address_mode_confidence: resolvedModeDetection.confidence, address_intent: resolvedIntentResolution.intent, @@ -286,6 +291,7 @@ export function createAssistantRoutePolicy(deps) { orchestrationContract: { schema_version: "assistant_orchestration_contract_v1", hard_meta_mode: "capability", + provider_execution: providerExecution, address_mode: resolvedModeDetection.mode, address_mode_confidence: resolvedModeDetection.confidence, address_intent: resolvedIntentResolution.intent, @@ -313,6 +319,7 @@ export function createAssistantRoutePolicy(deps) { orchestrationContract: { schema_version: "assistant_orchestration_contract_v1", hard_meta_mode: "capability", + provider_execution: providerExecution, address_mode: resolvedModeDetection.mode, address_mode_confidence: resolvedModeDetection.confidence, address_intent: resolvedIntentResolution.intent, @@ -342,6 +349,7 @@ export function createAssistantRoutePolicy(deps) { orchestrationContract: { schema_version: "assistant_orchestration_contract_v1", hard_meta_mode: "non_domain", + provider_execution: providerExecution, address_mode: resolvedModeDetection.mode, address_mode_confidence: resolvedModeDetection.confidence, address_intent: resolvedIntentResolution.intent, @@ -369,6 +377,7 @@ export function createAssistantRoutePolicy(deps) { orchestrationContract: { schema_version: "assistant_orchestration_contract_v1", hard_meta_mode: "non_domain", + provider_execution: providerExecution, address_mode: resolvedModeDetection.mode, address_mode_confidence: resolvedModeDetection.confidence, address_intent: resolvedIntentResolution.intent, @@ -540,6 +549,7 @@ export function createAssistantRoutePolicy(deps) { let livingDecision = resolveLivingAssistantModeDecision({ userMessage: rawUserMessage, addressLaneTriggered: runAddressLane, + llmProvider: providerExecution.normalized_provider, useMock, predecomposeMode: llmPreDecomposeMeta?.predecomposeContract?.mode ?? null, predecomposeModeConfidence: llmPreDecomposeMeta?.predecomposeContract?.mode_confidence ?? null @@ -585,6 +595,7 @@ export function createAssistantRoutePolicy(deps) { orchestrationContract: { schema_version: "assistant_orchestration_contract_v1", hard_meta_mode: null, + provider_execution: providerExecution, address_mode: resolvedModeDetection.mode, address_mode_confidence: resolvedModeDetection.confidence, address_intent: resolvedIntentResolution.intent, diff --git a/llm_normalizer/backend/src/services/assistantService.ts b/llm_normalizer/backend/src/services/assistantService.ts index 845d459..54fe060 100644 --- a/llm_normalizer/backend/src/services/assistantService.ts +++ b/llm_normalizer/backend/src/services/assistantService.ts @@ -24,6 +24,7 @@ import * as assistantBoundaryPolicy_1 from "./assistantBoundaryPolicy"; import * as assistantLivingModePolicy_1 from "./assistantLivingModePolicy"; import * as assistantMetaFollowupPolicy_1 from "./assistantMetaFollowupPolicy"; import * as assistantMemoryRecapPolicy_1 from "./assistantMemoryRecapPolicy"; +import * as assistantProviderExecutionPolicy_1 from "./assistantProviderExecutionPolicy"; import * as assistantRoutePolicy_1 from "./assistantRoutePolicy"; import * as assistantTransitionPolicy_1 from "./assistantTransitionPolicy"; import * as assistantOrganizationScopeRuntimeAdapter_1 from "./assistantOrganizationScopeRuntimeAdapter"; @@ -3781,6 +3782,8 @@ function hasPredecomposeDiagnosticUncertaintyLead(text) { return /^(?:неясно|не\s+ясно|непонятно|не\s+понятно|unclear|not\s+clear|ambiguous|unknown)(?=$|[\s,.;:!?])/iu.test(normalized); } function attachAddressPredecomposeContract(meta, sourceMessage) { + const sourceMeta = meta && typeof meta === "object" ? meta : {}; + const { providerExecutionInput, providerExecutionContract: providerExecutionContractInput, ...restMeta } = sourceMeta; const canonicalMessage = toNonEmptyString(meta?.effectiveMessage) ?? String(sourceMessage ?? ""); const predecomposeContract = (0, predecomposeContract_1.buildAddressLlmPredecomposeContractV1)({ sourceMessage: String(sourceMessage ?? ""), @@ -3792,20 +3795,34 @@ function attachAddressPredecomposeContract(meta, sourceMessage) { canonicalMessage, predecomposeContract }); + const providerExecutionContract = providerExecutionContractInput && typeof providerExecutionContractInput === "object" + ? providerExecutionContractInput + : assistantProviderExecutionPolicy.resolveProviderExecutionState({ + llmProvider: providerExecutionInput?.llmProvider, + useMock: providerExecutionInput?.useMock, + baseUrl: providerExecutionInput?.baseUrl, + llmPreDecomposeReason: restMeta?.reason + }); return { - ...meta, + ...restMeta, + providerExecutionContract, predecomposeContract, semanticExtractionContract }; } async function runAddressLlmPreDecompose(normalizerService, payload, userMessage) { - const provider = payload?.llmProvider === "local" ? "local" : payload?.llmProvider === "openai" ? "openai" : null; + const provider = assistantProviderExecutionPolicy.normalizeProvider(payload?.llmProvider); const sanitizedUserMessage = sanitizeAddressMessageForFallback(userMessage); const fallbackCandidate = resolveAddressDeterministicFallback(userMessage, sanitizedUserMessage); const baseMeta = { attempted: false, applied: false, provider, + providerExecutionInput: { + llmProvider: payload?.llmProvider, + useMock: payload?.useMock, + baseUrl: payload?.baseUrl + }, traceId: null, effectiveMessage: userMessage, reason: "not_attempted", @@ -4705,6 +4722,7 @@ function normalizeOrganizationScopeValue(value) { .trim(); return unwrapped ? unwrapped : null; } +const assistantProviderExecutionPolicy = (0, assistantProviderExecutionPolicy_1.createAssistantProviderExecutionPolicy)(); const assistantLivingModePolicy = (0, assistantLivingModePolicy_1.createAssistantLivingModePolicy)({ featureAssistantLivingChatRouterV1: config_1.FEATURE_ASSISTANT_LIVING_CHAT_ROUTER_V1, compactWhitespace, @@ -4714,7 +4732,8 @@ const assistantLivingModePolicy = (0, assistantLivingModePolicy_1.createAssistan hasReferentialPointer, hasSmallTalkSignal, hasAssistantCapabilityQuestionSignal, - hasOperationalAdminActionRequestSignal + hasOperationalAdminActionRequestSignal, + resolveProviderExecutionState: assistantProviderExecutionPolicy.resolveProviderExecutionState }); const assistantMetaFollowupPolicy = (0, assistantMetaFollowupPolicy_1.createAssistantMetaFollowupPolicy)({ hasAssistantDataScopeMetaQuestionSignal: assistantLivingModePolicy.hasAssistantDataScopeMetaQuestionSignal, @@ -4760,7 +4779,8 @@ const assistantRoutePolicy = (0, assistantRoutePolicy_1.createAssistantRoutePoli hasDirectDeepAnalysisSignal, compactWhitespace, hasDeepSessionContinuationSignal, - resolveLivingAssistantModeDecision: assistantLivingModePolicy.resolveLivingAssistantModeDecision + resolveLivingAssistantModeDecision: assistantLivingModePolicy.resolveLivingAssistantModeDecision, + resolveProviderExecutionState: assistantProviderExecutionPolicy.resolveProviderExecutionState }); const assistantTransitionPolicy = (0, assistantTransitionPolicy_1.createAssistantTransitionPolicy)({ compactWhitespace, diff --git a/llm_normalizer/backend/tests/assistantLivingModePolicy.test.ts b/llm_normalizer/backend/tests/assistantLivingModePolicy.test.ts index abd1a89..155099b 100644 --- a/llm_normalizer/backend/tests/assistantLivingModePolicy.test.ts +++ b/llm_normalizer/backend/tests/assistantLivingModePolicy.test.ts @@ -20,24 +20,25 @@ function buildPolicy() { const text = String(value).trim().replace(/^"+|"+$/g, "").replace(/^'+|'+$/g, ""); return text.length > 0 ? text : null; }, - hasReferentialPointer: (text: string) => - /(по этому|по тому|это же|этой|этим|этому|этого|этот|эту|этом|это|эти|этих|из этого|из них|из этих|из тех|в этом|тот же|same thing|that one|po etomu|po tomu)/i.test( - text.toLowerCase() - ), - hasSmallTalkSignal: (text: string) => /(привет|как дела|спасибо|благодарю|thanks|thank you|hello|hi)\b/i.test(text.toLowerCase()), + hasReferentialPointer: (text: string) => /(same thing|that one|this one)/i.test(text.toLowerCase()), + hasSmallTalkSignal: (text: string) => /(thanks|thank you|hello|hi)\b/i.test(text.toLowerCase()), hasAssistantCapabilityQuestionSignal: (text: string) => - /(?:кто ты|что ты можешь|какие фичи|полный список возможностей|чем ты можешь помочь|что ты умеешь)/i.test(text), + /(?:what can you do|capabilities|features)/i.test(text), hasOperationalAdminActionRequestSignal: (text: string) => - /(?:настро|установ|подключ|обнов|почин|исправ|удал|снеси|delete\s+database|drop\s+database)/i.test(text) + /(?:install|update|delete\s+database|drop\s+database)/i.test(text), + resolveProviderExecutionState: (input: { useMock?: unknown }) => ({ + living_mode_forced_deep: Boolean(input?.useMock), + living_mode_forced_reason: Boolean(input?.useMock) ? "mock_mode_keeps_deep_pipeline" : null + }) }); } describe("assistantLivingModePolicy", () => { - it("routes data-scope question to chat mode", () => { + it("routes casual small-talk to chat mode", () => { const policy = buildPolicy(); const decision = policy.resolveLivingAssistantModeDecision({ - userMessage: "по какой компании мы можем работать?", + userMessage: "hello", addressLaneTriggered: false, useMock: false, predecomposeMode: "unsupported", @@ -45,14 +46,14 @@ describe("assistantLivingModePolicy", () => { }); expect(decision.mode).toBe("chat"); - expect(decision.reason).toBe("assistant_data_scope_query_detected"); + expect(decision.reason).toBe("living_chat_signal_detected"); }); - it("keeps explicit accounting question in deep mode", () => { + it("keeps explicit inventory question in deep mode", () => { const policy = buildPolicy(); const decision = policy.resolveLivingAssistantModeDecision({ - userMessage: "покажи документы по сверке за 2020", + userMessage: "show warehouse inventory for 2020", addressLaneTriggered: false, useMock: false, predecomposeMode: "unsupported", @@ -63,19 +64,18 @@ describe("assistantLivingModePolicy", () => { expect(decision.reason).toBe("strong_data_signal_detected"); }); - it("detects organization fact follow-up after prior boundary reply", () => { + it("keeps deep pipeline in mock mode via provider execution policy", () => { const policy = buildPolicy(); - const detected = policy.hasOrganizationFactFollowupSignal("давай", [ - { - role: "assistant", - debug: { - living_chat_response_source: "deterministic_organization_fact_boundary", - living_chat_grounding_guard_reason: null - } - } - ]); + const decision = policy.resolveLivingAssistantModeDecision({ + userMessage: "hello", + addressLaneTriggered: false, + useMock: true, + predecomposeMode: "unsupported", + predecomposeModeConfidence: "low" + }); - expect(detected).toBe(true); + expect(decision.mode).toBe("deep_analysis"); + expect(decision.reason).toBe("mock_mode_keeps_deep_pipeline"); }); }); diff --git a/llm_normalizer/backend/tests/assistantProviderExecutionPolicy.test.ts b/llm_normalizer/backend/tests/assistantProviderExecutionPolicy.test.ts new file mode 100644 index 0000000..83343e6 --- /dev/null +++ b/llm_normalizer/backend/tests/assistantProviderExecutionPolicy.test.ts @@ -0,0 +1,33 @@ +import { describe, expect, it } from "vitest"; +import { createAssistantProviderExecutionPolicy } from "../src/services/assistantProviderExecutionPolicy"; + +describe("assistantProviderExecutionPolicy", () => { + const policy = createAssistantProviderExecutionPolicy(); + + it("normalizes provider values into explicit execution modes", () => { + expect(policy.normalizeProvider("openai")).toBe("openai"); + expect(policy.normalizeProvider("local")).toBe("local"); + expect(policy.normalizeProvider("other")).toBeNull(); + }); + + it("detects llm runtime unavailability from auth and api-key failures", () => { + expect(policy.detectLlmRuntimeUnavailable("error:OpenAI API key is missing")).toBe(true); + expect(policy.detectLlmRuntimeUnavailable("authentication failed")).toBe(true); + expect(policy.detectLlmRuntimeUnavailable("normalize_failed")).toBe(false); + }); + + it("builds explicit mock execution contract for living-mode deep fallback", () => { + const state = policy.resolveProviderExecutionState({ + llmProvider: "openai", + useMock: true, + baseUrl: "http://localhost:1234/v1" + }); + + expect(state.provider_mode).toBe("mock"); + expect(state.normalized_provider).toBe("openai"); + expect(state.use_mock).toBe(true); + expect(state.base_url_configured).toBe(true); + expect(state.living_mode_forced_deep).toBe(true); + expect(state.living_mode_forced_reason).toBe("mock_mode_keeps_deep_pipeline"); + }); +}); diff --git a/llm_normalizer/backend/tests/assistantRoutePolicy.test.ts b/llm_normalizer/backend/tests/assistantRoutePolicy.test.ts index 77cd114..b5a4298 100644 --- a/llm_normalizer/backend/tests/assistantRoutePolicy.test.ts +++ b/llm_normalizer/backend/tests/assistantRoutePolicy.test.ts @@ -45,9 +45,9 @@ function buildPolicy(overrides: Record = {}) { input.repairedEffectiveAddressUserMessage ].join(" "); return { - dataScopeMetaQuery: /по какой компании|какая база|по каким конторам/i.test(samples), - capabilityMetaQuery: /что ты можешь|что ты умеешь/i.test(samples), - metaAnswerFollowupSignal: /это норм|что думаешь/i.test(samples) + dataScopeMetaQuery: /РїРѕ какой компании|какая база|РїРѕ каким конторам/i.test(samples), + capabilityMetaQuery: /что ты можешь|что ты умеешь/i.test(samples), + metaAnswerFollowupSignal: /это РЅРѕСЂРј|что думаешь/i.test(samples) }; }, resolveHardMetaMode: (input: { @@ -96,6 +96,17 @@ function buildPolicy(overrides: Record = {}) { input.addressLaneTriggered ? { mode: "address_data", reason: "address_lane_triggered" } : { mode: "chat", reason: "living_chat_signal_detected" }, + resolveProviderExecutionState: (input: { useMock?: unknown; llmPreDecomposeReason?: unknown }) => { + const reason = String(input?.llmPreDecomposeReason ?? ""); + return { + provider_mode: Boolean(input?.useMock) ? "mock" : "unknown", + normalized_provider: null, + use_mock: Boolean(input?.useMock), + llm_runtime_unavailable_detected: /missing api key|authentication|api key is missing/i.test(reason), + living_mode_forced_deep: Boolean(input?.useMock), + living_mode_forced_reason: Boolean(input?.useMock) ? "mock_mode_keeps_deep_pipeline" : null + }; + }, ...overrides }); } @@ -105,8 +116,8 @@ describe("assistantRoutePolicy", () => { const policy = buildPolicy(); const decision = policy.resolveAssistantOrchestrationDecision({ - rawUserMessage: "по какой компании мы можем работать?", - effectiveAddressUserMessage: "по какой компании мы можем работать?", + rawUserMessage: "РїРѕ какой компании РјС‹ можем работать?", + effectiveAddressUserMessage: "РїРѕ какой компании РјС‹ можем работать?", followupContext: null, llmPreDecomposeMeta: null, useMock: false @@ -116,6 +127,7 @@ describe("assistantRoutePolicy", () => { expect(decision.toolGateReason).toBe("assistant_data_scope_query_detected"); expect(decision.livingMode).toBe("chat"); expect(decision.orchestrationContract?.hard_meta_mode).toBe("data_scope"); + expect(decision.orchestrationContract?.provider_execution?.provider_mode).toBe("unknown"); }); it("keeps supported address intent in address lane", () => { @@ -130,8 +142,8 @@ describe("assistantRoutePolicy", () => { }); const decision = policy.resolveAssistantOrchestrationDecision({ - rawUserMessage: "какие товары сейчас лежат на складе", - effectiveAddressUserMessage: "какие товары сейчас лежат на складе", + rawUserMessage: "какие товары сейчас лежат РЅР° складе", + effectiveAddressUserMessage: "какие товары сейчас лежат РЅР° складе", followupContext: null, llmPreDecomposeMeta: null, useMock: false @@ -153,8 +165,8 @@ describe("assistantRoutePolicy", () => { }); const decision = policy.resolveAssistantOrchestrationDecision({ - rawUserMessage: "а ты помнишь что мы обсуждали?", - effectiveAddressUserMessage: "а ты помнишь что мы обсуждали?", + rawUserMessage: "Р° ты помнишь что РјС‹ обсуждали?", + effectiveAddressUserMessage: "Р° ты помнишь что РјС‹ обсуждали?", followupContext: null, llmPreDecomposeMeta: { applied: false, @@ -174,4 +186,36 @@ describe("assistantRoutePolicy", () => { expect(decision.livingMode).toBe("chat"); expect(decision.livingReason).toBe("memory_recap_followup_detected"); }); + + it("does not force unsupported-intent fallback when predecompose runtime is unavailable", () => { + const policy = buildPolicy({ + hasStrongDataIntentSignal: () => true, + hasDataRetrievalRequestSignal: () => true, + resolveAddressToolGateDecision: () => ({ + runAddressLane: true, + decision: "run_address_lane", + reason: "address_mode_classifier_detected" + }) + }); + + const decision = policy.resolveAssistantOrchestrationDecision({ + rawUserMessage: "покажи документы по сверке", + effectiveAddressUserMessage: "покажи документы по сверке", + followupContext: { root_context_only: true }, + llmPreDecomposeMeta: { + reason: "error:OpenAI API key is missing", + predecomposeContract: { + mode: "unsupported", + mode_confidence: "low", + intent: "unknown", + intent_confidence: "low" + } + }, + useMock: false + }); + + expect(decision.toolGateReason).toBe("address_mode_classifier_detected"); + expect(decision.orchestrationContract?.unsupported_address_intent_fallback_to_deep).toBe(false); + expect(decision.orchestrationContract?.provider_execution?.llm_runtime_unavailable_detected).toBe(true); + }); }); diff --git a/llm_normalizer/data/autorun_generators/history.json b/llm_normalizer/data/autorun_generators/history.json index bcae254..a4af91c 100644 --- a/llm_normalizer/data/autorun_generators/history.json +++ b/llm_normalizer/data/autorun_generators/history.json @@ -1,4 +1,37 @@ [ + { + "generation_id": "gen-ag04170855-d13dd3", + "created_at": "2026-04-17T08:55:50+00:00", + "mode": "saved_user_sessions", + "title": "AGENT | Phase 6 provider/runtime replay across chat, meta, and address boundaries", + "count": 5, + "domain": "address_phase6_provider_axis_mix", + "questions": [ + "привет, как дела?", + "по какой компании мы сейчас работаем?", + "что ты можешь по 1С?", + "какие остатки на складе на март 2021", + "а исторические остатки тоже можешь?" + ], + "generated_by": "codex_agent", + "saved_case_set_file": "assistant_autogen_saved_user_sessions_20260417085550_gen-ag04170855-d13dd3.json", + "context": { + "llm_provider": null, + "model": null, + "assistant_prompt_version": null, + "decomposition_prompt_version": null, + "prompt_fingerprint": null, + "autogen_personality_id": null, + "autogen_personality_prompt": null, + "source_session_id": null, + "saved_session_file": "assistant_saved_session_20260417085550_gen-ag04170855-d13dd3.json", + "saved_case_set_kind": "agent_semantic_scenario", + "agent_run": true, + "agent_focus": "provider runtime axis hardening across chat meta and address boundaries", + "architecture_phase": "turnaround_11_phase6", + "source_spec_file": "X:\\1C\\NDC_1C\\docs\\orchestration\\address_truth_harness_phase6_provider_axis_mix.json" + } + }, { "generation_id": "gen-ag04170830-5f771d", "created_at": "2026-04-17T08:30:44+00:00", diff --git a/llm_normalizer/data/autorun_generators/saved_sessions/assistant_saved_session_20260417085550_gen-ag04170855-d13dd3.json b/llm_normalizer/data/autorun_generators/saved_sessions/assistant_saved_session_20260417085550_gen-ag04170855-d13dd3.json new file mode 100644 index 0000000..6d4b5db --- /dev/null +++ b/llm_normalizer/data/autorun_generators/saved_sessions/assistant_saved_session_20260417085550_gen-ag04170855-d13dd3.json @@ -0,0 +1,83 @@ +{ + "saved_at": "2026-04-17T08:55:50+00:00", + "generation_id": "gen-ag04170855-d13dd3", + "mode": "saved_user_sessions", + "title": "AGENT | Phase 6 provider/runtime replay across chat, meta, and address boundaries", + "agent_run": true, + "questions": [ + "привет, как дела?", + "по какой компании мы сейчас работаем?", + "что ты можешь по 1С?", + "какие остатки на складе на март 2021", + "а исторические остатки тоже можешь?" + ], + "metadata": { + "assistant_prompt_version": null, + "decomposition_prompt_version": null, + "prompt_fingerprint": null, + "agent_focus": "provider runtime axis hardening across chat meta and address boundaries", + "architecture_phase": "turnaround_11_phase6", + "source_spec_file": "X:\\1C\\NDC_1C\\docs\\orchestration\\address_truth_harness_phase6_provider_axis_mix.json" + }, + "source_session_id": null, + "session": { + "session_id": null, + "mode": "agent_semantic_run", + "items": [ + { + "message_id": "agent-user-001", + "role": "user", + "text": "привет, как дела?", + "created_at": "2026-04-17T08:55:50+00:00", + "reply_type": null, + "trace_id": null, + "debug": null + }, + { + "message_id": "agent-user-002", + "role": "user", + "text": "по какой компании мы сейчас работаем?", + "created_at": "2026-04-17T08:55:50+00:00", + "reply_type": null, + "trace_id": null, + "debug": null + }, + { + "message_id": "agent-user-003", + "role": "user", + "text": "что ты можешь по 1С?", + "created_at": "2026-04-17T08:55:50+00:00", + "reply_type": null, + "trace_id": null, + "debug": null + }, + { + "message_id": "agent-user-004", + "role": "user", + "text": "какие остатки на складе на март 2021", + "created_at": "2026-04-17T08:55:50+00:00", + "reply_type": null, + "trace_id": null, + "debug": null + }, + { + "message_id": "agent-user-005", + "role": "user", + "text": "а исторические остатки тоже можешь?", + "created_at": "2026-04-17T08:55:50+00:00", + "reply_type": null, + "trace_id": null, + "debug": null + } + ], + "agent_run": true, + "metadata": { + "assistant_prompt_version": null, + "decomposition_prompt_version": null, + "prompt_fingerprint": null, + "agent_focus": "provider runtime axis hardening across chat meta and address boundaries", + "architecture_phase": "turnaround_11_phase6", + "source_spec_file": "X:\\1C\\NDC_1C\\docs\\orchestration\\address_truth_harness_phase6_provider_axis_mix.json" + } + } +} diff --git a/llm_normalizer/data/eval_cases/assistant_autogen_saved_user_sessions_20260417085550_gen-ag04170855-d13dd3.json b/llm_normalizer/data/eval_cases/assistant_autogen_saved_user_sessions_20260417085550_gen-ag04170855-d13dd3.json new file mode 100644 index 0000000..721676e --- /dev/null +++ b/llm_normalizer/data/eval_cases/assistant_autogen_saved_user_sessions_20260417085550_gen-ag04170855-d13dd3.json @@ -0,0 +1,40 @@ +{ + "suite_id": "assistant_saved_session_gen-ag04170855-d13dd3", + "suite_version": "0.1.0", + "schema_version": "assistant_saved_session_suite_v0_1", + "generated_at": "2026-04-17T08:55:50+00:00", + "generation_id": "gen-ag04170855-d13dd3", + "mode": "saved_user_sessions", + "title": "AGENT | Phase 6 provider/runtime replay across chat, meta, and address boundaries", + "domain": "address_phase6_provider_axis_mix", + "scenario_count": 1, + "case_ids": [ + "SAVED-001" + ], + "cases": [ + { + "case_id": "SAVED-001", + "scenario_tag": "agent_saved_user_sessions", + "title": "AGENT | Phase 6 provider/runtime replay across chat, meta, and address boundaries", + "question_type": "followup", + "broadness_level": "medium", + "turns": [ + { + "user_message": "привет, как дела?" + }, + { + "user_message": "по какой компании мы сейчас работаем?" + }, + { + "user_message": "что ты можешь по 1С?" + }, + { + "user_message": "какие остатки на складе на март 2021" + }, + { + "user_message": "а исторические остатки тоже можешь?" + } + ] + } + ] +} diff --git a/llm_normalizer/data/eval_cases/assistant_saved_session_runtime_job--UpMq6CnP3.json b/llm_normalizer/data/eval_cases/assistant_saved_session_runtime_job--UpMq6CnP3.json new file mode 100644 index 0000000..6a29b72 --- /dev/null +++ b/llm_normalizer/data/eval_cases/assistant_saved_session_runtime_job--UpMq6CnP3.json @@ -0,0 +1,39 @@ +{ + "suite_id": "assistant_saved_session_runtime_job--UpMq6CnP3", + "suite_version": "0.1.0", + "schema_version": "assistant_saved_session_runtime_v0_1", + "title": "AGENT | Phase 5 meta and memory recap replay over interrupted address context", + "scenario_count": 1, + "case_ids": [ + "SAVED-001" + ], + "cases": [ + { + "case_id": "SAVED-001", + "scenario_tag": "saved_user_sessions_runtime", + "title": "AGENT | Phase 5 meta and memory recap replay over interrupted address context", + "question_type": "followup", + "broadness_level": "medium", + "turns": [ + { + "user_message": "какие остатки на складе на март 2021" + }, + { + "user_message": "а исторические остатки тоже можешь?" + }, + { + "user_message": "по какой компании мы сейчас работаем?" + }, + { + "user_message": "По выбранному объекту \"Столешница 600*3050*26 альмандин\": кто нам это поставил?" + }, + { + "user_message": "что ты умеешь?" + }, + { + "user_message": "а ты помнишь, что мы по этой позиции уже выяснили?" + } + ] + } + ] +} \ No newline at end of file