From db3bdd3487a6e5da52dc3abb074210cd47c107aa Mon Sep 17 00:00:00 2001 From: hyeokjun32 Date: Fri, 29 May 2026 01:23:00 +0900 Subject: [PATCH] Render AIGuard task event rollup evidence --- README.md | 2 + .../runtime_intelligence_gitlab_artifacts.md | 2 +- ...uard_runtime_operation_guard_analysis.json | 104 ++++++++++++++++++ inferedgelab/report/runtime_intelligence.py | 59 ++++++++++ ...ck_runtime_intelligence_artifact_bundle.py | 7 ++ ...check_runtime_intelligence_ci_artifacts.py | 2 + tests/test_report_generators.py | 51 +++++++++ .../test_runtime_intelligence_ci_template.py | 10 ++ ...ntime_intelligence_evidence_chain_smoke.py | 17 +++ 9 files changed, 253 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index c41c7bc..4c4d242 100644 --- a/README.md +++ b/README.md @@ -502,6 +502,8 @@ If the Orchestrator feed includes `runtime_task_event_summary`, Lab also renders If AIGuard also emits `edgeenv_orchestrator_operation_risk_summary`, Lab renders that deterministic evidence as a separate operation-risk summary row. This keeps the AIGuard warning explanation visible while preserving Lab as the final deployment decision owner. +If AIGuard emits `edgeenv_orchestrator_task_event_rollup`, Lab renders a separate task-event evidence row for affected tasks, deadline misses, fallback decisions, scheduler delay, and queue/drop reasons. This is deterministic review context only; it does not override EdgeEnv comparability or Lab deployment policy. + For the remote dispatch starter path, Lab surfaces AIGuard's compact remote runtime event summary, count alias, consistency state, `evidence_role=remote_dispatch_runtime_event_compact_summary`, `operation_boundary=remote dispatch starter evidence only`, and `production_remote_execution=false` markers. These values remain Lab-owned deployment review context: Lab can require review from the evidence, but it does not treat Orchestrator, EdgeEnv, or AIGuard as the final decision owner and does not present the path as production remote execution, long-lived worker readiness, or cloud orchestration. Markdown and HTML reports include a Runtime Intelligence Risk Summary that connects: diff --git a/docs/ci/runtime_intelligence_gitlab_artifacts.md b/docs/ci/runtime_intelligence_gitlab_artifacts.md index d4f6ceb..c75dacf 100644 --- a/docs/ci/runtime_intelligence_gitlab_artifacts.md +++ b/docs/ci/runtime_intelligence_gitlab_artifacts.md @@ -177,7 +177,7 @@ EdgeEnv handoff summary and AIGuard deterministic evidence agree on producer-lineage guard-alignment run IDs. This keeps the cross-repo marker check file-based and does not make AIGuard a deployment decision owner. -The artifact gate is implemented by `scripts/check_runtime_intelligence_artifact_bundle.py`. It checks the generated Markdown / HTML report for the required Runtime Intelligence rows, including Lab ownership, EdgeEnv comparability, telemetry coverage-gap markers, Orchestrator operation feed context, Orchestrator task event rollup, Orchestrator `operation_risk_summary` navigation context, AIGuard runtime operation anomalies, AIGuard `edgeenv_orchestrator_operation_risk_summary` evidence, remote dispatch starter event summary, `edgeenv_orchestrator_producer_lineage`, `runtime_history_seed_run_config_traceability`, `remote_execution_recovered_by_fallback`, and triggered deployment review rules. +The artifact gate is implemented by `scripts/check_runtime_intelligence_artifact_bundle.py`. It checks the generated Markdown / HTML report for the required Runtime Intelligence rows, including Lab ownership, EdgeEnv comparability, telemetry coverage-gap markers, Orchestrator operation feed context, Orchestrator task event rollup, Orchestrator `operation_risk_summary` navigation context, AIGuard runtime operation anomalies, AIGuard `edgeenv_orchestrator_operation_risk_summary` evidence, AIGuard `edgeenv_orchestrator_task_event_rollup` evidence, remote dispatch starter event summary, `edgeenv_orchestrator_producer_lineage`, `runtime_history_seed_run_config_traceability`, `remote_execution_recovered_by_fallback`, and triggered deployment review rules. The CI artifact gate is implemented by `scripts/check_runtime_intelligence_ci_artifacts.py`. It runs in the deployment-risk stage and verifies that the collected optional GitLab artifacts include the manifest gate summary, AIGuard handoff alignment artifact, report gate summary, Runtime Intelligence Risk Summary report, portfolio demo status, and the validated contract markers from the bundle manifest gate. This keeps the final CI gate file-based and deterministic without turning GitLab into a runtime control plane. The same CI artifact gate also checks the copied diff --git a/examples/runtime_intelligence_chain/aiguard_runtime_operation_guard_analysis.json b/examples/runtime_intelligence_chain/aiguard_runtime_operation_guard_analysis.json index eb80289..52204c8 100644 --- a/examples/runtime_intelligence_chain/aiguard_runtime_operation_guard_analysis.json +++ b/examples/runtime_intelligence_chain/aiguard_runtime_operation_guard_analysis.json @@ -1263,6 +1263,110 @@ } } }, + { + "type": "edgeenv_orchestrator_task_event_rollup", + "metric_name": "orchestrator_task_event_affected_task_count", + "observed_value": 2, + "baseline_value": 0, + "threshold": 1, + "delta": null, + "delta_pct": null, + "increase_factor": null, + "severity": "medium", + "status": "warning", + "explanation": "EdgeEnv preserved Orchestrator task event rollup with 2 task-level review target(s).", + "why_it_matters": "Task-level scheduler delay, deadline miss, and fallback context explains which workload produced runtime operation risk. AIGuard preserves this deterministic warning evidence while Lab remains the deployment decision owner.", + "suspected_causes": [ + "scheduler_delay_context", + "deadline_miss_context", + "fallback_policy_context", + "queue_pressure_context" + ], + "recommendation": "Review Orchestrator task event rollup, queue policy, deadline budget, and fallback behavior in the Lab report before deployment.", + "raw_context": { + "task_event_rollup": { + "summary": { + "vision_agent": { + "event_count": 9, + "event_type_counts": { + "schedule": 3, + "execution": 3, + "policy_decision": 3 + }, + "reason_counts": { + "deadline_missed": 1, + "queue_backlog_threshold_exceeded": 1 + }, + "policy_decision_reason_counts": { + "queue_backlog_threshold_exceeded": 1 + }, + "drop_reason_counts": {}, + "deadline_missed_count": 1, + "fallback_decision_count": 0, + "scheduler_delay_event_count": 1, + "max_scheduler_delay_cycles": 3, + "max_queue_wait_ms": 15.0, + "latest_event_index": 42, + "latest_event_type": "execution" + }, + "voice_command_agent": { + "event_count": 6, + "event_type_counts": { + "drop": 1, + "policy_decision": 1, + "schedule": 2, + "execution": 2 + }, + "reason_counts": { + "queue_backlog_threshold_exceeded": 1, + "load_shedding_backlog_threshold_exceeded": 1 + }, + "policy_decision_reason_counts": { + "queue_backlog_threshold_exceeded": 1 + }, + "drop_reason_counts": { + "load_shedding_backlog_threshold_exceeded": 1 + }, + "deadline_missed_count": 0, + "fallback_decision_count": 1, + "scheduler_delay_event_count": 0, + "max_scheduler_delay_cycles": 0, + "max_queue_wait_ms": 0.0, + "latest_event_index": 39, + "latest_event_type": "policy_decision" + } + }, + "tasks_with_deadline_miss": [ + "vision_agent" + ], + "tasks_with_fallback": [ + "voice_command_agent" + ], + "tasks_with_scheduler_delay": [ + "vision_agent" + ], + "affected_tasks": [ + "vision_agent", + "voice_command_agent" + ], + "reason_counts": { + "deadline_missed": 1, + "queue_backlog_threshold_exceeded": 2, + "load_shedding_backlog_threshold_exceeded": 1 + }, + "review_markers": [ + "deadline_miss", + "fallback", + "scheduler_delay", + "queue_pressure_reason" + ], + "boundary_markers_valid": true, + "decision_owner": "lab", + "scheduler_owner": "orchestrator", + "not_a_deployment_decision": true + } + } + }, { "type": "runtime_history_seed_run_config_traceability", "metric_name": "runtime_history_seed_run_config_runs", diff --git a/inferedgelab/report/runtime_intelligence.py b/inferedgelab/report/runtime_intelligence.py index d76ff28..f80d4ba 100644 --- a/inferedgelab/report/runtime_intelligence.py +++ b/inferedgelab/report/runtime_intelligence.py @@ -18,6 +18,10 @@ "edgeenv_orchestrator_operation_risk_summary" ) +ORCHESTRATOR_TASK_EVENT_ROLLUP_EVIDENCE_TYPE = ( + "edgeenv_orchestrator_task_event_rollup" +) + RUN_CONFIG_TRACEABILITY_EVIDENCE_TYPE = "runtime_history_seed_run_config_traceability" REMOTE_RUNTIME_EVENT_SUMMARY_MISMATCH_EVIDENCE_TYPE = ( @@ -467,6 +471,16 @@ def _append_aiguard_runtime_operation_rows( ) ) + task_event_rollup_label = _aiguard_task_event_rollup_label(evidence_items) + if task_event_rollup_label: + rows.append( + ( + "AIGuard task event rollup evidence", + task_event_rollup_label, + "AIGuard preserves task-level scheduler/deadline/fallback evidence as deterministic review context; Lab still owns the deployment decision.", + ) + ) + candidate_summary = guard_analysis.get("candidate_summary") if not isinstance(candidate_summary, dict): return @@ -612,6 +626,51 @@ def _aiguard_operation_risk_summary_label( return ", ".join(parts) +def _aiguard_task_event_rollup_label( + evidence_items: list[dict[str, Any]], +) -> str: + evidence = _find_evidence_item( + evidence_items, + ORCHESTRATOR_TASK_EVENT_ROLLUP_EVIDENCE_TYPE, + ) + if evidence is None: + return "" + + parts: list[str] = [] + status = evidence.get("status") + if status is not None: + parts.append(f"status={status}") + observed = evidence.get("observed_value") + if observed is not None: + parts.append(f"affected={_format_compact_value(observed)}") + + raw_context = evidence.get("raw_context") + if isinstance(raw_context, dict): + context = raw_context.get("task_event_rollup") + if isinstance(context, dict): + affected_tasks = _string_list(context.get("affected_tasks")) + if affected_tasks: + parts.append("tasks=" + ",".join(affected_tasks)) + deadline_tasks = _string_list(context.get("tasks_with_deadline_miss")) + if deadline_tasks: + parts.append("deadline=" + ",".join(deadline_tasks)) + fallback_tasks = _string_list(context.get("tasks_with_fallback")) + if fallback_tasks: + parts.append("fallback=" + ",".join(fallback_tasks)) + scheduler_delay_tasks = _string_list( + context.get("tasks_with_scheduler_delay") + ) + if scheduler_delay_tasks: + parts.append("scheduler_delay=" + ",".join(scheduler_delay_tasks)) + reason_counts = context.get("reason_counts") + if isinstance(reason_counts, dict) and reason_counts: + parts.append("reasons=" + _format_reason_count_label(reason_counts)) + boundary_valid = context.get("boundary_markers_valid") + if boundary_valid is not None: + parts.append(f"boundary_valid={boundary_valid}") + return ", ".join(parts) + + def _append_aiguard_remote_dispatch_rows( rows: list[tuple[str, str, str]], guard_analysis: dict[str, Any], diff --git a/scripts/check_runtime_intelligence_artifact_bundle.py b/scripts/check_runtime_intelligence_artifact_bundle.py index 71200aa..042f341 100644 --- a/scripts/check_runtime_intelligence_artifact_bundle.py +++ b/scripts/check_runtime_intelligence_artifact_bundle.py @@ -46,6 +46,11 @@ "aiguard_operation_risk_summary_type": ( "edgeenv_orchestrator_operation_risk_summary" ), + "aiguard_task_event_rollup_evidence": ( + "| AIGuard task event rollup evidence | " + "status=warning, affected=2" + ), + "aiguard_task_event_rollup_type": "edgeenv_orchestrator_task_event_rollup", "aiguard_remote_dispatch_summary": ( "| AIGuard remote dispatch event summary | " "events=3, final=succeeded, fallback_recovered=True |" @@ -100,6 +105,8 @@ "aiguard_operation_risk_summary_type": ( "edgeenv_orchestrator_operation_risk_summary" ), + "aiguard_task_event_rollup_evidence": "AIGuard task event rollup evidence", + "aiguard_task_event_rollup_type": "edgeenv_orchestrator_task_event_rollup", "aiguard_remote_dispatch_summary": "AIGuard remote dispatch event summary", "aiguard_remote_dispatch_label": ( "events=3, final=succeeded, fallback_recovered=True" diff --git a/scripts/check_runtime_intelligence_ci_artifacts.py b/scripts/check_runtime_intelligence_ci_artifacts.py index fc6a972..a06f334 100644 --- a/scripts/check_runtime_intelligence_ci_artifacts.py +++ b/scripts/check_runtime_intelligence_ci_artifacts.py @@ -147,6 +147,8 @@ def _validate_runtime_report(path: Path, errors: list[str]) -> None: "runtime_queue_overload, runtime_thermal_instability", "AIGuard operation risk summary evidence", "edgeenv_orchestrator_operation_risk_summary", + "AIGuard task event rollup evidence", + "edgeenv_orchestrator_task_event_rollup", "Runtime telemetry coverage gaps", "AIGuard producer-lineage guard alignment", "edgeenv_orchestrator_producer_lineage", diff --git a/tests/test_report_generators.py b/tests/test_report_generators.py index cbe32bd..ac9060d 100644 --- a/tests/test_report_generators.py +++ b/tests/test_report_generators.py @@ -474,6 +474,49 @@ def make_runtime_operation_guard_analysis() -> dict: } }, }, + { + "type": "edgeenv_orchestrator_task_event_rollup", + "metric_name": "orchestrator_task_event_affected_task_count", + "observed_value": 2, + "baseline_value": 0, + "threshold": 1, + "status": "warning", + "severity": "medium", + "why_it_matters": ( + "Task-level scheduler delay, deadline miss, and fallback " + "context explains which workload produced runtime operation risk." + ), + "suspected_causes": [ + "scheduler_delay_context", + "deadline_miss_context", + "fallback_policy_context", + "queue_pressure_context", + ], + "recommendation": ( + "Review Orchestrator task event rollup, queue policy, " + "deadline budget, and fallback behavior in the Lab report." + ), + "raw_context": { + "task_event_rollup": { + "tasks_with_deadline_miss": ["vision_agent"], + "tasks_with_fallback": ["voice_command_agent"], + "tasks_with_scheduler_delay": ["vision_agent"], + "affected_tasks": [ + "vision_agent", + "voice_command_agent", + ], + "reason_counts": { + "deadline_missed": 1, + "queue_backlog_threshold_exceeded": 2, + "load_shedding_backlog_threshold_exceeded": 1, + }, + "boundary_markers_valid": True, + "decision_owner": "lab", + "scheduler_owner": "orchestrator", + "not_a_deployment_decision": True, + } + }, + }, ], "candidate_summary": { "edgeenv_regression": { @@ -815,6 +858,14 @@ def test_generate_compare_markdown_summarizes_orchestrator_context_risk(): "input_mode=dummy, input_preprocess=none, power_mode=unknown, " "jetson_clocks=unknown, warmup=1, runs=10 |" ) in text + assert ( + "| AIGuard task event rollup evidence | " + "status=warning, affected=2, tasks=vision_agent,voice_command_agent, " + "deadline=vision_agent, fallback=voice_command_agent, " + "scheduler_delay=vision_agent, reasons=deadline_missed:1," + "queue_backlog_threshold_exceeded:2," + "load_shedding_backlog_threshold_exceeded:1, boundary_valid=True |" + ) in text assert "AIGuard does not own the final decision" in text diff --git a/tests/test_runtime_intelligence_ci_template.py b/tests/test_runtime_intelligence_ci_template.py index 1050f0c..61b31e4 100644 --- a/tests/test_runtime_intelligence_ci_template.py +++ b/tests/test_runtime_intelligence_ci_template.py @@ -97,6 +97,8 @@ def test_runtime_intelligence_ci_artifact_gate_passes_for_expected_outputs(tmp_p "runtime_queue_overload, runtime_thermal_instability", "AIGuard operation risk summary evidence", "edgeenv_orchestrator_operation_risk_summary", + "AIGuard task event rollup evidence", + "edgeenv_orchestrator_task_event_rollup", "Runtime telemetry coverage gaps", "AIGuard producer-lineage guard alignment", "edgeenv_orchestrator_producer_lineage", @@ -285,6 +287,8 @@ def test_runtime_intelligence_ci_artifact_gate_fails_for_missing_lab_marker_cont "runtime_queue_overload, runtime_thermal_instability", "AIGuard operation risk summary evidence", "edgeenv_orchestrator_operation_risk_summary", + "AIGuard task event rollup evidence", + "edgeenv_orchestrator_task_event_rollup", "Runtime telemetry coverage gaps", "AIGuard producer-lineage guard alignment", "edgeenv_orchestrator_producer_lineage", @@ -415,6 +419,8 @@ def test_runtime_intelligence_ci_artifact_gate_fails_for_missing_contract_marker "runtime_queue_overload, runtime_thermal_instability", "AIGuard operation risk summary evidence", "edgeenv_orchestrator_operation_risk_summary", + "AIGuard task event rollup evidence", + "edgeenv_orchestrator_task_event_rollup", "Runtime telemetry coverage gaps", "AIGuard producer-lineage guard alignment", "edgeenv_orchestrator_producer_lineage", @@ -489,6 +495,8 @@ def test_runtime_intelligence_ci_artifact_gate_fails_for_missing_coverage_gap_ma "runtime_queue_overload, runtime_thermal_instability", "AIGuard operation risk summary evidence", "edgeenv_orchestrator_operation_risk_summary", + "AIGuard task event rollup evidence", + "edgeenv_orchestrator_task_event_rollup", "Runtime telemetry coverage gaps", "AIGuard producer-lineage guard alignment", "edgeenv_orchestrator_producer_lineage", @@ -548,6 +556,8 @@ def test_runtime_intelligence_ci_artifact_gate_fails_for_failed_deployment_risk( "runtime_queue_overload, runtime_thermal_instability", "AIGuard operation risk summary evidence", "edgeenv_orchestrator_operation_risk_summary", + "AIGuard task event rollup evidence", + "edgeenv_orchestrator_task_event_rollup", "Runtime telemetry coverage gaps", "AIGuard producer-lineage guard alignment", "edgeenv_orchestrator_producer_lineage", diff --git a/tests/test_runtime_intelligence_evidence_chain_smoke.py b/tests/test_runtime_intelligence_evidence_chain_smoke.py index 20e412a..b4eed5a 100644 --- a/tests/test_runtime_intelligence_evidence_chain_smoke.py +++ b/tests/test_runtime_intelligence_evidence_chain_smoke.py @@ -97,6 +97,7 @@ def test_runtime_intelligence_chain_smoke_ingests_precomputed_guard_artifact(): evidence_types = {item["type"] for item in bundle["guard_analysis"]["evidence"]} assert "edgeenv_orchestrator_producer_lineage" in evidence_types assert "edgeenv_orchestrator_operation_risk_summary" in evidence_types + assert "edgeenv_orchestrator_task_event_rollup" in evidence_types assert "runtime_history_seed_run_config_traceability" in evidence_types assert "remote_execution_recovered_by_fallback" in evidence_types coverage_evidence = next( @@ -288,6 +289,16 @@ def test_runtime_intelligence_chain_smoke_ingests_precomputed_guard_artifact(): ) assert operation_risk_context["primary_health_reason"] == "worker_health_degraded" assert operation_risk_context["degraded_worker_ids"] == ["vision_agent"] + task_event_rollup_evidence = next( + item + for item in bundle["guard_analysis"]["evidence"] + if item["type"] == "edgeenv_orchestrator_task_event_rollup" + ) + assert task_event_rollup_evidence["status"] == "warning" + assert task_event_rollup_evidence["observed_value"] == 2 + assert task_event_rollup_evidence["raw_context"]["task_event_rollup"][ + "affected_tasks" + ] == ["vision_agent", "voice_command_agent"] assert operation_risk_context["device_local_event_count"] == 15.0 run_config_traceability_evidence = next( item @@ -461,6 +472,9 @@ def test_compare_cmd_runtime_intelligence_chain_writes_markdown_and_html( assert "AIGuard runtime operation anomalies" in markdown assert "AIGuard operation risk summary evidence" in markdown assert "edgeenv_orchestrator_operation_risk_summary" in markdown + assert "AIGuard task event rollup evidence" in markdown + assert "edgeenv_orchestrator_task_event_rollup" in markdown + assert "tasks=vision_agent,voice_command_agent" in markdown assert "status=warning, markers=4" in markdown assert "health=worker_health_degraded" in markdown assert "Orchestrator context attached runs" in markdown @@ -505,6 +519,9 @@ def test_compare_cmd_runtime_intelligence_chain_writes_markdown_and_html( assert "runtime_queue_overload, runtime_thermal_instability" in html assert "AIGuard operation risk summary evidence" in html assert "edgeenv_orchestrator_operation_risk_summary" in html + assert "AIGuard task event rollup evidence" in html + assert "edgeenv_orchestrator_task_event_rollup" in html + assert "tasks=vision_agent,voice_command_agent" in html assert "health=worker_health_degraded" in html assert "AIGuard remote dispatch event summary" in html assert "events=3, final=succeeded, fallback_recovered=True" in html