From b5f60db7c1ceb13a9f21d98eb4304e7bdffa239e Mon Sep 17 00:00:00 2001 From: hyeokjun32 Date: Fri, 29 May 2026 21:00:50 +0900 Subject: [PATCH] test: harden remote fallback marker metadata --- .../runtime_intelligence_gitlab_artifacts.md | 8 ++++- .../edgeenv_runtime_regression_lab_handoff.md | 3 +- .../aiguard_edgeenv_handoff_alignment.json | 3 +- .../aiguard_edgeenv_handoff_alignment.md | 2 +- .../bundle_manifest.json | 1 + .../edgeenv_lab_handoff_manifest.json | 1 + ...ck_runtime_intelligence_bundle_manifest.py | 28 ++++++++++++++++++ ...check_runtime_intelligence_ci_artifacts.py | 4 +++ ...st_runtime_intelligence_bundle_manifest.py | 29 +++++++++++++++++++ .../test_runtime_intelligence_ci_template.py | 13 ++++++--- 10 files changed, 84 insertions(+), 8 deletions(-) diff --git a/docs/ci/runtime_intelligence_gitlab_artifacts.md b/docs/ci/runtime_intelligence_gitlab_artifacts.md index 7e3cb32..85e17b7 100644 --- a/docs/ci/runtime_intelligence_gitlab_artifacts.md +++ b/docs/ci/runtime_intelligence_gitlab_artifacts.md @@ -172,7 +172,11 @@ packaging runs. That marker set preserves `Runtime Intelligence Risk Summary`, `AIGuard remote dispatch event summary`, `AIGuard remote event summary consistency`, `Remote fallback starter evidence`, +`lab=Remote fallback starter evidence; evidence=remote_execution_recovered_by_fallback`, `AIGuard producer-lineage guard alignment`, and `Lab remains the final deployment decision owner.`. +The gate summary also emits +`expected_report_markers: remote fallback Lab context row declared` so CI can +verify the row-value marker declaration from the bundle summary artifact. The smoke also includes the precomputed AIGuard `aiguard_edgeenv_handoff_alignment` artifact. That artifact verifies that the @@ -187,7 +191,9 @@ The CI artifact gate is implemented by `scripts/check_runtime_intelligence_ci_ar The same CI artifact gate also checks the copied `aiguard_edgeenv_handoff_alignment.json/.md` for Lab report marker context: `lab_expected_report_markers` must match the Lab-owned Runtime Intelligence -report marker set, `report_marker_context_role` must remain +report marker set, including the remote fallback row value marker +`lab=Remote fallback starter evidence; evidence=remote_execution_recovered_by_fallback`, +`report_marker_context_role` must remain `lab_report_contract_context`, and `aiguard_validates_expected_report_markers` must remain `false`. This keeps AIGuard as a deterministic external evidence provider while leaving report diff --git a/docs/portfolio/edgeenv_runtime_regression_lab_handoff.md b/docs/portfolio/edgeenv_runtime_regression_lab_handoff.md index 69c4a94..e238e86 100644 --- a/docs/portfolio/edgeenv_runtime_regression_lab_handoff.md +++ b/docs/portfolio/edgeenv_runtime_regression_lab_handoff.md @@ -175,7 +175,8 @@ Expected Lab behavior: - The same gate requires EdgeEnv-preserved Orchestrator producer markers to carry `source_repository=InferEdgeOrchestrator`, `artifact_role=orchestrator-supplemental-operation-context`, and `producer_contract=inferedge-orchestrator-edgeenv-runtime-telemetry-feed-v1`. - When EdgeEnv preservation context is present, Lab renders `Lab EdgeEnv preservation context` with `lab_report_preservation_context_present=True`, `lab_preservation=present`, and `lab_context=present` so the Runtime Intelligence report gate and entrypoint evidence index use the same Lab-owned marker vocabulary. - When an EdgeEnv handoff manifest is provided, the bundle gate requires EdgeEnv-produced file keys, external AIGuard file keys, source repository mapping, artifact roles, producer contracts, and boundary flags to match Lab's Runtime Intelligence bundle contract. -- The same manifest gate requires `expected_report_markers` to preserve these exact Lab-owned report markers: `Runtime Intelligence Risk Summary`, `Orchestrator operation feed context`, `Orchestrator task event rollup`, `Lab EdgeEnv preservation context`, `AIGuard task event rollup evidence`, `AIGuard runtime operation anomalies`, `AIGuard remote dispatch event summary`, `AIGuard remote event summary consistency`, `Remote fallback starter evidence`, `AIGuard producer-lineage guard alignment`, and `Lab remains the final deployment decision owner.`. +- The same manifest gate requires `expected_report_markers` to preserve these exact Lab-owned report markers: `Runtime Intelligence Risk Summary`, `Orchestrator operation feed context`, `Orchestrator task event rollup`, `Lab EdgeEnv preservation context`, `AIGuard task event rollup evidence`, `AIGuard runtime operation anomalies`, `AIGuard remote dispatch event summary`, `AIGuard remote event summary consistency`, `Remote fallback starter evidence`, `lab=Remote fallback starter evidence; evidence=remote_execution_recovered_by_fallback`, `AIGuard producer-lineage guard alignment`, and `Lab remains the final deployment decision owner.`. +- The same gate summary emits `expected_report_markers: remote fallback Lab context row declared`, keeping the remote fallback row-value marker visible in file-based CI artifacts without making CI a runtime control plane. - The same handoff gate verifies that the referenced `runtime_telemetry_history` artifact exists and preserves EdgeEnv history schema, telemetry coverage, and Runtime history seed ownership markers. - The same handoff gate verifies that missing telemetry entries remain evidence gaps while preserving Orchestrator producer markers, owner boundary flags, and EdgeEnv mapping hints when Orchestrator context is attached. - The same handoff gate validates `edgeenv_report_summary.producer_lineage_guard_alignment_run_ids` against the preserved EdgeEnv regression context so the Orchestrator-declared `edgeenv_orchestrator_producer_lineage` marker cannot disappear between EdgeEnv producer output and Lab bundle ingestion. diff --git a/examples/runtime_intelligence_chain/aiguard_edgeenv_handoff_alignment.json b/examples/runtime_intelligence_chain/aiguard_edgeenv_handoff_alignment.json index 893e2a5..887ee60 100644 --- a/examples/runtime_intelligence_chain/aiguard_edgeenv_handoff_alignment.json +++ b/examples/runtime_intelligence_chain/aiguard_edgeenv_handoff_alignment.json @@ -8,7 +8,7 @@ "guard_analysis_schema_version": "inferedge-aiguard-diagnosis-v1", "required_evidence_type_count": 7, "guard_evidence_type_count": 8, - "lab_expected_report_marker_count": 11, + "lab_expected_report_marker_count": 12, "lab_expected_report_markers": [ "Runtime Intelligence Risk Summary", "Orchestrator operation feed context", @@ -19,6 +19,7 @@ "AIGuard remote dispatch event summary", "AIGuard remote event summary consistency", "Remote fallback starter evidence", + "lab=Remote fallback starter evidence; evidence=remote_execution_recovered_by_fallback", "AIGuard producer-lineage guard alignment", "Lab remains the final deployment decision owner." ], diff --git a/examples/runtime_intelligence_chain/aiguard_edgeenv_handoff_alignment.md b/examples/runtime_intelligence_chain/aiguard_edgeenv_handoff_alignment.md index e20bef9..2f40f8f 100644 --- a/examples/runtime_intelligence_chain/aiguard_edgeenv_handoff_alignment.md +++ b/examples/runtime_intelligence_chain/aiguard_edgeenv_handoff_alignment.md @@ -4,7 +4,7 @@ - recommendation: alignment_satisfied - decision_owner: lab - diagnosis_owner: aiguard -- lab_expected_report_markers: Runtime Intelligence Risk Summary, Orchestrator operation feed context, Orchestrator task event rollup, Lab EdgeEnv preservation context, AIGuard task event rollup evidence, AIGuard runtime operation anomalies, AIGuard remote dispatch event summary, AIGuard remote event summary consistency, Remote fallback starter evidence, AIGuard producer-lineage guard alignment, Lab remains the final deployment decision owner. +- lab_expected_report_markers: Runtime Intelligence Risk Summary, Orchestrator operation feed context, Orchestrator task event rollup, Lab EdgeEnv preservation context, AIGuard task event rollup evidence, AIGuard runtime operation anomalies, AIGuard remote dispatch event summary, AIGuard remote event summary consistency, Remote fallback starter evidence, lab=Remote fallback starter evidence; evidence=remote_execution_recovered_by_fallback, AIGuard producer-lineage guard alignment, Lab remains the final deployment decision owner. - report_marker_context_role: lab_report_contract_context - aiguard_validates_expected_report_markers: False - required_evidence_types: runtime_telemetry_context_coverage, edgeenv_orchestrator_producer_lineage, edgeenv_orchestrator_task_event_rollup, runtime_history_seed_run_config_traceability, runtime_queue_overload, runtime_thermal_instability, remote_execution_recovered_by_fallback diff --git a/examples/runtime_intelligence_chain/bundle_manifest.json b/examples/runtime_intelligence_chain/bundle_manifest.json index 6647a9b..db6ab4e 100644 --- a/examples/runtime_intelligence_chain/bundle_manifest.json +++ b/examples/runtime_intelligence_chain/bundle_manifest.json @@ -55,6 +55,7 @@ "AIGuard remote dispatch event summary", "AIGuard remote event summary consistency", "Remote fallback starter evidence", + "lab=Remote fallback starter evidence; evidence=remote_execution_recovered_by_fallback", "AIGuard producer-lineage guard alignment", "Lab remains the final deployment decision owner." ] diff --git a/examples/runtime_intelligence_chain/edgeenv_lab_handoff_manifest.json b/examples/runtime_intelligence_chain/edgeenv_lab_handoff_manifest.json index d8291c0..dd192c5 100644 --- a/examples/runtime_intelligence_chain/edgeenv_lab_handoff_manifest.json +++ b/examples/runtime_intelligence_chain/edgeenv_lab_handoff_manifest.json @@ -76,6 +76,7 @@ "AIGuard remote dispatch event summary", "AIGuard remote event summary consistency", "Remote fallback starter evidence", + "lab=Remote fallback starter evidence; evidence=remote_execution_recovered_by_fallback", "AIGuard producer-lineage guard alignment", "Lab remains the final deployment decision owner." ], diff --git a/scripts/check_runtime_intelligence_bundle_manifest.py b/scripts/check_runtime_intelligence_bundle_manifest.py index 53b8de6..520c9db 100644 --- a/scripts/check_runtime_intelligence_bundle_manifest.py +++ b/scripts/check_runtime_intelligence_bundle_manifest.py @@ -132,6 +132,7 @@ "AIGuard remote dispatch event summary", "AIGuard remote event summary consistency", "Remote fallback starter evidence", + "lab=Remote fallback starter evidence; evidence=remote_execution_recovered_by_fallback", "AIGuard producer-lineage guard alignment", "Lab remains the final deployment decision owner.", } @@ -167,6 +168,7 @@ "aiguard_raw_context: missing_telemetry_orchestrator_context preserved", "aiguard_handoff_alignment: external required evidence types satisfied", "expected_report_markers: Runtime Intelligence report markers declared", + "expected_report_markers: remote fallback Lab context row declared", ) EDGEENV_HANDOFF_SUMMARY_CONTRACT_MARKERS = ( "edgeenv_handoff: lab_bundle_alignment validated", @@ -423,6 +425,32 @@ def _validate_edgeenv_handoff_alignment( "AIGuard evidence types", ) + expected_report_markers = alignment.get("expected_report_markers") + _record( + isinstance(expected_report_markers, list), + errors, + "EdgeEnv handoff lab_bundle_alignment.expected_report_markers " + "must be a list", + ) + if isinstance(expected_report_markers, list): + invalid_markers = [ + marker + for marker in expected_report_markers + if not isinstance(marker, str) or not marker + ] + _record( + not invalid_markers, + errors, + "EdgeEnv handoff lab_bundle_alignment.expected_report_markers " + "must contain non-empty strings", + ) + _record( + set(expected_report_markers) == REQUIRED_EXPECTED_REPORT_MARKERS, + errors, + "EdgeEnv handoff lab_bundle_alignment.expected_report_markers " + "must match Lab-required Runtime Intelligence report markers", + ) + source_repositories = alignment.get("source_repositories") _record( isinstance(source_repositories, dict), diff --git a/scripts/check_runtime_intelligence_ci_artifacts.py b/scripts/check_runtime_intelligence_ci_artifacts.py index f1f09e0..7e17dc8 100644 --- a/scripts/check_runtime_intelligence_ci_artifacts.py +++ b/scripts/check_runtime_intelligence_ci_artifacts.py @@ -57,6 +57,7 @@ "aiguard_raw_context: missing_telemetry_orchestrator_context preserved", "aiguard_handoff_alignment: external required evidence types satisfied", "expected_report_markers: Runtime Intelligence report markers declared", + "expected_report_markers: remote fallback Lab context row declared", "edgeenv_handoff: lab_bundle_alignment validated", "edgeenv_handoff: runtime_telemetry_history validated", "edgeenv_handoff: external AIGuard evidence requirements declared", @@ -75,6 +76,7 @@ "AIGuard remote dispatch event summary", "AIGuard remote event summary consistency", "Remote fallback starter evidence", + "lab=Remote fallback starter evidence; evidence=remote_execution_recovered_by_fallback", "AIGuard producer-lineage guard alignment", "Lab remains the final deployment decision owner.", ) @@ -291,6 +293,8 @@ def _validate_aiguard_handoff_alignment( "AIGuard runtime operation anomalies, AIGuard remote dispatch event summary, " "AIGuard remote event summary consistency, " "Remote fallback starter evidence, " + "lab=Remote fallback starter evidence; " + "evidence=remote_execution_recovered_by_fallback, " "AIGuard producer-lineage guard alignment, " "Lab remains the final deployment decision owner.", "report_marker_context_role: lab_report_contract_context", diff --git a/tests/test_runtime_intelligence_bundle_manifest.py b/tests/test_runtime_intelligence_bundle_manifest.py index 87a25c0..f3d837b 100644 --- a/tests/test_runtime_intelligence_bundle_manifest.py +++ b/tests/test_runtime_intelligence_bundle_manifest.py @@ -123,6 +123,10 @@ def test_runtime_intelligence_bundle_manifest_gate_cli_passes(tmp_path): "expected_report_markers: Runtime Intelligence report markers declared" in summary ) + assert ( + "expected_report_markers: remote fallback Lab context row declared" + in summary + ) def test_runtime_intelligence_docs_describe_expected_report_markers(): @@ -291,6 +295,31 @@ def test_runtime_intelligence_bundle_manifest_gate_fails_for_bad_edgeenv_handoff ) in summary +def test_runtime_intelligence_bundle_manifest_gate_fails_for_handoff_missing_report_marker( + tmp_path, +): + handoff = json.loads(EDGEENV_HANDOFF.read_text(encoding="utf-8")) + handoff["lab_bundle_alignment"]["expected_report_markers"].remove( + "lab=Remote fallback starter evidence; evidence=remote_execution_recovered_by_fallback" + ) + handoff_path = tmp_path / "edgeenv_lab_handoff_manifest.json" + handoff_path.write_text(json.dumps(handoff), encoding="utf-8") + summary_path = tmp_path / "bundle_manifest_gate_summary.md" + + result = manifest_gate( + manifest=str(MANIFEST), + edgeenv_handoff=str(handoff_path), + summary_out=str(summary_path), + ) + + assert result == 2 + summary = summary_path.read_text(encoding="utf-8") + assert ( + "lab_bundle_alignment.expected_report_markers must match " + "Lab-required Runtime Intelligence report markers" + ) in summary + + def test_runtime_intelligence_bundle_manifest_gate_fails_for_bad_edgeenv_summary_guard_alignment( tmp_path, ): diff --git a/tests/test_runtime_intelligence_ci_template.py b/tests/test_runtime_intelligence_ci_template.py index 80ad3e3..f841346 100644 --- a/tests/test_runtime_intelligence_ci_template.py +++ b/tests/test_runtime_intelligence_ci_template.py @@ -155,6 +155,7 @@ def test_runtime_intelligence_ci_artifact_gate_passes_for_expected_outputs(tmp_p "- aiguard_raw_context: missing_telemetry_orchestrator_context preserved", "- aiguard_handoff_alignment: external required evidence types satisfied", "- expected_report_markers: Runtime Intelligence report markers declared", + "- expected_report_markers: remote fallback Lab context row declared", "- edgeenv_handoff: lab_bundle_alignment validated", "- edgeenv_handoff: runtime_telemetry_history validated", "- edgeenv_handoff: remote_dispatch_boundary preserved", @@ -174,7 +175,7 @@ def test_runtime_intelligence_ci_artifact_gate_passes_for_expected_outputs(tmp_p (report_dir / "aiguard_edgeenv_handoff_alignment.json").write_text( '{"schema_version":"inferedge-aiguard-edgeenv-handoff-alignment-v1",' '"status":"passed","decision_owner":"lab","diagnosis_owner":"aiguard",' - '"lab_expected_report_marker_count":11,' + '"lab_expected_report_marker_count":12,' '"lab_expected_report_markers":[' '"Runtime Intelligence Risk Summary",' '"Orchestrator operation feed context",' @@ -185,6 +186,7 @@ def test_runtime_intelligence_ci_artifact_gate_passes_for_expected_outputs(tmp_p '"AIGuard remote dispatch event summary",' '"AIGuard remote event summary consistency",' '"Remote fallback starter evidence",' + '"lab=Remote fallback starter evidence; evidence=remote_execution_recovered_by_fallback",' '"AIGuard producer-lineage guard alignment",' '"Lab remains the final deployment decision owner."],' '"lab_report_marker_owner":"lab",' @@ -203,7 +205,7 @@ def test_runtime_intelligence_ci_artifact_gate_passes_for_expected_outputs(tmp_p "- status: passed", "- decision_owner: lab", "- diagnosis_owner: aiguard", - "- lab_expected_report_markers: Runtime Intelligence Risk Summary, Orchestrator operation feed context, Orchestrator task event rollup, Lab EdgeEnv preservation context, AIGuard task event rollup evidence, AIGuard runtime operation anomalies, AIGuard remote dispatch event summary, AIGuard remote event summary consistency, Remote fallback starter evidence, AIGuard producer-lineage guard alignment, Lab remains the final deployment decision owner.", + "- lab_expected_report_markers: Runtime Intelligence Risk Summary, Orchestrator operation feed context, Orchestrator task event rollup, Lab EdgeEnv preservation context, AIGuard task event rollup evidence, AIGuard runtime operation anomalies, AIGuard remote dispatch event summary, AIGuard remote event summary consistency, Remote fallback starter evidence, lab=Remote fallback starter evidence; evidence=remote_execution_recovered_by_fallback, AIGuard producer-lineage guard alignment, Lab remains the final deployment decision owner.", "- report_marker_context_role: lab_report_contract_context", "- aiguard_validates_expected_report_markers: False", "- handoff_producer_lineage_guard_alignment_run_ids: edgeenv-smoke-candidate, edgeenv-smoke-missing", @@ -356,6 +358,7 @@ def test_runtime_intelligence_ci_artifact_gate_fails_for_missing_lab_marker_cont "- aiguard_raw_context: missing_telemetry_orchestrator_context preserved", "- aiguard_handoff_alignment: external required evidence types satisfied", "- expected_report_markers: Runtime Intelligence report markers declared", + "- expected_report_markers: remote fallback Lab context row declared", "- edgeenv_handoff: lab_bundle_alignment validated", "- edgeenv_handoff: runtime_telemetry_history validated", "- edgeenv_handoff: remote_dispatch_boundary preserved", @@ -643,6 +646,7 @@ def test_runtime_intelligence_ci_artifact_gate_fails_for_failed_deployment_risk( "- aiguard_raw_context: missing_telemetry_orchestrator_context preserved", "- aiguard_handoff_alignment: external required evidence types satisfied", "- expected_report_markers: Runtime Intelligence report markers declared", + "- expected_report_markers: remote fallback Lab context row declared", "- edgeenv_handoff: lab_bundle_alignment validated", "- edgeenv_handoff: runtime_telemetry_history validated", "- edgeenv_handoff: remote_dispatch_boundary preserved", @@ -662,7 +666,7 @@ def test_runtime_intelligence_ci_artifact_gate_fails_for_failed_deployment_risk( (report_dir / "aiguard_edgeenv_handoff_alignment.json").write_text( '{"schema_version":"inferedge-aiguard-edgeenv-handoff-alignment-v1",' '"status":"passed","decision_owner":"lab","diagnosis_owner":"aiguard",' - '"lab_expected_report_marker_count":11,' + '"lab_expected_report_marker_count":12,' '"lab_expected_report_markers":[' '"Runtime Intelligence Risk Summary",' '"Orchestrator operation feed context",' @@ -673,6 +677,7 @@ def test_runtime_intelligence_ci_artifact_gate_fails_for_failed_deployment_risk( '"AIGuard remote dispatch event summary",' '"AIGuard remote event summary consistency",' '"Remote fallback starter evidence",' + '"lab=Remote fallback starter evidence; evidence=remote_execution_recovered_by_fallback",' '"AIGuard producer-lineage guard alignment",' '"Lab remains the final deployment decision owner."],' '"lab_report_marker_owner":"lab",' @@ -691,7 +696,7 @@ def test_runtime_intelligence_ci_artifact_gate_fails_for_failed_deployment_risk( "- status: passed", "- decision_owner: lab", "- diagnosis_owner: aiguard", - "- lab_expected_report_markers: Runtime Intelligence Risk Summary, Orchestrator operation feed context, Orchestrator task event rollup, Lab EdgeEnv preservation context, AIGuard task event rollup evidence, AIGuard runtime operation anomalies, AIGuard remote dispatch event summary, AIGuard remote event summary consistency, Remote fallback starter evidence, AIGuard producer-lineage guard alignment, Lab remains the final deployment decision owner.", + "- lab_expected_report_markers: Runtime Intelligence Risk Summary, Orchestrator operation feed context, Orchestrator task event rollup, Lab EdgeEnv preservation context, AIGuard task event rollup evidence, AIGuard runtime operation anomalies, AIGuard remote dispatch event summary, AIGuard remote event summary consistency, Remote fallback starter evidence, lab=Remote fallback starter evidence; evidence=remote_execution_recovered_by_fallback, AIGuard producer-lineage guard alignment, Lab remains the final deployment decision owner.", "- report_marker_context_role: lab_report_contract_context", "- aiguard_validates_expected_report_markers: False", "- handoff_producer_lineage_guard_alignment_run_ids: edgeenv-smoke-candidate, edgeenv-smoke-missing",