An extended four-act demo, and the noise it exposed

README.md

@@ -37,6 +37,17 @@
 
 <br>
 
+<p align="center">
+  <img src="docs/demo/aicertify-animated.svg" alt="A terminal session. aicertify score-card bert-base-uncased reports completeness 0.49 against a threshold of 0.8 and prints BELOW THRESHOLD, with a per-section bar chart. aicertify explain eu_ai_act then reports 150 fields you must declare, which no evaluator can observe, and 14 fields produced by evaluators, noting that asserting your own fairness score defeats the point." width="92%" />
+</p>
+
+<p align="center">
+  <sub>Scoring a real Hugging Face card, then asking what the EU AI Act actually needs.<br>
+  <a href="https://principledevolution.ai/aicertify">Watch the full four-minute walkthrough</a>, including running a contract and the metrics still missing an evaluator.</sub>
+</p>
+
+<br>
+
 **AICertify is the open execution and evidence layer for AI governance.** Describe an AI system in a contract, supply the facts only your organisation can know, attach or compute measured metrics, evaluate that evidence against versioned [GOPAL](https://github.com/Principled-Evolution/gopal) policies through [Open Policy Agent](https://www.openpolicyagent.org/), and generate dated PDF, Markdown, JSON, or HTML reports.
 
 **The goal is simple: move from “we have an AI policy” to evidence another engineer, auditor, or risk team can inspect and reproduce.**

aicertify/opa_core/extraction.py

@@ -359,10 +359,19 @@ def extract_policy_results_with_schema(
         return extracted_policies
 
     except Exception as e:
-        logger.error(f"Error validating OPA results against schema: {e}")
-        # Dont try to recover using a more flexible approach
-        logger.warning(
-            "Failed to validate OPA results against schema, returning empty list"
+        # Expected, and the common case. This path validates against the
+        # `report_output` schema, which four of gopal's policies define. Every
+        # other policy fails it, and the caller then reads the decision rules
+        # instead (see the note on extract_results_from_packages below).
+        #
+        # Logged at error until the #78 fallback landed, which meant a normal
+        # successful evaluation printed a pydantic validation report per policy
+        # and looked to a first-time user like a crash. The behaviour was
+        # already correct; only the log level was left behind.
+        logger.debug(
+            "No report_output schema match (%s); the caller falls back to "
+            "decision rules.",
+            e.__class__.__name__,
         )
         return []
 

docs/demo/aicertify-short.cast

@@ -0,0 +1,264 @@
+{"version": 2, "width": 100, "height": 30, "idle_time_limit": 3.0, "title": "AICertify: 150 declared, 14 measured"}
+[0.6, "o", "$ "]
+[0.6857, "o", "a"]
+[0.7481, "o", "i"]
+[0.8326, "o", "c"]
+[0.8772, "o", "e"]
+[0.9482, "o", "r"]
+[0.993, "o", "t"]
+[1.0405, "o", "i"]
+[1.1202, "o", "f"]
+[1.1796, "o", "y"]
+[1.2503, "o", " "]
+[1.3196, "o", "s"]
+[1.3809, "o", "c"]
+[1.45, "o", "o"]
+[1.5289, "o", "r"]
+[1.5857, "o", "e"]
+[1.6409, "o", "-"]
+[1.7185, "o", "c"]
+[1.8039, "o", "a"]
+[1.8679, "o", "r"]
+[1.9161, "o", "d"]
+[1.9636, "o", " "]
+[2.0197, "o", "b"]
+[2.0779, "o", "e"]
+[2.1447, "o", "r"]
+[2.2007, "o", "t"]
+[2.2876, "o", "-"]
+[2.3404, "o", "b"]
+[2.4044, "o", "a"]
+[2.4619, "o", "s"]
+[2.5101, "o", "e"]
+[2.5927, "o", "-"]
+[2.661, "o", "u"]
+[2.7296, "o", "n"]
+[2.7831, "o", "c"]
+[2.8522, "o", "a"]
+[2.8905, "o", "s"]
+[2.9325, "o", "e"]
+[2.9761, "o", "d"]
+[3.3961, "o", "\r\n"]
+[5.4561, "o", "\r\nbert-base-uncased\r\n  completeness  0.49   (threshold 0.8)\r\n"]
+[5.5461, "o", "  quality       0.66\r\n  BELOW THRESHOLD\r\n\r\n"]
+[5.6361, "o", "    intended_use             1.00  ####################\r\n    training_data            0.67  #############\r\n    caveats_recommendations  0.50  ##########\r\n"]
+[5.7261, "o", "    factors                  0.50  ##########\r\n    metrics                  0.50  ##########\r\n    quantitative_analyses    0.50  ##########\r\n"]
+[5.8161, "o", "    ethical_considerations   0.33  #######\r\n    model_details            0.33  ######\r\n    evaluation_data          0.00  \r\n"]
+[5.9061, "o", "\r\nA model card answers part of what Annex IV asks and then stops.\r\nThis is the documentation-completeness gap, not a compliance verdict.\r\n"]
+[5.9961, "o", "\r\nscored by GOPAL 2.0.0 (global/v1/documentation/model_card_score), via aicertify 0.7.0\r\n"]
+[6.8861, "o", "\u001b7\u001b[18A\r\u001b[16C\u001b[43;30m0\u001b[0m\u001b8"]
+[6.9141, "o", "\u001b7\u001b[18A\r\u001b[16C\u001b[43;30m0.\u001b[0m\u001b8"]
+[6.9421, "o", "\u001b7\u001b[18A\r\u001b[16C\u001b[43;30m0.4\u001b[0m\u001b8"]
+[6.9701, "o", "\u001b7\u001b[18A\r\u001b[16C\u001b[43;30m0.49\u001b[0m\u001b8"]
+[7.3381, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mB\u001b[0m\u001b8"]
+[7.3661, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBE\u001b[0m\u001b8"]
+[7.3941, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBEL\u001b[0m\u001b8"]
+[7.4221, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELO\u001b[0m\u001b8"]
+[7.4501, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW\u001b[0m\u001b8"]
+[7.4781, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW \u001b[0m\u001b8"]
+[7.5061, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW T\u001b[0m\u001b8"]
+[7.5341, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW TH\u001b[0m\u001b8"]
+[7.5621, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THR\u001b[0m\u001b8"]
+[7.5901, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THRE\u001b[0m\u001b8"]
+[7.6181, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THRES\u001b[0m\u001b8"]
+[7.6461, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THRESH\u001b[0m\u001b8"]
+[7.6741, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THRESHO\u001b[0m\u001b8"]
+[7.7021, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THRESHOL\u001b[0m\u001b8"]
+[7.7301, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THRESHOLD\u001b[0m\u001b8"]
+[13.4981, "o", "$ "]
+[13.5533, "o", "#"]
+[13.6042, "o", " "]
+[13.6675, "o", "s"]
+[13.7216, "o", "o"]
+[13.7668, "o", " "]
+[13.8414, "o", "w"]
+[13.901, "o", "h"]
+[13.98, "o", "a"]
+[14.036, "o", "t"]
+[14.2622, "o", " "]
+[14.3358, "o", "d"]
+[14.407, "o", "o"]
+[14.452, "o", "e"]
+[14.5034, "o", "s"]
+[14.656, "o", " "]
+[14.7291, "o", "t"]
+[14.8083, "o", "h"]
+[14.8639, "o", "e"]
+[14.9825, "o", " "]
+[15.0553, "o", "E"]
+[15.1175, "o", "U"]
+[15.2782, "o", " "]
+[15.3602, "o", "A"]
+[15.4087, "o", "I"]
+[15.4983, "o", " "]
+[15.5678, "o", "A"]
+[15.6207, "o", "c"]
+[15.7021, "o", "t"]
+[15.8514, "o", " "]
+[15.9182, "o", "a"]
+[15.9948, "o", "c"]
+[16.0738, "o", "t"]
+[16.1586, "o", "u"]
+[16.2018, "o", "a"]
+[16.2789, "o", "l"]
+[16.3431, "o", "k"]
+[16.7652, "o", "\b \b"]
+[16.9292, "o", "l"]
+[17.0206, "o", "y"]
+[17.1123, "o", " "]
+[17.1793, "o", "a"]
+[17.2189, "o", "s"]
+[17.2605, "o", "k"]
+[17.304, "o", " "]
+[17.3619, "o", "f"]
+[17.432, "o", "o"]
+[17.4911, "o", "r"]
+[17.5342, "o", "?"]
+[17.9542, "o", "\r\n"]
+[18.6142, "o", "$ "]
+[18.6746, "o", "a"]
+[18.7332, "o", "i"]
+[18.8196, "o", "c"]
+[18.8938, "o", "e"]
+[18.9528, "o", "r"]
+[19.0121, "o", "t"]
+[19.0838, "o", "i"]
+[19.1662, "o", "f"]
+[19.2485, "o", "y"]
+[19.4441, "o", " "]
+[19.5332, "o", "e"]
+[19.625, "o", "x"]
+[19.6876, "o", "p"]
+[19.7399, "o", "l"]
+[19.8257, "o", "a"]
+[19.8648, "o", "i"]
+[19.9051, "o", "n"]
+[19.9678, "o", " "]
+[20.0131, "o", "e"]
+[20.075, "o", "u"]
+[20.1434, "o", "_"]
+[20.1838, "o", "a"]
+[20.2341, "o", "i"]
+[20.2763, "o", "_"]
+[20.3213, "o", "a"]
+[20.3978, "o", "c"]
+[20.4606, "o", "t"]
+[20.5349, "o", " "]
+[20.587, "o", "|"]
+[20.7452, "o", " "]
+[20.7897, "o", "h"]
+[20.8367, "o", "e"]
+[20.8764, "o", "a"]
+[20.9218, "o", "d"]
+[20.9953, "o", " "]
+[21.0738, "o", "-"]
+[21.1456, "o", "1"]
+[21.2333, "o", "2"]
+[21.6533, "o", "\r\n"]
+[23.7133, "o", "\r\neu_ai_act  \u2014  29 policies\r\n\r\n"]
+[23.8033, "o", "Fields you must declare (150)\r\n  No evaluator can observe these. They are facts about your system,\r\n  your process, or your paperwork, so you assert them in the contract.\r\n"]
+[23.8933, "o", "\r\n  accuracy.declared_in_instructions              technical_robustness\r\n  accuracy.metrics_declared                      technical_robustness\r\n"]
+[23.9833, "o", "  assessment.completed                           conformity_assessment\r\n  assessment.harmonised_standards_applied        conformity_assessment\r\n  assessment.notified_body_involved              conformity_assessment\r\n"]
+[24.7733, "o", "\u001b7\u001b[9A\r\u001b[25C\u001b[43;30m1\u001b[0m\u001b8"]
+[24.8013, "o", "\u001b7\u001b[9A\r\u001b[25C\u001b[43;30m15\u001b[0m\u001b8"]
+[24.8293, "o", "\u001b7\u001b[9A\r\u001b[25C\u001b[43;30m150\u001b[0m\u001b8"]
+[27.7973, "o", "$ "]
+[27.8433, "o", "a"]
+[27.9297, "o", "i"]
+[28.0189, "o", "c"]
+[28.07, "o", "e"]
+[28.1519, "o", "r"]
+[28.219, "o", "t"]
+[28.2995, "o", "i"]
+[28.3838, "o", "f"]
+[28.4619, "o", "y"]
+[28.519, "o", " "]
+[28.6104, "o", "e"]
+[28.6649, "o", "x"]
+[28.7462, "o", "p"]
+[28.7963, "o", "l"]
+[28.8422, "o", "a"]
+[28.9155, "o", "i"]
+[28.9987, "o", "n"]
+[29.1715, "o", " "]
+[29.2417, "o", "e"]
+[29.281, "o", "u"]
+[29.321, "o", "_"]
+[29.4052, "o", "a"]
+[29.447, "o", "i"]
+[29.4953, "o", "_"]
+[29.5566, "o", "a"]
+[29.5965, "o", "c"]
+[29.6743, "o", "t"]
+[29.8469, "o", " "]
+[29.929, "o", "|"]
+[30.0082, "o", " "]
+[30.0598, "o", "g"]
+[30.125, "o", "r"]
+[30.1909, "o", "e"]
+[30.2435, "o", "p"]
+[30.3152, "o", " "]
+[30.3924, "o", "-"]
+[30.4637, "o", "A"]
+[30.5366, "o", "8"]
+[30.5898, "o", " "]
+[30.6284, "o", "\""]
+[30.6883, "o", "p"]
+[30.7435, "o", "r"]
+[30.7848, "o", "o"]
+[30.8245, "o", "d"]
+[30.9021, "o", "u"]
+[30.9473, "o", "c"]
+[30.9886, "o", "e"]
+[31.0293, "o", "d"]
+[31.0939, "o", " "]
+[31.1713, "o", "b"]
+[31.2282, "o", "y"]
+[31.2966, "o", " "]
+[31.368, "o", "e"]
+[31.4421, "o", "v"]
+[31.5281, "o", "a"]
+[31.5911, "o", "l"]
+[31.6516, "o", "u"]
+[31.7086, "o", "a"]
+[31.7847, "o", "t"]
+[31.8314, "o", "o"]
+[31.8898, "o", "r"]
+[31.9389, "o", "s"]
+[32.004, "o", "\""]
+[32.424, "o", "\r\n"]
+[34.484, "o", "Fields produced by evaluators (14)\r\n  Computed at evaluation time from your interactions. Do not hand-write\r\n  these: asserting your own fairness score defeats the point.\r\n"]
+[34.574, "o", "\r\n  metrics.content_safety.score                   eu_fairness [content_safety evaluator]\r\n  metrics.emotion_recognition.combined_score     emotion_recognition\r\n"]
+[34.664, "o", "  metrics.emotion_recognition.educational_context emotion_recognition\r\n  metrics.emotion_recognition.workplace_context  emotion_recognition\r\n  metrics.fairness.gender_bias                   eu_fairness [fairness evaluator]\r\n"]
+[35.454, "o", "\u001b7\u001b[9A\r\u001b[31C\u001b[43;30m1\u001b[0m\u001b8"]
+[35.482, "o", "\u001b7\u001b[9A\r\u001b[31C\u001b[43;30m14\u001b[0m\u001b8"]
+[35.85, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30md\u001b[0m\u001b8"]
+[35.878, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mde\u001b[0m\u001b8"]
+[35.906, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdef\u001b[0m\u001b8"]
+[35.934, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefe\u001b[0m\u001b8"]
+[35.962, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefea\u001b[0m\u001b8"]
+[35.99, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeat\u001b[0m\u001b8"]
+[36.018, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats\u001b[0m\u001b8"]
+[36.046, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats \u001b[0m\u001b8"]
+[36.074, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats t\u001b[0m\u001b8"]
+[36.102, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats th\u001b[0m\u001b8"]
+[36.13, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the\u001b[0m\u001b8"]
+[36.158, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the \u001b[0m\u001b8"]
+[36.186, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the p\u001b[0m\u001b8"]
+[36.214, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the po\u001b[0m\u001b8"]
+[36.242, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the poi\u001b[0m\u001b8"]
+[36.27, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the poin\u001b[0m\u001b8"]
+[36.298, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the point\u001b[0m\u001b8"]
+[42.066, "o", "$ "]
+[42.066, "o", "\u2588"]
+[42.596, "o", "\b \b"]
+[43.126, "o", "\u2588"]
+[43.656, "o", "\b \b"]
+[44.186, "o", "\u2588"]
+[44.716, "o", "\b \b"]
+[45.246, "o", "\u2588"]
+[45.776, "o", "\b \b"]
+[46.306, "o", "\u2588"]
+[46.836, "o", "\b \b"]
+[47.366, "o", "\u2588"]
+[47.896, "o", "\b \b"]