From 7bd89c592f9a6e92d4392aa30509529a8a10a75e Mon Sep 17 00:00:00 2001 From: "kapil.madan" Date: Sun, 30 Aug 2026 18:11:01 +0530 Subject: [PATCH] fix(demo): end on a named provision, not on a score Three changes from reviewing the recording. Act 3 ended on an empty contract failing all 29 policies. True, and the argument act 2 makes, but as an ending it reads as "the tool does not work" to anyone who has not followed the reasoning. It now answers the 150 declarations first, the way a reader would, and ends on: Article 14 Controls Not Satisfied: ['Article 14(1)-(2) oversight designed into the system'] A named provision and the specific control. That is what the product is actually differentiated on and what a reader can act on, where a count reads as a grade. Act 4, the metric gap report, is dropped. Enumerating the metrics nobody has written an evaluator for is a good thing to show a contributor and a poor thing to show someone deciding whether to adopt. The command stays in the contributing section of the website page, where the audience is right. The README cut is 36 seconds rather than 48. It autoplays above the fold, so it has to be watchable before a reader decides to scroll. The page cut keeps its framing line and runs about 70 seconds, where a reader has chosen to watch. Typos now fire on a quarter of commands rather than half, which reads as a person typing rather than as someone who cannot type. Highlights re-verified by replaying both casts through a VT emulator: every row carrying a highlighted phrase matches what the command actually printed. --- README.md | 2 +- docs/demo/aicertify-animated.svg | 478 ++++++++-------- docs/demo/aicertify-short.cast | 470 +++++++--------- docs/demo/aicertify.cast | 922 +++++++++++++++---------------- scripts/record-demo.py | 208 +++++-- 5 files changed, 1020 insertions(+), 1060 deletions(-) diff --git a/README.md b/README.md index 7bddd77..db0042e 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@

Scoring a real Hugging Face card, then asking what the EU AI Act actually needs.
- Watch the full four-minute walkthrough, including running a contract and the metrics still missing an evaluator. + Watch the full walkthrough, which goes on to answer the declarations and evaluate them.


diff --git a/docs/demo/aicertify-animated.svg b/docs/demo/aicertify-animated.svg index 5c4765e..de0df1d 100644 --- a/docs/demo/aicertify-animated.svg +++ b/docs/demo/aicertify-animated.svg @@ -1,4 +1,4 @@ - + aicertify: 150 declared, 14 measured $ aicertify score-card bert-base-uncased @@ -21,7 +21,6 @@ A model card answers part of what Annex IV asks and then stops. This is the documentation-completeness gap, not a compliance verdict. scored by GOPAL 2.0.0 (global/v1/documentation/model_card_score), via aicertify 0.7.0 -$ # so what does the EU AI Act actually ask for? $ aicertify explain eu_ai_act | head -12 eu_ai_act — 29 policies @@ -46,18 +45,18 @@ Fields produced by evaluators (14) metrics.emotion_recognition.workplace_context emotion_recognition metrics.fairness.gender_bias eu_fairness [fairness evaluator] $ - - + + aicertify: 150 declared, 14 measured - - - - - + + + + + $ a @@ -155,253 +154,216 @@ $ This is the documentation-completeness gap, not a compliance verdict. scored by GOPAL 2.0.0 (global/v1/documentation/model_card_score), via aicertify 0.7.0 $ -# -s -o -w -h -a -t -d -o -e -s -t -h -e -E -U -A -I -A -c -t -a -c -t -u -a -l -k -l -y -a -s -k -f -o -r -? -$ -a -i -c -e -r -t -i -f -y -e -x -p -l -a -i -n -e -u -_ -a -i -_ -a -c -t -| -h -e -a -d -- -1 -2 -eu_ai_act — 29 policies -F -i -e -l -d -s -y -o -u -m -u -s -t -d -e -c -l -a -r -e -( -1 -5 -0 -) - No evaluator can observe these. They are facts about your system, - your process, or your paperwork, so you assert them in the contract. - accuracy.declared_in_instructions technical_robustness - accuracy.metrics_declared technical_robustness - assessment.completed conformity_assessment - assessment.harmonised_standards_applied conformity_assessment - assessment.notified_body_involved conformity_assessment -$ -a -i -c -e -r -t -i -f -y -e -x -p -l -a -i -n -e -u -_ -a -i -_ -a -c -t -| -g -r -e -p -- -A -8 -" -p -r -o -d -u -c -e -d -b -y -e -v -a -l -u -a -t -o -r -s -" -F -i -e -l -d -s -p -r -o -d -u -c -e -d -b -y -e -v -a -l -u -a -t -o -r -s -( -1 -4 -) - Computed at evaluation time from your interactions. Do not hand-write -t -h -e -s -e -: -a -s -s -e -r -t -i -n -g -y -o -u -r -o -w -n -f -a -i -r -n -e -s -s -s -c -o -r -e -d -e -f -e -a -t -s -t -h -e -p -o -i -n -t -. - metrics.content_safety.score eu_fairness [content_safety evaluator] - metrics.emotion_recognition.combined_score emotion_recognition - metrics.emotion_recognition.educational_context emotion_recognition - metrics.emotion_recognition.workplace_context emotion_recognition - metrics.fairness.gender_bias eu_fairness [fairness evaluator] -$ -$ █ +a +i +c +e +r +t +i +f +y +e +x +p +l +a +i +n +e +u +_ +a +i +_ +a +c +t +| +h +e +a +d +- +1 +2 +eu_ai_act — 29 policies +F +i +e +l +d +s +y +o +u +m +u +s +t +d +e +c +l +a +r +e +( +1 +5 +0 +) + No evaluator can observe these. They are facts about your system, + your process, or your paperwork, so you assert them in the contract. + accuracy.declared_in_instructions technical_robustness + accuracy.metrics_declared technical_robustness + assessment.completed conformity_assessment + assessment.harmonised_standards_applied conformity_assessment + assessment.notified_body_involved conformity_assessment +$ +a +i +c +e +r +t +i +f +y +r +e +x +p +l +a +i +n +e +u +_ +a +i +_ +a +c +t +| +g +r +e +p +- +A +8 +" +p +r +o +d +u +c +e +d +b +y +e +v +a +l +u +a +t +o +r +s +" +F +i +e +l +d +s +p +r +o +d +u +c +e +d +b +y +e +v +a +l +u +a +t +o +r +s +( +1 +4 +) + Computed at evaluation time from your interactions. Do not hand-write +t +h +e +s +e +: +a +s +s +e +r +t +i +n +g +y +o +u +r +o +w +n +f +a +i +r +n +e +s +s +s +c +o +r +e +d +e +f +e +a +t +s +t +h +e +p +o +i +n +t +. + metrics.content_safety.score eu_fairness [content_safety evaluator] + metrics.emotion_recognition.combined_score emotion_recognition + metrics.emotion_recognition.educational_context emotion_recognition + metrics.emotion_recognition.workplace_context emotion_recognition + metrics.fairness.gender_bias eu_fairness [fairness evaluator] +$ +$ █ diff --git a/docs/demo/aicertify-short.cast b/docs/demo/aicertify-short.cast index c827fe8..ae21e11 100644 --- a/docs/demo/aicertify-short.cast +++ b/docs/demo/aicertify-short.cast @@ -1,264 +1,210 @@ {"version": 2, "width": 100, "height": 30, "idle_time_limit": 3.0, "title": "AICertify: 150 declared, 14 measured"} [0.6, "o", "$ "] -[0.6857, "o", "a"] -[0.7481, "o", "i"] -[0.8326, "o", "c"] -[0.8772, "o", "e"] -[0.9482, "o", "r"] -[0.993, "o", "t"] -[1.0405, "o", "i"] -[1.1202, "o", "f"] -[1.1796, "o", "y"] -[1.2503, "o", " "] -[1.3196, "o", "s"] -[1.3809, "o", "c"] -[1.45, "o", "o"] -[1.5289, "o", "r"] -[1.5857, "o", "e"] -[1.6409, "o", "-"] -[1.7185, "o", "c"] -[1.8039, "o", "a"] -[1.8679, "o", "r"] -[1.9161, "o", "d"] -[1.9636, "o", " "] -[2.0197, "o", "b"] -[2.0779, "o", "e"] -[2.1447, "o", "r"] -[2.2007, "o", "t"] -[2.2876, "o", "-"] -[2.3404, "o", "b"] -[2.4044, "o", "a"] -[2.4619, "o", "s"] -[2.5101, "o", "e"] -[2.5927, "o", "-"] -[2.661, "o", "u"] -[2.7296, "o", "n"] -[2.7831, "o", "c"] -[2.8522, "o", "a"] -[2.8905, "o", "s"] -[2.9325, "o", "e"] -[2.9761, "o", "d"] -[3.3961, "o", "\r\n"] -[5.4561, "o", "\r\nbert-base-uncased\r\n completeness 0.49 (threshold 0.8)\r\n"] -[5.5461, "o", " quality 0.66\r\n BELOW THRESHOLD\r\n\r\n"] -[5.6361, "o", " intended_use 1.00 ####################\r\n training_data 0.67 #############\r\n caveats_recommendations 0.50 ##########\r\n"] -[5.7261, "o", " factors 0.50 ##########\r\n metrics 0.50 ##########\r\n quantitative_analyses 0.50 ##########\r\n"] -[5.8161, "o", " ethical_considerations 0.33 #######\r\n model_details 0.33 ######\r\n evaluation_data 0.00 \r\n"] -[5.9061, "o", "\r\nA model card answers part of what Annex IV asks and then stops.\r\nThis is the documentation-completeness gap, not a compliance verdict.\r\n"] -[5.9961, "o", "\r\nscored by GOPAL 2.0.0 (global/v1/documentation/model_card_score), via aicertify 0.7.0\r\n"] -[6.8861, "o", "\u001b7\u001b[18A\r\u001b[16C\u001b[43;30m0\u001b[0m\u001b8"] -[6.9141, "o", "\u001b7\u001b[18A\r\u001b[16C\u001b[43;30m0.\u001b[0m\u001b8"] -[6.9421, "o", "\u001b7\u001b[18A\r\u001b[16C\u001b[43;30m0.4\u001b[0m\u001b8"] -[6.9701, "o", "\u001b7\u001b[18A\r\u001b[16C\u001b[43;30m0.49\u001b[0m\u001b8"] -[7.3381, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mB\u001b[0m\u001b8"] -[7.3661, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBE\u001b[0m\u001b8"] -[7.3941, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBEL\u001b[0m\u001b8"] -[7.4221, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELO\u001b[0m\u001b8"] -[7.4501, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW\u001b[0m\u001b8"] -[7.4781, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW \u001b[0m\u001b8"] -[7.5061, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW T\u001b[0m\u001b8"] -[7.5341, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW TH\u001b[0m\u001b8"] -[7.5621, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THR\u001b[0m\u001b8"] -[7.5901, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THRE\u001b[0m\u001b8"] -[7.6181, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THRES\u001b[0m\u001b8"] -[7.6461, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THRESH\u001b[0m\u001b8"] -[7.6741, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THRESHO\u001b[0m\u001b8"] -[7.7021, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THRESHOL\u001b[0m\u001b8"] -[7.7301, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THRESHOLD\u001b[0m\u001b8"] -[13.4981, "o", "$ "] -[13.5533, "o", "#"] -[13.6042, "o", " "] -[13.6675, "o", "s"] -[13.7216, "o", "o"] -[13.7668, "o", " "] -[13.8414, "o", "w"] -[13.901, "o", "h"] -[13.98, "o", "a"] -[14.036, "o", "t"] -[14.2622, "o", " "] -[14.3358, "o", "d"] -[14.407, "o", "o"] -[14.452, "o", "e"] -[14.5034, "o", "s"] -[14.656, "o", " "] -[14.7291, "o", "t"] -[14.8083, "o", "h"] -[14.8639, "o", "e"] -[14.9825, "o", " "] -[15.0553, "o", "E"] -[15.1175, "o", "U"] -[15.2782, "o", " "] -[15.3602, "o", "A"] -[15.4087, "o", "I"] -[15.4983, "o", " "] -[15.5678, "o", "A"] -[15.6207, "o", "c"] -[15.7021, "o", "t"] -[15.8514, "o", " "] -[15.9182, "o", "a"] -[15.9948, "o", "c"] -[16.0738, "o", "t"] -[16.1586, "o", "u"] -[16.2018, "o", "a"] -[16.2789, "o", "l"] -[16.3431, "o", "k"] -[16.7652, "o", "\b \b"] -[16.9292, "o", "l"] -[17.0206, "o", "y"] -[17.1123, "o", " "] -[17.1793, "o", "a"] -[17.2189, "o", "s"] -[17.2605, "o", "k"] -[17.304, "o", " "] -[17.3619, "o", "f"] -[17.432, "o", "o"] -[17.4911, "o", "r"] -[17.5342, "o", "?"] -[17.9542, "o", "\r\n"] -[18.6142, "o", "$ "] -[18.6746, "o", "a"] -[18.7332, "o", "i"] -[18.8196, "o", "c"] -[18.8938, "o", "e"] -[18.9528, "o", "r"] -[19.0121, "o", "t"] -[19.0838, "o", "i"] -[19.1662, "o", "f"] -[19.2485, "o", "y"] -[19.4441, "o", " "] -[19.5332, "o", "e"] -[19.625, "o", "x"] -[19.6876, "o", "p"] -[19.7399, "o", "l"] -[19.8257, "o", "a"] -[19.8648, "o", "i"] -[19.9051, "o", "n"] -[19.9678, "o", " "] -[20.0131, "o", "e"] -[20.075, "o", "u"] -[20.1434, "o", "_"] -[20.1838, "o", "a"] -[20.2341, "o", "i"] -[20.2763, "o", "_"] -[20.3213, "o", "a"] -[20.3978, "o", "c"] -[20.4606, "o", "t"] -[20.5349, "o", " "] -[20.587, "o", "|"] -[20.7452, "o", " "] -[20.7897, "o", "h"] -[20.8367, "o", "e"] -[20.8764, "o", "a"] -[20.9218, "o", "d"] -[20.9953, "o", " "] -[21.0738, "o", "-"] -[21.1456, "o", "1"] -[21.2333, "o", "2"] -[21.6533, "o", "\r\n"] -[23.7133, "o", "\r\neu_ai_act \u2014 29 policies\r\n\r\n"] -[23.8033, "o", "Fields you must declare (150)\r\n No evaluator can observe these. They are facts about your system,\r\n your process, or your paperwork, so you assert them in the contract.\r\n"] -[23.8933, "o", "\r\n accuracy.declared_in_instructions technical_robustness\r\n accuracy.metrics_declared technical_robustness\r\n"] -[23.9833, "o", " assessment.completed conformity_assessment\r\n assessment.harmonised_standards_applied conformity_assessment\r\n assessment.notified_body_involved conformity_assessment\r\n"] -[24.7733, "o", "\u001b7\u001b[9A\r\u001b[25C\u001b[43;30m1\u001b[0m\u001b8"] -[24.8013, "o", "\u001b7\u001b[9A\r\u001b[25C\u001b[43;30m15\u001b[0m\u001b8"] -[24.8293, "o", "\u001b7\u001b[9A\r\u001b[25C\u001b[43;30m150\u001b[0m\u001b8"] -[27.7973, "o", "$ "] -[27.8433, "o", "a"] -[27.9297, "o", "i"] -[28.0189, "o", "c"] -[28.07, "o", "e"] -[28.1519, "o", "r"] -[28.219, "o", "t"] -[28.2995, "o", "i"] -[28.3838, "o", "f"] -[28.4619, "o", "y"] -[28.519, "o", " "] -[28.6104, "o", "e"] -[28.6649, "o", "x"] -[28.7462, "o", "p"] -[28.7963, "o", "l"] -[28.8422, "o", "a"] -[28.9155, "o", "i"] -[28.9987, "o", "n"] -[29.1715, "o", " "] -[29.2417, "o", "e"] -[29.281, "o", "u"] -[29.321, "o", "_"] -[29.4052, "o", "a"] -[29.447, "o", "i"] -[29.4953, "o", "_"] -[29.5566, "o", "a"] -[29.5965, "o", "c"] -[29.6743, "o", "t"] -[29.8469, "o", " "] -[29.929, "o", "|"] -[30.0082, "o", " "] -[30.0598, "o", "g"] -[30.125, "o", "r"] -[30.1909, "o", "e"] -[30.2435, "o", "p"] -[30.3152, "o", " "] -[30.3924, "o", "-"] -[30.4637, "o", "A"] -[30.5366, "o", "8"] -[30.5898, "o", " "] -[30.6284, "o", "\""] -[30.6883, "o", "p"] -[30.7435, "o", "r"] -[30.7848, "o", "o"] -[30.8245, "o", "d"] -[30.9021, "o", "u"] -[30.9473, "o", "c"] -[30.9886, "o", "e"] -[31.0293, "o", "d"] -[31.0939, "o", " "] -[31.1713, "o", "b"] -[31.2282, "o", "y"] -[31.2966, "o", " "] -[31.368, "o", "e"] -[31.4421, "o", "v"] -[31.5281, "o", "a"] -[31.5911, "o", "l"] -[31.6516, "o", "u"] -[31.7086, "o", "a"] -[31.7847, "o", "t"] -[31.8314, "o", "o"] -[31.8898, "o", "r"] -[31.9389, "o", "s"] -[32.004, "o", "\""] -[32.424, "o", "\r\n"] -[34.484, "o", "Fields produced by evaluators (14)\r\n Computed at evaluation time from your interactions. Do not hand-write\r\n these: asserting your own fairness score defeats the point.\r\n"] -[34.574, "o", "\r\n metrics.content_safety.score eu_fairness [content_safety evaluator]\r\n metrics.emotion_recognition.combined_score emotion_recognition\r\n"] -[34.664, "o", " metrics.emotion_recognition.educational_context emotion_recognition\r\n metrics.emotion_recognition.workplace_context emotion_recognition\r\n metrics.fairness.gender_bias eu_fairness [fairness evaluator]\r\n"] -[35.454, "o", "\u001b7\u001b[9A\r\u001b[31C\u001b[43;30m1\u001b[0m\u001b8"] -[35.482, "o", "\u001b7\u001b[9A\r\u001b[31C\u001b[43;30m14\u001b[0m\u001b8"] -[35.85, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30md\u001b[0m\u001b8"] -[35.878, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mde\u001b[0m\u001b8"] -[35.906, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdef\u001b[0m\u001b8"] -[35.934, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefe\u001b[0m\u001b8"] -[35.962, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefea\u001b[0m\u001b8"] -[35.99, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeat\u001b[0m\u001b8"] -[36.018, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats\u001b[0m\u001b8"] -[36.046, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats \u001b[0m\u001b8"] -[36.074, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats t\u001b[0m\u001b8"] -[36.102, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats th\u001b[0m\u001b8"] -[36.13, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the\u001b[0m\u001b8"] -[36.158, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the \u001b[0m\u001b8"] -[36.186, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the p\u001b[0m\u001b8"] -[36.214, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the po\u001b[0m\u001b8"] -[36.242, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the poi\u001b[0m\u001b8"] -[36.27, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the poin\u001b[0m\u001b8"] -[36.298, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the point\u001b[0m\u001b8"] -[42.066, "o", "$ "] -[42.066, "o", "\u2588"] -[42.596, "o", "\b \b"] -[43.126, "o", "\u2588"] -[43.656, "o", "\b \b"] -[44.186, "o", "\u2588"] -[44.716, "o", "\b \b"] -[45.246, "o", "\u2588"] -[45.776, "o", "\b \b"] -[46.306, "o", "\u2588"] -[46.836, "o", "\b \b"] -[47.366, "o", "\u2588"] -[47.896, "o", "\b \b"] +[0.6912, "o", "a"] +[0.7544, "o", "i"] +[0.8074, "o", "c"] +[0.8904, "o", "e"] +[0.9385, "o", "r"] +[1.0087, "o", "t"] +[1.0861, "o", "i"] +[1.1464, "o", "f"] +[1.2163, "o", "y"] +[1.3685, "o", " "] +[1.4257, "o", "s"] +[1.469, "o", "c"] +[1.5596, "o", "o"] +[1.6319, "o", "r"] +[1.6926, "o", "e"] +[1.7633, "o", "-"] +[1.8453, "o", "c"] +[1.8879, "o", "a"] +[1.9697, "o", "r"] +[2.0137, "o", "d"] +[2.1456, "o", " "] +[2.23, "o", "b"] +[2.2811, "o", "e"] +[2.3311, "o", "r"] +[2.4192, "o", "t"] +[2.4648, "o", "-"] +[2.5295, "o", "b"] +[2.582, "o", "a"] +[2.6272, "o", "s"] +[2.705, "o", "e"] +[2.7881, "o", "-"] +[2.8655, "o", "u"] +[2.9474, "o", "n"] +[3.0034, "o", "c"] +[3.0755, "o", "a"] +[3.1184, "o", "s"] +[3.1855, "o", "e"] +[3.2576, "o", "d"] +[3.6776, "o", "\r\n"] +[5.7376, "o", "\r\nbert-base-uncased\r\n completeness 0.49 (threshold 0.8)\r\n"] +[5.8276, "o", " quality 0.66\r\n BELOW THRESHOLD\r\n\r\n"] +[5.9176, "o", " intended_use 1.00 ####################\r\n training_data 0.67 #############\r\n caveats_recommendations 0.50 ##########\r\n"] +[6.0076, "o", " factors 0.50 ##########\r\n metrics 0.50 ##########\r\n quantitative_analyses 0.50 ##########\r\n"] +[6.0976, "o", " ethical_considerations 0.33 #######\r\n model_details 0.33 ######\r\n evaluation_data 0.00 \r\n"] +[6.1876, "o", "\r\nA model card answers part of what Annex IV asks and then stops.\r\nThis is the documentation-completeness gap, not a compliance verdict.\r\n"] +[6.2776, "o", "\r\nscored by GOPAL 2.0.0 (global/v1/documentation/model_card_score), via aicertify 0.7.0\r\n"] +[7.1676, "o", "\u001b7\u001b[18A\r\u001b[16C\u001b[43;30m0\u001b[0m\u001b8"] +[7.1896, "o", "\u001b7\u001b[18A\r\u001b[16C\u001b[43;30m0.\u001b[0m\u001b8"] +[7.2116, "o", "\u001b7\u001b[18A\r\u001b[16C\u001b[43;30m0.4\u001b[0m\u001b8"] +[7.2336, "o", "\u001b7\u001b[18A\r\u001b[16C\u001b[43;30m0.49\u001b[0m\u001b8"] +[7.4956, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mB\u001b[0m\u001b8"] +[7.5176, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBE\u001b[0m\u001b8"] +[7.5396, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBEL\u001b[0m\u001b8"] +[7.5616, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELO\u001b[0m\u001b8"] +[7.5836, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW\u001b[0m\u001b8"] +[7.6056, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW \u001b[0m\u001b8"] +[7.6276, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW T\u001b[0m\u001b8"] +[7.6496, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW TH\u001b[0m\u001b8"] +[7.6716, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THR\u001b[0m\u001b8"] +[7.6936, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THRE\u001b[0m\u001b8"] +[7.7156, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THRES\u001b[0m\u001b8"] +[7.7376, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THRESH\u001b[0m\u001b8"] +[7.7596, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THRESHO\u001b[0m\u001b8"] +[7.7816, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THRESHOL\u001b[0m\u001b8"] +[7.8036, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THRESHOLD\u001b[0m\u001b8"] +[11.4656, "o", "$ "] +[11.5336, "o", "a"] +[11.5788, "o", "i"] +[11.6534, "o", "c"] +[11.712, "o", "e"] +[11.7642, "o", "r"] +[11.8347, "o", "t"] +[11.9041, "o", "i"] +[11.9898, "o", "f"] +[12.0522, "o", "y"] +[12.254, "o", " "] +[12.2988, "o", "e"] +[12.3462, "o", "x"] +[12.426, "o", "p"] +[12.4854, "o", "l"] +[12.5561, "o", "a"] +[12.6407, "o", "i"] +[12.7101, "o", "n"] +[12.7714, "o", " "] +[12.8502, "o", "e"] +[12.9071, "o", "u"] +[12.9623, "o", "_"] +[13.0399, "o", "a"] +[13.1253, "o", "i"] +[13.1893, "o", "_"] +[13.2375, "o", "a"] +[13.2849, "o", "c"] +[13.365, "o", "t"] +[13.4212, "o", " "] +[13.4879, "o", "|"] +[13.544, "o", " "] +[13.5967, "o", "h"] +[13.6607, "o", "e"] +[13.7182, "o", "a"] +[13.7664, "o", "d"] +[13.849, "o", " "] +[13.9176, "o", "-"] +[13.9711, "o", "1"] +[14.0401, "o", "2"] +[14.4601, "o", "\r\n"] +[16.5201, "o", "\r\neu_ai_act \u2014 29 policies\r\n\r\n"] +[16.6101, "o", "Fields you must declare (150)\r\n No evaluator can observe these. They are facts about your system,\r\n your process, or your paperwork, so you assert them in the contract.\r\n"] +[16.7001, "o", "\r\n accuracy.declared_in_instructions technical_robustness\r\n accuracy.metrics_declared technical_robustness\r\n"] +[16.7901, "o", " assessment.completed conformity_assessment\r\n assessment.harmonised_standards_applied conformity_assessment\r\n assessment.notified_body_involved conformity_assessment\r\n"] +[17.5801, "o", "\u001b7\u001b[9A\r\u001b[25C\u001b[43;30m1\u001b[0m\u001b8"] +[17.6021, "o", "\u001b7\u001b[9A\r\u001b[25C\u001b[43;30m15\u001b[0m\u001b8"] +[17.6241, "o", "\u001b7\u001b[9A\r\u001b[25C\u001b[43;30m150\u001b[0m\u001b8"] +[19.8861, "o", "$ "] +[19.9328, "o", "a"] +[19.9885, "o", "i"] +[20.0562, "o", "c"] +[20.1188, "o", "e"] +[20.2067, "o", "r"] +[20.2856, "o", "t"] +[20.353, "o", "i"] +[20.4393, "o", "f"] +[20.5046, "o", "y"] +[20.5794, "o", " "] +[20.6487, "o", "r"] +[20.9188, "o", "\b \b"] +[21.1028, "o", "e"] +[21.1488, "o", "x"] +[21.2315, "o", "p"] +[21.3216, "o", "l"] +[21.4096, "o", "a"] +[21.4889, "o", "i"] +[21.5723, "o", "n"] +[21.6689, "o", " "] +[21.7097, "o", "e"] +[21.7587, "o", "u"] +[21.8502, "o", "_"] +[21.9086, "o", "a"] +[21.9724, "o", "i"] +[22.0319, "o", "_"] +[22.0953, "o", "a"] +[22.1819, "o", "c"] +[22.2568, "o", "t"] +[22.3368, "o", " "] +[22.3774, "o", "|"] +[22.4359, "o", " "] +[22.5195, "o", "g"] +[22.5794, "o", "r"] +[22.6208, "o", "e"] +[22.6889, "o", "p"] +[22.7582, "o", " "] +[22.8177, "o", "-"] +[22.8905, "o", "A"] +[22.9497, "o", "8"] +[23.1584, "o", " "] +[23.2477, "o", "\""] +[23.2901, "o", "p"] +[23.3629, "o", "r"] +[23.4036, "o", "o"] +[23.4569, "o", "d"] +[23.5189, "o", "u"] +[23.604, "o", "c"] +[23.6869, "o", "e"] +[23.7259, "o", "d"] +[23.8104, "o", " "] +[23.8564, "o", "b"] +[23.9131, "o", "y"] +[24.0893, "o", " "] +[24.1714, "o", "e"] +[24.2187, "o", "v"] +[24.2903, "o", "a"] +[24.38, "o", "l"] +[24.4512, "o", "u"] +[24.49, "o", "a"] +[24.5682, "o", "t"] +[24.6222, "o", "o"] +[24.6827, "o", "r"] +[24.7723, "o", "s"] +[24.841, "o", "\""] +[25.261, "o", "\r\n"] +[27.321, "o", "Fields produced by evaluators (14)\r\n Computed at evaluation time from your interactions. Do not hand-write\r\n these: asserting your own fairness score defeats the point.\r\n"] +[27.411, "o", "\r\n metrics.content_safety.score eu_fairness [content_safety evaluator]\r\n metrics.emotion_recognition.combined_score emotion_recognition\r\n"] +[27.501, "o", " metrics.emotion_recognition.educational_context emotion_recognition\r\n metrics.emotion_recognition.workplace_context emotion_recognition\r\n metrics.fairness.gender_bias eu_fairness [fairness evaluator]\r\n"] +[28.291, "o", "\u001b7\u001b[9A\r\u001b[31C\u001b[43;30m1\u001b[0m\u001b8"] +[28.313, "o", "\u001b7\u001b[9A\r\u001b[31C\u001b[43;30m14\u001b[0m\u001b8"] +[28.575, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30md\u001b[0m\u001b8"] +[28.597, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mde\u001b[0m\u001b8"] +[28.619, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdef\u001b[0m\u001b8"] +[28.641, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefe\u001b[0m\u001b8"] +[28.663, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefea\u001b[0m\u001b8"] +[28.685, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeat\u001b[0m\u001b8"] +[28.707, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats\u001b[0m\u001b8"] +[28.729, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats \u001b[0m\u001b8"] +[28.751, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats t\u001b[0m\u001b8"] +[28.773, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats th\u001b[0m\u001b8"] +[28.795, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the\u001b[0m\u001b8"] +[28.817, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the \u001b[0m\u001b8"] +[28.839, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the p\u001b[0m\u001b8"] +[28.861, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the po\u001b[0m\u001b8"] +[28.883, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the poi\u001b[0m\u001b8"] +[28.905, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the poin\u001b[0m\u001b8"] +[28.927, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the point\u001b[0m\u001b8"] +[32.589, "o", "$ "] +[32.589, "o", "\u2588"] +[33.119, "o", "\b \b"] +[33.649, "o", "\u2588"] +[34.179, "o", "\b \b"] +[34.709, "o", "\u2588"] +[35.239, "o", "\b \b"] diff --git a/docs/demo/aicertify.cast b/docs/demo/aicertify.cast index 84700db..b096f36 100644 --- a/docs/demo/aicertify.cast +++ b/docs/demo/aicertify.cast @@ -49,486 +49,442 @@ [6.7427, "o", "\r\nA model card answers part of what Annex IV asks and then stops.\r\nThis is the documentation-completeness gap, not a compliance verdict.\r\n"] [6.8327, "o", "\r\nscored by GOPAL 2.0.0 (global/v1/documentation/model_card_score), via aicertify 0.7.0\r\n"] [7.7227, "o", "\u001b7\u001b[18A\r\u001b[16C\u001b[43;30m0\u001b[0m\u001b8"] -[7.7507, "o", "\u001b7\u001b[18A\r\u001b[16C\u001b[43;30m0.\u001b[0m\u001b8"] -[7.7787, "o", "\u001b7\u001b[18A\r\u001b[16C\u001b[43;30m0.4\u001b[0m\u001b8"] -[7.8067, "o", "\u001b7\u001b[18A\r\u001b[16C\u001b[43;30m0.49\u001b[0m\u001b8"] -[8.1747, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mB\u001b[0m\u001b8"] -[8.2027, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBE\u001b[0m\u001b8"] -[8.2307, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBEL\u001b[0m\u001b8"] -[8.2587, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELO\u001b[0m\u001b8"] -[8.2867, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW\u001b[0m\u001b8"] -[8.3147, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW \u001b[0m\u001b8"] -[8.3427, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW T\u001b[0m\u001b8"] -[8.3707, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW TH\u001b[0m\u001b8"] -[8.3987, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THR\u001b[0m\u001b8"] -[8.4267, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THRE\u001b[0m\u001b8"] -[8.4547, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THRES\u001b[0m\u001b8"] -[8.4827, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THRESH\u001b[0m\u001b8"] -[8.5107, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THRESHO\u001b[0m\u001b8"] -[8.5387, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THRESHOL\u001b[0m\u001b8"] -[8.5667, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THRESHOLD\u001b[0m\u001b8"] -[14.3347, "o", "$ "] -[14.3728, "o", "#"] -[14.4216, "o", " "] -[14.4751, "o", "s"] -[14.5457, "o", "o"] -[14.6038, "o", " "] -[14.6894, "o", "w"] -[14.7391, "o", "h"] -[14.8281, "o", "a"] -[14.8792, "o", "t"] -[14.9428, "o", " "] -[15.0337, "o", "d"] -[15.1247, "o", "o"] -[15.2054, "o", "e"] -[15.261, "o", "s"] -[15.3275, "o", " "] -[15.3931, "o", "t"] -[15.4326, "o", "h"] -[15.5002, "o", "e"] -[15.5741, "o", " "] -[15.6338, "o", "E"] -[15.6756, "o", "U"] -[15.7291, "o", " "] -[15.8146, "o", "A"] -[15.8902, "o", "I"] -[15.9434, "o", " "] -[16.0148, "o", "A"] -[16.0948, "o", "c"] -[16.146, "o", "t"] -[16.1997, "o", " "] -[16.2682, "o", "a"] -[16.308, "o", "c"] -[16.3465, "o", "t"] -[16.3926, "o", "u"] -[16.4409, "o", "a"] -[16.5198, "o", "l"] -[16.58, "o", "l"] -[16.62, "o", "y"] -[16.7305, "o", " "] -[16.8186, "o", "a"] -[16.8661, "o", "s"] -[16.936, "o", "k"] -[17.004, "o", " "] -[17.0739, "o", "f"] -[17.1166, "o", "o"] -[17.1599, "o", "r"] -[17.2375, "o", "?"] -[17.6575, "o", "\r\n"] -[18.3175, "o", "$ "] -[18.3736, "o", "a"] -[18.4514, "o", "i"] -[18.5393, "o", "c"] -[18.6178, "o", "e"] -[18.6876, "o", "r"] -[18.7719, "o", "t"] -[18.8375, "o", "i"] -[18.8849, "o", "f"] -[18.948, "o", "y"] -[19.12, "o", " "] -[19.2032, "o", "e"] -[19.2881, "o", "x"] -[19.3525, "o", "p"] -[19.4241, "o", "l"] -[19.4642, "o", "a"] -[19.5344, "o", "i"] -[19.6001, "o", "n"] -[19.6664, "o", " "] -[19.727, "o", "e"] -[19.7846, "o", "u"] -[19.8725, "o", "_"] -[19.9425, "o", "a"] -[19.9966, "o", "i"] -[20.0526, "o", "_"] -[20.1064, "o", "s"] -[20.5153, "o", "\b \b"] -[20.7183, "o", "a"] -[20.7907, "o", "c"] -[20.8689, "o", "t"] -[21.0413, "o", " "] -[21.0899, "o", "|"] -[21.169, "o", " "] -[21.2214, "o", "h"] -[21.2811, "o", "e"] -[21.3388, "o", "a"] -[21.3854, "o", "d"] -[21.4432, "o", " "] -[21.5249, "o", "-"] -[21.5912, "o", "1"] -[21.646, "o", "2"] -[22.066, "o", "\r\n"] -[24.126, "o", "\r\neu_ai_act \u2014 29 policies\r\n\r\n"] -[24.216, "o", "Fields you must declare (150)\r\n No evaluator can observe these. They are facts about your system,\r\n your process, or your paperwork, so you assert them in the contract.\r\n"] -[24.306, "o", "\r\n accuracy.declared_in_instructions technical_robustness\r\n accuracy.metrics_declared technical_robustness\r\n"] -[24.396, "o", " assessment.completed conformity_assessment\r\n assessment.harmonised_standards_applied conformity_assessment\r\n assessment.notified_body_involved conformity_assessment\r\n"] -[25.186, "o", "\u001b7\u001b[9A\r\u001b[25C\u001b[43;30m1\u001b[0m\u001b8"] -[25.214, "o", "\u001b7\u001b[9A\r\u001b[25C\u001b[43;30m15\u001b[0m\u001b8"] -[25.242, "o", "\u001b7\u001b[9A\r\u001b[25C\u001b[43;30m150\u001b[0m\u001b8"] -[28.21, "o", "$ "] -[28.2514, "o", "a"] -[28.3117, "o", "i"] -[28.3522, "o", "c"] -[28.4074, "o", "e"] -[28.4549, "o", "r"] -[28.5189, "o", "t"] -[28.5999, "o", "i"] -[28.6522, "o", "f"] -[28.7288, "o", "y"] -[28.7878, "o", " "] -[28.855, "o", "e"] -[28.9184, "o", "x"] -[28.9987, "o", "p"] -[29.0767, "o", "l"] -[29.1447, "o", "a"] -[29.2237, "o", "i"] -[29.272, "o", "n"] -[29.4492, "o", " "] -[29.5169, "o", "e"] -[29.5878, "o", "u"] -[29.6708, "o", "_"] -[29.7474, "o", "a"] -[29.8036, "o", "i"] -[29.8704, "o", "_"] -[29.9512, "o", "a"] -[30.0082, "o", "c"] -[30.0582, "o", "r"] -[30.4082, "o", "\b \b"] -[30.6003, "o", "t"] -[30.687, "o", " "] -[30.746, "o", "|"] -[30.9139, "o", " "] -[30.9762, "o", "g"] -[31.0533, "o", "r"] -[31.1333, "o", "e"] -[31.1925, "o", "p"] -[31.2527, "o", " "] -[31.3087, "o", "-"] -[31.3607, "o", "A"] -[31.4381, "o", "8"] -[31.6029, "o", " "] -[31.6572, "o", "\""] -[31.7332, "o", "p"] -[31.8124, "o", "r"] -[31.8912, "o", "o"] -[31.9601, "o", "d"] -[32.0083, "o", "u"] -[32.0643, "o", "c"] -[32.1034, "o", "e"] -[32.1912, "o", "d"] -[32.3943, "o", " "] -[32.446, "o", "b"] -[32.5336, "o", "y"] -[32.765, "o", " "] -[32.8458, "o", "e"] -[32.9108, "o", "v"] -[32.9632, "o", "a"] -[33.0423, "o", "l"] -[33.1011, "o", "u"] -[33.1434, "o", "a"] -[33.2204, "o", "t"] -[33.289, "o", "o"] -[33.3575, "o", "r"] -[33.4416, "o", "s"] -[33.5174, "o", "\""] -[33.9374, "o", "\r\n"] -[35.9974, "o", "Fields produced by evaluators (14)\r\n Computed at evaluation time from your interactions. Do not hand-write\r\n these: asserting your own fairness score defeats the point.\r\n"] -[36.0874, "o", "\r\n metrics.content_safety.score eu_fairness [content_safety evaluator]\r\n metrics.emotion_recognition.combined_score emotion_recognition\r\n"] -[36.1774, "o", " metrics.emotion_recognition.educational_context emotion_recognition\r\n metrics.emotion_recognition.workplace_context emotion_recognition\r\n metrics.fairness.gender_bias eu_fairness [fairness evaluator]\r\n"] -[36.9674, "o", "\u001b7\u001b[9A\r\u001b[31C\u001b[43;30m1\u001b[0m\u001b8"] -[36.9954, "o", "\u001b7\u001b[9A\r\u001b[31C\u001b[43;30m14\u001b[0m\u001b8"] -[37.3634, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30md\u001b[0m\u001b8"] -[37.3914, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mde\u001b[0m\u001b8"] -[37.4194, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdef\u001b[0m\u001b8"] -[37.4474, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefe\u001b[0m\u001b8"] -[37.4754, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefea\u001b[0m\u001b8"] -[37.5034, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeat\u001b[0m\u001b8"] -[37.5314, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats\u001b[0m\u001b8"] -[37.5594, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats \u001b[0m\u001b8"] -[37.5874, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats t\u001b[0m\u001b8"] -[37.6154, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats th\u001b[0m\u001b8"] -[37.6434, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the\u001b[0m\u001b8"] -[37.6714, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the \u001b[0m\u001b8"] -[37.6994, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the p\u001b[0m\u001b8"] -[37.7274, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the po\u001b[0m\u001b8"] -[37.7554, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the poi\u001b[0m\u001b8"] -[37.7834, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the poin\u001b[0m\u001b8"] -[37.8114, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the point\u001b[0m\u001b8"] -[43.5794, "o", "$ "] -[43.6385, "o", "a"] -[43.6917, "o", "i"] -[43.7727, "o", "c"] -[43.8634, "o", "e"] -[43.9259, "o", "r"] -[43.9854, "o", "t"] -[44.042, "o", "i"] -[44.1211, "o", "f"] -[44.16, "o", "y"] -[44.3086, "o", " "] -[44.3876, "o", "i"] -[44.4676, "o", "n"] -[44.5508, "o", "i"] -[44.6033, "o", "t"] -[44.6617, "o", "-"] -[44.734, "o", "c"] -[44.7916, "o", "o"] -[44.8625, "o", "n"] -[44.9534, "o", "t"] -[45.0147, "o", "r"] -[45.0767, "o", "a"] -[45.1332, "o", "c"] -[45.1976, "o", "t"] -[45.2691, "o", " "] -[45.3283, "o", "0"] -[45.6653, "o", "\b \b"] -[45.8623, "o", "-"] -[45.9455, "o", "-"] -[45.9921, "o", "p"] -[46.0499, "o", "o"] -[46.1131, "o", "l"] -[46.1849, "o", "i"] -[46.2709, "o", "c"] -[46.3439, "o", "y"] -[46.5273, "o", " "] -[46.5701, "o", "e"] -[46.6335, "o", "u"] -[46.7152, "o", "_"] -[46.807, "o", "a"] -[46.8723, "o", "i"] -[46.9585, "o", "_"] -[46.9969, "o", "a"] -[47.0447, "o", "c"] -[47.0982, "o", "t"] -[47.1756, "o", " "] -[47.2153, "o", ">"] -[47.2948, "o", " "] -[47.351, "o", "c"] -[47.4214, "o", "o"] -[47.4799, "o", "n"] -[47.5598, "o", "t"] -[47.6278, "o", "r"] -[47.6705, "o", "a"] -[47.734, "o", "c"] -[47.8129, "o", "t"] -[47.8936, "o", "."] -[47.9715, "o", "j"] -[48.0437, "o", "s"] -[48.1216, "o", "o"] -[48.2074, "o", "n"] -[48.6274, "o", "\r\n"] -[51.5874, "o", "$ "] -[51.6272, "o", "a"] -[51.7191, "o", "i"] -[51.7606, "o", "c"] -[51.8272, "o", "e"] -[51.8975, "o", "r"] -[51.9475, "o", "t"] -[52.0156, "o", "i"] -[52.1, "o", "f"] -[52.1779, "o", "y"] -[52.3785, "o", " "] -[52.4637, "o", "e"] -[52.5155, "o", "v"] -[52.5743, "o", "a"] -[52.6317, "o", "l"] -[52.6783, "o", "u"] -[52.7438, "o", "a"] -[52.8299, "o", "t"] -[52.8931, "o", "e"] -[52.9803, "o", " "] -[53.0393, "o", "-"] -[53.1041, "o", "-"] -[53.1831, "o", "c"] -[53.2314, "o", "o"] -[53.2756, "o", "n"] -[53.346, "o", "t"] -[53.433, "o", "r"] -[53.4931, "o", "a"] -[53.5609, "o", "c"] -[53.6446, "o", "t"] -[53.7168, "o", " "] -[53.7934, "o", "c"] -[53.8616, "o", "o"] -[53.9169, "o", "n"] -[53.9696, "o", "t"] -[54.0341, "o", "r"] -[54.1098, "o", "a"] -[54.1802, "o", "c"] -[54.2676, "o", "t"] -[54.3384, "o", "."] -[54.4258, "o", "j"] -[54.5065, "o", "s"] -[54.563, "o", "o"] -[54.6023, "o", "n"] -[54.6764, "o", " "] -[54.7407, "o", "-"] -[54.7891, "o", "-"] -[54.8412, "o", "p"] -[54.8814, "o", "o"] -[54.9483, "o", "l"] -[54.9985, "o", "i"] -[55.0459, "o", "c"] -[55.1294, "o", "y"] -[55.3245, "o", " "] -[55.3683, "o", "e"] -[55.4425, "o", "u"] -[55.4895, "o", "_"] -[55.5737, "o", "a"] -[55.627, "o", "i"] -[55.7134, "o", "_"] -[55.7583, "o", "a"] -[55.8154, "o", "c"] -[55.8796, "o", "t"] -[56.2996, "o", "\r\n"] -[58.3596, "o", "Evaluation Summary:\r\nContract ID: b7e60f21-6717-4ebd-b83f-dd6927dd5c52\r\nApplication: your-application\r\n"] -[58.4496, "o", "Report: ./reports/folder_report_your-application_2026-08-30_165312.md\r\nOPA Evaluation: Successful\r\n"] -[59.3396, "o", "$ "] -[59.4066, "o", "h"] -[59.4879, "o", "e"] -[59.5623, "o", "a"] -[59.652, "o", "d"] -[59.73, "o", " "] -[59.7905, "o", "-"] -[59.8812, "o", "8"] -[59.962, "o", " "] -[60.0213, "o", "e"] -[60.3724, "o", "\b \b"] -[60.5701, "o", "r"] -[60.6452, "o", "e"] -[60.7252, "o", "p"] -[60.7674, "o", "o"] -[60.8375, "o", "r"] -[60.8975, "o", "t"] -[60.9699, "o", "s"] -[61.0196, "o", "/"] -[61.08, "o", "*"] -[61.138, "o", "."] -[61.1952, "o", "m"] -[61.287, "o", "d"] -[61.707, "o", "\r\n"] -[62.167, "o", "# Evaluation Report\r\n\r\n**Analysis Date:** 2026-08-30 11:23:12\r\n"] -[62.257, "o", "\r\n## Evaluation Summary\r\n- **Total Policies:** 29\r\n"] -[62.347, "o", "- **Green Count:** 0\r\n- **Red Count:** 29\r\n"] -[63.137, "o", "\u001b7\u001b[1A\r\u001b[17C\u001b[43;30m2\u001b[0m\u001b8"] -[63.165, "o", "\u001b7\u001b[1A\r\u001b[17C\u001b[43;30m29\u001b[0m\u001b8"] -[63.533, "o", "\u001b7\u001b[2A\r\u001b[19C\u001b[43;30m0\u001b[0m\u001b8"] -[69.301, "o", "$ "] -[69.3882, "o", "#"] -[69.4743, "o", " "] -[69.5451, "o", "w"] -[69.6227, "o", "h"] -[69.7075, "o", "i"] -[69.7818, "o", "c"] -[69.828, "o", "h"] -[69.9057, "o", " "] -[69.9455, "o", "m"] -[69.9909, "o", "e"] -[70.0551, "o", "t"] -[70.0977, "o", "r"] -[70.1655, "o", "i"] -[70.2543, "o", "c"] -[70.2991, "o", "s"] -[70.4012, "o", " "] -[70.4616, "o", "c"] -[70.5501, "o", "a"] -[70.6038, "o", "n"] -[70.6874, "o", " "] -[70.7728, "o", "a"] -[70.8183, "o", "n"] -[70.9047, "o", " "] -[70.9633, "o", "e"] -[71.0446, "o", "v"] -[71.1189, "o", "a"] -[71.2031, "o", "l"] -[71.2921, "o", "u"] -[71.3711, "o", "a"] -[71.4169, "o", "t"] -[71.4596, "o", "o"] -[71.5339, "o", "r"] -[71.7245, "o", " "] -[71.7665, "o", "s"] -[71.8055, "o", "u"] -[71.8918, "o", "p"] -[71.9756, "o", "p"] -[72.03, "o", "l"] -[72.0698, "o", "y"] -[72.1141, "o", " "] -[72.1811, "o", "t"] -[72.2331, "o", "o"] -[72.3002, "o", "d"] -[72.3897, "o", "a"] -[72.4633, "o", "y"] -[72.5286, "o", "?"] -[72.9486, "o", "\r\n"] -[73.6086, "o", "$ "] -[73.6944, "o", "p"] -[73.7683, "o", "y"] -[73.8566, "o", "t"] -[73.9233, "o", "h"] -[74.0105, "o", "o"] -[74.0865, "o", "n"] -[74.2597, "o", " "] -[74.3029, "o", "s"] -[74.3827, "o", "c"] -[74.4661, "o", "r"] -[74.5439, "o", "i"] -[74.6249, "o", "p"] -[74.7139, "o", "t"] -[74.797, "o", "s"] -[74.8734, "o", "/"] -[74.9156, "o", "m"] -[74.9747, "o", "e"] -[75.0364, "o", "t"] -[75.1187, "o", "r"] -[75.1817, "o", "i"] -[75.261, "o", "c"] -[75.3511, "o", "_"] -[75.3951, "o", "g"] -[75.4514, "o", "a"] -[75.5039, "o", "p"] -[75.594, "o", "_"] -[75.6727, "o", "r"] -[75.7639, "o", "e"] -[75.8538, "o", "p"] -[75.9081, "o", "o"] -[75.9502, "o", "r"] -[76.0355, "o", "t"] -[76.0967, "o", "."] -[76.1828, "o", "p"] -[76.2418, "o", "y"] -[76.3131, "o", " "] -[76.3935, "o", "|"] -[76.5115, "o", " "] -[76.5943, "o", "t"] -[76.665, "o", "a"] -[76.7531, "o", "i"] -[76.8212, "o", "l"] -[76.8663, "o", " "] -[76.9249, "o", "-"] -[76.977, "o", "1"] -[77.0476, "o", "2"] -[77.4676, "o", "\r\n"] -[79.5276, "o", " calc metrics.model_card.completeness computed by global.v1.documentation.model_card_score, no evaluator needed\r\n GAP metrics.model_card.compliance_level no evaluator declares this\r\n calc metrics.model_card.quality computed by global.v1.documentation.model_card_score, no evaluator needed\r\n"] -[79.6176, "o", " calc metrics.model_card.section_scores computed by global.v1.documentation.model_card_score, no evaluator needed\r\n ok metrics.social_scoring.combined_score SocialScoringEvaluator\r\n ok metrics.social_scoring.detrimental_treatment SocialScoringEvaluator\r\n"] -[79.7076, "o", " ok metrics.social_scoring.score SocialScoringEvaluator\r\n ok metrics.toxicity.max_toxicity ContentSafetyEvaluator\r\n\r\n"] -[79.7976, "o", "TOTAL: 22 of 27 measured metrics can be supplied today.\r\nThe gaps are what you would need to write an evaluator for, or\r\nsupply by hand. See docs on writing one.\r\n"] -[80.6876, "o", "\u001b7\u001b[3A\r\u001b[7C\u001b[43;30m2\u001b[0m\u001b8"] -[80.7156, "o", "\u001b7\u001b[3A\r\u001b[7C\u001b[43;30m22\u001b[0m\u001b8"] -[80.7436, "o", "\u001b7\u001b[3A\r\u001b[7C\u001b[43;30m22 \u001b[0m\u001b8"] -[80.7716, "o", "\u001b7\u001b[3A\r\u001b[7C\u001b[43;30m22 o\u001b[0m\u001b8"] -[80.7996, "o", "\u001b7\u001b[3A\r\u001b[7C\u001b[43;30m22 of\u001b[0m\u001b8"] -[80.8276, "o", "\u001b7\u001b[3A\r\u001b[7C\u001b[43;30m22 of \u001b[0m\u001b8"] -[80.8556, "o", "\u001b7\u001b[3A\r\u001b[7C\u001b[43;30m22 of 2\u001b[0m\u001b8"] -[80.8836, "o", "\u001b7\u001b[3A\r\u001b[7C\u001b[43;30m22 of 27\u001b[0m\u001b8"] -[86.6516, "o", "$ "] -[86.6516, "o", "\u2588"] -[87.1816, "o", "\b \b"] -[87.7116, "o", "\u2588"] -[88.2416, "o", "\b \b"] -[88.7716, "o", "\u2588"] -[89.3016, "o", "\b \b"] -[89.8316, "o", "\u2588"] -[90.3616, "o", "\b \b"] -[90.8916, "o", "\u2588"] -[91.4216, "o", "\b \b"] -[91.9516, "o", "\u2588"] -[92.4816, "o", "\b \b"] +[7.7447, "o", "\u001b7\u001b[18A\r\u001b[16C\u001b[43;30m0.\u001b[0m\u001b8"] +[7.7667, "o", "\u001b7\u001b[18A\r\u001b[16C\u001b[43;30m0.4\u001b[0m\u001b8"] +[7.7887, "o", "\u001b7\u001b[18A\r\u001b[16C\u001b[43;30m0.49\u001b[0m\u001b8"] +[8.0507, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mB\u001b[0m\u001b8"] +[8.0727, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBE\u001b[0m\u001b8"] +[8.0947, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBEL\u001b[0m\u001b8"] +[8.1167, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELO\u001b[0m\u001b8"] +[8.1387, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW\u001b[0m\u001b8"] +[8.1607, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW \u001b[0m\u001b8"] +[8.1827, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW T\u001b[0m\u001b8"] +[8.2047, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW TH\u001b[0m\u001b8"] +[8.2267, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THR\u001b[0m\u001b8"] +[8.2487, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THRE\u001b[0m\u001b8"] +[8.2707, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THRES\u001b[0m\u001b8"] +[8.2927, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THRESH\u001b[0m\u001b8"] +[8.3147, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THRESHO\u001b[0m\u001b8"] +[8.3367, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THRESHOL\u001b[0m\u001b8"] +[8.3587, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THRESHOLD\u001b[0m\u001b8"] +[12.0207, "o", "$ "] +[12.0657, "o", "#"] +[12.2259, "o", " "] +[12.273, "o", "s"] +[12.3421, "o", "o"] +[12.386, "o", " "] +[12.4553, "o", "w"] +[12.5381, "o", "h"] +[12.5781, "o", "a"] +[12.6178, "o", "t"] +[12.8279, "o", " "] +[12.9118, "o", "d"] +[12.9917, "o", "o"] +[13.0304, "o", "e"] +[13.0755, "o", "s"] +[13.1443, "o", " "] +[13.2158, "o", "t"] +[13.2943, "o", "h"] +[13.3767, "o", "e"] +[13.4227, "o", " "] +[13.498, "o", "E"] +[13.5667, "o", "U"] +[13.7794, "o", " "] +[13.8358, "o", "A"] +[13.9237, "o", "I"] +[13.9733, "o", " "] +[14.0587, "o", "A"] +[14.0979, "o", "c"] +[14.1492, "o", "t"] +[14.2018, "o", " "] +[14.2508, "o", "a"] +[14.336, "o", "c"] +[14.4167, "o", "t"] +[14.488, "o", "u"] +[14.5766, "o", "a"] +[14.6189, "o", "l"] +[14.6578, "o", "l"] +[14.7179, "o", "y"] +[14.7898, "o", " "] +[14.8672, "o", "a"] +[14.9136, "o", "s"] +[14.9539, "o", "k"] +[15.0162, "o", " "] +[15.1041, "o", "f"] +[15.1825, "o", "o"] +[15.264, "o", "r"] +[15.3493, "o", "?"] +[15.7693, "o", "\r\n"] +[16.4293, "o", "$ "] +[16.5071, "o", "a"] +[16.595, "o", "i"] +[16.6735, "o", "c"] +[16.7433, "o", "e"] +[16.8276, "o", "r"] +[16.8932, "o", "t"] +[16.9406, "o", "i"] +[17.0036, "o", "f"] +[17.0918, "o", "y"] +[17.2885, "o", " "] +[17.3733, "o", "e"] +[17.4377, "o", "x"] +[17.5093, "o", "p"] +[17.5494, "o", "l"] +[17.6197, "o", "a"] +[17.6853, "o", "i"] +[17.7516, "o", "n"] +[17.8281, "o", " "] +[17.8857, "o", "e"] +[17.9735, "o", "u"] +[18.0436, "o", "_"] +[18.0977, "o", "a"] +[18.1537, "o", "i"] +[18.2074, "o", "_"] +[18.293, "o", "a"] +[18.3736, "o", "c"] +[18.4295, "o", "t"] +[18.5159, "o", " "] +[18.5942, "o", "|"] +[18.7666, "o", " "] +[18.8151, "o", "h"] +[18.8942, "o", "e"] +[18.9756, "o", "a"] +[19.028, "o", "d"] +[19.0877, "o", " "] +[19.1343, "o", "-"] +[19.1921, "o", "1"] +[19.2512, "o", "2"] +[19.6712, "o", "\r\n"] +[21.7312, "o", "\r\neu_ai_act \u2014 29 policies\r\n\r\n"] +[21.8212, "o", "Fields you must declare (150)\r\n No evaluator can observe these. They are facts about your system,\r\n your process, or your paperwork, so you assert them in the contract.\r\n"] +[21.9112, "o", "\r\n accuracy.declared_in_instructions technical_robustness\r\n accuracy.metrics_declared technical_robustness\r\n"] +[22.0012, "o", " assessment.completed conformity_assessment\r\n assessment.harmonised_standards_applied conformity_assessment\r\n assessment.notified_body_involved conformity_assessment\r\n"] +[22.7912, "o", "\u001b7\u001b[9A\r\u001b[25C\u001b[43;30m1\u001b[0m\u001b8"] +[22.8132, "o", "\u001b7\u001b[9A\r\u001b[25C\u001b[43;30m15\u001b[0m\u001b8"] +[22.8352, "o", "\u001b7\u001b[9A\r\u001b[25C\u001b[43;30m150\u001b[0m\u001b8"] +[25.0972, "o", "$ "] +[25.1634, "o", "a"] +[25.2183, "o", "i"] +[25.2671, "o", "c"] +[25.3228, "o", "e"] +[25.3639, "o", "r"] +[25.4126, "o", "t"] +[25.4704, "o", "i"] +[25.5614, "o", "f"] +[25.6472, "o", "y"] +[25.7778, "o", " "] +[25.8253, "o", "e"] +[25.8888, "o", "x"] +[25.9569, "o", "p"] +[26.0, "o", "l"] +[26.0723, "o", "a"] +[26.1582, "o", "i"] +[26.2323, "o", "n"] +[26.3201, "o", " "] +[26.4103, "o", "e"] +[26.4678, "o", "u"] +[26.5485, "o", "_"] +[26.6042, "o", "a"] +[26.6806, "o", "i"] +[26.7196, "o", "_"] +[26.8113, "o", "a"] +[26.8675, "o", "c"] +[26.9136, "o", "t"] +[27.103, "o", " "] +[27.1429, "o", "|"] +[27.3126, "o", " "] +[27.399, "o", "g"] +[27.4509, "o", "r"] +[27.5074, "o", "e"] +[27.5969, "o", "p"] +[27.7324, "o", " "] +[27.7827, "o", "-"] +[27.861, "o", "A"] +[27.9384, "o", "8"] +[28.0012, "o", " "] +[28.0516, "o", "\""] +[28.1285, "o", "p"] +[28.2122, "o", "r"] +[28.2653, "o", "o"] +[28.3485, "o", "d"] +[28.4387, "o", "u"] +[28.4781, "o", "c"] +[28.5557, "o", "e"] +[28.6193, "o", "d"] +[28.7785, "o", " "] +[28.8447, "o", "b"] +[28.9018, "o", "y"] +[29.0954, "o", " "] +[29.1461, "o", "e"] +[29.198, "o", "v"] +[29.2623, "o", "a"] +[29.3112, "o", "l"] +[29.3772, "o", "u"] +[29.4187, "o", "a"] +[29.4772, "o", "t"] +[29.5336, "o", "o"] +[29.5948, "o", "r"] +[29.686, "o", "s"] +[29.7272, "o", "\""] +[30.1472, "o", "\r\n"] +[32.2072, "o", "Fields produced by evaluators (14)\r\n Computed at evaluation time from your interactions. Do not hand-write\r\n these: asserting your own fairness score defeats the point.\r\n"] +[32.2972, "o", "\r\n metrics.content_safety.score eu_fairness [content_safety evaluator]\r\n metrics.emotion_recognition.combined_score emotion_recognition\r\n"] +[32.3872, "o", " metrics.emotion_recognition.educational_context emotion_recognition\r\n metrics.emotion_recognition.workplace_context emotion_recognition\r\n metrics.fairness.gender_bias eu_fairness [fairness evaluator]\r\n"] +[33.1772, "o", "\u001b7\u001b[9A\r\u001b[31C\u001b[43;30m1\u001b[0m\u001b8"] +[33.1992, "o", "\u001b7\u001b[9A\r\u001b[31C\u001b[43;30m14\u001b[0m\u001b8"] +[33.4612, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30md\u001b[0m\u001b8"] +[33.4832, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mde\u001b[0m\u001b8"] +[33.5052, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdef\u001b[0m\u001b8"] +[33.5272, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefe\u001b[0m\u001b8"] +[33.5492, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefea\u001b[0m\u001b8"] +[33.5712, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeat\u001b[0m\u001b8"] +[33.5932, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats\u001b[0m\u001b8"] +[33.6152, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats \u001b[0m\u001b8"] +[33.6372, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats t\u001b[0m\u001b8"] +[33.6592, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats th\u001b[0m\u001b8"] +[33.6812, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the\u001b[0m\u001b8"] +[33.7032, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the \u001b[0m\u001b8"] +[33.7252, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the p\u001b[0m\u001b8"] +[33.7472, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the po\u001b[0m\u001b8"] +[33.7692, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the poi\u001b[0m\u001b8"] +[33.7912, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the poin\u001b[0m\u001b8"] +[33.8132, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the point\u001b[0m\u001b8"] +[37.4752, "o", "$ "] +[37.5463, "o", "a"] +[37.6206, "o", "i"] +[37.6596, "o", "c"] +[37.7474, "o", "e"] +[37.8104, "o", "r"] +[37.8581, "o", "t"] +[37.9474, "o", "i"] +[38.0159, "o", "f"] +[38.075, "o", "y"] +[38.1282, "o", " "] +[38.2189, "o", "i"] +[38.2813, "o", "n"] +[38.3409, "o", "i"] +[38.3975, "o", "t"] +[38.4766, "o", "-"] +[38.5155, "o", "c"] +[38.5826, "o", "o"] +[38.6253, "o", "n"] +[38.6788, "o", "t"] +[38.7577, "o", "r"] +[38.8377, "o", "a"] +[38.9209, "o", "c"] +[38.9734, "o", "t"] +[39.0319, "o", " "] +[39.0894, "o", "-"] +[39.1603, "o", "-"] +[39.2512, "o", "p"] +[39.3126, "o", "o"] +[39.3746, "o", "l"] +[39.431, "o", "i"] +[39.4954, "o", "c"] +[39.567, "o", "y"] +[39.6386, "o", " "] +[39.6796, "o", "e"] +[39.7642, "o", "u"] +[39.8084, "o", "_"] +[39.8808, "o", "a"] +[39.9711, "o", "i"] +[40.0616, "o", "_"] +[40.1367, "o", "a"] +[40.2028, "o", "c"] +[40.2555, "o", "t"] +[40.3386, "o", " "] +[40.4272, "o", ">"] +[40.6372, "o", " "] +[40.6927, "o", "c"] +[40.7405, "o", "o"] +[40.7895, "o", "n"] +[40.8733, "o", "t"] +[40.9334, "o", "r"] +[40.974, "o", "a"] +[41.0523, "o", "c"] +[41.1215, "o", "t"] +[41.1714, "o", "."] +[41.2392, "o", "j"] +[41.3201, "o", "s"] +[41.3669, "o", "o"] +[41.4086, "o", "n"] +[41.8286, "o", "\r\n"] +[44.5886, "o", "$ "] +[44.6469, "o", "#"] +[44.7088, "o", " "] +[44.7842, "o", "1"] +[44.8482, "o", "5"] +[44.9291, "o", "0"] +[45.1603, "o", " "] +[45.2136, "o", "d"] +[45.2628, "o", "e"] +[45.3407, "o", "c"] +[45.3989, "o", "l"] +[45.4386, "o", "a"] +[45.5306, "o", "r"] +[45.572, "o", "a"] +[45.6386, "o", "t"] +[45.7089, "o", "i"] +[45.759, "o", "o"] +[45.827, "o", "n"] +[45.9115, "o", "s"] +[45.9893, "o", " "] +[46.0378, "o", "t"] +[46.1059, "o", "o"] +[46.2836, "o", " "] +[46.3409, "o", "a"] +[46.3876, "o", "n"] +[46.453, "o", "s"] +[46.5392, "o", "w"] +[46.6023, "o", "e"] +[46.6895, "o", "r"] +[46.7797, "o", "."] +[46.8387, "o", " "] +[46.9177, "o", "a"] +[46.966, "o", "n"] +[47.0102, "o", "s"] +[47.0805, "o", "w"] +[47.1676, "o", "e"] +[47.2276, "o", "r"] +[47.2954, "o", "e"] +[47.3792, "o", "d"] +[47.4513, "o", ","] +[47.5369, "o", " "] +[47.6051, "o", "t"] +[47.6604, "o", "h"] +[47.713, "o", "e"] +[47.7776, "o", "n"] +[47.8532, "o", ":"] +[48.2732, "o", "\r\n"] +[48.9332, "o", "$ "] +[49.0207, "o", "a"] +[49.0915, "o", "i"] +[49.1789, "o", "c"] +[49.2596, "o", "e"] +[49.3161, "o", "r"] +[49.3554, "o", "t"] +[49.4294, "o", "i"] +[49.5185, "o", "f"] +[49.5828, "o", "y"] +[49.6857, "o", " "] +[49.7526, "o", "e"] +[49.8028, "o", "v"] +[49.8502, "o", "a"] +[49.9337, "o", "l"] +[49.9906, "o", "u"] +[50.0313, "o", "a"] +[50.1126, "o", "t"] +[50.1565, "o", "e"] +[50.3747, "o", " "] +[50.428, "o", "-"] +[50.5144, "o", "-"] +[50.5593, "o", "c"] +[50.6164, "o", "o"] +[50.6806, "o", "n"] +[50.7367, "o", "t"] +[50.7848, "o", "r"] +[50.8641, "o", "a"] +[50.9435, "o", "c"] +[51.0048, "o", "t"] +[51.0925, "o", " "] +[51.1375, "o", "c"] +[51.2261, "o", "o"] +[51.2885, "o", "n"] +[51.3555, "o", "t"] +[51.4202, "o", "r"] +[51.4824, "o", "a"] +[51.5499, "o", "c"] +[51.6153, "o", "t"] +[51.6771, "o", "."] +[51.7523, "o", "j"] +[51.8322, "o", "s"] +[51.8745, "o", "o"] +[51.9445, "o", "n"] +[52.0045, "o", " "] +[52.0543, "o", "-"] +[52.1147, "o", "-"] +[52.1727, "o", "p"] +[52.2299, "o", "o"] +[52.3217, "o", "l"] +[52.3906, "o", "i"] +[52.4778, "o", "c"] +[52.5639, "o", "y"] +[52.6503, "o", " "] +[52.7279, "o", "e"] +[52.8127, "o", "u"] +[52.887, "o", "_"] +[52.9331, "o", "a"] +[53.0108, "o", "i"] +[53.0876, "o", "_"] +[53.1275, "o", "a"] +[53.1728, "o", "c"] +[53.2371, "o", "t"] +[53.6571, "o", "\r\n"] +[55.7171, "o", "Contract ID: 5692c240-fbd2-45af-a478-2142e4fd4a39\r\nApplication: acme-triage-assistant\r\nReport: ./reports/folder_report_acme-triage-assistant_2026-08-30_180617.md\r\n"] +[55.8071, "o", "OPA Evaluation: Successful\r\n"] +[56.6971, "o", "$ "] +[56.7549, "o", "g"] +[56.795, "o", "r"] +[56.8815, "o", "e"] +[56.941, "o", "p"] +[56.9869, "o", " "] +[57.0693, "o", "-"] +[57.1519, "o", "m"] +[57.2421, "o", "2"] +[57.4026, "o", " "] +[57.4602, "o", "\""] +[57.5062, "o", "N"] +[57.5444, "o", "o"] +[57.6073, "o", "t"] +[57.7154, "o", " "] +[57.7631, "o", "S"] +[57.8343, "o", "a"] +[57.8782, "o", "t"] +[57.9436, "o", "i"] +[58.0159, "o", "s"] +[58.0888, "o", "f"] +[58.148, "o", "i"] +[58.2315, "o", "e"] +[58.3191, "o", "d"] +[58.3625, "o", "\""] +[58.4392, "o", " "] +[58.4825, "o", "r"] +[58.5691, "o", "e"] +[58.6108, "o", "p"] +[58.6986, "o", "o"] +[58.7892, "o", "r"] +[58.864, "o", "t"] +[58.9313, "o", "s"] +[58.9941, "o", "/"] +[59.071, "o", "*"] +[59.1319, "o", "."] +[59.1711, "o", "m"] +[59.235, "o", "d"] +[59.655, "o", "\r\n"] +[60.115, "o", "- **Article 15 Controls Not Satisfied:** ['Article 15(4) feedback loops addressed where the syst\r\n- **Article 14 Controls Not Satisfied:** ['Article 14(1)-(2) oversight designed into the system'\r\n"] +[60.905, "o", "\u001b7\u001b[1A\r\u001b[43C\u001b[43;30mA\u001b[0m\u001b8"] +[60.927, "o", "\u001b7\u001b[1A\r\u001b[43C\u001b[43;30mAr\u001b[0m\u001b8"] +[60.949, "o", "\u001b7\u001b[1A\r\u001b[43C\u001b[43;30mArt\u001b[0m\u001b8"] +[60.971, "o", "\u001b7\u001b[1A\r\u001b[43C\u001b[43;30mArti\u001b[0m\u001b8"] +[60.993, "o", "\u001b7\u001b[1A\r\u001b[43C\u001b[43;30mArtic\u001b[0m\u001b8"] +[61.015, "o", "\u001b7\u001b[1A\r\u001b[43C\u001b[43;30mArticl\u001b[0m\u001b8"] +[61.037, "o", "\u001b7\u001b[1A\r\u001b[43C\u001b[43;30mArticle\u001b[0m\u001b8"] +[61.059, "o", "\u001b7\u001b[1A\r\u001b[43C\u001b[43;30mArticle \u001b[0m\u001b8"] +[61.081, "o", "\u001b7\u001b[1A\r\u001b[43C\u001b[43;30mArticle 1\u001b[0m\u001b8"] +[61.103, "o", "\u001b7\u001b[1A\r\u001b[43C\u001b[43;30mArticle 14\u001b[0m\u001b8"] +[61.125, "o", "\u001b7\u001b[1A\r\u001b[43C\u001b[43;30mArticle 14(\u001b[0m\u001b8"] +[61.147, "o", "\u001b7\u001b[1A\r\u001b[43C\u001b[43;30mArticle 14(1\u001b[0m\u001b8"] +[61.169, "o", "\u001b7\u001b[1A\r\u001b[43C\u001b[43;30mArticle 14(1)\u001b[0m\u001b8"] +[61.191, "o", "\u001b7\u001b[1A\r\u001b[43C\u001b[43;30mArticle 14(1)-\u001b[0m\u001b8"] +[61.213, "o", "\u001b7\u001b[1A\r\u001b[43C\u001b[43;30mArticle 14(1)-(\u001b[0m\u001b8"] +[61.235, "o", "\u001b7\u001b[1A\r\u001b[43C\u001b[43;30mArticle 14(1)-(2\u001b[0m\u001b8"] +[61.257, "o", "\u001b7\u001b[1A\r\u001b[43C\u001b[43;30mArticle 14(1)-(2)\u001b[0m\u001b8"] +[64.919, "o", "$ "] +[64.919, "o", "\u2588"] +[65.449, "o", "\b \b"] +[65.979, "o", "\u2588"] +[66.509, "o", "\b \b"] +[67.039, "o", "\u2588"] +[67.569, "o", "\b \b"] +[68.099, "o", "\u2588"] +[68.629, "o", "\b \b"] diff --git a/scripts/record-demo.py b/scripts/record-demo.py index 9bbfa6d..b5df954 100644 --- a/scripts/record-demo.py +++ b/scripts/record-demo.py @@ -2,9 +2,9 @@ """Compose the AICertify demo asciicast: real output, typed at a human pace. Every command here is actually run and its output captured, so nothing on the -screen is written by hand. What is synthesised is the *timing*. The four -commands take about three minutes between them, most of it downloading model -weights and waiting on OPA, and a cast at real speed is unwatchable. A `script` +screen is written by hand. What is synthesised is the *timing*. The commands +take about three minutes between them, most of it downloading model weights and +waiting on OPA, and a cast at real speed is unwatchable. A `script` capture would also have no typing in it at all, and output that materialises instantly reads as a screenshot rather than a session. @@ -12,26 +12,35 @@ the verdicts and the report are what the commands printed. The rhythm is invented, the way any screencast's rhythm is invented by the person recording. -Four acts, because AICertify does a different job from GOPAL and needs the +Three acts, because AICertify does a different job from GOPAL and needs the room. GOPAL decides things, so its demo is one beat: swap the model, the build fails. AICertify gathers things, and the interesting thing about it is how honest it is about the part it cannot gather. 1. score-card a real Hugging Face card, scored, in one screen 2. explain 150 fields you must declare, 14 an evaluator computes - 3. init/eval scaffold a contract, run it, get 0 of 29 - 4. gap report 22 of 27 metrics supplied, and the five that are not + 3. init/eval answer them, evaluate, and get a named article back -Act 3 ends on nothing passing. That is the point rather than an embarrassment: -the scaffold asserts none of the 150 declarations, so none of the obligations -that turn on them can be met. It is the same argument act 2 makes, arriving as -a verdict instead of a number. +Act 3 used to end on an empty contract failing all 29 policies. That is true +and it is the argument act 2 makes, but as an ending it reads as "the tool does +not work" to anyone who has not followed the reasoning. So it now answers the +declarations first and ends where the product is actually differentiated: not +on a score, but on `Article 14(1)-(2) oversight designed into the system`. A +named provision and the specific control, which is what you can act on. + +The gap report was act 4 and has been dropped. Enumerating the metrics nobody +has written an evaluator for is a good thing to show a contributor and a poor +thing to show someone deciding whether to adopt. It lives in the contributing +section of the page instead. Usage: record-demo.py [short.cast] -The short cast is acts 1 and 2 only, for the README, where a 90-second loop is -too long and JavaScript does not run. +The short cast is acts 1 and 2 only, about 36 seconds, for the README. That +loop autoplays above the fold and JavaScript does not run there, so it is a +CSS-animated SVG and it has to be short enough to watch before deciding to +scroll. The full cut runs about 70 seconds on the page, where a reader has +chosen to watch it. """ from __future__ import annotations @@ -59,18 +68,18 @@ ENTER_SETTLES = 0.16 # and the press registers before anything happens FIRST_BYTE = 0.30 # a command does not answer instantly THINKING = 1.9 # a command that clearly did some work -READ_SHORT = 2.6 -READ_LONG = 5.4 # long enough to read a verdict and understand it +READ_SHORT = 2.0 +READ_LONG = 3.4 # long enough to read a verdict and understand it BLINK = 0.53 -BLINKS = 6 +BLINKS = 4 CURSOR = "█" -SWEEP = 0.028 # per character, the pace of a pen moving +SWEEP = 0.022 # per character, the pace of a pen moving HL_ON = "\x1b[43;30m" HL_OFF = "\x1b[0m" SAVE, RESTORE = "\x1b7", "\x1b8" -TYPO_CHANCE = 0.55 # per command, not per character +TYPO_CHANCE = 0.25 # per command, not per character NOTICE_TYPO = (0.22, 0.46) AFTER_FIX = (0.08, 0.19) @@ -203,7 +212,7 @@ def rows(line: str) -> int: if needle in line and phrase in line: up = sum(rows(rest) for rest in lines[idx:]) self.highlight(up, line.index(phrase), phrase) - self.wait(0.34) + self.wait(0.24) break def write(self, dst: Path, title: str) -> None: @@ -275,22 +284,32 @@ def main() -> int: contract.exists() and contract.stat().st_size > 1000, "init-contract produced no usable contract", ) - # A declarations-only contract. The scaffold ships one placeholder - # interaction, and evaluating it pulls DeepEval into an LLM call that - # needs a provider key. The point of this act is the policy verdict, - # not the toxicity classifier, and act 4 covers the evaluators. + # Answer the declarations, the way a reader would before evaluating. + # The scaffold leaves all 150 null, and an unanswered contract fails + # everything: true, and the argument act 2 makes, but as an ending it + # reads as a broken tool rather than an unanswered questionnaire. + # + # Interactions are cleared because evaluating the placeholder one pulls + # DeepEval into an LLM call needing a provider key. This act is about + # the policy verdict, not the toxicity classifier. doc = json.loads(contract.read_text()) doc["interactions"] = [] + doc["application_name"] = "acme-triage-assistant" + answered = _answer_declarations(doc["context"]) contract.write_text(json.dumps(doc, indent=2)) + require( + answered > 100, + f"only {answered} declarations were answered; the scaffold shape " + "changed and act 3 would show an unanswered contract", + ) evaluate = shell( f"{PY} -m aicertify.cli evaluate --contract contract.json " f"--policy eu_ai_act --report-format markdown --output-dir ./reports " - "2>&1 | tail -5", + "2>&1 | tail -4", cwd=work, ) - summary = shell("head -8 reports/*.md", cwd=work) - gaps = shell(f"{PY} scripts/metric_gap_report.py 2>/dev/null | tail -12") + controls = shell('grep -m2 "Not Satisfied" reports/*.md | cut -c1-96', cwd=work) # Guards. Each of these is a fact the cast puts on screen, so if the # tool stops saying it the recording is a lie and must not be written. @@ -312,17 +331,12 @@ def main() -> int: "the line explaining why you must not hand-write metrics is gone", ) require( - "Red Count:** 29" in summary and "Green Count:** 0" in summary, - "an empty contract no longer fails all 29 policies, so act 3 " - "has lost its point", - ) - require( - "22 of 27" in gaps, - "the metric gap report no longer says 22 of 27; act 4's " - "contribution hook needs the real number", + "Article 14" in controls and "Not Satisfied" in controls, + "the report no longer names an unsatisfied Article 14 control, and " + "act 3 ends on a named provision rather than on a score", ) - acts = build(card, declare, measure, evaluate, summary, gaps) + acts = build(card, declare, measure, evaluate, controls) acts.write(full_dst, "AICertify: what it measures, and what it cannot") if short_dst: @@ -333,6 +347,87 @@ def main() -> int: shutil.rmtree(work, ignore_errors=True) +def _answer_declarations(context: dict) -> int: + """Fill every unanswered declaration with a plausible answer. + + The scaffold leaves all 150 as null. Types are taken from GOPAL's coverage + data where it has inferred one, and from the field name otherwise: anything + that reads as a quantity gets a number, anything plural gets a list, and the + rest are the yes/no assertions that make up most of the Act. + + These are not claims about a real system. They stand in for a compliance + officer having answered the questionnaire, which is the state act 3 needs + to show and the state the scaffold exists to reach. + """ + types: dict[str, str] = {} + coverage = ( + REPO / "aicertify" / "opa_policies" / "docs" / "coverage" / "coverage.json" + ) + if coverage.exists(): + data = json.loads(coverage.read_text()) + for framework in data.get("frameworks", []): + if "eu_ai_act" in (framework.get("id") or ""): + for policy in framework.get("policies", []): + types.update(policy.get("input_field_types", {})) + + quantities = ( + "completeness", + "score", + "months", + "days", + "count", + "risk", + "level", + "number", + "percentage", + "rate", + "size", + ) + collections_ = ( + "measures", + "fields", + "controls", + "reasons", + "sections", + "list", + "categories", + "purposes", + "records", + ) + + def answer(path: str): + kind = types.get(path) + if kind == "boolean": + return True + if kind == "number": + return 0.95 + if kind == "string": + return "documented" + if kind == "list": + return [] + leaf = path.rsplit(".", 1)[-1] + if any(word in leaf for word in quantities): + return 0.95 + if any(word in leaf for word in collections_): + return [] + return True + + filled = 0 + + def walk(node: dict, path: str = "") -> None: + nonlocal filled + for key, value in list(node.items()): + here = f"{path}.{key}" if path else key + if isinstance(value, dict): + walk(value, here) + elif value is None: + node[key] = answer(here) + filled += 1 + + walk(context) + return filled + + def act_one(cast: Cast, card: str) -> None: """A famous model, scored, with no setup at all.""" cast.wait(0.6) @@ -349,10 +444,16 @@ def act_one(cast: Cast, card: str) -> None: cast.wait(READ_LONG) -def act_two(cast: Cast, declare: str, measure: str) -> None: - """The split that the whole product turns on.""" - cast.command("# so what does the EU AI Act actually ask for?") - cast.wait(0.5) +def act_two(cast: Cast, declare: str, measure: str, narrate: bool = True) -> None: + """The split that the whole product turns on. + + `narrate` is off for the README cut. That loop autoplays above the fold and + has to be short enough that a reader sees the whole thing before deciding to + scroll, so it drops the framing line the page version can afford. + """ + if narrate: + cast.command("# so what does the EU AI Act actually ask for?") + cast.wait(0.5) cast.command("aicertify explain eu_ai_act | head -12") cast.block(declare, first_byte=THINKING) cast.wait(0.7) @@ -375,37 +476,32 @@ def act_two(cast: Cast, declare: str, measure: str) -> None: def build_short(card: str, declare: str, measure: str) -> Cast: cast = Cast() act_one(cast, card) - act_two(cast, declare, measure) - cast.blink() + act_two(cast, declare, measure, narrate=False) + cast.blink(3) return cast -def build(card, declare, measure, evaluate, summary, gaps) -> Cast: +def build(card, declare, measure, evaluate, controls) -> Cast: cast = Cast() act_one(cast, card) act_two(cast, declare, measure) - # Act 3: scaffold, run, and find out you have answered nothing. + # Act 3: answer them, evaluate, and get a provision back rather than a score. cast.command("aicertify init-contract --policy eu_ai_act > contract.json") - cast.wait(THINKING + 0.9) + cast.wait(THINKING + 0.7) + cast.command("# 150 declarations to answer. answered, then:") + cast.wait(0.5) cast.command("aicertify evaluate --contract contract.json --policy eu_ai_act") cast.block(evaluate, first_byte=THINKING) cast.wait(0.8) - cast.command("head -8 reports/*.md") - cast.block(summary) + # The ending. Not a count, which reads as a grade, but the named provision + # and the specific control, which is the thing you can act on and the thing + # a black-box compliance score cannot give you. + cast.command('grep -m2 "Not Satisfied" reports/*.md') + cast.block(controls) cast.wait(0.7) - cast.sweep(summary, [("Red Count", "29"), ("Green Count", "0")]) - cast.wait(READ_LONG) - - # Act 4: the tool enumerating its own gaps, which is a contributor's - # to-do list rather than a caveat. - cast.command("# which metrics can an evaluator supply today?") - cast.wait(0.5) - cast.command("python scripts/metric_gap_report.py | tail -12") - cast.block(gaps, first_byte=THINKING) - cast.wait(0.8) - cast.sweep(gaps, [("TOTAL", "22 of 27")]) + cast.sweep(controls, [("Article 14", "Article 14(1)-(2)")]) cast.wait(READ_LONG) cast.blink() return cast