diff --git a/README.md b/README.md index 7bddd77..db0042e 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@

Scoring a real Hugging Face card, then asking what the EU AI Act actually needs.
- Watch the full four-minute walkthrough, including running a contract and the metrics still missing an evaluator.
+ Watch the full walkthrough, which goes on to answer the declarations and evaluate them.


diff --git a/docs/demo/aicertify-animated.svg b/docs/demo/aicertify-animated.svg index 5c4765e..de0df1d 100644 --- a/docs/demo/aicertify-animated.svg +++ b/docs/demo/aicertify-animated.svg @@ -1,4 +1,4 @@ - + aicertify: 150 declared, 14 measured $ aicertify score-card bert-base-uncased @@ -21,7 +21,6 @@ A model card answers part of what Annex IV asks and then stops. This is the documentation-completeness gap, not a compliance verdict. scored by GOPAL 2.0.0 (global/v1/documentation/model_card_score), via aicertify 0.7.0 -$ # so what does the EU AI Act actually ask for? $ aicertify explain eu_ai_act | head -12 eu_ai_act — 29 policies @@ -46,18 +45,18 @@ Fields produced by evaluators (14) metrics.emotion_recognition.workplace_context emotion_recognition metrics.fairness.gender_bias eu_fairness [fairness evaluator] $ - - + + aicertify: 150 declared, 14 measured - - - - - + + + + + $ a @@ -155,253 +154,216 @@ $ This is the documentation-completeness gap, not a compliance verdict. scored by GOPAL 2.0.0 (global/v1/documentation/model_card_score), via aicertify 0.7.0 $ -# -s -o -w -h -a -t -d -o -e -s -t -h -e -E -U -A -I -A -c -t -a -c -t -u -a -l -k -l -y -a -s -k -f -o -r -? -$ -a -i -c -e -r -t -i -f -y -e -x -p -l -a -i -n -e -u -_ -a -i -_ -a -c -t -| -h -e -a -d -- -1 -2 -eu_ai_act — 29 policies -F -i -e -l -d -s -y -o -u -m -u -s -t -d -e -c -l -a -r -e -( -1 -5 -0 -) - No evaluator can observe these. They are facts about your system, - your process, or your paperwork, so you assert them in the contract. - accuracy.declared_in_instructions technical_robustness - accuracy.metrics_declared technical_robustness - assessment.completed conformity_assessment - assessment.harmonised_standards_applied conformity_assessment - assessment.notified_body_involved conformity_assessment -$ -a -i -c -e -r -t -i -f -y -e -x -p -l -a -i -n -e -u -_ -a -i -_ -a -c -t -| -g -r -e -p -- -A -8 -" -p -r -o -d -u -c -e -d -b -y -e -v -a -l -u -a -t -o -r -s -" -F -i -e -l -d -s -p -r -o -d -u -c -e -d -b -y -e -v -a -l -u -a -t -o -r -s -( -1 -4 -) - Computed at evaluation time from your interactions. Do not hand-write -t -h -e -s -e -: -a -s -s -e -r -t -i -n -g -y -o -u -r -o -w -n -f -a -i -r -n -e -s -s -s -c -o -r -e -d -e -f -e -a -t -s -t -h -e -p -o -i -n -t -. - metrics.content_safety.score eu_fairness [content_safety evaluator] - metrics.emotion_recognition.combined_score emotion_recognition - metrics.emotion_recognition.educational_context emotion_recognition - metrics.emotion_recognition.workplace_context emotion_recognition - metrics.fairness.gender_bias eu_fairness [fairness evaluator] -$ -$ +a +i +c +e +r +t +i +f +y +e +x +p +l +a +i +n +e +u +_ +a +i +_ +a +c +t +| +h +e +a +d +- +1 +2 +eu_ai_act — 29 policies +F +i +e +l +d +s +y +o +u +m +u +s +t +d +e +c +l +a +r +e +( +1 +5 +0 +) + No evaluator can observe these. They are facts about your system, + your process, or your paperwork, so you assert them in the contract. + accuracy.declared_in_instructions technical_robustness + accuracy.metrics_declared technical_robustness + assessment.completed conformity_assessment + assessment.harmonised_standards_applied conformity_assessment + assessment.notified_body_involved conformity_assessment +$ +a +i +c +e +r +t +i +f +y +r +e +x +p +l +a +i +n +e +u +_ +a +i +_ +a +c +t +| +g +r +e +p +- +A +8 +" +p +r +o +d +u +c +e +d +b +y +e +v +a +l +u +a +t +o +r +s +" +F +i +e +l +d +s +p +r +o +d +u +c +e +d +b +y +e +v +a +l +u +a +t +o +r +s +( +1 +4 +) + Computed at evaluation time from your interactions. Do not hand-write +t +h +e +s +e +: +a +s +s +e +r +t +i +n +g +y +o +u +r +o +w +n +f +a +i +r +n +e +s +s +s +c +o +r +e +d +e +f +e +a +t +s +t +h +e +p +o +i +n +t +. + metrics.content_safety.score eu_fairness [content_safety evaluator] + metrics.emotion_recognition.combined_score emotion_recognition + metrics.emotion_recognition.educational_context emotion_recognition + metrics.emotion_recognition.workplace_context emotion_recognition + metrics.fairness.gender_bias eu_fairness [fairness evaluator] +$ +$ diff --git a/docs/demo/aicertify-short.cast b/docs/demo/aicertify-short.cast index c827fe8..ae21e11 100644 --- a/docs/demo/aicertify-short.cast +++ b/docs/demo/aicertify-short.cast @@ -1,264 +1,210 @@ {"version": 2, "width": 100, "height": 30, "idle_time_limit": 3.0, "title": "AICertify: 150 declared, 14 measured"} [0.6, "o", "$ "] -[0.6857, "o", "a"] -[0.7481, "o", "i"] -[0.8326, "o", "c"] -[0.8772, "o", "e"] -[0.9482, "o", "r"] -[0.993, "o", "t"] -[1.0405, "o", "i"] -[1.1202, "o", "f"] -[1.1796, "o", "y"] -[1.2503, "o", " "] -[1.3196, "o", "s"] -[1.3809, "o", "c"] -[1.45, "o", "o"] -[1.5289, "o", "r"] -[1.5857, "o", "e"] -[1.6409, "o", "-"] -[1.7185, "o", "c"] -[1.8039, "o", "a"] -[1.8679, "o", "r"] -[1.9161, "o", "d"] -[1.9636, "o", " "] -[2.0197, "o", "b"] -[2.0779, "o", "e"] -[2.1447, "o", "r"] -[2.2007, "o", "t"] -[2.2876, "o", "-"] -[2.3404, "o", "b"] -[2.4044, "o", "a"] -[2.4619, "o", "s"] -[2.5101, "o", "e"] -[2.5927, "o", "-"] -[2.661, "o", "u"] -[2.7296, "o", "n"] -[2.7831, "o", "c"] -[2.8522, "o", "a"] -[2.8905, "o", "s"] -[2.9325, "o", "e"] -[2.9761, "o", "d"] -[3.3961, "o", "\r\n"] -[5.4561, "o", "\r\nbert-base-uncased\r\n completeness 0.49 (threshold 0.8)\r\n"] -[5.5461, "o", " quality 0.66\r\n BELOW THRESHOLD\r\n\r\n"] -[5.6361, "o", " intended_use 1.00 ####################\r\n training_data 0.67 #############\r\n caveats_recommendations 0.50 ##########\r\n"] -[5.7261, "o", " factors 0.50 ##########\r\n metrics 0.50 ##########\r\n quantitative_analyses 0.50 ##########\r\n"] -[5.8161, "o", " ethical_considerations 0.33 #######\r\n model_details 0.33 ######\r\n evaluation_data 0.00 \r\n"] -[5.9061, "o", "\r\nA model card answers part of what Annex IV asks and then stops.\r\nThis is the documentation-completeness gap, not a compliance verdict.\r\n"] -[5.9961, "o", "\r\nscored by GOPAL 2.0.0 (global/v1/documentation/model_card_score), via aicertify 0.7.0\r\n"] -[6.8861, "o", "\u001b7\u001b[18A\r\u001b[16C\u001b[43;30m0\u001b[0m\u001b8"] -[6.9141, "o", "\u001b7\u001b[18A\r\u001b[16C\u001b[43;30m0.\u001b[0m\u001b8"] -[6.9421, "o", "\u001b7\u001b[18A\r\u001b[16C\u001b[43;30m0.4\u001b[0m\u001b8"] -[6.9701, "o", "\u001b7\u001b[18A\r\u001b[16C\u001b[43;30m0.49\u001b[0m\u001b8"] -[7.3381, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mB\u001b[0m\u001b8"] -[7.3661, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBE\u001b[0m\u001b8"] -[7.3941, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBEL\u001b[0m\u001b8"] -[7.4221, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELO\u001b[0m\u001b8"] -[7.4501, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW\u001b[0m\u001b8"] -[7.4781, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW \u001b[0m\u001b8"] -[7.5061, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW T\u001b[0m\u001b8"] -[7.5341, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW TH\u001b[0m\u001b8"] -[7.5621, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THR\u001b[0m\u001b8"] -[7.5901, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THRE\u001b[0m\u001b8"] -[7.6181, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THRES\u001b[0m\u001b8"] -[7.6461, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THRESH\u001b[0m\u001b8"] -[7.6741, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THRESHO\u001b[0m\u001b8"] -[7.7021, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THRESHOL\u001b[0m\u001b8"] -[7.7301, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THRESHOLD\u001b[0m\u001b8"] -[13.4981, "o", "$ "] -[13.5533, "o", "#"] -[13.6042, "o", " "] -[13.6675, "o", "s"] -[13.7216, "o", "o"] -[13.7668, "o", " "] -[13.8414, "o", "w"] -[13.901, "o", "h"] -[13.98, "o", "a"] -[14.036, "o", "t"] -[14.2622, "o", " "] -[14.3358, "o", "d"] -[14.407, "o", "o"] -[14.452, "o", "e"] -[14.5034, "o", "s"] -[14.656, "o", " "] -[14.7291, "o", "t"] -[14.8083, "o", "h"] -[14.8639, "o", "e"] -[14.9825, "o", " "] -[15.0553, "o", "E"] -[15.1175, "o", "U"] -[15.2782, "o", " "] -[15.3602, "o", "A"] -[15.4087, "o", "I"] -[15.4983, "o", " "] -[15.5678, "o", "A"] -[15.6207, "o", "c"] -[15.7021, "o", "t"] -[15.8514, "o", " "] -[15.9182, "o", "a"] -[15.9948, "o", "c"] -[16.0738, "o", "t"] -[16.1586, "o", "u"] -[16.2018, "o", "a"] -[16.2789, "o", "l"] -[16.3431, "o", "k"] -[16.7652, "o", "\b \b"] -[16.9292, "o", "l"] -[17.0206, "o", "y"] -[17.1123, "o", " "] -[17.1793, "o", "a"] -[17.2189, "o", "s"] -[17.2605, "o", "k"] -[17.304, "o", " "] -[17.3619, "o", "f"] -[17.432, "o", "o"] -[17.4911, "o", "r"] -[17.5342, "o", "?"] -[17.9542, "o", "\r\n"] -[18.6142, "o", "$ "] -[18.6746, "o", "a"] -[18.7332, "o", "i"] -[18.8196, "o", "c"] -[18.8938, "o", "e"] -[18.9528, "o", "r"] -[19.0121, "o", "t"] -[19.0838, "o", "i"] -[19.1662, "o", "f"] -[19.2485, "o", "y"] -[19.4441, "o", " "] -[19.5332, "o", "e"] -[19.625, "o", "x"] -[19.6876, "o", "p"] -[19.7399, "o", "l"] -[19.8257, "o", "a"] -[19.8648, "o", "i"] -[19.9051, "o", "n"] -[19.9678, "o", " "] -[20.0131, "o", "e"] -[20.075, "o", "u"] -[20.1434, "o", "_"] -[20.1838, "o", "a"] -[20.2341, "o", "i"] -[20.2763, "o", "_"] -[20.3213, "o", "a"] -[20.3978, "o", "c"] -[20.4606, "o", "t"] -[20.5349, "o", " "] -[20.587, "o", "|"] -[20.7452, "o", " "] -[20.7897, "o", "h"] -[20.8367, "o", "e"] -[20.8764, "o", "a"] -[20.9218, "o", "d"] -[20.9953, "o", " "] -[21.0738, "o", "-"] -[21.1456, "o", "1"] -[21.2333, "o", "2"] -[21.6533, "o", "\r\n"] -[23.7133, "o", "\r\neu_ai_act \u2014 29 policies\r\n\r\n"] -[23.8033, "o", "Fields you must declare (150)\r\n No evaluator can observe these. They are facts about your system,\r\n your process, or your paperwork, so you assert them in the contract.\r\n"] -[23.8933, "o", "\r\n accuracy.declared_in_instructions technical_robustness\r\n accuracy.metrics_declared technical_robustness\r\n"] -[23.9833, "o", " assessment.completed conformity_assessment\r\n assessment.harmonised_standards_applied conformity_assessment\r\n assessment.notified_body_involved conformity_assessment\r\n"] -[24.7733, "o", "\u001b7\u001b[9A\r\u001b[25C\u001b[43;30m1\u001b[0m\u001b8"] -[24.8013, "o", "\u001b7\u001b[9A\r\u001b[25C\u001b[43;30m15\u001b[0m\u001b8"] -[24.8293, "o", "\u001b7\u001b[9A\r\u001b[25C\u001b[43;30m150\u001b[0m\u001b8"] -[27.7973, "o", "$ "] -[27.8433, "o", "a"] -[27.9297, "o", "i"] -[28.0189, "o", "c"] -[28.07, "o", "e"] -[28.1519, "o", "r"] -[28.219, "o", "t"] -[28.2995, "o", "i"] -[28.3838, "o", "f"] -[28.4619, "o", "y"] -[28.519, "o", " "] -[28.6104, "o", "e"] -[28.6649, "o", "x"] -[28.7462, "o", "p"] -[28.7963, "o", "l"] -[28.8422, "o", "a"] -[28.9155, "o", "i"] -[28.9987, "o", "n"] -[29.1715, "o", " "] -[29.2417, "o", "e"] -[29.281, "o", "u"] -[29.321, "o", "_"] -[29.4052, "o", "a"] -[29.447, "o", "i"] -[29.4953, "o", "_"] -[29.5566, "o", "a"] -[29.5965, "o", "c"] -[29.6743, "o", "t"] -[29.8469, "o", " "] -[29.929, "o", "|"] -[30.0082, "o", " "] -[30.0598, "o", "g"] -[30.125, "o", "r"] -[30.1909, "o", "e"] -[30.2435, "o", "p"] -[30.3152, "o", " "] -[30.3924, "o", "-"] -[30.4637, "o", "A"] -[30.5366, "o", "8"] -[30.5898, "o", " "] -[30.6284, "o", "\""] -[30.6883, "o", "p"] -[30.7435, "o", "r"] -[30.7848, "o", "o"] -[30.8245, "o", "d"] -[30.9021, "o", "u"] -[30.9473, "o", "c"] -[30.9886, "o", "e"] -[31.0293, "o", "d"] -[31.0939, "o", " "] -[31.1713, "o", "b"] -[31.2282, "o", "y"] -[31.2966, "o", " "] -[31.368, "o", "e"] -[31.4421, "o", "v"] -[31.5281, "o", "a"] -[31.5911, "o", "l"] -[31.6516, "o", "u"] -[31.7086, "o", "a"] -[31.7847, "o", "t"] -[31.8314, "o", "o"] -[31.8898, "o", "r"] -[31.9389, "o", "s"] -[32.004, "o", "\""] -[32.424, "o", "\r\n"] -[34.484, "o", "Fields produced by evaluators (14)\r\n Computed at evaluation time from your interactions. Do not hand-write\r\n these: asserting your own fairness score defeats the point.\r\n"] -[34.574, "o", "\r\n metrics.content_safety.score eu_fairness [content_safety evaluator]\r\n metrics.emotion_recognition.combined_score emotion_recognition\r\n"] -[34.664, "o", " metrics.emotion_recognition.educational_context emotion_recognition\r\n metrics.emotion_recognition.workplace_context emotion_recognition\r\n metrics.fairness.gender_bias eu_fairness [fairness evaluator]\r\n"] -[35.454, "o", "\u001b7\u001b[9A\r\u001b[31C\u001b[43;30m1\u001b[0m\u001b8"] -[35.482, "o", "\u001b7\u001b[9A\r\u001b[31C\u001b[43;30m14\u001b[0m\u001b8"] -[35.85, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30md\u001b[0m\u001b8"] -[35.878, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mde\u001b[0m\u001b8"] -[35.906, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdef\u001b[0m\u001b8"] -[35.934, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefe\u001b[0m\u001b8"] -[35.962, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefea\u001b[0m\u001b8"] -[35.99, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeat\u001b[0m\u001b8"] -[36.018, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats\u001b[0m\u001b8"] -[36.046, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats \u001b[0m\u001b8"] -[36.074, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats t\u001b[0m\u001b8"] -[36.102, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats th\u001b[0m\u001b8"] -[36.13, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the\u001b[0m\u001b8"] -[36.158, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the \u001b[0m\u001b8"] -[36.186, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the p\u001b[0m\u001b8"] -[36.214, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the po\u001b[0m\u001b8"] -[36.242, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the poi\u001b[0m\u001b8"] -[36.27, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the poin\u001b[0m\u001b8"] -[36.298, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the point\u001b[0m\u001b8"] -[42.066, "o", "$ "] -[42.066, "o", "\u2588"] -[42.596, "o", "\b \b"] -[43.126, "o", "\u2588"] -[43.656, "o", "\b \b"] -[44.186, "o", "\u2588"] -[44.716, "o", "\b \b"] -[45.246, "o", "\u2588"] -[45.776, "o", "\b \b"] -[46.306, "o", "\u2588"] -[46.836, "o", "\b \b"] -[47.366, "o", "\u2588"] -[47.896, "o", "\b \b"] +[0.6912, "o", "a"] +[0.7544, "o", "i"] +[0.8074, "o", "c"] +[0.8904, "o", "e"] +[0.9385, "o", "r"] +[1.0087, "o", "t"] +[1.0861, "o", "i"] +[1.1464, "o", "f"] +[1.2163, "o", "y"] +[1.3685, "o", " "] +[1.4257, "o", "s"] +[1.469, "o", "c"] +[1.5596, "o", "o"] +[1.6319, "o", "r"] +[1.6926, "o", "e"] +[1.7633, "o", "-"] +[1.8453, "o", "c"] +[1.8879, "o", "a"] +[1.9697, "o", "r"] +[2.0137, "o", "d"] +[2.1456, "o", " "] +[2.23, "o", "b"] +[2.2811, "o", "e"] +[2.3311, "o", "r"] +[2.4192, "o", "t"] +[2.4648, "o", "-"] +[2.5295, "o", "b"] +[2.582, "o", "a"] +[2.6272, "o", "s"] +[2.705, "o", "e"] +[2.7881, "o", "-"] +[2.8655, "o", "u"] +[2.9474, "o", "n"] +[3.0034, "o", "c"] +[3.0755, "o", "a"] +[3.1184, "o", "s"] +[3.1855, "o", "e"] +[3.2576, "o", "d"] +[3.6776, "o", "\r\n"] +[5.7376, "o", "\r\nbert-base-uncased\r\n completeness 0.49 (threshold 0.8)\r\n"] +[5.8276, "o", " quality 0.66\r\n BELOW THRESHOLD\r\n\r\n"] +[5.9176, "o", " intended_use 1.00 ####################\r\n training_data 0.67 #############\r\n caveats_recommendations 0.50 ##########\r\n"] +[6.0076, "o", " factors 0.50 ##########\r\n metrics 0.50 ##########\r\n quantitative_analyses 0.50 ##########\r\n"] +[6.0976, "o", " ethical_considerations 0.33 #######\r\n model_details 0.33 ######\r\n evaluation_data 0.00 \r\n"] +[6.1876, "o", "\r\nA model card answers part of what Annex IV asks and then stops.\r\nThis is the documentation-completeness gap, not a compliance verdict.\r\n"] +[6.2776, "o", "\r\nscored by GOPAL 2.0.0 (global/v1/documentation/model_card_score), via aicertify 0.7.0\r\n"] +[7.1676, "o", "\u001b7\u001b[18A\r\u001b[16C\u001b[43;30m0\u001b[0m\u001b8"] +[7.1896, "o", "\u001b7\u001b[18A\r\u001b[16C\u001b[43;30m0.\u001b[0m\u001b8"] +[7.2116, "o", "\u001b7\u001b[18A\r\u001b[16C\u001b[43;30m0.4\u001b[0m\u001b8"] +[7.2336, "o", "\u001b7\u001b[18A\r\u001b[16C\u001b[43;30m0.49\u001b[0m\u001b8"] +[7.4956, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mB\u001b[0m\u001b8"] +[7.5176, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBE\u001b[0m\u001b8"] +[7.5396, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBEL\u001b[0m\u001b8"] +[7.5616, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELO\u001b[0m\u001b8"] +[7.5836, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW\u001b[0m\u001b8"] +[7.6056, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW \u001b[0m\u001b8"] +[7.6276, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW T\u001b[0m\u001b8"] +[7.6496, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW TH\u001b[0m\u001b8"] +[7.6716, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THR\u001b[0m\u001b8"] +[7.6936, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THRE\u001b[0m\u001b8"] +[7.7156, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THRES\u001b[0m\u001b8"] +[7.7376, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THRESH\u001b[0m\u001b8"] +[7.7596, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THRESHO\u001b[0m\u001b8"] +[7.7816, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THRESHOL\u001b[0m\u001b8"] +[7.8036, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THRESHOLD\u001b[0m\u001b8"] +[11.4656, "o", "$ "] +[11.5336, "o", "a"] +[11.5788, "o", "i"] +[11.6534, "o", "c"] +[11.712, "o", "e"] +[11.7642, "o", "r"] +[11.8347, "o", "t"] +[11.9041, "o", "i"] +[11.9898, "o", "f"] +[12.0522, "o", "y"] +[12.254, "o", " "] +[12.2988, "o", "e"] +[12.3462, "o", "x"] +[12.426, "o", "p"] +[12.4854, "o", "l"] +[12.5561, "o", "a"] +[12.6407, "o", "i"] +[12.7101, "o", "n"] +[12.7714, "o", " "] +[12.8502, "o", "e"] +[12.9071, "o", "u"] +[12.9623, "o", "_"] +[13.0399, "o", "a"] +[13.1253, "o", "i"] +[13.1893, "o", "_"] +[13.2375, "o", "a"] +[13.2849, "o", "c"] +[13.365, "o", "t"] +[13.4212, "o", " "] +[13.4879, "o", "|"] +[13.544, "o", " "] +[13.5967, "o", "h"] +[13.6607, "o", "e"] +[13.7182, "o", "a"] +[13.7664, "o", "d"] +[13.849, "o", " "] +[13.9176, "o", "-"] +[13.9711, "o", "1"] +[14.0401, "o", "2"] +[14.4601, "o", "\r\n"] +[16.5201, "o", "\r\neu_ai_act \u2014 29 policies\r\n\r\n"] +[16.6101, "o", "Fields you must declare (150)\r\n No evaluator can observe these. They are facts about your system,\r\n your process, or your paperwork, so you assert them in the contract.\r\n"] +[16.7001, "o", "\r\n accuracy.declared_in_instructions technical_robustness\r\n accuracy.metrics_declared technical_robustness\r\n"] +[16.7901, "o", " assessment.completed conformity_assessment\r\n assessment.harmonised_standards_applied conformity_assessment\r\n assessment.notified_body_involved conformity_assessment\r\n"] +[17.5801, "o", "\u001b7\u001b[9A\r\u001b[25C\u001b[43;30m1\u001b[0m\u001b8"] +[17.6021, "o", "\u001b7\u001b[9A\r\u001b[25C\u001b[43;30m15\u001b[0m\u001b8"] +[17.6241, "o", "\u001b7\u001b[9A\r\u001b[25C\u001b[43;30m150\u001b[0m\u001b8"] +[19.8861, "o", "$ "] +[19.9328, "o", "a"] +[19.9885, "o", "i"] +[20.0562, "o", "c"] +[20.1188, "o", "e"] +[20.2067, "o", "r"] +[20.2856, "o", "t"] +[20.353, "o", "i"] +[20.4393, "o", "f"] +[20.5046, "o", "y"] +[20.5794, "o", " "] +[20.6487, "o", "r"] +[20.9188, "o", "\b \b"] +[21.1028, "o", "e"] +[21.1488, "o", "x"] +[21.2315, "o", "p"] +[21.3216, "o", "l"] +[21.4096, "o", "a"] +[21.4889, "o", "i"] +[21.5723, "o", "n"] +[21.6689, "o", " "] +[21.7097, "o", "e"] +[21.7587, "o", "u"] +[21.8502, "o", "_"] +[21.9086, "o", "a"] +[21.9724, "o", "i"] +[22.0319, "o", "_"] +[22.0953, "o", "a"] +[22.1819, "o", "c"] +[22.2568, "o", "t"] +[22.3368, "o", " "] +[22.3774, "o", "|"] +[22.4359, "o", " "] +[22.5195, "o", "g"] +[22.5794, "o", "r"] +[22.6208, "o", "e"] +[22.6889, "o", "p"] +[22.7582, "o", " "] +[22.8177, "o", "-"] +[22.8905, "o", "A"] +[22.9497, "o", "8"] +[23.1584, "o", " "] +[23.2477, "o", "\""] +[23.2901, "o", "p"] +[23.3629, "o", "r"] +[23.4036, "o", "o"] +[23.4569, "o", "d"] +[23.5189, "o", "u"] +[23.604, "o", "c"] +[23.6869, "o", "e"] +[23.7259, "o", "d"] +[23.8104, "o", " "] +[23.8564, "o", "b"] +[23.9131, "o", "y"] +[24.0893, "o", " "] +[24.1714, "o", "e"] +[24.2187, "o", "v"] +[24.2903, "o", "a"] +[24.38, "o", "l"] +[24.4512, "o", "u"] +[24.49, "o", "a"] +[24.5682, "o", "t"] +[24.6222, "o", "o"] +[24.6827, "o", "r"] +[24.7723, "o", "s"] +[24.841, "o", "\""] +[25.261, "o", "\r\n"] +[27.321, "o", "Fields produced by evaluators (14)\r\n Computed at evaluation time from your interactions. Do not hand-write\r\n these: asserting your own fairness score defeats the point.\r\n"] +[27.411, "o", "\r\n metrics.content_safety.score eu_fairness [content_safety evaluator]\r\n metrics.emotion_recognition.combined_score emotion_recognition\r\n"] +[27.501, "o", " metrics.emotion_recognition.educational_context emotion_recognition\r\n metrics.emotion_recognition.workplace_context emotion_recognition\r\n metrics.fairness.gender_bias eu_fairness [fairness evaluator]\r\n"] +[28.291, "o", "\u001b7\u001b[9A\r\u001b[31C\u001b[43;30m1\u001b[0m\u001b8"] +[28.313, "o", "\u001b7\u001b[9A\r\u001b[31C\u001b[43;30m14\u001b[0m\u001b8"] +[28.575, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30md\u001b[0m\u001b8"] +[28.597, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mde\u001b[0m\u001b8"] +[28.619, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdef\u001b[0m\u001b8"] +[28.641, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefe\u001b[0m\u001b8"] +[28.663, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefea\u001b[0m\u001b8"] +[28.685, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeat\u001b[0m\u001b8"] +[28.707, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats\u001b[0m\u001b8"] +[28.729, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats \u001b[0m\u001b8"] +[28.751, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats t\u001b[0m\u001b8"] +[28.773, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats th\u001b[0m\u001b8"] +[28.795, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the\u001b[0m\u001b8"] +[28.817, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the \u001b[0m\u001b8"] +[28.839, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the p\u001b[0m\u001b8"] +[28.861, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the po\u001b[0m\u001b8"] +[28.883, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the poi\u001b[0m\u001b8"] +[28.905, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the poin\u001b[0m\u001b8"] +[28.927, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the point\u001b[0m\u001b8"] +[32.589, "o", "$ "] +[32.589, "o", "\u2588"] +[33.119, "o", "\b \b"] +[33.649, "o", "\u2588"] +[34.179, "o", "\b \b"] +[34.709, "o", "\u2588"] +[35.239, "o", "\b \b"] diff --git a/docs/demo/aicertify.cast b/docs/demo/aicertify.cast index 84700db..b096f36 100644 --- a/docs/demo/aicertify.cast +++ b/docs/demo/aicertify.cast @@ -49,486 +49,442 @@ [6.7427, "o", "\r\nA model card answers part of what Annex IV asks and then stops.\r\nThis is the documentation-completeness gap, not a compliance verdict.\r\n"] [6.8327, "o", "\r\nscored by GOPAL 2.0.0 (global/v1/documentation/model_card_score), via aicertify 0.7.0\r\n"] [7.7227, "o", "\u001b7\u001b[18A\r\u001b[16C\u001b[43;30m0\u001b[0m\u001b8"] -[7.7507, "o", "\u001b7\u001b[18A\r\u001b[16C\u001b[43;30m0.\u001b[0m\u001b8"] -[7.7787, "o", "\u001b7\u001b[18A\r\u001b[16C\u001b[43;30m0.4\u001b[0m\u001b8"] -[7.8067, "o", "\u001b7\u001b[18A\r\u001b[16C\u001b[43;30m0.49\u001b[0m\u001b8"] -[8.1747, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mB\u001b[0m\u001b8"] -[8.2027, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBE\u001b[0m\u001b8"] -[8.2307, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBEL\u001b[0m\u001b8"] -[8.2587, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELO\u001b[0m\u001b8"] -[8.2867, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW\u001b[0m\u001b8"] -[8.3147, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW \u001b[0m\u001b8"] -[8.3427, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW T\u001b[0m\u001b8"] -[8.3707, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW TH\u001b[0m\u001b8"] -[8.3987, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THR\u001b[0m\u001b8"] -[8.4267, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THRE\u001b[0m\u001b8"] -[8.4547, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THRES\u001b[0m\u001b8"] -[8.4827, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THRESH\u001b[0m\u001b8"] -[8.5107, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THRESHO\u001b[0m\u001b8"] -[8.5387, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THRESHOL\u001b[0m\u001b8"] -[8.5667, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THRESHOLD\u001b[0m\u001b8"] -[14.3347, "o", "$ "] -[14.3728, "o", "#"] -[14.4216, "o", " "] -[14.4751, "o", "s"] -[14.5457, "o", "o"] -[14.6038, "o", " "] -[14.6894, "o", "w"] -[14.7391, "o", "h"] -[14.8281, "o", "a"] -[14.8792, "o", "t"] -[14.9428, "o", " "] -[15.0337, "o", "d"] -[15.1247, "o", "o"] -[15.2054, "o", "e"] -[15.261, "o", "s"] -[15.3275, "o", " "] -[15.3931, "o", "t"] -[15.4326, "o", "h"] -[15.5002, "o", "e"] -[15.5741, "o", " "] -[15.6338, "o", "E"] -[15.6756, "o", "U"] -[15.7291, "o", " "] -[15.8146, "o", "A"] -[15.8902, "o", "I"] -[15.9434, "o", " "] -[16.0148, "o", "A"] -[16.0948, "o", "c"] -[16.146, "o", "t"] -[16.1997, "o", " "] -[16.2682, "o", "a"] -[16.308, "o", "c"] -[16.3465, "o", "t"] -[16.3926, "o", "u"] -[16.4409, "o", "a"] -[16.5198, "o", "l"] -[16.58, "o", "l"] -[16.62, "o", "y"] -[16.7305, "o", " "] -[16.8186, "o", "a"] -[16.8661, "o", "s"] -[16.936, "o", "k"] -[17.004, "o", " "] -[17.0739, "o", "f"] -[17.1166, "o", "o"] -[17.1599, "o", "r"] -[17.2375, "o", "?"] -[17.6575, "o", "\r\n"] -[18.3175, "o", "$ "] -[18.3736, "o", "a"] -[18.4514, "o", "i"] -[18.5393, "o", "c"] -[18.6178, "o", "e"] -[18.6876, "o", "r"] -[18.7719, "o", "t"] -[18.8375, "o", "i"] -[18.8849, "o", "f"] -[18.948, "o", "y"] -[19.12, "o", " "] -[19.2032, "o", "e"] -[19.2881, "o", "x"] -[19.3525, "o", "p"] -[19.4241, "o", "l"] -[19.4642, "o", "a"] -[19.5344, "o", "i"] -[19.6001, "o", "n"] -[19.6664, "o", " "] -[19.727, "o", "e"] -[19.7846, "o", "u"] -[19.8725, "o", "_"] -[19.9425, "o", "a"] -[19.9966, "o", "i"] -[20.0526, "o", "_"] -[20.1064, "o", "s"] -[20.5153, "o", "\b \b"] -[20.7183, "o", "a"] -[20.7907, "o", "c"] -[20.8689, "o", "t"] -[21.0413, "o", " "] -[21.0899, "o", "|"] -[21.169, "o", " "] -[21.2214, "o", "h"] -[21.2811, "o", "e"] -[21.3388, "o", "a"] -[21.3854, "o", "d"] -[21.4432, "o", " "] -[21.5249, "o", "-"] -[21.5912, "o", "1"] -[21.646, "o", "2"] -[22.066, "o", "\r\n"] -[24.126, "o", "\r\neu_ai_act \u2014 29 policies\r\n\r\n"] -[24.216, "o", "Fields you must declare (150)\r\n No evaluator can observe these. They are facts about your system,\r\n your process, or your paperwork, so you assert them in the contract.\r\n"] -[24.306, "o", "\r\n accuracy.declared_in_instructions technical_robustness\r\n accuracy.metrics_declared technical_robustness\r\n"] -[24.396, "o", " assessment.completed conformity_assessment\r\n assessment.harmonised_standards_applied conformity_assessment\r\n assessment.notified_body_involved conformity_assessment\r\n"] -[25.186, "o", "\u001b7\u001b[9A\r\u001b[25C\u001b[43;30m1\u001b[0m\u001b8"] -[25.214, "o", "\u001b7\u001b[9A\r\u001b[25C\u001b[43;30m15\u001b[0m\u001b8"] -[25.242, "o", "\u001b7\u001b[9A\r\u001b[25C\u001b[43;30m150\u001b[0m\u001b8"] -[28.21, "o", "$ "] -[28.2514, "o", "a"] -[28.3117, "o", "i"] -[28.3522, "o", "c"] -[28.4074, "o", "e"] -[28.4549, "o", "r"] -[28.5189, "o", "t"] -[28.5999, "o", "i"] -[28.6522, "o", "f"] -[28.7288, "o", "y"] -[28.7878, "o", " "] -[28.855, "o", "e"] -[28.9184, "o", "x"] -[28.9987, "o", "p"] -[29.0767, "o", "l"] -[29.1447, "o", "a"] -[29.2237, "o", "i"] -[29.272, "o", "n"] -[29.4492, "o", " "] -[29.5169, "o", "e"] -[29.5878, "o", "u"] -[29.6708, "o", "_"] -[29.7474, "o", "a"] -[29.8036, "o", "i"] -[29.8704, "o", "_"] -[29.9512, "o", "a"] -[30.0082, "o", "c"] -[30.0582, "o", "r"] -[30.4082, "o", "\b \b"] -[30.6003, "o", "t"] -[30.687, "o", " "] -[30.746, "o", "|"] -[30.9139, "o", " "] -[30.9762, "o", "g"] -[31.0533, "o", "r"] -[31.1333, "o", "e"] -[31.1925, "o", "p"] -[31.2527, "o", " "] -[31.3087, "o", "-"] -[31.3607, "o", "A"] -[31.4381, "o", "8"] -[31.6029, "o", " "] -[31.6572, "o", "\""] -[31.7332, "o", "p"] -[31.8124, "o", "r"] -[31.8912, "o", "o"] -[31.9601, "o", "d"] -[32.0083, "o", "u"] -[32.0643, "o", "c"] -[32.1034, "o", "e"] -[32.1912, "o", "d"] -[32.3943, "o", " "] -[32.446, "o", "b"] -[32.5336, "o", "y"] -[32.765, "o", " "] -[32.8458, "o", "e"] -[32.9108, "o", "v"] -[32.9632, "o", "a"] -[33.0423, "o", "l"] -[33.1011, "o", "u"] -[33.1434, "o", "a"] -[33.2204, "o", "t"] -[33.289, "o", "o"] -[33.3575, "o", "r"] -[33.4416, "o", "s"] -[33.5174, "o", "\""] -[33.9374, "o", "\r\n"] -[35.9974, "o", "Fields produced by evaluators (14)\r\n Computed at evaluation time from your interactions. Do not hand-write\r\n these: asserting your own fairness score defeats the point.\r\n"] -[36.0874, "o", "\r\n metrics.content_safety.score eu_fairness [content_safety evaluator]\r\n metrics.emotion_recognition.combined_score emotion_recognition\r\n"] -[36.1774, "o", " metrics.emotion_recognition.educational_context emotion_recognition\r\n metrics.emotion_recognition.workplace_context emotion_recognition\r\n metrics.fairness.gender_bias eu_fairness [fairness evaluator]\r\n"] -[36.9674, "o", "\u001b7\u001b[9A\r\u001b[31C\u001b[43;30m1\u001b[0m\u001b8"] -[36.9954, "o", "\u001b7\u001b[9A\r\u001b[31C\u001b[43;30m14\u001b[0m\u001b8"] -[37.3634, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30md\u001b[0m\u001b8"] -[37.3914, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mde\u001b[0m\u001b8"] -[37.4194, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdef\u001b[0m\u001b8"] -[37.4474, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefe\u001b[0m\u001b8"] -[37.4754, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefea\u001b[0m\u001b8"] -[37.5034, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeat\u001b[0m\u001b8"] -[37.5314, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats\u001b[0m\u001b8"] -[37.5594, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats \u001b[0m\u001b8"] -[37.5874, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats t\u001b[0m\u001b8"] -[37.6154, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats th\u001b[0m\u001b8"] -[37.6434, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the\u001b[0m\u001b8"] -[37.6714, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the \u001b[0m\u001b8"] -[37.6994, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the p\u001b[0m\u001b8"] -[37.7274, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the po\u001b[0m\u001b8"] -[37.7554, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the poi\u001b[0m\u001b8"] -[37.7834, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the poin\u001b[0m\u001b8"] -[37.8114, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the point\u001b[0m\u001b8"] -[43.5794, "o", "$ "] -[43.6385, "o", "a"] -[43.6917, "o", "i"] -[43.7727, "o", "c"] -[43.8634, "o", "e"] -[43.9259, "o", "r"] -[43.9854, "o", "t"] -[44.042, "o", "i"] -[44.1211, "o", "f"] -[44.16, "o", "y"] -[44.3086, "o", " "] -[44.3876, "o", "i"] -[44.4676, "o", "n"] -[44.5508, "o", "i"] -[44.6033, "o", "t"] -[44.6617, "o", "-"] -[44.734, "o", "c"] -[44.7916, "o", "o"] -[44.8625, "o", "n"] -[44.9534, "o", "t"] -[45.0147, "o", "r"] -[45.0767, "o", "a"] -[45.1332, "o", "c"] -[45.1976, "o", "t"] -[45.2691, "o", " "] -[45.3283, "o", "0"] -[45.6653, "o", "\b \b"] -[45.8623, "o", "-"] -[45.9455, "o", "-"] -[45.9921, "o", "p"] -[46.0499, "o", "o"] -[46.1131, "o", "l"] -[46.1849, "o", "i"] -[46.2709, "o", "c"] -[46.3439, "o", "y"] -[46.5273, "o", " "] -[46.5701, "o", "e"] -[46.6335, "o", "u"] -[46.7152, "o", "_"] -[46.807, "o", "a"] -[46.8723, "o", "i"] -[46.9585, "o", "_"] -[46.9969, "o", "a"] -[47.0447, "o", "c"] -[47.0982, "o", "t"] -[47.1756, "o", " "] -[47.2153, "o", ">"] -[47.2948, "o", " "] -[47.351, "o", "c"] -[47.4214, "o", "o"] -[47.4799, "o", "n"] -[47.5598, "o", "t"] -[47.6278, "o", "r"] -[47.6705, "o", "a"] -[47.734, "o", "c"] -[47.8129, "o", "t"] -[47.8936, "o", "."] -[47.9715, "o", "j"] -[48.0437, "o", "s"] -[48.1216, "o", "o"] -[48.2074, "o", "n"] -[48.6274, "o", "\r\n"] -[51.5874, "o", "$ "] -[51.6272, "o", "a"] -[51.7191, "o", "i"] -[51.7606, "o", "c"] -[51.8272, "o", "e"] -[51.8975, "o", "r"] -[51.9475, "o", "t"] -[52.0156, "o", "i"] -[52.1, "o", "f"] -[52.1779, "o", "y"] -[52.3785, "o", " "] -[52.4637, "o", "e"] -[52.5155, "o", "v"] -[52.5743, "o", "a"] -[52.6317, "o", "l"] -[52.6783, "o", "u"] -[52.7438, "o", "a"] -[52.8299, "o", "t"] -[52.8931, "o", "e"] -[52.9803, "o", " "] -[53.0393, "o", "-"] -[53.1041, "o", "-"] -[53.1831, "o", "c"] -[53.2314, "o", "o"] -[53.2756, "o", "n"] -[53.346, "o", "t"] -[53.433, "o", "r"] -[53.4931, "o", "a"] -[53.5609, "o", "c"] -[53.6446, "o", "t"] -[53.7168, "o", " "] -[53.7934, "o", "c"] -[53.8616, "o", "o"] -[53.9169, "o", "n"] -[53.9696, "o", "t"] -[54.0341, "o", "r"] -[54.1098, "o", "a"] -[54.1802, "o", "c"] -[54.2676, "o", "t"] -[54.3384, "o", "."] -[54.4258, "o", "j"] -[54.5065, "o", "s"] -[54.563, "o", "o"] -[54.6023, "o", "n"] -[54.6764, "o", " "] -[54.7407, "o", "-"] -[54.7891, "o", "-"] -[54.8412, "o", "p"] -[54.8814, "o", "o"] -[54.9483, "o", "l"] -[54.9985, "o", "i"] -[55.0459, "o", "c"] -[55.1294, "o", "y"] -[55.3245, "o", " "] -[55.3683, "o", "e"] -[55.4425, "o", "u"] -[55.4895, "o", "_"] -[55.5737, "o", "a"] -[55.627, "o", "i"] -[55.7134, "o", "_"] -[55.7583, "o", "a"] -[55.8154, "o", "c"] -[55.8796, "o", "t"] -[56.2996, "o", "\r\n"] -[58.3596, "o", "Evaluation Summary:\r\nContract ID: b7e60f21-6717-4ebd-b83f-dd6927dd5c52\r\nApplication: your-application\r\n"] -[58.4496, "o", "Report: ./reports/folder_report_your-application_2026-08-30_165312.md\r\nOPA Evaluation: Successful\r\n"] -[59.3396, "o", "$ "] -[59.4066, "o", "h"] -[59.4879, "o", "e"] -[59.5623, "o", "a"] -[59.652, "o", "d"] -[59.73, "o", " "] -[59.7905, "o", "-"] -[59.8812, "o", "8"] -[59.962, "o", " "] -[60.0213, "o", "e"] -[60.3724, "o", "\b \b"] -[60.5701, "o", "r"] -[60.6452, "o", "e"] -[60.7252, "o", "p"] -[60.7674, "o", "o"] -[60.8375, "o", "r"] -[60.8975, "o", "t"] -[60.9699, "o", "s"] -[61.0196, "o", "/"] -[61.08, "o", "*"] -[61.138, "o", "."] -[61.1952, "o", "m"] -[61.287, "o", "d"] -[61.707, "o", "\r\n"] -[62.167, "o", "# Evaluation Report\r\n\r\n**Analysis Date:** 2026-08-30 11:23:12\r\n"] -[62.257, "o", "\r\n## Evaluation Summary\r\n- **Total Policies:** 29\r\n"] -[62.347, "o", "- **Green Count:** 0\r\n- **Red Count:** 29\r\n"] -[63.137, "o", "\u001b7\u001b[1A\r\u001b[17C\u001b[43;30m2\u001b[0m\u001b8"] -[63.165, "o", "\u001b7\u001b[1A\r\u001b[17C\u001b[43;30m29\u001b[0m\u001b8"] -[63.533, "o", "\u001b7\u001b[2A\r\u001b[19C\u001b[43;30m0\u001b[0m\u001b8"] -[69.301, "o", "$ "] -[69.3882, "o", "#"] -[69.4743, "o", " "] -[69.5451, "o", "w"] -[69.6227, "o", "h"] -[69.7075, "o", "i"] -[69.7818, "o", "c"] -[69.828, "o", "h"] -[69.9057, "o", " "] -[69.9455, "o", "m"] -[69.9909, "o", "e"] -[70.0551, "o", "t"] -[70.0977, "o", "r"] -[70.1655, "o", "i"] -[70.2543, "o", "c"] -[70.2991, "o", "s"] -[70.4012, "o", " "] -[70.4616, "o", "c"] -[70.5501, "o", "a"] -[70.6038, "o", "n"] -[70.6874, "o", " "] -[70.7728, "o", "a"] -[70.8183, "o", "n"] -[70.9047, "o", " "] -[70.9633, "o", "e"] -[71.0446, "o", "v"] -[71.1189, "o", "a"] -[71.2031, "o", "l"] -[71.2921, "o", "u"] -[71.3711, "o", "a"] -[71.4169, "o", "t"] -[71.4596, "o", "o"] -[71.5339, "o", "r"] -[71.7245, "o", " "] -[71.7665, "o", "s"] -[71.8055, "o", "u"] -[71.8918, "o", "p"] -[71.9756, "o", "p"] -[72.03, "o", "l"] -[72.0698, "o", "y"] -[72.1141, "o", " "] -[72.1811, "o", "t"] -[72.2331, "o", "o"] -[72.3002, "o", "d"] -[72.3897, "o", "a"] -[72.4633, "o", "y"] -[72.5286, "o", "?"] -[72.9486, "o", "\r\n"] -[73.6086, "o", "$ "] -[73.6944, "o", "p"] -[73.7683, "o", "y"] -[73.8566, "o", "t"] -[73.9233, "o", "h"] -[74.0105, "o", "o"] -[74.0865, "o", "n"] -[74.2597, "o", " "] -[74.3029, "o", "s"] -[74.3827, "o", "c"] -[74.4661, "o", "r"] -[74.5439, "o", "i"] -[74.6249, "o", "p"] -[74.7139, "o", "t"] -[74.797, "o", "s"] -[74.8734, "o", "/"] -[74.9156, "o", "m"] -[74.9747, "o", "e"] -[75.0364, "o", "t"] -[75.1187, "o", "r"] -[75.1817, "o", "i"] -[75.261, "o", "c"] -[75.3511, "o", "_"] -[75.3951, "o", "g"] -[75.4514, "o", "a"] -[75.5039, "o", "p"] -[75.594, "o", "_"] -[75.6727, "o", "r"] -[75.7639, "o", "e"] -[75.8538, "o", "p"] -[75.9081, "o", "o"] -[75.9502, "o", "r"] -[76.0355, "o", "t"] -[76.0967, "o", "."] -[76.1828, "o", "p"] -[76.2418, "o", "y"] -[76.3131, "o", " "] -[76.3935, "o", "|"] -[76.5115, "o", " "] -[76.5943, "o", "t"] -[76.665, "o", "a"] -[76.7531, "o", "i"] -[76.8212, "o", "l"] -[76.8663, "o", " "] -[76.9249, "o", "-"] -[76.977, "o", "1"] -[77.0476, "o", "2"] -[77.4676, "o", "\r\n"] -[79.5276, "o", " calc metrics.model_card.completeness computed by global.v1.documentation.model_card_score, no evaluator needed\r\n GAP metrics.model_card.compliance_level no evaluator declares this\r\n calc metrics.model_card.quality computed by global.v1.documentation.model_card_score, no evaluator needed\r\n"] -[79.6176, "o", " calc metrics.model_card.section_scores computed by global.v1.documentation.model_card_score, no evaluator needed\r\n ok metrics.social_scoring.combined_score SocialScoringEvaluator\r\n ok metrics.social_scoring.detrimental_treatment SocialScoringEvaluator\r\n"] -[79.7076, "o", " ok metrics.social_scoring.score SocialScoringEvaluator\r\n ok metrics.toxicity.max_toxicity ContentSafetyEvaluator\r\n\r\n"] -[79.7976, "o", "TOTAL: 22 of 27 measured metrics can be supplied today.\r\nThe gaps are what you would need to write an evaluator for, or\r\nsupply by hand. See docs on writing one.\r\n"] -[80.6876, "o", "\u001b7\u001b[3A\r\u001b[7C\u001b[43;30m2\u001b[0m\u001b8"] -[80.7156, "o", "\u001b7\u001b[3A\r\u001b[7C\u001b[43;30m22\u001b[0m\u001b8"] -[80.7436, "o", "\u001b7\u001b[3A\r\u001b[7C\u001b[43;30m22 \u001b[0m\u001b8"] -[80.7716, "o", "\u001b7\u001b[3A\r\u001b[7C\u001b[43;30m22 o\u001b[0m\u001b8"] -[80.7996, "o", "\u001b7\u001b[3A\r\u001b[7C\u001b[43;30m22 of\u001b[0m\u001b8"] -[80.8276, "o", "\u001b7\u001b[3A\r\u001b[7C\u001b[43;30m22 of \u001b[0m\u001b8"] -[80.8556, "o", "\u001b7\u001b[3A\r\u001b[7C\u001b[43;30m22 of 2\u001b[0m\u001b8"] -[80.8836, "o", "\u001b7\u001b[3A\r\u001b[7C\u001b[43;30m22 of 27\u001b[0m\u001b8"] -[86.6516, "o", "$ "] -[86.6516, "o", "\u2588"] -[87.1816, "o", "\b \b"] -[87.7116, "o", "\u2588"] -[88.2416, "o", "\b \b"] -[88.7716, "o", "\u2588"] -[89.3016, "o", "\b \b"] -[89.8316, "o", "\u2588"] -[90.3616, "o", "\b \b"] -[90.8916, "o", "\u2588"] -[91.4216, "o", "\b \b"] -[91.9516, "o", "\u2588"] -[92.4816, "o", "\b \b"] +[7.7447, "o", "\u001b7\u001b[18A\r\u001b[16C\u001b[43;30m0.\u001b[0m\u001b8"] +[7.7667, "o", "\u001b7\u001b[18A\r\u001b[16C\u001b[43;30m0.4\u001b[0m\u001b8"] +[7.7887, "o", "\u001b7\u001b[18A\r\u001b[16C\u001b[43;30m0.49\u001b[0m\u001b8"] +[8.0507, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mB\u001b[0m\u001b8"] +[8.0727, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBE\u001b[0m\u001b8"] +[8.0947, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBEL\u001b[0m\u001b8"] +[8.1167, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELO\u001b[0m\u001b8"] +[8.1387, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW\u001b[0m\u001b8"] +[8.1607, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW \u001b[0m\u001b8"] +[8.1827, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW T\u001b[0m\u001b8"] +[8.2047, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW TH\u001b[0m\u001b8"] +[8.2267, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THR\u001b[0m\u001b8"] +[8.2487, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THRE\u001b[0m\u001b8"] +[8.2707, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THRES\u001b[0m\u001b8"] +[8.2927, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THRESH\u001b[0m\u001b8"] +[8.3147, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THRESHO\u001b[0m\u001b8"] +[8.3367, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THRESHOL\u001b[0m\u001b8"] +[8.3587, "o", "\u001b7\u001b[16A\r\u001b[2C\u001b[43;30mBELOW THRESHOLD\u001b[0m\u001b8"] +[12.0207, "o", "$ "] +[12.0657, "o", "#"] +[12.2259, "o", " "] +[12.273, "o", "s"] +[12.3421, "o", "o"] +[12.386, "o", " "] +[12.4553, "o", "w"] +[12.5381, "o", "h"] +[12.5781, "o", "a"] +[12.6178, "o", "t"] +[12.8279, "o", " "] +[12.9118, "o", "d"] +[12.9917, "o", "o"] +[13.0304, "o", "e"] +[13.0755, "o", "s"] +[13.1443, "o", " "] +[13.2158, "o", "t"] +[13.2943, "o", "h"] +[13.3767, "o", "e"] +[13.4227, "o", " "] +[13.498, "o", "E"] +[13.5667, "o", "U"] +[13.7794, "o", " "] +[13.8358, "o", "A"] +[13.9237, "o", "I"] +[13.9733, "o", " "] +[14.0587, "o", "A"] +[14.0979, "o", "c"] +[14.1492, "o", "t"] +[14.2018, "o", " "] +[14.2508, "o", "a"] +[14.336, "o", "c"] +[14.4167, "o", "t"] +[14.488, "o", "u"] +[14.5766, "o", "a"] +[14.6189, "o", "l"] +[14.6578, "o", "l"] +[14.7179, "o", "y"] +[14.7898, "o", " "] +[14.8672, "o", "a"] +[14.9136, "o", "s"] +[14.9539, "o", "k"] +[15.0162, "o", " "] +[15.1041, "o", "f"] +[15.1825, "o", "o"] +[15.264, "o", "r"] +[15.3493, "o", "?"] +[15.7693, "o", "\r\n"] +[16.4293, "o", "$ "] +[16.5071, "o", "a"] +[16.595, "o", "i"] +[16.6735, "o", "c"] +[16.7433, "o", "e"] +[16.8276, "o", "r"] +[16.8932, "o", "t"] +[16.9406, "o", "i"] +[17.0036, "o", "f"] +[17.0918, "o", "y"] +[17.2885, "o", " "] +[17.3733, "o", "e"] +[17.4377, "o", "x"] +[17.5093, "o", "p"] +[17.5494, "o", "l"] +[17.6197, "o", "a"] +[17.6853, "o", "i"] +[17.7516, "o", "n"] +[17.8281, "o", " "] +[17.8857, "o", "e"] +[17.9735, "o", "u"] +[18.0436, "o", "_"] +[18.0977, "o", "a"] +[18.1537, "o", "i"] +[18.2074, "o", "_"] +[18.293, "o", "a"] +[18.3736, "o", "c"] +[18.4295, "o", "t"] +[18.5159, "o", " "] +[18.5942, "o", "|"] +[18.7666, "o", " "] +[18.8151, "o", "h"] +[18.8942, "o", "e"] +[18.9756, "o", "a"] +[19.028, "o", "d"] +[19.0877, "o", " "] +[19.1343, "o", "-"] +[19.1921, "o", "1"] +[19.2512, "o", "2"] +[19.6712, "o", "\r\n"] +[21.7312, "o", "\r\neu_ai_act \u2014 29 policies\r\n\r\n"] +[21.8212, "o", "Fields you must declare (150)\r\n No evaluator can observe these. They are facts about your system,\r\n your process, or your paperwork, so you assert them in the contract.\r\n"] +[21.9112, "o", "\r\n accuracy.declared_in_instructions technical_robustness\r\n accuracy.metrics_declared technical_robustness\r\n"] +[22.0012, "o", " assessment.completed conformity_assessment\r\n assessment.harmonised_standards_applied conformity_assessment\r\n assessment.notified_body_involved conformity_assessment\r\n"] +[22.7912, "o", "\u001b7\u001b[9A\r\u001b[25C\u001b[43;30m1\u001b[0m\u001b8"] +[22.8132, "o", "\u001b7\u001b[9A\r\u001b[25C\u001b[43;30m15\u001b[0m\u001b8"] +[22.8352, "o", "\u001b7\u001b[9A\r\u001b[25C\u001b[43;30m150\u001b[0m\u001b8"] +[25.0972, "o", "$ "] +[25.1634, "o", "a"] +[25.2183, "o", "i"] +[25.2671, "o", "c"] +[25.3228, "o", "e"] +[25.3639, "o", "r"] +[25.4126, "o", "t"] +[25.4704, "o", "i"] +[25.5614, "o", "f"] +[25.6472, "o", "y"] +[25.7778, "o", " "] +[25.8253, "o", "e"] +[25.8888, "o", "x"] +[25.9569, "o", "p"] +[26.0, "o", "l"] +[26.0723, "o", "a"] +[26.1582, "o", "i"] +[26.2323, "o", "n"] +[26.3201, "o", " "] +[26.4103, "o", "e"] +[26.4678, "o", "u"] +[26.5485, "o", "_"] +[26.6042, "o", "a"] +[26.6806, "o", "i"] +[26.7196, "o", "_"] +[26.8113, "o", "a"] +[26.8675, "o", "c"] +[26.9136, "o", "t"] +[27.103, "o", " "] +[27.1429, "o", "|"] +[27.3126, "o", " "] +[27.399, "o", "g"] +[27.4509, "o", "r"] +[27.5074, "o", "e"] +[27.5969, "o", "p"] +[27.7324, "o", " "] +[27.7827, "o", "-"] +[27.861, "o", "A"] +[27.9384, "o", "8"] +[28.0012, "o", " "] +[28.0516, "o", "\""] +[28.1285, "o", "p"] +[28.2122, "o", "r"] +[28.2653, "o", "o"] +[28.3485, "o", "d"] +[28.4387, "o", "u"] +[28.4781, "o", "c"] +[28.5557, "o", "e"] +[28.6193, "o", "d"] +[28.7785, "o", " "] +[28.8447, "o", "b"] +[28.9018, "o", "y"] +[29.0954, "o", " "] +[29.1461, "o", "e"] +[29.198, "o", "v"] +[29.2623, "o", "a"] +[29.3112, "o", "l"] +[29.3772, "o", "u"] +[29.4187, "o", "a"] +[29.4772, "o", "t"] +[29.5336, "o", "o"] +[29.5948, "o", "r"] +[29.686, "o", "s"] +[29.7272, "o", "\""] +[30.1472, "o", "\r\n"] +[32.2072, "o", "Fields produced by evaluators (14)\r\n Computed at evaluation time from your interactions. Do not hand-write\r\n these: asserting your own fairness score defeats the point.\r\n"] +[32.2972, "o", "\r\n metrics.content_safety.score eu_fairness [content_safety evaluator]\r\n metrics.emotion_recognition.combined_score emotion_recognition\r\n"] +[32.3872, "o", " metrics.emotion_recognition.educational_context emotion_recognition\r\n metrics.emotion_recognition.workplace_context emotion_recognition\r\n metrics.fairness.gender_bias eu_fairness [fairness evaluator]\r\n"] +[33.1772, "o", "\u001b7\u001b[9A\r\u001b[31C\u001b[43;30m1\u001b[0m\u001b8"] +[33.1992, "o", "\u001b7\u001b[9A\r\u001b[31C\u001b[43;30m14\u001b[0m\u001b8"] +[33.4612, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30md\u001b[0m\u001b8"] +[33.4832, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mde\u001b[0m\u001b8"] +[33.5052, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdef\u001b[0m\u001b8"] +[33.5272, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefe\u001b[0m\u001b8"] +[33.5492, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefea\u001b[0m\u001b8"] +[33.5712, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeat\u001b[0m\u001b8"] +[33.5932, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats\u001b[0m\u001b8"] +[33.6152, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats \u001b[0m\u001b8"] +[33.6372, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats t\u001b[0m\u001b8"] +[33.6592, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats th\u001b[0m\u001b8"] +[33.6812, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the\u001b[0m\u001b8"] +[33.7032, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the \u001b[0m\u001b8"] +[33.7252, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the p\u001b[0m\u001b8"] +[33.7472, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the po\u001b[0m\u001b8"] +[33.7692, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the poi\u001b[0m\u001b8"] +[33.7912, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the poin\u001b[0m\u001b8"] +[33.8132, "o", "\u001b7\u001b[7A\r\u001b[43C\u001b[43;30mdefeats the point\u001b[0m\u001b8"] +[37.4752, "o", "$ "] +[37.5463, "o", "a"] +[37.6206, "o", "i"] +[37.6596, "o", "c"] +[37.7474, "o", "e"] +[37.8104, "o", "r"] +[37.8581, "o", "t"] +[37.9474, "o", "i"] +[38.0159, "o", "f"] +[38.075, "o", "y"] +[38.1282, "o", " "] +[38.2189, "o", "i"] +[38.2813, "o", "n"] +[38.3409, "o", "i"] +[38.3975, "o", "t"] +[38.4766, "o", "-"] +[38.5155, "o", "c"] +[38.5826, "o", "o"] +[38.6253, "o", "n"] +[38.6788, "o", "t"] +[38.7577, "o", "r"] +[38.8377, "o", "a"] +[38.9209, "o", "c"] +[38.9734, "o", "t"] +[39.0319, "o", " "] +[39.0894, "o", "-"] +[39.1603, "o", "-"] +[39.2512, "o", "p"] +[39.3126, "o", "o"] +[39.3746, "o", "l"] +[39.431, "o", "i"] +[39.4954, "o", "c"] +[39.567, "o", "y"] +[39.6386, "o", " "] +[39.6796, "o", "e"] +[39.7642, "o", "u"] +[39.8084, "o", "_"] +[39.8808, "o", "a"] +[39.9711, "o", "i"] +[40.0616, "o", "_"] +[40.1367, "o", "a"] +[40.2028, "o", "c"] +[40.2555, "o", "t"] +[40.3386, "o", " "] +[40.4272, "o", ">"] +[40.6372, "o", " "] +[40.6927, "o", "c"] +[40.7405, "o", "o"] +[40.7895, "o", "n"] +[40.8733, "o", "t"] +[40.9334, "o", "r"] +[40.974, "o", "a"] +[41.0523, "o", "c"] +[41.1215, "o", "t"] +[41.1714, "o", "."] +[41.2392, "o", "j"] +[41.3201, "o", "s"] +[41.3669, "o", "o"] +[41.4086, "o", "n"] +[41.8286, "o", "\r\n"] +[44.5886, "o", "$ "] +[44.6469, "o", "#"] +[44.7088, "o", " "] +[44.7842, "o", "1"] +[44.8482, "o", "5"] +[44.9291, "o", "0"] +[45.1603, "o", " "] +[45.2136, "o", "d"] +[45.2628, "o", "e"] +[45.3407, "o", "c"] +[45.3989, "o", "l"] +[45.4386, "o", "a"] +[45.5306, "o", "r"] +[45.572, "o", "a"] +[45.6386, "o", "t"] +[45.7089, "o", "i"] +[45.759, "o", "o"] +[45.827, "o", "n"] +[45.9115, "o", "s"] +[45.9893, "o", " "] +[46.0378, "o", "t"] +[46.1059, "o", "o"] +[46.2836, "o", " "] +[46.3409, "o", "a"] +[46.3876, "o", "n"] +[46.453, "o", "s"] +[46.5392, "o", "w"] +[46.6023, "o", "e"] +[46.6895, "o", "r"] +[46.7797, "o", "."] +[46.8387, "o", " "] +[46.9177, "o", "a"] +[46.966, "o", "n"] +[47.0102, "o", "s"] +[47.0805, "o", "w"] +[47.1676, "o", "e"] +[47.2276, "o", "r"] +[47.2954, "o", "e"] +[47.3792, "o", "d"] +[47.4513, "o", ","] +[47.5369, "o", " "] +[47.6051, "o", "t"] +[47.6604, "o", "h"] +[47.713, "o", "e"] +[47.7776, "o", "n"] +[47.8532, "o", ":"] +[48.2732, "o", "\r\n"] +[48.9332, "o", "$ "] +[49.0207, "o", "a"] +[49.0915, "o", "i"] +[49.1789, "o", "c"] +[49.2596, "o", "e"] +[49.3161, "o", "r"] +[49.3554, "o", "t"] +[49.4294, "o", "i"] +[49.5185, "o", "f"] +[49.5828, "o", "y"] +[49.6857, "o", " "] +[49.7526, "o", "e"] +[49.8028, "o", "v"] +[49.8502, "o", "a"] +[49.9337, "o", "l"] +[49.9906, "o", "u"] +[50.0313, "o", "a"] +[50.1126, "o", "t"] +[50.1565, "o", "e"] +[50.3747, "o", " "] +[50.428, "o", "-"] +[50.5144, "o", "-"] +[50.5593, "o", "c"] +[50.6164, "o", "o"] +[50.6806, "o", "n"] +[50.7367, "o", "t"] +[50.7848, "o", "r"] +[50.8641, "o", "a"] +[50.9435, "o", "c"] +[51.0048, "o", "t"] +[51.0925, "o", " "] +[51.1375, "o", "c"] +[51.2261, "o", "o"] +[51.2885, "o", "n"] +[51.3555, "o", "t"] +[51.4202, "o", "r"] +[51.4824, "o", "a"] +[51.5499, "o", "c"] +[51.6153, "o", "t"] +[51.6771, "o", "."] +[51.7523, "o", "j"] +[51.8322, "o", "s"] +[51.8745, "o", "o"] +[51.9445, "o", "n"] +[52.0045, "o", " "] +[52.0543, "o", "-"] +[52.1147, "o", "-"] +[52.1727, "o", "p"] +[52.2299, "o", "o"] +[52.3217, "o", "l"] +[52.3906, "o", "i"] +[52.4778, "o", "c"] +[52.5639, "o", "y"] +[52.6503, "o", " "] +[52.7279, "o", "e"] +[52.8127, "o", "u"] +[52.887, "o", "_"] +[52.9331, "o", "a"] +[53.0108, "o", "i"] +[53.0876, "o", "_"] +[53.1275, "o", "a"] +[53.1728, "o", "c"] +[53.2371, "o", "t"] +[53.6571, "o", "\r\n"] +[55.7171, "o", "Contract ID: 5692c240-fbd2-45af-a478-2142e4fd4a39\r\nApplication: acme-triage-assistant\r\nReport: ./reports/folder_report_acme-triage-assistant_2026-08-30_180617.md\r\n"] +[55.8071, "o", "OPA Evaluation: Successful\r\n"] +[56.6971, "o", "$ "] +[56.7549, "o", "g"] +[56.795, "o", "r"] +[56.8815, "o", "e"] +[56.941, "o", "p"] +[56.9869, "o", " "] +[57.0693, "o", "-"] +[57.1519, "o", "m"] +[57.2421, "o", "2"] +[57.4026, "o", " "] +[57.4602, "o", "\""] +[57.5062, "o", "N"] +[57.5444, "o", "o"] +[57.6073, "o", "t"] +[57.7154, "o", " "] +[57.7631, "o", "S"] +[57.8343, "o", "a"] +[57.8782, "o", "t"] +[57.9436, "o", "i"] +[58.0159, "o", "s"] +[58.0888, "o", "f"] +[58.148, "o", "i"] +[58.2315, "o", "e"] +[58.3191, "o", "d"] +[58.3625, "o", "\""] +[58.4392, "o", " "] +[58.4825, "o", "r"] +[58.5691, "o", "e"] +[58.6108, "o", "p"] +[58.6986, "o", "o"] +[58.7892, "o", "r"] +[58.864, "o", "t"] +[58.9313, "o", "s"] +[58.9941, "o", "/"] +[59.071, "o", "*"] +[59.1319, "o", "."] +[59.1711, "o", "m"] +[59.235, "o", "d"] +[59.655, "o", "\r\n"] +[60.115, "o", "- **Article 15 Controls Not Satisfied:** ['Article 15(4) feedback loops addressed where the syst\r\n- **Article 14 Controls Not Satisfied:** ['Article 14(1)-(2) oversight designed into the system'\r\n"] +[60.905, "o", "\u001b7\u001b[1A\r\u001b[43C\u001b[43;30mA\u001b[0m\u001b8"] +[60.927, "o", "\u001b7\u001b[1A\r\u001b[43C\u001b[43;30mAr\u001b[0m\u001b8"] +[60.949, "o", "\u001b7\u001b[1A\r\u001b[43C\u001b[43;30mArt\u001b[0m\u001b8"] +[60.971, "o", "\u001b7\u001b[1A\r\u001b[43C\u001b[43;30mArti\u001b[0m\u001b8"] +[60.993, "o", "\u001b7\u001b[1A\r\u001b[43C\u001b[43;30mArtic\u001b[0m\u001b8"] +[61.015, "o", "\u001b7\u001b[1A\r\u001b[43C\u001b[43;30mArticl\u001b[0m\u001b8"] +[61.037, "o", "\u001b7\u001b[1A\r\u001b[43C\u001b[43;30mArticle\u001b[0m\u001b8"] +[61.059, "o", "\u001b7\u001b[1A\r\u001b[43C\u001b[43;30mArticle \u001b[0m\u001b8"] +[61.081, "o", "\u001b7\u001b[1A\r\u001b[43C\u001b[43;30mArticle 1\u001b[0m\u001b8"] +[61.103, "o", "\u001b7\u001b[1A\r\u001b[43C\u001b[43;30mArticle 14\u001b[0m\u001b8"] +[61.125, "o", "\u001b7\u001b[1A\r\u001b[43C\u001b[43;30mArticle 14(\u001b[0m\u001b8"] +[61.147, "o", "\u001b7\u001b[1A\r\u001b[43C\u001b[43;30mArticle 14(1\u001b[0m\u001b8"] +[61.169, "o", "\u001b7\u001b[1A\r\u001b[43C\u001b[43;30mArticle 14(1)\u001b[0m\u001b8"] +[61.191, "o", "\u001b7\u001b[1A\r\u001b[43C\u001b[43;30mArticle 14(1)-\u001b[0m\u001b8"] +[61.213, "o", "\u001b7\u001b[1A\r\u001b[43C\u001b[43;30mArticle 14(1)-(\u001b[0m\u001b8"] +[61.235, "o", "\u001b7\u001b[1A\r\u001b[43C\u001b[43;30mArticle 14(1)-(2\u001b[0m\u001b8"] +[61.257, "o", "\u001b7\u001b[1A\r\u001b[43C\u001b[43;30mArticle 14(1)-(2)\u001b[0m\u001b8"] +[64.919, "o", "$ "] +[64.919, "o", "\u2588"] +[65.449, "o", "\b \b"] +[65.979, "o", "\u2588"] +[66.509, "o", "\b \b"] +[67.039, "o", "\u2588"] +[67.569, "o", "\b \b"] +[68.099, "o", "\u2588"] +[68.629, "o", "\b \b"] diff --git a/scripts/record-demo.py b/scripts/record-demo.py index 9bbfa6d..b5df954 100644 --- a/scripts/record-demo.py +++ b/scripts/record-demo.py @@ -2,9 +2,9 @@ """Compose the AICertify demo asciicast: real output, typed at a human pace. Every command here is actually run and its output captured, so nothing on the -screen is written by hand. What is synthesised is the *timing*. The four -commands take about three minutes between them, most of it downloading model -weights and waiting on OPA, and a cast at real speed is unwatchable. A `script` +screen is written by hand. What is synthesised is the *timing*. The commands +take about three minutes between them, most of it downloading model weights and +waiting on OPA, and a cast at real speed is unwatchable. A `script` capture would also have no typing in it at all, and output that materialises instantly reads as a screenshot rather than a session. @@ -12,26 +12,35 @@ the verdicts and the report are what the commands printed. The rhythm is invented, the way any screencast's rhythm is invented by the person recording. -Four acts, because AICertify does a different job from GOPAL and needs the +Three acts, because AICertify does a different job from GOPAL and needs the room. GOPAL decides things, so its demo is one beat: swap the model, the build fails. AICertify gathers things, and the interesting thing about it is how honest it is about the part it cannot gather. 1. score-card a real Hugging Face card, scored, in one screen 2. explain 150 fields you must declare, 14 an evaluator computes - 3. init/eval scaffold a contract, run it, get 0 of 29 - 4. gap report 22 of 27 metrics supplied, and the five that are not + 3. init/eval answer them, evaluate, and get a named article back -Act 3 ends on nothing passing. That is the point rather than an embarrassment: -the scaffold asserts none of the 150 declarations, so none of the obligations -that turn on them can be met. It is the same argument act 2 makes, arriving as -a verdict instead of a number. +Act 3 used to end on an empty contract failing all 29 policies. That is true +and it is the argument act 2 makes, but as an ending it reads as "the tool does +not work" to anyone who has not followed the reasoning. So it now answers the +declarations first and ends where the product is actually differentiated: not +on a score, but on `Article 14(1)-(2) oversight designed into the system`. A +named provision and the specific control, which is what you can act on. + +The gap report was act 4 and has been dropped. Enumerating the metrics nobody +has written an evaluator for is a good thing to show a contributor and a poor +thing to show someone deciding whether to adopt. It lives in the contributing +section of the page instead. Usage: record-demo.py [short.cast] -The short cast is acts 1 and 2 only, for the README, where a 90-second loop is -too long and JavaScript does not run. +The short cast is acts 1 and 2 only, about 36 seconds, for the README. That +loop autoplays above the fold and JavaScript does not run there, so it is a +CSS-animated SVG and it has to be short enough to watch before deciding to +scroll. The full cut runs about 70 seconds on the page, where a reader has +chosen to watch it. """ from __future__ import annotations @@ -59,18 +68,18 @@ ENTER_SETTLES = 0.16 # and the press registers before anything happens FIRST_BYTE = 0.30 # a command does not answer instantly THINKING = 1.9 # a command that clearly did some work -READ_SHORT = 2.6 -READ_LONG = 5.4 # long enough to read a verdict and understand it +READ_SHORT = 2.0 +READ_LONG = 3.4 # long enough to read a verdict and understand it BLINK = 0.53 -BLINKS = 6 +BLINKS = 4 CURSOR = "█" -SWEEP = 0.028 # per character, the pace of a pen moving +SWEEP = 0.022 # per character, the pace of a pen moving HL_ON = "\x1b[43;30m" HL_OFF = "\x1b[0m" SAVE, RESTORE = "\x1b7", "\x1b8" -TYPO_CHANCE = 0.55 # per command, not per character +TYPO_CHANCE = 0.25 # per command, not per character NOTICE_TYPO = (0.22, 0.46) AFTER_FIX = (0.08, 0.19) @@ -203,7 +212,7 @@ def rows(line: str) -> int: if needle in line and phrase in line: up = sum(rows(rest) for rest in lines[idx:]) self.highlight(up, line.index(phrase), phrase) - self.wait(0.34) + self.wait(0.24) break def write(self, dst: Path, title: str) -> None: @@ -275,22 +284,32 @@ def main() -> int: contract.exists() and contract.stat().st_size > 1000, "init-contract produced no usable contract", ) - # A declarations-only contract. The scaffold ships one placeholder - # interaction, and evaluating it pulls DeepEval into an LLM call that - # needs a provider key. The point of this act is the policy verdict, - # not the toxicity classifier, and act 4 covers the evaluators. + # Answer the declarations, the way a reader would before evaluating. + # The scaffold leaves all 150 null, and an unanswered contract fails + # everything: true, and the argument act 2 makes, but as an ending it + # reads as a broken tool rather than an unanswered questionnaire. + # + # Interactions are cleared because evaluating the placeholder one pulls + # DeepEval into an LLM call needing a provider key. This act is about + # the policy verdict, not the toxicity classifier. doc = json.loads(contract.read_text()) doc["interactions"] = [] + doc["application_name"] = "acme-triage-assistant" + answered = _answer_declarations(doc["context"]) contract.write_text(json.dumps(doc, indent=2)) + require( + answered > 100, + f"only {answered} declarations were answered; the scaffold shape " + "changed and act 3 would show an unanswered contract", + ) evaluate = shell( f"{PY} -m aicertify.cli evaluate --contract contract.json " f"--policy eu_ai_act --report-format markdown --output-dir ./reports " - "2>&1 | tail -5", + "2>&1 | tail -4", cwd=work, ) - summary = shell("head -8 reports/*.md", cwd=work) - gaps = shell(f"{PY} scripts/metric_gap_report.py 2>/dev/null | tail -12") + controls = shell('grep -m2 "Not Satisfied" reports/*.md | cut -c1-96', cwd=work) # Guards. Each of these is a fact the cast puts on screen, so if the # tool stops saying it the recording is a lie and must not be written. @@ -312,17 +331,12 @@ def main() -> int: "the line explaining why you must not hand-write metrics is gone", ) require( - "Red Count:** 29" in summary and "Green Count:** 0" in summary, - "an empty contract no longer fails all 29 policies, so act 3 " - "has lost its point", - ) - require( - "22 of 27" in gaps, - "the metric gap report no longer says 22 of 27; act 4's " - "contribution hook needs the real number", + "Article 14" in controls and "Not Satisfied" in controls, + "the report no longer names an unsatisfied Article 14 control, and " + "act 3 ends on a named provision rather than on a score", ) - acts = build(card, declare, measure, evaluate, summary, gaps) + acts = build(card, declare, measure, evaluate, controls) acts.write(full_dst, "AICertify: what it measures, and what it cannot") if short_dst: @@ -333,6 +347,87 @@ def main() -> int: shutil.rmtree(work, ignore_errors=True) +def _answer_declarations(context: dict) -> int: + """Fill every unanswered declaration with a plausible answer. + + The scaffold leaves all 150 as null. Types are taken from GOPAL's coverage + data where it has inferred one, and from the field name otherwise: anything + that reads as a quantity gets a number, anything plural gets a list, and the + rest are the yes/no assertions that make up most of the Act. + + These are not claims about a real system. They stand in for a compliance + officer having answered the questionnaire, which is the state act 3 needs + to show and the state the scaffold exists to reach. + """ + types: dict[str, str] = {} + coverage = ( + REPO / "aicertify" / "opa_policies" / "docs" / "coverage" / "coverage.json" + ) + if coverage.exists(): + data = json.loads(coverage.read_text()) + for framework in data.get("frameworks", []): + if "eu_ai_act" in (framework.get("id") or ""): + for policy in framework.get("policies", []): + types.update(policy.get("input_field_types", {})) + + quantities = ( + "completeness", + "score", + "months", + "days", + "count", + "risk", + "level", + "number", + "percentage", + "rate", + "size", + ) + collections_ = ( + "measures", + "fields", + "controls", + "reasons", + "sections", + "list", + "categories", + "purposes", + "records", + ) + + def answer(path: str): + kind = types.get(path) + if kind == "boolean": + return True + if kind == "number": + return 0.95 + if kind == "string": + return "documented" + if kind == "list": + return [] + leaf = path.rsplit(".", 1)[-1] + if any(word in leaf for word in quantities): + return 0.95 + if any(word in leaf for word in collections_): + return [] + return True + + filled = 0 + + def walk(node: dict, path: str = "") -> None: + nonlocal filled + for key, value in list(node.items()): + here = f"{path}.{key}" if path else key + if isinstance(value, dict): + walk(value, here) + elif value is None: + node[key] = answer(here) + filled += 1 + + walk(context) + return filled + + def act_one(cast: Cast, card: str) -> None: """A famous model, scored, with no setup at all.""" cast.wait(0.6) @@ -349,10 +444,16 @@ def act_one(cast: Cast, card: str) -> None: cast.wait(READ_LONG) -def act_two(cast: Cast, declare: str, measure: str) -> None: - """The split that the whole product turns on.""" - cast.command("# so what does the EU AI Act actually ask for?") - cast.wait(0.5) +def act_two(cast: Cast, declare: str, measure: str, narrate: bool = True) -> None: + """The split that the whole product turns on. + + `narrate` is off for the README cut. That loop autoplays above the fold and + has to be short enough that a reader sees the whole thing before deciding to + scroll, so it drops the framing line the page version can afford. + """ + if narrate: + cast.command("# so what does the EU AI Act actually ask for?") + cast.wait(0.5) cast.command("aicertify explain eu_ai_act | head -12") cast.block(declare, first_byte=THINKING) cast.wait(0.7) @@ -375,37 +476,32 @@ def act_two(cast: Cast, declare: str, measure: str) -> None: def build_short(card: str, declare: str, measure: str) -> Cast: cast = Cast() act_one(cast, card) - act_two(cast, declare, measure) - cast.blink() + act_two(cast, declare, measure, narrate=False) + cast.blink(3) return cast -def build(card, declare, measure, evaluate, summary, gaps) -> Cast: +def build(card, declare, measure, evaluate, controls) -> Cast: cast = Cast() act_one(cast, card) act_two(cast, declare, measure) - # Act 3: scaffold, run, and find out you have answered nothing. + # Act 3: answer them, evaluate, and get a provision back rather than a score. cast.command("aicertify init-contract --policy eu_ai_act > contract.json") - cast.wait(THINKING + 0.9) + cast.wait(THINKING + 0.7) + cast.command("# 150 declarations to answer. answered, then:") + cast.wait(0.5) cast.command("aicertify evaluate --contract contract.json --policy eu_ai_act") cast.block(evaluate, first_byte=THINKING) cast.wait(0.8) - cast.command("head -8 reports/*.md") - cast.block(summary) + # The ending. Not a count, which reads as a grade, but the named provision + # and the specific control, which is the thing you can act on and the thing + # a black-box compliance score cannot give you. + cast.command('grep -m2 "Not Satisfied" reports/*.md') + cast.block(controls) cast.wait(0.7) - cast.sweep(summary, [("Red Count", "29"), ("Green Count", "0")]) - cast.wait(READ_LONG) - - # Act 4: the tool enumerating its own gaps, which is a contributor's - # to-do list rather than a caveat. - cast.command("# which metrics can an evaluator supply today?") - cast.wait(0.5) - cast.command("python scripts/metric_gap_report.py | tail -12") - cast.block(gaps, first_byte=THINKING) - cast.wait(0.8) - cast.sweep(gaps, [("TOTAL", "22 of 27")]) + cast.sweep(controls, [("Article 14", "Article 14(1)-(2)")]) cast.wait(READ_LONG) cast.blink() return cast