Skip to content

Commit 369f3c7

Browse files
bigboatengbigboateng
committed
eval: freeze range-large v7 pair manifest
1 parent a8d2d82 commit 369f3c7

4 files changed

Lines changed: 243 additions & 3 deletions

File tree

‎.github/workflows/deltawire-lab.yml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,5 +24,6 @@ jobs:
2424
go-version-file: labs/20-deltawire/go.mod
2525
- uses: actions/setup-python@v6
2626
with:
27-
python-version: "3.11"
27+
python-version: "3.12"
28+
- uses: astral-sh/setup-uv@v7
2829
- run: bash labs/20-deltawire/scripts/validate.sh

‎.github/workflows/deltawire-v7.yml‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ jobs:
3333
run: |
3434
cd labs/20-deltawire
3535
mkdir -p eval/.generated eval/pairs/range-large-v7-r1/environment/.generated
36-
GOWORK=off CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -o eval/.generated/deltawire-linux-amd64 ./cmd/deltawire
36+
GOWORK=off CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -buildvcs=false -o eval/.generated/deltawire-linux-amd64 ./cmd/deltawire
3737
cp eval/.generated/deltawire-linux-amd64 eval/pairs/range-large-v7-r1/environment/.generated/deltawire
3838
test "$(sha256sum eval/.generated/deltawire-linux-amd64 | cut -d' ' -f1)" = e5198d15000e093a2a28e57ad4a093dde8f66bcab21462549ddf9114064c25f4
3939
- name: Validate skill examples using the actual Linux binary
@@ -49,4 +49,7 @@ jobs:
4949
run: bash labs/20-deltawire/eval/scripts/v7/validate.sh
5050
- name: Prove workflow has no model invocation
5151
run: |
52-
! grep -E 'runner.py .*pair-v7|harbor run|GEMINI_API_KEY' .github/workflows/deltawire-v7.yml
52+
live_runner='runner.py .*pair'"-v7"
53+
harbor_live='harbor '"run"
54+
credential='GEMINI_API'"_KEY"
55+
! grep -E "$live_runner|$harbor_live|$credential" .github/workflows/deltawire-v7.yml

‎labs/20-deltawire/eval/manifests/deltawire-range-large-v7-pair.json‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
{
2+
"READY_FOR_72": false,
3+
"agent": "preinstalled-gemini-cli-v1",
4+
"agent_import": "agents.preinstalled_gemini_cli:PreinstalledGeminiCli",
5+
"binary_sha256": "e5198d15000e093a2a28e57ad4a093dde8f66bcab21462549ddf9114064c25f4",
6+
"expected_reported_model": "gemini-3.1-pro-preview",
7+
"failure_policy": "withhold_B0_unless_D1_passes",
8+
"full_run_started": false,
9+
"gemini_cli_version": "0.51.0",
10+
"harbor_command": [
11+
"uvx",
12+
"--from",
13+
"harbor==0.20.0",
14+
"harbor"
15+
],
16+
"harbor_source": [
17+
{
18+
"module": "harbor.agents.installed.gemini_cli",
19+
"sha256": "dc8ce81e8f41b78f00abb54cd91fd7840d147f442960ca32e68c7a760e060a2a"
20+
},
21+
{
22+
"module": "harbor.skills",
23+
"sha256": "45fa5d0f45dce8029c47eac7468dc8bdb01157cc2b7c9271503719fc157cc7f3"
24+
}
25+
],
26+
"harbor_version": "0.20.0",
27+
"image_id": "sha256:589fce58d7ddf09910876afe0198bcee0086a840af6de7c08515bbce3e8367a7",
28+
"image_ref": "deltawire-preflight-v6:runtime-v1",
29+
"infrastructure_commit": "a8d2d8260ceed56baf0d1cf8fcc2f40776e9f7b6",
30+
"input_hashes": {
31+
"labs/20-deltawire/eval/agents/preinstalled_gemini_cli.py": "dfd0d27258ac20b8778b9aacce5f302db93576d2f735c6645e489be6c6b96058",
32+
"labs/20-deltawire/eval/manifests/preflight-v6-terminal-evidence-lock.json": "76a6ac42fe2dfc41d1af2d73502c11724ed632a21e1d8d10b065e9bc619b7fcc",
33+
"labs/20-deltawire/eval/matrices/72-run-matrix.json": "f9dcddc4b7336418c7e2b8e7a7e51ca1e42b5a137255ff0823a732618f3568d5",
34+
"labs/20-deltawire/eval/pairs/range-large-v7-r1/authoritative-schema.json": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5",
35+
"labs/20-deltawire/eval/pairs/range-large-v7-r1/environment/.deltawire/config.json": "673b9cf053e3ba2263bfbc49034360ed8dba8d959bd167f8ff4b1f7cb4302b6f",
36+
"labs/20-deltawire/eval/pairs/range-large-v7-r1/environment/.deltawire/schemas/range-large.schema.json": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5",
37+
"labs/20-deltawire/eval/pairs/range-large-v7-r1/environment/Dockerfile": "cd768e64efba435604b265088bce6269e8dbd09e54bf51a4c5bf0dd94d10c05f",
38+
"labs/20-deltawire/eval/pairs/range-large-v7-r1/environment/deltawire-environment-expectations.json": "03fe7a40964e908f2fb3736add8b8bea890b407d837f2b185d92bd8d85d4d114",
39+
"labs/20-deltawire/eval/pairs/range-large-v7-r1/environment/environment_receipt.py": "3ab3bbcbfae2093ac658063d1e26f26b43c35dfdb81d8ddae5c9989f967ba623",
40+
"labs/20-deltawire/eval/pairs/range-large-v7-r1/environment/task-contract.json": "d3e1b7b3e5a4576540b3a2969d59f49818c7c0a12197b26d271e7e277b0054e5",
41+
"labs/20-deltawire/eval/pairs/range-large-v7-r1/environment/verify_plan_contract.py": "8aafe73ab47213027a258f9abbebbca4803fa53f658abeb7b405d4a79202d4af",
42+
"labs/20-deltawire/eval/pairs/range-large-v7-r1/instruction.md": "549f4047b1a4ba4f7a3f3dce9e60d1d70b141caa189aafdba942b15e6d022aee",
43+
"labs/20-deltawire/eval/pairs/range-large-v7-r1/skill/deltawire/SKILL.md": "e7d3a1e8ccef1b72338d5dac7a7b7ac412164846018ab55b7f2ecadf5a07a89a",
44+
"labs/20-deltawire/eval/pairs/range-large-v7-r1/skill/deltawire/examples/matrix.dw.json": "56d16a5b20086c8202cbc1bbdccf41d467a5d4c3806d701227ebcdbf72e54b1a",
45+
"labs/20-deltawire/eval/pairs/range-large-v7-r1/skill/deltawire/examples/range.dw.json": "b31e078409547f4fbb5a855f3bf0b1bd6f52386696c07401622f34d224fe148b",
46+
"labs/20-deltawire/eval/pairs/range-large-v7-r1/skill/deltawire/examples/rows.dw.json": "e172cc7d6224e4566a0cd46a73465392976338fea0e225512a089322ba81aca3",
47+
"labs/20-deltawire/eval/pairs/range-large-v7-r1/skill/deltawire/examples/variants.dw.json": "3183405b76e11874309728f3a4025687af6c54add8d2b97b94daff2d6ef8cdca",
48+
"labs/20-deltawire/eval/pairs/range-large-v7-r1/skill/deltawire/references/plan-v1.md": "f78d41266db73592526a3c5e7d6e4c883f549f99b01c6387c7f5337fc269dd5e",
49+
"labs/20-deltawire/eval/pairs/range-large-v7-r1/task-spec.json": "d3e1b7b3e5a4576540b3a2969d59f49818c7c0a12197b26d271e7e277b0054e5",
50+
"labs/20-deltawire/eval/pairs/range-large-v7-r1/task.toml": "4adc78ba9c59f8c743770ee9b1b31a55364e7303010e3c3f109da3b3d61d96e3",
51+
"labs/20-deltawire/eval/pairs/range-large-v7-r1/tests/semantic_oracle.py": "94b30fac117f40e977851966b5228e6c372bb48cbf2e5c29c24f428e1f98e91d",
52+
"labs/20-deltawire/eval/pairs/range-large-v7-r1/tests/task-spec.json": "d3e1b7b3e5a4576540b3a2969d59f49818c7c0a12197b26d271e7e277b0054e5",
53+
"labs/20-deltawire/eval/pairs/range-large-v7-r1/tests/test.sh": "6f7cfbefdf7eaf3a082fd0c88f264ba3d41bfe84fc53498df4464bef66dc116e",
54+
"labs/20-deltawire/eval/results/preflight-v7/skill-conformance.json": "4d0e142a1bc801642b5de8c77b9cbebc9680a42f6f64da95b3d48b8624fe7c46",
55+
"labs/20-deltawire/eval/runtime/preinstalled-gemini-v1.json": "19d9bf6ed82d3f6fa8e9bdad65bb74538ef9d7d161afb3f1f76771ac2e6b0c2d",
56+
"labs/20-deltawire/eval/scripts/semantic_oracle.py": "94b30fac117f40e977851966b5228e6c372bb48cbf2e5c29c24f428e1f98e91d",
57+
"labs/20-deltawire/eval/scripts/v6/artifact_manifest.py": "722f71bfbec56cbdbaea85a2f975aaa61f58d2dbbbd41e4a6b02741f3ac90ca0",
58+
"labs/20-deltawire/eval/scripts/v6/setup_receipt.py": "c57a5a614663e4b3b81f4e1b5929c3aadfbb3a319198c9700213da0308a3d634",
59+
"labs/20-deltawire/eval/scripts/v6/shell_observation.py": "e5269e06350147715505a76b8e83235abecd5af34eba1c8837e19b22b0e4c9fe",
60+
"labs/20-deltawire/eval/scripts/v6/trial_status.py": "47fea192f0853a3ea880d992c27934e87cd19bc84a7a2c01446d3e9a0be8379a",
61+
"labs/20-deltawire/eval/scripts/v6/verify_terminal_evidence_lock.py": "5adf6d90f641fb80fea894709f1a6e219e1b430454a11d19fc4c1c402bb0affe",
62+
"labs/20-deltawire/eval/scripts/v7/generate_manifest.py": "60fbcd9a31514b94f606e87b7df07c4442259a9ed15c00f169b344675596b1c6",
63+
"labs/20-deltawire/eval/scripts/v7/runner.py": "cc551f8f943f39c767c0cc88bc07f584972777b4d775ca586d3502eb653399b8",
64+
"labs/20-deltawire/eval/scripts/v7/skill_conformance.py": "31617b88fef9185635ea3bc78f60368df8a894212a804c4f5e1d60b2365d9491",
65+
"labs/20-deltawire/eval/scripts/v7/test_runner.py": "6e0ca10b09f1fd2624bbbe70892c9faf31ef630644f710ba9cd05fbaa92556ab",
66+
"labs/20-deltawire/eval/scripts/v7/test_skill.py": "6fd6b49e06de82e4af7a97a449a9981fd860dca96c806c069834cf96287d288e",
67+
"labs/20-deltawire/eval/scripts/v7/test_treatment.py": "4f2cb7744ce65d5cfe8ec1da8b1aca804609977a44c20fdbafa12397110409ce",
68+
"labs/20-deltawire/eval/scripts/v7/treatment.py": "5587514cd678223105a2aff6b1a01f58fff6b6e138d06b9461db0fc7f77e7ee8",
69+
"labs/20-deltawire/eval/scripts/v7/validate.py": "c96eadbac126fb6546816f83513e3a38dea0b29091b43fec620b929768e1e06a",
70+
"labs/20-deltawire/eval/scripts/v7/validate.sh": "a24c183a3824df10cf5dffcdd07c0bd336b7d51705a9e311127587d56aa87200",
71+
"labs/20-deltawire/eval/scripts/verify_plan_contract.py": "8aafe73ab47213027a258f9abbebbca4803fa53f658abeb7b405d4a79202d4af"
72+
},
73+
"integration_commit": "82a7963bdd98ca41435e756fb4212b3b37b19420",
74+
"model": "google/gemini-3.1-pro-preview",
75+
"order": [
76+
"D1",
77+
"B0"
78+
],
79+
"pair_id": "range-large/v7-r1",
80+
"result_root": "labs/20-deltawire/eval/results/range-large-v7-r1",
81+
"schema_version": "deltawire-range-large-v7-pair-manifest.v1",
82+
"six_run_canary_started": false,
83+
"skill": {
84+
"conformance_sha256": "4d0e142a1bc801642b5de8c77b9cbebc9680a42f6f64da95b3d48b8624fe7c46",
85+
"harbor_digest": "sha256:a841ca12dc687b1e467f90d5d9f5ccef120bc810742fab693e8e976135a75984",
86+
"reference_sha256": "f78d41266db73592526a3c5e7d6e4c883f549f99b01c6387c7f5337fc269dd5e",
87+
"skill_sha256": "e7d3a1e8ccef1b72338d5dac7a7b7ac412164846018ab55b7f2ecadf5a07a89a",
88+
"tree_sha256": "a841ca12dc687b1e467f90d5d9f5ccef120bc810742fab693e8e976135a75984"
89+
},
90+
"source_matrix_pair_id": "range-large/r1",
91+
"source_matrix_positions_zero_based": [
92+
36,
93+
37
94+
]
95+
}

‎labs/20-deltawire/eval/results/preflight-v7/skill-conformance.json‎

Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
{
2+
"binary_sha256": "e5198d15000e093a2a28e57ad4a093dde8f66bcab21462549ddf9114064c25f4",
3+
"checks": {
4+
"binary_exists": true,
5+
"credentials_absent": true,
6+
"examples_exact": true,
7+
"examples_pass": true,
8+
"negative_controls_pass": true,
9+
"reference_exists": true,
10+
"skill_file_exists": true,
11+
"zero_model_calls": true
12+
},
13+
"examples": [
14+
{
15+
"example": "matrix.dw.json",
16+
"example_sha256": "56d16a5b20086c8202cbc1bbdccf41d467a5d4c3806d701227ebcdbf72e54b1a",
17+
"expected_records": [
18+
{
19+
"method": "GET",
20+
"role": "guest"
21+
},
22+
{
23+
"method": "POST",
24+
"role": "guest"
25+
},
26+
{
27+
"method": "GET",
28+
"role": "member"
29+
},
30+
{
31+
"method": "POST",
32+
"role": "member"
33+
}
34+
],
35+
"record_count": 4,
36+
"rendered_sha256": [
37+
"01ee7ae45519e704708945436792560f537ae7e67b435623190e7eb3c1ad8660",
38+
"01ee7ae45519e704708945436792560f537ae7e67b435623190e7eb3c1ad8660"
39+
],
40+
"repeated_byte_determinism": true,
41+
"runs_pass": [
42+
true,
43+
true
44+
],
45+
"status": "pass"
46+
},
47+
{
48+
"example": "range.dw.json",
49+
"example_sha256": "b31e078409547f4fbb5a855f3bf0b1bd6f52386696c07401622f34d224fe148b",
50+
"expected_records": [
51+
{
52+
"index": 1
53+
},
54+
{
55+
"index": 2
56+
},
57+
{
58+
"index": 3
59+
}
60+
],
61+
"record_count": 3,
62+
"rendered_sha256": [
63+
"308b0a63452f680897077bfec12487712eaf512a7d49d454c011d209b11557d7",
64+
"308b0a63452f680897077bfec12487712eaf512a7d49d454c011d209b11557d7"
65+
],
66+
"repeated_byte_determinism": true,
67+
"runs_pass": [
68+
true,
69+
true
70+
],
71+
"status": "pass"
72+
},
73+
{
74+
"example": "rows.dw.json",
75+
"example_sha256": "e172cc7d6224e4566a0cd46a73465392976338fea0e225512a089322ba81aca3",
76+
"expected_records": [
77+
{
78+
"enabled": true,
79+
"name": "alpha",
80+
"rank": 1
81+
},
82+
{
83+
"enabled": true,
84+
"name": "beta",
85+
"rank": 2
86+
}
87+
],
88+
"record_count": 2,
89+
"rendered_sha256": [
90+
"77a7cabbfdbd71945310803ce919530ebf1b888d2e5c417c1344f0825fb63bad",
91+
"77a7cabbfdbd71945310803ce919530ebf1b888d2e5c417c1344f0825fb63bad"
92+
],
93+
"repeated_byte_determinism": true,
94+
"runs_pass": [
95+
true,
96+
true
97+
],
98+
"status": "pass"
99+
},
100+
{
101+
"example": "variants.dw.json",
102+
"example_sha256": "3183405b76e11874309728f3a4025687af6c54add8d2b97b94daff2d6ef8cdca",
103+
"expected_records": [
104+
{
105+
"active": true,
106+
"role": "member",
107+
"status": "deny"
108+
},
109+
{
110+
"active": false,
111+
"role": "member",
112+
"status": "allow"
113+
}
114+
],
115+
"record_count": 2,
116+
"rendered_sha256": [
117+
"e6ac2a1b8f43a7557f3d8b9b93e290618bbf0e44f6137515bdf7c61403d428c8",
118+
"e6ac2a1b8f43a7557f3d8b9b93e290618bbf0e44f6137515bdf7c61403d428c8"
119+
],
120+
"repeated_byte_determinism": true,
121+
"runs_pass": [
122+
true,
123+
true
124+
],
125+
"status": "pass"
126+
}
127+
],
128+
"harbor_skill_digest": "sha256:a841ca12dc687b1e467f90d5d9f5ccef120bc810742fab693e8e976135a75984",
129+
"negative_controls": {
130+
"checks": {
131+
"malformed_json_rejected": true,
132+
"unsupported_field_rejected": true
133+
},
134+
"status": "pass"
135+
},
136+
"reference_sha256": "f78d41266db73592526a3c5e7d6e4c883f549f99b01c6387c7f5337fc269dd5e",
137+
"schema_version": "deltawire-skill-conformance.v1",
138+
"skill_sha256": "e7d3a1e8ccef1b72338d5dac7a7b7ac412164846018ab55b7f2ecadf5a07a89a",
139+
"skill_tree_sha256": "a841ca12dc687b1e467f90d5d9f5ccef120bc810742fab693e8e976135a75984",
140+
"status": "pass"
141+
}

0 commit comments

Comments
 (0)