From a18a4a7ac572366c09c15641ec274cc6f15bfb5d Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Fri, 7 Aug 2026 13:39:40 -0500 Subject: [PATCH 01/16] Make release output dispatch container-safe --- .github/workflows/pr.yml | 14 +- .pre-commit-config.yaml | 21 +- README.md | 46 +++- release-build-output-dispatch/action.yml | 123 ++++++----- release-build-output/action.yml | 90 -------- release-build-output/config.schema.json | 206 ++++++++++++++++++ release-build-output/validate-config.sh | 119 ++++++++++ .../inline-package.json | 14 ++ .../package-file.json | 12 + tests/release_build_output_config_test.sh | 107 +++++++++ 10 files changed, 600 insertions(+), 152 deletions(-) delete mode 100644 release-build-output/action.yml create mode 100644 release-build-output/config.schema.json create mode 100755 release-build-output/validate-config.sh create mode 100644 tests/release-build-output-config/inline-package.json create mode 100644 tests/release-build-output-config/package-file.json create mode 100755 tests/release_build_output_config_test.sh diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 7b3ebce7..4c797303 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -22,6 +22,7 @@ jobs: - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1 - name: Test release build-output materialization run: | + ./tests/release_build_output_config_test.sh ./tests/release_build_output_descriptors_test.sh ./tests/release_build_output_prepare_test.sh ./tests/release_build_output_test.sh @@ -35,11 +36,14 @@ jobs: SHARED_ACTIONS_REPO: ${{ github.event.pull_request.head.repo.full_name }} SHARED_ACTIONS_REF: ${{ github.event.pull_request.head.sha }} with: - artifact-type: custom - output-directory: release-build-output-smoke - release-artifacts: '[{"path":"package.tar.gz"}]' - release-package: '{"ecosystem":"archive","name":"smoke","version":"1.0"}' - release-unit: archive:smoke + config: >- + { + "artifact_type": "custom", + "component_id": "archive:smoke", + "output_directory": "release-build-output-smoke", + "package": {"ecosystem": "archive", "name": "smoke", "version": "1.0"}, + "artifacts": [{"path": "package.tar.gz"}] + } source-artifact-name: release-build-output-dispatch-smoke source-sha: ${{ github.event.pull_request.head.sha }} - name: Verify release build-output dispatch smoke test diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 799d7320..4a7f1db5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,7 +3,10 @@ ci: autoupdate_schedule: quarterly - skip: [actionlint-docker] + # The runtime validator suite requires jq. pre-commit.ci validates the same + # valid fixtures against config.schema.json; GitHub Actions runs the full + # valid/invalid runtime suite on an Ubuntu runner with jq. + skip: [actionlint-docker, release-build-output-config] repos: - repo: https://github.com/pre-commit/pre-commit-hooks @@ -11,6 +14,7 @@ repos: hooks: - id: trailing-whitespace - id: check-added-large-files + - id: check-json - id: check-yaml - id: end-of-file-fixer - repo: https://github.com/zizmorcore/zizmor-pre-commit @@ -52,3 +56,18 @@ repos: hooks: - id: yamllint additional_dependencies: [pyyaml] + - repo: https://github.com/python-jsonschema/check-jsonschema + rev: 0.37.2 + hooks: + - id: check-jsonschema + name: validate release build-output config schema fixtures + args: [--schemafile, release-build-output/config.schema.json] + files: ^tests/release-build-output-config/.*\.json$ + - repo: local + hooks: + - id: release-build-output-config + name: validate release build-output configuration + entry: ./tests/release_build_output_config_test.sh + language: system + pass_filenames: false + files: ^(release-build-output/(config\.schema\.json|validate-config\.sh)|tests/release_build_output_config_test\.sh)$ diff --git a/README.md b/README.md index 4ea1b0ca..69f206a8 100644 --- a/README.md +++ b/README.md @@ -17,18 +17,50 @@ directory and uploads a companion artifact named `release-build-output.json`, `release-build-metadata.json`, provenance, and an SBOM record for every primary artifact. -Conda and wheel jobs can set `artifact-type` to `conda` or `wheel` and omit -`release-artifacts`; the implementation reads package metadata from the built -files. Custom bundles provide explicit artifact descriptors and either inline -package identity or a producer-created package JSON file. +Every producer passes one `config` JSON object. `artifact_type` is the +discriminator: `conda` and `wheel` derive artifact descriptors and package +metadata from the built files, while `custom` selects them explicitly. The +action validates the complete object before inspecting build outputs and +reports each invalid field as a pipeline error. Its schema is +[`release-build-output/config.schema.json`](release-build-output/config.schema.json). + +The custom configuration requires `component_id`, a non-empty `artifacts` +array, and exactly one of `package` or `package_file`. `output_directory` +defaults to `.`. A descriptor's `sbom`, `provenance`, and `signature` fields +select producer-supplied evidence; omitting `sbom` requests a generated +identity-only SPDX envelope, not a dependency SBOM. + +```yaml +- name: Create custom release build-output companion + uses: rapidsai/shared-actions/release-build-output-dispatch@main + with: + config: >- + { + "artifact_type": "custom", + "component_id": "maven:cuvs-java", + "output_directory": "java/cuvs-java/target", + "package_file": "cuvs-java.release-package.json", + "artifacts": [{"path": "cuvs-java-*-x86_64-cuda*.jar"}] + } + source-artifact-name: cuvs-java + source-sha: ${{ github.sha }} +``` + +The validator is also exercised by pre-commit against valid and invalid +fixtures. Pre-commit can validate the JSON structure, while the pipeline checks +that package files, primary artifacts, and evidence globs resolve after the +producer has run. ```yaml - name: Create release build-output companion uses: rapidsai/shared-actions/release-build-output-dispatch@main with: - artifact-type: wheel - output-directory: ${{ steps.package-name.outputs.WHEEL_OUTPUT_DIR }} - release-unit: wheel:example + config: >- + { + "artifact_type": "wheel", + "component_id": "wheel:example", + "output_directory": ${{ toJSON(steps.package-name.outputs.WHEEL_OUTPUT_DIR) }} + } source-artifact-name: ${{ steps.package-name.outputs.RAPIDS_PACKAGE_NAME }} source-sha: ${{ github.sha }} ``` diff --git a/release-build-output-dispatch/action.yml b/release-build-output-dispatch/action.yml index 154ba3d7..fc798f74 100644 --- a/release-build-output-dispatch/action.yml +++ b/release-build-output-dispatch/action.yml @@ -1,33 +1,12 @@ name: Dispatch release build output -description: Check out the selected shared-actions revision and create a release build-output companion. +description: Create a release build-output companion from the selected shared-actions revision. inputs: - artifact-type: - description: One of conda, wheel, or custom. + config: + description: >- + Release build-output JSON object containing artifact_type, component_id, output_directory, and custom artifact + selection and package identity when artifact_type is custom. required: true - release-unit: - description: Release-platform unit ID for every primary artifact in this bundle. - required: true - release-package: - description: JSON package fields shared by the bundle. - required: false - release-package-file: - description: Relative path to producer-created package JSON inside output-directory. - required: false - release-artifacts: - description: JSON artifact and evidence descriptors relative to output-directory. - required: false - output-directory: - description: Directory containing the primary files and any producer-supplied evidence. - required: true - manifest-name: - description: Filename to write inside output-directory. - required: false - default: release-build-output.json - metadata-name: - description: Filename for the build metadata envelope. - required: false - default: release-build-metadata.json source-artifact-name: description: Name of the GitHub Actions artifact bundle containing this output. required: true @@ -38,35 +17,81 @@ inputs: outputs: manifest-path: description: Absolute path to the generated manifest. - value: ${{ steps.release-build-output.outputs.manifest-path }} + value: ${{ steps.materialize.outputs.manifest-path }} metadata-path: description: Absolute path to the build metadata envelope. - value: ${{ steps.release-build-output.outputs.metadata-path }} + value: ${{ steps.materialize.outputs.metadata-path }} manifest-artifact-name: description: Name of the uploaded GitHub Actions companion artifact. - value: ${{ steps.release-build-output.outputs.manifest-artifact-name }} + value: ${{ steps.companion-name.outputs.name }} runs: using: composite steps: - - name: Check out shared-actions implementation - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: ${{ env.SHARED_ACTIONS_REPO || github.action_repository || 'rapidsai/shared-actions' }} - ref: ${{ env.SHARED_ACTIONS_REF || github.action_ref || 'main' }} - path: ./shared-actions - persist-credentials: false - - id: release-build-output - name: Create release build-output companion - uses: ./shared-actions/release-build-output + - id: implementation + name: Resolve release build-output implementation + shell: bash + env: + ACTION_PATH: ${{ github.action_path }} + HOST_WORKSPACE: ${{ github.workspace }} + run: | + host_root="$(dirname "$(dirname "${HOST_WORKSPACE}")")" + container_root="$(dirname "$(dirname "${PWD}")")" + action_path="${ACTION_PATH}" + if [[ ! -d "${action_path}" && "${action_path}" == "${host_root}/"* ]]; then + action_path="${container_root}/${action_path#"${host_root}/"}" + fi + implementation_path="$(dirname "${action_path}")/release-build-output" + if [[ ! -d "${implementation_path}" ]]; then + echo "release-build-output implementation not found at ${implementation_path}" >&2 + exit 1 + fi + echo "path=${implementation_path}" >>"${GITHUB_OUTPUT}" + - id: configuration + name: Validate release build-output configuration + shell: bash + env: + IMPLEMENTATION_PATH: ${{ steps.implementation.outputs.path }} + RELEASE_BUILD_OUTPUT_CONFIG: ${{ inputs.config }} + run: "${IMPLEMENTATION_PATH}/validate-config.sh" + - id: prepare + name: Describe release artifacts + shell: bash + env: + IMPLEMENTATION_PATH: ${{ steps.implementation.outputs.path }} + RELEASE_ARTIFACTS: ${{ steps.configuration.outputs.artifacts }} + RELEASE_ARTIFACT_TYPE: ${{ steps.configuration.outputs.artifact_type }} + RELEASE_OUTPUT_DIRECTORY: ${{ steps.configuration.outputs.output_directory }} + RELEASE_PACKAGE: ${{ steps.configuration.outputs.package }} + RELEASE_PACKAGE_FILE: ${{ steps.configuration.outputs.package_file }} + run: "${IMPLEMENTATION_PATH}/prepare.sh" + - id: materialize + name: Materialize release build-output records + shell: bash + env: + IMPLEMENTATION_PATH: ${{ steps.implementation.outputs.path }} + RELEASE_ARTIFACTS: ${{ steps.prepare.outputs.artifacts }} + RELEASE_MANIFEST_NAME: release-build-output.json + RELEASE_METADATA_NAME: release-build-metadata.json + RELEASE_OUTPUT_DIRECTORY: ${{ steps.configuration.outputs.output_directory }} + RELEASE_PACKAGE: ${{ steps.prepare.outputs.package }} + RELEASE_PACKAGE_FILE: ${{ steps.configuration.outputs.package_file }} + RELEASE_SOURCE_ARTIFACT_NAME: ${{ inputs.source-artifact-name }} + RELEASE_SOURCE_SHA: ${{ inputs.source-sha || github.sha }} + RELEASE_UNIT: ${{ steps.configuration.outputs.component_id }} + run: "${IMPLEMENTATION_PATH}/materialize.sh" + - id: companion-name + name: Set companion artifact name + shell: bash + env: + SOURCE_ARTIFACT_NAME: ${{ inputs.source-artifact-name }} + run: echo "name=release-build-output-${SOURCE_ARTIFACT_NAME}" >>"${GITHUB_OUTPUT}" + - name: Upload release build-output companion + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - artifact-type: ${{ inputs.artifact-type }} - release-unit: ${{ inputs.release-unit }} - release-package: ${{ inputs.release-package }} - release-package-file: ${{ inputs.release-package-file }} - release-artifacts: ${{ inputs.release-artifacts }} - output-directory: ${{ inputs.output-directory }} - manifest-name: ${{ inputs.manifest-name }} - metadata-name: ${{ inputs.metadata-name }} - source-artifact-name: ${{ inputs.source-artifact-name }} - source-sha: ${{ inputs.source-sha }} + if-no-files-found: error + name: ${{ steps.companion-name.outputs.name }} + path: | + ${{ steps.configuration.outputs.output_directory }}/release-build-output.json + ${{ steps.configuration.outputs.output_directory }}/release-build-metadata.json + ${{ steps.configuration.outputs.output_directory }}/release-evidence/** diff --git a/release-build-output/action.yml b/release-build-output/action.yml deleted file mode 100644 index 7c6965e3..00000000 --- a/release-build-output/action.yml +++ /dev/null @@ -1,90 +0,0 @@ -name: Create release build output companion -description: Validate a build artifact bundle and upload its release-platform manifest and evidence companion. - -inputs: - artifact-type: - description: One of conda, wheel, or custom. Conda and wheel descriptors are derived when release-artifacts is omitted. - required: true - release-unit: - description: Release-platform unit ID for every primary artifact in this bundle. - required: true - release-package: - description: JSON package fields shared by the bundle. Provide this or release-package-file for custom artifacts. - required: false - release-package-file: - description: Relative path to producer-created package JSON inside output-directory. - required: false - release-artifacts: - description: JSON artifact and evidence descriptors relative to output-directory. Required for custom artifacts. - required: false - output-directory: - description: Directory containing the primary files and any producer-supplied evidence. - required: true - manifest-name: - description: Filename to write inside output-directory. - required: false - default: release-build-output.json - metadata-name: - description: Filename for the build-environment and SBOM-classification envelope. - required: false - default: release-build-metadata.json - source-artifact-name: - description: Name of the GitHub Actions artifact bundle containing this output. - required: true - source-sha: - description: Source revision built by the producing job. Defaults to the current workflow SHA. - required: false - -outputs: - manifest-path: - description: Absolute path to the generated manifest. - value: ${{ steps.materialize.outputs.manifest-path }} - metadata-path: - description: Absolute path to the build metadata envelope. - value: ${{ steps.materialize.outputs.metadata-path }} - manifest-artifact-name: - description: Name of the uploaded GitHub Actions companion artifact. - value: ${{ steps.companion-name.outputs.name }} - -runs: - using: composite - steps: - - id: prepare - name: Describe release artifacts - shell: bash - env: - RELEASE_ARTIFACTS: ${{ inputs.release-artifacts }} - RELEASE_ARTIFACT_TYPE: ${{ inputs.artifact-type }} - RELEASE_OUTPUT_DIRECTORY: ${{ inputs.output-directory }} - RELEASE_PACKAGE: ${{ inputs.release-package }} - RELEASE_PACKAGE_FILE: ${{ inputs.release-package-file }} - run: ./shared-actions/release-build-output/prepare.sh - - id: materialize - name: Materialize release build-output records - shell: bash - env: - RELEASE_ARTIFACTS: ${{ steps.prepare.outputs.artifacts }} - RELEASE_MANIFEST_NAME: ${{ inputs.manifest-name }} - RELEASE_METADATA_NAME: ${{ inputs.metadata-name }} - RELEASE_OUTPUT_DIRECTORY: ${{ inputs.output-directory }} - RELEASE_PACKAGE: ${{ steps.prepare.outputs.package }} - RELEASE_PACKAGE_FILE: ${{ inputs.release-package-file }} - RELEASE_SOURCE_ARTIFACT_NAME: ${{ inputs.source-artifact-name }} - RELEASE_SOURCE_SHA: ${{ inputs.source-sha || github.sha }} - RELEASE_UNIT: ${{ inputs.release-unit }} - run: ./shared-actions/release-build-output/materialize.sh - - id: companion-name - name: Set companion artifact name - shell: bash - env: - SOURCE_ARTIFACT_NAME: ${{ inputs.source-artifact-name }} - run: echo "name=release-build-output-${SOURCE_ARTIFACT_NAME}" >>"${GITHUB_OUTPUT}" - - name: Upload release build-output companion - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - if-no-files-found: error - name: ${{ steps.companion-name.outputs.name }} - path: | - ${{ inputs.output-directory }}/${{ inputs.manifest-name }} - ${{ inputs.output-directory }}/${{ inputs.metadata-name }} - ${{ inputs.output-directory }}/release-evidence/** diff --git a/release-build-output/config.schema.json b/release-build-output/config.schema.json new file mode 100644 index 00000000..81d37d6f --- /dev/null +++ b/release-build-output/config.schema.json @@ -0,0 +1,206 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://rapids.ai/schemas/release-build-output-config-v1.json", + "title": "Release build-output configuration", + "description": "Producer configuration for one release build-output companion.", + "type": "object", + "additionalProperties": false, + "required": [ + "artifact_type", + "component_id" + ], + "properties": { + "artifact_type": { + "description": "How primary artifact descriptors and package identities are obtained.", + "type": "string", + "enum": [ + "conda", + "custom", + "wheel" + ] + }, + "component_id": { + "description": "Stable release component ID shared by the selected files and their matrix variants.", + "type": "string", + "minLength": 1, + "pattern": "^[^\\r\\n]+$" + }, + "output_directory": { + "description": "Base directory for artifact and evidence paths. Defaults to the job working directory.", + "type": "string", + "minLength": 1, + "pattern": "^[^\\r\\n]+$", + "default": "." + }, + "package": { + "$ref": "#/$defs/package" + }, + "package_file": { + "description": "Producer-created package identity JSON, relative to output_directory.", + "type": "string", + "minLength": 1, + "pattern": "^[^\\r\\n]+$" + }, + "artifacts": { + "description": "Primary release files and optional producer-supplied evidence sidecars.", + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/artifact" + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "artifact_type": { + "const": "custom" + } + }, + "required": [ + "artifact_type" + ] + }, + "then": { + "required": [ + "artifacts" + ], + "oneOf": [ + { + "required": [ + "package" + ], + "not": { + "required": [ + "package_file" + ] + } + }, + { + "required": [ + "package_file" + ], + "not": { + "required": [ + "package" + ] + } + } + ] + }, + "else": { + "required": [ + "output_directory" + ], + "not": { + "anyOf": [ + { + "required": [ + "artifacts" + ] + }, + { + "required": [ + "package" + ] + }, + { + "required": [ + "package_file" + ] + } + ] + } + } + } + ], + "$defs": { + "package": { + "type": "object", + "additionalProperties": false, + "required": [ + "ecosystem", + "name" + ], + "properties": { + "ecosystem": { + "type": "string", + "minLength": 1 + }, + "name": { + "type": "string", + "minLength": 1 + }, + "version": { + "type": "string" + }, + "build": { + "type": "string" + }, + "platform": { + "type": "string" + } + } + }, + "artifact": { + "type": "object", + "additionalProperties": false, + "required": [ + "path" + ], + "properties": { + "path": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\r\\n]+$" + }, + "sbom": { + "description": "Producer-supplied dependency SBOM path or glob.", + "type": "string", + "minLength": 1, + "pattern": "^[^\\r\\n]+$" + }, + "provenance": { + "description": "Producer-supplied provenance path or glob.", + "type": "string", + "minLength": 1, + "pattern": "^[^\\r\\n]+$" + }, + "signature": { + "description": "Producer-supplied signature path or glob.", + "type": "string", + "minLength": 1, + "pattern": "^[^\\r\\n]+$" + }, + "package": { + "description": "Per-artifact package identity overrides.", + "type": "object", + "additionalProperties": false, + "properties": { + "ecosystem": { + "type": "string", + "minLength": 1 + }, + "name": { + "type": "string", + "minLength": 1 + }, + "version": { + "type": "string", + "minLength": 1 + }, + "build": { + "type": "string", + "minLength": 1 + }, + "platform": { + "type": "string", + "minLength": 1 + } + } + } + } + } + } +} diff --git a/release-build-output/validate-config.sh b/release-build-output/validate-config.sh new file mode 100755 index 00000000..458f4e5e --- /dev/null +++ b/release-build-output/validate-config.sh @@ -0,0 +1,119 @@ +#!/usr/bin/env bash +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +set -euo pipefail + +emit_error() { + local message="$1" + printf '::error title=Invalid release-build-output configuration::%s\n' "${message}" >&2 +} + +config="${RELEASE_BUILD_OUTPUT_CONFIG:-}" +if [[ -z "${config}" ]]; then + emit_error "release-build-output must be a non-empty JSON object" + exit 1 +fi + +if ! compact_config="$(jq -ce . <<<"${config}" 2>/dev/null)"; then + parse_error="$(jq -ce . <<<"${config}" 2>&1 || true)" + emit_error "release-build-output must be valid JSON: ${parse_error}" + exit 1 +fi + +validation_errors="$(jq -r ' + def single_line_string: + type == "string" and length > 0 and (test("[\\r\\n]") | not); + def package_object: + type == "object" + and (keys - ["ecosystem", "name", "version", "build", "platform"] | length == 0) + and (.ecosystem | single_line_string) + and (.name | single_line_string) + and ((.version // "") | type == "string") + and ((.build // "") | type == "string") + and ((.platform // "") | type == "string"); + def artifact_package_object: + type == "object" + and (keys - ["ecosystem", "name", "version", "build", "platform"] | length == 0) + and (to_entries | map(.value | single_line_string) | all); + if type != "object" then + ["release-build-output must be a JSON object"] + else + (keys - ["artifact_type", "component_id", "output_directory", "package", "package_file", "artifacts"]) as $unknown + | [ + if ($unknown | length) > 0 then + "unknown field(s): " + ($unknown | join(", ")) + else empty end, + if (.artifact_type == "conda" or .artifact_type == "custom" or .artifact_type == "wheel") then empty + else "artifact_type must be one of: conda, custom, wheel" end, + if (.component_id | single_line_string) then empty + else "component_id must be a non-empty, single-line string" end, + if (has("output_directory") | not) or (.output_directory | single_line_string) then empty + else "output_directory must be a non-empty, single-line string when supplied" end + ] + + if .artifact_type == "custom" then + [ + if ([has("package"), has("package_file")] | map(select(.)) | length) == 1 then empty + else "exactly one of package or package_file is required for custom artifacts" end, + if (has("package") | not) or (.package | package_object) then empty + else "package must contain non-empty ecosystem and name strings and only version, build, or platform as optional string fields" end, + if (has("package_file") | not) or (.package_file | single_line_string) then empty + else "package_file must be a non-empty, single-line path relative to output_directory" end, + if (.artifacts | type == "array" and length > 0) then empty + else "artifacts must be a non-empty array for custom artifacts" end + ] + elif .artifact_type == "conda" or .artifact_type == "wheel" then + [ + if (.output_directory | single_line_string) then empty + else "output_directory is required for conda and wheel artifacts" end, + if has("artifacts") or has("package") or has("package_file") then + "artifacts, package, and package_file are only valid when artifact_type is custom" + else empty end + ] + else [] end + + if .artifact_type == "custom" and (.artifacts | type) == "array" then + [ + .artifacts | to_entries[] + | .key as $index + | .value as $artifact + | if ($artifact | type) != "object" then + "artifacts[\($index)] must be an object" + else + ($artifact | keys - ["path", "sbom", "provenance", "signature", "package"]) as $artifact_unknown + | if ($artifact_unknown | length) > 0 then + "artifacts[\($index)] has unknown field(s): " + ($artifact_unknown | join(", ")) + else empty end, + if ($artifact.path | single_line_string) then empty + else "artifacts[\($index)].path must be a non-empty, single-line string" end, + ($artifact | to_entries[] + | select(.key == "sbom" or .key == "provenance" or .key == "signature") + | select((.value | single_line_string) | not) + | "artifacts[\($index)].\(.key) must be a non-empty, single-line string"), + if ($artifact | has("package") | not) or ($artifact.package | artifact_package_object) then empty + else "artifacts[\($index)].package must contain only non-empty string identity overrides" end + end + ] + else [] end + end + | .[] +' <<<"${compact_config}")" + +if [[ -n "${validation_errors}" ]]; then + while IFS= read -r validation_error; do + emit_error "${validation_error}" + done <<<"${validation_errors}" + exit 1 +fi + +if [[ -z "${GITHUB_OUTPUT:-}" ]]; then + emit_error "GITHUB_OUTPUT is required" + exit 1 +fi + +{ + printf 'artifact_type=%s\n' "$(jq -r '.artifact_type' <<<"${compact_config}")" + printf 'component_id=%s\n' "$(jq -r '.component_id' <<<"${compact_config}")" + printf 'output_directory=%s\n' "$(jq -r '.output_directory // "."' <<<"${compact_config}")" + printf 'package=%s\n' "$(jq -c '.package // empty' <<<"${compact_config}")" + printf 'package_file=%s\n' "$(jq -r '.package_file // empty' <<<"${compact_config}")" + printf 'artifacts=%s\n' "$(jq -c '.artifacts // empty' <<<"${compact_config}")" +} >>"${GITHUB_OUTPUT}" diff --git a/tests/release-build-output-config/inline-package.json b/tests/release-build-output-config/inline-package.json new file mode 100644 index 00000000..3443ea3b --- /dev/null +++ b/tests/release-build-output-config/inline-package.json @@ -0,0 +1,14 @@ +{ + "artifact_type": "custom", + "component_id": "archive:smoke", + "package": { + "ecosystem": "archive", + "name": "smoke", + "version": "1.0" + }, + "artifacts": [ + { + "path": "smoke.tar.gz" + } + ] +} diff --git a/tests/release-build-output-config/package-file.json b/tests/release-build-output-config/package-file.json new file mode 100644 index 00000000..de25059e --- /dev/null +++ b/tests/release-build-output-config/package-file.json @@ -0,0 +1,12 @@ +{ + "artifact_type": "custom", + "component_id": "maven:cuvs-java", + "output_directory": "java/cuvs-java/target", + "package_file": "cuvs-java.release-package.json", + "artifacts": [ + { + "path": "cuvs-java-*-x86_64-cuda*.jar", + "sbom": "cuvs-java.spdx.json" + } + ] +} diff --git a/tests/release_build_output_config_test.sh b/tests/release_build_output_config_test.sh new file mode 100755 index 00000000..d77375e8 --- /dev/null +++ b/tests/release_build_output_config_test.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +set -euo pipefail + +repository_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +validator="${repository_root}/release-build-output/validate-config.sh" +temporary_directory="$(mktemp -d)" +trap 'rm -rf "${temporary_directory}"' EXIT + +assert_invalid() { + local name="$1" + local config="$2" + local expected="$3" + local error_file="${temporary_directory}/${name}.error" + + if RELEASE_BUILD_OUTPUT_CONFIG="${config}" GITHUB_OUTPUT="${temporary_directory}/${name}.output" \ + "${validator}" 2>"${error_file}"; then + echo "validate-config.sh unexpectedly accepted ${name}" >&2 + exit 1 + fi + if ! grep -F -- "${expected}" "${error_file}" >/dev/null; then + echo "validate-config.sh did not explain ${name}; output was:" >&2 + sed 's/^/ /' "${error_file}" >&2 + exit 1 + fi +} + +valid_output="${temporary_directory}/valid.output" +RELEASE_BUILD_OUTPUT_CONFIG="$(<"${repository_root}/tests/release-build-output-config/package-file.json")" \ + GITHUB_OUTPUT="${valid_output}" "${validator}" + +grep -Fx 'component_id=maven:cuvs-java' "${valid_output}" +grep -Fx 'artifact_type=custom' "${valid_output}" +grep -Fx 'output_directory=java/cuvs-java/target' "${valid_output}" +grep -Fx 'package=' "${valid_output}" +grep -Fx 'package_file=cuvs-java.release-package.json' "${valid_output}" +grep -Fx 'artifacts=[{"path":"cuvs-java-*-x86_64-cuda*.jar","sbom":"cuvs-java.spdx.json"}]' "${valid_output}" + +inline_output="${temporary_directory}/inline.output" +RELEASE_BUILD_OUTPUT_CONFIG="$(<"${repository_root}/tests/release-build-output-config/inline-package.json")" \ + GITHUB_OUTPUT="${inline_output}" "${validator}" + +grep -Fx 'output_directory=.' "${inline_output}" +grep -Fx 'package={"ecosystem":"archive","name":"smoke","version":"1.0"}' "${inline_output}" +grep -Fx 'package_file=' "${inline_output}" + +assert_invalid \ + malformed-json \ + '{"component_id":' \ + 'release-build-output must be valid JSON' + +assert_invalid \ + not-an-object \ + '[]' \ + 'release-build-output must be a JSON object' + +assert_invalid \ + missing-fields \ + '{}' \ + 'component_id must be a non-empty, single-line string' +grep -F 'artifact_type must be one of: conda, custom, wheel' "${temporary_directory}/missing-fields.error" >/dev/null + +assert_invalid \ + custom-missing-fields \ + '{"artifact_type":"custom","component_id":"archive:smoke"}' \ + 'exactly one of package or package_file is required for custom artifacts' +grep -F 'artifacts must be a non-empty array for custom artifacts' "${temporary_directory}/custom-missing-fields.error" >/dev/null + +assert_invalid \ + unknown-field \ + '{"artifact_type":"custom","component_id":"archive:smoke","package_file":"package.json","artifacts":[{"path":"smoke.tar.gz"}],"unit":"archive:smoke"}' \ + 'unknown field(s): unit' + +assert_invalid \ + conflicting-package-source \ + '{"artifact_type":"custom","component_id":"archive:smoke","package":{"ecosystem":"archive","name":"smoke"},"package_file":"package.json","artifacts":[{"path":"smoke.tar.gz"}]}' \ + 'exactly one of package or package_file is required for custom artifacts' + +assert_invalid \ + malformed-artifact \ + '{"artifact_type":"custom","component_id":"archive:smoke","package_file":"package.json","artifacts":[{"file":"smoke.tar.gz","sbom":false}]}' \ + 'artifacts[0] has unknown field(s): file' +grep -F 'artifacts[0].path must be a non-empty, single-line string' "${temporary_directory}/malformed-artifact.error" >/dev/null +grep -F 'artifacts[0].sbom must be a non-empty, single-line string' "${temporary_directory}/malformed-artifact.error" >/dev/null + +assert_invalid \ + standard-missing-output-directory \ + '{"artifact_type":"conda","component_id":"conda:smoke"}' \ + 'output_directory is required for conda and wheel artifacts' + +assert_invalid \ + standard-custom-fields \ + '{"artifact_type":"wheel","component_id":"wheel:smoke","output_directory":"dist","artifacts":[{"path":"smoke.whl"}]}' \ + 'artifacts, package, and package_file are only valid when artifact_type is custom' + +standard_output="${temporary_directory}/standard.output" +RELEASE_BUILD_OUTPUT_CONFIG='{ + "artifact_type": "wheel", + "component_id": "wheel:kvikio", + "output_directory": "dist" +}' GITHUB_OUTPUT="${standard_output}" "${validator}" + +grep -Fx 'artifact_type=wheel' "${standard_output}" +grep -Fx 'component_id=wheel:kvikio' "${standard_output}" +grep -Fx 'output_directory=dist' "${standard_output}" +grep -Fx 'artifacts=' "${standard_output}" From 2e1001979ff1ee4ec8f78f6ec04a3880f51831b1 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Mon, 17 Aug 2026 11:35:25 -0500 Subject: [PATCH 02/16] Link release output config schema --- release-build-output-dispatch/action.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/release-build-output-dispatch/action.yml b/release-build-output-dispatch/action.yml index fc798f74..85cf7e97 100644 --- a/release-build-output-dispatch/action.yml +++ b/release-build-output-dispatch/action.yml @@ -5,7 +5,8 @@ inputs: config: description: >- Release build-output JSON object containing artifact_type, component_id, output_directory, and custom artifact - selection and package identity when artifact_type is custom. + selection and package identity when artifact_type is custom. Schema and field documentation: + https://github.com/rapidsai/shared-actions/blob/main/release-build-output/config.schema.json required: true source-artifact-name: description: Name of the GitHub Actions artifact bundle containing this output. From 318371e84cee75250f30f46bd1a905bf92d73df1 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Mon, 17 Aug 2026 20:22:48 -0500 Subject: [PATCH 03/16] Document release build output companions --- README.md | 60 ++------------------ release-build-output/README.md | 101 +++++++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+), 56 deletions(-) create mode 100644 release-build-output/README.md diff --git a/README.md b/README.md index 69f206a8..697fd7f8 100644 --- a/README.md +++ b/README.md @@ -13,63 +13,11 @@ A dispatch action is one that: `release-build-output-dispatch` validates a producer's local build artifact directory and uploads a companion artifact named -`release-build-output-`. The companion contains -`release-build-output.json`, `release-build-metadata.json`, provenance, and an -SBOM record for every primary artifact. - -Every producer passes one `config` JSON object. `artifact_type` is the -discriminator: `conda` and `wheel` derive artifact descriptors and package -metadata from the built files, while `custom` selects them explicitly. The -action validates the complete object before inspecting build outputs and -reports each invalid field as a pipeline error. Its schema is -[`release-build-output/config.schema.json`](release-build-output/config.schema.json). - -The custom configuration requires `component_id`, a non-empty `artifacts` -array, and exactly one of `package` or `package_file`. `output_directory` -defaults to `.`. A descriptor's `sbom`, `provenance`, and `signature` fields -select producer-supplied evidence; omitting `sbom` requests a generated -identity-only SPDX envelope, not a dependency SBOM. +`release-build-output-`. -```yaml -- name: Create custom release build-output companion - uses: rapidsai/shared-actions/release-build-output-dispatch@main - with: - config: >- - { - "artifact_type": "custom", - "component_id": "maven:cuvs-java", - "output_directory": "java/cuvs-java/target", - "package_file": "cuvs-java.release-package.json", - "artifacts": [{"path": "cuvs-java-*-x86_64-cuda*.jar"}] - } - source-artifact-name: cuvs-java - source-sha: ${{ github.sha }} -``` - -The validator is also exercised by pre-commit against valid and invalid -fixtures. Pre-commit can validate the JSON structure, while the pipeline checks -that package files, primary artifacts, and evidence globs resolve after the -producer has run. - -```yaml -- name: Create release build-output companion - uses: rapidsai/shared-actions/release-build-output-dispatch@main - with: - config: >- - { - "artifact_type": "wheel", - "component_id": "wheel:example", - "output_directory": ${{ toJSON(steps.package-name.outputs.WHEEL_OUTPUT_DIR) }} - } - source-artifact-name: ${{ steps.package-name.outputs.RAPIDS_PACKAGE_NAME }} - source-sha: ${{ github.sha }} -``` - -A descriptor-selected producer SBOM is classified as `producer-dependency`. -When no SBOM is supplied, the action generates an SPDX artifact-identity -envelope and classifies it as `generated-identity`. The generated envelope -contains the primary artifact's identity and SHA-256 but no dependency -inventory; it must not be treated as dependency coverage. +See the [`release-build-output` documentation](release-build-output/README.md) +for configuration, examples, source-revision handling, companion contents, and +evidence semantics. The dispatch wrapper honors `SHARED_ACTIONS_REPO` and `SHARED_ACTIONS_REF`. When neither is set, it checks out the same repository and ref used to invoke diff --git a/release-build-output/README.md b/release-build-output/README.md new file mode 100644 index 00000000..8f6d6935 --- /dev/null +++ b/release-build-output/README.md @@ -0,0 +1,101 @@ +# Release build-output companions + +`release-build-output-dispatch` records the exact files produced by a package +build and uploads a companion GitHub Actions artifact named +`release-build-output-`. Release tooling uses the +companion to associate primary artifacts with their build identity and +available evidence without reconstructing the build later. + +The companion contains: + +```text +. +├── release-build-metadata.json +├── release-build-output.json +└── release-evidence + ├── .provenance.json + └── .spdx.json +``` + +## Configuration + +Every producer passes one `config` JSON object. Its canonical schema and field +documentation are in [`config.schema.json`](config.schema.json). + +The `artifact_type` field determines how the action discovers primary files: + +- `conda` and `wheel` derive artifact descriptors and package identity from the + built packages; +- `custom` requires explicit artifact descriptors and exactly one of `package` + or `package_file`. + +`component_id` is a stable release-catalog identifier such as `conda:cudf` or +`maven:cuvs-java`; it is not a generated UUID. Standard RAPIDS Conda and wheel +workflows construct it as `:`. Custom producers +must supply the release unit selected for that package family. + +The action validates the complete configuration before inspecting build +outputs. It then verifies properties that depend on produced files, including +whether package, primary-artifact, and evidence paths resolve unambiguously. + +## Source revision + +`source-sha` must identify the repository revision actually checked out and +built. RAPIDS shared workflows pass `${{ env.RAPIDS_SHA }}`: + +- Conda build workflows set `RAPIDS_SHA` to `git rev-parse HEAD` immediately + after checkout. +- Wheel and custom workflows use `rapids-github-info`; it uses `inputs.sha` + when supplied and otherwise sets `RAPIDS_SHA` to `git rev-parse HEAD`. + +This distinction matters when a reusable workflow checks out a repository or +revision different from the workflow event. Direct callers should likewise +resolve the checked-out commit instead of assuming `${{ github.sha }}` names +the built source. + +## Standard package example + +```yaml +- name: Create wheel release build-output companion + uses: rapidsai/shared-actions/release-build-output-dispatch@main + with: + config: >- + { + "artifact_type": "wheel", + "component_id": "wheel:example", + "output_directory": ${{ toJSON(steps.package-name.outputs.WHEEL_OUTPUT_DIR) }} + } + source-artifact-name: ${{ steps.package-name.outputs.RAPIDS_PACKAGE_NAME }} + source-sha: ${{ env.RAPIDS_SHA }} +``` + +## Custom package example + +```yaml +- name: Create custom release build-output companion + uses: rapidsai/shared-actions/release-build-output-dispatch@main + with: + config: >- + { + "artifact_type": "custom", + "component_id": "maven:cuvs-java", + "output_directory": "java/cuvs-java/target", + "package_file": "cuvs-java.release-package.json", + "artifacts": [{"path": "cuvs-java-*-x86_64-cuda*.jar"}] + } + source-artifact-name: cuvs-java + source-sha: ${{ env.RAPIDS_SHA }} +``` + +## Evidence semantics + +A descriptor-selected producer SBOM is classified as `producer-dependency`. +It is evidence supplied by the producer and may contain a dependency inventory. + +When no SBOM is selected, the action generates an SPDX artifact-identity +envelope classified as `generated-identity`. It records package identity and +the primary artifact SHA-256, but contains no dependency or source-license +inventory. It must not be reported as producer-supplied dependency coverage. + +Producer-supplied SBOM, provenance, and signature sidecars are copied under +`release-evidence/` so the companion remains independently consumable. From e28ffe5a933bcb73b407d8a79871bd1a8f121631 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Mon, 17 Aug 2026 21:20:24 -0500 Subject: [PATCH 04/16] Use release catalog key terminology --- .github/workflows/pr.yml | 4 +-- .pre-commit-config.yaml | 9 ++++-- release-build-output-dispatch/action.yml | 5 ++-- release-build-output/README.md | 14 ++++++---- release-build-output/config.schema.json | 6 ++-- release-build-output/materialize.sh | 16 +++++------ release-build-output/validate-config.sh | 8 +++--- .../inline-package.json | 2 +- .../package-file.json | 2 +- tests/release_build_output_config_test.sh | 24 ++++++++-------- tests/release_build_output_test.sh | 28 +++++++++---------- 11 files changed, 61 insertions(+), 57 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 4c797303..8cbc2db5 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -39,7 +39,7 @@ jobs: config: >- { "artifact_type": "custom", - "component_id": "archive:smoke", + "release_catalog_key": "archive:smoke", "output_directory": "release-build-output-smoke", "package": {"ecosystem": "archive", "name": "smoke", "version": "1.0"}, "artifacts": [{"path": "package.tar.gz"}] @@ -49,7 +49,7 @@ jobs: - name: Verify release build-output dispatch smoke test run: | jq -e ' - .artifacts[0].unit_id == "archive:smoke" + .artifacts[0].release_catalog_key == "archive:smoke" and .artifacts[0].path == "package.tar.gz" ' release-build-output-smoke/release-build-output.json >/dev/null jq -e ' diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4a7f1db5..92f9986c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,9 +3,12 @@ ci: autoupdate_schedule: quarterly - # The runtime validator suite requires jq. pre-commit.ci validates the same - # valid fixtures against config.schema.json; GitHub Actions runs the full - # valid/invalid runtime suite on an Ubuntu runner with jq. + # These hooks require tools unavailable in pre-commit.ci: + # + # * actionlint-docker requires Docker + # * release-build-output-config requires jq + # + # Both run in other CI jobs with controlled runtime dependencies. skip: [actionlint-docker, release-build-output-config] repos: diff --git a/release-build-output-dispatch/action.yml b/release-build-output-dispatch/action.yml index 85cf7e97..9af3d5de 100644 --- a/release-build-output-dispatch/action.yml +++ b/release-build-output-dispatch/action.yml @@ -4,8 +4,7 @@ description: Create a release build-output companion from the selected shared-ac inputs: config: description: >- - Release build-output JSON object containing artifact_type, component_id, output_directory, and custom artifact - selection and package identity when artifact_type is custom. Schema and field documentation: + JSON string with configuration for this action. Schema and field documentation: https://github.com/rapidsai/shared-actions/blob/main/release-build-output/config.schema.json required: true source-artifact-name: @@ -79,7 +78,7 @@ runs: RELEASE_PACKAGE_FILE: ${{ steps.configuration.outputs.package_file }} RELEASE_SOURCE_ARTIFACT_NAME: ${{ inputs.source-artifact-name }} RELEASE_SOURCE_SHA: ${{ inputs.source-sha || github.sha }} - RELEASE_UNIT: ${{ steps.configuration.outputs.component_id }} + RELEASE_CATALOG_KEY: ${{ steps.configuration.outputs.release_catalog_key }} run: "${IMPLEMENTATION_PATH}/materialize.sh" - id: companion-name name: Set companion artifact name diff --git a/release-build-output/README.md b/release-build-output/README.md index 8f6d6935..0899b292 100644 --- a/release-build-output/README.md +++ b/release-build-output/README.md @@ -29,10 +29,12 @@ The `artifact_type` field determines how the action discovers primary files: - `custom` requires explicit artifact descriptors and exactly one of `package` or `package_file`. -`component_id` is a stable release-catalog identifier such as `conda:cudf` or -`maven:cuvs-java`; it is not a generated UUID. Standard RAPIDS Conda and wheel -workflows construct it as `:`. Custom producers -must supply the release unit selected for that package family. +`release_catalog_key` identifies the release-catalog entry that owns these +artifacts. Every file and matrix variant in the same publishable artifact set +uses the same key. Standard RAPIDS Conda and wheel workflows construct it as +`:`, such as `conda:cudf`; custom producers select +an existing catalog key, such as `maven:cuvs-java`. Do not generate a UUID or a +per-build value. The action validates the complete configuration before inspecting build outputs. It then verifies properties that depend on produced files, including @@ -62,7 +64,7 @@ the built source. config: >- { "artifact_type": "wheel", - "component_id": "wheel:example", + "release_catalog_key": "wheel:example", "output_directory": ${{ toJSON(steps.package-name.outputs.WHEEL_OUTPUT_DIR) }} } source-artifact-name: ${{ steps.package-name.outputs.RAPIDS_PACKAGE_NAME }} @@ -78,7 +80,7 @@ the built source. config: >- { "artifact_type": "custom", - "component_id": "maven:cuvs-java", + "release_catalog_key": "maven:cuvs-java", "output_directory": "java/cuvs-java/target", "package_file": "cuvs-java.release-package.json", "artifacts": [{"path": "cuvs-java-*-x86_64-cuda*.jar"}] diff --git a/release-build-output/config.schema.json b/release-build-output/config.schema.json index 81d37d6f..9efd2ba6 100644 --- a/release-build-output/config.schema.json +++ b/release-build-output/config.schema.json @@ -7,7 +7,7 @@ "additionalProperties": false, "required": [ "artifact_type", - "component_id" + "release_catalog_key" ], "properties": { "artifact_type": { @@ -19,8 +19,8 @@ "wheel" ] }, - "component_id": { - "description": "Stable release component ID shared by the selected files and their matrix variants.", + "release_catalog_key": { + "description": "Key of the release-catalog entry that owns these artifacts. Reuse it for every file and matrix variant in the same publishable artifact set. Standard RAPIDS workflows use :, such as conda:cudf; custom producers select an existing catalog key, such as maven:cuvs-java. This is not a generated UUID or per-build value.", "type": "string", "minLength": 1, "pattern": "^[^\\r\\n]+$" diff --git a/release-build-output/materialize.sh b/release-build-output/materialize.sh index 673dd6ba..a500e744 100755 --- a/release-build-output/materialize.sh +++ b/release-build-output/materialize.sh @@ -12,7 +12,7 @@ require_nonempty() { fi } -require_nonempty "RELEASE_UNIT" "${RELEASE_UNIT:-}" +require_nonempty "RELEASE_CATALOG_KEY" "${RELEASE_CATALOG_KEY:-}" require_nonempty "RELEASE_OUTPUT_DIRECTORY" "${RELEASE_OUTPUT_DIRECTORY:-}" require_nonempty "RELEASE_MANIFEST_NAME" "${RELEASE_MANIFEST_NAME:-}" require_nonempty "RELEASE_METADATA_NAME" "${RELEASE_METADATA_NAME:-}" @@ -248,7 +248,7 @@ write_generated_provenance() { predicate: { buildDefinition: { buildType: "https://rapids.ai/release-platform/build-output/v1", - externalParameters: {release_unit: env.RELEASE_UNIT, package: $package}, + externalParameters: {release_catalog_key: env.RELEASE_CATALOG_KEY, package: $package}, resolvedDependencies: [{ uri: ("git+https://github.com/" + $repository + "@" + $source_sha), digest: {gitCommit: $source_sha} @@ -310,12 +310,12 @@ while IFS= read -r descriptor; do write_generated_provenance "${primary_path}" "${package}" "${provenance_path}" fi artifact="$(jq -cn \ - --arg unit_id "${RELEASE_UNIT}" \ + --arg release_catalog_key "${RELEASE_CATALOG_KEY}" \ --arg path "${primary_path}" \ --arg sbom "${sbom_path}" \ --arg provenance "${provenance_path}" \ --argjson package "${package}" \ - '{unit_id: $unit_id, path: $path, sbom: $sbom, provenance: $provenance, package: $package}')" + '{release_catalog_key: $release_catalog_key, path: $path, sbom: $sbom, provenance: $provenance, package: $package}')" if [[ -n "${signature_pattern}" ]]; then supplied_signature_path="$(resolve_one_file signature "${signature_pattern}")" signature_path="$(copy_supplied_evidence "${primary_path}" "${supplied_signature_path}" "signature")" @@ -332,8 +332,8 @@ while IFS= read -r descriptor; do mv "${temporary_manifest}.next" "${temporary_manifest}" done < <(jq -c '.[]' <<<"${RELEASE_ARTIFACTS}") -if ! jq -e '.artifacts as $items | ($items | map([.unit_id, .path] | join("\u0000")) | unique | length) == ($items | length)' "${temporary_manifest}" >/dev/null; then - echo "release-artifacts contains duplicate unit/path entries" >&2 +if ! jq -e '.artifacts as $items | ($items | map([.release_catalog_key, .path] | join("\u0000")) | unique | length) == ($items | length)' "${temporary_manifest}" >/dev/null; then + echo "release-artifacts contains duplicate release_catalog_key/path entries" >&2 exit 1 fi @@ -345,13 +345,13 @@ jq -n -S \ --arg run_attempt "${GITHUB_RUN_ATTEMPT:-}" \ --arg run_id "${GITHUB_RUN_ID:-}" \ --arg sha "${source_sha}" \ - --arg unit_id "${RELEASE_UNIT}" \ + --arg release_catalog_key "${RELEASE_CATALOG_KEY}" \ --arg workflow_ref "${GITHUB_WORKFLOW_REF:-}" \ --argjson artifact_metadata "${artifact_metadata}" \ '{ schema_version: 1, producer: "shared-workflows", - release_unit: $unit_id, + release_catalog_key: $release_catalog_key, source_artifact: $artifact_name, build_output_manifest: $manifest_name, build_environment: { diff --git a/release-build-output/validate-config.sh b/release-build-output/validate-config.sh index 458f4e5e..bf7aa946 100755 --- a/release-build-output/validate-config.sh +++ b/release-build-output/validate-config.sh @@ -38,15 +38,15 @@ validation_errors="$(jq -r ' if type != "object" then ["release-build-output must be a JSON object"] else - (keys - ["artifact_type", "component_id", "output_directory", "package", "package_file", "artifacts"]) as $unknown + (keys - ["artifact_type", "release_catalog_key", "output_directory", "package", "package_file", "artifacts"]) as $unknown | [ if ($unknown | length) > 0 then "unknown field(s): " + ($unknown | join(", ")) else empty end, if (.artifact_type == "conda" or .artifact_type == "custom" or .artifact_type == "wheel") then empty else "artifact_type must be one of: conda, custom, wheel" end, - if (.component_id | single_line_string) then empty - else "component_id must be a non-empty, single-line string" end, + if (.release_catalog_key | single_line_string) then empty + else "release_catalog_key must be a non-empty, single-line string" end, if (has("output_directory") | not) or (.output_directory | single_line_string) then empty else "output_directory must be a non-empty, single-line string when supplied" end ] @@ -111,7 +111,7 @@ fi { printf 'artifact_type=%s\n' "$(jq -r '.artifact_type' <<<"${compact_config}")" - printf 'component_id=%s\n' "$(jq -r '.component_id' <<<"${compact_config}")" + printf 'release_catalog_key=%s\n' "$(jq -r '.release_catalog_key' <<<"${compact_config}")" printf 'output_directory=%s\n' "$(jq -r '.output_directory // "."' <<<"${compact_config}")" printf 'package=%s\n' "$(jq -c '.package // empty' <<<"${compact_config}")" printf 'package_file=%s\n' "$(jq -r '.package_file // empty' <<<"${compact_config}")" diff --git a/tests/release-build-output-config/inline-package.json b/tests/release-build-output-config/inline-package.json index 3443ea3b..80d6b6d3 100644 --- a/tests/release-build-output-config/inline-package.json +++ b/tests/release-build-output-config/inline-package.json @@ -1,6 +1,6 @@ { "artifact_type": "custom", - "component_id": "archive:smoke", + "release_catalog_key": "archive:smoke", "package": { "ecosystem": "archive", "name": "smoke", diff --git a/tests/release-build-output-config/package-file.json b/tests/release-build-output-config/package-file.json index de25059e..67776f18 100644 --- a/tests/release-build-output-config/package-file.json +++ b/tests/release-build-output-config/package-file.json @@ -1,6 +1,6 @@ { "artifact_type": "custom", - "component_id": "maven:cuvs-java", + "release_catalog_key": "maven:cuvs-java", "output_directory": "java/cuvs-java/target", "package_file": "cuvs-java.release-package.json", "artifacts": [ diff --git a/tests/release_build_output_config_test.sh b/tests/release_build_output_config_test.sh index d77375e8..2f451c4b 100755 --- a/tests/release_build_output_config_test.sh +++ b/tests/release_build_output_config_test.sh @@ -30,7 +30,7 @@ valid_output="${temporary_directory}/valid.output" RELEASE_BUILD_OUTPUT_CONFIG="$(<"${repository_root}/tests/release-build-output-config/package-file.json")" \ GITHUB_OUTPUT="${valid_output}" "${validator}" -grep -Fx 'component_id=maven:cuvs-java' "${valid_output}" +grep -Fx 'release_catalog_key=maven:cuvs-java' "${valid_output}" grep -Fx 'artifact_type=custom' "${valid_output}" grep -Fx 'output_directory=java/cuvs-java/target' "${valid_output}" grep -Fx 'package=' "${valid_output}" @@ -47,7 +47,7 @@ grep -Fx 'package_file=' "${inline_output}" assert_invalid \ malformed-json \ - '{"component_id":' \ + '{"release_catalog_key":' \ 'release-build-output must be valid JSON' assert_invalid \ @@ -58,50 +58,50 @@ assert_invalid \ assert_invalid \ missing-fields \ '{}' \ - 'component_id must be a non-empty, single-line string' + 'release_catalog_key must be a non-empty, single-line string' grep -F 'artifact_type must be one of: conda, custom, wheel' "${temporary_directory}/missing-fields.error" >/dev/null assert_invalid \ custom-missing-fields \ - '{"artifact_type":"custom","component_id":"archive:smoke"}' \ + '{"artifact_type":"custom","release_catalog_key":"archive:smoke"}' \ 'exactly one of package or package_file is required for custom artifacts' grep -F 'artifacts must be a non-empty array for custom artifacts' "${temporary_directory}/custom-missing-fields.error" >/dev/null assert_invalid \ unknown-field \ - '{"artifact_type":"custom","component_id":"archive:smoke","package_file":"package.json","artifacts":[{"path":"smoke.tar.gz"}],"unit":"archive:smoke"}' \ - 'unknown field(s): unit' + '{"artifact_type":"custom","release_catalog_key":"archive:smoke","package_file":"package.json","artifacts":[{"path":"smoke.tar.gz"}],"component_id":"archive:smoke"}' \ + 'unknown field(s): component_id' assert_invalid \ conflicting-package-source \ - '{"artifact_type":"custom","component_id":"archive:smoke","package":{"ecosystem":"archive","name":"smoke"},"package_file":"package.json","artifacts":[{"path":"smoke.tar.gz"}]}' \ + '{"artifact_type":"custom","release_catalog_key":"archive:smoke","package":{"ecosystem":"archive","name":"smoke"},"package_file":"package.json","artifacts":[{"path":"smoke.tar.gz"}]}' \ 'exactly one of package or package_file is required for custom artifacts' assert_invalid \ malformed-artifact \ - '{"artifact_type":"custom","component_id":"archive:smoke","package_file":"package.json","artifacts":[{"file":"smoke.tar.gz","sbom":false}]}' \ + '{"artifact_type":"custom","release_catalog_key":"archive:smoke","package_file":"package.json","artifacts":[{"file":"smoke.tar.gz","sbom":false}]}' \ 'artifacts[0] has unknown field(s): file' grep -F 'artifacts[0].path must be a non-empty, single-line string' "${temporary_directory}/malformed-artifact.error" >/dev/null grep -F 'artifacts[0].sbom must be a non-empty, single-line string' "${temporary_directory}/malformed-artifact.error" >/dev/null assert_invalid \ standard-missing-output-directory \ - '{"artifact_type":"conda","component_id":"conda:smoke"}' \ + '{"artifact_type":"conda","release_catalog_key":"conda:smoke"}' \ 'output_directory is required for conda and wheel artifacts' assert_invalid \ standard-custom-fields \ - '{"artifact_type":"wheel","component_id":"wheel:smoke","output_directory":"dist","artifacts":[{"path":"smoke.whl"}]}' \ + '{"artifact_type":"wheel","release_catalog_key":"wheel:smoke","output_directory":"dist","artifacts":[{"path":"smoke.whl"}]}' \ 'artifacts, package, and package_file are only valid when artifact_type is custom' standard_output="${temporary_directory}/standard.output" RELEASE_BUILD_OUTPUT_CONFIG='{ "artifact_type": "wheel", - "component_id": "wheel:kvikio", + "release_catalog_key": "wheel:kvikio", "output_directory": "dist" }' GITHUB_OUTPUT="${standard_output}" "${validator}" grep -Fx 'artifact_type=wheel' "${standard_output}" -grep -Fx 'component_id=wheel:kvikio' "${standard_output}" +grep -Fx 'release_catalog_key=wheel:kvikio' "${standard_output}" grep -Fx 'output_directory=dist' "${standard_output}" grep -Fx 'artifacts=' "${standard_output}" diff --git a/tests/release_build_output_test.sh b/tests/release_build_output_test.sh index 2a5dc6e1..bdbf8320 100755 --- a/tests/release_build_output_test.sh +++ b/tests/release_build_output_test.sh @@ -32,10 +32,10 @@ RELEASE_PACKAGE='' RELEASE_PACKAGE_FILE="cuvs-java-package.json" RELEASE_SOURCE_ARTIFACT_NAME="cuvs-java-cuda12.9.1" RELEASE_SOURCE_SHA="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" -RELEASE_UNIT="maven:cuvs-java" +RELEASE_CATALOG_KEY="maven:cuvs-java" export GITHUB_REPOSITORY GITHUB_RUN_ATTEMPT GITHUB_RUN_ID GITHUB_SHA GITHUB_WORKFLOW_REF export RELEASE_ARTIFACTS RELEASE_MANIFEST_NAME RELEASE_METADATA_NAME RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE RELEASE_PACKAGE_FILE -export RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_UNIT +export RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY "${repository_root}/release-build-output/materialize.sh" @@ -46,14 +46,14 @@ jq -e ' .schema_version == 1 and .producer == "release-platform" and (.artifacts | length == 1) - and .artifacts[0].unit_id == "maven:cuvs-java" + and .artifacts[0].release_catalog_key == "maven:cuvs-java" and .artifacts[0].path == "cuvs-java-26.08.0.jar" and .artifacts[0].package.name == "ai.rapids:cuvs-java" ' "${manifest_path}" >/dev/null jq -e ' .schema_version == 1 and .producer == "shared-workflows" - and .release_unit == "maven:cuvs-java" + and .release_catalog_key == "maven:cuvs-java" and .source_artifact == "cuvs-java-cuda12.9.1" and .build_output_manifest == "release-build-output.json" and .build_environment.repository == "rapidsai/cuvs" @@ -91,8 +91,8 @@ RELEASE_PACKAGE="$(jq -cn '{ecosystem: "conda", name: "kvikio"}')" RELEASE_PACKAGE_FILE='' RELEASE_SOURCE_ARTIFACT_NAME="kvikio_conda_python_kvikio_x86_64_abi3_cu12" RELEASE_SOURCE_SHA="bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" -RELEASE_UNIT="conda:kvikio" -export RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE RELEASE_PACKAGE_FILE RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_UNIT +RELEASE_CATALOG_KEY="conda:kvikio" +export RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE RELEASE_PACKAGE_FILE RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY "${repository_root}/release-build-output/materialize.sh" @@ -101,7 +101,7 @@ generated_metadata_path="${generated_directory}/release-build-metadata.json" generated_sbom_path="$(jq -r '.artifacts[0].sbom' "${generated_manifest_path}")" generated_provenance_path="$(jq -r '.artifacts[0].provenance' "${generated_manifest_path}")" jq -e ' - .artifacts[0].unit_id == "conda:kvikio" + .artifacts[0].release_catalog_key == "conda:kvikio" and .artifacts[0].path == "linux-64/kvikio-26.08.00a32-cuda12_260714_2f567060.conda" and .artifacts[0].package == {ecosystem: "conda", name: "kvikio", version: "26.08.00a32"} ' "${generated_manifest_path}" >/dev/null @@ -112,11 +112,11 @@ jq -e ' ' "${generated_directory}/${generated_sbom_path}" >/dev/null jq -e ' .predicateType == "https://slsa.dev/provenance/v1" - and .predicate.buildDefinition.externalParameters.release_unit == "conda:kvikio" + and .predicate.buildDefinition.externalParameters.release_catalog_key == "conda:kvikio" and .predicate.buildDefinition.resolvedDependencies[0].digest.gitCommit == "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" ' "${generated_directory}/${generated_provenance_path}" >/dev/null jq -e ' - .release_unit == "conda:kvikio" + .release_catalog_key == "conda:kvikio" and .source_artifact == "kvikio_conda_python_kvikio_x86_64_abi3_cu12" and .build_environment.sha == "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" and .metadata.artifacts == [{path: "linux-64/kvikio-26.08.00a32-cuda12_260714_2f567060.conda", sbom_kind: "generated-identity"}] @@ -131,13 +131,13 @@ RELEASE_OUTPUT_DIRECTORY="${wheel_directory}" RELEASE_PACKAGE="$(jq -cn '{ecosystem: "wheel", name: "libkvikio-cu12"}')" RELEASE_SOURCE_ARTIFACT_NAME="kvikio_wheel_cpp_libkvikio_x86_64_cu12" RELEASE_SOURCE_SHA="cccccccccccccccccccccccccccccccccccccccc" -RELEASE_UNIT="wheel:kvikio" -export RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_UNIT +RELEASE_CATALOG_KEY="wheel:kvikio" +export RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY "${repository_root}/release-build-output/materialize.sh" jq -e ' - .artifacts[0].unit_id == "wheel:kvikio" + .artifacts[0].release_catalog_key == "wheel:kvikio" and .artifacts[0].path == "libkvikio_cu12-26.8.0a32-py3-none-manylinux_2_28_x86_64.whl" and .artifacts[0].package == {ecosystem: "wheel", name: "libkvikio-cu12", version: "26.8.0a32"} ' "${wheel_directory}/release-build-output.json" >/dev/null @@ -155,8 +155,8 @@ RELEASE_PACKAGE="$(jq -cn '{ecosystem: "archive", name: "bundle"}')" RELEASE_PACKAGE_FILE='' RELEASE_SOURCE_ARTIFACT_NAME="bundle-archive" RELEASE_SOURCE_SHA="dddddddddddddddddddddddddddddddddddddddd" -RELEASE_UNIT="archive:bundle" -export RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE RELEASE_PACKAGE_FILE RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_UNIT +RELEASE_CATALOG_KEY="archive:bundle" +export RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE RELEASE_PACKAGE_FILE RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY if "${repository_root}/release-build-output/materialize.sh" 2>"${temporary_directory}/missing-version-error"; then echo "materialize.sh unexpectedly accepted a custom artifact without a package version" >&2 From 45e84f709fde5e3f1b5960ed0510599c5f388333 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Mon, 17 Aug 2026 21:40:07 -0500 Subject: [PATCH 05/16] Rename release catalog feature --- .github/workflows/pr.yml | 30 +++++++++---------- .pre-commit-config.yaml | 18 +++++------ README.md | 8 ++--- .../action.yml | 30 +++++++++---------- .../README.md | 18 +++++------ .../config.schema.json | 6 ++-- .../describe-conda.sh | 0 .../describe-wheels.sh | 0 .../materialize.sh | 8 ++--- .../prepare.sh | 0 .../validate-config.sh | 10 +++---- .../inline-package.json | 0 .../package-file.json | 0 ...test.sh => release_catalog_config_test.sh} | 14 ++++----- ...sh => release_catalog_descriptors_test.sh} | 4 +-- ...est.sh => release_catalog_prepare_test.sh} | 6 ++-- ...output_test.sh => release_catalog_test.sh} | 28 ++++++++--------- 17 files changed, 90 insertions(+), 90 deletions(-) rename {release-build-output-dispatch => release-catalog-dispatch}/action.yml (81%) rename {release-build-output => release-catalog}/README.md (87%) rename {release-build-output => release-catalog}/config.schema.json (95%) rename {release-build-output => release-catalog}/describe-conda.sh (100%) rename {release-build-output => release-catalog}/describe-wheels.sh (100%) rename {release-build-output => release-catalog}/materialize.sh (98%) rename {release-build-output => release-catalog}/prepare.sh (100%) rename {release-build-output => release-catalog}/validate-config.sh (93%) rename tests/{release-build-output-config => release-catalog-config}/inline-package.json (100%) rename tests/{release-build-output-config => release-catalog-config}/package-file.json (100%) rename tests/{release_build_output_config_test.sh => release_catalog_config_test.sh} (88%) rename tests/{release_build_output_descriptors_test.sh => release_catalog_descriptors_test.sh} (92%) rename tests/{release_build_output_prepare_test.sh => release_catalog_prepare_test.sh} (90%) rename tests/{release_build_output_test.sh => release_catalog_test.sh} (89%) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 8cbc2db5..54a32531 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -20,18 +20,18 @@ jobs: fetch-depth: 1 persist-credentials: false - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1 - - name: Test release build-output materialization + - name: Test release catalog materialization run: | - ./tests/release_build_output_config_test.sh - ./tests/release_build_output_descriptors_test.sh - ./tests/release_build_output_prepare_test.sh - ./tests/release_build_output_test.sh - - name: Prepare release build-output dispatch smoke test + ./tests/release_catalog_config_test.sh + ./tests/release_catalog_descriptors_test.sh + ./tests/release_catalog_prepare_test.sh + ./tests/release_catalog_test.sh + - name: Prepare release catalog dispatch smoke test run: | - mkdir -p release-build-output-smoke - printf '%s\n' smoke >release-build-output-smoke/package.tar.gz - - name: Run release build-output dispatch smoke test - uses: ./release-build-output-dispatch + mkdir -p release-catalog-smoke + printf '%s\n' smoke >release-catalog-smoke/package.tar.gz + - name: Run release catalog dispatch smoke test + uses: ./release-catalog-dispatch env: SHARED_ACTIONS_REPO: ${{ github.event.pull_request.head.repo.full_name }} SHARED_ACTIONS_REF: ${{ github.event.pull_request.head.sha }} @@ -40,18 +40,18 @@ jobs: { "artifact_type": "custom", "release_catalog_key": "archive:smoke", - "output_directory": "release-build-output-smoke", + "output_directory": "release-catalog-smoke", "package": {"ecosystem": "archive", "name": "smoke", "version": "1.0"}, "artifacts": [{"path": "package.tar.gz"}] } - source-artifact-name: release-build-output-dispatch-smoke + source-artifact-name: release-catalog-dispatch-smoke source-sha: ${{ github.event.pull_request.head.sha }} - - name: Verify release build-output dispatch smoke test + - name: Verify release catalog dispatch smoke test run: | jq -e ' .artifacts[0].release_catalog_key == "archive:smoke" and .artifacts[0].path == "package.tar.gz" - ' release-build-output-smoke/release-build-output.json >/dev/null + ' release-catalog-smoke/release-catalog-entries.json >/dev/null jq -e ' .metadata.artifacts == [{path: "package.tar.gz", sbom_kind: "generated-identity"}] - ' release-build-output-smoke/release-build-metadata.json >/dev/null + ' release-catalog-smoke/release-catalog-metadata.json >/dev/null diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 92f9986c..6cf849f3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,10 +6,10 @@ ci: # These hooks require tools unavailable in pre-commit.ci: # # * actionlint-docker requires Docker - # * release-build-output-config requires jq + # * release-catalog-config requires jq # # Both run in other CI jobs with controlled runtime dependencies. - skip: [actionlint-docker, release-build-output-config] + skip: [actionlint-docker, release-catalog-config] repos: - repo: https://github.com/pre-commit/pre-commit-hooks @@ -63,14 +63,14 @@ repos: rev: 0.37.2 hooks: - id: check-jsonschema - name: validate release build-output config schema fixtures - args: [--schemafile, release-build-output/config.schema.json] - files: ^tests/release-build-output-config/.*\.json$ + name: validate release catalog config schema fixtures + args: [--schemafile, release-catalog/config.schema.json] + files: ^tests/release-catalog-config/.*\.json$ - repo: local hooks: - - id: release-build-output-config - name: validate release build-output configuration - entry: ./tests/release_build_output_config_test.sh + - id: release-catalog-config + name: validate release catalog configuration + entry: ./tests/release_catalog_config_test.sh language: system pass_filenames: false - files: ^(release-build-output/(config\.schema\.json|validate-config\.sh)|tests/release_build_output_config_test\.sh)$ + files: ^(release-catalog/(config\.schema\.json|validate-config\.sh)|tests/release_catalog_config_test\.sh)$ diff --git a/README.md b/README.md index 697fd7f8..88248366 100644 --- a/README.md +++ b/README.md @@ -9,13 +9,13 @@ A dispatch action is one that: * clones the shared-actions repository (repo/ref changeable using env vars) * runs (dispatches to) another action within the clone, using a relative path -## Release build-output companions +## Release catalog companions -`release-build-output-dispatch` validates a producer's local build artifact +`release-catalog-dispatch` validates a producer's local build artifact directory and uploads a companion artifact named -`release-build-output-`. +`release-catalog-`. -See the [`release-build-output` documentation](release-build-output/README.md) +See the [release catalog documentation](release-catalog/README.md) for configuration, examples, source-revision handling, companion contents, and evidence semantics. diff --git a/release-build-output-dispatch/action.yml b/release-catalog-dispatch/action.yml similarity index 81% rename from release-build-output-dispatch/action.yml rename to release-catalog-dispatch/action.yml index 9af3d5de..b06951a5 100644 --- a/release-build-output-dispatch/action.yml +++ b/release-catalog-dispatch/action.yml @@ -1,11 +1,11 @@ -name: Dispatch release build output -description: Create a release build-output companion from the selected shared-actions revision. +name: Dispatch release catalog +description: Create a release catalog companion from the selected shared-actions revision. inputs: config: description: >- JSON string with configuration for this action. Schema and field documentation: - https://github.com/rapidsai/shared-actions/blob/main/release-build-output/config.schema.json + https://github.com/rapidsai/shared-actions/blob/main/release-catalog/config.schema.json required: true source-artifact-name: description: Name of the GitHub Actions artifact bundle containing this output. @@ -29,7 +29,7 @@ runs: using: composite steps: - id: implementation - name: Resolve release build-output implementation + name: Resolve release catalog implementation shell: bash env: ACTION_PATH: ${{ github.action_path }} @@ -41,18 +41,18 @@ runs: if [[ ! -d "${action_path}" && "${action_path}" == "${host_root}/"* ]]; then action_path="${container_root}/${action_path#"${host_root}/"}" fi - implementation_path="$(dirname "${action_path}")/release-build-output" + implementation_path="$(dirname "${action_path}")/release-catalog" if [[ ! -d "${implementation_path}" ]]; then - echo "release-build-output implementation not found at ${implementation_path}" >&2 + echo "release-catalog implementation not found at ${implementation_path}" >&2 exit 1 fi echo "path=${implementation_path}" >>"${GITHUB_OUTPUT}" - id: configuration - name: Validate release build-output configuration + name: Validate release catalog configuration shell: bash env: IMPLEMENTATION_PATH: ${{ steps.implementation.outputs.path }} - RELEASE_BUILD_OUTPUT_CONFIG: ${{ inputs.config }} + RELEASE_CATALOG_CONFIG: ${{ inputs.config }} run: "${IMPLEMENTATION_PATH}/validate-config.sh" - id: prepare name: Describe release artifacts @@ -66,13 +66,13 @@ runs: RELEASE_PACKAGE_FILE: ${{ steps.configuration.outputs.package_file }} run: "${IMPLEMENTATION_PATH}/prepare.sh" - id: materialize - name: Materialize release build-output records + name: Materialize release catalog records shell: bash env: IMPLEMENTATION_PATH: ${{ steps.implementation.outputs.path }} RELEASE_ARTIFACTS: ${{ steps.prepare.outputs.artifacts }} - RELEASE_MANIFEST_NAME: release-build-output.json - RELEASE_METADATA_NAME: release-build-metadata.json + RELEASE_MANIFEST_NAME: release-catalog-entries.json + RELEASE_METADATA_NAME: release-catalog-metadata.json RELEASE_OUTPUT_DIRECTORY: ${{ steps.configuration.outputs.output_directory }} RELEASE_PACKAGE: ${{ steps.prepare.outputs.package }} RELEASE_PACKAGE_FILE: ${{ steps.configuration.outputs.package_file }} @@ -85,13 +85,13 @@ runs: shell: bash env: SOURCE_ARTIFACT_NAME: ${{ inputs.source-artifact-name }} - run: echo "name=release-build-output-${SOURCE_ARTIFACT_NAME}" >>"${GITHUB_OUTPUT}" - - name: Upload release build-output companion + run: echo "name=release-catalog-${SOURCE_ARTIFACT_NAME}" >>"${GITHUB_OUTPUT}" + - name: Upload release catalog companion uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: if-no-files-found: error name: ${{ steps.companion-name.outputs.name }} path: | - ${{ steps.configuration.outputs.output_directory }}/release-build-output.json - ${{ steps.configuration.outputs.output_directory }}/release-build-metadata.json + ${{ steps.configuration.outputs.output_directory }}/release-catalog-entries.json + ${{ steps.configuration.outputs.output_directory }}/release-catalog-metadata.json ${{ steps.configuration.outputs.output_directory }}/release-evidence/** diff --git a/release-build-output/README.md b/release-catalog/README.md similarity index 87% rename from release-build-output/README.md rename to release-catalog/README.md index 0899b292..22cf77e2 100644 --- a/release-build-output/README.md +++ b/release-catalog/README.md @@ -1,8 +1,8 @@ -# Release build-output companions +# Release catalog companions -`release-build-output-dispatch` records the exact files produced by a package +`release-catalog-dispatch` records the exact files produced by a package build and uploads a companion GitHub Actions artifact named -`release-build-output-`. Release tooling uses the +`release-catalog-`. Release tooling uses the companion to associate primary artifacts with their build identity and available evidence without reconstructing the build later. @@ -10,8 +10,8 @@ The companion contains: ```text . -├── release-build-metadata.json -├── release-build-output.json +├── release-catalog-metadata.json +├── release-catalog-entries.json └── release-evidence ├── .provenance.json └── .spdx.json @@ -58,8 +58,8 @@ the built source. ## Standard package example ```yaml -- name: Create wheel release build-output companion - uses: rapidsai/shared-actions/release-build-output-dispatch@main +- name: Create wheel release catalog companion + uses: rapidsai/shared-actions/release-catalog-dispatch@main with: config: >- { @@ -74,8 +74,8 @@ the built source. ## Custom package example ```yaml -- name: Create custom release build-output companion - uses: rapidsai/shared-actions/release-build-output-dispatch@main +- name: Create custom release catalog companion + uses: rapidsai/shared-actions/release-catalog-dispatch@main with: config: >- { diff --git a/release-build-output/config.schema.json b/release-catalog/config.schema.json similarity index 95% rename from release-build-output/config.schema.json rename to release-catalog/config.schema.json index 9efd2ba6..6c6b77a3 100644 --- a/release-build-output/config.schema.json +++ b/release-catalog/config.schema.json @@ -1,8 +1,8 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://rapids.ai/schemas/release-build-output-config-v1.json", - "title": "Release build-output configuration", - "description": "Producer configuration for one release build-output companion.", + "$id": "https://rapids.ai/schemas/release-catalog-config-v1.json", + "title": "Release catalog configuration", + "description": "Producer configuration for one release catalog companion.", "type": "object", "additionalProperties": false, "required": [ diff --git a/release-build-output/describe-conda.sh b/release-catalog/describe-conda.sh similarity index 100% rename from release-build-output/describe-conda.sh rename to release-catalog/describe-conda.sh diff --git a/release-build-output/describe-wheels.sh b/release-catalog/describe-wheels.sh similarity index 100% rename from release-build-output/describe-wheels.sh rename to release-catalog/describe-wheels.sh diff --git a/release-build-output/materialize.sh b/release-catalog/materialize.sh similarity index 98% rename from release-build-output/materialize.sh rename to release-catalog/materialize.sh index a500e744..80b2b97a 100755 --- a/release-build-output/materialize.sh +++ b/release-catalog/materialize.sh @@ -92,7 +92,7 @@ fi manifest_path="${output_directory}/${RELEASE_MANIFEST_NAME}" metadata_path="${output_directory}/${RELEASE_METADATA_NAME}" -temporary_manifest="$(mktemp "${output_directory}/.release-build-output.XXXXXX")" +temporary_manifest="$(mktemp "${output_directory}/.release-catalog.XXXXXX")" trap 'rm -f "${temporary_manifest}"' EXIT printf '%s\n' '{"schema_version":1,"producer":"release-platform","artifacts":[]}' >"${temporary_manifest}" @@ -204,7 +204,7 @@ write_generated_sbom() { name: ("RAPIDS release artifact " + $artifact_path), documentNamespace: ("https://rapids.ai/release-platform/spdx/" + $artifact_digest), creationInfo: { - creators: ["Tool: rapidsai/shared-workflows release-build-output"], + creators: ["Tool: rapidsai/shared-workflows release catalog"], created: (now | strftime("%Y-%m-%dT%H:%M:%SZ")) }, documentDescribes: ["SPDXRef-Artifact"], @@ -247,7 +247,7 @@ write_generated_provenance() { predicateType: "https://slsa.dev/provenance/v1", predicate: { buildDefinition: { - buildType: "https://rapids.ai/release-platform/build-output/v1", + buildType: "https://rapids.ai/release-platform/catalog-record/v1", externalParameters: {release_catalog_key: env.RELEASE_CATALOG_KEY, package: $package}, resolvedDependencies: [{ uri: ("git+https://github.com/" + $repository + "@" + $source_sha), @@ -353,7 +353,7 @@ jq -n -S \ producer: "shared-workflows", release_catalog_key: $release_catalog_key, source_artifact: $artifact_name, - build_output_manifest: $manifest_name, + catalog_record_manifest: $manifest_name, build_environment: { repository: $repository, sha: $sha, diff --git a/release-build-output/prepare.sh b/release-catalog/prepare.sh similarity index 100% rename from release-build-output/prepare.sh rename to release-catalog/prepare.sh diff --git a/release-build-output/validate-config.sh b/release-catalog/validate-config.sh similarity index 93% rename from release-build-output/validate-config.sh rename to release-catalog/validate-config.sh index bf7aa946..12916159 100755 --- a/release-build-output/validate-config.sh +++ b/release-catalog/validate-config.sh @@ -5,18 +5,18 @@ set -euo pipefail emit_error() { local message="$1" - printf '::error title=Invalid release-build-output configuration::%s\n' "${message}" >&2 + printf '::error title=Invalid release catalog configuration::%s\n' "${message}" >&2 } -config="${RELEASE_BUILD_OUTPUT_CONFIG:-}" +config="${RELEASE_CATALOG_CONFIG:-}" if [[ -z "${config}" ]]; then - emit_error "release-build-output must be a non-empty JSON object" + emit_error "release catalog configuration must be a non-empty JSON object" exit 1 fi if ! compact_config="$(jq -ce . <<<"${config}" 2>/dev/null)"; then parse_error="$(jq -ce . <<<"${config}" 2>&1 || true)" - emit_error "release-build-output must be valid JSON: ${parse_error}" + emit_error "release catalog configuration must be valid JSON: ${parse_error}" exit 1 fi @@ -36,7 +36,7 @@ validation_errors="$(jq -r ' and (keys - ["ecosystem", "name", "version", "build", "platform"] | length == 0) and (to_entries | map(.value | single_line_string) | all); if type != "object" then - ["release-build-output must be a JSON object"] + ["release catalog configuration must be a JSON object"] else (keys - ["artifact_type", "release_catalog_key", "output_directory", "package", "package_file", "artifacts"]) as $unknown | [ diff --git a/tests/release-build-output-config/inline-package.json b/tests/release-catalog-config/inline-package.json similarity index 100% rename from tests/release-build-output-config/inline-package.json rename to tests/release-catalog-config/inline-package.json diff --git a/tests/release-build-output-config/package-file.json b/tests/release-catalog-config/package-file.json similarity index 100% rename from tests/release-build-output-config/package-file.json rename to tests/release-catalog-config/package-file.json diff --git a/tests/release_build_output_config_test.sh b/tests/release_catalog_config_test.sh similarity index 88% rename from tests/release_build_output_config_test.sh rename to tests/release_catalog_config_test.sh index 2f451c4b..36d92c7c 100755 --- a/tests/release_build_output_config_test.sh +++ b/tests/release_catalog_config_test.sh @@ -4,7 +4,7 @@ set -euo pipefail repository_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -validator="${repository_root}/release-build-output/validate-config.sh" +validator="${repository_root}/release-catalog/validate-config.sh" temporary_directory="$(mktemp -d)" trap 'rm -rf "${temporary_directory}"' EXIT @@ -14,7 +14,7 @@ assert_invalid() { local expected="$3" local error_file="${temporary_directory}/${name}.error" - if RELEASE_BUILD_OUTPUT_CONFIG="${config}" GITHUB_OUTPUT="${temporary_directory}/${name}.output" \ + if RELEASE_CATALOG_CONFIG="${config}" GITHUB_OUTPUT="${temporary_directory}/${name}.output" \ "${validator}" 2>"${error_file}"; then echo "validate-config.sh unexpectedly accepted ${name}" >&2 exit 1 @@ -27,7 +27,7 @@ assert_invalid() { } valid_output="${temporary_directory}/valid.output" -RELEASE_BUILD_OUTPUT_CONFIG="$(<"${repository_root}/tests/release-build-output-config/package-file.json")" \ +RELEASE_CATALOG_CONFIG="$(<"${repository_root}/tests/release-catalog-config/package-file.json")" \ GITHUB_OUTPUT="${valid_output}" "${validator}" grep -Fx 'release_catalog_key=maven:cuvs-java' "${valid_output}" @@ -38,7 +38,7 @@ grep -Fx 'package_file=cuvs-java.release-package.json' "${valid_output}" grep -Fx 'artifacts=[{"path":"cuvs-java-*-x86_64-cuda*.jar","sbom":"cuvs-java.spdx.json"}]' "${valid_output}" inline_output="${temporary_directory}/inline.output" -RELEASE_BUILD_OUTPUT_CONFIG="$(<"${repository_root}/tests/release-build-output-config/inline-package.json")" \ +RELEASE_CATALOG_CONFIG="$(<"${repository_root}/tests/release-catalog-config/inline-package.json")" \ GITHUB_OUTPUT="${inline_output}" "${validator}" grep -Fx 'output_directory=.' "${inline_output}" @@ -48,12 +48,12 @@ grep -Fx 'package_file=' "${inline_output}" assert_invalid \ malformed-json \ '{"release_catalog_key":' \ - 'release-build-output must be valid JSON' + 'release catalog configuration must be valid JSON' assert_invalid \ not-an-object \ '[]' \ - 'release-build-output must be a JSON object' + 'release catalog configuration must be a JSON object' assert_invalid \ missing-fields \ @@ -95,7 +95,7 @@ assert_invalid \ 'artifacts, package, and package_file are only valid when artifact_type is custom' standard_output="${temporary_directory}/standard.output" -RELEASE_BUILD_OUTPUT_CONFIG='{ +RELEASE_CATALOG_CONFIG='{ "artifact_type": "wheel", "release_catalog_key": "wheel:kvikio", "output_directory": "dist" diff --git a/tests/release_build_output_descriptors_test.sh b/tests/release_catalog_descriptors_test.sh similarity index 92% rename from tests/release_build_output_descriptors_test.sh rename to tests/release_catalog_descriptors_test.sh index c6b6c46c..1c608e51 100755 --- a/tests/release_build_output_descriptors_test.sh +++ b/tests/release_catalog_descriptors_test.sh @@ -20,7 +20,7 @@ printf '%s\n' \ zip -qr "${wheel_output_directory}/libkvikio_cu12-26.8.0a32-py3-none-manylinux_2_28_x86_64.whl" . ) -wheel_descriptors="$("${repository_root}/release-build-output/describe-wheels.sh" "${wheel_output_directory}")" +wheel_descriptors="$("${repository_root}/release-catalog/describe-wheels.sh" "${wheel_output_directory}")" jq -e ' . == [{ path: "libkvikio_cu12-26.8.0a32-py3-none-manylinux_2_28_x86_64.whl", @@ -57,7 +57,7 @@ zstd -q -f "${temporary_directory}/info-librmm.tar" -o "${temporary_directory}/i info-librmm.tar.zst ) -conda_descriptors="$("${repository_root}/release-build-output/describe-conda.sh" "${conda_output_directory}")" +conda_descriptors="$("${repository_root}/release-catalog/describe-conda.sh" "${conda_output_directory}")" jq -e ' length == 2 and any(.[]; diff --git a/tests/release_build_output_prepare_test.sh b/tests/release_catalog_prepare_test.sh similarity index 90% rename from tests/release_build_output_prepare_test.sh rename to tests/release_catalog_prepare_test.sh index 3598e1e5..a84eae03 100755 --- a/tests/release_build_output_prepare_test.sh +++ b/tests/release_catalog_prepare_test.sh @@ -26,7 +26,7 @@ RELEASE_PACKAGE='' RELEASE_PACKAGE_FILE='' export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_ARTIFACT_TYPE RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE RELEASE_PACKAGE_FILE -"${repository_root}/release-build-output/prepare.sh" +"${repository_root}/release-catalog/prepare.sh" grep -Fx 'package={"ecosystem":"wheel","name":"bundle"}' "${GITHUB_OUTPUT}" artifacts="$(sed -n 's/^artifacts=//p' "${GITHUB_OUTPUT}")" @@ -44,7 +44,7 @@ RELEASE_PACKAGE='' RELEASE_PACKAGE_FILE=release-package.json export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_ARTIFACT_TYPE RELEASE_PACKAGE RELEASE_PACKAGE_FILE -"${repository_root}/release-build-output/prepare.sh" +"${repository_root}/release-catalog/prepare.sh" grep -Fx 'artifacts=[{"path":"bundle.tar.gz","sbom":"bundle.spdx.json"}]' "${GITHUB_OUTPUT}" grep -Fx 'package=' "${GITHUB_OUTPUT}" @@ -52,7 +52,7 @@ grep -Fx 'package=' "${GITHUB_OUTPUT}" GITHUB_OUTPUT="${temporary_directory}/invalid-output" RELEASE_ARTIFACT_TYPE=archive export GITHUB_OUTPUT RELEASE_ARTIFACT_TYPE -if "${repository_root}/release-build-output/prepare.sh" 2>"${temporary_directory}/invalid-error"; then +if "${repository_root}/release-catalog/prepare.sh" 2>"${temporary_directory}/invalid-error"; then echo "prepare.sh unexpectedly accepted an invalid artifact type" >&2 exit 1 fi diff --git a/tests/release_build_output_test.sh b/tests/release_catalog_test.sh similarity index 89% rename from tests/release_build_output_test.sh rename to tests/release_catalog_test.sh index bdbf8320..51d93836 100755 --- a/tests/release_build_output_test.sh +++ b/tests/release_catalog_test.sh @@ -25,8 +25,8 @@ GITHUB_SHA="0123456789012345678901234567890123456789" GITHUB_WORKFLOW_REF="rapidsai/cuvs/.github/workflows/build.yaml@refs/heads/release/26.08" export GITHUB_OUTPUT RELEASE_ARTIFACTS="$(jq -cn '[{path: "cuvs-java-*.jar", sbom: "cuvs-java-*.spdx.json", provenance: "cuvs-java-*.provenance.jsonl", signature: "cuvs-java-*.jar.asc"}]')" -RELEASE_MANIFEST_NAME="release-build-output.json" -RELEASE_METADATA_NAME="release-build-metadata.json" +RELEASE_MANIFEST_NAME="release-catalog-entries.json" +RELEASE_METADATA_NAME="release-catalog-metadata.json" RELEASE_OUTPUT_DIRECTORY="${bundle_directory}" RELEASE_PACKAGE='' RELEASE_PACKAGE_FILE="cuvs-java-package.json" @@ -37,11 +37,11 @@ export GITHUB_REPOSITORY GITHUB_RUN_ATTEMPT GITHUB_RUN_ID GITHUB_SHA GITHUB_WORK export RELEASE_ARTIFACTS RELEASE_MANIFEST_NAME RELEASE_METADATA_NAME RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE RELEASE_PACKAGE_FILE export RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY -"${repository_root}/release-build-output/materialize.sh" +"${repository_root}/release-catalog/materialize.sh" canonical_bundle_directory="$(realpath "${bundle_directory}")" -manifest_path="${canonical_bundle_directory}/release-build-output.json" -metadata_path="${canonical_bundle_directory}/release-build-metadata.json" +manifest_path="${canonical_bundle_directory}/release-catalog-entries.json" +metadata_path="${canonical_bundle_directory}/release-catalog-metadata.json" jq -e ' .schema_version == 1 and .producer == "release-platform" @@ -55,7 +55,7 @@ jq -e ' and .producer == "shared-workflows" and .release_catalog_key == "maven:cuvs-java" and .source_artifact == "cuvs-java-cuda12.9.1" - and .build_output_manifest == "release-build-output.json" + and .catalog_record_manifest == "release-catalog-entries.json" and .build_environment.repository == "rapidsai/cuvs" and .build_environment.sha == "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" and .metadata.artifacts == [{path: "cuvs-java-26.08.0.jar", sbom_kind: "producer-dependency"}] @@ -79,7 +79,7 @@ cp "${manifest_path}" "${metadata_path}" "${isolated_companion_directory}/" cp -R "${canonical_bundle_directory}/release-evidence" "${isolated_companion_directory}/" while IFS= read -r evidence_path; do test -f "${isolated_companion_directory}/${evidence_path}" -done < <(jq -r '.artifacts[] | .sbom, .provenance, (.signature // empty)' "${isolated_companion_directory}/release-build-output.json") +done < <(jq -r '.artifacts[] | .sbom, .provenance, (.signature // empty)' "${isolated_companion_directory}/release-catalog-entries.json") generated_directory="${temporary_directory}/generated-bundle" mkdir -p "${generated_directory}/linux-64" @@ -94,10 +94,10 @@ RELEASE_SOURCE_SHA="bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" RELEASE_CATALOG_KEY="conda:kvikio" export RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE RELEASE_PACKAGE_FILE RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY -"${repository_root}/release-build-output/materialize.sh" +"${repository_root}/release-catalog/materialize.sh" -generated_manifest_path="${generated_directory}/release-build-output.json" -generated_metadata_path="${generated_directory}/release-build-metadata.json" +generated_manifest_path="${generated_directory}/release-catalog-entries.json" +generated_metadata_path="${generated_directory}/release-catalog-metadata.json" generated_sbom_path="$(jq -r '.artifacts[0].sbom' "${generated_manifest_path}")" generated_provenance_path="$(jq -r '.artifacts[0].provenance' "${generated_manifest_path}")" jq -e ' @@ -134,16 +134,16 @@ RELEASE_SOURCE_SHA="cccccccccccccccccccccccccccccccccccccccc" RELEASE_CATALOG_KEY="wheel:kvikio" export RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY -"${repository_root}/release-build-output/materialize.sh" +"${repository_root}/release-catalog/materialize.sh" jq -e ' .artifacts[0].release_catalog_key == "wheel:kvikio" and .artifacts[0].path == "libkvikio_cu12-26.8.0a32-py3-none-manylinux_2_28_x86_64.whl" and .artifacts[0].package == {ecosystem: "wheel", name: "libkvikio-cu12", version: "26.8.0a32"} -' "${wheel_directory}/release-build-output.json" >/dev/null +' "${wheel_directory}/release-catalog-entries.json" >/dev/null jq -e ' .metadata.artifacts == [{path: "libkvikio_cu12-26.8.0a32-py3-none-manylinux_2_28_x86_64.whl", sbom_kind: "generated-identity"}] -' "${wheel_directory}/release-build-metadata.json" >/dev/null +' "${wheel_directory}/release-catalog-metadata.json" >/dev/null missing_version_directory="${temporary_directory}/missing-version-bundle" mkdir -p "${missing_version_directory}" @@ -158,7 +158,7 @@ RELEASE_SOURCE_SHA="dddddddddddddddddddddddddddddddddddddddd" RELEASE_CATALOG_KEY="archive:bundle" export RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE RELEASE_PACKAGE_FILE RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY -if "${repository_root}/release-build-output/materialize.sh" 2>"${temporary_directory}/missing-version-error"; then +if "${repository_root}/release-catalog/materialize.sh" 2>"${temporary_directory}/missing-version-error"; then echo "materialize.sh unexpectedly accepted a custom artifact without a package version" >&2 exit 1 fi From 11d9aa68d10674af2c6ef149e07b9ff0f0b97599 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Mon, 17 Aug 2026 22:07:44 -0500 Subject: [PATCH 06/16] clarify release catalog key uniqueness/reuse --- release-catalog/README.md | 23 ++++++++++++++++++----- release-catalog/config.schema.json | 2 +- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/release-catalog/README.md b/release-catalog/README.md index 22cf77e2..0a940f18 100644 --- a/release-catalog/README.md +++ b/release-catalog/README.md @@ -30,11 +30,24 @@ The `artifact_type` field determines how the action discovers primary files: or `package_file`. `release_catalog_key` identifies the release-catalog entry that owns these -artifacts. Every file and matrix variant in the same publishable artifact set -uses the same key. Standard RAPIDS Conda and wheel workflows construct it as -`:`, such as `conda:cudf`; custom producers select -an existing catalog key, such as `maven:cuvs-java`. Do not generate a UUID or a -per-build value. +artifacts. Every file from every matrix variant in the same publishable artifact +set uses the same key, and multiple files are aggregated. Standard RAPIDS Conda +and wheel workflows construct it as `:`, such as +`conda:cudf`; custom producers select an existing catalog key, such as +`maven:cuvs-java`. Do not generate a UUID or a per-build value. + +Use a distinct `release_catalog_key` only when the release catalog intentionally gives the +outputs different release policy. Examples include: + +* different versioning +* validation requirements +* dependency ordering +* publication destinations +* promotion strategy. + +Multiple package names from one repository do not by themselves justify separate +keys: for example, `cudf` and `dask-cudf` Conda packages remain part of +`conda:cudf` when they share one release policy. The action validates the complete configuration before inspecting build outputs. It then verifies properties that depend on produced files, including diff --git a/release-catalog/config.schema.json b/release-catalog/config.schema.json index 6c6b77a3..9883c7d7 100644 --- a/release-catalog/config.schema.json +++ b/release-catalog/config.schema.json @@ -20,7 +20,7 @@ ] }, "release_catalog_key": { - "description": "Key of the release-catalog entry that owns these artifacts. Reuse it for every file and matrix variant in the same publishable artifact set. Standard RAPIDS workflows use :, such as conda:cudf; custom producers select an existing catalog key, such as maven:cuvs-java. This is not a generated UUID or per-build value.", + "description": "Release-catalog entry name that owns these artifacts. Default for conda/wheels is :, such as conda:cudf. Custom producers specify their own, such as maven:cuvs-java.", "type": "string", "minLength": 1, "pattern": "^[^\\r\\n]+$" From c4b135aba0b7c5e9fa97f384505efeb17b8c5aa9 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Mon, 17 Aug 2026 23:07:38 -0500 Subject: [PATCH 07/16] merge release catalog metadata into entries --- .github/workflows/pr.yml | 10 ++-- release-catalog-dispatch/action.yml | 17 +++--- release-catalog/README.md | 9 +++- release-catalog/materialize.sh | 51 ++++++------------ tests/release_catalog_test.sh | 83 ++++++++++++----------------- 5 files changed, 68 insertions(+), 102 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 54a32531..2a578473 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -49,9 +49,9 @@ jobs: - name: Verify release catalog dispatch smoke test run: | jq -e ' - .artifacts[0].release_catalog_key == "archive:smoke" - and .artifacts[0].path == "package.tar.gz" + .producer == "shared-workflows" + and .source.artifact == "release-catalog-dispatch-smoke" + and .entries[0].release_catalog_key == "archive:smoke" + and .entries[0].path == "package.tar.gz" + and .entries[0].sbom_kind == "generated-identity" ' release-catalog-smoke/release-catalog-entries.json >/dev/null - jq -e ' - .metadata.artifacts == [{path: "package.tar.gz", sbom_kind: "generated-identity"}] - ' release-catalog-smoke/release-catalog-metadata.json >/dev/null diff --git a/release-catalog-dispatch/action.yml b/release-catalog-dispatch/action.yml index b06951a5..e95c3844 100644 --- a/release-catalog-dispatch/action.yml +++ b/release-catalog-dispatch/action.yml @@ -15,13 +15,10 @@ inputs: required: false outputs: - manifest-path: - description: Absolute path to the generated manifest. - value: ${{ steps.materialize.outputs.manifest-path }} - metadata-path: - description: Absolute path to the build metadata envelope. - value: ${{ steps.materialize.outputs.metadata-path }} - manifest-artifact-name: + entries-path: + description: Absolute path to the generated release catalog entries. + value: ${{ steps.materialize.outputs.entries-path }} + companion-artifact-name: description: Name of the uploaded GitHub Actions companion artifact. value: ${{ steps.companion-name.outputs.name }} @@ -66,13 +63,12 @@ runs: RELEASE_PACKAGE_FILE: ${{ steps.configuration.outputs.package_file }} run: "${IMPLEMENTATION_PATH}/prepare.sh" - id: materialize - name: Materialize release catalog records + name: Materialize release catalog entries shell: bash env: IMPLEMENTATION_PATH: ${{ steps.implementation.outputs.path }} RELEASE_ARTIFACTS: ${{ steps.prepare.outputs.artifacts }} - RELEASE_MANIFEST_NAME: release-catalog-entries.json - RELEASE_METADATA_NAME: release-catalog-metadata.json + RELEASE_ENTRIES_NAME: release-catalog-entries.json RELEASE_OUTPUT_DIRECTORY: ${{ steps.configuration.outputs.output_directory }} RELEASE_PACKAGE: ${{ steps.prepare.outputs.package }} RELEASE_PACKAGE_FILE: ${{ steps.configuration.outputs.package_file }} @@ -93,5 +89,4 @@ runs: name: ${{ steps.companion-name.outputs.name }} path: | ${{ steps.configuration.outputs.output_directory }}/release-catalog-entries.json - ${{ steps.configuration.outputs.output_directory }}/release-catalog-metadata.json ${{ steps.configuration.outputs.output_directory }}/release-evidence/** diff --git a/release-catalog/README.md b/release-catalog/README.md index 0a940f18..2c05c416 100644 --- a/release-catalog/README.md +++ b/release-catalog/README.md @@ -10,13 +10,18 @@ The companion contains: ```text . -├── release-catalog-metadata.json ├── release-catalog-entries.json └── release-evidence ├── .provenance.json └── .spdx.json ``` +`release-catalog-entries.json` is one atomic job-level envelope. Its `source` +object records the source artifact and build context, while its `entries` array +contains the release catalog entries produced by that job. The release platform +validates and aggregates entry arrays from selected builds into the release +catalog; there is no separate metadata document to keep synchronized. + ## Configuration Every producer passes one `config` JSON object. Its canonical schema and field @@ -29,7 +34,7 @@ The `artifact_type` field determines how the action discovers primary files: - `custom` requires explicit artifact descriptors and exactly one of `package` or `package_file`. -`release_catalog_key` identifies the release-catalog entry that owns these +`release_catalog_key` identifies the release catalog entry that owns these artifacts. Every file from every matrix variant in the same publishable artifact set uses the same key, and multiple files are aggregated. Standard RAPIDS Conda and wheel workflows construct it as `:`, such as diff --git a/release-catalog/materialize.sh b/release-catalog/materialize.sh index 80b2b97a..f800d03a 100755 --- a/release-catalog/materialize.sh +++ b/release-catalog/materialize.sh @@ -14,8 +14,7 @@ require_nonempty() { require_nonempty "RELEASE_CATALOG_KEY" "${RELEASE_CATALOG_KEY:-}" require_nonempty "RELEASE_OUTPUT_DIRECTORY" "${RELEASE_OUTPUT_DIRECTORY:-}" -require_nonempty "RELEASE_MANIFEST_NAME" "${RELEASE_MANIFEST_NAME:-}" -require_nonempty "RELEASE_METADATA_NAME" "${RELEASE_METADATA_NAME:-}" +require_nonempty "RELEASE_ENTRIES_NAME" "${RELEASE_ENTRIES_NAME:-}" require_nonempty "RELEASE_ARTIFACTS" "${RELEASE_ARTIFACTS:-}" require_nonempty "RELEASE_SOURCE_ARTIFACT_NAME" "${RELEASE_SOURCE_ARTIFACT_NAME:-}" @@ -40,12 +39,7 @@ require_plain_filename() { fi } -require_plain_filename "manifest-name" "${RELEASE_MANIFEST_NAME}" -require_plain_filename "metadata-name" "${RELEASE_METADATA_NAME}" -if [[ "${RELEASE_MANIFEST_NAME}" == "${RELEASE_METADATA_NAME}" ]]; then - echo "manifest-name and metadata-name must differ" >&2 - exit 1 -fi +require_plain_filename "entries-name" "${RELEASE_ENTRIES_NAME}" if [[ ! -d "${RELEASE_OUTPUT_DIRECTORY}" ]]; then echo "output-directory does not exist or is not a directory: ${RELEASE_OUTPUT_DIRECTORY}" >&2 @@ -90,12 +84,11 @@ if ! jq -e 'type == "array" and length > 0' <<<"${RELEASE_ARTIFACTS}" >/dev/null exit 1 fi -manifest_path="${output_directory}/${RELEASE_MANIFEST_NAME}" -metadata_path="${output_directory}/${RELEASE_METADATA_NAME}" +entries_path="${output_directory}/${RELEASE_ENTRIES_NAME}" temporary_manifest="$(mktemp "${output_directory}/.release-catalog.XXXXXX")" trap 'rm -f "${temporary_manifest}"' EXIT -printf '%s\n' '{"schema_version":1,"producer":"release-platform","artifacts":[]}' >"${temporary_manifest}" +printf '%s\n' '{"entries":[]}' >"${temporary_manifest}" resolve_one_file() { local field="$1" @@ -263,7 +256,6 @@ write_generated_provenance() { } shopt -s globstar nullglob -artifact_metadata='[]' while IFS= read -r descriptor; do if ! jq -e ' type == "object" @@ -309,59 +301,48 @@ while IFS= read -r descriptor; do provenance_path="$(generated_evidence_path "${primary_path}" "provenance")" write_generated_provenance "${primary_path}" "${package}" "${provenance_path}" fi - artifact="$(jq -cn \ + entry="$(jq -cn \ --arg release_catalog_key "${RELEASE_CATALOG_KEY}" \ --arg path "${primary_path}" \ --arg sbom "${sbom_path}" \ + --arg sbom_kind "${sbom_kind}" \ --arg provenance "${provenance_path}" \ --argjson package "${package}" \ - '{release_catalog_key: $release_catalog_key, path: $path, sbom: $sbom, provenance: $provenance, package: $package}')" + '{release_catalog_key: $release_catalog_key, path: $path, sbom: $sbom, sbom_kind: $sbom_kind, provenance: $provenance, package: $package}')" if [[ -n "${signature_pattern}" ]]; then supplied_signature_path="$(resolve_one_file signature "${signature_pattern}")" signature_path="$(copy_supplied_evidence "${primary_path}" "${supplied_signature_path}" "signature")" - artifact="$(jq -c --arg signature "${signature_path}" '. + {signature: $signature}' <<<"${artifact}")" + entry="$(jq -c --arg signature "${signature_path}" '. + {signature: $signature}' <<<"${entry}")" fi - artifact_metadata="$(jq -cn \ - --arg path "${primary_path}" \ - --arg sbom_kind "${sbom_kind}" \ - --argjson artifacts "${artifact_metadata}" \ - '$artifacts + [{path: $path, sbom_kind: $sbom_kind}]')" - - jq --argjson artifact "${artifact}" '.artifacts += [$artifact]' "${temporary_manifest}" >"${temporary_manifest}.next" + jq --argjson entry "${entry}" '.entries += [$entry]' "${temporary_manifest}" >"${temporary_manifest}.next" mv "${temporary_manifest}.next" "${temporary_manifest}" done < <(jq -c '.[]' <<<"${RELEASE_ARTIFACTS}") -if ! jq -e '.artifacts as $items | ($items | map([.release_catalog_key, .path] | join("\u0000")) | unique | length) == ($items | length)' "${temporary_manifest}" >/dev/null; then +if ! jq -e '.entries as $items | ($items | map([.release_catalog_key, .path] | join("\u0000")) | unique | length) == ($items | length)' "${temporary_manifest}" >/dev/null; then echo "release-artifacts contains duplicate release_catalog_key/path entries" >&2 exit 1 fi -jq -S . "${temporary_manifest}" >"${manifest_path}" jq -n -S \ --arg artifact_name "${RELEASE_SOURCE_ARTIFACT_NAME}" \ - --arg manifest_name "${RELEASE_MANIFEST_NAME}" \ --arg repository "${GITHUB_REPOSITORY:-}" \ --arg run_attempt "${GITHUB_RUN_ATTEMPT:-}" \ --arg run_id "${GITHUB_RUN_ID:-}" \ --arg sha "${source_sha}" \ - --arg release_catalog_key "${RELEASE_CATALOG_KEY}" \ --arg workflow_ref "${GITHUB_WORKFLOW_REF:-}" \ - --argjson artifact_metadata "${artifact_metadata}" \ + --argjson entries "$(jq -c '.entries' "${temporary_manifest}")" \ '{ schema_version: 1, producer: "shared-workflows", - release_catalog_key: $release_catalog_key, - source_artifact: $artifact_name, - catalog_record_manifest: $manifest_name, - build_environment: { + source: { + artifact: $artifact_name, repository: $repository, sha: $sha, workflow_ref: $workflow_ref, run_id: $run_id, run_attempt: $run_attempt }, - metadata: {artifacts: $artifact_metadata} - }' >"${metadata_path}" -echo "manifest-path=${manifest_path}" >>"${GITHUB_OUTPUT}" -echo "metadata-path=${metadata_path}" >>"${GITHUB_OUTPUT}" + entries: $entries + }' >"${entries_path}" +echo "entries-path=${entries_path}" >>"${GITHUB_OUTPUT}" diff --git a/tests/release_catalog_test.sh b/tests/release_catalog_test.sh index 51d93836..a390b98c 100755 --- a/tests/release_catalog_test.sh +++ b/tests/release_catalog_test.sh @@ -25,8 +25,7 @@ GITHUB_SHA="0123456789012345678901234567890123456789" GITHUB_WORKFLOW_REF="rapidsai/cuvs/.github/workflows/build.yaml@refs/heads/release/26.08" export GITHUB_OUTPUT RELEASE_ARTIFACTS="$(jq -cn '[{path: "cuvs-java-*.jar", sbom: "cuvs-java-*.spdx.json", provenance: "cuvs-java-*.provenance.jsonl", signature: "cuvs-java-*.jar.asc"}]')" -RELEASE_MANIFEST_NAME="release-catalog-entries.json" -RELEASE_METADATA_NAME="release-catalog-metadata.json" +RELEASE_ENTRIES_NAME="release-catalog-entries.json" RELEASE_OUTPUT_DIRECTORY="${bundle_directory}" RELEASE_PACKAGE='' RELEASE_PACKAGE_FILE="cuvs-java-package.json" @@ -34,38 +33,31 @@ RELEASE_SOURCE_ARTIFACT_NAME="cuvs-java-cuda12.9.1" RELEASE_SOURCE_SHA="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" RELEASE_CATALOG_KEY="maven:cuvs-java" export GITHUB_REPOSITORY GITHUB_RUN_ATTEMPT GITHUB_RUN_ID GITHUB_SHA GITHUB_WORKFLOW_REF -export RELEASE_ARTIFACTS RELEASE_MANIFEST_NAME RELEASE_METADATA_NAME RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE RELEASE_PACKAGE_FILE +export RELEASE_ARTIFACTS RELEASE_ENTRIES_NAME RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE RELEASE_PACKAGE_FILE export RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY "${repository_root}/release-catalog/materialize.sh" canonical_bundle_directory="$(realpath "${bundle_directory}")" -manifest_path="${canonical_bundle_directory}/release-catalog-entries.json" -metadata_path="${canonical_bundle_directory}/release-catalog-metadata.json" -jq -e ' - .schema_version == 1 - and .producer == "release-platform" - and (.artifacts | length == 1) - and .artifacts[0].release_catalog_key == "maven:cuvs-java" - and .artifacts[0].path == "cuvs-java-26.08.0.jar" - and .artifacts[0].package.name == "ai.rapids:cuvs-java" -' "${manifest_path}" >/dev/null +entries_path="${canonical_bundle_directory}/release-catalog-entries.json" +test ! -e "${canonical_bundle_directory}/release-catalog-metadata.json" jq -e ' .schema_version == 1 and .producer == "shared-workflows" - and .release_catalog_key == "maven:cuvs-java" - and .source_artifact == "cuvs-java-cuda12.9.1" - and .catalog_record_manifest == "release-catalog-entries.json" - and .build_environment.repository == "rapidsai/cuvs" - and .build_environment.sha == "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" - and .metadata.artifacts == [{path: "cuvs-java-26.08.0.jar", sbom_kind: "producer-dependency"}] -' "${metadata_path}" >/dev/null -grep -Fx "manifest-path=${manifest_path}" "${GITHUB_OUTPUT}" -grep -Fx "metadata-path=${metadata_path}" "${GITHUB_OUTPUT}" - -supplied_sbom_path="$(jq -r '.artifacts[0].sbom' "${manifest_path}")" -supplied_provenance_path="$(jq -r '.artifacts[0].provenance' "${manifest_path}")" -supplied_signature_path="$(jq -r '.artifacts[0].signature' "${manifest_path}")" + and .source.artifact == "cuvs-java-cuda12.9.1" + and .source.repository == "rapidsai/cuvs" + and .source.sha == "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + and (.entries | length == 1) + and .entries[0].release_catalog_key == "maven:cuvs-java" + and .entries[0].path == "cuvs-java-26.08.0.jar" + and .entries[0].package.name == "ai.rapids:cuvs-java" + and .entries[0].sbom_kind == "producer-dependency" +' "${entries_path}" >/dev/null +grep -Fx "entries-path=${entries_path}" "${GITHUB_OUTPUT}" + +supplied_sbom_path="$(jq -r '.entries[0].sbom' "${entries_path}")" +supplied_provenance_path="$(jq -r '.entries[0].provenance' "${entries_path}")" +supplied_signature_path="$(jq -r '.entries[0].signature' "${entries_path}")" [[ "${supplied_sbom_path}" == release-evidence/*/sbom-* ]] [[ "${supplied_provenance_path}" == release-evidence/*/provenance-* ]] [[ "${supplied_signature_path}" == release-evidence/*/signature-* ]] @@ -75,11 +67,11 @@ grep -Fx signature "${canonical_bundle_directory}/${supplied_signature_path}" isolated_companion_directory="${temporary_directory}/isolated-companion" mkdir -p "${isolated_companion_directory}" -cp "${manifest_path}" "${metadata_path}" "${isolated_companion_directory}/" +cp "${entries_path}" "${isolated_companion_directory}/" cp -R "${canonical_bundle_directory}/release-evidence" "${isolated_companion_directory}/" while IFS= read -r evidence_path; do test -f "${isolated_companion_directory}/${evidence_path}" -done < <(jq -r '.artifacts[] | .sbom, .provenance, (.signature // empty)' "${isolated_companion_directory}/release-catalog-entries.json") +done < <(jq -r '.entries[] | .sbom, .provenance, (.signature // empty)' "${isolated_companion_directory}/release-catalog-entries.json") generated_directory="${temporary_directory}/generated-bundle" mkdir -p "${generated_directory}/linux-64" @@ -96,15 +88,17 @@ export RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE RELEASE_PACKAG "${repository_root}/release-catalog/materialize.sh" -generated_manifest_path="${generated_directory}/release-catalog-entries.json" -generated_metadata_path="${generated_directory}/release-catalog-metadata.json" -generated_sbom_path="$(jq -r '.artifacts[0].sbom' "${generated_manifest_path}")" -generated_provenance_path="$(jq -r '.artifacts[0].provenance' "${generated_manifest_path}")" +generated_entries_path="${generated_directory}/release-catalog-entries.json" +generated_sbom_path="$(jq -r '.entries[0].sbom' "${generated_entries_path}")" +generated_provenance_path="$(jq -r '.entries[0].provenance' "${generated_entries_path}")" jq -e ' - .artifacts[0].release_catalog_key == "conda:kvikio" - and .artifacts[0].path == "linux-64/kvikio-26.08.00a32-cuda12_260714_2f567060.conda" - and .artifacts[0].package == {ecosystem: "conda", name: "kvikio", version: "26.08.00a32"} -' "${generated_manifest_path}" >/dev/null + .source.artifact == "kvikio_conda_python_kvikio_x86_64_abi3_cu12" + and .source.sha == "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + and .entries[0].release_catalog_key == "conda:kvikio" + and .entries[0].path == "linux-64/kvikio-26.08.00a32-cuda12_260714_2f567060.conda" + and .entries[0].package == {ecosystem: "conda", name: "kvikio", version: "26.08.00a32"} + and .entries[0].sbom_kind == "generated-identity" +' "${generated_entries_path}" >/dev/null jq -e ' .spdxVersion == "SPDX-2.3" and .packages[0].name == "kvikio" @@ -115,13 +109,6 @@ jq -e ' and .predicate.buildDefinition.externalParameters.release_catalog_key == "conda:kvikio" and .predicate.buildDefinition.resolvedDependencies[0].digest.gitCommit == "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" ' "${generated_directory}/${generated_provenance_path}" >/dev/null -jq -e ' - .release_catalog_key == "conda:kvikio" - and .source_artifact == "kvikio_conda_python_kvikio_x86_64_abi3_cu12" - and .build_environment.sha == "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" - and .metadata.artifacts == [{path: "linux-64/kvikio-26.08.00a32-cuda12_260714_2f567060.conda", sbom_kind: "generated-identity"}] -' "${generated_metadata_path}" >/dev/null - wheel_directory="${temporary_directory}/wheel-bundle" mkdir -p "${wheel_directory}" printf '%s\n' wheel >"${wheel_directory}/libkvikio_cu12-26.8.0a32-py3-none-manylinux_2_28_x86_64.whl" @@ -137,13 +124,11 @@ export RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE RELEASE_SOURCE "${repository_root}/release-catalog/materialize.sh" jq -e ' - .artifacts[0].release_catalog_key == "wheel:kvikio" - and .artifacts[0].path == "libkvikio_cu12-26.8.0a32-py3-none-manylinux_2_28_x86_64.whl" - and .artifacts[0].package == {ecosystem: "wheel", name: "libkvikio-cu12", version: "26.8.0a32"} + .entries[0].release_catalog_key == "wheel:kvikio" + and .entries[0].path == "libkvikio_cu12-26.8.0a32-py3-none-manylinux_2_28_x86_64.whl" + and .entries[0].package == {ecosystem: "wheel", name: "libkvikio-cu12", version: "26.8.0a32"} + and .entries[0].sbom_kind == "generated-identity" ' "${wheel_directory}/release-catalog-entries.json" >/dev/null -jq -e ' - .metadata.artifacts == [{path: "libkvikio_cu12-26.8.0a32-py3-none-manylinux_2_28_x86_64.whl", sbom_kind: "generated-identity"}] -' "${wheel_directory}/release-catalog-metadata.json" >/dev/null missing_version_directory="${temporary_directory}/missing-version-bundle" mkdir -p "${missing_version_directory}" From 6cdff151c40c97551fdf2abd5d977580f94ed7e1 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 18 Aug 2026 09:40:11 -0500 Subject: [PATCH 08/16] simplify custom package identity input --- release-catalog-dispatch/action.yml | 7 +- release-catalog/README.md | 36 ++++- release-catalog/config.schema.json | 105 ++------------ release-catalog/materialize.sh | 134 +++++++----------- release-catalog/prepare.sh | 14 +- release-catalog/validate-config.sh | 35 ++--- .../inline-package.json | 14 -- ...e-file.json => package-identity-file.json} | 2 +- tests/release_catalog_config_test.sh | 32 ++--- tests/release_catalog_prepare_test.sh | 12 +- tests/release_catalog_test.sh | 33 +++-- 11 files changed, 145 insertions(+), 279 deletions(-) delete mode 100644 tests/release-catalog-config/inline-package.json rename tests/release-catalog-config/{package-file.json => package-identity-file.json} (77%) diff --git a/release-catalog-dispatch/action.yml b/release-catalog-dispatch/action.yml index e95c3844..d53309ca 100644 --- a/release-catalog-dispatch/action.yml +++ b/release-catalog-dispatch/action.yml @@ -59,8 +59,7 @@ runs: RELEASE_ARTIFACTS: ${{ steps.configuration.outputs.artifacts }} RELEASE_ARTIFACT_TYPE: ${{ steps.configuration.outputs.artifact_type }} RELEASE_OUTPUT_DIRECTORY: ${{ steps.configuration.outputs.output_directory }} - RELEASE_PACKAGE: ${{ steps.configuration.outputs.package }} - RELEASE_PACKAGE_FILE: ${{ steps.configuration.outputs.package_file }} + RELEASE_PACKAGE_IDENTITY_FILE: ${{ steps.configuration.outputs.package_identity_file }} run: "${IMPLEMENTATION_PATH}/prepare.sh" - id: materialize name: Materialize release catalog entries @@ -68,10 +67,10 @@ runs: env: IMPLEMENTATION_PATH: ${{ steps.implementation.outputs.path }} RELEASE_ARTIFACTS: ${{ steps.prepare.outputs.artifacts }} + RELEASE_ARTIFACT_TYPE: ${{ steps.configuration.outputs.artifact_type }} RELEASE_ENTRIES_NAME: release-catalog-entries.json RELEASE_OUTPUT_DIRECTORY: ${{ steps.configuration.outputs.output_directory }} - RELEASE_PACKAGE: ${{ steps.prepare.outputs.package }} - RELEASE_PACKAGE_FILE: ${{ steps.configuration.outputs.package_file }} + RELEASE_PACKAGE_IDENTITY_FILE: ${{ steps.configuration.outputs.package_identity_file }} RELEASE_SOURCE_ARTIFACT_NAME: ${{ inputs.source-artifact-name }} RELEASE_SOURCE_SHA: ${{ inputs.source-sha || github.sha }} RELEASE_CATALOG_KEY: ${{ steps.configuration.outputs.release_catalog_key }} diff --git a/release-catalog/README.md b/release-catalog/README.md index 2c05c416..b1a7785c 100644 --- a/release-catalog/README.md +++ b/release-catalog/README.md @@ -31,8 +31,8 @@ The `artifact_type` field determines how the action discovers primary files: - `conda` and `wheel` derive artifact descriptors and package identity from the built packages; -- `custom` requires explicit artifact descriptors and exactly one of `package` - or `package_file`. +- `custom` requires explicit artifact descriptors and a + `package_identity_file` created by the producer during the build. `release_catalog_key` identifies the release catalog entry that owns these artifacts. Every file from every matrix variant in the same publishable artifact @@ -54,9 +54,10 @@ Multiple package names from one repository do not by themselves justify separate keys: for example, `cudf` and `dask-cudf` Conda packages remain part of `conda:cudf` when they share one release policy. -The action validates the complete configuration before inspecting build -outputs. It then verifies properties that depend on produced files, including -whether package, primary-artifact, and evidence paths resolve unambiguously. +The action validates the configuration before inspecting build outputs. It then +verifies properties that depend on produced files, including the package +identity contents and whether primary-artifact and evidence paths resolve +unambiguously. ## Source revision @@ -100,13 +101,26 @@ the built source. "artifact_type": "custom", "release_catalog_key": "maven:cuvs-java", "output_directory": "java/cuvs-java/target", - "package_file": "cuvs-java.release-package.json", + "package_identity_file": "cuvs-java.release-package-identity.json", "artifacts": [{"path": "cuvs-java-*-x86_64-cuda*.jar"}] } source-artifact-name: cuvs-java source-sha: ${{ env.RAPIDS_SHA }} ``` +Before the action runs, the producer creates +`java/cuvs-java/target/cuvs-java.release-package-identity.json`. The +`package_identity_file` value is the path to that file relative to +`output_directory`. For example: + +```json +{ + "ecosystem": "maven", + "name": "ai.rapids:cuvs-java", + "version": "26.08.0" +} +``` + ## Evidence semantics A descriptor-selected producer SBOM is classified as `producer-dependency`. @@ -119,3 +133,13 @@ inventory. It must not be reported as producer-supplied dependency coverage. Producer-supplied SBOM, provenance, and signature sidecars are copied under `release-evidence/` so the companion remains independently consumable. + +Concrete producer-supplied evidence examples include: + +- an official [SPDX 2.3 dependency SBOM](https://github.com/spdx/spdx-examples/blob/2181917ef6ff74de89252ee785583c27a38d6199/presentations/OSS-NA-2023/SPDXVersion2.3/03-SBOMwDependency.json); +- an official [SLSA provenance v1 statement](https://github.com/slsa-framework/github-actions-buildtypes/blob/5f855ef0106dad3ee0e0f1046dc31b3b65152956/workflow/v1/example.json); +- a Maven Central [detached ASCII-armored signature](https://repo1.maven.org/maven2/org/apache/commons/commons-lang3/3.17.0/commons-lang3-3.17.0.jar.asc). + +These examples illustrate the expected purpose of the files, not required +serialization formats. The action copies producer-supplied evidence as opaque +sidecars and does not validate their contents or require these formats. diff --git a/release-catalog/config.schema.json b/release-catalog/config.schema.json index 9883c7d7..5610a1d3 100644 --- a/release-catalog/config.schema.json +++ b/release-catalog/config.schema.json @@ -11,7 +11,7 @@ ], "properties": { "artifact_type": { - "description": "How primary artifact descriptors and package identities are obtained.", + "description": "Metadata can be extracted from the artifact for conda/wheel types. More info needed for custom type.", "type": "string", "enum": [ "conda", @@ -20,7 +20,7 @@ ] }, "release_catalog_key": { - "description": "Release-catalog entry name that owns these artifacts. Default for conda/wheels is :, such as conda:cudf. Custom producers specify their own, such as maven:cuvs-java.", + "description": "Release-catalog entry name that owns these artifacts. Default for conda & wheels is :, such as conda:cudf. Custom jobs specify their own, such as maven:cuvs-java.", "type": "string", "minLength": 1, "pattern": "^[^\\r\\n]+$" @@ -32,17 +32,14 @@ "pattern": "^[^\\r\\n]+$", "default": "." }, - "package": { - "$ref": "#/$defs/package" - }, - "package_file": { - "description": "Producer-created package identity JSON, relative to output_directory.", + "package_identity_file": { + "description": "Path, relative to output_directory, to producer-created package identity JSON. Required for custom artifacts; Conda and wheel identities are extracted from the artifacts. The file must contain ecosystem, name, and version and may contain build and platform.", "type": "string", "minLength": 1, "pattern": "^[^\\r\\n]+$" }, "artifacts": { - "description": "Primary release files and optional producer-supplied evidence sidecars.", + "description": "Primary release files and optional SBoM evidence sidecar files", "type": "array", "minItems": 1, "items": { @@ -64,29 +61,8 @@ }, "then": { "required": [ - "artifacts" - ], - "oneOf": [ - { - "required": [ - "package" - ], - "not": { - "required": [ - "package_file" - ] - } - }, - { - "required": [ - "package_file" - ], - "not": { - "required": [ - "package" - ] - } - } + "artifacts", + "package_identity_file" ] }, "else": { @@ -102,12 +78,7 @@ }, { "required": [ - "package" - ] - }, - { - "required": [ - "package_file" + "package_identity_file" ] } ] @@ -116,33 +87,6 @@ } ], "$defs": { - "package": { - "type": "object", - "additionalProperties": false, - "required": [ - "ecosystem", - "name" - ], - "properties": { - "ecosystem": { - "type": "string", - "minLength": 1 - }, - "name": { - "type": "string", - "minLength": 1 - }, - "version": { - "type": "string" - }, - "build": { - "type": "string" - }, - "platform": { - "type": "string" - } - } - }, "artifact": { "type": "object", "additionalProperties": false, @@ -156,49 +100,22 @@ "pattern": "^[^\\r\\n]+$" }, "sbom": { - "description": "Producer-supplied dependency SBOM path or glob.", + "description": "Dependency SBOM path or glob. Example file: https://github.com/spdx/spdx-examples/blob/2181917ef6ff74de89252ee785583c27a38d6199/presentations/OSS-NA-2023/SPDXVersion2.3/03-SBOMwDependency.json", "type": "string", "minLength": 1, "pattern": "^[^\\r\\n]+$" }, "provenance": { - "description": "Producer-supplied provenance path or glob.", + "description": "Provenance path or glob. Example file: https://github.com/slsa-framework/github-actions-buildtypes/blob/5f855ef0106dad3ee0e0f1046dc31b3b65152956/workflow/v1/example.json", "type": "string", "minLength": 1, "pattern": "^[^\\r\\n]+$" }, "signature": { - "description": "Producer-supplied signature path or glob.", + "description": "Signature path or glob. Example file: https://repo1.maven.org/maven2/org/apache/commons/commons-lang3/3.17.0/commons-lang3-3.17.0.jar.asc", "type": "string", "minLength": 1, "pattern": "^[^\\r\\n]+$" - }, - "package": { - "description": "Per-artifact package identity overrides.", - "type": "object", - "additionalProperties": false, - "properties": { - "ecosystem": { - "type": "string", - "minLength": 1 - }, - "name": { - "type": "string", - "minLength": 1 - }, - "version": { - "type": "string", - "minLength": 1 - }, - "build": { - "type": "string", - "minLength": 1 - }, - "platform": { - "type": "string", - "minLength": 1 - } - } } } } diff --git a/release-catalog/materialize.sh b/release-catalog/materialize.sh index f800d03a..24c45fcf 100755 --- a/release-catalog/materialize.sh +++ b/release-catalog/materialize.sh @@ -13,20 +13,12 @@ require_nonempty() { } require_nonempty "RELEASE_CATALOG_KEY" "${RELEASE_CATALOG_KEY:-}" +require_nonempty "RELEASE_ARTIFACT_TYPE" "${RELEASE_ARTIFACT_TYPE:-}" require_nonempty "RELEASE_OUTPUT_DIRECTORY" "${RELEASE_OUTPUT_DIRECTORY:-}" require_nonempty "RELEASE_ENTRIES_NAME" "${RELEASE_ENTRIES_NAME:-}" require_nonempty "RELEASE_ARTIFACTS" "${RELEASE_ARTIFACTS:-}" require_nonempty "RELEASE_SOURCE_ARTIFACT_NAME" "${RELEASE_SOURCE_ARTIFACT_NAME:-}" -if [[ -n "${RELEASE_PACKAGE:-}" && -n "${RELEASE_PACKAGE_FILE:-}" ]]; then - echo "release-package and release-package-file are mutually exclusive" >&2 - exit 1 -fi -if [[ -z "${RELEASE_PACKAGE:-}" && -z "${RELEASE_PACKAGE_FILE:-}" ]]; then - echo "one of release-package or release-package-file is required" >&2 - exit 1 -fi - source_sha="${RELEASE_SOURCE_SHA:-${GITHUB_SHA:-}}" require_nonempty "RELEASE_SOURCE_SHA or GITHUB_SHA" "${source_sha}" @@ -56,26 +48,40 @@ ensure_relative_pattern() { } output_directory="$(realpath "${RELEASE_OUTPUT_DIRECTORY}")" -if [[ -n "${RELEASE_PACKAGE_FILE:-}" ]]; then - ensure_relative_pattern "release-package-file" "${RELEASE_PACKAGE_FILE}" - package_file_path="$(realpath "${output_directory}/${RELEASE_PACKAGE_FILE}")" - if [[ "${package_file_path}" != "${output_directory}"/* || ! -f "${package_file_path}" ]]; then - echo "release-package-file must resolve to one file inside output-directory: ${RELEASE_PACKAGE_FILE}" >&2 +package_identity='' +case "${RELEASE_ARTIFACT_TYPE}" in + custom) + require_nonempty "RELEASE_PACKAGE_IDENTITY_FILE" "${RELEASE_PACKAGE_IDENTITY_FILE:-}" + ensure_relative_pattern "package-identity-file" "${RELEASE_PACKAGE_IDENTITY_FILE}" + package_identity_path="$(realpath "${output_directory}/${RELEASE_PACKAGE_IDENTITY_FILE}")" + if [[ "${package_identity_path}" != "${output_directory}"/* || ! -f "${package_identity_path}" ]]; then + echo "package-identity-file must resolve to one file inside output-directory: ${RELEASE_PACKAGE_IDENTITY_FILE}" >&2 + exit 1 + fi + package_identity="$(jq -c . "${package_identity_path}")" + ;; + conda | wheel) + if [[ -n "${RELEASE_PACKAGE_IDENTITY_FILE:-}" ]]; then + echo "package-identity-file is only valid for custom artifacts" >&2 + exit 1 + fi + ;; + *) + echo "artifact-type must be one of: conda, custom, wheel" >&2 exit 1 - fi - RELEASE_PACKAGE="$(jq -c . "${package_file_path}")" -fi + ;; +esac -if ! jq -e ' +if [[ "${RELEASE_ARTIFACT_TYPE}" == "custom" ]] && ! jq -e ' type == "object" and (keys - ["ecosystem", "name", "version", "build", "platform"] | length == 0) and (.ecosystem | type == "string" and length > 0) and (.name | type == "string" and length > 0) - and ((.version // "") | type == "string") - and ((.build // "") | type == "string") - and ((.platform // "") | type == "string") -' <<<"${RELEASE_PACKAGE}" >/dev/null; then - echo "release-package must be a package object with ecosystem and name; version may be supplied or derived per artifact" >&2 + and (.version | type == "string" and length > 0) + and ((has("build") | not) or (.build | type == "string" and length > 0)) + and ((has("platform") | not) or (.platform | type == "string" and length > 0)) +' <<<"${package_identity}" >/dev/null; then + echo "package identity file must contain non-empty ecosystem, name, and version strings and only optional build or platform strings" >&2 exit 1 fi @@ -113,49 +119,6 @@ resolve_one_file() { printf '%s\n' "${resolved#"${output_directory}/"}" } -derive_package_version() { - local ecosystem="$1" - local package_name="$2" - local artifact_path="$3" - local filename - filename="$(basename "${artifact_path}")" - - case "${ecosystem}" in - conda) - local conda_prefix="${package_name}-" - if [[ "${filename}" != "${conda_prefix}"* ]]; then - echo "Conda artifact filename does not start with package name '${package_name}': ${filename}" >&2 - exit 1 - fi - local conda_remainder="${filename#"${conda_prefix}"}" - local conda_version="${conda_remainder%%-*}" - if [[ -z "${conda_version}" || "${conda_version}" == "${conda_remainder}" ]]; then - echo "cannot derive Conda package version from artifact filename: ${filename}" >&2 - exit 1 - fi - printf '%s\n' "${conda_version}" - ;; - wheel) - local wheel_prefix="${package_name//-/_}-" - if [[ "${filename}" != "${wheel_prefix}"* ]]; then - echo "wheel artifact filename does not start with normalized package name '${package_name}': ${filename}" >&2 - exit 1 - fi - local wheel_remainder="${filename#"${wheel_prefix}"}" - local wheel_version="${wheel_remainder%%-*}" - if [[ -z "${wheel_version}" || "${wheel_version}" == "${wheel_remainder}" ]]; then - echo "cannot derive wheel package version from artifact filename: ${filename}" >&2 - exit 1 - fi - printf '%s\n' "${wheel_version}" - ;; - *) - echo "release-package version is required for ${ecosystem} artifacts" >&2 - exit 1 - ;; - esac -} - generated_evidence_path() { local primary_path="$1" local kind="$2" @@ -257,18 +220,34 @@ write_generated_provenance() { shopt -s globstar nullglob while IFS= read -r descriptor; do - if ! jq -e ' + if ! jq -e --arg artifact_type "${RELEASE_ARTIFACT_TYPE}" ' type == "object" - and (keys - ["path", "sbom", "provenance", "signature", "package"] | length == 0) + and ( + if $artifact_type == "custom" then + (keys - ["path", "sbom", "provenance", "signature"] | length == 0) + else + (keys - ["path", "sbom", "provenance", "signature", "package"] | length == 0) + end + ) and (.path | type == "string" and length > 0) and ((.sbom // "") | type == "string") and ((.provenance // "") | type == "string") and ((.signature // "") | type == "string") - and ((.package // {}) | type == "object") - and ((.package // {} | keys - ["ecosystem", "name", "version", "build", "platform"]) | length == 0) - and ((.package // {} | to_entries | map(.value | type == "string" and length > 0) | all)) + and ( + if $artifact_type == "custom" then + has("package") | not + else + (.package | type == "object") + and (.package | keys - ["ecosystem", "name", "version", "build", "platform"] | length == 0) + and (.package.ecosystem | type == "string" and length > 0) + and (.package.name | type == "string" and length > 0) + and (.package.version | type == "string" and length > 0) + and ((.package | has("build") | not) or (.package.build | type == "string" and length > 0)) + and ((.package | has("platform") | not) or (.package.platform | type == "string" and length > 0)) + end + ) ' <<<"${descriptor}" >/dev/null; then - echo "every release-artifacts entry must contain path and optional SBOM/provenance/signature/package overrides" >&2 + echo "release artifact descriptor is invalid for ${RELEASE_ARTIFACT_TYPE}: ${descriptor}" >&2 exit 1 fi @@ -276,13 +255,10 @@ while IFS= read -r descriptor; do sbom_pattern="$(jq -r '.sbom // empty' <<<"${descriptor}")" provenance_pattern="$(jq -r '.provenance // empty' <<<"${descriptor}")" signature_pattern="$(jq -r '.signature // empty' <<<"${descriptor}")" - package_override="$(jq -c '.package // {}' <<<"${descriptor}")" - - package="$(jq -cn --argjson base "${RELEASE_PACKAGE}" --argjson override "${package_override}" '$base + $override')" - package_version="$(jq -r '.version // empty' <<<"${package}")" - if [[ -z "${package_version}" ]]; then - package_version="$(derive_package_version "$(jq -r '.ecosystem' <<<"${package}")" "$(jq -r '.name' <<<"${package}")" "${primary_path}")" - package="$(jq -c --arg version "${package_version}" '. + {version: $version}' <<<"${package}")" + if [[ "${RELEASE_ARTIFACT_TYPE}" == "custom" ]]; then + package="${package_identity}" + else + package="$(jq -c '.package' <<<"${descriptor}")" fi if [[ -n "${sbom_pattern}" ]]; then diff --git a/release-catalog/prepare.sh b/release-catalog/prepare.sh index f3aa97f4..4a803d28 100755 --- a/release-catalog/prepare.sh +++ b/release-catalog/prepare.sh @@ -17,27 +17,21 @@ require_nonempty "RELEASE_OUTPUT_DIRECTORY" "${RELEASE_OUTPUT_DIRECTORY:-}" script_directory="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" artifacts="${RELEASE_ARTIFACTS:-}" -package="${RELEASE_PACKAGE:-}" case "${RELEASE_ARTIFACT_TYPE}" in conda) if [[ -z "${artifacts}" ]]; then artifacts="$("${script_directory}/describe-conda.sh" "${RELEASE_OUTPUT_DIRECTORY}")" fi - if [[ -z "${package}" && -z "${RELEASE_PACKAGE_FILE:-}" ]]; then - package='{"ecosystem":"conda","name":"bundle"}' - fi ;; wheel) if [[ -z "${artifacts}" ]]; then artifacts="$("${script_directory}/describe-wheels.sh" "${RELEASE_OUTPUT_DIRECTORY}")" fi - if [[ -z "${package}" && -z "${RELEASE_PACKAGE_FILE:-}" ]]; then - package='{"ecosystem":"wheel","name":"bundle"}' - fi ;; custom) require_nonempty "RELEASE_ARTIFACTS" "${artifacts}" + require_nonempty "RELEASE_PACKAGE_IDENTITY_FILE" "${RELEASE_PACKAGE_IDENTITY_FILE:-}" ;; *) echo "artifact-type must be one of: conda, custom, wheel" >&2 @@ -45,10 +39,4 @@ case "${RELEASE_ARTIFACT_TYPE}" in ;; esac -if [[ -z "${package}" && -z "${RELEASE_PACKAGE_FILE:-}" ]]; then - echo "one of release-package or release-package-file is required for custom artifacts" >&2 - exit 1 -fi - printf 'artifacts=%s\n' "${artifacts}" >>"${GITHUB_OUTPUT}" -printf 'package=%s\n' "${package}" >>"${GITHUB_OUTPUT}" diff --git a/release-catalog/validate-config.sh b/release-catalog/validate-config.sh index 12916159..9cbfafd1 100755 --- a/release-catalog/validate-config.sh +++ b/release-catalog/validate-config.sh @@ -23,22 +23,10 @@ fi validation_errors="$(jq -r ' def single_line_string: type == "string" and length > 0 and (test("[\\r\\n]") | not); - def package_object: - type == "object" - and (keys - ["ecosystem", "name", "version", "build", "platform"] | length == 0) - and (.ecosystem | single_line_string) - and (.name | single_line_string) - and ((.version // "") | type == "string") - and ((.build // "") | type == "string") - and ((.platform // "") | type == "string"); - def artifact_package_object: - type == "object" - and (keys - ["ecosystem", "name", "version", "build", "platform"] | length == 0) - and (to_entries | map(.value | single_line_string) | all); if type != "object" then ["release catalog configuration must be a JSON object"] else - (keys - ["artifact_type", "release_catalog_key", "output_directory", "package", "package_file", "artifacts"]) as $unknown + (keys - ["artifact_type", "release_catalog_key", "output_directory", "package_identity_file", "artifacts"]) as $unknown | [ if ($unknown | length) > 0 then "unknown field(s): " + ($unknown | join(", ")) @@ -52,12 +40,8 @@ validation_errors="$(jq -r ' ] + if .artifact_type == "custom" then [ - if ([has("package"), has("package_file")] | map(select(.)) | length) == 1 then empty - else "exactly one of package or package_file is required for custom artifacts" end, - if (has("package") | not) or (.package | package_object) then empty - else "package must contain non-empty ecosystem and name strings and only version, build, or platform as optional string fields" end, - if (has("package_file") | not) or (.package_file | single_line_string) then empty - else "package_file must be a non-empty, single-line path relative to output_directory" end, + if (.package_identity_file | single_line_string) then empty + else "package_identity_file is required for custom artifacts and must be a non-empty, single-line path relative to output_directory" end, if (.artifacts | type == "array" and length > 0) then empty else "artifacts must be a non-empty array for custom artifacts" end ] @@ -65,8 +49,8 @@ validation_errors="$(jq -r ' [ if (.output_directory | single_line_string) then empty else "output_directory is required for conda and wheel artifacts" end, - if has("artifacts") or has("package") or has("package_file") then - "artifacts, package, and package_file are only valid when artifact_type is custom" + if has("artifacts") or has("package_identity_file") then + "artifacts and package_identity_file are only valid when artifact_type is custom" else empty end ] else [] end @@ -78,7 +62,7 @@ validation_errors="$(jq -r ' | if ($artifact | type) != "object" then "artifacts[\($index)] must be an object" else - ($artifact | keys - ["path", "sbom", "provenance", "signature", "package"]) as $artifact_unknown + ($artifact | keys - ["path", "sbom", "provenance", "signature"]) as $artifact_unknown | if ($artifact_unknown | length) > 0 then "artifacts[\($index)] has unknown field(s): " + ($artifact_unknown | join(", ")) else empty end, @@ -87,9 +71,7 @@ validation_errors="$(jq -r ' ($artifact | to_entries[] | select(.key == "sbom" or .key == "provenance" or .key == "signature") | select((.value | single_line_string) | not) - | "artifacts[\($index)].\(.key) must be a non-empty, single-line string"), - if ($artifact | has("package") | not) or ($artifact.package | artifact_package_object) then empty - else "artifacts[\($index)].package must contain only non-empty string identity overrides" end + | "artifacts[\($index)].\(.key) must be a non-empty, single-line string") end ] else [] end @@ -113,7 +95,6 @@ fi printf 'artifact_type=%s\n' "$(jq -r '.artifact_type' <<<"${compact_config}")" printf 'release_catalog_key=%s\n' "$(jq -r '.release_catalog_key' <<<"${compact_config}")" printf 'output_directory=%s\n' "$(jq -r '.output_directory // "."' <<<"${compact_config}")" - printf 'package=%s\n' "$(jq -c '.package // empty' <<<"${compact_config}")" - printf 'package_file=%s\n' "$(jq -r '.package_file // empty' <<<"${compact_config}")" + printf 'package_identity_file=%s\n' "$(jq -r '.package_identity_file // empty' <<<"${compact_config}")" printf 'artifacts=%s\n' "$(jq -c '.artifacts // empty' <<<"${compact_config}")" } >>"${GITHUB_OUTPUT}" diff --git a/tests/release-catalog-config/inline-package.json b/tests/release-catalog-config/inline-package.json deleted file mode 100644 index 80d6b6d3..00000000 --- a/tests/release-catalog-config/inline-package.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "artifact_type": "custom", - "release_catalog_key": "archive:smoke", - "package": { - "ecosystem": "archive", - "name": "smoke", - "version": "1.0" - }, - "artifacts": [ - { - "path": "smoke.tar.gz" - } - ] -} diff --git a/tests/release-catalog-config/package-file.json b/tests/release-catalog-config/package-identity-file.json similarity index 77% rename from tests/release-catalog-config/package-file.json rename to tests/release-catalog-config/package-identity-file.json index 67776f18..0c6bacd8 100644 --- a/tests/release-catalog-config/package-file.json +++ b/tests/release-catalog-config/package-identity-file.json @@ -2,7 +2,7 @@ "artifact_type": "custom", "release_catalog_key": "maven:cuvs-java", "output_directory": "java/cuvs-java/target", - "package_file": "cuvs-java.release-package.json", + "package_identity_file": "cuvs-java.release-package-identity.json", "artifacts": [ { "path": "cuvs-java-*-x86_64-cuda*.jar", diff --git a/tests/release_catalog_config_test.sh b/tests/release_catalog_config_test.sh index 36d92c7c..ccfe3959 100755 --- a/tests/release_catalog_config_test.sh +++ b/tests/release_catalog_config_test.sh @@ -27,24 +27,15 @@ assert_invalid() { } valid_output="${temporary_directory}/valid.output" -RELEASE_CATALOG_CONFIG="$(<"${repository_root}/tests/release-catalog-config/package-file.json")" \ +RELEASE_CATALOG_CONFIG="$(<"${repository_root}/tests/release-catalog-config/package-identity-file.json")" \ GITHUB_OUTPUT="${valid_output}" "${validator}" grep -Fx 'release_catalog_key=maven:cuvs-java' "${valid_output}" grep -Fx 'artifact_type=custom' "${valid_output}" grep -Fx 'output_directory=java/cuvs-java/target' "${valid_output}" -grep -Fx 'package=' "${valid_output}" -grep -Fx 'package_file=cuvs-java.release-package.json' "${valid_output}" +grep -Fx 'package_identity_file=cuvs-java.release-package-identity.json' "${valid_output}" grep -Fx 'artifacts=[{"path":"cuvs-java-*-x86_64-cuda*.jar","sbom":"cuvs-java.spdx.json"}]' "${valid_output}" -inline_output="${temporary_directory}/inline.output" -RELEASE_CATALOG_CONFIG="$(<"${repository_root}/tests/release-catalog-config/inline-package.json")" \ - GITHUB_OUTPUT="${inline_output}" "${validator}" - -grep -Fx 'output_directory=.' "${inline_output}" -grep -Fx 'package={"ecosystem":"archive","name":"smoke","version":"1.0"}' "${inline_output}" -grep -Fx 'package_file=' "${inline_output}" - assert_invalid \ malformed-json \ '{"release_catalog_key":' \ @@ -64,22 +55,27 @@ grep -F 'artifact_type must be one of: conda, custom, wheel' "${temporary_direct assert_invalid \ custom-missing-fields \ '{"artifact_type":"custom","release_catalog_key":"archive:smoke"}' \ - 'exactly one of package or package_file is required for custom artifacts' + 'package_identity_file is required for custom artifacts' grep -F 'artifacts must be a non-empty array for custom artifacts' "${temporary_directory}/custom-missing-fields.error" >/dev/null assert_invalid \ unknown-field \ - '{"artifact_type":"custom","release_catalog_key":"archive:smoke","package_file":"package.json","artifacts":[{"path":"smoke.tar.gz"}],"component_id":"archive:smoke"}' \ + '{"artifact_type":"custom","release_catalog_key":"archive:smoke","package_identity_file":"package.json","artifacts":[{"path":"smoke.tar.gz"}],"component_id":"archive:smoke"}' \ 'unknown field(s): component_id' assert_invalid \ - conflicting-package-source \ - '{"artifact_type":"custom","release_catalog_key":"archive:smoke","package":{"ecosystem":"archive","name":"smoke"},"package_file":"package.json","artifacts":[{"path":"smoke.tar.gz"}]}' \ - 'exactly one of package or package_file is required for custom artifacts' + inline-package \ + '{"artifact_type":"custom","release_catalog_key":"archive:smoke","package_identity_file":"package.json","package":{"ecosystem":"archive","name":"smoke","version":"1.0"},"artifacts":[{"path":"smoke.tar.gz"}]}' \ + 'unknown field(s): package' + +assert_invalid \ + per-artifact-package \ + '{"artifact_type":"custom","release_catalog_key":"archive:smoke","package_identity_file":"package.json","artifacts":[{"path":"smoke.tar.gz","package":{"ecosystem":"archive","name":"smoke","version":"1.0"}}]}' \ + 'artifacts[0] has unknown field(s): package' assert_invalid \ malformed-artifact \ - '{"artifact_type":"custom","release_catalog_key":"archive:smoke","package_file":"package.json","artifacts":[{"file":"smoke.tar.gz","sbom":false}]}' \ + '{"artifact_type":"custom","release_catalog_key":"archive:smoke","package_identity_file":"package.json","artifacts":[{"file":"smoke.tar.gz","sbom":false}]}' \ 'artifacts[0] has unknown field(s): file' grep -F 'artifacts[0].path must be a non-empty, single-line string' "${temporary_directory}/malformed-artifact.error" >/dev/null grep -F 'artifacts[0].sbom must be a non-empty, single-line string' "${temporary_directory}/malformed-artifact.error" >/dev/null @@ -92,7 +88,7 @@ assert_invalid \ assert_invalid \ standard-custom-fields \ '{"artifact_type":"wheel","release_catalog_key":"wheel:smoke","output_directory":"dist","artifacts":[{"path":"smoke.whl"}]}' \ - 'artifacts, package, and package_file are only valid when artifact_type is custom' + 'artifacts and package_identity_file are only valid when artifact_type is custom' standard_output="${temporary_directory}/standard.output" RELEASE_CATALOG_CONFIG='{ diff --git a/tests/release_catalog_prepare_test.sh b/tests/release_catalog_prepare_test.sh index a84eae03..399df071 100755 --- a/tests/release_catalog_prepare_test.sh +++ b/tests/release_catalog_prepare_test.sh @@ -22,13 +22,11 @@ GITHUB_OUTPUT="${temporary_directory}/wheel-output" RELEASE_ARTIFACTS='' RELEASE_ARTIFACT_TYPE=wheel RELEASE_OUTPUT_DIRECTORY="${temporary_directory}" -RELEASE_PACKAGE='' -RELEASE_PACKAGE_FILE='' -export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_ARTIFACT_TYPE RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE RELEASE_PACKAGE_FILE +RELEASE_PACKAGE_IDENTITY_FILE='' +export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_ARTIFACT_TYPE RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE_IDENTITY_FILE "${repository_root}/release-catalog/prepare.sh" -grep -Fx 'package={"ecosystem":"wheel","name":"bundle"}' "${GITHUB_OUTPUT}" artifacts="$(sed -n 's/^artifacts=//p' "${GITHUB_OUTPUT}")" jq -e ' . == [{ @@ -40,14 +38,12 @@ jq -e ' GITHUB_OUTPUT="${temporary_directory}/custom-output" RELEASE_ARTIFACTS="$(jq -cn '[{path: "bundle.tar.gz", sbom: "bundle.spdx.json"}]')" RELEASE_ARTIFACT_TYPE=custom -RELEASE_PACKAGE='' -RELEASE_PACKAGE_FILE=release-package.json -export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_ARTIFACT_TYPE RELEASE_PACKAGE RELEASE_PACKAGE_FILE +RELEASE_PACKAGE_IDENTITY_FILE=release-package-identity.json +export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_ARTIFACT_TYPE RELEASE_PACKAGE_IDENTITY_FILE "${repository_root}/release-catalog/prepare.sh" grep -Fx 'artifacts=[{"path":"bundle.tar.gz","sbom":"bundle.spdx.json"}]' "${GITHUB_OUTPUT}" -grep -Fx 'package=' "${GITHUB_OUTPUT}" GITHUB_OUTPUT="${temporary_directory}/invalid-output" RELEASE_ARTIFACT_TYPE=archive diff --git a/tests/release_catalog_test.sh b/tests/release_catalog_test.sh index a390b98c..db347cf3 100755 --- a/tests/release_catalog_test.sh +++ b/tests/release_catalog_test.sh @@ -15,7 +15,7 @@ printf '%s\n' provenance >"${bundle_directory}/cuvs-java-26.08.0.provenance.json printf '%s\n' signature >"${bundle_directory}/cuvs-java-26.08.0.jar.asc" jq -n \ '{ecosystem: "maven", name: "ai.rapids:cuvs-java", version: "26.08.0"}' \ - >"${bundle_directory}/cuvs-java-package.json" + >"${bundle_directory}/cuvs-java-package-identity.json" GITHUB_OUTPUT="${temporary_directory}/github-output" GITHUB_REPOSITORY="rapidsai/cuvs" @@ -25,15 +25,15 @@ GITHUB_SHA="0123456789012345678901234567890123456789" GITHUB_WORKFLOW_REF="rapidsai/cuvs/.github/workflows/build.yaml@refs/heads/release/26.08" export GITHUB_OUTPUT RELEASE_ARTIFACTS="$(jq -cn '[{path: "cuvs-java-*.jar", sbom: "cuvs-java-*.spdx.json", provenance: "cuvs-java-*.provenance.jsonl", signature: "cuvs-java-*.jar.asc"}]')" +RELEASE_ARTIFACT_TYPE=custom RELEASE_ENTRIES_NAME="release-catalog-entries.json" RELEASE_OUTPUT_DIRECTORY="${bundle_directory}" -RELEASE_PACKAGE='' -RELEASE_PACKAGE_FILE="cuvs-java-package.json" +RELEASE_PACKAGE_IDENTITY_FILE="cuvs-java-package-identity.json" RELEASE_SOURCE_ARTIFACT_NAME="cuvs-java-cuda12.9.1" RELEASE_SOURCE_SHA="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" RELEASE_CATALOG_KEY="maven:cuvs-java" export GITHUB_REPOSITORY GITHUB_RUN_ATTEMPT GITHUB_RUN_ID GITHUB_SHA GITHUB_WORKFLOW_REF -export RELEASE_ARTIFACTS RELEASE_ENTRIES_NAME RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE RELEASE_PACKAGE_FILE +export RELEASE_ARTIFACTS RELEASE_ARTIFACT_TYPE RELEASE_ENTRIES_NAME RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE_IDENTITY_FILE export RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY "${repository_root}/release-catalog/materialize.sh" @@ -77,14 +77,14 @@ generated_directory="${temporary_directory}/generated-bundle" mkdir -p "${generated_directory}/linux-64" printf '%s\n' conda >"${generated_directory}/linux-64/kvikio-26.08.00a32-cuda12_260714_2f567060.conda" -RELEASE_ARTIFACTS="$(jq -cn '[{path: "linux-64/kvikio-*.conda"}]')" +RELEASE_ARTIFACTS="$(jq -cn '[{path: "linux-64/kvikio-*.conda", package: {ecosystem: "conda", name: "kvikio", version: "26.08.00a32"}}]')" +RELEASE_ARTIFACT_TYPE=conda RELEASE_OUTPUT_DIRECTORY="${generated_directory}" -RELEASE_PACKAGE="$(jq -cn '{ecosystem: "conda", name: "kvikio"}')" -RELEASE_PACKAGE_FILE='' +RELEASE_PACKAGE_IDENTITY_FILE='' RELEASE_SOURCE_ARTIFACT_NAME="kvikio_conda_python_kvikio_x86_64_abi3_cu12" RELEASE_SOURCE_SHA="bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" RELEASE_CATALOG_KEY="conda:kvikio" -export RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE RELEASE_PACKAGE_FILE RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY +export RELEASE_ARTIFACTS RELEASE_ARTIFACT_TYPE RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE_IDENTITY_FILE RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY "${repository_root}/release-catalog/materialize.sh" @@ -113,13 +113,13 @@ wheel_directory="${temporary_directory}/wheel-bundle" mkdir -p "${wheel_directory}" printf '%s\n' wheel >"${wheel_directory}/libkvikio_cu12-26.8.0a32-py3-none-manylinux_2_28_x86_64.whl" -RELEASE_ARTIFACTS="$(jq -cn '[{path: "libkvikio_cu12-*.whl"}]')" +RELEASE_ARTIFACTS="$(jq -cn '[{path: "libkvikio_cu12-*.whl", package: {ecosystem: "wheel", name: "libkvikio-cu12", version: "26.8.0a32"}}]')" +RELEASE_ARTIFACT_TYPE=wheel RELEASE_OUTPUT_DIRECTORY="${wheel_directory}" -RELEASE_PACKAGE="$(jq -cn '{ecosystem: "wheel", name: "libkvikio-cu12"}')" RELEASE_SOURCE_ARTIFACT_NAME="kvikio_wheel_cpp_libkvikio_x86_64_cu12" RELEASE_SOURCE_SHA="cccccccccccccccccccccccccccccccccccccccc" RELEASE_CATALOG_KEY="wheel:kvikio" -export RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY +export RELEASE_ARTIFACTS RELEASE_ARTIFACT_TYPE RELEASE_OUTPUT_DIRECTORY RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY "${repository_root}/release-catalog/materialize.sh" @@ -133,18 +133,21 @@ jq -e ' missing_version_directory="${temporary_directory}/missing-version-bundle" mkdir -p "${missing_version_directory}" printf '%s\n' archive >"${missing_version_directory}/bundle.tar.gz" +jq -n \ + '{ecosystem: "archive", name: "bundle"}' \ + >"${missing_version_directory}/bundle-package-identity.json" RELEASE_ARTIFACTS="$(jq -cn '[{path: "bundle.tar.gz"}]')" +RELEASE_ARTIFACT_TYPE=custom RELEASE_OUTPUT_DIRECTORY="${missing_version_directory}" -RELEASE_PACKAGE="$(jq -cn '{ecosystem: "archive", name: "bundle"}')" -RELEASE_PACKAGE_FILE='' +RELEASE_PACKAGE_IDENTITY_FILE='bundle-package-identity.json' RELEASE_SOURCE_ARTIFACT_NAME="bundle-archive" RELEASE_SOURCE_SHA="dddddddddddddddddddddddddddddddddddddddd" RELEASE_CATALOG_KEY="archive:bundle" -export RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE RELEASE_PACKAGE_FILE RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY +export RELEASE_ARTIFACTS RELEASE_ARTIFACT_TYPE RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE_IDENTITY_FILE RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY if "${repository_root}/release-catalog/materialize.sh" 2>"${temporary_directory}/missing-version-error"; then echo "materialize.sh unexpectedly accepted a custom artifact without a package version" >&2 exit 1 fi -grep -Fx 'release-package version is required for archive artifacts' "${temporary_directory}/missing-version-error" +grep -Fx 'package identity file must contain non-empty ecosystem, name, and version strings and only optional build or platform strings' "${temporary_directory}/missing-version-error" From bf3b4c4096135a19218ecf488af4ea6638f9c751 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 18 Aug 2026 10:53:49 -0500 Subject: [PATCH 09/16] resolve release catalog identity per artifact --- release-catalog-dispatch/action.yml | 4 - release-catalog/README.md | 24 ++-- release-catalog/config.schema.json | 63 ++-------- release-catalog/describe-conda.sh | 17 ++- release-catalog/describe-wheels.sh | 17 ++- release-catalog/materialize.sh | 89 +++++-------- release-catalog/prepare.sh | 118 +++++++++++++++--- release-catalog/validate-config.sh | 26 +--- .../package-identity-file.json | 3 +- tests/release_catalog_config_test.sh | 43 +++---- tests/release_catalog_prepare_test.sh | 109 +++++++++++++--- tests/release_catalog_test.sh | 45 ++++--- 12 files changed, 324 insertions(+), 234 deletions(-) diff --git a/release-catalog-dispatch/action.yml b/release-catalog-dispatch/action.yml index d53309ca..c38cb790 100644 --- a/release-catalog-dispatch/action.yml +++ b/release-catalog-dispatch/action.yml @@ -57,9 +57,7 @@ runs: env: IMPLEMENTATION_PATH: ${{ steps.implementation.outputs.path }} RELEASE_ARTIFACTS: ${{ steps.configuration.outputs.artifacts }} - RELEASE_ARTIFACT_TYPE: ${{ steps.configuration.outputs.artifact_type }} RELEASE_OUTPUT_DIRECTORY: ${{ steps.configuration.outputs.output_directory }} - RELEASE_PACKAGE_IDENTITY_FILE: ${{ steps.configuration.outputs.package_identity_file }} run: "${IMPLEMENTATION_PATH}/prepare.sh" - id: materialize name: Materialize release catalog entries @@ -67,10 +65,8 @@ runs: env: IMPLEMENTATION_PATH: ${{ steps.implementation.outputs.path }} RELEASE_ARTIFACTS: ${{ steps.prepare.outputs.artifacts }} - RELEASE_ARTIFACT_TYPE: ${{ steps.configuration.outputs.artifact_type }} RELEASE_ENTRIES_NAME: release-catalog-entries.json RELEASE_OUTPUT_DIRECTORY: ${{ steps.configuration.outputs.output_directory }} - RELEASE_PACKAGE_IDENTITY_FILE: ${{ steps.configuration.outputs.package_identity_file }} RELEASE_SOURCE_ARTIFACT_NAME: ${{ inputs.source-artifact-name }} RELEASE_SOURCE_SHA: ${{ inputs.source-sha || github.sha }} RELEASE_CATALOG_KEY: ${{ steps.configuration.outputs.release_catalog_key }} diff --git a/release-catalog/README.md b/release-catalog/README.md index b1a7785c..8b765b2a 100644 --- a/release-catalog/README.md +++ b/release-catalog/README.md @@ -27,12 +27,12 @@ catalog; there is no separate metadata document to keep synchronized. Every producer passes one `config` JSON object. Its canonical schema and field documentation are in [`config.schema.json`](config.schema.json). -The `artifact_type` field determines how the action discovers primary files: - -- `conda` and `wheel` derive artifact descriptors and package identity from the - built packages; -- `custom` requires explicit artifact descriptors and a - `package_identity_file` created by the producer during the build. +When `artifacts` is omitted, the action discovers every Conda and wheel output +in `output_directory` and extracts identity from each package independently. +When `artifacts` is supplied, each descriptor is resolved independently: +supported Conda and wheel files are parsed, while any other artifact requires +its own `package_identity_file`. Evidence paths also belong to the individual +artifact descriptor. `release_catalog_key` identifies the release catalog entry that owns these artifacts. Every file from every matrix variant in the same publishable artifact @@ -82,7 +82,6 @@ the built source. with: config: >- { - "artifact_type": "wheel", "release_catalog_key": "wheel:example", "output_directory": ${{ toJSON(steps.package-name.outputs.WHEEL_OUTPUT_DIR) }} } @@ -98,11 +97,12 @@ the built source. with: config: >- { - "artifact_type": "custom", "release_catalog_key": "maven:cuvs-java", "output_directory": "java/cuvs-java/target", - "package_identity_file": "cuvs-java.release-package-identity.json", - "artifacts": [{"path": "cuvs-java-*-x86_64-cuda*.jar"}] + "artifacts": [{ + "path": "cuvs-java-*-x86_64-cuda*.jar", + "package_identity_file": "cuvs-java.release-package-identity.json" + }] } source-artifact-name: cuvs-java source-sha: ${{ env.RAPIDS_SHA }} @@ -110,8 +110,8 @@ the built source. Before the action runs, the producer creates `java/cuvs-java/target/cuvs-java.release-package-identity.json`. The -`package_identity_file` value is the path to that file relative to -`output_directory`. For example: +artifact descriptor's `package_identity_file` value is the path to that file +relative to `output_directory`. For example: ```json { diff --git a/release-catalog/config.schema.json b/release-catalog/config.schema.json index 5610a1d3..c8402e1d 100644 --- a/release-catalog/config.schema.json +++ b/release-catalog/config.schema.json @@ -6,19 +6,9 @@ "type": "object", "additionalProperties": false, "required": [ - "artifact_type", "release_catalog_key" ], "properties": { - "artifact_type": { - "description": "Metadata can be extracted from the artifact for conda/wheel types. More info needed for custom type.", - "type": "string", - "enum": [ - "conda", - "custom", - "wheel" - ] - }, "release_catalog_key": { "description": "Release-catalog entry name that owns these artifacts. Default for conda & wheels is :, such as conda:cudf. Custom jobs specify their own, such as maven:cuvs-java.", "type": "string", @@ -32,14 +22,8 @@ "pattern": "^[^\\r\\n]+$", "default": "." }, - "package_identity_file": { - "description": "Path, relative to output_directory, to producer-created package identity JSON. Required for custom artifacts; Conda and wheel identities are extracted from the artifacts. The file must contain ecosystem, name, and version and may contain build and platform.", - "type": "string", - "minLength": 1, - "pattern": "^[^\\r\\n]+$" - }, "artifacts": { - "description": "Primary release files and optional SBoM evidence sidecar files", + "description": "Optional explicit primary release files and their identity or evidence sidecars. When omitted, Conda and wheel artifacts are discovered in output_directory.", "type": "array", "minItems": 1, "items": { @@ -47,45 +31,6 @@ } } }, - "allOf": [ - { - "if": { - "properties": { - "artifact_type": { - "const": "custom" - } - }, - "required": [ - "artifact_type" - ] - }, - "then": { - "required": [ - "artifacts", - "package_identity_file" - ] - }, - "else": { - "required": [ - "output_directory" - ], - "not": { - "anyOf": [ - { - "required": [ - "artifacts" - ] - }, - { - "required": [ - "package_identity_file" - ] - } - ] - } - } - } - ], "$defs": { "artifact": { "type": "object", @@ -99,6 +44,12 @@ "minLength": 1, "pattern": "^[^\\r\\n]+$" }, + "package_identity_file": { + "description": "Path, relative to output_directory, to producer-created package identity JSON for this artifact. Required only when identity cannot be extracted from the artifact. The file must contain ecosystem, name, and version and may contain build and platform.", + "type": "string", + "minLength": 1, + "pattern": "^[^\\r\\n]+$" + }, "sbom": { "description": "Dependency SBOM path or glob. Example file: https://github.com/spdx/spdx-examples/blob/2181917ef6ff74de89252ee785583c27a38d6199/presentations/OSS-NA-2023/SPDXVersion2.3/03-SBOMwDependency.json", "type": "string", diff --git a/release-catalog/describe-conda.sh b/release-catalog/describe-conda.sh index 6449addd..23bdcf47 100755 --- a/release-catalog/describe-conda.sh +++ b/release-catalog/describe-conda.sh @@ -3,16 +3,25 @@ set -euo pipefail -if [[ "$#" -ne 1 || ! -d "$1" ]]; then - echo "usage: $0 CONDA_OUTPUT_DIRECTORY" >&2 +if [[ "$#" -lt 1 || ! -d "$1" ]]; then + echo "usage: $0 CONDA_OUTPUT_DIRECTORY [CONDA_PACKAGE ...]" >&2 exit 1 fi output_directory="$(realpath "$1")" descriptors='[]' package_count=0 +package_paths=() -while IFS= read -r package_path; do +if [[ "$#" -gt 1 ]]; then + package_paths=("${@:2}") +else + while IFS= read -r package_path; do + package_paths+=("${package_path}") + done < <(find "${output_directory}" -type f \( -name '*.conda' -o -name '*.tar.bz2' \) -print | sort) +fi + +for package_path in "${package_paths[@]}"; do package_path="$(realpath "${package_path}")" if [[ "${package_path}" != "${output_directory}"/* ]]; then echo "Conda package must resolve inside output directory: ${package_path}" >&2 @@ -59,7 +68,7 @@ while IFS= read -r package_path; do --argjson current "${descriptors}" \ '$current + [{path: $path, package: $package}]')" package_count=$((package_count + 1)) -done < <(find "${output_directory}" -type f \( -name '*.conda' -o -name '*.tar.bz2' \) -print | sort) +done if [[ "${package_count}" -eq 0 ]]; then echo "Conda output directory contains no .conda or .tar.bz2 packages: ${output_directory}" >&2 diff --git a/release-catalog/describe-wheels.sh b/release-catalog/describe-wheels.sh index bc6b925e..f440055f 100755 --- a/release-catalog/describe-wheels.sh +++ b/release-catalog/describe-wheels.sh @@ -3,16 +3,25 @@ set -euo pipefail -if [[ "$#" -ne 1 || ! -d "$1" ]]; then - echo "usage: $0 WHEEL_OUTPUT_DIRECTORY" >&2 +if [[ "$#" -lt 1 || ! -d "$1" ]]; then + echo "usage: $0 WHEEL_OUTPUT_DIRECTORY [WHEEL ...]" >&2 exit 1 fi output_directory="$(realpath "$1")" descriptors='[]' wheel_count=0 +wheel_paths=() + +if [[ "$#" -gt 1 ]]; then + wheel_paths=("${@:2}") +else + while IFS= read -r wheel_path; do + wheel_paths+=("${wheel_path}") + done < <(find "${output_directory}" -type f -name '*.whl' -print | sort) +fi -while IFS= read -r wheel_path; do +for wheel_path in "${wheel_paths[@]}"; do wheel_path="$(realpath "${wheel_path}")" if [[ "${wheel_path}" != "${output_directory}"/* ]]; then echo "wheel must resolve inside output directory: ${wheel_path}" >&2 @@ -44,7 +53,7 @@ while IFS= read -r wheel_path; do --argjson current "${descriptors}" \ '$current + [{path: $path, package: {ecosystem: "wheel", name: $name, version: $version}}]')" wheel_count=$((wheel_count + 1)) -done < <(find "${output_directory}" -type f -name '*.whl' -print | sort) +done if [[ "${wheel_count}" -eq 0 ]]; then echo "wheel output directory contains no .whl files: ${output_directory}" >&2 diff --git a/release-catalog/materialize.sh b/release-catalog/materialize.sh index 24c45fcf..ffc9d92b 100755 --- a/release-catalog/materialize.sh +++ b/release-catalog/materialize.sh @@ -13,7 +13,6 @@ require_nonempty() { } require_nonempty "RELEASE_CATALOG_KEY" "${RELEASE_CATALOG_KEY:-}" -require_nonempty "RELEASE_ARTIFACT_TYPE" "${RELEASE_ARTIFACT_TYPE:-}" require_nonempty "RELEASE_OUTPUT_DIRECTORY" "${RELEASE_OUTPUT_DIRECTORY:-}" require_nonempty "RELEASE_ENTRIES_NAME" "${RELEASE_ENTRIES_NAME:-}" require_nonempty "RELEASE_ARTIFACTS" "${RELEASE_ARTIFACTS:-}" @@ -48,42 +47,6 @@ ensure_relative_pattern() { } output_directory="$(realpath "${RELEASE_OUTPUT_DIRECTORY}")" -package_identity='' -case "${RELEASE_ARTIFACT_TYPE}" in - custom) - require_nonempty "RELEASE_PACKAGE_IDENTITY_FILE" "${RELEASE_PACKAGE_IDENTITY_FILE:-}" - ensure_relative_pattern "package-identity-file" "${RELEASE_PACKAGE_IDENTITY_FILE}" - package_identity_path="$(realpath "${output_directory}/${RELEASE_PACKAGE_IDENTITY_FILE}")" - if [[ "${package_identity_path}" != "${output_directory}"/* || ! -f "${package_identity_path}" ]]; then - echo "package-identity-file must resolve to one file inside output-directory: ${RELEASE_PACKAGE_IDENTITY_FILE}" >&2 - exit 1 - fi - package_identity="$(jq -c . "${package_identity_path}")" - ;; - conda | wheel) - if [[ -n "${RELEASE_PACKAGE_IDENTITY_FILE:-}" ]]; then - echo "package-identity-file is only valid for custom artifacts" >&2 - exit 1 - fi - ;; - *) - echo "artifact-type must be one of: conda, custom, wheel" >&2 - exit 1 - ;; -esac - -if [[ "${RELEASE_ARTIFACT_TYPE}" == "custom" ]] && ! jq -e ' - type == "object" - and (keys - ["ecosystem", "name", "version", "build", "platform"] | length == 0) - and (.ecosystem | type == "string" and length > 0) - and (.name | type == "string" and length > 0) - and (.version | type == "string" and length > 0) - and ((has("build") | not) or (.build | type == "string" and length > 0)) - and ((has("platform") | not) or (.platform | type == "string" and length > 0)) -' <<<"${package_identity}" >/dev/null; then - echo "package identity file must contain non-empty ecosystem, name, and version strings and only optional build or platform strings" >&2 - exit 1 -fi if ! jq -e 'type == "array" and length > 0' <<<"${RELEASE_ARTIFACTS}" >/dev/null; then echo "release-artifacts must be a non-empty JSON array" >&2 @@ -119,6 +82,18 @@ resolve_one_file() { printf '%s\n' "${resolved#"${output_directory}/"}" } +validate_package_identity() { + jq -e ' + type == "object" + and (keys - ["ecosystem", "name", "version", "build", "platform"] | length == 0) + and (.ecosystem | type == "string" and length > 0) + and (.name | type == "string" and length > 0) + and (.version | type == "string" and length > 0) + and ((has("build") | not) or (.build | type == "string" and length > 0)) + and ((has("platform") | not) or (.platform | type == "string" and length > 0)) + ' >/dev/null +} + generated_evidence_path() { local primary_path="$1" local kind="$2" @@ -220,34 +195,17 @@ write_generated_provenance() { shopt -s globstar nullglob while IFS= read -r descriptor; do - if ! jq -e --arg artifact_type "${RELEASE_ARTIFACT_TYPE}" ' + if ! jq -e ' type == "object" - and ( - if $artifact_type == "custom" then - (keys - ["path", "sbom", "provenance", "signature"] | length == 0) - else - (keys - ["path", "sbom", "provenance", "signature", "package"] | length == 0) - end - ) + and (keys - ["path", "package_identity_file", "sbom", "provenance", "signature", "package"] | length == 0) and (.path | type == "string" and length > 0) + and ((.package_identity_file // "") | type == "string") and ((.sbom // "") | type == "string") and ((.provenance // "") | type == "string") and ((.signature // "") | type == "string") - and ( - if $artifact_type == "custom" then - has("package") | not - else - (.package | type == "object") - and (.package | keys - ["ecosystem", "name", "version", "build", "platform"] | length == 0) - and (.package.ecosystem | type == "string" and length > 0) - and (.package.name | type == "string" and length > 0) - and (.package.version | type == "string" and length > 0) - and ((.package | has("build") | not) or (.package.build | type == "string" and length > 0)) - and ((.package | has("platform") | not) or (.package.platform | type == "string" and length > 0)) - end - ) + and ([has("package"), has("package_identity_file")] | map(select(.)) | length == 1) ' <<<"${descriptor}" >/dev/null; then - echo "release artifact descriptor is invalid for ${RELEASE_ARTIFACT_TYPE}: ${descriptor}" >&2 + echo "release artifact descriptor must contain path, exactly one package identity source, and optional evidence paths: ${descriptor}" >&2 exit 1 fi @@ -255,11 +213,20 @@ while IFS= read -r descriptor; do sbom_pattern="$(jq -r '.sbom // empty' <<<"${descriptor}")" provenance_pattern="$(jq -r '.provenance // empty' <<<"${descriptor}")" signature_pattern="$(jq -r '.signature // empty' <<<"${descriptor}")" - if [[ "${RELEASE_ARTIFACT_TYPE}" == "custom" ]]; then - package="${package_identity}" + package_identity_file="$(jq -r '.package_identity_file // empty' <<<"${descriptor}")" + if [[ -n "${package_identity_file}" ]]; then + package_identity_path="$(resolve_one_file package_identity_file "${package_identity_file}")" + if ! package="$(jq -ce . "${output_directory}/${package_identity_path}" 2>/dev/null)"; then + echo "package_identity_file must contain valid JSON: ${package_identity_file}" >&2 + exit 1 + fi else package="$(jq -c '.package' <<<"${descriptor}")" fi + if ! validate_package_identity <<<"${package}"; then + echo "package identity for ${primary_path} must contain non-empty ecosystem, name, and version strings and only optional build or platform strings" >&2 + exit 1 + fi if [[ -n "${sbom_pattern}" ]]; then supplied_sbom_path="$(resolve_one_file sbom "${sbom_pattern}")" diff --git a/release-catalog/prepare.sh b/release-catalog/prepare.sh index 4a803d28..9246974e 100755 --- a/release-catalog/prepare.sh +++ b/release-catalog/prepare.sh @@ -12,31 +12,109 @@ require_nonempty() { fi } -require_nonempty "RELEASE_ARTIFACT_TYPE" "${RELEASE_ARTIFACT_TYPE:-}" require_nonempty "RELEASE_OUTPUT_DIRECTORY" "${RELEASE_OUTPUT_DIRECTORY:-}" script_directory="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" artifacts="${RELEASE_ARTIFACTS:-}" +if [[ ! -d "${RELEASE_OUTPUT_DIRECTORY}" ]]; then + echo "output-directory does not exist or is not a directory: ${RELEASE_OUTPUT_DIRECTORY}" >&2 + exit 1 +fi +output_directory="$(realpath "${RELEASE_OUTPUT_DIRECTORY}")" -case "${RELEASE_ARTIFACT_TYPE}" in - conda) - if [[ -z "${artifacts}" ]]; then - artifacts="$("${script_directory}/describe-conda.sh" "${RELEASE_OUTPUT_DIRECTORY}")" - fi - ;; - wheel) - if [[ -z "${artifacts}" ]]; then - artifacts="$("${script_directory}/describe-wheels.sh" "${RELEASE_OUTPUT_DIRECTORY}")" - fi - ;; - custom) - require_nonempty "RELEASE_ARTIFACTS" "${artifacts}" - require_nonempty "RELEASE_PACKAGE_IDENTITY_FILE" "${RELEASE_PACKAGE_IDENTITY_FILE:-}" - ;; - *) - echo "artifact-type must be one of: conda, custom, wheel" >&2 +ensure_relative_pattern() { + local field="$1" + local pattern="$2" + if [[ "${pattern}" == /* || "${pattern}" == */../* || "${pattern}" == ../* || "${pattern}" == *"/.." ]]; then + echo "${field} must be a relative path inside output-directory: ${pattern}" >&2 + exit 1 + fi +} + +resolve_primary_file() { + local pattern="$1" + local -a matches=() + + ensure_relative_pattern "artifact path" "${pattern}" + while IFS= read -r match; do + matches+=("${match}") + done < <(compgen -G "${output_directory}/${pattern}" || true) + if [[ "${#matches[@]}" -ne 1 || ! -f "${matches[0]:-}" ]]; then + echo "artifact path must resolve to exactly one file: ${pattern}" >&2 + exit 1 + fi + + local resolved + resolved="$(realpath "${matches[0]}")" + if [[ "${resolved}" != "${output_directory}"/* ]]; then + echo "artifact path must resolve inside output-directory: ${pattern}" >&2 exit 1 - ;; -esac + fi + printf '%s\n' "${resolved}" +} + +if [[ -z "${artifacts}" ]]; then + conda_descriptors='[]' + wheel_descriptors='[]' + if [[ -n "$(find "${output_directory}" -type f \( -name '*.conda' -o -name '*.tar.bz2' \) -print -quit)" ]]; then + conda_descriptors="$("${script_directory}/describe-conda.sh" "${output_directory}")" + fi + if [[ -n "$(find "${output_directory}" -type f -name '*.whl' -print -quit)" ]]; then + wheel_descriptors="$("${script_directory}/describe-wheels.sh" "${output_directory}")" + fi + artifacts="$(jq -cn --argjson conda "${conda_descriptors}" --argjson wheel "${wheel_descriptors}" '$conda + $wheel')" + if [[ "$(jq 'length' <<<"${artifacts}")" -eq 0 ]]; then + echo "output-directory contains no detectable Conda or wheel artifacts; explicitly selected artifacts require package_identity_file when their identity cannot be parsed" >&2 + exit 1 + fi +else + prepared_artifacts='[]' + while IFS= read -r descriptor; do + primary_file="$(resolve_primary_file "$(jq -r '.path' <<<"${descriptor}")")" + identity_file="$(jq -r '.package_identity_file // empty' <<<"${descriptor}")" + package='' + + case "${primary_file}" in + *.whl) + if [[ -n "${identity_file}" ]]; then + echo "package_identity_file is not allowed when wheel identity can be extracted: $(jq -r '.path' <<<"${descriptor}")" >&2 + exit 1 + fi + package="$("${script_directory}/describe-wheels.sh" "${output_directory}" "${primary_file}" | jq -c '.[0].package')" + ;; + *.conda) + if [[ -n "${identity_file}" ]]; then + echo "package_identity_file is not allowed when Conda identity can be extracted: $(jq -r '.path' <<<"${descriptor}")" >&2 + exit 1 + fi + package="$("${script_directory}/describe-conda.sh" "${output_directory}" "${primary_file}" | jq -c '.[0].package')" + ;; + *.tar.bz2) + if conda_descriptor="$("${script_directory}/describe-conda.sh" "${output_directory}" "${primary_file}" 2>/dev/null)"; then + if [[ -n "${identity_file}" ]]; then + echo "package_identity_file is not allowed when Conda identity can be extracted: $(jq -r '.path' <<<"${descriptor}")" >&2 + exit 1 + fi + package="$(jq -c '.[0].package' <<<"${conda_descriptor}")" + elif [[ -z "${identity_file}" ]]; then + echo "artifact is not a valid Conda package and requires package_identity_file: $(jq -r '.path' <<<"${descriptor}")" >&2 + exit 1 + fi + ;; + *) + if [[ -z "${identity_file}" ]]; then + echo "artifact identity cannot be extracted; package_identity_file is required: $(jq -r '.path' <<<"${descriptor}")" >&2 + exit 1 + fi + ;; + esac + + if [[ -n "${package}" ]]; then + descriptor="$(jq -c --argjson package "${package}" '. + {package: $package}' <<<"${descriptor}")" + fi + prepared_artifacts="$(jq -cn --argjson current "${prepared_artifacts}" --argjson descriptor "${descriptor}" '$current + [$descriptor]')" + done < <(jq -c '.[]' <<<"${artifacts}") + artifacts="${prepared_artifacts}" +fi printf 'artifacts=%s\n' "${artifacts}" >>"${GITHUB_OUTPUT}" diff --git a/release-catalog/validate-config.sh b/release-catalog/validate-config.sh index 9cbfafd1..042a424a 100755 --- a/release-catalog/validate-config.sh +++ b/release-catalog/validate-config.sh @@ -26,35 +26,23 @@ validation_errors="$(jq -r ' if type != "object" then ["release catalog configuration must be a JSON object"] else - (keys - ["artifact_type", "release_catalog_key", "output_directory", "package_identity_file", "artifacts"]) as $unknown + (keys - ["release_catalog_key", "output_directory", "artifacts"]) as $unknown | [ if ($unknown | length) > 0 then "unknown field(s): " + ($unknown | join(", ")) else empty end, - if (.artifact_type == "conda" or .artifact_type == "custom" or .artifact_type == "wheel") then empty - else "artifact_type must be one of: conda, custom, wheel" end, if (.release_catalog_key | single_line_string) then empty else "release_catalog_key must be a non-empty, single-line string" end, if (has("output_directory") | not) or (.output_directory | single_line_string) then empty else "output_directory must be a non-empty, single-line string when supplied" end ] - + if .artifact_type == "custom" then + + if has("artifacts") then [ - if (.package_identity_file | single_line_string) then empty - else "package_identity_file is required for custom artifacts and must be a non-empty, single-line path relative to output_directory" end, if (.artifacts | type == "array" and length > 0) then empty - else "artifacts must be a non-empty array for custom artifacts" end - ] - elif .artifact_type == "conda" or .artifact_type == "wheel" then - [ - if (.output_directory | single_line_string) then empty - else "output_directory is required for conda and wheel artifacts" end, - if has("artifacts") or has("package_identity_file") then - "artifacts and package_identity_file are only valid when artifact_type is custom" - else empty end + else "artifacts must be a non-empty array when supplied" end ] else [] end - + if .artifact_type == "custom" and (.artifacts | type) == "array" then + + if (.artifacts | type) == "array" then [ .artifacts | to_entries[] | .key as $index @@ -62,14 +50,14 @@ validation_errors="$(jq -r ' | if ($artifact | type) != "object" then "artifacts[\($index)] must be an object" else - ($artifact | keys - ["path", "sbom", "provenance", "signature"]) as $artifact_unknown + ($artifact | keys - ["path", "package_identity_file", "sbom", "provenance", "signature"]) as $artifact_unknown | if ($artifact_unknown | length) > 0 then "artifacts[\($index)] has unknown field(s): " + ($artifact_unknown | join(", ")) else empty end, if ($artifact.path | single_line_string) then empty else "artifacts[\($index)].path must be a non-empty, single-line string" end, ($artifact | to_entries[] - | select(.key == "sbom" or .key == "provenance" or .key == "signature") + | select(.key == "package_identity_file" or .key == "sbom" or .key == "provenance" or .key == "signature") | select((.value | single_line_string) | not) | "artifacts[\($index)].\(.key) must be a non-empty, single-line string") end @@ -92,9 +80,7 @@ if [[ -z "${GITHUB_OUTPUT:-}" ]]; then fi { - printf 'artifact_type=%s\n' "$(jq -r '.artifact_type' <<<"${compact_config}")" printf 'release_catalog_key=%s\n' "$(jq -r '.release_catalog_key' <<<"${compact_config}")" printf 'output_directory=%s\n' "$(jq -r '.output_directory // "."' <<<"${compact_config}")" - printf 'package_identity_file=%s\n' "$(jq -r '.package_identity_file // empty' <<<"${compact_config}")" printf 'artifacts=%s\n' "$(jq -c '.artifacts // empty' <<<"${compact_config}")" } >>"${GITHUB_OUTPUT}" diff --git a/tests/release-catalog-config/package-identity-file.json b/tests/release-catalog-config/package-identity-file.json index 0c6bacd8..46068c04 100644 --- a/tests/release-catalog-config/package-identity-file.json +++ b/tests/release-catalog-config/package-identity-file.json @@ -1,11 +1,10 @@ { - "artifact_type": "custom", "release_catalog_key": "maven:cuvs-java", "output_directory": "java/cuvs-java/target", - "package_identity_file": "cuvs-java.release-package-identity.json", "artifacts": [ { "path": "cuvs-java-*-x86_64-cuda*.jar", + "package_identity_file": "cuvs-java.release-package-identity.json", "sbom": "cuvs-java.spdx.json" } ] diff --git a/tests/release_catalog_config_test.sh b/tests/release_catalog_config_test.sh index ccfe3959..c0776a4d 100755 --- a/tests/release_catalog_config_test.sh +++ b/tests/release_catalog_config_test.sh @@ -31,10 +31,8 @@ RELEASE_CATALOG_CONFIG="$(<"${repository_root}/tests/release-catalog-config/pack GITHUB_OUTPUT="${valid_output}" "${validator}" grep -Fx 'release_catalog_key=maven:cuvs-java' "${valid_output}" -grep -Fx 'artifact_type=custom' "${valid_output}" grep -Fx 'output_directory=java/cuvs-java/target' "${valid_output}" -grep -Fx 'package_identity_file=cuvs-java.release-package-identity.json' "${valid_output}" -grep -Fx 'artifacts=[{"path":"cuvs-java-*-x86_64-cuda*.jar","sbom":"cuvs-java.spdx.json"}]' "${valid_output}" +grep -Fx 'artifacts=[{"path":"cuvs-java-*-x86_64-cuda*.jar","package_identity_file":"cuvs-java.release-package-identity.json","sbom":"cuvs-java.spdx.json"}]' "${valid_output}" assert_invalid \ malformed-json \ @@ -50,54 +48,53 @@ assert_invalid \ missing-fields \ '{}' \ 'release_catalog_key must be a non-empty, single-line string' -grep -F 'artifact_type must be one of: conda, custom, wheel' "${temporary_directory}/missing-fields.error" >/dev/null assert_invalid \ - custom-missing-fields \ - '{"artifact_type":"custom","release_catalog_key":"archive:smoke"}' \ - 'package_identity_file is required for custom artifacts' -grep -F 'artifacts must be a non-empty array for custom artifacts' "${temporary_directory}/custom-missing-fields.error" >/dev/null + top-level-package-identity-file \ + '{"release_catalog_key":"archive:smoke","package_identity_file":"package.json"}' \ + 'unknown field(s): package_identity_file' assert_invalid \ unknown-field \ - '{"artifact_type":"custom","release_catalog_key":"archive:smoke","package_identity_file":"package.json","artifacts":[{"path":"smoke.tar.gz"}],"component_id":"archive:smoke"}' \ + '{"release_catalog_key":"archive:smoke","artifacts":[{"path":"smoke.tar.gz","package_identity_file":"package.json"}],"component_id":"archive:smoke"}' \ 'unknown field(s): component_id' +assert_invalid \ + artifact-type \ + '{"artifact_type":"custom","release_catalog_key":"archive:smoke","artifacts":[{"path":"smoke.tar.gz","package_identity_file":"package.json"}]}' \ + 'unknown field(s): artifact_type' + assert_invalid \ inline-package \ - '{"artifact_type":"custom","release_catalog_key":"archive:smoke","package_identity_file":"package.json","package":{"ecosystem":"archive","name":"smoke","version":"1.0"},"artifacts":[{"path":"smoke.tar.gz"}]}' \ + '{"release_catalog_key":"archive:smoke","package":{"ecosystem":"archive","name":"smoke","version":"1.0"},"artifacts":[{"path":"smoke.tar.gz","package_identity_file":"package.json"}]}' \ 'unknown field(s): package' assert_invalid \ per-artifact-package \ - '{"artifact_type":"custom","release_catalog_key":"archive:smoke","package_identity_file":"package.json","artifacts":[{"path":"smoke.tar.gz","package":{"ecosystem":"archive","name":"smoke","version":"1.0"}}]}' \ + '{"release_catalog_key":"archive:smoke","artifacts":[{"path":"smoke.tar.gz","package":{"ecosystem":"archive","name":"smoke","version":"1.0"}}]}' \ 'artifacts[0] has unknown field(s): package' assert_invalid \ malformed-artifact \ - '{"artifact_type":"custom","release_catalog_key":"archive:smoke","package_identity_file":"package.json","artifacts":[{"file":"smoke.tar.gz","sbom":false}]}' \ + '{"release_catalog_key":"archive:smoke","artifacts":[{"file":"smoke.tar.gz","package_identity_file":false,"sbom":false}]}' \ 'artifacts[0] has unknown field(s): file' grep -F 'artifacts[0].path must be a non-empty, single-line string' "${temporary_directory}/malformed-artifact.error" >/dev/null +grep -F 'artifacts[0].package_identity_file must be a non-empty, single-line string' "${temporary_directory}/malformed-artifact.error" >/dev/null grep -F 'artifacts[0].sbom must be a non-empty, single-line string' "${temporary_directory}/malformed-artifact.error" >/dev/null -assert_invalid \ - standard-missing-output-directory \ - '{"artifact_type":"conda","release_catalog_key":"conda:smoke"}' \ - 'output_directory is required for conda and wheel artifacts' - -assert_invalid \ - standard-custom-fields \ - '{"artifact_type":"wheel","release_catalog_key":"wheel:smoke","output_directory":"dist","artifacts":[{"path":"smoke.whl"}]}' \ - 'artifacts and package_identity_file are only valid when artifact_type is custom' +default_output="${temporary_directory}/default.output" +RELEASE_CATALOG_CONFIG='{"release_catalog_key":"conda:smoke"}' \ + GITHUB_OUTPUT="${default_output}" "${validator}" +grep -Fx 'release_catalog_key=conda:smoke' "${default_output}" +grep -Fx 'output_directory=.' "${default_output}" +grep -Fx 'artifacts=' "${default_output}" standard_output="${temporary_directory}/standard.output" RELEASE_CATALOG_CONFIG='{ - "artifact_type": "wheel", "release_catalog_key": "wheel:kvikio", "output_directory": "dist" }' GITHUB_OUTPUT="${standard_output}" "${validator}" -grep -Fx 'artifact_type=wheel' "${standard_output}" grep -Fx 'release_catalog_key=wheel:kvikio' "${standard_output}" grep -Fx 'output_directory=dist' "${standard_output}" grep -Fx 'artifacts=' "${standard_output}" diff --git a/tests/release_catalog_prepare_test.sh b/tests/release_catalog_prepare_test.sh index 399df071..f83e90db 100755 --- a/tests/release_catalog_prepare_test.sh +++ b/tests/release_catalog_prepare_test.sh @@ -20,10 +20,8 @@ printf '%s\n' \ GITHUB_OUTPUT="${temporary_directory}/wheel-output" RELEASE_ARTIFACTS='' -RELEASE_ARTIFACT_TYPE=wheel RELEASE_OUTPUT_DIRECTORY="${temporary_directory}" -RELEASE_PACKAGE_IDENTITY_FILE='' -export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_ARTIFACT_TYPE RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE_IDENTITY_FILE +export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY "${repository_root}/release-catalog/prepare.sh" @@ -35,21 +33,104 @@ jq -e ' }] ' <<<"${artifacts}" >/dev/null +conda_output_directory="${temporary_directory}/conda" +conda_staging_directory="${temporary_directory}/conda-staging" +mkdir -p "${conda_output_directory}/noarch" "${conda_staging_directory}/info" +jq -n \ + '{name: "rapids-dask-dependency", version: "26.08.0", build: "py_0", subdir: "noarch"}' \ + >"${conda_staging_directory}/info/index.json" +tar -cjf \ + "${conda_output_directory}/noarch/rapids-dask-dependency-26.08.0-py_0.tar.bz2" \ + -C "${conda_staging_directory}" \ + info/index.json + +GITHUB_OUTPUT="${temporary_directory}/conda-output" +RELEASE_OUTPUT_DIRECTORY="${conda_output_directory}" +export GITHUB_OUTPUT RELEASE_OUTPUT_DIRECTORY + +"${repository_root}/release-catalog/prepare.sh" + +artifacts="$(sed -n 's/^artifacts=//p' "${GITHUB_OUTPUT}")" +jq -e ' + . == [{ + path: "noarch/rapids-dask-dependency-26.08.0-py_0.tar.bz2", + package: { + ecosystem: "conda", + name: "rapids-dask-dependency", + version: "26.08.0", + build: "py_0", + platform: "noarch" + } + }] +' <<<"${artifacts}" >/dev/null + GITHUB_OUTPUT="${temporary_directory}/custom-output" -RELEASE_ARTIFACTS="$(jq -cn '[{path: "bundle.tar.gz", sbom: "bundle.spdx.json"}]')" -RELEASE_ARTIFACT_TYPE=custom -RELEASE_PACKAGE_IDENTITY_FILE=release-package-identity.json -export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_ARTIFACT_TYPE RELEASE_PACKAGE_IDENTITY_FILE +RELEASE_OUTPUT_DIRECTORY="${temporary_directory}" +printf '%s\n' bundle >"${temporary_directory}/bundle.tar.gz" +jq -n '{ecosystem: "archive", name: "bundle", version: "1.0"}' \ + >"${temporary_directory}/release-package-identity.json" +RELEASE_ARTIFACTS="$(jq -cn '[{path: "bundle.tar.gz", package_identity_file: "release-package-identity.json", sbom: "bundle.spdx.json"}]')" +export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY "${repository_root}/release-catalog/prepare.sh" -grep -Fx 'artifacts=[{"path":"bundle.tar.gz","sbom":"bundle.spdx.json"}]' "${GITHUB_OUTPUT}" +grep -Fx 'artifacts=[{"path":"bundle.tar.gz","package_identity_file":"release-package-identity.json","sbom":"bundle.spdx.json"}]' "${GITHUB_OUTPUT}" + +GITHUB_OUTPUT="${temporary_directory}/explicit-wheel-output" +RELEASE_OUTPUT_DIRECTORY="${temporary_directory}" +RELEASE_ARTIFACTS="$(jq -cn '[{path: "libkvikio_cu12-*.whl", signature: "libkvikio.sig"}]')" +export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY +"${repository_root}/release-catalog/prepare.sh" +artifacts="$(sed -n 's/^artifacts=//p' "${GITHUB_OUTPUT}")" +jq -e ' + . == [{ + path: "libkvikio_cu12-*.whl", + signature: "libkvikio.sig", + package: {ecosystem: "wheel", name: "libkvikio-cu12", version: "26.8.0"} + }] +' <<<"${artifacts}" >/dev/null + +mixed_directory="${temporary_directory}/mixed" +mkdir -p "${mixed_directory}" +cp "${temporary_directory}/libkvikio_cu12-26.8.0-py3-none-manylinux_2_28_x86_64.whl" "${mixed_directory}/" +cp "${conda_output_directory}/noarch/rapids-dask-dependency-26.08.0-py_0.tar.bz2" "${mixed_directory}/" +GITHUB_OUTPUT="${temporary_directory}/mixed-output" +RELEASE_ARTIFACTS='' +RELEASE_OUTPUT_DIRECTORY="${mixed_directory}" +export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY +"${repository_root}/release-catalog/prepare.sh" +artifacts="$(sed -n 's/^artifacts=//p' "${GITHUB_OUTPUT}")" +jq -e ' + length == 2 + and ([.[].package.ecosystem] | sort) == ["conda", "wheel"] +' <<<"${artifacts}" >/dev/null + +empty_directory="${temporary_directory}/empty" +mkdir -p "${empty_directory}" +GITHUB_OUTPUT="${temporary_directory}/empty-output" +RELEASE_OUTPUT_DIRECTORY="${empty_directory}" +export GITHUB_OUTPUT RELEASE_OUTPUT_DIRECTORY +if "${repository_root}/release-catalog/prepare.sh" 2>"${temporary_directory}/empty-error"; then + echo "prepare.sh unexpectedly accepted an output directory without detectable artifacts" >&2 + exit 1 +fi +grep -Fx 'output-directory contains no detectable Conda or wheel artifacts; explicitly selected artifacts require package_identity_file when their identity cannot be parsed' "${temporary_directory}/empty-error" + +GITHUB_OUTPUT="${temporary_directory}/unsupported-output" +RELEASE_OUTPUT_DIRECTORY="${temporary_directory}" +RELEASE_ARTIFACTS="$(jq -cn '[{path: "bundle.tar.gz"}]')" +export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY +if "${repository_root}/release-catalog/prepare.sh" 2>"${temporary_directory}/unsupported-error"; then + echo "prepare.sh unexpectedly accepted an unsupported artifact without identity" >&2 + exit 1 +fi +grep -Fx 'artifact identity cannot be extracted; package_identity_file is required: bundle.tar.gz' "${temporary_directory}/unsupported-error" -GITHUB_OUTPUT="${temporary_directory}/invalid-output" -RELEASE_ARTIFACT_TYPE=archive -export GITHUB_OUTPUT RELEASE_ARTIFACT_TYPE -if "${repository_root}/release-catalog/prepare.sh" 2>"${temporary_directory}/invalid-error"; then - echo "prepare.sh unexpectedly accepted an invalid artifact type" >&2 +GITHUB_OUTPUT="${temporary_directory}/conflicting-identity-output" +RELEASE_ARTIFACTS="$(jq -cn '[{path: "libkvikio_cu12-*.whl", package_identity_file: "release-package-identity.json"}]')" +export GITHUB_OUTPUT RELEASE_ARTIFACTS +if "${repository_root}/release-catalog/prepare.sh" 2>"${temporary_directory}/conflicting-identity-error"; then + echo "prepare.sh unexpectedly accepted an identity file for a parseable artifact" >&2 exit 1 fi -grep -Fx 'artifact-type must be one of: conda, custom, wheel' "${temporary_directory}/invalid-error" +grep -Fx 'package_identity_file is not allowed when wheel identity can be extracted: libkvikio_cu12-*.whl' "${temporary_directory}/conflicting-identity-error" diff --git a/tests/release_catalog_test.sh b/tests/release_catalog_test.sh index db347cf3..50badf06 100755 --- a/tests/release_catalog_test.sh +++ b/tests/release_catalog_test.sh @@ -24,16 +24,14 @@ GITHUB_RUN_ID="1234" GITHUB_SHA="0123456789012345678901234567890123456789" GITHUB_WORKFLOW_REF="rapidsai/cuvs/.github/workflows/build.yaml@refs/heads/release/26.08" export GITHUB_OUTPUT -RELEASE_ARTIFACTS="$(jq -cn '[{path: "cuvs-java-*.jar", sbom: "cuvs-java-*.spdx.json", provenance: "cuvs-java-*.provenance.jsonl", signature: "cuvs-java-*.jar.asc"}]')" -RELEASE_ARTIFACT_TYPE=custom +RELEASE_ARTIFACTS="$(jq -cn '[{path: "cuvs-java-*.jar", package_identity_file: "cuvs-java-package-identity.json", sbom: "cuvs-java-*.spdx.json", provenance: "cuvs-java-*.provenance.jsonl", signature: "cuvs-java-*.jar.asc"}]')" RELEASE_ENTRIES_NAME="release-catalog-entries.json" RELEASE_OUTPUT_DIRECTORY="${bundle_directory}" -RELEASE_PACKAGE_IDENTITY_FILE="cuvs-java-package-identity.json" RELEASE_SOURCE_ARTIFACT_NAME="cuvs-java-cuda12.9.1" RELEASE_SOURCE_SHA="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" RELEASE_CATALOG_KEY="maven:cuvs-java" export GITHUB_REPOSITORY GITHUB_RUN_ATTEMPT GITHUB_RUN_ID GITHUB_SHA GITHUB_WORKFLOW_REF -export RELEASE_ARTIFACTS RELEASE_ARTIFACT_TYPE RELEASE_ENTRIES_NAME RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE_IDENTITY_FILE +export RELEASE_ARTIFACTS RELEASE_ENTRIES_NAME RELEASE_OUTPUT_DIRECTORY export RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY "${repository_root}/release-catalog/materialize.sh" @@ -73,18 +71,40 @@ while IFS= read -r evidence_path; do test -f "${isolated_companion_directory}/${evidence_path}" done < <(jq -r '.entries[] | .sbom, .provenance, (.signature // empty)' "${isolated_companion_directory}/release-catalog-entries.json") +multiple_identity_directory="${temporary_directory}/multiple-identities" +mkdir -p "${multiple_identity_directory}" +printf '%s\n' first >"${multiple_identity_directory}/first.tar.gz" +printf '%s\n' second >"${multiple_identity_directory}/second.jar" +jq -n '{ecosystem: "archive", name: "first", version: "1.0"}' \ + >"${multiple_identity_directory}/first.identity.json" +jq -n '{ecosystem: "maven", name: "example:second", version: "2.0"}' \ + >"${multiple_identity_directory}/second.identity.json" +RELEASE_ARTIFACTS="$(jq -cn '[ + {path: "first.tar.gz", package_identity_file: "first.identity.json"}, + {path: "second.jar", package_identity_file: "second.identity.json"} +]')" +RELEASE_OUTPUT_DIRECTORY="${multiple_identity_directory}" +RELEASE_SOURCE_ARTIFACT_NAME="multiple-identities" +RELEASE_SOURCE_SHA="eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" +RELEASE_CATALOG_KEY="archive:multiple" +export RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY +"${repository_root}/release-catalog/materialize.sh" +jq -e ' + (.entries | length == 2) + and .entries[0].package == {ecosystem: "archive", name: "first", version: "1.0"} + and .entries[1].package == {ecosystem: "maven", name: "example:second", version: "2.0"} +' "${multiple_identity_directory}/release-catalog-entries.json" >/dev/null + generated_directory="${temporary_directory}/generated-bundle" mkdir -p "${generated_directory}/linux-64" printf '%s\n' conda >"${generated_directory}/linux-64/kvikio-26.08.00a32-cuda12_260714_2f567060.conda" RELEASE_ARTIFACTS="$(jq -cn '[{path: "linux-64/kvikio-*.conda", package: {ecosystem: "conda", name: "kvikio", version: "26.08.00a32"}}]')" -RELEASE_ARTIFACT_TYPE=conda RELEASE_OUTPUT_DIRECTORY="${generated_directory}" -RELEASE_PACKAGE_IDENTITY_FILE='' RELEASE_SOURCE_ARTIFACT_NAME="kvikio_conda_python_kvikio_x86_64_abi3_cu12" RELEASE_SOURCE_SHA="bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" RELEASE_CATALOG_KEY="conda:kvikio" -export RELEASE_ARTIFACTS RELEASE_ARTIFACT_TYPE RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE_IDENTITY_FILE RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY +export RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY "${repository_root}/release-catalog/materialize.sh" @@ -114,12 +134,11 @@ mkdir -p "${wheel_directory}" printf '%s\n' wheel >"${wheel_directory}/libkvikio_cu12-26.8.0a32-py3-none-manylinux_2_28_x86_64.whl" RELEASE_ARTIFACTS="$(jq -cn '[{path: "libkvikio_cu12-*.whl", package: {ecosystem: "wheel", name: "libkvikio-cu12", version: "26.8.0a32"}}]')" -RELEASE_ARTIFACT_TYPE=wheel RELEASE_OUTPUT_DIRECTORY="${wheel_directory}" RELEASE_SOURCE_ARTIFACT_NAME="kvikio_wheel_cpp_libkvikio_x86_64_cu12" RELEASE_SOURCE_SHA="cccccccccccccccccccccccccccccccccccccccc" RELEASE_CATALOG_KEY="wheel:kvikio" -export RELEASE_ARTIFACTS RELEASE_ARTIFACT_TYPE RELEASE_OUTPUT_DIRECTORY RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY +export RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY "${repository_root}/release-catalog/materialize.sh" @@ -137,17 +156,15 @@ jq -n \ '{ecosystem: "archive", name: "bundle"}' \ >"${missing_version_directory}/bundle-package-identity.json" -RELEASE_ARTIFACTS="$(jq -cn '[{path: "bundle.tar.gz"}]')" -RELEASE_ARTIFACT_TYPE=custom +RELEASE_ARTIFACTS="$(jq -cn '[{path: "bundle.tar.gz", package_identity_file: "bundle-package-identity.json"}]')" RELEASE_OUTPUT_DIRECTORY="${missing_version_directory}" -RELEASE_PACKAGE_IDENTITY_FILE='bundle-package-identity.json' RELEASE_SOURCE_ARTIFACT_NAME="bundle-archive" RELEASE_SOURCE_SHA="dddddddddddddddddddddddddddddddddddddddd" RELEASE_CATALOG_KEY="archive:bundle" -export RELEASE_ARTIFACTS RELEASE_ARTIFACT_TYPE RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE_IDENTITY_FILE RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY +export RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY if "${repository_root}/release-catalog/materialize.sh" 2>"${temporary_directory}/missing-version-error"; then echo "materialize.sh unexpectedly accepted a custom artifact without a package version" >&2 exit 1 fi -grep -Fx 'package identity file must contain non-empty ecosystem, name, and version strings and only optional build or platform strings' "${temporary_directory}/missing-version-error" +grep -Fx 'package identity for bundle.tar.gz must contain non-empty ecosystem, name, and version strings and only optional build or platform strings' "${temporary_directory}/missing-version-error" From ed0c8f1fdeeff3348e50ea8fe5032c74b18542a7 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 18 Aug 2026 11:16:23 -0500 Subject: [PATCH 10/16] rename release catalog artifact directory --- release-catalog-dispatch/action.yml | 8 ++-- release-catalog/README.md | 8 ++-- release-catalog/config.schema.json | 6 +-- release-catalog/describe-conda.sh | 14 +++--- release-catalog/describe-wheels.sh | 14 +++--- release-catalog/materialize.sh | 44 +++++++++---------- release-catalog/prepare.sh | 32 +++++++------- release-catalog/validate-config.sh | 8 ++-- .../package-identity-file.json | 2 +- tests/release_catalog_config_test.sh | 8 ++-- tests/release_catalog_prepare_test.sh | 32 +++++++------- tests/release_catalog_test.sh | 20 ++++----- 12 files changed, 98 insertions(+), 98 deletions(-) diff --git a/release-catalog-dispatch/action.yml b/release-catalog-dispatch/action.yml index c38cb790..e2cd123b 100644 --- a/release-catalog-dispatch/action.yml +++ b/release-catalog-dispatch/action.yml @@ -57,7 +57,7 @@ runs: env: IMPLEMENTATION_PATH: ${{ steps.implementation.outputs.path }} RELEASE_ARTIFACTS: ${{ steps.configuration.outputs.artifacts }} - RELEASE_OUTPUT_DIRECTORY: ${{ steps.configuration.outputs.output_directory }} + RELEASE_ARTIFACT_DIRECTORY: ${{ steps.configuration.outputs.artifact_directory }} run: "${IMPLEMENTATION_PATH}/prepare.sh" - id: materialize name: Materialize release catalog entries @@ -66,7 +66,7 @@ runs: IMPLEMENTATION_PATH: ${{ steps.implementation.outputs.path }} RELEASE_ARTIFACTS: ${{ steps.prepare.outputs.artifacts }} RELEASE_ENTRIES_NAME: release-catalog-entries.json - RELEASE_OUTPUT_DIRECTORY: ${{ steps.configuration.outputs.output_directory }} + RELEASE_ARTIFACT_DIRECTORY: ${{ steps.configuration.outputs.artifact_directory }} RELEASE_SOURCE_ARTIFACT_NAME: ${{ inputs.source-artifact-name }} RELEASE_SOURCE_SHA: ${{ inputs.source-sha || github.sha }} RELEASE_CATALOG_KEY: ${{ steps.configuration.outputs.release_catalog_key }} @@ -83,5 +83,5 @@ runs: if-no-files-found: error name: ${{ steps.companion-name.outputs.name }} path: | - ${{ steps.configuration.outputs.output_directory }}/release-catalog-entries.json - ${{ steps.configuration.outputs.output_directory }}/release-evidence/** + ${{ steps.configuration.outputs.artifact_directory }}/release-catalog-entries.json + ${{ steps.configuration.outputs.artifact_directory }}/release-evidence/** diff --git a/release-catalog/README.md b/release-catalog/README.md index 8b765b2a..8e26c08e 100644 --- a/release-catalog/README.md +++ b/release-catalog/README.md @@ -28,7 +28,7 @@ Every producer passes one `config` JSON object. Its canonical schema and field documentation are in [`config.schema.json`](config.schema.json). When `artifacts` is omitted, the action discovers every Conda and wheel output -in `output_directory` and extracts identity from each package independently. +in `artifact_directory` and extracts identity from each package independently. When `artifacts` is supplied, each descriptor is resolved independently: supported Conda and wheel files are parsed, while any other artifact requires its own `package_identity_file`. Evidence paths also belong to the individual @@ -83,7 +83,7 @@ the built source. config: >- { "release_catalog_key": "wheel:example", - "output_directory": ${{ toJSON(steps.package-name.outputs.WHEEL_OUTPUT_DIR) }} + "artifact_directory": ${{ toJSON(steps.package-name.outputs.WHEEL_OUTPUT_DIR) }} } source-artifact-name: ${{ steps.package-name.outputs.RAPIDS_PACKAGE_NAME }} source-sha: ${{ env.RAPIDS_SHA }} @@ -98,7 +98,7 @@ the built source. config: >- { "release_catalog_key": "maven:cuvs-java", - "output_directory": "java/cuvs-java/target", + "artifact_directory": "java/cuvs-java/target", "artifacts": [{ "path": "cuvs-java-*-x86_64-cuda*.jar", "package_identity_file": "cuvs-java.release-package-identity.json" @@ -111,7 +111,7 @@ the built source. Before the action runs, the producer creates `java/cuvs-java/target/cuvs-java.release-package-identity.json`. The artifact descriptor's `package_identity_file` value is the path to that file -relative to `output_directory`. For example: +relative to `artifact_directory`. For example: ```json { diff --git a/release-catalog/config.schema.json b/release-catalog/config.schema.json index c8402e1d..0f099e1b 100644 --- a/release-catalog/config.schema.json +++ b/release-catalog/config.schema.json @@ -15,7 +15,7 @@ "minLength": 1, "pattern": "^[^\\r\\n]+$" }, - "output_directory": { + "artifact_directory": { "description": "Base directory for artifact and evidence paths. Defaults to the job working directory.", "type": "string", "minLength": 1, @@ -23,7 +23,7 @@ "default": "." }, "artifacts": { - "description": "Optional explicit primary release files and their identity or evidence sidecars. When omitted, Conda and wheel artifacts are discovered in output_directory.", + "description": "Optional explicit primary release files and their identity or evidence sidecars. When omitted, Conda and wheel artifacts are discovered in artifact_directory.", "type": "array", "minItems": 1, "items": { @@ -45,7 +45,7 @@ "pattern": "^[^\\r\\n]+$" }, "package_identity_file": { - "description": "Path, relative to output_directory, to producer-created package identity JSON for this artifact. Required only when identity cannot be extracted from the artifact. The file must contain ecosystem, name, and version and may contain build and platform.", + "description": "Path, relative to artifact_directory, to producer-created package identity JSON for this artifact. Required only when identity cannot be extracted from the artifact. The file must contain ecosystem, name, and version and may contain build and platform.", "type": "string", "minLength": 1, "pattern": "^[^\\r\\n]+$" diff --git a/release-catalog/describe-conda.sh b/release-catalog/describe-conda.sh index 23bdcf47..868175f6 100755 --- a/release-catalog/describe-conda.sh +++ b/release-catalog/describe-conda.sh @@ -4,11 +4,11 @@ set -euo pipefail if [[ "$#" -lt 1 || ! -d "$1" ]]; then - echo "usage: $0 CONDA_OUTPUT_DIRECTORY [CONDA_PACKAGE ...]" >&2 + echo "usage: $0 CONDA_ARTIFACT_DIRECTORY [CONDA_PACKAGE ...]" >&2 exit 1 fi -output_directory="$(realpath "$1")" +artifact_directory="$(realpath "$1")" descriptors='[]' package_count=0 package_paths=() @@ -18,16 +18,16 @@ if [[ "$#" -gt 1 ]]; then else while IFS= read -r package_path; do package_paths+=("${package_path}") - done < <(find "${output_directory}" -type f \( -name '*.conda' -o -name '*.tar.bz2' \) -print | sort) + done < <(find "${artifact_directory}" -type f \( -name '*.conda' -o -name '*.tar.bz2' \) -print | sort) fi for package_path in "${package_paths[@]}"; do package_path="$(realpath "${package_path}")" - if [[ "${package_path}" != "${output_directory}"/* ]]; then - echo "Conda package must resolve inside output directory: ${package_path}" >&2 + if [[ "${package_path}" != "${artifact_directory}"/* ]]; then + echo "Conda package must resolve inside artifact directory: ${package_path}" >&2 exit 1 fi - relative_path="${package_path#"${output_directory}/"}" + relative_path="${package_path#"${artifact_directory}/"}" case "${package_path}" in *.conda) @@ -71,7 +71,7 @@ for package_path in "${package_paths[@]}"; do done if [[ "${package_count}" -eq 0 ]]; then - echo "Conda output directory contains no .conda or .tar.bz2 packages: ${output_directory}" >&2 + echo "Conda artifact directory contains no .conda or .tar.bz2 packages: ${artifact_directory}" >&2 exit 1 fi diff --git a/release-catalog/describe-wheels.sh b/release-catalog/describe-wheels.sh index f440055f..8cf2e8ad 100755 --- a/release-catalog/describe-wheels.sh +++ b/release-catalog/describe-wheels.sh @@ -4,11 +4,11 @@ set -euo pipefail if [[ "$#" -lt 1 || ! -d "$1" ]]; then - echo "usage: $0 WHEEL_OUTPUT_DIRECTORY [WHEEL ...]" >&2 + echo "usage: $0 WHEEL_ARTIFACT_DIRECTORY [WHEEL ...]" >&2 exit 1 fi -output_directory="$(realpath "$1")" +artifact_directory="$(realpath "$1")" descriptors='[]' wheel_count=0 wheel_paths=() @@ -18,16 +18,16 @@ if [[ "$#" -gt 1 ]]; then else while IFS= read -r wheel_path; do wheel_paths+=("${wheel_path}") - done < <(find "${output_directory}" -type f -name '*.whl' -print | sort) + done < <(find "${artifact_directory}" -type f -name '*.whl' -print | sort) fi for wheel_path in "${wheel_paths[@]}"; do wheel_path="$(realpath "${wheel_path}")" - if [[ "${wheel_path}" != "${output_directory}"/* ]]; then - echo "wheel must resolve inside output directory: ${wheel_path}" >&2 + if [[ "${wheel_path}" != "${artifact_directory}"/* ]]; then + echo "wheel must resolve inside artifact directory: ${wheel_path}" >&2 exit 1 fi - relative_path="${wheel_path#"${output_directory}/"}" + relative_path="${wheel_path#"${artifact_directory}/"}" metadata_members=() while IFS= read -r metadata_member; do @@ -56,7 +56,7 @@ for wheel_path in "${wheel_paths[@]}"; do done if [[ "${wheel_count}" -eq 0 ]]; then - echo "wheel output directory contains no .whl files: ${output_directory}" >&2 + echo "wheel artifact directory contains no .whl files: ${artifact_directory}" >&2 exit 1 fi diff --git a/release-catalog/materialize.sh b/release-catalog/materialize.sh index ffc9d92b..57317441 100755 --- a/release-catalog/materialize.sh +++ b/release-catalog/materialize.sh @@ -13,7 +13,7 @@ require_nonempty() { } require_nonempty "RELEASE_CATALOG_KEY" "${RELEASE_CATALOG_KEY:-}" -require_nonempty "RELEASE_OUTPUT_DIRECTORY" "${RELEASE_OUTPUT_DIRECTORY:-}" +require_nonempty "RELEASE_ARTIFACT_DIRECTORY" "${RELEASE_ARTIFACT_DIRECTORY:-}" require_nonempty "RELEASE_ENTRIES_NAME" "${RELEASE_ENTRIES_NAME:-}" require_nonempty "RELEASE_ARTIFACTS" "${RELEASE_ARTIFACTS:-}" require_nonempty "RELEASE_SOURCE_ARTIFACT_NAME" "${RELEASE_SOURCE_ARTIFACT_NAME:-}" @@ -32,8 +32,8 @@ require_plain_filename() { require_plain_filename "entries-name" "${RELEASE_ENTRIES_NAME}" -if [[ ! -d "${RELEASE_OUTPUT_DIRECTORY}" ]]; then - echo "output-directory does not exist or is not a directory: ${RELEASE_OUTPUT_DIRECTORY}" >&2 +if [[ ! -d "${RELEASE_ARTIFACT_DIRECTORY}" ]]; then + echo "artifact-directory does not exist or is not a directory: ${RELEASE_ARTIFACT_DIRECTORY}" >&2 exit 1 fi @@ -41,20 +41,20 @@ ensure_relative_pattern() { local field="$1" local pattern="$2" if [[ "${pattern}" == /* || "${pattern}" == */../* || "${pattern}" == ../* || "${pattern}" == *"/.." ]]; then - echo "${field} must be a relative path inside output-directory: ${pattern}" >&2 + echo "${field} must be a relative path inside artifact-directory: ${pattern}" >&2 exit 1 fi } -output_directory="$(realpath "${RELEASE_OUTPUT_DIRECTORY}")" +artifact_directory="$(realpath "${RELEASE_ARTIFACT_DIRECTORY}")" if ! jq -e 'type == "array" and length > 0' <<<"${RELEASE_ARTIFACTS}" >/dev/null; then echo "release-artifacts must be a non-empty JSON array" >&2 exit 1 fi -entries_path="${output_directory}/${RELEASE_ENTRIES_NAME}" -temporary_manifest="$(mktemp "${output_directory}/.release-catalog.XXXXXX")" +entries_path="${artifact_directory}/${RELEASE_ENTRIES_NAME}" +temporary_manifest="$(mktemp "${artifact_directory}/.release-catalog.XXXXXX")" trap 'rm -f "${temporary_manifest}"' EXIT printf '%s\n' '{"entries":[]}' >"${temporary_manifest}" @@ -67,7 +67,7 @@ resolve_one_file() { ensure_relative_pattern "${field}" "${pattern}" while IFS= read -r match; do matches+=("${match}") - done < <(compgen -G "${output_directory}/${pattern}" || true) + done < <(compgen -G "${artifact_directory}/${pattern}" || true) if [[ "${#matches[@]}" -ne 1 || ! -f "${matches[0]:-}" ]]; then echo "${field} pattern must resolve to exactly one file: ${pattern}" >&2 exit 1 @@ -75,11 +75,11 @@ resolve_one_file() { local resolved resolved="$(realpath "${matches[0]}")" - if [[ "${resolved}" != "${output_directory}"/* ]]; then - echo "${field} must resolve inside output-directory: ${pattern}" >&2 + if [[ "${resolved}" != "${artifact_directory}"/* ]]; then + echo "${field} must resolve inside artifact-directory: ${pattern}" >&2 exit 1 fi - printf '%s\n' "${resolved#"${output_directory}/"}" + printf '%s\n' "${resolved#"${artifact_directory}/"}" } validate_package_identity() { @@ -98,7 +98,7 @@ generated_evidence_path() { local primary_path="$1" local kind="$2" local artifact_digest - artifact_digest="$(sha256sum "${output_directory}/${primary_path}" | awk '{print $1}')" + artifact_digest="$(sha256sum "${artifact_directory}/${primary_path}" | awk '{print $1}')" local artifact_label="${primary_path//\//_}" printf 'release-evidence/%s.%s.%s.json\n' "${artifact_label}" "${artifact_digest}" "${kind}" } @@ -108,12 +108,12 @@ copy_supplied_evidence() { local supplied_path="$2" local kind="$3" local artifact_digest - artifact_digest="$(sha256sum "${output_directory}/${primary_path}" | awk '{print $1}')" + artifact_digest="$(sha256sum "${artifact_directory}/${primary_path}" | awk '{print $1}')" local artifact_label="${primary_path//\//_}" local destination destination="release-evidence/${artifact_label}.${artifact_digest}/${kind}-$(basename "${supplied_path}")" - mkdir -p "$(dirname "${output_directory}/${destination}")" - cp "${output_directory}/${supplied_path}" "${output_directory}/${destination}" + mkdir -p "$(dirname "${artifact_directory}/${destination}")" + cp "${artifact_directory}/${supplied_path}" "${artifact_directory}/${destination}" printf '%s\n' "${destination}" } @@ -122,8 +122,8 @@ write_generated_sbom() { local package="$2" local destination="$3" local artifact_digest - artifact_digest="$(sha256sum "${output_directory}/${primary_path}" | awk '{print $1}')" - mkdir -p "$(dirname "${output_directory}/${destination}")" + artifact_digest="$(sha256sum "${artifact_directory}/${primary_path}" | awk '{print $1}')" + mkdir -p "$(dirname "${artifact_directory}/${destination}")" jq -n -S \ --arg artifact_digest "${artifact_digest}" \ --arg artifact_path "${primary_path}" \ @@ -153,7 +153,7 @@ write_generated_sbom() { relatedSpdxElement: "SPDXRef-Artifact" }], comment: "Artifact-identity SBOM envelope. A producer-supplied dependency SBOM may replace this record." - }' >"${output_directory}/${destination}" + }' >"${artifact_directory}/${destination}" } write_generated_provenance() { @@ -161,8 +161,8 @@ write_generated_provenance() { local package="$2" local destination="$3" local artifact_digest - artifact_digest="$(sha256sum "${output_directory}/${primary_path}" | awk '{print $1}')" - mkdir -p "$(dirname "${output_directory}/${destination}")" + artifact_digest="$(sha256sum "${artifact_directory}/${primary_path}" | awk '{print $1}')" + mkdir -p "$(dirname "${artifact_directory}/${destination}")" jq -n -S \ --arg artifact_digest "${artifact_digest}" \ --arg artifact_path "${primary_path}" \ @@ -190,7 +190,7 @@ write_generated_provenance() { metadata: {invocationId: ("https://github.com/" + $repository + "/actions/runs/" + $run_id + "/attempts/" + $run_attempt)} } } - }' >"${output_directory}/${destination}" + }' >"${artifact_directory}/${destination}" } shopt -s globstar nullglob @@ -216,7 +216,7 @@ while IFS= read -r descriptor; do package_identity_file="$(jq -r '.package_identity_file // empty' <<<"${descriptor}")" if [[ -n "${package_identity_file}" ]]; then package_identity_path="$(resolve_one_file package_identity_file "${package_identity_file}")" - if ! package="$(jq -ce . "${output_directory}/${package_identity_path}" 2>/dev/null)"; then + if ! package="$(jq -ce . "${artifact_directory}/${package_identity_path}" 2>/dev/null)"; then echo "package_identity_file must contain valid JSON: ${package_identity_file}" >&2 exit 1 fi diff --git a/release-catalog/prepare.sh b/release-catalog/prepare.sh index 9246974e..241e8724 100755 --- a/release-catalog/prepare.sh +++ b/release-catalog/prepare.sh @@ -12,21 +12,21 @@ require_nonempty() { fi } -require_nonempty "RELEASE_OUTPUT_DIRECTORY" "${RELEASE_OUTPUT_DIRECTORY:-}" +require_nonempty "RELEASE_ARTIFACT_DIRECTORY" "${RELEASE_ARTIFACT_DIRECTORY:-}" script_directory="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" artifacts="${RELEASE_ARTIFACTS:-}" -if [[ ! -d "${RELEASE_OUTPUT_DIRECTORY}" ]]; then - echo "output-directory does not exist or is not a directory: ${RELEASE_OUTPUT_DIRECTORY}" >&2 +if [[ ! -d "${RELEASE_ARTIFACT_DIRECTORY}" ]]; then + echo "artifact-directory does not exist or is not a directory: ${RELEASE_ARTIFACT_DIRECTORY}" >&2 exit 1 fi -output_directory="$(realpath "${RELEASE_OUTPUT_DIRECTORY}")" +artifact_directory="$(realpath "${RELEASE_ARTIFACT_DIRECTORY}")" ensure_relative_pattern() { local field="$1" local pattern="$2" if [[ "${pattern}" == /* || "${pattern}" == */../* || "${pattern}" == ../* || "${pattern}" == *"/.." ]]; then - echo "${field} must be a relative path inside output-directory: ${pattern}" >&2 + echo "${field} must be a relative path inside artifact-directory: ${pattern}" >&2 exit 1 fi } @@ -38,7 +38,7 @@ resolve_primary_file() { ensure_relative_pattern "artifact path" "${pattern}" while IFS= read -r match; do matches+=("${match}") - done < <(compgen -G "${output_directory}/${pattern}" || true) + done < <(compgen -G "${artifact_directory}/${pattern}" || true) if [[ "${#matches[@]}" -ne 1 || ! -f "${matches[0]:-}" ]]; then echo "artifact path must resolve to exactly one file: ${pattern}" >&2 exit 1 @@ -46,8 +46,8 @@ resolve_primary_file() { local resolved resolved="$(realpath "${matches[0]}")" - if [[ "${resolved}" != "${output_directory}"/* ]]; then - echo "artifact path must resolve inside output-directory: ${pattern}" >&2 + if [[ "${resolved}" != "${artifact_directory}"/* ]]; then + echo "artifact path must resolve inside artifact-directory: ${pattern}" >&2 exit 1 fi printf '%s\n' "${resolved}" @@ -56,15 +56,15 @@ resolve_primary_file() { if [[ -z "${artifacts}" ]]; then conda_descriptors='[]' wheel_descriptors='[]' - if [[ -n "$(find "${output_directory}" -type f \( -name '*.conda' -o -name '*.tar.bz2' \) -print -quit)" ]]; then - conda_descriptors="$("${script_directory}/describe-conda.sh" "${output_directory}")" + if [[ -n "$(find "${artifact_directory}" -type f \( -name '*.conda' -o -name '*.tar.bz2' \) -print -quit)" ]]; then + conda_descriptors="$("${script_directory}/describe-conda.sh" "${artifact_directory}")" fi - if [[ -n "$(find "${output_directory}" -type f -name '*.whl' -print -quit)" ]]; then - wheel_descriptors="$("${script_directory}/describe-wheels.sh" "${output_directory}")" + if [[ -n "$(find "${artifact_directory}" -type f -name '*.whl' -print -quit)" ]]; then + wheel_descriptors="$("${script_directory}/describe-wheels.sh" "${artifact_directory}")" fi artifacts="$(jq -cn --argjson conda "${conda_descriptors}" --argjson wheel "${wheel_descriptors}" '$conda + $wheel')" if [[ "$(jq 'length' <<<"${artifacts}")" -eq 0 ]]; then - echo "output-directory contains no detectable Conda or wheel artifacts; explicitly selected artifacts require package_identity_file when their identity cannot be parsed" >&2 + echo "artifact-directory contains no detectable Conda or wheel artifacts; explicitly selected artifacts require package_identity_file when their identity cannot be parsed" >&2 exit 1 fi else @@ -80,17 +80,17 @@ else echo "package_identity_file is not allowed when wheel identity can be extracted: $(jq -r '.path' <<<"${descriptor}")" >&2 exit 1 fi - package="$("${script_directory}/describe-wheels.sh" "${output_directory}" "${primary_file}" | jq -c '.[0].package')" + package="$("${script_directory}/describe-wheels.sh" "${artifact_directory}" "${primary_file}" | jq -c '.[0].package')" ;; *.conda) if [[ -n "${identity_file}" ]]; then echo "package_identity_file is not allowed when Conda identity can be extracted: $(jq -r '.path' <<<"${descriptor}")" >&2 exit 1 fi - package="$("${script_directory}/describe-conda.sh" "${output_directory}" "${primary_file}" | jq -c '.[0].package')" + package="$("${script_directory}/describe-conda.sh" "${artifact_directory}" "${primary_file}" | jq -c '.[0].package')" ;; *.tar.bz2) - if conda_descriptor="$("${script_directory}/describe-conda.sh" "${output_directory}" "${primary_file}" 2>/dev/null)"; then + if conda_descriptor="$("${script_directory}/describe-conda.sh" "${artifact_directory}" "${primary_file}" 2>/dev/null)"; then if [[ -n "${identity_file}" ]]; then echo "package_identity_file is not allowed when Conda identity can be extracted: $(jq -r '.path' <<<"${descriptor}")" >&2 exit 1 diff --git a/release-catalog/validate-config.sh b/release-catalog/validate-config.sh index 042a424a..6a5496ec 100755 --- a/release-catalog/validate-config.sh +++ b/release-catalog/validate-config.sh @@ -26,15 +26,15 @@ validation_errors="$(jq -r ' if type != "object" then ["release catalog configuration must be a JSON object"] else - (keys - ["release_catalog_key", "output_directory", "artifacts"]) as $unknown + (keys - ["release_catalog_key", "artifact_directory", "artifacts"]) as $unknown | [ if ($unknown | length) > 0 then "unknown field(s): " + ($unknown | join(", ")) else empty end, if (.release_catalog_key | single_line_string) then empty else "release_catalog_key must be a non-empty, single-line string" end, - if (has("output_directory") | not) or (.output_directory | single_line_string) then empty - else "output_directory must be a non-empty, single-line string when supplied" end + if (has("artifact_directory") | not) or (.artifact_directory | single_line_string) then empty + else "artifact_directory must be a non-empty, single-line string when supplied" end ] + if has("artifacts") then [ @@ -81,6 +81,6 @@ fi { printf 'release_catalog_key=%s\n' "$(jq -r '.release_catalog_key' <<<"${compact_config}")" - printf 'output_directory=%s\n' "$(jq -r '.output_directory // "."' <<<"${compact_config}")" + printf 'artifact_directory=%s\n' "$(jq -r '.artifact_directory // "."' <<<"${compact_config}")" printf 'artifacts=%s\n' "$(jq -c '.artifacts // empty' <<<"${compact_config}")" } >>"${GITHUB_OUTPUT}" diff --git a/tests/release-catalog-config/package-identity-file.json b/tests/release-catalog-config/package-identity-file.json index 46068c04..99047495 100644 --- a/tests/release-catalog-config/package-identity-file.json +++ b/tests/release-catalog-config/package-identity-file.json @@ -1,6 +1,6 @@ { "release_catalog_key": "maven:cuvs-java", - "output_directory": "java/cuvs-java/target", + "artifact_directory": "java/cuvs-java/target", "artifacts": [ { "path": "cuvs-java-*-x86_64-cuda*.jar", diff --git a/tests/release_catalog_config_test.sh b/tests/release_catalog_config_test.sh index c0776a4d..8bfcce06 100755 --- a/tests/release_catalog_config_test.sh +++ b/tests/release_catalog_config_test.sh @@ -31,7 +31,7 @@ RELEASE_CATALOG_CONFIG="$(<"${repository_root}/tests/release-catalog-config/pack GITHUB_OUTPUT="${valid_output}" "${validator}" grep -Fx 'release_catalog_key=maven:cuvs-java' "${valid_output}" -grep -Fx 'output_directory=java/cuvs-java/target' "${valid_output}" +grep -Fx 'artifact_directory=java/cuvs-java/target' "${valid_output}" grep -Fx 'artifacts=[{"path":"cuvs-java-*-x86_64-cuda*.jar","package_identity_file":"cuvs-java.release-package-identity.json","sbom":"cuvs-java.spdx.json"}]' "${valid_output}" assert_invalid \ @@ -86,15 +86,15 @@ default_output="${temporary_directory}/default.output" RELEASE_CATALOG_CONFIG='{"release_catalog_key":"conda:smoke"}' \ GITHUB_OUTPUT="${default_output}" "${validator}" grep -Fx 'release_catalog_key=conda:smoke' "${default_output}" -grep -Fx 'output_directory=.' "${default_output}" +grep -Fx 'artifact_directory=.' "${default_output}" grep -Fx 'artifacts=' "${default_output}" standard_output="${temporary_directory}/standard.output" RELEASE_CATALOG_CONFIG='{ "release_catalog_key": "wheel:kvikio", - "output_directory": "dist" + "artifact_directory": "dist" }' GITHUB_OUTPUT="${standard_output}" "${validator}" grep -Fx 'release_catalog_key=wheel:kvikio' "${standard_output}" -grep -Fx 'output_directory=dist' "${standard_output}" +grep -Fx 'artifact_directory=dist' "${standard_output}" grep -Fx 'artifacts=' "${standard_output}" diff --git a/tests/release_catalog_prepare_test.sh b/tests/release_catalog_prepare_test.sh index f83e90db..f897badc 100755 --- a/tests/release_catalog_prepare_test.sh +++ b/tests/release_catalog_prepare_test.sh @@ -20,8 +20,8 @@ printf '%s\n' \ GITHUB_OUTPUT="${temporary_directory}/wheel-output" RELEASE_ARTIFACTS='' -RELEASE_OUTPUT_DIRECTORY="${temporary_directory}" -export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY +RELEASE_ARTIFACT_DIRECTORY="${temporary_directory}" +export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY "${repository_root}/release-catalog/prepare.sh" @@ -45,8 +45,8 @@ tar -cjf \ info/index.json GITHUB_OUTPUT="${temporary_directory}/conda-output" -RELEASE_OUTPUT_DIRECTORY="${conda_output_directory}" -export GITHUB_OUTPUT RELEASE_OUTPUT_DIRECTORY +RELEASE_ARTIFACT_DIRECTORY="${conda_output_directory}" +export GITHUB_OUTPUT RELEASE_ARTIFACT_DIRECTORY "${repository_root}/release-catalog/prepare.sh" @@ -65,21 +65,21 @@ jq -e ' ' <<<"${artifacts}" >/dev/null GITHUB_OUTPUT="${temporary_directory}/custom-output" -RELEASE_OUTPUT_DIRECTORY="${temporary_directory}" +RELEASE_ARTIFACT_DIRECTORY="${temporary_directory}" printf '%s\n' bundle >"${temporary_directory}/bundle.tar.gz" jq -n '{ecosystem: "archive", name: "bundle", version: "1.0"}' \ >"${temporary_directory}/release-package-identity.json" RELEASE_ARTIFACTS="$(jq -cn '[{path: "bundle.tar.gz", package_identity_file: "release-package-identity.json", sbom: "bundle.spdx.json"}]')" -export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY +export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY "${repository_root}/release-catalog/prepare.sh" grep -Fx 'artifacts=[{"path":"bundle.tar.gz","package_identity_file":"release-package-identity.json","sbom":"bundle.spdx.json"}]' "${GITHUB_OUTPUT}" GITHUB_OUTPUT="${temporary_directory}/explicit-wheel-output" -RELEASE_OUTPUT_DIRECTORY="${temporary_directory}" +RELEASE_ARTIFACT_DIRECTORY="${temporary_directory}" RELEASE_ARTIFACTS="$(jq -cn '[{path: "libkvikio_cu12-*.whl", signature: "libkvikio.sig"}]')" -export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY +export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY "${repository_root}/release-catalog/prepare.sh" artifacts="$(sed -n 's/^artifacts=//p' "${GITHUB_OUTPUT}")" jq -e ' @@ -96,8 +96,8 @@ cp "${temporary_directory}/libkvikio_cu12-26.8.0-py3-none-manylinux_2_28_x86_64. cp "${conda_output_directory}/noarch/rapids-dask-dependency-26.08.0-py_0.tar.bz2" "${mixed_directory}/" GITHUB_OUTPUT="${temporary_directory}/mixed-output" RELEASE_ARTIFACTS='' -RELEASE_OUTPUT_DIRECTORY="${mixed_directory}" -export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY +RELEASE_ARTIFACT_DIRECTORY="${mixed_directory}" +export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY "${repository_root}/release-catalog/prepare.sh" artifacts="$(sed -n 's/^artifacts=//p' "${GITHUB_OUTPUT}")" jq -e ' @@ -108,18 +108,18 @@ jq -e ' empty_directory="${temporary_directory}/empty" mkdir -p "${empty_directory}" GITHUB_OUTPUT="${temporary_directory}/empty-output" -RELEASE_OUTPUT_DIRECTORY="${empty_directory}" -export GITHUB_OUTPUT RELEASE_OUTPUT_DIRECTORY +RELEASE_ARTIFACT_DIRECTORY="${empty_directory}" +export GITHUB_OUTPUT RELEASE_ARTIFACT_DIRECTORY if "${repository_root}/release-catalog/prepare.sh" 2>"${temporary_directory}/empty-error"; then - echo "prepare.sh unexpectedly accepted an output directory without detectable artifacts" >&2 + echo "prepare.sh unexpectedly accepted an artifact directory without detectable artifacts" >&2 exit 1 fi -grep -Fx 'output-directory contains no detectable Conda or wheel artifacts; explicitly selected artifacts require package_identity_file when their identity cannot be parsed' "${temporary_directory}/empty-error" +grep -Fx 'artifact-directory contains no detectable Conda or wheel artifacts; explicitly selected artifacts require package_identity_file when their identity cannot be parsed' "${temporary_directory}/empty-error" GITHUB_OUTPUT="${temporary_directory}/unsupported-output" -RELEASE_OUTPUT_DIRECTORY="${temporary_directory}" +RELEASE_ARTIFACT_DIRECTORY="${temporary_directory}" RELEASE_ARTIFACTS="$(jq -cn '[{path: "bundle.tar.gz"}]')" -export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY +export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY if "${repository_root}/release-catalog/prepare.sh" 2>"${temporary_directory}/unsupported-error"; then echo "prepare.sh unexpectedly accepted an unsupported artifact without identity" >&2 exit 1 diff --git a/tests/release_catalog_test.sh b/tests/release_catalog_test.sh index 50badf06..5278994b 100755 --- a/tests/release_catalog_test.sh +++ b/tests/release_catalog_test.sh @@ -26,12 +26,12 @@ GITHUB_WORKFLOW_REF="rapidsai/cuvs/.github/workflows/build.yaml@refs/heads/relea export GITHUB_OUTPUT RELEASE_ARTIFACTS="$(jq -cn '[{path: "cuvs-java-*.jar", package_identity_file: "cuvs-java-package-identity.json", sbom: "cuvs-java-*.spdx.json", provenance: "cuvs-java-*.provenance.jsonl", signature: "cuvs-java-*.jar.asc"}]')" RELEASE_ENTRIES_NAME="release-catalog-entries.json" -RELEASE_OUTPUT_DIRECTORY="${bundle_directory}" +RELEASE_ARTIFACT_DIRECTORY="${bundle_directory}" RELEASE_SOURCE_ARTIFACT_NAME="cuvs-java-cuda12.9.1" RELEASE_SOURCE_SHA="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" RELEASE_CATALOG_KEY="maven:cuvs-java" export GITHUB_REPOSITORY GITHUB_RUN_ATTEMPT GITHUB_RUN_ID GITHUB_SHA GITHUB_WORKFLOW_REF -export RELEASE_ARTIFACTS RELEASE_ENTRIES_NAME RELEASE_OUTPUT_DIRECTORY +export RELEASE_ARTIFACTS RELEASE_ENTRIES_NAME RELEASE_ARTIFACT_DIRECTORY export RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY "${repository_root}/release-catalog/materialize.sh" @@ -83,11 +83,11 @@ RELEASE_ARTIFACTS="$(jq -cn '[ {path: "first.tar.gz", package_identity_file: "first.identity.json"}, {path: "second.jar", package_identity_file: "second.identity.json"} ]')" -RELEASE_OUTPUT_DIRECTORY="${multiple_identity_directory}" +RELEASE_ARTIFACT_DIRECTORY="${multiple_identity_directory}" RELEASE_SOURCE_ARTIFACT_NAME="multiple-identities" RELEASE_SOURCE_SHA="eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" RELEASE_CATALOG_KEY="archive:multiple" -export RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY +export RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY "${repository_root}/release-catalog/materialize.sh" jq -e ' (.entries | length == 2) @@ -100,11 +100,11 @@ mkdir -p "${generated_directory}/linux-64" printf '%s\n' conda >"${generated_directory}/linux-64/kvikio-26.08.00a32-cuda12_260714_2f567060.conda" RELEASE_ARTIFACTS="$(jq -cn '[{path: "linux-64/kvikio-*.conda", package: {ecosystem: "conda", name: "kvikio", version: "26.08.00a32"}}]')" -RELEASE_OUTPUT_DIRECTORY="${generated_directory}" +RELEASE_ARTIFACT_DIRECTORY="${generated_directory}" RELEASE_SOURCE_ARTIFACT_NAME="kvikio_conda_python_kvikio_x86_64_abi3_cu12" RELEASE_SOURCE_SHA="bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" RELEASE_CATALOG_KEY="conda:kvikio" -export RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY +export RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY "${repository_root}/release-catalog/materialize.sh" @@ -134,11 +134,11 @@ mkdir -p "${wheel_directory}" printf '%s\n' wheel >"${wheel_directory}/libkvikio_cu12-26.8.0a32-py3-none-manylinux_2_28_x86_64.whl" RELEASE_ARTIFACTS="$(jq -cn '[{path: "libkvikio_cu12-*.whl", package: {ecosystem: "wheel", name: "libkvikio-cu12", version: "26.8.0a32"}}]')" -RELEASE_OUTPUT_DIRECTORY="${wheel_directory}" +RELEASE_ARTIFACT_DIRECTORY="${wheel_directory}" RELEASE_SOURCE_ARTIFACT_NAME="kvikio_wheel_cpp_libkvikio_x86_64_cu12" RELEASE_SOURCE_SHA="cccccccccccccccccccccccccccccccccccccccc" RELEASE_CATALOG_KEY="wheel:kvikio" -export RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY +export RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY "${repository_root}/release-catalog/materialize.sh" @@ -157,11 +157,11 @@ jq -n \ >"${missing_version_directory}/bundle-package-identity.json" RELEASE_ARTIFACTS="$(jq -cn '[{path: "bundle.tar.gz", package_identity_file: "bundle-package-identity.json"}]')" -RELEASE_OUTPUT_DIRECTORY="${missing_version_directory}" +RELEASE_ARTIFACT_DIRECTORY="${missing_version_directory}" RELEASE_SOURCE_ARTIFACT_NAME="bundle-archive" RELEASE_SOURCE_SHA="dddddddddddddddddddddddddddddddddddddddd" RELEASE_CATALOG_KEY="archive:bundle" -export RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY +export RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY if "${repository_root}/release-catalog/materialize.sh" 2>"${temporary_directory}/missing-version-error"; then echo "materialize.sh unexpectedly accepted a custom artifact without a package version" >&2 From 8a4baa9dfc21960c1ab5d64af02f7c5e9828e029 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 18 Aug 2026 11:53:25 -0500 Subject: [PATCH 11/16] consolidate release catalog implementation --- release-catalog-dispatch/action.yml | 10 +- release-catalog/describe-conda.sh | 78 -------- release-catalog/describe-wheels.sh | 63 ------ release-catalog/materialize.sh | 185 ++++++++++++++++-- release-catalog/prepare.sh | 120 ------------ ...t.sh => release_catalog_discovery_test.sh} | 64 +++--- ...> release_catalog_package_parsing_test.sh} | 42 +++- tests/release_catalog_test.sh | 16 +- 8 files changed, 252 insertions(+), 326 deletions(-) delete mode 100755 release-catalog/describe-conda.sh delete mode 100755 release-catalog/describe-wheels.sh delete mode 100755 release-catalog/prepare.sh rename tests/{release_catalog_prepare_test.sh => release_catalog_discovery_test.sh} (66%) rename tests/{release_catalog_descriptors_test.sh => release_catalog_package_parsing_test.sh} (60%) diff --git a/release-catalog-dispatch/action.yml b/release-catalog-dispatch/action.yml index e2cd123b..9c2e9257 100644 --- a/release-catalog-dispatch/action.yml +++ b/release-catalog-dispatch/action.yml @@ -51,20 +51,12 @@ runs: IMPLEMENTATION_PATH: ${{ steps.implementation.outputs.path }} RELEASE_CATALOG_CONFIG: ${{ inputs.config }} run: "${IMPLEMENTATION_PATH}/validate-config.sh" - - id: prepare - name: Describe release artifacts - shell: bash - env: - IMPLEMENTATION_PATH: ${{ steps.implementation.outputs.path }} - RELEASE_ARTIFACTS: ${{ steps.configuration.outputs.artifacts }} - RELEASE_ARTIFACT_DIRECTORY: ${{ steps.configuration.outputs.artifact_directory }} - run: "${IMPLEMENTATION_PATH}/prepare.sh" - id: materialize name: Materialize release catalog entries shell: bash env: IMPLEMENTATION_PATH: ${{ steps.implementation.outputs.path }} - RELEASE_ARTIFACTS: ${{ steps.prepare.outputs.artifacts }} + RELEASE_ARTIFACTS: ${{ steps.configuration.outputs.artifacts }} RELEASE_ENTRIES_NAME: release-catalog-entries.json RELEASE_ARTIFACT_DIRECTORY: ${{ steps.configuration.outputs.artifact_directory }} RELEASE_SOURCE_ARTIFACT_NAME: ${{ inputs.source-artifact-name }} diff --git a/release-catalog/describe-conda.sh b/release-catalog/describe-conda.sh deleted file mode 100755 index 868175f6..00000000 --- a/release-catalog/describe-conda.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - -set -euo pipefail - -if [[ "$#" -lt 1 || ! -d "$1" ]]; then - echo "usage: $0 CONDA_ARTIFACT_DIRECTORY [CONDA_PACKAGE ...]" >&2 - exit 1 -fi - -artifact_directory="$(realpath "$1")" -descriptors='[]' -package_count=0 -package_paths=() - -if [[ "$#" -gt 1 ]]; then - package_paths=("${@:2}") -else - while IFS= read -r package_path; do - package_paths+=("${package_path}") - done < <(find "${artifact_directory}" -type f \( -name '*.conda' -o -name '*.tar.bz2' \) -print | sort) -fi - -for package_path in "${package_paths[@]}"; do - package_path="$(realpath "${package_path}")" - if [[ "${package_path}" != "${artifact_directory}"/* ]]; then - echo "Conda package must resolve inside artifact directory: ${package_path}" >&2 - exit 1 - fi - relative_path="${package_path#"${artifact_directory}/"}" - - case "${package_path}" in - *.conda) - info_members=() - while IFS= read -r info_member; do - info_members+=("${info_member}") - done < <(unzip -Z1 "${package_path}" | awk '/^info-.*\.tar\.zst$/') - if [[ "${#info_members[@]}" -ne 1 ]]; then - echo ".conda package must contain exactly one info-*.tar.zst member: ${relative_path}" >&2 - exit 1 - fi - index_json="$(unzip -p "${package_path}" "${info_members[0]}" | zstd -dc | tar -xOf - info/index.json)" - ;; - *.tar.bz2) - index_json="$(tar -xOjf "${package_path}" info/index.json)" - ;; - *) - echo "unsupported Conda package extension: ${relative_path}" >&2 - exit 1 - ;; - esac - - if ! jq -e ' - type == "object" - and (.name | type == "string" and length > 0) - and (.version | type == "string" and length > 0) - and (.build | type == "string" and length > 0) - and (.subdir | type == "string" and length > 0) - ' <<<"${index_json}" >/dev/null; then - echo "Conda info/index.json must contain exact name, version, build, and subdir fields: ${relative_path}" >&2 - exit 1 - fi - - package="$(jq -c '{ecosystem: "conda", name, version, build, platform: .subdir}' <<<"${index_json}")" - descriptors="$(jq -cn \ - --arg path "${relative_path}" \ - --argjson package "${package}" \ - --argjson current "${descriptors}" \ - '$current + [{path: $path, package: $package}]')" - package_count=$((package_count + 1)) -done - -if [[ "${package_count}" -eq 0 ]]; then - echo "Conda artifact directory contains no .conda or .tar.bz2 packages: ${artifact_directory}" >&2 - exit 1 -fi - -printf '%s\n' "${descriptors}" diff --git a/release-catalog/describe-wheels.sh b/release-catalog/describe-wheels.sh deleted file mode 100755 index 8cf2e8ad..00000000 --- a/release-catalog/describe-wheels.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - -set -euo pipefail - -if [[ "$#" -lt 1 || ! -d "$1" ]]; then - echo "usage: $0 WHEEL_ARTIFACT_DIRECTORY [WHEEL ...]" >&2 - exit 1 -fi - -artifact_directory="$(realpath "$1")" -descriptors='[]' -wheel_count=0 -wheel_paths=() - -if [[ "$#" -gt 1 ]]; then - wheel_paths=("${@:2}") -else - while IFS= read -r wheel_path; do - wheel_paths+=("${wheel_path}") - done < <(find "${artifact_directory}" -type f -name '*.whl' -print | sort) -fi - -for wheel_path in "${wheel_paths[@]}"; do - wheel_path="$(realpath "${wheel_path}")" - if [[ "${wheel_path}" != "${artifact_directory}"/* ]]; then - echo "wheel must resolve inside artifact directory: ${wheel_path}" >&2 - exit 1 - fi - relative_path="${wheel_path#"${artifact_directory}/"}" - - metadata_members=() - while IFS= read -r metadata_member; do - metadata_members+=("${metadata_member}") - done < <(unzip -Z1 "${wheel_path}" | awk '/\.dist-info\/METADATA$/') - if [[ "${#metadata_members[@]}" -ne 1 ]]; then - echo "wheel must contain exactly one .dist-info/METADATA file: ${relative_path}" >&2 - exit 1 - fi - - metadata="$(unzip -p "${wheel_path}" "${metadata_members[0]}")" - package_name="$(awk 'tolower($0) ~ /^name:[[:space:]]*/ {sub(/^[^:]*:[[:space:]]*/, ""); sub(/\r$/, ""); print; exit}' <<<"${metadata}")" - package_version="$(awk 'tolower($0) ~ /^version:[[:space:]]*/ {sub(/^[^:]*:[[:space:]]*/, ""); sub(/\r$/, ""); print; exit}' <<<"${metadata}")" - if [[ -z "${package_name}" || -z "${package_version}" ]]; then - echo "wheel metadata must contain non-empty Name and Version fields: ${relative_path}" >&2 - exit 1 - fi - - descriptors="$(jq -cn \ - --arg path "${relative_path}" \ - --arg name "${package_name}" \ - --arg version "${package_version}" \ - --argjson current "${descriptors}" \ - '$current + [{path: $path, package: {ecosystem: "wheel", name: $name, version: $version}}]')" - wheel_count=$((wheel_count + 1)) -done - -if [[ "${wheel_count}" -eq 0 ]]; then - echo "wheel artifact directory contains no .whl files: ${artifact_directory}" >&2 - exit 1 -fi - -printf '%s\n' "${descriptors}" diff --git a/release-catalog/materialize.sh b/release-catalog/materialize.sh index 57317441..1a353a15 100755 --- a/release-catalog/materialize.sh +++ b/release-catalog/materialize.sh @@ -15,7 +15,6 @@ require_nonempty() { require_nonempty "RELEASE_CATALOG_KEY" "${RELEASE_CATALOG_KEY:-}" require_nonempty "RELEASE_ARTIFACT_DIRECTORY" "${RELEASE_ARTIFACT_DIRECTORY:-}" require_nonempty "RELEASE_ENTRIES_NAME" "${RELEASE_ENTRIES_NAME:-}" -require_nonempty "RELEASE_ARTIFACTS" "${RELEASE_ARTIFACTS:-}" require_nonempty "RELEASE_SOURCE_ARTIFACT_NAME" "${RELEASE_SOURCE_ARTIFACT_NAME:-}" source_sha="${RELEASE_SOURCE_SHA:-${GITHUB_SHA:-}}" @@ -48,17 +47,6 @@ ensure_relative_pattern() { artifact_directory="$(realpath "${RELEASE_ARTIFACT_DIRECTORY}")" -if ! jq -e 'type == "array" and length > 0' <<<"${RELEASE_ARTIFACTS}" >/dev/null; then - echo "release-artifacts must be a non-empty JSON array" >&2 - exit 1 -fi - -entries_path="${artifact_directory}/${RELEASE_ENTRIES_NAME}" -temporary_manifest="$(mktemp "${artifact_directory}/.release-catalog.XXXXXX")" -trap 'rm -f "${temporary_manifest}"' EXIT - -printf '%s\n' '{"entries":[]}' >"${temporary_manifest}" - resolve_one_file() { local field="$1" local pattern="$2" @@ -82,6 +70,175 @@ resolve_one_file() { printf '%s\n' "${resolved#"${artifact_directory}/"}" } +describe_wheel_package() { + local wheel_path="$1" + local relative_path="${wheel_path#"${artifact_directory}/"}" + local -a metadata_members=() + local metadata_member + while IFS= read -r metadata_member; do + metadata_members+=("${metadata_member}") + done < <(unzip -Z1 "${wheel_path}" | awk '/\.dist-info\/METADATA$/') + if [[ "${#metadata_members[@]}" -ne 1 ]]; then + echo "wheel must contain exactly one .dist-info/METADATA file: ${relative_path}" >&2 + exit 1 + fi + + local metadata package_name package_version + metadata="$(unzip -p "${wheel_path}" "${metadata_members[0]}")" + package_name="$(awk 'tolower($0) ~ /^name:[[:space:]]*/ {sub(/^[^:]*:[[:space:]]*/, ""); sub(/\r$/, ""); print; exit}' <<<"${metadata}")" + package_version="$(awk 'tolower($0) ~ /^version:[[:space:]]*/ {sub(/^[^:]*:[[:space:]]*/, ""); sub(/\r$/, ""); print; exit}' <<<"${metadata}")" + if [[ -z "${package_name}" || -z "${package_version}" ]]; then + echo "wheel metadata must contain non-empty Name and Version fields: ${relative_path}" >&2 + exit 1 + fi + + jq -cn --arg name "${package_name}" --arg version "${package_version}" \ + '{ecosystem: "wheel", name: $name, version: $version}' +} + +describe_conda_package() { + local package_path="$1" + local relative_path="${package_path#"${artifact_directory}/"}" + local index_json + case "${package_path}" in + *.conda) + local -a info_members=() + local info_member + while IFS= read -r info_member; do + info_members+=("${info_member}") + done < <(unzip -Z1 "${package_path}" | awk '/^info-.*\.tar\.zst$/') + if [[ "${#info_members[@]}" -ne 1 ]]; then + echo ".conda package must contain exactly one info-*.tar.zst member: ${relative_path}" >&2 + exit 1 + fi + index_json="$(unzip -p "${package_path}" "${info_members[0]}" | zstd -dc | tar -xOf - info/index.json)" + ;; + *.tar.bz2) + index_json="$(tar -xOjf "${package_path}" info/index.json)" + ;; + *) + echo "unsupported Conda package extension: ${relative_path}" >&2 + exit 1 + ;; + esac + + if ! jq -e ' + type == "object" + and (.name | type == "string" and length > 0) + and (.version | type == "string" and length > 0) + and (.build | type == "string" and length > 0) + and (.subdir | type == "string" and length > 0) + ' <<<"${index_json}" >/dev/null; then + echo "Conda info/index.json must contain exact name, version, build, and subdir fields: ${relative_path}" >&2 + exit 1 + fi + + jq -c '{ecosystem: "conda", name, version, build, platform: .subdir}' <<<"${index_json}" +} + +prepare_artifacts() { + local configured_artifacts="${RELEASE_ARTIFACTS:-}" + local prepared_artifacts='[]' + local conda_package descriptor package primary_path primary_file identity_file + + if [[ -z "${configured_artifacts}" ]]; then + local -a detected_files=() + while IFS= read -r primary_file; do + detected_files+=("${primary_file}") + done < <(find "${artifact_directory}" -type f \( -name '*.conda' -o -name '*.tar.bz2' -o -name '*.whl' \) -print | sort) + + for primary_file in "${detected_files[@]}"; do + primary_path="${primary_file#"${artifact_directory}/"}" + case "${primary_file}" in + *.whl) + if ! package="$(describe_wheel_package "${primary_file}")"; then + return 1 + fi + ;; + *) + if ! package="$(describe_conda_package "${primary_file}")"; then + return 1 + fi + ;; + esac + descriptor="$(jq -cn --arg path "${primary_path}" --argjson package "${package}" '{path: $path, package: $package}')" + prepared_artifacts="$(jq -cn --argjson current "${prepared_artifacts}" --argjson descriptor "${descriptor}" '$current + [$descriptor]')" + done + else + if ! jq -e 'type == "array" and length > 0' <<<"${configured_artifacts}" >/dev/null; then + echo "release-artifacts must be a non-empty JSON array when supplied" >&2 + exit 1 + fi + while IFS= read -r descriptor; do + if jq -e 'has("package")' <<<"${descriptor}" >/dev/null; then + echo "package identity must not be supplied inline: ${descriptor}" >&2 + return 1 + fi + primary_path="$(resolve_one_file path "$(jq -r '.path' <<<"${descriptor}")")" + primary_file="${artifact_directory}/${primary_path}" + identity_file="$(jq -r '.package_identity_file // empty' <<<"${descriptor}")" + package='' + + case "${primary_file}" in + *.whl) + if [[ -n "${identity_file}" ]]; then + echo "package_identity_file is not allowed when wheel identity can be extracted: $(jq -r '.path' <<<"${descriptor}")" >&2 + exit 1 + fi + if ! package="$(describe_wheel_package "${primary_file}")"; then + return 1 + fi + ;; + *.conda) + if [[ -n "${identity_file}" ]]; then + echo "package_identity_file is not allowed when Conda identity can be extracted: $(jq -r '.path' <<<"${descriptor}")" >&2 + exit 1 + fi + if ! package="$(describe_conda_package "${primary_file}")"; then + return 1 + fi + ;; + *.tar.bz2) + if conda_package="$(describe_conda_package "${primary_file}" 2>/dev/null)"; then + if [[ -n "${identity_file}" ]]; then + echo "package_identity_file is not allowed when Conda identity can be extracted: $(jq -r '.path' <<<"${descriptor}")" >&2 + exit 1 + fi + package="${conda_package}" + elif [[ -z "${identity_file}" ]]; then + echo "artifact is not a valid Conda package and requires package_identity_file: $(jq -r '.path' <<<"${descriptor}")" >&2 + exit 1 + fi + ;; + *) + if [[ -z "${identity_file}" ]]; then + echo "artifact identity cannot be extracted; package_identity_file is required: $(jq -r '.path' <<<"${descriptor}")" >&2 + exit 1 + fi + ;; + esac + + if [[ -n "${package}" ]]; then + descriptor="$(jq -c --argjson package "${package}" '. + {package: $package}' <<<"${descriptor}")" + fi + prepared_artifacts="$(jq -cn --argjson current "${prepared_artifacts}" --argjson descriptor "${descriptor}" '$current + [$descriptor]')" + done < <(jq -c '.[]' <<<"${configured_artifacts}") + fi + + if [[ "$(jq 'length' <<<"${prepared_artifacts}")" -eq 0 ]]; then + echo "artifact-directory contains no detectable Conda or wheel artifacts; explicitly selected artifacts require package_identity_file when their identity cannot be parsed" >&2 + exit 1 + fi + printf '%s\n' "${prepared_artifacts}" +} + +artifacts="$(prepare_artifacts)" +entries_path="${artifact_directory}/${RELEASE_ENTRIES_NAME}" +temporary_manifest="$(mktemp "${artifact_directory}/.release-catalog.XXXXXX")" +trap 'rm -f "${temporary_manifest}"' EXIT + +printf '%s\n' '{"entries":[]}' >"${temporary_manifest}" + validate_package_identity() { jq -e ' type == "object" @@ -133,6 +290,8 @@ write_generated_sbom() { dataLicense: "CC0-1.0", SPDXID: "SPDXRef-DOCUMENT", name: ("RAPIDS release artifact " + $artifact_path), + # This is a unique identifier, not a retrieval URL; SPDX requires an + # absolute URI for the document namespace but does not require it to resolve. documentNamespace: ("https://rapids.ai/release-platform/spdx/" + $artifact_digest), creationInfo: { creators: ["Tool: rapidsai/shared-workflows release catalog"], @@ -260,7 +419,7 @@ while IFS= read -r descriptor; do jq --argjson entry "${entry}" '.entries += [$entry]' "${temporary_manifest}" >"${temporary_manifest}.next" mv "${temporary_manifest}.next" "${temporary_manifest}" -done < <(jq -c '.[]' <<<"${RELEASE_ARTIFACTS}") +done < <(jq -c '.[]' <<<"${artifacts}") if ! jq -e '.entries as $items | ($items | map([.release_catalog_key, .path] | join("\u0000")) | unique | length) == ($items | length)' "${temporary_manifest}" >/dev/null; then echo "release-artifacts contains duplicate release_catalog_key/path entries" >&2 diff --git a/release-catalog/prepare.sh b/release-catalog/prepare.sh deleted file mode 100755 index 241e8724..00000000 --- a/release-catalog/prepare.sh +++ /dev/null @@ -1,120 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - -set -euo pipefail - -require_nonempty() { - local name="$1" - local value="$2" - if [[ -z "${value}" ]]; then - echo "${name} must be a non-empty string" >&2 - exit 1 - fi -} - -require_nonempty "RELEASE_ARTIFACT_DIRECTORY" "${RELEASE_ARTIFACT_DIRECTORY:-}" - -script_directory="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -artifacts="${RELEASE_ARTIFACTS:-}" -if [[ ! -d "${RELEASE_ARTIFACT_DIRECTORY}" ]]; then - echo "artifact-directory does not exist or is not a directory: ${RELEASE_ARTIFACT_DIRECTORY}" >&2 - exit 1 -fi -artifact_directory="$(realpath "${RELEASE_ARTIFACT_DIRECTORY}")" - -ensure_relative_pattern() { - local field="$1" - local pattern="$2" - if [[ "${pattern}" == /* || "${pattern}" == */../* || "${pattern}" == ../* || "${pattern}" == *"/.." ]]; then - echo "${field} must be a relative path inside artifact-directory: ${pattern}" >&2 - exit 1 - fi -} - -resolve_primary_file() { - local pattern="$1" - local -a matches=() - - ensure_relative_pattern "artifact path" "${pattern}" - while IFS= read -r match; do - matches+=("${match}") - done < <(compgen -G "${artifact_directory}/${pattern}" || true) - if [[ "${#matches[@]}" -ne 1 || ! -f "${matches[0]:-}" ]]; then - echo "artifact path must resolve to exactly one file: ${pattern}" >&2 - exit 1 - fi - - local resolved - resolved="$(realpath "${matches[0]}")" - if [[ "${resolved}" != "${artifact_directory}"/* ]]; then - echo "artifact path must resolve inside artifact-directory: ${pattern}" >&2 - exit 1 - fi - printf '%s\n' "${resolved}" -} - -if [[ -z "${artifacts}" ]]; then - conda_descriptors='[]' - wheel_descriptors='[]' - if [[ -n "$(find "${artifact_directory}" -type f \( -name '*.conda' -o -name '*.tar.bz2' \) -print -quit)" ]]; then - conda_descriptors="$("${script_directory}/describe-conda.sh" "${artifact_directory}")" - fi - if [[ -n "$(find "${artifact_directory}" -type f -name '*.whl' -print -quit)" ]]; then - wheel_descriptors="$("${script_directory}/describe-wheels.sh" "${artifact_directory}")" - fi - artifacts="$(jq -cn --argjson conda "${conda_descriptors}" --argjson wheel "${wheel_descriptors}" '$conda + $wheel')" - if [[ "$(jq 'length' <<<"${artifacts}")" -eq 0 ]]; then - echo "artifact-directory contains no detectable Conda or wheel artifacts; explicitly selected artifacts require package_identity_file when their identity cannot be parsed" >&2 - exit 1 - fi -else - prepared_artifacts='[]' - while IFS= read -r descriptor; do - primary_file="$(resolve_primary_file "$(jq -r '.path' <<<"${descriptor}")")" - identity_file="$(jq -r '.package_identity_file // empty' <<<"${descriptor}")" - package='' - - case "${primary_file}" in - *.whl) - if [[ -n "${identity_file}" ]]; then - echo "package_identity_file is not allowed when wheel identity can be extracted: $(jq -r '.path' <<<"${descriptor}")" >&2 - exit 1 - fi - package="$("${script_directory}/describe-wheels.sh" "${artifact_directory}" "${primary_file}" | jq -c '.[0].package')" - ;; - *.conda) - if [[ -n "${identity_file}" ]]; then - echo "package_identity_file is not allowed when Conda identity can be extracted: $(jq -r '.path' <<<"${descriptor}")" >&2 - exit 1 - fi - package="$("${script_directory}/describe-conda.sh" "${artifact_directory}" "${primary_file}" | jq -c '.[0].package')" - ;; - *.tar.bz2) - if conda_descriptor="$("${script_directory}/describe-conda.sh" "${artifact_directory}" "${primary_file}" 2>/dev/null)"; then - if [[ -n "${identity_file}" ]]; then - echo "package_identity_file is not allowed when Conda identity can be extracted: $(jq -r '.path' <<<"${descriptor}")" >&2 - exit 1 - fi - package="$(jq -c '.[0].package' <<<"${conda_descriptor}")" - elif [[ -z "${identity_file}" ]]; then - echo "artifact is not a valid Conda package and requires package_identity_file: $(jq -r '.path' <<<"${descriptor}")" >&2 - exit 1 - fi - ;; - *) - if [[ -z "${identity_file}" ]]; then - echo "artifact identity cannot be extracted; package_identity_file is required: $(jq -r '.path' <<<"${descriptor}")" >&2 - exit 1 - fi - ;; - esac - - if [[ -n "${package}" ]]; then - descriptor="$(jq -c --argjson package "${package}" '. + {package: $package}' <<<"${descriptor}")" - fi - prepared_artifacts="$(jq -cn --argjson current "${prepared_artifacts}" --argjson descriptor "${descriptor}" '$current + [$descriptor]')" - done < <(jq -c '.[]' <<<"${artifacts}") - artifacts="${prepared_artifacts}" -fi - -printf 'artifacts=%s\n' "${artifacts}" >>"${GITHUB_OUTPUT}" diff --git a/tests/release_catalog_prepare_test.sh b/tests/release_catalog_discovery_test.sh similarity index 66% rename from tests/release_catalog_prepare_test.sh rename to tests/release_catalog_discovery_test.sh index f897badc..711a8089 100755 --- a/tests/release_catalog_prepare_test.sh +++ b/tests/release_catalog_discovery_test.sh @@ -7,6 +7,12 @@ repository_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" temporary_directory="$(mktemp -d)" trap 'rm -rf "${temporary_directory}"' EXIT +RELEASE_CATALOG_KEY="test:discovery" +RELEASE_ENTRIES_NAME="release-catalog-entries.json" +RELEASE_SOURCE_ARTIFACT_NAME="discovery-test" +RELEASE_SOURCE_SHA="0123456789012345678901234567890123456789" +export RELEASE_CATALOG_KEY RELEASE_ENTRIES_NAME RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA + mkdir -p "${temporary_directory}/wheel/libkvikio_cu12-26.8.0.dist-info" printf '%s\n' \ 'Metadata-Version: 2.1' \ @@ -23,15 +29,14 @@ RELEASE_ARTIFACTS='' RELEASE_ARTIFACT_DIRECTORY="${temporary_directory}" export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY -"${repository_root}/release-catalog/prepare.sh" +"${repository_root}/release-catalog/materialize.sh" -artifacts="$(sed -n 's/^artifacts=//p' "${GITHUB_OUTPUT}")" jq -e ' - . == [{ + [.entries[] | {path, package}] == [{ path: "libkvikio_cu12-26.8.0-py3-none-manylinux_2_28_x86_64.whl", package: {ecosystem: "wheel", name: "libkvikio-cu12", version: "26.8.0"} }] -' <<<"${artifacts}" >/dev/null +' "${temporary_directory}/release-catalog-entries.json" >/dev/null conda_output_directory="${temporary_directory}/conda" conda_staging_directory="${temporary_directory}/conda-staging" @@ -48,11 +53,10 @@ GITHUB_OUTPUT="${temporary_directory}/conda-output" RELEASE_ARTIFACT_DIRECTORY="${conda_output_directory}" export GITHUB_OUTPUT RELEASE_ARTIFACT_DIRECTORY -"${repository_root}/release-catalog/prepare.sh" +"${repository_root}/release-catalog/materialize.sh" -artifacts="$(sed -n 's/^artifacts=//p' "${GITHUB_OUTPUT}")" jq -e ' - . == [{ + [.entries[] | {path, package}] == [{ path: "noarch/rapids-dask-dependency-26.08.0-py_0.tar.bz2", package: { ecosystem: "conda", @@ -62,33 +66,36 @@ jq -e ' platform: "noarch" } }] -' <<<"${artifacts}" >/dev/null +' "${conda_output_directory}/release-catalog-entries.json" >/dev/null GITHUB_OUTPUT="${temporary_directory}/custom-output" RELEASE_ARTIFACT_DIRECTORY="${temporary_directory}" printf '%s\n' bundle >"${temporary_directory}/bundle.tar.gz" +printf '%s\n' sbom >"${temporary_directory}/bundle.spdx.json" jq -n '{ecosystem: "archive", name: "bundle", version: "1.0"}' \ >"${temporary_directory}/release-package-identity.json" RELEASE_ARTIFACTS="$(jq -cn '[{path: "bundle.tar.gz", package_identity_file: "release-package-identity.json", sbom: "bundle.spdx.json"}]')" export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY -"${repository_root}/release-catalog/prepare.sh" +"${repository_root}/release-catalog/materialize.sh" -grep -Fx 'artifacts=[{"path":"bundle.tar.gz","package_identity_file":"release-package-identity.json","sbom":"bundle.spdx.json"}]' "${GITHUB_OUTPUT}" +jq -e ' + .entries[0].path == "bundle.tar.gz" + and .entries[0].package == {ecosystem: "archive", name: "bundle", version: "1.0"} + and .entries[0].sbom_kind == "producer-dependency" +' "${temporary_directory}/release-catalog-entries.json" >/dev/null GITHUB_OUTPUT="${temporary_directory}/explicit-wheel-output" RELEASE_ARTIFACT_DIRECTORY="${temporary_directory}" +printf '%s\n' signature >"${temporary_directory}/libkvikio.sig" RELEASE_ARTIFACTS="$(jq -cn '[{path: "libkvikio_cu12-*.whl", signature: "libkvikio.sig"}]')" export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY -"${repository_root}/release-catalog/prepare.sh" -artifacts="$(sed -n 's/^artifacts=//p' "${GITHUB_OUTPUT}")" +"${repository_root}/release-catalog/materialize.sh" jq -e ' - . == [{ - path: "libkvikio_cu12-*.whl", - signature: "libkvikio.sig", - package: {ecosystem: "wheel", name: "libkvikio-cu12", version: "26.8.0"} - }] -' <<<"${artifacts}" >/dev/null + .entries[0].path == "libkvikio_cu12-26.8.0-py3-none-manylinux_2_28_x86_64.whl" + and .entries[0].package == {ecosystem: "wheel", name: "libkvikio-cu12", version: "26.8.0"} + and (.entries[0].signature | startswith("release-evidence/")) +' "${temporary_directory}/release-catalog-entries.json" >/dev/null mixed_directory="${temporary_directory}/mixed" mkdir -p "${mixed_directory}" @@ -98,20 +105,19 @@ GITHUB_OUTPUT="${temporary_directory}/mixed-output" RELEASE_ARTIFACTS='' RELEASE_ARTIFACT_DIRECTORY="${mixed_directory}" export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY -"${repository_root}/release-catalog/prepare.sh" -artifacts="$(sed -n 's/^artifacts=//p' "${GITHUB_OUTPUT}")" +"${repository_root}/release-catalog/materialize.sh" jq -e ' - length == 2 - and ([.[].package.ecosystem] | sort) == ["conda", "wheel"] -' <<<"${artifacts}" >/dev/null + (.entries | length == 2) + and ([.entries[].package.ecosystem] | sort) == ["conda", "wheel"] +' "${mixed_directory}/release-catalog-entries.json" >/dev/null empty_directory="${temporary_directory}/empty" mkdir -p "${empty_directory}" GITHUB_OUTPUT="${temporary_directory}/empty-output" RELEASE_ARTIFACT_DIRECTORY="${empty_directory}" export GITHUB_OUTPUT RELEASE_ARTIFACT_DIRECTORY -if "${repository_root}/release-catalog/prepare.sh" 2>"${temporary_directory}/empty-error"; then - echo "prepare.sh unexpectedly accepted an artifact directory without detectable artifacts" >&2 +if "${repository_root}/release-catalog/materialize.sh" 2>"${temporary_directory}/empty-error"; then + echo "materialize.sh unexpectedly accepted an artifact directory without detectable artifacts" >&2 exit 1 fi grep -Fx 'artifact-directory contains no detectable Conda or wheel artifacts; explicitly selected artifacts require package_identity_file when their identity cannot be parsed' "${temporary_directory}/empty-error" @@ -120,8 +126,8 @@ GITHUB_OUTPUT="${temporary_directory}/unsupported-output" RELEASE_ARTIFACT_DIRECTORY="${temporary_directory}" RELEASE_ARTIFACTS="$(jq -cn '[{path: "bundle.tar.gz"}]')" export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY -if "${repository_root}/release-catalog/prepare.sh" 2>"${temporary_directory}/unsupported-error"; then - echo "prepare.sh unexpectedly accepted an unsupported artifact without identity" >&2 +if "${repository_root}/release-catalog/materialize.sh" 2>"${temporary_directory}/unsupported-error"; then + echo "materialize.sh unexpectedly accepted an unsupported artifact without identity" >&2 exit 1 fi grep -Fx 'artifact identity cannot be extracted; package_identity_file is required: bundle.tar.gz' "${temporary_directory}/unsupported-error" @@ -129,8 +135,8 @@ grep -Fx 'artifact identity cannot be extracted; package_identity_file is requir GITHUB_OUTPUT="${temporary_directory}/conflicting-identity-output" RELEASE_ARTIFACTS="$(jq -cn '[{path: "libkvikio_cu12-*.whl", package_identity_file: "release-package-identity.json"}]')" export GITHUB_OUTPUT RELEASE_ARTIFACTS -if "${repository_root}/release-catalog/prepare.sh" 2>"${temporary_directory}/conflicting-identity-error"; then - echo "prepare.sh unexpectedly accepted an identity file for a parseable artifact" >&2 +if "${repository_root}/release-catalog/materialize.sh" 2>"${temporary_directory}/conflicting-identity-error"; then + echo "materialize.sh unexpectedly accepted an identity file for a parseable artifact" >&2 exit 1 fi grep -Fx 'package_identity_file is not allowed when wheel identity can be extracted: libkvikio_cu12-*.whl' "${temporary_directory}/conflicting-identity-error" diff --git a/tests/release_catalog_descriptors_test.sh b/tests/release_catalog_package_parsing_test.sh similarity index 60% rename from tests/release_catalog_descriptors_test.sh rename to tests/release_catalog_package_parsing_test.sh index 1c608e51..a54a4f53 100755 --- a/tests/release_catalog_descriptors_test.sh +++ b/tests/release_catalog_package_parsing_test.sh @@ -7,6 +7,14 @@ repository_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" temporary_directory="$(mktemp -d)" trap 'rm -rf "${temporary_directory}"' EXIT +GITHUB_OUTPUT="${temporary_directory}/github-output" +RELEASE_ARTIFACTS='' +RELEASE_CATALOG_KEY="test:package-parsing" +RELEASE_ENTRIES_NAME="release-catalog-entries.json" +RELEASE_SOURCE_ARTIFACT_NAME="package-parsing-test" +RELEASE_SOURCE_SHA="0123456789012345678901234567890123456789" +export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_CATALOG_KEY RELEASE_ENTRIES_NAME RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA + wheel_output_directory="${temporary_directory}/wheels" wheel_staging_directory="${temporary_directory}/wheel-staging" mkdir -p "${wheel_output_directory}" "${wheel_staging_directory}/libkvikio_cu12-26.8.0.dist-info" @@ -20,13 +28,15 @@ printf '%s\n' \ zip -qr "${wheel_output_directory}/libkvikio_cu12-26.8.0a32-py3-none-manylinux_2_28_x86_64.whl" . ) -wheel_descriptors="$("${repository_root}/release-catalog/describe-wheels.sh" "${wheel_output_directory}")" +RELEASE_ARTIFACT_DIRECTORY="${wheel_output_directory}" +export RELEASE_ARTIFACT_DIRECTORY +"${repository_root}/release-catalog/materialize.sh" jq -e ' - . == [{ + [.entries[] | {path, package}] == [{ path: "libkvikio_cu12-26.8.0a32-py3-none-manylinux_2_28_x86_64.whl", package: {ecosystem: "wheel", name: "libkvikio-cu12", version: "26.8.0a32"} }] -' <<<"${wheel_descriptors}" >/dev/null +' "${wheel_output_directory}/release-catalog-entries.json" >/dev/null conda_output_directory="${temporary_directory}/conda" tar_bz2_staging_directory="${temporary_directory}/tar-bz2-staging" @@ -57,10 +67,13 @@ zstd -q -f "${temporary_directory}/info-librmm.tar" -o "${temporary_directory}/i info-librmm.tar.zst ) -conda_descriptors="$("${repository_root}/release-catalog/describe-conda.sh" "${conda_output_directory}")" +GITHUB_OUTPUT="${temporary_directory}/conda-github-output" +RELEASE_ARTIFACT_DIRECTORY="${conda_output_directory}" +export GITHUB_OUTPUT RELEASE_ARTIFACT_DIRECTORY +"${repository_root}/release-catalog/materialize.sh" jq -e ' - length == 2 - and any(.[]; + (.entries | length == 2) + and any(.entries[]; .path == "linux-64/librmm-26.08.00a32-cuda12_260714_2f567060.conda" and .package == { ecosystem: "conda", @@ -70,7 +83,7 @@ jq -e ' platform: "linux-64" } ) - and any(.[]; + and any(.entries[]; .path == "noarch/rapids-dask-dependency-26.08.0-py_0.tar.bz2" and .package == { ecosystem: "conda", @@ -80,4 +93,17 @@ jq -e ' platform: "noarch" } ) -' <<<"${conda_descriptors}" >/dev/null +' "${conda_output_directory}/release-catalog-entries.json" >/dev/null + +invalid_wheel_directory="${temporary_directory}/invalid-wheel" +mkdir -p "${invalid_wheel_directory}" +printf '%s\n' invalid >"${invalid_wheel_directory}/invalid.whl" +GITHUB_OUTPUT="${temporary_directory}/invalid-wheel-output" +RELEASE_ARTIFACT_DIRECTORY="${invalid_wheel_directory}" +export GITHUB_OUTPUT RELEASE_ARTIFACT_DIRECTORY +if "${repository_root}/release-catalog/materialize.sh" 2>"${temporary_directory}/invalid-wheel-error"; then + echo "materialize.sh unexpectedly accepted an invalid wheel" >&2 + exit 1 +fi +grep -F 'wheel must contain exactly one .dist-info/METADATA file: invalid.whl' \ + "${temporary_directory}/invalid-wheel-error" >/dev/null diff --git a/tests/release_catalog_test.sh b/tests/release_catalog_test.sh index 5278994b..02042c87 100755 --- a/tests/release_catalog_test.sh +++ b/tests/release_catalog_test.sh @@ -97,9 +97,11 @@ jq -e ' generated_directory="${temporary_directory}/generated-bundle" mkdir -p "${generated_directory}/linux-64" -printf '%s\n' conda >"${generated_directory}/linux-64/kvikio-26.08.00a32-cuda12_260714_2f567060.conda" +printf '%s\n' conda >"${generated_directory}/linux-64/kvikio-26.08.00a32-cuda12_260714_2f567060.bin" +jq -n '{ecosystem: "conda", name: "kvikio", version: "26.08.00a32"}' \ + >"${generated_directory}/kvikio.identity.json" -RELEASE_ARTIFACTS="$(jq -cn '[{path: "linux-64/kvikio-*.conda", package: {ecosystem: "conda", name: "kvikio", version: "26.08.00a32"}}]')" +RELEASE_ARTIFACTS="$(jq -cn '[{path: "linux-64/kvikio-*.bin", package_identity_file: "kvikio.identity.json"}]')" RELEASE_ARTIFACT_DIRECTORY="${generated_directory}" RELEASE_SOURCE_ARTIFACT_NAME="kvikio_conda_python_kvikio_x86_64_abi3_cu12" RELEASE_SOURCE_SHA="bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" @@ -115,7 +117,7 @@ jq -e ' .source.artifact == "kvikio_conda_python_kvikio_x86_64_abi3_cu12" and .source.sha == "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" and .entries[0].release_catalog_key == "conda:kvikio" - and .entries[0].path == "linux-64/kvikio-26.08.00a32-cuda12_260714_2f567060.conda" + and .entries[0].path == "linux-64/kvikio-26.08.00a32-cuda12_260714_2f567060.bin" and .entries[0].package == {ecosystem: "conda", name: "kvikio", version: "26.08.00a32"} and .entries[0].sbom_kind == "generated-identity" ' "${generated_entries_path}" >/dev/null @@ -131,9 +133,11 @@ jq -e ' ' "${generated_directory}/${generated_provenance_path}" >/dev/null wheel_directory="${temporary_directory}/wheel-bundle" mkdir -p "${wheel_directory}" -printf '%s\n' wheel >"${wheel_directory}/libkvikio_cu12-26.8.0a32-py3-none-manylinux_2_28_x86_64.whl" +printf '%s\n' wheel >"${wheel_directory}/libkvikio_cu12-26.8.0a32.bin" +jq -n '{ecosystem: "wheel", name: "libkvikio-cu12", version: "26.8.0a32"}' \ + >"${wheel_directory}/libkvikio.identity.json" -RELEASE_ARTIFACTS="$(jq -cn '[{path: "libkvikio_cu12-*.whl", package: {ecosystem: "wheel", name: "libkvikio-cu12", version: "26.8.0a32"}}]')" +RELEASE_ARTIFACTS="$(jq -cn '[{path: "libkvikio_cu12-*.bin", package_identity_file: "libkvikio.identity.json"}]')" RELEASE_ARTIFACT_DIRECTORY="${wheel_directory}" RELEASE_SOURCE_ARTIFACT_NAME="kvikio_wheel_cpp_libkvikio_x86_64_cu12" RELEASE_SOURCE_SHA="cccccccccccccccccccccccccccccccccccccccc" @@ -144,7 +148,7 @@ export RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY RELEASE_SOURCE_ARTIFACT_NAME jq -e ' .entries[0].release_catalog_key == "wheel:kvikio" - and .entries[0].path == "libkvikio_cu12-26.8.0a32-py3-none-manylinux_2_28_x86_64.whl" + and .entries[0].path == "libkvikio_cu12-26.8.0a32.bin" and .entries[0].package == {ecosystem: "wheel", name: "libkvikio-cu12", version: "26.8.0a32"} and .entries[0].sbom_kind == "generated-identity" ' "${wheel_directory}/release-catalog-entries.json" >/dev/null From 37cc653375326dba03e60187004ee7c0c8881f96 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 18 Aug 2026 12:14:20 -0500 Subject: [PATCH 12/16] simplify release catalog action interface --- release-catalog-dispatch/action.yml | 26 ++--------------- release-catalog/README.md | 13 ++++----- release-catalog/materialize.sh | 5 ++-- tests/release_catalog_discovery_test.sh | 28 +++++++----------- tests/release_catalog_package_parsing_test.sh | 11 +++---- tests/release_catalog_test.sh | 29 ++++++++++--------- 6 files changed, 41 insertions(+), 71 deletions(-) diff --git a/release-catalog-dispatch/action.yml b/release-catalog-dispatch/action.yml index 9c2e9257..77758741 100644 --- a/release-catalog-dispatch/action.yml +++ b/release-catalog-dispatch/action.yml @@ -8,20 +8,8 @@ inputs: https://github.com/rapidsai/shared-actions/blob/main/release-catalog/config.schema.json required: true source-artifact-name: - description: Name of the GitHub Actions artifact bundle containing this output. + description: Name of the existing GitHub Actions artifact whose files are being cataloged. required: true - source-sha: - description: Source revision built by the producing job. - required: false - -outputs: - entries-path: - description: Absolute path to the generated release catalog entries. - value: ${{ steps.materialize.outputs.entries-path }} - companion-artifact-name: - description: Name of the uploaded GitHub Actions companion artifact. - value: ${{ steps.companion-name.outputs.name }} - runs: using: composite steps: @@ -51,8 +39,7 @@ runs: IMPLEMENTATION_PATH: ${{ steps.implementation.outputs.path }} RELEASE_CATALOG_CONFIG: ${{ inputs.config }} run: "${IMPLEMENTATION_PATH}/validate-config.sh" - - id: materialize - name: Materialize release catalog entries + - name: Materialize release catalog entries shell: bash env: IMPLEMENTATION_PATH: ${{ steps.implementation.outputs.path }} @@ -60,20 +47,13 @@ runs: RELEASE_ENTRIES_NAME: release-catalog-entries.json RELEASE_ARTIFACT_DIRECTORY: ${{ steps.configuration.outputs.artifact_directory }} RELEASE_SOURCE_ARTIFACT_NAME: ${{ inputs.source-artifact-name }} - RELEASE_SOURCE_SHA: ${{ inputs.source-sha || github.sha }} RELEASE_CATALOG_KEY: ${{ steps.configuration.outputs.release_catalog_key }} run: "${IMPLEMENTATION_PATH}/materialize.sh" - - id: companion-name - name: Set companion artifact name - shell: bash - env: - SOURCE_ARTIFACT_NAME: ${{ inputs.source-artifact-name }} - run: echo "name=release-catalog-${SOURCE_ARTIFACT_NAME}" >>"${GITHUB_OUTPUT}" - name: Upload release catalog companion uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: if-no-files-found: error - name: ${{ steps.companion-name.outputs.name }} + name: release-catalog-${{ inputs.source-artifact-name }} path: | ${{ steps.configuration.outputs.artifact_directory }}/release-catalog-entries.json ${{ steps.configuration.outputs.artifact_directory }}/release-evidence/** diff --git a/release-catalog/README.md b/release-catalog/README.md index 8e26c08e..e0d2d576 100644 --- a/release-catalog/README.md +++ b/release-catalog/README.md @@ -61,8 +61,8 @@ unambiguously. ## Source revision -`source-sha` must identify the repository revision actually checked out and -built. RAPIDS shared workflows pass `${{ env.RAPIDS_SHA }}`: +The action requires `RAPIDS_SHA` in the job environment. It must identify the +repository revision actually checked out and built: - Conda build workflows set `RAPIDS_SHA` to `git rev-parse HEAD` immediately after checkout. @@ -70,9 +70,10 @@ built. RAPIDS shared workflows pass `${{ env.RAPIDS_SHA }}`: when supplied and otherwise sets `RAPIDS_SHA` to `git rev-parse HEAD`. This distinction matters when a reusable workflow checks out a repository or -revision different from the workflow event. Direct callers should likewise -resolve the checked-out commit instead of assuming `${{ github.sha }}` names -the built source. +revision different from the workflow event. Direct callers must likewise set +`RAPIDS_SHA` to the checked-out commit instead of assuming `${{ github.sha }}` +names the built source. Variables written to `GITHUB_ENV` are available to the +action when it runs as a subsequent job step. ## Standard package example @@ -86,7 +87,6 @@ the built source. "artifact_directory": ${{ toJSON(steps.package-name.outputs.WHEEL_OUTPUT_DIR) }} } source-artifact-name: ${{ steps.package-name.outputs.RAPIDS_PACKAGE_NAME }} - source-sha: ${{ env.RAPIDS_SHA }} ``` ## Custom package example @@ -105,7 +105,6 @@ the built source. }] } source-artifact-name: cuvs-java - source-sha: ${{ env.RAPIDS_SHA }} ``` Before the action runs, the producer creates diff --git a/release-catalog/materialize.sh b/release-catalog/materialize.sh index 1a353a15..62500806 100755 --- a/release-catalog/materialize.sh +++ b/release-catalog/materialize.sh @@ -17,8 +17,8 @@ require_nonempty "RELEASE_ARTIFACT_DIRECTORY" "${RELEASE_ARTIFACT_DIRECTORY:-}" require_nonempty "RELEASE_ENTRIES_NAME" "${RELEASE_ENTRIES_NAME:-}" require_nonempty "RELEASE_SOURCE_ARTIFACT_NAME" "${RELEASE_SOURCE_ARTIFACT_NAME:-}" -source_sha="${RELEASE_SOURCE_SHA:-${GITHUB_SHA:-}}" -require_nonempty "RELEASE_SOURCE_SHA or GITHUB_SHA" "${source_sha}" +source_sha="${RAPIDS_SHA:-}" +require_nonempty "RAPIDS_SHA" "${source_sha}" require_plain_filename() { local label="$1" @@ -447,4 +447,3 @@ jq -n -S \ }, entries: $entries }' >"${entries_path}" -echo "entries-path=${entries_path}" >>"${GITHUB_OUTPUT}" diff --git a/tests/release_catalog_discovery_test.sh b/tests/release_catalog_discovery_test.sh index 711a8089..ed24007e 100755 --- a/tests/release_catalog_discovery_test.sh +++ b/tests/release_catalog_discovery_test.sh @@ -10,8 +10,8 @@ trap 'rm -rf "${temporary_directory}"' EXIT RELEASE_CATALOG_KEY="test:discovery" RELEASE_ENTRIES_NAME="release-catalog-entries.json" RELEASE_SOURCE_ARTIFACT_NAME="discovery-test" -RELEASE_SOURCE_SHA="0123456789012345678901234567890123456789" -export RELEASE_CATALOG_KEY RELEASE_ENTRIES_NAME RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA +RAPIDS_SHA="0123456789012345678901234567890123456789" +export RAPIDS_SHA RELEASE_CATALOG_KEY RELEASE_ENTRIES_NAME RELEASE_SOURCE_ARTIFACT_NAME mkdir -p "${temporary_directory}/wheel/libkvikio_cu12-26.8.0.dist-info" printf '%s\n' \ @@ -24,10 +24,9 @@ printf '%s\n' \ zip -qr "${temporary_directory}/libkvikio_cu12-26.8.0-py3-none-manylinux_2_28_x86_64.whl" . ) -GITHUB_OUTPUT="${temporary_directory}/wheel-output" RELEASE_ARTIFACTS='' RELEASE_ARTIFACT_DIRECTORY="${temporary_directory}" -export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY +export RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY "${repository_root}/release-catalog/materialize.sh" @@ -49,9 +48,8 @@ tar -cjf \ -C "${conda_staging_directory}" \ info/index.json -GITHUB_OUTPUT="${temporary_directory}/conda-output" RELEASE_ARTIFACT_DIRECTORY="${conda_output_directory}" -export GITHUB_OUTPUT RELEASE_ARTIFACT_DIRECTORY +export RELEASE_ARTIFACT_DIRECTORY "${repository_root}/release-catalog/materialize.sh" @@ -68,14 +66,13 @@ jq -e ' }] ' "${conda_output_directory}/release-catalog-entries.json" >/dev/null -GITHUB_OUTPUT="${temporary_directory}/custom-output" RELEASE_ARTIFACT_DIRECTORY="${temporary_directory}" printf '%s\n' bundle >"${temporary_directory}/bundle.tar.gz" printf '%s\n' sbom >"${temporary_directory}/bundle.spdx.json" jq -n '{ecosystem: "archive", name: "bundle", version: "1.0"}' \ >"${temporary_directory}/release-package-identity.json" RELEASE_ARTIFACTS="$(jq -cn '[{path: "bundle.tar.gz", package_identity_file: "release-package-identity.json", sbom: "bundle.spdx.json"}]')" -export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY +export RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY "${repository_root}/release-catalog/materialize.sh" @@ -85,11 +82,10 @@ jq -e ' and .entries[0].sbom_kind == "producer-dependency" ' "${temporary_directory}/release-catalog-entries.json" >/dev/null -GITHUB_OUTPUT="${temporary_directory}/explicit-wheel-output" RELEASE_ARTIFACT_DIRECTORY="${temporary_directory}" printf '%s\n' signature >"${temporary_directory}/libkvikio.sig" RELEASE_ARTIFACTS="$(jq -cn '[{path: "libkvikio_cu12-*.whl", signature: "libkvikio.sig"}]')" -export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY +export RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY "${repository_root}/release-catalog/materialize.sh" jq -e ' .entries[0].path == "libkvikio_cu12-26.8.0-py3-none-manylinux_2_28_x86_64.whl" @@ -101,10 +97,9 @@ mixed_directory="${temporary_directory}/mixed" mkdir -p "${mixed_directory}" cp "${temporary_directory}/libkvikio_cu12-26.8.0-py3-none-manylinux_2_28_x86_64.whl" "${mixed_directory}/" cp "${conda_output_directory}/noarch/rapids-dask-dependency-26.08.0-py_0.tar.bz2" "${mixed_directory}/" -GITHUB_OUTPUT="${temporary_directory}/mixed-output" RELEASE_ARTIFACTS='' RELEASE_ARTIFACT_DIRECTORY="${mixed_directory}" -export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY +export RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY "${repository_root}/release-catalog/materialize.sh" jq -e ' (.entries | length == 2) @@ -113,28 +108,25 @@ jq -e ' empty_directory="${temporary_directory}/empty" mkdir -p "${empty_directory}" -GITHUB_OUTPUT="${temporary_directory}/empty-output" RELEASE_ARTIFACT_DIRECTORY="${empty_directory}" -export GITHUB_OUTPUT RELEASE_ARTIFACT_DIRECTORY +export RELEASE_ARTIFACT_DIRECTORY if "${repository_root}/release-catalog/materialize.sh" 2>"${temporary_directory}/empty-error"; then echo "materialize.sh unexpectedly accepted an artifact directory without detectable artifacts" >&2 exit 1 fi grep -Fx 'artifact-directory contains no detectable Conda or wheel artifacts; explicitly selected artifacts require package_identity_file when their identity cannot be parsed' "${temporary_directory}/empty-error" -GITHUB_OUTPUT="${temporary_directory}/unsupported-output" RELEASE_ARTIFACT_DIRECTORY="${temporary_directory}" RELEASE_ARTIFACTS="$(jq -cn '[{path: "bundle.tar.gz"}]')" -export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY +export RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY if "${repository_root}/release-catalog/materialize.sh" 2>"${temporary_directory}/unsupported-error"; then echo "materialize.sh unexpectedly accepted an unsupported artifact without identity" >&2 exit 1 fi grep -Fx 'artifact identity cannot be extracted; package_identity_file is required: bundle.tar.gz' "${temporary_directory}/unsupported-error" -GITHUB_OUTPUT="${temporary_directory}/conflicting-identity-output" RELEASE_ARTIFACTS="$(jq -cn '[{path: "libkvikio_cu12-*.whl", package_identity_file: "release-package-identity.json"}]')" -export GITHUB_OUTPUT RELEASE_ARTIFACTS +export RELEASE_ARTIFACTS if "${repository_root}/release-catalog/materialize.sh" 2>"${temporary_directory}/conflicting-identity-error"; then echo "materialize.sh unexpectedly accepted an identity file for a parseable artifact" >&2 exit 1 diff --git a/tests/release_catalog_package_parsing_test.sh b/tests/release_catalog_package_parsing_test.sh index a54a4f53..3b037c4d 100755 --- a/tests/release_catalog_package_parsing_test.sh +++ b/tests/release_catalog_package_parsing_test.sh @@ -7,13 +7,12 @@ repository_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" temporary_directory="$(mktemp -d)" trap 'rm -rf "${temporary_directory}"' EXIT -GITHUB_OUTPUT="${temporary_directory}/github-output" RELEASE_ARTIFACTS='' RELEASE_CATALOG_KEY="test:package-parsing" RELEASE_ENTRIES_NAME="release-catalog-entries.json" RELEASE_SOURCE_ARTIFACT_NAME="package-parsing-test" -RELEASE_SOURCE_SHA="0123456789012345678901234567890123456789" -export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_CATALOG_KEY RELEASE_ENTRIES_NAME RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA +RAPIDS_SHA="0123456789012345678901234567890123456789" +export RAPIDS_SHA RELEASE_ARTIFACTS RELEASE_CATALOG_KEY RELEASE_ENTRIES_NAME RELEASE_SOURCE_ARTIFACT_NAME wheel_output_directory="${temporary_directory}/wheels" wheel_staging_directory="${temporary_directory}/wheel-staging" @@ -67,9 +66,8 @@ zstd -q -f "${temporary_directory}/info-librmm.tar" -o "${temporary_directory}/i info-librmm.tar.zst ) -GITHUB_OUTPUT="${temporary_directory}/conda-github-output" RELEASE_ARTIFACT_DIRECTORY="${conda_output_directory}" -export GITHUB_OUTPUT RELEASE_ARTIFACT_DIRECTORY +export RELEASE_ARTIFACT_DIRECTORY "${repository_root}/release-catalog/materialize.sh" jq -e ' (.entries | length == 2) @@ -98,9 +96,8 @@ jq -e ' invalid_wheel_directory="${temporary_directory}/invalid-wheel" mkdir -p "${invalid_wheel_directory}" printf '%s\n' invalid >"${invalid_wheel_directory}/invalid.whl" -GITHUB_OUTPUT="${temporary_directory}/invalid-wheel-output" RELEASE_ARTIFACT_DIRECTORY="${invalid_wheel_directory}" -export GITHUB_OUTPUT RELEASE_ARTIFACT_DIRECTORY +export RELEASE_ARTIFACT_DIRECTORY if "${repository_root}/release-catalog/materialize.sh" 2>"${temporary_directory}/invalid-wheel-error"; then echo "materialize.sh unexpectedly accepted an invalid wheel" >&2 exit 1 diff --git a/tests/release_catalog_test.sh b/tests/release_catalog_test.sh index 02042c87..1383f32d 100755 --- a/tests/release_catalog_test.sh +++ b/tests/release_catalog_test.sh @@ -17,22 +17,20 @@ jq -n \ '{ecosystem: "maven", name: "ai.rapids:cuvs-java", version: "26.08.0"}' \ >"${bundle_directory}/cuvs-java-package-identity.json" -GITHUB_OUTPUT="${temporary_directory}/github-output" GITHUB_REPOSITORY="rapidsai/cuvs" GITHUB_RUN_ATTEMPT="1" GITHUB_RUN_ID="1234" GITHUB_SHA="0123456789012345678901234567890123456789" GITHUB_WORKFLOW_REF="rapidsai/cuvs/.github/workflows/build.yaml@refs/heads/release/26.08" -export GITHUB_OUTPUT RELEASE_ARTIFACTS="$(jq -cn '[{path: "cuvs-java-*.jar", package_identity_file: "cuvs-java-package-identity.json", sbom: "cuvs-java-*.spdx.json", provenance: "cuvs-java-*.provenance.jsonl", signature: "cuvs-java-*.jar.asc"}]')" RELEASE_ENTRIES_NAME="release-catalog-entries.json" RELEASE_ARTIFACT_DIRECTORY="${bundle_directory}" RELEASE_SOURCE_ARTIFACT_NAME="cuvs-java-cuda12.9.1" -RELEASE_SOURCE_SHA="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +RAPIDS_SHA="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" RELEASE_CATALOG_KEY="maven:cuvs-java" export GITHUB_REPOSITORY GITHUB_RUN_ATTEMPT GITHUB_RUN_ID GITHUB_SHA GITHUB_WORKFLOW_REF export RELEASE_ARTIFACTS RELEASE_ENTRIES_NAME RELEASE_ARTIFACT_DIRECTORY -export RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY +export RAPIDS_SHA RELEASE_SOURCE_ARTIFACT_NAME RELEASE_CATALOG_KEY "${repository_root}/release-catalog/materialize.sh" @@ -51,7 +49,6 @@ jq -e ' and .entries[0].package.name == "ai.rapids:cuvs-java" and .entries[0].sbom_kind == "producer-dependency" ' "${entries_path}" >/dev/null -grep -Fx "entries-path=${entries_path}" "${GITHUB_OUTPUT}" supplied_sbom_path="$(jq -r '.entries[0].sbom' "${entries_path}")" supplied_provenance_path="$(jq -r '.entries[0].provenance' "${entries_path}")" @@ -85,9 +82,9 @@ RELEASE_ARTIFACTS="$(jq -cn '[ ]')" RELEASE_ARTIFACT_DIRECTORY="${multiple_identity_directory}" RELEASE_SOURCE_ARTIFACT_NAME="multiple-identities" -RELEASE_SOURCE_SHA="eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" +RAPIDS_SHA="eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" RELEASE_CATALOG_KEY="archive:multiple" -export RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY +export RAPIDS_SHA RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY RELEASE_SOURCE_ARTIFACT_NAME RELEASE_CATALOG_KEY "${repository_root}/release-catalog/materialize.sh" jq -e ' (.entries | length == 2) @@ -104,9 +101,9 @@ jq -n '{ecosystem: "conda", name: "kvikio", version: "26.08.00a32"}' \ RELEASE_ARTIFACTS="$(jq -cn '[{path: "linux-64/kvikio-*.bin", package_identity_file: "kvikio.identity.json"}]')" RELEASE_ARTIFACT_DIRECTORY="${generated_directory}" RELEASE_SOURCE_ARTIFACT_NAME="kvikio_conda_python_kvikio_x86_64_abi3_cu12" -RELEASE_SOURCE_SHA="bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" +RAPIDS_SHA="bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" RELEASE_CATALOG_KEY="conda:kvikio" -export RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY +export RAPIDS_SHA RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY RELEASE_SOURCE_ARTIFACT_NAME RELEASE_CATALOG_KEY "${repository_root}/release-catalog/materialize.sh" @@ -140,9 +137,9 @@ jq -n '{ecosystem: "wheel", name: "libkvikio-cu12", version: "26.8.0a32"}' \ RELEASE_ARTIFACTS="$(jq -cn '[{path: "libkvikio_cu12-*.bin", package_identity_file: "libkvikio.identity.json"}]')" RELEASE_ARTIFACT_DIRECTORY="${wheel_directory}" RELEASE_SOURCE_ARTIFACT_NAME="kvikio_wheel_cpp_libkvikio_x86_64_cu12" -RELEASE_SOURCE_SHA="cccccccccccccccccccccccccccccccccccccccc" +RAPIDS_SHA="cccccccccccccccccccccccccccccccccccccccc" RELEASE_CATALOG_KEY="wheel:kvikio" -export RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY +export RAPIDS_SHA RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY RELEASE_SOURCE_ARTIFACT_NAME RELEASE_CATALOG_KEY "${repository_root}/release-catalog/materialize.sh" @@ -163,12 +160,18 @@ jq -n \ RELEASE_ARTIFACTS="$(jq -cn '[{path: "bundle.tar.gz", package_identity_file: "bundle-package-identity.json"}]')" RELEASE_ARTIFACT_DIRECTORY="${missing_version_directory}" RELEASE_SOURCE_ARTIFACT_NAME="bundle-archive" -RELEASE_SOURCE_SHA="dddddddddddddddddddddddddddddddddddddddd" +RAPIDS_SHA="dddddddddddddddddddddddddddddddddddddddd" RELEASE_CATALOG_KEY="archive:bundle" -export RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY +export RAPIDS_SHA RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY RELEASE_SOURCE_ARTIFACT_NAME RELEASE_CATALOG_KEY if "${repository_root}/release-catalog/materialize.sh" 2>"${temporary_directory}/missing-version-error"; then echo "materialize.sh unexpectedly accepted a custom artifact without a package version" >&2 exit 1 fi grep -Fx 'package identity for bundle.tar.gz must contain non-empty ecosystem, name, and version strings and only optional build or platform strings' "${temporary_directory}/missing-version-error" + +if (unset RAPIDS_SHA; "${repository_root}/release-catalog/materialize.sh") 2>"${temporary_directory}/missing-rapids-sha-error"; then + echo "materialize.sh unexpectedly accepted a missing RAPIDS_SHA" >&2 + exit 1 +fi +grep -Fx 'RAPIDS_SHA must be a non-empty string' "${temporary_directory}/missing-rapids-sha-error" From 7983299c01b890c92a88142905d4449c1cb4142c Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 18 Aug 2026 13:29:37 -0500 Subject: [PATCH 13/16] defer producer-supplied release evidence --- README.md | 2 +- release-catalog/README.md | 39 +++++-------- release-catalog/config.schema.json | 22 +------ release-catalog/materialize.sh | 57 ++++--------------- release-catalog/validate-config.sh | 4 +- .../package-identity-file.json | 3 +- tests/release_catalog_config_test.sh | 5 +- tests/release_catalog_discovery_test.sh | 10 ++-- tests/release_catalog_test.sh | 19 +------ 9 files changed, 39 insertions(+), 122 deletions(-) diff --git a/README.md b/README.md index 88248366..40f29a86 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ directory and uploads a companion artifact named See the [release catalog documentation](release-catalog/README.md) for configuration, examples, source-revision handling, companion contents, and -evidence semantics. +generated identity evidence semantics. The dispatch wrapper honors `SHARED_ACTIONS_REPO` and `SHARED_ACTIONS_REF`. When neither is set, it checks out the same repository and ref used to invoke diff --git a/release-catalog/README.md b/release-catalog/README.md index e0d2d576..f17bb2ef 100644 --- a/release-catalog/README.md +++ b/release-catalog/README.md @@ -3,8 +3,8 @@ `release-catalog-dispatch` records the exact files produced by a package build and uploads a companion GitHub Actions artifact named `release-catalog-`. Release tooling uses the -companion to associate primary artifacts with their build identity and -available evidence without reconstructing the build later. +companion to associate primary artifacts with their build identity and generated +identity evidence without reconstructing the build later. The companion contains: @@ -31,8 +31,7 @@ When `artifacts` is omitted, the action discovers every Conda and wheel output in `artifact_directory` and extracts identity from each package independently. When `artifacts` is supplied, each descriptor is resolved independently: supported Conda and wheel files are parsed, while any other artifact requires -its own `package_identity_file`. Evidence paths also belong to the individual -artifact descriptor. +its own `package_identity_file`. `release_catalog_key` identifies the release catalog entry that owns these artifacts. Every file from every matrix variant in the same publishable artifact @@ -56,8 +55,7 @@ keys: for example, `cudf` and `dask-cudf` Conda packages remain part of The action validates the configuration before inspecting build outputs. It then verifies properties that depend on produced files, including the package -identity contents and whether primary-artifact and evidence paths resolve -unambiguously. +identity contents and whether primary-artifact paths resolve unambiguously. ## Source revision @@ -120,25 +118,14 @@ relative to `artifact_directory`. For example: } ``` -## Evidence semantics +## Generated identity evidence -A descriptor-selected producer SBOM is classified as `producer-dependency`. -It is evidence supplied by the producer and may contain a dependency inventory. +For every artifact, the action generates an SPDX artifact-identity envelope +classified as `generated-identity` and a build-context provenance statement. +They record the artifact SHA-256, package identity, source revision, and workflow +context, but contain no dependency or source-license inventory. They must not be +reported as producer-supplied dependency coverage. -When no SBOM is selected, the action generates an SPDX artifact-identity -envelope classified as `generated-identity`. It records package identity and -the primary artifact SHA-256, but contains no dependency or source-license -inventory. It must not be reported as producer-supplied dependency coverage. - -Producer-supplied SBOM, provenance, and signature sidecars are copied under -`release-evidence/` so the companion remains independently consumable. - -Concrete producer-supplied evidence examples include: - -- an official [SPDX 2.3 dependency SBOM](https://github.com/spdx/spdx-examples/blob/2181917ef6ff74de89252ee785583c27a38d6199/presentations/OSS-NA-2023/SPDXVersion2.3/03-SBOMwDependency.json); -- an official [SLSA provenance v1 statement](https://github.com/slsa-framework/github-actions-buildtypes/blob/5f855ef0106dad3ee0e0f1046dc31b3b65152956/workflow/v1/example.json); -- a Maven Central [detached ASCII-armored signature](https://repo1.maven.org/maven2/org/apache/commons/commons-lang3/3.17.0/commons-lang3-3.17.0.jar.asc). - -These examples illustrate the expected purpose of the files, not required -serialization formats. The action copies producer-supplied evidence as opaque -sidecars and does not validate their contents or require these formats. +Producer-supplied SBOM, provenance, and signature inputs are intentionally +deferred until a future revision defines a generic, unambiguous way to associate +them with detected artifacts. diff --git a/release-catalog/config.schema.json b/release-catalog/config.schema.json index 0f099e1b..7df259e0 100644 --- a/release-catalog/config.schema.json +++ b/release-catalog/config.schema.json @@ -16,14 +16,14 @@ "pattern": "^[^\\r\\n]+$" }, "artifact_directory": { - "description": "Base directory for artifact and evidence paths. Defaults to the job working directory.", + "description": "Base directory for artifact paths. Defaults to the job working directory.", "type": "string", "minLength": 1, "pattern": "^[^\\r\\n]+$", "default": "." }, "artifacts": { - "description": "Optional explicit primary release files and their identity or evidence sidecars. When omitted, Conda and wheel artifacts are discovered in artifact_directory.", + "description": "Optional explicit primary release files. When omitted, Conda and wheel artifacts are discovered in artifact_directory.", "type": "array", "minItems": 1, "items": { @@ -49,24 +49,6 @@ "type": "string", "minLength": 1, "pattern": "^[^\\r\\n]+$" - }, - "sbom": { - "description": "Dependency SBOM path or glob. Example file: https://github.com/spdx/spdx-examples/blob/2181917ef6ff74de89252ee785583c27a38d6199/presentations/OSS-NA-2023/SPDXVersion2.3/03-SBOMwDependency.json", - "type": "string", - "minLength": 1, - "pattern": "^[^\\r\\n]+$" - }, - "provenance": { - "description": "Provenance path or glob. Example file: https://github.com/slsa-framework/github-actions-buildtypes/blob/5f855ef0106dad3ee0e0f1046dc31b3b65152956/workflow/v1/example.json", - "type": "string", - "minLength": 1, - "pattern": "^[^\\r\\n]+$" - }, - "signature": { - "description": "Signature path or glob. Example file: https://repo1.maven.org/maven2/org/apache/commons/commons-lang3/3.17.0/commons-lang3-3.17.0.jar.asc", - "type": "string", - "minLength": 1, - "pattern": "^[^\\r\\n]+$" } } } diff --git a/release-catalog/materialize.sh b/release-catalog/materialize.sh index 62500806..1bde11ee 100755 --- a/release-catalog/materialize.sh +++ b/release-catalog/materialize.sh @@ -260,20 +260,6 @@ generated_evidence_path() { printf 'release-evidence/%s.%s.%s.json\n' "${artifact_label}" "${artifact_digest}" "${kind}" } -copy_supplied_evidence() { - local primary_path="$1" - local supplied_path="$2" - local kind="$3" - local artifact_digest - artifact_digest="$(sha256sum "${artifact_directory}/${primary_path}" | awk '{print $1}')" - local artifact_label="${primary_path//\//_}" - local destination - destination="release-evidence/${artifact_label}.${artifact_digest}/${kind}-$(basename "${supplied_path}")" - mkdir -p "$(dirname "${artifact_directory}/${destination}")" - cp "${artifact_directory}/${supplied_path}" "${artifact_directory}/${destination}" - printf '%s\n' "${destination}" -} - write_generated_sbom() { local primary_path="$1" local package="$2" @@ -311,7 +297,7 @@ write_generated_sbom() { relationshipType: "DESCRIBES", relatedSpdxElement: "SPDXRef-Artifact" }], - comment: "Artifact-identity SBOM envelope. A producer-supplied dependency SBOM may replace this record." + comment: "Artifact-identity SBOM envelope. It does not contain a dependency inventory." }' >"${artifact_directory}/${destination}" } @@ -356,22 +342,16 @@ shopt -s globstar nullglob while IFS= read -r descriptor; do if ! jq -e ' type == "object" - and (keys - ["path", "package_identity_file", "sbom", "provenance", "signature", "package"] | length == 0) + and (keys - ["path", "package_identity_file", "package"] | length == 0) and (.path | type == "string" and length > 0) and ((.package_identity_file // "") | type == "string") - and ((.sbom // "") | type == "string") - and ((.provenance // "") | type == "string") - and ((.signature // "") | type == "string") and ([has("package"), has("package_identity_file")] | map(select(.)) | length == 1) ' <<<"${descriptor}" >/dev/null; then - echo "release artifact descriptor must contain path, exactly one package identity source, and optional evidence paths: ${descriptor}" >&2 + echo "release artifact descriptor must contain path and exactly one package identity source: ${descriptor}" >&2 exit 1 fi primary_path="$(resolve_one_file path "$(jq -r '.path' <<<"${descriptor}")")" - sbom_pattern="$(jq -r '.sbom // empty' <<<"${descriptor}")" - provenance_pattern="$(jq -r '.provenance // empty' <<<"${descriptor}")" - signature_pattern="$(jq -r '.signature // empty' <<<"${descriptor}")" package_identity_file="$(jq -r '.package_identity_file // empty' <<<"${descriptor}")" if [[ -n "${package_identity_file}" ]]; then package_identity_path="$(resolve_one_file package_identity_file "${package_identity_file}")" @@ -387,35 +367,20 @@ while IFS= read -r descriptor; do exit 1 fi - if [[ -n "${sbom_pattern}" ]]; then - supplied_sbom_path="$(resolve_one_file sbom "${sbom_pattern}")" - sbom_path="$(copy_supplied_evidence "${primary_path}" "${supplied_sbom_path}" "sbom")" - sbom_kind="producer-dependency" - else - sbom_path="$(generated_evidence_path "${primary_path}" "spdx")" - write_generated_sbom "${primary_path}" "${package}" "${sbom_path}" - sbom_kind="generated-identity" - fi - if [[ -n "${provenance_pattern}" ]]; then - supplied_provenance_path="$(resolve_one_file provenance "${provenance_pattern}")" - provenance_path="$(copy_supplied_evidence "${primary_path}" "${supplied_provenance_path}" "provenance")" - else - provenance_path="$(generated_evidence_path "${primary_path}" "provenance")" - write_generated_provenance "${primary_path}" "${package}" "${provenance_path}" - fi + sbom_path="$(generated_evidence_path "${primary_path}" "spdx")" + write_generated_sbom "${primary_path}" "${package}" "${sbom_path}" + provenance_path="$(generated_evidence_path "${primary_path}" "provenance")" + write_generated_provenance "${primary_path}" "${package}" "${provenance_path}" + + # TODO: Accept producer-supplied SBOM, provenance, and signature evidence after + # the release catalog defines a generic artifact-to-evidence association. entry="$(jq -cn \ --arg release_catalog_key "${RELEASE_CATALOG_KEY}" \ --arg path "${primary_path}" \ --arg sbom "${sbom_path}" \ - --arg sbom_kind "${sbom_kind}" \ --arg provenance "${provenance_path}" \ --argjson package "${package}" \ - '{release_catalog_key: $release_catalog_key, path: $path, sbom: $sbom, sbom_kind: $sbom_kind, provenance: $provenance, package: $package}')" - if [[ -n "${signature_pattern}" ]]; then - supplied_signature_path="$(resolve_one_file signature "${signature_pattern}")" - signature_path="$(copy_supplied_evidence "${primary_path}" "${supplied_signature_path}" "signature")" - entry="$(jq -c --arg signature "${signature_path}" '. + {signature: $signature}' <<<"${entry}")" - fi + '{release_catalog_key: $release_catalog_key, path: $path, sbom: $sbom, sbom_kind: "generated-identity", provenance: $provenance, package: $package}')" jq --argjson entry "${entry}" '.entries += [$entry]' "${temporary_manifest}" >"${temporary_manifest}.next" mv "${temporary_manifest}.next" "${temporary_manifest}" diff --git a/release-catalog/validate-config.sh b/release-catalog/validate-config.sh index 6a5496ec..5ed529e9 100755 --- a/release-catalog/validate-config.sh +++ b/release-catalog/validate-config.sh @@ -50,14 +50,14 @@ validation_errors="$(jq -r ' | if ($artifact | type) != "object" then "artifacts[\($index)] must be an object" else - ($artifact | keys - ["path", "package_identity_file", "sbom", "provenance", "signature"]) as $artifact_unknown + ($artifact | keys - ["path", "package_identity_file"]) as $artifact_unknown | if ($artifact_unknown | length) > 0 then "artifacts[\($index)] has unknown field(s): " + ($artifact_unknown | join(", ")) else empty end, if ($artifact.path | single_line_string) then empty else "artifacts[\($index)].path must be a non-empty, single-line string" end, ($artifact | to_entries[] - | select(.key == "package_identity_file" or .key == "sbom" or .key == "provenance" or .key == "signature") + | select(.key == "package_identity_file") | select((.value | single_line_string) | not) | "artifacts[\($index)].\(.key) must be a non-empty, single-line string") end diff --git a/tests/release-catalog-config/package-identity-file.json b/tests/release-catalog-config/package-identity-file.json index 99047495..c3b55a05 100644 --- a/tests/release-catalog-config/package-identity-file.json +++ b/tests/release-catalog-config/package-identity-file.json @@ -4,8 +4,7 @@ "artifacts": [ { "path": "cuvs-java-*-x86_64-cuda*.jar", - "package_identity_file": "cuvs-java.release-package-identity.json", - "sbom": "cuvs-java.spdx.json" + "package_identity_file": "cuvs-java.release-package-identity.json" } ] } diff --git a/tests/release_catalog_config_test.sh b/tests/release_catalog_config_test.sh index 8bfcce06..cf615f90 100755 --- a/tests/release_catalog_config_test.sh +++ b/tests/release_catalog_config_test.sh @@ -32,7 +32,7 @@ RELEASE_CATALOG_CONFIG="$(<"${repository_root}/tests/release-catalog-config/pack grep -Fx 'release_catalog_key=maven:cuvs-java' "${valid_output}" grep -Fx 'artifact_directory=java/cuvs-java/target' "${valid_output}" -grep -Fx 'artifacts=[{"path":"cuvs-java-*-x86_64-cuda*.jar","package_identity_file":"cuvs-java.release-package-identity.json","sbom":"cuvs-java.spdx.json"}]' "${valid_output}" +grep -Fx 'artifacts=[{"path":"cuvs-java-*-x86_64-cuda*.jar","package_identity_file":"cuvs-java.release-package-identity.json"}]' "${valid_output}" assert_invalid \ malformed-json \ @@ -77,10 +77,9 @@ assert_invalid \ assert_invalid \ malformed-artifact \ '{"release_catalog_key":"archive:smoke","artifacts":[{"file":"smoke.tar.gz","package_identity_file":false,"sbom":false}]}' \ - 'artifacts[0] has unknown field(s): file' + 'artifacts[0] has unknown field(s): file, sbom' grep -F 'artifacts[0].path must be a non-empty, single-line string' "${temporary_directory}/malformed-artifact.error" >/dev/null grep -F 'artifacts[0].package_identity_file must be a non-empty, single-line string' "${temporary_directory}/malformed-artifact.error" >/dev/null -grep -F 'artifacts[0].sbom must be a non-empty, single-line string' "${temporary_directory}/malformed-artifact.error" >/dev/null default_output="${temporary_directory}/default.output" RELEASE_CATALOG_CONFIG='{"release_catalog_key":"conda:smoke"}' \ diff --git a/tests/release_catalog_discovery_test.sh b/tests/release_catalog_discovery_test.sh index ed24007e..fce7a1e6 100755 --- a/tests/release_catalog_discovery_test.sh +++ b/tests/release_catalog_discovery_test.sh @@ -68,10 +68,9 @@ jq -e ' RELEASE_ARTIFACT_DIRECTORY="${temporary_directory}" printf '%s\n' bundle >"${temporary_directory}/bundle.tar.gz" -printf '%s\n' sbom >"${temporary_directory}/bundle.spdx.json" jq -n '{ecosystem: "archive", name: "bundle", version: "1.0"}' \ >"${temporary_directory}/release-package-identity.json" -RELEASE_ARTIFACTS="$(jq -cn '[{path: "bundle.tar.gz", package_identity_file: "release-package-identity.json", sbom: "bundle.spdx.json"}]')" +RELEASE_ARTIFACTS="$(jq -cn '[{path: "bundle.tar.gz", package_identity_file: "release-package-identity.json"}]')" export RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY "${repository_root}/release-catalog/materialize.sh" @@ -79,18 +78,17 @@ export RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY jq -e ' .entries[0].path == "bundle.tar.gz" and .entries[0].package == {ecosystem: "archive", name: "bundle", version: "1.0"} - and .entries[0].sbom_kind == "producer-dependency" + and .entries[0].sbom_kind == "generated-identity" ' "${temporary_directory}/release-catalog-entries.json" >/dev/null RELEASE_ARTIFACT_DIRECTORY="${temporary_directory}" -printf '%s\n' signature >"${temporary_directory}/libkvikio.sig" -RELEASE_ARTIFACTS="$(jq -cn '[{path: "libkvikio_cu12-*.whl", signature: "libkvikio.sig"}]')" +RELEASE_ARTIFACTS="$(jq -cn '[{path: "libkvikio_cu12-*.whl"}]')" export RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY "${repository_root}/release-catalog/materialize.sh" jq -e ' .entries[0].path == "libkvikio_cu12-26.8.0-py3-none-manylinux_2_28_x86_64.whl" and .entries[0].package == {ecosystem: "wheel", name: "libkvikio-cu12", version: "26.8.0"} - and (.entries[0].signature | startswith("release-evidence/")) + and .entries[0].sbom_kind == "generated-identity" ' "${temporary_directory}/release-catalog-entries.json" >/dev/null mixed_directory="${temporary_directory}/mixed" diff --git a/tests/release_catalog_test.sh b/tests/release_catalog_test.sh index 1383f32d..86ce8c89 100755 --- a/tests/release_catalog_test.sh +++ b/tests/release_catalog_test.sh @@ -10,9 +10,6 @@ trap 'rm -rf "${temporary_directory}"' EXIT bundle_directory="${temporary_directory}/bundle" mkdir -p "${bundle_directory}" printf '%s\n' jar >"${bundle_directory}/cuvs-java-26.08.0.jar" -printf '%s\n' sbom >"${bundle_directory}/cuvs-java-26.08.0.spdx.json" -printf '%s\n' provenance >"${bundle_directory}/cuvs-java-26.08.0.provenance.jsonl" -printf '%s\n' signature >"${bundle_directory}/cuvs-java-26.08.0.jar.asc" jq -n \ '{ecosystem: "maven", name: "ai.rapids:cuvs-java", version: "26.08.0"}' \ >"${bundle_directory}/cuvs-java-package-identity.json" @@ -22,7 +19,7 @@ GITHUB_RUN_ATTEMPT="1" GITHUB_RUN_ID="1234" GITHUB_SHA="0123456789012345678901234567890123456789" GITHUB_WORKFLOW_REF="rapidsai/cuvs/.github/workflows/build.yaml@refs/heads/release/26.08" -RELEASE_ARTIFACTS="$(jq -cn '[{path: "cuvs-java-*.jar", package_identity_file: "cuvs-java-package-identity.json", sbom: "cuvs-java-*.spdx.json", provenance: "cuvs-java-*.provenance.jsonl", signature: "cuvs-java-*.jar.asc"}]')" +RELEASE_ARTIFACTS="$(jq -cn '[{path: "cuvs-java-*.jar", package_identity_file: "cuvs-java-package-identity.json"}]')" RELEASE_ENTRIES_NAME="release-catalog-entries.json" RELEASE_ARTIFACT_DIRECTORY="${bundle_directory}" RELEASE_SOURCE_ARTIFACT_NAME="cuvs-java-cuda12.9.1" @@ -47,26 +44,16 @@ jq -e ' and .entries[0].release_catalog_key == "maven:cuvs-java" and .entries[0].path == "cuvs-java-26.08.0.jar" and .entries[0].package.name == "ai.rapids:cuvs-java" - and .entries[0].sbom_kind == "producer-dependency" + and .entries[0].sbom_kind == "generated-identity" ' "${entries_path}" >/dev/null -supplied_sbom_path="$(jq -r '.entries[0].sbom' "${entries_path}")" -supplied_provenance_path="$(jq -r '.entries[0].provenance' "${entries_path}")" -supplied_signature_path="$(jq -r '.entries[0].signature' "${entries_path}")" -[[ "${supplied_sbom_path}" == release-evidence/*/sbom-* ]] -[[ "${supplied_provenance_path}" == release-evidence/*/provenance-* ]] -[[ "${supplied_signature_path}" == release-evidence/*/signature-* ]] -grep -Fx sbom "${canonical_bundle_directory}/${supplied_sbom_path}" -grep -Fx provenance "${canonical_bundle_directory}/${supplied_provenance_path}" -grep -Fx signature "${canonical_bundle_directory}/${supplied_signature_path}" - isolated_companion_directory="${temporary_directory}/isolated-companion" mkdir -p "${isolated_companion_directory}" cp "${entries_path}" "${isolated_companion_directory}/" cp -R "${canonical_bundle_directory}/release-evidence" "${isolated_companion_directory}/" while IFS= read -r evidence_path; do test -f "${isolated_companion_directory}/${evidence_path}" -done < <(jq -r '.entries[] | .sbom, .provenance, (.signature // empty)' "${isolated_companion_directory}/release-catalog-entries.json") +done < <(jq -r '.entries[] | .sbom, .provenance' "${isolated_companion_directory}/release-catalog-entries.json") multiple_identity_directory="${temporary_directory}/multiple-identities" mkdir -p "${multiple_identity_directory}" From 9837bf6bd889a2d6b03a5f433f4fa62d48201b48 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 18 Aug 2026 14:54:00 -0500 Subject: [PATCH 14/16] manual README edits --- release-catalog/README.md | 124 +++++++++++++++++------------ release-catalog/config.schema.json | 2 +- 2 files changed, 76 insertions(+), 50 deletions(-) diff --git a/release-catalog/README.md b/release-catalog/README.md index f17bb2ef..fb2cdc4d 100644 --- a/release-catalog/README.md +++ b/release-catalog/README.md @@ -4,7 +4,11 @@ build and uploads a companion GitHub Actions artifact named `release-catalog-`. Release tooling uses the companion to associate primary artifacts with their build identity and generated -identity evidence without reconstructing the build later. +identity evidence without reconstructing the build later. This evidence supports: + +- [assembling releases that can be tested and verified before formal tagging](https://github.com/rapidsai/release-scripts/issues/102) +- using build-time metadata to triage CVE scan records more quickly (internal + GitLab project, `nspect-manager`) The companion contains: @@ -22,58 +26,48 @@ contains the release catalog entries produced by that job. The release platform validates and aggregates entry arrays from selected builds into the release catalog; there is no separate metadata document to keep synchronized. +Each matrix job produces its own companion artifact. The release platform merges +their entries into one composite release catalog. + ## Configuration -Every producer passes one `config` JSON object. Its canonical schema and field -documentation are in [`config.schema.json`](config.schema.json). +Every build job that calls this action passes one `config` JSON object. Its +canonical schema and field documentation are in +[`config.schema.json`](config.schema.json). + +The action validates the configuration before inspecting build outputs. It then +verifies properties that depend on produced files, including the package +identity contents and whether primary-artifact paths resolve unambiguously. -When `artifacts` is omitted, the action discovers every Conda and wheel output -in `artifact_directory` and extracts identity from each package independently. -When `artifacts` is supplied, each descriptor is resolved independently: -supported Conda and wheel files are parsed, while any other artifact requires -its own `package_identity_file`. +### `release_catalog_key` -`release_catalog_key` identifies the release catalog entry that owns these -artifacts. Every file from every matrix variant in the same publishable artifact -set uses the same key, and multiple files are aggregated. Standard RAPIDS Conda -and wheel workflows construct it as `:`, such as -`conda:cudf`; custom producers select an existing catalog key, such as -`maven:cuvs-java`. Do not generate a UUID or a per-build value. +The `release_catalog_key` is used to group artifacts in the catalog. Every file +from every matrix variant in the same publishable artifact set uses the same +key. The release platform aggregates multiple `release-catalog-entries.json` +files and coalesces entries with the same `release_catalog_key`. Standard RAPIDS +conda and wheel workflows construct the key as `:`, +such as `conda:cudf`. Custom producers select a key, such as `maven:cuvs-java`, +that represents their release policy. -Use a distinct `release_catalog_key` only when the release catalog intentionally gives the -outputs different release policy. Examples include: +Use a distinct `release_catalog_key` only when the outputs intentionally have +different release policies. Reasons include differences in: -* different versioning -* validation requirements -* dependency ordering -* publication destinations -* promotion strategy. +- versioning schemes +- validation requirements +- dependency ordering +- publication destinations +- promotion strategy Multiple package names from one repository do not by themselves justify separate keys: for example, `cudf` and `dask-cudf` Conda packages remain part of `conda:cudf` when they share one release policy. -The action validates the configuration before inspecting build outputs. It then -verifies properties that depend on produced files, including the package -identity contents and whether primary-artifact paths resolve unambiguously. - -## Source revision - -The action requires `RAPIDS_SHA` in the job environment. It must identify the -repository revision actually checked out and built: +### `artifacts` -- Conda build workflows set `RAPIDS_SHA` to `git rev-parse HEAD` immediately - after checkout. -- Wheel and custom workflows use `rapids-github-info`; it uses `inputs.sha` - when supplied and otherwise sets `RAPIDS_SHA` to `git rev-parse HEAD`. - -This distinction matters when a reusable workflow checks out a repository or -revision different from the workflow event. Direct callers must likewise set -`RAPIDS_SHA` to the checked-out commit instead of assuming `${{ github.sha }}` -names the built source. Variables written to `GITHUB_ENV` are available to the -action when it runs as a subsequent job step. - -## Standard package example +This is a list of objects, where each object corresponds to exactly one artifact +(filename). Standard Conda and wheel workflows do not explicitly specify +`artifacts`. Instead, the action discovers every Conda and wheel output in +`artifact_directory` and creates corresponding artifact entries. ```yaml - name: Create wheel release catalog companion @@ -87,7 +81,8 @@ action when it runs as a subsequent job step. source-artifact-name: ${{ steps.package-name.outputs.RAPIDS_PACKAGE_NAME }} ``` -## Custom package example +For artifacts other than conda and wheels, path and package_identity_file must +be specified: ```yaml - name: Create custom release catalog companion @@ -105,10 +100,21 @@ action when it runs as a subsequent job step. source-artifact-name: cuvs-java ``` -Before the action runs, the producer creates -`java/cuvs-java/target/cuvs-java.release-package-identity.json`. The -artifact descriptor's `package_identity_file` value is the path to that file -relative to `artifact_directory`. For example: +Wildcards are allowed to allow for variance in filenames, but each wildcard must +resolve to only one file. In other words, each list object corresponds to +exactly one artifact, and any ambiguity is an error. + +#### Package identity file + +Package identity is high-level information about an artifact outside of its +filename. It requires `ecosystem`, `name`, and `version` and may include `build` +and `platform`. The action implicitly parses this information from conda and +wheel artifacts, but other artifact formats require the producer to provide it +explicitly. + +The workflow that calls release-catalog must create each package identity file before this +action runs. An artifact descriptor's `package_identity_file` is the path to +that file relative to `artifact_directory`. For example: ```json { @@ -118,6 +124,25 @@ relative to `artifact_directory`. For example: } ``` +Multiple artifact descriptors may reference the same identity file when those +artifacts have the same package identity. + +## Prerequisite state + +The action requires `RAPIDS_SHA` in the job environment. It must identify the +repository revision actually checked out and built: + +- Conda build workflows set `RAPIDS_SHA` to `git rev-parse HEAD` immediately + after checkout. +- Wheel and custom workflows use `rapids-github-info`; it uses `inputs.sha` + when supplied and otherwise sets `RAPIDS_SHA` to `git rev-parse HEAD`. + +This distinction matters when a reusable workflow checks out a repository or +revision different from the workflow event. Direct callers must likewise set +`RAPIDS_SHA` to the checked-out commit instead of assuming `${{ github.sha }}` +names the built source. Variables written to `GITHUB_ENV` are available to the +action when it runs as a subsequent job step. + ## Generated identity evidence For every artifact, the action generates an SPDX artifact-identity envelope @@ -126,6 +151,7 @@ They record the artifact SHA-256, package identity, source revision, and workflo context, but contain no dependency or source-license inventory. They must not be reported as producer-supplied dependency coverage. -Producer-supplied SBOM, provenance, and signature inputs are intentionally -deferred until a future revision defines a generic, unambiguous way to associate -them with detected artifacts. +Producer-supplied dependency SBOMs and richer build provenance, such as build +dependencies and compiler flags, would make package contents easier to +understand without downloading them. Associating that input-side evidence with +artifacts is left for future work. diff --git a/release-catalog/config.schema.json b/release-catalog/config.schema.json index 7df259e0..11165ee5 100644 --- a/release-catalog/config.schema.json +++ b/release-catalog/config.schema.json @@ -45,7 +45,7 @@ "pattern": "^[^\\r\\n]+$" }, "package_identity_file": { - "description": "Path, relative to artifact_directory, to producer-created package identity JSON for this artifact. Required only when identity cannot be extracted from the artifact. The file must contain ecosystem, name, and version and may contain build and platform.", + "description": "Path, relative to artifact_directory, to producer-created package identity JSON for this artifact. Required only when identity cannot be extracted from the artifact. See https://github.com/rapidsai/shared-actions/blob/main/release-catalog/README.md#package-identity-file", "type": "string", "minLength": 1, "pattern": "^[^\\r\\n]+$" From 7c0a65593ef4474c24aead778b78a23f6892d7b6 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 18 Aug 2026 14:58:27 -0500 Subject: [PATCH 15/16] fix release catalog CI smoke test --- .github/workflows/pr.yml | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 2a578473..de53b856 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -23,29 +23,33 @@ jobs: - name: Test release catalog materialization run: | ./tests/release_catalog_config_test.sh - ./tests/release_catalog_descriptors_test.sh - ./tests/release_catalog_prepare_test.sh + ./tests/release_catalog_discovery_test.sh + ./tests/release_catalog_package_parsing_test.sh ./tests/release_catalog_test.sh - name: Prepare release catalog dispatch smoke test run: | mkdir -p release-catalog-smoke printf '%s\n' smoke >release-catalog-smoke/package.tar.gz + jq -n \ + '{ecosystem: "archive", name: "smoke", version: "1.0"}' \ + >release-catalog-smoke/package-identity.json - name: Run release catalog dispatch smoke test uses: ./release-catalog-dispatch env: + RAPIDS_SHA: ${{ github.sha }} SHARED_ACTIONS_REPO: ${{ github.event.pull_request.head.repo.full_name }} SHARED_ACTIONS_REF: ${{ github.event.pull_request.head.sha }} with: config: >- { - "artifact_type": "custom", "release_catalog_key": "archive:smoke", - "output_directory": "release-catalog-smoke", - "package": {"ecosystem": "archive", "name": "smoke", "version": "1.0"}, - "artifacts": [{"path": "package.tar.gz"}] + "artifact_directory": "release-catalog-smoke", + "artifacts": [{ + "path": "package.tar.gz", + "package_identity_file": "package-identity.json" + }] } source-artifact-name: release-catalog-dispatch-smoke - source-sha: ${{ github.event.pull_request.head.sha }} - name: Verify release catalog dispatch smoke test run: | jq -e ' From 32a308ab234f7381dd611056747b4c1ac30d2a34 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 18 Aug 2026 15:06:10 -0500 Subject: [PATCH 16/16] more readme edits --- release-catalog/README.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/release-catalog/README.md b/release-catalog/README.md index fb2cdc4d..c37275df 100644 --- a/release-catalog/README.md +++ b/release-catalog/README.md @@ -81,7 +81,7 @@ This is a list of objects, where each object corresponds to exactly one artifact source-artifact-name: ${{ steps.package-name.outputs.RAPIDS_PACKAGE_NAME }} ``` -For artifacts other than conda and wheels, path and package_identity_file must +For artifacts other than conda and wheels, `path` and `package_identity_file` must be specified: ```yaml @@ -110,11 +110,10 @@ Package identity is high-level information about an artifact outside of its filename. It requires `ecosystem`, `name`, and `version` and may include `build` and `platform`. The action implicitly parses this information from conda and wheel artifacts, but other artifact formats require the producer to provide it -explicitly. - -The workflow that calls release-catalog must create each package identity file before this -action runs. An artifact descriptor's `package_identity_file` is the path to -that file relative to `artifact_directory`. For example: +explicitly using the `package_identity_file` parameter. Any referenced package +identity file must be created before this action runs. An artifact descriptor's +`package_identity_file` is the path to that file relative to +`artifact_directory`. Example contents of a package_identity_file: ```json {