From 7e229bcd3a2123d984539d62b9ec9bfc94f5334b Mon Sep 17 00:00:00 2001 From: Matt Kornfield Date: Thu, 14 May 2026 16:46:50 +0000 Subject: [PATCH] chore: overhaul lints and fixes --- .gitignore | 1 + Makefile | 8 +- tools/lint/lint-all.sh | 77 +++--- tools/lint/lint-auth-config.sh | 5 +- tools/lint/lint-cli.sh | 2 +- tools/lint/lint-common.sh | 276 +++++++++++++++++++ tools/lint/lint-config-reference-docs.sh | 2 +- tools/lint/lint-copyright-headers.sh | 5 +- tools/lint/lint-fix-auth-config.sh | 9 + tools/lint/lint-fix-cli.sh | 8 + tools/lint/lint-fix-config-reference-docs.sh | 8 + tools/lint/lint-fix-copyright-headers.sh | 8 + tools/lint/lint-fix-helm.sh | 15 + tools/lint/lint-fix-merge-conflict.sh | 11 + tools/lint/lint-fix-openapi.sh | 9 + tools/lint/lint-fix-python-sdk.sh | 9 + tools/lint/lint-fix-python-style.sh | 9 + tools/lint/lint-fix-python-types.sh | 11 + tools/lint/lint-fix-sdk-vendored.sh | 10 + tools/lint/lint-fix.sh | 106 ++++--- tools/lint/lint-helm.sh | 5 +- tools/lint/lint-licenses.sh | 5 +- tools/lint/lint-merge-conflict.sh | 5 +- tools/lint/lint-openapi.sh | 9 +- tools/lint/lint-python-sdk.sh | 5 +- tools/lint/lint-python-style.sh | 9 +- tools/lint/lint-python-types.sh | 5 +- tools/lint/lint-sdk-vendored.sh | 4 +- tools/lint/lint-web-sdk.sh | 5 +- 29 files changed, 542 insertions(+), 99 deletions(-) create mode 100755 tools/lint/lint-common.sh create mode 100755 tools/lint/lint-fix-auth-config.sh create mode 100755 tools/lint/lint-fix-cli.sh create mode 100755 tools/lint/lint-fix-config-reference-docs.sh create mode 100755 tools/lint/lint-fix-copyright-headers.sh create mode 100755 tools/lint/lint-fix-helm.sh create mode 100755 tools/lint/lint-fix-merge-conflict.sh create mode 100755 tools/lint/lint-fix-openapi.sh create mode 100755 tools/lint/lint-fix-python-sdk.sh create mode 100755 tools/lint/lint-fix-python-style.sh create mode 100755 tools/lint/lint-fix-python-types.sh create mode 100755 tools/lint/lint-fix-sdk-vendored.sh diff --git a/.gitignore b/.gitignore index 7ebc58e7..495a8202 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,7 @@ venv* **/node_modules # Linter output files diff.txt +.lint-failures temp __pycache__ diff --git a/Makefile b/Makefile index e832e673..6f974597 100644 --- a/Makefile +++ b/Makefile @@ -192,13 +192,17 @@ check-copyright-headers: $(CMD_COPYRIGHT_HEADER_FIXER) --check .PHONY: lint -lint: ## Run all linters (licenses, openapi, config docs, python style/types/sdk, vendored SDK, CLI, auth config) +lint: ## Run lint scripts in tools/lint; optionally pass LINTS="lint-openapi lint-python-style" bash tools/lint/lint-all.sh .PHONY: lint-fix -lint-fix: ## Auto-fix lint issues in dependency order (openapi → stainless → style → cli → vendor → licenses → config-docs) +lint-fix: ## Run corresponding lint-fix scripts in dependency order; optionally pass LINTS="lint-openapi" bash tools/lint/lint-fix.sh +.PHONY: lint-fix-failed +lint-fix-failed: ## Run lint-fix scripts for failures recorded by the last make lint + bash tools/lint/lint-fix.sh --failed + .PHONY: vendor vendor: ## Vendor packages into the SDK and generate wrapper metadata uv run --no-sync nemo-platform-sdk-tools vendor all-from-configs \ diff --git a/tools/lint/lint-all.sh b/tools/lint/lint-all.sh index 8524b03d..ef6307fc 100755 --- a/tools/lint/lint-all.sh +++ b/tools/lint/lint-all.sh @@ -2,41 +2,31 @@ set -uo pipefail # Run all lint scripts serially, report summary, exit with failure if any failed. SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_ROOT="${CI_PROJECT_DIR:-$(cd "${SCRIPT_DIR}/../.." && pwd)}" +source "${SCRIPT_DIR}/lint-common.sh" cd "${PROJECT_ROOT}" || exit 1 -declare -a scripts=( - "lint-licenses:tools/lint/lint-licenses.sh" - "lint-openapi:tools/lint/lint-openapi.sh" - "lint-config-reference-docs:tools/lint/lint-config-reference-docs.sh" - "lint-python-style:tools/lint/lint-python-style.sh" - "lint-python-types:tools/lint/lint-python-types.sh" - "lint-python-sdk:tools/lint/lint-python-sdk.sh" - "lint-sdk-vendored:tools/lint/lint-sdk-vendored.sh" - "lint-web-sdk:tools/lint/lint-web-sdk.sh" - "lint-cli:tools/lint/lint-cli.sh" - "lint-auth-config:tools/lint/lint-auth-config.sh" - "lint-merge-conflict:tools/lint/lint-merge-conflict.sh" - "lint-copyright-headers:tools/lint/lint-copyright-headers.sh" -) +validate_lint_fix_pairs || exit 1 -is_no_fix_lint() { - local lint_name="$1" - case "${lint_name}" in - lint-python-types|lint-merge-conflict) - return 0 - ;; - *) - return 1 - ;; - esac -} +selected_output="$(selected_lint_names "${LINT_SCRIPT_NAMES[@]}")" || exit 1 +declare -a lint_names=() +if [[ -n "${selected_output}" ]]; then + mapfile -t lint_names <<< "${selected_output}" +fi + +if [[ ${#lint_names[@]} -eq 0 ]]; then + echo "No lint scripts selected." + write_lint_failures || exit 1 + exit 0 +fi + +if [[ -n "${LINTS:-}" ]]; then + echo "Selected lints: ${lint_names[*]}" +fi declare -a failed=() declare -a timing_lines=() -for entry in "${scripts[@]}"; do - name="${entry%%:*}" - path="${entry#*:}" +for name in "${lint_names[@]}"; do + path="$(lint_script_path "${name}")" start=$(date +%s) if bash "${path}"; then echo "[PASS] ${name}" @@ -52,7 +42,7 @@ done echo "" echo "--- Lint summary ---" -echo "Passed: $((${#scripts[@]} - ${#failed[@]}))" +echo "Passed: $((${#lint_names[@]} - ${#failed[@]}))" echo "Failed: ${#failed[@]}" echo "" echo "Timings:" @@ -62,26 +52,21 @@ for line in "${timing_lines[@]}"; do printf " %-40s %s\n" "${name}" "${details}" done if [[ ${#failed[@]} -gt 0 ]]; then + write_lint_failures "${failed[@]}" || exit 1 echo "Failed lints: ${failed[*]}" + echo "Recorded failed lints in $(lint_failure_file_display_path)." echo "" - # Check if any failed lint has an auto-fix command - has_fix=false - for name in "${failed[@]}"; do - if ! is_no_fix_lint "${name}"; then - has_fix=true - break - fi - done - if [[ "${has_fix}" == "true" ]]; then - echo "To fix auto-fixable issues, run:" - echo " make lint-fix" - echo "" - fi + echo "To run fixes for the recorded failures in dependency order:" + echo " make lint-fix-failed" + echo "" + echo "To run fixes for a specific subset:" + echo " make lint-fix LINTS=\"${failed[*]}\"" + echo "" + echo "Failed lint -> corresponding fix script:" for name in "${failed[@]}"; do - if is_no_fix_lint "${name}"; then - echo " ${name}: (see output above for manual fix)" - fi + printf " %-30s %s\n" "${name}" "$(lint_fix_script_display_path "${name}")" done exit 1 fi +write_lint_failures || exit 1 exit 0 diff --git a/tools/lint/lint-auth-config.sh b/tools/lint/lint-auth-config.sh index 6ebdcf69..d45a76fc 100755 --- a/tools/lint/lint-auth-config.sh +++ b/tools/lint/lint-auth-config.sh @@ -1,4 +1,7 @@ #!/usr/bin/env bash set -euo pipefail # Verify auth static config and OpenAPI are in sync. -uv run python services/core/auth/scripts/auth-tools.py check +uv run python services/core/auth/scripts/auth-tools.py check || { + echo "Run 'tools/lint/lint-fix-auth-config.sh' to update auth config and generated auth docs." + exit 1 +} diff --git a/tools/lint/lint-cli.sh b/tools/lint/lint-cli.sh index 20843127..8c516db4 100755 --- a/tools/lint/lint-cli.sh +++ b/tools/lint/lint-cli.sh @@ -8,6 +8,6 @@ cd "${PROJECT_ROOT}" make update-cli git add "${PROJECT_ROOT}/sdk/python/" git diff --cached --exit-code "${PROJECT_ROOT}/sdk/python/" > "${PROJECT_ROOT}/diff.txt" || { - echo "Run 'make update-cli' to update the CLI." + echo "Run 'tools/lint/lint-fix-cli.sh' to update the CLI." exit 1 } diff --git a/tools/lint/lint-common.sh b/tools/lint/lint-common.sh new file mode 100755 index 00000000..b49321ad --- /dev/null +++ b/tools/lint/lint-common.sh @@ -0,0 +1,276 @@ +#!/usr/bin/env bash + +# Shared lint registry. Lint/fix pairing is mechanical: +# tools/lint/lint-foo.sh -> tools/lint/lint-fix-foo.sh + +LINT_TOOLS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="${CI_PROJECT_DIR:-$(cd "${LINT_TOOLS_DIR}/../.." && pwd)}" +LINT_FAILURES_FILE="${LINT_FAILURES_FILE:-${PROJECT_ROOT}/.lint-failures}" + +# Lints run by `make lint`, in reporting order. +declare -a LINT_SCRIPT_NAMES=( + "lint-licenses" + "lint-openapi" + "lint-config-reference-docs" + "lint-python-style" + "lint-python-types" + "lint-python-sdk" + "lint-sdk-vendored" + "lint-web-sdk" + "lint-cli" + "lint-auth-config" + "lint-merge-conflict" + "lint-copyright-headers" +) + +# Fixes run by `make lint-fix`, expressed as lint names so the script path is +# always the corresponding lint-fix script. The order is dependency-aware. +declare -a LINT_FIX_ORDER=( + "lint-openapi" + "lint-web-sdk" + "lint-python-sdk" + "lint-python-style" + "lint-cli" + "lint-sdk-vendored" + "lint-licenses" + "lint-auth-config" + "lint-config-reference-docs" + "lint-copyright-headers" + "lint-python-types" + "lint-merge-conflict" +) + +lint_script_name_from_file() { + local file_name="$1" + printf '%s\n' "${file_name%.sh}" +} + +lint_script_path() { + local lint_name="$1" + printf '%s/%s.sh\n' "${LINT_TOOLS_DIR}" "${lint_name}" +} + +lint_script_display_path() { + local lint_name="$1" + printf 'tools/lint/%s.sh\n' "${lint_name}" +} + +lint_failure_file_display_path() { + printf '%s\n' "${LINT_FAILURES_FILE#${PROJECT_ROOT}/}" +} + +lint_fix_script_name() { + local lint_name="$1" + printf 'lint-fix-%s\n' "${lint_name#lint-}" +} + +lint_fix_script_path() { + local lint_name="$1" + lint_script_path "$(lint_fix_script_name "${lint_name}")" +} + +lint_fix_script_display_path() { + local lint_name="$1" + lint_script_display_path "$(lint_fix_script_name "${lint_name}")" +} + +is_runnable_lint_name() { + local lint_name="$1" + case "${lint_name}" in + lint-all|lint-common|lint-fix|lint-fix-*) + return 1 + ;; + esac + [[ -f "$(lint_script_path "${lint_name}")" ]] +} + +normalize_lint_name() { + local raw_name="$1" + local lint_name + raw_name="${raw_name#./}" + raw_name="${raw_name#tools/lint/}" + raw_name="${raw_name%.sh}" + + case "${raw_name}" in + lint-fix-*) + lint_name="lint-${raw_name#lint-fix-}" + ;; + lint-*) + lint_name="${raw_name}" + ;; + *) + lint_name="lint-${raw_name}" + ;; + esac + + if ! is_runnable_lint_name "${lint_name}"; then + echo "Unknown lint '${raw_name}'. Expected a tools/lint/lint-*.sh script name." >&2 + return 1 + fi + + printf '%s\n' "${lint_name}" +} + +normalize_lint_names_from_text() { + local raw_lints="$1" + local raw_name + local lint_name + local seen=" " + + raw_lints="${raw_lints//,/ }" + for raw_name in ${raw_lints}; do + lint_name="$(normalize_lint_name "${raw_name}")" || return 1 + case "${seen}" in + *" ${lint_name} "*) + continue + ;; + esac + seen+="${lint_name} " + printf '%s\n' "${lint_name}" + done +} + +selected_lint_names() { + if [[ -n "${LINTS:-}" ]]; then + normalize_lint_names_from_text "${LINTS}" + return + fi + + printf '%s\n' "$@" +} + +lint_name_in_list() { + local needle="$1" + shift + local lint_name + for lint_name in "$@"; do + if [[ "${lint_name}" == "${needle}" ]]; then + return 0 + fi + done + return 1 +} + +ordered_fix_lint_names() { + local -a requested_lints=("$@") + local lint_name + + for lint_name in "${LINT_FIX_ORDER[@]}"; do + if lint_name_in_list "${lint_name}" "${requested_lints[@]}"; then + printf '%s\n' "${lint_name}" + fi + done + + for lint_name in "${requested_lints[@]}"; do + if ! lint_name_in_list "${lint_name}" "${LINT_FIX_ORDER[@]}"; then + printf '%s\n' "${lint_name}" + fi + done +} + +write_lint_failures() { + if [[ $# -eq 0 ]]; then + rm -f "${LINT_FAILURES_FILE}" + return + fi + + printf '%s\n' "$@" > "${LINT_FAILURES_FILE}" +} + +read_lint_failures() { + local raw_lints + if [[ ! -s "${LINT_FAILURES_FILE}" ]]; then + return 0 + fi + + raw_lints="$(tr '\n' ' ' < "${LINT_FAILURES_FILE}")" + normalize_lint_names_from_text "${raw_lints}" +} + +all_lint_script_names() { + local path + local file_name + for path in "${LINT_TOOLS_DIR}"/lint-*.sh; do + [[ -f "${path}" ]] || continue + file_name="$(basename "${path}")" + case "${file_name}" in + lint-all.sh|lint-common.sh|lint-fix.sh|lint-fix-*.sh) + continue + ;; + esac + lint_script_name_from_file "${file_name}" + done | sort +} + +validate_lint_fix_pairs() { + local lint_name + local lint_path + local fix_path + local file_name + local path + local ordered_lint_name + local found + local -a missing_lints=() + local -a missing_fixes=() + local -a missing_fix_order=() + local -a orphan_fixes=() + + for lint_name in "${LINT_SCRIPT_NAMES[@]}" "${LINT_FIX_ORDER[@]}"; do + lint_path="$(lint_script_path "${lint_name}")" + if [[ ! -f "${lint_path}" ]]; then + missing_lints+=("$(lint_script_display_path "${lint_name}")") + fi + done + + while IFS= read -r lint_name; do + [[ -n "${lint_name}" ]] || continue + fix_path="$(lint_fix_script_path "${lint_name}")" + if [[ ! -f "${fix_path}" ]]; then + missing_fixes+=("$(lint_script_display_path "${lint_name}") -> $(lint_fix_script_display_path "${lint_name}")") + fi + done < <(all_lint_script_names) + + for lint_name in "${LINT_SCRIPT_NAMES[@]}"; do + found=false + for ordered_lint_name in "${LINT_FIX_ORDER[@]}"; do + if [[ "${lint_name}" == "${ordered_lint_name}" ]]; then + found=true + break + fi + done + if [[ "${found}" == "false" ]]; then + missing_fix_order+=("${lint_name} -> $(lint_fix_script_display_path "${lint_name}")") + fi + done + + for path in "${LINT_TOOLS_DIR}"/lint-fix-*.sh; do + [[ -f "${path}" ]] || continue + file_name="$(basename "${path}")" + lint_name="lint-${file_name#lint-fix-}" + lint_name="${lint_name%.sh}" + lint_path="$(lint_script_path "${lint_name}")" + if [[ ! -f "${lint_path}" ]]; then + orphan_fixes+=("$(lint_script_display_path "${lint_name}") <- tools/lint/${file_name}") + fi + done + + if [[ ${#missing_lints[@]} -gt 0 || ${#missing_fixes[@]} -gt 0 || ${#missing_fix_order[@]} -gt 0 || ${#orphan_fixes[@]} -gt 0 ]]; then + if [[ ${#missing_lints[@]} -gt 0 ]]; then + echo "Missing lint scripts referenced by tools/lint/lint-common.sh:" + printf ' %s\n' "${missing_lints[@]}" + fi + if [[ ${#missing_fixes[@]} -gt 0 ]]; then + echo "Missing corresponding lint-fix scripts:" + printf ' %s\n' "${missing_fixes[@]}" + fi + if [[ ${#missing_fix_order[@]} -gt 0 ]]; then + echo "Lints run by make lint but missing from LINT_FIX_ORDER:" + printf ' %s\n' "${missing_fix_order[@]}" + fi + if [[ ${#orphan_fixes[@]} -gt 0 ]]; then + echo "Lint-fix scripts without corresponding lint scripts:" + printf ' %s\n' "${orphan_fixes[@]}" + fi + return 1 + fi +} diff --git a/tools/lint/lint-config-reference-docs.sh b/tools/lint/lint-config-reference-docs.sh index c9ea0c2f..9ddec964 100755 --- a/tools/lint/lint-config-reference-docs.sh +++ b/tools/lint/lint-config-reference-docs.sh @@ -3,6 +3,6 @@ set -euo pipefail # Verify config reference doc is up to date. uv run --frozen generate-config-docs git diff --exit-code docs/set-up/config-reference.md || { - echo "Config reference doc is out of date. Run 'uv run generate-config-docs' and commit docs/set-up/config-reference.md" + echo "Config reference doc is out of date. Run 'tools/lint/lint-fix-config-reference-docs.sh' and commit docs/set-up/config-reference.md." exit 1 } diff --git a/tools/lint/lint-copyright-headers.sh b/tools/lint/lint-copyright-headers.sh index 024d7599..acbb9c8a 100755 --- a/tools/lint/lint-copyright-headers.sh +++ b/tools/lint/lint-copyright-headers.sh @@ -5,4 +5,7 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="${CI_PROJECT_DIR:-$(cd "${SCRIPT_DIR}/../.." && pwd)}" cd "${PROJECT_ROOT}" || exit 1 -make check-copyright-headers +make check-copyright-headers || { + echo "Run 'tools/lint/lint-fix-copyright-headers.sh' to update copyright headers." + exit 1 +} diff --git a/tools/lint/lint-fix-auth-config.sh b/tools/lint/lint-fix-auth-config.sh new file mode 100755 index 00000000..4862c4ac --- /dev/null +++ b/tools/lint/lint-fix-auth-config.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euo pipefail +# Update auth static config and generated auth permission docs. +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="${CI_PROJECT_DIR:-$(cd "${SCRIPT_DIR}/../.." && pwd)}" +cd "${PROJECT_ROOT}" || exit 1 + +uv run python services/core/auth/scripts/auth-tools.py update +uv run python services/core/auth/scripts/auth-tools.py generate-docs diff --git a/tools/lint/lint-fix-cli.sh b/tools/lint/lint-fix-cli.sh new file mode 100755 index 00000000..e313e5d7 --- /dev/null +++ b/tools/lint/lint-fix-cli.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -euo pipefail +# Regenerate CLI commands, vendored CLI package metadata, and CLI reference docs. +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="${CI_PROJECT_DIR:-$(cd "${SCRIPT_DIR}/../.." && pwd)}" +cd "${PROJECT_ROOT}" || exit 1 + +make update-cli diff --git a/tools/lint/lint-fix-config-reference-docs.sh b/tools/lint/lint-fix-config-reference-docs.sh new file mode 100755 index 00000000..ff3d57de --- /dev/null +++ b/tools/lint/lint-fix-config-reference-docs.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -euo pipefail +# Regenerate the platform config reference documentation. +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="${CI_PROJECT_DIR:-$(cd "${SCRIPT_DIR}/../.." && pwd)}" +cd "${PROJECT_ROOT}" || exit 1 + +uv run --frozen generate-config-docs diff --git a/tools/lint/lint-fix-copyright-headers.sh b/tools/lint/lint-fix-copyright-headers.sh new file mode 100755 index 00000000..03df9087 --- /dev/null +++ b/tools/lint/lint-fix-copyright-headers.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -euo pipefail +# Apply copyright headers. +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="${CI_PROJECT_DIR:-$(cd "${SCRIPT_DIR}/../.." && pwd)}" +cd "${PROJECT_ROOT}" || exit 1 + +make update-copyright-headers diff --git a/tools/lint/lint-fix-helm.sh b/tools/lint/lint-fix-helm.sh new file mode 100755 index 00000000..edd5e918 --- /dev/null +++ b/tools/lint/lint-fix-helm.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -euo pipefail +# Refresh Helm chart dependencies. Template and schema failures still require manual edits. +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="${CI_PROJECT_DIR:-$(cd "${SCRIPT_DIR}/../.." && pwd)}" +cd "${PROJECT_ROOT}" || exit 1 + +HELM_FOLDER=${HELM_FOLDER:-deploy/helm/platform} + +if [[ ! -d "${HELM_FOLDER}" ]]; then + echo "Helm folder not found: ${HELM_FOLDER}" >&2 + exit 1 +fi + +helm dependency update "${HELM_FOLDER}" diff --git a/tools/lint/lint-fix-merge-conflict.sh b/tools/lint/lint-fix-merge-conflict.sh new file mode 100755 index 00000000..6a5ebb7e --- /dev/null +++ b/tools/lint/lint-fix-merge-conflict.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -euo pipefail +# There is no safe automatic edit for merge conflict markers; rerun the lint. +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="${CI_PROJECT_DIR:-$(cd "${SCRIPT_DIR}/../.." && pwd)}" +cd "${PROJECT_ROOT}" || exit 1 + +bash tools/lint/lint-merge-conflict.sh || { + echo "No automatic fix is available for merge conflict markers." + exit 1 +} diff --git a/tools/lint/lint-fix-openapi.sh b/tools/lint/lint-fix-openapi.sh new file mode 100755 index 00000000..5b04628b --- /dev/null +++ b/tools/lint/lint-fix-openapi.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euo pipefail +# Regenerate OpenAPI specifications. +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="${CI_PROJECT_DIR:-$(cd "${SCRIPT_DIR}/../.." && pwd)}" +cd "${PROJECT_ROOT}" || exit 1 + +make refresh-openapi +rm -rf openapicheck diff --git a/tools/lint/lint-fix-python-sdk.sh b/tools/lint/lint-fix-python-sdk.sh new file mode 100755 index 00000000..29923a71 --- /dev/null +++ b/tools/lint/lint-fix-python-sdk.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euo pipefail +# Sync the Python SDK with the current OpenAPI spec when it is out of date. +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="${CI_PROJECT_DIR:-$(cd "${SCRIPT_DIR}/../.." && pwd)}" +cd "${PROJECT_ROOT}" || exit 1 + +OUTPUT_DIR="${TMPDIR:-/tmp}/nmp-sdk-lint" +uv run --frozen nemo-platform-sdk-tools is-up-to-date --output-dir "${OUTPUT_DIR}" || make stainless diff --git a/tools/lint/lint-fix-python-style.sh b/tools/lint/lint-fix-python-style.sh new file mode 100755 index 00000000..1046bc2e --- /dev/null +++ b/tools/lint/lint-fix-python-style.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euo pipefail +# Apply Python formatting and ruff auto-fixes. +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="${CI_PROJECT_DIR:-$(cd "${SCRIPT_DIR}/../.." && pwd)}" +cd "${PROJECT_ROOT}" || exit 1 + +uv run ruff check --fix +uv run ruff format diff --git a/tools/lint/lint-fix-python-types.sh b/tools/lint/lint-fix-python-types.sh new file mode 100755 index 00000000..22052d30 --- /dev/null +++ b/tools/lint/lint-fix-python-types.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -euo pipefail +# There is no safe automatic type fix; rerun the lint so remaining issues are visible. +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="${CI_PROJECT_DIR:-$(cd "${SCRIPT_DIR}/../.." && pwd)}" +cd "${PROJECT_ROOT}" || exit 1 + +bash tools/lint/lint-python-types.sh || { + echo "No automatic fix is available for Python type errors." + exit 1 +} diff --git a/tools/lint/lint-fix-sdk-vendored.sh b/tools/lint/lint-fix-sdk-vendored.sh new file mode 100755 index 00000000..b8231220 --- /dev/null +++ b/tools/lint/lint-fix-sdk-vendored.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -euo pipefail +# Regenerate vendored SDK package wrappers and CLI reference docs. +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="${CI_PROJECT_DIR:-$(cd "${SCRIPT_DIR}/../.." && pwd)}" +cd "${PROJECT_ROOT}" || exit 1 +export PATH="$HOME/.local/bin:$PATH" + +make vendor +make generate-cli-reference-docs diff --git a/tools/lint/lint-fix.sh b/tools/lint/lint-fix.sh index 776c1d2c..b7a18b70 100755 --- a/tools/lint/lint-fix.sh +++ b/tools/lint/lint-fix.sh @@ -1,46 +1,78 @@ #!/usr/bin/env bash set -euo pipefail -# Run all auto-fix commands in dependency order: -# 1. OpenAPI spec regeneration (other steps depend on this) -# 2. Web SDK regeneration (Orval reads openapi/ga/individual/platform.openapi.yaml) -# 3. Stainless sync (pulls updated Python SDK from Stainless; openapi already done in step 1) -# 4. Python style (ruff; run before vendoring so generated files aren't re-linted) -# 5. CLI command generation (the vendoring and docs are handled by the next step) -# 6. Vendor all packages (covers nemo_platform_ext too) + CLI reference docs -# 7. License update (may change after vendoring) -# 8. Config reference docs (independent, but run after structural changes) -# 9. Auth docs (regenerate permissions reference from static-authz.yaml) -# -# Note: update-sdk = build-policy + refresh-openapi + stainless + update-cli, so we use -# stainless directly here to avoid re-running refresh-openapi and update-cli redundantly. -# Note: update-cli = generate-cli-commands + vendor-nemo-platform-ext + generate-cli-reference-docs, -# but vendor-nemo-platform-ext is a subset of make vendor and generate-cli-reference-docs would -# run twice. So we run generate-cli-commands alone, then let make vendor cover all vendoring. +# Run corresponding lint-fix scripts in dependency order. Each entry in +# LINT_FIX_ORDER is a lint name and maps mechanically to lint-fix-.sh. SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_ROOT="${CI_PROJECT_DIR:-$(cd "${SCRIPT_DIR}/../.." && pwd)}" +source "${SCRIPT_DIR}/lint-common.sh" cd "${PROJECT_ROOT}" || exit 1 -declare -a steps=( - "refresh-openapi:make refresh-openapi" - "web-sdk:bash tools/lint/lint-fix-web-sdk.sh" - "stainless:uv run --frozen nemo-platform-sdk-tools is-up-to-date --output-dir \"${TMPDIR:-/tmp}/nmp-sdk-lint\" || make stainless" - "python-style:uv run ruff format && uv run ruff check --fix" - "generate-cli-commands:make generate-cli-commands" - "vendor+cli-reference-docs:make vendor && make generate-cli-reference-docs" - "update-licenses:bash tools/lint/lint-fix-licenses.sh" - "auth-config:uv run python services/core/auth/scripts/auth-tools.py update" - "generate-config-docs:uv run generate-config-docs" - "generate-auth-docs:uv run python services/core/auth/scripts/auth-tools.py generate-docs" -) +use_failed=false +while [[ $# -gt 0 ]]; do + case "$1" in + --failed) + use_failed=true + ;; + -h|--help) + echo "Usage: $0 [--failed]" + echo "" + echo "Environment:" + echo " LINTS=\"lint-openapi lint-python-style\" Run fixes for a subset." + echo " LINT_FAILURES_FILE=.lint-failures Override failed-lint state file." + exit 0 + ;; + *) + echo "Unknown argument: $1" >&2 + exit 1 + ;; + esac + shift +done + +validate_lint_fix_pairs || exit 1 + +declare -a lint_names=() +declare -a requested_lints=() +if [[ "${use_failed}" == "true" ]]; then + if [[ -n "${LINTS:-}" ]]; then + echo "Use either --failed or LINTS=..., not both." >&2 + exit 1 + fi + + failed_output="$(read_lint_failures)" || exit 1 + if [[ -z "${failed_output}" ]]; then + echo "No recorded lint failures in $(lint_failure_file_display_path). Run 'make lint' first." + exit 0 + fi + + mapfile -t requested_lints <<< "${failed_output}" + ordered_output="$(ordered_fix_lint_names "${requested_lints[@]}")" || exit 1 + if [[ -n "${ordered_output}" ]]; then + mapfile -t lint_names <<< "${ordered_output}" + fi + echo "Selected fixes from $(lint_failure_file_display_path): ${lint_names[*]}" +elif [[ -n "${LINTS:-}" ]]; then + requested_output="$(normalize_lint_names_from_text "${LINTS}")" || exit 1 + if [[ -n "${requested_output}" ]]; then + mapfile -t requested_lints <<< "${requested_output}" + fi + ordered_output="$(ordered_fix_lint_names "${requested_lints[@]}")" || exit 1 + if [[ -n "${ordered_output}" ]]; then + mapfile -t lint_names <<< "${ordered_output}" + fi + echo "Selected fixes: ${lint_names[*]}" +else + lint_names=("${LINT_FIX_ORDER[@]}") +fi declare -a failed=() declare -a timing_rows=() -for entry in "${steps[@]}"; do - name="${entry%%:*}" - cmd="${entry#*:}" - echo ">>> ${name}: ${cmd}" +for lint_name in "${lint_names[@]}"; do + name="$(lint_fix_script_name "${lint_name}")" + path="$(lint_fix_script_path "${lint_name}")" + display_path="$(lint_fix_script_display_path "${lint_name}")" + echo ">>> ${lint_name} -> ${display_path}" start=$(date +%s) - if eval "${cmd}"; then + if bash "${path}"; then echo "[DONE] ${name}" result="DONE" else @@ -54,7 +86,7 @@ for entry in "${steps[@]}"; do done echo "--- Fix summary ---" -echo "Completed: $((${#steps[@]} - ${#failed[@]}))" +echo "Completed: $((${#lint_names[@]} - ${#failed[@]}))" echo "Failed: ${#failed[@]}" echo "" echo "Timings:" @@ -66,4 +98,8 @@ if [[ ${#failed[@]} -gt 0 ]]; then echo "Failed steps: ${failed[*]}" exit 1 fi +if [[ "${use_failed}" == "true" ]]; then + echo "" + echo "Run 'make lint' to verify and refresh $(lint_failure_file_display_path)." +fi exit 0 diff --git a/tools/lint/lint-helm.sh b/tools/lint/lint-helm.sh index cd7c8cc7..4e880ba0 100755 --- a/tools/lint/lint-helm.sh +++ b/tools/lint/lint-helm.sh @@ -13,7 +13,10 @@ KUBECONFORM_CACHE="${KUBECONFORM_CACHE:-${PROJECT_ROOT}/.kubeconform-cache}" mkdir -p "${KUBECONFORM_CACHE}" # Fetch chart dependencies so subchart templates (e.g. postgresql) are available during lint/template -helm dependency update "${HELM_FOLDER}" +helm dependency update "${HELM_FOLDER}" || { + echo "Run 'tools/lint/lint-fix-helm.sh' to refresh Helm chart dependencies." + exit 1 +} # Lint the Helm chart helm lint --strict "${HELM_FOLDER}" diff --git a/tools/lint/lint-licenses.sh b/tools/lint/lint-licenses.sh index befc583f..c77b30f4 100755 --- a/tools/lint/lint-licenses.sh +++ b/tools/lint/lint-licenses.sh @@ -1,6 +1,9 @@ #!/usr/bin/env bash set -euo pipefail # Verify third-party licenses are up to date. -make check-licenses +make check-licenses || { + echo "Run 'tools/lint/lint-fix-licenses.sh' to update license files." + exit 1 +} # This only runs if the diff doesn't exit early git restore third_party diff --git a/tools/lint/lint-merge-conflict.sh b/tools/lint/lint-merge-conflict.sh index 12503d5c..da18b39b 100755 --- a/tools/lint/lint-merge-conflict.sh +++ b/tools/lint/lint-merge-conflict.sh @@ -1,3 +1,6 @@ #!/usr/bin/env bash set -euo pipefail -uv run pre-commit run check-merge-conflict +uv run pre-commit run check-merge-conflict || { + echo "Run 'tools/lint/lint-fix-merge-conflict.sh' after manually resolving conflict markers." + exit 1 +} diff --git a/tools/lint/lint-openapi.sh b/tools/lint/lint-openapi.sh index 8a98ddef..ce1cc418 100755 --- a/tools/lint/lint-openapi.sh +++ b/tools/lint/lint-openapi.sh @@ -9,5 +9,10 @@ mkdir -p openapicheck cp openapi/openapi.yaml openapicheck/openapi.yaml cp openapi/ga/openapi.yaml openapicheck/openapi.ga.yaml script/generate-openapi-spec.sh -diff openapi/openapi.yaml openapicheck/openapi.yaml -diff openapi/ga/openapi.yaml openapicheck/openapi.ga.yaml +diff_status=0 +diff openapi/openapi.yaml openapicheck/openapi.yaml || diff_status=1 +diff openapi/ga/openapi.yaml openapicheck/openapi.ga.yaml || diff_status=1 +if [[ ${diff_status} -ne 0 ]]; then + echo "OpenAPI specs are out of date. Run 'tools/lint/lint-fix-openapi.sh' to regenerate them." + exit 1 +fi diff --git a/tools/lint/lint-python-sdk.sh b/tools/lint/lint-python-sdk.sh index 22757857..eac8a3e6 100755 --- a/tools/lint/lint-python-sdk.sh +++ b/tools/lint/lint-python-sdk.sh @@ -2,4 +2,7 @@ set -euo pipefail # Verify Python SDK is up to date with OpenAPI spec. OUTPUT_DIR="${CI_PROJECT_DIR:-$(pwd)}/python-sdk-lint" -uv run --frozen nemo-platform-sdk-tools is-up-to-date --output-dir "${OUTPUT_DIR}" +uv run --frozen nemo-platform-sdk-tools is-up-to-date --output-dir "${OUTPUT_DIR}" || { + echo "Run 'tools/lint/lint-fix-python-sdk.sh' to update the Python SDK." + exit 1 +} diff --git a/tools/lint/lint-python-style.sh b/tools/lint/lint-python-style.sh index 8b3c6016..25446efa 100755 --- a/tools/lint/lint-python-style.sh +++ b/tools/lint/lint-python-style.sh @@ -2,5 +2,10 @@ set -euo pipefail # Check Python style with ruff (lint and format). # Uses the ruff version pinned in pyproject.toml dev dependencies. -uv run ruff check -uv run ruff format --check +status=0 +uv run ruff check || status=1 +uv run ruff format --check || status=1 +if [[ ${status} -ne 0 ]]; then + echo "Run 'tools/lint/lint-fix-python-style.sh' to apply Python style fixes." + exit 1 +fi diff --git a/tools/lint/lint-python-types.sh b/tools/lint/lint-python-types.sh index ae6077ae..acd0eab3 100755 --- a/tools/lint/lint-python-types.sh +++ b/tools/lint/lint-python-types.sh @@ -24,4 +24,7 @@ for rule in "${ci_ignored_rules[@]}"; do ignore_args+=(--ignore "$rule") done -uv run --frozen ty check "${ignore_args[@]}" +uv run --frozen ty check "${ignore_args[@]}" || { + echo "Run 'tools/lint/lint-fix-python-types.sh' after manually fixing type errors." + exit 1 +} diff --git a/tools/lint/lint-sdk-vendored.sh b/tools/lint/lint-sdk-vendored.sh index cda36b02..3f1a5702 100755 --- a/tools/lint/lint-sdk-vendored.sh +++ b/tools/lint/lint-sdk-vendored.sh @@ -9,13 +9,13 @@ export PATH="$HOME/.local/bin:$PATH" make vendor git add "${PROJECT_ROOT}/sdk/python/" "${PROJECT_ROOT}/packages/nemo_platform/pyproject.toml" git diff --cached --exit-code "${PROJECT_ROOT}/sdk/python/" "${PROJECT_ROOT}/packages/nemo_platform/pyproject.toml" > "${PROJECT_ROOT}/diff.txt" || { - echo "Run 'make vendor' to sync packages with the SDK and wrapper." + echo "Run 'tools/lint/lint-fix-sdk-vendored.sh' to sync packages with the SDK and wrapper." exit 1 } make generate-cli-reference-docs git add "${PROJECT_ROOT}/docs/cli" git diff --cached --exit-code "${PROJECT_ROOT}/docs/cli" > "${PROJECT_ROOT}/diff.txt" || { - echo "Run 'make generate-cli-reference-docs' to sync cli docs." + echo "Run 'tools/lint/lint-fix-sdk-vendored.sh' to sync CLI docs." exit 1 } diff --git a/tools/lint/lint-web-sdk.sh b/tools/lint/lint-web-sdk.sh index d0db9cf3..908c9c31 100755 --- a/tools/lint/lint-web-sdk.sh +++ b/tools/lint/lint-web-sdk.sh @@ -16,4 +16,7 @@ if ! command -v pnpm >/dev/null 2>&1; then exit 0 fi -exec pnpm gen:check +pnpm gen:check || { + echo "Run 'tools/lint/lint-fix-web-sdk.sh' to regenerate the web SDK." + exit 1 +}