From 47bf3f274bc24e1fee68a8c0372a8d7f99e8c606 Mon Sep 17 00:00:00 2001 From: anan Date: Tue, 26 May 2026 07:45:43 +0000 Subject: [PATCH] Test ci merged vpto tests. --- .github/workflows/ci.yml | 290 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 281 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c0ef0625b..d9174a912 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -94,6 +94,8 @@ jobs: runs-on: ubuntu-22.04 env: LLVM_COMMIT: cd708029e0b2869e80abe31ddb175f7c35361f90 + LLVM_REPO_VPTO: https://github.com/vpto-dev/llvm-project.git + LLVM_REF_VPTO: feature-vpto LLVM_BUILD_DIR: ${{ github.workspace }}/llvm-project/llvm/build-assert LLVM_DIR: ${{ github.workspace }}/llvm-project/llvm/build-assert PTO_BUILD_DIR: ${{ github.workspace }}/build-assert @@ -101,13 +103,33 @@ jobs: MLIR_PYTHONPATH: ${{ github.workspace }}/llvm-project/llvm/build-assert/tools/mlir/python_packages/mlir_core LLVM_CACHE_FLAVOR: assert-shared-mlirpy-hardening-v1 steps: - - name: Checkout + - name: Detect PR source fork + shell: bash + run: | + set -euo pipefail + HEAD_REPO="${{ github.event.pull_request.head.repo.full_name }}" + if [[ "${HEAD_REF}" == feature-vpto-* ]]; then + echo "IS_VPTO_FORK=true" >> "${GITHUB_ENV}" + echo "::notice::PR from vpto fork (${HEAD_REPO}) — using vpto LLVM branch" + else + echo "IS_VPTO_FORK=false" >> "${GITHUB_ENV}" + fi + + - name: Checkout (default) + if: env.IS_VPTO_FORK != 'true' uses: actions/checkout@v4 with: repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} ref: ${{ github.event.pull_request.head.sha || github.sha }} fetch-depth: 1 persist-credentials: false + + - name: Checkout (vpto fork) + if: env.IS_VPTO_FORK == 'true' + uses: actions/checkout@v4 + with: + fetch-depth: 1 + persist-credentials: false - name: Install dependencies run: | @@ -149,20 +171,43 @@ jobs: llvm-project/llvm/build-assert key: llvm-${{ runner.os }}-${{ env.LLVM_COMMIT }}-${{ env.LLVM_CACHE_FLAVOR }} + - name: Resolve LLVM source SHA + shell: bash + run: | + set -euo pipefail + if [[ "${IS_VPTO_FORK}" == "true" ]]; then + LLVM_SOURCE_SHA="$(git ls-remote "${LLVM_REPO_VPTO}" "refs/heads/${LLVM_REF_VPTO}" | awk '{print $1}')" + [[ -n "${LLVM_SOURCE_SHA}" ]] || { + echo "ERROR: failed to resolve ${LLVM_REPO_VPTO} ${LLVM_REF_VPTO}" >&2 + exit 1 + } + else + LLVM_SOURCE_SHA="${LLVM_COMMIT}" + fi + echo "LLVM_SOURCE_SHA=${LLVM_SOURCE_SHA}" >> "${GITHUB_ENV}" + - name: Prepare LLVM source (no rebuild) run: | + set -euo pipefail mkdir -p llvm-project cd llvm-project - - # cache 只保存 build 目录，这里补一个最小 git repo 供 cmake/ninja 使用 - if [ ! -d .git ]; then - git init - git remote add origin https://github.com/llvm/llvm-project.git + if [[ "${IS_VPTO_FORK}" == "true" ]]; then + if [ ! -d .git ]; then + git init + git remote add origin "${LLVM_REPO_VPTO}" + fi + git remote set-url origin "${LLVM_REPO_VPTO}" + git fetch --depth 1 origin "${LLVM_REF_VPTO}" + git checkout --force FETCH_HEAD + else + if [ ! -d .git ]; then + git init + git remote add origin https://github.com/llvm/llvm-project.git + fi + git fetch --depth 1 origin tag llvmorg-19.1.7 + git checkout "${LLVM_COMMIT}" fi - git fetch --depth 1 origin tag llvmorg-19.1.7 - git checkout "${LLVM_COMMIT}" - - name: Build LLVM/MLIR (only if cache miss) if: steps.cache-llvm.outputs.cache-hit != 'true' run: | @@ -271,6 +316,233 @@ jobs: path: ${{ env.PAYLOAD_TGZ }} if-no-files-found: error + vpto-sim-validation: + runs-on: [self-hosted, Linux, X64, label-1] + timeout-minutes: 120 + concurrency: + group: vpto-sim-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + # only worked for mly pr -> main + if: >- + ${{ + (github.event_name == 'workflow_dispatch' || github.event_name == 'pull_request') && + startsWith(github.event.pull_request.head.ref, 'feature-vpto-') + }} + env: + LLVM_REPO: https://github.com/vpto-dev/llvm-project.git + LLVM_REF: feature-vpto + VPTO_SIM_WORKSPACE: ${{ github.workspace }}/.work/vpto-sim-ci + TILELANG_DSL_WORKSPACE: ${{ github.workspace }}/.work/tilelang-dsl-ci + TILELANG_DSL_UT_WORKSPACE: ${{ github.workspace }}/.work/tilelang-dsl-ut-ci + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 1 + persist-credentials: false + + - name: Resolve LLVM directories + shell: bash + env: + TOOL_CACHE: ${{ runner.tool_cache }} + run: | + set -euo pipefail + echo "LLVM_ROOT=${TOOL_CACHE}/llvm-project" >> "${GITHUB_ENV}" + echo "LLVM_DIR=${TOOL_CACHE}/llvm-project/llvm/build-shared" >> "${GITHUB_ENV}" + echo "MLIR_PYTHONPATH=${TOOL_CACHE}/llvm-project/llvm/build-shared/tools/mlir/python_packages/mlir_core" >> "${GITHUB_ENV}" + + - name: Ensure runner dependencies + shell: bash + run: | + set -euo pipefail + missing_tools=() + for tool in python3 git cmake ninja make; do + if ! command -v "${tool}" >/dev/null 2>&1; then + missing_tools+=("${tool}") + fi + done + + if [[ "${#missing_tools[@]}" -gt 0 ]]; then + if command -v sudo >/dev/null 2>&1 && command -v apt-get >/dev/null 2>&1; then + sudo apt-get update + sudo apt-get install -y python3 python3-pip git cmake ninja-build make + else + echo "ERROR: missing required tools on self-hosted runner: ${missing_tools[*]}" >&2 + echo "ERROR: automatic installation requires sudo + apt-get" >&2 + exit 1 + fi + fi + + python3 -m pip --version >/dev/null 2>&1 || { + if command -v sudo >/dev/null 2>&1 && command -v apt-get >/dev/null 2>&1; then + sudo apt-get update + sudo apt-get install -y python3-pip + else + echo "ERROR: python3-pip is required on self-hosted runner" >&2 + exit 1 + fi + } + + need_pip_install=0 + python3 -c "import numpy" >/dev/null 2>&1 || need_pip_install=1 + python3 -m pybind11 --cmakedir >/dev/null 2>&1 || need_pip_install=1 + python3 -c "import ml_dtypes" >/dev/null 2>&1 || need_pip_install=1 + + if [[ "${need_pip_install}" -eq 1 ]]; then + python3 -m pip install --upgrade pip + python3 -m pip install 'pybind11<3' numpy ml-dtypes + fi + + - name: Clean CI work dirs + shell: bash + run: | + set -euo pipefail + rm -rf "${GITHUB_WORKSPACE}/build" + rm -rf "${VPTO_SIM_WORKSPACE}" + rm -rf "${TILELANG_DSL_WORKSPACE}" + + - name: Prepare LLVM source (no rebuild) + shell: bash + run: | + set -euo pipefail + mkdir -p "${LLVM_ROOT}" + cd "${LLVM_ROOT}" + + if [ ! -d .git ]; then + git init + git remote add origin "${LLVM_REPO}" + fi + + git remote set-url origin "${LLVM_REPO}" + git fetch --depth 1 origin "${LLVM_REF}" + git checkout --force FETCH_HEAD + + - name: Build LLVM/MLIR + shell: bash + run: | + set -euo pipefail + cd "${LLVM_ROOT}" + cmake -G Ninja -S llvm -B llvm/build-shared \ + -DLLVM_ENABLE_PROJECTS="mlir;clang" \ + -DBUILD_SHARED_LIBS=ON \ + -DMLIR_ENABLE_BINDINGS_PYTHON=ON \ + -DPython3_EXECUTABLE=python3 \ + -DCMAKE_BUILD_TYPE=Release \ + -DLLVM_TARGETS_TO_BUILD="host" + + ninja -C llvm/build-shared + + - name: Build PTOAS + shell: bash + run: | + set -euo pipefail + export PYBIND11_CMAKE_DIR="$(python3 -m pybind11 --cmakedir)" + cmake -G Ninja -S . -B build \ + -DLLVM_DIR="${LLVM_DIR}/lib/cmake/llvm" \ + -DMLIR_DIR="${LLVM_DIR}/lib/cmake/mlir" \ + -DPython3_EXECUTABLE=python3 \ + -DPython3_FIND_STRATEGY=LOCATION \ + -Dpybind11_DIR="${PYBIND11_CMAKE_DIR}" \ + -DMLIR_ENABLE_BINDINGS_PYTHON=ON \ + -DMLIR_PYTHON_PACKAGE_DIR="${LLVM_DIR}/tools/mlir/python_packages/mlir_core" \ + -DCMAKE_BUILD_TYPE=Release + ninja -C build ptoas + + - name: Resolve simulator environment + shell: bash + run: | + set -euo pipefail + + detect_ascend_home() { + for d in \ + "${ASCEND_HOME_PATH:-}" \ + /usr/local/Ascend/cann \ + /usr/local/Ascend/cann-* \ + /usr/local/Ascend/ascend-toolkit/latest + do + [[ -n "${d}" && -d "${d}" ]] || continue + printf '%s\n' "${d}" + return 0 + done + return 1 + } + + ASCEND_HOME_PATH_DETECTED="$(detect_ascend_home || true)" + if [[ -z "${ASCEND_HOME_PATH_DETECTED}" ]]; then + echo "ERROR: failed to detect ASCEND_HOME_PATH on self-hosted runner" >&2 + exit 1 + fi + + echo "ASCEND_HOME_PATH=${ASCEND_HOME_PATH_DETECTED}" >> "${GITHUB_ENV}" + echo "PTOAS_BIN=${GITHUB_WORKSPACE}/build/tools/ptoas/ptoas" >> "${GITHUB_ENV}" + + - name: Run VPTO SIM validation + if: ${{ true }} + shell: bash + run: | + set -euo pipefail + mkdir -p "${VPTO_SIM_WORKSPACE}" + WORK_SPACE="${VPTO_SIM_WORKSPACE}" \ + ASCEND_HOME_PATH="${ASCEND_HOME_PATH}" \ + PTOAS_BIN="${PTOAS_BIN}" \ + DEVICE=SIM \ + JOBS="${JOBS:-32}" \ + bash test/vpto/scripts/run_host_vpto_validation_parallel.sh + + - name: Run TileLang DSL CI + shell: bash + run: | + set -euo pipefail + mkdir -p "${TILELANG_DSL_WORKSPACE}" + if [[ "${{ github.event_name }}" == "pull_request" ]]; then + ASCEND_HOME_PATH="${ASCEND_HOME_PATH}" \ + PTOAS_BIN="${PTOAS_BIN}" \ + bash test/tilelang_st/script/run_ci.sh -r sim -v a5 --jobs 64 --smoke \ + 2>&1 | tee "${TILELANG_DSL_WORKSPACE}/run_ci.log" + else + ASCEND_HOME_PATH="${ASCEND_HOME_PATH}" \ + PTOAS_BIN="${PTOAS_BIN}" \ + bash test/tilelang_st/script/run_ci.sh -r sim -v a5 --jobs 64 \ + 2>&1 | tee "${TILELANG_DSL_WORKSPACE}/run_ci.log" + fi + + - name: Upload TileLang DSL logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: tilelang-dsl-ci-${{ github.run_id }} + path: | + ${{ env.TILELANG_DSL_WORKSPACE }}/run_ci.log + if-no-files-found: warn + + - name: Run TileLang DSL unit tests + shell: bash + run: | + set -euo pipefail + mkdir -p "${TILELANG_DSL_UT_WORKSPACE}" + cd tilelang-dsl + PYTHONPATH=python python3 -m unittest discover -s tests -p 'test_*.py' \ + 2>&1 | tee "${TILELANG_DSL_UT_WORKSPACE}/unittest.log" + + - name: Upload TileLang DSL unit test logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: tilelang-dsl-ut-ci-${{ github.run_id }} + path: | + ${{ env.TILELANG_DSL_UT_WORKSPACE }}/unittest.log + if-no-files-found: warn + + - name: Upload VPTO SIM logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: vpto-sim-validation-${{ github.run_id }} + path: | + ${{ env.VPTO_SIM_WORKSPACE }}/parallel-runner.log + ${{ env.VPTO_SIM_WORKSPACE }}/parallel-summary.tsv + if-no-files-found: warn + remote-npu-validation: needs: build-and-test runs-on: ubuntu-22.04