[CI]add acc & build

.github/workflows/alignment_model_accuracy.yml

@@ -0,0 +1,171 @@
+name: Alignment Model Accuracy
+
+on:
+  pull_request:
+  workflow_dispatch:
+
+concurrency:
+  group: Alignment-${{ github.workflow }}-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
+env:
+  BRANCH: ${{ github.event.pull_request.base.ref || github.ref_name }}
+  COMMIT_ID: ${{ github.event.pull_request.head.sha || github.sha }}
+  PR_ID: ${{ github.event.pull_request.number }}
+  work_dir: ${{ github.workspace }}
+  python_version: 3.12
+  TASK: Megatron-LM-${{ github.sha }}-alignment
+  CE_name: alignment-Megatron-LM
+  no_proxy: "localhost,bj.bcebos.com,su.bcebos.com,bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  alignment_model_accuracy:
+    name: alignment_model_accuracy
+    runs-on:
+      group: h20-2c
+    steps:
+      - name: Determine the runner
+        run: |
+          runner_name=`(echo $PWD|awk -F '/' '{print $3}')`
+          echo $runner_name
+          wget -q https://xly-devops.bj.bcebos.com/utils.sh
+          source utils.sh
+          determine_gpu_runner ${runner_name}
+      - name: Determine Image Name
+        run: |
+          echo "IMAGE_NAME=ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:ubuntu24-cuda130-py312-dev" >> "$GITHUB_ENV"
+      - name: Run Container
+        run: |
+          container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
+          echo "container_name=${container_name}" >> "$GITHUB_ENV"
+          docker run -d -t --name ${container_name} --net=host --gpus all --shm-size=32G \
+            -v /dev/shm:/dev/shm \
+            -v $work_dir/../../..:$work_dir/../../.. \
+            -v $work_dir:/workspace \
+            -v /home/.cache/pip:/home/.cache/pip \
+            -v /ssd4/MODELDATA:/home/.cache \
+            -e "BRANCH=$BRANCH" \
+            -e "COMMIT_ID=$COMMIT_ID" \
+            -e "PR_ID=$PR_ID" \
+            -e "work_dir=$work_dir" \
+            -e no_proxy \
+            -e CE_name \
+            -e python_version \
+            -w /workspace $IMAGE_NAME
+      - name: Checkout Code
+        run: |
+          docker exec -t $container_name /bin/bash -c '
+            rm -rf * .[^.]*
+            source $work_dir/../../../proxy
+            echo "Download PaddleFleet form https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddleFleet.tar"
+            wget -q --no-proxy  https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddleFleet.tar --no-check-certificate
+            rm -rf PaddleFleet && tar xf PaddleFleet.tar && rm -rf PaddleFleet.tar
+            cd PaddleFleet && git pull && cd -
+
+            echo "Download Megatron-LM form https://paddle-github-action.bj.bcebos.com/whl/Megatron-LM.tar.gz"
+            wget -q --no-proxy https://paddle-github-action.bj.bcebos.com/whl/Megatron-LM.tar.gz --no-check-certificate
+            rm -rf Megatron-LM && tar zxf Megatron-LM.tar.gz && rm -rf Megatron-LM.tar.gz
+            cd Megatron-LM
+            git config --global --add safe.directory /workspace/Megatron-LM
+            git pull
+            git submodule update --init --recursive --force
+            if [ -n "$PR_ID" ] && [ "$PR_ID" != "0" ]; then
+              git fetch origin pull/${PR_ID}/head
+              git checkout -b PR_${PR_ID} FETCH_HEAD
+              git remote add upstream https://github.com/PFCCLab/Megatron-LM.git
+              echo "Checking out ${BRANCH}..."
+              git fetch upstream ${BRANCH}:${BRANCH}
+              git merge ${BRANCH} --no-edit
+              git diff --numstat ${BRANCH} -- | awk "{print \$NF}"
+            else
+              echo "Not in a pull_request event. Skipping PR-specific operations."
+            fi
+            git log --pretty=oneline -10
+          '
+      - name: Change python version
+        run: |
+          docker exec -i -e "work_dir=$work_dir" -e "python_version=$python_version" $container_name /bin/bash <<-DOCKER_EOF
+          echo "Change python version to $python_version"
+          set -e
+          source $work_dir/../../../proxy
+          cp ${work_dir}/../../../Miniforge3-Linux-x86_64.sh /tmp/miniforge.sh
+          bash /tmp/miniforge.sh -b -p /opt/conda
+          /opt/conda/bin/conda create -n py_\$python_version python=\$python_version -y \
+            --override-channels \
+            -c conda-forge
+          . /opt/conda/etc/profile.d/conda.sh
+          conda activate py_\$python_version
+          python --version
+          DOCKER_EOF
+      - name: Get Whl
+        run: |
+          docker exec -t $container_name /bin/bash -c '
+          . /opt/conda/etc/profile.d/conda.sh
+          conda activate py_$python_version
+          python --version
+          ldconfig
+
+          BOS=https://paddle-github-action.bj.bcebos.com
+          echo "::group::Download paddlefleet / paddlefleet_ops / ms_swift / mcore-bridge wheels from BOS"
+          cd /workspace
+          for url in \
+            $BOS/PaddleFleet/develop/latest/paddlefleet-0.0.0-py3-none-linux_x86_64.whl \
+            $BOS/PaddleFleet/develop/latest/cu130/paddle-release/paddlefleet_ops-0.0.0-cp312-cp312-linux_x86_64.whl \
+            $BOS/whl/ms_swift-0.0.0-py3-none-any.whl \
+            $BOS/whl/mcore_bridge-0.0.0-py3-none-any.whl ; do
+            echo "Downloading $url"
+            wget -q --no-proxy --no-check-certificate --tries=3 --timeout=60 "$url"
+          done
+          ls -l /workspace/
+          echo "::endgroup::"
+
+          echo "::group::Build megatron-core wheel"
+          cd /workspace/Megatron-LM
+          pip config set global.cache-dir "/home/.cache/pip"
+          bash scripts/dependence/build.sh
+          ls -l /workspace/upload/
+          echo "::endgroup::"
+          '
+      - name: alignment_model_accuracy
+        run: |
+          docker exec -t $container_name /bin/bash -c '
+          . /opt/conda/etc/profile.d/conda.sh
+          conda activate py_$python_version
+          python --version
+          ldconfig
+          source $work_dir/../../../proxy
+          export PROXY_URL="${http_proxy}"
+          export PADDLEFLEET_WHEEL_PATH="/workspace/paddlefleet-0.0.0-py3-none-linux_x86_64.whl"
+          export PADDLEFLEET_OPS_WHEEL_PATH="/workspace/paddlefleet_ops-0.0.0-cp312-cp312-linux_x86_64.whl"
+          export MEGATRON_CORE_WHEEL_PATH=/workspace/upload/megatron_core-0.0.0-cp312-cp312-linux_x86_64.whl
+          export MS_SWIFT_WHEEL_PATH=/workspace/ms_swift-0.0.0-py3-none-any.whl
+          export MCORE_BRIDGE_WHEEL_PATH=/workspace/mcore_bridge-0.0.0-py3-none-any.whl
+          # The BOS wheels are republished under a fixed 0.0.0 filename
+          export UV_SKIP_WHEEL_FILENAME_CHECK=1
+
+          for whl in "$PADDLEFLEET_WHEEL_PATH" "$PADDLEFLEET_OPS_WHEEL_PATH" \
+            "$MS_SWIFT_WHEEL_PATH" "$MEGATRON_CORE_WHEEL_PATH" \
+            "$MCORE_BRIDGE_WHEEL_PATH"; do
+            [ -f "$whl" ] || { echo "::error:: missing wheel: $whl"; exit 1; }
+            echo "using $whl"
+          done
+          python -m pip install uv
+          bash PaddleFleet/scripts/alignment_model_accuracy/setup_venvs.sh
+          bash -x PaddleFleet/scripts/alignment_model_accuracy/run_alignment_test.sh
+
+          model_acc_align_exit_code=$?
+          if [[ "$model_acc_align_exit_code" != "0" ]]; then
+            echo -e "::error:: \033[31m Model accuracy align failed.\033[0m"
+            exit 1
+          else
+            echo -e "\033[32m Model accuracy align succeeded.\033[0m"
+          fi
+          '
+      - name: Terminate And Delete the Container
+        if: always()
+        run: |
+          docker rm -f $container_name 2>/dev/null || true

.github/workflows/build.yml

@@ -0,0 +1,144 @@
+name: Build Whl
+
+on:
+  workflow_dispatch:
+    inputs:
+      commit:
+        description: 'Commit to build (empty = HEAD of the selected ref)'
+        required: false
+        type: string
+        default: ''
+      update_latest:
+        description: 'Also publish the latest (0.0.0) whl'
+        required: false
+        type: boolean
+        default: true
+  schedule:
+    - cron: '0 16 * * *'
+
+env:
+  BRANCH: ${{ github.ref_name }}
+  COMMIT_ID: ${{ inputs.commit || github.sha }}
+  TARGET_COMMIT: ${{ inputs.commit }}
+  UPDATE_LATEST: ${{ github.event.inputs.update_latest || 'true' }}
+  work_dir: ${{ github.workspace }}
+  python_version: 3.12
+  TASK: Megatron-LM-${{ github.sha }}-build
+  CE_name: build-megatron-lm
+  no_proxy: "localhost,bj.bcebos.com,su.bcebos.com,bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  build:
+    name: build
+    runs-on:
+      group: h20-2c
+    steps:
+      - name: Determine the runner
+        run: |
+          runner_name=`(echo $PWD|awk -F '/' '{print $3}')`
+          echo $runner_name
+          wget -q https://xly-devops.bj.bcebos.com/utils.sh
+          source utils.sh
+          determine_gpu_runner ${runner_name}
+      - name: Determine Image Name
+        run: |
+          echo "IMAGE_NAME=ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:cuda130-dev" >> "$GITHUB_ENV"
+      - name: Run Container
+        run: |
+          container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
+          echo "container_name=${container_name}" >> "$GITHUB_ENV"
+          docker run -d -t --name ${container_name} --net=host -v /dev/shm:/dev/shm --shm-size=32G \
+            -v $work_dir/../../..:$work_dir/../../.. \
+            -v $work_dir:/workspace \
+            -v /home/.cache/pip:/home/.cache/pip \
+            -e "BRANCH=$BRANCH" \
+            -e "COMMIT_ID=$COMMIT_ID" \
+            -e "TARGET_COMMIT=$TARGET_COMMIT" \
+            -e "UPDATE_LATEST=$UPDATE_LATEST" \
+            -e work_dir \
+            -e no_proxy \
+            -e CE_name \
+            -e FLAGS_dynamic_static_unified_comm \
+            -e python_version \
+            -w /workspace $IMAGE_NAME
+      - name: Checkout Code
+        run: |
+          docker exec -t $container_name /bin/bash -c '
+            rm -rf * .[^.]*
+            echo "Download Megatron-LM form https://paddle-github-action.bj.bcebos.com/whl/Megatron-LM.tar.gz"
+            wget -q --no-proxy https://paddle-github-action.bj.bcebos.com/whl/Megatron-LM.tar.gz --no-check-certificate
+            rm -rf Megatron-LM && tar --warning=no-unknown-keyword -zxf Megatron-LM.tar.gz && rm -rf Megatron-LM.tar.gz
+            source $work_dir/../../../proxy
+            cd Megatron-LM
+            git pull
+            if [ -n "$TARGET_COMMIT" ]; then
+              echo "Checkout requested commit: $TARGET_COMMIT"
+              git checkout "$TARGET_COMMIT"
+            fi
+            git log --pretty=oneline -10
+          '
+      - name: Change python version
+        run: |
+          docker exec -i -e "work_dir=$work_dir" -e "python_version=$python_version" $container_name /bin/bash <<-DOCKER_EOF
+          echo "Change python version to $python_version"
+          set -e
+          cp ${work_dir}/../../../Miniforge3-Linux-x86_64.sh /tmp/miniforge.sh
+          bash /tmp/miniforge.sh -b -p /opt/conda
+          /opt/conda/bin/conda create -n py_\$python_version python=\$python_version -y \
+            --override-channels \
+            -c conda-forge
+          . /opt/conda/etc/profile.d/conda.sh
+          conda activate py_\$python_version
+          python --version
+          DOCKER_EOF
+      - name: Build Whl
+        run: |
+          docker exec -t $container_name /bin/bash -c '
+          . /opt/conda/etc/profile.d/conda.sh
+          conda activate py_$python_version
+          python --version
+          ldconfig
+          pip config set global.cache-dir "/home/.cache/pip"
+          set -e
+          cd /workspace/Megatron-LM && git config --global --add safe.directory $PWD
+          bash scripts/dependence/build.sh
+          '
+      - name: Upload Products
+        env:
+          bos_file: ${{ github.workspace }}/bos_retry/BosClient.py
+        if: always()
+        run: |
+          docker exec -t $container_name /bin/bash -c '
+          echo "::group::Install bce-python-sdk"
+          python -m pip install bce-python-sdk==0.8.74
+          echo "::endgroup::"
+          export AK=paddle
+          export SK=paddle
+          if [ ! -f "${{ env.bos_file }}" ]; then
+            wget -q --no-proxy -O /workspace/bos_retry.tar.gz https://xly-devops.bj.bcebos.com/home/bos_retry.tar.gz --no-check-certificate
+            mkdir /workspace/bos_retry
+            tar xf /workspace/bos_retry.tar.gz -C /workspace/bos_retry
+          fi
+          cd /workspace/upload
+          for FILE in /workspace/upload/*.whl; do
+            file=$(basename "$FILE")
+            python ${{ env.bos_file }} $file paddle-github-action/whl
+            echo "$file: https://paddle-github-action.bj.bcebos.com/whl/$file"
+          done
+          TAR_FILE=/workspace/upload/Megatron-LM.tar.gz
+          if [ -e "$TAR_FILE" ]; then
+            file=$(basename "$TAR_FILE")
+            python ${{ env.bos_file }} $file paddle-github-action/whl
+            echo "$file: https://paddle-github-action.bj.bcebos.com/whl/$file"
+          else
+            echo "Megatron-LM.tar.gz not found, skip upload"
+          fi
+          '
+      - name: Terminate And Delete the Container
+        if: always()
+        run: |
+          docker rm -f $container_name 2>/dev/null || true

scripts/dependence/build.sh

@@ -0,0 +1,80 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+export megatron_dir=/workspace/Megatron-LM
+mkdir -p /workspace/build_logs
+export log_path=/workspace/build_logs
+mkdir -p /workspace/upload
+upload_path=/workspace/upload
+
+python -m pip config --user set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple
+python -m pip config --user set global.trusted-host pypi.tuna.tsinghua.edu.cn
+
+megatron_tar (){
+    cd /workspace
+    # Megatron-LM.tar only include the main branch
+    if [ -n "$TARGET_COMMIT" ]; then
+        echo "TARGET_COMMIT=$TARGET_COMMIT specified, skip refreshing Megatron-LM.tar.gz"
+    elif [ -n "$BRANCH" ] && [ "$BRANCH" = "main" ]; then
+        echo "Checkout branch $BRANCH"
+        tar -zcf Megatron-LM.tar.gz Megatron-LM/
+        mv Megatron-LM.tar.gz ${upload_path}/
+    else
+        echo "No BRANCH specified, skip checkout"
+    fi
+}
+
+megatron_build (){
+    cd $megatron_dir
+    rm -rf di/
+    rm -rf dist/
+    rm -rf megatron_core.egg-info/
+
+    python -m pip install --upgrade pip
+    python -m pip install build "setuptools>=80" pybind11 packaging
+    NO_VCS_VERSION=1 python -m build --wheel --no-isolation
+
+    echo "install_megatron_develop_whl"
+    python -m pip install --ignore-installed dist/megatron_core-*.whl --no-cache-dir --force-reinstall --no-dependencies
+
+    commit=${COMMIT_ID:-unknown}
+    commit=${commit:0:7}
+
+    whl_file=$(ls $megatron_dir/dist/megatron_core-*.whl)
+    base_name=$(basename $whl_file)
+    new_name=$(echo $base_name | sed "s/^\(megatron_core-[0-9.]*\)-/\1+${commit}-/")
+    echo "commit whl: $new_name"
+    cp "$whl_file" "${upload_path}/${new_name}"
+
+    zero_name=$(echo $base_name | sed "s/^megatron_core-[^-]*-/megatron_core-0.0.0-/")
+    if [ "${UPDATE_LATEST:-true}" = "true" ]; then
+        echo "latest whl: $base_name"
+        cp "$whl_file" "${upload_path}/${base_name}"
+        echo "0.0.0 whl: $zero_name"
+        cp "$whl_file" "${upload_path}/${zero_name}"
+    else
+        echo "UPDATE_LATEST=$UPDATE_LATEST, skip publishing $base_name and $zero_name"
+    fi
+}
+
+# main
+cd ${megatron_dir}
+echo -e "\033[32m ---- make Megatron-LM.tar.gz  \033[0m"
+megatron_tar
+echo -e "\033[32m ---- build Megatron-LM whl  \033[0m"
+megatron_build
+