diff --git a/.github/workflows/codex-pgo-training.sh b/.github/workflows/codex-pgo-training.sh new file mode 100644 index 00000000000000..070d8b54cae486 --- /dev/null +++ b/.github/workflows/codex-pgo-training.sh @@ -0,0 +1,60 @@ +#!/bin/sh + +# Keep this workload short and biased toward the local Git operations Codex +# invokes frequently. The full Git test suite is too slow for each release +# target and would weight test-harness paths more heavily than status, diff, +# clone, fetch, and repository maintenance. + +set -eu + +git_bin="$PWD/bin-wrappers/git" +training_dir=$(mktemp -d "${TMPDIR:-/tmp}/codex-git-pgo.XXXXXX") +repo="$training_dir/repo" +clone="$training_dir/clone" + +cleanup () { + rm -rf "$training_dir" +} +trap cleanup EXIT HUP INT TERM + +mkdir -p "$training_dir/home" +export HOME="$training_dir/home" +export GIT_CONFIG_NOSYSTEM=1 +export GIT_TERMINAL_PROMPT=0 + +"$git_bin" clone --quiet --no-local "$PWD" "$repo" +"$git_bin" -C "$repo" config user.name "Codex Git PGO" +"$git_bin" -C "$repo" config user.email "codex-git-pgo@openai.com" + +i=0 +while test "$i" -lt 256 +do + dir="$repo/training/$((i % 16))" + mkdir -p "$dir" + printf '%s\n' "$i" >"$dir/file-$i" + i=$((i + 1)) +done + +"$git_bin" -C "$repo" status --porcelain=v2 --branch >/dev/null +"$git_bin" -C "$repo" status --porcelain=v2 --branch --untracked-files=all >/dev/null +"$git_bin" -C "$repo" ls-files --others --exclude-standard >/dev/null +"$git_bin" -C "$repo" add training +"$git_bin" -C "$repo" diff --cached --stat >/dev/null +"$git_bin" -C "$repo" commit --quiet -m "add training files" + +printf 'changed\n' >>"$repo/training/0/file-0" +rm "$repo/training/1/file-1" +mkdir -p "$repo/untracked" +printf 'new\n' >"$repo/untracked/file" + +"$git_bin" -C "$repo" status --porcelain=v2 --branch >/dev/null +"$git_bin" -C "$repo" diff --stat >/dev/null +"$git_bin" -C "$repo" diff --name-status >/dev/null +"$git_bin" -C "$repo" ls-files --stage >/dev/null +"$git_bin" -C "$repo" log --oneline --decorate -20 >/dev/null +"$git_bin" -C "$repo" rev-list --objects --all >/dev/null +"$git_bin" -C "$repo" for-each-ref --format='%(refname) %(objectname)' >/dev/null +"$git_bin" -C "$repo" repack -ad +"$git_bin" clone --quiet --no-local "$repo" "$clone" +"$git_bin" -C "$clone" status --porcelain=v2 --branch >/dev/null +"$git_bin" -C "$clone" fetch --quiet "$repo" diff --git a/.github/workflows/codex-release.yml b/.github/workflows/codex-release.yml new file mode 100644 index 00000000000000..778a67abf28b4f --- /dev/null +++ b/.github/workflows/codex-release.yml @@ -0,0 +1,582 @@ +name: Codex Git release + +on: + push: + branches: + - codex + - codex-unstable + +permissions: + contents: read + +concurrency: + group: codex-git-release-${{ github.sha }} + cancel-in-progress: false + +jobs: + publication: + name: Verify controller publication + runs-on: ubuntu-24.04 + if: github.event.deleted == false + outputs: + published: ${{ steps.verify.outputs.published }} + steps: + - name: Check the published controller output + id: verify + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + case "$GITHUB_REF" in + refs/heads/codex) + output_key=codex.output-tip + ;; + refs/heads/codex-unstable) + output_key=codex-unstable.output-tip + ;; + *) + printf 'unexpected release ref: %s\n' "$GITHUB_REF" >&2 + exit 1 + ;; + esac + + meta=$(gh api \ + "repos/$GITHUB_REPOSITORY/git/ref/heads/meta" \ + --jq '.object.sha') + recorded=$(gh api \ + "repos/$GITHUB_REPOSITORY/contents/codex.config?ref=$meta" \ + -H 'Accept: application/vnd.github.raw+json' | + git config --no-includes --file /dev/stdin \ + --get "$output_key") + + if test "$GITHUB_SHA" = "$recorded" + then + printf 'published=true\n' >>"$GITHUB_OUTPUT" + printf 'Releasing controller-published commit %s.\n' "$GITHUB_SHA" + else + printf 'published=false\n' >>"$GITHUB_OUTPUT" + printf 'Skipping non-controller publication %s.\n' "$GITHUB_SHA" + fi + + version: + name: Determine version + needs: publication + if: needs.publication.outputs.published == 'true' + runs-on: ubuntu-24.04 + outputs: + describe: ${{ steps.version.outputs.describe }} + upstream_tag: ${{ steps.version.outputs.upstream_tag }} + version: ${{ steps.version.outputs.version }} + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Derive OpenAI version from git describe + id: version + shell: bash + run: | + set -euo pipefail + describe="$(git describe \ + --match 'v[0-9]*' \ + --exclude 'v*-openai.*' \ + --long \ + --always \ + --abbrev=12 \ + "$GITHUB_SHA")" + + if [[ "$describe" =~ ^(.+)-([0-9]+)-g([0-9a-f]+)$ ]] + then + upstream_tag="${BASH_REMATCH[1]}" + version="$upstream_tag-openai.${BASH_REMATCH[2]}.g${BASH_REMATCH[3]}" + else + upstream_tag= + version="openai-$describe" + fi + git check-ref-format "refs/tags/$version" + printf 'describe=%s\n' "$describe" | tee -a "$GITHUB_OUTPUT" + printf 'upstream_tag=%s\n' "$upstream_tag" | tee -a "$GITHUB_OUTPUT" + printf 'version=%s\n' "$version" | tee -a "$GITHUB_OUTPUT" + + build: + name: ${{ matrix.name }} + needs: version + runs-on: ${{ matrix.os }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + include: + - name: macOS arm64 + os: macos-15 + target_platform: macOS + asset_platform: macOS + arch: arm64 + binary: /tmp/build/git/bin/git + file_pattern: Mach-O 64-bit executable arm64 + has_gcm: false + lto: thin + profile_format: LLVM + llvm_profdata: xcrun llvm-profdata + max_tar_bytes: 67108864 + - name: macOS x64 + os: macos-15-intel + target_platform: macOS + asset_platform: macOS + arch: x64 + binary: /tmp/build/git/bin/git + file_pattern: Mach-O 64-bit executable x86_64 + has_gcm: false + lto: thin + profile_format: LLVM + llvm_profdata: xcrun llvm-profdata + max_tar_bytes: 67108864 + # Keep arm64 builds native so release smoke tests can execute them. + - name: Linux arm64 + os: ubuntu-22.04-arm + target_platform: ubuntu + asset_platform: ubuntu + arch: arm64 + binary: /tmp/build/git/bin/git + file_pattern: ELF 64-bit.*ARM aarch64 + has_gcm: false + lto: auto + profile_format: GCC + max_tar_bytes: 67108864 + - name: Linux x64 + os: ubuntu-22.04 + target_platform: ubuntu + asset_platform: ubuntu + arch: x64 + binary: /tmp/build/git/bin/git + file_pattern: ELF 64-bit.*x86-64 + has_gcm: false + lto: auto + profile_format: GCC + max_tar_bytes: 67108864 + - name: Windows arm64 + os: windows-11-arm + target_platform: win32 + asset_platform: windows + arch: arm64 + binary: /tmp/build/git/clangarm64/bin/git.exe + file_pattern: PE32\+.*ARM64 + has_gcm: true + lto: thin + profile_format: LLVM + llvm_profdata: llvm-profdata + max_tar_bytes: 134217728 + sdk_arch: aarch64 + sdk_flavor: full + mingw_dir: clangarm64 + mingit_arch: arm64 + mingit_filename: MinGit-2.55.0.2-arm64.zip + mingit_url: https://github.com/git-for-windows/git/releases/download/v2.55.0.windows.2/MinGit-2.55.0.2-arm64.zip + mingit_sha256: 0b2b81fdce284efd174cbb51b886ccea2fd271679c4b5c21f07d9e03bae51413 + - name: Windows x64 + os: windows-2025 + target_platform: win32 + asset_platform: windows + arch: x64 + binary: /tmp/build/git/mingw64/bin/git.exe + file_pattern: PE32\+.*x86-64 + has_gcm: true + lto: auto + profile_format: GCC + max_tar_bytes: 134217728 + sdk_arch: x86_64 + sdk_flavor: full + mingw_dir: mingw64 + mingit_arch: amd64 + mingit_filename: MinGit-2.55.0.2-64-bit.zip + mingit_url: https://github.com/git-for-windows/git/releases/download/v2.55.0.windows.2/MinGit-2.55.0.2-64-bit.zip + mingit_sha256: e3ea2944cea4b3fabcd69c7c1669ef69b1b66c05ac7806d81224d0abad2dec31 + + steps: + # Keep the packaging contract, dependency pins, and platform build logic + # aligned with the artifacts already consumed by Codex and GitHub Desktop. + - name: Check out Dugite Native + uses: actions/checkout@v6 + with: + repository: dreynaud-oai/dugite-native + ref: b6f4473557acb85433fdf9deffe0854a34fd9cc5 + path: dugite-native + fetch-depth: 0 + persist-credentials: false + + - name: Check out this Git revision + uses: actions/checkout@v6 + with: + ref: ${{ github.sha }} + path: dugite-native/git + fetch-depth: 1 + persist-credentials: false + + - name: Give the source an immutable package version + shell: bash + working-directory: dugite-native/git + env: + VERSION: ${{ needs.version.outputs.version }} + run: | + git \ + -c 'user.name=github-actions[bot]' \ + -c 'user.email=41898282+github-actions[bot]@users.noreply.github.com' \ + tag -a "$VERSION" -m "$VERSION" + + - name: Install OpenAI build configuration + shell: bash + working-directory: dugite-native/git + run: cp config.mak.openai config.mak + + # Match Dugite Native's compatibility choice for its macOS x64 build. + - name: Select Xcode 16.4 + if: matrix.target_platform == 'macOS' && matrix.arch == 'x64' + run: | + sudo xcode-select -s /Applications/Xcode_16.4.app/Contents/Developer/ + sudo rm -rf /Library/Developer/CommandLineTools + + - name: Install Linux build dependencies + if: matrix.target_platform == 'ubuntu' + run: | + sudo apt-get update + sudo apt-get install -y \ + autoconf \ + automake \ + build-essential \ + ca-certificates \ + curl \ + gettext \ + jq \ + lsb-release \ + pkg-config + + - name: Install Linux x64 build dependencies + if: matrix.target_platform == 'ubuntu' && matrix.arch == 'x64' + run: | + sudo apt-get install -y \ + libcurl4-gnutls-dev \ + libexpat1-dev \ + libssl-dev \ + zlib1g-dev + + - name: Install Linux arm64 build dependencies + if: matrix.target_platform == 'ubuntu' && matrix.arch == 'arm64' + run: | + sudo apt-get install -y \ + binutils-aarch64-linux-gnu \ + gcc-aarch64-linux-gnu \ + libcurl4-gnutls-dev \ + libexpat1-dev \ + libssl-dev \ + zlib1g-dev + + # Dugite Native currently pins MinGit 2.53. Keep its build script and + # dependency schema, but match the runtime to the Git series we compile. + - name: Select the matching MinGit runtime + if: matrix.target_platform == 'win32' + shell: bash + working-directory: dugite-native + env: + MINGIT_ARCH: ${{ matrix.mingit_arch }} + MINGIT_FILENAME: ${{ matrix.mingit_filename }} + MINGIT_SHA256: ${{ matrix.mingit_sha256 }} + MINGIT_URL: ${{ matrix.mingit_url }} + MINGIT_VERSION: v2.55.0 + SOURCE_UPSTREAM_TAG: ${{ needs.version.outputs.upstream_tag }} + run: | + set -euo pipefail + test "$SOURCE_UPSTREAM_TAG" = "$MINGIT_VERSION" + updated="$(mktemp)" + jq \ + --arg arch "$MINGIT_ARCH" \ + --arg checksum "$MINGIT_SHA256" \ + --arg filename "$MINGIT_FILENAME" \ + --arg url "$MINGIT_URL" \ + --arg version "$MINGIT_VERSION" \ + '.git.version = $version | + (.git.files[] | + select(.platform == "windows" and .arch == $arch)) |= + (.filename = $filename | + .url = $url | + .checksum = $checksum)' \ + dependencies.json >"$updated" + mv "$updated" dependencies.json + + # Codex does not configure or invoke GCM on macOS or Linux. The + # self-contained .NET payload accounts for most of those bundles, while + # Windows MinGit configures credential.helper=manager and must retain it. + - name: Omit unused GCM from POSIX bundles + if: matrix.target_platform != 'win32' + shell: bash + working-directory: dugite-native + run: | + set -euo pipefail + updated="$(mktemp)" + jq '."git-credential-manager".files = []' \ + dependencies.json >"$updated" + mv "$updated" dependencies.json + + # Build an instrumented Git, run a representative local workload, then + # rebuild with its profile. Keep Git's optional Rust library disabled + # until its Makefile can direct Cargo at these targets. + - name: Build the Dugite Native distribution + shell: bash + working-directory: dugite-native + env: + NO_RUST: 1 + OPENAI_LLVM_PROFDATA: ${{ matrix.llvm_profdata }} + OPENAI_LTO: ${{ matrix.lto }} + OPENAI_PROFILE: BUILD + OPENAI_PROFILE_FORMAT: ${{ matrix.profile_format }} + TARGET_PLATFORM: ${{ matrix.target_platform }} + TARGET_ARCH: ${{ matrix.arch }} + run: | + set -euo pipefail + script/build.sh + + - name: Set up Git for Windows SDK + if: matrix.target_platform == 'win32' + uses: git-for-windows/setup-git-for-windows-sdk@v2 + with: + architecture: ${{ matrix.sdk_arch }} + flavor: ${{ matrix.sdk_flavor }} + cache: false + + # Dugite Native compiles its Git submodule on macOS and Linux. On + # Windows it starts from MinGit, so replace MinGit's Git programs with + # the build from this repository while retaining the portable runtime. + # MinGit omits dashed builtin aliases; installing them as copies would + # add hundreds of redundant MiB to the archive. + - name: Install this Git build into the Windows distribution + if: matrix.target_platform == 'win32' + shell: bash + working-directory: dugite-native/git + env: + MINGW_DIR: ${{ matrix.mingw_dir }} + OPENAI_LLVM_PROFDATA: ${{ matrix.llvm_profdata }} + OPENAI_LTO: ${{ matrix.lto }} + OPENAI_PROFILE_FORMAT: ${{ matrix.profile_format }} + run: | + set -euo pipefail + + make_args=( + "prefix=/$MINGW_DIR" + NO_PERL=YesPlease + NO_RUST=YesPlease + NO_TCLTK=YesPlease + NO_GETTEXT=YesPlease + NO_INSTALL_HARDLINKS=YesPlease + NO_CROSS_DIRECTORY_HARDLINKS=YesPlease + SKIP_DASHED_BUILT_INS=YesPlease + ) + jobs="${NUMBER_OF_PROCESSORS:-2}" + make -j"$jobs" "${make_args[@]}" OPENAI_PROFILE=BUILD all + make "${make_args[@]}" OPENAI_PROFILE=USE DESTDIR=/tmp/build/git strip install + + - name: Verify distribution layout and provenance + shell: bash + env: + TARGET_PLATFORM: ${{ matrix.target_platform }} + MINGW_DIR: ${{ matrix.mingw_dir }} + GIT_BINARY: ${{ matrix.binary }} + FILE_PATTERN: ${{ matrix.file_pattern }} + HAS_GCM: ${{ matrix.has_gcm }} + LTO: ${{ matrix.lto }} + PROFILE_FORMAT: ${{ matrix.profile_format }} + run: | + set -euo pipefail + if test "$TARGET_PLATFORM" = win32 + then + test -f /tmp/build/git/cmd/git.exe + test -f "/tmp/build/git/$MINGW_DIR/libexec/git-core/git-lfs.exe" + test -d "/tmp/build/git/$MINGW_DIR/share/git-core/templates" + test ! -e "/tmp/build/git/$MINGW_DIR/libexec/git-core/git-add.exe" + if test "$HAS_GCM" = true + then + test -f "/tmp/build/git/$MINGW_DIR/bin/git-credential-manager.exe" + fi + else + test -x /tmp/build/git/libexec/git-core/git-lfs + test -d /tmp/build/git/share/git-core/templates + if test "$HAS_GCM" = true + then + test -x /tmp/build/git/libexec/git-core/git-credential-manager + else + test ! -e /tmp/build/git/libexec/git-core/git-credential-manager + fi + fi + test -f /tmp/build/git/etc/gitconfig + + file "$GIT_BINARY" | tee /tmp/git-file-type + grep -E "$FILE_PATTERN" /tmp/git-file-type + strings "$GIT_BINARY" | grep -F "$GITHUB_SHA" + grep -F -- "-flto=$LTO" dugite-native/git/GIT-CFLAGS + if test "$PROFILE_FORMAT" = LLVM + then + grep -F -- "-fprofile-instr-use=" dugite-native/git/GIT-CFLAGS + else + grep -F -- "-fprofile-use=" dugite-native/git/GIT-CFLAGS + fi + + - name: Smoke-test the native distribution + shell: bash + env: + TARGET_PLATFORM: ${{ matrix.target_platform }} + MINGW_DIR: ${{ matrix.mingw_dir }} + HAS_GCM: ${{ matrix.has_gcm }} + run: | + set -euo pipefail + smoke=/tmp/codex-git-smoke + mkdir -p "$smoke/home" + + if test "$TARGET_PLATFORM" = win32 + then + git_binary=/tmp/build/git/cmd/git.exe + git_env=( + "PATH=/tmp/build/git/cmd:/tmp/build/git/$MINGW_DIR/bin:/tmp/build/git/usr/bin:$PATH" + ) + else + git_binary=/tmp/build/git/bin/git + git_env=( + GIT_CONFIG_SYSTEM=/tmp/build/git/etc/gitconfig + GIT_EXEC_PATH=/tmp/build/git/libexec/git-core + GIT_TEMPLATE_DIR=/tmp/build/git/share/git-core/templates + ) + if test "$TARGET_PLATFORM" = ubuntu + then + git_env+=( + GIT_SSL_CAINFO=/tmp/build/git/ssl/cacert.pem + PREFIX=/tmp/build/git + ) + fi + fi + git_env+=("HOME=$smoke/home" GIT_TERMINAL_PROMPT=0) + + build_options="$(env "${git_env[@]}" "$git_binary" --version --build-options)" + printf '%s\n' "$build_options" + grep -F "built from commit: $GITHUB_SHA" <<<"$build_options" + env "${git_env[@]}" "$git_binary" lfs version + if test "$HAS_GCM" = true + then + env "${git_env[@]}" "$git_binary" credential-manager --version + fi + + env "${git_env[@]}" "$git_binary" init --quiet "$smoke/repo" + echo test >"$smoke/repo/file" + env "${git_env[@]}" "$git_binary" -C "$smoke/repo" add file + env "${git_env[@]}" "$git_binary" -C "$smoke/repo" \ + -c user.name='Codex Git CI' \ + -c user.email='codex-git-ci@openai.com' \ + commit --quiet -m initial + test -z "$(env "${git_env[@]}" "$git_binary" -C "$smoke/repo" status --porcelain)" + + - name: Package with Dugite Native + shell: bash + working-directory: dugite-native + env: + ASSET_PLATFORM: ${{ matrix.asset_platform }} + TARGET_PLATFORM: ${{ matrix.target_platform }} + TARGET_ARCH: ${{ matrix.arch }} + VERSION: ${{ needs.version.outputs.version }} + MAX_TAR_BYTES: ${{ matrix.max_tar_bytes }} + run: | + set -euo pipefail + script/package.sh + + for extension in tar.gz lzma + do + matches=( + output/dugite-native-"$VERSION"-*-"$ASSET_PLATFORM"-"$TARGET_ARCH.$extension" + ) + test "${#matches[@]}" -eq 1 + test -f "${matches[0]}" + + destination="output/git-$VERSION-$ASSET_PLATFORM-$TARGET_ARCH.$extension" + mv "${matches[0]}" "$destination" + mv "${matches[0]}.sha256" "$destination.sha256" + done + + for checksum in output/*.sha256 + do + archive="${checksum%.sha256}" + expected="$(tr -d '\r\n' <"$checksum")" + if command -v sha256sum >/dev/null 2>&1 + then + actual="$(sha256sum "$archive" | awk '{print $1}')" + else + actual="$(shasum -a 256 "$archive" | awk '{print $1}')" + fi + test "$actual" = "$expected" + done + + tarball="output/git-$VERSION-$ASSET_PLATFORM-$TARGET_ARCH.tar.gz" + tar_bytes="$(wc -c <"$tarball")" + printf '%s bytes: %s\n' "$tar_bytes" "$tarball" + test "$tar_bytes" -le "$MAX_TAR_BYTES" + + - name: Upload release assets + uses: actions/upload-artifact@v7 + with: + name: git-${{ matrix.asset_platform }}-${{ matrix.arch }} + path: dugite-native/output/git-* + if-no-files-found: error + retention-days: 7 + + # The arm64 SDK puts its target Git first on PATH, but action cleanup + # runs on the x64 host and therefore needs the runner's native Git. + - name: Restore native Git for action cleanup + if: always() && matrix.target_platform == 'win32' + shell: pwsh + run: | + "C:\Program Files\Git\cmd" | + Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + + release: + name: Publish GitHub prerelease + needs: + - version + - build + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + contents: write + steps: + - name: Download release assets + uses: actions/download-artifact@v8 + with: + pattern: git-* + path: artifacts + merge-multiple: true + + - name: Publish immutable prerelease + env: + GH_TOKEN: ${{ github.token }} + SOURCE_DESCRIPTION: ${{ needs.version.outputs.describe }} + SOURCE_REF: ${{ github.ref }} + VERSION: ${{ needs.version.outputs.version }} + run: | + set -euo pipefail + assets=(artifacts/git-*) + release_notes=$( + printf '%s\n' \ + "source_ref=$SOURCE_REF" \ + "source_sha=$GITHUB_SHA" \ + "" \ + "OpenAI Git release artifacts for $SOURCE_DESCRIPTION, built from $GITHUB_SHA for Codex." + ) + + if gh release view "$VERSION" --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1 + then + gh release upload "$VERSION" "${assets[@]}" \ + --repo "$GITHUB_REPOSITORY" \ + --clobber + else + gh release create "$VERSION" "${assets[@]}" \ + --repo "$GITHUB_REPOSITORY" \ + --target "$GITHUB_SHA" \ + --title "$VERSION" \ + --notes "$release_notes" \ + --prerelease + fi diff --git a/.github/workflows/codex.yml b/.github/workflows/codex.yml new file mode 100644 index 00000000000000..650cfd55baef14 --- /dev/null +++ b/.github/workflows/codex.yml @@ -0,0 +1,243 @@ +name: Refresh codex + +on: + schedule: + - cron: '*/5 * * * *' + workflow_dispatch: + inputs: + operation: + description: Refresh, scan, remove, or reorder a pinned topic + type: choice + options: + - refresh + - scan + - remove + - reorder + default: refresh + lane: + description: codex or codex-unstable for a plan operation + required: false + type: string + topic: + description: Exact topic branch for a plan operation + required: false + type: string + after: + description: Existing topic or root for reorder + required: false + type: string + plan_branch: + description: Optional codex-plan/* branch name + required: false + type: string + pull_request_target: + branches: + - meta + types: + - opened + - reopened + - synchronize + - ready_for_review + +permissions: + actions: read + contents: read + pull-requests: read + +jobs: + refresh: + if: >- + github.event_name == 'workflow_dispatch' && + github.ref == 'refs/heads/codex' && + inputs.operation == 'refresh' + uses: openai/git/.github/workflows/codex.yml@meta + topic_plan_scan: + name: Find one approved topic plan + if: >- + github.event_name == 'schedule' || + (github.event_name == 'workflow_dispatch' && + github.ref == 'refs/heads/codex' && + inputs.operation == 'scan') + runs-on: ubuntu-24.04 + permissions: + contents: read + pull-requests: read + concurrency: + group: codex-topic-plan-scan + cancel-in-progress: false + outputs: + lane: ${{ steps.reviewed.outputs.lane }} + topic: ${{ steps.reviewed.outputs.topic }} + source_tip: ${{ steps.reviewed.outputs.source_tip }} + review_pr: ${{ steps.reviewed.outputs.review_pr }} + env: + GH_TOKEN: ${{ github.token }} + steps: + - name: Pin trusted meta + id: meta + run: | + set -euo pipefail + test "$GITHUB_REPOSITORY" = openai/git + test "$GITHUB_REF" = refs/heads/codex + sha=$(gh api "repos/$GITHUB_REPOSITORY/git/ref/heads/meta" \ + --jq .object.sha) + case "$sha" in + ''|*[!0-9a-f]*) exit 1 ;; + esac + test "${#sha}" = 40 + printf 'sha=%s\n' "$sha" >>"$GITHUB_OUTPUT" + + - name: Check out trusted meta + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + repository: ${{ github.repository }} + ref: ${{ steps.meta.outputs.sha }} + fetch-depth: 0 + persist-credentials: false + + - name: Find one exact approved topic PR + id: reviewed + env: + META_SHA: ${{ steps.meta.outputs.sha }} + run: | + set -euo pipefail + + die () { + printf '%s\n' "$*" >&2 + exit 1 + } + + test "$GITHUB_REPOSITORY" = openai/git + test "$GITHUB_REF" = refs/heads/codex || + die "topic scan must run from the trusted default branch" + test "$(git rev-parse HEAD)" = "$META_SHA" || + die "trusted checkout does not match pinned meta" + gh auth setup-git + mkdir -p "$RUNNER_TEMP/codex-plan-scan" + for lane in codex codex-unstable + do + case "$lane" in + codex) plan=codex.plan ;; + codex-unstable) plan=codex-unstable.plan ;; + esac + test -f "$plan" || + die "trusted meta has no $plan" + gh pr list --repo "$GITHUB_REPOSITORY" --state open \ + --base "$lane" --limit 1000 \ + --json number,isDraft,headRefName,headRefOid,headRepository,reviewDecision | + jq -r --arg lane "$lane" ' + .[] | + select(.isDraft | not) | + select(.reviewDecision == "APPROVED") | + select(.headRepository.nameWithOwner == "openai/git") | + [$lane, .headRefName, .headRefOid, + (.number | tostring)] | @tsv + ' + done | sort -k4,4n >"$RUNNER_TEMP/codex-plan-scan/candidates" + + while IFS=$'\t' read -r lane topic source_tip review_pr + do + test -n "$review_pr" || continue + case "$review_pr" in + *[!0-9]*) die "approved topic PR has invalid number '$review_pr'" ;; + esac + case "$source_tip" in + *[!0-9a-f]*|'') die "approved topic PR has invalid source SHA" ;; + esac + test "${#source_tip}" = 40 || + die "approved topic PR has invalid source SHA" + git check-ref-format "refs/heads/$topic" >/dev/null 2>&1 || + die "approved topic PR has invalid branch '$topic'" + case "$topic" in + ??/codex/*) ;; + *) continue ;; + esac + suffix=${topic#??/codex/} + case "$suffix" in + ''|*/*|*-wip|*-stale) continue ;; + esac + case "$lane" in + codex) + case "$topic" in + *-unstable) continue ;; + esac + plan=codex.plan + ;; + codex-unstable) + case "$topic" in + *-unstable) ;; + *) continue ;; + esac + plan=codex-unstable.plan + ;; + *) die "approved topic PR has invalid lane '$lane'" ;; + esac + pinned=$(git config --no-includes \ + --file "$plan" \ + --get "branch.$topic.source-tip" || :) + test "$pinned" = "$source_tip" && continue + short=$(printf '%.12s' "$source_tip") + slug=${topic##*/} + plan_branch=codex-plan/$lane-$slug-$short + pending=$(gh pr list --repo "$GITHUB_REPOSITORY" \ + --state open --base meta --head "$plan_branch" \ + --json number --jq '.[0].number // empty') || + die "could not inspect pending Codex plan PR" + test -n "$pending" && continue + if ! sh .github/workflows/codex-branch.sh propose-plan \ + --remote origin --lane "$lane" --topic "$topic" \ + --action auto --source-tip "$source_tip" \ + --review-pr "$review_pr" --expected-meta "$META_SHA" \ + --no-push >/dev/null + then + printf 'skipping approved topic PR #%s: preflight failed\n' \ + "$review_pr" >&2 + continue + fi + { + printf 'lane=%s\n' "$lane" + printf 'topic=%s\n' "$topic" + printf 'source_tip=%s\n' "$source_tip" + printf 'review_pr=%s\n' "$review_pr" + } >>"$GITHUB_OUTPUT" + exit 0 + done <"$RUNNER_TEMP/codex-plan-scan/candidates" + topic_plan_propose: + name: Propose reviewed topic plan + needs: topic_plan_scan + if: needs.topic_plan_scan.outputs.review_pr != '' + permissions: + contents: read + pull-requests: read + uses: openai/git/.github/workflows/codex-plan-propose.yml@meta + with: + lane: ${{ needs.topic_plan_scan.outputs.lane }} + topic: ${{ needs.topic_plan_scan.outputs.topic }} + action: auto + source_tip: ${{ needs.topic_plan_scan.outputs.source_tip }} + review_pr: ${{ needs.topic_plan_scan.outputs.review_pr }} + policy_plan_propose: + name: Propose explicit plan policy + if: >- + github.event_name == 'workflow_dispatch' && + github.ref == 'refs/heads/codex' && + (inputs.operation == 'remove' || inputs.operation == 'reorder') + permissions: + contents: read + pull-requests: read + uses: openai/git/.github/workflows/codex-plan-propose.yml@meta + with: + lane: ${{ inputs.lane }} + topic: ${{ inputs.topic }} + action: ${{ inputs.operation }} + after: ${{ inputs.after }} + plan_branch: ${{ inputs.plan_branch }} + plan_admission: + name: Codex plan admission + if: >- + github.event_name == 'pull_request_target' && + github.event.pull_request.base.ref == 'meta' + permissions: + contents: read + pull-requests: write + uses: openai/git/.github/workflows/codex-plan-admission.yml@meta diff --git a/Documentation/config.adoc b/Documentation/config.adoc index 1ef72de62f2ba6..a6b83c014288cd 100644 --- a/Documentation/config.adoc +++ b/Documentation/config.adoc @@ -518,6 +518,8 @@ include::config/includeif.adoc[] include::config/index.adoc[] +include::config/indexpack.adoc[] + include::config/init.adoc[] include::config/instaweb.adoc[] diff --git a/Documentation/config/indexpack.adoc b/Documentation/config/indexpack.adoc new file mode 100644 index 00000000000000..a56beccd390f1a --- /dev/null +++ b/Documentation/config/indexpack.adoc @@ -0,0 +1,21 @@ +indexPack.hashThreads:: + Experimental number of workers used to hash full blobs during the + first pass of linkgit:git-index-pack[1]. The default, zero, disables + the workers. Values from 1 through 32 are accepted. Parsing and + inflation remain serial, and normal collision/content validation runs + on the main thread. This is independent of `pack.threads` and of + concurrent packfile-URI downloads. Strict, fsck, and promisor modes, + and builds without thread support, use the existing serial path. + +indexPack.hashBufferSize:: + Maximum retained blob-buffer bytes for `indexPack.hashThreads`, + including the terminating byte, the producer's reserved buffer, and + completed results awaiting validation. Defaults to 64 MiB. At most + twice the number of hash workers can be outstanding. Objects that do + not fit use the existing serial path; the limit is not a bound on + Git's total memory use. Usual `k`, `m`, and `g` suffixes are accepted. + +indexPack.hashMinSize:: + Minimum full-blob size eligible for `indexPack.hashThreads`. + Defaults to 64 KiB. Blobs above `core.bigFileThreshold` retain their + existing streaming path. Usual `k`, `m`, and `g` suffixes are accepted. diff --git a/Documentation/git-http-fetch.adoc b/Documentation/git-http-fetch.adoc index 2200f073c47120..45e0d3d07c73cf 100644 --- a/Documentation/git-http-fetch.adoc +++ b/Documentation/git-http-fetch.adoc @@ -48,13 +48,15 @@ commit-id:: line (which is not expected in this case), 'git http-fetch' fetches the packfile directly at the given URL and uses index-pack to generate corresponding .idx and .keep files. - The hash is used to determine the name of the temporary file and is - arbitrary. The output of index-pack is printed to stdout. Requires - --index-pack-args. + The hash is used to determine the name of the temporary file. It need + not be the pack hash, but it must uniquely identify the pack contents + for resumption. The output of index-pack is printed to stdout. Requires + one or more --index-pack-arg options. ---index-pack-args=:: - For internal use only. The command to run on the contents of the - downloaded pack. Arguments are URL-encoded separated by spaces. +--index-pack-arg=:: + For internal use only. The first instance specifies the command run on + the contents of the downloaded pack. Subsequent instances specify its + arguments. --recover:: Verify that everything reachable from target is fetched. Used after diff --git a/Documentation/git-index-pack.adoc b/Documentation/git-index-pack.adoc index 18036953c06b22..0cf251ac9bb65d 100644 --- a/Documentation/git-index-pack.adoc +++ b/Documentation/git-index-pack.adoc @@ -148,6 +148,11 @@ accessible through promisor objects. + Requires to not be specified. +CONFIGURATION +------------- + +include::config/indexpack.adoc[] + NOTES ----- diff --git a/builtin/gc.c b/builtin/gc.c index 49c8474fade8ed..e9572940dcb22d 100644 --- a/builtin/gc.c +++ b/builtin/gc.c @@ -1593,7 +1593,8 @@ static int maintenance_task_geometric_repack(struct maintenance_run_opts *opts, child.odb_to_close = the_repository->objects; strvec_pushl(&child.args, "repack", "-d", "-l", NULL); - if (geometry.split < geometry.pack_nr) + if (geometry.split < geometry.pack_nr || + geometry.promisor_split < geometry.promisor_pack_nr) strvec_pushf(&child.args, "--geometric=%d", geometry.split_factor); else @@ -1648,7 +1649,7 @@ static int geometric_repack_auto_condition(struct gc_config *cfg UNUSED) * When we'd merge at least two packs with one another we always * perform the repack. */ - if (geometry.split) { + if (geometry.split || geometry.promisor_split) { ret = 1; goto out; } diff --git a/builtin/index-pack.c b/builtin/index-pack.c index bc86925ad04340..9b6a0abbb2b44f 100644 --- a/builtin/index-pack.c +++ b/builtin/index-pack.c @@ -31,6 +31,7 @@ #include "run-command.h" #include "setup.h" #include "strvec.h" +#include "trace2.h" static const char index_pack_usage[] = "git index-pack [-v] [-o ] [--keep | --keep=] [--[no-]rev-index] [--verify] [--strict[==...]] [--fsck-objects[==...]] ( | --stdin [--fix-thin] [])"; @@ -43,6 +44,43 @@ struct object_entry { signed char real_type; }; +/* + * Hash workers own no repository state. They only hash immutable full-blob + * buffers; the main thread retires their results and performs the usual + * collision/content checks before releasing those buffers. + */ +struct first_pass_hash_job { + struct object_entry *obj; + struct object_id oid; + void *data; + int done; +}; + +struct first_pass_hash_pool { + pthread_t *threads; + pthread_mutex_t mutex; + pthread_cond_t work_ready; + pthread_cond_t result_ready; + struct first_pass_hash_job *queue; + size_t nr_threads, queue_size; + size_t first, next_work, nr, pending; + size_t buffered, jobs; + int stop; +}; + +#define FIRST_PASS_HASH_MAX_THREADS 32 + +static int first_pass_hash_threads; +static size_t first_pass_hash_buffer_size = 64 * 1024 * 1024; +static size_t first_pass_hash_min_size = 64 * 1024; +static struct first_pass_hash_pool first_pass_hash_pool; + +static struct first_pass_hash_job *reserve_first_pass_hash(struct object_entry *obj); +static void submit_first_pass_hash(struct first_pass_hash_job *job, + struct object_entry *obj, void *data); +static void start_first_pass_hash(void); +static void finish_first_pass_hash(void); + struct object_stat { unsigned delta_depth; int base_object_no; @@ -479,7 +517,7 @@ static void *unpack_entry_data(off_t offset, size_t size, char hdr[32]; int hdrlen; - if (!is_delta_type(type)) { + if (!is_delta_type(type) && oid) { hdrlen = format_object_header(hdr, sizeof(hdr), type, size); git_hash_init(&c, the_hash_algo); git_hash_update(&c, hdr, hdrlen); @@ -520,7 +558,8 @@ static void *unpack_entry_data(off_t offset, size_t size, static void *unpack_raw_entry(struct object_entry *obj, off_t *ofs_offset, struct object_id *ref_oid, - struct object_id *oid) + struct object_id *oid, + struct first_pass_hash_job **hash_job) { unsigned char *p; size_t size, c; @@ -582,7 +621,9 @@ static void *unpack_raw_entry(struct object_entry *obj, } obj->hdr_size = consumed_bytes - obj->idx.offset; - data = unpack_entry_data(obj->idx.offset, obj->size, obj->type, oid); + *hash_job = reserve_first_pass_hash(obj); + data = unpack_entry_data(obj->idx.offset, obj->size, obj->type, + *hash_job ? NULL : oid); obj->idx.crc32 = input_crc32; return data; } @@ -978,6 +1019,171 @@ static void sha1_object(const void *data, struct object_entry *obj_entry, free(new_data); } +static void *first_pass_hash_worker(void *data UNUSED) +{ + struct first_pass_hash_pool *pool = &first_pass_hash_pool; + + trace2_thread_start("index-pack-hash"); + for (;;) { + struct first_pass_hash_job *job; + + pthread_mutex_lock(&pool->mutex); + while (!pool->pending && !pool->stop) + pthread_cond_wait(&pool->work_ready, &pool->mutex); + if (!pool->pending) { + pthread_mutex_unlock(&pool->mutex); + break; + } + job = &pool->queue[pool->next_work]; + pool->next_work = (pool->next_work + 1) % pool->queue_size; + pool->pending--; + pthread_mutex_unlock(&pool->mutex); + + /* This uses the same collision-detecting hash as the serial path. */ + hash_object_file(the_hash_algo, job->data, job->obj->size, + OBJ_BLOB, &job->oid); + + pthread_mutex_lock(&pool->mutex); + job->done = 1; + pthread_cond_signal(&pool->result_ready); + pthread_mutex_unlock(&pool->mutex); + } + trace2_thread_exit(); + return NULL; +} + +static void retire_first_pass_hash(void) +{ + struct first_pass_hash_pool *pool = &first_pass_hash_pool; + struct first_pass_hash_job *job; + size_t allocation; + + pthread_mutex_lock(&pool->mutex); + if (!pool->nr) + BUG("no queued first-pass hash to retire"); + job = &pool->queue[pool->first]; + while (!job->done) + pthread_cond_wait(&pool->result_ready, &pool->mutex); + pool->first = (pool->first + 1) % pool->queue_size; + pool->nr--; + pthread_mutex_unlock(&pool->mutex); + + /* All ODB and object-cache access stays on the main thread. */ + oidcpy(&job->obj->idx.oid, &job->oid); + sha1_object(job->data, NULL, job->obj->size, OBJ_BLOB, + &job->obj->idx.oid); + allocation = st_add(job->obj->size, 1); + free(job->data); + pool->buffered -= allocation; +} + +static struct first_pass_hash_job *reserve_first_pass_hash(struct object_entry *obj) +{ + struct first_pass_hash_pool *pool = &first_pass_hash_pool; + size_t allocation; + + if (!pool->nr_threads || obj->type != OBJ_BLOB || + obj->size < first_pass_hash_min_size || + obj->size >= first_pass_hash_buffer_size || + obj->size > repo_settings_get_big_file_threshold(the_repository)) + return NULL; + + allocation = st_add(obj->size, 1); + while (pool->nr == pool->queue_size || + allocation > first_pass_hash_buffer_size - pool->buffered) + retire_first_pass_hash(); + + /* Reserve before the producer allocates the inflated blob. */ + pool->buffered += allocation; + return &pool->queue[(pool->first + pool->nr) % pool->queue_size]; +} + +static void submit_first_pass_hash(struct first_pass_hash_job *job, + struct object_entry *obj, void *data) +{ + struct first_pass_hash_pool *pool = &first_pass_hash_pool; + + assert(pool->nr_threads && data && obj->type == OBJ_BLOB); + job->obj = obj; + job->data = data; + job->done = 0; + + pthread_mutex_lock(&pool->mutex); + pool->nr++; + pool->pending++; + pool->jobs++; + pthread_cond_signal(&pool->work_ready); + pthread_mutex_unlock(&pool->mutex); +} + +static void stop_first_pass_hash(size_t nr_threads) +{ + struct first_pass_hash_pool *pool = &first_pass_hash_pool; + size_t i; + + pthread_mutex_lock(&pool->mutex); + pool->stop = 1; + pthread_cond_broadcast(&pool->work_ready); + pthread_mutex_unlock(&pool->mutex); + for (i = 0; i < nr_threads; i++) + pthread_join(pool->threads[i], NULL); + pthread_cond_destroy(&pool->result_ready); + pthread_cond_destroy(&pool->work_ready); + pthread_mutex_destroy(&pool->mutex); + free(pool->queue); + free(pool->threads); +} + +static void start_first_pass_hash(void) +{ + struct first_pass_hash_pool *pool = &first_pass_hash_pool; + size_t i; + int ret; + + /* These modes share fsck/object-cache state; retain their serial path. */ + if (!HAVE_THREADS || !first_pass_hash_threads || strict || + do_fsck_object || record_outgoing_links || + first_pass_hash_min_size >= first_pass_hash_buffer_size) { + trace2_data_intmax("index-pack", the_repository, + "first_pass_hash/threads", 0); + return; + } + + pool->nr_threads = first_pass_hash_threads; + pool->queue_size = st_mult(pool->nr_threads, 2); + CALLOC_ARRAY(pool->threads, pool->nr_threads); + CALLOC_ARRAY(pool->queue, pool->queue_size); + pthread_mutex_init(&pool->mutex, NULL); + pthread_cond_init(&pool->work_ready, NULL); + pthread_cond_init(&pool->result_ready, NULL); + for (i = 0; i < pool->nr_threads; i++) { + ret = pthread_create(&pool->threads[i], NULL, + first_pass_hash_worker, NULL); + if (ret) { + stop_first_pass_hash(i); + die(_("unable to create index-pack hash thread: %s"), + strerror(ret)); + } + } + trace2_data_intmax("index-pack", the_repository, + "first_pass_hash/threads", pool->nr_threads); +} + +static void finish_first_pass_hash(void) +{ + struct first_pass_hash_pool *pool = &first_pass_hash_pool; + + if (!pool->nr_threads) + return; + while (pool->nr) + retire_first_pass_hash(); + + stop_first_pass_hash(pool->nr_threads); + trace2_data_intmax("index-pack", the_repository, + "first_pass_hash/jobs", pool->jobs); + pool->nr_threads = 0; +} + /* * Ensure that this node has been reconstructed and return its contents. * @@ -1255,6 +1461,8 @@ static void parse_pack_objects(unsigned char *hash) struct stat st; struct git_hash_ctx tmp_ctx; + start_first_pass_hash(); + if (verbose) progress = start_progress( the_repository, @@ -1263,9 +1471,10 @@ static void parse_pack_objects(unsigned char *hash) nr_objects); for (i = 0; i < nr_objects; i++) { struct object_entry *obj = &objects[i]; + struct first_pass_hash_job *hash_job; void *data = unpack_raw_entry(obj, &ofs_delta->offset, &ref_delta_oid, - &obj->idx.oid); + &obj->idx.oid, &hash_job); obj->real_type = obj->type; if (obj->type == OBJ_OFS_DELTA) { nr_ofs_deltas++; @@ -1276,6 +1485,9 @@ static void parse_pack_objects(unsigned char *hash) oidcpy(&ref_deltas[nr_ref_deltas].oid, &ref_delta_oid); ref_deltas[nr_ref_deltas].obj_no = i; nr_ref_deltas++; + } else if (hash_job) { + submit_first_pass_hash(hash_job, obj, data); + data = NULL; } else if (!data) { /* large blobs, check later */ obj->real_type = OBJ_BAD; @@ -1287,6 +1499,7 @@ static void parse_pack_objects(unsigned char *hash) display_progress(progress, i+1); } objects[i].idx.offset = consumed_bytes; + finish_first_pass_hash(); stop_progress(&progress); /* Check pack integrity */ @@ -1667,6 +1880,23 @@ static int git_index_pack_config(const char *k, const char *v, { struct pack_idx_option *opts = cb; + if (!strcmp(k, "indexpack.hashthreads")) { + first_pass_hash_threads = git_config_int(k, v, ctx->kvi); + if (first_pass_hash_threads < 0 || + first_pass_hash_threads > FIRST_PASS_HASH_MAX_THREADS) + die(_("%s must be between 0 and %d"), + k, FIRST_PASS_HASH_MAX_THREADS); + return 0; + } + if (!strcmp(k, "indexpack.hashbuffersize")) { + first_pass_hash_buffer_size = git_config_ulong(k, v, ctx->kvi); + return 0; + } + if (!strcmp(k, "indexpack.hashminsize")) { + first_pass_hash_min_size = git_config_ulong(k, v, ctx->kvi); + return 0; + } + if (!strcmp(k, "pack.indexversion")) { opts->version = git_config_int(k, v, ctx->kvi); if (opts->version > 2) diff --git a/config.mak.openai b/config.mak.openai new file mode 100644 index 00000000000000..0ae133c8bb2ecf --- /dev/null +++ b/config.mak.openai @@ -0,0 +1,74 @@ +# OpenAI release build settings. +# +# The Codex release workflow copies this file to config.mak before +# building. Keep release-only optimizations here instead of in the +# upstream Makefile. + +OPENAI_PROFILE_DIR := $(CURDIR) +OPENAI_PROFILE_RAW := $(OPENAI_PROFILE_DIR)/default_%m_%p.profraw +OPENAI_PROFILE_DATA := $(OPENAI_PROFILE_DIR)/default.profdata +OPENAI_PROFILE_TRAINING ?= .github/workflows/codex-pgo-training.sh +OPENAI_LLVM_PROFDATA ?= llvm-profdata + +ifdef OPENAI_LTO +CFLAGS_APPEND += -flto=$(OPENAI_LTO) +LDFLAGS_APPEND += -flto=$(OPENAI_LTO) +endif + +ifeq ("$(OPENAI_PROFILE_FORMAT)","LLVM") +ifeq ("$(OPENAI_PROFILE)","GEN") + BASIC_CFLAGS += -fprofile-instr-generate=$(OPENAI_PROFILE_RAW) + BASIC_CFLAGS += -DNO_NORETURN=1 + export CCACHE_DISABLE = t + V = 1 +else +ifneq ("$(OPENAI_PROFILE)","") + BASIC_CFLAGS += -fprofile-instr-use=$(OPENAI_PROFILE_DATA) + BASIC_CFLAGS += -Wno-profile-instr-unprofiled -DNO_NORETURN=1 + export CCACHE_DISABLE = t + V = 1 +endif +endif +else +ifeq ("$(OPENAI_PROFILE)","GEN") + BASIC_CFLAGS += -fprofile-generate=$(OPENAI_PROFILE_DIR) + BASIC_CFLAGS += -DNO_NORETURN=1 + EXTLIBS += -lgcov + export CCACHE_DISABLE = t + V = 1 +else +ifneq ("$(OPENAI_PROFILE)","") + BASIC_CFLAGS += -fprofile-use=$(OPENAI_PROFILE_DIR) + BASIC_CFLAGS += -fprofile-correction -DNO_NORETURN=1 + export CCACHE_DISABLE = t + V = 1 +endif +endif +endif + +# Every C object already waits for Git's forced GIT-CFLAGS target. Gate that +# target so "make strip install" finishes PGO before its normal prerequisites +# can start compiling with profile-use flags. +ifeq "$(OPENAI_PROFILE)" "BUILD" +GIT-CFLAGS: openai-profile +endif + +openai-profile: profile-clean openai-profile-clean + $(MAKE) OPENAI_PROFILE=GEN all + $(SHELL_PATH) $(OPENAI_PROFILE_TRAINING) + $(MAKE) OPENAI_PROFILE= openai-profile-merge + $(MAKE) OPENAI_PROFILE=USE all + +openai-profile-merge: +ifeq ("$(OPENAI_PROFILE_FORMAT)","LLVM") + $(OPENAI_LLVM_PROFDATA) merge \ + -output=$(OPENAI_PROFILE_DATA) \ + $(OPENAI_PROFILE_DIR)/*.profraw +endif + +clean: openai-profile-clean + +openai-profile-clean: + $(RM) $(OPENAI_PROFILE_DIR)/*.profraw $(OPENAI_PROFILE_DATA) + +.PHONY: openai-profile openai-profile-merge openai-profile-clean diff --git a/fetch-pack.c b/fetch-pack.c index 922a9b25812c68..626f799712ec2e 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -1854,9 +1854,10 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args, } for (i = 0; i < packfile_uris.nr; i++) { + bool created_keep; int j; struct child_process cmd = CHILD_PROCESS_INIT; - char packname[GIT_MAX_HEXSZ + 1]; + char packhash[GIT_MAX_HEXSZ + 1]; const char *uri = packfile_uris.items[i].string + the_hash_algo->hexsz + 1; @@ -1874,16 +1875,17 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args, if (start_command(&cmd)) die("fetch-pack: unable to spawn http-fetch"); - if (read_in_full(cmd.out, packname, 5) < 0 || - memcmp(packname, "keep\t", 5)) - die("fetch-pack: expected keep then TAB at start of http-fetch output"); + if (read_in_full(cmd.out, packhash, 5) != 5 || + (memcmp(packhash, "keep\t", 5) && + memcmp(packhash, "pack\t", 5))) + die("fetch-pack: expected pack or keep then TAB at start of http-fetch output"); + created_keep = !memcmp(packhash, "keep\t", 5); - if (read_in_full(cmd.out, packname, - the_hash_algo->hexsz + 1) < 0 || - packname[the_hash_algo->hexsz] != '\n') - die("fetch-pack: expected hash then LF at end of http-fetch output"); - - packname[the_hash_algo->hexsz] = '\0'; + if (read_in_full(cmd.out, packhash, + the_hash_algo->hexsz + 1) != the_hash_algo->hexsz + 1 || + packhash[the_hash_algo->hexsz] != '\n') + die("fetch-pack: expected hash then LF in http-fetch output"); + packhash[the_hash_algo->hexsz] = '\0'; parse_gitmodules_oids(cmd.out, &fsck_options.gitmodules_found); @@ -1892,16 +1894,17 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args, if (finish_command(&cmd)) die("fetch-pack: unable to finish http-fetch"); - if (memcmp(packfile_uris.items[i].string, packname, + if (memcmp(packfile_uris.items[i].string, packhash, the_hash_algo->hexsz)) die("fetch-pack: pack downloaded from %s does not match expected hash %.*s", uri, (int) the_hash_algo->hexsz, packfile_uris.items[i].string); - string_list_append_nodup(pack_lockfiles, - xstrfmt("%s/pack/pack-%s.keep", - repo_get_object_directory(the_repository), - packname)); + if (created_keep) + string_list_append_nodup(pack_lockfiles, + xstrfmt("%s/pack/pack-%s.keep", + repo_get_object_directory(the_repository), + packhash)); } string_list_clear(&packfile_uris, 0); strvec_clear(&index_pack_args); diff --git a/http-fetch.c b/http-fetch.c index f9b6ecb0616fe0..05f68f306a5821 100644 --- a/http-fetch.c +++ b/http-fetch.c @@ -70,7 +70,8 @@ static void fetch_single_packfile(struct object_id *packfile_hash, if (start_active_slot(preq->slot)) { run_active_slot(preq->slot); - if (results.curl_result != CURLE_OK) { + if (results.curl_result != CURLE_OK && + results.http_code != 416) { struct url_info url; char *nurl = url_normalize(preq->url, &url); if (!nurl || !git_env_bool("GIT_TRACE_REDACT", 1)) { @@ -155,7 +156,7 @@ int cmd_main(int argc, const char **argv) if (packfile) { if (!index_pack_args.nr) - die(_("the option '%s' requires '%s'"), "--packfile", "--index-pack-args"); + die(_("the option '%s' requires '%s'"), "--packfile", "--index-pack-arg"); fetch_single_packfile(&packfile_hash, argv[arg], index_pack_args.v); @@ -164,7 +165,7 @@ int cmd_main(int argc, const char **argv) } if (index_pack_args.nr) - die(_("the option '%s' requires '%s'"), "--index-pack-args", "--packfile"); + die(_("the option '%s' requires '%s'"), "--index-pack-arg", "--packfile"); if (commits_on_stdin) { commits = walker_targets_stdin(&commit_id, &write_ref); diff --git a/http-push.c b/http-push.c index 94a1fac9ab0fcd..786a2e9c0d0546 100644 --- a/http-push.c +++ b/http-push.c @@ -595,7 +595,8 @@ static void finish_request(struct transfer_request *request) } else if (request->state == RUN_FETCH_PACKED) { int fail = 1; - if (request->curl_result != CURLE_OK) { + if (request->curl_result != CURLE_OK && + request->http_code != 416) { fprintf(stderr, "Unable to get pack file %s\n%s", request->url, curl_errorstr); } else { diff --git a/http-walker.c b/http-walker.c index b58a3b2a92be38..abafca84d65441 100644 --- a/http-walker.c +++ b/http-walker.c @@ -451,7 +451,8 @@ static int http_fetch_pack(struct walker *walker, struct alt_base *repo, if (start_active_slot(preq->slot)) { run_active_slot(preq->slot); - if (results.curl_result != CURLE_OK) { + if (results.curl_result != CURLE_OK && + results.http_code != 416) { error("Unable to get pack file %s\n%s", preq->url, curl_errorstr); goto abort; diff --git a/http.c b/http.c index caccf2108e4479..a0d399b2745ae6 100644 --- a/http.c +++ b/http.c @@ -2688,10 +2688,13 @@ int finish_http_pack_request(struct http_pack_request *preq) int tmpfile_fd; int ret = 0; + /* Another downloader may unlink the staging path while we index it. */ + tmpfile_fd = xdup(fileno(preq->packfile)); fclose(preq->packfile); preq->packfile = NULL; - - tmpfile_fd = xopen(preq->tmpfile.buf, O_RDONLY); + if (lseek(tmpfile_fd, 0, SEEK_SET) < 0) + die_errno("unable to seek local file %s for pack", + preq->tmpfile.buf); ip.git_cmd = 1; ip.in = tmpfile_fd; @@ -2704,13 +2707,8 @@ int finish_http_pack_request(struct http_pack_request *preq) else ip.no_stdout = 1; - if (run_command(&ip)) { + if (run_command(&ip)) ret = -1; - goto cleanup; - } - -cleanup: - close(tmpfile_fd); unlink(preq->tmpfile.buf); return ret; } @@ -2738,22 +2736,45 @@ struct http_pack_request *new_http_pack_request( struct http_pack_request *new_direct_http_pack_request( const unsigned char *packed_git_hash, char *url) { - off_t prev_posn = 0; + off_t prev_posn; struct http_pack_request *preq; + int fd; CALLOC_ARRAY(preq, 1); strbuf_init(&preq->tmpfile, 0); - preq->url = url; odb_pack_name(the_repository, &preq->tmpfile, packed_git_hash, "pack"); strbuf_addstr(&preq->tmpfile, ".temp"); - preq->packfile = fopen(preq->tmpfile.buf, "a"); - if (!preq->packfile) { - error("Unable to open local file %s for pack", - preq->tmpfile.buf); + /* + * MinGW's non-append O_RDWR open grants FILE_SHARE_DELETE only for an + * existing file; reopen a newly created file so others may unlink it. + */ + for (;;) { + fd = open(preq->tmpfile.buf, O_RDWR); + if (fd >= 0 || errno != ENOENT) + break; + fd = open(preq->tmpfile.buf, O_RDWR | O_CREAT | O_EXCL, 0666); + if (fd >= 0) { + close(fd); + continue; + } + if (errno != EEXIST) + break; + } + if (fd < 0) { + error_errno("unable to open local file %s for pack", + preq->tmpfile.buf); goto abort; } + prev_posn = lseek(fd, 0, SEEK_END); + if (prev_posn < 0) { + error_errno("unable to seek local file %s for pack", + preq->tmpfile.buf); + close(fd); + goto abort; + } + preq->packfile = xfdopen(fd, "w"); preq->slot = get_active_slot(); preq->headers = object_request_headers(); @@ -2762,12 +2783,7 @@ struct http_pack_request *new_direct_http_pack_request( curl_easy_setopt(preq->slot->curl, CURLOPT_URL, preq->url); curl_easy_setopt(preq->slot->curl, CURLOPT_HTTPHEADER, preq->headers); - /* - * If there is data present from a previous transfer attempt, - * resume where it left off - */ - prev_posn = ftello(preq->packfile); - if (prev_posn>0) { + if (prev_posn > 0) { if (http_is_verbose) fprintf(stderr, "Resuming fetch of pack %s at byte %"PRIuMAX"\n", diff --git a/t/meson.build b/t/meson.build index a25f37d2f5ae7d..65e26a371bd8f3 100644 --- a/t/meson.build +++ b/t/meson.build @@ -639,6 +639,7 @@ integration_tests = [ 't5334-incremental-multi-pack-index.sh', 't5335-compact-multi-pack-index.sh', 't5351-unpack-large-objects.sh', + 't5352-index-pack-hash-pipeline.sh', 't5400-send-pack.sh', 't5401-update-hooks.sh', 't5402-post-merge-hook.sh', @@ -1172,6 +1173,7 @@ benchmarks = [ 'perf/p5326-multi-pack-bitmaps.sh', 'perf/p5332-multi-pack-reuse.sh', 'perf/p5333-pseudo-merge-bitmaps.sh', + 'perf/p5352-index-pack-hash-pipeline.sh', 'perf/p5550-fetch-tags.sh', 'perf/p5551-fetch-rescan.sh', 'perf/p5600-partial-clone.sh', diff --git a/t/perf/p5352-index-pack-hash-pipeline.sh b/t/perf/p5352-index-pack-hash-pipeline.sh new file mode 100755 index 00000000000000..c9af7eafc5e6da --- /dev/null +++ b/t/perf/p5352-index-pack-hash-pipeline.sh @@ -0,0 +1,49 @@ +#!/bin/sh + +test_description='Test index-pack first-pass hash worker performance + +GIT_PERF_5352_NR_BLOBS controls the number of full blobs in the input pack. +GIT_PERF_5352_BLOB_SIZE controls the size of each blob in bytes. +Keep the blob size between 65536 and 67108863 to exercise the worker path. +' + +. ./perf-lib.sh + +test_perf_fresh_repo + +: ${GIT_PERF_5352_NR_BLOBS:=128} +: ${GIT_PERF_5352_BLOB_SIZE:=1048576} + +test_expect_success 'create a pack of full blobs' ' + for i in $(test_seq 1 "$GIT_PERF_5352_NR_BLOBS") + do + test-tool genrandom "index-pack-hash-$i" \ + "$GIT_PERF_5352_BLOB_SIZE" | + git hash-object -w --stdin || return 1 + done >oids && + git pack-objects --stdout --window=0 input.pack +' + +test_size 'pack size' ' + test_file_size input.pack +' + +test_perf 'index-pack, serial hash' \ + --setup 'rm -rf repo.git && git init --bare -q repo.git' ' + git -C repo.git -c indexPack.hashThreads=0 \ + index-pack --threads=1 --stdin /dev/null +' + +test_perf 'index-pack, one hash worker' --prereq PTHREADS \ + --setup 'rm -rf repo.git && git init --bare -q repo.git' ' + git -C repo.git -c indexPack.hashThreads=1 \ + index-pack --threads=1 --stdin /dev/null +' + +test_perf 'index-pack, two hash workers' --prereq PTHREADS \ + --setup 'rm -rf repo.git && git init --bare -q repo.git' ' + git -C repo.git -c indexPack.hashThreads=2 \ + index-pack --threads=1 --stdin /dev/null +' + +test_done diff --git a/t/t5331-pack-objects-stdin.sh b/t/t5331-pack-objects-stdin.sh index c74b5861af322f..2a983e28ac43e7 100755 --- a/t/t5331-pack-objects-stdin.sh +++ b/t/t5331-pack-objects-stdin.sh @@ -368,7 +368,8 @@ test_expect_success '--stdin-packs does not perform backfill fetch' ' git -C remote config set --local uploadpack.allowfilter 1 && git -C remote config set --local uploadpack.allowanysha1inwant 1 && - git clone --filter=tree:0 "file://$(pwd)/remote" client && + git -c maintenance.auto=false clone --filter=tree:0 \ + "file://$(pwd)/remote" client && ( cd client && ls .git/objects/pack/*.promisor | sed "s|.*/||; s/\.promisor$/.pack/" >packs && diff --git a/t/t5352-index-pack-hash-pipeline.sh b/t/t5352-index-pack-hash-pipeline.sh new file mode 100755 index 00000000000000..6b1642c2a7f561 --- /dev/null +++ b/t/t5352-index-pack-hash-pipeline.sh @@ -0,0 +1,177 @@ +#!/bin/sh + +test_description='bounded first-pass index-pack hash workers' + +. ./test-lib.sh +. "$TEST_DIRECTORY"/lib-pack.sh + +run_index () { + name=$1 && + input=$2 && + shift 2 && + git init --bare "$name.git" && + GIT_TRACE2_EVENT="$TRASH_DIRECTORY/$name.trace" \ + git -C "$name.git" "$@" <"$input" >"$name.out" +} + +trace_value () { + key=$1 && + value=$2 && + file=$3 && + test_grep "\"key\":\"first_pass_hash/$key\",\"value\":\"$value\"" "$file" +} + +compare_pack_files () { + left=$1 && + right=$2 && + pack_hash=$(cut -f2 "$left.out") && + test_cmp "$left.out" "$right.out" && + for suffix in pack idx rev + do + test_cmp_bin "$left.git/objects/pack/pack-$pack_hash.$suffix" \ + "$right.git/objects/pack/pack-$pack_hash.$suffix" || return 1 + done +} + +test_expect_success 'make a pack of full blobs' ' + for i in $(test_seq 1 8) + do + test-tool genrandom "hash-pipeline-$i" 65536 >"blob-$i" && + git hash-object -w "blob-$i" || return 1 + done >oids && + git pack-objects --stdout --window=0 input.pack && + run_index serial input.pack index-pack --stdin --fix-thin && + trace_value threads 0 serial.trace +' + +test_expect_success PTHREADS 'workers preserve pack, index and reverse index' ' + run_index parallel input.pack \ + -c indexPack.hashThreads=2 \ + -c indexPack.hashBufferSize=131074 \ + index-pack --stdin --fix-thin && + compare_pack_files serial parallel && + trace_value threads 2 parallel.trace && + trace_value jobs 8 parallel.trace +' + +test_expect_success PTHREADS 'an object must fit including its trailing byte' ' + run_index too-small input.pack \ + -c indexPack.hashThreads=2 \ + -c indexPack.hashBufferSize=65536 \ + -c indexPack.hashMinSize=0 \ + index-pack --stdin --fix-thin && + compare_pack_files serial too-small && + trace_value jobs 0 too-small.trace +' + +test_expect_success PTHREADS 'streamed large blobs retain the serial path' ' + run_index streamed input.pack \ + -c indexPack.hashThreads=2 \ + -c indexPack.hashMinSize=0 \ + -c core.bigFileThreshold=1 \ + index-pack --stdin --fix-thin && + compare_pack_files serial streamed && + trace_value jobs 0 streamed.trace +' + +test_expect_success PTHREADS 'validation modes retain the serial path' ' + for mode in strict fsck-objects promisor + do + run_index "$mode" input.pack \ + -c indexPack.hashThreads=2 \ + -c indexPack.hashMinSize=0 \ + index-pack --stdin --fix-thin "--$mode" && + trace_value threads 0 "$mode.trace" || return 1 + done +' + +test_expect_success PTHREADS 'existing-object collision check is not skipped' ' + git init --bare collision.git && + a=$(git -C collision.git hash-object -w ../blob-1) && + b=$(git -C collision.git hash-object -w ../blob-2) && + a_path=$(echo "$a" | sed "s!^..!&/!") && + b_path=$(echo "$b" | sed "s!^..!&/!") && + cp -f "collision.git/objects/$b_path" "collision.git/objects/$a_path" && + test_env GIT_TRACE2_EVENT="$TRASH_DIRECTORY/collision.trace" \ + test_must_fail git -C collision.git \ + -c indexPack.hashThreads=2 \ + index-pack --stdin --fix-thin collision.err && + trace_value threads 2 collision.trace && + test_grep "SHA1 COLLISION FOUND" collision.err +' + +test_expect_success PTHREADS 'duplicate full blobs preserve native acceptance' ' + { + pack_header 2 && + pack_obj "$EMPTY_BLOB" && + pack_obj "$EMPTY_BLOB" + } >duplicates.pack && + pack_trailer duplicates.pack && + run_index duplicate-serial duplicates.pack index-pack --stdin && + run_index duplicate-parallel duplicates.pack \ + -c indexPack.hashThreads=2 \ + -c indexPack.hashBufferSize=2 \ + -c indexPack.hashMinSize=0 \ + index-pack --stdin && + compare_pack_files duplicate-serial duplicate-parallel && + trace_value jobs 2 duplicate-parallel.trace && + git init --bare duplicate-strict.git && + test_must_fail git -C duplicate-strict.git \ + -c indexPack.hashThreads=2 \ + -c indexPack.hashMinSize=0 \ + index-pack --strict --stdin ref.pack && + pack_trailer ref.pack && + pack_obj "$A" "$B" >ref-entry && + { + pack_header 2 && + pack_obj "$B" && + # The full B entry is eleven bytes; the delta data is five. + printf "\145\013" && + dd if=ref-entry bs=1 skip=$((1 + $(test_oid rawsz))) + } >ofs.pack && + pack_trailer ofs.pack && + for kind in ref ofs + do + run_index "$kind" "$kind.pack" \ + -c indexPack.hashThreads=2 \ + -c indexPack.hashMinSize=0 \ + index-pack --stdin --fix-thin && + trace_value jobs 1 "$kind.trace" && + git -C "$kind.git" cat-file blob "$A" >actual && + test "$(git hash-object actual)" = "$A" || return 1 + done +' + +test_expect_success PTHREADS 'bad input cannot publish an index' ' + length=$(wc -c actual && + test_must_be_empty actual || return 1 + done +' + +test_expect_success 'invalid worker counts are rejected' ' + test_must_fail git -c indexPack.hashThreads=-1 index-pack input.pack && + test_must_fail git -c indexPack.hashThreads=33 index-pack input.pack +' + +test_done diff --git a/t/t5550-http-fetch-dumb.sh b/t/t5550-http-fetch-dumb.sh index f00eeae48f8554..b5758f1c9ca1c2 100755 --- a/t/t5550-http-fetch-dumb.sh +++ b/t/t5550-http-fetch-dumb.sh @@ -293,6 +293,210 @@ test_expect_success 'http-fetch --packfile' ' git -C packfileclient cat-file -e "$HASH" ' +test_expect_success 'http-fetch --packfile accepts an already complete partial' ' + git init packfileclient-complete && + p=$(cd "$HTTPD_DOCUMENT_ROOT_PATH"/repo_pack.git && + ls objects/pack/pack-*.pack) && + packhash=$(basename "$p" .pack) && + packhash=${packhash#pack-} && + tmpfile="packfileclient-complete/.git/objects/pack/pack-$packhash.pack.temp" && + cp "$HTTPD_DOCUMENT_ROOT_PATH/repo_pack.git/$p" "$tmpfile" && + chmod u+w "$tmpfile" && + GIT_TRACE_CURL="$TRASH_DIRECTORY/complete.trace" \ + git -C packfileclient-complete http-fetch --packfile="$packhash" \ + --index-pack-arg=index-pack \ + --index-pack-arg=--stdin --index-pack-arg=--keep \ + "$HTTPD_URL/dumb/repo_pack.git/$p" >out && + test_grep "416 Requested Range Not Satisfiable" complete.trace && + test_path_is_missing "$tmpfile" && + git -C packfileclient-complete cat-file -e "$HASH" +' + +test_expect_success 'http-fetch --packfile resumes a partial download' ' + git init packfileclient-resume && + p=$(cd "$HTTPD_DOCUMENT_ROOT_PATH"/repo_pack.git && + ls objects/pack/pack-*.pack) && + tmpfile="packfileclient-resume/.git/objects/pack/pack-$ARBITRARY.pack.temp" && + test_copy_bytes 64 <"$HTTPD_DOCUMENT_ROOT_PATH/repo_pack.git/$p" >"$tmpfile" && + GIT_TRACE_CURL="$TRASH_DIRECTORY/resume.trace" \ + git -C packfileclient-resume http-fetch --packfile="$ARBITRARY" \ + --index-pack-arg=index-pack --index-pack-arg=--stdin \ + --index-pack-arg=--keep \ + "$HTTPD_URL/dumb/repo_pack.git/$p" >out && + test_grep "Range: bytes=64-" resume.trace && + test_path_is_missing "$tmpfile" && + git -C packfileclient-resume cat-file -e "$HASH" +' + +test_expect_success 'http-fetch --packfile permits unlink while indexing' ' + git init packfileclient-unlink && + p=$(cd "$HTTPD_DOCUMENT_ROOT_PATH"/repo_pack.git && + ls objects/pack/pack-*.pack) && + tmpfile="packfileclient-unlink/.git/objects/pack/pack-$ARBITRARY.pack.temp" && + write_script git-unlink-index-pack <<-\EOF && + test -f "$GIT_TEST_PACK_TEMP" || exit 1 + rm "$GIT_TEST_PACK_TEMP" || exit 1 + exec git index-pack "$@" + EOF + test_when_finished "rm -f git-unlink-index-pack" && + PATH="$TRASH_DIRECTORY:$PATH" \ + GIT_TEST_PACK_TEMP="$TRASH_DIRECTORY/$tmpfile" \ + git -C packfileclient-unlink http-fetch --packfile="$ARBITRARY" \ + --index-pack-arg=unlink-index-pack \ + --index-pack-arg=--stdin --index-pack-arg=--keep \ + "$HTTPD_URL/dumb/repo_pack.git/$p" >out && + test_path_is_missing "$tmpfile" && + git -C packfileclient-unlink cat-file -e "$HASH" +' + +test_expect_success PERL,PIPE 'concurrent http-fetch --packfile cannot corrupt an overlapping download' ' + git init packfileclient-overlap && + blob=$(test-tool genrandom pack-overlap 2m | + git -C "$HTTPD_DOCUMENT_ROOT_PATH"/repo_pack.git \ + hash-object -w --stdin) && + packhash=$(printf "%s\n" "$blob" | + git -C "$HTTPD_DOCUMENT_ROOT_PATH"/repo_pack.git \ + pack-objects "$TRASH_DIRECTORY/overlap-pack") && + pack="$TRASH_DIRECTORY/overlap-pack-$packhash.pack" && + tmpfile="packfileclient-overlap/.git/objects/pack/pack-$packhash.pack.temp" && + mkfifo server-ready first-ready && + exec 7<>server-ready && + exec 8<>first-ready && + write_script slow-pack-server "$PERL_PATH" <<-\EOF && + use strict; + use warnings; + use IO::Socket::INET; + + my ($packfile, $server_ready, $first_ready) = @ARGV; + my $completed = 0; + END { + if (!$completed) { + signal_ready($server_ready, "failed"); + signal_ready($first_ready, "failed"); + } + } + + $SIG{ALRM} = sub { die "timed out serving concurrent pack requests\n" }; + alarm 60; + + open(my $in, "<:raw", $packfile) or die "open $packfile: $!"; + my $pack = do { local $/; <$in> }; + close($in) or die "close $packfile: $!"; + my $server = IO::Socket::INET->new(LocalAddr => "127.0.0.1", + LocalPort => 0, Proto => "tcp", Listen => 2, ReuseAddr => 1) + or die "listen: $!"; + + sub signal_ready { + my ($file, $value) = @_; + open(my $out, ">", $file) or die "open $file: $!"; + print $out "$value\n" or die "write $file: $!"; + close($out) or die "close $file: $!"; + } + + sub write_all { + my ($out, $data) = @_; + my $offset = 0; + while ($offset < length($data)) { + my $written = syswrite($out, $data, + length($data) - $offset, $offset); + defined($written) && $written or die "write response: $!"; + $offset += $written; + } + } + + sub start_response { + my $out = $server->accept() or die "accept: $!"; + <$out> or die "read request: $!"; + my $start = 0; + while (<$out>) { + last if /^\r?\n$/; + $start = $1 if /^Range: bytes=(\d+)-/i; + } + $start < length($pack) or die "invalid range $start"; + my $length = length($pack) - $start; + my $middle = int($length / 2); + my $status = $start ? "206 Partial Content" : "200 OK"; + my $headers = "HTTP/1.1 $status\r\n" . + "Content-Length: $length\r\n" . + ($start ? "Content-Range: bytes $start-" . + (length($pack) - 1) . "/" . length($pack) . "\r\n" : "") . + "Connection: close\r\n\r\n"; + write_all($out, $headers); + write_all($out, substr($pack, $start, $middle)); + return ($out, $start + $middle); + } + + signal_ready($server_ready, $server->sockport()); + my ($first, $first_pos) = start_response(); + signal_ready($first_ready, "ready"); + my ($second, $second_pos) = start_response(); + write_all($first, substr($pack, $first_pos)); + write_all($second, substr($pack, $second_pos)); + close($first) or die "close first response: $!"; + close($second) or die "close second response: $!"; + $completed = 1; + alarm 0; + EOF + { + "$TRASH_DIRECTORY/slow-pack-server" "$pack" \ + "$TRASH_DIRECTORY/server-ready" \ + "$TRASH_DIRECTORY/first-ready" >server.log 2>&1 & + server_pid=$! + } && + test_when_finished " + kill $server_pid 2>/dev/null || : + wait $server_pid 2>/dev/null || : + exec 7>&- + exec 8>&- + rm -f server-ready first-ready slow-pack-server + " && + read port <&7 && + url="http://127.0.0.1:$port/pack" && + { + ( + if ! GIT_TRACE_CURL="$TRASH_DIRECTORY/overlap-first.trace" \ + GIT_TRACE_CURL_NO_DATA=1 \ + git -C packfileclient-overlap http-fetch --packfile="$packhash" \ + --index-pack-arg=index-pack \ + --index-pack-arg=--stdin --index-pack-arg=--keep \ + "$url" >first.out + then + echo failed >"$TRASH_DIRECTORY/first-ready" && + exit 1 + fi + ) & + first_pid=$! + } && + test_when_finished " + kill $first_pid 2>/dev/null || : + wait $first_pid 2>/dev/null || : + " && + read ready <&8 && + test "$ready" = ready && + test_path_is_file "$tmpfile" && + { + GIT_TRACE_CURL="$TRASH_DIRECTORY/overlap-second.trace" \ + GIT_TRACE_CURL_NO_DATA=1 \ + git -C packfileclient-overlap http-fetch --packfile="$packhash" \ + --index-pack-arg=index-pack \ + --index-pack-arg=--stdin --index-pack-arg=--keep \ + "$url" >second.out & + second_pid=$! + } && + test_when_finished " + kill $second_pid 2>/dev/null || : + wait $second_pid 2>/dev/null || : + " && + wait "$second_pid" && + wait "$first_pid" && + wait "$server_pid" && + printf "keep\t%s\npack\t%s\n" "$packhash" "$packhash" | sort >expect && + sort first.out second.out >actual && + test_cmp expect actual && + test_path_is_missing "$tmpfile" && + git -C packfileclient-overlap cat-file -e "$blob" +' + test_expect_success 'fetch notices corrupt pack' ' cp -R "$HTTPD_DOCUMENT_ROOT_PATH"/repo_pack.git "$HTTPD_DOCUMENT_ROOT_PATH"/repo_bad1.git && (cd "$HTTPD_DOCUMENT_ROOT_PATH"/repo_bad1.git && diff --git a/t/t5702-protocol-v2.sh b/t/t5702-protocol-v2.sh index 74a2b7730bf3da..0f05286de8b4df 100755 --- a/t/t5702-protocol-v2.sh +++ b/t/t5702-protocol-v2.sh @@ -1291,6 +1291,37 @@ test_expect_success 'packfile URIs with fetch instead of clone' ' fetch "$HTTPD_URL/smart/http_parent" ' +test_expect_success 'packfile URI preserves an existing keep file' ' + P="$HTTPD_DOCUMENT_ROOT_PATH/http_parent" && + rm -rf "$P" http_child keep.expect && + + git init "$P" && + git -C "$P" config uploadpack.allowsidebandall true && + + echo my-blob >"$P/my-blob" && + git -C "$P" add my-blob && + git -C "$P" commit -m x && + configure_exclusion "$P" my-blob >h && + + git init http_child && + packhash=$(cat packh) && + keep="http_child/.git/objects/pack/pack-$packhash.keep" && + echo pre-existing >"$keep" && + cp "$keep" keep.expect && + + GIT_TEST_SIDEBAND_ALL=1 \ + git -C http_child -c protocol.version=2 \ + -c fetch.uriprotocols=http,https \ + fetch "$HTTPD_URL/smart/http_parent" && + + test_path_is_file \ + "http_child/.git/objects/pack/pack-$packhash.pack" && + test_path_is_file \ + "http_child/.git/objects/pack/pack-$packhash.idx" && + test_cmp keep.expect "$keep" && + git -C http_child cat-file -e "$(cat h)" +' + test_expect_success 'fetching with valid packfile URI but invalid hash fails' ' P="$HTTPD_DOCUMENT_ROOT_PATH/http_parent" && rm -rf "$P" http_child log && diff --git a/t/t7900-maintenance.sh b/t/t7900-maintenance.sh index a8d691719da062..fb5f2d89028738 100755 --- a/t/t7900-maintenance.sh +++ b/t/t7900-maintenance.sh @@ -659,6 +659,51 @@ test_expect_success 'geometric repacking task' ' ) ' +objdir=.git/objects +packdir=$objdir/pack + +pack_promisor () { + p="$(echo "$@" | git pack-objects --revs $packdir/pack)" && + touch "$packdir/pack-$p.promisor" && + echo "$p" +} + +test_expect_success 'geometric repacking task handles promisor packs' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + git config set maintenance.auto false && + git remote add promisor garbage && + git config set remote.promisor.promisor true && + + for n in $(test_seq 6) + do + test_commit $n || return 1 + done && + + A="$(pack_promisor 1)" && + B="$(pack_promisor 1..2)" && + C="$(pack_promisor 2..6)" && + git prune-packed && + + ls $packdir/pack-*.promisor | sort >promisors.before && + GIT_TRACE2_EVENT="$(pwd)/trace2.txt" \ + git maintenance run --quiet --task=geometric-repack && + ls $packdir/pack-*.promisor | sort >promisors.after && + + test_subcommand git repack -d -l --geometric=2 \ + --quiet --write-midx expect && + comm -23 promisors.before promisors.after >actual && + + test_cmp expect actual + ) +' + test_geometric_repack_needed () { NEEDED="$1" GEOMETRIC_CONFIG="$2" && @@ -714,6 +759,29 @@ test_expect_success 'geometric repacking with --auto' ' ) ' +test_expect_success 'geometric repacking with --auto handles promisor packs' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + git config set maintenance.auto false && + git remote add promisor garbage && + git config set remote.promisor.promisor true && + + for n in $(test_seq 6) + do + test_commit $n || return 1 + done && + + pack_promisor 1 >/dev/null && + pack_promisor 1..2 >/dev/null && + pack_promisor 2..6 >/dev/null && + git prune-packed && + + test_geometric_repack_needed true auto=9000 + ) +' + test_expect_success 'geometric repacking honors configured split factor' ' test_when_finished "rm -rf repo" && git init repo &&