index-pack: hash full blobs in a bounded worker pool

.github/workflows/codex-release.yml

@@ -200,8 +200,8 @@ jobs:
       - name: Check out Dugite Native
         uses: actions/checkout@v6
         with:
-          repository: desktop/dugite-native
-          ref: f97e50add48cdcff053a69d95aa343a4a4a258c2
+          repository: dreynaud-oai/dugite-native
+          ref: b6f4473557acb85433fdf9deffe0854a34fd9cc5
           path: dugite-native
           fetch-depth: 0
           persist-credentials: false

.github/workflows/codex.yml

@@ -0,0 +1,243 @@
+name: Refresh codex
+
+on:
+  schedule:
+    - cron: '*/5 * * * *'
+  workflow_dispatch:
+    inputs:
+      operation:
+        description: Refresh, scan, remove, or reorder a pinned topic
+        type: choice
+        options:
+          - refresh
+          - scan
+          - remove
+          - reorder
+        default: refresh
+      lane:
+        description: codex or codex-unstable for a plan operation
+        required: false
+        type: string
+      topic:
+        description: Exact topic branch for a plan operation
+        required: false
+        type: string
+      after:
+        description: Existing topic or root for reorder
+        required: false
+        type: string
+      plan_branch:
+        description: Optional codex-plan/* branch name
+        required: false
+        type: string
+  pull_request_target:
+    branches:
+      - meta
+    types:
+      - opened
+      - reopened
+      - synchronize
+      - ready_for_review
+
+permissions:
+  actions: read
+  contents: read
+  pull-requests: read
+
+jobs:
+  refresh:
+    if: >-
+      github.event_name == 'workflow_dispatch' &&
+      github.ref == 'refs/heads/codex' &&
+      inputs.operation == 'refresh'
+    uses: openai/git/.github/workflows/codex.yml@meta
+  topic_plan_scan:
+    name: Find one approved topic plan
+    if: >-
+      github.event_name == 'schedule' ||
+      (github.event_name == 'workflow_dispatch' &&
+       github.ref == 'refs/heads/codex' &&
+       inputs.operation == 'scan')
+    runs-on: ubuntu-24.04
+    permissions:
+      contents: read
+      pull-requests: read
+    concurrency:
+      group: codex-topic-plan-scan
+      cancel-in-progress: false
+    outputs:
+      lane: ${{ steps.reviewed.outputs.lane }}
+      topic: ${{ steps.reviewed.outputs.topic }}
+      source_tip: ${{ steps.reviewed.outputs.source_tip }}
+      review_pr: ${{ steps.reviewed.outputs.review_pr }}
+    env:
+      GH_TOKEN: ${{ github.token }}
+    steps:
+      - name: Pin trusted meta
+        id: meta
+        run: |
+          set -euo pipefail
+          test "$GITHUB_REPOSITORY" = openai/git
+          test "$GITHUB_REF" = refs/heads/codex
+          sha=$(gh api "repos/$GITHUB_REPOSITORY/git/ref/heads/meta" \
+            --jq .object.sha)
+          case "$sha" in
+          ''|*[!0-9a-f]*) exit 1 ;;
+          esac
+          test "${#sha}" = 40
+          printf 'sha=%s\n' "$sha" >>"$GITHUB_OUTPUT"
+
+      - name: Check out trusted meta
+        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
+        with:
+          repository: ${{ github.repository }}
+          ref: ${{ steps.meta.outputs.sha }}
+          fetch-depth: 0
+          persist-credentials: false
+
+      - name: Find one exact approved topic PR
+        id: reviewed
+        env:
+          META_SHA: ${{ steps.meta.outputs.sha }}
+        run: |
+          set -euo pipefail
+
+          die () {
+            printf '%s\n' "$*" >&2
+            exit 1
+          }
+
+          test "$GITHUB_REPOSITORY" = openai/git
+          test "$GITHUB_REF" = refs/heads/codex ||
+            die "topic scan must run from the trusted default branch"
+          test "$(git rev-parse HEAD)" = "$META_SHA" ||
+            die "trusted checkout does not match pinned meta"
+          gh auth setup-git
+          mkdir -p "$RUNNER_TEMP/codex-plan-scan"
+          for lane in codex codex-unstable
+          do
+            case "$lane" in
+            codex) plan=codex.plan ;;
+            codex-unstable) plan=codex-unstable.plan ;;
+            esac
+            test -f "$plan" ||
+              die "trusted meta has no $plan"
+            gh pr list --repo "$GITHUB_REPOSITORY" --state open \
+              --base "$lane" --limit 1000 \
+              --json number,isDraft,headRefName,headRefOid,headRepository,reviewDecision |
+              jq -r --arg lane "$lane" '
+                .[] |
+                select(.isDraft | not) |
+                select(.reviewDecision == "APPROVED") |
+                select(.headRepository.nameWithOwner == "openai/git") |
+                [$lane, .headRefName, .headRefOid,
+                 (.number | tostring)] | @tsv
+              '
+          done | sort -k4,4n >"$RUNNER_TEMP/codex-plan-scan/candidates"
+
+          while IFS=$'\t' read -r lane topic source_tip review_pr
+          do
+            test -n "$review_pr" || continue
+            case "$review_pr" in
+            *[!0-9]*) die "approved topic PR has invalid number '$review_pr'" ;;
+            esac
+            case "$source_tip" in
+            *[!0-9a-f]*|'') die "approved topic PR has invalid source SHA" ;;
+            esac
+            test "${#source_tip}" = 40 ||
+              die "approved topic PR has invalid source SHA"
+            git check-ref-format "refs/heads/$topic" >/dev/null 2>&1 ||
+              die "approved topic PR has invalid branch '$topic'"
+            case "$topic" in
+            ??/codex/*) ;;
+            *) continue ;;
+            esac
+            suffix=${topic#??/codex/}
+            case "$suffix" in
+            ''|*/*|*-wip|*-stale) continue ;;
+            esac
+            case "$lane" in
+            codex)
+              case "$topic" in
+              *-unstable) continue ;;
+              esac
+              plan=codex.plan
+              ;;
+            codex-unstable)
+              case "$topic" in
+              *-unstable) ;;
+              *) continue ;;
+              esac
+              plan=codex-unstable.plan
+              ;;
+            *) die "approved topic PR has invalid lane '$lane'" ;;
+            esac
+            pinned=$(git config --no-includes \
+              --file "$plan" \
+              --get "branch.$topic.source-tip" || :)
+            test "$pinned" = "$source_tip" && continue
+            short=$(printf '%.12s' "$source_tip")
+            slug=${topic##*/}
+            plan_branch=codex-plan/$lane-$slug-$short
+            pending=$(gh pr list --repo "$GITHUB_REPOSITORY" \
+              --state open --base meta --head "$plan_branch" \
+              --json number --jq '.[0].number // empty') ||
+              die "could not inspect pending Codex plan PR"
+            test -n "$pending" && continue
+            if ! sh .github/workflows/codex-branch.sh propose-plan \
+              --remote origin --lane "$lane" --topic "$topic" \
+              --action auto --source-tip "$source_tip" \
+              --review-pr "$review_pr" --expected-meta "$META_SHA" \
+              --no-push >/dev/null
+            then
+              printf 'skipping approved topic PR #%s: preflight failed\n' \
+                "$review_pr" >&2
+              continue
+            fi
+            {
+              printf 'lane=%s\n' "$lane"
+              printf 'topic=%s\n' "$topic"
+              printf 'source_tip=%s\n' "$source_tip"
+              printf 'review_pr=%s\n' "$review_pr"
+            } >>"$GITHUB_OUTPUT"
+            exit 0
+          done <"$RUNNER_TEMP/codex-plan-scan/candidates"
+  topic_plan_propose:
+    name: Propose reviewed topic plan
+    needs: topic_plan_scan
+    if: needs.topic_plan_scan.outputs.review_pr != ''
+    permissions:
+      contents: read
+      pull-requests: read
+    uses: openai/git/.github/workflows/codex-plan-propose.yml@meta
+    with:
+      lane: ${{ needs.topic_plan_scan.outputs.lane }}
+      topic: ${{ needs.topic_plan_scan.outputs.topic }}
+      action: auto
+      source_tip: ${{ needs.topic_plan_scan.outputs.source_tip }}
+      review_pr: ${{ needs.topic_plan_scan.outputs.review_pr }}
+  policy_plan_propose:
+    name: Propose explicit plan policy
+    if: >-
+      github.event_name == 'workflow_dispatch' &&
+      github.ref == 'refs/heads/codex' &&
+      (inputs.operation == 'remove' || inputs.operation == 'reorder')
+    permissions:
+      contents: read
+      pull-requests: read
+    uses: openai/git/.github/workflows/codex-plan-propose.yml@meta
+    with:
+      lane: ${{ inputs.lane }}
+      topic: ${{ inputs.topic }}
+      action: ${{ inputs.operation }}
+      after: ${{ inputs.after }}
+      plan_branch: ${{ inputs.plan_branch }}
+  plan_admission:
+    name: Codex plan admission
+    if: >-
+      github.event_name == 'pull_request_target' &&
+      github.event.pull_request.base.ref == 'meta'
+    permissions:
+      contents: read
+      pull-requests: write
+    uses: openai/git/.github/workflows/codex-plan-admission.yml@meta

Documentation/config.adoc

@@ -518,6 +518,8 @@ include::config/includeif.adoc[]
 
 include::config/index.adoc[]
 
+include::config/indexpack.adoc[]
+
 include::config/init.adoc[]
 
 include::config/instaweb.adoc[]

Documentation/config/indexpack.adoc

@@ -0,0 +1,21 @@
+indexPack.hashThreads::
+	Experimental number of workers used to hash full blobs during the
+	first pass of linkgit:git-index-pack[1]. The default, zero, disables
+	the workers. Values from 1 through 32 are accepted. Parsing and
+	inflation remain serial, and normal collision/content validation runs
+	on the main thread. This is independent of `pack.threads` and of
+	concurrent packfile-URI downloads. Strict, fsck, and promisor modes,
+	and builds without thread support, use the existing serial path.
+
+indexPack.hashBufferSize::
+	Maximum retained blob-buffer bytes for `indexPack.hashThreads`,
+	including the terminating byte, the producer's reserved buffer, and
+	completed results awaiting validation. Defaults to 64 MiB. At most
+	twice the number of hash workers can be outstanding. Objects that do
+	not fit use the existing serial path; the limit is not a bound on
+	Git's total memory use. Usual `k`, `m`, and `g` suffixes are accepted.
+
+indexPack.hashMinSize::
+	Minimum full-blob size eligible for `indexPack.hashThreads`.
+	Defaults to 64 KiB. Blobs above `core.bigFileThreshold` retain their
+	existing streaming path. Usual `k`, `m`, and `g` suffixes are accepted.

Documentation/git-http-fetch.adoc

@@ -48,13 +48,15 @@ commit-id::
 	line (which is not expected in
 	this case), 'git http-fetch' fetches the packfile directly at the given
 	URL and uses index-pack to generate corresponding .idx and .keep files.
-	The hash is used to determine the name of the temporary file and is
-	arbitrary. The output of index-pack is printed to stdout. Requires
-	--index-pack-args.
+	The hash is used to determine the name of the temporary file. It need
+	not be the pack hash, but it must uniquely identify the pack contents
+	for resumption. The output of index-pack is printed to stdout. Requires
+	one or more --index-pack-arg options.
 
---index-pack-args=<args>::
-	For internal use only. The command to run on the contents of the
-	downloaded pack. Arguments are URL-encoded separated by spaces.
+--index-pack-arg=<arg>::
+	For internal use only. The first instance specifies the command run on
+	the contents of the downloaded pack. Subsequent instances specify its
+	arguments.
 
 --recover::
 	Verify that everything reachable from target is fetched.  Used after

Documentation/git-index-pack.adoc

@@ -148,6 +148,11 @@ accessible through promisor objects.
 +
 Requires <pack-file> to not be specified.
 
+CONFIGURATION
+-------------
+
+include::config/indexpack.adoc[]
+
 NOTES
 -----
 

builtin/gc.c

@@ -1593,7 +1593,8 @@ static int maintenance_task_geometric_repack(struct maintenance_run_opts *opts,
 	child.odb_to_close = the_repository->objects;
 
 	strvec_pushl(&child.args, "repack", "-d", "-l", NULL);
-	if (geometry.split < geometry.pack_nr)
+	if (geometry.split < geometry.pack_nr ||
+	    geometry.promisor_split < geometry.promisor_pack_nr)
 		strvec_pushf(&child.args, "--geometric=%d",
 			     geometry.split_factor);
 	else
@@ -1648,7 +1649,7 @@ static int geometric_repack_auto_condition(struct gc_config *cfg UNUSED)
 	 * When we'd merge at least two packs with one another we always
 	 * perform the repack.
 	 */
-	if (geometry.split) {
+	if (geometry.split || geometry.promisor_split) {
 		ret = 1;
 		goto out;
 	}