CIHelper: optionally clone the Git mailing list mirror

A couple of GitGitGadget's common operations require a local clone of
the Git mailing list mirror.

For obvious reasons, the idea here is to initialize a partial, shallow
clone, so that missing objects (if any) will be retrieved on demand, and
no time is wasted on a full clone every time GitGitGadget wants to look
at mails.

The not-so-obvious idea is to un-shallow the clone using the same
strategy (and in fact, the same function) as for the Git "worktree":
When I tried to simply delete the `shallow` file during the development
of this patch, assuming that Git's partial clone functionality would
gracefully fetch any missing commit e.g. when computing a merge base,
reality taught me the harsh lesson that it would fail hard in such
scenarios. Instead, the `git fetch --unshallow` command is called with
`--filter=tree:0` so that _only_ the missing commits (and all of them)
are fetched. This still seems to strike a good compromise between saving
time on the clone and saving on network round-trips when fetching
missing Git objects.

Signed-off-by: Johannes Schindelin <johannes.schindelin@gmx.de>

Author: dscho

.vscode/settings.json

@@ -96,6 +96,7 @@
         "CIsmtphost",
         "CIsmtppass",
         "CIsmtpopts",
+        "mailrepo",
         "parsePRURLInput",
         "users\\.noreply\\.github\\.com",
     ],

lib/ci-helper.ts

@@ -46,6 +46,7 @@ export class CIHelper {
     private notesPushToken: string | undefined;
     private smtpOptions?: ISMTPOptions;
     protected maxCommitsExceptions: string[];
+    protected mailingListMirror: string | undefined;
 
     public static async getConfig(configFile?: string): Promise<IConfig> {
         return configFile ? await getExternalConfig(configFile) : getConfig();
@@ -66,7 +67,7 @@ export class CIHelper {
         this.urlRepo = `${this.urlBase}${this.config.repo.name}/`;
     }
 
-    public async setupGitHubAction(): Promise<void> {
+    public async setupGitHubAction(setupOptions?: { needsMailingListMirror?: boolean }): Promise<void> {
         // help dugite realize where `git` is...
         const gitExecutable = os.type() === "Windows_NT" ? "git.exe" : "git";
         const stripSuffix = `bin${path.sep}${gitExecutable}`;
@@ -150,6 +151,58 @@ export class CIHelper {
             console.timeEnd(`Making ${workDir} non-shallow`);
         };
         await unshallow(this.workDir);
+
+        if (setupOptions?.needsMailingListMirror) {
+            this.mailingListMirror = "mailing-list-mirror.git";
+            const epoch = this.config.mailrepo.public_inbox_epoch ?? 1;
+
+            // eslint-disable-next-line security/detect-non-literal-fs-filename
+            if (!fs.existsSync(this.mailingListMirror)) {
+                await git(["init", "--bare", "--initial-branch", this.config.mailrepo.branch, this.mailingListMirror]);
+            }
+
+            // First fetch from GitGitGadget's mirror, which supports partial clones
+            for (const [key, value] of [
+                ["remote.mirror.url", this.config.mailrepo.mirrorURL || this.config.mailrepo.url],
+                ["remote.mirror.promisor", "true"],
+                ["remote.mirror.partialCloneFilter", "blob:none"],
+            ]) {
+                await git(["config", key, value], { workDir: this.mailingListMirror });
+            }
+            console.time("fetch mailing list mirror");
+            await git(
+                [
+                    "-c",
+                    "remote.mirror.promisor=false", // let's fetch the mails with all of their contents
+                    "fetch",
+                    "mirror",
+                    "--depth=50",
+                    "+REF:REF".replace("REF", this.config.mailrepo.mirrorRef || `refs/heads/lore-${epoch}`),
+                ],
+                {
+                    workDir: this.mailingListMirror,
+                },
+            );
+            console.timeEnd("fetch mailing list mirror");
+
+            // Now update the head branch from the authoritative repository
+            console.time(`update from ${this.config.mailrepo.url}`);
+            await git(["config", "remote.origin.url", `${this.config.mailrepo.url.replace(/\/*$/, "")}/${epoch}`], {
+                workDir: this.mailingListMirror,
+            });
+            await git(
+                [
+                    "fetch",
+                    "origin",
+                    `+refs/heads/${this.config.mailrepo.branch}:refs/heads/${this.config.mailrepo.branch}`,
+                ],
+                {
+                    workDir: this.mailingListMirror,
+                },
+            );
+            console.timeEnd(`update from ${this.config.mailrepo.url}`);
+            await unshallow(this.mailingListMirror);
+        }
     }
 
     public parsePRCommentURLInput(): { owner: string; repo: string; prNumber: number; commentId: number } {
@@ -938,7 +991,13 @@ export class CIHelper {
         }
     }
 
-    public async handleNewMails(mailArchiveGitDir: string, onlyPRs?: Set<number>): Promise<boolean> {
+    public async handleNewMails(mailArchiveGitDir?: string, onlyPRs?: Set<number>): Promise<boolean> {
+        if (!mailArchiveGitDir) {
+            mailArchiveGitDir = this.mailingListMirror;
+            if (!mailArchiveGitDir) {
+                throw new Error("No mail archive directory specified (forgot to run `setupGitHubAction()`?)");
+            }
+        }
         await git(["fetch"], { workDir: mailArchiveGitDir });
         const prFilter = !onlyPRs
             ? undefined

lib/gitgitgadget-config.ts

@@ -19,6 +19,9 @@ const defaultConfig: IConfig = {
         branch: "master",
         host: "lore.kernel.org",
         url: "https://lore.kernel.org/git/",
+        public_inbox_epoch: 1,
+        mirrorURL: "https://github.com/gitgitgadget/git-mailing-list-mirror",
+        mirrorRef: "refs/heads/lore-1",
         descriptiveName: "lore.kernel/git",
     },
     mail: {

lib/project-config.ts

@@ -27,6 +27,9 @@ export interface IConfig {
         branch: string;
         host: string;
         url: string;
+        public_inbox_epoch?: number;
+        mirrorURL?: string;
+        mirrorRef?: string;
         descriptiveName: string;
     };
     mail: {