REP-6882 Track mismatch duration (#189)

Verifier historically considered all mismatches coequally significant. For logging this is unhelpful: ideally users should see those mismatches that are recurring in successive generations. This changeset takes a step toward that by tracking how long a given mismatch has appeared in Verifier checks and surfacing the “longest-lived” mismatches in the log.

To do this, two pieces of information are now tracked for a mismatch:
- when it was first seen
- how long the most _recent_ occurrence happened after that first sight (i.e., the mismatch’s “duration”)

These get saved into both the mismatch (for logging purposes). The recheck entry tracks just the first-seen time.

When Verifier creates a new task from recheck entries, that task also contains the relevant document’s first-mismatch time. _If any non-mismatch rechecks exist for the document, the mismatch time is omitted from the task._ This is because those non-mismatch rechecks will have come from change events, which means the document has changed, so any further mismatch will be a “new” one. (In this case, of course, hopefully the change event means the mismatch is now fixed!)

If, on recheck, the document is still mismatched, the mismatch’s duration is recomputed (i.e., the time since the mismatch’s first-seen time), and that new duration is saved with the new recheck entry. (If there’s no mismatch, then there’s no recheck.)

Finally: when logging mismatches, Verifier now also shows the mismatch’s duration and sorts the displayed mismatches longest-duration-first. To offset this additional “noise” in the logs, Verifier no longer shows destination namespaces in document mismatch logs.

This changeset also includes some tangential-but-opportune logging improvements:
- BUG FIX: Verifier no longer tells people to query the tasks collection. Instead, the status line refers to the documentation (which contains an aggregation command for this).
- BUG FIX: Verifier no longer logs context cancellation errors when its logging is interrupted.
- Logs now distinguish missing-on-dst documents from extra-on-dst. Previously the logs labeled both scenarios as “missing”, which was always confusing.
- Log entries’ final status lines are now more nuanced. If all shown logs are 0s in duration, for example, the status line no longer says simply, “Mismatches found”. Instead Verifier now says that all mismatches are “new” and will (hopefully) be resolved.
- Some headers are now abbreviated & made more consistent.

Ancillary, implementation-level changes:
- getDocIdFromComparison() now asserts rather than ignoring errors. (It should never fail.)
- This includes a new library to make writing MongoDB aggregations far more ergonomic in Go.

Author: FGasper

README.md

@@ -156,7 +156,7 @@ The verifier will now check to completion to make sure that there are no inconsi
 
 # Investigation of Mismatches
 
-The verifier records any mismatches it finds in its metadata’s `mismatches`
+The verifier records mismatches in its metadata’s `mismatches`
 collection. Mismatches are indexed by verification task ID. To find a given
 generation’s mismatches, aggregate like this on the metadata cluster:
 
@@ -166,7 +166,7 @@ generation’s mismatches, aggregate like this on the metadata cluster:
     db.verification_tasks.aggregate(
         { $match: {
             generation: <whichever generation>,
-            status: "failed",
+            status: {$in: ["failed", "mismatch"]},
         } },
         { $lookup: {
             from: "mismatches",

agg/accum/accumulators.go

@@ -0,0 +1,70 @@
+// Package accum exposes helper types for accumulation operators.
+package accum
+
+import "go.mongodb.org/mongo-driver/v2/bson"
+
+type Sum [1]any
+
+var _ bson.Marshaler = Sum{}
+
+func (s Sum) MarshalBSON() ([]byte, error) {
+	return bson.Marshal(bson.D{{"$sum", s[0]}})
+}
+
+//----------------------------------------------------------------------
+
+type Push [1]any
+
+var _ bson.Marshaler = Push{}
+
+func (p Push) MarshalBSON() ([]byte, error) {
+	return bson.Marshal(bson.D{{"$push", p[0]}})
+}
+
+//----------------------------------------------------------------------
+
+type Max [1]any
+
+var _ bson.Marshaler = Max{}
+
+func (m Max) MarshalBSON() ([]byte, error) {
+	return bson.Marshal(bson.D{{"$max", m[0]}})
+}
+
+//----------------------------------------------------------------------
+
+type FirstN struct {
+	N     any
+	Input any
+}
+
+var _ bson.Marshaler = FirstN{}
+
+func (t FirstN) MarshalBSON() ([]byte, error) {
+	return bson.Marshal(bson.D{
+		{"$firstN", bson.D{
+			{"n", t.N},
+			{"input", t.Input},
+		}},
+	})
+}
+
+//----------------------------------------------------------------------
+
+type TopN struct {
+	N      any
+	SortBy bson.D
+	Output any
+}
+
+var _ bson.Marshaler = TopN{}
+
+func (t TopN) MarshalBSON() ([]byte, error) {
+	return bson.Marshal(bson.D{
+		{"$topN", bson.D{
+			{"n", t.N},
+			{"sortBy", t.SortBy},
+			{"output", t.Output},
+		}},
+	})
+}

agg/agg.go

@@ -0,0 +1,220 @@
+// Package agg provides convenience types for aggregation operators.
+// This yields two major advantages over using bson.D or bson.M:
+// - simpler syntax
+// - auto-completion (i.e., via gopls)
+//
+// Guiding principles are:
+//   - Prefer [1]any for 1-arg operators (e.g., `$bsonSize`).
+//   - Prefer [2]any for binary operators whose arguments don’t benefit
+//     from naming. (e.g., $eq)
+//   - Prefer struct types for operators with named parameters.
+//   - Prefer struct types for operators whose documentation gives names,
+//     even if those names aren’t sent to the server.
+//   - Use functions sparingly, e.g., for “tuple” operators like `$in`.
+//   - Use Go type `any` for generic expressions.
+package agg
+
+import (
+	"go.mongodb.org/mongo-driver/v2/bson"
+)
+
+type Eq [2]any
+
+var _ bson.Marshaler = Eq{}
+
+func (e Eq) MarshalBSON() ([]byte, error) {
+	return bson.Marshal(bson.D{{"$eq", [2]any(e)}})
+}
+
+// ---------------------------------------------
+
+type Gt [2]any
+
+func (g Gt) MarshalBSON() ([]byte, error) {
+	return bson.Marshal(bson.D{{"$gt", [2]any(g)}})
+}
+
+// ---------------------------------------------
+
+func In[T any](needle any, haystack []T) bson.D {
+	return bson.D{{"$in", bson.A{needle, haystack}}}
+}
+
+// ---------------------------------------------
+
+type BSONSize [1]any
+
+var _ bson.Marshaler = BSONSize{}
+
+func (b BSONSize) MarshalBSON() ([]byte, error) {
+	return bson.Marshal(bson.D{{"$bsonSize", b[0]}})
+}
+
+// ---------------------------------------------
+
+type Type [1]any
+
+var _ bson.Marshaler = Type{}
+
+func (t Type) MarshalBSON() ([]byte, error) {
+	return bson.Marshal(bson.D{{"$type", t[0]}})
+}
+
+// ---------------------------------------------
+
+type Not [1]any
+
+var _ bson.Marshaler = Not{}
+
+func (n Not) MarshalBSON() ([]byte, error) {
+	return bson.Marshal(bson.D{{"$not", n[0]}})
+}
+
+// ---------------------------------------------
+
+type And []any
+
+var _ bson.Marshaler = And{}
+
+func (a And) MarshalBSON() ([]byte, error) {
+	return bson.Marshal(bson.D{
+		{"$and", []any(a)},
+	})
+}
+
+// ---------------------------------------------
+
+type Or []any
+
+var _ bson.Marshaler = Or{}
+
+func (o Or) MarshalBSON() ([]byte, error) {
+	return bson.Marshal(bson.D{
+		{"$or", []any(o)},
+	})
+}
+
+// ---------------------------------------------
+
+type MergeObjects []any
+
+var _ bson.Marshaler = MergeObjects{}
+
+func (m MergeObjects) MarshalBSON() ([]byte, error) {
+	return bson.Marshal(bson.D{
+		{"$mergeObjects", []any(m)},
+	})
+}
+
+// ---------------------------------------------
+
+type GetField struct {
+	Input, Field any
+}
+
+var _ bson.Marshaler = GetField{}
+
+func (gf GetField) MarshalBSON() ([]byte, error) {
+	return bson.Marshal(
+		bson.D{
+			{"$getField", bson.D{
+				{"input", gf.Input},
+				{"field", gf.Field},
+			}},
+		},
+	)
+}
+
+// ---------------------------------------------
+
+type Cond struct {
+	If, Then, Else any
+}
+
+var _ bson.Marshaler = Cond{}
+
+func (c Cond) D() bson.D {
+	return bson.D{
+		{"$cond", bson.D{
+			{"if", c.If},
+			{"then", c.Then},
+			{"else", c.Else},
+		}},
+	}
+}
+
+func (c Cond) MarshalBSON() ([]byte, error) {
+	return bson.Marshal(c.D())
+}
+
+// ---------------------------------------------
+
+type Switch struct {
+	Branches []SwitchCase
+	Default  any
+}
+
+var _ bson.Marshaler = Switch{}
+
+type SwitchCase struct {
+	Case any
+	Then any
+}
+
+func (s Switch) D() bson.D {
+	return bson.D{{"$switch", bson.D{
+		{"branches", s.Branches},
+		{"default", s.Default},
+	}}}
+}
+
+func (s Switch) MarshalBSON() ([]byte, error) {
+	return bson.Marshal(s.D())
+}
+
+// ---------------------------------------------
+
+type Map struct {
+	Input, As, In any
+}
+
+var _ bson.Marshaler = Map{}
+
+func (m Map) D() bson.D {
+	return bson.D{
+		{"$map", bson.D{
+			{"input", m.Input},
+			{"as", m.As},
+			{"in", m.In},
+		}},
+	}
+}
+
+func (m Map) MarshalBSON() ([]byte, error) {
+	return bson.Marshal(m.D())
+}
+
+// ------------------------------------------
+
+type Filter struct {
+	Input, As, Cond, Limit any
+}
+
+var _ bson.Marshaler = Filter{}
+
+func (f Filter) D() bson.D {
+	d := bson.D{
+		{"input", f.Input},
+		{"as", f.As},
+		{"cond", f.Cond},
+	}
+
+	if f.Limit != nil {
+		d = append(d, bson.E{"limit", f.Limit})
+	}
+	return bson.D{{"$filter", d}}
+}
+
+func (f Filter) MarshalBSON() ([]byte, error) {
+	return bson.Marshal(f.D())
+}

agg/array.go

@@ -0,0 +1,42 @@
+package agg
+
+import (
+	"slices"
+
+	"go.mongodb.org/mongo-driver/v2/bson"
+)
+
+type Slice struct {
+	Array    any
+	Position *any
+	N        any
+}
+
+func (s Slice) MarshalBSON() ([]byte, error) {
+	args := []any{s.Array, s.N}
+	if s.Position != nil {
+		args = slices.Insert(args, 1, *s.Position)
+	}
+
+	return bson.Marshal(bson.D{
+		{"$slice", args},
+	})
+}
+
+type ArrayElemAt struct {
+	Array any
+	Index any
+}
+
+var _ bson.Marshaler = ArrayElemAt{}
+
+func (a ArrayElemAt) D() bson.D {
+	return bson.D{{"$arrayElemAt", bson.A{
+		a.Array,
+		a.Index,
+	}}}
+}
+
+func (a ArrayElemAt) MarshalBSON() ([]byte, error) {
+	return bson.Marshal(a.D())
+}

agg/helpers/exist.go

@@ -0,0 +1,14 @@
+// Package helpers exposes functions that express common operations
+// that don’t map to a single aggregation operator.
+package helpers
+
+import (
+	"github.com/10gen/migration-verifier/agg"
+	"go.mongodb.org/mongo-driver/v2/bson"
+)
+
+type Exists [1]any
+
+func (e Exists) MarshalBSON() ([]byte, error) {
+	return bson.Marshal(agg.Not{agg.Eq{"missing", agg.Type{e[0]}}})
+}

agg/math.go

@@ -0,0 +1,11 @@
+package agg
+
+import "go.mongodb.org/mongo-driver/v2/bson"
+
+type Subtract [2]any
+
+var _ bson.Marshaler = Subtract{}
+
+func (s Subtract) MarshalBSON() ([]byte, error) {
+	return bson.Marshal(bson.D{{"$subtract", [2]any(s)}})
+}