Milestone 3: lexer — scan.l port, base_yylex filter, Scan/SplitWithScanner/HashXXH3_64

PLAN.md

@@ -373,10 +373,13 @@ gate; the corpus harness exists before the first line of the lexer.
    as `todo`. **Gate:** goldens reproducible byte-for-byte across two
    regeneration runs — met, and re-verified weekly by the `regenerate` CI
    workflow. See "As-built notes" below for measured counts and deferrals.
-3. **Lexer.** `scan.l` port + `base_yylex` filter layer; `Scan`,
-   `SplitWithScanner`, `HashXXH3_64` (+ `internal/xxh3`) ship. **Gate:**
-   token streams byte-identical to the oracle across the entire corpus,
-   including scanner error messages and cursor positions.
+3. **Lexer.** ✅ *Landed 2026-08-15.* `scan.l` port + `base_yylex` filter
+   layer; `Scan`, `SplitWithScanner`, `HashXXH3_64` (+ `internal/xxh3`)
+   ship. **Gate:** token streams byte-identical to the oracle across the
+   entire corpus, including scanner error messages and cursor positions —
+   met: all 43,373 scan cases and all 8 split_scanner cases pass; xxh3 is
+   verified against 126 oracle-generated vectors covering every length
+   class and seed path. See "As-built notes (milestone 3)".
 4. **Expressions + SELECT.** `a_expr`/`b_expr`/`c_expr` precedence machinery,
    constants and typecasts, `func_expr` (including `json_*`, `xml*`,
    aggregate `FILTER`/`WITHIN GROUP`, window functions), `SELECT` end-to-end:
@@ -448,6 +451,34 @@ Measured at the pin, where the plan's estimates differ:
   **`plpgsql_regress/`** (milestone 11), and **summary** golden extraction
   (milestone 10). The stretch tarball tier remains stretch.
 
+### As-built notes (milestone 3)
+
+- The scanner (`internal/lexer`) is pull-based (`Scanner.Next`), one token
+  per call like `core_yylex`; `Scan`/`SplitWithScanner` drain it eagerly,
+  but milestone 4's parser must pull lazily so grammar errors can win over
+  scanner errors that lie further right, as they do under bison.
+- Flex's longest-match/rule-order discipline is reproduced structurally
+  except for the numeric-literal rules, whose fourteen overlapping patterns
+  (`decinteger` … `real_junk`, including flex backtracking inside the
+  `*_junk` rules) are resolved by computing every candidate match length
+  and picking the winner (`internal/lexer/numbers.go`).
+- Two pinned-oracle behaviors worth knowing: token End offsets come from
+  the patch-03 `yyllocend` for multi-rule tokens
+  (`pg_query_scan.c` uses it for SCONST/USCONST/BCONST/XCONST/IDENT/
+  UIDENT/C_COMMENT and `yylloc + yyleng` for the rest — with the eager
+  scanner both reduce to "position after the token"), and a comment
+  *between* a `base_yylex` merge pair blocks the merge (the lookahead is a
+  raw `core_yylex` call, so pg_query_go v6.2.2 rejects
+  `SELECT 1 WHERE 1 NOT /* c */ IN (2)` where vanilla PostgreSQL does not;
+  the filter reproduces this, pinned by a unit test).
+- Scanner error cursor positions are character-based
+  (`pg_mbstrlen_with_len` semantics, per-lead-byte stride, not rune
+  count); the "at or near" text runs from the error location to the end of
+  the current match (flex's hold-char NUL), or to end of input in `<<EOF>>`
+  rules.
+- `internal/xxh3` implements only `XXH3_64bits_withSeed` (scalar paths),
+  the sole entry point the API needs.
+
 ## Regeneration (the PostgreSQL-upgrade story)
 
 Everything derived is derived by committed tooling from the pin:

compat_test.go

@@ -37,8 +37,6 @@ func TestNotImplementedErrors(t *testing.T) {
 	check("Parse", err)
 	_, err = pg_query.ParseToJSON("SELECT 1")
 	check("ParseToJSON", err)
-	_, err = pg_query.Scan("SELECT 1")
-	check("Scan", err)
 	_, err = pg_query.Deparse(&pg_query.ParseResult{})
 	check("Deparse", err)
 	_, err = pg_query.Normalize("SELECT 1")
@@ -49,8 +47,6 @@ func TestNotImplementedErrors(t *testing.T) {
 	check("Fingerprint", err)
 	_, err = pg_query.FingerprintToUInt64("SELECT 1")
 	check("FingerprintToUInt64", err)
-	_, err = pg_query.SplitWithScanner("SELECT 1", false)
-	check("SplitWithScanner", err)
 	_, err = pg_query.SplitWithParser("SELECT 1", false)
 	check("SplitWithParser", err)
 	_, err = pg_query.IsUtilityStmt("SELECT 1")
@@ -59,13 +55,73 @@ func TestNotImplementedErrors(t *testing.T) {
 	check("ParsePlPgSqlToJSON", err)
 	_, err = pg_query.Summary("SELECT 1", 0)
 	check("Summary", err)
+}
 
-	defer func() {
-		if recover() == nil {
-			t.Fatal("HashXXH3_64: expected a not-implemented panic")
+// TestScanTokens pins one token stream end-to-end through the public Scan
+// API (pg_query_go's TestScan is a smoke test; the corpus goldens carry the
+// exhaustive coverage).
+func TestScanTokens(t *testing.T) {
+	result, err := pg_query.Scan("SELECT update AS left /* comment */ FROM between")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if result.Version != 170007 {
+		t.Fatalf("ScanResult.Version = %d, want 170007", result.Version)
+	}
+	type tok struct {
+		start, end  int32
+		token       pg_query.Token
+		keywordKind pg_query.KeywordKind
+	}
+	expected := []tok{
+		{0, 6, pg_query.Token_SELECT, pg_query.KeywordKind_RESERVED_KEYWORD},
+		{7, 13, pg_query.Token_UPDATE, pg_query.KeywordKind_UNRESERVED_KEYWORD},
+		{14, 16, pg_query.Token_AS, pg_query.KeywordKind_RESERVED_KEYWORD},
+		{17, 21, pg_query.Token_LEFT, pg_query.KeywordKind_TYPE_FUNC_NAME_KEYWORD},
+		{22, 35, pg_query.Token_C_COMMENT, pg_query.KeywordKind_NO_KEYWORD},
+		{36, 40, pg_query.Token_FROM, pg_query.KeywordKind_RESERVED_KEYWORD},
+		{41, 48, pg_query.Token_BETWEEN, pg_query.KeywordKind_COL_NAME_KEYWORD},
+	}
+	if len(result.Tokens) != len(expected) {
+		t.Fatalf("got %d tokens, want %d", len(result.Tokens), len(expected))
+	}
+	for i, e := range expected {
+		g := result.Tokens[i]
+		if g.Start != e.start || g.End != e.end || g.Token != e.token || g.KeywordKind != e.keywordKind {
+			t.Errorf("token %d = (%d %d %v %v), want (%d %d %v %v)",
+				i, g.Start, g.End, g.Token, g.KeywordKind, e.start, e.end, e.token, e.keywordKind)
 		}
-	}()
-	pg_query.HashXXH3_64([]byte("x"), 0)
+	}
+}
+
+// TestSplitWithScanner mirrors pg_query_go's scanner-based split tests.
+func TestSplitWithScanner(t *testing.T) {
+	stmts, err := pg_query.SplitWithScanner("SELECT /* comment with ; */ 1; SELECT 2", true)
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := []string{"SELECT /* comment with ; */ 1", "SELECT 2"}
+	if len(stmts) != len(want) || stmts[0] != want[0] || stmts[1] != want[1] {
+		t.Fatalf("SplitWithScanner = %q, want %q", stmts, want)
+	}
+}
+
+// TestHashXXH3_64 carries pg_query_go's exact hard-coded vectors
+// (fingerprint_test.go, v6.2.2).
+func TestHashXXH3_64(t *testing.T) {
+	for _, v := range []struct {
+		input    string
+		seed     uint64
+		expected uint64
+	}{
+		{"TEST", 0, 11717748491247689214},
+		{"TEST", 42, 10412276358662179996},
+		{"Something else", 0, 14679351602596009561},
+	} {
+		if got := pg_query.HashXXH3_64([]byte(v.input), v.seed); got != v.expected {
+			t.Errorf("HashXXH3_64(%q, %d) = %d, want %d", v.input, v.seed, got, v.expected)
+		}
+	}
 }
 
 // TestMakeFuncs smoke-checks the verbatim-ported constructors build the same

internal/lexer/errors.go

@@ -0,0 +1,133 @@
+package lexer
+
+import "fmt"
+
+// yyerror is scan.l's scanner_yyerror: the reported text runs from the error
+// location (yylloc) to the end of the current match — flex's temporary NUL
+// at the match end is what terminates the C "%s" — or to the end of input
+// for the <<EOF>> rules. Callers must have s.pos at the match end.
+func (s *Scanner) yyerror(loc int, message string) *Error {
+	if loc >= len(s.input) {
+		return &Error{
+			Message:   message + " at end of input",
+			Filename:  "scan.l",
+			Funcname:  "scanner_yyerror",
+			Cursorpos: s.cursorpos(loc),
+		}
+	}
+	return &Error{
+		Message:   message + ` at or near "` + s.input[loc:s.pos] + `"`,
+		Filename:  "scan.l",
+		Funcname:  "scanner_yyerror",
+		Cursorpos: s.cursorpos(loc),
+	}
+}
+
+// ereport is a direct ereport(ERROR, ... lexer_errposition()) from inside a
+// scan.l rule action: no "at or near" decoration, and the C __func__ is the
+// flex-generated core_yylex.
+func (s *Scanner) ereport(loc int, message string) *Error {
+	return &Error{
+		Message:   message,
+		Filename:  "scan.l",
+		Funcname:  "core_yylex",
+		Cursorpos: s.cursorpos(loc),
+	}
+}
+
+// cursorpos is scanner_errposition: the 1-based character (not byte) number
+// of the given byte offset, counted with pg_mbstrlen_with_len's per-lead-byte
+// stride.
+func (s *Scanner) cursorpos(loc int) int {
+	count := 0
+	for i := 0; i < loc && i < len(s.input); count++ {
+		i += mblen(s.input[i])
+	}
+	return count + 1
+}
+
+// encodingError is mbutils.c's report_invalid_encoding for the UTF-8 server
+// encoding: raised via pg_verifymbstr when an escape produced a NUL or
+// high-bit byte and the resulting literal is not valid UTF-8. No error
+// position is attached (scan.l installs no errposition callback around it).
+func encodingError(seq []byte) *Error {
+	limit := len(seq)
+	if l := mblen(seq[0]); l < limit {
+		limit = l
+	}
+	if limit > 8 {
+		limit = 8
+	}
+	msg := `invalid byte sequence for encoding "UTF8":`
+	for j := 0; j < limit; j++ {
+		msg += fmt.Sprintf(" 0x%02x", seq[j])
+	}
+	return &Error{
+		Message:  msg,
+		Filename: "src_backend_utils_mb_mbutils.c",
+		Funcname: "report_invalid_encoding",
+	}
+}
+
+// verifyUTF8 is pg_verifymbstr for the UTF-8 server encoding
+// (pg_utf8_verifystr semantics, one character at a time).
+func verifyUTF8(b []byte) *Error {
+	i := 0
+	for i < len(b) {
+		c := b[i]
+		if c < 0x80 {
+			if c == 0 {
+				return encodingError(b[i:])
+			}
+			i++
+			continue
+		}
+		l := mblen(c)
+		if i+l > len(b) || !utf8CharLegal(b[i:i+l]) {
+			return encodingError(b[i:])
+		}
+		i += l
+	}
+	return nil
+}
+
+// utf8CharLegal is wchar.c's pg_utf8_islegal.
+func utf8CharLegal(source []byte) bool {
+	switch len(source) {
+	case 1:
+		return source[0] < 0x80
+	case 2:
+		if source[0] < 0xC2 {
+			return false
+		}
+		return source[1] >= 0x80 && source[1] <= 0xBF
+	case 3:
+		if source[2] < 0x80 || source[2] > 0xBF {
+			return false
+		}
+		switch source[0] {
+		case 0xE0:
+			return source[1] >= 0xA0 && source[1] <= 0xBF
+		case 0xED:
+			return source[1] >= 0x80 && source[1] <= 0x9F
+		default:
+			return source[1] >= 0x80 && source[1] <= 0xBF
+		}
+	case 4:
+		if source[3] < 0x80 || source[3] > 0xBF {
+			return false
+		}
+		if source[2] < 0x80 || source[2] > 0xBF {
+			return false
+		}
+		switch source[0] {
+		case 0xF0:
+			return source[1] >= 0x90 && source[1] <= 0xBF
+		case 0xF4:
+			return source[1] >= 0x80 && source[1] <= 0x8F
+		default:
+			return source[1] >= 0x80 && source[1] <= 0xBF
+		}
+	}
+	return false
+}