Milestone 4: expressions + SELECT — parser, JSON emitter, ParseToJSON

PLAN.md

@@ -380,12 +380,15 @@ gate; the corpus harness exists before the first line of the lexer.
    met: all 43,373 scan cases and all 8 split_scanner cases pass; xxh3 is
    verified against 126 oracle-generated vectors covering every length
    class and seed path. See "As-built notes (milestone 3)".
-4. **Expressions + SELECT.** `a_expr`/`b_expr`/`c_expr` precedence machinery,
-   constants and typecasts, `func_expr` (including `json_*`, `xml*`,
-   aggregate `FILTER`/`WITHIN GROUP`, window functions), `SELECT` end-to-end:
-   target list, `FROM` (joins, `LATERAL`, table functions, `TABLESAMPLE`),
-   grouping sets, set operations, CTEs (`MATERIALIZED`, `SEARCH`/`CYCLE`),
-   `VALUES`, locking clauses. The largest single chunk of work.
+4. **Expressions + SELECT.** ✅ *Landed 2026-08-15.* `a_expr`/`b_expr`/
+   `c_expr` precedence machinery, constants and typecasts, `func_expr`
+   (including `json_*`, `xml*`, aggregate `FILTER`/`WITHIN GROUP`, window
+   functions), `SELECT` end-to-end: target list, `FROM` (joins, `LATERAL`,
+   table functions, `TABLESAMPLE`, `XMLTABLE`, `JSON_TABLE`), grouping
+   sets, set operations, CTEs (`MATERIALIZED`, `SEARCH`/`CYCLE`), `VALUES`,
+   locking clauses. The largest single chunk of work. `ParseToJSON` ships
+   here too (the generated-equivalent JSON emitter plus `Parse`/
+   `ParseToProtobuf`). See "As-built notes (milestone 4)".
 5. **DML.** `INSERT` (`ON CONFLICT`, `OVERRIDING`), `UPDATE`, `DELETE`,
    `MERGE`, `RETURNING`, `COPY`, `PREPARE`/`EXECUTE`, cursors.
 6. **DDL, part 1.** `CREATE`/`ALTER TABLE` (the second-biggest grammar
@@ -479,6 +482,49 @@ Measured at the pin, where the plan's estimates differ:
 - `internal/xxh3` implements only `XXH3_64bits_withSeed` (scalar paths),
   the sole entry point the API needs.
 
+### As-built notes (milestone 4)
+
+- The JSON emitter (`internal/emit`) is a protobuf-descriptor-driven walker
+  rather than generated per-node code: the pinned proto was itself generated
+  from the C struct metadata, so field declaration order, `json_name`, and
+  field kinds already encode everything `pg_query_outfuncs_json.c` does. The
+  hand-written special cases (`A_Const`, the five value nodes, `List`
+  variants, `_outToken` escaping, two always-emitted empty-string fields)
+  mirror the reference file. Byte-parity is proven corpus-wide by
+  `internal/emit.TestGoldenRoundTrip`, which protojson-decodes all 42,970
+  tree goldens and re-emits them byte-identically. Generated per-node
+  emitters remain an option for milestone 12 if profiling wants them.
+- The parser (`internal/parse`) reproduces bison's conflict resolution by
+  precedence climbing: left-assoc operators parse their right operand one
+  level up, `%nonassoc` levels error when chained within one climb, and the
+  `%prec` annotations are reproduced at their call sites. Two behaviors
+  worth calling out: `a_expr subquery_Op sub_type` enters the loop at the
+  operator token's own precedence but reduces at `%prec Op` (so
+  `1 = 2 = ANY(...)` fails at the second `=` while `= ANY(...) + 1` binds
+  the `+` outside), and qual-requiring JOINs absorb further joins into
+  their right operand (bison shifts because the rule cannot reduce until
+  its join_qual) while CROSS/NATURAL joins stay left-associative.
+- LALR keeps sub-select and parenthesized-expression paths alive
+  simultaneously; the recursive-descent port uses bounded backtracking
+  (token-position mark/reset) in exactly four places: `(`-headed c_expr /
+  in_expr / table_ref (select_with_parens trials, pre-gated by a
+  skip-the-parens lookahead for a SELECT/VALUES/TABLE/WITH head), and the
+  OVERLAY/SUBSTRING/JSON_OBJECT special-vs-generic argument forms.
+- The `NOT_LA`/`WITH_LA`/`FORMAT_LA` merges arrive pre-applied from the
+  milestone-3 filter; grammar errors report through the same
+  scanner_yyerror path the C stack uses (`base_yyerror` → `parser_yyerror`
+  → `scanner_yyerror`), so syntax errors carry `scan.l` error data and the
+  merged tokens report only their first word, exactly as parser.c's
+  hold-char poke arranges.
+- Corpus effect: 16,300+ parse cases came off the todo list. Every
+  remaining parse todo either starts with a statement type from milestones
+  5-7 or embeds one (DML inside CTEs); the only pure-SELECT stragglers are
+  negative cases whose error texts come from those same unimplemented
+  productions.
+- `cmd/difftest` currently ships the interim `-summary`/`-show` failure
+  classifier used to drive the milestone; the mutation fuzzer replaces it
+  in milestone 12.
+
 ## Regeneration (the PostgreSQL-upgrade story)
 
 Everything derived is derived by committed tooling from the pin:

cmd/difftest/main.go

@@ -0,0 +1,175 @@
+// Command difftest will run mutation differential fuzzing against the live
+// oracle (milestone 12). For now it provides the -summary mode used during
+// grammar bring-up: it walks the parse corpus's remaining todo cases, runs
+// them through the current parser, and classifies the failures so the
+// development loop can attack the biggest buckets first.
+package main
+
+import (
+	"errors"
+	"flag"
+	"fmt"
+	"os"
+	"path/filepath"
+	"regexp"
+	"sort"
+	"strings"
+
+	pg_query "github.com/sqlc-dev/oliphant"
+	"github.com/sqlc-dev/oliphant/internal/testfile"
+	"github.com/sqlc-dev/oliphant/parser"
+)
+
+func main() {
+	var (
+		summary = flag.Bool("summary", false, "classify remaining parse todo failures")
+		show    = flag.String("show", "", "print the first N failing cases whose bucket matches this substring")
+		limit   = flag.Int("n", 5, "how many cases to print with -show")
+		root    = flag.String("root", "parser/testdata/parse", "corpus root")
+	)
+	flag.Parse()
+	if !*summary && *show == "" {
+		fmt.Fprintln(os.Stderr, "usage: difftest -summary | -show <bucket> [-n N]")
+		os.Exit(2)
+	}
+
+	buckets := map[string]int{}
+	type sample struct{ file, name, input, got, want string }
+	var samples []sample
+
+	var files []string
+	filepath.WalkDir(*root, func(path string, d os.DirEntry, err error) error {
+		if err == nil && !d.IsDir() && strings.HasSuffix(path, ".test") {
+			files = append(files, path)
+		}
+		return nil
+	})
+	for _, path := range files {
+		cases, err := testfile.Read(path)
+		if err != nil {
+			panic(err)
+		}
+		meta, err := testfile.ReadMetadata(path)
+		if err != nil {
+			panic(err)
+		}
+		todo := map[string]bool{}
+		for _, n := range meta.Todo {
+			todo[n] = true
+		}
+		for _, c := range cases {
+			if !todo[c.Name] {
+				continue
+			}
+			got := evaluate(c.Input)
+			if got == c.Expected {
+				continue
+			}
+			b := classify(got, c.Expected)
+			buckets[b]++
+			if *show != "" && strings.Contains(b, *show) && len(samples) < *limit {
+				samples = append(samples, sample{path, c.Name, c.Input, got, c.Expected})
+			}
+		}
+	}
+
+	if *summary {
+		type kv struct {
+			k string
+			v int
+		}
+		var list []kv
+		total := 0
+		for k, v := range buckets {
+			list = append(list, kv{k, v})
+			total += v
+		}
+		sort.Slice(list, func(i, j int) bool { return list[i].v > list[j].v })
+		for _, e := range list {
+			fmt.Printf("%7d  %s\n", e.v, e.k)
+		}
+		fmt.Printf("%7d  TOTAL failing\n", total)
+	}
+	for _, s := range samples {
+		fmt.Printf("=== %s %s\ninput:\n%s\ngot:\n%s\nwant:\n%s\n\n", s.file, s.name, s.input, s.got, s.want)
+	}
+}
+
+func evaluate(input string) string {
+	out, err := pg_query.ParseToJSON(input)
+	if err != nil {
+		if pe, ok := errAsParser(err); ok {
+			return testfile.RenderError(testfile.ErrorExpectation{
+				Message:   pe.Message,
+				Cursorpos: pe.Cursorpos,
+				Filename:  pe.Filename,
+				Funcname:  pe.Funcname,
+				Context:   pe.Context,
+			})
+		}
+		return "UNIMPLEMENTED: " + err.Error()
+	}
+	return out
+}
+
+var firstDiffRe = regexp.MustCompile(`"([A-Za-z_0-9]+)":`)
+
+// classify produces a coarse failure bucket.
+func classify(got, want string) string {
+	gotErr := strings.HasPrefix(got, "ERROR: ")
+	wantErr := strings.HasPrefix(want, "ERROR: ")
+	switch {
+	case gotErr && wantErr:
+		g := strings.SplitN(got, "\n", 2)[0]
+		w := strings.SplitN(want, "\n", 2)[0]
+		if g == w {
+			return "error-detail-mismatch (message equal)"
+		}
+		return "error-vs-error message differs"
+	case gotErr && !wantErr:
+		// We reject what the oracle accepts: the interesting bucket.
+		msg := strings.SplitN(got, "\n", 2)[0]
+		if m := regexp.MustCompile(`at or near "(.*)"$`).FindStringSubmatch(msg); m != nil {
+			word := m[1]
+			if len(word) > 20 {
+				word = word[:20]
+			}
+			return `reject: at or near "` + word + `"`
+		}
+		return "reject: " + msg
+	case !gotErr && wantErr:
+		return "accept-but-should-reject"
+	default:
+		// Tree mismatch: name the first differing JSON key.
+		i := 0
+		for i < len(got) && i < len(want) && got[i] == want[i] {
+			i++
+		}
+		start := i
+		if start > 40 {
+			start -= 40
+		} else {
+			start = 0
+		}
+		ctx := want[start:min(len(want), i+20)]
+		if m := firstDiffRe.FindAllString(ctx, -1); len(m) > 0 {
+			return "tree-diff near " + m[len(m)-1]
+		}
+		return "tree-diff"
+	}
+}
+
+func min(a, b int) int {
+	if a < b {
+		return a
+	}
+	return b
+}
+
+func errAsParser(err error) (*parser.Error, bool) {
+	var pe *parser.Error
+	if errors.As(err, &pe) {
+		return pe, true
+	}
+	return nil, false
+}

compat_test.go

@@ -33,11 +33,7 @@ func TestNotImplementedErrors(t *testing.T) {
 		}
 	}
 
-	_, err := pg_query.Parse("SELECT 1")
-	check("Parse", err)
-	_, err = pg_query.ParseToJSON("SELECT 1")
-	check("ParseToJSON", err)
-	_, err = pg_query.Deparse(&pg_query.ParseResult{})
+	_, err := pg_query.Deparse(&pg_query.ParseResult{})
 	check("Deparse", err)
 	_, err = pg_query.Normalize("SELECT 1")
 	check("Normalize", err)