From 5eca0ab7b0d3c3a6b468ca506ee635b19827775d Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 26 Aug 2026 13:38:56 +0000 Subject: [PATCH 1/3] Resolve dotted and array paths in request receiver topic_identifier The receiver looked up topic_identifier with a flat lookup: topic = output.headers[identifier] || output.body?.[identifier]; 23 providers declare an identifier that nests the event type, such as data.type, events[].eventType or entry[].changes[].field. None of them resolve through a flat lookup, so capturing any of those providers with yarn dev:receiver named every sample untitled-. Measured against one real sample per affected provider, replayed through the receiver in an isolated tree: before 0 of 22 named correctly, 22 untitled after 19 of 22 named correctly Supported forms are a plain key, a dotted path, and [] array segments including nested ones. A literal key that exists is matched before the value is treated as a path, so a header whose real name contains a dot still resolves. Only a scalar can name a file; anything else falls back to untitled- rather than writing [object Object]. The three that still do not resolve are stale configs rather than resolver limitations, and are left alone deliberately since each needs a decision about the right value: mailgun declares event-data.event, sample has event at top level twitter declares data.event_type, sample has direct_message_events at top level xero topic is a composite of eventCategory/eventType, so no single identifier can produce CREDITNOTE/CREATE compile.ts is unaffected: it reads the topic key already stored in each sample file and never consults topic_identifier. No published output changes. README documents the supported path forms. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01N3a91JneFov5QaEUrcXXog --- README.md | 17 +++++++++++++++++ requestReceiver.ts | 46 +++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index f921c87..565892e 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,23 @@ resolves wins, so the most specific one goes first: } ``` +A `topic_identifier` may be a plain key or a path into the body, so +providers that nest the event type are named correctly on capture: + +| Form | Example | Resolves | +|---|---|---| +| plain key | `event` | `body.event`, or the header of that name | +| dotted path | `data.type` | `body.data.type` | +| array segment | `events[].eventType` | first element of `body.events` | +| nested arrays | `entry[].changes[].field` | first element at each level | + +A header or body key whose literal name contains a dot is matched before +the value is treated as a path, so real keys always win. + +Only a scalar can name a file. If a path resolves to an object or array the +sample falls back to `untitled-`, which is a signal that the +identifier is wrong for that payload rather than something to work around. + `provenance` is optional and records, per version, how that version's samples were obtained: diff --git a/requestReceiver.ts b/requestReceiver.ts index 09896c4..94c8bf1 100644 --- a/requestReceiver.ts +++ b/requestReceiver.ts @@ -29,6 +29,50 @@ app.use( }) ); +// Resolve a topic_identifier against a headers or body object. +// +// Most identifiers are a plain key ("x-shopify-topic", "event"), but many +// providers nest the event type ("data.type", "events[].eventType"), so a +// flat lookup alone leaves those captures named "untitled-". +// +// Supported forms: +// event a plain key +// data.type a dotted path +// events[].eventType an array segment; the first element is used +// entry[].changes[].field nested array segments +// +// A literal key that exists is preferred over path interpretation, so a header +// whose real name contains a dot still resolves. +const resolveTopic = (source: any, identifier: string): string | undefined => { + if (source == null) return undefined; + + if (typeof source === "object" && source[identifier] !== undefined) { + return scalarOrUndefined(source[identifier]); + } + + let current = source; + for (const segment of identifier.split(".")) { + if (current == null) return undefined; + + const is_array = segment.endsWith("[]"); + current = current[is_array ? segment.slice(0, -2) : segment]; + + if (is_array) { + if (!Array.isArray(current)) return undefined; + current = current[0]; + } + } + + return scalarOrUndefined(current); +}; + +// Only a scalar can name a file. Anything else means the path landed somewhere +// unintended, and falling back to "untitled-" is more honest than "[object Object]". +const scalarOrUndefined = (value: any): string | undefined => + typeof value === "string" || typeof value === "number" + ? String(value) + : undefined; + const outputToFile = (output: any, provider: string, version: string) => { if (!fs.existsSync(path.join(process.cwd(), "providers", provider))) { console.warn( @@ -67,7 +111,7 @@ const outputToFile = (output: any, provider: string, version: string) => { let topic; for (const identifier of topic_identifiers) { - topic = output.headers[identifier] || output.body?.[identifier]; + topic = resolveTopic(output.headers, identifier) ?? resolveTopic(output.body, identifier); if (topic) break; } From 6933e40de40b7d0adba90d7cf5e8bc003b76f83f Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 28 Aug 2026 16:04:32 +0000 Subject: [PATCH 2/3] Add unit tests for the topic resolver The resolver has six branches that all fail silently: a wrong answer becomes a filename, not an exception. The PR's measured before/after came from a throwaway script, so nothing in the repo would catch a regression. requestReceiver.ts calls app.listen() at import, so a test importing it would hang. The two pure functions move to topic.ts unchanged, with their original comment verbatim, and requestReceiver.ts imports from there. 14 tests over plain keys, dotted paths, single and nested array segments, an array segment under a dotted path, literal keys that contain a dot, paths landing on objects, empty arrays, broken paths, null sources, and numeric coercion. Verified the tests can fail: changing current[0] to current[1] in the array branch turns 3 of them red. `tsc --noEmit` reports the same two pre-existing errors in scripts/ as before, and `yarn compile` produces an identical tree, so the extraction changes no published output. Uses node:test with the ts-node already in devDependencies, so the repo gains a `yarn test` script and no new dependency. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01N3a91JneFov5QaEUrcXXog --- README.md | 3 ++ package.json | 3 +- requestReceiver.ts | 45 +-------------------------- topic.test.ts | 76 ++++++++++++++++++++++++++++++++++++++++++++++ topic.ts | 47 ++++++++++++++++++++++++++++ 5 files changed, 129 insertions(+), 45 deletions(-) create mode 100644 topic.test.ts create mode 100644 topic.ts diff --git a/README.md b/README.md index 565892e..70f2442 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,9 @@ Only a scalar can name a file. If a path resolves to an object or array the sample falls back to `untitled-`, which is a signal that the identifier is wrong for that payload rather than something to work around. +The resolver is covered by unit tests in `topic.test.ts`. Run them with +`yarn test`. + `provenance` is optional and records, per version, how that version's samples were obtained: diff --git a/package.json b/package.json index 57c3799..d02891a 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,8 @@ "dev:receiver": "nodemon --exec ./node_modules/.bin/ts-node requestReceiver.ts --watch *.ts", "setup:scrapfly": "./node_modules/.bin/ts-node scripts/scrapfly/setup.ts", "capture:scrapfly": "./node_modules/.bin/ts-node scripts/scrapfly/capture.ts", - "generate:ordinal": "./node_modules/.bin/ts-node scripts/ordinal/docs.ts" + "generate:ordinal": "./node_modules/.bin/ts-node scripts/ordinal/docs.ts", + "test": "node --require ts-node/register --test \"*.test.ts\"" }, "devDependencies": { "@types/express": "^4.17.17", diff --git a/requestReceiver.ts b/requestReceiver.ts index 94c8bf1..59f045e 100644 --- a/requestReceiver.ts +++ b/requestReceiver.ts @@ -2,6 +2,7 @@ import express from "express"; import * as fs from "fs"; import * as path from "path"; import crypto from "crypto"; +import { resolveTopic } from "./topic"; const app = express(); const port = process.env.PORT || 9001; @@ -29,50 +30,6 @@ app.use( }) ); -// Resolve a topic_identifier against a headers or body object. -// -// Most identifiers are a plain key ("x-shopify-topic", "event"), but many -// providers nest the event type ("data.type", "events[].eventType"), so a -// flat lookup alone leaves those captures named "untitled-". -// -// Supported forms: -// event a plain key -// data.type a dotted path -// events[].eventType an array segment; the first element is used -// entry[].changes[].field nested array segments -// -// A literal key that exists is preferred over path interpretation, so a header -// whose real name contains a dot still resolves. -const resolveTopic = (source: any, identifier: string): string | undefined => { - if (source == null) return undefined; - - if (typeof source === "object" && source[identifier] !== undefined) { - return scalarOrUndefined(source[identifier]); - } - - let current = source; - for (const segment of identifier.split(".")) { - if (current == null) return undefined; - - const is_array = segment.endsWith("[]"); - current = current[is_array ? segment.slice(0, -2) : segment]; - - if (is_array) { - if (!Array.isArray(current)) return undefined; - current = current[0]; - } - } - - return scalarOrUndefined(current); -}; - -// Only a scalar can name a file. Anything else means the path landed somewhere -// unintended, and falling back to "untitled-" is more honest than "[object Object]". -const scalarOrUndefined = (value: any): string | undefined => - typeof value === "string" || typeof value === "number" - ? String(value) - : undefined; - const outputToFile = (output: any, provider: string, version: string) => { if (!fs.existsSync(path.join(process.cwd(), "providers", provider))) { console.warn( diff --git a/topic.test.ts b/topic.test.ts new file mode 100644 index 0000000..b399d76 --- /dev/null +++ b/topic.test.ts @@ -0,0 +1,76 @@ +import { strict as assert } from "node:assert"; +import { describe, it } from "node:test"; + +import { resolveTopic, scalarOrUndefined } from "./topic"; + +describe("resolveTopic", () => { + it("resolves a plain key", () => { + assert.equal(resolveTopic({ event: "charge.succeeded" }, "event"), "charge.succeeded"); + }); + + it("resolves a dotted path", () => { + assert.equal(resolveTopic({ data: { type: "invoice.paid" } }, "data.type"), "invoice.paid"); + }); + + it("resolves an array segment using the first element", () => { + const body = { events: [{ eventType: "user.created" }, { eventType: "user.deleted" }] }; + assert.equal(resolveTopic(body, "events[].eventType"), "user.created"); + }); + + it("resolves nested array segments", () => { + const body = { entry: [{ changes: [{ field: "messages" }] }] }; + assert.equal(resolveTopic(body, "entry[].changes[].field"), "messages"); + }); + + it("resolves an array segment nested under a dotted path", () => { + const body = { data: { events: [{ eventType: "user.lifecycle.activate" }] } }; + assert.equal(resolveTopic(body, "data.events[].eventType"), "user.lifecycle.activate"); + }); + + // A header called "x-thing.type" is a real key, not a path into "x-thing". + it("prefers a literal key over interpreting it as a path", () => { + const headers = { "data.type": "literal", data: { type: "viaPath" } }; + assert.equal(resolveTopic(headers, "data.type"), "literal"); + }); + + it("returns undefined when the path lands on an object", () => { + assert.equal(resolveTopic({ data: { type: { nested: true } } }, "data.type"), undefined); + }); + + it("returns undefined when an array segment finds no array", () => { + assert.equal(resolveTopic({ events: { eventType: "x" } }, "events[].eventType"), undefined); + }); + + it("returns undefined for an empty array", () => { + assert.equal(resolveTopic({ events: [] }, "events[].eventType"), undefined); + }); + + it("returns undefined when the path breaks part way", () => { + assert.equal(resolveTopic({ data: null }, "data.type"), undefined); + assert.equal(resolveTopic({}, "a.b.c"), undefined); + }); + + it("returns undefined for a null or undefined source", () => { + assert.equal(resolveTopic(null, "event"), undefined); + assert.equal(resolveTopic(undefined, "event"), undefined); + }); + + it("coerces a numeric value to a string", () => { + assert.equal(resolveTopic({ data: { type: 42 } }, "data.type"), "42"); + }); +}); + +describe("scalarOrUndefined", () => { + it("passes through strings and stringifies numbers", () => { + assert.equal(scalarOrUndefined("a"), "a"); + assert.equal(scalarOrUndefined(0), "0"); + }); + + it("rejects anything that would stringify to [object Object]", () => { + assert.equal(scalarOrUndefined({}), undefined); + assert.equal(scalarOrUndefined([]), undefined); + assert.equal(scalarOrUndefined(null), undefined); + assert.equal(scalarOrUndefined(undefined), undefined); + assert.equal(scalarOrUndefined(true), undefined); + }); +}); diff --git a/topic.ts b/topic.ts new file mode 100644 index 0000000..fc83d30 --- /dev/null +++ b/topic.ts @@ -0,0 +1,47 @@ +// Resolve a topic_identifier against a headers or body object. +// +// Most identifiers are a plain key ("x-shopify-topic", "event"), but many +// providers nest the event type ("data.type", "events[].eventType"), so a +// flat lookup alone leaves those captures named "untitled-". +// +// Supported forms: +// event a plain key +// data.type a dotted path +// events[].eventType an array segment; the first element is used +// entry[].changes[].field nested array segments +// +// A literal key that exists is preferred over path interpretation, so a header +// whose real name contains a dot still resolves. +export const resolveTopic = ( + source: any, + identifier: string +): string | undefined => { + if (source == null) return undefined; + + if (typeof source === "object" && source[identifier] !== undefined) { + return scalarOrUndefined(source[identifier]); + } + + let current = source; + for (const segment of identifier.split(".")) { + if (current == null) return undefined; + + const is_array = segment.endsWith("[]"); + current = current[is_array ? segment.slice(0, -2) : segment]; + + if (is_array) { + if (!Array.isArray(current)) return undefined; + current = current[0]; + } + } + + return scalarOrUndefined(current); +}; + +// Only a scalar can name a file. Anything else means the path landed somewhere +// unintended, and falling back to "untitled-" is more honest than +// "[object Object]". +export const scalarOrUndefined = (value: any): string | undefined => + typeof value === "string" || typeof value === "number" + ? String(value) + : undefined; From 53ca0de71fb84c56bf206abbc4c6aa129abb8e68 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 28 Aug 2026 16:27:59 +0000 Subject: [PATCH 3/3] Run the tests in CI The repo had no workflows, so `yarn test` only ever ran if someone remembered to. A test nobody runs is a comment. Triggers on pull requests and pushes to main. Node 22 with the yarn cache, `yarn install --frozen-lockfile` so the lockfile is authoritative, then `yarn test`. Verified by running the same sequence against a clean export of this tree: install from the lockfile, then 14 of 14 passing. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01N3a91JneFov5QaEUrcXXog --- .github/workflows/test.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..6fa7391 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,23 @@ +name: Tests + +on: + push: + branches: [main] + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: yarn + + - name: Install dependencies + run: yarn install --frozen-lockfile + + - name: Run tests + run: yarn test