From d1931daeedd7f71c8932cbc4535db73f18bcbb25 Mon Sep 17 00:00:00 2001 From: bb7133 Date: Thu, 3 Sep 2026 16:24:57 +0800 Subject: [PATCH 1/4] docs: freeze Rust offline replayer ownership --- docs/design/rust-offline-replayer.md | 204 +++++++++++++++++++++++++++ 1 file changed, 204 insertions(+) create mode 100644 docs/design/rust-offline-replayer.md diff --git a/docs/design/rust-offline-replayer.md b/docs/design/rust-offline-replayer.md new file mode 100644 index 000000000..dfd9b20f6 --- /dev/null +++ b/docs/design/rust-offline-replayer.md @@ -0,0 +1,204 @@ +# Rust offline replayer ownership + +This document freezes the `CP-REPLAYER` (`#152`) ownership boundary before +the Go `cmd/replayer` implementation moves to Rust. The target is a separate +`tiproxy-replayer` executable. It is deliberately not linked into the +`tiproxy-rs` SQL-serving process and does not weaken `PARITY-EXCL-001`. + +## Ownership and cutover + +The Rust executable owns the standalone one-shot replayer and its replay-only +service mode. The one-shot CLI keeps the existing flag names, defaults, and +validation. Service mode keeps these endpoints and response shapes: + +- `POST /api/traffic/replay`; +- `POST /api/traffic/cancel`; +- `GET /api/traffic/show`. + +The standalone server does not advertise or accept traffic capture. Capture +and capture-file production belong to `CP-CAPTURE` (`#151`); the integrated +control API belongs to `CP-ADMIN` (`#150`). The old Go command is removed only +after the Rust release artifact and the evidence below are green. The Go +`pkg/sqlreplay/capture` package and the temporarily integrated Go traffic API +are not made Rust owners by this slice and remain covered by +`PARITY-EXCL-001` until their owning issues cut them over. + +The implementation is split into: + +- `replayer-core`: bounded config, format decoders, storage, ordering, + scheduling, checkpointing, direct MySQL execution, and reports; +- `tiproxy-replayer`: CLI, signals, replay-only HTTP lifecycle, and version + output. + +Neither crate depends on `control-proto`, the legacy bridge, the topology +owner, or the SQL-serving dataplane. There is no replay payload over IPC or +FFI. A replayer process has exactly one job owner and at most one active job. + +## Input contracts + +### Native capture format + +Native files preserve the current file and record contract: + +- `traffic-2006-01-02T15-04-05.999.log[.gz]`, ordered by the timestamp in the + filename and then by full path; +- optional `meta` JSON with version `v1`, duration, command counts, filtered + counts, and optional `aes256-ctr` method; +- one record headed by `# Time`, `# Conn_ID`, optional `# Cmd_type`, + `# Success`, `# Captured_ps_id`, `# Prepared_stmt`, and `# Payload_len`, + followed by exactly `Payload_len` binary bytes and one newline; +- the decoded payload starts with the MySQL command byte. Query is the default + when `Cmd_type` is absent. + +Timestamps use RFC3339Nano. Unknown fields are ignored for forward +compatibility, but duplicate required fields, invalid integers/timestamps, +negative lengths, truncated payloads, oversized records, or a missing final +newline fail closed with path and byte-offset context. The default per-record +payload cap is 64 MiB and is never allocated before the length is validated. + +### Audit log plugin format + +The plugin decoder accepts the current bracketed key/value grammar and its +timestamp layout `2006/01/02 15:04:05.999 -07:00`. It preserves current +handling of `GENERAL`, `TABLE_ACCESS`, and `CONNECTION` events; per-decoder +connection remapping; retry filtering; case-insensitive user allowlists; +duplicate-write suppression; current database; prepared statement text/ID; +and execute parameters. Malformed quoting, nesting, timestamps, or a line +larger than 64 MiB fails closed rather than scanning without a bound. + +### Audit log extension format + +The extension decoder accepts `_LOG_TIME` plus the quoted tuple `EVENT` +contract for `QUERY`, `EXECUTE`, `CONNECTION`, and `DISCONNECT`. It preserves +the current `always`/`never` prepared-close strategies, skips redacted +parameters, and supports the existing end-time filter. `directed` close and +retry filtering remain rejected for this format. + +### Ordering and filtering + +Multiple readers are merged by `(command_start_timestamp, connection_id, +source_ordinal, record_ordinal)`. The first two keys preserve the observable Go +contract; the final keys make previously unstable exact ties deterministic. +Audit inputs use the configured bounded reorder buffer. Native input rejects +multiple roots and a non-directed prepared-close strategy. + +`--read-only` uses a TiDB-aware bounded tokenizer/normalizer, not a generic SQL +parser. It freezes Go `Normalize` / `NormalizeDigest` output for the statements +used by reporting and the current read-only lexical decision. Tokens inside +comments, quoted strings, identifiers, and executable comments cannot become +keywords. Invalid UTF-8, unterminated constructs, nesting/token-count overflow, +and inputs over 64 MiB are classified explicitly and never panic or loop. + +## Storage contract + +`replayer-core` exposes a TiProxy-owned object-store trait implemented by +Apache OpenDAL. The dependency is exact-pinned, has default features disabled, +and enables only `fs`, `s3`, `gcs`, `azblob`, `oss`, and `cos` plus the Tokio +executor. Provider clients never escape this module. + +The URI adapter freezes the Go query parameters, provider root/path rules, +ordered pagination, and directory listing behavior. Credentials are accepted +only through the provider-specific configuration boundary and are redacted +from errors, diagnostics, job JSON, and logs. Transient reads/lists use the +bounded, cancellation-aware retry classifier from the CP-002 policy; invalid +configuration, permission failures, missing objects, corrupt data, and decode +failures are terminal. + +Local checkpoint and metadata writes use create-in-the-same-directory, +`sync_all`, atomic rename, and parent-directory sync. Remote publication uses a +unique temporary object followed by a conditional final write when the +provider supports it; providers without a compare/create primitive reject +shared-writer mode. A replayer never treats a partially published object as a +traffic file. + +Native streams apply AES-256-CTR and gzip in the same layer order as Go. The +first 16 bytes are the random IV and the key is exactly the first 32 bytes of +the key file. Short keys, short IVs, invalid gzip streams, wrong metadata, and +trailing/truncated records fail closed. Plaintext keys and credentials are +zeroized after construction. + +Dynamic input initially supports the Go-compatible local and S3 modes. A +directory is assigned with FNV-1a modulo `replayer-count`; `replayer-index` +must be lower than that count. Listings are paginated and deduplicated. EOF +polling is driven by a cancellation-aware timer rather than an unconditional +sleep. + +## Replay engine contract + +The engine maintains a bounded queue per logical captured connection and one +socket owner per live replay connection. It preserves per-connection order, +uses the first captured timestamp and `--speed` to derive absolute deadlines, +and sends late commands immediately. Buffer, connection, pending-command, and +slowdown thresholds are explicit; overflow cancels the job with a typed error. + +Direct MySQL connections support TCP and backend TLS with bounded dial, +handshake, packet, and response deadlines. Authentication supports the TiDB +paths used by the Go replayer (`mysql_native_password` and +`caching_sha2_password`, including TLS full-auth and auth switch). Passwords, +scrambles, SQL payloads, and storage credentials are absent from `Debug` and +error output. + +Each command writes the exact captured payload and drains the complete response +before that connection advances. `COM_STMT_PREPARE` records the replay ID and +parameter count. `EXECUTE`, `FETCH`, `CLOSE`, `RESET`, and `SEND_LONG_DATA` +rewrite only the four-byte statement ID, using the captured-to-replay mapping. +The configured prepared-close strategy remains `directed`, `always`, or +`never`. A backend disconnect retires only its socket; the next command +reconnects and restores the command's current database. `--ignore-errs` +controls command failures, never configuration, corruption, authentication, +or ownership failures. + +`--dry-run` performs the complete read/decrypt/decompress/decode/order/filter +path without a backend. It produces the same command/filter counts and +checkpoint frontier as execution mode. + +Checkpoints contain the current command start/end timestamp frontier and an +input identity digest. Loading a checkpoint with different inputs, format, +filters, partitioning, or encryption metadata fails closed. SIGINT/SIGTERM +stops new scheduling, drains only when graceful cancellation was requested, +publishes the final checkpoint/report atomically, joins all owned tasks, and +then exits. + +## Reports and service mode + +Job JSON preserves the observable fields `type`, `status`, `start_time`, +`end_time`, `progress`, `error`, `last_cmd_ts`, `input`, `username`, `format`, +`speed`, `readonly`, and `addr`; URLs are credential-redacted. History is +bounded to ten jobs. Progress never decreases. Only one job may be running or +stopping. + +Failed commands are grouped by TiDB-compatible normalized digest and written +to the existing `tiproxy_traffic_replay.fail` and `other_errors` schemas. SQL +output is opt-in, written with bounded records, and never includes passwords or +cloud credentials. Service-mode form parsing keeps current names and defaults, +rejects unknown or duplicate scalar fields, and bounds request bodies. Cancel +supports immediate and graceful modes with one absolute timeout. + +## Differential and fault evidence + +The ownership move requires all of the following from the exact PR head: + +1. A Go oracle and the production Rust decoders consume the same native, + plugin, and extension corpus. Semantic command observations, normalized + output/digests, filtering, ordering, prepared mappings, counts, and public + errors compare exactly. The corpus includes TiDB syntax, malformed quoting, + invalid UTF-8, boundary lengths, timestamp ties, encryption, and gzip. +2. Local and MinIO/OpenDAL runs compare URI mapping, multi-page ordering, + not-found/permission/transient errors, credential redaction, cancellation, + and atomic publication. A page-token and retry-class mutation must be killed. +3. A real TiDB run covers plain and TLS authentication, query/init-db/quit, + prepare/execute/long-data/reset/fetch/close, read-only filtering, reconnect, + ignore-errors, SQL/report output, and equal per-connection order at 0.5x, + 1x, and 2x. A statement-ID and deadline mutation must be killed. +4. A process-death run kills the replayer during decode and during an in-flight + response, restarts from the checkpoint, and proves no command before the + committed frontier is replayed twice. Dynamic-input owner partitioning is + stable across restart. A non-atomic-checkpoint mutation must be killed. +5. Rust format, lint, unit, doc, release (amd64/arm64), supply-chain, stale-lock, + negative, and `PARITY-EXCL-001` gates are green. The released artifact + reports the repository version/commit/build time and the Go `cmd/replayer` + build target resolves to the Rust artifact after cutover. + +The PR must list the stopped Go command/manager ownership and the remaining +Go traffic surface. No completion claim is made from fixture-only or dry-run +evidence. From 558b843355cc0d4a8802dabc95bc210c0d91b5bb Mon Sep 17 00:00:00 2001 From: bb7133 Date: Thu, 3 Sep 2026 16:29:53 +0800 Subject: [PATCH 2/4] docs: clarify replayer ownership and restart bounds --- docs/design/rust-offline-replayer.md | 31 +++++++++++++++++++++------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/docs/design/rust-offline-replayer.md b/docs/design/rust-offline-replayer.md index dfd9b20f6..083d682dc 100644 --- a/docs/design/rust-offline-replayer.md +++ b/docs/design/rust-offline-replayer.md @@ -33,6 +33,11 @@ The implementation is split into: Neither crate depends on `control-proto`, the legacy bridge, the topology owner, or the SQL-serving dataplane. There is no replay payload over IPC or FFI. A replayer process has exactly one job owner and at most one active job. +That invariant is process-local: during the migration window the standalone +Rust process and the integrated Go traffic API are not mechanically fenced +from each other. Deployments must not point both at the same replay input, +checkpoint, and target at once. Cross-process coordination is an operational +precondition, not an exactly-once lock supplied by this executable. ## Input contracts @@ -105,11 +110,15 @@ configuration, permission failures, missing objects, corrupt data, and decode failures are terminal. Local checkpoint and metadata writes use create-in-the-same-directory, -`sync_all`, atomic rename, and parent-directory sync. Remote publication uses a -unique temporary object followed by a conditional final write when the -provider supports it; providers without a compare/create primitive reject -shared-writer mode. A replayer never treats a partially published object as a -traffic file. +`sync_all`, atomic rename, and parent-directory sync. The current Go checkpoint +and SQL-output flags are filesystem paths, so `CP-REPLAYER` uses OpenDAL cloud +services only for read/list/stat and rejects remote checkpoint or output URLs. +It therefore has no shared remote writer and makes no cross-provider atomic +publication claim. If a later owner adds remote publication, it must query +OpenDAL's `Capability.write_with_if_not_exists` at runtime and use a provider's +native atomic conditional-create primitive; read-then-write emulation is not +acceptable. Providers without that capability must reject shared-writer mode. +Capture-side remote publication remains owned by `CP-CAPTURE` (`#151`). Native streams apply AES-256-CTR and gzip in the same layer order as Go. The first 16 bytes are the random IV and the key is exactly the first 32 bytes of @@ -157,7 +166,12 @@ input identity digest. Loading a checkpoint with different inputs, format, filters, partitioning, or encryption metadata fails closed. SIGINT/SIGTERM stops new scheduling, drains only when graceful cancellation was requested, publishes the final checkpoint/report atomically, joins all owned tasks, and -then exits. +then exits. The restart contract is explicitly bounded at-least-once, not +exactly-once: a command whose request was sent but whose response/frontier was +not committed at process death may execute once again after restart. Bare +MySQL supplies no idempotency token or transaction with which the replayer +could rule that out, so write traffic can have a duplicate effect at the +frontier. ## Reports and service mode @@ -192,8 +206,9 @@ The ownership move requires all of the following from the exact PR head: 1x, and 2x. A statement-ID and deadline mutation must be killed. 4. A process-death run kills the replayer during decode and during an in-flight response, restarts from the checkpoint, and proves no command before the - committed frontier is replayed twice. Dynamic-input owner partitioning is - stable across restart. A non-atomic-checkpoint mutation must be killed. + committed frontier is replayed twice while allowing only the one boundary + in-flight command to repeat. Dynamic-input owner partitioning is stable + across restart. A non-atomic-checkpoint mutation must be killed. 5. Rust format, lint, unit, doc, release (amd64/arm64), supply-chain, stale-lock, negative, and `PARITY-EXCL-001` gates are green. The released artifact reports the repository version/commit/build time and the Go `cmd/replayer` From d205fe71e3121c012d179d3e19b01d1d864bd8fd Mon Sep 17 00:00:00 2001 From: bb7133 Date: Thu, 3 Sep 2026 19:17:34 +0800 Subject: [PATCH 3/4] replayer: add bounded Rust dry-run foundation --- .github/workflows/rust.yml | 4 + AGENTS.md | 2 + Makefile | 5 +- rust/.cargo/audit.toml | 6 +- rust/Cargo.lock | 608 +++++++- rust/Cargo.toml | 10 + rust/crates/replayer-core/Cargo.toml | 29 + .../examples/cp_replayer_observer.rs | 181 +++ rust/crates/replayer-core/src/checkpoint.rs | 236 +++ rust/crates/replayer-core/src/command.rs | 261 ++++ rust/crates/replayer-core/src/config.rs | 239 ++++ rust/crates/replayer-core/src/decode/audit.rs | 1273 +++++++++++++++++ rust/crates/replayer-core/src/decode/mod.rs | 23 + .../crates/replayer-core/src/decode/native.rs | 306 ++++ .../crates/replayer-core/src/decode/quoted.rs | 128 ++ rust/crates/replayer-core/src/engine.rs | 453 ++++++ rust/crates/replayer-core/src/error.rs | 88 ++ rust/crates/replayer-core/src/lib.rs | 32 + rust/crates/replayer-core/src/storage.rs | 580 ++++++++ rust/crates/replayer-core/tests/dry_run.rs | 155 ++ rust/crates/tiproxy-replayer/Cargo.toml | 22 + rust/crates/tiproxy-replayer/src/main.rs | 184 +++ rust/deny.toml | 11 +- rust/vulnerability-exceptions.toml | 18 + tests/replayer/differential/corpus.v1.json | 86 ++ .../replayer/differential/go-observer/main.go | 248 ++++ tests/replayer/differential/run.sh | 45 + 27 files changed, 5207 insertions(+), 26 deletions(-) create mode 100644 rust/crates/replayer-core/Cargo.toml create mode 100644 rust/crates/replayer-core/examples/cp_replayer_observer.rs create mode 100644 rust/crates/replayer-core/src/checkpoint.rs create mode 100644 rust/crates/replayer-core/src/command.rs create mode 100644 rust/crates/replayer-core/src/config.rs create mode 100644 rust/crates/replayer-core/src/decode/audit.rs create mode 100644 rust/crates/replayer-core/src/decode/mod.rs create mode 100644 rust/crates/replayer-core/src/decode/native.rs create mode 100644 rust/crates/replayer-core/src/decode/quoted.rs create mode 100644 rust/crates/replayer-core/src/engine.rs create mode 100644 rust/crates/replayer-core/src/error.rs create mode 100644 rust/crates/replayer-core/src/lib.rs create mode 100644 rust/crates/replayer-core/src/storage.rs create mode 100644 rust/crates/replayer-core/tests/dry_run.rs create mode 100644 rust/crates/tiproxy-replayer/Cargo.toml create mode 100644 rust/crates/tiproxy-replayer/src/main.rs create mode 100644 tests/replayer/differential/corpus.v1.json create mode 100644 tests/replayer/differential/go-observer/main.go create mode 100755 tests/replayer/differential/run.sh diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index d6ccc9746..73e6f3423 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -32,6 +32,7 @@ on: - "scripts/generate-control-proto.sh" - "tests/controlplane/**" - "tests/dataplane/capture-format/**" + - "tests/replayer/**" push: branches: - main @@ -56,6 +57,7 @@ on: - "scripts/generate-control-proto.sh" - "tests/controlplane/**" - "tests/dataplane/capture-format/**" + - "tests/replayer/**" # The control-plane evidence jobs below do not run on pull requests. They # re-prove live etcd / Go-oracle behaviour that a pull request rarely # changes, and re-running them on every push to a branch was the bulk of @@ -131,6 +133,8 @@ jobs: run: make rust-lint - name: Run unit tests run: make rust-test + - name: Compare Go and Rust replay decoders and kill a mutation + run: make replayer-differential-evidence - name: Run doc tests run: make rust-doc-test - name: Reject a stale lockfile diff --git a/AGENTS.md b/AGENTS.md index 480511994..9e41be58c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -78,6 +78,7 @@ When adding or modifying features, prefer extending existing packages before cre - `pkg/util/` - Utilities, including buffered IO, etcd wrapper, HTTP wrapper, simple MySQL lexer, and others. - `rust/crates/mysql-wire/` - MySQL wire-format types and codec boundaries; it does not own sockets or routing decisions. - `rust/crates/proxy-io/` - Client/backend transport ownership, including future TLS, compression, and PROXY protocol support. +- `rust/crates/replayer-core/` - Bounded offline traffic replay primitives: input storage, native/audit decoding, deterministic ordering, filtering, and local checkpoints; it has no control-plane or SQL-serving dependency. - `rust/crates/session-core/` - Protocol-independent session lifecycle and migration state. - `rust/crates/control-router/` - Staged namespace-scoped routing selection, exact config/topology/health capture, and reservation accounting. Production dataplane/binary wiring remains prohibited until full routing parity is complete. - `rust/crates/control-routing/` - Protocol-independent route values, pure group matching/port-conflict primitives, and Go-compatible time arithmetic; it owns no reservation or production selector yet and carries no wire payloads. @@ -94,6 +95,7 @@ When adding or modifying features, prefer extending existing packages before cre - `rust/crates/dataplane/` - Rust dataplane orchestration across wire, transport, session, and control components. - `rust/crates/differential-runner/` - Payload-free Rust consumer for the immutable Go protocol corpus; it drives merged wire/session parsers and emits sharded semantic observations. - `rust/crates/tiproxy-rs/` - Rust dataplane executable and build/version metadata. +- `rust/crates/tiproxy-replayer/` - Standalone Rust offline traffic replayer composition root; replay payloads never enter the SQL-serving process or control bridge. - `tests/compatibility/` - Versioned client-driver and MySQL capability contract shared by the Go/Rust dataplane integration tests. - `tests/controlplane/cproute/api-differential/` - One test-only external router API input/output runner and its frozen acceptance/retirement inventories; synthetic adapter smoke does not qualify recorded-trace acceptance. Its `metrics/` package shares value-only query packets and decoding between recording and API replay; it owns no factor state. Its `clock/` package supplies the shared public event clock to test-build overlays only. - `tests/controlplane/` - Versioned Go control-plane contract and fault catalogs plus the payload-free observation comparator used by every Rust control-plane migration slice. diff --git a/Makefile b/Makefile index 1955be188..5f795f40e 100644 --- a/Makefile +++ b/Makefile @@ -43,7 +43,7 @@ CARGO_DENY_VERSION := 0.20.2 RUST_TOOL_ROOT ?= $(GOBIN)/rust-tools RUST_TOOL_BIN := $(RUST_TOOL_ROOT)/bin -.PHONY: cmd_% test lint parity-drift parity-drift-weekly docker docker-release golangci-lint gocovmerge clean rust-build rust-test rust-doc-test rust-lint rust-release rust-install-tools rust-supply-chain rust-negative-tests control-proto-generate control-proto-generate-check controlplane-contracts controlplane-differential controlplane-differential-self-test controlplane-cp001-evidence controlplane-cplog-evidence controlplane-cpadmin-evidence controlplane-cpdiag-evidence controlplane-cpctl-evidence controlplane-cpmeter-evidence controlplane-cp002-evidence controlplane-cp003-evidence controlplane-cp003-authority-evidence controlplane-cp004-evidence controlplane-cproute-evidence controlplane-cpmetrics-evidence controlplane-cpmetrics-applied-evidence controlplane-cpmetrics-collector-evidence controlplane-cpmetrics-factor-evidence controlplane-cproute-resource-evidence controlplane-cproute-migration-evidence controlplane-cproute-balance-evidence controlplane-cproute-worker-evidence controlplane-cproute-shadow-evidence controlplane-cproute-recorder-evidence controlplane-cproute-read-foundation-evidence controlplane-cproute-time-evidence controlplane-cproute-native-evidence controlplane-cptopo-evidence dataplane-differential dataplane-differential-coverage dataplane-differential-mutation dataplane-integration dataplane-t2-integration dataplane-t3-integration dataplane-t4-integration dataplane-t4-qualification dataplane-integration-go dataplane-integration-self-test +.PHONY: cmd_% test lint parity-drift parity-drift-weekly docker docker-release golangci-lint gocovmerge clean rust-build rust-test rust-doc-test rust-lint rust-release rust-install-tools rust-supply-chain rust-negative-tests control-proto-generate control-proto-generate-check controlplane-contracts controlplane-differential controlplane-differential-self-test controlplane-cp001-evidence controlplane-cplog-evidence controlplane-cpadmin-evidence controlplane-cpdiag-evidence controlplane-cpctl-evidence controlplane-cpmeter-evidence controlplane-cp002-evidence controlplane-cp003-evidence controlplane-cp003-authority-evidence controlplane-cp004-evidence controlplane-cproute-evidence controlplane-cpmetrics-evidence controlplane-cpmetrics-applied-evidence controlplane-cpmetrics-collector-evidence controlplane-cpmetrics-factor-evidence controlplane-cproute-resource-evidence controlplane-cproute-migration-evidence controlplane-cproute-balance-evidence controlplane-cproute-worker-evidence controlplane-cproute-shadow-evidence controlplane-cproute-recorder-evidence controlplane-cproute-read-foundation-evidence controlplane-cproute-time-evidence controlplane-cproute-native-evidence controlplane-cptopo-evidence replayer-differential-evidence dataplane-differential dataplane-differential-coverage dataplane-differential-mutation dataplane-integration dataplane-t2-integration dataplane-t3-integration dataplane-t4-integration dataplane-t4-qualification dataplane-integration-go dataplane-integration-self-test default: cmd @@ -209,6 +209,9 @@ controlplane-cproute-migration-evidence: controlplane-cptopo-evidence: $(RUST_BUILD_ENV) tests/controlplane/cptopo/run.sh +replayer-differential-evidence: + $(RUST_BUILD_ENV) tests/replayer/differential/run.sh + DIFFERENTIAL_SHARD_INDEX ?= 0 DIFFERENTIAL_SHARD_COUNT ?= 4 diff --git a/rust/.cargo/audit.toml b/rust/.cargo/audit.toml index 24beb2d28..249c6b02a 100644 --- a/rust/.cargo/audit.toml +++ b/rust/.cargo/audit.toml @@ -1,5 +1,9 @@ [advisories] -ignore = [] +ignore = [ + "RUSTSEC-2023-0071", + "RUSTSEC-2026-0194", + "RUSTSEC-2026-0195", +] informational_warnings = ["unmaintained", "unsound"] severity_threshold = "low" diff --git a/rust/Cargo.lock b/rust/Cargo.lock index a39515ee2..fdd79d6e9 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -17,6 +17,17 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures 0.2.17", +] + [[package]] name = "aho-corasick" version = "1.1.5" @@ -231,6 +242,17 @@ dependencies = [ "url", ] +[[package]] +name = "backon" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cffb0e931875b666fc4fcb20fee52e9bbd1ef836fd9e9e04ec21555f9f85f7ef" +dependencies = [ + "fastrand", + "gloo-timers", + "tokio", +] + [[package]] name = "backtrace" version = "0.3.76" @@ -258,6 +280,12 @@ version = "0.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + [[package]] name = "bit-vec" version = "0.9.1" @@ -279,6 +307,15 @@ version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "block-buffer" version = "0.12.1" @@ -288,6 +325,15 @@ dependencies = [ "hybrid-array", ] +[[package]] +name = "block-padding" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" +dependencies = [ + "generic-array", +] + [[package]] name = "bumpalo" version = "3.20.3" @@ -306,6 +352,15 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" +[[package]] +name = "cbc" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" +dependencies = [ + "cipher", +] + [[package]] name = "cc" version = "1.4.4" @@ -335,7 +390,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.3.1", "rand_core 0.10.1", ] @@ -346,7 +401,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" dependencies = [ "iana-time-zone", + "js-sys", "num-traits", + "wasm-bindgen", "windows-link", ] @@ -377,30 +434,54 @@ dependencies = [ "half", ] +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common 0.1.7", + "inout", +] + [[package]] name = "clap" -version = "4.6.6" +version = "4.5.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca" +checksum = "4c26d721170e0295f191a69bd9a1f93efcdb0aff38684b61ab5750468972e5f5" dependencies = [ "clap_builder", + "clap_derive", ] [[package]] name = "clap_builder" -version = "4.6.6" +version = "4.5.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889" +checksum = "75835f0c7bf681bfd05abe44e965760fea999a5286c6eb2d59883634fd02011a" dependencies = [ "anstyle", "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.5.49" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a0b5487afeab2deb2ff4e03a807ad1a03ac532ff5a2cee5d86884440c7f7671" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", ] [[package]] name = "clap_lex" -version = "1.1.0" +version = "0.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" +checksum = "c3e64b0cc0439b12df2fa678eae89a1c56a529fd067a9115f7827f1fffd22b32" [[package]] name = "cmov" @@ -418,6 +499,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + [[package]] name = "const-oid" version = "0.10.2" @@ -548,7 +635,7 @@ dependencies = [ "getrandom 0.2.17", "http", "openssl", - "quick-xml", + "quick-xml 0.41.0", "regex", "reqsign-aliyun-oss", "reqsign-aws-core", @@ -648,6 +735,15 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + [[package]] name = "cpufeatures" version = "0.3.1" @@ -657,6 +753,15 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32c" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" +dependencies = [ + "rustc_version", +] + [[package]] name = "crc32fast" version = "1.5.1" @@ -711,6 +816,16 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "crypto-common" version = "0.2.2" @@ -720,6 +835,15 @@ dependencies = [ "hybrid-array", ] +[[package]] +name = "ctr" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" +dependencies = [ + "cipher", +] + [[package]] name = "ctutils" version = "0.4.2" @@ -791,6 +915,17 @@ dependencies = [ "thiserror", ] +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid 0.9.6", + "pem-rfc7468", + "zeroize", +] + [[package]] name = "der-parser" version = "10.0.0" @@ -814,15 +949,27 @@ dependencies = [ "serde_core", ] +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer 0.10.4", + "const-oid 0.9.6", + "crypto-common 0.1.7", + "subtle", +] + [[package]] name = "digest" version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ - "block-buffer", - "const-oid", - "crypto-common", + "block-buffer 0.12.1", + "const-oid 0.10.2", + "crypto-common 0.2.2", "ctutils", ] @@ -1059,6 +1206,16 @@ dependencies = [ "slab", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.2.17" @@ -1104,6 +1261,18 @@ version = "0.32.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" +[[package]] +name = "gloo-timers" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb143cf96099802033e0d4f4963b19fd2e0b728bcf076cd9cf7f6634f092994" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", +] + [[package]] name = "h2" version = "0.4.19" @@ -1176,13 +1345,31 @@ dependencies = [ "url", ] +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest 0.10.7", +] + [[package]] name = "hmac" version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" dependencies = [ - "digest", + "digest 0.11.3", +] + +[[package]] +name = "home" +version = "0.5.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" +dependencies = [ + "windows-sys 0.61.2", ] [[package]] @@ -1460,6 +1647,16 @@ dependencies = [ "hashbrown 0.17.1", ] +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "block-padding", + "generic-array", +] + [[package]] name = "io-uring" version = "0.7.14" @@ -1625,11 +1822,29 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "jsonwebtoken" +version = "9.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a87cc7a48537badeae96744432de36f4be2b4a34a05a5ef32e9dd8a1c169dde" +dependencies = [ + "base64 0.22.1", + "js-sys", + "pem", + "ring", + "serde", + "serde_json", + "simple_asn1", +] + [[package]] name = "lazy_static" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +dependencies = [ + "spin", +] [[package]] name = "legacy-router-shadow" @@ -1650,6 +1865,12 @@ version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + [[package]] name = "linux-raw-sys" version = "0.4.15" @@ -1680,6 +1901,16 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest 0.10.7", +] + [[package]] name = "mea" version = "0.6.7" @@ -1761,6 +1992,22 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-bigint-dig" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e661dda6640fad38e827a6d4a310ff4763082116fe217f279885c97f511bb0b7" +dependencies = [ + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand 0.8.8", + "smallvec", + "zeroize", +] + [[package]] name = "num-conv" version = "0.2.2" @@ -1776,6 +2023,16 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-iter" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b" +dependencies = [ + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -1783,6 +2040,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", + "libm", ] [[package]] @@ -1819,6 +2077,36 @@ version = "11.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" +[[package]] +name = "opendal" +version = "0.53.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f947c4efbca344c1a125753366033c8107f552b2e3f8251815ed1908f116ca3e" +dependencies = [ + "anyhow", + "async-trait", + "backon", + "base64 0.22.1", + "bytes", + "chrono", + "crc32c", + "futures", + "getrandom 0.2.17", + "http", + "http-body", + "log", + "md-5", + "percent-encoding", + "quick-xml 0.37.5", + "reqsign", + "reqwest", + "serde", + "serde_json", + "sha2 0.10.9", + "tokio", + "uuid", +] + [[package]] name = "openssl" version = "0.10.81" @@ -1898,6 +2186,16 @@ version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" +[[package]] +name = "pbkdf2" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +dependencies = [ + "digest 0.10.7", + "hmac 0.12.1", +] + [[package]] name = "pem" version = "3.0.6" @@ -1908,6 +2206,15 @@ dependencies = [ "serde_core", ] +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + [[package]] name = "percent-encoding" version = "2.3.2" @@ -1956,6 +2263,44 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der", + "pkcs8", + "spki", +] + +[[package]] +name = "pkcs5" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e847e2c91a18bfa887dd028ec33f2fe6f25db77db3619024764914affe8b69a6" +dependencies = [ + "aes", + "cbc", + "der", + "pbkdf2", + "scrypt", + "sha2 0.10.9", + "spki", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "pkcs5", + "rand_core 0.6.4", + "spki", +] + [[package]] name = "pkg-config" version = "0.3.34" @@ -2113,6 +2458,16 @@ dependencies = [ "pulldown-cmark", ] +[[package]] +name = "quick-xml" +version = "0.37.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "331e97a1af0bf59823e6eadffe373d7b27f485be8748f71471c662c1f269b7fb" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "quick-xml" version = "0.41.0" @@ -2200,13 +2555,24 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "rand" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + [[package]] name = "rand" version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" dependencies = [ - "rand_chacha", + "rand_chacha 0.9.0", "rand_core 0.9.5", ] @@ -2221,6 +2587,16 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + [[package]] name = "rand_chacha" version = "0.9.0" @@ -2231,6 +2607,15 @@ dependencies = [ "rand_core 0.9.5", ] +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + [[package]] name = "rand_core" version = "0.9.5" @@ -2298,6 +2683,57 @@ version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" +[[package]] +name = "replayer-core" +version = "0.1.0" +dependencies = [ + "aes", + "ctr", + "flate2", + "futures-util", + "opendal", + "ring", + "serde", + "serde_json", + "thiserror", + "time", + "tokio", + "url", + "zeroize", +] + +[[package]] +name = "reqsign" +version = "0.16.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43451dbf3590a7590684c25fb8d12ecdcc90ed3ac123433e500447c7d77ed701" +dependencies = [ + "anyhow", + "async-trait", + "base64 0.22.1", + "chrono", + "form_urlencoded", + "getrandom 0.2.17", + "hex", + "hmac 0.12.1", + "home", + "http", + "jsonwebtoken", + "log", + "once_cell", + "percent-encoding", + "quick-xml 0.37.5", + "rand 0.8.8", + "reqwest", + "rsa", + "rust-ini", + "serde", + "serde_json", + "sha1 0.10.7", + "sha2 0.10.9", + "tokio", +] + [[package]] name = "reqsign-aliyun-oss" version = "3.1.5" @@ -2327,13 +2763,13 @@ dependencies = [ "http", "log", "percent-encoding", - "quick-xml", + "quick-xml 0.41.0", "reqsign-core", "rust-ini", "serde", "serde_json", "serde_urlencoded", - "sha1", + "sha1 0.11.0", ] [[package]] @@ -2345,7 +2781,7 @@ dependencies = [ "bytes", "http", "log", - "quick-xml", + "quick-xml 0.41.0", "reqsign-aws-core", "reqsign-core", "serde", @@ -2362,14 +2798,14 @@ dependencies = [ "bytes", "futures", "hex", - "hmac", + "hmac 0.13.0", "http", "jiff", "log", "mea", "percent-encoding", - "sha1", - "sha2", + "sha1 0.11.0", + "sha2 0.11.0", "windows-sys 0.61.2", ] @@ -2442,6 +2878,27 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rsa" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" +dependencies = [ + "const-oid 0.9.6", + "digest 0.10.7", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core 0.6.4", + "sha2 0.10.9", + "signature", + "spki", + "subtle", + "zeroize", +] + [[package]] name = "rust-ini" version = "0.21.3" @@ -2555,6 +3012,15 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "salsa20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97a22f5af31f73a954c10289c93e8a50cc23d971e80ee446f1f6f7137a088213" +dependencies = [ + "cipher", +] + [[package]] name = "same-file" version = "1.0.6" @@ -2573,6 +3039,17 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "scrypt" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0516a385866c09368f0b5bcd1caff3366aace790fcd46e2bb032697bb172fd1f" +dependencies = [ + "pbkdf2", + "salsa20", + "sha2 0.10.9", +] + [[package]] name = "security-framework" version = "3.7.0" @@ -2676,6 +3153,17 @@ dependencies = [ "zeroize", ] +[[package]] +name = "sha1" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + [[package]] name = "sha1" version = "0.11.0" @@ -2683,8 +3171,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" dependencies = [ "cfg-if", - "cpufeatures", - "digest", + "cpufeatures 0.3.1", + "digest 0.11.3", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", ] [[package]] @@ -2694,8 +3193,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" dependencies = [ "cfg-if", - "cpufeatures", - "digest", + "cpufeatures 0.3.1", + "digest 0.11.3", ] [[package]] @@ -2714,6 +3213,16 @@ dependencies = [ "libc", ] +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest 0.10.7", + "rand_core 0.6.4", +] + [[package]] name = "simd_cesu8" version = "1.2.0" @@ -2730,6 +3239,18 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" +[[package]] +name = "simple_asn1" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d585997b0ac10be3c5ee635f1bab02d512760d14b7c468801ac8a01d9ae5f1d" +dependencies = [ + "num-bigint", + "num-traits", + "thiserror", + "time", +] + [[package]] name = "slab" version = "0.4.12" @@ -2752,12 +3273,34 @@ dependencies = [ "windows-sys 0.60.2", ] +[[package]] +name = "spin" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + [[package]] name = "subtle" version = "2.6.1" @@ -2913,6 +3456,18 @@ dependencies = [ "session-core", ] +[[package]] +name = "tiproxy-replayer" +version = "0.1.0" +dependencies = [ + "clap", + "replayer-core", + "serde_json", + "time", + "tokio", + "zeroize", +] + [[package]] name = "tiproxy-rs" version = "0.1.0" @@ -3305,6 +3860,7 @@ checksum = "2ef6dac1e96601b4fb3acccccff2139741fcb757cb9a36089bf5be91cfb285ce" dependencies = [ "getrandom 0.4.3", "js-sys", + "serde_core", "wasm-bindgen", ] @@ -3314,6 +3870,12 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + [[package]] name = "walkdir" version = "2.5.0" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 2baf77137..aad4a5e3f 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -15,7 +15,9 @@ members = [ "crates/legacy-router-shadow", "crates/mysql-wire", "crates/proxy-io", + "crates/replayer-core", "crates/session-core", + "crates/tiproxy-replayer", "crates/tiproxy-rs", ] exclude = ["ci/fixtures"] @@ -29,6 +31,7 @@ license = "Apache-2.0" repository = "https://github.com/pingcap/tiproxy" [workspace.dependencies] +aes = { version = "=0.8.4", default-features = false } axum = { version = "=0.8.9", default-features = false } hyper = { version = "=1.11.1", default-features = false, features = ["server", "http1", "http2"] } hyper-util = { version = "=0.1.20", default-features = false, features = ["server", "server-auto", "server-graceful", "service", "tokio"] } @@ -49,7 +52,9 @@ reqsign-aws-core = { version = "=3.1.1", default-features = false } reqsign-aws-v4 = { version = "=3.3.0", default-features = false } reqsign-aliyun-oss = { version = "=3.1.5", default-features = false } reqsign-tencent-cos = { version = "=3.0.6", default-features = false } +clap = { version = "=4.5.51", default-features = false, features = ["derive", "error-context", "help", "std", "suggestions", "usage"] } crc32fast = { version = "=1.5.1", default-features = false } +ctr = { version = "=0.9.2", default-features = false } getrandom = "=0.2.17" control-admin = { path = "crates/control-admin", version = "0.1.0" } control-config = { path = "crates/control-config", version = "0.1.0" } @@ -72,7 +77,10 @@ if-addrs = { version = "=0.13.4", default-features = false } libc = { version = "=0.2.189", default-features = false } legacy-router-shadow = { path = "crates/legacy-router-shadow", version = "0.1.0" } mysql-wire = { path = "crates/mysql-wire", version = "0.1.0" } +opendal = { version = "=0.53.3", default-features = false, features = ["executors-tokio", "services-azblob", "services-cos", "services-fs", "services-gcs", "services-oss", "services-s3"] } proxy-io = { path = "crates/proxy-io", version = "0.1.0" } +replayer-core = { path = "crates/replayer-core", version = "0.1.0" } +ring = { version = "=0.17.14", default-features = false, features = ["std"] } serde = { version = "=1.0.228", default-features = false, features = ["derive", "std"] } serde_json = { version = "=1.0.151", default-features = false, features = ["std"] } session-core = { path = "crates/session-core", version = "0.1.0" } @@ -87,11 +95,13 @@ socket2 = { version = "=0.6.1", default-features = false, features = ["all"] } tokio-rustls = { version = "=0.26.4", default-features = false, features = ["ring", "tls12"] } tokio = { version = "=1.47.1", default-features = false, features = ["io-util", "macros", "net", "rt", "rt-multi-thread", "sync", "time"] } thiserror = { version = "=2.0.17", default-features = false } +time = { version = "=0.3.55", default-features = false, features = ["formatting", "parsing", "serde", "std"] } toml = { version = "=0.8.23", default-features = false, features = ["display", "parse"] } tonic = { version = "=0.14.6", default-features = false, features = ["codegen", "server", "transport"] } tonic-prost = { version = "=0.14.6", default-features = false } x509-parser = { version = "=0.18.1", default-features = false } zeroize = { version = "=1.9.0", default-features = false, features = ["alloc"] } +url = { version = "=2.5.7", default-features = false } zstd-pure-rs = { version = "=0.1.2", default-features = false, features = ["std"] } zstd-zero = { version = "=0.1.0", default-features = false } diff --git a/rust/crates/replayer-core/Cargo.toml b/rust/crates/replayer-core/Cargo.toml new file mode 100644 index 000000000..9e5e19979 --- /dev/null +++ b/rust/crates/replayer-core/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "replayer-core" +description = "TiProxy offline traffic replay engine" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true + +[features] +default = [] + +[dependencies] +aes.workspace = true +ctr.workspace = true +flate2.workspace = true +futures-util.workspace = true +opendal.workspace = true +ring.workspace = true +serde.workspace = true +serde_json.workspace = true +thiserror.workspace = true +time.workspace = true +tokio.workspace = true +url.workspace = true +zeroize.workspace = true + +[lints] +workspace = true diff --git a/rust/crates/replayer-core/examples/cp_replayer_observer.rs b/rust/crates/replayer-core/examples/cp_replayer_observer.rs new file mode 100644 index 000000000..ad8522d10 --- /dev/null +++ b/rust/crates/replayer-core/examples/cp_replayer_observer.rs @@ -0,0 +1,181 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Emits canonical Rust decoder observations for the checked-in Go/Rust corpus. + +use std::error::Error; +use std::fs::File; +use std::io::{self, Write}; + +use replayer_core::decode::{AuditDecoder, NativeDecoder}; +use replayer_core::{Command, PreparedCloseStrategy, TrafficFormat}; +use serde::{Deserialize, Serialize}; + +#[derive(Deserialize)] +struct Corpus { + cases: Vec, +} + +#[derive(Deserialize)] +struct Case { + id: String, + format: TrafficFormat, + prepared_close: PreparedCloseStrategy, + filter_retries: bool, + user_allowlist: Vec, + source_ordinal: u64, + records: Vec, + lines: Vec, +} + +#[derive(Deserialize)] +struct NativeRecord { + headers: Vec<(String, String)>, + payload_hex: String, +} + +#[derive(Serialize)] +struct Observation { + case_id: String, + index: u64, + payload_hex: String, + start_unix_nanos: i64, + end_unix_nanos: i64, + connection_id: u64, + upstream_connection_id: u64, + command: u8, + current_database: String, + captured_statement_id: u32, + prepared_statement: String, + statement_type: String, + succeeded: bool, +} + +fn main() -> Result<(), Box> { + let corpus_path = std::env::var("CP_REPLAYER_CORPUS")?; + let corpus: Corpus = serde_json::from_reader(File::open(corpus_path)?)?; + let mut observations = Vec::new(); + for case in corpus.cases { + let commands = decode_case(&case)?; + for (index, command) in commands.into_iter().enumerate() { + observations.push(observe(&case.id, index, command)?); + } + } + if std::env::var_os("CP_REPLAYER_MUTATE_CONNECTION_ID").is_some() + && let Some(first) = observations.first_mut() + { + first.connection_id = first.connection_id.saturating_add(1); + } + serde_json::to_writer(io::stdout().lock(), &observations)?; + io::stdout().lock().write_all(b"\n")?; + Ok(()) +} + +fn decode_case(case: &Case) -> Result, Box> { + let input = build_input(case)?; + match case.format { + TrafficFormat::Native => { + let mut decoder = NativeDecoder::new(&input, &case.id, 64 * 1024 * 1024, None); + let mut commands = Vec::new(); + while let Some(mut command) = decoder.next_command()? { + command.source_ordinal = case.source_ordinal; + commands.push(command); + } + Ok(commands) + } + TrafficFormat::AuditLogPlugin | TrafficFormat::AuditLogExtension => { + let mut decoder = AuditDecoder::new( + case.format, + &case.id, + 64 * 1024 * 1024, + case.prepared_close, + None, + None, + case.filter_retries, + &case.user_allowlist, + case.source_ordinal, + )?; + Ok(decoder.decode_all(&input)?) + } + } +} + +fn build_input(case: &Case) -> Result, Box> { + let mut input = Vec::new(); + if case.format == TrafficFormat::Native { + for record in &case.records { + for (key, value) in &record.headers { + writeln!(input, "# {key}: {value}")?; + } + let payload = decode_hex(&record.payload_hex)?; + writeln!(input, "# Payload_len: {}", payload.len())?; + input.extend_from_slice(&payload); + input.push(b'\n'); + } + } else { + for line in &case.lines { + input.extend_from_slice(line.as_bytes()); + input.push(b'\n'); + } + } + Ok(input) +} + +fn observe(case_id: &str, index: usize, command: Command) -> Result> { + Ok(Observation { + case_id: case_id.to_owned(), + index: u64::try_from(index)?, + payload_hex: encode_hex(&command.payload), + start_unix_nanos: i64::try_from(command.start_time.unix_timestamp_nanos())?, + end_unix_nanos: command + .end_time + .map_or(Ok(0), |value| i64::try_from(value.unix_timestamp_nanos()))?, + connection_id: command.connection_id, + upstream_connection_id: command.upstream_connection_id, + command: command.command.byte(), + current_database: command.current_database, + captured_statement_id: command.captured_statement_id.unwrap_or_default(), + prepared_statement: command.prepared_statement.unwrap_or_default(), + statement_type: command.statement_type.unwrap_or_default(), + succeeded: command.succeeded, + }) +} + +fn decode_hex(value: &str) -> Result, io::Error> { + if !value.len().is_multiple_of(2) { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "hex payload has odd length", + )); + } + value + .as_bytes() + .chunks_exact(2) + .map(|pair| { + let text = std::str::from_utf8(pair) + .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?; + u8::from_str_radix(text, 16) + .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error)) + }) + .collect() +} + +fn encode_hex(value: &[u8]) -> String { + let mut output = String::with_capacity(value.len().saturating_mul(2)); + for byte in value { + use std::fmt::Write as _; + let _ = write!(output, "{byte:02x}"); + } + output +} diff --git a/rust/crates/replayer-core/src/checkpoint.rs b/rust/crates/replayer-core/src/checkpoint.rs new file mode 100644 index 000000000..9aa07c21b --- /dev/null +++ b/rust/crates/replayer-core/src/checkpoint.rs @@ -0,0 +1,236 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Versioned, atomic local replay checkpoints. + +use std::fs::{File, OpenOptions}; +use std::io::Write; +use std::path::Path; +use std::sync::atomic::{AtomicU64, Ordering}; + +use ring::digest::{SHA256, digest}; +use serde::{Deserialize, Serialize}; + +use crate::ReplayError; + +const CHECKPOINT_SCHEMA_VERSION: u32 = 1; +static TEMPORARY_SEQUENCE: AtomicU64 = AtomicU64::new(0); + +/// Digest binding a checkpoint to the exact replay input and filters. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct InputIdentity { + /// Lowercase SHA-256 hexadecimal digest. + pub sha256: String, +} + +impl InputIdentity { + /// Hashes a canonical byte representation. + #[must_use] + pub fn from_canonical_bytes(bytes: &[u8]) -> Self { + let value = digest(&SHA256, bytes); + let mut sha256 = String::with_capacity(value.as_ref().len() * 2); + for byte in value.as_ref() { + use std::fmt::Write as _; + let _ = write!(sha256, "{byte:02x}"); + } + Self { sha256 } + } +} + +/// Durable replay frontier. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct Checkpoint { + schema_version: u32, + /// Exact replay identity. + pub input_identity: InputIdentity, + /// Last committed command start time in Unix nanoseconds. + pub command_start_unix_nanos: i128, + /// Last committed command end time in Unix nanoseconds. + pub command_end_unix_nanos: i128, + /// Credential-redacted logical source path. + pub source: String, + /// Replay-local logical connection identifier. + pub connection_id: u64, + /// Stable input-source ordinal used to break timestamp ties. + pub source_ordinal: u64, + /// Stable source-local record ordinal. + pub record_ordinal: u64, + /// Number of commands committed before and including this frontier. + pub committed_commands: u64, + /// Number of read-only-filtered commands before and including this frontier. + pub filtered_commands: u64, +} + +impl Checkpoint { + /// Creates a versioned checkpoint. + #[must_use] + #[allow(clippy::too_many_arguments)] + pub const fn new( + input_identity: InputIdentity, + command_start_unix_nanos: i128, + command_end_unix_nanos: i128, + source: String, + connection_id: u64, + source_ordinal: u64, + record_ordinal: u64, + committed_commands: u64, + filtered_commands: u64, + ) -> Self { + Self { + schema_version: CHECKPOINT_SCHEMA_VERSION, + input_identity, + command_start_unix_nanos, + command_end_unix_nanos, + source, + connection_id, + source_ordinal, + record_ordinal, + committed_commands, + filtered_commands, + } + } + + /// Loads and validates a checkpoint identity. Empty and missing files are + /// treated as no checkpoint, matching the Go command. + /// + /// # Errors + /// + /// Returns an error when the file cannot be read, is malformed, has an + /// unsupported schema, or belongs to a different replay input. + pub fn load(path: &Path, expected: &InputIdentity) -> Result, ReplayError> { + let bytes = match std::fs::read(path) { + Ok(bytes) => bytes, + Err(source) if source.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(source) => { + return Err(ReplayError::Io { + operation: "read checkpoint", + path: path.to_path_buf(), + source, + }); + } + }; + if bytes.is_empty() { + return Ok(None); + } + let checkpoint: Self = serde_json::from_slice(&bytes) + .map_err(|error| ReplayError::Checkpoint(format!("invalid JSON: {error}")))?; + if checkpoint.schema_version != CHECKPOINT_SCHEMA_VERSION { + return Err(ReplayError::Checkpoint(format!( + "unsupported schema version {}", + checkpoint.schema_version + ))); + } + if &checkpoint.input_identity != expected { + return Err(ReplayError::Checkpoint( + "input identity does not match this replay".to_owned(), + )); + } + Ok(Some(checkpoint)) + } + + /// Persists with same-directory create, file sync, rename, and directory + /// sync. No remote path is accepted by this API. + /// + /// # Errors + /// + /// Returns an error when the checkpoint cannot be encoded or durably + /// written to the requested local path. + pub fn save_atomic(&self, path: &Path) -> Result<(), ReplayError> { + let parent = path.parent().unwrap_or_else(|| Path::new(".")); + let file_name = path + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| { + ReplayError::Checkpoint("checkpoint path has no file name".to_owned()) + })?; + let sequence = TEMPORARY_SEQUENCE.fetch_add(1, Ordering::Relaxed); + let temporary = parent.join(format!( + ".{file_name}.{}.{}.tmp", + std::process::id(), + sequence + )); + let result = self.write_temporary(&temporary).and_then(|()| { + std::fs::rename(&temporary, path).map_err(|source| ReplayError::Io { + operation: "rename checkpoint", + path: path.to_path_buf(), + source, + })?; + let directory = File::open(parent).map_err(|source| ReplayError::Io { + operation: "open checkpoint directory", + path: parent.to_path_buf(), + source, + })?; + directory.sync_all().map_err(|source| ReplayError::Io { + operation: "sync checkpoint directory", + path: parent.to_path_buf(), + source, + }) + }); + if result.is_err() { + let _ = std::fs::remove_file(&temporary); + } + result + } + + fn write_temporary(&self, path: &Path) -> Result<(), ReplayError> { + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .open(path) + .map_err(|source| ReplayError::Io { + operation: "create checkpoint temporary", + path: path.to_path_buf(), + source, + })?; + serde_json::to_writer(&mut file, self) + .map_err(|error| ReplayError::Checkpoint(format!("encode JSON: {error}")))?; + file.write_all(b"\n").map_err(|source| ReplayError::Io { + operation: "write checkpoint temporary", + path: path.to_path_buf(), + source, + })?; + file.sync_all().map_err(|source| ReplayError::Io { + operation: "sync checkpoint temporary", + path: path.to_path_buf(), + source, + }) + } +} + +#[cfg(test)] +#[allow(clippy::expect_used)] +mod tests { + use super::*; + + #[test] + fn atomic_round_trip_and_identity_fence() { + let directory = std::env::temp_dir().join(format!( + "tiproxy-replayer-checkpoint-{}", + std::process::id() + )); + std::fs::create_dir_all(&directory).expect("create temp directory"); + let path = directory.join("checkpoint.json"); + let identity = InputIdentity::from_canonical_bytes(b"input-a"); + let checkpoint = + Checkpoint::new(identity.clone(), 10, 11, "input".to_owned(), 7, 1, 2, 3, 1); + checkpoint.save_atomic(&path).expect("save checkpoint"); + assert_eq!( + Checkpoint::load(&path, &identity).expect("load"), + Some(checkpoint) + ); + let other = InputIdentity::from_canonical_bytes(b"input-b"); + assert!(Checkpoint::load(&path, &other).is_err()); + std::fs::remove_dir_all(directory).expect("remove temp directory"); + } +} diff --git a/rust/crates/replayer-core/src/command.rs b/rust/crates/replayer-core/src/command.rs new file mode 100644 index 000000000..d2ff2ed7b --- /dev/null +++ b/rust/crates/replayer-core/src/command.rs @@ -0,0 +1,261 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Replay command model shared by all input decoders. + +use std::fmt; +use std::str::FromStr; + +use serde::{Deserialize, Serialize}; +use time::OffsetDateTime; + +use crate::ReplayError; + +/// Traffic input format accepted by the replayer. +#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum TrafficFormat { + /// `TiProxy`'s binary-preserving native capture format. + #[default] + Native, + /// `TiDB` audit-log plugin format. + AuditLogPlugin, + /// `TiDB` audit-log extension format. + AuditLogExtension, +} + +impl TrafficFormat { + /// Whether the format is one of `TiDB`'s audit formats. + #[must_use] + pub const fn is_audit(self) -> bool { + matches!(self, Self::AuditLogPlugin | Self::AuditLogExtension) + } +} + +impl fmt::Display for TrafficFormat { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::Native => "native", + Self::AuditLogPlugin => "audit_log_plugin", + Self::AuditLogExtension => "audit_log_extension", + }) + } +} + +impl FromStr for TrafficFormat { + type Err = ReplayError; + + fn from_str(value: &str) -> Result { + match value { + "" | "native" => Ok(Self::Native), + "audit_log_plugin" => Ok(Self::AuditLogPlugin), + "audit_log_extension" => Ok(Self::AuditLogExtension), + other => Err(ReplayError::Config(format!( + "invalid traffic file format {other}" + ))), + } + } +} + +/// Prepared-statement close policy. +#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum PreparedCloseStrategy { + /// Replay only captured close commands. + #[default] + Directed, + /// Close immediately after each execution. + Always, + /// Reuse each distinct statement for the life of the connection. + Never, +} + +impl fmt::Display for PreparedCloseStrategy { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::Directed => "directed", + Self::Always => "always", + Self::Never => "never", + }) + } +} + +impl FromStr for PreparedCloseStrategy { + type Err = ReplayError; + + fn from_str(value: &str) -> Result { + match value { + "directed" => Ok(Self::Directed), + "always" => Ok(Self::Always), + "never" => Ok(Self::Never), + other => Err(ReplayError::Config(format!( + "invalid prepared statement close strategy {other}" + ))), + } + } +} + +/// One-byte `MySQL` command code. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(transparent)] +pub struct CommandCode(u8); + +impl CommandCode { + /// `COM_QUIT`. + pub const QUIT: Self = Self(0x01); + /// `COM_INIT_DB`. + pub const INIT_DB: Self = Self(0x02); + /// `COM_QUERY`. + pub const QUERY: Self = Self(0x03); + /// `COM_CREATE_DB`. + pub const CREATE_DB: Self = Self(0x05); + /// `COM_DROP_DB`. + pub const DROP_DB: Self = Self(0x06); + /// `COM_DELAYED_INSERT`. + pub const DELAYED_INSERT: Self = Self(0x10); + /// `COM_STMT_PREPARE`. + pub const STMT_PREPARE: Self = Self(0x16); + /// `COM_STMT_EXECUTE`. + pub const STMT_EXECUTE: Self = Self(0x17); + /// `COM_STMT_SEND_LONG_DATA`. + pub const STMT_SEND_LONG_DATA: Self = Self(0x18); + /// `COM_STMT_CLOSE`. + pub const STMT_CLOSE: Self = Self(0x19); + /// `COM_STMT_RESET`. + pub const STMT_RESET: Self = Self(0x1a); + /// `COM_STMT_FETCH`. + pub const STMT_FETCH: Self = Self(0x1c); + + /// Returns the wire byte. + #[must_use] + pub const fn byte(self) -> u8 { + self.0 + } + + /// Parses the names emitted by the Go native encoder. + /// + /// # Errors + /// + /// Returns an error for a command name outside the Go command table. + pub fn from_go_name(value: &str) -> Result { + const NAMES: [&str; 32] = [ + "Sleep", + "Quit", + "InitDB", + "Query", + "FieldList", + "CreateDB", + "DropDB", + "Refresh", + "(DEPRECATED)Shutdown", + "Statistics", + "ProcessInfo", + "Connect", + "ProcessKill", + "Debug", + "Ping", + "Time", + "DelayedInsert", + "ChangeUser", + "BinlogDump", + "TableDump", + "ConnectOut", + "RegisterSlave", + "StmtPrepare", + "StmtExecute", + "StmtSendLongData", + "StmtClose", + "StmtReset", + "SetOption", + "StmtFetch", + "Daemon", + "BinlogDumpGtid", + "ResetConnect", + ]; + NAMES + .iter() + .position(|name| *name == value) + .and_then(|index| u8::try_from(index).ok()) + .map(Self) + .ok_or_else(|| format!("unknown MySQL command {value}")) + } +} + +/// One decoded replay command. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct Command { + /// Command payload including the one-byte command code. + pub payload: Vec, + /// Captured command start time. + #[serde(with = "time::serde::rfc3339")] + pub start_time: OffsetDateTime, + /// Captured command end time, when the input provides it. + #[serde(with = "time::serde::rfc3339::option")] + pub end_time: Option, + /// Replay-local logical connection identifier. + pub connection_id: u64, + /// Original `TiDB` connection identifier. + pub upstream_connection_id: u64, + /// `MySQL` command code. + pub command: CommandCode, + /// Current database that must be restored after reconnect. + pub current_database: String, + /// Captured prepared-statement identifier. + pub captured_statement_id: Option, + /// Prepared SQL text when known. + pub prepared_statement: Option, + /// Audit statement type when known. + pub statement_type: Option, + /// Whether the captured command succeeded. + pub succeeded: bool, + /// Credential-redacted logical source path. + pub source: String, + /// One-based source line for diagnostics. + pub line: u64, + /// Stable input-source ordinal used to break timestamp ties. + pub source_ordinal: u64, + /// Stable record ordinal within one source. + pub record_ordinal: u64, +} + +impl Command { + /// Constructs a minimal command while preserving the required invariants. + #[must_use] + pub fn new( + payload: Vec, + start_time: OffsetDateTime, + connection_id: u64, + source: impl Into, + line: u64, + ) -> Option { + let command = payload.first().copied().map(CommandCode)?; + Some(Self { + payload, + start_time, + end_time: None, + connection_id, + upstream_connection_id: connection_id, + command, + current_database: String::new(), + captured_statement_id: None, + prepared_statement: None, + statement_type: None, + succeeded: true, + source: source.into(), + line, + source_ordinal: 0, + record_ordinal: 0, + }) + } +} diff --git a/rust/crates/replayer-core/src/config.rs b/rust/crates/replayer-core/src/config.rs new file mode 100644 index 000000000..93eb15427 --- /dev/null +++ b/rust/crates/replayer-core/src/config.rs @@ -0,0 +1,239 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Replay configuration and compatibility validation. + +use std::path::PathBuf; +use std::time::Duration; + +use serde::{Deserialize, Serialize}; +use time::OffsetDateTime; + +use crate::{PreparedCloseStrategy, ReplayError, TrafficFormat}; + +/// Maximum native payload and audit-log line size. +pub const DEFAULT_RECORD_LIMIT: usize = 64 * 1024 * 1024; + +/// Validated replay job configuration. +#[derive(Clone, Debug, Deserialize, Serialize)] +#[allow(clippy::struct_excessive_bools)] +pub struct ReplayConfig { + /// Comma-separated input roots for audit logs, or one native root. + pub input: String, + /// Replay speed multiplier. + pub speed: f64, + /// Backend username. + pub username: String, + /// Backend address. + pub address: String, + /// Input format. + pub format: TrafficFormat, + /// Replay only commands classified as read-only. + pub read_only: bool, + /// Absolute replay start time. + #[serde(with = "time::serde::rfc3339")] + pub start_time: OffsetDateTime, + /// Ignore recoverable command failures. + pub ignore_errors: bool, + /// Audit reorder buffer size. + pub reorder_buffer: usize, + /// Prepared-statement close policy. + pub prepared_close: PreparedCloseStrategy, + /// Perform the entire input path without connecting to `TiDB`. + pub dry_run: bool, + /// Optional local checkpoint path. + pub checkpoint_path: Option, + /// Watch for new traffic directories. + pub dynamic_input: bool, + /// Number of dynamic-input replayer owners. + pub replayer_count: u64, + /// This process's dynamic-input owner index. + pub replayer_index: u64, + /// Optional local SQL output path. + pub output_path: Option, + /// Filter audit commands marked as retries. + pub filter_command_with_retry: bool, + /// Case-insensitive audit-log user allowlist. + pub user_allowlist: Vec, + /// Continue watching after the current input reaches EOF. + pub wait_on_eof: bool, + /// Optional command start-time frontier. + #[serde(with = "time::serde::rfc3339::option")] + pub command_start_time: Option, + /// Optional command end-time frontier. + #[serde(with = "time::serde::rfc3339::option")] + pub command_end_time: Option, + /// Optional AES-256 key path. + pub key_file: Option, + /// Per-record allocation limit. + pub record_limit: usize, +} + +impl ReplayConfig { + /// Validates the public Go-compatible replay configuration. + /// + /// # Errors + /// + /// Returns an error when fields violate the Go CLI compatibility rules or + /// the local safety bounds. + pub fn validate(&mut self, now: OffsetDateTime) -> Result<(), ReplayError> { + if self.input.trim().is_empty() { + return Err(ReplayError::Config("input is required".to_owned())); + } + let input_count = self.input.split(',').count(); + if input_count > 1 && !self.format.is_audit() { + return Err(ReplayError::Config( + "only `audit_log_plugin` format supports multiple input files".to_owned(), + )); + } + if !self.dry_run && self.username.is_empty() { + return Err(ReplayError::Config("username is required".to_owned())); + } + if self.speed == 0.0 { + self.speed = 1.0; + } else if !(0.1..=10.0).contains(&self.speed) || !self.speed.is_finite() { + return Err(ReplayError::Config( + "speed should be between 0.1 and 10".to_owned(), + )); + } + if self.start_time + Duration::from_secs(60) < now { + return Err(ReplayError::Config( + "start time should not be in the past".to_owned(), + )); + } + if self.format == TrafficFormat::Native + && self.prepared_close != PreparedCloseStrategy::Directed + { + return Err(ReplayError::Config( + "only `directed` prepared statement close strategy is supported for `native` format" + .to_owned(), + )); + } + if !self.format.is_audit() && self.command_end_time.is_some() { + return Err(ReplayError::Config( + "command end time is only supported for audit formats".to_owned(), + )); + } + if self.format == TrafficFormat::AuditLogExtension { + if self.prepared_close == PreparedCloseStrategy::Directed { + return Err(ReplayError::Config( + "prepared statement directed close strategy is not supported for audit log plugin v2 format" + .to_owned(), + )); + } + if self.filter_command_with_retry { + return Err(ReplayError::Config( + "filtering commands with retry is not supported for audit log plugin v2 format" + .to_owned(), + )); + } + } + if self.dynamic_input { + if input_count != 1 { + return Err(ReplayError::Config( + "dynamic input cannot be enabled with more than one input".to_owned(), + )); + } + if self.replayer_count == 0 || self.replayer_index >= self.replayer_count { + return Err(ReplayError::Config( + "dynamic input requires a valid replayer count and index".to_owned(), + )); + } + } + if self.record_limit == 0 || self.record_limit > DEFAULT_RECORD_LIMIT { + return Err(ReplayError::Config(format!( + "record limit must be between 1 and {DEFAULT_RECORD_LIMIT}" + ))); + } + Ok(()) + } + + /// Returns normalized input roots without accepting empty members. + /// + /// # Errors + /// + /// Returns an error when any comma-separated input root is empty. + pub fn inputs(&self) -> Result, ReplayError> { + self.input + .split(',') + .map(str::trim) + .map(|input| { + if input.is_empty() { + Err(ReplayError::Config( + "input contains an empty root".to_owned(), + )) + } else { + Ok(input) + } + }) + .collect() + } +} + +#[cfg(test)] +#[allow(clippy::expect_used)] +mod tests { + use super::*; + + fn config(now: OffsetDateTime) -> ReplayConfig { + ReplayConfig { + input: "capture".to_owned(), + speed: 1.0, + username: "root".to_owned(), + address: "127.0.0.1:4000".to_owned(), + format: TrafficFormat::Native, + read_only: false, + start_time: now, + ignore_errors: false, + reorder_buffer: 100_000, + prepared_close: PreparedCloseStrategy::Directed, + dry_run: true, + checkpoint_path: None, + dynamic_input: false, + replayer_count: 1, + replayer_index: 0, + output_path: None, + filter_command_with_retry: false, + user_allowlist: Vec::new(), + wait_on_eof: false, + command_start_time: None, + command_end_time: None, + key_file: None, + record_limit: DEFAULT_RECORD_LIMIT, + } + } + + #[test] + fn matches_go_validation_edges() { + let now = OffsetDateTime::UNIX_EPOCH; + let mut valid = config(now); + valid.speed = 0.0; + assert!(valid.validate(now).is_ok()); + assert!((valid.speed - 1.0).abs() < f64::EPSILON); + + let mut multiple_native = config(now); + multiple_native.input = "one,two".to_owned(); + assert!(multiple_native.validate(now).is_err()); + + let mut bad_dynamic = config(now); + bad_dynamic.dynamic_input = true; + bad_dynamic.replayer_count = 1; + bad_dynamic.replayer_index = 1; + assert!(bad_dynamic.validate(now).is_err()); + + let mut extension = config(now); + extension.format = TrafficFormat::AuditLogExtension; + assert!(extension.validate(now).is_err()); + } +} diff --git a/rust/crates/replayer-core/src/decode/audit.rs b/rust/crates/replayer-core/src/decode/audit.rs new file mode 100644 index 000000000..377126049 --- /dev/null +++ b/rust/crates/replayer-core/src/decode/audit.rs @@ -0,0 +1,1273 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::{HashMap, HashSet, VecDeque}; +use std::time::Duration; + +use time::OffsetDateTime; + +use crate::decode::parse_go_quoted; +use crate::{Command, CommandCode, PreparedCloseStrategy, ReplayError, TrafficFormat}; + +const AUDIT_TIME_FORMAT: &str = concat!( + "[year]/[month]/[day] [hour]:[minute]:[second].", + "[subsecond digits:3] [offset_hour sign:mandatory]:[offset_minute]" +); + +#[derive(Clone, Debug)] +enum Parameter { + Null, + Signed(i64), + Unsigned(u64), + Float(f32), + Double(f64), + String(Vec), +} + +#[derive(Clone, Debug, Default)] +struct ConnectionState { + replay_id: u64, + last_statement_id: u32, + prepared_ids: HashSet, + prepared_by_sql: HashMap, + last_write: Option, +} + +#[derive(Clone, Debug)] +struct WriteSignature { + command: String, + sql: String, + params: String, + statement_type: String, + start: OffsetDateTime, + end: OffsetDateTime, +} + +/// Stateful decoder for `TiDB` audit plugin and audit extension records. +pub struct AuditDecoder { + format: TrafficFormat, + path: String, + record_limit: usize, + strategy: PreparedCloseStrategy, + command_start_time: Option, + command_end_time: Option, + filter_retries: bool, + user_allowlist: HashSet, + source_ordinal: u64, + next_connection_id: u64, + record_ordinal: u64, + connections: HashMap, + pending: VecDeque, +} + +impl AuditDecoder { + /// Creates an audit decoder. The source ordinal must fit the Go allocator's + /// ten-bit decoder partition. + /// + /// # Errors + /// + /// Returns an error for a non-audit format or an exhausted source-ordinal + /// partition. + #[allow(clippy::too_many_arguments)] + pub fn new( + format: TrafficFormat, + path: impl Into, + record_limit: usize, + strategy: PreparedCloseStrategy, + command_start_time: Option, + command_end_time: Option, + filter_retries: bool, + user_allowlist: &[String], + source_ordinal: u64, + ) -> Result { + if !format.is_audit() { + return Err(ReplayError::Config( + "audit decoder requires an audit format".to_owned(), + )); + } + if source_ordinal >= 1024 { + return Err(ReplayError::Config(format!( + "decoder source ordinal {source_ordinal} is too large" + ))); + } + Ok(Self { + format, + path: path.into(), + record_limit, + strategy, + command_start_time, + command_end_time, + filter_retries, + user_allowlist: user_allowlist + .iter() + .map(|user| user.trim().to_ascii_lowercase()) + .filter(|user| !user.is_empty()) + .collect(), + source_ordinal, + next_connection_id: source_ordinal << 54, + record_ordinal: 0, + connections: HashMap::new(), + pending: VecDeque::new(), + }) + } + + /// Decodes every bounded line in one decrypted input file. + /// + /// # Errors + /// + /// Returns an error when a line exceeds the configured bound or violates + /// the selected audit grammar. + pub fn decode_all(&mut self, input: &[u8]) -> Result, ReplayError> { + let mut output = Vec::new(); + let mut offset = 0_usize; + let mut line = 0_u64; + while offset < input.len() { + let relative_end = input[offset..] + .iter() + .position(|byte| *byte == b'\n') + .unwrap_or(input.len() - offset); + if relative_end > self.record_limit { + return Err(ReplayError::decode( + &self.path, + offset, + "audit line exceeds record limit", + )); + } + let end = offset + relative_end; + line = line.saturating_add(1); + let text = std::str::from_utf8(&input[offset..end]).map_err(|_| { + ReplayError::decode(&self.path, offset, "audit line is not valid UTF-8") + })?; + self.record_ordinal = self.record_ordinal.saturating_add(1); + let commands = self.decode_line(text, offset, line)?; + output.extend(commands); + offset = if end < input.len() { end + 1 } else { end }; + } + while let Some(command) = self.pending.pop_front() { + output.push(command); + } + Ok(output) + } + + fn decode_line( + &mut self, + line_text: &str, + offset: usize, + line: u64, + ) -> Result, ReplayError> { + let fields = parse_bracket_fields(line_text) + .map_err(|message| ReplayError::decode(&self.path, offset, message))?; + let upstream_connection_id = required(&fields, "CONNECTION_ID", &self.path, offset)? + .parse::() + .map_err(|_| ReplayError::decode(&self.path, offset, "invalid CONNECTION_ID"))?; + match self.format { + TrafficFormat::AuditLogPlugin => { + self.decode_plugin(&fields, upstream_connection_id, offset, line) + } + TrafficFormat::AuditLogExtension => { + self.decode_extension(&fields, upstream_connection_id, offset, line) + } + TrafficFormat::Native => Err(ReplayError::decode( + &self.path, + offset, + "native input routed to audit decoder", + )), + } + } + + fn decode_plugin( + &mut self, + fields: &HashMap, + upstream_connection_id: u64, + offset: usize, + line: u64, + ) -> Result, ReplayError> { + let end_time = parse_audit_time(required(fields, "TIMESTAMP", &self.path, offset)?) + .map_err(|message| ReplayError::decode(&self.path, offset, message))?; + let start_time = match fields.get("COST_TIME").filter(|value| !value.is_empty()) { + Some(cost) => { + let micros = cost + .parse::() + .map_err(|_| ReplayError::decode(&self.path, offset, "invalid COST_TIME"))?; + end_time - duration_from_cost_micros(f64::from(micros), &self.path, offset)? + } + None => end_time, + }; + if self + .command_start_time + .is_some_and(|frontier| start_time < frontier) + || self + .command_end_time + .is_some_and(|frontier| end_time < frontier) + { + return Ok(Vec::new()); + } + if !self.user_allowed(fields) { + return Ok(Vec::new()); + } + let replay_id = self.ensure_connection(upstream_connection_id)?; + let class = required(fields, "EVENT_CLASS", &self.path, offset)?; + if class == "CONNECTION" { + if fields.get("EVENT_SUBCLASS").map(String::as_str) == Some("Disconnect") { + self.connections.remove(&upstream_connection_id); + return Ok(vec![build_command( + vec![CommandCode::QUIT.byte()], + start_time, + end_time, + replay_id, + upstream_connection_id, + fields, + &self.path, + line, + self.source_ordinal, + self.record_ordinal, + )?]); + } + return Ok(Vec::new()); + } + if class != "GENERAL" && class != "TABLE_ACCESS" { + return Err(ReplayError::decode( + &self.path, + offset, + format!("unknown EVENT_CLASS {class}"), + )); + } + if fields.get("EVENT").map(String::as_str) != Some("COMPLETED") { + return Ok(Vec::new()); + } + let encoded_command = fields.get("COMMAND").map_or("", String::as_str); + let command_name = if encoded_command.starts_with('"') { + parse_go_quoted(encoded_command) + .map_err(|message| ReplayError::decode(&self.path, offset, message))? + } else { + encoded_command.to_owned() + }; + let sql = match command_name.as_str() { + "Query" | "Execute" => Some( + unquote_if_needed(required(fields, "SQL_TEXT", &self.path, offset)?) + .map_err(|message| ReplayError::decode(&self.path, offset, message))?, + ), + _ => None, + }; + if matches!(command_name.as_str(), "Query" | "Execute") + && self.filter_retries + && fields.get("RETRY").map(String::as_str) == Some("true") + { + return Ok(Vec::new()); + } + if let Some(sql) = sql.as_deref() + && !self.filter_retries + && (command_name != "Execute" || fields.contains_key("EXECUTE_PARAMS")) + && self.is_duplicate_write( + upstream_connection_id, + fields, + &command_name, + sql, + start_time, + end_time, + )? + { + return Ok(Vec::new()); + } + self.commands_for_event( + fields, + upstream_connection_id, + start_time, + end_time, + line, + &command_name, + sql.as_deref(), + false, + offset, + ) + } + + fn decode_extension( + &mut self, + fields: &HashMap, + upstream_connection_id: u64, + offset: usize, + line: u64, + ) -> Result, ReplayError> { + let end_time = parse_audit_time(required(fields, "_LOG_TIME", &self.path, offset)?) + .map_err(|message| ReplayError::decode(&self.path, offset, message))?; + if self + .command_end_time + .is_some_and(|frontier| end_time < frontier) + { + return Ok(Vec::new()); + } + if !self.user_allowed(fields) { + return Ok(Vec::new()); + } + let replay_id = self.ensure_connection(upstream_connection_id)?; + let encoded_event = required(fields, "EVENT", &self.path, offset)?; + let event = parse_go_quoted(encoded_event) + .map_err(|message| ReplayError::decode(&self.path, offset, message))?; + let event = event + .strip_prefix('[') + .and_then(|value| value.strip_suffix(']')) + .ok_or_else(|| ReplayError::decode(&self.path, offset, "invalid EVENT tuple"))?; + let events: Vec<&str> = event.split(',').collect(); + match events.first().copied() { + Some("CONNECTION") if events.get(1).copied() == Some("DISCONNECT") => { + self.connections.remove(&upstream_connection_id); + Ok(vec![build_command( + vec![CommandCode::QUIT.byte()], + end_time, + end_time, + replay_id, + upstream_connection_id, + fields, + &self.path, + line, + self.source_ordinal, + self.record_ordinal, + )?]) + } + Some("QUERY") => { + let sql = match fields.get("SQL_TEXT") { + Some(value) if !value.is_empty() => unquote_if_needed(value) + .map_err(|message| ReplayError::decode(&self.path, offset, message))?, + _ => String::new(), + }; + let command_name = if events.get(1).copied() == Some("EXECUTE") { + "Execute" + } else { + "Query" + }; + self.commands_for_event( + fields, + upstream_connection_id, + end_time, + end_time, + line, + command_name, + Some(&sql), + true, + offset, + ) + } + _ => Ok(Vec::new()), + } + } + + #[allow(clippy::too_many_arguments, clippy::too_many_lines)] + fn commands_for_event( + &mut self, + fields: &HashMap, + upstream_connection_id: u64, + start_time: OffsetDateTime, + end_time: OffsetDateTime, + line: u64, + command_name: &str, + sql: Option<&str>, + extension: bool, + offset: usize, + ) -> Result, ReplayError> { + let mut state = self + .connections + .remove(&upstream_connection_id) + .ok_or_else(|| ReplayError::Config("audit connection state is missing".to_owned()))?; + let mut commands = Vec::with_capacity(3); + match command_name { + "Query" => { + let sql = sql.unwrap_or_default(); + let mut payload = vec![CommandCode::QUERY.byte()]; + payload.extend_from_slice(sql.as_bytes()); + commands.push(build_command( + payload, + start_time, + end_time, + state.replay_id, + upstream_connection_id, + fields, + &self.path, + line, + self.source_ordinal, + self.record_ordinal, + )?); + } + "Close stmt" => { + if self.strategy == PreparedCloseStrategy::Directed { + let statement_id = parse_statement_id(fields, &self.path, offset)?; + if state.prepared_ids.remove(&statement_id) { + commands.push(prepared_command( + close_payload(statement_id), + start_time, + end_time, + state.replay_id, + upstream_connection_id, + fields, + &self.path, + line, + self.source_ordinal, + self.record_ordinal, + statement_id, + None, + )?); + } + } + } + "Execute" => { + let Some(encoded_parameters) = fields.get("EXECUTE_PARAMS") else { + self.connections.insert(upstream_connection_id, state); + return Ok(Vec::new()); + }; + let parameters = if extension { + parse_extension_parameters(encoded_parameters) + } else { + parse_plugin_parameters(encoded_parameters) + } + .map_err(|message| ReplayError::decode(&self.path, offset, message))?; + let sql = sql.unwrap_or_default(); + let (statement_id, should_prepare) = match self.strategy { + PreparedCloseStrategy::Always => { + state.last_statement_id = state.last_statement_id.saturating_add(1); + (state.last_statement_id, true) + } + PreparedCloseStrategy::Directed => { + let statement_id = parse_statement_id(fields, &self.path, offset)?; + (statement_id, state.prepared_ids.insert(statement_id)) + } + PreparedCloseStrategy::Never => { + if let Some(statement_id) = state.prepared_by_sql.get(sql) { + (*statement_id, false) + } else { + state.last_statement_id = state.last_statement_id.saturating_add(1); + state + .prepared_by_sql + .insert(sql.to_owned(), state.last_statement_id); + (state.last_statement_id, true) + } + } + }; + if should_prepare { + let mut payload = vec![CommandCode::STMT_PREPARE.byte()]; + payload.extend_from_slice(sql.as_bytes()); + commands.push(prepared_command( + payload, + start_time, + end_time, + state.replay_id, + upstream_connection_id, + fields, + &self.path, + line, + self.source_ordinal, + self.record_ordinal, + statement_id, + Some(sql), + )?); + } + commands.push(prepared_command( + execute_payload(statement_id, ¶meters)?, + start_time, + end_time, + state.replay_id, + upstream_connection_id, + fields, + &self.path, + line, + self.source_ordinal, + self.record_ordinal, + statement_id, + Some(sql), + )?); + if self.strategy == PreparedCloseStrategy::Always { + commands.push(prepared_command( + close_payload(statement_id), + start_time, + end_time, + state.replay_id, + upstream_connection_id, + fields, + &self.path, + line, + self.source_ordinal, + self.record_ordinal, + statement_id, + Some(sql), + )?); + } + } + _ => {} + } + self.connections.insert(upstream_connection_id, state); + Ok(commands) + } + + fn ensure_connection(&mut self, upstream: u64) -> Result { + if let Some(state) = self.connections.get(&upstream) { + return Ok(state.replay_id); + } + self.next_connection_id = self.next_connection_id.checked_add(1).ok_or_else(|| { + ReplayError::Config("audit connection identifier space exhausted".to_owned()) + })?; + self.connections.insert( + upstream, + ConnectionState { + replay_id: self.next_connection_id, + ..ConnectionState::default() + }, + ); + Ok(self.next_connection_id) + } + + fn user_allowed(&self, fields: &HashMap) -> bool { + self.user_allowlist.is_empty() + || self.user_allowlist.contains( + &fields + .get("USER") + .map_or("", String::as_str) + .trim() + .to_ascii_lowercase(), + ) + } + + fn is_duplicate_write( + &mut self, + upstream: u64, + fields: &HashMap, + command: &str, + sql: &str, + start: OffsetDateTime, + end: OffsetDateTime, + ) -> Result { + let params = fields.get("EXECUTE_PARAMS").cloned().unwrap_or_default(); + let statement_type = fields.get("SQL_STATEMENTS").cloned().unwrap_or_default(); + let is_write = matches!( + statement_type.as_str(), + "Insert" | "Update" | "Delete" | "Replace" + ) || (statement_type == "Select" + && sql.to_ascii_lowercase().contains("for update")); + let state = self + .connections + .get_mut(&upstream) + .ok_or_else(|| ReplayError::Config("audit connection state is missing".to_owned()))?; + if !is_write { + state.last_write = None; + return Ok(false); + } + let duplicate = state.last_write.as_ref().is_some_and(|last| { + last.command == command + && last.sql == sql + && last.params == params + && last.statement_type == statement_type + && last.end >= start + && start - last.start <= Duration::from_millis(1) + }); + if !duplicate { + state.last_write = Some(WriteSignature { + command: command.to_owned(), + sql: sql.to_owned(), + params, + statement_type, + start, + end, + }); + } + Ok(duplicate) + } +} + +/// Parses the `TiDB` audit logger's bounded bracketed field grammar. +/// +/// # Errors +/// +/// Returns an error for duplicate fields, unbalanced delimiters, unterminated +/// quoted values, or invalid UTF-8 boundaries. +pub fn parse_bracket_fields(line: &str) -> Result, String> { + let bytes = line.as_bytes(); + let mut fields = HashMap::new(); + let mut index = 0_usize; + let mut first = true; + while index < bytes.len() { + if bytes[index] != b'[' { + index += 1; + continue; + } + let field_start = index + 1; + let mut cursor = field_start; + let mut separator = None; + let mut quote = None; + while cursor < bytes.len() { + let byte = bytes[cursor]; + if let Some(delimiter) = quote { + if byte == b'\\' { + cursor = cursor + .checked_add(2) + .ok_or_else(|| "quoted field offset overflow".to_owned())?; + continue; + } + if byte == delimiter { + quote = None; + } + } else { + match byte { + b'"' | b'\'' => quote = Some(byte), + b'=' if separator.is_none() => separator = Some(cursor), + b']' => break, + _ => {} + } + } + cursor += 1; + } + if quote.is_some() { + return Err("unterminated quote in audit field".to_owned()); + } + if cursor >= bytes.len() { + return Err("unterminated bracket in audit line".to_owned()); + } + let value_start = separator.map_or(field_start, |position| position + 1); + let value = line + .get(value_start..cursor) + .ok_or_else(|| "audit field is not on UTF-8 boundaries".to_owned())?; + if first { + fields.insert("_LOG_TIME".to_owned(), value.to_owned()); + first = false; + } else if let Some(position) = separator { + if position == field_start { + return Err("empty audit field key".to_owned()); + } + let key = line + .get(field_start..position) + .ok_or_else(|| "audit key is not on UTF-8 boundaries".to_owned())?; + if fields.insert(key.to_owned(), value.to_owned()).is_some() { + return Err(format!("duplicate audit field {key}")); + } + } + index = cursor + 1; + } + Ok(fields) +} + +fn parse_audit_time(value: &str) -> Result { + let format = time::format_description::parse_borrowed::<2>(AUDIT_TIME_FORMAT) + .map_err(|_| "internal audit timestamp format is invalid".to_owned())?; + OffsetDateTime::parse(value, &format).map_err(|_| format!("invalid audit timestamp {value}")) +} + +#[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] +fn duration_from_cost_micros( + micros: f64, + path: &str, + offset: usize, +) -> Result { + const MAX_DURATION_NANOS: f64 = 9_223_372_036_854_775_807.0; + let nanos = micros * 1_000.0; + if !nanos.is_finite() || !(0.0..=MAX_DURATION_NANOS).contains(&nanos) { + return Err(ReplayError::decode( + path, + offset, + "COST_TIME is outside the supported range", + )); + } + // Go converts COST_TIME * time.Microsecond to an integer duration by + // truncating toward zero. The bounds above make this cast explicit. + Ok(Duration::from_nanos(nanos as u64)) +} + +fn required<'a>( + fields: &'a HashMap, + key: &str, + path: &str, + offset: usize, +) -> Result<&'a str, ReplayError> { + fields + .get(key) + .filter(|value| !value.is_empty()) + .map(String::as_str) + .ok_or_else(|| ReplayError::decode(path, offset, format!("missing {key}"))) +} + +fn unquote_if_needed(value: &str) -> Result { + if value.starts_with('"') { + parse_go_quoted(value) + } else if value.is_empty() { + Err("empty SQL or command value".to_owned()) + } else { + Ok(value.to_owned()) + } +} + +fn parse_statement_id( + fields: &HashMap, + path: &str, + offset: usize, +) -> Result { + required(fields, "PREPARED_STMT_ID", path, offset)? + .parse::() + .map_err(|_| ReplayError::decode(path, offset, "invalid PREPARED_STMT_ID")) +} + +#[allow(clippy::too_many_arguments)] +fn build_command( + payload: Vec, + start: OffsetDateTime, + end: OffsetDateTime, + replay_connection_id: u64, + upstream_connection_id: u64, + fields: &HashMap, + path: &str, + line: u64, + source_ordinal: u64, + record_ordinal: u64, +) -> Result { + let mut command = Command::new(payload, start, replay_connection_id, path.to_owned(), line) + .ok_or_else(|| ReplayError::decode(path, 0, "empty generated command"))?; + command.end_time = Some(end); + command.upstream_connection_id = upstream_connection_id; + command.current_database = fields.get("CURRENT_DB").cloned().unwrap_or_default(); + command.statement_type = fields.get("SQL_STATEMENTS").cloned(); + command.source_ordinal = source_ordinal; + command.record_ordinal = record_ordinal; + Ok(command) +} + +#[allow(clippy::too_many_arguments)] +fn prepared_command( + payload: Vec, + start: OffsetDateTime, + end: OffsetDateTime, + replay_connection_id: u64, + upstream_connection_id: u64, + fields: &HashMap, + path: &str, + line: u64, + source_ordinal: u64, + record_ordinal: u64, + statement_id: u32, + sql: Option<&str>, +) -> Result { + let mut command = build_command( + payload, + start, + end, + replay_connection_id, + upstream_connection_id, + fields, + path, + line, + source_ordinal, + record_ordinal, + )?; + command.captured_statement_id = Some(statement_id); + command.prepared_statement = sql.map(str::to_owned); + Ok(command) +} + +fn parse_plugin_parameters(input: &str) -> Result, String> { + let decoded = parse_go_quoted(input).map_err(|error| format!("invalid params: {error}"))?; + let body = decoded + .strip_prefix('[') + .and_then(|value| value.strip_suffix(']')) + .ok_or_else(|| "params have no surrounding brackets".to_owned())?; + if body.is_empty() { + return Ok(Vec::new()); + } + let values = split_quoted_list(body)?; + values + .into_iter() + .map(|value| { + let decoded = parse_go_quoted(value)?; + let (kind, value) = decoded + .split_once(' ') + .ok_or_else(|| format!("no space in param {decoded}"))?; + match kind { + "KindNull" => Ok(Parameter::Null), + "KindInt64" => value + .parse::() + .map(Parameter::Signed) + .map_err(|_| format!("invalid signed param {value}")), + "KindUint64" => value + .parse::() + .map(Parameter::Unsigned) + .map_err(|_| format!("invalid unsigned param {value}")), + "KindFloat32" => value + .parse::() + .map(Parameter::Float) + .map_err(|_| format!("invalid float param {value}")), + "KindFloat64" | "KindMysqlDecimal" => value + .parse::() + .map(Parameter::Double) + .map_err(|_| format!("invalid double param {value}")), + "KindString" => parse_go_quoted(&format!("\"{value}\"")) + .map(|value| Parameter::String(value.into_bytes())), + "KindBytes" => parse_go_quoted(&format!("\"{value}\"")) + .map(|value| Parameter::String(value.into_bytes())), + "KindBinaryLiteral" | "KindMysqlBit" | "KindMysqlSet" | "KindMysqlTime" + | "KindMysqlJSON" => Ok(Parameter::String(value.as_bytes().to_vec())), + "KindMysqlDuration" | "KindMysqlEnum" | "KindInterface" | "KindMinNotNull" + | "KindMaxValue" | "KindRaw" => Err(format!("unsupported param type {kind}")), + _ => Err(format!("unknown param type {kind}")), + } + }) + .collect() +} + +fn parse_extension_parameters(input: &str) -> Result, String> { + let decoded = parse_go_quoted(input).map_err(|error| format!("invalid params: {error}"))?; + let body = decoded + .strip_prefix('[') + .and_then(|value| value.strip_suffix(']')) + .ok_or_else(|| "params have no surrounding brackets".to_owned())?; + if body.is_empty() { + return Ok(Vec::new()); + } + split_extension_list(body).map(|values| { + values + .into_iter() + .map(|value| Parameter::String(value.into_bytes())) + .collect() + }) +} + +fn split_quoted_list(input: &str) -> Result, String> { + let bytes = input.as_bytes(); + let mut values = Vec::new(); + let mut index = 0_usize; + while index < bytes.len() { + while bytes.get(index).is_some_and(u8::is_ascii_whitespace) + || bytes.get(index) == Some(&b',') + { + index += 1; + } + if index >= bytes.len() { + break; + } + if bytes[index] != b'"' { + return Err("expected quoted audit parameter".to_owned()); + } + let start = index; + index += 1; + let mut closed = false; + while index < bytes.len() { + match bytes[index] { + b'\\' => index += 2, + b'"' => { + index += 1; + values.push( + input + .get(start..index) + .ok_or_else(|| "parameter is not on UTF-8 boundaries".to_owned())?, + ); + closed = true; + break; + } + _ => index += 1, + } + } + if !closed { + return Err("unterminated quoted audit parameter".to_owned()); + } + } + Ok(values) +} + +fn split_extension_list(input: &str) -> Result, String> { + let bytes = input.as_bytes(); + let mut values = Vec::new(); + let mut index = 0_usize; + while index < bytes.len() { + while bytes + .get(index) + .is_some_and(|byte| *byte == b',' || byte.is_ascii_whitespace()) + { + index += 1; + } + if index >= bytes.len() { + break; + } + if bytes[index] == b'"' { + let start = index; + index += 1; + while index < bytes.len() { + match bytes[index] { + b'\\' => index += 2, + b'"' => { + index += 1; + let quoted = input + .get(start..index) + .ok_or_else(|| "parameter is not on UTF-8 boundaries".to_owned())?; + values.push(parse_go_quoted(quoted)?); + break; + } + _ => index += 1, + } + } + if index > bytes.len() || bytes.get(index.saturating_sub(1)) != Some(&b'"') { + return Err("unterminated extension parameter".to_owned()); + } + } else { + let start = index; + while index < bytes.len() && bytes[index] != b',' { + index += 1; + } + let value = input + .get(start..index) + .ok_or_else(|| "parameter is not on UTF-8 boundaries".to_owned())? + .trim(); + values.push(value.to_owned()); + } + } + Ok(values) +} + +fn execute_payload(statement_id: u32, parameters: &[Parameter]) -> Result, ReplayError> { + let bitmap_length = parameters.len().div_ceil(8); + let capacity = 10_usize + .checked_add(bitmap_length) + .and_then(|base| base.checked_add(usize::from(!parameters.is_empty()))) + .and_then(|base| base.checked_add(parameters.len().saturating_mul(10))) + .ok_or_else(|| ReplayError::Config("execute payload size overflow".to_owned()))?; + let mut output = Vec::with_capacity(capacity); + output.push(CommandCode::STMT_EXECUTE.byte()); + output.extend_from_slice(&statement_id.to_le_bytes()); + output.push(0); + output.extend_from_slice(&1_u32.to_le_bytes()); + if parameters.is_empty() { + return Ok(output); + } + let bitmap_offset = output.len(); + output.resize(bitmap_offset + bitmap_length, 0); + output.push(1); + for (index, parameter) in parameters.iter().enumerate() { + let (field_type, unsigned) = match parameter { + Parameter::Null => { + output[bitmap_offset + index / 8] |= 1 << (index % 8); + (6_u8, 0_u8) + } + Parameter::Signed(_) => (8, 0), + Parameter::Unsigned(_) => (8, 0x80), + Parameter::Float(_) => (4, 0), + Parameter::Double(_) => (5, 0), + Parameter::String(_) => (0xfe, 0), + }; + output.extend_from_slice(&[field_type, unsigned]); + } + for parameter in parameters { + match parameter { + Parameter::Null => {} + Parameter::Signed(value) => output.extend_from_slice(&value.to_le_bytes()), + Parameter::Unsigned(value) => output.extend_from_slice(&value.to_le_bytes()), + Parameter::Float(value) => output.extend_from_slice(&value.to_le_bytes()), + Parameter::Double(value) => output.extend_from_slice(&value.to_le_bytes()), + Parameter::String(value) => append_length_encoded(&mut output, value)?, + } + } + Ok(output) +} + +fn append_length_encoded(output: &mut Vec, value: &[u8]) -> Result<(), ReplayError> { + let length = u64::try_from(value.len()) + .map_err(|_| ReplayError::Config("parameter length does not fit u64".to_owned()))?; + match length { + 0..=250 => output.push( + u8::try_from(length) + .map_err(|_| ReplayError::Config("parameter length exceeds u8".to_owned()))?, + ), + 251..=65_535 => { + output.push(0xfc); + output.extend_from_slice( + &u16::try_from(length) + .map_err(|_| ReplayError::Config("parameter length exceeds u16".to_owned()))? + .to_le_bytes(), + ); + } + 65_536..=16_777_215 => { + output.push(0xfd); + let bytes = length.to_le_bytes(); + output.extend_from_slice(&bytes[..3]); + } + _ => { + output.push(0xfe); + output.extend_from_slice(&length.to_le_bytes()); + } + } + output.extend_from_slice(value); + Ok(()) +} + +fn close_payload(statement_id: u32) -> Vec { + let mut output = vec![CommandCode::STMT_CLOSE.byte()]; + output.extend_from_slice(&statement_id.to_le_bytes()); + output +} + +#[cfg(test)] +#[allow(clippy::expect_used)] +mod tests { + use super::*; + + #[test] + fn parses_nested_brackets_and_quotes() { + let fields = parse_bracket_fields( + r#"[2025/09/06 17:03:53.720 +08:00] [SQL_TEXT="select \"[=]\""] [CONNECTION_ID=9]"#, + ) + .expect("valid fields"); + assert_eq!( + fields.get("_LOG_TIME").map(String::as_str), + Some("2025/09/06 17:03:53.720 +08:00") + ); + assert_eq!( + fields.get("SQL_TEXT").map(String::as_str), + Some(r#""select \"[=]\"""#) + ); + assert_eq!(fields.get("CONNECTION_ID").map(String::as_str), Some("9")); + } + + #[test] + fn rejects_unterminated_and_duplicate_fields() { + assert!(parse_bracket_fields("[a=b").is_err()); + assert!(parse_bracket_fields("[time] [a=b] [a=c]").is_err()); + } + + #[test] + fn plugin_query_and_retry_filter_match_contract() { + let line = concat!( + "[2025/09/06 17:03:53.720 +08:00] ", + "[TIMESTAMP=2025/09/06 17:03:53.720 +08:00] ", + "[EVENT_CLASS=GENERAL] [EVENT=COMPLETED] [COMMAND=Query] ", + "[SQL_TEXT=SELECT 1] [SQL_STATEMENTS=Select] [CONNECTION_ID=9] ", + "[USER=root] [CURRENT_DB=test] [RETRY=false]\n" + ); + let mut decoder = AuditDecoder::new( + TrafficFormat::AuditLogPlugin, + "audit.log", + 4096, + PreparedCloseStrategy::Directed, + None, + None, + true, + &["ROOT".to_owned()], + 3, + ) + .expect("decoder"); + let commands = decoder.decode_all(line.as_bytes()).expect("decode"); + assert_eq!(commands.len(), 1); + assert_eq!(commands[0].payload, b"\x03SELECT 1"); + assert_eq!(commands[0].connection_id, (3_u64 << 54) + 1); + assert_eq!(commands[0].current_database, "test"); + } + + #[test] + fn extension_always_emits_prepare_execute_close() { + let line = concat!( + "[2026/01/08 19:44:11.114 +08:00] ", + "[EVENT=\"[QUERY,EXECUTE,SELECT]\"] [USER=root] [CONNECTION_ID=7] ", + "[CURRENT_DB=test] [SQL_TEXT=\"SELECT ?\"] [EXECUTE_PARAMS=\"[1]\"]\n" + ); + let mut decoder = AuditDecoder::new( + TrafficFormat::AuditLogExtension, + "audit.log", + 4096, + PreparedCloseStrategy::Always, + None, + None, + false, + &[], + 0, + ) + .expect("decoder"); + let commands = decoder.decode_all(line.as_bytes()).expect("decode"); + assert_eq!(commands.len(), 3); + assert_eq!(commands[0].command, CommandCode::STMT_PREPARE); + assert_eq!(commands[1].command, CommandCode::STMT_EXECUTE); + assert_eq!(commands[2].command, CommandCode::STMT_CLOSE); + assert_eq!( + commands[1].payload, + vec![0x17, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0xfe, 0, 1, b'1'] + ); + } + + #[test] + fn ignored_plugin_events_still_advance_connection_allocator() { + let lines = concat!( + "[2025/09/06 17:03:53.720 +08:00] ", + "[TIMESTAMP=2025/09/06 17:03:53.720 +08:00] ", + "[EVENT_CLASS=GENERAL] [EVENT=STARTING] [COMMAND=Query] ", + "[SQL_TEXT=SELECT 1] [SQL_STATEMENTS=Select] [CONNECTION_ID=40]\n", + "[2025/09/06 17:03:53.720 +08:00] ", + "[TIMESTAMP=2025/09/06 17:03:53.720 +08:00] ", + "[EVENT_CLASS=CONNECTION] [EVENT_SUBCLASS=Connected] [CONNECTION_ID=41]\n", + "[2025/09/06 17:03:53.720 +08:00] ", + "[TIMESTAMP=2025/09/06 17:03:53.720 +08:00] ", + "[EVENT_CLASS=GENERAL] [EVENT=COMPLETED] [COMMAND=Query] ", + "[SQL_TEXT=SELECT 2] [SQL_STATEMENTS=Select] [CONNECTION_ID=42]\n" + ); + let mut decoder = AuditDecoder::new( + TrafficFormat::AuditLogPlugin, + "audit.log", + 4096, + PreparedCloseStrategy::Directed, + None, + None, + false, + &[], + 2, + ) + .expect("decoder"); + let commands = decoder.decode_all(lines.as_bytes()).expect("decode"); + assert_eq!(commands.len(), 1); + assert_eq!(commands[0].connection_id, (2_u64 << 54) + 3); + } + + #[test] + fn disconnect_uses_existing_replay_connection_id() { + let lines = concat!( + "[2025/09/06 17:03:53.720 +08:00] ", + "[TIMESTAMP=2025/09/06 17:03:53.720 +08:00] ", + "[EVENT_CLASS=GENERAL] [EVENT=COMPLETED] [COMMAND=Query] ", + "[SQL_TEXT=SELECT 1] [SQL_STATEMENTS=Select] [CONNECTION_ID=9]\n", + "[2025/09/06 17:03:53.721 +08:00] ", + "[TIMESTAMP=2025/09/06 17:03:53.721 +08:00] ", + "[EVENT_CLASS=CONNECTION] [EVENT_SUBCLASS=Disconnect] [CONNECTION_ID=9]\n", + "[2025/09/06 17:03:53.722 +08:00] ", + "[TIMESTAMP=2025/09/06 17:03:53.722 +08:00] ", + "[EVENT_CLASS=GENERAL] [EVENT=COMPLETED] [COMMAND=Query] ", + "[SQL_TEXT=SELECT 2] [SQL_STATEMENTS=Select] [CONNECTION_ID=9]\n" + ); + let mut decoder = AuditDecoder::new( + TrafficFormat::AuditLogPlugin, + "audit.log", + 4096, + PreparedCloseStrategy::Directed, + None, + None, + false, + &[], + 0, + ) + .expect("decoder"); + let commands = decoder.decode_all(lines.as_bytes()).expect("decode"); + assert_eq!(commands.len(), 3); + assert_eq!(commands[0].connection_id, commands[1].connection_id); + assert_eq!(commands[1].command, CommandCode::QUIT); + assert_ne!(commands[1].connection_id, commands[2].connection_id); + } + + #[test] + fn intervening_read_prevents_non_adjacent_write_deduplication() { + let lines = concat!( + "[2025/09/06 17:03:53.720 +08:00] ", + "[TIMESTAMP=2025/09/06 17:03:53.720 +08:00] ", + "[EVENT_CLASS=GENERAL] [EVENT=COMPLETED] [COMMAND=Query] ", + "[SQL_TEXT=INSERT INTO t VALUES (1)] [SQL_STATEMENTS=Insert] [CONNECTION_ID=9]\n", + "[2025/09/06 17:03:53.720 +08:00] ", + "[TIMESTAMP=2025/09/06 17:03:53.720 +08:00] ", + "[EVENT_CLASS=GENERAL] [EVENT=COMPLETED] [COMMAND=Query] ", + "[SQL_TEXT=SELECT 1] [SQL_STATEMENTS=Select] [CONNECTION_ID=9]\n", + "[2025/09/06 17:03:53.720 +08:00] ", + "[TIMESTAMP=2025/09/06 17:03:53.720 +08:00] ", + "[EVENT_CLASS=GENERAL] [EVENT=COMPLETED] [COMMAND=Query] ", + "[SQL_TEXT=INSERT INTO t VALUES (1)] [SQL_STATEMENTS=Insert] [CONNECTION_ID=9]\n" + ); + let mut decoder = AuditDecoder::new( + TrafficFormat::AuditLogPlugin, + "audit.log", + 4096, + PreparedCloseStrategy::Directed, + None, + None, + false, + &[], + 0, + ) + .expect("decoder"); + let commands = decoder.decode_all(lines.as_bytes()).expect("decode"); + assert_eq!(commands.len(), 3); + } + + #[test] + fn retry_filter_does_not_suppress_prepared_close() { + let lines = concat!( + "[2025/09/06 17:03:53.720 +08:00] ", + "[TIMESTAMP=2025/09/06 17:03:53.720 +08:00] ", + "[EVENT_CLASS=GENERAL] [EVENT=COMPLETED] [COMMAND=Execute] ", + "[SQL_TEXT=SELECT ?] [SQL_STATEMENTS=Select] [CONNECTION_ID=9] ", + "[PREPARED_STMT_ID=7] [EXECUTE_PARAMS=\"[]\"] [RETRY=false]\n", + "[2025/09/06 17:03:53.721 +08:00] ", + "[TIMESTAMP=2025/09/06 17:03:53.721 +08:00] ", + "[EVENT_CLASS=GENERAL] [EVENT=COMPLETED] [COMMAND=\"Close stmt\"] ", + "[CONNECTION_ID=9] [PREPARED_STMT_ID=7] [RETRY=true]\n" + ); + let mut decoder = AuditDecoder::new( + TrafficFormat::AuditLogPlugin, + "audit.log", + 4096, + PreparedCloseStrategy::Directed, + None, + None, + true, + &[], + 0, + ) + .expect("decoder"); + let commands = decoder.decode_all(lines.as_bytes()).expect("decode"); + assert_eq!(commands.len(), 3); + assert_eq!(commands[1].payload, vec![0x17, 7, 0, 0, 0, 0, 1, 0, 0, 0]); + assert_eq!(commands[2].command, CommandCode::STMT_CLOSE); + } + + #[test] + fn plugin_validates_timestamp_before_user_filter() { + let line = concat!( + "[not-a-time] [TIMESTAMP=not-a-time] [EVENT_CLASS=GENERAL] ", + "[EVENT=COMPLETED] [COMMAND=Query] [SQL_TEXT=SELECT 1] ", + "[SQL_STATEMENTS=Select] [CONNECTION_ID=9] [USER=someone-else]\n" + ); + let mut decoder = AuditDecoder::new( + TrafficFormat::AuditLogPlugin, + "audit.log", + 4096, + PreparedCloseStrategy::Directed, + None, + None, + false, + &["root".to_owned()], + 0, + ) + .expect("decoder"); + assert!(decoder.decode_all(line.as_bytes()).is_err()); + } + + #[test] + fn empty_plugin_command_is_ignored_after_allocating_connection() { + let lines = concat!( + "[2025/09/06 17:03:53.720 +08:00] ", + "[TIMESTAMP=2025/09/06 17:03:53.720 +08:00] ", + "[EVENT_CLASS=GENERAL] [EVENT=COMPLETED] [CONNECTION_ID=8]\n", + "[2025/09/06 17:03:53.721 +08:00] ", + "[TIMESTAMP=2025/09/06 17:03:53.721 +08:00] ", + "[EVENT_CLASS=GENERAL] [EVENT=COMPLETED] [COMMAND=Query] ", + "[SQL_TEXT=SELECT 1] [SQL_STATEMENTS=Select] [CONNECTION_ID=9]\n" + ); + let mut decoder = AuditDecoder::new( + TrafficFormat::AuditLogPlugin, + "audit.log", + 4096, + PreparedCloseStrategy::Directed, + None, + None, + false, + &[], + 0, + ) + .expect("decoder"); + let commands = decoder.decode_all(lines.as_bytes()).expect("decode"); + assert_eq!(commands.len(), 1); + assert_eq!(commands[0].connection_id, 2); + } +} diff --git a/rust/crates/replayer-core/src/decode/mod.rs b/rust/crates/replayer-core/src/decode/mod.rs new file mode 100644 index 000000000..751a0e057 --- /dev/null +++ b/rust/crates/replayer-core/src/decode/mod.rs @@ -0,0 +1,23 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Bounded traffic-format decoders. + +mod audit; +mod native; +mod quoted; + +pub use audit::{AuditDecoder, parse_bracket_fields}; +pub use native::NativeDecoder; +pub(crate) use quoted::parse_go_quoted; diff --git a/rust/crates/replayer-core/src/decode/native.rs b/rust/crates/replayer-core/src/decode/native.rs new file mode 100644 index 000000000..585407ad1 --- /dev/null +++ b/rust/crates/replayer-core/src/decode/native.rs @@ -0,0 +1,306 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::HashSet; + +use time::OffsetDateTime; +use time::format_description::well_known::Rfc3339; + +use crate::decode::parse_go_quoted; +use crate::{Command, CommandCode, ReplayError}; + +/// Stateful decoder for `TiProxy` native capture records. +pub struct NativeDecoder<'a> { + input: &'a [u8], + path: String, + offset: usize, + line: u64, + record_limit: usize, + command_start_time: Option, + record_ordinal: u64, +} + +impl<'a> NativeDecoder<'a> { + /// Creates a bounded decoder over one uncompressed, decrypted file. + #[must_use] + pub fn new( + input: &'a [u8], + path: impl Into, + record_limit: usize, + command_start_time: Option, + ) -> Self { + Self { + input, + path: path.into(), + offset: 0, + line: 0, + record_limit, + command_start_time, + record_ordinal: 0, + } + } + + /// Decodes the next record, or `None` at a clean file boundary. + /// + /// # Errors + /// + /// Returns an error when the next record violates the bounded native + /// capture grammar. + pub fn next_command(&mut self) -> Result, ReplayError> { + loop { + if self.offset == self.input.len() { + return Ok(None); + } + let command = self.decode_one()?; + if self + .command_start_time + .is_some_and(|frontier| command.start_time <= frontier) + { + continue; + } + return Ok(Some(command)); + } + } + + #[allow(clippy::too_many_lines)] + fn decode_one(&mut self) -> Result { + let record_offset = self.offset; + let mut seen = HashSet::new(); + let mut start_time = None; + let mut connection_id = None; + let mut command = CommandCode::QUERY; + let mut succeeded = true; + let mut captured_statement_id = None; + let mut prepared_statement = None; + + loop { + let (line_offset, line) = self.read_header_line()?; + let line = std::str::from_utf8(line).map_err(|_| { + ReplayError::decode(&self.path, line_offset, "header is not valid UTF-8") + })?; + let body = line.strip_prefix("# ").ok_or_else(|| { + ReplayError::decode(&self.path, line_offset, "line does not start with '# '") + })?; + let (key, value) = body.split_once(": ").ok_or_else(|| { + ReplayError::decode(&self.path, line_offset, "header does not contain ': '") + })?; + if value.is_empty() { + return Err(ReplayError::decode( + &self.path, + line_offset, + format!("{key} value is empty"), + )); + } + let known = matches!( + key, + "Time" + | "Conn_ID" + | "Cmd_type" + | "Success" + | "Captured_ps_id" + | "Prepared_stmt" + | "Payload_len" + ); + if known && !seen.insert(key.to_owned()) { + return Err(ReplayError::decode( + &self.path, + line_offset, + format!("duplicate {key} header"), + )); + } + match key { + "Time" => { + start_time = Some(OffsetDateTime::parse(value, &Rfc3339).map_err(|_| { + ReplayError::decode(&self.path, line_offset, "invalid RFC3339 Time") + })?); + } + "Conn_ID" => { + let value = value.parse::().map_err(|_| { + ReplayError::decode(&self.path, line_offset, "invalid Conn_ID") + })?; + if value == 0 { + return Err(ReplayError::decode( + &self.path, + line_offset, + "Conn_ID must be positive", + )); + } + connection_id = Some(value); + } + "Cmd_type" => { + command = CommandCode::from_go_name(value) + .map_err(|message| ReplayError::decode(&self.path, line_offset, message))?; + } + "Success" => succeeded = value == "true", + "Captured_ps_id" => { + captured_statement_id = Some(value.parse::().map_err(|_| { + ReplayError::decode(&self.path, line_offset, "invalid Captured_ps_id") + })?); + } + "Prepared_stmt" => { + prepared_statement = Some(parse_go_quoted(value).map_err(|message| { + ReplayError::decode(&self.path, line_offset, message) + })?); + } + "Payload_len" => { + let length = value.parse::().map_err(|_| { + ReplayError::decode(&self.path, line_offset, "invalid Payload_len") + })?; + if length > self.record_limit { + return Err(ReplayError::decode( + &self.path, + line_offset, + format!("Payload_len {length} exceeds limit {}", self.record_limit), + )); + } + let payload_offset = self.offset; + let payload_end = payload_offset.checked_add(length).ok_or_else(|| { + ReplayError::decode(&self.path, payload_offset, "Payload_len overflow") + })?; + if payload_end >= self.input.len() { + return Err(ReplayError::decode( + &self.path, + payload_offset, + "truncated payload or missing final newline", + )); + } + if self.input[payload_end] != b'\n' { + return Err(ReplayError::decode( + &self.path, + payload_end, + "payload is not followed by newline", + )); + } + let mut payload = Vec::with_capacity(length + 1); + payload.push(command.byte()); + payload.extend_from_slice(&self.input[payload_offset..payload_end]); + self.offset = payload_end + 1; + self.line = self.line.saturating_add(1); + self.record_ordinal = self.record_ordinal.saturating_add(1); + let start_time = start_time.ok_or_else(|| { + ReplayError::decode(&self.path, record_offset, "missing Time header") + })?; + let connection_id = connection_id.ok_or_else(|| { + ReplayError::decode(&self.path, record_offset, "missing Conn_ID header") + })?; + let mut decoded = Command::new( + payload, + start_time, + connection_id, + self.path.clone(), + self.line, + ) + .ok_or_else(|| { + ReplayError::decode(&self.path, payload_offset, "empty payload") + })?; + decoded.command = command; + decoded.succeeded = succeeded; + decoded.captured_statement_id = captured_statement_id; + decoded.prepared_statement = prepared_statement; + decoded.record_ordinal = self.record_ordinal; + return Ok(decoded); + } + _ => {} + } + } + } + + fn read_header_line(&mut self) -> Result<(usize, &'a [u8]), ReplayError> { + let start = self.offset; + let relative_end = self.input[start..] + .iter() + .position(|byte| *byte == b'\n') + .ok_or_else(|| ReplayError::decode(&self.path, start, "unterminated header line"))?; + if relative_end > self.record_limit { + return Err(ReplayError::decode( + &self.path, + start, + "header line exceeds record limit", + )); + } + let end = start + relative_end; + self.offset = end + 1; + self.line = self.line.saturating_add(1); + Ok((start, &self.input[start..end])) + } +} + +#[cfg(test)] +#[allow(clippy::expect_used)] +mod tests { + use super::*; + + #[test] + fn decodes_binary_native_record_and_ties() { + let input = concat!( + "# Time: 2026-01-08T19:44:11.099+08:00\n", + "# Conn_ID: 42\n", + "# Cmd_type: StmtExecute\n", + "# Captured_ps_id: 7\n", + r#"# Prepared_stmt: "select \n?""#, + "\n", + "# Payload_len: 5\n", + ); + let mut bytes = input.as_bytes().to_vec(); + bytes.extend_from_slice(&[7, 0, 0, 0, 0]); + bytes.push(b'\n'); + let mut decoder = NativeDecoder::new(&bytes, "traffic.log", 64, None); + let command = decoder + .next_command() + .expect("decode succeeds") + .expect("record"); + assert_eq!(command.command, CommandCode::STMT_EXECUTE); + assert_eq!(command.payload, vec![0x17, 7, 0, 0, 0, 0]); + assert_eq!(command.captured_statement_id, Some(7)); + assert_eq!(command.prepared_statement.as_deref(), Some("select \n?")); + assert!(decoder.next_command().expect("EOF succeeds").is_none()); + } + + #[test] + fn rejects_oversized_before_allocation() { + let bytes = b"# Time: 2026-01-08T19:44:11Z\n# Conn_ID: 1\n# Payload_len: 65\n"; + let mut decoder = NativeDecoder::new(bytes, "traffic.log", 64, None); + let error = decoder.next_command().expect_err("oversize rejected"); + assert!(error.to_string().contains("exceeds limit 64")); + } + + #[test] + fn rejects_duplicate_and_missing_newline() { + let duplicate = b"# Time: 2026-01-08T19:44:11Z\n# Time: 2026-01-08T19:44:12Z\n"; + let mut decoder = NativeDecoder::new(duplicate, "traffic.log", 64, None); + assert!(decoder.next_command().is_err()); + + let truncated = b"# Time: 2026-01-08T19:44:11Z\n# Conn_ID: 1\n# Payload_len: 1\nx"; + let mut decoder = NativeDecoder::new(truncated, "traffic.log", 64, None); + assert!(decoder.next_command().is_err()); + } + + #[test] + fn ignores_duplicate_unknown_headers_for_forward_compatibility() { + let input = concat!( + "# Future: one\n", + "# Future: two\n", + "# Time: 2026-01-08T19:44:11Z\n", + "# Conn_ID: 1\n", + "# Payload_len: 0\n", + "\n" + ); + let mut decoder = NativeDecoder::new(input.as_bytes(), "traffic.log", 64, None); + let command = decoder + .next_command() + .expect("decode succeeds") + .expect("record"); + assert_eq!(command.payload, vec![CommandCode::QUERY.byte()]); + } +} diff --git a/rust/crates/replayer-core/src/decode/quoted.rs b/rust/crates/replayer-core/src/decode/quoted.rs new file mode 100644 index 000000000..7bb096a6b --- /dev/null +++ b/rust/crates/replayer-core/src/decode/quoted.rs @@ -0,0 +1,128 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pub(crate) fn parse_go_quoted(input: &str) -> Result { + let bytes = input.as_bytes(); + if bytes.len() < 2 || bytes.first() != Some(&b'"') || bytes.last() != Some(&b'"') { + return Err("value must be a Go-quoted string".to_owned()); + } + let mut output = Vec::with_capacity(bytes.len().saturating_sub(2)); + let mut index = 1; + while index + 1 < bytes.len() { + if bytes[index] != b'\\' { + output.push(bytes[index]); + index += 1; + continue; + } + index += 1; + let escaped = *bytes + .get(index) + .ok_or_else(|| "value has a trailing escape".to_owned())?; + match escaped { + b'a' => output.push(0x07), + b'b' => output.push(0x08), + b'f' => output.push(0x0c), + b'n' => output.push(b'\n'), + b'r' => output.push(b'\r'), + b't' => output.push(b'\t'), + b'v' => output.push(0x0b), + b'\\' | b'"' | b'\'' => output.push(escaped), + b'x' => { + output.push( + u8::try_from(parse_hex(bytes, index + 1, 2)?) + .map_err(|_| "hex escape is larger than one byte".to_owned())?, + ); + index += 2; + } + b'u' => { + append_unicode(&mut output, parse_hex(bytes, index + 1, 4)?)?; + index += 4; + } + b'U' => { + append_unicode(&mut output, parse_hex(bytes, index + 1, 8)?)?; + index += 8; + } + b'0'..=b'7' => { + let end = index + .checked_add(3) + .ok_or_else(|| "octal escape overflow".to_owned())?; + let digits = bytes + .get(index..end) + .ok_or_else(|| "short octal escape".to_owned())?; + let mut value = 0_u16; + for digit in digits { + if !(b'0'..=b'7').contains(digit) { + return Err("invalid octal escape".to_owned()); + } + value = value * 8 + u16::from(*digit - b'0'); + } + if value > u16::from(u8::MAX) { + return Err("octal escape is larger than one byte".to_owned()); + } + output.push( + u8::try_from(value) + .map_err(|_| "octal escape is larger than one byte".to_owned())?, + ); + index += 2; + } + other => return Err(format!("unsupported escape \\{}", char::from(other))), + } + index += 1; + } + String::from_utf8(output).map_err(|_| "quoted value is not valid UTF-8".to_owned()) +} + +fn parse_hex(bytes: &[u8], start: usize, count: usize) -> Result { + let end = start + .checked_add(count) + .ok_or_else(|| "hex escape overflow".to_owned())?; + let digits = bytes + .get(start..end) + .ok_or_else(|| "short hex escape".to_owned())?; + let mut value = 0_u32; + for digit in digits { + value = value + .checked_mul(16) + .and_then(|current| char::from(*digit).to_digit(16).map(|next| current + next)) + .ok_or_else(|| "invalid hex escape".to_owned())?; + } + Ok(value) +} + +fn append_unicode(output: &mut Vec, value: u32) -> Result<(), String> { + let value = char::from_u32(value).ok_or_else(|| "invalid Unicode escape".to_owned())?; + let mut encoded = [0_u8; 4]; + output.extend_from_slice(value.encode_utf8(&mut encoded).as_bytes()); + Ok(()) +} + +#[cfg(test)] +#[allow(clippy::expect_used)] +mod tests { + use super::*; + + #[test] + fn accepts_go_specific_escapes() { + assert_eq!( + parse_go_quoted(r#""a\x00\u4e2d\U0001f60a\141""#).expect("valid"), + "a\0中😊a" + ); + } + + #[test] + fn rejects_invalid_utf8_and_short_escapes() { + assert!(parse_go_quoted(r#""\xff""#).is_err()); + assert!(parse_go_quoted(r#""\x0""#).is_err()); + } +} diff --git a/rust/crates/replayer-core/src/engine.rs b/rust/crates/replayer-core/src/engine.rs new file mode 100644 index 000000000..0039ca9e5 --- /dev/null +++ b/rust/crates/replayer-core/src/engine.rs @@ -0,0 +1,453 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Deterministic dry-run replay path. + +use std::collections::{HashMap, HashSet}; + +use serde::{Deserialize, Serialize}; + +use crate::checkpoint::{Checkpoint, InputIdentity}; +use crate::config::ReplayConfig; +use crate::decode::{AuditDecoder, NativeDecoder}; +use crate::storage::InputRoot; +use crate::{Command, CommandCode, ReplayError, TrafficFormat}; + +/// Semantic output of a complete dry run. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct DryRunSummary { + /// Number of decoded commands before replay-side filtering. + pub decoded_commands: u64, + /// Number of commands accepted by replay-side filtering. + pub replayed_commands: u64, + /// Number of replay-side filtered commands. + pub filtered_commands: u64, + /// Number of deterministic input files consumed. + pub input_files: u64, + /// Last committed command timestamp in RFC3339. + pub last_command_time: Option, + /// Exact input identity used by the checkpoint. + pub input_identity: InputIdentity, +} + +/// Runs the full local/OpenDAL read, decrypt, decompress, decode, ordering, +/// filtering, and atomic-checkpoint path without a backend connection. +/// +/// # Errors +/// +/// Returns an error when configuration, storage, decoding, filtering, or +/// checkpoint persistence fails closed. +#[allow(clippy::too_many_lines)] +pub async fn dry_run(config: &ReplayConfig) -> Result { + if !config.dry_run { + return Err(ReplayError::Config( + "dry_run requires dry-run configuration".to_owned(), + )); + } + let inputs = config.inputs()?; + let mut commands = Vec::new(); + let mut identity_parts = Vec::new(); + let mut input_files = 0_u64; + for (source_index, input) in inputs.iter().enumerate() { + let source_ordinal = u64::try_from(source_index) + .map_err(|_| ReplayError::Config("too many input roots".to_owned()))?; + let root = InputRoot::open(input)?; + let metadata = if config.format == TrafficFormat::Native { + root.metadata().await? + } else { + None + }; + let entries = root.list(config.format).await?; + identity_parts.extend_from_slice(root.safe_root().as_bytes()); + identity_parts.push(0); + if let Some(metadata) = metadata.as_ref() { + identity_parts.extend_from_slice( + &serde_json::to_vec(metadata).map_err(|error| { + ReplayError::Checkpoint(format!("encode metadata: {error}")) + })?, + ); + } + let mut audit = if config.format.is_audit() { + Some(AuditDecoder::new( + config.format, + root.safe_root(), + config.record_limit, + config.prepared_close, + config.command_start_time, + config.command_end_time, + config.filter_command_with_retry, + &config.user_allowlist, + source_ordinal, + )?) + } else { + None + }; + for entry in entries { + input_files = input_files.saturating_add(1); + identity_parts.extend_from_slice(entry.path.as_bytes()); + identity_parts.extend_from_slice(&entry.content_length.to_le_bytes()); + identity_parts.push(0); + let bytes = root + .read_decoded(&entry, metadata.as_ref(), config.key_file.as_deref()) + .await?; + let content_identity = InputIdentity::from_canonical_bytes(&bytes); + identity_parts.extend_from_slice(content_identity.sha256.as_bytes()); + identity_parts.push(0); + match config.format { + TrafficFormat::Native => { + let mut decoder = NativeDecoder::new( + &bytes, + &entry.safe_path, + config.record_limit, + config.command_start_time, + ); + while let Some(mut command) = decoder.next_command()? { + command.source_ordinal = source_ordinal; + commands.push(command); + } + } + TrafficFormat::AuditLogPlugin | TrafficFormat::AuditLogExtension => { + let decoder = audit.as_mut().ok_or_else(|| { + ReplayError::Config("audit decoder was not constructed".to_owned()) + })?; + commands.extend(decoder.decode_all(&bytes)?); + } + } + } + } + identity_parts.extend_from_slice( + &serde_json::to_vec(&IdentityConfig::from(config)) + .map_err(|error| ReplayError::Checkpoint(format!("encode identity: {error}")))?, + ); + let input_identity = InputIdentity::from_canonical_bytes(&identity_parts); + let checkpoint = match config.checkpoint_path.as_deref() { + Some(path) => Checkpoint::load(path, &input_identity)?, + None => None, + }; + commands.sort_by(|left, right| { + left.start_time + .cmp(&right.start_time) + .then_with(|| left.connection_id.cmp(&right.connection_id)) + .then_with(|| left.source_ordinal.cmp(&right.source_ordinal)) + .then_with(|| left.record_ordinal.cmp(&right.record_ordinal)) + }); + if let Some(frontier) = checkpoint.as_ref() { + commands.retain(|command| { + ( + command.start_time.unix_timestamp_nanos(), + command.connection_id, + command.source_ordinal, + command.record_ordinal, + ) > ( + frontier.command_start_unix_nanos, + frontier.connection_id, + frontier.source_ordinal, + frontier.record_ordinal, + ) + }); + } + let decoded_commands = u64::try_from(commands.len()).unwrap_or(u64::MAX); + let mut filtered_commands = 0_u64; + let mut read_only_statements = HashMap::new(); + let mut last = None; + for command in &commands { + let accepted = !config.read_only + || command_is_read_only(command, &mut read_only_statements, config.record_limit)?; + if !accepted { + filtered_commands = filtered_commands.saturating_add(1); + } + last = Some(command); + } + let replayed_commands = decoded_commands.saturating_sub(filtered_commands); + if let (Some(path), Some(command)) = (config.checkpoint_path.as_deref(), last) { + let checkpoint = Checkpoint::new( + input_identity.clone(), + command.start_time.unix_timestamp_nanos(), + command + .end_time + .unwrap_or(command.start_time) + .unix_timestamp_nanos(), + command.source.clone(), + command.connection_id, + command.source_ordinal, + command.record_ordinal, + checkpoint + .as_ref() + .map_or(0, |value| value.committed_commands) + .saturating_add(replayed_commands), + checkpoint + .as_ref() + .map_or(0, |value| value.filtered_commands) + .saturating_add(filtered_commands), + ); + checkpoint.save_atomic(path)?; + } + Ok(DryRunSummary { + decoded_commands, + replayed_commands, + filtered_commands, + input_files, + last_command_time: last.and_then(|command| { + command + .start_time + .format(&time::format_description::well_known::Rfc3339) + .ok() + }), + input_identity, + }) +} + +#[derive(Serialize)] +struct IdentityConfig<'a> { + format: TrafficFormat, + read_only: bool, + prepared_close: crate::PreparedCloseStrategy, + filter_command_with_retry: bool, + user_allowlist: &'a [String], + command_start_time: Option, + command_end_time: Option, + dynamic_input: bool, + replayer_count: u64, + replayer_index: u64, +} + +impl<'a> From<&'a ReplayConfig> for IdentityConfig<'a> { + fn from(config: &'a ReplayConfig) -> Self { + Self { + format: config.format, + read_only: config.read_only, + prepared_close: config.prepared_close, + filter_command_with_retry: config.filter_command_with_retry, + user_allowlist: &config.user_allowlist, + command_start_time: config + .command_start_time + .map(time::OffsetDateTime::unix_timestamp_nanos), + command_end_time: config + .command_end_time + .map(time::OffsetDateTime::unix_timestamp_nanos), + dynamic_input: config.dynamic_input, + replayer_count: config.replayer_count, + replayer_index: config.replayer_index, + } + } +} + +fn command_is_read_only( + command: &Command, + prepared: &mut HashMap>, + limit: usize, +) -> Result { + match command.command { + CommandCode::QUERY | CommandCode::STMT_PREPARE => { + let sql = std::str::from_utf8(command.payload.get(1..).unwrap_or_default()).map_err( + |_| ReplayError::decode(&command.source, 0, "SQL payload is not valid UTF-8"), + )?; + let read_only = sql_is_read_only(sql, limit)?; + if command.command == CommandCode::STMT_PREPARE + && read_only + && let Some(statement_id) = command.captured_statement_id + { + prepared + .entry(command.connection_id) + .or_default() + .insert(statement_id); + } + Ok(read_only) + } + CommandCode::STMT_CLOSE => Ok(command.captured_statement_id.is_some_and(|statement_id| { + prepared + .get_mut(&command.connection_id) + .is_some_and(|statements| statements.remove(&statement_id)) + })), + CommandCode::STMT_EXECUTE + | CommandCode::STMT_SEND_LONG_DATA + | CommandCode::STMT_RESET + | CommandCode::STMT_FETCH => { + Ok(command.captured_statement_id.is_some_and(|statement_id| { + prepared + .get(&command.connection_id) + .is_some_and(|statements| statements.contains(&statement_id)) + })) + } + CommandCode::CREATE_DB | CommandCode::DROP_DB | CommandCode::DELAYED_INSERT => Ok(false), + CommandCode::QUIT => { + prepared.remove(&command.connection_id); + Ok(true) + } + _ => Ok(true), + } +} + +fn sql_is_read_only(sql: &str, limit: usize) -> Result { + let tokens = tokenize(sql, limit)?; + let first = tokens.first().map(String::as_str).unwrap_or_default(); + match first { + "SELECT" => Ok(!tokens + .windows(2) + .any(|pair| pair[0] == "FOR" && pair[1] == "UPDATE")), + "SHOW" | "WITH" | "USE" | "DESC" | "DESCRIBE" | "TABLE" | "DO" | "BEGIN" | "COMMIT" + | "ROLLBACK" => Ok(true), + "START" => Ok(tokens.get(1).map(String::as_str) == Some("TRANSACTION")), + "SET" => { + let second = tokens.get(1).map(String::as_str).unwrap_or_default(); + Ok(matches!( + second, + "SESSION_STATES" + | "SESSION" + | "NAMES" + | "CHAR" + | "CHARSET" + | "CHARACTER" + | "TRANSACTION" + ) || second.starts_with('@') && !second.starts_with("@@GLOBAL")) + } + _ => Ok(false), + } +} + +fn tokenize(sql: &str, limit: usize) -> Result, ReplayError> { + if sql.len() > limit { + return Err(ReplayError::Config("SQL exceeds record limit".to_owned())); + } + let bytes = sql.as_bytes(); + let mut tokens = Vec::new(); + let mut index = 0_usize; + while index < bytes.len() { + match bytes[index] { + byte if byte.is_ascii_whitespace() || matches!(byte, b'(' | b')' | b',' | b';') => { + index += 1; + } + b'#' => { + index = skip_line_comment(bytes, index + 1); + } + b'-' if bytes.get(index + 1) == Some(&b'-') => { + index = skip_line_comment(bytes, index + 2); + } + b'/' if bytes.get(index + 1) == Some(&b'*') => { + index = skip_block_comment(bytes, index + 2)?; + } + quote @ (b'\'' | b'"' | b'`') => { + index = skip_quoted(bytes, index + 1, quote)?; + } + _ => { + let start = index; + while index < bytes.len() + && !bytes[index].is_ascii_whitespace() + && !matches!( + bytes[index], + b'(' | b')' | b',' | b';' | b'\'' | b'"' | b'`' + ) + { + index += 1; + } + let token = sql.get(start..index).ok_or_else(|| { + ReplayError::Config("SQL token is not on UTF-8 boundaries".to_owned()) + })?; + tokens.push(token.to_ascii_uppercase()); + if tokens.len() > 1_000_000 { + return Err(ReplayError::Config("SQL token limit exceeded".to_owned())); + } + } + } + } + Ok(tokens) +} + +fn skip_line_comment(bytes: &[u8], mut index: usize) -> usize { + while index < bytes.len() && bytes[index] != b'\n' { + index += 1; + } + index +} + +fn skip_block_comment(bytes: &[u8], mut index: usize) -> Result { + while index + 1 < bytes.len() { + if bytes[index] == b'*' && bytes[index + 1] == b'/' { + return Ok(index + 2); + } + index += 1; + } + Err(ReplayError::Config( + "unterminated SQL block comment".to_owned(), + )) +} + +fn skip_quoted(bytes: &[u8], mut index: usize, quote: u8) -> Result { + while index < bytes.len() { + if bytes[index] == b'\\' { + index = index.saturating_add(2); + continue; + } + if bytes[index] == quote { + if bytes.get(index + 1) == Some("e) { + index += 2; + continue; + } + return Ok(index + 1); + } + index += 1; + } + Err(ReplayError::Config( + "unterminated SQL quoted construct".to_owned(), + )) +} + +#[cfg(test)] +#[allow(clippy::expect_used)] +mod tests { + use super::*; + use time::OffsetDateTime; + + #[test] + fn readonly_filter_ignores_literals_and_comments() { + assert!(sql_is_read_only("/* FOR UPDATE */ SELECT 'for update'", 1024).expect("classify")); + assert!(!sql_is_read_only("SELECT 1 FOR UPDATE", 1024).expect("classify")); + assert!(sql_is_read_only("SET @@session.foo=1", 1024).expect("classify")); + assert!(!sql_is_read_only("SET @@global.foo=1", 1024).expect("classify")); + } + + #[test] + fn malformed_sql_fails_closed() { + assert!(sql_is_read_only("SELECT 'oops", 1024).is_err()); + assert!(sql_is_read_only("/* oops", 1024).is_err()); + } + + #[test] + fn prepared_readonly_state_is_scoped_to_connection() { + let timestamp = OffsetDateTime::UNIX_EPOCH; + let mut prepared = HashMap::new(); + let mut prepare = Command::new(b"\x16SELECT ?".to_vec(), timestamp, 1, "fixture", 1) + .expect("prepare command"); + prepare.captured_statement_id = Some(7); + assert!(command_is_read_only(&prepare, &mut prepared, 1024).expect("classify prepare")); + + let mut other_connection_execute = + Command::new(vec![0x17, 7, 0, 0, 0], timestamp, 2, "fixture", 2) + .expect("execute command"); + other_connection_execute.captured_statement_id = Some(7); + assert!( + !command_is_read_only(&other_connection_execute, &mut prepared, 1024) + .expect("classify execute") + ); + + let mut same_connection_execute = + Command::new(vec![0x17, 7, 0, 0, 0], timestamp, 1, "fixture", 3) + .expect("execute command"); + same_connection_execute.captured_statement_id = Some(7); + assert!( + command_is_read_only(&same_connection_execute, &mut prepared, 1024) + .expect("classify execute") + ); + } +} diff --git a/rust/crates/replayer-core/src/error.rs b/rust/crates/replayer-core/src/error.rs new file mode 100644 index 000000000..7c2e0d70b --- /dev/null +++ b/rust/crates/replayer-core/src/error.rs @@ -0,0 +1,88 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Public, credential-redacted replay errors. + +use std::io; +use std::path::PathBuf; + +use thiserror::Error; + +/// Error returned by replay configuration, storage, decoding, or persistence. +#[derive(Debug, Error)] +pub enum ReplayError { + /// Invalid user configuration. + #[error("invalid replay configuration: {0}")] + Config(String), + /// A storage operation failed. The path is already credential-redacted. + #[error("storage {operation} failed for {path}: {message}")] + Storage { + /// Operation name. + operation: &'static str, + /// Safe logical path. + path: String, + /// Provider classification without endpoint credentials. + message: String, + }, + /// An input record was malformed. + #[error("decode failed at {path}:{offset}: {message}")] + Decode { + /// Safe logical input path. + path: String, + /// Zero-based byte offset. + offset: u64, + /// Failure detail. + message: String, + }, + /// A local filesystem operation failed. + #[error("local {operation} failed for {path:?}: {source}")] + Io { + /// Operation name. + operation: &'static str, + /// Local path. + path: PathBuf, + /// Underlying I/O failure. + #[source] + source: io::Error, + }, + /// A checkpoint cannot be used for this replay configuration. + #[error("checkpoint rejected: {0}")] + Checkpoint(String), +} + +impl ReplayError { + /// Builds a decode error with stable path and offset context. + #[must_use] + pub fn decode(path: impl Into, offset: usize, message: impl Into) -> Self { + Self::Decode { + path: path.into(), + offset: u64::try_from(offset).unwrap_or(u64::MAX), + message: message.into(), + } + } + + /// Builds a redacted storage error from an `OpenDAL` failure. + #[must_use] + pub fn storage( + operation: &'static str, + path: impl Into, + source: &opendal::Error, + ) -> Self { + Self::Storage { + operation, + path: path.into(), + message: source.kind().to_string(), + } + } +} diff --git a/rust/crates/replayer-core/src/lib.rs b/rust/crates/replayer-core/src/lib.rs new file mode 100644 index 000000000..0fc74b7dd --- /dev/null +++ b/rust/crates/replayer-core/src/lib.rs @@ -0,0 +1,32 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Bounded input, decoding, and checkpoint primitives for `TiProxy`'s offline +//! traffic replayer. + +#![forbid(unsafe_code)] + +pub mod checkpoint; +pub mod command; +pub mod config; +pub mod decode; +pub mod engine; +pub mod error; +pub mod storage; + +pub use checkpoint::{Checkpoint, InputIdentity}; +pub use command::{Command, CommandCode, PreparedCloseStrategy, TrafficFormat}; +pub use config::ReplayConfig; +pub use engine::{DryRunSummary, dry_run}; +pub use error::ReplayError; diff --git a/rust/crates/replayer-core/src/storage.rs b/rust/crates/replayer-core/src/storage.rs new file mode 100644 index 000000000..93bee58c7 --- /dev/null +++ b/rust/crates/replayer-core/src/storage.rs @@ -0,0 +1,580 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Credential-redacted `OpenDAL` input ownership. + +use std::collections::HashMap; +use std::io::Read; +use std::path::{Path, PathBuf}; + +use aes::Aes256; +use ctr::cipher::{KeyIvInit, StreamCipher}; +use flate2::read::GzDecoder; +use futures_util::TryStreamExt; +use opendal::{ErrorKind, Operator, Scheme}; +use serde::{Deserialize, Serialize}; +use time::PrimitiveDateTime; +use url::Url; +use zeroize::Zeroizing; + +use crate::{ReplayError, TrafficFormat}; + +const MAX_STORED_FILE_BYTES: u64 = 512 * 1024 * 1024; +const MAX_EXPANDED_FILE_BYTES: u64 = 512 * 1024 * 1024; +const MAX_METADATA_BYTES: u64 = 64 * 1024; +const STORAGE_READ_CHUNK_BYTES: usize = 1024 * 1024; + +type Aes256Ctr = ctr::Ctr128BE; + +/// Native capture metadata. +#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "PascalCase")] +pub struct CaptureMetadata { + /// Format version. + pub version: String, + /// Capture duration in nanoseconds. + pub duration: i64, + /// Number of captured commands. + pub cmds: u64, + /// Number of capture-side filtered commands. + #[serde(default)] + pub filtered_cmds: u64, + /// Optional encryption method. + #[serde(default)] + pub encrypt_method: String, +} + +/// One replay input file returned in deterministic order. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct InputEntry { + /// Operator-relative object path. + pub path: String, + /// Credential-redacted logical path. + pub safe_path: String, + /// Stored content length. + pub content_length: u64, +} + +/// OpenDAL-backed read/list/stat owner for one input root. +#[derive(Clone)] +pub struct InputRoot { + operator: Operator, + safe_root: String, +} + +impl std::fmt::Debug for InputRoot { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("InputRoot") + .field("safe_root", &self.safe_root) + .finish_non_exhaustive() + } +} + +impl InputRoot { + /// Parses one local or supported object-store root. Query credentials are + /// consumed into provider configuration and never retained in diagnostics. + /// + /// # Errors + /// + /// Returns an error for invalid roots, unsupported providers, or provider + /// configuration failures. + pub fn open(input: &str) -> Result { + if !input.contains("://") { + return Self::open_local(Path::new(input)); + } + let parsed = Url::parse(input) + .map_err(|_| ReplayError::Config("input URL is invalid".to_owned()))?; + if parsed.scheme() == "file" { + let path = parsed + .to_file_path() + .map_err(|()| ReplayError::Config("file URL is invalid".to_owned()))?; + return Self::open_local(&path); + } + let scheme = match parsed.scheme() { + "s3" => Scheme::S3, + "gcs" | "gs" => Scheme::Gcs, + "azblob" | "azure" => Scheme::Azblob, + "oss" => Scheme::Oss, + "cos" => Scheme::Cos, + other => { + return Err(ReplayError::Config(format!( + "unsupported input scheme {other}" + ))); + } + }; + let authority = parsed + .host_str() + .filter(|host| !host.is_empty()) + .ok_or_else(|| ReplayError::Config("object-store URL has no bucket".to_owned()))?; + let mut config = HashMap::new(); + match scheme { + Scheme::Azblob => { + config.insert("container".to_owned(), authority.to_owned()); + } + _ => { + config.insert("bucket".to_owned(), authority.to_owned()); + } + } + let root = parsed.path().trim_matches('/'); + config.insert("root".to_owned(), format!("/{root}")); + for (key, value) in parsed.query_pairs() { + if let Some((normalized, value)) = provider_query_option(scheme, &key, &value)? { + config.insert(normalized, value); + } + } + let operator = Operator::via_iter(scheme, config) + .map_err(|error| ReplayError::storage("configure", safe_uri(&parsed), &error))?; + Ok(Self { + operator, + safe_root: safe_uri(&parsed), + }) + } + + fn open_local(path: &Path) -> Result { + let path = if path.is_absolute() { + path.to_path_buf() + } else { + std::env::current_dir() + .map_err(|source| ReplayError::Io { + operation: "resolve input root", + path: path.to_path_buf(), + source, + })? + .join(path) + }; + let root = path.canonicalize().map_err(|source| ReplayError::Io { + operation: "canonicalize input root", + path: path.clone(), + source, + })?; + let root_text = root + .to_str() + .ok_or_else(|| ReplayError::Config("input root is not valid UTF-8".to_owned()))? + .to_owned(); + let operator = Operator::via_iter(Scheme::Fs, [("root".to_owned(), root_text.clone())]) + .map_err(|error| ReplayError::storage("configure", &root_text, &error))?; + Ok(Self { + operator, + safe_root: root_text, + }) + } + + /// Reads optional native metadata. Missing metadata is compatible with Go. + /// + /// # Errors + /// + /// Returns an error when metadata storage or JSON decoding fails. + pub async fn metadata(&self) -> Result, ReplayError> { + match self.operator.stat("meta").await { + Ok(metadata) => { + if metadata.content_length() > MAX_METADATA_BYTES { + return Err(ReplayError::decode( + self.safe_path("meta"), + 0, + format!("metadata exceeds {MAX_METADATA_BYTES} bytes"), + )); + } + let buffer = self + .read_object_bounded("meta", &self.safe_path("meta"), MAX_METADATA_BYTES) + .await?; + let metadata: CaptureMetadata = + serde_json::from_slice(buffer.as_slice()).map_err(|error| { + ReplayError::decode( + self.safe_path("meta"), + 0, + format!("invalid meta JSON: {error}"), + ) + })?; + if metadata.version != "v1" { + return Err(ReplayError::decode( + self.safe_path("meta"), + 0, + format!("unsupported native capture version {}", metadata.version), + )); + } + Ok(Some(metadata)) + } + Err(error) if error.kind() == ErrorKind::NotFound => Ok(None), + Err(error) => Err(ReplayError::storage("stat", self.safe_path("meta"), &error)), + } + } + + /// Lists replay inputs recursively in timestamp/path order. + /// + /// # Errors + /// + /// Returns an error when listing or metadata lookup fails, or when an + /// input exceeds the stored-file bound. + #[allow(clippy::case_sensitive_file_extension_comparisons)] + pub async fn list(&self, format: TrafficFormat) -> Result, ReplayError> { + let prefix = if format.is_audit() { + "tidb-audit-" + } else { + "traffic-" + }; + let entries = self + .operator + .list_with("") + .recursive(true) + .await + .map_err(|error| ReplayError::storage("list", &self.safe_root, &error))?; + let mut output = Vec::new(); + for entry in entries { + let path = entry.path(); + if capture_timestamp(path, prefix).is_none() { + continue; + } + let metadata = self + .operator + .stat(path) + .await + .map_err(|error| ReplayError::storage("stat", self.safe_path(path), &error))?; + if metadata.is_file() && metadata.content_length() <= MAX_STORED_FILE_BYTES { + output.push(InputEntry { + path: path.to_owned(), + safe_path: self.safe_path(path), + content_length: metadata.content_length(), + }); + } else if metadata.is_file() { + return Err(ReplayError::decode( + self.safe_path(path), + 0, + format!("stored file exceeds {MAX_STORED_FILE_BYTES} bytes"), + )); + } + } + output.sort_by(|left, right| { + file_timestamp_key(&left.path) + .cmp(file_timestamp_key(&right.path)) + .then_with(|| left.path.cmp(&right.path)) + }); + Ok(output) + } + + /// Reads, decompresses, and decrypts one bounded input file. + /// + /// # Errors + /// + /// Returns an error when the object cannot be read, expanded within its + /// bound, or decrypted with the configured metadata and key. + #[allow(clippy::case_sensitive_file_extension_comparisons)] + pub async fn read_decoded( + &self, + entry: &InputEntry, + metadata: Option<&CaptureMetadata>, + key_file: Option<&Path>, + ) -> Result, ReplayError> { + let stored = self + .read_object_bounded(&entry.path, &entry.safe_path, MAX_STORED_FILE_BYTES) + .await?; + let expanded = if entry.path.ends_with(".gz") { + let mut decoder = GzDecoder::new(stored.as_slice()); + read_bounded(&mut decoder, MAX_EXPANDED_FILE_BYTES, &entry.safe_path)? + } else { + stored + }; + decrypt(expanded, metadata, key_file, &entry.safe_path) + } + + /// Credential-redacted root identifier. + #[must_use] + pub fn safe_root(&self) -> &str { + &self.safe_root + } + + fn safe_path(&self, path: &str) -> String { + if path.is_empty() { + self.safe_root.clone() + } else { + format!("{}/{path}", self.safe_root.trim_end_matches('/')) + } + } + + async fn read_object_bounded( + &self, + path: &str, + safe_path: &str, + limit: u64, + ) -> Result, ReplayError> { + let reader = self + .operator + .reader_with(path) + .chunk(STORAGE_READ_CHUNK_BYTES) + .await + .map_err(|error| ReplayError::storage("read", safe_path, &error))?; + let mut stream = reader + .into_stream(..) + .await + .map_err(|error| ReplayError::storage("read", safe_path, &error))?; + let mut output = Vec::new(); + while let Some(buffer) = stream + .try_next() + .await + .map_err(|error| ReplayError::storage("read", safe_path, &error))? + { + let next_len = output + .len() + .checked_add(buffer.len()) + .ok_or_else(|| ReplayError::decode(safe_path, 0, "stored size overflow"))?; + reject_oversize(next_len, limit, safe_path, "stored object")?; + for bytes in buffer { + output.extend_from_slice(&bytes); + } + } + Ok(output) + } +} + +fn reject_oversize(actual: usize, limit: u64, path: &str, kind: &str) -> Result<(), ReplayError> { + if u64::try_from(actual).unwrap_or(u64::MAX) > limit { + return Err(ReplayError::decode( + path, + 0, + format!("{kind} exceeds {limit} bytes"), + )); + } + Ok(()) +} + +fn read_bounded(reader: &mut impl Read, limit: u64, path: &str) -> Result, ReplayError> { + let mut output = Vec::new(); + reader + .take(limit.saturating_add(1)) + .read_to_end(&mut output) + .map_err(|source| ReplayError::Io { + operation: "decompress input", + path: PathBuf::from(path), + source, + })?; + if u64::try_from(output.len()).unwrap_or(u64::MAX) > limit { + return Err(ReplayError::decode( + path, + 0, + format!("expanded file exceeds {limit} bytes"), + )); + } + Ok(output) +} + +fn decrypt( + mut input: Vec, + metadata: Option<&CaptureMetadata>, + key_file: Option<&Path>, + path: &str, +) -> Result, ReplayError> { + let method = metadata.map_or("", |metadata| metadata.encrypt_method.as_str()); + match method.to_ascii_lowercase().as_str() { + "" | "plaintext" => Ok(input), + "aes256-ctr" => { + let key_path = key_file.ok_or_else(|| { + ReplayError::Config("security.encryption-key-path is not set".to_owned()) + })?; + let key = + Zeroizing::new(std::fs::read(key_path).map_err(|source| ReplayError::Io { + operation: "read encryption key", + path: key_path.to_path_buf(), + source, + })?); + if key.len() < 32 { + return Err(ReplayError::Config(format!( + "invalid AES-256 key length {}, expecting at least 32", + key.len() + ))); + } + if input.len() < 16 { + return Err(ReplayError::decode( + path, + 0, + "encrypted input has a short IV", + )); + } + let (iv, body) = input.split_at_mut(16); + let mut cipher = Aes256Ctr::new_from_slices(&key[..32], iv) + .map_err(|_| ReplayError::Config("invalid AES-256 key or IV".to_owned()))?; + cipher.apply_keystream(body); + Ok(body.to_vec()) + } + other => Err(ReplayError::Config(format!( + "unsupported encrypt method {other}" + ))), + } +} + +fn provider_query_option( + scheme: Scheme, + key: &str, + value: &str, +) -> Result, ReplayError> { + let canonical = key.replace('-', "_"); + if canonical == "provider" { + return Ok(None); + } + if scheme == Scheme::S3 && canonical == "force_path_style" { + let force_path_style = value.parse::().map_err(|_| { + ReplayError::Config("force-path-style must be true or false".to_owned()) + })?; + return Ok(Some(( + "enable_virtual_host_style".to_owned(), + (!force_path_style).to_string(), + ))); + } + let normalized = match (scheme, canonical.as_str()) { + (Scheme::S3 | Scheme::Oss, "access_key" | "access_key_id") => "access_key_id", + (Scheme::Cos, "access_key" | "access_key_id" | "secret_id") => "secret_id", + (Scheme::S3, "secret_access_key") => "secret_access_key", + (Scheme::Oss, "secret_access_key" | "access_key_secret") => "access_key_secret", + (Scheme::Cos, "secret_access_key" | "secret_key") => "secret_key", + (Scheme::Gcs, "credentials_file") => "credential_path", + _ => canonical.as_str(), + }; + Ok(Some((normalized.to_owned(), value.to_owned()))) +} + +fn safe_uri(url: &Url) -> String { + let host = url.host_str().unwrap_or_default(); + format!("{}://{}{}", url.scheme(), host, url.path()) +} + +fn file_timestamp_key(path: &str) -> &str { + let name = path.rsplit('/').next().unwrap_or(path); + name.strip_prefix("traffic-") + .or_else(|| name.strip_prefix("tidb-audit-")) + .and_then(|value| value.strip_suffix(".gz").or(Some(value))) + .and_then(|value| value.strip_suffix(".log")) + .unwrap_or(name) +} + +fn capture_timestamp<'a>(name: &'a str, prefix: &str) -> Option<&'a str> { + let name = name.strip_prefix(prefix)?; + let name = name.strip_suffix(".gz").unwrap_or(name); + let timestamp = name.strip_suffix(".log")?; + if let Some((_, fraction)) = timestamp.split_once('.') + && (fraction.is_empty() + || fraction.len() > 3 + || !fraction.bytes().all(|byte| byte.is_ascii_digit())) + { + return None; + } + let description = if timestamp.contains('.') { + "[year]-[month]-[day]T[hour]-[minute]-[second].[subsecond]" + } else { + "[year]-[month]-[day]T[hour]-[minute]-[second]" + }; + let format = time::format_description::parse_borrowed::<2>(description).ok()?; + PrimitiveDateTime::parse(timestamp, &format).ok()?; + Some(timestamp) +} + +#[cfg(test)] +#[allow(clippy::expect_used)] +mod tests { + use super::*; + + #[tokio::test] + async fn local_list_is_timestamp_then_path_ordered() { + let directory = + std::env::temp_dir().join(format!("tiproxy-replayer-storage-{}", std::process::id())); + std::fs::create_dir_all(directory.join("nested")).expect("create directory"); + std::fs::write(directory.join("traffic-2026-01-01T00-00-01.000.log"), b"a") + .expect("write file"); + std::fs::write( + directory + .join("nested") + .join("traffic-2026-01-01T00-00-00.000.log"), + b"ignored nested input", + ) + .expect("write file"); + std::fs::write(directory.join("traffic-2026-01-01T00-00-00.000.log"), b"b") + .expect("write file"); + std::fs::write(directory.join("traffic-2026-01-01T00-00-02.log"), b"c") + .expect("write file"); + std::fs::write(directory.join("traffic-not-a-time.log"), b"bad") + .expect("write invalid file"); + std::fs::write(directory.join("traffic-2026-99-01T00-00-03.log"), b"bad") + .expect("write invalid date"); + std::fs::write(directory.join("ignore.txt"), b"x").expect("write ignored file"); + let root = InputRoot::open(directory.to_str().expect("UTF-8 path")).expect("open root"); + let files = root.list(TrafficFormat::Native).await.expect("list files"); + assert_eq!(files.len(), 3); + assert!(files[0].path.contains("00-00-00.000")); + assert!(files[1].path.contains("00-00-01.000")); + assert!(files[2].path.contains("00-00-02.log")); + std::fs::remove_dir_all(directory).expect("remove directory"); + } + + #[tokio::test] + async fn native_metadata_rejects_unknown_version() { + let directory = + std::env::temp_dir().join(format!("tiproxy-replayer-meta-{}", std::process::id())); + if directory.exists() { + std::fs::remove_dir_all(&directory).expect("remove stale directory"); + } + std::fs::create_dir_all(&directory).expect("create directory"); + std::fs::write( + directory.join("meta"), + br#"{"Version":"v2","Duration":0,"Cmds":0}"#, + ) + .expect("write metadata"); + let root = InputRoot::open(directory.to_str().expect("UTF-8 path")).expect("open root"); + let error = root.metadata().await.expect_err("unknown version"); + assert!( + error + .to_string() + .contains("unsupported native capture version v2") + ); + std::fs::remove_dir_all(directory).expect("remove directory"); + } + + #[test] + fn provider_options_map_to_native_opendal_fields() { + assert_eq!( + provider_query_option(Scheme::S3, "force-path-style", "true").expect("option"), + Some(("enable_virtual_host_style".to_owned(), "false".to_owned())) + ); + assert_eq!( + provider_query_option(Scheme::Oss, "secret-access-key", "secret").expect("option"), + Some(("access_key_secret".to_owned(), "secret".to_owned())) + ); + assert_eq!( + provider_query_option(Scheme::Cos, "access-key", "id").expect("option"), + Some(("secret_id".to_owned(), "id".to_owned())) + ); + assert_eq!( + provider_query_option(Scheme::Gcs, "credentials-file", "/tmp/key").expect("option"), + Some(("credential_path".to_owned(), "/tmp/key".to_owned())) + ); + assert_eq!( + provider_query_option(Scheme::S3, "provider", "minio").expect("option"), + None + ); + assert!(provider_query_option(Scheme::S3, "force-path-style", "maybe").is_err()); + } + + #[test] + fn uri_diagnostics_drop_credentials() { + let parsed = + Url::parse("s3://bucket/prefix?access-key=secret-one&secret-access-key=secret-two") + .expect("URL"); + let safe = safe_uri(&parsed); + assert_eq!(safe, "s3://bucket/prefix"); + assert!(!safe.contains("secret")); + } + + #[test] + fn post_read_bounds_reject_object_growth() { + assert!(reject_oversize(65, 64, "traffic.log", "stored file").is_err()); + assert!(reject_oversize(64, 64, "traffic.log", "stored file").is_ok()); + } +} diff --git a/rust/crates/replayer-core/tests/dry_run.rs b/rust/crates/replayer-core/tests/dry_run.rs new file mode 100644 index 000000000..f60deb641 --- /dev/null +++ b/rust/crates/replayer-core/tests/dry_run.rs @@ -0,0 +1,155 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! End-to-end dry-run storage and checkpoint contract tests. + +use std::error::Error; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; + +use aes::Aes256; +use ctr::cipher::{KeyIvInit, StreamCipher}; +use flate2::Compression; +use flate2::write::GzEncoder; +use replayer_core::config::DEFAULT_RECORD_LIMIT; +use replayer_core::{Checkpoint, PreparedCloseStrategy, ReplayConfig, TrafficFormat, dry_run}; +use time::OffsetDateTime; + +type TestResult = Result<(), Box>; +type Aes256Ctr = ctr::Ctr128BE; + +static DIRECTORY_SEQUENCE: AtomicU64 = AtomicU64::new(0); + +fn temporary_directory(name: &str) -> Result { + let sequence = DIRECTORY_SEQUENCE.fetch_add(1, Ordering::Relaxed); + let path = std::env::temp_dir().join(format!( + "tiproxy-replayer-{name}-{}-{sequence}", + std::process::id() + )); + std::fs::create_dir_all(&path)?; + Ok(path) +} + +fn native_record(timestamp: &str, connection_id: u64, sql: &str) -> Vec { + format!( + "# Time: {timestamp}\n# Conn_ID: {connection_id}\n# Payload_len: {}\n{sql}\n", + sql.len() + ) + .into_bytes() +} + +fn dry_run_config(input: &Path, checkpoint: Option) -> ReplayConfig { + ReplayConfig { + input: input.to_string_lossy().into_owned(), + speed: 1.0, + username: "root".to_owned(), + address: "127.0.0.1:4000".to_owned(), + format: TrafficFormat::Native, + read_only: false, + start_time: OffsetDateTime::now_utc(), + ignore_errors: false, + reorder_buffer: 100_000, + prepared_close: PreparedCloseStrategy::Directed, + dry_run: true, + checkpoint_path: checkpoint, + dynamic_input: false, + replayer_count: 1, + replayer_index: 0, + output_path: None, + filter_command_with_retry: false, + user_allowlist: Vec::new(), + wait_on_eof: false, + command_start_time: None, + command_end_time: None, + key_file: None, + record_limit: DEFAULT_RECORD_LIMIT, + } +} + +#[tokio::test] +async fn checkpoint_uses_full_tie_frontier_and_content_identity() -> TestResult { + let directory = temporary_directory("checkpoint")?; + let traffic = directory.join("traffic-2026-01-08T19-44-11.099.log"); + let checkpoint_path = directory.join("checkpoint.json"); + let timestamp = "2026-01-08T19:44:11.099+08:00"; + let mut input = native_record(timestamp, 2, "SELECT 1"); + input.extend(native_record(timestamp, 1, "SELECT 1")); + std::fs::write(&traffic, &input)?; + + let config = dry_run_config(&directory, Some(checkpoint_path.clone())); + let first = dry_run(&config).await?; + assert_eq!(first.decoded_commands, 2); + assert_eq!(first.replayed_commands, 2); + let checkpoint = Checkpoint::load(&checkpoint_path, &first.input_identity)? + .ok_or("checkpoint was not persisted")?; + assert_eq!(checkpoint.connection_id, 2); + assert_eq!(checkpoint.source_ordinal, 0); + assert_eq!(checkpoint.record_ordinal, 1); + + let resumed = dry_run(&config).await?; + assert_eq!(resumed.decoded_commands, 0); + assert_eq!(resumed.replayed_commands, 0); + + let mut changed = native_record(timestamp, 2, "SELECT 2"); + changed.extend(native_record(timestamp, 1, "SELECT 1")); + assert_eq!(changed.len(), input.len()); + std::fs::write(&traffic, changed)?; + let mismatch = dry_run(&config).await; + assert!(mismatch.is_err()); + assert!( + mismatch + .err() + .is_some_and(|error| error.to_string().contains("input identity")) + ); + + std::fs::remove_dir_all(directory)?; + Ok(()) +} + +#[tokio::test] +async fn gzip_then_aes_ctr_matches_native_storage_layers() -> TestResult { + let directory = temporary_directory("encrypted")?; + let key_path = directory.join("key"); + let key = [0x5a_u8; 32]; + std::fs::write(&key_path, key)?; + std::fs::write( + directory.join("meta"), + br#"{"Version":"v1","Duration":1,"Cmds":1,"EncryptMethod":"aes256-ctr"}"#, + )?; + + let plaintext = native_record("2026-01-08T19:44:11.099+08:00", 1, "SELECT 1"); + let iv = [0x24_u8; 16]; + let mut encrypted = plaintext; + let mut cipher = Aes256Ctr::new_from_slices(&key, &iv)?; + cipher.apply_keystream(&mut encrypted); + let mut stored = iv.to_vec(); + stored.extend(encrypted); + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(&stored)?; + std::fs::write( + directory.join("traffic-2026-01-08T19-44-11.099.log.gz"), + encoder.finish()?, + )?; + + let mut config = dry_run_config(&directory, None); + config.key_file = Some(key_path); + let summary = dry_run(&config).await?; + assert_eq!(summary.input_files, 1); + assert_eq!(summary.decoded_commands, 1); + assert_eq!(summary.replayed_commands, 1); + + std::fs::remove_dir_all(directory)?; + Ok(()) +} diff --git a/rust/crates/tiproxy-replayer/Cargo.toml b/rust/crates/tiproxy-replayer/Cargo.toml new file mode 100644 index 000000000..7fc96b12c --- /dev/null +++ b/rust/crates/tiproxy-replayer/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "tiproxy-replayer" +description = "TiProxy standalone offline traffic replayer" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true + +[features] +default = [] + +[dependencies] +clap.workspace = true +replayer-core.workspace = true +serde_json.workspace = true +time.workspace = true +tokio = { workspace = true, features = ["signal"] } +zeroize.workspace = true + +[lints] +workspace = true diff --git a/rust/crates/tiproxy-replayer/src/main.rs b/rust/crates/tiproxy-replayer/src/main.rs new file mode 100644 index 000000000..263ffc8a0 --- /dev/null +++ b/rust/crates/tiproxy-replayer/src/main.rs @@ -0,0 +1,184 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Standalone composition root for the Rust offline replayer. + +#![forbid(unsafe_code)] + +use std::path::PathBuf; +use std::process::ExitCode; + +use clap::Parser; +use replayer_core::config::DEFAULT_RECORD_LIMIT; +use replayer_core::{PreparedCloseStrategy, ReplayConfig, TrafficFormat, dry_run}; +use time::OffsetDateTime; +use time::format_description::well_known::Rfc3339; +use zeroize::Zeroizing; + +/// Standalone `TiProxy` traffic replayer. +#[derive(Parser)] +#[command(name = "tiproxy-replayer", version, about)] +#[allow(clippy::struct_excessive_bools)] +struct Options { + /// Downstream `TiDB` address. + #[arg(long, default_value = "127.0.0.1:4000")] + addr: String, + /// Directory or object-store root containing traffic files. + #[arg(long, default_value = "")] + input: String, + /// Replay speed multiplier. + #[arg(long, default_value_t = 1.0)] + speed: f64, + /// `TiDB` username. + #[arg(long, default_value = "root")] + username: String, + /// `TiDB` password. Never included in debug or output. + #[arg(long, default_value = "")] + password: String, + /// Replay only read-only statements. + #[arg(long)] + read_only: bool, + /// Input format: `native`, `audit_log_plugin`, or `audit_log_extension`. + #[arg(long, default_value = "")] + format: TrafficFormat, + /// Log output path. + #[arg(long, default_value = "")] + log_file: String, + /// Skip commands at or before this RFC3339 start time. + #[arg(long, value_parser = parse_timestamp)] + command_start_time: Option, + /// Skip audit commands ending before this RFC3339 time. + #[arg(long, value_parser = parse_timestamp)] + command_end_time: Option, + /// Ignore recoverable replay command errors. + #[arg(long)] + ignore_errs: bool, + /// Audit reorder buffer size; zero disables reordering. + #[arg(long, default_value_t = 100_000)] + bufsize: usize, + /// Optional pprof/API listen address retained for CLI compatibility. + #[arg(long, default_value = "")] + pprof_addr: String, + /// Prepared close strategy: directed, always, or never. + #[arg(long, default_value = "directed")] + ps_close: PreparedCloseStrategy, + /// Decode and filter without connecting to `TiDB`. + #[arg(long)] + dry_run: bool, + /// Local atomic checkpoint path. + #[arg(long)] + checkpoint_path: Option, + /// Watch for newly published traffic directories. + #[arg(long)] + dynamic_input: bool, + /// Total dynamic-input replayer count. + #[arg(long, default_value_t = 1)] + replayer_count: u64, + /// This dynamic-input replayer index. + #[arg(long, default_value_t = 0)] + replayer_index: u64, + /// Optional local replayed-SQL output path. + #[arg(long)] + output_path: Option, + /// Run the replay-only HTTP service. + #[arg(long)] + service_mode: bool, + /// Log level retained for CLI compatibility. + #[arg(long, default_value = "info")] + log_level: String, + /// Absolute RFC3339 replay start time; defaults to process start. + #[arg(long, value_parser = parse_timestamp)] + start_time: Option, + /// Filter audit commands marked as retries. + #[arg(long)] + filter_command_with_retry: bool, + /// Case-insensitive audit user allowlist. + #[arg(long, value_delimiter = ',')] + user_allowlist: Vec, + /// Wait for new files after EOF. + #[arg(long)] + wait_on_eof: bool, + /// AES-256 key file for encrypted native captures. + #[arg(long)] + key_file: Option, +} + +#[tokio::main] +async fn main() -> ExitCode { + let options = Options::parse(); + match run(options).await { + Ok(()) => ExitCode::SUCCESS, + Err(error) => { + eprintln!("tiproxy-replayer stopped: {error}"); + ExitCode::from(1) + } + } +} + +async fn run(options: Options) -> Result<(), String> { + let now = OffsetDateTime::now_utc(); + drop(Zeroizing::new(options.password)); + drop((options.log_file, options.pprof_addr, options.log_level)); + if options.service_mode { + return Err("service mode is not enabled in the current implementation slice".to_owned()); + } + if !options.dry_run { + return Err( + "backend execution is not enabled in the current implementation slice; use --dry-run" + .to_owned(), + ); + } + if options.dynamic_input || options.wait_on_eof { + return Err("dynamic input is not enabled in the current implementation slice".to_owned()); + } + if options.output_path.is_some() { + return Err("SQL output is not enabled in the current implementation slice".to_owned()); + } + let mut config = ReplayConfig { + input: options.input, + speed: options.speed, + username: options.username, + address: options.addr, + format: options.format, + read_only: options.read_only, + start_time: options.start_time.unwrap_or(now), + ignore_errors: options.ignore_errs, + reorder_buffer: options.bufsize, + prepared_close: options.ps_close, + dry_run: options.dry_run, + checkpoint_path: options.checkpoint_path, + dynamic_input: options.dynamic_input, + replayer_count: options.replayer_count, + replayer_index: options.replayer_index, + output_path: options.output_path, + filter_command_with_retry: options.filter_command_with_retry, + user_allowlist: options.user_allowlist, + wait_on_eof: options.wait_on_eof, + command_start_time: options.command_start_time, + command_end_time: options.command_end_time, + key_file: options.key_file, + record_limit: DEFAULT_RECORD_LIMIT, + }; + config.validate(now).map_err(|error| error.to_string())?; + let summary = dry_run(&config).await.map_err(|error| error.to_string())?; + println!( + "{}", + serde_json::to_string_pretty(&summary).map_err(|error| error.to_string())? + ); + Ok(()) +} + +fn parse_timestamp(value: &str) -> Result { + OffsetDateTime::parse(value, &Rfc3339).map_err(|error| error.to_string()) +} diff --git a/rust/deny.toml b/rust/deny.toml index fe14100b8..e3b6db608 100644 --- a/rust/deny.toml +++ b/rust/deny.toml @@ -6,7 +6,16 @@ targets = [ all-features = true [advisories] -ignore = [] +# OpenDAL 0.53.3's S3/GCS signers retain these transitive advisories. This +# foundation is an operator-run, dry-run-only binary with trusted storage +# endpoints; service mode and backend execution fail closed. Revisit the pin +# before either deferred mode is enabled. OpenDAL 0.58 fixes quick-xml but has +# MSRV 1.91, above this workspace's 1.89 toolchain. +ignore = [ + "RUSTSEC-2023-0071", + "RUSTSEC-2026-0194", + "RUSTSEC-2026-0195", +] [licenses] allow = [ diff --git a/rust/vulnerability-exceptions.toml b/rust/vulnerability-exceptions.toml index a0757b6df..9ac37aa0f 100644 --- a/rust/vulnerability-exceptions.toml +++ b/rust/vulnerability-exceptions.toml @@ -9,3 +9,21 @@ schema_version = 1 # owner = "@github-handle-or-team" # rationale = "Why exposure is bounded and what remediation is in progress." # expires = 2026-12-31 + +[[exceptions]] +advisory = "RUSTSEC-2023-0071" +owner = "@bb7133" +rationale = "OpenDAL 0.53.3 reaches rsa through request signing, but this slice is operator-run dry-run-only with trusted storage endpoints; upgrade OpenDAL before enabling service or backend modes." +expires = 2027-03-01 + +[[exceptions]] +advisory = "RUSTSEC-2026-0194" +owner = "@bb7133" +rationale = "OpenDAL 0.53.3 reaches quick-xml through cloud provider support, but this slice is operator-run dry-run-only with trusted storage endpoints; upgrade OpenDAL before enabling service or backend modes." +expires = 2027-03-01 + +[[exceptions]] +advisory = "RUSTSEC-2026-0195" +owner = "@bb7133" +rationale = "OpenDAL 0.53.3 reaches quick-xml through cloud provider support, but this slice is operator-run dry-run-only with trusted storage endpoints; upgrade OpenDAL before enabling service or backend modes." +expires = 2027-03-01 diff --git a/tests/replayer/differential/corpus.v1.json b/tests/replayer/differential/corpus.v1.json new file mode 100644 index 000000000..85bafb763 --- /dev/null +++ b/tests/replayer/differential/corpus.v1.json @@ -0,0 +1,86 @@ +{ + "cases": [ + { + "id": "native-binary", + "format": "native", + "prepared_close": "directed", + "filter_retries": false, + "user_allowlist": [], + "source_ordinal": 0, + "records": [ + { + "headers": [ + ["Future", "one"], + ["Future", "two"], + ["Time", "2026-01-08T19:44:11.099+08:00"], + ["Conn_ID", "42"] + ], + "payload_hex": "53454c4543542031" + }, + { + "headers": [ + ["Time", "2026-01-08T19:44:11.100+08:00"], + ["Conn_ID", "42"], + ["Cmd_type", "StmtPrepare"], + ["Captured_ps_id", "7"] + ], + "payload_hex": "53454c454354203f" + }, + { + "headers": [ + ["Time", "2026-01-08T19:44:11.101+08:00"], + ["Conn_ID", "42"], + ["Cmd_type", "StmtExecute"], + ["Captured_ps_id", "7"], + ["Prepared_stmt", "\"SELECT ?\""] + ], + "payload_hex": "070000000001000000" + } + ], + "lines": [] + }, + { + "id": "plugin-allocation-and-dedup", + "format": "audit_log_plugin", + "prepared_close": "directed", + "filter_retries": false, + "user_allowlist": ["ROOT"], + "source_ordinal": 1, + "records": [], + "lines": [ + "[2025/09/06 17:03:53.719 +08:00] [TIMESTAMP=2025/09/06 17:03:53.719 +08:00] [EVENT_CLASS=GENERAL] [EVENT=STARTING] [COMMAND=Query] [SQL_TEXT=SELECT 0] [SQL_STATEMENTS=Select] [CONNECTION_ID=40] [USER=root]", + "[2025/09/06 17:03:53.720 +08:00] [TIMESTAMP=2025/09/06 17:03:53.720 +08:00] [EVENT_CLASS=GENERAL] [EVENT=COMPLETED] [COMMAND=Query] [SQL_TEXT=INSERT INTO t VALUES (1)] [SQL_STATEMENTS=Insert] [CONNECTION_ID=9] [USER=root] [CURRENT_DB=test]", + "[2025/09/06 17:03:53.720 +08:00] [TIMESTAMP=2025/09/06 17:03:53.720 +08:00] [EVENT_CLASS=GENERAL] [EVENT=COMPLETED] [COMMAND=Query] [SQL_TEXT=INSERT INTO t VALUES (1)] [SQL_STATEMENTS=Insert] [CONNECTION_ID=9] [USER=root] [CURRENT_DB=test]", + "[2025/09/06 17:03:53.720 +08:00] [TIMESTAMP=2025/09/06 17:03:53.720 +08:00] [EVENT_CLASS=GENERAL] [EVENT=COMPLETED] [COMMAND=Query] [SQL_TEXT=SELECT 1] [SQL_STATEMENTS=Select] [CONNECTION_ID=9] [USER=root] [CURRENT_DB=test]", + "[2025/09/06 17:03:53.720 +08:00] [TIMESTAMP=2025/09/06 17:03:53.720 +08:00] [EVENT_CLASS=GENERAL] [EVENT=COMPLETED] [COMMAND=Query] [SQL_TEXT=INSERT INTO t VALUES (1)] [SQL_STATEMENTS=Insert] [CONNECTION_ID=9] [USER=root] [CURRENT_DB=test]" + ] + }, + { + "id": "plugin-retry-and-close", + "format": "audit_log_plugin", + "prepared_close": "directed", + "filter_retries": true, + "user_allowlist": [], + "source_ordinal": 2, + "records": [], + "lines": [ + "[2025/09/06 17:03:53.721 +08:00] [TIMESTAMP=2025/09/06 17:03:53.721 +08:00] [EVENT_CLASS=GENERAL] [EVENT=COMPLETED] [COMMAND=Execute] [SQL_TEXT=SELECT ?] [SQL_STATEMENTS=Select] [CONNECTION_ID=9] [PREPARED_STMT_ID=7] [EXECUTE_PARAMS=\"[]\"] [RETRY=false]", + "[2025/09/06 17:03:53.722 +08:00] [TIMESTAMP=2025/09/06 17:03:53.722 +08:00] [EVENT_CLASS=GENERAL] [EVENT=COMPLETED] [COMMAND=\"Close stmt\"] [CONNECTION_ID=9] [PREPARED_STMT_ID=7] [RETRY=true]", + "[2025/09/06 17:03:53.723 +08:00] [TIMESTAMP=2025/09/06 17:03:53.723 +08:00] [EVENT_CLASS=CONNECTION] [EVENT_SUBCLASS=Disconnect] [CONNECTION_ID=9]" + ] + }, + { + "id": "extension-prepare-execute-close", + "format": "audit_log_extension", + "prepared_close": "always", + "filter_retries": false, + "user_allowlist": [], + "source_ordinal": 3, + "records": [], + "lines": [ + "[2026/01/08 19:44:11.114 +08:00] [EVENT=\"[QUERY,EXECUTE,SELECT]\"] [USER=root] [CONNECTION_ID=7] [CURRENT_DB=test] [SQL_TEXT=\"SELECT ?\"] [EXECUTE_PARAMS=\"[1]\"]", + "[2026/01/08 19:44:11.115 +08:00] [EVENT=\"[CONNECTION,DISCONNECT]\"] [USER=root] [CONNECTION_ID=7]" + ] + } + ] +} diff --git a/tests/replayer/differential/go-observer/main.go b/tests/replayer/differential/go-observer/main.go new file mode 100644 index 000000000..47993498f --- /dev/null +++ b/tests/replayer/differential/go-observer/main.go @@ -0,0 +1,248 @@ +// Copyright 2026 PingCAP, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bytes" + "encoding/hex" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "os" + "reflect" + "time" + + replaycmd "github.com/pingcap/tiproxy/pkg/sqlreplay/cmd" + "go.uber.org/zap" +) + +type corpus struct { + Cases []replayCase `json:"cases"` +} + +type replayCase struct { + ID string `json:"id"` + Format string `json:"format"` + PreparedClose string `json:"prepared_close"` + FilterRetries bool `json:"filter_retries"` + UserAllowlist []string `json:"user_allowlist"` + SourceOrdinal int `json:"source_ordinal"` + Records []nativeRecord `json:"records"` + Lines []string `json:"lines"` +} + +type nativeRecord struct { + Headers [][2]string `json:"headers"` + PayloadHex string `json:"payload_hex"` +} + +type observation struct { + CaseID string `json:"case_id"` + Index uint64 `json:"index"` + PayloadHex string `json:"payload_hex"` + StartUnixNanos int64 `json:"start_unix_nanos"` + EndUnixNanos int64 `json:"end_unix_nanos"` + ConnectionID uint64 `json:"connection_id"` + UpstreamConnectionID uint64 `json:"upstream_connection_id"` + Command byte `json:"command"` + CurrentDatabase string `json:"current_database"` + CapturedStatementID uint32 `json:"captured_statement_id"` + PreparedStatement string `json:"prepared_statement"` + StatementType string `json:"statement_type"` + Succeeded bool `json:"succeeded"` +} + +func main() { + corpusPath := flag.String("corpus", "", "shared replay corpus") + baselinePath := flag.String("baseline", "", "baseline observation JSON") + candidatePath := flag.String("candidate", "", "candidate observation JSON") + flag.Parse() + if *baselinePath != "" || *candidatePath != "" { + if *baselinePath == "" || *candidatePath == "" { + fatal(errors.New("both -baseline and -candidate are required")) + } + if err := compare(*baselinePath, *candidatePath); err != nil { + _, _ = fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + _, _ = fmt.Fprintln(os.Stdout, `{"equal":true}`) + return + } + if *corpusPath == "" { + fatal(errors.New("-corpus is required")) + } + if err := observeCorpus(*corpusPath, os.Stdout); err != nil { + fatal(err) + } +} + +func observeCorpus(path string, output io.Writer) error { + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + var fixture corpus + if err := json.NewDecoder(file).Decode(&fixture); err != nil { + return err + } + observations := make([]observation, 0) + for _, testCase := range fixture.Cases { + input, err := buildInput(testCase) + if err != nil { + return fmt.Errorf("build %s: %w", testCase.ID, err) + } + decoder := replaycmd.NewCmdDecoder( + replaycmd.TrafficFormat(testCase.Format), + replaycmd.NewDeDup(), + zap.NewNop(), + ) + if audit, ok := decoder.(replaycmd.AuditLogDecoder); ok { + audit.SetPSCloseStrategy(replaycmd.PSCloseStrategy(testCase.PreparedClose)) + allocator, err := replaycmd.NewConnIDAllocator(testCase.SourceOrdinal) + if err != nil { + return err + } + audit.SetIDAllocator(allocator) + audit.SetUserAllowlist(testCase.UserAllowlist) + if testCase.FilterRetries { + audit.EnableFilterCommandWithRetry() + } + } + reader := &memoryLineReader{name: testCase.ID, input: input} + for index := uint64(0); ; index++ { + command, err := decoder.Decode(reader) + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return fmt.Errorf("decode %s: %w", testCase.ID, err) + } + upstreamID := command.UpstreamConnID + if upstreamID == 0 { + upstreamID = command.ConnID + } + observations = append(observations, observation{ + CaseID: testCase.ID, + Index: index, + PayloadHex: hex.EncodeToString(command.Payload), + StartUnixNanos: command.StartTs.UnixNano(), + EndUnixNanos: unixNanosOrZero(command.EndTs), + ConnectionID: command.ConnID, + UpstreamConnectionID: upstreamID, + Command: command.Type.Byte(), + CurrentDatabase: command.CurDB, + CapturedStatementID: command.CapturedPsID, + PreparedStatement: command.PreparedStmt, + StatementType: command.StmtType, + Succeeded: command.Success, + }) + } + } + return json.NewEncoder(output).Encode(observations) +} + +func buildInput(testCase replayCase) ([]byte, error) { + var input bytes.Buffer + if testCase.Format == string(replaycmd.FormatNative) { + for _, record := range testCase.Records { + for _, header := range record.Headers { + if _, err := fmt.Fprintf(&input, "# %s: %s\n", header[0], header[1]); err != nil { + return nil, err + } + } + payload, err := hex.DecodeString(record.PayloadHex) + if err != nil { + return nil, err + } + if _, err := fmt.Fprintf(&input, "# Payload_len: %d\n", len(payload)); err != nil { + return nil, err + } + input.Write(payload) + input.WriteByte('\n') + } + } else { + for _, line := range testCase.Lines { + input.WriteString(line) + input.WriteByte('\n') + } + } + return input.Bytes(), nil +} + +func unixNanosOrZero(value time.Time) int64 { + if value.IsZero() { + return 0 + } + return value.UnixNano() +} + +type memoryLineReader struct { + name string + input []byte + offset int + line int +} + +func (r *memoryLineReader) String() string { return r.name } + +func (r *memoryLineReader) ReadLine() ([]byte, string, int, error) { + if r.offset == len(r.input) { + return nil, r.name, r.line, io.EOF + } + relativeEnd := bytes.IndexByte(r.input[r.offset:], '\n') + if relativeEnd < 0 { + return nil, r.name, r.line, io.ErrUnexpectedEOF + } + start := r.offset + r.offset += relativeEnd + 1 + r.line++ + return r.input[start : r.offset-1], r.name, r.line, nil +} + +func (r *memoryLineReader) Read(output []byte) (string, int, error) { + if len(r.input)-r.offset < len(output) { + return r.name, r.line, io.ErrUnexpectedEOF + } + copy(output, r.input[r.offset:r.offset+len(output)]) + r.offset += len(output) + return r.name, r.line, nil +} + +func (r *memoryLineReader) Close() {} + +func compare(baselinePath, candidatePath string) error { + read := func(path string) (any, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var value any + if err := json.Unmarshal(data, &value); err != nil { + return nil, err + } + return value, nil + } + baseline, err := read(baselinePath) + if err != nil { + return err + } + candidate, err := read(candidatePath) + if err != nil { + return err + } + if !reflect.DeepEqual(baseline, candidate) { + return fmt.Errorf("decoder observations differ: baseline=%s candidate=%s", baselinePath, candidatePath) + } + return nil +} + +func fatal(err error) { + _, _ = fmt.Fprintln(os.Stderr, err) + os.Exit(2) +} + +var _ replaycmd.LineReader = (*memoryLineReader)(nil) diff --git a/tests/replayer/differential/run.sh b/tests/replayer/differential/run.sh new file mode 100755 index 000000000..d38458814 --- /dev/null +++ b/tests/replayer/differential/run.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# Copyright 2026 PingCAP, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd) +corpus="$repo_root/tests/replayer/differential/corpus.v1.json" +tmp_dir=$(mktemp -d) + +cleanup() { + rm -rf "$tmp_dir" +} +trap cleanup EXIT INT TERM + +cd "$repo_root" +go run ./tests/replayer/differential/go-observer -corpus "$corpus" >"$tmp_dir/go.json" +CP_REPLAYER_CORPUS="$corpus" cargo run --locked --quiet --manifest-path rust/Cargo.toml \ + -p replayer-core --example cp_replayer_observer >"$tmp_dir/rust.json" +go run ./tests/replayer/differential/go-observer \ + -baseline "$tmp_dir/go.json" -candidate "$tmp_dir/rust.json" + +CP_REPLAYER_CORPUS="$corpus" CP_REPLAYER_MUTATE_CONNECTION_ID=1 \ + cargo run --locked --quiet --manifest-path rust/Cargo.toml \ + -p replayer-core --example cp_replayer_observer >"$tmp_dir/rust-mutated.json" +set +e +go run ./tests/replayer/differential/go-observer \ + -baseline "$tmp_dir/go.json" -candidate "$tmp_dir/rust-mutated.json" +status=$? +set -e +if [[ "$status" -eq 0 ]]; then + echo "CP-REPLAYER connection-ID mutation was not killed" >&2 + exit 1 +fi From c431d5a6391c617aef31e4991a6bb2dfc1ebb831 Mon Sep 17 00:00:00 2001 From: bb7133 Date: Thu, 3 Sep 2026 20:28:47 +0800 Subject: [PATCH 4/4] fix: harden replayer foundation semantics --- docs/design/rust-offline-replayer.md | 67 +++++-- rust/crates/replayer-core/src/checkpoint.rs | 40 ++++- rust/crates/replayer-core/src/command.rs | 4 + rust/crates/replayer-core/src/config.rs | 21 ++- rust/crates/replayer-core/src/decode/audit.rs | 165 ++++++++++++------ rust/crates/replayer-core/src/decode/mod.rs | 2 +- .../crates/replayer-core/src/decode/native.rs | 4 +- .../crates/replayer-core/src/decode/quoted.rs | 22 ++- rust/crates/replayer-core/src/engine.rs | 8 +- rust/crates/replayer-core/src/storage.rs | 67 ++++++- rust/crates/replayer-core/tests/dry_run.rs | 35 ++++ rust/vulnerability-exceptions.toml | 6 +- tests/replayer/differential/corpus.v1.json | 12 ++ 13 files changed, 348 insertions(+), 105 deletions(-) diff --git a/docs/design/rust-offline-replayer.md b/docs/design/rust-offline-replayer.md index 083d682dc..c5b9ea718 100644 --- a/docs/design/rust-offline-replayer.md +++ b/docs/design/rust-offline-replayer.md @@ -39,6 +39,15 @@ from each other. Deployments must not point both at the same replay input, checkpoint, and target at once. Cross-process coordination is an operational precondition, not an exactly-once lock supplied by this executable. +The first implementation PR is deliberately a **dry-run foundation**, not the +ownership cutover described by this target contract. It enables bounded input, +decode, filter, deterministic materialized ordering, and checkpoint evidence. +Backend execution, replay scheduling, SQL/report output, dynamic input, +service mode, and the Go-command cutover remain fail-closed until the relevant +evidence below is present. Where this document says the replayer "supports" or +"owns" those deferred paths, it describes the required cutover state rather +than behavior exposed by the foundation binary. + ## Input contracts ### Native capture format @@ -76,14 +85,18 @@ larger than 64 MiB fails closed rather than scanning without a bound. The extension decoder accepts `_LOG_TIME` plus the quoted tuple `EVENT` contract for `QUERY`, `EXECUTE`, `CONNECTION`, and `DISCONNECT`. It preserves the current `always`/`never` prepared-close strategies, skips redacted -parameters, and supports the existing end-time filter. `directed` close and -retry filtering remain rejected for this format. +parameters, and supports the existing end-time filter. For Go compatibility, +the command-start filter maps onto that same frontier; an explicit command-end +filter takes precedence. `directed` close and retry filtering remain rejected +for this format. ### Ordering and filtering Multiple readers are merged by `(command_start_timestamp, connection_id, -source_ordinal, record_ordinal)`. The first two keys preserve the observable Go -contract; the final keys make previously unstable exact ties deterministic. +source_ordinal, record_ordinal, command_ordinal)`. The first two keys preserve +the observable Go contract; the final keys make previously unstable exact ties +deterministic, including `PREPARE` / `EXECUTE` / `CLOSE` commands expanded from +one audit record. Audit inputs use the configured bounded reorder buffer. Native input rejects multiple roots and a non-directed prepared-close strategy. @@ -101,13 +114,17 @@ Apache OpenDAL. The dependency is exact-pinned, has default features disabled, and enables only `fs`, `s3`, `gcs`, `azblob`, `oss`, and `cos` plus the Tokio executor. Provider clients never escape this module. -The URI adapter freezes the Go query parameters, provider root/path rules, -ordered pagination, and directory listing behavior. Credentials are accepted -only through the provider-specific configuration boundary and are redacted -from errors, diagnostics, job JSON, and logs. Transient reads/lists use the -bounded, cancellation-aware retry classifier from the CP-002 policy; invalid +The URI adapter accepts a provider-specific allowlist of the Go-compatible +read-side query parameters and rejects unknown, duplicate, bucket/container, +and root overrides before constructing OpenDAL. It preserves provider +root/path rules, ordered pagination, and directory listing behavior. +Credentials are accepted only through that configuration boundary and are +redacted from errors, diagnostics, job JSON, and logs. The foundation performs +one bounded storage attempt and fails closed. Before service, backend, or +dynamic-input mode is enabled, transient reads/lists must use the bounded, +cancellation-aware retry classifier from the CP-002 policy; invalid configuration, permission failures, missing objects, corrupt data, and decode -failures are terminal. +failures remain terminal. Local checkpoint and metadata writes use create-in-the-same-directory, `sync_all`, atomic rename, and parent-directory sync. The current Go checkpoint @@ -123,8 +140,9 @@ Capture-side remote publication remains owned by `CP-CAPTURE` (`#151`). Native streams apply AES-256-CTR and gzip in the same layer order as Go. The first 16 bytes are the random IV and the key is exactly the first 32 bytes of the key file. Short keys, short IVs, invalid gzip streams, wrong metadata, and -trailing/truncated records fail closed. Plaintext keys and credentials are -zeroized after construction. +trailing/truncated records fail closed. Local AES key-file contents use a +zeroizing buffer. URI credentials are redacted from diagnostics, but complete +provider-config zeroization is a cutover gate before untrusted service mode. Dynamic input initially supports the Go-compatible local and S3 modes. A directory is assigned with FNV-1a modulo `replayer-count`; `replayer-index` @@ -157,13 +175,19 @@ reconnects and restores the command's current database. `--ignore-errs` controls command failures, never configuration, corruption, authentication, or ownership failures. -`--dry-run` performs the complete read/decrypt/decompress/decode/order/filter -path without a backend. It produces the same command/filter counts and -checkpoint frontier as execution mode. +`--dry-run` performs the read/decrypt/decompress/decode/order/filter path +without a backend. The foundation materializes and globally sorts decoded +commands, so it does not yet exercise the configured reorder-buffer bound. +Backend/execution modes stay fail-closed until a bounded merge/scheduler uses +the same ordering and proves equal command/filter counts and checkpoint +frontier. -Checkpoints contain the current command start/end timestamp frontier and an -input identity digest. Loading a checkpoint with different inputs, format, -filters, partitioning, or encryption metadata fails closed. SIGINT/SIGTERM +Checkpoints contain schema-v2's full durable frontier +`(command_start_timestamp, connection_id, source_ordinal, record_ordinal, +command_ordinal)`, the command end timestamp, and an input identity digest that +also binds the frontier schema. Loading a checkpoint with an older schema or +different inputs, format, filters, partitioning, or encryption metadata fails +closed. SIGINT/SIGTERM stops new scheduling, drains only when graceful cancellation was requested, publishes the final checkpoint/report atomically, joins all owned tasks, and then exits. The restart contract is explicitly bounded at-least-once, not @@ -190,7 +214,12 @@ supports immediate and graceful modes with one absolute timeout. ## Differential and fault evidence -The ownership move requires all of the following from the exact PR head: +The ownership move requires all of the following from the exact cutover PR +head. The current foundation evidence covers well-formed native, audit-plugin, +and audit-extension command observations (including allocation, duplicate and +retry filtering, prepared expansion, and binary `KindBytes`), plus Rust-local +malformed/boundary, gzip/AES, storage-mapping, and checkpoint tests. It does not +claim the complete corpus or live/provider/restart coverage listed below. 1. A Go oracle and the production Rust decoders consume the same native, plugin, and extension corpus. Semantic command observations, normalized diff --git a/rust/crates/replayer-core/src/checkpoint.rs b/rust/crates/replayer-core/src/checkpoint.rs index 9aa07c21b..c57d99eed 100644 --- a/rust/crates/replayer-core/src/checkpoint.rs +++ b/rust/crates/replayer-core/src/checkpoint.rs @@ -24,7 +24,7 @@ use serde::{Deserialize, Serialize}; use crate::ReplayError; -const CHECKPOINT_SCHEMA_VERSION: u32 = 1; +pub(crate) const CHECKPOINT_SCHEMA_VERSION: u32 = 2; static TEMPORARY_SEQUENCE: AtomicU64 = AtomicU64::new(0); /// Digest binding a checkpoint to the exact replay input and filters. @@ -66,6 +66,8 @@ pub struct Checkpoint { pub source_ordinal: u64, /// Stable source-local record ordinal. pub record_ordinal: u64, + /// Stable command ordinal within a source-local record. + pub command_ordinal: u32, /// Number of commands committed before and including this frontier. pub committed_commands: u64, /// Number of read-only-filtered commands before and including this frontier. @@ -84,6 +86,7 @@ impl Checkpoint { connection_id: u64, source_ordinal: u64, record_ordinal: u64, + command_ordinal: u32, committed_commands: u64, filtered_commands: u64, ) -> Self { @@ -96,6 +99,7 @@ impl Checkpoint { connection_id, source_ordinal, record_ordinal, + command_ordinal, committed_commands, filtered_commands, } @@ -147,7 +151,7 @@ impl Checkpoint { /// Returns an error when the checkpoint cannot be encoded or durably /// written to the requested local path. pub fn save_atomic(&self, path: &Path) -> Result<(), ReplayError> { - let parent = path.parent().unwrap_or_else(|| Path::new(".")); + let parent = checkpoint_parent(path); let file_name = path .file_name() .and_then(|name| name.to_str()) @@ -208,6 +212,12 @@ impl Checkpoint { } } +fn checkpoint_parent(path: &Path) -> &Path { + path.parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .unwrap_or_else(|| Path::new(".")) +} + #[cfg(test)] #[allow(clippy::expect_used)] mod tests { @@ -222,8 +232,18 @@ mod tests { std::fs::create_dir_all(&directory).expect("create temp directory"); let path = directory.join("checkpoint.json"); let identity = InputIdentity::from_canonical_bytes(b"input-a"); - let checkpoint = - Checkpoint::new(identity.clone(), 10, 11, "input".to_owned(), 7, 1, 2, 3, 1); + let checkpoint = Checkpoint::new( + identity.clone(), + 10, + 11, + "input".to_owned(), + 7, + 1, + 2, + 0, + 3, + 1, + ); checkpoint.save_atomic(&path).expect("save checkpoint"); assert_eq!( Checkpoint::load(&path, &identity).expect("load"), @@ -233,4 +253,16 @@ mod tests { assert!(Checkpoint::load(&path, &other).is_err()); std::fs::remove_dir_all(directory).expect("remove temp directory"); } + + #[test] + fn bare_filename_uses_current_directory_for_sync() { + assert_eq!( + checkpoint_parent(Path::new("checkpoint.json")), + Path::new(".") + ); + assert_eq!( + checkpoint_parent(Path::new("state/checkpoint.json")), + Path::new("state") + ); + } } diff --git a/rust/crates/replayer-core/src/command.rs b/rust/crates/replayer-core/src/command.rs index d2ff2ed7b..7360c0fbd 100644 --- a/rust/crates/replayer-core/src/command.rs +++ b/rust/crates/replayer-core/src/command.rs @@ -227,6 +227,9 @@ pub struct Command { pub source_ordinal: u64, /// Stable record ordinal within one source. pub record_ordinal: u64, + /// Stable command ordinal within a record that expands to multiple commands. + #[serde(default)] + pub command_ordinal: u32, } impl Command { @@ -256,6 +259,7 @@ impl Command { line, source_ordinal: 0, record_ordinal: 0, + command_ordinal: 0, }) } } diff --git a/rust/crates/replayer-core/src/config.rs b/rust/crates/replayer-core/src/config.rs index 93eb15427..23f4a2764 100644 --- a/rust/crates/replayer-core/src/config.rs +++ b/rust/crates/replayer-core/src/config.rs @@ -15,7 +15,6 @@ //! Replay configuration and compatibility validation. use std::path::PathBuf; -use std::time::Duration; use serde::{Deserialize, Serialize}; use time::OffsetDateTime; @@ -107,7 +106,11 @@ impl ReplayConfig { "speed should be between 0.1 and 10".to_owned(), )); } - if self.start_time + Duration::from_secs(60) < now { + let latest_allowed_now = self + .start_time + .checked_add(time::Duration::seconds(60)) + .ok_or_else(|| ReplayError::Config("start time grace period overflows".to_owned()))?; + if latest_allowed_now < now { return Err(ReplayError::Config( "start time should not be in the past".to_owned(), )); @@ -236,4 +239,18 @@ mod tests { extension.format = TrafficFormat::AuditLogExtension; assert!(extension.validate(now).is_err()); } + + #[test] + fn start_time_grace_period_overflow_fails_closed() { + let maximum = time::Date::MAX + .with_hms(23, 59, 59) + .expect("valid maximum time") + .assume_utc(); + let mut overflow = config(maximum); + overflow.start_time = maximum; + let error = overflow + .validate(maximum) + .expect_err("overflow must fail closed"); + assert!(error.to_string().contains("grace period overflows")); + } } diff --git a/rust/crates/replayer-core/src/decode/audit.rs b/rust/crates/replayer-core/src/decode/audit.rs index 377126049..da0b986b6 100644 --- a/rust/crates/replayer-core/src/decode/audit.rs +++ b/rust/crates/replayer-core/src/decode/audit.rs @@ -17,7 +17,7 @@ use std::time::Duration; use time::OffsetDateTime; -use crate::decode::parse_go_quoted; +use crate::decode::{parse_go_quoted, parse_go_quoted_utf8}; use crate::{Command, CommandCode, PreparedCloseStrategy, ReplayError, TrafficFormat}; const AUDIT_TIME_FORMAT: &str = concat!( @@ -150,7 +150,16 @@ impl AuditDecoder { ReplayError::decode(&self.path, offset, "audit line is not valid UTF-8") })?; self.record_ordinal = self.record_ordinal.saturating_add(1); - let commands = self.decode_line(text, offset, line)?; + let mut commands = self.decode_line(text, offset, line)?; + for (command_ordinal, command) in commands.iter_mut().enumerate() { + command.command_ordinal = u32::try_from(command_ordinal).map_err(|_| { + ReplayError::decode( + &self.path, + offset, + "too many commands expanded from one audit record", + ) + })?; + } output.extend(commands); offset = if end < input.len() { end + 1 } else { end }; } @@ -248,7 +257,7 @@ impl AuditDecoder { } let encoded_command = fields.get("COMMAND").map_or("", String::as_str); let command_name = if encoded_command.starts_with('"') { - parse_go_quoted(encoded_command) + parse_go_quoted_utf8(encoded_command) .map_err(|message| ReplayError::decode(&self.path, offset, message))? } else { encoded_command.to_owned() @@ -302,10 +311,11 @@ impl AuditDecoder { ) -> Result, ReplayError> { let end_time = parse_audit_time(required(fields, "_LOG_TIME", &self.path, offset)?) .map_err(|message| ReplayError::decode(&self.path, offset, message))?; - if self - .command_end_time - .is_some_and(|frontier| end_time < frontier) - { + // The Go extension decoder maps command-start-time onto its sole + // end-time frontier. A non-zero command-end-time is then applied + // later and takes precedence. + let frontier = self.command_end_time.or(self.command_start_time); + if frontier.is_some_and(|frontier| end_time < frontier) { return Ok(Vec::new()); } if !self.user_allowed(fields) { @@ -313,7 +323,7 @@ impl AuditDecoder { } let replay_id = self.ensure_connection(upstream_connection_id)?; let encoded_event = required(fields, "EVENT", &self.path, offset)?; - let event = parse_go_quoted(encoded_event) + let event = parse_go_quoted_utf8(encoded_event) .map_err(|message| ReplayError::decode(&self.path, offset, message))?; let event = event .strip_prefix('[') @@ -694,7 +704,7 @@ fn required<'a>( fn unquote_if_needed(value: &str) -> Result { if value.starts_with('"') { - parse_go_quoted(value) + parse_go_quoted_utf8(value) } else if value.is_empty() { Err("empty SQL or command value".to_owned()) } else { @@ -769,11 +779,9 @@ fn prepared_command( } fn parse_plugin_parameters(input: &str) -> Result, String> { - let decoded = parse_go_quoted(input).map_err(|error| format!("invalid params: {error}"))?; - let body = decoded - .strip_prefix('[') - .and_then(|value| value.strip_suffix(']')) - .ok_or_else(|| "params have no surrounding brackets".to_owned())?; + let decoded = + parse_go_quoted(input.as_bytes()).map_err(|error| format!("invalid params: {error}"))?; + let body = bracket_body(&decoded)?; if body.is_empty() { return Ok(Vec::new()); } @@ -782,33 +790,36 @@ fn parse_plugin_parameters(input: &str) -> Result, String> { .into_iter() .map(|value| { let decoded = parse_go_quoted(value)?; - let (kind, value) = decoded - .split_once(' ') - .ok_or_else(|| format!("no space in param {decoded}"))?; + let separator = decoded + .iter() + .position(|byte| *byte == b' ') + .ok_or_else(|| { + format!("no space in param {}", String::from_utf8_lossy(&decoded)) + })?; + let kind = std::str::from_utf8(&decoded[..separator]) + .map_err(|_| "parameter kind is not valid UTF-8".to_owned())?; + let value = &decoded[separator + 1..]; match kind { "KindNull" => Ok(Parameter::Null), - "KindInt64" => value + "KindInt64" => parameter_text(value)? .parse::() .map(Parameter::Signed) - .map_err(|_| format!("invalid signed param {value}")), - "KindUint64" => value + .map_err(|_| "invalid signed param".to_owned()), + "KindUint64" => parameter_text(value)? .parse::() .map(Parameter::Unsigned) - .map_err(|_| format!("invalid unsigned param {value}")), - "KindFloat32" => value + .map_err(|_| "invalid unsigned param".to_owned()), + "KindFloat32" => parameter_text(value)? .parse::() .map(Parameter::Float) - .map_err(|_| format!("invalid float param {value}")), - "KindFloat64" | "KindMysqlDecimal" => value + .map_err(|_| "invalid float param".to_owned()), + "KindFloat64" | "KindMysqlDecimal" => parameter_text(value)? .parse::() .map(Parameter::Double) - .map_err(|_| format!("invalid double param {value}")), - "KindString" => parse_go_quoted(&format!("\"{value}\"")) - .map(|value| Parameter::String(value.into_bytes())), - "KindBytes" => parse_go_quoted(&format!("\"{value}\"")) - .map(|value| Parameter::String(value.into_bytes())), + .map_err(|_| "invalid double param".to_owned()), + "KindString" | "KindBytes" => parse_parameter_bytes(value).map(Parameter::String), "KindBinaryLiteral" | "KindMysqlBit" | "KindMysqlSet" | "KindMysqlTime" - | "KindMysqlJSON" => Ok(Parameter::String(value.as_bytes().to_vec())), + | "KindMysqlJSON" => Ok(Parameter::String(value.to_vec())), "KindMysqlDuration" | "KindMysqlEnum" | "KindInterface" | "KindMinNotNull" | "KindMaxValue" | "KindRaw" => Err(format!("unsupported param type {kind}")), _ => Err(format!("unknown param type {kind}")), @@ -818,24 +829,36 @@ fn parse_plugin_parameters(input: &str) -> Result, String> { } fn parse_extension_parameters(input: &str) -> Result, String> { - let decoded = parse_go_quoted(input).map_err(|error| format!("invalid params: {error}"))?; - let body = decoded - .strip_prefix('[') - .and_then(|value| value.strip_suffix(']')) - .ok_or_else(|| "params have no surrounding brackets".to_owned())?; + let decoded = + parse_go_quoted(input.as_bytes()).map_err(|error| format!("invalid params: {error}"))?; + let body = bracket_body(&decoded)?; if body.is_empty() { return Ok(Vec::new()); } - split_extension_list(body).map(|values| { - values - .into_iter() - .map(|value| Parameter::String(value.into_bytes())) - .collect() - }) + split_extension_list(body).map(|values| values.into_iter().map(Parameter::String).collect()) +} + +fn bracket_body(input: &[u8]) -> Result<&[u8], String> { + if input.first() != Some(&b'[') || input.last() != Some(&b']') { + return Err("params have no surrounding brackets".to_owned()); + } + Ok(&input[1..input.len() - 1]) } -fn split_quoted_list(input: &str) -> Result, String> { - let bytes = input.as_bytes(); +fn parameter_text(input: &[u8]) -> Result<&str, String> { + std::str::from_utf8(input).map_err(|_| "numeric parameter is not valid UTF-8".to_owned()) +} + +fn parse_parameter_bytes(input: &[u8]) -> Result, String> { + let mut quoted = Vec::with_capacity(input.len().saturating_add(2)); + quoted.push(b'"'); + quoted.extend_from_slice(input); + quoted.push(b'"'); + parse_go_quoted("ed) +} + +fn split_quoted_list(input: &[u8]) -> Result, String> { + let bytes = input; let mut values = Vec::new(); let mut index = 0_usize; while index < bytes.len() { @@ -858,11 +881,7 @@ fn split_quoted_list(input: &str) -> Result, String> { b'\\' => index += 2, b'"' => { index += 1; - values.push( - input - .get(start..index) - .ok_or_else(|| "parameter is not on UTF-8 boundaries".to_owned())?, - ); + values.push(&input[start..index]); closed = true; break; } @@ -876,8 +895,8 @@ fn split_quoted_list(input: &str) -> Result, String> { Ok(values) } -fn split_extension_list(input: &str) -> Result, String> { - let bytes = input.as_bytes(); +fn split_extension_list(input: &[u8]) -> Result>, String> { + let bytes = input; let mut values = Vec::new(); let mut index = 0_usize; while index < bytes.len() { @@ -898,9 +917,7 @@ fn split_extension_list(input: &str) -> Result, String> { b'\\' => index += 2, b'"' => { index += 1; - let quoted = input - .get(start..index) - .ok_or_else(|| "parameter is not on UTF-8 boundaries".to_owned())?; + let quoted = &input[start..index]; values.push(parse_go_quoted(quoted)?); break; } @@ -915,11 +932,7 @@ fn split_extension_list(input: &str) -> Result, String> { while index < bytes.len() && bytes[index] != b',' { index += 1; } - let value = input - .get(start..index) - .ok_or_else(|| "parameter is not on UTF-8 boundaries".to_owned())? - .trim(); - values.push(value.to_owned()); + values.push(input[start..index].trim_ascii().to_vec()); } } Ok(values) @@ -1092,6 +1105,42 @@ mod tests { ); } + #[test] + fn extension_maps_command_start_time_to_end_time_frontier() { + let lines = concat!( + "[2026/01/08 19:44:11.099 +08:00] ", + "[EVENT=\"[QUERY,QUERY_DDL]\"] [USER=root] [CONNECTION_ID=7] ", + "[CURRENT_DB=test] [SQL_TEXT=\"CREATE TABLE t (id INT)\"]\n", + "[2026/01/08 19:44:11.110 +08:00] ", + "[EVENT=\"[QUERY,QUERY_DML,INSERT]\"] [USER=root] [CONNECTION_ID=7] ", + "[CURRENT_DB=test] [SQL_TEXT=\"INSERT INTO t VALUES (1)\"]\n" + ); + let frontier = parse_audit_time("2026/01/08 19:44:11.110 +08:00").expect("valid frontier"); + let mut decoder = AuditDecoder::new( + TrafficFormat::AuditLogExtension, + "audit.log", + 4096, + PreparedCloseStrategy::Never, + Some(frontier), + None, + false, + &[], + 0, + ) + .expect("decoder"); + let commands = decoder.decode_all(lines.as_bytes()).expect("decode"); + assert_eq!(commands.len(), 1); + assert_eq!(commands[0].payload, b"\x03INSERT INTO t VALUES (1)"); + } + + #[test] + fn plugin_kind_bytes_preserves_non_utf8_payload() { + let parameters = parse_plugin_parameters(r#""[\"KindBytes \\\\xff\"]""#) + .expect("binary KindBytes parameter"); + let payload = execute_payload(7, ¶meters).expect("execute payload"); + assert_eq!(payload.last(), Some(&0xff)); + } + #[test] fn ignored_plugin_events_still_advance_connection_allocator() { let lines = concat!( diff --git a/rust/crates/replayer-core/src/decode/mod.rs b/rust/crates/replayer-core/src/decode/mod.rs index 751a0e057..ac16c0b61 100644 --- a/rust/crates/replayer-core/src/decode/mod.rs +++ b/rust/crates/replayer-core/src/decode/mod.rs @@ -20,4 +20,4 @@ mod quoted; pub use audit::{AuditDecoder, parse_bracket_fields}; pub use native::NativeDecoder; -pub(crate) use quoted::parse_go_quoted; +pub(crate) use quoted::{parse_go_quoted, parse_go_quoted_utf8}; diff --git a/rust/crates/replayer-core/src/decode/native.rs b/rust/crates/replayer-core/src/decode/native.rs index 585407ad1..b0c94331d 100644 --- a/rust/crates/replayer-core/src/decode/native.rs +++ b/rust/crates/replayer-core/src/decode/native.rs @@ -17,7 +17,7 @@ use std::collections::HashSet; use time::OffsetDateTime; use time::format_description::well_known::Rfc3339; -use crate::decode::parse_go_quoted; +use crate::decode::parse_go_quoted_utf8; use crate::{Command, CommandCode, ReplayError}; /// Stateful decoder for `TiProxy` native capture records. @@ -149,7 +149,7 @@ impl<'a> NativeDecoder<'a> { })?); } "Prepared_stmt" => { - prepared_statement = Some(parse_go_quoted(value).map_err(|message| { + prepared_statement = Some(parse_go_quoted_utf8(value).map_err(|message| { ReplayError::decode(&self.path, line_offset, message) })?); } diff --git a/rust/crates/replayer-core/src/decode/quoted.rs b/rust/crates/replayer-core/src/decode/quoted.rs index 7bb096a6b..0949ea67d 100644 --- a/rust/crates/replayer-core/src/decode/quoted.rs +++ b/rust/crates/replayer-core/src/decode/quoted.rs @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -pub(crate) fn parse_go_quoted(input: &str) -> Result { - let bytes = input.as_bytes(); +pub(crate) fn parse_go_quoted(input: &[u8]) -> Result, String> { + let bytes = input; if bytes.len() < 2 || bytes.first() != Some(&b'"') || bytes.last() != Some(&b'"') { return Err("value must be a Go-quoted string".to_owned()); } @@ -80,7 +80,12 @@ pub(crate) fn parse_go_quoted(input: &str) -> Result { } index += 1; } - String::from_utf8(output).map_err(|_| "quoted value is not valid UTF-8".to_owned()) + Ok(output) +} + +pub(crate) fn parse_go_quoted_utf8(input: &str) -> Result { + String::from_utf8(parse_go_quoted(input.as_bytes())?) + .map_err(|_| "quoted value is not valid UTF-8".to_owned()) } fn parse_hex(bytes: &[u8], start: usize, count: usize) -> Result { @@ -115,14 +120,15 @@ mod tests { #[test] fn accepts_go_specific_escapes() { assert_eq!( - parse_go_quoted(r#""a\x00\u4e2d\U0001f60a\141""#).expect("valid"), - "a\0中😊a" + parse_go_quoted(br#""a\x00\u4e2d\U0001f60a\141""#).expect("valid"), + "a\0中😊a".as_bytes() ); } #[test] - fn rejects_invalid_utf8_and_short_escapes() { - assert!(parse_go_quoted(r#""\xff""#).is_err()); - assert!(parse_go_quoted(r#""\x0""#).is_err()); + fn preserves_binary_and_rejects_short_escapes() { + assert_eq!(parse_go_quoted(br#""\xff""#).expect("binary"), [0xff]); + assert!(parse_go_quoted(br#""\x0""#).is_err()); + assert!(parse_go_quoted_utf8(r#""\xff""#).is_err()); } } diff --git a/rust/crates/replayer-core/src/engine.rs b/rust/crates/replayer-core/src/engine.rs index 0039ca9e5..7feb73371 100644 --- a/rust/crates/replayer-core/src/engine.rs +++ b/rust/crates/replayer-core/src/engine.rs @@ -18,7 +18,7 @@ use std::collections::{HashMap, HashSet}; use serde::{Deserialize, Serialize}; -use crate::checkpoint::{Checkpoint, InputIdentity}; +use crate::checkpoint::{CHECKPOINT_SCHEMA_VERSION, Checkpoint, InputIdentity}; use crate::config::ReplayConfig; use crate::decode::{AuditDecoder, NativeDecoder}; use crate::storage::InputRoot; @@ -141,6 +141,7 @@ pub async fn dry_run(config: &ReplayConfig) -> Result Result ( frontier.command_start_unix_nanos, frontier.connection_id, frontier.source_ordinal, frontier.record_ordinal, + frontier.command_ordinal, ) }); } @@ -182,6 +185,7 @@ pub async fn dry_run(config: &ReplayConfig) -> Result Result { + checkpoint_frontier_schema_version: u32, format: TrafficFormat, read_only: bool, prepared_close: crate::PreparedCloseStrategy, @@ -225,6 +230,7 @@ struct IdentityConfig<'a> { impl<'a> From<&'a ReplayConfig> for IdentityConfig<'a> { fn from(config: &'a ReplayConfig) -> Self { Self { + checkpoint_frontier_schema_version: CHECKPOINT_SCHEMA_VERSION, format: config.format, read_only: config.read_only, prepared_close: config.prepared_close, diff --git a/rust/crates/replayer-core/src/storage.rs b/rust/crates/replayer-core/src/storage.rs index 93bee58c7..3d9571cbb 100644 --- a/rust/crates/replayer-core/src/storage.rs +++ b/rust/crates/replayer-core/src/storage.rs @@ -14,7 +14,7 @@ //! Credential-redacted `OpenDAL` input ownership. -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::io::Read; use std::path::{Path, PathBuf}; @@ -129,8 +129,18 @@ impl InputRoot { } let root = parsed.path().trim_matches('/'); config.insert("root".to_owned(), format!("/{root}")); + let mut seen_options = HashSet::new(); for (key, value) in parsed.query_pairs() { - if let Some((normalized, value)) = provider_query_option(scheme, &key, &value)? { + let option = provider_query_option(scheme, &key, &value)?; + let normalized = option + .as_ref() + .map_or_else(|| "provider".to_owned(), |(key, _)| key.clone()); + if !seen_options.insert(normalized.clone()) { + return Err(ReplayError::Config(format!( + "duplicate object-store query option {normalized}" + ))); + } + if let Some((normalized, value)) = option { config.insert(normalized, value); } } @@ -417,8 +427,8 @@ fn provider_query_option( key: &str, value: &str, ) -> Result, ReplayError> { - let canonical = key.replace('-', "_"); - if canonical == "provider" { + let canonical = key.to_ascii_lowercase().replace('-', "_"); + if canonical == "provider" && matches!(scheme, Scheme::S3 | Scheme::Oss) { return Ok(None); } if scheme == Scheme::S3 && canonical == "force_path_style" { @@ -431,13 +441,25 @@ fn provider_query_option( ))); } let normalized = match (scheme, canonical.as_str()) { + ( + Scheme::S3, + "endpoint" | "region" | "secret_access_key" | "session_token" | "role_arn" + | "external_id" | "profile", + ) + | (Scheme::Azblob, "endpoint" | "account_name" | "account_key" | "sas_token") => { + canonical.as_str() + } (Scheme::S3 | Scheme::Oss, "access_key" | "access_key_id") => "access_key_id", - (Scheme::Cos, "access_key" | "access_key_id" | "secret_id") => "secret_id", - (Scheme::S3, "secret_access_key") => "secret_access_key", + (Scheme::Oss | Scheme::Cos | Scheme::Gcs, "endpoint") => "endpoint", (Scheme::Oss, "secret_access_key" | "access_key_secret") => "access_key_secret", + (Scheme::Cos, "access_key" | "access_key_id" | "secret_id") => "secret_id", (Scheme::Cos, "secret_access_key" | "secret_key") => "secret_key", (Scheme::Gcs, "credentials_file") => "credential_path", - _ => canonical.as_str(), + _ => { + return Err(ReplayError::Config(format!( + "unsupported object-store query option {canonical} for {scheme:?}" + ))); + } }; Ok(Some((normalized.to_owned(), value.to_owned()))) } @@ -560,6 +582,37 @@ mod tests { None ); assert!(provider_query_option(Scheme::S3, "force-path-style", "maybe").is_err()); + assert!(provider_query_option(Scheme::S3, "root", "/override").is_err()); + assert!(provider_query_option(Scheme::S3, "bucket", "override").is_err()); + assert!(provider_query_option(Scheme::S3, "unknown", "value").is_err()); + assert!(provider_query_option(Scheme::Gcs, "provider", "ignored").is_err()); + } + + #[test] + fn object_store_query_rejects_alias_duplicates_and_root_override() { + let duplicate = InputRoot::open("s3://bucket/prefix?access-key=a&access_key_id=b") + .expect_err("aliases must not use last-wins semantics"); + assert!( + duplicate + .to_string() + .contains("duplicate object-store query option") + ); + + let override_root = InputRoot::open("s3://bucket/prefix?root=/elsewhere") + .expect_err("root override must fail closed"); + assert!( + override_root + .to_string() + .contains("unsupported object-store query option root") + ); + + let override_container = InputRoot::open("azblob://container/prefix?container=other") + .expect_err("container override must fail closed"); + assert!( + override_container + .to_string() + .contains("unsupported object-store query option container") + ); } #[test] diff --git a/rust/crates/replayer-core/tests/dry_run.rs b/rust/crates/replayer-core/tests/dry_run.rs index f60deb641..6a6432878 100644 --- a/rust/crates/replayer-core/tests/dry_run.rs +++ b/rust/crates/replayer-core/tests/dry_run.rs @@ -97,6 +97,7 @@ async fn checkpoint_uses_full_tie_frontier_and_content_identity() -> TestResult assert_eq!(checkpoint.connection_id, 2); assert_eq!(checkpoint.source_ordinal, 0); assert_eq!(checkpoint.record_ordinal, 1); + assert_eq!(checkpoint.command_ordinal, 0); let resumed = dry_run(&config).await?; assert_eq!(resumed.decoded_commands, 0); @@ -118,6 +119,40 @@ async fn checkpoint_uses_full_tie_frontier_and_content_identity() -> TestResult Ok(()) } +#[tokio::test] +async fn checkpoint_frontier_distinguishes_commands_expanded_from_one_record() -> TestResult { + let directory = temporary_directory("checkpoint-command-ordinal")?; + let traffic = directory.join("tidb-audit-2026-01-08T19-44-11.099.log"); + let checkpoint_path = directory.join("checkpoint.json"); + std::fs::write( + &traffic, + concat!( + "[2026/01/08 19:44:11.114 +08:00] ", + "[TIMESTAMP=2026/01/08 19:44:11.114 +08:00] ", + "[EVENT_CLASS=GENERAL] [EVENT=COMPLETED] [COMMAND=Execute] ", + "[SQL_TEXT=SELECT ?] [SQL_STATEMENTS=Select] [CONNECTION_ID=7] ", + "[EXECUTE_PARAMS=\"[]\"] [CURRENT_DB=test]\n" + ), + )?; + + let mut config = dry_run_config(&directory, Some(checkpoint_path.clone())); + config.format = TrafficFormat::AuditLogPlugin; + config.prepared_close = PreparedCloseStrategy::Always; + let first = dry_run(&config).await?; + assert_eq!(first.decoded_commands, 3); + let checkpoint = Checkpoint::load(&checkpoint_path, &first.input_identity)? + .ok_or("checkpoint was not persisted")?; + assert_eq!(checkpoint.record_ordinal, 1); + assert_eq!(checkpoint.command_ordinal, 2); + + let resumed = dry_run(&config).await?; + assert_eq!(resumed.decoded_commands, 0); + assert_eq!(resumed.replayed_commands, 0); + + std::fs::remove_dir_all(directory)?; + Ok(()) +} + #[tokio::test] async fn gzip_then_aes_ctr_matches_native_storage_layers() -> TestResult { let directory = temporary_directory("encrypted")?; diff --git a/rust/vulnerability-exceptions.toml b/rust/vulnerability-exceptions.toml index 9ac37aa0f..826877d35 100644 --- a/rust/vulnerability-exceptions.toml +++ b/rust/vulnerability-exceptions.toml @@ -14,16 +14,16 @@ schema_version = 1 advisory = "RUSTSEC-2023-0071" owner = "@bb7133" rationale = "OpenDAL 0.53.3 reaches rsa through request signing, but this slice is operator-run dry-run-only with trusted storage endpoints; upgrade OpenDAL before enabling service or backend modes." -expires = 2027-03-01 +expires = 2026-12-01 [[exceptions]] advisory = "RUSTSEC-2026-0194" owner = "@bb7133" rationale = "OpenDAL 0.53.3 reaches quick-xml through cloud provider support, but this slice is operator-run dry-run-only with trusted storage endpoints; upgrade OpenDAL before enabling service or backend modes." -expires = 2027-03-01 +expires = 2026-12-01 [[exceptions]] advisory = "RUSTSEC-2026-0195" owner = "@bb7133" rationale = "OpenDAL 0.53.3 reaches quick-xml through cloud provider support, but this slice is operator-run dry-run-only with trusted storage endpoints; upgrade OpenDAL before enabling service or backend modes." -expires = 2027-03-01 +expires = 2026-12-01 diff --git a/tests/replayer/differential/corpus.v1.json b/tests/replayer/differential/corpus.v1.json index 85bafb763..e3d4d9d35 100644 --- a/tests/replayer/differential/corpus.v1.json +++ b/tests/replayer/differential/corpus.v1.json @@ -81,6 +81,18 @@ "[2026/01/08 19:44:11.114 +08:00] [EVENT=\"[QUERY,EXECUTE,SELECT]\"] [USER=root] [CONNECTION_ID=7] [CURRENT_DB=test] [SQL_TEXT=\"SELECT ?\"] [EXECUTE_PARAMS=\"[1]\"]", "[2026/01/08 19:44:11.115 +08:00] [EVENT=\"[CONNECTION,DISCONNECT]\"] [USER=root] [CONNECTION_ID=7]" ] + }, + { + "id": "plugin-binary-kind-bytes", + "format": "audit_log_plugin", + "prepared_close": "always", + "filter_retries": false, + "user_allowlist": [], + "source_ordinal": 4, + "records": [], + "lines": [ + "[2025/09/06 17:03:53.724 +08:00] [TIMESTAMP=2025/09/06 17:03:53.724 +08:00] [EVENT_CLASS=GENERAL] [EVENT=COMPLETED] [COMMAND=Execute] [SQL_TEXT=SELECT ?] [SQL_STATEMENTS=Select] [CONNECTION_ID=10] [EXECUTE_PARAMS=\"[\\\"KindBytes \\\\\\\\xff\\\"]\"] [RETRY=false]" + ] } ] }