diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index d6ccc9746..73e6f3423 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -32,6 +32,7 @@ on: - "scripts/generate-control-proto.sh" - "tests/controlplane/**" - "tests/dataplane/capture-format/**" + - "tests/replayer/**" push: branches: - main @@ -56,6 +57,7 @@ on: - "scripts/generate-control-proto.sh" - "tests/controlplane/**" - "tests/dataplane/capture-format/**" + - "tests/replayer/**" # The control-plane evidence jobs below do not run on pull requests. They # re-prove live etcd / Go-oracle behaviour that a pull request rarely # changes, and re-running them on every push to a branch was the bulk of @@ -131,6 +133,8 @@ jobs: run: make rust-lint - name: Run unit tests run: make rust-test + - name: Compare Go and Rust replay decoders and kill a mutation + run: make replayer-differential-evidence - name: Run doc tests run: make rust-doc-test - name: Reject a stale lockfile diff --git a/AGENTS.md b/AGENTS.md index 480511994..9e41be58c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -78,6 +78,7 @@ When adding or modifying features, prefer extending existing packages before cre - `pkg/util/` - Utilities, including buffered IO, etcd wrapper, HTTP wrapper, simple MySQL lexer, and others. - `rust/crates/mysql-wire/` - MySQL wire-format types and codec boundaries; it does not own sockets or routing decisions. - `rust/crates/proxy-io/` - Client/backend transport ownership, including future TLS, compression, and PROXY protocol support. +- `rust/crates/replayer-core/` - Bounded offline traffic replay primitives: input storage, native/audit decoding, deterministic ordering, filtering, and local checkpoints; it has no control-plane or SQL-serving dependency. - `rust/crates/session-core/` - Protocol-independent session lifecycle and migration state. - `rust/crates/control-router/` - Staged namespace-scoped routing selection, exact config/topology/health capture, and reservation accounting. Production dataplane/binary wiring remains prohibited until full routing parity is complete. - `rust/crates/control-routing/` - Protocol-independent route values, pure group matching/port-conflict primitives, and Go-compatible time arithmetic; it owns no reservation or production selector yet and carries no wire payloads. @@ -94,6 +95,7 @@ When adding or modifying features, prefer extending existing packages before cre - `rust/crates/dataplane/` - Rust dataplane orchestration across wire, transport, session, and control components. - `rust/crates/differential-runner/` - Payload-free Rust consumer for the immutable Go protocol corpus; it drives merged wire/session parsers and emits sharded semantic observations. - `rust/crates/tiproxy-rs/` - Rust dataplane executable and build/version metadata. +- `rust/crates/tiproxy-replayer/` - Standalone Rust offline traffic replayer composition root; replay payloads never enter the SQL-serving process or control bridge. - `tests/compatibility/` - Versioned client-driver and MySQL capability contract shared by the Go/Rust dataplane integration tests. - `tests/controlplane/cproute/api-differential/` - One test-only external router API input/output runner and its frozen acceptance/retirement inventories; synthetic adapter smoke does not qualify recorded-trace acceptance. Its `metrics/` package shares value-only query packets and decoding between recording and API replay; it owns no factor state. Its `clock/` package supplies the shared public event clock to test-build overlays only. - `tests/controlplane/` - Versioned Go control-plane contract and fault catalogs plus the payload-free observation comparator used by every Rust control-plane migration slice. diff --git a/Makefile b/Makefile index 1955be188..5f795f40e 100644 --- a/Makefile +++ b/Makefile @@ -43,7 +43,7 @@ CARGO_DENY_VERSION := 0.20.2 RUST_TOOL_ROOT ?= $(GOBIN)/rust-tools RUST_TOOL_BIN := $(RUST_TOOL_ROOT)/bin -.PHONY: cmd_% test lint parity-drift parity-drift-weekly docker docker-release golangci-lint gocovmerge clean rust-build rust-test rust-doc-test rust-lint rust-release rust-install-tools rust-supply-chain rust-negative-tests control-proto-generate control-proto-generate-check controlplane-contracts controlplane-differential controlplane-differential-self-test controlplane-cp001-evidence controlplane-cplog-evidence controlplane-cpadmin-evidence controlplane-cpdiag-evidence controlplane-cpctl-evidence controlplane-cpmeter-evidence controlplane-cp002-evidence controlplane-cp003-evidence controlplane-cp003-authority-evidence controlplane-cp004-evidence controlplane-cproute-evidence controlplane-cpmetrics-evidence controlplane-cpmetrics-applied-evidence controlplane-cpmetrics-collector-evidence controlplane-cpmetrics-factor-evidence controlplane-cproute-resource-evidence controlplane-cproute-migration-evidence controlplane-cproute-balance-evidence controlplane-cproute-worker-evidence controlplane-cproute-shadow-evidence controlplane-cproute-recorder-evidence controlplane-cproute-read-foundation-evidence controlplane-cproute-time-evidence controlplane-cproute-native-evidence controlplane-cptopo-evidence dataplane-differential dataplane-differential-coverage dataplane-differential-mutation dataplane-integration dataplane-t2-integration dataplane-t3-integration dataplane-t4-integration dataplane-t4-qualification dataplane-integration-go dataplane-integration-self-test +.PHONY: cmd_% test lint parity-drift parity-drift-weekly docker docker-release golangci-lint gocovmerge clean rust-build rust-test rust-doc-test rust-lint rust-release rust-install-tools rust-supply-chain rust-negative-tests control-proto-generate control-proto-generate-check controlplane-contracts controlplane-differential controlplane-differential-self-test controlplane-cp001-evidence controlplane-cplog-evidence controlplane-cpadmin-evidence controlplane-cpdiag-evidence controlplane-cpctl-evidence controlplane-cpmeter-evidence controlplane-cp002-evidence controlplane-cp003-evidence controlplane-cp003-authority-evidence controlplane-cp004-evidence controlplane-cproute-evidence controlplane-cpmetrics-evidence controlplane-cpmetrics-applied-evidence controlplane-cpmetrics-collector-evidence controlplane-cpmetrics-factor-evidence controlplane-cproute-resource-evidence controlplane-cproute-migration-evidence controlplane-cproute-balance-evidence controlplane-cproute-worker-evidence controlplane-cproute-shadow-evidence controlplane-cproute-recorder-evidence controlplane-cproute-read-foundation-evidence controlplane-cproute-time-evidence controlplane-cproute-native-evidence controlplane-cptopo-evidence replayer-differential-evidence dataplane-differential dataplane-differential-coverage dataplane-differential-mutation dataplane-integration dataplane-t2-integration dataplane-t3-integration dataplane-t4-integration dataplane-t4-qualification dataplane-integration-go dataplane-integration-self-test default: cmd @@ -209,6 +209,9 @@ controlplane-cproute-migration-evidence: controlplane-cptopo-evidence: $(RUST_BUILD_ENV) tests/controlplane/cptopo/run.sh +replayer-differential-evidence: + $(RUST_BUILD_ENV) tests/replayer/differential/run.sh + DIFFERENTIAL_SHARD_INDEX ?= 0 DIFFERENTIAL_SHARD_COUNT ?= 4 diff --git a/docs/design/rust-offline-replayer.md b/docs/design/rust-offline-replayer.md new file mode 100644 index 000000000..c5b9ea718 --- /dev/null +++ b/docs/design/rust-offline-replayer.md @@ -0,0 +1,248 @@ +# Rust offline replayer ownership + +This document freezes the `CP-REPLAYER` (`#152`) ownership boundary before +the Go `cmd/replayer` implementation moves to Rust. The target is a separate +`tiproxy-replayer` executable. It is deliberately not linked into the +`tiproxy-rs` SQL-serving process and does not weaken `PARITY-EXCL-001`. + +## Ownership and cutover + +The Rust executable owns the standalone one-shot replayer and its replay-only +service mode. The one-shot CLI keeps the existing flag names, defaults, and +validation. Service mode keeps these endpoints and response shapes: + +- `POST /api/traffic/replay`; +- `POST /api/traffic/cancel`; +- `GET /api/traffic/show`. + +The standalone server does not advertise or accept traffic capture. Capture +and capture-file production belong to `CP-CAPTURE` (`#151`); the integrated +control API belongs to `CP-ADMIN` (`#150`). The old Go command is removed only +after the Rust release artifact and the evidence below are green. The Go +`pkg/sqlreplay/capture` package and the temporarily integrated Go traffic API +are not made Rust owners by this slice and remain covered by +`PARITY-EXCL-001` until their owning issues cut them over. + +The implementation is split into: + +- `replayer-core`: bounded config, format decoders, storage, ordering, + scheduling, checkpointing, direct MySQL execution, and reports; +- `tiproxy-replayer`: CLI, signals, replay-only HTTP lifecycle, and version + output. + +Neither crate depends on `control-proto`, the legacy bridge, the topology +owner, or the SQL-serving dataplane. There is no replay payload over IPC or +FFI. A replayer process has exactly one job owner and at most one active job. +That invariant is process-local: during the migration window the standalone +Rust process and the integrated Go traffic API are not mechanically fenced +from each other. Deployments must not point both at the same replay input, +checkpoint, and target at once. Cross-process coordination is an operational +precondition, not an exactly-once lock supplied by this executable. + +The first implementation PR is deliberately a **dry-run foundation**, not the +ownership cutover described by this target contract. It enables bounded input, +decode, filter, deterministic materialized ordering, and checkpoint evidence. +Backend execution, replay scheduling, SQL/report output, dynamic input, +service mode, and the Go-command cutover remain fail-closed until the relevant +evidence below is present. Where this document says the replayer "supports" or +"owns" those deferred paths, it describes the required cutover state rather +than behavior exposed by the foundation binary. + +## Input contracts + +### Native capture format + +Native files preserve the current file and record contract: + +- `traffic-2006-01-02T15-04-05.999.log[.gz]`, ordered by the timestamp in the + filename and then by full path; +- optional `meta` JSON with version `v1`, duration, command counts, filtered + counts, and optional `aes256-ctr` method; +- one record headed by `# Time`, `# Conn_ID`, optional `# Cmd_type`, + `# Success`, `# Captured_ps_id`, `# Prepared_stmt`, and `# Payload_len`, + followed by exactly `Payload_len` binary bytes and one newline; +- the decoded payload starts with the MySQL command byte. Query is the default + when `Cmd_type` is absent. + +Timestamps use RFC3339Nano. Unknown fields are ignored for forward +compatibility, but duplicate required fields, invalid integers/timestamps, +negative lengths, truncated payloads, oversized records, or a missing final +newline fail closed with path and byte-offset context. The default per-record +payload cap is 64 MiB and is never allocated before the length is validated. + +### Audit log plugin format + +The plugin decoder accepts the current bracketed key/value grammar and its +timestamp layout `2006/01/02 15:04:05.999 -07:00`. It preserves current +handling of `GENERAL`, `TABLE_ACCESS`, and `CONNECTION` events; per-decoder +connection remapping; retry filtering; case-insensitive user allowlists; +duplicate-write suppression; current database; prepared statement text/ID; +and execute parameters. Malformed quoting, nesting, timestamps, or a line +larger than 64 MiB fails closed rather than scanning without a bound. + +### Audit log extension format + +The extension decoder accepts `_LOG_TIME` plus the quoted tuple `EVENT` +contract for `QUERY`, `EXECUTE`, `CONNECTION`, and `DISCONNECT`. It preserves +the current `always`/`never` prepared-close strategies, skips redacted +parameters, and supports the existing end-time filter. For Go compatibility, +the command-start filter maps onto that same frontier; an explicit command-end +filter takes precedence. `directed` close and retry filtering remain rejected +for this format. + +### Ordering and filtering + +Multiple readers are merged by `(command_start_timestamp, connection_id, +source_ordinal, record_ordinal, command_ordinal)`. The first two keys preserve +the observable Go contract; the final keys make previously unstable exact ties +deterministic, including `PREPARE` / `EXECUTE` / `CLOSE` commands expanded from +one audit record. +Audit inputs use the configured bounded reorder buffer. Native input rejects +multiple roots and a non-directed prepared-close strategy. + +`--read-only` uses a TiDB-aware bounded tokenizer/normalizer, not a generic SQL +parser. It freezes Go `Normalize` / `NormalizeDigest` output for the statements +used by reporting and the current read-only lexical decision. Tokens inside +comments, quoted strings, identifiers, and executable comments cannot become +keywords. Invalid UTF-8, unterminated constructs, nesting/token-count overflow, +and inputs over 64 MiB are classified explicitly and never panic or loop. + +## Storage contract + +`replayer-core` exposes a TiProxy-owned object-store trait implemented by +Apache OpenDAL. The dependency is exact-pinned, has default features disabled, +and enables only `fs`, `s3`, `gcs`, `azblob`, `oss`, and `cos` plus the Tokio +executor. Provider clients never escape this module. + +The URI adapter accepts a provider-specific allowlist of the Go-compatible +read-side query parameters and rejects unknown, duplicate, bucket/container, +and root overrides before constructing OpenDAL. It preserves provider +root/path rules, ordered pagination, and directory listing behavior. +Credentials are accepted only through that configuration boundary and are +redacted from errors, diagnostics, job JSON, and logs. The foundation performs +one bounded storage attempt and fails closed. Before service, backend, or +dynamic-input mode is enabled, transient reads/lists must use the bounded, +cancellation-aware retry classifier from the CP-002 policy; invalid +configuration, permission failures, missing objects, corrupt data, and decode +failures remain terminal. + +Local checkpoint and metadata writes use create-in-the-same-directory, +`sync_all`, atomic rename, and parent-directory sync. The current Go checkpoint +and SQL-output flags are filesystem paths, so `CP-REPLAYER` uses OpenDAL cloud +services only for read/list/stat and rejects remote checkpoint or output URLs. +It therefore has no shared remote writer and makes no cross-provider atomic +publication claim. If a later owner adds remote publication, it must query +OpenDAL's `Capability.write_with_if_not_exists` at runtime and use a provider's +native atomic conditional-create primitive; read-then-write emulation is not +acceptable. Providers without that capability must reject shared-writer mode. +Capture-side remote publication remains owned by `CP-CAPTURE` (`#151`). + +Native streams apply AES-256-CTR and gzip in the same layer order as Go. The +first 16 bytes are the random IV and the key is exactly the first 32 bytes of +the key file. Short keys, short IVs, invalid gzip streams, wrong metadata, and +trailing/truncated records fail closed. Local AES key-file contents use a +zeroizing buffer. URI credentials are redacted from diagnostics, but complete +provider-config zeroization is a cutover gate before untrusted service mode. + +Dynamic input initially supports the Go-compatible local and S3 modes. A +directory is assigned with FNV-1a modulo `replayer-count`; `replayer-index` +must be lower than that count. Listings are paginated and deduplicated. EOF +polling is driven by a cancellation-aware timer rather than an unconditional +sleep. + +## Replay engine contract + +The engine maintains a bounded queue per logical captured connection and one +socket owner per live replay connection. It preserves per-connection order, +uses the first captured timestamp and `--speed` to derive absolute deadlines, +and sends late commands immediately. Buffer, connection, pending-command, and +slowdown thresholds are explicit; overflow cancels the job with a typed error. + +Direct MySQL connections support TCP and backend TLS with bounded dial, +handshake, packet, and response deadlines. Authentication supports the TiDB +paths used by the Go replayer (`mysql_native_password` and +`caching_sha2_password`, including TLS full-auth and auth switch). Passwords, +scrambles, SQL payloads, and storage credentials are absent from `Debug` and +error output. + +Each command writes the exact captured payload and drains the complete response +before that connection advances. `COM_STMT_PREPARE` records the replay ID and +parameter count. `EXECUTE`, `FETCH`, `CLOSE`, `RESET`, and `SEND_LONG_DATA` +rewrite only the four-byte statement ID, using the captured-to-replay mapping. +The configured prepared-close strategy remains `directed`, `always`, or +`never`. A backend disconnect retires only its socket; the next command +reconnects and restores the command's current database. `--ignore-errs` +controls command failures, never configuration, corruption, authentication, +or ownership failures. + +`--dry-run` performs the read/decrypt/decompress/decode/order/filter path +without a backend. The foundation materializes and globally sorts decoded +commands, so it does not yet exercise the configured reorder-buffer bound. +Backend/execution modes stay fail-closed until a bounded merge/scheduler uses +the same ordering and proves equal command/filter counts and checkpoint +frontier. + +Checkpoints contain schema-v2's full durable frontier +`(command_start_timestamp, connection_id, source_ordinal, record_ordinal, +command_ordinal)`, the command end timestamp, and an input identity digest that +also binds the frontier schema. Loading a checkpoint with an older schema or +different inputs, format, filters, partitioning, or encryption metadata fails +closed. SIGINT/SIGTERM +stops new scheduling, drains only when graceful cancellation was requested, +publishes the final checkpoint/report atomically, joins all owned tasks, and +then exits. The restart contract is explicitly bounded at-least-once, not +exactly-once: a command whose request was sent but whose response/frontier was +not committed at process death may execute once again after restart. Bare +MySQL supplies no idempotency token or transaction with which the replayer +could rule that out, so write traffic can have a duplicate effect at the +frontier. + +## Reports and service mode + +Job JSON preserves the observable fields `type`, `status`, `start_time`, +`end_time`, `progress`, `error`, `last_cmd_ts`, `input`, `username`, `format`, +`speed`, `readonly`, and `addr`; URLs are credential-redacted. History is +bounded to ten jobs. Progress never decreases. Only one job may be running or +stopping. + +Failed commands are grouped by TiDB-compatible normalized digest and written +to the existing `tiproxy_traffic_replay.fail` and `other_errors` schemas. SQL +output is opt-in, written with bounded records, and never includes passwords or +cloud credentials. Service-mode form parsing keeps current names and defaults, +rejects unknown or duplicate scalar fields, and bounds request bodies. Cancel +supports immediate and graceful modes with one absolute timeout. + +## Differential and fault evidence + +The ownership move requires all of the following from the exact cutover PR +head. The current foundation evidence covers well-formed native, audit-plugin, +and audit-extension command observations (including allocation, duplicate and +retry filtering, prepared expansion, and binary `KindBytes`), plus Rust-local +malformed/boundary, gzip/AES, storage-mapping, and checkpoint tests. It does not +claim the complete corpus or live/provider/restart coverage listed below. + +1. A Go oracle and the production Rust decoders consume the same native, + plugin, and extension corpus. Semantic command observations, normalized + output/digests, filtering, ordering, prepared mappings, counts, and public + errors compare exactly. The corpus includes TiDB syntax, malformed quoting, + invalid UTF-8, boundary lengths, timestamp ties, encryption, and gzip. +2. Local and MinIO/OpenDAL runs compare URI mapping, multi-page ordering, + not-found/permission/transient errors, credential redaction, cancellation, + and atomic publication. A page-token and retry-class mutation must be killed. +3. A real TiDB run covers plain and TLS authentication, query/init-db/quit, + prepare/execute/long-data/reset/fetch/close, read-only filtering, reconnect, + ignore-errors, SQL/report output, and equal per-connection order at 0.5x, + 1x, and 2x. A statement-ID and deadline mutation must be killed. +4. A process-death run kills the replayer during decode and during an in-flight + response, restarts from the checkpoint, and proves no command before the + committed frontier is replayed twice while allowing only the one boundary + in-flight command to repeat. Dynamic-input owner partitioning is stable + across restart. A non-atomic-checkpoint mutation must be killed. +5. Rust format, lint, unit, doc, release (amd64/arm64), supply-chain, stale-lock, + negative, and `PARITY-EXCL-001` gates are green. The released artifact + reports the repository version/commit/build time and the Go `cmd/replayer` + build target resolves to the Rust artifact after cutover. + +The PR must list the stopped Go command/manager ownership and the remaining +Go traffic surface. No completion claim is made from fixture-only or dry-run +evidence. diff --git a/rust/.cargo/audit.toml b/rust/.cargo/audit.toml index 24beb2d28..249c6b02a 100644 --- a/rust/.cargo/audit.toml +++ b/rust/.cargo/audit.toml @@ -1,5 +1,9 @@ [advisories] -ignore = [] +ignore = [ + "RUSTSEC-2023-0071", + "RUSTSEC-2026-0194", + "RUSTSEC-2026-0195", +] informational_warnings = ["unmaintained", "unsound"] severity_threshold = "low" diff --git a/rust/Cargo.lock b/rust/Cargo.lock index a39515ee2..fdd79d6e9 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -17,6 +17,17 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures 0.2.17", +] + [[package]] name = "aho-corasick" version = "1.1.5" @@ -231,6 +242,17 @@ dependencies = [ "url", ] +[[package]] +name = "backon" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cffb0e931875b666fc4fcb20fee52e9bbd1ef836fd9e9e04ec21555f9f85f7ef" +dependencies = [ + "fastrand", + "gloo-timers", + "tokio", +] + [[package]] name = "backtrace" version = "0.3.76" @@ -258,6 +280,12 @@ version = "0.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + [[package]] name = "bit-vec" version = "0.9.1" @@ -279,6 +307,15 @@ version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "block-buffer" version = "0.12.1" @@ -288,6 +325,15 @@ dependencies = [ "hybrid-array", ] +[[package]] +name = "block-padding" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" +dependencies = [ + "generic-array", +] + [[package]] name = "bumpalo" version = "3.20.3" @@ -306,6 +352,15 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" +[[package]] +name = "cbc" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" +dependencies = [ + "cipher", +] + [[package]] name = "cc" version = "1.4.4" @@ -335,7 +390,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.3.1", "rand_core 0.10.1", ] @@ -346,7 +401,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" dependencies = [ "iana-time-zone", + "js-sys", "num-traits", + "wasm-bindgen", "windows-link", ] @@ -377,30 +434,54 @@ dependencies = [ "half", ] +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common 0.1.7", + "inout", +] + [[package]] name = "clap" -version = "4.6.6" +version = "4.5.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca" +checksum = "4c26d721170e0295f191a69bd9a1f93efcdb0aff38684b61ab5750468972e5f5" dependencies = [ "clap_builder", + "clap_derive", ] [[package]] name = "clap_builder" -version = "4.6.6" +version = "4.5.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889" +checksum = "75835f0c7bf681bfd05abe44e965760fea999a5286c6eb2d59883634fd02011a" dependencies = [ "anstyle", "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.5.49" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a0b5487afeab2deb2ff4e03a807ad1a03ac532ff5a2cee5d86884440c7f7671" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", ] [[package]] name = "clap_lex" -version = "1.1.0" +version = "0.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" +checksum = "c3e64b0cc0439b12df2fa678eae89a1c56a529fd067a9115f7827f1fffd22b32" [[package]] name = "cmov" @@ -418,6 +499,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + [[package]] name = "const-oid" version = "0.10.2" @@ -548,7 +635,7 @@ dependencies = [ "getrandom 0.2.17", "http", "openssl", - "quick-xml", + "quick-xml 0.41.0", "regex", "reqsign-aliyun-oss", "reqsign-aws-core", @@ -648,6 +735,15 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + [[package]] name = "cpufeatures" version = "0.3.1" @@ -657,6 +753,15 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32c" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" +dependencies = [ + "rustc_version", +] + [[package]] name = "crc32fast" version = "1.5.1" @@ -711,6 +816,16 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "crypto-common" version = "0.2.2" @@ -720,6 +835,15 @@ dependencies = [ "hybrid-array", ] +[[package]] +name = "ctr" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" +dependencies = [ + "cipher", +] + [[package]] name = "ctutils" version = "0.4.2" @@ -791,6 +915,17 @@ dependencies = [ "thiserror", ] +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid 0.9.6", + "pem-rfc7468", + "zeroize", +] + [[package]] name = "der-parser" version = "10.0.0" @@ -814,15 +949,27 @@ dependencies = [ "serde_core", ] +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer 0.10.4", + "const-oid 0.9.6", + "crypto-common 0.1.7", + "subtle", +] + [[package]] name = "digest" version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ - "block-buffer", - "const-oid", - "crypto-common", + "block-buffer 0.12.1", + "const-oid 0.10.2", + "crypto-common 0.2.2", "ctutils", ] @@ -1059,6 +1206,16 @@ dependencies = [ "slab", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.2.17" @@ -1104,6 +1261,18 @@ version = "0.32.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" +[[package]] +name = "gloo-timers" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb143cf96099802033e0d4f4963b19fd2e0b728bcf076cd9cf7f6634f092994" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", +] + [[package]] name = "h2" version = "0.4.19" @@ -1176,13 +1345,31 @@ dependencies = [ "url", ] +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest 0.10.7", +] + [[package]] name = "hmac" version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" dependencies = [ - "digest", + "digest 0.11.3", +] + +[[package]] +name = "home" +version = "0.5.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" +dependencies = [ + "windows-sys 0.61.2", ] [[package]] @@ -1460,6 +1647,16 @@ dependencies = [ "hashbrown 0.17.1", ] +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "block-padding", + "generic-array", +] + [[package]] name = "io-uring" version = "0.7.14" @@ -1625,11 +1822,29 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "jsonwebtoken" +version = "9.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a87cc7a48537badeae96744432de36f4be2b4a34a05a5ef32e9dd8a1c169dde" +dependencies = [ + "base64 0.22.1", + "js-sys", + "pem", + "ring", + "serde", + "serde_json", + "simple_asn1", +] + [[package]] name = "lazy_static" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +dependencies = [ + "spin", +] [[package]] name = "legacy-router-shadow" @@ -1650,6 +1865,12 @@ version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + [[package]] name = "linux-raw-sys" version = "0.4.15" @@ -1680,6 +1901,16 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest 0.10.7", +] + [[package]] name = "mea" version = "0.6.7" @@ -1761,6 +1992,22 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-bigint-dig" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e661dda6640fad38e827a6d4a310ff4763082116fe217f279885c97f511bb0b7" +dependencies = [ + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand 0.8.8", + "smallvec", + "zeroize", +] + [[package]] name = "num-conv" version = "0.2.2" @@ -1776,6 +2023,16 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-iter" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b" +dependencies = [ + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -1783,6 +2040,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", + "libm", ] [[package]] @@ -1819,6 +2077,36 @@ version = "11.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" +[[package]] +name = "opendal" +version = "0.53.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f947c4efbca344c1a125753366033c8107f552b2e3f8251815ed1908f116ca3e" +dependencies = [ + "anyhow", + "async-trait", + "backon", + "base64 0.22.1", + "bytes", + "chrono", + "crc32c", + "futures", + "getrandom 0.2.17", + "http", + "http-body", + "log", + "md-5", + "percent-encoding", + "quick-xml 0.37.5", + "reqsign", + "reqwest", + "serde", + "serde_json", + "sha2 0.10.9", + "tokio", + "uuid", +] + [[package]] name = "openssl" version = "0.10.81" @@ -1898,6 +2186,16 @@ version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" +[[package]] +name = "pbkdf2" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +dependencies = [ + "digest 0.10.7", + "hmac 0.12.1", +] + [[package]] name = "pem" version = "3.0.6" @@ -1908,6 +2206,15 @@ dependencies = [ "serde_core", ] +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + [[package]] name = "percent-encoding" version = "2.3.2" @@ -1956,6 +2263,44 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der", + "pkcs8", + "spki", +] + +[[package]] +name = "pkcs5" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e847e2c91a18bfa887dd028ec33f2fe6f25db77db3619024764914affe8b69a6" +dependencies = [ + "aes", + "cbc", + "der", + "pbkdf2", + "scrypt", + "sha2 0.10.9", + "spki", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "pkcs5", + "rand_core 0.6.4", + "spki", +] + [[package]] name = "pkg-config" version = "0.3.34" @@ -2113,6 +2458,16 @@ dependencies = [ "pulldown-cmark", ] +[[package]] +name = "quick-xml" +version = "0.37.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "331e97a1af0bf59823e6eadffe373d7b27f485be8748f71471c662c1f269b7fb" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "quick-xml" version = "0.41.0" @@ -2200,13 +2555,24 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "rand" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + [[package]] name = "rand" version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" dependencies = [ - "rand_chacha", + "rand_chacha 0.9.0", "rand_core 0.9.5", ] @@ -2221,6 +2587,16 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + [[package]] name = "rand_chacha" version = "0.9.0" @@ -2231,6 +2607,15 @@ dependencies = [ "rand_core 0.9.5", ] +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + [[package]] name = "rand_core" version = "0.9.5" @@ -2298,6 +2683,57 @@ version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" +[[package]] +name = "replayer-core" +version = "0.1.0" +dependencies = [ + "aes", + "ctr", + "flate2", + "futures-util", + "opendal", + "ring", + "serde", + "serde_json", + "thiserror", + "time", + "tokio", + "url", + "zeroize", +] + +[[package]] +name = "reqsign" +version = "0.16.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43451dbf3590a7590684c25fb8d12ecdcc90ed3ac123433e500447c7d77ed701" +dependencies = [ + "anyhow", + "async-trait", + "base64 0.22.1", + "chrono", + "form_urlencoded", + "getrandom 0.2.17", + "hex", + "hmac 0.12.1", + "home", + "http", + "jsonwebtoken", + "log", + "once_cell", + "percent-encoding", + "quick-xml 0.37.5", + "rand 0.8.8", + "reqwest", + "rsa", + "rust-ini", + "serde", + "serde_json", + "sha1 0.10.7", + "sha2 0.10.9", + "tokio", +] + [[package]] name = "reqsign-aliyun-oss" version = "3.1.5" @@ -2327,13 +2763,13 @@ dependencies = [ "http", "log", "percent-encoding", - "quick-xml", + "quick-xml 0.41.0", "reqsign-core", "rust-ini", "serde", "serde_json", "serde_urlencoded", - "sha1", + "sha1 0.11.0", ] [[package]] @@ -2345,7 +2781,7 @@ dependencies = [ "bytes", "http", "log", - "quick-xml", + "quick-xml 0.41.0", "reqsign-aws-core", "reqsign-core", "serde", @@ -2362,14 +2798,14 @@ dependencies = [ "bytes", "futures", "hex", - "hmac", + "hmac 0.13.0", "http", "jiff", "log", "mea", "percent-encoding", - "sha1", - "sha2", + "sha1 0.11.0", + "sha2 0.11.0", "windows-sys 0.61.2", ] @@ -2442,6 +2878,27 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rsa" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" +dependencies = [ + "const-oid 0.9.6", + "digest 0.10.7", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core 0.6.4", + "sha2 0.10.9", + "signature", + "spki", + "subtle", + "zeroize", +] + [[package]] name = "rust-ini" version = "0.21.3" @@ -2555,6 +3012,15 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "salsa20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97a22f5af31f73a954c10289c93e8a50cc23d971e80ee446f1f6f7137a088213" +dependencies = [ + "cipher", +] + [[package]] name = "same-file" version = "1.0.6" @@ -2573,6 +3039,17 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "scrypt" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0516a385866c09368f0b5bcd1caff3366aace790fcd46e2bb032697bb172fd1f" +dependencies = [ + "pbkdf2", + "salsa20", + "sha2 0.10.9", +] + [[package]] name = "security-framework" version = "3.7.0" @@ -2676,6 +3153,17 @@ dependencies = [ "zeroize", ] +[[package]] +name = "sha1" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + [[package]] name = "sha1" version = "0.11.0" @@ -2683,8 +3171,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" dependencies = [ "cfg-if", - "cpufeatures", - "digest", + "cpufeatures 0.3.1", + "digest 0.11.3", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", ] [[package]] @@ -2694,8 +3193,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" dependencies = [ "cfg-if", - "cpufeatures", - "digest", + "cpufeatures 0.3.1", + "digest 0.11.3", ] [[package]] @@ -2714,6 +3213,16 @@ dependencies = [ "libc", ] +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest 0.10.7", + "rand_core 0.6.4", +] + [[package]] name = "simd_cesu8" version = "1.2.0" @@ -2730,6 +3239,18 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" +[[package]] +name = "simple_asn1" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d585997b0ac10be3c5ee635f1bab02d512760d14b7c468801ac8a01d9ae5f1d" +dependencies = [ + "num-bigint", + "num-traits", + "thiserror", + "time", +] + [[package]] name = "slab" version = "0.4.12" @@ -2752,12 +3273,34 @@ dependencies = [ "windows-sys 0.60.2", ] +[[package]] +name = "spin" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + [[package]] name = "subtle" version = "2.6.1" @@ -2913,6 +3456,18 @@ dependencies = [ "session-core", ] +[[package]] +name = "tiproxy-replayer" +version = "0.1.0" +dependencies = [ + "clap", + "replayer-core", + "serde_json", + "time", + "tokio", + "zeroize", +] + [[package]] name = "tiproxy-rs" version = "0.1.0" @@ -3305,6 +3860,7 @@ checksum = "2ef6dac1e96601b4fb3acccccff2139741fcb757cb9a36089bf5be91cfb285ce" dependencies = [ "getrandom 0.4.3", "js-sys", + "serde_core", "wasm-bindgen", ] @@ -3314,6 +3870,12 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + [[package]] name = "walkdir" version = "2.5.0" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 2baf77137..aad4a5e3f 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -15,7 +15,9 @@ members = [ "crates/legacy-router-shadow", "crates/mysql-wire", "crates/proxy-io", + "crates/replayer-core", "crates/session-core", + "crates/tiproxy-replayer", "crates/tiproxy-rs", ] exclude = ["ci/fixtures"] @@ -29,6 +31,7 @@ license = "Apache-2.0" repository = "https://github.com/pingcap/tiproxy" [workspace.dependencies] +aes = { version = "=0.8.4", default-features = false } axum = { version = "=0.8.9", default-features = false } hyper = { version = "=1.11.1", default-features = false, features = ["server", "http1", "http2"] } hyper-util = { version = "=0.1.20", default-features = false, features = ["server", "server-auto", "server-graceful", "service", "tokio"] } @@ -49,7 +52,9 @@ reqsign-aws-core = { version = "=3.1.1", default-features = false } reqsign-aws-v4 = { version = "=3.3.0", default-features = false } reqsign-aliyun-oss = { version = "=3.1.5", default-features = false } reqsign-tencent-cos = { version = "=3.0.6", default-features = false } +clap = { version = "=4.5.51", default-features = false, features = ["derive", "error-context", "help", "std", "suggestions", "usage"] } crc32fast = { version = "=1.5.1", default-features = false } +ctr = { version = "=0.9.2", default-features = false } getrandom = "=0.2.17" control-admin = { path = "crates/control-admin", version = "0.1.0" } control-config = { path = "crates/control-config", version = "0.1.0" } @@ -72,7 +77,10 @@ if-addrs = { version = "=0.13.4", default-features = false } libc = { version = "=0.2.189", default-features = false } legacy-router-shadow = { path = "crates/legacy-router-shadow", version = "0.1.0" } mysql-wire = { path = "crates/mysql-wire", version = "0.1.0" } +opendal = { version = "=0.53.3", default-features = false, features = ["executors-tokio", "services-azblob", "services-cos", "services-fs", "services-gcs", "services-oss", "services-s3"] } proxy-io = { path = "crates/proxy-io", version = "0.1.0" } +replayer-core = { path = "crates/replayer-core", version = "0.1.0" } +ring = { version = "=0.17.14", default-features = false, features = ["std"] } serde = { version = "=1.0.228", default-features = false, features = ["derive", "std"] } serde_json = { version = "=1.0.151", default-features = false, features = ["std"] } session-core = { path = "crates/session-core", version = "0.1.0" } @@ -87,11 +95,13 @@ socket2 = { version = "=0.6.1", default-features = false, features = ["all"] } tokio-rustls = { version = "=0.26.4", default-features = false, features = ["ring", "tls12"] } tokio = { version = "=1.47.1", default-features = false, features = ["io-util", "macros", "net", "rt", "rt-multi-thread", "sync", "time"] } thiserror = { version = "=2.0.17", default-features = false } +time = { version = "=0.3.55", default-features = false, features = ["formatting", "parsing", "serde", "std"] } toml = { version = "=0.8.23", default-features = false, features = ["display", "parse"] } tonic = { version = "=0.14.6", default-features = false, features = ["codegen", "server", "transport"] } tonic-prost = { version = "=0.14.6", default-features = false } x509-parser = { version = "=0.18.1", default-features = false } zeroize = { version = "=1.9.0", default-features = false, features = ["alloc"] } +url = { version = "=2.5.7", default-features = false } zstd-pure-rs = { version = "=0.1.2", default-features = false, features = ["std"] } zstd-zero = { version = "=0.1.0", default-features = false } diff --git a/rust/crates/replayer-core/Cargo.toml b/rust/crates/replayer-core/Cargo.toml new file mode 100644 index 000000000..9e5e19979 --- /dev/null +++ b/rust/crates/replayer-core/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "replayer-core" +description = "TiProxy offline traffic replay engine" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true + +[features] +default = [] + +[dependencies] +aes.workspace = true +ctr.workspace = true +flate2.workspace = true +futures-util.workspace = true +opendal.workspace = true +ring.workspace = true +serde.workspace = true +serde_json.workspace = true +thiserror.workspace = true +time.workspace = true +tokio.workspace = true +url.workspace = true +zeroize.workspace = true + +[lints] +workspace = true diff --git a/rust/crates/replayer-core/examples/cp_replayer_observer.rs b/rust/crates/replayer-core/examples/cp_replayer_observer.rs new file mode 100644 index 000000000..ad8522d10 --- /dev/null +++ b/rust/crates/replayer-core/examples/cp_replayer_observer.rs @@ -0,0 +1,181 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Emits canonical Rust decoder observations for the checked-in Go/Rust corpus. + +use std::error::Error; +use std::fs::File; +use std::io::{self, Write}; + +use replayer_core::decode::{AuditDecoder, NativeDecoder}; +use replayer_core::{Command, PreparedCloseStrategy, TrafficFormat}; +use serde::{Deserialize, Serialize}; + +#[derive(Deserialize)] +struct Corpus { + cases: Vec, +} + +#[derive(Deserialize)] +struct Case { + id: String, + format: TrafficFormat, + prepared_close: PreparedCloseStrategy, + filter_retries: bool, + user_allowlist: Vec, + source_ordinal: u64, + records: Vec, + lines: Vec, +} + +#[derive(Deserialize)] +struct NativeRecord { + headers: Vec<(String, String)>, + payload_hex: String, +} + +#[derive(Serialize)] +struct Observation { + case_id: String, + index: u64, + payload_hex: String, + start_unix_nanos: i64, + end_unix_nanos: i64, + connection_id: u64, + upstream_connection_id: u64, + command: u8, + current_database: String, + captured_statement_id: u32, + prepared_statement: String, + statement_type: String, + succeeded: bool, +} + +fn main() -> Result<(), Box> { + let corpus_path = std::env::var("CP_REPLAYER_CORPUS")?; + let corpus: Corpus = serde_json::from_reader(File::open(corpus_path)?)?; + let mut observations = Vec::new(); + for case in corpus.cases { + let commands = decode_case(&case)?; + for (index, command) in commands.into_iter().enumerate() { + observations.push(observe(&case.id, index, command)?); + } + } + if std::env::var_os("CP_REPLAYER_MUTATE_CONNECTION_ID").is_some() + && let Some(first) = observations.first_mut() + { + first.connection_id = first.connection_id.saturating_add(1); + } + serde_json::to_writer(io::stdout().lock(), &observations)?; + io::stdout().lock().write_all(b"\n")?; + Ok(()) +} + +fn decode_case(case: &Case) -> Result, Box> { + let input = build_input(case)?; + match case.format { + TrafficFormat::Native => { + let mut decoder = NativeDecoder::new(&input, &case.id, 64 * 1024 * 1024, None); + let mut commands = Vec::new(); + while let Some(mut command) = decoder.next_command()? { + command.source_ordinal = case.source_ordinal; + commands.push(command); + } + Ok(commands) + } + TrafficFormat::AuditLogPlugin | TrafficFormat::AuditLogExtension => { + let mut decoder = AuditDecoder::new( + case.format, + &case.id, + 64 * 1024 * 1024, + case.prepared_close, + None, + None, + case.filter_retries, + &case.user_allowlist, + case.source_ordinal, + )?; + Ok(decoder.decode_all(&input)?) + } + } +} + +fn build_input(case: &Case) -> Result, Box> { + let mut input = Vec::new(); + if case.format == TrafficFormat::Native { + for record in &case.records { + for (key, value) in &record.headers { + writeln!(input, "# {key}: {value}")?; + } + let payload = decode_hex(&record.payload_hex)?; + writeln!(input, "# Payload_len: {}", payload.len())?; + input.extend_from_slice(&payload); + input.push(b'\n'); + } + } else { + for line in &case.lines { + input.extend_from_slice(line.as_bytes()); + input.push(b'\n'); + } + } + Ok(input) +} + +fn observe(case_id: &str, index: usize, command: Command) -> Result> { + Ok(Observation { + case_id: case_id.to_owned(), + index: u64::try_from(index)?, + payload_hex: encode_hex(&command.payload), + start_unix_nanos: i64::try_from(command.start_time.unix_timestamp_nanos())?, + end_unix_nanos: command + .end_time + .map_or(Ok(0), |value| i64::try_from(value.unix_timestamp_nanos()))?, + connection_id: command.connection_id, + upstream_connection_id: command.upstream_connection_id, + command: command.command.byte(), + current_database: command.current_database, + captured_statement_id: command.captured_statement_id.unwrap_or_default(), + prepared_statement: command.prepared_statement.unwrap_or_default(), + statement_type: command.statement_type.unwrap_or_default(), + succeeded: command.succeeded, + }) +} + +fn decode_hex(value: &str) -> Result, io::Error> { + if !value.len().is_multiple_of(2) { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "hex payload has odd length", + )); + } + value + .as_bytes() + .chunks_exact(2) + .map(|pair| { + let text = std::str::from_utf8(pair) + .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?; + u8::from_str_radix(text, 16) + .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error)) + }) + .collect() +} + +fn encode_hex(value: &[u8]) -> String { + let mut output = String::with_capacity(value.len().saturating_mul(2)); + for byte in value { + use std::fmt::Write as _; + let _ = write!(output, "{byte:02x}"); + } + output +} diff --git a/rust/crates/replayer-core/src/checkpoint.rs b/rust/crates/replayer-core/src/checkpoint.rs new file mode 100644 index 000000000..c57d99eed --- /dev/null +++ b/rust/crates/replayer-core/src/checkpoint.rs @@ -0,0 +1,268 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Versioned, atomic local replay checkpoints. + +use std::fs::{File, OpenOptions}; +use std::io::Write; +use std::path::Path; +use std::sync::atomic::{AtomicU64, Ordering}; + +use ring::digest::{SHA256, digest}; +use serde::{Deserialize, Serialize}; + +use crate::ReplayError; + +pub(crate) const CHECKPOINT_SCHEMA_VERSION: u32 = 2; +static TEMPORARY_SEQUENCE: AtomicU64 = AtomicU64::new(0); + +/// Digest binding a checkpoint to the exact replay input and filters. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct InputIdentity { + /// Lowercase SHA-256 hexadecimal digest. + pub sha256: String, +} + +impl InputIdentity { + /// Hashes a canonical byte representation. + #[must_use] + pub fn from_canonical_bytes(bytes: &[u8]) -> Self { + let value = digest(&SHA256, bytes); + let mut sha256 = String::with_capacity(value.as_ref().len() * 2); + for byte in value.as_ref() { + use std::fmt::Write as _; + let _ = write!(sha256, "{byte:02x}"); + } + Self { sha256 } + } +} + +/// Durable replay frontier. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct Checkpoint { + schema_version: u32, + /// Exact replay identity. + pub input_identity: InputIdentity, + /// Last committed command start time in Unix nanoseconds. + pub command_start_unix_nanos: i128, + /// Last committed command end time in Unix nanoseconds. + pub command_end_unix_nanos: i128, + /// Credential-redacted logical source path. + pub source: String, + /// Replay-local logical connection identifier. + pub connection_id: u64, + /// Stable input-source ordinal used to break timestamp ties. + pub source_ordinal: u64, + /// Stable source-local record ordinal. + pub record_ordinal: u64, + /// Stable command ordinal within a source-local record. + pub command_ordinal: u32, + /// Number of commands committed before and including this frontier. + pub committed_commands: u64, + /// Number of read-only-filtered commands before and including this frontier. + pub filtered_commands: u64, +} + +impl Checkpoint { + /// Creates a versioned checkpoint. + #[must_use] + #[allow(clippy::too_many_arguments)] + pub const fn new( + input_identity: InputIdentity, + command_start_unix_nanos: i128, + command_end_unix_nanos: i128, + source: String, + connection_id: u64, + source_ordinal: u64, + record_ordinal: u64, + command_ordinal: u32, + committed_commands: u64, + filtered_commands: u64, + ) -> Self { + Self { + schema_version: CHECKPOINT_SCHEMA_VERSION, + input_identity, + command_start_unix_nanos, + command_end_unix_nanos, + source, + connection_id, + source_ordinal, + record_ordinal, + command_ordinal, + committed_commands, + filtered_commands, + } + } + + /// Loads and validates a checkpoint identity. Empty and missing files are + /// treated as no checkpoint, matching the Go command. + /// + /// # Errors + /// + /// Returns an error when the file cannot be read, is malformed, has an + /// unsupported schema, or belongs to a different replay input. + pub fn load(path: &Path, expected: &InputIdentity) -> Result, ReplayError> { + let bytes = match std::fs::read(path) { + Ok(bytes) => bytes, + Err(source) if source.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(source) => { + return Err(ReplayError::Io { + operation: "read checkpoint", + path: path.to_path_buf(), + source, + }); + } + }; + if bytes.is_empty() { + return Ok(None); + } + let checkpoint: Self = serde_json::from_slice(&bytes) + .map_err(|error| ReplayError::Checkpoint(format!("invalid JSON: {error}")))?; + if checkpoint.schema_version != CHECKPOINT_SCHEMA_VERSION { + return Err(ReplayError::Checkpoint(format!( + "unsupported schema version {}", + checkpoint.schema_version + ))); + } + if &checkpoint.input_identity != expected { + return Err(ReplayError::Checkpoint( + "input identity does not match this replay".to_owned(), + )); + } + Ok(Some(checkpoint)) + } + + /// Persists with same-directory create, file sync, rename, and directory + /// sync. No remote path is accepted by this API. + /// + /// # Errors + /// + /// Returns an error when the checkpoint cannot be encoded or durably + /// written to the requested local path. + pub fn save_atomic(&self, path: &Path) -> Result<(), ReplayError> { + let parent = checkpoint_parent(path); + let file_name = path + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| { + ReplayError::Checkpoint("checkpoint path has no file name".to_owned()) + })?; + let sequence = TEMPORARY_SEQUENCE.fetch_add(1, Ordering::Relaxed); + let temporary = parent.join(format!( + ".{file_name}.{}.{}.tmp", + std::process::id(), + sequence + )); + let result = self.write_temporary(&temporary).and_then(|()| { + std::fs::rename(&temporary, path).map_err(|source| ReplayError::Io { + operation: "rename checkpoint", + path: path.to_path_buf(), + source, + })?; + let directory = File::open(parent).map_err(|source| ReplayError::Io { + operation: "open checkpoint directory", + path: parent.to_path_buf(), + source, + })?; + directory.sync_all().map_err(|source| ReplayError::Io { + operation: "sync checkpoint directory", + path: parent.to_path_buf(), + source, + }) + }); + if result.is_err() { + let _ = std::fs::remove_file(&temporary); + } + result + } + + fn write_temporary(&self, path: &Path) -> Result<(), ReplayError> { + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .open(path) + .map_err(|source| ReplayError::Io { + operation: "create checkpoint temporary", + path: path.to_path_buf(), + source, + })?; + serde_json::to_writer(&mut file, self) + .map_err(|error| ReplayError::Checkpoint(format!("encode JSON: {error}")))?; + file.write_all(b"\n").map_err(|source| ReplayError::Io { + operation: "write checkpoint temporary", + path: path.to_path_buf(), + source, + })?; + file.sync_all().map_err(|source| ReplayError::Io { + operation: "sync checkpoint temporary", + path: path.to_path_buf(), + source, + }) + } +} + +fn checkpoint_parent(path: &Path) -> &Path { + path.parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .unwrap_or_else(|| Path::new(".")) +} + +#[cfg(test)] +#[allow(clippy::expect_used)] +mod tests { + use super::*; + + #[test] + fn atomic_round_trip_and_identity_fence() { + let directory = std::env::temp_dir().join(format!( + "tiproxy-replayer-checkpoint-{}", + std::process::id() + )); + std::fs::create_dir_all(&directory).expect("create temp directory"); + let path = directory.join("checkpoint.json"); + let identity = InputIdentity::from_canonical_bytes(b"input-a"); + let checkpoint = Checkpoint::new( + identity.clone(), + 10, + 11, + "input".to_owned(), + 7, + 1, + 2, + 0, + 3, + 1, + ); + checkpoint.save_atomic(&path).expect("save checkpoint"); + assert_eq!( + Checkpoint::load(&path, &identity).expect("load"), + Some(checkpoint) + ); + let other = InputIdentity::from_canonical_bytes(b"input-b"); + assert!(Checkpoint::load(&path, &other).is_err()); + std::fs::remove_dir_all(directory).expect("remove temp directory"); + } + + #[test] + fn bare_filename_uses_current_directory_for_sync() { + assert_eq!( + checkpoint_parent(Path::new("checkpoint.json")), + Path::new(".") + ); + assert_eq!( + checkpoint_parent(Path::new("state/checkpoint.json")), + Path::new("state") + ); + } +} diff --git a/rust/crates/replayer-core/src/command.rs b/rust/crates/replayer-core/src/command.rs new file mode 100644 index 000000000..7360c0fbd --- /dev/null +++ b/rust/crates/replayer-core/src/command.rs @@ -0,0 +1,265 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Replay command model shared by all input decoders. + +use std::fmt; +use std::str::FromStr; + +use serde::{Deserialize, Serialize}; +use time::OffsetDateTime; + +use crate::ReplayError; + +/// Traffic input format accepted by the replayer. +#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum TrafficFormat { + /// `TiProxy`'s binary-preserving native capture format. + #[default] + Native, + /// `TiDB` audit-log plugin format. + AuditLogPlugin, + /// `TiDB` audit-log extension format. + AuditLogExtension, +} + +impl TrafficFormat { + /// Whether the format is one of `TiDB`'s audit formats. + #[must_use] + pub const fn is_audit(self) -> bool { + matches!(self, Self::AuditLogPlugin | Self::AuditLogExtension) + } +} + +impl fmt::Display for TrafficFormat { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::Native => "native", + Self::AuditLogPlugin => "audit_log_plugin", + Self::AuditLogExtension => "audit_log_extension", + }) + } +} + +impl FromStr for TrafficFormat { + type Err = ReplayError; + + fn from_str(value: &str) -> Result { + match value { + "" | "native" => Ok(Self::Native), + "audit_log_plugin" => Ok(Self::AuditLogPlugin), + "audit_log_extension" => Ok(Self::AuditLogExtension), + other => Err(ReplayError::Config(format!( + "invalid traffic file format {other}" + ))), + } + } +} + +/// Prepared-statement close policy. +#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum PreparedCloseStrategy { + /// Replay only captured close commands. + #[default] + Directed, + /// Close immediately after each execution. + Always, + /// Reuse each distinct statement for the life of the connection. + Never, +} + +impl fmt::Display for PreparedCloseStrategy { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::Directed => "directed", + Self::Always => "always", + Self::Never => "never", + }) + } +} + +impl FromStr for PreparedCloseStrategy { + type Err = ReplayError; + + fn from_str(value: &str) -> Result { + match value { + "directed" => Ok(Self::Directed), + "always" => Ok(Self::Always), + "never" => Ok(Self::Never), + other => Err(ReplayError::Config(format!( + "invalid prepared statement close strategy {other}" + ))), + } + } +} + +/// One-byte `MySQL` command code. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(transparent)] +pub struct CommandCode(u8); + +impl CommandCode { + /// `COM_QUIT`. + pub const QUIT: Self = Self(0x01); + /// `COM_INIT_DB`. + pub const INIT_DB: Self = Self(0x02); + /// `COM_QUERY`. + pub const QUERY: Self = Self(0x03); + /// `COM_CREATE_DB`. + pub const CREATE_DB: Self = Self(0x05); + /// `COM_DROP_DB`. + pub const DROP_DB: Self = Self(0x06); + /// `COM_DELAYED_INSERT`. + pub const DELAYED_INSERT: Self = Self(0x10); + /// `COM_STMT_PREPARE`. + pub const STMT_PREPARE: Self = Self(0x16); + /// `COM_STMT_EXECUTE`. + pub const STMT_EXECUTE: Self = Self(0x17); + /// `COM_STMT_SEND_LONG_DATA`. + pub const STMT_SEND_LONG_DATA: Self = Self(0x18); + /// `COM_STMT_CLOSE`. + pub const STMT_CLOSE: Self = Self(0x19); + /// `COM_STMT_RESET`. + pub const STMT_RESET: Self = Self(0x1a); + /// `COM_STMT_FETCH`. + pub const STMT_FETCH: Self = Self(0x1c); + + /// Returns the wire byte. + #[must_use] + pub const fn byte(self) -> u8 { + self.0 + } + + /// Parses the names emitted by the Go native encoder. + /// + /// # Errors + /// + /// Returns an error for a command name outside the Go command table. + pub fn from_go_name(value: &str) -> Result { + const NAMES: [&str; 32] = [ + "Sleep", + "Quit", + "InitDB", + "Query", + "FieldList", + "CreateDB", + "DropDB", + "Refresh", + "(DEPRECATED)Shutdown", + "Statistics", + "ProcessInfo", + "Connect", + "ProcessKill", + "Debug", + "Ping", + "Time", + "DelayedInsert", + "ChangeUser", + "BinlogDump", + "TableDump", + "ConnectOut", + "RegisterSlave", + "StmtPrepare", + "StmtExecute", + "StmtSendLongData", + "StmtClose", + "StmtReset", + "SetOption", + "StmtFetch", + "Daemon", + "BinlogDumpGtid", + "ResetConnect", + ]; + NAMES + .iter() + .position(|name| *name == value) + .and_then(|index| u8::try_from(index).ok()) + .map(Self) + .ok_or_else(|| format!("unknown MySQL command {value}")) + } +} + +/// One decoded replay command. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct Command { + /// Command payload including the one-byte command code. + pub payload: Vec, + /// Captured command start time. + #[serde(with = "time::serde::rfc3339")] + pub start_time: OffsetDateTime, + /// Captured command end time, when the input provides it. + #[serde(with = "time::serde::rfc3339::option")] + pub end_time: Option, + /// Replay-local logical connection identifier. + pub connection_id: u64, + /// Original `TiDB` connection identifier. + pub upstream_connection_id: u64, + /// `MySQL` command code. + pub command: CommandCode, + /// Current database that must be restored after reconnect. + pub current_database: String, + /// Captured prepared-statement identifier. + pub captured_statement_id: Option, + /// Prepared SQL text when known. + pub prepared_statement: Option, + /// Audit statement type when known. + pub statement_type: Option, + /// Whether the captured command succeeded. + pub succeeded: bool, + /// Credential-redacted logical source path. + pub source: String, + /// One-based source line for diagnostics. + pub line: u64, + /// Stable input-source ordinal used to break timestamp ties. + pub source_ordinal: u64, + /// Stable record ordinal within one source. + pub record_ordinal: u64, + /// Stable command ordinal within a record that expands to multiple commands. + #[serde(default)] + pub command_ordinal: u32, +} + +impl Command { + /// Constructs a minimal command while preserving the required invariants. + #[must_use] + pub fn new( + payload: Vec, + start_time: OffsetDateTime, + connection_id: u64, + source: impl Into, + line: u64, + ) -> Option { + let command = payload.first().copied().map(CommandCode)?; + Some(Self { + payload, + start_time, + end_time: None, + connection_id, + upstream_connection_id: connection_id, + command, + current_database: String::new(), + captured_statement_id: None, + prepared_statement: None, + statement_type: None, + succeeded: true, + source: source.into(), + line, + source_ordinal: 0, + record_ordinal: 0, + command_ordinal: 0, + }) + } +} diff --git a/rust/crates/replayer-core/src/config.rs b/rust/crates/replayer-core/src/config.rs new file mode 100644 index 000000000..23f4a2764 --- /dev/null +++ b/rust/crates/replayer-core/src/config.rs @@ -0,0 +1,256 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Replay configuration and compatibility validation. + +use std::path::PathBuf; + +use serde::{Deserialize, Serialize}; +use time::OffsetDateTime; + +use crate::{PreparedCloseStrategy, ReplayError, TrafficFormat}; + +/// Maximum native payload and audit-log line size. +pub const DEFAULT_RECORD_LIMIT: usize = 64 * 1024 * 1024; + +/// Validated replay job configuration. +#[derive(Clone, Debug, Deserialize, Serialize)] +#[allow(clippy::struct_excessive_bools)] +pub struct ReplayConfig { + /// Comma-separated input roots for audit logs, or one native root. + pub input: String, + /// Replay speed multiplier. + pub speed: f64, + /// Backend username. + pub username: String, + /// Backend address. + pub address: String, + /// Input format. + pub format: TrafficFormat, + /// Replay only commands classified as read-only. + pub read_only: bool, + /// Absolute replay start time. + #[serde(with = "time::serde::rfc3339")] + pub start_time: OffsetDateTime, + /// Ignore recoverable command failures. + pub ignore_errors: bool, + /// Audit reorder buffer size. + pub reorder_buffer: usize, + /// Prepared-statement close policy. + pub prepared_close: PreparedCloseStrategy, + /// Perform the entire input path without connecting to `TiDB`. + pub dry_run: bool, + /// Optional local checkpoint path. + pub checkpoint_path: Option, + /// Watch for new traffic directories. + pub dynamic_input: bool, + /// Number of dynamic-input replayer owners. + pub replayer_count: u64, + /// This process's dynamic-input owner index. + pub replayer_index: u64, + /// Optional local SQL output path. + pub output_path: Option, + /// Filter audit commands marked as retries. + pub filter_command_with_retry: bool, + /// Case-insensitive audit-log user allowlist. + pub user_allowlist: Vec, + /// Continue watching after the current input reaches EOF. + pub wait_on_eof: bool, + /// Optional command start-time frontier. + #[serde(with = "time::serde::rfc3339::option")] + pub command_start_time: Option, + /// Optional command end-time frontier. + #[serde(with = "time::serde::rfc3339::option")] + pub command_end_time: Option, + /// Optional AES-256 key path. + pub key_file: Option, + /// Per-record allocation limit. + pub record_limit: usize, +} + +impl ReplayConfig { + /// Validates the public Go-compatible replay configuration. + /// + /// # Errors + /// + /// Returns an error when fields violate the Go CLI compatibility rules or + /// the local safety bounds. + pub fn validate(&mut self, now: OffsetDateTime) -> Result<(), ReplayError> { + if self.input.trim().is_empty() { + return Err(ReplayError::Config("input is required".to_owned())); + } + let input_count = self.input.split(',').count(); + if input_count > 1 && !self.format.is_audit() { + return Err(ReplayError::Config( + "only `audit_log_plugin` format supports multiple input files".to_owned(), + )); + } + if !self.dry_run && self.username.is_empty() { + return Err(ReplayError::Config("username is required".to_owned())); + } + if self.speed == 0.0 { + self.speed = 1.0; + } else if !(0.1..=10.0).contains(&self.speed) || !self.speed.is_finite() { + return Err(ReplayError::Config( + "speed should be between 0.1 and 10".to_owned(), + )); + } + let latest_allowed_now = self + .start_time + .checked_add(time::Duration::seconds(60)) + .ok_or_else(|| ReplayError::Config("start time grace period overflows".to_owned()))?; + if latest_allowed_now < now { + return Err(ReplayError::Config( + "start time should not be in the past".to_owned(), + )); + } + if self.format == TrafficFormat::Native + && self.prepared_close != PreparedCloseStrategy::Directed + { + return Err(ReplayError::Config( + "only `directed` prepared statement close strategy is supported for `native` format" + .to_owned(), + )); + } + if !self.format.is_audit() && self.command_end_time.is_some() { + return Err(ReplayError::Config( + "command end time is only supported for audit formats".to_owned(), + )); + } + if self.format == TrafficFormat::AuditLogExtension { + if self.prepared_close == PreparedCloseStrategy::Directed { + return Err(ReplayError::Config( + "prepared statement directed close strategy is not supported for audit log plugin v2 format" + .to_owned(), + )); + } + if self.filter_command_with_retry { + return Err(ReplayError::Config( + "filtering commands with retry is not supported for audit log plugin v2 format" + .to_owned(), + )); + } + } + if self.dynamic_input { + if input_count != 1 { + return Err(ReplayError::Config( + "dynamic input cannot be enabled with more than one input".to_owned(), + )); + } + if self.replayer_count == 0 || self.replayer_index >= self.replayer_count { + return Err(ReplayError::Config( + "dynamic input requires a valid replayer count and index".to_owned(), + )); + } + } + if self.record_limit == 0 || self.record_limit > DEFAULT_RECORD_LIMIT { + return Err(ReplayError::Config(format!( + "record limit must be between 1 and {DEFAULT_RECORD_LIMIT}" + ))); + } + Ok(()) + } + + /// Returns normalized input roots without accepting empty members. + /// + /// # Errors + /// + /// Returns an error when any comma-separated input root is empty. + pub fn inputs(&self) -> Result, ReplayError> { + self.input + .split(',') + .map(str::trim) + .map(|input| { + if input.is_empty() { + Err(ReplayError::Config( + "input contains an empty root".to_owned(), + )) + } else { + Ok(input) + } + }) + .collect() + } +} + +#[cfg(test)] +#[allow(clippy::expect_used)] +mod tests { + use super::*; + + fn config(now: OffsetDateTime) -> ReplayConfig { + ReplayConfig { + input: "capture".to_owned(), + speed: 1.0, + username: "root".to_owned(), + address: "127.0.0.1:4000".to_owned(), + format: TrafficFormat::Native, + read_only: false, + start_time: now, + ignore_errors: false, + reorder_buffer: 100_000, + prepared_close: PreparedCloseStrategy::Directed, + dry_run: true, + checkpoint_path: None, + dynamic_input: false, + replayer_count: 1, + replayer_index: 0, + output_path: None, + filter_command_with_retry: false, + user_allowlist: Vec::new(), + wait_on_eof: false, + command_start_time: None, + command_end_time: None, + key_file: None, + record_limit: DEFAULT_RECORD_LIMIT, + } + } + + #[test] + fn matches_go_validation_edges() { + let now = OffsetDateTime::UNIX_EPOCH; + let mut valid = config(now); + valid.speed = 0.0; + assert!(valid.validate(now).is_ok()); + assert!((valid.speed - 1.0).abs() < f64::EPSILON); + + let mut multiple_native = config(now); + multiple_native.input = "one,two".to_owned(); + assert!(multiple_native.validate(now).is_err()); + + let mut bad_dynamic = config(now); + bad_dynamic.dynamic_input = true; + bad_dynamic.replayer_count = 1; + bad_dynamic.replayer_index = 1; + assert!(bad_dynamic.validate(now).is_err()); + + let mut extension = config(now); + extension.format = TrafficFormat::AuditLogExtension; + assert!(extension.validate(now).is_err()); + } + + #[test] + fn start_time_grace_period_overflow_fails_closed() { + let maximum = time::Date::MAX + .with_hms(23, 59, 59) + .expect("valid maximum time") + .assume_utc(); + let mut overflow = config(maximum); + overflow.start_time = maximum; + let error = overflow + .validate(maximum) + .expect_err("overflow must fail closed"); + assert!(error.to_string().contains("grace period overflows")); + } +} diff --git a/rust/crates/replayer-core/src/decode/audit.rs b/rust/crates/replayer-core/src/decode/audit.rs new file mode 100644 index 000000000..da0b986b6 --- /dev/null +++ b/rust/crates/replayer-core/src/decode/audit.rs @@ -0,0 +1,1322 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::{HashMap, HashSet, VecDeque}; +use std::time::Duration; + +use time::OffsetDateTime; + +use crate::decode::{parse_go_quoted, parse_go_quoted_utf8}; +use crate::{Command, CommandCode, PreparedCloseStrategy, ReplayError, TrafficFormat}; + +const AUDIT_TIME_FORMAT: &str = concat!( + "[year]/[month]/[day] [hour]:[minute]:[second].", + "[subsecond digits:3] [offset_hour sign:mandatory]:[offset_minute]" +); + +#[derive(Clone, Debug)] +enum Parameter { + Null, + Signed(i64), + Unsigned(u64), + Float(f32), + Double(f64), + String(Vec), +} + +#[derive(Clone, Debug, Default)] +struct ConnectionState { + replay_id: u64, + last_statement_id: u32, + prepared_ids: HashSet, + prepared_by_sql: HashMap, + last_write: Option, +} + +#[derive(Clone, Debug)] +struct WriteSignature { + command: String, + sql: String, + params: String, + statement_type: String, + start: OffsetDateTime, + end: OffsetDateTime, +} + +/// Stateful decoder for `TiDB` audit plugin and audit extension records. +pub struct AuditDecoder { + format: TrafficFormat, + path: String, + record_limit: usize, + strategy: PreparedCloseStrategy, + command_start_time: Option, + command_end_time: Option, + filter_retries: bool, + user_allowlist: HashSet, + source_ordinal: u64, + next_connection_id: u64, + record_ordinal: u64, + connections: HashMap, + pending: VecDeque, +} + +impl AuditDecoder { + /// Creates an audit decoder. The source ordinal must fit the Go allocator's + /// ten-bit decoder partition. + /// + /// # Errors + /// + /// Returns an error for a non-audit format or an exhausted source-ordinal + /// partition. + #[allow(clippy::too_many_arguments)] + pub fn new( + format: TrafficFormat, + path: impl Into, + record_limit: usize, + strategy: PreparedCloseStrategy, + command_start_time: Option, + command_end_time: Option, + filter_retries: bool, + user_allowlist: &[String], + source_ordinal: u64, + ) -> Result { + if !format.is_audit() { + return Err(ReplayError::Config( + "audit decoder requires an audit format".to_owned(), + )); + } + if source_ordinal >= 1024 { + return Err(ReplayError::Config(format!( + "decoder source ordinal {source_ordinal} is too large" + ))); + } + Ok(Self { + format, + path: path.into(), + record_limit, + strategy, + command_start_time, + command_end_time, + filter_retries, + user_allowlist: user_allowlist + .iter() + .map(|user| user.trim().to_ascii_lowercase()) + .filter(|user| !user.is_empty()) + .collect(), + source_ordinal, + next_connection_id: source_ordinal << 54, + record_ordinal: 0, + connections: HashMap::new(), + pending: VecDeque::new(), + }) + } + + /// Decodes every bounded line in one decrypted input file. + /// + /// # Errors + /// + /// Returns an error when a line exceeds the configured bound or violates + /// the selected audit grammar. + pub fn decode_all(&mut self, input: &[u8]) -> Result, ReplayError> { + let mut output = Vec::new(); + let mut offset = 0_usize; + let mut line = 0_u64; + while offset < input.len() { + let relative_end = input[offset..] + .iter() + .position(|byte| *byte == b'\n') + .unwrap_or(input.len() - offset); + if relative_end > self.record_limit { + return Err(ReplayError::decode( + &self.path, + offset, + "audit line exceeds record limit", + )); + } + let end = offset + relative_end; + line = line.saturating_add(1); + let text = std::str::from_utf8(&input[offset..end]).map_err(|_| { + ReplayError::decode(&self.path, offset, "audit line is not valid UTF-8") + })?; + self.record_ordinal = self.record_ordinal.saturating_add(1); + let mut commands = self.decode_line(text, offset, line)?; + for (command_ordinal, command) in commands.iter_mut().enumerate() { + command.command_ordinal = u32::try_from(command_ordinal).map_err(|_| { + ReplayError::decode( + &self.path, + offset, + "too many commands expanded from one audit record", + ) + })?; + } + output.extend(commands); + offset = if end < input.len() { end + 1 } else { end }; + } + while let Some(command) = self.pending.pop_front() { + output.push(command); + } + Ok(output) + } + + fn decode_line( + &mut self, + line_text: &str, + offset: usize, + line: u64, + ) -> Result, ReplayError> { + let fields = parse_bracket_fields(line_text) + .map_err(|message| ReplayError::decode(&self.path, offset, message))?; + let upstream_connection_id = required(&fields, "CONNECTION_ID", &self.path, offset)? + .parse::() + .map_err(|_| ReplayError::decode(&self.path, offset, "invalid CONNECTION_ID"))?; + match self.format { + TrafficFormat::AuditLogPlugin => { + self.decode_plugin(&fields, upstream_connection_id, offset, line) + } + TrafficFormat::AuditLogExtension => { + self.decode_extension(&fields, upstream_connection_id, offset, line) + } + TrafficFormat::Native => Err(ReplayError::decode( + &self.path, + offset, + "native input routed to audit decoder", + )), + } + } + + fn decode_plugin( + &mut self, + fields: &HashMap, + upstream_connection_id: u64, + offset: usize, + line: u64, + ) -> Result, ReplayError> { + let end_time = parse_audit_time(required(fields, "TIMESTAMP", &self.path, offset)?) + .map_err(|message| ReplayError::decode(&self.path, offset, message))?; + let start_time = match fields.get("COST_TIME").filter(|value| !value.is_empty()) { + Some(cost) => { + let micros = cost + .parse::() + .map_err(|_| ReplayError::decode(&self.path, offset, "invalid COST_TIME"))?; + end_time - duration_from_cost_micros(f64::from(micros), &self.path, offset)? + } + None => end_time, + }; + if self + .command_start_time + .is_some_and(|frontier| start_time < frontier) + || self + .command_end_time + .is_some_and(|frontier| end_time < frontier) + { + return Ok(Vec::new()); + } + if !self.user_allowed(fields) { + return Ok(Vec::new()); + } + let replay_id = self.ensure_connection(upstream_connection_id)?; + let class = required(fields, "EVENT_CLASS", &self.path, offset)?; + if class == "CONNECTION" { + if fields.get("EVENT_SUBCLASS").map(String::as_str) == Some("Disconnect") { + self.connections.remove(&upstream_connection_id); + return Ok(vec![build_command( + vec![CommandCode::QUIT.byte()], + start_time, + end_time, + replay_id, + upstream_connection_id, + fields, + &self.path, + line, + self.source_ordinal, + self.record_ordinal, + )?]); + } + return Ok(Vec::new()); + } + if class != "GENERAL" && class != "TABLE_ACCESS" { + return Err(ReplayError::decode( + &self.path, + offset, + format!("unknown EVENT_CLASS {class}"), + )); + } + if fields.get("EVENT").map(String::as_str) != Some("COMPLETED") { + return Ok(Vec::new()); + } + let encoded_command = fields.get("COMMAND").map_or("", String::as_str); + let command_name = if encoded_command.starts_with('"') { + parse_go_quoted_utf8(encoded_command) + .map_err(|message| ReplayError::decode(&self.path, offset, message))? + } else { + encoded_command.to_owned() + }; + let sql = match command_name.as_str() { + "Query" | "Execute" => Some( + unquote_if_needed(required(fields, "SQL_TEXT", &self.path, offset)?) + .map_err(|message| ReplayError::decode(&self.path, offset, message))?, + ), + _ => None, + }; + if matches!(command_name.as_str(), "Query" | "Execute") + && self.filter_retries + && fields.get("RETRY").map(String::as_str) == Some("true") + { + return Ok(Vec::new()); + } + if let Some(sql) = sql.as_deref() + && !self.filter_retries + && (command_name != "Execute" || fields.contains_key("EXECUTE_PARAMS")) + && self.is_duplicate_write( + upstream_connection_id, + fields, + &command_name, + sql, + start_time, + end_time, + )? + { + return Ok(Vec::new()); + } + self.commands_for_event( + fields, + upstream_connection_id, + start_time, + end_time, + line, + &command_name, + sql.as_deref(), + false, + offset, + ) + } + + fn decode_extension( + &mut self, + fields: &HashMap, + upstream_connection_id: u64, + offset: usize, + line: u64, + ) -> Result, ReplayError> { + let end_time = parse_audit_time(required(fields, "_LOG_TIME", &self.path, offset)?) + .map_err(|message| ReplayError::decode(&self.path, offset, message))?; + // The Go extension decoder maps command-start-time onto its sole + // end-time frontier. A non-zero command-end-time is then applied + // later and takes precedence. + let frontier = self.command_end_time.or(self.command_start_time); + if frontier.is_some_and(|frontier| end_time < frontier) { + return Ok(Vec::new()); + } + if !self.user_allowed(fields) { + return Ok(Vec::new()); + } + let replay_id = self.ensure_connection(upstream_connection_id)?; + let encoded_event = required(fields, "EVENT", &self.path, offset)?; + let event = parse_go_quoted_utf8(encoded_event) + .map_err(|message| ReplayError::decode(&self.path, offset, message))?; + let event = event + .strip_prefix('[') + .and_then(|value| value.strip_suffix(']')) + .ok_or_else(|| ReplayError::decode(&self.path, offset, "invalid EVENT tuple"))?; + let events: Vec<&str> = event.split(',').collect(); + match events.first().copied() { + Some("CONNECTION") if events.get(1).copied() == Some("DISCONNECT") => { + self.connections.remove(&upstream_connection_id); + Ok(vec![build_command( + vec![CommandCode::QUIT.byte()], + end_time, + end_time, + replay_id, + upstream_connection_id, + fields, + &self.path, + line, + self.source_ordinal, + self.record_ordinal, + )?]) + } + Some("QUERY") => { + let sql = match fields.get("SQL_TEXT") { + Some(value) if !value.is_empty() => unquote_if_needed(value) + .map_err(|message| ReplayError::decode(&self.path, offset, message))?, + _ => String::new(), + }; + let command_name = if events.get(1).copied() == Some("EXECUTE") { + "Execute" + } else { + "Query" + }; + self.commands_for_event( + fields, + upstream_connection_id, + end_time, + end_time, + line, + command_name, + Some(&sql), + true, + offset, + ) + } + _ => Ok(Vec::new()), + } + } + + #[allow(clippy::too_many_arguments, clippy::too_many_lines)] + fn commands_for_event( + &mut self, + fields: &HashMap, + upstream_connection_id: u64, + start_time: OffsetDateTime, + end_time: OffsetDateTime, + line: u64, + command_name: &str, + sql: Option<&str>, + extension: bool, + offset: usize, + ) -> Result, ReplayError> { + let mut state = self + .connections + .remove(&upstream_connection_id) + .ok_or_else(|| ReplayError::Config("audit connection state is missing".to_owned()))?; + let mut commands = Vec::with_capacity(3); + match command_name { + "Query" => { + let sql = sql.unwrap_or_default(); + let mut payload = vec![CommandCode::QUERY.byte()]; + payload.extend_from_slice(sql.as_bytes()); + commands.push(build_command( + payload, + start_time, + end_time, + state.replay_id, + upstream_connection_id, + fields, + &self.path, + line, + self.source_ordinal, + self.record_ordinal, + )?); + } + "Close stmt" => { + if self.strategy == PreparedCloseStrategy::Directed { + let statement_id = parse_statement_id(fields, &self.path, offset)?; + if state.prepared_ids.remove(&statement_id) { + commands.push(prepared_command( + close_payload(statement_id), + start_time, + end_time, + state.replay_id, + upstream_connection_id, + fields, + &self.path, + line, + self.source_ordinal, + self.record_ordinal, + statement_id, + None, + )?); + } + } + } + "Execute" => { + let Some(encoded_parameters) = fields.get("EXECUTE_PARAMS") else { + self.connections.insert(upstream_connection_id, state); + return Ok(Vec::new()); + }; + let parameters = if extension { + parse_extension_parameters(encoded_parameters) + } else { + parse_plugin_parameters(encoded_parameters) + } + .map_err(|message| ReplayError::decode(&self.path, offset, message))?; + let sql = sql.unwrap_or_default(); + let (statement_id, should_prepare) = match self.strategy { + PreparedCloseStrategy::Always => { + state.last_statement_id = state.last_statement_id.saturating_add(1); + (state.last_statement_id, true) + } + PreparedCloseStrategy::Directed => { + let statement_id = parse_statement_id(fields, &self.path, offset)?; + (statement_id, state.prepared_ids.insert(statement_id)) + } + PreparedCloseStrategy::Never => { + if let Some(statement_id) = state.prepared_by_sql.get(sql) { + (*statement_id, false) + } else { + state.last_statement_id = state.last_statement_id.saturating_add(1); + state + .prepared_by_sql + .insert(sql.to_owned(), state.last_statement_id); + (state.last_statement_id, true) + } + } + }; + if should_prepare { + let mut payload = vec![CommandCode::STMT_PREPARE.byte()]; + payload.extend_from_slice(sql.as_bytes()); + commands.push(prepared_command( + payload, + start_time, + end_time, + state.replay_id, + upstream_connection_id, + fields, + &self.path, + line, + self.source_ordinal, + self.record_ordinal, + statement_id, + Some(sql), + )?); + } + commands.push(prepared_command( + execute_payload(statement_id, ¶meters)?, + start_time, + end_time, + state.replay_id, + upstream_connection_id, + fields, + &self.path, + line, + self.source_ordinal, + self.record_ordinal, + statement_id, + Some(sql), + )?); + if self.strategy == PreparedCloseStrategy::Always { + commands.push(prepared_command( + close_payload(statement_id), + start_time, + end_time, + state.replay_id, + upstream_connection_id, + fields, + &self.path, + line, + self.source_ordinal, + self.record_ordinal, + statement_id, + Some(sql), + )?); + } + } + _ => {} + } + self.connections.insert(upstream_connection_id, state); + Ok(commands) + } + + fn ensure_connection(&mut self, upstream: u64) -> Result { + if let Some(state) = self.connections.get(&upstream) { + return Ok(state.replay_id); + } + self.next_connection_id = self.next_connection_id.checked_add(1).ok_or_else(|| { + ReplayError::Config("audit connection identifier space exhausted".to_owned()) + })?; + self.connections.insert( + upstream, + ConnectionState { + replay_id: self.next_connection_id, + ..ConnectionState::default() + }, + ); + Ok(self.next_connection_id) + } + + fn user_allowed(&self, fields: &HashMap) -> bool { + self.user_allowlist.is_empty() + || self.user_allowlist.contains( + &fields + .get("USER") + .map_or("", String::as_str) + .trim() + .to_ascii_lowercase(), + ) + } + + fn is_duplicate_write( + &mut self, + upstream: u64, + fields: &HashMap, + command: &str, + sql: &str, + start: OffsetDateTime, + end: OffsetDateTime, + ) -> Result { + let params = fields.get("EXECUTE_PARAMS").cloned().unwrap_or_default(); + let statement_type = fields.get("SQL_STATEMENTS").cloned().unwrap_or_default(); + let is_write = matches!( + statement_type.as_str(), + "Insert" | "Update" | "Delete" | "Replace" + ) || (statement_type == "Select" + && sql.to_ascii_lowercase().contains("for update")); + let state = self + .connections + .get_mut(&upstream) + .ok_or_else(|| ReplayError::Config("audit connection state is missing".to_owned()))?; + if !is_write { + state.last_write = None; + return Ok(false); + } + let duplicate = state.last_write.as_ref().is_some_and(|last| { + last.command == command + && last.sql == sql + && last.params == params + && last.statement_type == statement_type + && last.end >= start + && start - last.start <= Duration::from_millis(1) + }); + if !duplicate { + state.last_write = Some(WriteSignature { + command: command.to_owned(), + sql: sql.to_owned(), + params, + statement_type, + start, + end, + }); + } + Ok(duplicate) + } +} + +/// Parses the `TiDB` audit logger's bounded bracketed field grammar. +/// +/// # Errors +/// +/// Returns an error for duplicate fields, unbalanced delimiters, unterminated +/// quoted values, or invalid UTF-8 boundaries. +pub fn parse_bracket_fields(line: &str) -> Result, String> { + let bytes = line.as_bytes(); + let mut fields = HashMap::new(); + let mut index = 0_usize; + let mut first = true; + while index < bytes.len() { + if bytes[index] != b'[' { + index += 1; + continue; + } + let field_start = index + 1; + let mut cursor = field_start; + let mut separator = None; + let mut quote = None; + while cursor < bytes.len() { + let byte = bytes[cursor]; + if let Some(delimiter) = quote { + if byte == b'\\' { + cursor = cursor + .checked_add(2) + .ok_or_else(|| "quoted field offset overflow".to_owned())?; + continue; + } + if byte == delimiter { + quote = None; + } + } else { + match byte { + b'"' | b'\'' => quote = Some(byte), + b'=' if separator.is_none() => separator = Some(cursor), + b']' => break, + _ => {} + } + } + cursor += 1; + } + if quote.is_some() { + return Err("unterminated quote in audit field".to_owned()); + } + if cursor >= bytes.len() { + return Err("unterminated bracket in audit line".to_owned()); + } + let value_start = separator.map_or(field_start, |position| position + 1); + let value = line + .get(value_start..cursor) + .ok_or_else(|| "audit field is not on UTF-8 boundaries".to_owned())?; + if first { + fields.insert("_LOG_TIME".to_owned(), value.to_owned()); + first = false; + } else if let Some(position) = separator { + if position == field_start { + return Err("empty audit field key".to_owned()); + } + let key = line + .get(field_start..position) + .ok_or_else(|| "audit key is not on UTF-8 boundaries".to_owned())?; + if fields.insert(key.to_owned(), value.to_owned()).is_some() { + return Err(format!("duplicate audit field {key}")); + } + } + index = cursor + 1; + } + Ok(fields) +} + +fn parse_audit_time(value: &str) -> Result { + let format = time::format_description::parse_borrowed::<2>(AUDIT_TIME_FORMAT) + .map_err(|_| "internal audit timestamp format is invalid".to_owned())?; + OffsetDateTime::parse(value, &format).map_err(|_| format!("invalid audit timestamp {value}")) +} + +#[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] +fn duration_from_cost_micros( + micros: f64, + path: &str, + offset: usize, +) -> Result { + const MAX_DURATION_NANOS: f64 = 9_223_372_036_854_775_807.0; + let nanos = micros * 1_000.0; + if !nanos.is_finite() || !(0.0..=MAX_DURATION_NANOS).contains(&nanos) { + return Err(ReplayError::decode( + path, + offset, + "COST_TIME is outside the supported range", + )); + } + // Go converts COST_TIME * time.Microsecond to an integer duration by + // truncating toward zero. The bounds above make this cast explicit. + Ok(Duration::from_nanos(nanos as u64)) +} + +fn required<'a>( + fields: &'a HashMap, + key: &str, + path: &str, + offset: usize, +) -> Result<&'a str, ReplayError> { + fields + .get(key) + .filter(|value| !value.is_empty()) + .map(String::as_str) + .ok_or_else(|| ReplayError::decode(path, offset, format!("missing {key}"))) +} + +fn unquote_if_needed(value: &str) -> Result { + if value.starts_with('"') { + parse_go_quoted_utf8(value) + } else if value.is_empty() { + Err("empty SQL or command value".to_owned()) + } else { + Ok(value.to_owned()) + } +} + +fn parse_statement_id( + fields: &HashMap, + path: &str, + offset: usize, +) -> Result { + required(fields, "PREPARED_STMT_ID", path, offset)? + .parse::() + .map_err(|_| ReplayError::decode(path, offset, "invalid PREPARED_STMT_ID")) +} + +#[allow(clippy::too_many_arguments)] +fn build_command( + payload: Vec, + start: OffsetDateTime, + end: OffsetDateTime, + replay_connection_id: u64, + upstream_connection_id: u64, + fields: &HashMap, + path: &str, + line: u64, + source_ordinal: u64, + record_ordinal: u64, +) -> Result { + let mut command = Command::new(payload, start, replay_connection_id, path.to_owned(), line) + .ok_or_else(|| ReplayError::decode(path, 0, "empty generated command"))?; + command.end_time = Some(end); + command.upstream_connection_id = upstream_connection_id; + command.current_database = fields.get("CURRENT_DB").cloned().unwrap_or_default(); + command.statement_type = fields.get("SQL_STATEMENTS").cloned(); + command.source_ordinal = source_ordinal; + command.record_ordinal = record_ordinal; + Ok(command) +} + +#[allow(clippy::too_many_arguments)] +fn prepared_command( + payload: Vec, + start: OffsetDateTime, + end: OffsetDateTime, + replay_connection_id: u64, + upstream_connection_id: u64, + fields: &HashMap, + path: &str, + line: u64, + source_ordinal: u64, + record_ordinal: u64, + statement_id: u32, + sql: Option<&str>, +) -> Result { + let mut command = build_command( + payload, + start, + end, + replay_connection_id, + upstream_connection_id, + fields, + path, + line, + source_ordinal, + record_ordinal, + )?; + command.captured_statement_id = Some(statement_id); + command.prepared_statement = sql.map(str::to_owned); + Ok(command) +} + +fn parse_plugin_parameters(input: &str) -> Result, String> { + let decoded = + parse_go_quoted(input.as_bytes()).map_err(|error| format!("invalid params: {error}"))?; + let body = bracket_body(&decoded)?; + if body.is_empty() { + return Ok(Vec::new()); + } + let values = split_quoted_list(body)?; + values + .into_iter() + .map(|value| { + let decoded = parse_go_quoted(value)?; + let separator = decoded + .iter() + .position(|byte| *byte == b' ') + .ok_or_else(|| { + format!("no space in param {}", String::from_utf8_lossy(&decoded)) + })?; + let kind = std::str::from_utf8(&decoded[..separator]) + .map_err(|_| "parameter kind is not valid UTF-8".to_owned())?; + let value = &decoded[separator + 1..]; + match kind { + "KindNull" => Ok(Parameter::Null), + "KindInt64" => parameter_text(value)? + .parse::() + .map(Parameter::Signed) + .map_err(|_| "invalid signed param".to_owned()), + "KindUint64" => parameter_text(value)? + .parse::() + .map(Parameter::Unsigned) + .map_err(|_| "invalid unsigned param".to_owned()), + "KindFloat32" => parameter_text(value)? + .parse::() + .map(Parameter::Float) + .map_err(|_| "invalid float param".to_owned()), + "KindFloat64" | "KindMysqlDecimal" => parameter_text(value)? + .parse::() + .map(Parameter::Double) + .map_err(|_| "invalid double param".to_owned()), + "KindString" | "KindBytes" => parse_parameter_bytes(value).map(Parameter::String), + "KindBinaryLiteral" | "KindMysqlBit" | "KindMysqlSet" | "KindMysqlTime" + | "KindMysqlJSON" => Ok(Parameter::String(value.to_vec())), + "KindMysqlDuration" | "KindMysqlEnum" | "KindInterface" | "KindMinNotNull" + | "KindMaxValue" | "KindRaw" => Err(format!("unsupported param type {kind}")), + _ => Err(format!("unknown param type {kind}")), + } + }) + .collect() +} + +fn parse_extension_parameters(input: &str) -> Result, String> { + let decoded = + parse_go_quoted(input.as_bytes()).map_err(|error| format!("invalid params: {error}"))?; + let body = bracket_body(&decoded)?; + if body.is_empty() { + return Ok(Vec::new()); + } + split_extension_list(body).map(|values| values.into_iter().map(Parameter::String).collect()) +} + +fn bracket_body(input: &[u8]) -> Result<&[u8], String> { + if input.first() != Some(&b'[') || input.last() != Some(&b']') { + return Err("params have no surrounding brackets".to_owned()); + } + Ok(&input[1..input.len() - 1]) +} + +fn parameter_text(input: &[u8]) -> Result<&str, String> { + std::str::from_utf8(input).map_err(|_| "numeric parameter is not valid UTF-8".to_owned()) +} + +fn parse_parameter_bytes(input: &[u8]) -> Result, String> { + let mut quoted = Vec::with_capacity(input.len().saturating_add(2)); + quoted.push(b'"'); + quoted.extend_from_slice(input); + quoted.push(b'"'); + parse_go_quoted("ed) +} + +fn split_quoted_list(input: &[u8]) -> Result, String> { + let bytes = input; + let mut values = Vec::new(); + let mut index = 0_usize; + while index < bytes.len() { + while bytes.get(index).is_some_and(u8::is_ascii_whitespace) + || bytes.get(index) == Some(&b',') + { + index += 1; + } + if index >= bytes.len() { + break; + } + if bytes[index] != b'"' { + return Err("expected quoted audit parameter".to_owned()); + } + let start = index; + index += 1; + let mut closed = false; + while index < bytes.len() { + match bytes[index] { + b'\\' => index += 2, + b'"' => { + index += 1; + values.push(&input[start..index]); + closed = true; + break; + } + _ => index += 1, + } + } + if !closed { + return Err("unterminated quoted audit parameter".to_owned()); + } + } + Ok(values) +} + +fn split_extension_list(input: &[u8]) -> Result>, String> { + let bytes = input; + let mut values = Vec::new(); + let mut index = 0_usize; + while index < bytes.len() { + while bytes + .get(index) + .is_some_and(|byte| *byte == b',' || byte.is_ascii_whitespace()) + { + index += 1; + } + if index >= bytes.len() { + break; + } + if bytes[index] == b'"' { + let start = index; + index += 1; + while index < bytes.len() { + match bytes[index] { + b'\\' => index += 2, + b'"' => { + index += 1; + let quoted = &input[start..index]; + values.push(parse_go_quoted(quoted)?); + break; + } + _ => index += 1, + } + } + if index > bytes.len() || bytes.get(index.saturating_sub(1)) != Some(&b'"') { + return Err("unterminated extension parameter".to_owned()); + } + } else { + let start = index; + while index < bytes.len() && bytes[index] != b',' { + index += 1; + } + values.push(input[start..index].trim_ascii().to_vec()); + } + } + Ok(values) +} + +fn execute_payload(statement_id: u32, parameters: &[Parameter]) -> Result, ReplayError> { + let bitmap_length = parameters.len().div_ceil(8); + let capacity = 10_usize + .checked_add(bitmap_length) + .and_then(|base| base.checked_add(usize::from(!parameters.is_empty()))) + .and_then(|base| base.checked_add(parameters.len().saturating_mul(10))) + .ok_or_else(|| ReplayError::Config("execute payload size overflow".to_owned()))?; + let mut output = Vec::with_capacity(capacity); + output.push(CommandCode::STMT_EXECUTE.byte()); + output.extend_from_slice(&statement_id.to_le_bytes()); + output.push(0); + output.extend_from_slice(&1_u32.to_le_bytes()); + if parameters.is_empty() { + return Ok(output); + } + let bitmap_offset = output.len(); + output.resize(bitmap_offset + bitmap_length, 0); + output.push(1); + for (index, parameter) in parameters.iter().enumerate() { + let (field_type, unsigned) = match parameter { + Parameter::Null => { + output[bitmap_offset + index / 8] |= 1 << (index % 8); + (6_u8, 0_u8) + } + Parameter::Signed(_) => (8, 0), + Parameter::Unsigned(_) => (8, 0x80), + Parameter::Float(_) => (4, 0), + Parameter::Double(_) => (5, 0), + Parameter::String(_) => (0xfe, 0), + }; + output.extend_from_slice(&[field_type, unsigned]); + } + for parameter in parameters { + match parameter { + Parameter::Null => {} + Parameter::Signed(value) => output.extend_from_slice(&value.to_le_bytes()), + Parameter::Unsigned(value) => output.extend_from_slice(&value.to_le_bytes()), + Parameter::Float(value) => output.extend_from_slice(&value.to_le_bytes()), + Parameter::Double(value) => output.extend_from_slice(&value.to_le_bytes()), + Parameter::String(value) => append_length_encoded(&mut output, value)?, + } + } + Ok(output) +} + +fn append_length_encoded(output: &mut Vec, value: &[u8]) -> Result<(), ReplayError> { + let length = u64::try_from(value.len()) + .map_err(|_| ReplayError::Config("parameter length does not fit u64".to_owned()))?; + match length { + 0..=250 => output.push( + u8::try_from(length) + .map_err(|_| ReplayError::Config("parameter length exceeds u8".to_owned()))?, + ), + 251..=65_535 => { + output.push(0xfc); + output.extend_from_slice( + &u16::try_from(length) + .map_err(|_| ReplayError::Config("parameter length exceeds u16".to_owned()))? + .to_le_bytes(), + ); + } + 65_536..=16_777_215 => { + output.push(0xfd); + let bytes = length.to_le_bytes(); + output.extend_from_slice(&bytes[..3]); + } + _ => { + output.push(0xfe); + output.extend_from_slice(&length.to_le_bytes()); + } + } + output.extend_from_slice(value); + Ok(()) +} + +fn close_payload(statement_id: u32) -> Vec { + let mut output = vec![CommandCode::STMT_CLOSE.byte()]; + output.extend_from_slice(&statement_id.to_le_bytes()); + output +} + +#[cfg(test)] +#[allow(clippy::expect_used)] +mod tests { + use super::*; + + #[test] + fn parses_nested_brackets_and_quotes() { + let fields = parse_bracket_fields( + r#"[2025/09/06 17:03:53.720 +08:00] [SQL_TEXT="select \"[=]\""] [CONNECTION_ID=9]"#, + ) + .expect("valid fields"); + assert_eq!( + fields.get("_LOG_TIME").map(String::as_str), + Some("2025/09/06 17:03:53.720 +08:00") + ); + assert_eq!( + fields.get("SQL_TEXT").map(String::as_str), + Some(r#""select \"[=]\"""#) + ); + assert_eq!(fields.get("CONNECTION_ID").map(String::as_str), Some("9")); + } + + #[test] + fn rejects_unterminated_and_duplicate_fields() { + assert!(parse_bracket_fields("[a=b").is_err()); + assert!(parse_bracket_fields("[time] [a=b] [a=c]").is_err()); + } + + #[test] + fn plugin_query_and_retry_filter_match_contract() { + let line = concat!( + "[2025/09/06 17:03:53.720 +08:00] ", + "[TIMESTAMP=2025/09/06 17:03:53.720 +08:00] ", + "[EVENT_CLASS=GENERAL] [EVENT=COMPLETED] [COMMAND=Query] ", + "[SQL_TEXT=SELECT 1] [SQL_STATEMENTS=Select] [CONNECTION_ID=9] ", + "[USER=root] [CURRENT_DB=test] [RETRY=false]\n" + ); + let mut decoder = AuditDecoder::new( + TrafficFormat::AuditLogPlugin, + "audit.log", + 4096, + PreparedCloseStrategy::Directed, + None, + None, + true, + &["ROOT".to_owned()], + 3, + ) + .expect("decoder"); + let commands = decoder.decode_all(line.as_bytes()).expect("decode"); + assert_eq!(commands.len(), 1); + assert_eq!(commands[0].payload, b"\x03SELECT 1"); + assert_eq!(commands[0].connection_id, (3_u64 << 54) + 1); + assert_eq!(commands[0].current_database, "test"); + } + + #[test] + fn extension_always_emits_prepare_execute_close() { + let line = concat!( + "[2026/01/08 19:44:11.114 +08:00] ", + "[EVENT=\"[QUERY,EXECUTE,SELECT]\"] [USER=root] [CONNECTION_ID=7] ", + "[CURRENT_DB=test] [SQL_TEXT=\"SELECT ?\"] [EXECUTE_PARAMS=\"[1]\"]\n" + ); + let mut decoder = AuditDecoder::new( + TrafficFormat::AuditLogExtension, + "audit.log", + 4096, + PreparedCloseStrategy::Always, + None, + None, + false, + &[], + 0, + ) + .expect("decoder"); + let commands = decoder.decode_all(line.as_bytes()).expect("decode"); + assert_eq!(commands.len(), 3); + assert_eq!(commands[0].command, CommandCode::STMT_PREPARE); + assert_eq!(commands[1].command, CommandCode::STMT_EXECUTE); + assert_eq!(commands[2].command, CommandCode::STMT_CLOSE); + assert_eq!( + commands[1].payload, + vec![0x17, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0xfe, 0, 1, b'1'] + ); + } + + #[test] + fn extension_maps_command_start_time_to_end_time_frontier() { + let lines = concat!( + "[2026/01/08 19:44:11.099 +08:00] ", + "[EVENT=\"[QUERY,QUERY_DDL]\"] [USER=root] [CONNECTION_ID=7] ", + "[CURRENT_DB=test] [SQL_TEXT=\"CREATE TABLE t (id INT)\"]\n", + "[2026/01/08 19:44:11.110 +08:00] ", + "[EVENT=\"[QUERY,QUERY_DML,INSERT]\"] [USER=root] [CONNECTION_ID=7] ", + "[CURRENT_DB=test] [SQL_TEXT=\"INSERT INTO t VALUES (1)\"]\n" + ); + let frontier = parse_audit_time("2026/01/08 19:44:11.110 +08:00").expect("valid frontier"); + let mut decoder = AuditDecoder::new( + TrafficFormat::AuditLogExtension, + "audit.log", + 4096, + PreparedCloseStrategy::Never, + Some(frontier), + None, + false, + &[], + 0, + ) + .expect("decoder"); + let commands = decoder.decode_all(lines.as_bytes()).expect("decode"); + assert_eq!(commands.len(), 1); + assert_eq!(commands[0].payload, b"\x03INSERT INTO t VALUES (1)"); + } + + #[test] + fn plugin_kind_bytes_preserves_non_utf8_payload() { + let parameters = parse_plugin_parameters(r#""[\"KindBytes \\\\xff\"]""#) + .expect("binary KindBytes parameter"); + let payload = execute_payload(7, ¶meters).expect("execute payload"); + assert_eq!(payload.last(), Some(&0xff)); + } + + #[test] + fn ignored_plugin_events_still_advance_connection_allocator() { + let lines = concat!( + "[2025/09/06 17:03:53.720 +08:00] ", + "[TIMESTAMP=2025/09/06 17:03:53.720 +08:00] ", + "[EVENT_CLASS=GENERAL] [EVENT=STARTING] [COMMAND=Query] ", + "[SQL_TEXT=SELECT 1] [SQL_STATEMENTS=Select] [CONNECTION_ID=40]\n", + "[2025/09/06 17:03:53.720 +08:00] ", + "[TIMESTAMP=2025/09/06 17:03:53.720 +08:00] ", + "[EVENT_CLASS=CONNECTION] [EVENT_SUBCLASS=Connected] [CONNECTION_ID=41]\n", + "[2025/09/06 17:03:53.720 +08:00] ", + "[TIMESTAMP=2025/09/06 17:03:53.720 +08:00] ", + "[EVENT_CLASS=GENERAL] [EVENT=COMPLETED] [COMMAND=Query] ", + "[SQL_TEXT=SELECT 2] [SQL_STATEMENTS=Select] [CONNECTION_ID=42]\n" + ); + let mut decoder = AuditDecoder::new( + TrafficFormat::AuditLogPlugin, + "audit.log", + 4096, + PreparedCloseStrategy::Directed, + None, + None, + false, + &[], + 2, + ) + .expect("decoder"); + let commands = decoder.decode_all(lines.as_bytes()).expect("decode"); + assert_eq!(commands.len(), 1); + assert_eq!(commands[0].connection_id, (2_u64 << 54) + 3); + } + + #[test] + fn disconnect_uses_existing_replay_connection_id() { + let lines = concat!( + "[2025/09/06 17:03:53.720 +08:00] ", + "[TIMESTAMP=2025/09/06 17:03:53.720 +08:00] ", + "[EVENT_CLASS=GENERAL] [EVENT=COMPLETED] [COMMAND=Query] ", + "[SQL_TEXT=SELECT 1] [SQL_STATEMENTS=Select] [CONNECTION_ID=9]\n", + "[2025/09/06 17:03:53.721 +08:00] ", + "[TIMESTAMP=2025/09/06 17:03:53.721 +08:00] ", + "[EVENT_CLASS=CONNECTION] [EVENT_SUBCLASS=Disconnect] [CONNECTION_ID=9]\n", + "[2025/09/06 17:03:53.722 +08:00] ", + "[TIMESTAMP=2025/09/06 17:03:53.722 +08:00] ", + "[EVENT_CLASS=GENERAL] [EVENT=COMPLETED] [COMMAND=Query] ", + "[SQL_TEXT=SELECT 2] [SQL_STATEMENTS=Select] [CONNECTION_ID=9]\n" + ); + let mut decoder = AuditDecoder::new( + TrafficFormat::AuditLogPlugin, + "audit.log", + 4096, + PreparedCloseStrategy::Directed, + None, + None, + false, + &[], + 0, + ) + .expect("decoder"); + let commands = decoder.decode_all(lines.as_bytes()).expect("decode"); + assert_eq!(commands.len(), 3); + assert_eq!(commands[0].connection_id, commands[1].connection_id); + assert_eq!(commands[1].command, CommandCode::QUIT); + assert_ne!(commands[1].connection_id, commands[2].connection_id); + } + + #[test] + fn intervening_read_prevents_non_adjacent_write_deduplication() { + let lines = concat!( + "[2025/09/06 17:03:53.720 +08:00] ", + "[TIMESTAMP=2025/09/06 17:03:53.720 +08:00] ", + "[EVENT_CLASS=GENERAL] [EVENT=COMPLETED] [COMMAND=Query] ", + "[SQL_TEXT=INSERT INTO t VALUES (1)] [SQL_STATEMENTS=Insert] [CONNECTION_ID=9]\n", + "[2025/09/06 17:03:53.720 +08:00] ", + "[TIMESTAMP=2025/09/06 17:03:53.720 +08:00] ", + "[EVENT_CLASS=GENERAL] [EVENT=COMPLETED] [COMMAND=Query] ", + "[SQL_TEXT=SELECT 1] [SQL_STATEMENTS=Select] [CONNECTION_ID=9]\n", + "[2025/09/06 17:03:53.720 +08:00] ", + "[TIMESTAMP=2025/09/06 17:03:53.720 +08:00] ", + "[EVENT_CLASS=GENERAL] [EVENT=COMPLETED] [COMMAND=Query] ", + "[SQL_TEXT=INSERT INTO t VALUES (1)] [SQL_STATEMENTS=Insert] [CONNECTION_ID=9]\n" + ); + let mut decoder = AuditDecoder::new( + TrafficFormat::AuditLogPlugin, + "audit.log", + 4096, + PreparedCloseStrategy::Directed, + None, + None, + false, + &[], + 0, + ) + .expect("decoder"); + let commands = decoder.decode_all(lines.as_bytes()).expect("decode"); + assert_eq!(commands.len(), 3); + } + + #[test] + fn retry_filter_does_not_suppress_prepared_close() { + let lines = concat!( + "[2025/09/06 17:03:53.720 +08:00] ", + "[TIMESTAMP=2025/09/06 17:03:53.720 +08:00] ", + "[EVENT_CLASS=GENERAL] [EVENT=COMPLETED] [COMMAND=Execute] ", + "[SQL_TEXT=SELECT ?] [SQL_STATEMENTS=Select] [CONNECTION_ID=9] ", + "[PREPARED_STMT_ID=7] [EXECUTE_PARAMS=\"[]\"] [RETRY=false]\n", + "[2025/09/06 17:03:53.721 +08:00] ", + "[TIMESTAMP=2025/09/06 17:03:53.721 +08:00] ", + "[EVENT_CLASS=GENERAL] [EVENT=COMPLETED] [COMMAND=\"Close stmt\"] ", + "[CONNECTION_ID=9] [PREPARED_STMT_ID=7] [RETRY=true]\n" + ); + let mut decoder = AuditDecoder::new( + TrafficFormat::AuditLogPlugin, + "audit.log", + 4096, + PreparedCloseStrategy::Directed, + None, + None, + true, + &[], + 0, + ) + .expect("decoder"); + let commands = decoder.decode_all(lines.as_bytes()).expect("decode"); + assert_eq!(commands.len(), 3); + assert_eq!(commands[1].payload, vec![0x17, 7, 0, 0, 0, 0, 1, 0, 0, 0]); + assert_eq!(commands[2].command, CommandCode::STMT_CLOSE); + } + + #[test] + fn plugin_validates_timestamp_before_user_filter() { + let line = concat!( + "[not-a-time] [TIMESTAMP=not-a-time] [EVENT_CLASS=GENERAL] ", + "[EVENT=COMPLETED] [COMMAND=Query] [SQL_TEXT=SELECT 1] ", + "[SQL_STATEMENTS=Select] [CONNECTION_ID=9] [USER=someone-else]\n" + ); + let mut decoder = AuditDecoder::new( + TrafficFormat::AuditLogPlugin, + "audit.log", + 4096, + PreparedCloseStrategy::Directed, + None, + None, + false, + &["root".to_owned()], + 0, + ) + .expect("decoder"); + assert!(decoder.decode_all(line.as_bytes()).is_err()); + } + + #[test] + fn empty_plugin_command_is_ignored_after_allocating_connection() { + let lines = concat!( + "[2025/09/06 17:03:53.720 +08:00] ", + "[TIMESTAMP=2025/09/06 17:03:53.720 +08:00] ", + "[EVENT_CLASS=GENERAL] [EVENT=COMPLETED] [CONNECTION_ID=8]\n", + "[2025/09/06 17:03:53.721 +08:00] ", + "[TIMESTAMP=2025/09/06 17:03:53.721 +08:00] ", + "[EVENT_CLASS=GENERAL] [EVENT=COMPLETED] [COMMAND=Query] ", + "[SQL_TEXT=SELECT 1] [SQL_STATEMENTS=Select] [CONNECTION_ID=9]\n" + ); + let mut decoder = AuditDecoder::new( + TrafficFormat::AuditLogPlugin, + "audit.log", + 4096, + PreparedCloseStrategy::Directed, + None, + None, + false, + &[], + 0, + ) + .expect("decoder"); + let commands = decoder.decode_all(lines.as_bytes()).expect("decode"); + assert_eq!(commands.len(), 1); + assert_eq!(commands[0].connection_id, 2); + } +} diff --git a/rust/crates/replayer-core/src/decode/mod.rs b/rust/crates/replayer-core/src/decode/mod.rs new file mode 100644 index 000000000..ac16c0b61 --- /dev/null +++ b/rust/crates/replayer-core/src/decode/mod.rs @@ -0,0 +1,23 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Bounded traffic-format decoders. + +mod audit; +mod native; +mod quoted; + +pub use audit::{AuditDecoder, parse_bracket_fields}; +pub use native::NativeDecoder; +pub(crate) use quoted::{parse_go_quoted, parse_go_quoted_utf8}; diff --git a/rust/crates/replayer-core/src/decode/native.rs b/rust/crates/replayer-core/src/decode/native.rs new file mode 100644 index 000000000..b0c94331d --- /dev/null +++ b/rust/crates/replayer-core/src/decode/native.rs @@ -0,0 +1,306 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::HashSet; + +use time::OffsetDateTime; +use time::format_description::well_known::Rfc3339; + +use crate::decode::parse_go_quoted_utf8; +use crate::{Command, CommandCode, ReplayError}; + +/// Stateful decoder for `TiProxy` native capture records. +pub struct NativeDecoder<'a> { + input: &'a [u8], + path: String, + offset: usize, + line: u64, + record_limit: usize, + command_start_time: Option, + record_ordinal: u64, +} + +impl<'a> NativeDecoder<'a> { + /// Creates a bounded decoder over one uncompressed, decrypted file. + #[must_use] + pub fn new( + input: &'a [u8], + path: impl Into, + record_limit: usize, + command_start_time: Option, + ) -> Self { + Self { + input, + path: path.into(), + offset: 0, + line: 0, + record_limit, + command_start_time, + record_ordinal: 0, + } + } + + /// Decodes the next record, or `None` at a clean file boundary. + /// + /// # Errors + /// + /// Returns an error when the next record violates the bounded native + /// capture grammar. + pub fn next_command(&mut self) -> Result, ReplayError> { + loop { + if self.offset == self.input.len() { + return Ok(None); + } + let command = self.decode_one()?; + if self + .command_start_time + .is_some_and(|frontier| command.start_time <= frontier) + { + continue; + } + return Ok(Some(command)); + } + } + + #[allow(clippy::too_many_lines)] + fn decode_one(&mut self) -> Result { + let record_offset = self.offset; + let mut seen = HashSet::new(); + let mut start_time = None; + let mut connection_id = None; + let mut command = CommandCode::QUERY; + let mut succeeded = true; + let mut captured_statement_id = None; + let mut prepared_statement = None; + + loop { + let (line_offset, line) = self.read_header_line()?; + let line = std::str::from_utf8(line).map_err(|_| { + ReplayError::decode(&self.path, line_offset, "header is not valid UTF-8") + })?; + let body = line.strip_prefix("# ").ok_or_else(|| { + ReplayError::decode(&self.path, line_offset, "line does not start with '# '") + })?; + let (key, value) = body.split_once(": ").ok_or_else(|| { + ReplayError::decode(&self.path, line_offset, "header does not contain ': '") + })?; + if value.is_empty() { + return Err(ReplayError::decode( + &self.path, + line_offset, + format!("{key} value is empty"), + )); + } + let known = matches!( + key, + "Time" + | "Conn_ID" + | "Cmd_type" + | "Success" + | "Captured_ps_id" + | "Prepared_stmt" + | "Payload_len" + ); + if known && !seen.insert(key.to_owned()) { + return Err(ReplayError::decode( + &self.path, + line_offset, + format!("duplicate {key} header"), + )); + } + match key { + "Time" => { + start_time = Some(OffsetDateTime::parse(value, &Rfc3339).map_err(|_| { + ReplayError::decode(&self.path, line_offset, "invalid RFC3339 Time") + })?); + } + "Conn_ID" => { + let value = value.parse::().map_err(|_| { + ReplayError::decode(&self.path, line_offset, "invalid Conn_ID") + })?; + if value == 0 { + return Err(ReplayError::decode( + &self.path, + line_offset, + "Conn_ID must be positive", + )); + } + connection_id = Some(value); + } + "Cmd_type" => { + command = CommandCode::from_go_name(value) + .map_err(|message| ReplayError::decode(&self.path, line_offset, message))?; + } + "Success" => succeeded = value == "true", + "Captured_ps_id" => { + captured_statement_id = Some(value.parse::().map_err(|_| { + ReplayError::decode(&self.path, line_offset, "invalid Captured_ps_id") + })?); + } + "Prepared_stmt" => { + prepared_statement = Some(parse_go_quoted_utf8(value).map_err(|message| { + ReplayError::decode(&self.path, line_offset, message) + })?); + } + "Payload_len" => { + let length = value.parse::().map_err(|_| { + ReplayError::decode(&self.path, line_offset, "invalid Payload_len") + })?; + if length > self.record_limit { + return Err(ReplayError::decode( + &self.path, + line_offset, + format!("Payload_len {length} exceeds limit {}", self.record_limit), + )); + } + let payload_offset = self.offset; + let payload_end = payload_offset.checked_add(length).ok_or_else(|| { + ReplayError::decode(&self.path, payload_offset, "Payload_len overflow") + })?; + if payload_end >= self.input.len() { + return Err(ReplayError::decode( + &self.path, + payload_offset, + "truncated payload or missing final newline", + )); + } + if self.input[payload_end] != b'\n' { + return Err(ReplayError::decode( + &self.path, + payload_end, + "payload is not followed by newline", + )); + } + let mut payload = Vec::with_capacity(length + 1); + payload.push(command.byte()); + payload.extend_from_slice(&self.input[payload_offset..payload_end]); + self.offset = payload_end + 1; + self.line = self.line.saturating_add(1); + self.record_ordinal = self.record_ordinal.saturating_add(1); + let start_time = start_time.ok_or_else(|| { + ReplayError::decode(&self.path, record_offset, "missing Time header") + })?; + let connection_id = connection_id.ok_or_else(|| { + ReplayError::decode(&self.path, record_offset, "missing Conn_ID header") + })?; + let mut decoded = Command::new( + payload, + start_time, + connection_id, + self.path.clone(), + self.line, + ) + .ok_or_else(|| { + ReplayError::decode(&self.path, payload_offset, "empty payload") + })?; + decoded.command = command; + decoded.succeeded = succeeded; + decoded.captured_statement_id = captured_statement_id; + decoded.prepared_statement = prepared_statement; + decoded.record_ordinal = self.record_ordinal; + return Ok(decoded); + } + _ => {} + } + } + } + + fn read_header_line(&mut self) -> Result<(usize, &'a [u8]), ReplayError> { + let start = self.offset; + let relative_end = self.input[start..] + .iter() + .position(|byte| *byte == b'\n') + .ok_or_else(|| ReplayError::decode(&self.path, start, "unterminated header line"))?; + if relative_end > self.record_limit { + return Err(ReplayError::decode( + &self.path, + start, + "header line exceeds record limit", + )); + } + let end = start + relative_end; + self.offset = end + 1; + self.line = self.line.saturating_add(1); + Ok((start, &self.input[start..end])) + } +} + +#[cfg(test)] +#[allow(clippy::expect_used)] +mod tests { + use super::*; + + #[test] + fn decodes_binary_native_record_and_ties() { + let input = concat!( + "# Time: 2026-01-08T19:44:11.099+08:00\n", + "# Conn_ID: 42\n", + "# Cmd_type: StmtExecute\n", + "# Captured_ps_id: 7\n", + r#"# Prepared_stmt: "select \n?""#, + "\n", + "# Payload_len: 5\n", + ); + let mut bytes = input.as_bytes().to_vec(); + bytes.extend_from_slice(&[7, 0, 0, 0, 0]); + bytes.push(b'\n'); + let mut decoder = NativeDecoder::new(&bytes, "traffic.log", 64, None); + let command = decoder + .next_command() + .expect("decode succeeds") + .expect("record"); + assert_eq!(command.command, CommandCode::STMT_EXECUTE); + assert_eq!(command.payload, vec![0x17, 7, 0, 0, 0, 0]); + assert_eq!(command.captured_statement_id, Some(7)); + assert_eq!(command.prepared_statement.as_deref(), Some("select \n?")); + assert!(decoder.next_command().expect("EOF succeeds").is_none()); + } + + #[test] + fn rejects_oversized_before_allocation() { + let bytes = b"# Time: 2026-01-08T19:44:11Z\n# Conn_ID: 1\n# Payload_len: 65\n"; + let mut decoder = NativeDecoder::new(bytes, "traffic.log", 64, None); + let error = decoder.next_command().expect_err("oversize rejected"); + assert!(error.to_string().contains("exceeds limit 64")); + } + + #[test] + fn rejects_duplicate_and_missing_newline() { + let duplicate = b"# Time: 2026-01-08T19:44:11Z\n# Time: 2026-01-08T19:44:12Z\n"; + let mut decoder = NativeDecoder::new(duplicate, "traffic.log", 64, None); + assert!(decoder.next_command().is_err()); + + let truncated = b"# Time: 2026-01-08T19:44:11Z\n# Conn_ID: 1\n# Payload_len: 1\nx"; + let mut decoder = NativeDecoder::new(truncated, "traffic.log", 64, None); + assert!(decoder.next_command().is_err()); + } + + #[test] + fn ignores_duplicate_unknown_headers_for_forward_compatibility() { + let input = concat!( + "# Future: one\n", + "# Future: two\n", + "# Time: 2026-01-08T19:44:11Z\n", + "# Conn_ID: 1\n", + "# Payload_len: 0\n", + "\n" + ); + let mut decoder = NativeDecoder::new(input.as_bytes(), "traffic.log", 64, None); + let command = decoder + .next_command() + .expect("decode succeeds") + .expect("record"); + assert_eq!(command.payload, vec![CommandCode::QUERY.byte()]); + } +} diff --git a/rust/crates/replayer-core/src/decode/quoted.rs b/rust/crates/replayer-core/src/decode/quoted.rs new file mode 100644 index 000000000..0949ea67d --- /dev/null +++ b/rust/crates/replayer-core/src/decode/quoted.rs @@ -0,0 +1,134 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pub(crate) fn parse_go_quoted(input: &[u8]) -> Result, String> { + let bytes = input; + if bytes.len() < 2 || bytes.first() != Some(&b'"') || bytes.last() != Some(&b'"') { + return Err("value must be a Go-quoted string".to_owned()); + } + let mut output = Vec::with_capacity(bytes.len().saturating_sub(2)); + let mut index = 1; + while index + 1 < bytes.len() { + if bytes[index] != b'\\' { + output.push(bytes[index]); + index += 1; + continue; + } + index += 1; + let escaped = *bytes + .get(index) + .ok_or_else(|| "value has a trailing escape".to_owned())?; + match escaped { + b'a' => output.push(0x07), + b'b' => output.push(0x08), + b'f' => output.push(0x0c), + b'n' => output.push(b'\n'), + b'r' => output.push(b'\r'), + b't' => output.push(b'\t'), + b'v' => output.push(0x0b), + b'\\' | b'"' | b'\'' => output.push(escaped), + b'x' => { + output.push( + u8::try_from(parse_hex(bytes, index + 1, 2)?) + .map_err(|_| "hex escape is larger than one byte".to_owned())?, + ); + index += 2; + } + b'u' => { + append_unicode(&mut output, parse_hex(bytes, index + 1, 4)?)?; + index += 4; + } + b'U' => { + append_unicode(&mut output, parse_hex(bytes, index + 1, 8)?)?; + index += 8; + } + b'0'..=b'7' => { + let end = index + .checked_add(3) + .ok_or_else(|| "octal escape overflow".to_owned())?; + let digits = bytes + .get(index..end) + .ok_or_else(|| "short octal escape".to_owned())?; + let mut value = 0_u16; + for digit in digits { + if !(b'0'..=b'7').contains(digit) { + return Err("invalid octal escape".to_owned()); + } + value = value * 8 + u16::from(*digit - b'0'); + } + if value > u16::from(u8::MAX) { + return Err("octal escape is larger than one byte".to_owned()); + } + output.push( + u8::try_from(value) + .map_err(|_| "octal escape is larger than one byte".to_owned())?, + ); + index += 2; + } + other => return Err(format!("unsupported escape \\{}", char::from(other))), + } + index += 1; + } + Ok(output) +} + +pub(crate) fn parse_go_quoted_utf8(input: &str) -> Result { + String::from_utf8(parse_go_quoted(input.as_bytes())?) + .map_err(|_| "quoted value is not valid UTF-8".to_owned()) +} + +fn parse_hex(bytes: &[u8], start: usize, count: usize) -> Result { + let end = start + .checked_add(count) + .ok_or_else(|| "hex escape overflow".to_owned())?; + let digits = bytes + .get(start..end) + .ok_or_else(|| "short hex escape".to_owned())?; + let mut value = 0_u32; + for digit in digits { + value = value + .checked_mul(16) + .and_then(|current| char::from(*digit).to_digit(16).map(|next| current + next)) + .ok_or_else(|| "invalid hex escape".to_owned())?; + } + Ok(value) +} + +fn append_unicode(output: &mut Vec, value: u32) -> Result<(), String> { + let value = char::from_u32(value).ok_or_else(|| "invalid Unicode escape".to_owned())?; + let mut encoded = [0_u8; 4]; + output.extend_from_slice(value.encode_utf8(&mut encoded).as_bytes()); + Ok(()) +} + +#[cfg(test)] +#[allow(clippy::expect_used)] +mod tests { + use super::*; + + #[test] + fn accepts_go_specific_escapes() { + assert_eq!( + parse_go_quoted(br#""a\x00\u4e2d\U0001f60a\141""#).expect("valid"), + "a\0中😊a".as_bytes() + ); + } + + #[test] + fn preserves_binary_and_rejects_short_escapes() { + assert_eq!(parse_go_quoted(br#""\xff""#).expect("binary"), [0xff]); + assert!(parse_go_quoted(br#""\x0""#).is_err()); + assert!(parse_go_quoted_utf8(r#""\xff""#).is_err()); + } +} diff --git a/rust/crates/replayer-core/src/engine.rs b/rust/crates/replayer-core/src/engine.rs new file mode 100644 index 000000000..7feb73371 --- /dev/null +++ b/rust/crates/replayer-core/src/engine.rs @@ -0,0 +1,459 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Deterministic dry-run replay path. + +use std::collections::{HashMap, HashSet}; + +use serde::{Deserialize, Serialize}; + +use crate::checkpoint::{CHECKPOINT_SCHEMA_VERSION, Checkpoint, InputIdentity}; +use crate::config::ReplayConfig; +use crate::decode::{AuditDecoder, NativeDecoder}; +use crate::storage::InputRoot; +use crate::{Command, CommandCode, ReplayError, TrafficFormat}; + +/// Semantic output of a complete dry run. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct DryRunSummary { + /// Number of decoded commands before replay-side filtering. + pub decoded_commands: u64, + /// Number of commands accepted by replay-side filtering. + pub replayed_commands: u64, + /// Number of replay-side filtered commands. + pub filtered_commands: u64, + /// Number of deterministic input files consumed. + pub input_files: u64, + /// Last committed command timestamp in RFC3339. + pub last_command_time: Option, + /// Exact input identity used by the checkpoint. + pub input_identity: InputIdentity, +} + +/// Runs the full local/OpenDAL read, decrypt, decompress, decode, ordering, +/// filtering, and atomic-checkpoint path without a backend connection. +/// +/// # Errors +/// +/// Returns an error when configuration, storage, decoding, filtering, or +/// checkpoint persistence fails closed. +#[allow(clippy::too_many_lines)] +pub async fn dry_run(config: &ReplayConfig) -> Result { + if !config.dry_run { + return Err(ReplayError::Config( + "dry_run requires dry-run configuration".to_owned(), + )); + } + let inputs = config.inputs()?; + let mut commands = Vec::new(); + let mut identity_parts = Vec::new(); + let mut input_files = 0_u64; + for (source_index, input) in inputs.iter().enumerate() { + let source_ordinal = u64::try_from(source_index) + .map_err(|_| ReplayError::Config("too many input roots".to_owned()))?; + let root = InputRoot::open(input)?; + let metadata = if config.format == TrafficFormat::Native { + root.metadata().await? + } else { + None + }; + let entries = root.list(config.format).await?; + identity_parts.extend_from_slice(root.safe_root().as_bytes()); + identity_parts.push(0); + if let Some(metadata) = metadata.as_ref() { + identity_parts.extend_from_slice( + &serde_json::to_vec(metadata).map_err(|error| { + ReplayError::Checkpoint(format!("encode metadata: {error}")) + })?, + ); + } + let mut audit = if config.format.is_audit() { + Some(AuditDecoder::new( + config.format, + root.safe_root(), + config.record_limit, + config.prepared_close, + config.command_start_time, + config.command_end_time, + config.filter_command_with_retry, + &config.user_allowlist, + source_ordinal, + )?) + } else { + None + }; + for entry in entries { + input_files = input_files.saturating_add(1); + identity_parts.extend_from_slice(entry.path.as_bytes()); + identity_parts.extend_from_slice(&entry.content_length.to_le_bytes()); + identity_parts.push(0); + let bytes = root + .read_decoded(&entry, metadata.as_ref(), config.key_file.as_deref()) + .await?; + let content_identity = InputIdentity::from_canonical_bytes(&bytes); + identity_parts.extend_from_slice(content_identity.sha256.as_bytes()); + identity_parts.push(0); + match config.format { + TrafficFormat::Native => { + let mut decoder = NativeDecoder::new( + &bytes, + &entry.safe_path, + config.record_limit, + config.command_start_time, + ); + while let Some(mut command) = decoder.next_command()? { + command.source_ordinal = source_ordinal; + commands.push(command); + } + } + TrafficFormat::AuditLogPlugin | TrafficFormat::AuditLogExtension => { + let decoder = audit.as_mut().ok_or_else(|| { + ReplayError::Config("audit decoder was not constructed".to_owned()) + })?; + commands.extend(decoder.decode_all(&bytes)?); + } + } + } + } + identity_parts.extend_from_slice( + &serde_json::to_vec(&IdentityConfig::from(config)) + .map_err(|error| ReplayError::Checkpoint(format!("encode identity: {error}")))?, + ); + let input_identity = InputIdentity::from_canonical_bytes(&identity_parts); + let checkpoint = match config.checkpoint_path.as_deref() { + Some(path) => Checkpoint::load(path, &input_identity)?, + None => None, + }; + commands.sort_by(|left, right| { + left.start_time + .cmp(&right.start_time) + .then_with(|| left.connection_id.cmp(&right.connection_id)) + .then_with(|| left.source_ordinal.cmp(&right.source_ordinal)) + .then_with(|| left.record_ordinal.cmp(&right.record_ordinal)) + .then_with(|| left.command_ordinal.cmp(&right.command_ordinal)) + }); + if let Some(frontier) = checkpoint.as_ref() { + commands.retain(|command| { + ( + command.start_time.unix_timestamp_nanos(), + command.connection_id, + command.source_ordinal, + command.record_ordinal, + command.command_ordinal, + ) > ( + frontier.command_start_unix_nanos, + frontier.connection_id, + frontier.source_ordinal, + frontier.record_ordinal, + frontier.command_ordinal, + ) + }); + } + let decoded_commands = u64::try_from(commands.len()).unwrap_or(u64::MAX); + let mut filtered_commands = 0_u64; + let mut read_only_statements = HashMap::new(); + let mut last = None; + for command in &commands { + let accepted = !config.read_only + || command_is_read_only(command, &mut read_only_statements, config.record_limit)?; + if !accepted { + filtered_commands = filtered_commands.saturating_add(1); + } + last = Some(command); + } + let replayed_commands = decoded_commands.saturating_sub(filtered_commands); + if let (Some(path), Some(command)) = (config.checkpoint_path.as_deref(), last) { + let checkpoint = Checkpoint::new( + input_identity.clone(), + command.start_time.unix_timestamp_nanos(), + command + .end_time + .unwrap_or(command.start_time) + .unix_timestamp_nanos(), + command.source.clone(), + command.connection_id, + command.source_ordinal, + command.record_ordinal, + command.command_ordinal, + checkpoint + .as_ref() + .map_or(0, |value| value.committed_commands) + .saturating_add(replayed_commands), + checkpoint + .as_ref() + .map_or(0, |value| value.filtered_commands) + .saturating_add(filtered_commands), + ); + checkpoint.save_atomic(path)?; + } + Ok(DryRunSummary { + decoded_commands, + replayed_commands, + filtered_commands, + input_files, + last_command_time: last.and_then(|command| { + command + .start_time + .format(&time::format_description::well_known::Rfc3339) + .ok() + }), + input_identity, + }) +} + +#[derive(Serialize)] +struct IdentityConfig<'a> { + checkpoint_frontier_schema_version: u32, + format: TrafficFormat, + read_only: bool, + prepared_close: crate::PreparedCloseStrategy, + filter_command_with_retry: bool, + user_allowlist: &'a [String], + command_start_time: Option, + command_end_time: Option, + dynamic_input: bool, + replayer_count: u64, + replayer_index: u64, +} + +impl<'a> From<&'a ReplayConfig> for IdentityConfig<'a> { + fn from(config: &'a ReplayConfig) -> Self { + Self { + checkpoint_frontier_schema_version: CHECKPOINT_SCHEMA_VERSION, + format: config.format, + read_only: config.read_only, + prepared_close: config.prepared_close, + filter_command_with_retry: config.filter_command_with_retry, + user_allowlist: &config.user_allowlist, + command_start_time: config + .command_start_time + .map(time::OffsetDateTime::unix_timestamp_nanos), + command_end_time: config + .command_end_time + .map(time::OffsetDateTime::unix_timestamp_nanos), + dynamic_input: config.dynamic_input, + replayer_count: config.replayer_count, + replayer_index: config.replayer_index, + } + } +} + +fn command_is_read_only( + command: &Command, + prepared: &mut HashMap>, + limit: usize, +) -> Result { + match command.command { + CommandCode::QUERY | CommandCode::STMT_PREPARE => { + let sql = std::str::from_utf8(command.payload.get(1..).unwrap_or_default()).map_err( + |_| ReplayError::decode(&command.source, 0, "SQL payload is not valid UTF-8"), + )?; + let read_only = sql_is_read_only(sql, limit)?; + if command.command == CommandCode::STMT_PREPARE + && read_only + && let Some(statement_id) = command.captured_statement_id + { + prepared + .entry(command.connection_id) + .or_default() + .insert(statement_id); + } + Ok(read_only) + } + CommandCode::STMT_CLOSE => Ok(command.captured_statement_id.is_some_and(|statement_id| { + prepared + .get_mut(&command.connection_id) + .is_some_and(|statements| statements.remove(&statement_id)) + })), + CommandCode::STMT_EXECUTE + | CommandCode::STMT_SEND_LONG_DATA + | CommandCode::STMT_RESET + | CommandCode::STMT_FETCH => { + Ok(command.captured_statement_id.is_some_and(|statement_id| { + prepared + .get(&command.connection_id) + .is_some_and(|statements| statements.contains(&statement_id)) + })) + } + CommandCode::CREATE_DB | CommandCode::DROP_DB | CommandCode::DELAYED_INSERT => Ok(false), + CommandCode::QUIT => { + prepared.remove(&command.connection_id); + Ok(true) + } + _ => Ok(true), + } +} + +fn sql_is_read_only(sql: &str, limit: usize) -> Result { + let tokens = tokenize(sql, limit)?; + let first = tokens.first().map(String::as_str).unwrap_or_default(); + match first { + "SELECT" => Ok(!tokens + .windows(2) + .any(|pair| pair[0] == "FOR" && pair[1] == "UPDATE")), + "SHOW" | "WITH" | "USE" | "DESC" | "DESCRIBE" | "TABLE" | "DO" | "BEGIN" | "COMMIT" + | "ROLLBACK" => Ok(true), + "START" => Ok(tokens.get(1).map(String::as_str) == Some("TRANSACTION")), + "SET" => { + let second = tokens.get(1).map(String::as_str).unwrap_or_default(); + Ok(matches!( + second, + "SESSION_STATES" + | "SESSION" + | "NAMES" + | "CHAR" + | "CHARSET" + | "CHARACTER" + | "TRANSACTION" + ) || second.starts_with('@') && !second.starts_with("@@GLOBAL")) + } + _ => Ok(false), + } +} + +fn tokenize(sql: &str, limit: usize) -> Result, ReplayError> { + if sql.len() > limit { + return Err(ReplayError::Config("SQL exceeds record limit".to_owned())); + } + let bytes = sql.as_bytes(); + let mut tokens = Vec::new(); + let mut index = 0_usize; + while index < bytes.len() { + match bytes[index] { + byte if byte.is_ascii_whitespace() || matches!(byte, b'(' | b')' | b',' | b';') => { + index += 1; + } + b'#' => { + index = skip_line_comment(bytes, index + 1); + } + b'-' if bytes.get(index + 1) == Some(&b'-') => { + index = skip_line_comment(bytes, index + 2); + } + b'/' if bytes.get(index + 1) == Some(&b'*') => { + index = skip_block_comment(bytes, index + 2)?; + } + quote @ (b'\'' | b'"' | b'`') => { + index = skip_quoted(bytes, index + 1, quote)?; + } + _ => { + let start = index; + while index < bytes.len() + && !bytes[index].is_ascii_whitespace() + && !matches!( + bytes[index], + b'(' | b')' | b',' | b';' | b'\'' | b'"' | b'`' + ) + { + index += 1; + } + let token = sql.get(start..index).ok_or_else(|| { + ReplayError::Config("SQL token is not on UTF-8 boundaries".to_owned()) + })?; + tokens.push(token.to_ascii_uppercase()); + if tokens.len() > 1_000_000 { + return Err(ReplayError::Config("SQL token limit exceeded".to_owned())); + } + } + } + } + Ok(tokens) +} + +fn skip_line_comment(bytes: &[u8], mut index: usize) -> usize { + while index < bytes.len() && bytes[index] != b'\n' { + index += 1; + } + index +} + +fn skip_block_comment(bytes: &[u8], mut index: usize) -> Result { + while index + 1 < bytes.len() { + if bytes[index] == b'*' && bytes[index + 1] == b'/' { + return Ok(index + 2); + } + index += 1; + } + Err(ReplayError::Config( + "unterminated SQL block comment".to_owned(), + )) +} + +fn skip_quoted(bytes: &[u8], mut index: usize, quote: u8) -> Result { + while index < bytes.len() { + if bytes[index] == b'\\' { + index = index.saturating_add(2); + continue; + } + if bytes[index] == quote { + if bytes.get(index + 1) == Some("e) { + index += 2; + continue; + } + return Ok(index + 1); + } + index += 1; + } + Err(ReplayError::Config( + "unterminated SQL quoted construct".to_owned(), + )) +} + +#[cfg(test)] +#[allow(clippy::expect_used)] +mod tests { + use super::*; + use time::OffsetDateTime; + + #[test] + fn readonly_filter_ignores_literals_and_comments() { + assert!(sql_is_read_only("/* FOR UPDATE */ SELECT 'for update'", 1024).expect("classify")); + assert!(!sql_is_read_only("SELECT 1 FOR UPDATE", 1024).expect("classify")); + assert!(sql_is_read_only("SET @@session.foo=1", 1024).expect("classify")); + assert!(!sql_is_read_only("SET @@global.foo=1", 1024).expect("classify")); + } + + #[test] + fn malformed_sql_fails_closed() { + assert!(sql_is_read_only("SELECT 'oops", 1024).is_err()); + assert!(sql_is_read_only("/* oops", 1024).is_err()); + } + + #[test] + fn prepared_readonly_state_is_scoped_to_connection() { + let timestamp = OffsetDateTime::UNIX_EPOCH; + let mut prepared = HashMap::new(); + let mut prepare = Command::new(b"\x16SELECT ?".to_vec(), timestamp, 1, "fixture", 1) + .expect("prepare command"); + prepare.captured_statement_id = Some(7); + assert!(command_is_read_only(&prepare, &mut prepared, 1024).expect("classify prepare")); + + let mut other_connection_execute = + Command::new(vec![0x17, 7, 0, 0, 0], timestamp, 2, "fixture", 2) + .expect("execute command"); + other_connection_execute.captured_statement_id = Some(7); + assert!( + !command_is_read_only(&other_connection_execute, &mut prepared, 1024) + .expect("classify execute") + ); + + let mut same_connection_execute = + Command::new(vec![0x17, 7, 0, 0, 0], timestamp, 1, "fixture", 3) + .expect("execute command"); + same_connection_execute.captured_statement_id = Some(7); + assert!( + command_is_read_only(&same_connection_execute, &mut prepared, 1024) + .expect("classify execute") + ); + } +} diff --git a/rust/crates/replayer-core/src/error.rs b/rust/crates/replayer-core/src/error.rs new file mode 100644 index 000000000..7c2e0d70b --- /dev/null +++ b/rust/crates/replayer-core/src/error.rs @@ -0,0 +1,88 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Public, credential-redacted replay errors. + +use std::io; +use std::path::PathBuf; + +use thiserror::Error; + +/// Error returned by replay configuration, storage, decoding, or persistence. +#[derive(Debug, Error)] +pub enum ReplayError { + /// Invalid user configuration. + #[error("invalid replay configuration: {0}")] + Config(String), + /// A storage operation failed. The path is already credential-redacted. + #[error("storage {operation} failed for {path}: {message}")] + Storage { + /// Operation name. + operation: &'static str, + /// Safe logical path. + path: String, + /// Provider classification without endpoint credentials. + message: String, + }, + /// An input record was malformed. + #[error("decode failed at {path}:{offset}: {message}")] + Decode { + /// Safe logical input path. + path: String, + /// Zero-based byte offset. + offset: u64, + /// Failure detail. + message: String, + }, + /// A local filesystem operation failed. + #[error("local {operation} failed for {path:?}: {source}")] + Io { + /// Operation name. + operation: &'static str, + /// Local path. + path: PathBuf, + /// Underlying I/O failure. + #[source] + source: io::Error, + }, + /// A checkpoint cannot be used for this replay configuration. + #[error("checkpoint rejected: {0}")] + Checkpoint(String), +} + +impl ReplayError { + /// Builds a decode error with stable path and offset context. + #[must_use] + pub fn decode(path: impl Into, offset: usize, message: impl Into) -> Self { + Self::Decode { + path: path.into(), + offset: u64::try_from(offset).unwrap_or(u64::MAX), + message: message.into(), + } + } + + /// Builds a redacted storage error from an `OpenDAL` failure. + #[must_use] + pub fn storage( + operation: &'static str, + path: impl Into, + source: &opendal::Error, + ) -> Self { + Self::Storage { + operation, + path: path.into(), + message: source.kind().to_string(), + } + } +} diff --git a/rust/crates/replayer-core/src/lib.rs b/rust/crates/replayer-core/src/lib.rs new file mode 100644 index 000000000..0fc74b7dd --- /dev/null +++ b/rust/crates/replayer-core/src/lib.rs @@ -0,0 +1,32 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Bounded input, decoding, and checkpoint primitives for `TiProxy`'s offline +//! traffic replayer. + +#![forbid(unsafe_code)] + +pub mod checkpoint; +pub mod command; +pub mod config; +pub mod decode; +pub mod engine; +pub mod error; +pub mod storage; + +pub use checkpoint::{Checkpoint, InputIdentity}; +pub use command::{Command, CommandCode, PreparedCloseStrategy, TrafficFormat}; +pub use config::ReplayConfig; +pub use engine::{DryRunSummary, dry_run}; +pub use error::ReplayError; diff --git a/rust/crates/replayer-core/src/storage.rs b/rust/crates/replayer-core/src/storage.rs new file mode 100644 index 000000000..3d9571cbb --- /dev/null +++ b/rust/crates/replayer-core/src/storage.rs @@ -0,0 +1,633 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Credential-redacted `OpenDAL` input ownership. + +use std::collections::{HashMap, HashSet}; +use std::io::Read; +use std::path::{Path, PathBuf}; + +use aes::Aes256; +use ctr::cipher::{KeyIvInit, StreamCipher}; +use flate2::read::GzDecoder; +use futures_util::TryStreamExt; +use opendal::{ErrorKind, Operator, Scheme}; +use serde::{Deserialize, Serialize}; +use time::PrimitiveDateTime; +use url::Url; +use zeroize::Zeroizing; + +use crate::{ReplayError, TrafficFormat}; + +const MAX_STORED_FILE_BYTES: u64 = 512 * 1024 * 1024; +const MAX_EXPANDED_FILE_BYTES: u64 = 512 * 1024 * 1024; +const MAX_METADATA_BYTES: u64 = 64 * 1024; +const STORAGE_READ_CHUNK_BYTES: usize = 1024 * 1024; + +type Aes256Ctr = ctr::Ctr128BE; + +/// Native capture metadata. +#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "PascalCase")] +pub struct CaptureMetadata { + /// Format version. + pub version: String, + /// Capture duration in nanoseconds. + pub duration: i64, + /// Number of captured commands. + pub cmds: u64, + /// Number of capture-side filtered commands. + #[serde(default)] + pub filtered_cmds: u64, + /// Optional encryption method. + #[serde(default)] + pub encrypt_method: String, +} + +/// One replay input file returned in deterministic order. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct InputEntry { + /// Operator-relative object path. + pub path: String, + /// Credential-redacted logical path. + pub safe_path: String, + /// Stored content length. + pub content_length: u64, +} + +/// OpenDAL-backed read/list/stat owner for one input root. +#[derive(Clone)] +pub struct InputRoot { + operator: Operator, + safe_root: String, +} + +impl std::fmt::Debug for InputRoot { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("InputRoot") + .field("safe_root", &self.safe_root) + .finish_non_exhaustive() + } +} + +impl InputRoot { + /// Parses one local or supported object-store root. Query credentials are + /// consumed into provider configuration and never retained in diagnostics. + /// + /// # Errors + /// + /// Returns an error for invalid roots, unsupported providers, or provider + /// configuration failures. + pub fn open(input: &str) -> Result { + if !input.contains("://") { + return Self::open_local(Path::new(input)); + } + let parsed = Url::parse(input) + .map_err(|_| ReplayError::Config("input URL is invalid".to_owned()))?; + if parsed.scheme() == "file" { + let path = parsed + .to_file_path() + .map_err(|()| ReplayError::Config("file URL is invalid".to_owned()))?; + return Self::open_local(&path); + } + let scheme = match parsed.scheme() { + "s3" => Scheme::S3, + "gcs" | "gs" => Scheme::Gcs, + "azblob" | "azure" => Scheme::Azblob, + "oss" => Scheme::Oss, + "cos" => Scheme::Cos, + other => { + return Err(ReplayError::Config(format!( + "unsupported input scheme {other}" + ))); + } + }; + let authority = parsed + .host_str() + .filter(|host| !host.is_empty()) + .ok_or_else(|| ReplayError::Config("object-store URL has no bucket".to_owned()))?; + let mut config = HashMap::new(); + match scheme { + Scheme::Azblob => { + config.insert("container".to_owned(), authority.to_owned()); + } + _ => { + config.insert("bucket".to_owned(), authority.to_owned()); + } + } + let root = parsed.path().trim_matches('/'); + config.insert("root".to_owned(), format!("/{root}")); + let mut seen_options = HashSet::new(); + for (key, value) in parsed.query_pairs() { + let option = provider_query_option(scheme, &key, &value)?; + let normalized = option + .as_ref() + .map_or_else(|| "provider".to_owned(), |(key, _)| key.clone()); + if !seen_options.insert(normalized.clone()) { + return Err(ReplayError::Config(format!( + "duplicate object-store query option {normalized}" + ))); + } + if let Some((normalized, value)) = option { + config.insert(normalized, value); + } + } + let operator = Operator::via_iter(scheme, config) + .map_err(|error| ReplayError::storage("configure", safe_uri(&parsed), &error))?; + Ok(Self { + operator, + safe_root: safe_uri(&parsed), + }) + } + + fn open_local(path: &Path) -> Result { + let path = if path.is_absolute() { + path.to_path_buf() + } else { + std::env::current_dir() + .map_err(|source| ReplayError::Io { + operation: "resolve input root", + path: path.to_path_buf(), + source, + })? + .join(path) + }; + let root = path.canonicalize().map_err(|source| ReplayError::Io { + operation: "canonicalize input root", + path: path.clone(), + source, + })?; + let root_text = root + .to_str() + .ok_or_else(|| ReplayError::Config("input root is not valid UTF-8".to_owned()))? + .to_owned(); + let operator = Operator::via_iter(Scheme::Fs, [("root".to_owned(), root_text.clone())]) + .map_err(|error| ReplayError::storage("configure", &root_text, &error))?; + Ok(Self { + operator, + safe_root: root_text, + }) + } + + /// Reads optional native metadata. Missing metadata is compatible with Go. + /// + /// # Errors + /// + /// Returns an error when metadata storage or JSON decoding fails. + pub async fn metadata(&self) -> Result, ReplayError> { + match self.operator.stat("meta").await { + Ok(metadata) => { + if metadata.content_length() > MAX_METADATA_BYTES { + return Err(ReplayError::decode( + self.safe_path("meta"), + 0, + format!("metadata exceeds {MAX_METADATA_BYTES} bytes"), + )); + } + let buffer = self + .read_object_bounded("meta", &self.safe_path("meta"), MAX_METADATA_BYTES) + .await?; + let metadata: CaptureMetadata = + serde_json::from_slice(buffer.as_slice()).map_err(|error| { + ReplayError::decode( + self.safe_path("meta"), + 0, + format!("invalid meta JSON: {error}"), + ) + })?; + if metadata.version != "v1" { + return Err(ReplayError::decode( + self.safe_path("meta"), + 0, + format!("unsupported native capture version {}", metadata.version), + )); + } + Ok(Some(metadata)) + } + Err(error) if error.kind() == ErrorKind::NotFound => Ok(None), + Err(error) => Err(ReplayError::storage("stat", self.safe_path("meta"), &error)), + } + } + + /// Lists replay inputs recursively in timestamp/path order. + /// + /// # Errors + /// + /// Returns an error when listing or metadata lookup fails, or when an + /// input exceeds the stored-file bound. + #[allow(clippy::case_sensitive_file_extension_comparisons)] + pub async fn list(&self, format: TrafficFormat) -> Result, ReplayError> { + let prefix = if format.is_audit() { + "tidb-audit-" + } else { + "traffic-" + }; + let entries = self + .operator + .list_with("") + .recursive(true) + .await + .map_err(|error| ReplayError::storage("list", &self.safe_root, &error))?; + let mut output = Vec::new(); + for entry in entries { + let path = entry.path(); + if capture_timestamp(path, prefix).is_none() { + continue; + } + let metadata = self + .operator + .stat(path) + .await + .map_err(|error| ReplayError::storage("stat", self.safe_path(path), &error))?; + if metadata.is_file() && metadata.content_length() <= MAX_STORED_FILE_BYTES { + output.push(InputEntry { + path: path.to_owned(), + safe_path: self.safe_path(path), + content_length: metadata.content_length(), + }); + } else if metadata.is_file() { + return Err(ReplayError::decode( + self.safe_path(path), + 0, + format!("stored file exceeds {MAX_STORED_FILE_BYTES} bytes"), + )); + } + } + output.sort_by(|left, right| { + file_timestamp_key(&left.path) + .cmp(file_timestamp_key(&right.path)) + .then_with(|| left.path.cmp(&right.path)) + }); + Ok(output) + } + + /// Reads, decompresses, and decrypts one bounded input file. + /// + /// # Errors + /// + /// Returns an error when the object cannot be read, expanded within its + /// bound, or decrypted with the configured metadata and key. + #[allow(clippy::case_sensitive_file_extension_comparisons)] + pub async fn read_decoded( + &self, + entry: &InputEntry, + metadata: Option<&CaptureMetadata>, + key_file: Option<&Path>, + ) -> Result, ReplayError> { + let stored = self + .read_object_bounded(&entry.path, &entry.safe_path, MAX_STORED_FILE_BYTES) + .await?; + let expanded = if entry.path.ends_with(".gz") { + let mut decoder = GzDecoder::new(stored.as_slice()); + read_bounded(&mut decoder, MAX_EXPANDED_FILE_BYTES, &entry.safe_path)? + } else { + stored + }; + decrypt(expanded, metadata, key_file, &entry.safe_path) + } + + /// Credential-redacted root identifier. + #[must_use] + pub fn safe_root(&self) -> &str { + &self.safe_root + } + + fn safe_path(&self, path: &str) -> String { + if path.is_empty() { + self.safe_root.clone() + } else { + format!("{}/{path}", self.safe_root.trim_end_matches('/')) + } + } + + async fn read_object_bounded( + &self, + path: &str, + safe_path: &str, + limit: u64, + ) -> Result, ReplayError> { + let reader = self + .operator + .reader_with(path) + .chunk(STORAGE_READ_CHUNK_BYTES) + .await + .map_err(|error| ReplayError::storage("read", safe_path, &error))?; + let mut stream = reader + .into_stream(..) + .await + .map_err(|error| ReplayError::storage("read", safe_path, &error))?; + let mut output = Vec::new(); + while let Some(buffer) = stream + .try_next() + .await + .map_err(|error| ReplayError::storage("read", safe_path, &error))? + { + let next_len = output + .len() + .checked_add(buffer.len()) + .ok_or_else(|| ReplayError::decode(safe_path, 0, "stored size overflow"))?; + reject_oversize(next_len, limit, safe_path, "stored object")?; + for bytes in buffer { + output.extend_from_slice(&bytes); + } + } + Ok(output) + } +} + +fn reject_oversize(actual: usize, limit: u64, path: &str, kind: &str) -> Result<(), ReplayError> { + if u64::try_from(actual).unwrap_or(u64::MAX) > limit { + return Err(ReplayError::decode( + path, + 0, + format!("{kind} exceeds {limit} bytes"), + )); + } + Ok(()) +} + +fn read_bounded(reader: &mut impl Read, limit: u64, path: &str) -> Result, ReplayError> { + let mut output = Vec::new(); + reader + .take(limit.saturating_add(1)) + .read_to_end(&mut output) + .map_err(|source| ReplayError::Io { + operation: "decompress input", + path: PathBuf::from(path), + source, + })?; + if u64::try_from(output.len()).unwrap_or(u64::MAX) > limit { + return Err(ReplayError::decode( + path, + 0, + format!("expanded file exceeds {limit} bytes"), + )); + } + Ok(output) +} + +fn decrypt( + mut input: Vec, + metadata: Option<&CaptureMetadata>, + key_file: Option<&Path>, + path: &str, +) -> Result, ReplayError> { + let method = metadata.map_or("", |metadata| metadata.encrypt_method.as_str()); + match method.to_ascii_lowercase().as_str() { + "" | "plaintext" => Ok(input), + "aes256-ctr" => { + let key_path = key_file.ok_or_else(|| { + ReplayError::Config("security.encryption-key-path is not set".to_owned()) + })?; + let key = + Zeroizing::new(std::fs::read(key_path).map_err(|source| ReplayError::Io { + operation: "read encryption key", + path: key_path.to_path_buf(), + source, + })?); + if key.len() < 32 { + return Err(ReplayError::Config(format!( + "invalid AES-256 key length {}, expecting at least 32", + key.len() + ))); + } + if input.len() < 16 { + return Err(ReplayError::decode( + path, + 0, + "encrypted input has a short IV", + )); + } + let (iv, body) = input.split_at_mut(16); + let mut cipher = Aes256Ctr::new_from_slices(&key[..32], iv) + .map_err(|_| ReplayError::Config("invalid AES-256 key or IV".to_owned()))?; + cipher.apply_keystream(body); + Ok(body.to_vec()) + } + other => Err(ReplayError::Config(format!( + "unsupported encrypt method {other}" + ))), + } +} + +fn provider_query_option( + scheme: Scheme, + key: &str, + value: &str, +) -> Result, ReplayError> { + let canonical = key.to_ascii_lowercase().replace('-', "_"); + if canonical == "provider" && matches!(scheme, Scheme::S3 | Scheme::Oss) { + return Ok(None); + } + if scheme == Scheme::S3 && canonical == "force_path_style" { + let force_path_style = value.parse::().map_err(|_| { + ReplayError::Config("force-path-style must be true or false".to_owned()) + })?; + return Ok(Some(( + "enable_virtual_host_style".to_owned(), + (!force_path_style).to_string(), + ))); + } + let normalized = match (scheme, canonical.as_str()) { + ( + Scheme::S3, + "endpoint" | "region" | "secret_access_key" | "session_token" | "role_arn" + | "external_id" | "profile", + ) + | (Scheme::Azblob, "endpoint" | "account_name" | "account_key" | "sas_token") => { + canonical.as_str() + } + (Scheme::S3 | Scheme::Oss, "access_key" | "access_key_id") => "access_key_id", + (Scheme::Oss | Scheme::Cos | Scheme::Gcs, "endpoint") => "endpoint", + (Scheme::Oss, "secret_access_key" | "access_key_secret") => "access_key_secret", + (Scheme::Cos, "access_key" | "access_key_id" | "secret_id") => "secret_id", + (Scheme::Cos, "secret_access_key" | "secret_key") => "secret_key", + (Scheme::Gcs, "credentials_file") => "credential_path", + _ => { + return Err(ReplayError::Config(format!( + "unsupported object-store query option {canonical} for {scheme:?}" + ))); + } + }; + Ok(Some((normalized.to_owned(), value.to_owned()))) +} + +fn safe_uri(url: &Url) -> String { + let host = url.host_str().unwrap_or_default(); + format!("{}://{}{}", url.scheme(), host, url.path()) +} + +fn file_timestamp_key(path: &str) -> &str { + let name = path.rsplit('/').next().unwrap_or(path); + name.strip_prefix("traffic-") + .or_else(|| name.strip_prefix("tidb-audit-")) + .and_then(|value| value.strip_suffix(".gz").or(Some(value))) + .and_then(|value| value.strip_suffix(".log")) + .unwrap_or(name) +} + +fn capture_timestamp<'a>(name: &'a str, prefix: &str) -> Option<&'a str> { + let name = name.strip_prefix(prefix)?; + let name = name.strip_suffix(".gz").unwrap_or(name); + let timestamp = name.strip_suffix(".log")?; + if let Some((_, fraction)) = timestamp.split_once('.') + && (fraction.is_empty() + || fraction.len() > 3 + || !fraction.bytes().all(|byte| byte.is_ascii_digit())) + { + return None; + } + let description = if timestamp.contains('.') { + "[year]-[month]-[day]T[hour]-[minute]-[second].[subsecond]" + } else { + "[year]-[month]-[day]T[hour]-[minute]-[second]" + }; + let format = time::format_description::parse_borrowed::<2>(description).ok()?; + PrimitiveDateTime::parse(timestamp, &format).ok()?; + Some(timestamp) +} + +#[cfg(test)] +#[allow(clippy::expect_used)] +mod tests { + use super::*; + + #[tokio::test] + async fn local_list_is_timestamp_then_path_ordered() { + let directory = + std::env::temp_dir().join(format!("tiproxy-replayer-storage-{}", std::process::id())); + std::fs::create_dir_all(directory.join("nested")).expect("create directory"); + std::fs::write(directory.join("traffic-2026-01-01T00-00-01.000.log"), b"a") + .expect("write file"); + std::fs::write( + directory + .join("nested") + .join("traffic-2026-01-01T00-00-00.000.log"), + b"ignored nested input", + ) + .expect("write file"); + std::fs::write(directory.join("traffic-2026-01-01T00-00-00.000.log"), b"b") + .expect("write file"); + std::fs::write(directory.join("traffic-2026-01-01T00-00-02.log"), b"c") + .expect("write file"); + std::fs::write(directory.join("traffic-not-a-time.log"), b"bad") + .expect("write invalid file"); + std::fs::write(directory.join("traffic-2026-99-01T00-00-03.log"), b"bad") + .expect("write invalid date"); + std::fs::write(directory.join("ignore.txt"), b"x").expect("write ignored file"); + let root = InputRoot::open(directory.to_str().expect("UTF-8 path")).expect("open root"); + let files = root.list(TrafficFormat::Native).await.expect("list files"); + assert_eq!(files.len(), 3); + assert!(files[0].path.contains("00-00-00.000")); + assert!(files[1].path.contains("00-00-01.000")); + assert!(files[2].path.contains("00-00-02.log")); + std::fs::remove_dir_all(directory).expect("remove directory"); + } + + #[tokio::test] + async fn native_metadata_rejects_unknown_version() { + let directory = + std::env::temp_dir().join(format!("tiproxy-replayer-meta-{}", std::process::id())); + if directory.exists() { + std::fs::remove_dir_all(&directory).expect("remove stale directory"); + } + std::fs::create_dir_all(&directory).expect("create directory"); + std::fs::write( + directory.join("meta"), + br#"{"Version":"v2","Duration":0,"Cmds":0}"#, + ) + .expect("write metadata"); + let root = InputRoot::open(directory.to_str().expect("UTF-8 path")).expect("open root"); + let error = root.metadata().await.expect_err("unknown version"); + assert!( + error + .to_string() + .contains("unsupported native capture version v2") + ); + std::fs::remove_dir_all(directory).expect("remove directory"); + } + + #[test] + fn provider_options_map_to_native_opendal_fields() { + assert_eq!( + provider_query_option(Scheme::S3, "force-path-style", "true").expect("option"), + Some(("enable_virtual_host_style".to_owned(), "false".to_owned())) + ); + assert_eq!( + provider_query_option(Scheme::Oss, "secret-access-key", "secret").expect("option"), + Some(("access_key_secret".to_owned(), "secret".to_owned())) + ); + assert_eq!( + provider_query_option(Scheme::Cos, "access-key", "id").expect("option"), + Some(("secret_id".to_owned(), "id".to_owned())) + ); + assert_eq!( + provider_query_option(Scheme::Gcs, "credentials-file", "/tmp/key").expect("option"), + Some(("credential_path".to_owned(), "/tmp/key".to_owned())) + ); + assert_eq!( + provider_query_option(Scheme::S3, "provider", "minio").expect("option"), + None + ); + assert!(provider_query_option(Scheme::S3, "force-path-style", "maybe").is_err()); + assert!(provider_query_option(Scheme::S3, "root", "/override").is_err()); + assert!(provider_query_option(Scheme::S3, "bucket", "override").is_err()); + assert!(provider_query_option(Scheme::S3, "unknown", "value").is_err()); + assert!(provider_query_option(Scheme::Gcs, "provider", "ignored").is_err()); + } + + #[test] + fn object_store_query_rejects_alias_duplicates_and_root_override() { + let duplicate = InputRoot::open("s3://bucket/prefix?access-key=a&access_key_id=b") + .expect_err("aliases must not use last-wins semantics"); + assert!( + duplicate + .to_string() + .contains("duplicate object-store query option") + ); + + let override_root = InputRoot::open("s3://bucket/prefix?root=/elsewhere") + .expect_err("root override must fail closed"); + assert!( + override_root + .to_string() + .contains("unsupported object-store query option root") + ); + + let override_container = InputRoot::open("azblob://container/prefix?container=other") + .expect_err("container override must fail closed"); + assert!( + override_container + .to_string() + .contains("unsupported object-store query option container") + ); + } + + #[test] + fn uri_diagnostics_drop_credentials() { + let parsed = + Url::parse("s3://bucket/prefix?access-key=secret-one&secret-access-key=secret-two") + .expect("URL"); + let safe = safe_uri(&parsed); + assert_eq!(safe, "s3://bucket/prefix"); + assert!(!safe.contains("secret")); + } + + #[test] + fn post_read_bounds_reject_object_growth() { + assert!(reject_oversize(65, 64, "traffic.log", "stored file").is_err()); + assert!(reject_oversize(64, 64, "traffic.log", "stored file").is_ok()); + } +} diff --git a/rust/crates/replayer-core/tests/dry_run.rs b/rust/crates/replayer-core/tests/dry_run.rs new file mode 100644 index 000000000..6a6432878 --- /dev/null +++ b/rust/crates/replayer-core/tests/dry_run.rs @@ -0,0 +1,190 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! End-to-end dry-run storage and checkpoint contract tests. + +use std::error::Error; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; + +use aes::Aes256; +use ctr::cipher::{KeyIvInit, StreamCipher}; +use flate2::Compression; +use flate2::write::GzEncoder; +use replayer_core::config::DEFAULT_RECORD_LIMIT; +use replayer_core::{Checkpoint, PreparedCloseStrategy, ReplayConfig, TrafficFormat, dry_run}; +use time::OffsetDateTime; + +type TestResult = Result<(), Box>; +type Aes256Ctr = ctr::Ctr128BE; + +static DIRECTORY_SEQUENCE: AtomicU64 = AtomicU64::new(0); + +fn temporary_directory(name: &str) -> Result { + let sequence = DIRECTORY_SEQUENCE.fetch_add(1, Ordering::Relaxed); + let path = std::env::temp_dir().join(format!( + "tiproxy-replayer-{name}-{}-{sequence}", + std::process::id() + )); + std::fs::create_dir_all(&path)?; + Ok(path) +} + +fn native_record(timestamp: &str, connection_id: u64, sql: &str) -> Vec { + format!( + "# Time: {timestamp}\n# Conn_ID: {connection_id}\n# Payload_len: {}\n{sql}\n", + sql.len() + ) + .into_bytes() +} + +fn dry_run_config(input: &Path, checkpoint: Option) -> ReplayConfig { + ReplayConfig { + input: input.to_string_lossy().into_owned(), + speed: 1.0, + username: "root".to_owned(), + address: "127.0.0.1:4000".to_owned(), + format: TrafficFormat::Native, + read_only: false, + start_time: OffsetDateTime::now_utc(), + ignore_errors: false, + reorder_buffer: 100_000, + prepared_close: PreparedCloseStrategy::Directed, + dry_run: true, + checkpoint_path: checkpoint, + dynamic_input: false, + replayer_count: 1, + replayer_index: 0, + output_path: None, + filter_command_with_retry: false, + user_allowlist: Vec::new(), + wait_on_eof: false, + command_start_time: None, + command_end_time: None, + key_file: None, + record_limit: DEFAULT_RECORD_LIMIT, + } +} + +#[tokio::test] +async fn checkpoint_uses_full_tie_frontier_and_content_identity() -> TestResult { + let directory = temporary_directory("checkpoint")?; + let traffic = directory.join("traffic-2026-01-08T19-44-11.099.log"); + let checkpoint_path = directory.join("checkpoint.json"); + let timestamp = "2026-01-08T19:44:11.099+08:00"; + let mut input = native_record(timestamp, 2, "SELECT 1"); + input.extend(native_record(timestamp, 1, "SELECT 1")); + std::fs::write(&traffic, &input)?; + + let config = dry_run_config(&directory, Some(checkpoint_path.clone())); + let first = dry_run(&config).await?; + assert_eq!(first.decoded_commands, 2); + assert_eq!(first.replayed_commands, 2); + let checkpoint = Checkpoint::load(&checkpoint_path, &first.input_identity)? + .ok_or("checkpoint was not persisted")?; + assert_eq!(checkpoint.connection_id, 2); + assert_eq!(checkpoint.source_ordinal, 0); + assert_eq!(checkpoint.record_ordinal, 1); + assert_eq!(checkpoint.command_ordinal, 0); + + let resumed = dry_run(&config).await?; + assert_eq!(resumed.decoded_commands, 0); + assert_eq!(resumed.replayed_commands, 0); + + let mut changed = native_record(timestamp, 2, "SELECT 2"); + changed.extend(native_record(timestamp, 1, "SELECT 1")); + assert_eq!(changed.len(), input.len()); + std::fs::write(&traffic, changed)?; + let mismatch = dry_run(&config).await; + assert!(mismatch.is_err()); + assert!( + mismatch + .err() + .is_some_and(|error| error.to_string().contains("input identity")) + ); + + std::fs::remove_dir_all(directory)?; + Ok(()) +} + +#[tokio::test] +async fn checkpoint_frontier_distinguishes_commands_expanded_from_one_record() -> TestResult { + let directory = temporary_directory("checkpoint-command-ordinal")?; + let traffic = directory.join("tidb-audit-2026-01-08T19-44-11.099.log"); + let checkpoint_path = directory.join("checkpoint.json"); + std::fs::write( + &traffic, + concat!( + "[2026/01/08 19:44:11.114 +08:00] ", + "[TIMESTAMP=2026/01/08 19:44:11.114 +08:00] ", + "[EVENT_CLASS=GENERAL] [EVENT=COMPLETED] [COMMAND=Execute] ", + "[SQL_TEXT=SELECT ?] [SQL_STATEMENTS=Select] [CONNECTION_ID=7] ", + "[EXECUTE_PARAMS=\"[]\"] [CURRENT_DB=test]\n" + ), + )?; + + let mut config = dry_run_config(&directory, Some(checkpoint_path.clone())); + config.format = TrafficFormat::AuditLogPlugin; + config.prepared_close = PreparedCloseStrategy::Always; + let first = dry_run(&config).await?; + assert_eq!(first.decoded_commands, 3); + let checkpoint = Checkpoint::load(&checkpoint_path, &first.input_identity)? + .ok_or("checkpoint was not persisted")?; + assert_eq!(checkpoint.record_ordinal, 1); + assert_eq!(checkpoint.command_ordinal, 2); + + let resumed = dry_run(&config).await?; + assert_eq!(resumed.decoded_commands, 0); + assert_eq!(resumed.replayed_commands, 0); + + std::fs::remove_dir_all(directory)?; + Ok(()) +} + +#[tokio::test] +async fn gzip_then_aes_ctr_matches_native_storage_layers() -> TestResult { + let directory = temporary_directory("encrypted")?; + let key_path = directory.join("key"); + let key = [0x5a_u8; 32]; + std::fs::write(&key_path, key)?; + std::fs::write( + directory.join("meta"), + br#"{"Version":"v1","Duration":1,"Cmds":1,"EncryptMethod":"aes256-ctr"}"#, + )?; + + let plaintext = native_record("2026-01-08T19:44:11.099+08:00", 1, "SELECT 1"); + let iv = [0x24_u8; 16]; + let mut encrypted = plaintext; + let mut cipher = Aes256Ctr::new_from_slices(&key, &iv)?; + cipher.apply_keystream(&mut encrypted); + let mut stored = iv.to_vec(); + stored.extend(encrypted); + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(&stored)?; + std::fs::write( + directory.join("traffic-2026-01-08T19-44-11.099.log.gz"), + encoder.finish()?, + )?; + + let mut config = dry_run_config(&directory, None); + config.key_file = Some(key_path); + let summary = dry_run(&config).await?; + assert_eq!(summary.input_files, 1); + assert_eq!(summary.decoded_commands, 1); + assert_eq!(summary.replayed_commands, 1); + + std::fs::remove_dir_all(directory)?; + Ok(()) +} diff --git a/rust/crates/tiproxy-replayer/Cargo.toml b/rust/crates/tiproxy-replayer/Cargo.toml new file mode 100644 index 000000000..7fc96b12c --- /dev/null +++ b/rust/crates/tiproxy-replayer/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "tiproxy-replayer" +description = "TiProxy standalone offline traffic replayer" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true + +[features] +default = [] + +[dependencies] +clap.workspace = true +replayer-core.workspace = true +serde_json.workspace = true +time.workspace = true +tokio = { workspace = true, features = ["signal"] } +zeroize.workspace = true + +[lints] +workspace = true diff --git a/rust/crates/tiproxy-replayer/src/main.rs b/rust/crates/tiproxy-replayer/src/main.rs new file mode 100644 index 000000000..263ffc8a0 --- /dev/null +++ b/rust/crates/tiproxy-replayer/src/main.rs @@ -0,0 +1,184 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Standalone composition root for the Rust offline replayer. + +#![forbid(unsafe_code)] + +use std::path::PathBuf; +use std::process::ExitCode; + +use clap::Parser; +use replayer_core::config::DEFAULT_RECORD_LIMIT; +use replayer_core::{PreparedCloseStrategy, ReplayConfig, TrafficFormat, dry_run}; +use time::OffsetDateTime; +use time::format_description::well_known::Rfc3339; +use zeroize::Zeroizing; + +/// Standalone `TiProxy` traffic replayer. +#[derive(Parser)] +#[command(name = "tiproxy-replayer", version, about)] +#[allow(clippy::struct_excessive_bools)] +struct Options { + /// Downstream `TiDB` address. + #[arg(long, default_value = "127.0.0.1:4000")] + addr: String, + /// Directory or object-store root containing traffic files. + #[arg(long, default_value = "")] + input: String, + /// Replay speed multiplier. + #[arg(long, default_value_t = 1.0)] + speed: f64, + /// `TiDB` username. + #[arg(long, default_value = "root")] + username: String, + /// `TiDB` password. Never included in debug or output. + #[arg(long, default_value = "")] + password: String, + /// Replay only read-only statements. + #[arg(long)] + read_only: bool, + /// Input format: `native`, `audit_log_plugin`, or `audit_log_extension`. + #[arg(long, default_value = "")] + format: TrafficFormat, + /// Log output path. + #[arg(long, default_value = "")] + log_file: String, + /// Skip commands at or before this RFC3339 start time. + #[arg(long, value_parser = parse_timestamp)] + command_start_time: Option, + /// Skip audit commands ending before this RFC3339 time. + #[arg(long, value_parser = parse_timestamp)] + command_end_time: Option, + /// Ignore recoverable replay command errors. + #[arg(long)] + ignore_errs: bool, + /// Audit reorder buffer size; zero disables reordering. + #[arg(long, default_value_t = 100_000)] + bufsize: usize, + /// Optional pprof/API listen address retained for CLI compatibility. + #[arg(long, default_value = "")] + pprof_addr: String, + /// Prepared close strategy: directed, always, or never. + #[arg(long, default_value = "directed")] + ps_close: PreparedCloseStrategy, + /// Decode and filter without connecting to `TiDB`. + #[arg(long)] + dry_run: bool, + /// Local atomic checkpoint path. + #[arg(long)] + checkpoint_path: Option, + /// Watch for newly published traffic directories. + #[arg(long)] + dynamic_input: bool, + /// Total dynamic-input replayer count. + #[arg(long, default_value_t = 1)] + replayer_count: u64, + /// This dynamic-input replayer index. + #[arg(long, default_value_t = 0)] + replayer_index: u64, + /// Optional local replayed-SQL output path. + #[arg(long)] + output_path: Option, + /// Run the replay-only HTTP service. + #[arg(long)] + service_mode: bool, + /// Log level retained for CLI compatibility. + #[arg(long, default_value = "info")] + log_level: String, + /// Absolute RFC3339 replay start time; defaults to process start. + #[arg(long, value_parser = parse_timestamp)] + start_time: Option, + /// Filter audit commands marked as retries. + #[arg(long)] + filter_command_with_retry: bool, + /// Case-insensitive audit user allowlist. + #[arg(long, value_delimiter = ',')] + user_allowlist: Vec, + /// Wait for new files after EOF. + #[arg(long)] + wait_on_eof: bool, + /// AES-256 key file for encrypted native captures. + #[arg(long)] + key_file: Option, +} + +#[tokio::main] +async fn main() -> ExitCode { + let options = Options::parse(); + match run(options).await { + Ok(()) => ExitCode::SUCCESS, + Err(error) => { + eprintln!("tiproxy-replayer stopped: {error}"); + ExitCode::from(1) + } + } +} + +async fn run(options: Options) -> Result<(), String> { + let now = OffsetDateTime::now_utc(); + drop(Zeroizing::new(options.password)); + drop((options.log_file, options.pprof_addr, options.log_level)); + if options.service_mode { + return Err("service mode is not enabled in the current implementation slice".to_owned()); + } + if !options.dry_run { + return Err( + "backend execution is not enabled in the current implementation slice; use --dry-run" + .to_owned(), + ); + } + if options.dynamic_input || options.wait_on_eof { + return Err("dynamic input is not enabled in the current implementation slice".to_owned()); + } + if options.output_path.is_some() { + return Err("SQL output is not enabled in the current implementation slice".to_owned()); + } + let mut config = ReplayConfig { + input: options.input, + speed: options.speed, + username: options.username, + address: options.addr, + format: options.format, + read_only: options.read_only, + start_time: options.start_time.unwrap_or(now), + ignore_errors: options.ignore_errs, + reorder_buffer: options.bufsize, + prepared_close: options.ps_close, + dry_run: options.dry_run, + checkpoint_path: options.checkpoint_path, + dynamic_input: options.dynamic_input, + replayer_count: options.replayer_count, + replayer_index: options.replayer_index, + output_path: options.output_path, + filter_command_with_retry: options.filter_command_with_retry, + user_allowlist: options.user_allowlist, + wait_on_eof: options.wait_on_eof, + command_start_time: options.command_start_time, + command_end_time: options.command_end_time, + key_file: options.key_file, + record_limit: DEFAULT_RECORD_LIMIT, + }; + config.validate(now).map_err(|error| error.to_string())?; + let summary = dry_run(&config).await.map_err(|error| error.to_string())?; + println!( + "{}", + serde_json::to_string_pretty(&summary).map_err(|error| error.to_string())? + ); + Ok(()) +} + +fn parse_timestamp(value: &str) -> Result { + OffsetDateTime::parse(value, &Rfc3339).map_err(|error| error.to_string()) +} diff --git a/rust/deny.toml b/rust/deny.toml index fe14100b8..e3b6db608 100644 --- a/rust/deny.toml +++ b/rust/deny.toml @@ -6,7 +6,16 @@ targets = [ all-features = true [advisories] -ignore = [] +# OpenDAL 0.53.3's S3/GCS signers retain these transitive advisories. This +# foundation is an operator-run, dry-run-only binary with trusted storage +# endpoints; service mode and backend execution fail closed. Revisit the pin +# before either deferred mode is enabled. OpenDAL 0.58 fixes quick-xml but has +# MSRV 1.91, above this workspace's 1.89 toolchain. +ignore = [ + "RUSTSEC-2023-0071", + "RUSTSEC-2026-0194", + "RUSTSEC-2026-0195", +] [licenses] allow = [ diff --git a/rust/vulnerability-exceptions.toml b/rust/vulnerability-exceptions.toml index a0757b6df..826877d35 100644 --- a/rust/vulnerability-exceptions.toml +++ b/rust/vulnerability-exceptions.toml @@ -9,3 +9,21 @@ schema_version = 1 # owner = "@github-handle-or-team" # rationale = "Why exposure is bounded and what remediation is in progress." # expires = 2026-12-31 + +[[exceptions]] +advisory = "RUSTSEC-2023-0071" +owner = "@bb7133" +rationale = "OpenDAL 0.53.3 reaches rsa through request signing, but this slice is operator-run dry-run-only with trusted storage endpoints; upgrade OpenDAL before enabling service or backend modes." +expires = 2026-12-01 + +[[exceptions]] +advisory = "RUSTSEC-2026-0194" +owner = "@bb7133" +rationale = "OpenDAL 0.53.3 reaches quick-xml through cloud provider support, but this slice is operator-run dry-run-only with trusted storage endpoints; upgrade OpenDAL before enabling service or backend modes." +expires = 2026-12-01 + +[[exceptions]] +advisory = "RUSTSEC-2026-0195" +owner = "@bb7133" +rationale = "OpenDAL 0.53.3 reaches quick-xml through cloud provider support, but this slice is operator-run dry-run-only with trusted storage endpoints; upgrade OpenDAL before enabling service or backend modes." +expires = 2026-12-01 diff --git a/tests/replayer/differential/corpus.v1.json b/tests/replayer/differential/corpus.v1.json new file mode 100644 index 000000000..e3d4d9d35 --- /dev/null +++ b/tests/replayer/differential/corpus.v1.json @@ -0,0 +1,98 @@ +{ + "cases": [ + { + "id": "native-binary", + "format": "native", + "prepared_close": "directed", + "filter_retries": false, + "user_allowlist": [], + "source_ordinal": 0, + "records": [ + { + "headers": [ + ["Future", "one"], + ["Future", "two"], + ["Time", "2026-01-08T19:44:11.099+08:00"], + ["Conn_ID", "42"] + ], + "payload_hex": "53454c4543542031" + }, + { + "headers": [ + ["Time", "2026-01-08T19:44:11.100+08:00"], + ["Conn_ID", "42"], + ["Cmd_type", "StmtPrepare"], + ["Captured_ps_id", "7"] + ], + "payload_hex": "53454c454354203f" + }, + { + "headers": [ + ["Time", "2026-01-08T19:44:11.101+08:00"], + ["Conn_ID", "42"], + ["Cmd_type", "StmtExecute"], + ["Captured_ps_id", "7"], + ["Prepared_stmt", "\"SELECT ?\""] + ], + "payload_hex": "070000000001000000" + } + ], + "lines": [] + }, + { + "id": "plugin-allocation-and-dedup", + "format": "audit_log_plugin", + "prepared_close": "directed", + "filter_retries": false, + "user_allowlist": ["ROOT"], + "source_ordinal": 1, + "records": [], + "lines": [ + "[2025/09/06 17:03:53.719 +08:00] [TIMESTAMP=2025/09/06 17:03:53.719 +08:00] [EVENT_CLASS=GENERAL] [EVENT=STARTING] [COMMAND=Query] [SQL_TEXT=SELECT 0] [SQL_STATEMENTS=Select] [CONNECTION_ID=40] [USER=root]", + "[2025/09/06 17:03:53.720 +08:00] [TIMESTAMP=2025/09/06 17:03:53.720 +08:00] [EVENT_CLASS=GENERAL] [EVENT=COMPLETED] [COMMAND=Query] [SQL_TEXT=INSERT INTO t VALUES (1)] [SQL_STATEMENTS=Insert] [CONNECTION_ID=9] [USER=root] [CURRENT_DB=test]", + "[2025/09/06 17:03:53.720 +08:00] [TIMESTAMP=2025/09/06 17:03:53.720 +08:00] [EVENT_CLASS=GENERAL] [EVENT=COMPLETED] [COMMAND=Query] [SQL_TEXT=INSERT INTO t VALUES (1)] [SQL_STATEMENTS=Insert] [CONNECTION_ID=9] [USER=root] [CURRENT_DB=test]", + "[2025/09/06 17:03:53.720 +08:00] [TIMESTAMP=2025/09/06 17:03:53.720 +08:00] [EVENT_CLASS=GENERAL] [EVENT=COMPLETED] [COMMAND=Query] [SQL_TEXT=SELECT 1] [SQL_STATEMENTS=Select] [CONNECTION_ID=9] [USER=root] [CURRENT_DB=test]", + "[2025/09/06 17:03:53.720 +08:00] [TIMESTAMP=2025/09/06 17:03:53.720 +08:00] [EVENT_CLASS=GENERAL] [EVENT=COMPLETED] [COMMAND=Query] [SQL_TEXT=INSERT INTO t VALUES (1)] [SQL_STATEMENTS=Insert] [CONNECTION_ID=9] [USER=root] [CURRENT_DB=test]" + ] + }, + { + "id": "plugin-retry-and-close", + "format": "audit_log_plugin", + "prepared_close": "directed", + "filter_retries": true, + "user_allowlist": [], + "source_ordinal": 2, + "records": [], + "lines": [ + "[2025/09/06 17:03:53.721 +08:00] [TIMESTAMP=2025/09/06 17:03:53.721 +08:00] [EVENT_CLASS=GENERAL] [EVENT=COMPLETED] [COMMAND=Execute] [SQL_TEXT=SELECT ?] [SQL_STATEMENTS=Select] [CONNECTION_ID=9] [PREPARED_STMT_ID=7] [EXECUTE_PARAMS=\"[]\"] [RETRY=false]", + "[2025/09/06 17:03:53.722 +08:00] [TIMESTAMP=2025/09/06 17:03:53.722 +08:00] [EVENT_CLASS=GENERAL] [EVENT=COMPLETED] [COMMAND=\"Close stmt\"] [CONNECTION_ID=9] [PREPARED_STMT_ID=7] [RETRY=true]", + "[2025/09/06 17:03:53.723 +08:00] [TIMESTAMP=2025/09/06 17:03:53.723 +08:00] [EVENT_CLASS=CONNECTION] [EVENT_SUBCLASS=Disconnect] [CONNECTION_ID=9]" + ] + }, + { + "id": "extension-prepare-execute-close", + "format": "audit_log_extension", + "prepared_close": "always", + "filter_retries": false, + "user_allowlist": [], + "source_ordinal": 3, + "records": [], + "lines": [ + "[2026/01/08 19:44:11.114 +08:00] [EVENT=\"[QUERY,EXECUTE,SELECT]\"] [USER=root] [CONNECTION_ID=7] [CURRENT_DB=test] [SQL_TEXT=\"SELECT ?\"] [EXECUTE_PARAMS=\"[1]\"]", + "[2026/01/08 19:44:11.115 +08:00] [EVENT=\"[CONNECTION,DISCONNECT]\"] [USER=root] [CONNECTION_ID=7]" + ] + }, + { + "id": "plugin-binary-kind-bytes", + "format": "audit_log_plugin", + "prepared_close": "always", + "filter_retries": false, + "user_allowlist": [], + "source_ordinal": 4, + "records": [], + "lines": [ + "[2025/09/06 17:03:53.724 +08:00] [TIMESTAMP=2025/09/06 17:03:53.724 +08:00] [EVENT_CLASS=GENERAL] [EVENT=COMPLETED] [COMMAND=Execute] [SQL_TEXT=SELECT ?] [SQL_STATEMENTS=Select] [CONNECTION_ID=10] [EXECUTE_PARAMS=\"[\\\"KindBytes \\\\\\\\xff\\\"]\"] [RETRY=false]" + ] + } + ] +} diff --git a/tests/replayer/differential/go-observer/main.go b/tests/replayer/differential/go-observer/main.go new file mode 100644 index 000000000..47993498f --- /dev/null +++ b/tests/replayer/differential/go-observer/main.go @@ -0,0 +1,248 @@ +// Copyright 2026 PingCAP, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bytes" + "encoding/hex" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "os" + "reflect" + "time" + + replaycmd "github.com/pingcap/tiproxy/pkg/sqlreplay/cmd" + "go.uber.org/zap" +) + +type corpus struct { + Cases []replayCase `json:"cases"` +} + +type replayCase struct { + ID string `json:"id"` + Format string `json:"format"` + PreparedClose string `json:"prepared_close"` + FilterRetries bool `json:"filter_retries"` + UserAllowlist []string `json:"user_allowlist"` + SourceOrdinal int `json:"source_ordinal"` + Records []nativeRecord `json:"records"` + Lines []string `json:"lines"` +} + +type nativeRecord struct { + Headers [][2]string `json:"headers"` + PayloadHex string `json:"payload_hex"` +} + +type observation struct { + CaseID string `json:"case_id"` + Index uint64 `json:"index"` + PayloadHex string `json:"payload_hex"` + StartUnixNanos int64 `json:"start_unix_nanos"` + EndUnixNanos int64 `json:"end_unix_nanos"` + ConnectionID uint64 `json:"connection_id"` + UpstreamConnectionID uint64 `json:"upstream_connection_id"` + Command byte `json:"command"` + CurrentDatabase string `json:"current_database"` + CapturedStatementID uint32 `json:"captured_statement_id"` + PreparedStatement string `json:"prepared_statement"` + StatementType string `json:"statement_type"` + Succeeded bool `json:"succeeded"` +} + +func main() { + corpusPath := flag.String("corpus", "", "shared replay corpus") + baselinePath := flag.String("baseline", "", "baseline observation JSON") + candidatePath := flag.String("candidate", "", "candidate observation JSON") + flag.Parse() + if *baselinePath != "" || *candidatePath != "" { + if *baselinePath == "" || *candidatePath == "" { + fatal(errors.New("both -baseline and -candidate are required")) + } + if err := compare(*baselinePath, *candidatePath); err != nil { + _, _ = fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + _, _ = fmt.Fprintln(os.Stdout, `{"equal":true}`) + return + } + if *corpusPath == "" { + fatal(errors.New("-corpus is required")) + } + if err := observeCorpus(*corpusPath, os.Stdout); err != nil { + fatal(err) + } +} + +func observeCorpus(path string, output io.Writer) error { + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + var fixture corpus + if err := json.NewDecoder(file).Decode(&fixture); err != nil { + return err + } + observations := make([]observation, 0) + for _, testCase := range fixture.Cases { + input, err := buildInput(testCase) + if err != nil { + return fmt.Errorf("build %s: %w", testCase.ID, err) + } + decoder := replaycmd.NewCmdDecoder( + replaycmd.TrafficFormat(testCase.Format), + replaycmd.NewDeDup(), + zap.NewNop(), + ) + if audit, ok := decoder.(replaycmd.AuditLogDecoder); ok { + audit.SetPSCloseStrategy(replaycmd.PSCloseStrategy(testCase.PreparedClose)) + allocator, err := replaycmd.NewConnIDAllocator(testCase.SourceOrdinal) + if err != nil { + return err + } + audit.SetIDAllocator(allocator) + audit.SetUserAllowlist(testCase.UserAllowlist) + if testCase.FilterRetries { + audit.EnableFilterCommandWithRetry() + } + } + reader := &memoryLineReader{name: testCase.ID, input: input} + for index := uint64(0); ; index++ { + command, err := decoder.Decode(reader) + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return fmt.Errorf("decode %s: %w", testCase.ID, err) + } + upstreamID := command.UpstreamConnID + if upstreamID == 0 { + upstreamID = command.ConnID + } + observations = append(observations, observation{ + CaseID: testCase.ID, + Index: index, + PayloadHex: hex.EncodeToString(command.Payload), + StartUnixNanos: command.StartTs.UnixNano(), + EndUnixNanos: unixNanosOrZero(command.EndTs), + ConnectionID: command.ConnID, + UpstreamConnectionID: upstreamID, + Command: command.Type.Byte(), + CurrentDatabase: command.CurDB, + CapturedStatementID: command.CapturedPsID, + PreparedStatement: command.PreparedStmt, + StatementType: command.StmtType, + Succeeded: command.Success, + }) + } + } + return json.NewEncoder(output).Encode(observations) +} + +func buildInput(testCase replayCase) ([]byte, error) { + var input bytes.Buffer + if testCase.Format == string(replaycmd.FormatNative) { + for _, record := range testCase.Records { + for _, header := range record.Headers { + if _, err := fmt.Fprintf(&input, "# %s: %s\n", header[0], header[1]); err != nil { + return nil, err + } + } + payload, err := hex.DecodeString(record.PayloadHex) + if err != nil { + return nil, err + } + if _, err := fmt.Fprintf(&input, "# Payload_len: %d\n", len(payload)); err != nil { + return nil, err + } + input.Write(payload) + input.WriteByte('\n') + } + } else { + for _, line := range testCase.Lines { + input.WriteString(line) + input.WriteByte('\n') + } + } + return input.Bytes(), nil +} + +func unixNanosOrZero(value time.Time) int64 { + if value.IsZero() { + return 0 + } + return value.UnixNano() +} + +type memoryLineReader struct { + name string + input []byte + offset int + line int +} + +func (r *memoryLineReader) String() string { return r.name } + +func (r *memoryLineReader) ReadLine() ([]byte, string, int, error) { + if r.offset == len(r.input) { + return nil, r.name, r.line, io.EOF + } + relativeEnd := bytes.IndexByte(r.input[r.offset:], '\n') + if relativeEnd < 0 { + return nil, r.name, r.line, io.ErrUnexpectedEOF + } + start := r.offset + r.offset += relativeEnd + 1 + r.line++ + return r.input[start : r.offset-1], r.name, r.line, nil +} + +func (r *memoryLineReader) Read(output []byte) (string, int, error) { + if len(r.input)-r.offset < len(output) { + return r.name, r.line, io.ErrUnexpectedEOF + } + copy(output, r.input[r.offset:r.offset+len(output)]) + r.offset += len(output) + return r.name, r.line, nil +} + +func (r *memoryLineReader) Close() {} + +func compare(baselinePath, candidatePath string) error { + read := func(path string) (any, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var value any + if err := json.Unmarshal(data, &value); err != nil { + return nil, err + } + return value, nil + } + baseline, err := read(baselinePath) + if err != nil { + return err + } + candidate, err := read(candidatePath) + if err != nil { + return err + } + if !reflect.DeepEqual(baseline, candidate) { + return fmt.Errorf("decoder observations differ: baseline=%s candidate=%s", baselinePath, candidatePath) + } + return nil +} + +func fatal(err error) { + _, _ = fmt.Fprintln(os.Stderr, err) + os.Exit(2) +} + +var _ replaycmd.LineReader = (*memoryLineReader)(nil) diff --git a/tests/replayer/differential/run.sh b/tests/replayer/differential/run.sh new file mode 100755 index 000000000..d38458814 --- /dev/null +++ b/tests/replayer/differential/run.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# Copyright 2026 PingCAP, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd) +corpus="$repo_root/tests/replayer/differential/corpus.v1.json" +tmp_dir=$(mktemp -d) + +cleanup() { + rm -rf "$tmp_dir" +} +trap cleanup EXIT INT TERM + +cd "$repo_root" +go run ./tests/replayer/differential/go-observer -corpus "$corpus" >"$tmp_dir/go.json" +CP_REPLAYER_CORPUS="$corpus" cargo run --locked --quiet --manifest-path rust/Cargo.toml \ + -p replayer-core --example cp_replayer_observer >"$tmp_dir/rust.json" +go run ./tests/replayer/differential/go-observer \ + -baseline "$tmp_dir/go.json" -candidate "$tmp_dir/rust.json" + +CP_REPLAYER_CORPUS="$corpus" CP_REPLAYER_MUTATE_CONNECTION_ID=1 \ + cargo run --locked --quiet --manifest-path rust/Cargo.toml \ + -p replayer-core --example cp_replayer_observer >"$tmp_dir/rust-mutated.json" +set +e +go run ./tests/replayer/differential/go-observer \ + -baseline "$tmp_dir/go.json" -candidate "$tmp_dir/rust-mutated.json" +status=$? +set -e +if [[ "$status" -eq 0 ]]; then + echo "CP-REPLAYER connection-ID mutation was not killed" >&2 + exit 1 +fi