From cd8d3f2090392fd578cf5ddbaa9b1fa5c5cf54dd Mon Sep 17 00:00:00 2001 From: mlevkov Date: Sun, 2 Aug 2026 01:28:29 -0700 Subject: [PATCH 1/2] feat(connectors): add the HTTP source webhook gateway connector Iggy has no way to receive a webhook. Every provider that pushes events over HTTP needs something in front of it, and today that means running a separate service whose only job is to accept a POST and republish it. This connector removes that hop: it runs an embedded HTTP server, accepts authenticated POST bodies, and produces them to the instance's stream and topic as raw bytes. One plugin .so is loaded once no matter how many source entries reference it, so the listener cannot live on any single instance. It lives in a process-global registry keyed by listen address: the first open binds the public and admin ports, later opens validate their body limit, admin address, management token and instance name against the running listener before joining, and the last close releases both ports. Mismatches fail that instance's open rather than silently handing it a listener its configuration does not describe. A single port can therefore serve many providers, each routed to its own topic. Requests resolve against an ArcSwap route table that is rebuilt whole on every control-plane change, so one atomic load yields both the endpoint's auth rules and the destination bridge. Secret paths carry 128 bits in the URL itself, on the model of a Slack webhook, with optional bearer or HMAC on top; HMAC is verified over the raw body in constant time. Revoked endpoints answer 404 alongside paths that never existed, so a leaked URL cannot be used to confirm it was once live. Endpoints can be registered, re-keyed and revoked at runtime through a token-guarded API on the admin listener, because revoking a compromised endpoint is time-critical and provisioning one per tenant is inherently programmatic. Those endpoints ride the SDK's ConnectorState, and state is attached only to an empty batch: the runtime saves state solely on the success branch of the Iggy send, and an empty send always succeeds, so a mutation cannot be lost to an unrelated send failure. Revocation writes a tombstone that outranks TOML on restore, so a stale config file cannot resurrect an endpoint an operator revoked. Delivery is best-effort in both directions and the README says so first, before anything else: HTTP 200 means accepted into an in-memory buffer, and both the loss and duplicate windows are enumerated with what mitigates each. A full bridge answers 429 with Retry-After rather than blocking, since holding the connection open would turn a slow Iggy into a retry storm. Gateway metrics on the admin listener cover accept-to-200 latency, which the runtime's own stage histograms begin too late to see. Part of the webhook gateway design accepted in #3039. The backpressure chain is only complete once the bounded runtime forwarding channel from #3795 lands; until then a full bridge signals an arrival burst rather than a slow Iggy, which the README documents. Co-authored-by: Claude --- Cargo.lock | 27 + Cargo.toml | 3 + .../connectors/http_source_github.toml | 60 + .../connectors/http_source_partner.toml | 55 + core/connectors/sources/README.md | 3 +- .../connectors/sources/http_source/Cargo.toml | 59 + core/connectors/sources/http_source/README.md | 302 ++++ .../sources/http_source/config.toml | 65 + .../sources/http_source/src/auth.rs | 245 +++ .../connectors/sources/http_source/src/lib.rs | 1036 ++++++++++++ .../sources/http_source/src/management.rs | 869 ++++++++++ .../sources/http_source/src/metrics.rs | 433 +++++ .../sources/http_source/src/routes.rs | 410 +++++ .../sources/http_source/src/server.rs | 1504 +++++++++++++++++ .../sources/http_source/src/state.rs | 510 ++++++ .../sources/http_source/src/types.rs | 247 +++ core/integration/Cargo.toml | 3 + .../tests/connectors/fixtures/http/mod.rs | 5 + .../tests/connectors/fixtures/http/source.rs | 110 ++ .../tests/connectors/fixtures/mod.rs | 6 +- .../tests/connectors/http/http_source.rs | 496 ++++++ core/integration/tests/connectors/http/mod.rs | 3 + .../tests/connectors/http/source.toml | 20 + .../http/source_config/http_github.toml | 53 + .../http/source_config/http_partner.toml | 47 + 25 files changed, 6568 insertions(+), 3 deletions(-) create mode 100644 core/connectors/runtime/example_config/connectors/http_source_github.toml create mode 100644 core/connectors/runtime/example_config/connectors/http_source_partner.toml create mode 100644 core/connectors/sources/http_source/Cargo.toml create mode 100644 core/connectors/sources/http_source/README.md create mode 100644 core/connectors/sources/http_source/config.toml create mode 100644 core/connectors/sources/http_source/src/auth.rs create mode 100644 core/connectors/sources/http_source/src/lib.rs create mode 100644 core/connectors/sources/http_source/src/management.rs create mode 100644 core/connectors/sources/http_source/src/metrics.rs create mode 100644 core/connectors/sources/http_source/src/routes.rs create mode 100644 core/connectors/sources/http_source/src/server.rs create mode 100644 core/connectors/sources/http_source/src/state.rs create mode 100644 core/connectors/sources/http_source/src/types.rs create mode 100644 core/integration/tests/connectors/fixtures/http/source.rs create mode 100644 core/integration/tests/connectors/http/http_source.rs create mode 100644 core/integration/tests/connectors/http/source.toml create mode 100644 core/integration/tests/connectors/http/source_config/http_github.toml create mode 100644 core/integration/tests/connectors/http/source_config/http_partner.toml diff --git a/Cargo.lock b/Cargo.lock index 0cf2039bd2..0f8fdb876d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6993,6 +6993,31 @@ dependencies = [ "tracing", ] +[[package]] +name = "iggy_connector_http_source" +version = "0.4.1-edge.1" +dependencies = [ + "arc-swap", + "async-trait", + "axum", + "crossfire", + "dashmap", + "hex", + "iggy_common", + "iggy_connector_sdk", + "prometheus-client", + "rand 0.10.2", + "reqwest 0.13.4", + "ring", + "rmp-serde", + "secrecy", + "serde", + "serde_json", + "tokio", + "toml 1.1.3+spec-1.1.0", + "tracing", +] + [[package]] name = "iggy_connector_iceberg_sink" version = "0.5.0-edge.1" @@ -7462,6 +7487,7 @@ dependencies = [ "figment", "futures", "harness_derive", + "hex", "humantime", "iggy", "iggy-cli", @@ -7481,6 +7507,7 @@ dependencies = [ "reqwest 0.13.4", "reqwest-middleware", "reqwest-retry", + "ring", "rmcp", "rust-s3", "secrecy", diff --git a/Cargo.toml b/Cargo.toml index f33c0eddc7..6bc61d5cdc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -48,6 +48,7 @@ members = [ "core/connectors/sinks/stdout_sink", "core/connectors/sinks/surrealdb_sink", "core/connectors/sources/elasticsearch_source", + "core/connectors/sources/http_source", "core/connectors/sources/influxdb_source", "core/connectors/sources/postgres_source", "core/connectors/sources/random_source", @@ -86,6 +87,7 @@ aligned-vec = "0.6.4" anyhow = "1.0.104" apache-avro = "0.21.0" apple-native-keyring-store = { version = "1.0.1", features = ["keychain"] } +arc-swap = "1.7.1" # "std" is load-bearing: it cascades to rand_core/getrandom, which the # crypto module's `OsRng` import needs even when no sibling crate in the # build graph happens to enable it via feature unification. @@ -190,6 +192,7 @@ gloo = "0.12" governor = "0.10.4" harness_derive = { path = "core/harness_derive" } hash32 = "1.0.0" +hex = "0.4.3" hostname = "0.4.2" http = "1.4.2" human-repr = "1.1.0" diff --git a/core/connectors/runtime/example_config/connectors/http_source_github.toml b/core/connectors/runtime/example_config/connectors/http_source_github.toml new file mode 100644 index 0000000000..f58005be52 --- /dev/null +++ b/core/connectors/runtime/example_config/connectors/http_source_github.toml @@ -0,0 +1,60 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Instance 1 of a shared webhook listener. Pair it with +# http_source_partner.toml to see two instances serving one port. +# +# The public listener binds loopback here so running the examples cannot +# expose a port; production wants 0.0.0.0 behind a load balancer. + +type = "source" +key = "http_github" +enabled = true +version = 0 +name = "HTTP source (GitHub)" +path = "/target/release/libiggy_connector_http_source" +plugin_config_format = "toml" +verbose = false + +[[streams]] +stream = "webhooks" +topic = "github_events" +schema = "raw" +batch_length = 100 +linger_time = "5ms" + +[plugin_config] +listen_addr = "127.0.0.1:9090" +admin_listen_addr = "127.0.0.1:9091" +instance_name = "http_github" +topic_path = "github_events" +auth_bearer_token = "replace_with_secret_token" +management_token = "replace_with_secret_token" +max_body_size_bytes = 1048576 +buffer_capacity = 10000 +max_batch_size = 500 +include_http_metadata = true +forward_headers = ["X-GitHub-Delivery", "X-Request-ID"] + +[[plugin_config.endpoints]] +# Replace this: it is published in this repository, so anyone can reach it. +# Generate your own with `openssl rand -hex 16`. +endpoint_id = "a3f8c2e1b9d04f7a8e6c1d2b3a4f5e6d" +auth_type = "hmac-sha256" +auth_secret = "replace_with_webhook_signing_secret" +hmac_header = "X-Hub-Signature-256" +hmac_prefix = "sha256=" diff --git a/core/connectors/runtime/example_config/connectors/http_source_partner.toml b/core/connectors/runtime/example_config/connectors/http_source_partner.toml new file mode 100644 index 0000000000..8beb4ece6e --- /dev/null +++ b/core/connectors/runtime/example_config/connectors/http_source_partner.toml @@ -0,0 +1,55 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Instance 2, joining the listener bound by http_source_github.toml. +# listen_addr, admin_listen_addr, max_body_size_bytes, and management_token +# must match that file exactly, or this instance's open fails. +# +# No topic_path: this instance serves secret-path endpoints only. + +type = "source" +key = "http_partner" +enabled = true +version = 0 +name = "HTTP source (partner)" +path = "/target/release/libiggy_connector_http_source" +plugin_config_format = "toml" +verbose = false + +[[streams]] +stream = "webhooks" +topic = "partner_events" +schema = "raw" +batch_length = 100 +linger_time = "5ms" + +[plugin_config] +listen_addr = "127.0.0.1:9090" +admin_listen_addr = "127.0.0.1:9091" +instance_name = "http_partner" +management_token = "replace_with_secret_token" +max_body_size_bytes = 1048576 +buffer_capacity = 10000 +max_batch_size = 500 +include_http_metadata = true + +[[plugin_config.endpoints]] +# Replace this: it is published in this repository, so anyone can reach it. +# Generate your own with `openssl rand -hex 16`. +endpoint_id = "0b7d9e2f4a6c8e1d3b5f7a9c2e4d6f81" +auth_type = "bearer" +auth_secret = "replace_with_partner_token" diff --git a/core/connectors/sources/README.md b/core/connectors/sources/README.md index 34989aef00..0bb7207e36 100644 --- a/core/connectors/sources/README.md +++ b/core/connectors/sources/README.md @@ -9,6 +9,7 @@ Source connectors are responsible for ingesting data from external sources into | Source | Description | | ------ | ----------- | | **elasticsearch_source** | Polls documents from Elasticsearch indices with timestamp-based tracking | +| **http_source** | Webhook gateway: an embedded HTTP server shared by every instance, with per-endpoint bearer/HMAC auth and a management API for endpoints registered at runtime | | **influxdb_source** | Polls InfluxDB with cursor-based timestamp tracking; supports V2 (Flux, annotated CSV) and V3 (SQL, JSONL) | | **postgres_source** | Reads rows from PostgreSQL tables with multiple strategies: delete after read, mark as processed, or timestamp tracking | | **random_source** | Generates random test messages (useful for testing and development) | @@ -69,7 +70,7 @@ enabled = true # Toggle source on/off version = 0 name = "Random source" # Name of the source path = "libiggy_connector_random_source" # Path to the source connector -config_format = "toml" +plugin_config_format = "toml" verbose = false # Log message processing at info level instead of debug benchmark = false # Emit per-batch timing events on `iggy_connectors::benchmark` target diff --git a/core/connectors/sources/http_source/Cargo.toml b/core/connectors/sources/http_source/Cargo.toml new file mode 100644 index 0000000000..2ce8b9ea0f --- /dev/null +++ b/core/connectors/sources/http_source/Cargo.toml @@ -0,0 +1,59 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[package] +name = "iggy_connector_http_source" +version = "0.4.1-edge.1" +description = "Iggy is the persistent message streaming platform written in Rust, supporting QUIC, TCP and HTTP transport protocols, capable of processing millions of messages per second." +edition = "2024" +license = "Apache-2.0" +keywords = ["iggy", "messaging", "streaming"] +categories = ["command-line-utilities", "database", "network-programming"] +homepage = "https://iggy.apache.org" +documentation = "https://iggy.apache.org/docs" +repository = "https://github.com/apache/iggy" +readme = "../../README.md" +publish = false + +[package.metadata.cargo-machete] +ignored = ["dashmap"] + +[lib] +crate-type = ["cdylib", "lib"] + +[dependencies] +arc-swap = { workspace = true } +async-trait = { workspace = true } +axum = { workspace = true } +crossfire = { workspace = true } +dashmap = { workspace = true } +hex = { workspace = true } +iggy_common = { workspace = true } +iggy_connector_sdk = { workspace = true } +prometheus-client = { workspace = true } +rand = { workspace = true } +ring = { workspace = true } +secrecy = { workspace = true } +serde = { workspace = true } +tokio = { workspace = true } +tracing = { workspace = true } + +[dev-dependencies] +reqwest = { workspace = true } +rmp-serde = { workspace = true } +serde_json = { workspace = true } +toml = { workspace = true } diff --git a/core/connectors/sources/http_source/README.md b/core/connectors/sources/http_source/README.md new file mode 100644 index 0000000000..87100223d1 --- /dev/null +++ b/core/connectors/sources/http_source/README.md @@ -0,0 +1,302 @@ +# HTTP Source Connector (Webhook Gateway) + +The HTTP source connector turns Apache Iggy into a webhook receiver. It runs an embedded HTTP server, accepts `POST` bodies on authenticated paths, and produces them to the instance's configured stream and topic as raw bytes. + +Every instance of this plugin shares one listener, so a single port can serve many providers, each routed to its own topic. + +## Delivery semantics (read this first) + +> **Delivery guarantee: best-effort.** HTTP 200 means "accepted into the connector's in-memory buffer", not "durably stored in Iggy". This is weaker than at-most-once: a message can be lost after acknowledgment, with no record beyond a log line and a metric. + +Loss windows, explicitly: + +1. **Process crash** between HTTP 200 and the producer send. Buffered messages are volatile. +2. **Producer failure.** The runtime's `source_forwarding_loop` logs `producer.send()` errors and continues; there is no feedback channel back to the HTTP handler. This is structural in the current poll-based SDK, not a defect in this connector. +3. **Shutdown.** Narrowed by #3321, which closes the plugin before tearing down the forwarding channel so in-flight batches drain. Messages still in the bridge when the poll task stops are lost; the connector logs the count and increments `http_source_dropped_on_close_total`. + +What mitigates this in practice is the caller: webhook senders such as GitHub, Stripe, and Twilio retry on timeout and 5xx, so the sender side is at-least-once up to the moment this connector returns 200. The connector's job is to make the post-200 window as small and as observable as possible. + +**Duplicates are also possible.** The same retries that mitigate loss create a duplicate window: + +1. A caller times out after the connector enqueued the message but before the 200 reached it, retries, and the payload lands twice. +2. A load balancer or proxy retries a POST after a hiccup downstream of a successful enqueue. + +So the guarantee is best-effort in both directions: no silent-loss guarantee and no dedup guarantee. Consumers that need effectively-once processing should dedupe on the provider's delivery id (`X-GitHub-Delivery`, `svix-id`, Stripe's `event.id` in the body), which is why forwarding those headers is the default recommendation. + +Stronger guarantees need an SDK change, not a connector change: at-least-once by construction requires the runtime to hand the connector a producer handle so it can await the send before answering 200. See #3039. + +## Configuration + +Two instances sharing one listener, each routing to its own topic. Each block is a separate connector configuration file. + +```toml +# instance 1: GitHub webhooks -> webhooks/github_events +type = "source" +key = "http_github" +enabled = true +version = 0 +name = "HTTP source (GitHub)" +path = "../../target/release/libiggy_connector_http_source" +verbose = false + +[[streams]] +stream = "webhooks" +topic = "github_events" +schema = "raw" +batch_length = 100 +linger_time = "5ms" + +[plugin_config] +listen_addr = "0.0.0.0:9090" +admin_listen_addr = "127.0.0.1:9091" +instance_name = "http_github" +topic_path = "github_events" +auth_bearer_token = "global-webhook-secret" +management_token = "admin-secret" +max_body_size_bytes = 1048576 +buffer_capacity = 10000 +max_batch_size = 500 +include_http_metadata = true +forward_headers = ["X-GitHub-Delivery", "X-Request-ID"] + +[[plugin_config.endpoints]] +endpoint_id = "a3f8c2e1b9d04f7a8e6c1d2b3a4f5e6d" +auth_type = "hmac-sha256" +auth_secret = "whsec_github_example" +hmac_header = "X-Hub-Signature-256" +hmac_prefix = "sha256=" +``` + +```toml +# instance 2: partner webhooks -> webhooks/partner_events, joining the same listener +type = "source" +key = "http_partner" +enabled = true +version = 0 +name = "HTTP source (partner)" +path = "../../target/release/libiggy_connector_http_source" + +[[streams]] +stream = "webhooks" +topic = "partner_events" +schema = "raw" + +[plugin_config] +listen_addr = "0.0.0.0:9090" +admin_listen_addr = "127.0.0.1:9091" +instance_name = "http_partner" +management_token = "admin-secret" +# no topic_path: this instance serves secret-path endpoints only + +[[plugin_config.endpoints]] +endpoint_id = "0b7d9e2f4a6c8e1d3b5f7a9c2e4d6f81" +auth_type = "bearer" +auth_secret = "partner-token" +``` + +The stream's `schema` **must be `raw`**. This connector always produces raw bodies; with `schema = "json"` the runtime's encoder rejects every message and the batch is dropped. + +### Options + +| Option | Type | Default | Description | +| ------ | ---- | ------- | ----------- | +| `listen_addr` | string | required | Public listener. Every instance sharing it must configure the identical value. | +| `admin_listen_addr` | string | `127.0.0.1:9091` | Management API, admin health, and metrics. Never route this through a public load balancer. | +| `instance_name` | string | runtime id | Identifies the instance in message headers and on the admin listener. The default is the plugin's numeric runtime id, assigned in load order and **not stable across restarts** — set it explicitly in production. | +| `topic_path` | string | none | Exposes `POST /topics/{topic_path}`. Unset leaves only secret-path endpoints. | +| `auth_bearer_token` | string | none | Guards the named topic path. Unset leaves it unauthenticated, for deployments behind an authenticating gateway. | +| `management_token` | string | none | Enables `/admin/endpoints`. Unset means the management API does not exist. | +| `max_body_size_bytes` | usize | `1048576` | Request body limit, enforced by the `Bytes` extractor. Routing wins over it: an oversized POST to an unknown path answers 404, not 413. On a known path the body is buffered before authentication, which HMAC-over-raw-body requires. Must match across instances sharing a listener. | +| `buffer_capacity` | usize | `10000` | Messages the instance bridge holds. A full bridge answers 429, though see Backpressure: until #3795 lands that signals an arrival burst, not a slow Iggy. | +| `max_batch_size` | usize | `500` | Maximum messages a single `poll()` returns. | +| `include_http_metadata` | bool | `true` | Adds instance, peer address, and receive time as message headers. | +| `forward_headers` | array | `[]` | Request headers copied onto the message. Invalid names fail `open()`, as do `Authorization`, `Proxy-Authorization`, and `Cookie` — forwarding a reusable credential would copy it onto every message and persist it in the log. | +| `endpoints` | array | `[]` | Static secret-path endpoints. | +| `verbose_logging` | bool | `false` | Log per-batch detail at info instead of debug. | + +### Endpoint options + +| Option | Type | Default | Description | +| ------ | ---- | ------- | ----------- | +| `endpoint_id` | string | required | Exactly 32 lowercase hex characters. Generate with `openssl rand -hex 16`. | +| `auth_type` | string | `none` | `none`, `bearer`, `hmac-sha256`, or `hmac-sha1`. | +| `auth_secret` | string | none | Required unless `auth_type` is `none`. | +| `hmac_header` | string | `X-Hub-Signature-256` | Header carrying the signature. | +| `hmac_prefix` | string | `sha256=` | Prefix stripped before hex-decoding. Use `""` for a bare hex signature. | +| `expires_at` | u64 | none | Unix seconds. Requests arriving at or after this answer 410. | + +`HttpSourceConfig` deliberately does not implement `Serialize`, so this connector cannot write a credential out by accident. That does **not** protect the values in your TOML: the runtime keeps plugin configuration as raw JSON and serves it verbatim from `GET /sources/{key}/configs/plugin` (and inside `/configs` and `/configs/active`), so anyone who can reach the runtime's control API can read every secret configured here. Treat that API as privileged. (`/stats` carries no plugin configuration.) + +## Routing + +Two kinds of path, both `POST`: + +```text +POST /topics/{topic_path} named path, one per instance, guarded by auth_bearer_token +POST /e/{endpoint_id} secret path, many per instance, each with its own auth +``` + +A secret-path URL is itself the credential: 32 hex characters is 128 bits of entropy, the same model as a Slack webhook URL. Treat these URLs as secrets and prefer adding an HMAC on top for providers that support one. + +Paths must be unique across every instance sharing a listener. Two instances claiming the same `topic_path` or the same `endpoint_id` fail the second instance's `open()` rather than letting one silently take the other's traffic. + +## Request and response contract + +```text +POST /e/a3f8c2e1b9d04f7a8e6c1d2b3a4f5e6d +X-Hub-Signature-256: sha256=... +Content-Type: application/json + +{"event": "push", "repository": "apache/iggy"} +``` + +| Status | Condition | Body | +| ------ | --------- | ---- | +| 200 | Accepted into the bridge | `{"status":"queued"}` | +| 401 | Bearer or HMAC validation failed | `{"error":"unauthorized"}` | +| 404 | Unknown path, or a revoked endpoint | `{"error":"not found"}` | +| 410 | Endpoint past `expires_at` | `{"error":"gone"}` | +| 400 | Malformed request body, e.g. the client reset mid-send | `{"error":"bad request"}` | +| 413 | Body over `max_body_size_bytes` | `{"error":"payload too large"}` | +| 429 | Bridge full | `{"error":"service temporarily unavailable"}` plus `Retry-After: 1` | +| 503 | `GET /health` with no instance serving | `{"status":"unavailable"}` | + +Revoked endpoints answer 404 rather than 410 or 403 on purpose: a leaked URL must not be usable to confirm that it was once live. Error bodies carry no internals; diagnostics live on the admin listener. + +`GET /health` on the public listener answers 200 while at least one instance is serving and 503 otherwise, which is what a load balancer should watch. + +HMAC signatures are validated over the raw request body exactly as received, never over a re-serialized form, and compared in constant time. + +The supported shape is a hex digest of the body behind a fixed prefix, which covers GitHub (`X-Hub-Signature-256: sha256=`) and most generic partner webhooks. Set `hmac_prefix = ""` for a bare hex signature. + +Providers that sign a composed string rather than the body alone are **not** supported in v1. Stripe signs `{timestamp}.{body}` and packs it into `Stripe-Signature: t=...,v1=...`; Twilio signs the URL plus sorted form parameters as base64. For those, use `bearer` or an endpoint with `auth_type = "none"`, forward the signature header, and verify it downstream. + +## Message headers + +With `include_http_metadata = true` each message carries: + +| Header | Value | +| ------ | ----- | +| `iggy_source_instance` | `instance_name`, or the connector id if unset | +| `iggy_http_remote_addr` | Peer IP address | +| `iggy_http_received_at` | Accept time, microseconds since the Unix epoch | + +Anything listed in `forward_headers` is copied alongside, under its own name. Iggy rejects header values over 255 bytes, and real values such as `User-Agent` routinely exceed that, so forwarded values are truncated to fit rather than failing the message. Truncations and drops are counted in `http_source_headers_clamped_total` and `http_source_headers_dropped_total`. + +Forwarding the provider's delivery id is the recommended default, because it is the dedup key a consumer needs to close the duplicate window described above. + +## Sharing one listener + +The first instance to open binds both ports. Later instances validate their configuration against the running listener and join it. `listen_addr`, `admin_listen_addr`, `max_body_size_bytes`, and `management_token` must agree; a mismatch fails that instance's `open()` with a message naming the field, rather than silently handing it a listener its configuration does not describe. + +`instance_name` must also be unique on the listener, since it is how the management API addresses an instance and how every metric series is labelled. Instances that leave it unset get their distinct numeric plugin id, so they cannot collide. + +Closing an instance deregisters its routes immediately, so its paths answer 404 while its siblings keep serving. The last instance to close shuts both listeners down gracefully and releases the ports, which the runtime's stop-then-start restart flow depends on. + +The registry is per process. Several runtime processes behind a load balancer will not converge on dynamically registered endpoints; that is a fleet-coordination problem left to a future version. Single-process, multi-instance deployments are consistent by construction. + +## Dynamic endpoint management + +Set `management_token` to enable `/admin/endpoints` on the admin listener. Without it the API is not mounted at all and every path under it answers 404. Every call needs `Authorization: Bearer `. + +```text +POST /admin/endpoints register, id generated server-side -> 201 +GET /admin/endpoints list every endpoint, secrets omitted -> 200 +GET /admin/endpoints/{id} one endpoint -> 200 +PATCH /admin/endpoints/{id} rotate auth_secret in place -> 200 +DELETE /admin/endpoints/{id} revoke -> 204 +``` + +```bash +curl -sS -X POST http://127.0.0.1:9091/admin/endpoints \ + -H 'Authorization: Bearer admin-secret' \ + -H 'Content-Type: application/json' \ + -d '{"instance":"http_github","auth_type":"hmac-sha256","auth_secret":"whsec_new"}' +# {"endpoint_id":"9f2c...","path":"/e/9f2c..."} +``` + +Rotation deliberately keeps the path: a webhook sender configures the URL once, so changing the shared secret must not force it to be reconfigured. It applies to dynamic endpoints only — a static endpoint's secret lives in TOML, so rotating one answers 409 and points you at the file, because restoring prefers TOML and would silently revert the change on the next restart. + +| Status | Condition | +| ------ | --------- | +| 400 | Empty `auth_secret`, or an `expires_at` already in the past | +| 401 | Missing or wrong `management_token` | +| 404 | Unknown endpoint, unknown `instance`, or the API is not configured | +| 409 | Rotating a static endpoint, or a generated id collision | +| 500 | The route table could not be rebuilt | +| 503 | The owning instance closed while the request was in flight | + +Revocation writes a tombstone rather than deleting the entry. The tombstone persists, so a restart against a stale TOML file cannot resurrect an endpoint someone revoked. + +The 204 means the endpoint stopped serving *now*, in memory. Durability follows the same path as registration below, with one asymmetry worth knowing: losing an unsaved registration fails closed, but losing an unsaved revocation fails **open** — the endpoint would come back after a restart. Check `submitted` on `GET /admin/endpoints/{id}` before treating a revocation as final, and if the connector cannot reach Iggy, remove the endpoint from the TOML too. + +Dynamic endpoints ride the SDK's `ConnectorState`, which the runtime writes after the next successful send. A management response therefore means "accepted", not "durable": if Iggy is unreachable, the endpoint is live in memory but not yet on disk, and a crash in that window loses it. + +`GET /admin/endpoints` reports `submitted` per endpoint. It is named that, and not `persisted`, on purpose: the plugin hands state to the runtime and gets no acknowledgement back across the FFI, so `submitted: true` means the registry reached the runtime, not that the write landed. Watch the connector's status for save failures. + +## Backpressure + +The chain below is the **target** behaviour and is not yet complete: it needs the bounded runtime forwarding channel from #3795. + +The bridge is bounded today, so 429 does fire on an arrival burst the poll loop cannot keep up with. What is missing is the coupling: until #3795 lands, `poll()` drains into an unbounded runtime channel, so a slow Iggy does not propagate back into 429 and shows up as memory growth instead. + +```text +Iggy slow -> forwarding loop blocks -> bounded channel fills -> poll() stalls + -> instance bridge fills -> HTTP 429 + Retry-After: 1 +``` + +The handler never blocks on a full bridge. Waiting would hold connections open and, once the sender times out, produce a retry storm; a fast 429 tells the sender exactly what to do. + +| Traffic | `buffer_capacity` | Rationale | +| ------- | ----------------- | --------- | +| Under 100 req/s | 1000 | Small footprint | +| 100 to 1000 req/s | 10000 (default) | Absorbs roughly ten seconds of burst | +| Over 1000 req/s | 50000 to 100000 | Sustained bursts; tune against the buffer metrics | + +The bridge is bounded by message count, not bytes, so worst-case memory is `buffer_capacity * max_body_size_bytes`. At the defaults that is about 10 GB. Size the two together. + +## Observability + +`GET /admin/health` returns per-instance JSON: queue depth and capacity, serving endpoint counts by origin plus expired and revoked counts, `state_submitted`, and header loss counters. + +`GET /admin/metrics` returns Prometheus text format. The runtime's own stage histograms begin at `poll()`, so they cannot see accept-to-200 latency; these fill that gap. + +| Metric | Type | Labels | Notes | +| ------ | ---- | ------ | ----- | +| `http_source_requests_total` | counter | `instance`, `kind`, `status` | | +| `http_source_request_duration_seconds` | histogram | `instance`, `status` | | +| `http_source_rejected_full_total` | counter | `instance` | | +| `http_source_dropped_on_close_total` | counter | `instance` | | +| `http_source_headers_clamped_total` | counter | `instance` | | +| `http_source_headers_dropped_total` | counter | `instance` | | +| `http_source_buffer_used` | gauge | `instance` | | +| `http_source_buffer_capacity` | gauge | `instance` | | +| `http_source_endpoints_active` | gauge | `instance`, `kind` | endpoints that would accept a request now: neither revoked nor past `expires_at` | + +`kind` is `named` or `secret` for requests and `static` or `dynamic` for endpoints. `status` is the response class, `2xx`, `4xx`, or `5xx`, rather than the exact code, so a caller cannot inflate cardinality by probing. Only requests to a genuinely unknown path are counted under `instance="unrouted"`, which is where a scan for live endpoint ids shows up; a revoked or expired endpoint is still metered against the instance that owns it. + +A metric with no series yet is absent from the scrape rather than reported as zero, which is how Prometheus client libraries represent labelled families. Write dashboard queries accordingly. + +## Operational notes + +**Protect the state directory.** Once anything writes state, the file holds every endpoint's secret in the clear, static and dynamic alike, in the runtime's state path. That is the same at-rest posture as the TOML those endpoints would otherwise live in, but it means the state directory needs `chmod 700` and the same handling as any credential store. + +**Revocation tombstones accumulate.** They are retained deliberately, so a revocation survives a restart and stays auditable, and nothing evicts them. + +Each is roughly a hundred bytes and the whole registry is rewritten on every mutation, so a deployment that churns endpoints continuously will see the state file grow over time. An instance whose endpoints are all static writes no state file until something mutates its registry. Revoking a static endpoint through the management API does exactly that, and the tombstone it writes is what stops the TOML entry coming back. + +**`dropped_on_close` disappears with its listener.** The counter lives on the shared listener's registry, so it survives one instance of several leaving — but when the *last* instance closes, the listener and its metrics go with it. In a single-instance deployment the `warn!` log line is the only surviving record of messages lost at shutdown. + +**Sampled gauges are dropped when an instance leaves**, so `buffer_used` and `endpoints_active` do not linger at a stale value for an instance that no longer exists. The counters persist, as counters should. + +**Put a reverse proxy in front of the public listener.** It sets no header-read timeout, no idle timeout, and no connection cap, so a client that opens a socket and stops writing holds a task and a file descriptor indefinitely. The descriptor limit is shared with the rest of the runtime process, so exhaustion is not contained to this connector. + +**Keep the admin listener private.** It defaults to loopback. The management API is token-guarded, but health and metrics are not, and they expose instance names and traffic volumes. + +## Limitations + +- Best-effort delivery in both directions, as described above. +- Endpoints registered through the management API do not converge across runtime processes. +- The bridge is bounded by message count, not by bytes. +- `POST` only. There is no support for provider handshakes that require answering a `GET` challenge. +- HMAC validation covers hex-digest-of-body schemes. Composed-string schemes such as Stripe's and Twilio's need downstream verification. diff --git a/core/connectors/sources/http_source/config.toml b/core/connectors/sources/http_source/config.toml new file mode 100644 index 0000000000..6dff6742f7 --- /dev/null +++ b/core/connectors/sources/http_source/config.toml @@ -0,0 +1,65 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +type = "source" +key = "http_github" +enabled = true +version = 0 +name = "HTTP source (GitHub)" +path = "../../target/release/libiggy_connector_http_source" +verbose = false + +[[streams]] +stream = "webhooks" +topic = "github_events" +schema = "raw" +batch_length = 100 +linger_time = "5ms" + +[plugin_config] +# Public listener. Instances that name the same address share one listener. +# Naming a different address binds a second listener instead, which then fails +# on the shared admin port - so keep this identical across instances that are +# meant to share. +listen_addr = "0.0.0.0:9090" +# Management API, admin health, and metrics. Never route this through the +# public load balancer. +admin_listen_addr = "127.0.0.1:9091" +instance_name = "http_github" +# Exposes POST /topics/github_events. Omit for secret-path endpoints only. +topic_path = "github_events" +# Guards the named path. Omit when an authenticating gateway fronts it. +auth_bearer_token = "global-webhook-secret" +# Enables POST/GET/PATCH/DELETE /admin/endpoints. Omit to leave the +# management API unmounted. +management_token = "admin-secret" +max_body_size_bytes = 1048576 +buffer_capacity = 10000 +max_batch_size = 500 +include_http_metadata = true +# The provider's delivery id is the dedup key a consumer needs, because +# delivery is best-effort in both directions. See the README. +forward_headers = ["X-GitHub-Delivery", "X-Request-ID"] + +# Static secret-path endpoint: POST /e/{endpoint_id}. The URL is itself the +# credential, so generate it with `openssl rand -hex 16`. +[[plugin_config.endpoints]] +endpoint_id = "a3f8c2e1b9d04f7a8e6c1d2b3a4f5e6d" +auth_type = "hmac-sha256" +auth_secret = "whsec_github_example" +hmac_header = "X-Hub-Signature-256" +hmac_prefix = "sha256=" diff --git a/core/connectors/sources/http_source/src/auth.rs b/core/connectors/sources/http_source/src/auth.rs new file mode 100644 index 0000000000..576feca082 --- /dev/null +++ b/core/connectors/sources/http_source/src/auth.rs @@ -0,0 +1,245 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use ring::hmac; +use secrecy::{ExposeSecret, SecretString}; +use serde::{Deserialize, Serialize}; + +/// HMAC algorithms accepted for signature validation. SHA-256 covers GitHub, +/// Stripe, and most modern providers; SHA-1 exists only for legacy senders. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum HmacAlgorithm { + HmacSha256, + HmacSha1, +} + +impl HmacAlgorithm { + fn ring_algorithm(self) -> hmac::Algorithm { + match self { + Self::HmacSha256 => hmac::HMAC_SHA256, + Self::HmacSha1 => hmac::HMAC_SHA1_FOR_LEGACY_USE_ONLY, + } + } +} + +/// Validates `Authorization: Bearer ` in constant time. +pub fn validate_bearer(authorization_header: Option<&str>, expected_token: &SecretString) -> bool { + let Some(header_value) = authorization_header else { + return false; + }; + let Some(presented_token) = strip_bearer(header_value) else { + return false; + }; + constant_time_eq( + presented_token.as_bytes(), + expected_token.expose_secret().as_bytes(), + ) +} + +/// Splits `Bearer `. RFC 7235 makes the scheme case-insensitive, so a +/// sender using `bearer` is legitimate and must not be turned away. +pub fn strip_bearer(header_value: &str) -> Option<&str> { + let (scheme, token) = header_value.split_once(' ')?; + scheme.eq_ignore_ascii_case("Bearer").then_some(token) +} + +/// Compares two secrets without leaking their contents through timing. +pub fn secrets_match(left: &SecretString, right: &SecretString) -> bool { + constant_time_eq( + left.expose_secret().as_bytes(), + right.expose_secret().as_bytes(), + ) +} + +// ring deprecated its direct comparison helper; `hmac::verify` compares +// tags in constant time, so equal inputs iff the tag over one verifies +// against the other. +fn constant_time_eq(left: &[u8], right: &[u8]) -> bool { + let key = hmac::Key::new(hmac::HMAC_SHA256, &[]); + let left_tag = hmac::sign(&key, left); + hmac::verify(&key, right, left_tag.as_ref()).is_ok() +} + +/// Validates an HMAC signature over the raw request body bytes, never a +/// re-serialized form (whitespace or key-order changes would break the hash). +/// `ring::hmac::verify` compares in constant time. +pub fn validate_hmac( + body: &[u8], + signature_header: Option<&str>, + signature_prefix: &str, + secret: &SecretString, + algorithm: HmacAlgorithm, +) -> bool { + let Some(header_value) = signature_header else { + return false; + }; + let Some(signature_hex) = header_value.strip_prefix(signature_prefix) else { + return false; + }; + let Ok(expected_signature) = hex::decode(signature_hex) else { + return false; + }; + let key = hmac::Key::new( + algorithm.ring_algorithm(), + secret.expose_secret().as_bytes(), + ); + hmac::verify(&key, body, &expected_signature).is_ok() +} + +#[cfg(test)] +mod tests { + use super::*; + + const SECRET: &str = "whsec_test_secret"; + const BODY: &[u8] = br#"{"event": "push", "repository": "apache/iggy"}"#; + + fn secret() -> SecretString { + SecretString::from(SECRET) + } + + fn github_style_signature(body: &[u8], algorithm: HmacAlgorithm) -> String { + let key = hmac::Key::new(algorithm.ring_algorithm(), SECRET.as_bytes()); + let tag = hmac::sign(&key, body); + hex::encode(tag.as_ref()) + } + + #[test] + fn given_valid_token_when_bearer_validated_should_accept() { + let header = format!("Bearer {SECRET}"); + assert!(validate_bearer(Some(&header), &secret())); + } + + #[test] + fn given_wrong_token_when_bearer_validated_should_reject() { + assert!(!validate_bearer(Some("Bearer wrong"), &secret())); + } + + #[test] + fn given_missing_header_when_bearer_validated_should_reject() { + assert!(!validate_bearer(None, &secret())); + } + + #[test] + fn given_lowercase_scheme_when_bearer_validated_should_accept() { + let header = format!("bearer {SECRET}"); + assert!( + validate_bearer(Some(&header), &secret()), + "RFC 7235 makes the auth scheme case-insensitive" + ); + } + + #[test] + fn given_wrong_scheme_when_bearer_validated_should_reject() { + let header = format!("Basic {SECRET}"); + assert!(!validate_bearer(Some(&header), &secret())); + } + + #[test] + fn given_valid_sha256_signature_when_hmac_validated_should_accept() { + let signature = format!( + "sha256={}", + github_style_signature(BODY, HmacAlgorithm::HmacSha256) + ); + assert!(validate_hmac( + BODY, + Some(&signature), + "sha256=", + &secret(), + HmacAlgorithm::HmacSha256, + )); + } + + #[test] + fn given_valid_sha1_signature_when_hmac_validated_should_accept() { + let signature = format!( + "sha1={}", + github_style_signature(BODY, HmacAlgorithm::HmacSha1) + ); + assert!(validate_hmac( + BODY, + Some(&signature), + "sha1=", + &secret(), + HmacAlgorithm::HmacSha1, + )); + } + + #[test] + fn given_tampered_body_when_hmac_validated_should_reject() { + let signature = format!( + "sha256={}", + github_style_signature(BODY, HmacAlgorithm::HmacSha256) + ); + assert!(!validate_hmac( + br#"{"event": "push", "repository": "attacker/repo"}"#, + Some(&signature), + "sha256=", + &secret(), + HmacAlgorithm::HmacSha256, + )); + } + + #[test] + fn given_missing_signature_when_hmac_validated_should_reject() { + assert!(!validate_hmac( + BODY, + None, + "sha256=", + &secret(), + HmacAlgorithm::HmacSha256, + )); + } + + #[test] + fn given_wrong_prefix_when_hmac_validated_should_reject() { + let signature = format!( + "sha1={}", + github_style_signature(BODY, HmacAlgorithm::HmacSha256) + ); + assert!(!validate_hmac( + BODY, + Some(&signature), + "sha256=", + &secret(), + HmacAlgorithm::HmacSha256, + )); + } + + #[test] + fn given_malformed_hex_when_hmac_validated_should_reject() { + assert!(!validate_hmac( + BODY, + Some("sha256=not-hex-at-all"), + "sha256=", + &secret(), + HmacAlgorithm::HmacSha256, + )); + } + + #[test] + fn given_empty_prefix_when_hmac_validated_should_accept_raw_hex() { + let signature = github_style_signature(BODY, HmacAlgorithm::HmacSha256); + assert!(validate_hmac( + BODY, + Some(&signature), + "", + &secret(), + HmacAlgorithm::HmacSha256, + )); + } +} diff --git a/core/connectors/sources/http_source/src/lib.rs b/core/connectors/sources/http_source/src/lib.rs new file mode 100644 index 0000000000..f8821a7e0b --- /dev/null +++ b/core/connectors/sources/http_source/src/lib.rs @@ -0,0 +1,1036 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub mod auth; +pub mod management; +pub mod metrics; +pub mod routes; +pub mod server; +pub mod state; +pub mod types; + +use arc_swap::{ArcSwap, Guard}; +use async_trait::async_trait; +use axum::http::HeaderName; +use iggy_common::HeaderKey; +use iggy_connector_sdk::{ + ConnectorState, Error, ProducedMessages, Schema, Source, source_connector, +}; +use secrecy::{ExposeSecret, SecretString}; +use serde::{Deserialize, Serialize}; +use std::net::SocketAddr; +use std::str::FromStr; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Duration; +use tokio::sync::{Mutex, Notify}; +use tracing::{debug, info}; + +use crate::auth::HmacAlgorithm; +use crate::state::EndpointRegistry; +use crate::types::{EndpointId, QueuedMessage, unix_now_seconds}; + +pub const CONNECTOR_NAME: &str = "HTTP source"; + +pub const DEFAULT_ADMIN_LISTEN_ADDR: &str = "127.0.0.1:9091"; +pub const DEFAULT_MAX_BODY_SIZE_BYTES: usize = 1024 * 1024; +pub const DEFAULT_BUFFER_CAPACITY: usize = 10_000; +pub const DEFAULT_MAX_BATCH_SIZE: usize = 500; +pub const DEFAULT_HMAC_HEADER: &str = "X-Hub-Signature-256"; +pub const DEFAULT_HMAC_PREFIX: &str = "sha256="; + +const IDLE_POLL_INTERVAL: Duration = Duration::from_millis(100); + +/// HTTP handler side of an instance's bridge. The pair comes from +/// `bounded_async` because the `poll()` side needs `recv().await`; the handler +/// side only ever `try_send`s. +pub type MessageSender = crossfire::MAsyncTx>; + +/// `poll()` side of an instance's bridge. +pub type MessageReceiver = crossfire::AsyncRx>; + +source_connector!(HttpSource); + +/// Webhook gateway source: accepts HTTP POST requests on a listener shared +/// by every instance of this plugin and produces the raw bodies to the +/// instance's configured stream/topic. +#[derive(Debug)] +pub struct HttpSource { + pub id: u32, + shared: Arc, + /// Deliberately not on [`SharedState`]: handlers hold that behind an `Arc` + /// for as long as the listener serves them, and a receiver reachable from + /// there would outlive the source it belongs to. + /// + /// The mutex only exists to make the single-consumer receiver `Sync`, as + /// [`Source`] requires. The runtime drives exactly one `poll()` at a time, + /// so it is never contended despite being held across the wait. + receiver: Mutex, +} + +/// Everything an HTTP handler needs from one instance. +/// +/// Handler tasks hold `Arc` and never `Arc`: the SDK +/// tears a source down with `Arc::try_unwrap`, so a stray clone of the source +/// itself would turn `close()` into a leak. +#[derive(Debug)] +pub struct SharedState { + pub id: u32, + pub config: HttpSourceConfig, + pub sender: MessageSender, + /// Resolved once from `instance_name`, falling back to the connector ID. + pub instance_name: String, + /// Each configured request header paired with the Iggy header key it lands + /// under, resolved once so the request path neither parses nor validates. + pub(crate) forward_headers: Vec<(HeaderName, HeaderKey)>, + registry: ArcSwap, + registry_dirty: AtomicBool, + /// Serializes registry writers, which are all control-plane. The request + /// path only ever loads the `ArcSwap` and never touches this. + registry_writer: Mutex<()>, + /// Wakes `poll()` when a management mutation needs a state flush and no + /// webhook traffic would otherwise arrive to carry it. + pub state_flush: Notify, +} + +impl SharedState { + /// Wait-free snapshot of the registry, one atomic load per request. + pub fn registry(&self) -> Guard> { + self.registry.load() + } + + /// Applies a control-plane mutation and arms the next state flush. + pub async fn mutate_registry(&self, mutation: impl FnOnce(&mut EndpointRegistry) -> R) -> R { + let outcome = { + let _writer = self.registry_writer.lock().await; + let mut next = EndpointRegistry::clone(&self.registry.load()); + let outcome = mutation(&mut next); + self.registry.store(Arc::new(next)); + self.registry_dirty.store(true, Ordering::Release); + outcome + }; + // Notified after the gate is free, so the woken poll finds it open + // rather than bouncing off `try_lock` and relying on the re-arm there. + self.state_flush.notify_one(); + outcome + } + + /// Whether a mutation is still waiting to be handed to the runtime. + pub fn has_pending_state(&self) -> bool { + self.registry_dirty.load(Ordering::Acquire) + } + + /// Hands the registry to the runtime for persistence, once per mutation. + /// + /// Only ever called for an empty batch. The runtime saves state solely on + /// the success branch of the Iggy send, and an empty send always succeeds, + /// so attaching state to a batch that could fail would let the save be + /// skipped while this side had already cleared the flag and marked the + /// registry submitted - losing a revocation tombstone with no trace. + /// + /// Static-only instances never arm the flag, so their polls return + /// `state: None` and the runtime writes no state file at all. + pub fn take_dirty_state(&self) -> Option { + // `try_lock`, never `lock().await`. `poll()` calls this holding a + // batch that has already left the bridge, and the SDK drops the poll + // future on shutdown; an await here would lose those messages + // uncounted. Contention comes only from a concurrent management call, + // and losing that race just leaves the flag armed for the next poll. + let Ok(_writer) = self.registry_writer.try_lock() else { + // Mid-mutation. If the permit that woke this poll was theirs we + // have just consumed it, so re-arm rather than sleep on a flush + // that no further traffic would ever carry. + self.state_flush.notify_one(); + return None; + }; + if !self.registry_dirty.swap(false, Ordering::AcqRel) { + return None; + } + let snapshot = self.registry.load_full(); + let Some(state) = snapshot.to_connector_state(self.id) else { + // Serialization logged the cause. Re-arm and re-notify, or the + // retry this promises would wait for unrelated traffic. + self.registry_dirty.store(true, Ordering::Release); + self.state_flush.notify_one(); + return None; + }; + let mut persisted = EndpointRegistry::clone(&snapshot); + persisted.mark_submitted(); + self.registry.store(Arc::new(persisted)); + Some(state) + } +} + +/// Deliberately not `Serialize`. The runtime keeps plugin configuration as raw +/// JSON and never serializes this struct, so nothing needs it — and without it +/// the compiler guarantees a credential cannot be written out by some future +/// caller. `SecretString` has no `Serialize` impl for exactly that reason. +#[derive(Debug, Clone, Deserialize)] +pub struct HttpSourceConfig { + /// Public listener shared by all instances of this plugin; every + /// instance must configure the identical address. + pub listen_addr: String, + /// Management + observability listener; never route it through the + /// public load balancer. + #[serde(default = "default_admin_listen_addr")] + pub admin_listen_addr: String, + #[serde(default = "default_max_body_size_bytes")] + pub max_body_size_bytes: usize, + /// Capacity of this instance's HTTP-to-poll bridge, in messages. A full + /// bridge answers 429. + #[serde(default = "default_buffer_capacity")] + pub buffer_capacity: usize, + /// Maximum messages returned by a single `poll()`. + #[serde(default = "default_max_batch_size")] + pub max_batch_size: usize, + /// Path segment exposed as `POST /topics/{topic_path}`. Unset disables + /// the named path, leaving only secret-path endpoints. + #[serde(default)] + pub topic_path: Option, + /// Identifies this instance in forwarded message headers and on the admin + /// listener. Defaults to the connector ID: the runtime keeps the connector + /// key on its own side of the FFI, so the plugin cannot read it. + #[serde(default)] + pub instance_name: Option, + /// Guards the named topic path; unset leaves that path unauthenticated + /// (for deployments fronted by an authenticating gateway). + #[serde(default)] + pub auth_bearer_token: Option, + /// Enables `/admin/endpoints`; unset disables dynamic management. + #[serde(default)] + pub management_token: Option, + #[serde(default = "default_true")] + pub include_http_metadata: bool, + /// HTTP request headers forwarded as Iggy message headers, clamped to + /// the 255-byte `HeaderValue` limit. + #[serde(default)] + pub forward_headers: Vec, + /// Statically configured secret-path endpoints. + #[serde(default)] + pub endpoints: Vec, + #[serde(default)] + pub verbose_logging: Option, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct StaticEndpointConfig { + pub endpoint_id: EndpointId, + #[serde(default)] + pub auth_type: EndpointAuthType, + #[serde(default)] + pub auth_secret: Option, + #[serde(default = "default_hmac_header")] + pub hmac_header: String, + #[serde(default = "default_hmac_prefix")] + pub hmac_prefix: String, + /// Unix seconds; requests arriving at or after this answer 410 Gone. + #[serde(default)] + pub expires_at: Option, +} + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum EndpointAuthType { + #[default] + None, + Bearer, + HmacSha256, + HmacSha1, +} + +impl EndpointAuthType { + pub fn hmac_algorithm(self) -> Option { + match self { + Self::HmacSha256 => Some(HmacAlgorithm::HmacSha256), + Self::HmacSha1 => Some(HmacAlgorithm::HmacSha1), + Self::None | Self::Bearer => None, + } + } +} + +impl HttpSourceConfig { + fn validate(&self) -> Result<(), Error> { + self.listen_addr.parse::().map_err(|error| { + Error::InvalidConfigValue(format!("listen_addr '{}': {error}", self.listen_addr)) + })?; + self.admin_listen_addr + .parse::() + .map_err(|error| { + Error::InvalidConfigValue(format!( + "admin_listen_addr '{}': {error}", + self.admin_listen_addr + )) + })?; + if self.buffer_capacity == 0 { + return Err(Error::InvalidConfigValue( + "buffer_capacity must be at least 1".to_string(), + )); + } + if self.max_batch_size == 0 { + return Err(Error::InvalidConfigValue( + "max_batch_size must be at least 1".to_string(), + )); + } + if self.max_body_size_bytes == 0 { + return Err(Error::InvalidConfigValue( + "max_body_size_bytes must be at least 1".to_string(), + )); + } + if let Some(instance_name) = &self.instance_name + && HeaderKey::try_from(instance_name.as_str()).is_err() + { + return Err(Error::InvalidConfigValue(format!( + "instance_name '{instance_name}' must be a valid Iggy header key: non-empty and at most 255 bytes" + ))); + } + for header in &self.forward_headers { + // Forwarding a reusable credential would copy it onto every + // message and into the log Iggy persists. A per-body signature + // header is fine; these are not. + if matches!( + header.to_ascii_lowercase().as_str(), + "authorization" | "proxy-authorization" | "cookie" + ) { + return Err(Error::InvalidConfigValue(format!( + "forward_headers entry '{header}' would copy a credential onto every message" + ))); + } + if HeaderName::from_str(header).is_err() { + return Err(Error::InvalidConfigValue(format!( + "forward_headers entry '{header}' is not a valid HTTP header name" + ))); + } + if HeaderKey::try_from(header.as_str()).is_err() { + return Err(Error::InvalidConfigValue(format!( + "forward_headers entry '{header}' is not a valid Iggy header key" + ))); + } + } + if let Some(topic_path) = &self.topic_path + && (topic_path.is_empty() || topic_path.contains('/')) + { + return Err(Error::InvalidConfigValue(format!( + "topic_path '{topic_path}' must be a single non-empty path segment" + ))); + } + for header in [ + ("auth_bearer_token", &self.auth_bearer_token), + ("management_token", &self.management_token), + ] { + if let (field, Some(secret)) = header + && secret.expose_secret().is_empty() + { + return Err(Error::InvalidConfigValue(format!( + "{field} must not be empty; omit it entirely to disable that guard" + ))); + } + } + for endpoint in &self.endpoints { + if endpoint.auth_type != EndpointAuthType::None + && !endpoint + .auth_secret + .as_ref() + .is_some_and(|secret| !secret.expose_secret().is_empty()) + { + // An empty key is perfectly valid for HMAC, so accepting one + // would leave `auth_type` advertising a second factor that + // anyone holding the URL can compute. + // Prefix only: this becomes `last_error`, which the runtime + // logs and serves over its control API, and the operator will + // fix the secret while keeping the id. + return Err(Error::InvalidConfigValue(format!( + "endpoint {} declares auth_type {:?} but no non-empty auth_secret", + endpoint.endpoint_id.log_prefix(), + endpoint.auth_type + ))); + } + } + Ok(()) + } +} + +impl HttpSource { + pub fn new(id: u32, config: HttpSourceConfig, state: Option) -> Self { + let registry = EndpointRegistry::restore(&config.endpoints, state, id); + let (sender, receiver) = crossfire::mpsc::bounded_async(config.buffer_capacity); + let instance_name = config + .instance_name + .clone() + .unwrap_or_else(|| id.to_string()); + // Entries that fail to resolve are rejected by `validate()` in + // `open()`, so the instance never serves traffic with a silent gap. + let forward_headers = config + .forward_headers + .iter() + .filter_map(|header| { + let name = HeaderName::from_str(header).ok()?; + let key = HeaderKey::try_from(header.as_str()).ok()?; + Some((name, key)) + }) + .collect(); + let shared = SharedState { + id, + config, + sender, + instance_name, + forward_headers, + registry: ArcSwap::from_pointee(registry), + registry_dirty: AtomicBool::new(false), + registry_writer: Mutex::new(()), + state_flush: Notify::new(), + }; + HttpSource { + id, + shared: Arc::new(shared), + receiver: Mutex::new(receiver), + } + } + + pub fn shared(&self) -> &Arc { + &self.shared + } +} + +#[async_trait] +impl Source for HttpSource { + async fn open(&mut self) -> Result<(), Error> { + self.shared.config.validate()?; + server::join(Arc::clone(&self.shared)).await?; + info!( + "Opened {CONNECTOR_NAME} connector ID: {}, listen address: {}, endpoints: {}, named path: {:?}", + self.id, + self.shared.config.listen_addr, + self.shared.registry().serving_count(unix_now_seconds()), + self.shared.config.topic_path, + ); + Ok(()) + } + + async fn poll(&self) -> Result { + let max_batch_size = self.shared.config.max_batch_size; + let mut messages = Vec::with_capacity(max_batch_size); + let receiver = self.receiver.lock().await; + tokio::select! { + // The SDK races poll() against its own shutdown watch, so blocking + // here until traffic arrives is what keeps an idle gateway off the + // CPU. crossfire documents recv() as cancellation-safe. + received = receiver.recv() => match received { + Ok(message) => { + messages.push(message.into()); + while messages.len() < max_batch_size { + let Ok(message) = receiver.try_recv() else { + break; + }; + messages.push(message.into()); + } + } + // Reachable only once every sender is gone. Idle rather than + // spin the SDK's poll loop. + Err(_) => tokio::time::sleep(IDLE_POLL_INTERVAL).await, + }, + _ = self.shared.state_flush.notified() => {} + } + + if !messages.is_empty() { + let count = messages.len(); + if self.shared.config.verbose_logging.unwrap_or(false) { + info!( + "Polled {count} messages for {CONNECTOR_NAME} connector ID: {}", + self.id + ); + } else { + debug!( + "Polled {count} messages for {CONNECTOR_NAME} connector ID: {}", + self.id + ); + } + } + + // State rides an empty batch and nothing else, so the send it depends + // on cannot fail. Under traffic that means deferring to a later poll; + // re-arming the notify is what stops it waiting on traffic to arrive. + let state = if messages.is_empty() { + self.shared.take_dirty_state() + } else { + if self.shared.has_pending_state() { + self.shared.state_flush.notify_one(); + } + None + }; + + Ok(ProducedMessages { + schema: Schema::Raw, + messages, + state, + }) + } + + async fn close(&mut self) -> Result<(), Error> { + // The SDK stops the poll task before calling this, so anything still + // in the bridge is already unreachable. Deregistering first is what + // stops new requests from being accepted into a queue nobody drains. + server::leave(&self.shared).await; + info!("Closed {CONNECTOR_NAME} connector ID: {}", self.id); + Ok(()) + } +} + +fn default_admin_listen_addr() -> String { + DEFAULT_ADMIN_LISTEN_ADDR.to_string() +} + +fn default_max_body_size_bytes() -> usize { + DEFAULT_MAX_BODY_SIZE_BYTES +} + +fn default_buffer_capacity() -> usize { + DEFAULT_BUFFER_CAPACITY +} + +fn default_max_batch_size() -> usize { + DEFAULT_MAX_BATCH_SIZE +} + +fn default_hmac_header() -> String { + DEFAULT_HMAC_HEADER.to_string() +} + +fn default_hmac_prefix() -> String { + DEFAULT_HMAC_PREFIX.to_string() +} + +fn default_true() -> bool { + true +} + +/// Fixtures shared by the routing and state test modules. +#[cfg(test)] +pub(crate) mod test_support { + use super::*; + + pub const ENDPOINT_ONE: &str = "a3f8c2e1b9d04f7a8e6c1d2b3a4f5e6d"; + pub const ENDPOINT_TWO: &str = "b4092d3fa1c85e6b7d0f2a1c3e4b5d6a"; + + pub fn endpoint_id(raw_id: &str) -> EndpointId { + raw_id.parse().expect("test endpoint id must be valid") + } + + pub fn static_endpoint(raw_id: &str) -> StaticEndpointConfig { + StaticEndpointConfig { + endpoint_id: endpoint_id(raw_id), + auth_type: EndpointAuthType::HmacSha256, + auth_secret: Some(SecretString::from("whsec_static")), + hmac_header: DEFAULT_HMAC_HEADER.to_string(), + hmac_prefix: DEFAULT_HMAC_PREFIX.to_string(), + expires_at: None, + } + } + + pub fn config(topic_path: Option<&str>, endpoint_ids: &[&str]) -> HttpSourceConfig { + HttpSourceConfig { + listen_addr: "127.0.0.1:9090".to_string(), + admin_listen_addr: DEFAULT_ADMIN_LISTEN_ADDR.to_string(), + max_body_size_bytes: DEFAULT_MAX_BODY_SIZE_BYTES, + buffer_capacity: DEFAULT_BUFFER_CAPACITY, + max_batch_size: DEFAULT_MAX_BATCH_SIZE, + topic_path: topic_path.map(str::to_string), + instance_name: None, + auth_bearer_token: None, + management_token: None, + include_http_metadata: true, + forward_headers: Vec::new(), + endpoints: endpoint_ids + .iter() + .map(|raw_id| static_endpoint(raw_id)) + .collect(), + verbose_logging: None, + } + } + + /// Reserves an ephemeral port and hands it back. Every test that binds + /// needs its own, because the listener registry is process-global and the + /// test binary runs its cases in parallel. + pub fn free_port() -> u16 { + std::net::TcpListener::bind("127.0.0.1:0") + .expect("the loopback interface must offer a port") + .local_addr() + .expect("a bound listener has an address") + .port() + } + + /// No connection pooling: an idle keep-alive socket would hold graceful + /// shutdown open until the timeout and slow every teardown to a crawl. + pub fn client() -> reqwest::Client { + reqwest::Client::builder() + .pool_max_idle_per_host(0) + .build() + .expect("the test client must build") + } + + /// Builds one instance's shared state through the real constructor. The + /// source itself is dropped, taking the bridge receiver with it: routing + /// and state tests resolve and mutate, they never send. + pub fn instance(id: u32, topic_path: Option<&str>, endpoint_ids: &[&str]) -> Arc { + let source = HttpSource::new(id, config(topic_path, endpoint_ids), None); + Arc::clone(source.shared()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_support::{ENDPOINT_ONE, ENDPOINT_TWO}; + use std::time::Instant; + + fn minimal_config_json() -> &'static str { + r#"{"listen_addr": "0.0.0.0:9090"}"# + } + + fn parse(config_json: &str) -> HttpSourceConfig { + serde_json::from_str(config_json).expect("config must deserialize") + } + + #[test] + fn given_minimal_config_when_deserialized_should_apply_defaults() { + let config = parse(minimal_config_json()); + assert_eq!(config.admin_listen_addr, DEFAULT_ADMIN_LISTEN_ADDR); + assert_eq!(config.max_body_size_bytes, DEFAULT_MAX_BODY_SIZE_BYTES); + assert_eq!(config.buffer_capacity, DEFAULT_BUFFER_CAPACITY); + assert_eq!(config.max_batch_size, DEFAULT_MAX_BATCH_SIZE); + assert!(config.include_http_metadata); + assert!(config.topic_path.is_none()); + assert!(config.auth_bearer_token.is_none()); + assert!(config.management_token.is_none()); + assert!(config.forward_headers.is_empty()); + assert!(config.endpoints.is_empty()); + } + + #[test] + fn given_minimal_config_when_validated_should_accept() { + assert!(parse(minimal_config_json()).validate().is_ok()); + } + + #[test] + fn given_unparsable_listen_addr_when_validated_should_reject() { + let config = parse(r#"{"listen_addr": "not-an-address"}"#); + assert!(matches!( + config.validate(), + Err(Error::InvalidConfigValue(message)) if message.contains("listen_addr") + )); + } + + #[test] + fn given_zero_buffer_capacity_when_validated_should_reject() { + let config = parse(r#"{"listen_addr": "0.0.0.0:9090", "buffer_capacity": 0}"#); + assert!(matches!( + config.validate(), + Err(Error::InvalidConfigValue(message)) if message.contains("buffer_capacity") + )); + } + + #[test] + fn given_topic_path_with_slash_when_validated_should_reject() { + let config = parse(r#"{"listen_addr": "0.0.0.0:9090", "topic_path": "a/b"}"#); + assert!(matches!( + config.validate(), + Err(Error::InvalidConfigValue(message)) if message.contains("topic_path") + )); + } + + #[test] + fn given_endpoint_with_auth_but_no_secret_when_validated_should_reject() { + let config = parse( + r#"{ + "listen_addr": "0.0.0.0:9090", + "endpoints": [{ + "endpoint_id": "a3f8c2e1b9d04f7a8e6c1d2b3a4f5e6d", + "auth_type": "hmac-sha256" + }] + }"#, + ); + assert!(matches!( + config.validate(), + Err(Error::InvalidConfigValue(message)) if message.contains("auth_secret") + )); + } + + #[test] + fn given_endpoint_config_when_deserialized_should_apply_hmac_defaults() { + let config = parse( + r#"{ + "listen_addr": "0.0.0.0:9090", + "endpoints": [{ + "endpoint_id": "a3f8c2e1b9d04f7a8e6c1d2b3a4f5e6d", + "auth_type": "hmac-sha256", + "auth_secret": "whsec_test" + }] + }"#, + ); + let endpoint = &config.endpoints[0]; + assert_eq!(endpoint.hmac_header, DEFAULT_HMAC_HEADER); + assert_eq!(endpoint.hmac_prefix, DEFAULT_HMAC_PREFIX); + assert_eq!( + endpoint.auth_type.hmac_algorithm(), + Some(HmacAlgorithm::HmacSha256) + ); + assert!(config.validate().is_ok()); + } + + #[test] + fn given_zero_max_batch_size_when_validated_should_reject() { + let config = parse(r#"{"listen_addr": "0.0.0.0:9090", "max_batch_size": 0}"#); + assert!(matches!( + config.validate(), + Err(Error::InvalidConfigValue(message)) if message.contains("max_batch_size") + )); + } + + #[test] + fn given_zero_max_body_size_when_validated_should_reject() { + let config = parse(r#"{"listen_addr": "0.0.0.0:9090", "max_body_size_bytes": 0}"#); + assert!(matches!( + config.validate(), + Err(Error::InvalidConfigValue(message)) if message.contains("max_body_size_bytes") + )); + } + + #[test] + fn given_unparsable_admin_addr_when_validated_should_reject() { + let config = parse(r#"{"listen_addr": "0.0.0.0:9090", "admin_listen_addr": "nope"}"#); + assert!(matches!( + config.validate(), + Err(Error::InvalidConfigValue(message)) if message.contains("admin_listen_addr") + )); + } + + #[test] + fn given_empty_instance_name_when_validated_should_reject() { + let config = parse(r#"{"listen_addr": "0.0.0.0:9090", "instance_name": ""}"#); + assert!(matches!( + config.validate(), + Err(Error::InvalidConfigValue(message)) if message.contains("instance_name") + )); + } + + #[test] + fn given_oversized_instance_name_when_validated_should_reject() { + // It becomes a HeaderKey, and Iggy caps those at 255 bytes. Without + // this check the identity header would vanish from every message. + let long_name = "n".repeat(256); + let config = parse(&format!( + r#"{{"listen_addr": "0.0.0.0:9090", "instance_name": "{long_name}"}}"# + )); + assert!(matches!( + config.validate(), + Err(Error::InvalidConfigValue(message)) if message.contains("instance_name") + )); + } + + #[test] + fn given_invalid_forward_header_when_validated_should_reject() { + // `new()` silently filters entries that fail to resolve, on the stated + // assumption that validate() rejects them first. That is the test. + let config = + parse(r#"{"listen_addr": "0.0.0.0:9090", "forward_headers": ["not a header"]}"#); + assert!(matches!( + config.validate(), + Err(Error::InvalidConfigValue(message)) if message.contains("forward_headers") + )); + } + + #[test] + fn given_empty_endpoint_secret_when_validated_should_reject() { + // An empty key is valid for HMAC, so accepting one would advertise a + // second factor that anyone holding the URL can compute. + let config = parse( + r#"{ + "listen_addr": "0.0.0.0:9090", + "endpoints": [{ + "endpoint_id": "a3f8c2e1b9d04f7a8e6c1d2b3a4f5e6d", + "auth_type": "hmac-sha256", + "auth_secret": "" + }] + }"#, + ); + assert!(matches!( + config.validate(), + Err(Error::InvalidConfigValue(message)) if message.contains("auth_secret") + )); + } + + #[test] + fn given_empty_management_token_when_validated_should_reject() { + let config = parse(r#"{"listen_addr": "0.0.0.0:9090", "management_token": ""}"#); + assert!(matches!( + config.validate(), + Err(Error::InvalidConfigValue(message)) if message.contains("management_token") + )); + } + + #[test] + fn given_credential_forward_header_when_validated_should_reject() { + for header in ["Authorization", "cookie", "PROXY-AUTHORIZATION"] { + let config = parse(&format!( + r#"{{"listen_addr": "0.0.0.0:9090", "forward_headers": ["{header}"]}}"# + )); + assert!( + matches!( + config.validate(), + Err(Error::InvalidConfigValue(message)) if message.contains("credential") + ), + "{header} would be copied onto every message and persisted in the log" + ); + } + } + + #[test] + fn given_invalid_endpoint_id_when_deserialized_should_reject() { + let result = serde_json::from_str::( + r#"{ + "listen_addr": "0.0.0.0:9090", + "endpoints": [{"endpoint_id": "too-short"}] + }"#, + ); + assert!(result.is_err(), "invalid endpoint_id must fail at parse"); + } + + /// The shipped configurations are documentation, and documentation that + /// no longer deserializes is worse than none. + #[test] + fn given_shipped_configs_when_parsed_should_deserialize_and_validate() { + for (name, raw) in [ + ("config.toml", include_str!("../config.toml")), + ( + "example_config/http_source_github.toml", + include_str!("../../../runtime/example_config/connectors/http_source_github.toml"), + ), + ( + "example_config/http_source_partner.toml", + include_str!("../../../runtime/example_config/connectors/http_source_partner.toml"), + ), + ] { + let document: toml::Value = + toml::from_str(raw).unwrap_or_else(|error| panic!("{name} must parse: {error}")); + let plugin_config = document + .get("plugin_config") + .unwrap_or_else(|| panic!("{name} must carry a plugin_config table")) + .clone(); + let config: HttpSourceConfig = plugin_config + .try_into() + .unwrap_or_else(|error| panic!("{name} plugin_config must deserialize: {error}")); + config + .validate() + .unwrap_or_else(|error| panic!("{name} must pass validation: {error}")); + } + } + + fn queued(payload: &str) -> QueuedMessage { + QueuedMessage { + payload: payload.as_bytes().to_vec(), + headers: None, + received_at: Instant::now(), + } + } + + #[tokio::test] + async fn given_persisted_state_when_constructed_should_restore_into_the_shared_registry() { + let mut persisted = EndpointRegistry::default(); + assert!(persisted.insert(crate::routes::Endpoint { + endpoint_id: test_support::endpoint_id(ENDPOINT_TWO), + auth_type: EndpointAuthType::Bearer, + auth_secret: Some(SecretString::from("whsec_dynamic")), + hmac_header: DEFAULT_HMAC_HEADER.to_string(), + hmac_prefix: DEFAULT_HMAC_PREFIX.to_string(), + expires_at: None, + origin: crate::routes::EndpointOrigin::Dynamic, + state: crate::routes::EndpointState::Active, + submitted: false, + })); + + let source = HttpSource::new( + 1, + test_support::config(None, &[ENDPOINT_ONE]), + persisted.to_connector_state(1), + ); + + let registry = source.shared.registry(); + assert!( + registry.endpoint(ENDPOINT_TWO).is_some(), + "the constructor must hand persisted state to the registry, not drop it" + ); + assert!(registry.endpoint(ENDPOINT_ONE).is_some()); + } + + #[tokio::test] + async fn given_static_only_instance_when_state_taken_should_stay_none() { + let source = HttpSource::new(1, test_support::config(None, &[ENDPOINT_ONE]), None); + + assert!( + source.shared.take_dirty_state().is_none(), + "a static-only instance must behave like a stateless source" + ); + } + + #[tokio::test] + async fn given_registry_mutation_when_state_taken_should_return_it_once() { + let source = HttpSource::new(1, test_support::config(None, &[ENDPOINT_ONE]), None); + source + .shared + .mutate_registry(|registry| registry.revoke(ENDPOINT_ONE, "rotated".to_string(), 42)) + .await; + + assert!(source.shared.take_dirty_state().is_some()); + assert!( + source.shared.take_dirty_state().is_none(), + "an unchanged registry must not be rewritten every poll" + ); + } + + #[tokio::test] + async fn given_state_taken_when_registry_read_should_report_endpoints_submitted() { + let source = HttpSource::new(1, test_support::config(None, &[ENDPOINT_ONE]), None); + source + .shared + .mutate_registry(|registry| registry.revoke(ENDPOINT_ONE, "rotated".to_string(), 42)) + .await; + + assert!( + !source + .shared + .registry() + .endpoint(ENDPOINT_ONE) + .expect("endpoint must exist") + .submitted + ); + source.shared.take_dirty_state(); + assert!( + source + .shared + .registry() + .endpoint(ENDPOINT_ONE) + .expect("endpoint must exist") + .submitted + ); + } + + #[tokio::test] + async fn given_pending_mutation_when_polled_should_flush_state_on_an_empty_batch() { + let source = HttpSource::new(1, test_support::config(None, &[ENDPOINT_ONE]), None); + source + .shared + .mutate_registry(|registry| registry.revoke(ENDPOINT_ONE, "rotated".to_string(), 42)) + .await; + + let produced = source.poll().await.expect("poll must succeed"); + + assert!(produced.messages.is_empty()); + assert!( + produced.state.is_some(), + "a mutation must reach the runtime without waiting for traffic" + ); + } + + #[tokio::test] + async fn given_pending_mutation_and_traffic_when_polled_should_defer_the_flush() { + let mut config = test_support::config(None, &[ENDPOINT_ONE]); + config.max_batch_size = 1; + let source = HttpSource::new(1, config, None); + source + .shared + .mutate_registry(|registry| registry.revoke(ENDPOINT_ONE, "rotated".to_string(), 42)) + .await; + source + .shared + .sender + .try_send(queued("one")) + .expect("bridge must accept"); + + // Consume the permit the mutation armed, so `select!` has exactly one + // ready branch and the batch-carrying poll is deterministic. Asserting + // across both polls instead would only catch the regression when the + // random branch order happened to cooperate. + source.shared.state_flush.notified().await; + + let carried = source.poll().await.expect("poll must succeed"); + assert_eq!(carried.messages.len(), 1); + assert!( + carried.state.is_none(), + "state must never ride a batch whose send can fail, or a failed send loses it silently" + ); + assert!( + source.shared.has_pending_state(), + "and the deferred flush must still be armed" + ); + + let flushed = source.poll().await.expect("poll must succeed"); + assert!(flushed.messages.is_empty()); + assert!( + flushed.state.is_some(), + "the re-arm must carry it without waiting for further traffic" + ); + } + + #[tokio::test] + async fn given_queued_messages_when_polled_should_drain_up_to_max_batch_size() { + let mut config = test_support::config(None, &[]); + config.max_batch_size = 2; + let source = HttpSource::new(1, config, None); + for payload in ["one", "two", "three"] { + source + .shared + .sender + .try_send(queued(payload)) + .expect("bridge must accept within capacity"); + } + + let first = source.poll().await.expect("poll must succeed"); + let second = source.poll().await.expect("poll must succeed"); + + assert_eq!(first.messages.len(), 2); + assert_eq!(second.messages.len(), 1); + assert_eq!(first.messages[0].payload, b"one"); + assert!(matches!(first.schema, Schema::Raw)); + assert!(first.state.is_none()); + } + + #[tokio::test] + async fn given_full_bridge_when_message_sent_should_reject() { + let mut config = test_support::config(None, &[]); + config.buffer_capacity = 1; + let source = HttpSource::new(1, config, None); + + assert!(source.shared.sender.try_send(queued("one")).is_ok()); + assert!( + source.shared.sender.try_send(queued("two")).is_err(), + "a full bridge is what turns into a 429" + ); + } + + // Shutdown drains whatever the handlers already accepted, and crossfire's + // docs stop short of promising it. + #[tokio::test] + async fn given_dropped_senders_when_received_should_drain_buffered_messages() { + let (sender, receiver) = crossfire::mpsc::bounded_async::(4); + sender.try_send(queued("one")).expect("capacity is free"); + sender.try_send(queued("two")).expect("capacity is free"); + drop(sender); + + assert_eq!(receiver.recv().await.expect("buffered").payload, b"one"); + assert_eq!(receiver.recv().await.expect("buffered").payload, b"two"); + assert!(receiver.recv().await.is_err(), "then the bridge is closed"); + } +} diff --git a/core/connectors/sources/http_source/src/management.rs b/core/connectors/sources/http_source/src/management.rs new file mode 100644 index 0000000000..2596dc9710 --- /dev/null +++ b/core/connectors/sources/http_source/src/management.rs @@ -0,0 +1,869 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! The endpoint management API, served on the admin listener only. +//! +//! The operations here are the ones a TOML edit plus a restart is the wrong +//! tool for: revoking a compromised endpoint is time-critical, and a platform +//! provisioning a webhook URL per tenant is inherently programmatic. +//! +//! Disabled unless `management_token` is set, and never reachable from the +//! public listener. Mutations become durable on the next successful poll, so +//! a response here means "accepted", not "written"; `GET /admin/endpoints` +//! reports `submitted` per endpoint for callers that need the difference. + +use axum::extract::{Path, State}; +use axum::http::{HeaderMap, StatusCode, header}; +use axum::response::{IntoResponse, Response}; +use axum::routing::{get, post}; +use axum::{Json, Router}; +use rand::RngExt; +use secrecy::{ExposeSecret, SecretString}; +use serde::{Deserialize, Serialize}; +use std::sync::Arc; +use tracing::{error, info, warn}; + +use crate::auth::{secrets_match, strip_bearer}; +use crate::routes::{Endpoint, EndpointOrigin, EndpointState}; +use crate::server::{ServerState, error_response, refresh_routes}; +use crate::types::{EndpointId, unix_now_seconds}; +use crate::{CONNECTOR_NAME, EndpointAuthType, SharedState}; + +/// Bytes of entropy behind a generated endpoint id. The URL is the bearer +/// token for a secret-path endpoint, so it carries the whole secret. +const ENDPOINT_ID_BYTES: usize = 16; + +pub(crate) fn router(state: Arc) -> Router> { + if state.management_token.is_none() { + info!( + "Dynamic endpoint management is disabled for the {CONNECTOR_NAME} listener on {}: no management_token is configured", + state.listen_addr + ); + return Router::new(); + } + info!( + "Enabled dynamic endpoint management for the {CONNECTOR_NAME} listener on {}", + state.listen_addr + ); + Router::new() + .route( + "/admin/endpoints", + post(register_endpoint).get(list_endpoints), + ) + .route( + "/admin/endpoints/{endpoint_id}", + get(get_endpoint) + .patch(rotate_secret) + .delete(revoke_endpoint), + ) +} + +async fn register_endpoint( + State(state): State>, + request_headers: HeaderMap, + body: Result, axum::extract::rejection::JsonRejection>, +) -> Response { + if let Some(response) = denied(&state, &request_headers) { + return response; + } + let Json(request) = match body { + Ok(body) => body, + Err(rejection) => return error_response(rejection.status(), "invalid request body"), + }; + + let Some(instance) = state.instance(&request.instance) else { + return error_response(StatusCode::NOT_FOUND, "unknown instance"); + }; + if request.auth_type != EndpointAuthType::None && !is_usable(&request.auth_secret) { + return error_response( + StatusCode::BAD_REQUEST, + "a non-empty auth_secret is required", + ); + } + if request + .expires_at + .is_some_and(|expires_at| expires_at <= unix_now_seconds()) + { + return error_response(StatusCode::BAD_REQUEST, "expires_at is already past"); + } + + let endpoint_id = generate_endpoint_id(); + let endpoint = Endpoint { + endpoint_id: endpoint_id.clone(), + auth_type: request.auth_type, + auth_secret: request.auth_secret, + hmac_header: request.hmac_header, + hmac_prefix: request.hmac_prefix, + expires_at: request.expires_at, + origin: EndpointOrigin::Dynamic, + state: EndpointState::Active, + submitted: false, + }; + + if !instance + .mutate_registry(|registry| registry.insert(endpoint)) + .await + { + // 128 bits of entropy makes this unreachable in practice; refusing to + // overwrite is what keeps it from silently retargeting live traffic. + warn!( + "Generated a colliding endpoint id for {CONNECTOR_NAME} connector ID: {instance_id}, refusing the registration", + instance_id = instance.id + ); + return error_response(StatusCode::CONFLICT, "endpoint id collision"); + } + if let Some(failure) = republish(&state).await { + // Undo the insert. Left in place it would be persisted on the next + // flush and come back live after a restart, despite the caller having + // been told the registration failed. + if !instance + .mutate_registry(|registry| registry.remove(endpoint_id.as_str())) + .await + { + error!( + "Failed to roll back endpoint {} on {CONNECTOR_NAME} connector ID: {}; it may be persisted and served after a restart despite this registration failing", + endpoint_id.log_prefix(), + instance.id + ); + } + return failure; + } + if !still_joined(&state, &instance) { + // The instance closed while we were mutating it. Its registry is no + // longer polled or projected, so the URL we would hand back is dead. + // No rollback: that registry is already unreachable, and the poll task + // that could have persisted it is stopped. + return error_response(StatusCode::SERVICE_UNAVAILABLE, "instance is closing"); + } + + // Logged by prefix only: the full id is the credential for a secret-path + // endpoint, and process logs reach a far wider audience than the state + // directory the README scopes secrets to. + info!( + "Registered endpoint {} for {CONNECTOR_NAME} connector ID: {}", + endpoint_id.log_prefix(), + instance.id + ); + ( + StatusCode::CREATED, + Json(RegisteredEndpoint { + path: format!("/e/{endpoint_id}"), + endpoint_id, + }), + ) + .into_response() +} + +/// Replaces an endpoint's secret without changing its URL. +/// +/// Deliberate: a webhook sender configures the URL once, so rotating the +/// shared secret must not force it to be reconfigured. +async fn rotate_secret( + State(state): State>, + Path(endpoint_id): Path, + request_headers: HeaderMap, + body: Result, axum::extract::rejection::JsonRejection>, +) -> Response { + if let Some(response) = denied(&state, &request_headers) { + return response; + } + let Json(request) = match body { + Ok(body) => body, + Err(rejection) => return error_response(rejection.status(), "invalid request body"), + }; + if request.auth_secret.expose_secret().is_empty() { + // An empty HMAC key validates any signature the holder of the URL can + // compute, so rotating to one silently removes the second factor. + return error_response(StatusCode::BAD_REQUEST, "auth_secret must not be empty"); + } + let Some(instance) = owner_of(&state, &endpoint_id) else { + return error_response(StatusCode::NOT_FOUND, "not found"); + }; + // Restoring prefers TOML over any still-active persisted entry, so a + // rotated static secret would silently revert on the next restart and the + // operator would believe a leaked secret had been replaced. + if instance + .registry() + .endpoint(&endpoint_id) + .is_some_and(|endpoint| endpoint.origin == EndpointOrigin::Static) + { + return error_response( + StatusCode::CONFLICT, + "a static endpoint's secret lives in TOML; edit auth_secret there and restart the instance", + ); + } + + let rotated = instance + .mutate_registry(|registry| { + let Some(endpoint) = registry.endpoint_mut(&endpoint_id) else { + return false; + }; + if !endpoint.is_active() { + return false; + } + endpoint.auth_secret = Some(request.auth_secret.clone()); + endpoint.submitted = false; + true + }) + .await; + if !rotated { + return error_response(StatusCode::NOT_FOUND, "not found"); + } + if let Some(failure) = republish_or_close(&state).await { + return failure; + } + if !still_joined(&state, &instance) { + return error_response(StatusCode::SERVICE_UNAVAILABLE, "instance is closing"); + } + + info!( + "Rotated the secret for endpoint {} on {CONNECTOR_NAME} connector ID: {}", + EndpointId::log_prefix_of(&endpoint_id), + instance.id + ); + summary_response(&instance, &endpoint_id) +} + +async fn revoke_endpoint( + State(state): State>, + Path(endpoint_id): Path, + request_headers: HeaderMap, + body: Option>, +) -> Response { + if let Some(response) = denied(&state, &request_headers) { + return response; + } + let reason = body + .and_then(|Json(request)| request.reason) + .unwrap_or_else(|| "unspecified".to_string()); + let Some(instance) = owner_of(&state, &endpoint_id) else { + return error_response(StatusCode::NOT_FOUND, "not found"); + }; + + let revoked = instance + .mutate_registry(|registry| registry.revoke(&endpoint_id, reason, unix_now_seconds())) + .await; + if !revoked { + return error_response(StatusCode::NOT_FOUND, "not found"); + } + if let Some(failure) = republish_or_close(&state).await { + return failure; + } + // Most important here of the three: a revocation reported 204 but landed + // on a departed instance is never persisted, so the compromised endpoint + // comes back after the restart the operator is probably about to do. + if !still_joined(&state, &instance) { + return error_response(StatusCode::SERVICE_UNAVAILABLE, "instance is closing"); + } + + info!( + "Revoked endpoint {} on {CONNECTOR_NAME} connector ID: {}", + EndpointId::log_prefix_of(&endpoint_id), + instance.id + ); + StatusCode::NO_CONTENT.into_response() +} + +async fn list_endpoints( + State(state): State>, + request_headers: HeaderMap, +) -> Response { + if let Some(response) = denied(&state, &request_headers) { + return response; + } + let endpoints: Vec = state + .instances() + .iter() + .flat_map(|instance| { + instance + .registry() + .endpoints() + .map(|endpoint| EndpointSummary::new(instance, endpoint)) + .collect::>() + }) + .collect(); + Json(endpoints).into_response() +} + +async fn get_endpoint( + State(state): State>, + Path(endpoint_id): Path, + request_headers: HeaderMap, +) -> Response { + if let Some(response) = denied(&state, &request_headers) { + return response; + } + let Some(instance) = owner_of(&state, &endpoint_id) else { + return error_response(StatusCode::NOT_FOUND, "not found"); + }; + summary_response(&instance, &endpoint_id) +} + +/// Guards every management call with the shared token, answering with the +/// rejection to send when the caller may not proceed. +/// +/// The token is required rather than optional here: the router is only +/// mounted when one is configured, so reaching a handler without one would +/// mean the listener is serving an endpoint it never meant to expose. +fn denied(state: &ServerState, request_headers: &HeaderMap) -> Option { + let Some(expected) = &state.management_token else { + return Some(error_response(StatusCode::NOT_FOUND, "not found")); + }; + let presented = request_headers + .get(header::AUTHORIZATION) + .and_then(|value| value.to_str().ok()) + .and_then(strip_bearer) + .map(SecretString::from); + match presented { + Some(presented) if secrets_match(&presented, expected) => None, + _ => Some(error_response(StatusCode::UNAUTHORIZED, "unauthorized")), + } +} + +fn is_usable(secret: &Option) -> bool { + secret + .as_ref() + .is_some_and(|secret| !secret.expose_secret().is_empty()) +} + +fn owner_of(state: &ServerState, endpoint_id: &str) -> Option> { + state + .instances() + .into_iter() + .find(|instance| instance.registry().endpoint(endpoint_id).is_some()) +} + +/// Whether the instance a handler resolved earlier is still the one joined +/// under that name. A handler awaits between resolving and mutating, and an +/// instance can close in between - the mutation would then land on a registry +/// nobody polls and the caller would be told it succeeded. +fn still_joined(state: &ServerState, instance: &Arc) -> bool { + state + .instance(&instance.instance_name) + .is_some_and(|current| Arc::ptr_eq(¤t, instance)) +} + +/// Projects the mutated registry into the shared route table. +/// +/// A failure here leaves the registry ahead of the routes, so it answers 500 +/// rather than pretending the endpoint is live. +async fn republish(state: &ServerState) -> Option { + let Err(error) = refresh_routes(&state.listen_addr).await else { + return None; + }; + warn!( + "Failed to republish {CONNECTOR_NAME} routes on {}. {error}", + state.listen_addr + ); + Some(error_response( + StatusCode::INTERNAL_SERVER_ERROR, + "route update failed", + )) +} + +/// Republish for a mutation that takes access AWAY. If the table cannot be +/// rebuilt, the old one is still serving the endpoint the operator just +/// revoked or re-keyed, so serve nothing rather than a credential they +/// believe is dead. +async fn republish_or_close(state: &ServerState) -> Option { + let failure = republish(state).await?; + state.serve_nothing(); + Some(failure) +} + +fn summary_response(instance: &Arc, endpoint_id: &str) -> Response { + let registry = instance.registry(); + match registry.endpoint(endpoint_id) { + Some(endpoint) => Json(EndpointSummary::new(instance, endpoint)).into_response(), + None => error_response(StatusCode::NOT_FOUND, "not found"), + } +} + +fn generate_endpoint_id() -> EndpointId { + let bytes: [u8; ENDPOINT_ID_BYTES] = rand::rng().random(); + hex::encode(bytes) + .parse() + .expect("hex of 16 bytes is 32 lowercase hex characters") +} + +#[derive(Debug, Deserialize)] +struct RegisterRequest { + instance: String, + #[serde(default)] + auth_type: EndpointAuthType, + #[serde(default)] + auth_secret: Option, + #[serde(default = "crate::default_hmac_header")] + hmac_header: String, + #[serde(default = "crate::default_hmac_prefix")] + hmac_prefix: String, + #[serde(default)] + expires_at: Option, +} + +#[derive(Debug, Deserialize)] +struct RotateRequest { + auth_secret: SecretString, +} + +#[derive(Debug, Deserialize)] +struct RevokeRequest { + #[serde(default)] + reason: Option, +} + +#[derive(Debug, Serialize)] +struct RegisteredEndpoint { + endpoint_id: EndpointId, + path: String, +} + +/// The operator-facing view of an endpoint. Carries no secret, which is why +/// it exists rather than serializing [`Endpoint`] directly. +#[derive(Debug, Serialize)] +struct EndpointSummary { + endpoint_id: EndpointId, + instance: String, + state: &'static str, + origin: &'static str, + auth_type: EndpointAuthType, + expires_at: Option, + /// False until the batch carrying this endpoint has been handed to the + /// runtime. Not `persisted`: the plugin gets no acknowledgement that the + /// runtime's write landed, so this is submission, not durability. + submitted: bool, + revoked_at: Option, + revoked_reason: Option, +} + +impl EndpointSummary { + fn new(instance: &Arc, endpoint: &Endpoint) -> Self { + let (state, revoked_at, revoked_reason) = match &endpoint.state { + EndpointState::Active => ("active", None, None), + EndpointState::Revoked { reason, revoked_at } => { + ("revoked", Some(*revoked_at), Some(reason.clone())) + } + }; + EndpointSummary { + endpoint_id: endpoint.endpoint_id.clone(), + instance: instance.instance_name.clone(), + state, + origin: match endpoint.origin { + EndpointOrigin::Static => "static", + EndpointOrigin::Dynamic => "dynamic", + }, + auth_type: endpoint.auth_type, + expires_at: endpoint.expires_at, + submitted: endpoint.submitted, + revoked_at, + revoked_reason, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::HttpSource; + use crate::test_support::{ENDPOINT_ONE, client, free_port}; + use iggy_connector_sdk::Source; + use serde_json::{Value, json}; + + const TOKEN: &str = "mgmt-secret"; + + struct Fixture { + source: HttpSource, + public: String, + admin: String, + } + + impl Fixture { + async fn start(management_token: Option<&str>) -> Self { + let mut config = crate::test_support::config(Some("github"), &[ENDPOINT_ONE]); + config.listen_addr = format!("127.0.0.1:{}", free_port()); + config.admin_listen_addr = format!("127.0.0.1:{}", free_port()); + config.instance_name = Some("http_github".to_string()); + config.management_token = management_token.map(SecretString::from); + let public = format!("http://{}", config.listen_addr); + let admin = format!("http://{}", config.admin_listen_addr); + + let mut source = HttpSource::new(1, config, None); + source.open().await.expect("open must succeed"); + Fixture { + source, + public, + admin, + } + } + + async fn register(&self, body: Value) -> reqwest::Response { + client() + .post(format!("{}/admin/endpoints", self.admin)) + .header(header::AUTHORIZATION, format!("Bearer {TOKEN}")) + .json(&body) + .send() + .await + .expect("the request must reach the admin listener") + } + + async fn close(mut self) { + self.source.close().await.expect("close must succeed"); + } + } + + async fn post_signed(url: &str, secret: &str, body: &'static str) -> reqwest::Response { + let key = ring::hmac::Key::new(ring::hmac::HMAC_SHA256, secret.as_bytes()); + let signature = format!( + "sha256={}", + hex::encode(ring::hmac::sign(&key, body.as_bytes()).as_ref()) + ); + client() + .post(url) + .header(crate::DEFAULT_HMAC_HEADER, signature) + .body(body) + .send() + .await + .expect("the request must reach the listener") + } + + fn hmac_endpoint() -> Value { + json!({ + "instance": "http_github", + "auth_type": "hmac-sha256", + "auth_secret": "whsec_dynamic", + }) + } + + #[tokio::test] + async fn given_no_management_token_when_endpoints_called_should_answer_not_found() { + let fixture = Fixture::start(None).await; + + let response = client() + .get(format!("{}/admin/endpoints", fixture.admin)) + .send() + .await + .expect("the request must reach the admin listener"); + + assert_eq!(response.status(), StatusCode::NOT_FOUND); + assert!( + response.text().await.expect("a body").is_empty(), + "an unconfigured management API must not exist, not merely refuse: axum's \ + fallback has no body, whereas a handler answering 404 would render one" + ); + fixture.close().await; + } + + #[tokio::test] + async fn given_missing_token_when_endpoints_called_should_answer_unauthorized() { + let fixture = Fixture::start(Some(TOKEN)).await; + + let response = client() + .get(format!("{}/admin/endpoints", fixture.admin)) + .send() + .await + .expect("the request must reach the admin listener"); + + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + fixture.close().await; + } + + #[tokio::test] + async fn given_wrong_token_when_endpoints_called_should_answer_unauthorized() { + let fixture = Fixture::start(Some(TOKEN)).await; + + let response = client() + .get(format!("{}/admin/endpoints", fixture.admin)) + .header(header::AUTHORIZATION, "Bearer not-the-token") + .send() + .await + .expect("the request must reach the admin listener"); + + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + fixture.close().await; + } + + #[tokio::test] + async fn given_valid_request_when_endpoint_registered_should_generate_a_secret_path() { + let fixture = Fixture::start(Some(TOKEN)).await; + + let response = fixture.register(hmac_endpoint()).await; + + assert_eq!(response.status(), StatusCode::CREATED); + let body: Value = response.json().await.expect("the response must be JSON"); + let endpoint_id = body["endpoint_id"].as_str().expect("an id is returned"); + assert_eq!(endpoint_id.len(), EndpointId::LENGTH); + assert!(endpoint_id.parse::().is_ok()); + assert_eq!(body["path"], json!(format!("/e/{endpoint_id}"))); + fixture.close().await; + } + + #[tokio::test] + async fn given_registered_endpoint_when_revoked_should_stop_accepting_webhooks() { + let fixture = Fixture::start(Some(TOKEN)).await; + let created: Value = fixture + .register(json!({"instance": "http_github"})) + .await + .json() + .await + .expect("the response must be JSON"); + let endpoint_id = created["endpoint_id"] + .as_str() + .expect("an id is returned") + .to_string(); + let webhook = format!("{}/e/{endpoint_id}", fixture.public); + + let accepted = client() + .post(&webhook) + .body("{}") + .send() + .await + .expect("the request must reach the listener"); + let revoked = client() + .delete(format!("{}/admin/endpoints/{endpoint_id}", fixture.admin)) + .header(header::AUTHORIZATION, format!("Bearer {TOKEN}")) + .json(&json!({"reason": "compromised"})) + .send() + .await + .expect("the request must reach the admin listener"); + let after = client() + .post(&webhook) + .body("{}") + .send() + .await + .expect("the request must reach the listener"); + + assert_eq!(accepted.status(), StatusCode::OK); + assert_eq!(revoked.status(), StatusCode::NO_CONTENT); + assert_eq!( + after.status(), + StatusCode::NOT_FOUND, + "a revocation must take effect without a restart" + ); + fixture.close().await; + } + + #[tokio::test] + async fn given_registered_endpoint_when_secret_rotated_should_keep_the_same_path() { + let fixture = Fixture::start(Some(TOKEN)).await; + let created: Value = fixture + .register(hmac_endpoint()) + .await + .json() + .await + .expect("the response must be JSON"); + let endpoint_id = created["endpoint_id"] + .as_str() + .expect("an id is returned") + .to_string(); + + let rotated = client() + .patch(format!("{}/admin/endpoints/{endpoint_id}", fixture.admin)) + .header(header::AUTHORIZATION, format!("Bearer {TOKEN}")) + .json(&json!({"auth_secret": "whsec_rotated"})) + .send() + .await + .expect("the request must reach the admin listener"); + + assert_eq!(rotated.status(), StatusCode::OK); + let body: Value = rotated.json().await.expect("the response must be JSON"); + assert_eq!( + body["endpoint_id"], + json!(endpoint_id), + "a sender configures the URL once, so rotation must not move it" + ); + + let webhook = format!("{}/e/{endpoint_id}", fixture.public); + let with_new = post_signed(&webhook, "whsec_rotated", "{}").await; + let with_old = post_signed(&webhook, "whsec_dynamic", "{}").await; + assert_eq!( + with_new.status(), + StatusCode::OK, + "the rotated secret must be the one that now validates" + ); + assert_eq!( + with_old.status(), + StatusCode::UNAUTHORIZED, + "and the replaced secret must stop working" + ); + fixture.close().await; + } + + #[tokio::test] + async fn given_static_endpoint_when_secret_rotated_should_refuse() { + let fixture = Fixture::start(Some(TOKEN)).await; + + let rotated = client() + .patch(format!("{}/admin/endpoints/{ENDPOINT_ONE}", fixture.admin)) + .header(header::AUTHORIZATION, format!("Bearer {TOKEN}")) + .json(&json!({"auth_secret": "whsec_rotated"})) + .send() + .await + .expect("the request must reach the admin listener"); + + assert_eq!( + rotated.status(), + StatusCode::CONFLICT, + "restore prefers TOML for an active static endpoint, so a rotation here would \ + silently revert on the next restart and leave a leaked secret in service" + ); + fixture.close().await; + } + + #[tokio::test] + async fn given_empty_secret_when_rotated_should_refuse() { + let fixture = Fixture::start(Some(TOKEN)).await; + let created: Value = fixture + .register(hmac_endpoint()) + .await + .json() + .await + .expect("the response must be JSON"); + let endpoint_id = created["endpoint_id"].as_str().expect("an id").to_string(); + + let rotated = client() + .patch(format!("{}/admin/endpoints/{endpoint_id}", fixture.admin)) + .header(header::AUTHORIZATION, format!("Bearer {TOKEN}")) + .json(&json!({"auth_secret": ""})) + .send() + .await + .expect("the request must reach the admin listener"); + + assert_eq!( + rotated.status(), + StatusCode::BAD_REQUEST, + "an empty HMAC key validates any signature the URL holder can compute" + ); + fixture.close().await; + } + + #[tokio::test] + async fn given_revoked_endpoint_when_secret_rotated_should_answer_not_found() { + let fixture = Fixture::start(Some(TOKEN)).await; + let created: Value = fixture + .register(hmac_endpoint()) + .await + .json() + .await + .expect("the response must be JSON"); + let endpoint_id = created["endpoint_id"] + .as_str() + .expect("an id is returned") + .to_string(); + client() + .delete(format!("{}/admin/endpoints/{endpoint_id}", fixture.admin)) + .header(header::AUTHORIZATION, format!("Bearer {TOKEN}")) + .send() + .await + .expect("the request must reach the admin listener"); + + let rotated = client() + .patch(format!("{}/admin/endpoints/{endpoint_id}", fixture.admin)) + .header(header::AUTHORIZATION, format!("Bearer {TOKEN}")) + .json(&json!({"auth_secret": "whsec_rotated"})) + .send() + .await + .expect("the request must reach the admin listener"); + + assert_eq!( + rotated.status(), + StatusCode::NOT_FOUND, + "a tombstone must not be revivable through rotation" + ); + fixture.close().await; + } + + #[tokio::test] + async fn given_auth_type_without_secret_when_registered_should_reject() { + let fixture = Fixture::start(Some(TOKEN)).await; + + let response = fixture + .register(json!({"instance": "http_github", "auth_type": "hmac-sha256"})) + .await; + + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + fixture.close().await; + } + + #[tokio::test] + async fn given_past_expiry_when_registered_should_reject() { + let fixture = Fixture::start(Some(TOKEN)).await; + + let response = fixture + .register(json!({"instance": "http_github", "expires_at": 1})) + .await; + + assert_eq!( + response.status(), + StatusCode::BAD_REQUEST, + "registering an endpoint that is already Gone is an operator error" + ); + fixture.close().await; + } + + #[tokio::test] + async fn given_unknown_instance_when_registered_should_answer_not_found() { + let fixture = Fixture::start(Some(TOKEN)).await; + + let response = fixture.register(json!({"instance": "http_stripe"})).await; + + assert_eq!(response.status(), StatusCode::NOT_FOUND); + fixture.close().await; + } + + #[tokio::test] + async fn given_untyped_body_when_registered_should_reject() { + let fixture = Fixture::start(Some(TOKEN)).await; + + let response = fixture + .register(json!({"instance": "http_github", "auth_type": "totally-made-up"})) + .await; + + assert!( + response.status().is_client_error(), + "an unknown auth_type must be rejected at parse, not coerced into a default" + ); + fixture.close().await; + } + + #[tokio::test] + async fn given_listed_endpoints_when_read_should_never_return_secrets() { + let fixture = Fixture::start(Some(TOKEN)).await; + fixture.register(hmac_endpoint()).await; + + let response = client() + .get(format!("{}/admin/endpoints", fixture.admin)) + .header(header::AUTHORIZATION, format!("Bearer {TOKEN}")) + .send() + .await + .expect("the request must reach the admin listener"); + + assert_eq!(response.status(), StatusCode::OK); + let body = response + .text() + .await + .expect("the response must have a body"); + assert!(!body.contains("whsec_dynamic")); + assert!(!body.contains("whsec_static")); + assert!(body.contains("\"origin\":\"dynamic\"")); + assert!(body.contains("\"origin\":\"static\"")); + assert!( + body.contains("\"submitted\":false"), + "a caller must be able to tell an accepted mutation from a durable one" + ); + fixture.close().await; + } +} diff --git a/core/connectors/sources/http_source/src/metrics.rs b/core/connectors/sources/http_source/src/metrics.rs new file mode 100644 index 0000000000..b523bdbb91 --- /dev/null +++ b/core/connectors/sources/http_source/src/metrics.rs @@ -0,0 +1,433 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Gateway-side Prometheus metrics, served on the admin listener. +//! +//! The runtime's own stage histograms start at `poll()`, so they cannot see +//! the number that matters most for a webhook gateway: how long a sender +//! waited between TCP accept and its 200. This connector is an HTTP server, +//! so it measures that itself. +//! +//! Names carry an `http_source_` prefix to keep them clear of the runtime's +//! `iggy_connector_` family. One registry per shared listener, with every +//! series labelled by instance. + +use prometheus_client::encoding::text::encode; +use prometheus_client::encoding::{EncodeLabelSet, EncodeLabelValue, LabelValueEncoder}; +use prometheus_client::metrics::counter::Counter; +use prometheus_client::metrics::family::Family; +use prometheus_client::metrics::gauge::Gauge; +use prometheus_client::metrics::histogram::Histogram; +use prometheus_client::registry::Registry; +use std::fmt; +use std::sync::Arc; +use std::time::Duration; +use tracing::error; + +use crate::SharedState; +use crate::routes::EndpointOrigin; +use crate::types::unix_now_seconds; + +/// Instance label for requests that never resolved to one, so a scan for +/// live endpoint ids still shows up rather than going uncounted. +pub const UNROUTED: &str = "unrouted"; + +/// Sub-millisecond at the low end: accepting a webhook is a route lookup, a +/// signature check, and a channel send, so the interesting range is tight. +const REQUEST_BUCKETS_SECONDS: [f64; 12] = [ + 50e-6, 100e-6, 250e-6, 500e-6, 1e-3, 2.5e-3, 5e-3, 10e-3, 25e-3, 50e-3, 100e-3, 500e-3, +]; + +/// Which surface a request arrived on. +#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)] +pub enum PathKind { + Named, + Secret, +} + +/// Response class, coarse on purpose: a per-status-code label would let a +/// caller inflate cardinality by probing. +#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)] +pub enum StatusClass { + Success, + ClientError, + ServerError, +} + +impl From for StatusClass { + fn from(status: u16) -> Self { + match status { + 200..=399 => Self::Success, + 400..=499 => Self::ClientError, + _ => Self::ServerError, + } + } +} + +#[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelSet)] +pub struct InstanceLabel { + pub instance: String, +} + +#[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelSet)] +pub struct RequestLabels { + pub instance: String, + pub kind: PathKind, + pub status: StatusClass, +} + +#[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelSet)] +pub struct DurationLabels { + pub instance: String, + pub status: StatusClass, +} + +#[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelSet)] +pub struct EndpointLabels { + pub instance: String, + pub kind: EndpointOrigin, +} + +#[derive(Debug)] +pub struct Metrics { + registry: Registry, + requests: Family, + request_duration_seconds: Family Histogram>, + rejected_full: Family, + dropped_on_close: Family, + headers_clamped: Family, + headers_dropped: Family, + buffer_used: Family, + buffer_capacity: Family, + endpoints_active: Family, +} + +impl Metrics { + pub fn new() -> Self { + let mut registry = Registry::default(); + let requests = Family::::default(); + let request_duration_seconds: Family Histogram> = + Family::new_with_constructor(request_histogram); + let rejected_full = Family::::default(); + let dropped_on_close = Family::::default(); + let headers_clamped = Family::::default(); + let headers_dropped = Family::::default(); + let buffer_used = Family::::default(); + let buffer_capacity = Family::::default(); + let endpoints_active = Family::::default(); + + registry.register( + "http_source_requests", + "Webhook requests by path kind and response class", + requests.clone(), + ); + registry.register( + "http_source_request_duration_seconds", + "Time from request accepted to response, in seconds", + request_duration_seconds.clone(), + ); + registry.register( + "http_source_rejected_full", + "Requests answered 429 because the instance bridge was full", + rejected_full.clone(), + ); + registry.register( + "http_source_dropped_on_close", + "Accepted messages still queued when the instance closed", + dropped_on_close.clone(), + ); + registry.register( + "http_source_headers_clamped", + "Forwarded header values truncated to the Iggy 255-byte limit", + headers_clamped.clone(), + ); + registry.register( + "http_source_headers_dropped", + "Forwarded header values Iggy would have rejected outright", + headers_dropped.clone(), + ); + registry.register( + "http_source_buffer_used", + "Messages queued in the instance bridge", + buffer_used.clone(), + ); + registry.register( + "http_source_buffer_capacity", + "Configured capacity of the instance bridge", + buffer_capacity.clone(), + ); + registry.register( + "http_source_endpoints_active", + "Secret-path endpoints currently accepting requests, by origin", + endpoints_active.clone(), + ); + + Metrics { + registry, + requests, + request_duration_seconds, + rejected_full, + dropped_on_close, + headers_clamped, + headers_dropped, + buffer_used, + buffer_capacity, + endpoints_active, + } + } + + /// Labels are built per call rather than cached per instance: one small + /// `String` clone is noise next to copying the request body, and the + /// cache would have to cover every kind and status combination. + pub fn record_request(&self, instance: &str, kind: PathKind, status: u16, elapsed: Duration) { + let status = StatusClass::from(status); + self.requests + .get_or_create(&RequestLabels { + instance: instance.to_owned(), + kind, + status, + }) + .inc(); + self.request_duration_seconds + .get_or_create(&DurationLabels { + instance: instance.to_owned(), + status, + }) + .observe(elapsed.as_secs_f64()); + } + + pub fn record_rejected_full(&self, instance: &str) { + self.rejected_full.get_or_create(&label(instance)).inc(); + } + + pub fn record_dropped_on_close(&self, instance: &str, dropped: u64) { + self.dropped_on_close + .get_or_create(&label(instance)) + .inc_by(dropped); + } + + pub fn record_headers(&self, instance: &str, clamped: u64, dropped: u64) { + if clamped > 0 { + self.headers_clamped + .get_or_create(&label(instance)) + .inc_by(clamped); + } + if dropped > 0 { + self.headers_dropped + .get_or_create(&label(instance)) + .inc_by(dropped); + } + } + + /// Reads without creating. `get_or_create` would materialise a zero + /// series as a side effect, so an admin health check would make counters + /// appear in every later scrape - contradicting the documented rule that + /// an untouched family is absent rather than zero. + pub fn headers_clamped(&self, instance: &str) -> u64 { + self.headers_clamped + .get(&label(instance)) + .map_or(0, |counter| counter.get()) + } + + pub fn headers_dropped(&self, instance: &str) -> u64 { + self.headers_dropped + .get(&label(instance)) + .map_or(0, |counter| counter.get()) + } + + /// Drops a departed instance's sampled gauges. + /// + /// Counters stay: they are cumulative and their last value remains true. + /// Gauges are instantaneous, so leaving them behind would report a queue + /// depth for an instance that no longer exists. + pub fn forget_instance(&self, instance: &str) { + let label = label(instance); + self.buffer_used.remove(&label); + self.buffer_capacity.remove(&label); + for origin in [EndpointOrigin::Static, EndpointOrigin::Dynamic] { + self.endpoints_active.remove(&EndpointLabels { + instance: instance.to_owned(), + kind: origin, + }); + } + } + + /// Refreshes the sampled gauges and renders the Prometheus text format. + /// + /// The gauges are read from the instances at scrape time rather than + /// maintained on the hot path: queue depth and endpoint counts are only + /// meaningful as instantaneous values, and nothing observes them between + /// scrapes. + pub fn encode(&self, instances: &[Arc]) -> String { + for instance in instances { + let label = label(&instance.instance_name); + self.buffer_used + .get_or_create(&label) + .set(instance.sender.len() as i64); + self.buffer_capacity + .get_or_create(&label) + .set(instance.config.buffer_capacity as i64); + + let registry = instance.registry(); + let now = unix_now_seconds(); + for origin in [EndpointOrigin::Static, EndpointOrigin::Dynamic] { + self.endpoints_active + .get_or_create(&EndpointLabels { + instance: instance.instance_name.clone(), + kind: origin, + }) + .set(registry.serving_count_by_origin(origin, now) as i64); + } + } + + let mut buffer = String::new(); + if let Err(error) = encode(&mut buffer, &self.registry) { + error!( + "Failed to encode {} metrics. {error}", + crate::CONNECTOR_NAME + ); + } + buffer + } +} + +impl Default for Metrics { + fn default() -> Self { + Metrics::new() + } +} + +fn label(instance: &str) -> InstanceLabel { + InstanceLabel { + instance: instance.to_owned(), + } +} + +fn request_histogram() -> Histogram { + Histogram::new(REQUEST_BUCKETS_SECONDS.iter().copied()) +} + +impl EncodeLabelValue for PathKind { + fn encode(&self, encoder: &mut LabelValueEncoder) -> Result<(), fmt::Error> { + match self { + Self::Named => "named", + Self::Secret => "secret", + } + .encode(encoder) + } +} + +impl EncodeLabelValue for StatusClass { + fn encode(&self, encoder: &mut LabelValueEncoder) -> Result<(), fmt::Error> { + match self { + Self::Success => "2xx", + Self::ClientError => "4xx", + Self::ServerError => "5xx", + } + .encode(encoder) + } +} + +impl EncodeLabelValue for EndpointOrigin { + fn encode(&self, encoder: &mut LabelValueEncoder) -> Result<(), fmt::Error> { + match self { + Self::Static => "static", + Self::Dynamic => "dynamic", + } + .encode(encoder) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn given_status_codes_when_classified_should_collapse_to_three_classes() { + assert_eq!(StatusClass::from(200), StatusClass::Success); + assert_eq!(StatusClass::from(204), StatusClass::Success); + assert_eq!(StatusClass::from(404), StatusClass::ClientError); + assert_eq!(StatusClass::from(429), StatusClass::ClientError); + assert_eq!(StatusClass::from(500), StatusClass::ServerError); + } + + #[test] + fn given_recorded_requests_when_encoded_should_carry_prefixed_labelled_series() { + let metrics = Metrics::new(); + + metrics.record_request( + "http_github", + PathKind::Secret, + 200, + Duration::from_micros(80), + ); + metrics.record_request( + "http_github", + PathKind::Named, + 401, + Duration::from_micros(40), + ); + metrics.record_rejected_full("http_github"); + + let encoded = metrics.encode(&[]); + assert!(encoded.contains( + "http_source_requests_total{instance=\"http_github\",kind=\"secret\",status=\"2xx\"} 1" + )); + assert!(encoded.contains( + "http_source_requests_total{instance=\"http_github\",kind=\"named\",status=\"4xx\"} 1" + )); + assert!( + encoded.contains("http_source_rejected_full_total{instance=\"http_github\"} 1"), + "a 429 needs its own series: it is backpressure, not a caller error" + ); + assert!(encoded.contains("http_source_request_duration_seconds_bucket")); + } + + #[test] + fn given_header_losses_when_recorded_should_count_clamps_and_drops_apart() { + let metrics = Metrics::new(); + + metrics.record_headers("http_github", 2, 5); + metrics.record_headers("http_github", 1, 0); + + // Distinct totals on purpose: equal ones would survive the two + // counters being swapped, which is the bug this test exists to catch. + assert_eq!(metrics.headers_clamped("http_github"), 3); + assert_eq!(metrics.headers_dropped("http_github"), 5); + } + + #[test] + fn given_untouched_families_when_encoded_should_omit_them_until_first_use() { + let metrics = Metrics::new(); + + let empty = metrics.encode(&[]); + assert!( + empty.trim() == "# EOF", + "a labelled family has no series to emit before it is touched, so \ + dashboards must treat these as absent rather than zero" + ); + + metrics.record_rejected_full("http_github"); + + assert!( + metrics + .encode(&[]) + .contains("# TYPE http_source_rejected_full counter"), + "and it must appear the moment there is something to report" + ); + } +} diff --git a/core/connectors/sources/http_source/src/routes.rs b/core/connectors/sources/http_source/src/routes.rs new file mode 100644 index 0000000000..fcdd52a047 --- /dev/null +++ b/core/connectors/sources/http_source/src/routes.rs @@ -0,0 +1,410 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Route table for a shared listener. +//! +//! Every instance joining a listener contributes its named topic path and its +//! secret-path endpoints. The merged table is rebuilt and swapped whole on +//! join, leave, and management mutations, so a request resolves its auth +//! requirements and its destination bridge from a single atomic load. + +use secrecy::SecretString; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::collections::hash_map::Entry; +use std::fmt::{self, Display, Formatter}; +use std::sync::Arc; + +use crate::types::EndpointId; +use crate::{EndpointAuthType, SharedState, StaticEndpointConfig}; + +/// Lifecycle state of a secret-path endpoint. +/// +/// Revocation writes a tombstone rather than deleting the entry: the tombstone +/// persists through a restart, so a stale TOML entry can never resurrect an +/// endpoint an operator revoked. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum EndpointState { + Active, + Revoked { reason: String, revoked_at: u64 }, +} + +/// Where an endpoint came from, which decides whether a TOML edit plus an +/// instance restart or a management API call is the way to change it. +#[derive(Debug, Clone, Copy, Default, Hash, PartialEq, Eq, Serialize, Deserialize)] +pub enum EndpointOrigin { + #[default] + Static, + Dynamic, +} + +/// A secret-path endpoint served at `POST /e/{endpoint_id}`. +/// +/// Persisted verbatim into the runtime state directory, HMAC and bearer +/// secrets included, so endpoints created through the management API survive a +/// restart. That file is as sensitive as the TOML it mirrors, which is why the +/// README requires `chmod 700` on the state path. +/// Fields added after the first release must be APPENDED and carry +/// `#[serde(default)]`. `ConnectorState` uses rmp's compact codec, where a +/// struct is a positional array, so a new field without a default makes every +/// existing state file fail to decode, and `EndpointRegistry::restore` turns a +/// decode failure into "tombstones lost, revoked endpoints served again". +/// +/// `auth_type` and `state` deliberately have no default: for those two, a +/// missing element must fail the decode rather than read as an active, +/// unauthenticated endpoint. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Endpoint { + pub endpoint_id: EndpointId, + // No default: an absent auth_type must fail the decode, not silently + // read as `None` and drop the second factor. + pub auth_type: EndpointAuthType, + #[serde(default, serialize_with = "crate::state::serialize_secret_to_state")] + pub auth_secret: Option, + #[serde(default)] + pub hmac_header: String, + #[serde(default)] + pub hmac_prefix: String, + /// Unix seconds; requests arriving at or after this answer 410 Gone. + #[serde(default)] + pub expires_at: Option, + #[serde(default)] + pub origin: EndpointOrigin, + // No default: `EndpointState::default()` would be `Active`, so a record + // short by one element would resurrect a revoked endpoint - turning a + // loud decode failure into a silent, fail-open one. + pub state: EndpointState, + /// Whether this endpoint has been handed to the runtime for persistence. + /// Reset on every mutation, never itself persisted: a restored endpoint is + /// durable by definition. Not named `persisted`, because the plugin gets + /// no acknowledgement that the runtime's write actually landed. + #[serde(skip)] + pub submitted: bool, +} + +impl Endpoint { + pub fn is_active(&self) -> bool { + matches!(self.state, EndpointState::Active) + } + + /// Whether a request arriving now would be accepted. + pub fn is_serving(&self, now_seconds: u64) -> bool { + self.is_active() && !self.is_expired(now_seconds) + } + + pub fn is_expired(&self, now_seconds: u64) -> bool { + self.expires_at + .is_some_and(|expires_at| now_seconds >= expires_at) + } + + pub fn revoke(&mut self, reason: String, revoked_at: u64) { + self.state = EndpointState::Revoked { reason, revoked_at }; + self.submitted = false; + } +} + +impl From<&StaticEndpointConfig> for Endpoint { + fn from(config: &StaticEndpointConfig) -> Self { + Endpoint { + endpoint_id: config.endpoint_id.clone(), + auth_type: config.auth_type, + auth_secret: config.auth_secret.clone(), + hmac_header: config.hmac_header.clone(), + hmac_prefix: config.hmac_prefix.clone(), + expires_at: config.expires_at, + origin: EndpointOrigin::Static, + state: EndpointState::Active, + // Declared in TOML, so it needs no state file to come back. + submitted: true, + } + } +} + +/// Immutable snapshot of every path one shared listener serves. +#[derive(Debug, Default)] +pub struct RouteTable { + secret_paths: HashMap, + named_paths: HashMap>, +} + +/// A resolved secret path: the endpoint's own auth rules plus the instance +/// whose bridge receives the body. +#[derive(Debug)] +pub struct RouteEntry { + pub instance: Arc, + pub endpoint: Endpoint, +} + +/// Outcome of resolving a secret path. `Revoked` and `Unknown` both answer 404 +/// so the table never leaks which endpoints once existed; they stay distinct +/// here so the handler can log and meter them apart. +#[derive(Debug)] +pub enum RouteLookup<'a> { + Active(&'a RouteEntry), + /// Carries its entry even though the response hides it, so the request can + /// still be metered against the instance that owns the endpoint. + Revoked(&'a RouteEntry), + Expired(&'a RouteEntry), + Unknown, +} + +/// Two instances claiming the same path. Fails the join rather than letting +/// whichever instance opened last silently steal another's traffic. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum RouteConflict { + EndpointId { + endpoint_id: EndpointId, + held_by: u32, + claimed_by: u32, + }, + TopicPath { + topic_path: String, + held_by: u32, + claimed_by: u32, + }, +} + +impl RouteTable { + /// Projects every joined instance's registry into one lookup table. + pub fn build(instances: &[Arc]) -> Result { + let mut table = RouteTable::default(); + for instance in instances { + if let Some(topic_path) = &instance.config.topic_path { + match table.named_paths.entry(topic_path.clone()) { + Entry::Occupied(occupied) => { + return Err(RouteConflict::TopicPath { + topic_path: topic_path.clone(), + held_by: occupied.get().id, + claimed_by: instance.id, + }); + } + Entry::Vacant(vacant) => { + vacant.insert(Arc::clone(instance)); + } + } + } + for endpoint in instance.registry().endpoints() { + match table.secret_paths.entry(endpoint.endpoint_id.clone()) { + Entry::Occupied(occupied) => { + return Err(RouteConflict::EndpointId { + endpoint_id: endpoint.endpoint_id.clone(), + held_by: occupied.get().instance.id, + claimed_by: instance.id, + }); + } + Entry::Vacant(vacant) => { + vacant.insert(RouteEntry { + instance: Arc::clone(instance), + endpoint: endpoint.clone(), + }); + } + } + } + } + Ok(table) + } + + pub fn lookup_secret_path(&self, endpoint_id: &str, now_seconds: u64) -> RouteLookup<'_> { + let Some(entry) = self.secret_paths.get(endpoint_id) else { + return RouteLookup::Unknown; + }; + if !entry.endpoint.is_active() { + return RouteLookup::Revoked(entry); + } + if entry.endpoint.is_expired(now_seconds) { + return RouteLookup::Expired(entry); + } + RouteLookup::Active(entry) + } + + pub fn lookup_named_path(&self, topic_path: &str) -> Option<&Arc> { + self.named_paths.get(topic_path) + } + + pub fn secret_path_count(&self) -> usize { + self.secret_paths.len() + } + + pub fn named_path_count(&self) -> usize { + self.named_paths.len() + } +} + +impl Display for RouteConflict { + fn fmt(&self, formatter: &mut Formatter<'_>) -> fmt::Result { + match self { + Self::EndpointId { + endpoint_id, + held_by, + claimed_by, + } => write!( + formatter, + "endpoint_id {} is already served by connector ID: {held_by}, claimed by connector ID: {claimed_by}", + endpoint_id.log_prefix() + ), + Self::TopicPath { + topic_path, + held_by, + claimed_by, + } => write!( + formatter, + "topic_path '{topic_path}' is already served by connector ID: {held_by}, claimed by connector ID: {claimed_by}" + ), + } + } +} + +impl std::error::Error for RouteConflict {} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_support::{ENDPOINT_ONE, ENDPOINT_TWO, endpoint_id, instance}; + + const NOW: u64 = 1_800_000_000; + + #[test] + fn given_two_instances_when_table_built_should_serve_both_paths() { + let first = instance(1, Some("github"), &[ENDPOINT_ONE]); + let second = instance(2, Some("stripe"), &[ENDPOINT_TWO]); + + let table = RouteTable::build(&[first, second]).expect("distinct paths must not conflict"); + + assert_eq!(table.named_path_count(), 2); + assert_eq!(table.secret_path_count(), 2); + assert!(table.lookup_named_path("github").is_some()); + assert!(matches!( + table.lookup_secret_path(ENDPOINT_TWO, NOW), + RouteLookup::Active(_) + )); + } + + #[test] + fn given_instance_without_topic_path_when_table_built_should_serve_secret_paths_only() { + let table = RouteTable::build(&[instance(1, None, &[ENDPOINT_ONE])]) + .expect("secret-path-only instance must build"); + + assert_eq!(table.named_path_count(), 0); + assert_eq!(table.secret_path_count(), 1); + } + + #[test] + fn given_duplicate_topic_path_when_table_built_should_reject() { + let first = instance(1, Some("github"), &[ENDPOINT_ONE]); + let second = instance(7, Some("github"), &[ENDPOINT_TWO]); + + let conflict = RouteTable::build(&[first, second]) + .expect_err("a stolen topic path must fail the join"); + + assert_eq!( + conflict, + RouteConflict::TopicPath { + topic_path: "github".to_string(), + held_by: 1, + claimed_by: 7, + } + ); + } + + #[test] + fn given_duplicate_endpoint_id_when_table_built_should_reject() { + let first = instance(1, Some("github"), &[ENDPOINT_ONE]); + let second = instance(7, Some("stripe"), &[ENDPOINT_ONE]); + + let conflict = RouteTable::build(&[first, second]) + .expect_err("a stolen endpoint id must fail the join"); + + assert_eq!( + conflict, + RouteConflict::EndpointId { + endpoint_id: endpoint_id(ENDPOINT_ONE), + held_by: 1, + claimed_by: 7, + } + ); + } + + #[test] + fn given_unknown_endpoint_id_when_looked_up_should_report_unknown() { + let table = RouteTable::build(&[instance(1, None, &[ENDPOINT_ONE])]).expect("must build"); + + assert!(matches!( + table.lookup_secret_path(ENDPOINT_TWO, NOW), + RouteLookup::Unknown + )); + } + + #[tokio::test] + async fn given_revoked_endpoint_when_looked_up_should_report_revoked() { + let source = instance(1, None, &[ENDPOINT_ONE]); + source + .mutate_registry(|registry| registry.revoke(ENDPOINT_ONE, "compromised".to_string(), 1)) + .await; + + let table = RouteTable::build(&[source]).expect("must build"); + + assert!(matches!( + table.lookup_secret_path(ENDPOINT_ONE, NOW), + RouteLookup::Revoked(_) + )); + } + + #[tokio::test] + async fn given_expired_endpoint_when_looked_up_should_report_expired() { + let source = instance(1, None, &[ENDPOINT_ONE]); + source + .mutate_registry(|registry| { + registry + .endpoint_mut(ENDPOINT_ONE) + .expect("static endpoint is registered") + .expires_at = Some(NOW); + }) + .await; + + let table = RouteTable::build(&[source]).expect("must build"); + + assert!(matches!( + table.lookup_secret_path(ENDPOINT_ONE, NOW - 1), + RouteLookup::Active(_) + )); + assert!(matches!( + table.lookup_secret_path(ENDPOINT_ONE, NOW), + RouteLookup::Expired(_) + )); + } + + #[tokio::test] + async fn given_revoked_and_expired_endpoint_when_looked_up_should_prefer_revoked() { + let source = instance(1, None, &[ENDPOINT_ONE]); + source + .mutate_registry(|registry| { + let endpoint = registry + .endpoint_mut(ENDPOINT_ONE) + .expect("static endpoint is registered"); + endpoint.expires_at = Some(NOW); + endpoint.revoke("compromised".to_string(), NOW); + }) + .await; + + let table = RouteTable::build(&[source]).expect("must build"); + + assert!(matches!( + table.lookup_secret_path(ENDPOINT_ONE, NOW + 1), + RouteLookup::Revoked(_) + )); + } +} diff --git a/core/connectors/sources/http_source/src/server.rs b/core/connectors/sources/http_source/src/server.rs new file mode 100644 index 0000000000..a4c2bb9a0a --- /dev/null +++ b/core/connectors/sources/http_source/src/server.rs @@ -0,0 +1,1504 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! The listener every instance of this plugin shares, and the request path. +//! +//! One `.so` is loaded once no matter how many `[[source]]` entries reference +//! it, so the listeners live in a process-global registry keyed by listen +//! address rather than on any one instance. The first `open()` binds; later +//! ones validate their config against the running server and join; the last +//! `close()` releases the ports, which the runtime's stop-then-start restart +//! flow depends on. + +use arc_swap::ArcSwap; +use axum::Router; +use axum::body::Bytes; +use axum::extract::rejection::BytesRejection; +use axum::extract::{ConnectInfo, DefaultBodyLimit, Path, State}; +use axum::http::{HeaderMap, StatusCode, header}; +use axum::response::{IntoResponse, Response}; +use axum::routing::{get, post}; +use axum::{Json, serve}; +use iggy_common::{HeaderKey, HeaderValue}; +use iggy_connector_sdk::Error; +use secrecy::SecretString; +use serde::Serialize; +use std::collections::{BTreeMap, HashMap}; +use std::net::SocketAddr; +use std::str::FromStr; +use std::sync::{Arc, LazyLock}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; +use tokio::net::TcpListener; +use tokio::sync::{Mutex, watch}; +use tokio::task::JoinHandle; +use tracing::{debug, error, info, warn}; + +use crate::auth::{secrets_match, validate_bearer, validate_hmac}; +use crate::metrics::{Metrics, PathKind, UNROUTED}; +use crate::routes::{Endpoint, EndpointOrigin, RouteLookup, RouteTable}; +use crate::types::{QueuedMessage, clamp_header_value, unix_now_seconds}; +use crate::{CONNECTOR_NAME, EndpointAuthType, HttpSourceConfig, SharedState, management}; + +/// Iggy header carrying the instance an accepted request was routed to. +pub const INSTANCE_HEADER: &str = "iggy_source_instance"; +/// Iggy header carrying the peer address the request arrived from. +pub const REMOTE_ADDR_HEADER: &str = "iggy_http_remote_addr"; +/// Iggy header carrying accept time, in microseconds since the Unix epoch. +pub const RECEIVED_AT_HEADER: &str = "iggy_http_received_at"; + +/// How long the last `close()` waits for in-flight requests before abandoning +/// the listener tasks. Bounded so a wedged connection cannot stall shutdown. +const SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(5); + +/// Registers an instance with the listener for its configured address, +/// binding that listener if this is the first instance to ask for it. +pub async fn join(instance: Arc) -> Result<(), Error> { + let listen_addr = instance.config.listen_addr.clone(); + let mut servers = SERVERS.lock().await; + + if let Some(server) = servers.get_mut(&listen_addr) { + if server.draining { + return Err(Error::InitError(format!( + "The {CONNECTOR_NAME} listener on {listen_addr} is shutting down; retry the open once its port is released" + ))); + } + server.ensure_compatible(&instance.config)?; + let mut instances = server.state.instances(); + // Names address instances in the management API and label every + // metric series, so a duplicate would silently route registrations to + // whichever instance happens to sit first in the list. + if instances + .iter() + .any(|joined| joined.instance_name == instance.instance_name) + { + return Err(Error::InvalidConfigValue(format!( + "instance_name '{}' is already used by another instance on {listen_addr}", + instance.instance_name + ))); + } + instances.push(Arc::clone(&instance)); + server.state.publish(instances)?; + info!( + "Joined {CONNECTOR_NAME} connector ID: {} to the listener on {listen_addr}", + instance.id + ); + return Ok(()); + } + + // Build the routes before binding: a config that cannot produce a valid + // table must not leave a port bound behind it. + let state = Arc::new(ServerState::new(&instance.config)); + state.publish(vec![Arc::clone(&instance)])?; + let server = SharedServer::start(&instance.config, state).await?; + servers.insert(listen_addr.clone(), server); + info!( + "Bound the {CONNECTOR_NAME} listener on {listen_addr} for connector ID: {}, admin listener on {}", + instance.id, instance.config.admin_listen_addr + ); + Ok(()) +} + +/// Deregisters an instance's routes, and shuts the listeners down once the +/// last instance has left. +pub async fn leave(instance: &Arc) { + let listen_addr = &instance.config.listen_addr; + let mut servers = SERVERS.lock().await; + let Some(server) = servers.get_mut(listen_addr) else { + return; + }; + + let remaining: Vec> = server + .state + .instances() + .into_iter() + .filter(|joined| joined.id != instance.id) + .collect(); + let remaining_count = remaining.len(); + if let Err(error) = server.state.publish(remaining) { + // Dropping an instance cannot introduce a collision, so this is + // unreachable; serve nothing rather than stale routes if it happens. + error!( + "Failed to rebuild {CONNECTOR_NAME} routes after connector ID: {} left. {error}", + instance.id + ); + server.state.serve_nothing(); + } + info!( + "Deregistered {CONNECTOR_NAME} routes for connector ID: {}, instances left on {listen_addr}: {remaining_count}", + instance.id + ); + + server + .state + .metrics + .forget_instance(&instance.instance_name); + + // Counted after the routes are gone, which narrows the window but does + // not close it: a handler that already loaded the old table can still + // enqueue after this read. The SDK stops the poll task before close(), so + // whatever is queued here is unreachable either way. + let dropped = instance.sender.len() as u64; + if dropped > 0 { + warn!( + "Dropped {dropped} queued messages closing {CONNECTOR_NAME} connector ID: {}", + instance.id + ); + server + .state + .metrics + .record_dropped_on_close(&instance.instance_name, dropped); + } + + if remaining_count == 0 { + // Mark the entry draining and take the tasks, but leave the entry in + // place while releasing the guard. Holding it across shutdown would + // deadlock against an in-flight management request parked on this + // lock; removing it outright would let a concurrent `join()` bind a + // port the listener is still draining. The tombstone does neither. + server.draining = true; + let signal = server.shutdown.clone(); + let tasks = std::mem::take(&mut server.tasks); + drop(servers); + + SharedServer::stop(signal, tasks, listen_addr).await; + SERVERS.lock().await.remove(listen_addr); + } +} + +/// Every listener this process has bound, keyed by public listen address. +/// +/// An async mutex because the guard is held across the bind and across the +/// graceful shutdown await. Both are open/close operations, never requests. +static SERVERS: LazyLock>> = + LazyLock::new(|| Mutex::new(HashMap::new())); + +struct SharedServer { + admin_listen_addr: String, + max_body_size_bytes: usize, + state: Arc, + shutdown: watch::Sender<()>, + tasks: Vec>, + /// Set while the listener is draining. The entry stays in the registry so + /// a concurrent `join()` cannot bind a port that is still held, but it can + /// no longer be joined. + draining: bool, +} + +/// The part of a shared server the request handlers see. +#[derive(Debug)] +pub(crate) struct ServerState { + pub(crate) listen_addr: String, + /// Taken from the first instance to bind. Every instance joining the same + /// listener must present the same token, so this is unambiguous. + pub(crate) management_token: Option, + pub(crate) metrics: Metrics, + routes: ArcSwap, + instances: ArcSwap>>, + started_at: Instant, +} + +impl ServerState { + fn new(config: &HttpSourceConfig) -> Self { + ServerState { + listen_addr: config.listen_addr.clone(), + management_token: config.management_token.clone(), + metrics: Metrics::new(), + routes: ArcSwap::from_pointee(RouteTable::default()), + instances: ArcSwap::from_pointee(Vec::new()), + started_at: Instant::now(), + } + } + + pub(crate) fn instances(&self) -> Vec> { + (**self.instances.load()).clone() + } + + pub(crate) fn instance(&self, instance_name: &str) -> Option> { + self.instances + .load() + .iter() + .find(|instance| instance.instance_name == instance_name) + .map(Arc::clone) + } + + /// Serves nothing until the next successful publish. Used when a mutation + /// removes access but the table cannot be rebuilt: stale routes would keep + /// honouring a credential the operator believes is gone. + pub(crate) fn serve_nothing(&self) { + let dropped = self.routes.load(); + error!( + "Serving no {CONNECTOR_NAME} routes on {}: dropped {} secret paths and {} named paths across {} instances until the next successful publish", + self.listen_addr, + dropped.secret_path_count(), + dropped.named_path_count(), + self.instances.load().len() + ); + self.routes.store(Arc::new(RouteTable::default())); + } + + /// Swaps in a new instance set and the routes it projects to, or leaves + /// both untouched if the instances collide on a path. + fn publish(&self, instances: Vec>) -> Result<(), Error> { + let table = RouteTable::build(&instances) + .map_err(|conflict| Error::InvalidConfigValue(conflict.to_string()))?; + self.instances.store(Arc::new(instances)); + self.routes.store(Arc::new(table)); + Ok(()) + } +} + +/// Reprojects a listener's route table after a management mutation. +/// +/// Takes `SERVERS` - the listener map, not the per-instance registry gate, +/// which the caller has already released. A mutation and a join can therefore +/// still interleave; that is safe because the rebuild reprojects from whatever +/// instance set is current rather than patching the previous table. +/// +/// Blocking on the registry here is safe because `leave()` releases it before +/// awaiting graceful shutdown; holding it across that await would deadlock, +/// since shutdown waits for the very request that is parked here. +pub(crate) async fn refresh_routes(listen_addr: &str) -> Result<(), Error> { + let servers = SERVERS.lock().await; + let Some(server) = servers.get(listen_addr) else { + return Err(Error::InitError(format!( + "No {CONNECTOR_NAME} listener is bound to {listen_addr}" + ))); + }; + server.state.publish(server.state.instances()) +} + +impl SharedServer { + async fn start(config: &HttpSourceConfig, state: Arc) -> Result { + let public = bind(&config.listen_addr).await?; + let admin = bind(&config.admin_listen_addr).await?; + let (shutdown, _) = watch::channel(()); + + // The one place this plugin owns background tasks. The runtime cannot + // drive an HTTP listener for us, so the last close() shuts them down + // explicitly rather than leaving them to outlive the connector. + let tasks = vec![ + tokio::spawn(run( + public, + public_router(Arc::clone(&state), config.max_body_size_bytes), + shutdown.subscribe(), + "public", + )), + tokio::spawn(run( + admin, + admin_router(state.clone()), + shutdown.subscribe(), + "admin", + )), + ]; + + Ok(SharedServer { + admin_listen_addr: config.admin_listen_addr.clone(), + max_body_size_bytes: config.max_body_size_bytes, + state, + shutdown, + tasks, + draining: false, + }) + } + + /// Refuses a join whose settings disagree with the running listener. + /// + /// Fails closed on purpose: first-instance-wins would leave an operator + /// with a body limit or admin address their TOML says they do not have. + fn ensure_compatible(&self, config: &HttpSourceConfig) -> Result<(), Error> { + if self.admin_listen_addr != config.admin_listen_addr { + return Err(Error::InvalidConfigValue(format!( + "admin_listen_addr '{}' does not match '{}' on the listener already bound to {}", + config.admin_listen_addr, self.admin_listen_addr, config.listen_addr + ))); + } + if self.max_body_size_bytes != config.max_body_size_bytes { + return Err(Error::InvalidConfigValue(format!( + "max_body_size_bytes {} does not match {} on the listener already bound to {}", + config.max_body_size_bytes, self.max_body_size_bytes, config.listen_addr + ))); + } + // The management API guards one shared listener, so instances cannot + // hold different opinions about who may call it. + if !same_token(&self.state.management_token, &config.management_token) { + return Err(Error::InvalidConfigValue(format!( + "management_token does not match the one on the listener already bound to {}", + config.listen_addr + ))); + } + Ok(()) + } + + async fn stop(signal: watch::Sender<()>, mut tasks: Vec>, listen_addr: &str) { + let _ = signal.send(()); + + // One deadline over both listeners: applied per task in a loop, the + // real bound would be twice SHUTDOWN_TIMEOUT. + let graceful = tokio::time::timeout(SHUTDOWN_TIMEOUT, async { + for task in &mut tasks { + if let Err(error) = task.await { + warn!("A {CONNECTOR_NAME} listener task on {listen_addr} failed. {error}"); + } + } + }) + .await; + + if graceful.is_err() { + warn!( + "A {CONNECTOR_NAME} listener on {listen_addr} did not stop within {SHUTDOWN_TIMEOUT:?}, aborting" + ); + // `abort()` only schedules cancellation. Awaiting the handle is + // what guarantees the future - and the TcpListener inside it - has + // actually been dropped, so the runtime's stop-then-start restart + // cannot race the bind and fail with EADDRINUSE. + for task in &mut tasks { + // Skip the ones the graceful loop already drove to completion: + // awaiting a finished `JoinHandle` a second time panics, and + // this runs inside the FFI close of a dlopened plugin. + if task.is_finished() { + continue; + } + task.abort(); + match task.await { + Err(error) if error.is_cancelled() => { + warn!("Aborted a wedged {CONNECTOR_NAME} listener task on {listen_addr}") + } + Err(error) => { + warn!("A {CONNECTOR_NAME} listener task on {listen_addr} panicked. {error}") + } + Ok(()) => {} + } + } + } + info!("Released the {CONNECTOR_NAME} listener on {listen_addr}"); + } +} + +async fn bind(address: &str) -> Result { + TcpListener::bind(address).await.map_err(|error| { + Error::InitError(format!( + "Failed to bind the {CONNECTOR_NAME} listener to {address}. {error}" + )) + }) +} + +async fn run( + listener: TcpListener, + router: Router, + mut shutdown: watch::Receiver<()>, + label: &str, +) { + let service = router.into_make_service_with_connect_info::(); + let result = serve(listener, service) + .with_graceful_shutdown(async move { + let _ = shutdown.changed().await; + }) + .await; + if let Err(error) = result { + error!("The {CONNECTOR_NAME} {label} listener stopped. {error}"); + } +} + +fn public_router(state: Arc, max_body_size_bytes: usize) -> Router { + Router::new() + .route("/topics/{topic_path}", post(handle_named_path)) + .route("/e/{endpoint_id}", post(handle_secret_path)) + .route("/health", get(handle_health)) + .layer(DefaultBodyLimit::max(max_body_size_bytes)) + .with_state(state) +} + +fn admin_router(state: Arc) -> Router { + Router::new() + .route("/admin/health", get(handle_admin_health)) + .route("/admin/metrics", get(handle_admin_metrics)) + .merge(management::router(Arc::clone(&state))) + .with_state(state) +} + +fn same_token(left: &Option, right: &Option) -> bool { + match (left, right) { + (None, None) => true, + (Some(left), Some(right)) => secrets_match(left, right), + _ => false, + } +} + +async fn handle_named_path( + State(state): State>, + Path(topic_path): Path, + ConnectInfo(remote_addr): ConnectInfo, + request_headers: HeaderMap, + body: Result, +) -> Response { + let started = Instant::now(); + let (instance_name, response) = + named_path_outcome(&state, &topic_path, remote_addr, &request_headers, body); + state.metrics.record_request( + &instance_name, + PathKind::Named, + response.status().as_u16(), + started.elapsed(), + ); + response +} + +async fn handle_secret_path( + State(state): State>, + Path(endpoint_id): Path, + ConnectInfo(remote_addr): ConnectInfo, + request_headers: HeaderMap, + body: Result, +) -> Response { + let started = Instant::now(); + let (instance_name, response) = + secret_path_outcome(&state, &endpoint_id, remote_addr, &request_headers, body); + state.metrics.record_request( + &instance_name, + PathKind::Secret, + response.status().as_u16(), + started.elapsed(), + ); + response +} + +/// Returns the instance the request resolved to alongside the response, so +/// the caller can label the metrics. Requests that never resolved are still +/// counted, under [`UNROUTED`]. +fn named_path_outcome( + state: &ServerState, + topic_path: &str, + remote_addr: SocketAddr, + request_headers: &HeaderMap, + body: Result, +) -> (String, Response) { + let routes = state.routes.load(); + let Some(instance) = routes.lookup_named_path(topic_path) else { + return ( + UNROUTED.to_owned(), + error_response(StatusCode::NOT_FOUND, "not found"), + ); + }; + let name = instance.instance_name.clone(); + let body = match body { + Ok(body) => body, + Err(rejection) => return (name, rejected_body_response(rejection)), + }; + if let Some(expected) = &instance.config.auth_bearer_token + && !validate_bearer(bearer_header(request_headers), expected) + { + return ( + name, + error_response(StatusCode::UNAUTHORIZED, "unauthorized"), + ); + } + let response = enqueue(instance, request_headers, remote_addr, body, &state.metrics); + (name, response) +} + +fn secret_path_outcome( + state: &ServerState, + endpoint_id: &str, + remote_addr: SocketAddr, + request_headers: &HeaderMap, + body: Result, +) -> (String, Response) { + let routes = state.routes.load(); + let entry = match routes.lookup_secret_path(endpoint_id, unix_now_seconds()) { + RouteLookup::Active(entry) => entry, + // Revoked endpoints answer as if they never existed, so a leaked URL + // cannot be used to confirm it was once live. The metric still names + // the owning instance: only a genuinely unknown path is `unrouted`. + RouteLookup::Revoked(entry) => { + return ( + entry.instance.instance_name.clone(), + error_response(StatusCode::NOT_FOUND, "not found"), + ); + } + RouteLookup::Expired(entry) => { + return ( + entry.instance.instance_name.clone(), + error_response(StatusCode::GONE, "gone"), + ); + } + RouteLookup::Unknown => { + return ( + UNROUTED.to_owned(), + error_response(StatusCode::NOT_FOUND, "not found"), + ); + } + }; + let name = entry.instance.instance_name.clone(); + let body = match body { + Ok(body) => body, + Err(rejection) => return (name, rejected_body_response(rejection)), + }; + if !authorize(&entry.endpoint, request_headers, &body) { + return ( + name, + error_response(StatusCode::UNAUTHORIZED, "unauthorized"), + ); + } + let response = enqueue( + &entry.instance, + request_headers, + remote_addr, + body, + &state.metrics, + ); + (name, response) +} + +/// Prometheus text format. Unguarded like `/admin/health`: the admin +/// listener defaults to loopback and scrapers do not carry bearer tokens. +async fn handle_admin_metrics(State(state): State>) -> String { + state.metrics.encode(&state.instances()) +} + +/// Readiness for a load balancer: unavailable until an instance is serving. +async fn handle_health(State(state): State>) -> Response { + if state.instances.load().is_empty() { + return ( + StatusCode::SERVICE_UNAVAILABLE, + Json(StatusResponse { + status: "unavailable", + }), + ) + .into_response(); + } + Json(StatusResponse { status: "ok" }).into_response() +} + +async fn handle_admin_health(State(state): State>) -> Response { + let instances = state + .instances + .load() + .iter() + .map(|instance| { + let registry = instance.registry(); + let now = unix_now_seconds(); + InstanceHealth { + instance: instance.instance_name.clone(), + topic_path: instance.config.topic_path.clone(), + buffer_used: instance.sender.len(), + buffer_capacity: instance.config.buffer_capacity, + endpoints_static: registry.serving_count_by_origin(EndpointOrigin::Static, now), + endpoints_dynamic: registry.serving_count_by_origin(EndpointOrigin::Dynamic, now), + endpoints_expired: registry.expired_count(now), + endpoints_revoked: registry.revoked_count(), + named_path: instance.config.topic_path.is_some(), + state_submitted: registry.all_submitted(), + dropped_headers: state.metrics.headers_dropped(&instance.instance_name), + clamped_headers: state.metrics.headers_clamped(&instance.instance_name), + } + }) + .collect(); + + Json(AdminHealth { + status: "ok", + instances, + uptime_secs: state.started_at.elapsed().as_secs(), + }) + .into_response() +} + +/// Hands an accepted request to its instance's bridge, or rejects it. +/// +/// Never blocks on a full bridge: waiting would turn a slow Iggy into a pile +/// of held-open connections and, once the sender times out, a retry storm. +fn enqueue( + instance: &Arc, + request_headers: &HeaderMap, + remote_addr: SocketAddr, + body: Bytes, + metrics: &Metrics, +) -> Response { + let (headers, clamped, dropped) = message_headers(instance, request_headers, remote_addr); + let message = QueuedMessage { + payload: body.to_vec(), + headers, + received_at: Instant::now(), + }; + if instance.sender.try_send(message).is_err() { + metrics.record_rejected_full(&instance.instance_name); + debug!( + "Rejected a request for {CONNECTOR_NAME} connector ID: {}, bridge is full at {} messages", + instance.id, instance.config.buffer_capacity + ); + return ( + StatusCode::TOO_MANY_REQUESTS, + [(header::RETRY_AFTER, "1")], + Json(ErrorResponse { + error: "service temporarily unavailable", + }), + ) + .into_response(); + } + // Only now: a rejected request produced no message, so its header losses + // would otherwise be counted against messages that never existed. + metrics.record_headers(&instance.instance_name, clamped, dropped); + Json(StatusResponse { status: "queued" }).into_response() +} + +fn authorize(endpoint: &Endpoint, request_headers: &HeaderMap, body: &[u8]) -> bool { + match endpoint.auth_type { + EndpointAuthType::None => true, + EndpointAuthType::Bearer => endpoint + .auth_secret + .as_ref() + .is_some_and(|secret| validate_bearer(bearer_header(request_headers), secret)), + EndpointAuthType::HmacSha256 | EndpointAuthType::HmacSha1 => { + let (Some(secret), Some(algorithm)) = ( + endpoint.auth_secret.as_ref(), + endpoint.auth_type.hmac_algorithm(), + ) else { + return false; + }; + validate_hmac( + body, + header_str(request_headers, &endpoint.hmac_header), + &endpoint.hmac_prefix, + secret, + algorithm, + ) + } + } +} + +/// Builds the Iggy headers an accepted request rides with. +/// +/// Values Iggy would reject are dropped rather than failing the message: a +/// webhook body is worth more than a `User-Agent`. +/// +/// Returns the headers to attach plus the number of forwarded values that were +/// truncated and dropped. The counts are returned rather than recorded here so +/// a request that is ultimately rejected does not report losses for a message +/// that was never produced. +fn message_headers( + instance: &Arc, + request_headers: &HeaderMap, + remote_addr: SocketAddr, +) -> (Option>, u64, u64) { + let mut headers = BTreeMap::new(); + let mut dropped = 0; + let mut clamped = 0; + if instance.config.include_http_metadata { + insert_header(&mut headers, INSTANCE_HEADER, &instance.instance_name); + insert_header( + &mut headers, + REMOTE_ADDR_HEADER, + &remote_addr.ip().to_string(), + ); + insert_header(&mut headers, RECEIVED_AT_HEADER, &received_at_micros()); + } + + for (name, key) in &instance.forward_headers { + let Some(present) = request_headers.get(name) else { + continue; + }; + // Present but unrepresentable is a loss; absent is not. `to_str` + // rejects any byte outside visible ASCII, which a UTF-8 User-Agent + // routinely carries. + let Ok(raw) = present.to_str() else { + dropped += 1; + continue; + }; + let Some(value) = clamp_header_value(raw) else { + dropped += 1; + continue; + }; + if value.len() < raw.len() { + clamped += 1; + } + match HeaderValue::from_str(value) { + Ok(value) => { + headers.insert(key.clone(), value); + } + Err(_) => dropped += 1, + } + } + ((!headers.is_empty()).then_some(headers), clamped, dropped) +} + +fn insert_header(headers: &mut BTreeMap, key: &str, value: &str) { + let (Ok(key), Ok(value)) = (HeaderKey::try_from(key), HeaderValue::from_str(value)) else { + return; + }; + headers.insert(key, value); +} + +fn received_at_micros() -> String { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|elapsed| elapsed.as_micros()) + .unwrap_or_default() + .to_string() +} + +fn bearer_header(request_headers: &HeaderMap) -> Option<&str> { + request_headers + .get(header::AUTHORIZATION) + .and_then(|value| value.to_str().ok()) +} + +fn header_str<'a>(request_headers: &'a HeaderMap, name: &str) -> Option<&'a str> { + request_headers + .get(name) + .and_then(|value| value.to_str().ok()) +} + +fn rejected_body_response(rejection: BytesRejection) -> Response { + let status = rejection.status(); + let message = if status == StatusCode::PAYLOAD_TOO_LARGE { + "payload too large" + } else { + "bad request" + }; + error_response(status, message) +} + +pub(crate) fn error_response(status: StatusCode, error: &'static str) -> Response { + (status, Json(ErrorResponse { error })).into_response() +} + +#[derive(Serialize)] +struct StatusResponse { + status: &'static str, +} + +#[derive(Serialize)] +struct ErrorResponse { + error: &'static str, +} + +#[derive(Serialize)] +struct AdminHealth { + status: &'static str, + instances: Vec, + uptime_secs: u64, +} + +#[derive(Serialize)] +struct InstanceHealth { + instance: String, + topic_path: Option, + buffer_used: usize, + buffer_capacity: usize, + endpoints_static: usize, + endpoints_dynamic: usize, + endpoints_expired: usize, + endpoints_revoked: usize, + named_path: bool, + state_submitted: bool, + dropped_headers: u64, + clamped_headers: u64, +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::HttpSource; + use crate::test_support::{ENDPOINT_ONE, ENDPOINT_TWO, client, free_port}; + use iggy_connector_sdk::Source; + use ring::hmac; + + const STATIC_SECRET: &str = "whsec_static"; + + fn config(public_port: u16, admin_port: u16, endpoints: &[&str]) -> HttpSourceConfig { + let mut config = crate::test_support::config(Some("github"), endpoints); + config.listen_addr = format!("127.0.0.1:{public_port}"); + config.admin_listen_addr = format!("127.0.0.1:{admin_port}"); + config + } + + async fn open(id: u32, config: HttpSourceConfig) -> HttpSource { + let mut source = HttpSource::new(id, config, None); + source.open().await.expect("open must succeed"); + source + } + + fn signature(body: &[u8]) -> String { + let key = hmac::Key::new(hmac::HMAC_SHA256, STATIC_SECRET.as_bytes()); + format!("sha256={}", hex::encode(hmac::sign(&key, body).as_ref())) + } + + async fn post_signed(base: &str, endpoint_id: &str, body: &'static str) -> reqwest::Response { + client() + .post(format!("{base}/e/{endpoint_id}")) + .header(crate::DEFAULT_HMAC_HEADER, signature(body.as_bytes())) + .body(body) + .send() + .await + .expect("the request must reach the listener") + } + + fn base_url(source: &HttpSource) -> String { + format!("http://{}", source.shared.config.listen_addr) + } + + #[tokio::test] + async fn given_valid_signature_when_posted_to_secret_path_should_queue() { + let mut source = open(1, config(free_port(), free_port(), &[ENDPOINT_ONE])).await; + + let response = post_signed(&base_url(&source), ENDPOINT_ONE, "{\"event\":\"push\"}").await; + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(source.shared.sender.len(), 1); + close(&mut source).await; + } + + #[tokio::test] + async fn given_tampered_signature_when_posted_should_answer_unauthorized() { + let mut source = open(1, config(free_port(), free_port(), &[ENDPOINT_ONE])).await; + + let response = client() + .post(format!("{}/e/{ENDPOINT_ONE}", base_url(&source))) + .header(crate::DEFAULT_HMAC_HEADER, signature(b"a different body")) + .body("{\"event\":\"push\"}") + .send() + .await + .expect("the request must reach the listener"); + + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + assert_eq!(source.shared.sender.len(), 0); + close(&mut source).await; + } + + #[tokio::test] + async fn given_unknown_endpoint_when_posted_should_answer_not_found() { + let mut source = open(1, config(free_port(), free_port(), &[ENDPOINT_ONE])).await; + + let response = post_signed(&base_url(&source), ENDPOINT_TWO, "{}").await; + + assert_eq!(response.status(), StatusCode::NOT_FOUND); + close(&mut source).await; + } + + #[tokio::test] + async fn given_revoked_endpoint_when_posted_should_answer_not_found() { + let mut source = open(1, config(free_port(), free_port(), &[ENDPOINT_ONE])).await; + source + .shared + .mutate_registry(|registry| registry.revoke(ENDPOINT_ONE, "compromised".to_string(), 1)) + .await; + rebuild_routes(&source).await; + + let response = post_signed(&base_url(&source), ENDPOINT_ONE, "{}").await; + + assert_eq!( + response.status(), + StatusCode::NOT_FOUND, + "a revoked endpoint must not be distinguishable from one that never existed" + ); + close(&mut source).await; + } + + #[tokio::test] + async fn given_expired_endpoint_when_posted_should_answer_gone() { + let mut source = open(1, config(free_port(), free_port(), &[ENDPOINT_ONE])).await; + source + .shared + .mutate_registry(|registry| { + registry + .endpoint_mut(ENDPOINT_ONE) + .expect("the static endpoint is registered") + .expires_at = Some(1); + }) + .await; + rebuild_routes(&source).await; + + let response = post_signed(&base_url(&source), ENDPOINT_ONE, "{}").await; + + assert_eq!(response.status(), StatusCode::GONE); + close(&mut source).await; + } + + #[tokio::test] + async fn given_oversized_body_when_posted_should_answer_payload_too_large() { + let mut config = config(free_port(), free_port(), &[ENDPOINT_ONE]); + config.max_body_size_bytes = 16; + let mut source = open(1, config).await; + + let response = client() + .post(format!("{}/e/{ENDPOINT_ONE}", base_url(&source))) + .body("x".repeat(1024)) + .send() + .await + .expect("the request must reach the listener"); + + assert_eq!(response.status(), StatusCode::PAYLOAD_TOO_LARGE); + close(&mut source).await; + } + + #[tokio::test] + async fn given_full_bridge_when_posted_should_answer_too_many_requests() { + let mut config = config(free_port(), free_port(), &[ENDPOINT_ONE]); + config.buffer_capacity = 1; + let mut source = open(1, config).await; + let base = base_url(&source); + + assert_eq!( + post_signed(&base, ENDPOINT_ONE, "{}").await.status(), + StatusCode::OK + ); + let response = post_signed(&base, ENDPOINT_ONE, "{}").await; + + assert_eq!(response.status(), StatusCode::TOO_MANY_REQUESTS); + assert_eq!( + response + .headers() + .get(header::RETRY_AFTER) + .and_then(|value| value.to_str().ok()), + Some("1"), + "a sender needs to be told when to come back, not just refused" + ); + close(&mut source).await; + } + + #[tokio::test] + async fn given_bearer_token_when_named_path_posted_should_enforce_it() { + let mut config = config(free_port(), free_port(), &[]); + config.auth_bearer_token = Some(secrecy::SecretString::from("global-secret")); + let mut source = open(1, config).await; + let url = format!("{}/topics/github", base_url(&source)); + + let unauthorized = client() + .post(&url) + .body("{}") + .send() + .await + .expect("the request must reach the listener"); + let authorized = client() + .post(&url) + .header(header::AUTHORIZATION, "Bearer global-secret") + .body("{}") + .send() + .await + .expect("the request must reach the listener"); + + assert_eq!(unauthorized.status(), StatusCode::UNAUTHORIZED); + assert_eq!(authorized.status(), StatusCode::OK); + assert_eq!(source.shared.sender.len(), 1); + close(&mut source).await; + } + + #[tokio::test] + async fn given_forwarded_headers_when_posted_should_ride_on_the_message() { + let mut config = config(free_port(), free_port(), &[ENDPOINT_ONE]); + config.forward_headers = vec!["x-github-delivery".to_string()]; + let mut source = open(1, config).await; + + client() + .post(format!("{}/e/{ENDPOINT_ONE}", base_url(&source))) + .header(crate::DEFAULT_HMAC_HEADER, signature(b"{}")) + .header("x-github-delivery", "72d3162e-cc78-11e3-81ab-4c9367dc0958") + .body("{}") + .send() + .await + .expect("the request must reach the listener"); + + let message = source + .receiver + .lock() + .await + .try_recv() + .expect("the request must have been queued"); + let headers = message.headers.expect("metadata forwarding is on"); + assert_eq!( + headers + .get(&HeaderKey::try_from("x-github-delivery").expect("valid key")) + .map(|value| value.as_str().expect("utf-8 value")), + Some("72d3162e-cc78-11e3-81ab-4c9367dc0958") + ); + assert!( + headers.contains_key(&HeaderKey::try_from(INSTANCE_HEADER).expect("valid key")), + "instance identity must ride along so a consumer can tell sources apart" + ); + close(&mut source).await; + } + + #[tokio::test] + async fn given_oversized_forwarded_header_when_posted_should_clamp_and_count() { + let mut config = config(free_port(), free_port(), &[ENDPOINT_ONE]); + config.forward_headers = vec!["user-agent".to_string()]; + let mut source = open(1, config).await; + + client() + .post(format!("{}/e/{ENDPOINT_ONE}", base_url(&source))) + .header(crate::DEFAULT_HMAC_HEADER, signature(b"{}")) + .header(header::USER_AGENT, "u".repeat(400)) + .body("{}") + .send() + .await + .expect("the request must reach the listener"); + + let message = source + .receiver + .lock() + .await + .try_recv() + .expect("the request must have been queued"); + let headers = message.headers.expect("metadata forwarding is on"); + let forwarded = headers + .get(&HeaderKey::try_from("user-agent").expect("valid key")) + .and_then(|value| value.as_str().ok()) + .expect("an oversized value is clamped, not dropped"); + assert_eq!(forwarded.len(), crate::types::MAX_HEADER_VALUE_BYTES); + let (clamped, dropped) = metrics_snapshot(&source, &source.shared.instance_name).await; + assert_eq!(clamped, 1); + assert_eq!(dropped, 0, "an oversized value is clamped, never dropped"); + close(&mut source).await; + } + + #[tokio::test] + async fn given_serving_instance_when_admin_health_read_should_report_counts() { + let mut config = config(free_port(), free_port(), &[ENDPOINT_ONE]); + config.instance_name = Some("http_github".to_string()); + config.buffer_capacity = 7; + let admin = format!("http://{}", config.admin_listen_addr); + let mut source = open(1, config).await; + post_signed(&base_url(&source), ENDPOINT_ONE, "{}").await; + + let body: serde_json::Value = client() + .get(format!("{admin}/admin/health")) + .send() + .await + .expect("the request must reach the admin listener") + .json() + .await + .expect("admin health must be JSON"); + + assert_eq!(body["status"], "ok"); + let instance = &body["instances"][0]; + assert_eq!(instance["instance"], "http_github"); + assert_eq!(instance["buffer_used"], 1); + assert_eq!(instance["buffer_capacity"], 7); + assert_eq!(instance["endpoints_static"], 1); + assert_eq!(instance["endpoints_dynamic"], 0); + assert_eq!(instance["endpoints_revoked"], 0); + assert_eq!(instance["named_path"], true); + assert_eq!(instance["state_submitted"], true); + close(&mut source).await; + } + + #[tokio::test] + async fn given_revoked_endpoint_when_admin_health_read_should_report_it_as_revoked() { + let mut config = config(free_port(), free_port(), &[ENDPOINT_ONE]); + config.instance_name = Some("http_github".to_string()); + let admin = format!("http://{}", config.admin_listen_addr); + let mut source = open(1, config).await; + source + .shared + .mutate_registry(|registry| registry.revoke(ENDPOINT_ONE, "compromised".to_string(), 1)) + .await; + + let body: serde_json::Value = client() + .get(format!("{admin}/admin/health")) + .send() + .await + .expect("the request must reach the admin listener") + .json() + .await + .expect("admin health must be JSON"); + + let instance = &body["instances"][0]; + assert_eq!(instance["endpoints_static"], 0); + assert_eq!(instance["endpoints_revoked"], 1); + assert_eq!( + instance["state_submitted"], false, + "a mutation not yet handed to the runtime must not read as durable" + ); + close(&mut source).await; + } + + #[tokio::test] + async fn given_expired_endpoint_when_admin_health_read_should_not_count_it_as_serving() { + let mut config = config(free_port(), free_port(), &[ENDPOINT_ONE]); + config.instance_name = Some("http_github".to_string()); + let admin = format!("http://{}", config.admin_listen_addr); + let mut source = open(1, config).await; + source + .shared + .mutate_registry(|registry| { + registry + .endpoint_mut(ENDPOINT_ONE) + .expect("the static endpoint is registered") + .expires_at = Some(1); + }) + .await; + + let body: serde_json::Value = client() + .get(format!("{admin}/admin/health")) + .send() + .await + .expect("the request must reach the admin listener") + .json() + .await + .expect("admin health must be JSON"); + + let instance = &body["instances"][0]; + assert_eq!( + instance["endpoints_static"], 0, + "an endpoint that answers 410 to everything is not serving" + ); + assert_eq!(instance["endpoints_expired"], 1); + assert_eq!(instance["endpoints_revoked"], 0); + close(&mut source).await; + } + + #[tokio::test] + async fn given_metadata_enabled_when_posted_should_carry_peer_and_receive_time() { + let mut source = open(1, config(free_port(), free_port(), &[ENDPOINT_ONE])).await; + let before = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("the clock is after 1970") + .as_micros(); + + post_signed(&base_url(&source), ENDPOINT_ONE, "{}").await; + + let message = source + .receiver + .lock() + .await + .try_recv() + .expect("the request must have been queued"); + let headers = message.headers.expect("metadata forwarding is on"); + assert_eq!( + headers + .get(&HeaderKey::try_from(REMOTE_ADDR_HEADER).expect("valid key")) + .and_then(|value| value.as_str().ok()), + Some("127.0.0.1") + ); + let received_at: u128 = headers + .get(&HeaderKey::try_from(RECEIVED_AT_HEADER).expect("valid key")) + .and_then(|value| value.as_str().ok()) + .expect("receive time must be present") + .parse() + .expect("receive time must be numeric microseconds"); + assert!( + received_at >= before, + "receive time must not predate the request" + ); + close(&mut source).await; + } + + #[tokio::test] + async fn given_metadata_disabled_when_posted_should_carry_no_headers() { + let mut config = config(free_port(), free_port(), &[ENDPOINT_ONE]); + config.include_http_metadata = false; + let mut source = open(1, config).await; + + post_signed(&base_url(&source), ENDPOINT_ONE, "{}").await; + + let message = source + .receiver + .lock() + .await + .try_recv() + .expect("the request must have been queued"); + assert!( + message.headers.is_none(), + "no metadata and no forward_headers must mean no header map at all" + ); + close(&mut source).await; + } + + #[tokio::test] + async fn given_two_instances_when_joined_should_share_one_listener() { + let public_port = free_port(); + let admin_port = free_port(); + let mut first = open(1, config(public_port, admin_port, &[ENDPOINT_ONE])).await; + let mut second_config = config(public_port, admin_port, &[ENDPOINT_TWO]); + second_config.topic_path = Some("stripe".to_string()); + let mut second = open(2, second_config).await; + let base = base_url(&first); + + assert_eq!( + post_signed(&base, ENDPOINT_ONE, "{}").await.status(), + StatusCode::OK + ); + assert_eq!( + post_signed(&base, ENDPOINT_TWO, "{}").await.status(), + StatusCode::OK + ); + assert_eq!(first.shared.sender.len(), 1); + assert_eq!(second.shared.sender.len(), 1); + + close(&mut first).await; + assert_eq!( + post_signed(&base, ENDPOINT_ONE, "{}").await.status(), + StatusCode::NOT_FOUND, + "one instance leaving must not take the listener down with it" + ); + assert_eq!( + post_signed(&base, ENDPOINT_TWO, "{}").await.status(), + StatusCode::OK + ); + close(&mut second).await; + } + + #[tokio::test] + async fn given_mismatched_body_limit_when_joined_should_reject() { + let public_port = free_port(); + let admin_port = free_port(); + let mut first = open(1, config(public_port, admin_port, &[ENDPOINT_ONE])).await; + let mut second_config = config(public_port, admin_port, &[ENDPOINT_TWO]); + second_config.topic_path = Some("stripe".to_string()); + second_config.max_body_size_bytes = 1; + + let mut second = HttpSource::new(2, second_config, None); + let error = second + .open() + .await + .expect_err("a listener cannot serve two body limits at once"); + + assert!(matches!( + error, + Error::InvalidConfigValue(message) if message.contains("max_body_size_bytes") + )); + close(&mut first).await; + } + + #[tokio::test] + async fn given_mismatched_admin_address_when_joined_should_reject() { + let public_port = free_port(); + let mut first = open(1, config(public_port, free_port(), &[ENDPOINT_ONE])).await; + let mut second_config = config(public_port, free_port(), &[ENDPOINT_TWO]); + second_config.topic_path = Some("stripe".to_string()); + + let mut second = HttpSource::new(2, second_config, None); + let error = second + .open() + .await + .expect_err("an instance must not silently get an admin listener it did not configure"); + + assert!(matches!( + error, + Error::InvalidConfigValue(message) if message.contains("admin_listen_addr") + )); + close(&mut first).await; + } + + #[tokio::test] + async fn given_mismatched_management_token_when_joined_should_reject() { + let public_port = free_port(); + let admin_port = free_port(); + let mut first_config = config(public_port, admin_port, &[ENDPOINT_ONE]); + first_config.management_token = Some(SecretString::from("mgmt-secret")); + let mut first = open(1, first_config).await; + let mut second_config = config(public_port, admin_port, &[ENDPOINT_TWO]); + second_config.topic_path = Some("stripe".to_string()); + + let mut second = HttpSource::new(2, second_config, None); + let error = second + .open() + .await + .expect_err("one listener cannot answer to two management tokens"); + + assert!(matches!( + error, + Error::InvalidConfigValue(message) if message.contains("management_token") + )); + close(&mut first).await; + } + + #[tokio::test] + async fn given_colliding_topic_path_when_joined_should_reject() { + let public_port = free_port(); + let admin_port = free_port(); + let mut first = open(1, config(public_port, admin_port, &[ENDPOINT_ONE])).await; + + let mut second = HttpSource::new(2, config(public_port, admin_port, &[ENDPOINT_TWO]), None); + let error = second + .open() + .await + .expect_err("two instances cannot claim the same topic path"); + + assert!(matches!( + error, + Error::InvalidConfigValue(message) if message.contains("topic_path") + )); + assert_eq!( + post_signed(&base_url(&first), ENDPOINT_TWO, "{}") + .await + .status(), + StatusCode::NOT_FOUND, + "a rejected join must leave no routes behind" + ); + close(&mut first).await; + } + + #[tokio::test] + async fn given_last_instance_when_left_should_release_the_port() { + let public_port = free_port(); + let mut source = open(1, config(public_port, free_port(), &[ENDPOINT_ONE])).await; + close(&mut source).await; + + TcpListener::bind(format!("127.0.0.1:{public_port}")) + .await + .expect("the last close must release the port for the runtime's restart flow"); + } + + #[tokio::test] + async fn given_served_traffic_when_metrics_scraped_should_report_it() { + let mut config = config(free_port(), free_port(), &[ENDPOINT_ONE]); + config.instance_name = Some("http_github".to_string()); + config.buffer_capacity = 4; + let admin = format!("http://{}", config.admin_listen_addr); + let mut source = open(1, config).await; + let base = base_url(&source); + + post_signed(&base, ENDPOINT_ONE, "{}").await; + post_signed(&base, ENDPOINT_TWO, "{}").await; + + let scraped = client() + .get(format!("{admin}/admin/metrics")) + .send() + .await + .expect("the request must reach the admin listener") + .text() + .await + .expect("the scrape must have a body"); + + assert!(scraped.contains( + "http_source_requests_total{instance=\"http_github\",kind=\"secret\",status=\"2xx\"} 1" + )); + assert!( + scraped.contains( + "http_source_requests_total{instance=\"unrouted\",kind=\"secret\",status=\"4xx\"} 1" + ), + "a probe for live endpoint ids must be countable even though it resolves to nothing" + ); + assert!(scraped.contains("http_source_buffer_used{instance=\"http_github\"} 1")); + assert!(scraped.contains("http_source_buffer_capacity{instance=\"http_github\"} 4")); + assert!( + scraped.contains( + "http_source_endpoints_active{instance=\"http_github\",kind=\"static\"} 1" + ) + ); + close(&mut source).await; + } + + #[tokio::test] + async fn given_queued_messages_when_instance_closes_should_count_the_loss() { + let public_port = free_port(); + let admin_port = free_port(); + let mut first_config = config(public_port, admin_port, &[ENDPOINT_ONE]); + first_config.instance_name = Some("http_github".to_string()); + let admin = format!("http://{}", first_config.admin_listen_addr); + let mut first = open(1, first_config).await; + // A sibling keeps the listener alive so the counter survives to be + // scraped; the last instance leaving takes the whole registry with it. + let mut second_config = config(public_port, admin_port, &[ENDPOINT_TWO]); + second_config.topic_path = Some("stripe".to_string()); + second_config.instance_name = Some("http_stripe".to_string()); + let mut second = open(2, second_config).await; + + post_signed(&base_url(&first), ENDPOINT_ONE, "{}").await; + close(&mut first).await; + + let scraped = client() + .get(format!("{admin}/admin/metrics")) + .send() + .await + .expect("the request must reach the admin listener") + .text() + .await + .expect("the scrape must have a body"); + + assert!( + scraped.contains("http_source_dropped_on_close_total{instance=\"http_github\"} 1"), + "messages accepted but never polled are lost, and the loss must be visible" + ); + close(&mut second).await; + } + + #[tokio::test] + async fn given_serving_instance_when_health_checked_should_report_ok() { + let mut source = open(1, config(free_port(), free_port(), &[])).await; + + let response = client() + .get(format!("{}/health", base_url(&source))) + .send() + .await + .expect("the request must reach the listener"); + + assert_eq!(response.status(), StatusCode::OK); + close(&mut source).await; + } + + #[tokio::test] + async fn given_no_instances_when_health_checked_should_report_unavailable() { + let mut source = open(1, config(free_port(), free_port(), &[])).await; + // The window a load balancer must see: routes are gone but the last + // instance has not finished releasing the listener yet. + publish(&source, Vec::new()).await; + + let response = client() + .get(format!("{}/health", base_url(&source))) + .send() + .await + .expect("the request must reach the listener"); + + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + close(&mut source).await; + } + + /// Republishes the instance set so a registry mutation reaches the table. + /// The management API does this for real; these tests reach for it after + /// mutating the registry directly. + async fn rebuild_routes(source: &HttpSource) { + let instances = { + let servers = SERVERS.lock().await; + servers + .get(&source.shared.config.listen_addr) + .expect("the instance is joined") + .state + .instances() + }; + publish(source, instances).await; + } + + async fn publish(source: &HttpSource, instances: Vec>) { + let servers = SERVERS.lock().await; + servers + .get(&source.shared.config.listen_addr) + .expect("the instance is joined") + .state + .publish(instances) + .expect("republishing a known-good instance set cannot collide"); + } + + /// The metrics live on the listener, not the instance, so a test has to + /// go through the registry to read them. + async fn metrics_snapshot(source: &HttpSource, instance: &str) -> (u64, u64) { + let servers = SERVERS.lock().await; + let metrics = &servers + .get(&source.shared.config.listen_addr) + .expect("the instance is joined") + .state + .metrics; + ( + metrics.headers_clamped(instance), + metrics.headers_dropped(instance), + ) + } + + async fn close(source: &mut HttpSource) { + source.close().await.expect("close must succeed"); + } +} diff --git a/core/connectors/sources/http_source/src/state.rs b/core/connectors/sources/http_source/src/state.rs new file mode 100644 index 0000000000..73a04f4b5c --- /dev/null +++ b/core/connectors/sources/http_source/src/state.rs @@ -0,0 +1,510 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Endpoint registry and its round trip through `ConnectorState`. +//! +//! The registry is per instance and authoritative: the route table is a +//! projection of it, and it is the unit the runtime persists after a +//! successful send. Endpoints created through the management API exist only +//! here, so without this round trip a restart would drop them. +//! +//! Persisting them means writing their bearer tokens and HMAC secrets to the +//! runtime state directory in the clear. That is the same at-rest posture as +//! the TOML those endpoints would otherwise live in; the README requires +//! `chmod 700` on the state path. + +use iggy_connector_sdk::ConnectorState; +use secrecy::{ExposeSecret, SecretString}; +use serde::{Deserialize, Serialize, Serializer}; +use std::collections::BTreeMap; +use std::collections::btree_map::Entry; +use tracing::{info, warn}; + +use crate::routes::{Endpoint, EndpointOrigin}; +use crate::types::EndpointId; +use crate::{CONNECTOR_NAME, StaticEndpointConfig}; + +/// Every secret-path endpoint one instance owns, static and dynamic alike. +/// +/// Ordered so the encoding is deterministic; an unordered map would produce +/// different bytes for an identical registry. +#[derive(Debug, Default, Clone, Serialize, Deserialize)] +pub struct EndpointRegistry { + #[serde(default)] + endpoints: BTreeMap, +} + +impl EndpointRegistry { + /// Merges the TOML endpoints with whatever the runtime persisted. + /// + /// TOML wins for endpoints it declares: editing a static endpoint and + /// restarting the instance is the documented way to change it. The one + /// exception is a revocation tombstone, which always wins, so an operator + /// who revoked a compromised endpoint does not get it back by restarting + /// against a TOML file nobody remembered to edit. + pub fn restore( + static_endpoints: &[StaticEndpointConfig], + state: Option, + connector_id: u32, + ) -> Self { + let mut endpoints: BTreeMap = static_endpoints + .iter() + .map(|config| (config.endpoint_id.clone(), Endpoint::from(config))) + .collect(); + let static_count = endpoints.len(); + + let had_state = state.is_some(); + let Some(persisted) = state + .and_then(|state| state.deserialize::(CONNECTOR_NAME, connector_id)) + else { + if had_state { + // Unreadable state is not a clean start: every revocation + // tombstone is gone, so an endpoint revoked because it was + // compromised is about to come back live from TOML. + warn!( + "Discarded an unreadable registry for {CONNECTOR_NAME} connector ID: {connector_id}; revocation tombstones are lost and static endpoints will be served again, static endpoints: {static_count}" + ); + } else { + info!( + "Started {CONNECTOR_NAME} connector ID: {connector_id} with no persisted registry, static endpoints: {static_count}" + ); + } + return EndpointRegistry { endpoints }; + }; + + let mut restored = 0; + let mut tombstones = 0; + for (endpoint_id, mut endpoint) in persisted.endpoints { + endpoint.submitted = true; + let revoked = !endpoint.is_active(); + match endpoints.entry(endpoint_id) { + Entry::Occupied(mut occupied) if revoked => { + occupied.insert(endpoint); + tombstones += 1; + } + Entry::Occupied(_) => {} + Entry::Vacant(vacant) => { + vacant.insert(endpoint); + if revoked { + tombstones += 1; + } else { + restored += 1; + } + } + } + } + info!( + "Restored registry for {CONNECTOR_NAME} connector ID: {connector_id}, static endpoints: {static_count}, dynamic endpoints: {restored}, revoked: {tombstones}" + ); + + EndpointRegistry { endpoints } + } + + pub fn endpoints(&self) -> impl Iterator { + self.endpoints.values() + } + + pub fn endpoint(&self, endpoint_id: &str) -> Option<&Endpoint> { + self.endpoints.get(endpoint_id) + } + + pub fn endpoint_mut(&mut self, endpoint_id: &str) -> Option<&mut Endpoint> { + self.endpoints.get_mut(endpoint_id) + } + + /// Registers a new endpoint, refusing to overwrite an existing one so a + /// generated-id collision can never silently retarget live traffic. + pub fn insert(&mut self, endpoint: Endpoint) -> bool { + match self.endpoints.entry(endpoint.endpoint_id.clone()) { + Entry::Occupied(_) => false, + Entry::Vacant(vacant) => { + vacant.insert(endpoint); + true + } + } + } + + /// Drops an endpoint outright, as opposed to tombstoning it. Only for + /// undoing a registration that never became reachable; a live endpoint is + /// always revoked instead, so the tombstone survives a restart. + pub fn remove(&mut self, endpoint_id: &str) -> bool { + self.endpoints.remove(endpoint_id).is_some() + } + + pub fn revoke(&mut self, endpoint_id: &str, reason: String, revoked_at: u64) -> bool { + let Some(endpoint) = self.endpoints.get_mut(endpoint_id) else { + return false; + }; + if !endpoint.is_active() { + return false; + } + endpoint.revoke(reason, revoked_at); + true + } + + /// Flags the whole registry as handed to the runtime for persistence. + /// + /// The plugin never learns whether the save itself succeeded: the runtime + /// writes state only after the batch carrying it lands in Iggy, and no + /// acknowledgement comes back across the FFI. That is why the flag is + /// `submitted` rather than `persisted` - it is the strongest claim this + /// side of the boundary can honestly make. + pub fn mark_submitted(&mut self) { + for endpoint in self.endpoints.values_mut() { + endpoint.submitted = true; + } + } + + pub fn len(&self) -> usize { + self.endpoints.len() + } + + pub fn is_empty(&self) -> bool { + self.endpoints.is_empty() + } + + /// Endpoints that would accept a request right now: neither revoked nor + /// past their expiry. An expired endpoint is still `Active` in lifecycle + /// terms but answers 410, so counting it as serving would mislead. + pub fn serving_count(&self, now_seconds: u64) -> usize { + self.endpoints + .values() + .filter(|endpoint| endpoint.is_serving(now_seconds)) + .count() + } + + pub fn serving_count_by_origin(&self, origin: EndpointOrigin, now_seconds: u64) -> usize { + self.endpoints + .values() + .filter(|endpoint| endpoint.is_serving(now_seconds) && endpoint.origin == origin) + .count() + } + + pub fn expired_count(&self, now_seconds: u64) -> usize { + self.endpoints + .values() + .filter(|endpoint| endpoint.is_active() && endpoint.is_expired(now_seconds)) + .count() + } + + pub fn revoked_count(&self) -> usize { + self.endpoints + .values() + .filter(|endpoint| !endpoint.is_active()) + .count() + } + + /// Whether every endpoint has been handed to the runtime, which is what + /// the admin listener reports as `state_submitted`. + pub fn all_submitted(&self) -> bool { + self.endpoints.values().all(|endpoint| endpoint.submitted) + } + + pub fn to_connector_state(&self, connector_id: u32) -> Option { + ConnectorState::serialize(self, CONNECTOR_NAME, connector_id) + } +} + +/// Writes an endpoint secret in the clear, for `ConnectorState` only. +/// +/// A local helper rather than the shared +/// [`iggy_common::serde_secret::serialize_optional_secret`] so that the one +/// place in this crate that deliberately writes a secret is greppable and +/// cannot be reached by accident: `HttpSourceConfig` does not implement +/// `Serialize` at all, precisely so a credential cannot leak that way. +pub fn serialize_secret_to_state( + secret: &Option, + serializer: S, +) -> Result +where + S: Serializer, +{ + match secret { + Some(secret) => serializer.serialize_some(secret.expose_secret()), + None => serializer.serialize_none(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::EndpointAuthType; + use crate::routes::EndpointState; + use crate::test_support::{ENDPOINT_ONE, ENDPOINT_TWO, endpoint_id, static_endpoint}; + + fn dynamic_endpoint(raw_id: &str) -> Endpoint { + Endpoint { + endpoint_id: endpoint_id(raw_id), + auth_type: EndpointAuthType::Bearer, + auth_secret: Some(SecretString::from("whsec_dynamic")), + hmac_header: crate::DEFAULT_HMAC_HEADER.to_string(), + hmac_prefix: crate::DEFAULT_HMAC_PREFIX.to_string(), + expires_at: None, + origin: EndpointOrigin::Dynamic, + state: EndpointState::Active, + submitted: false, + } + } + + fn registry_state(registry: &EndpointRegistry) -> ConnectorState { + registry + .to_connector_state(1) + .expect("registry must serialize") + } + + #[test] + fn given_persisted_state_should_restore_dynamic_endpoints() { + let mut original = EndpointRegistry::default(); + assert!(original.insert(dynamic_endpoint(ENDPOINT_TWO))); + + let restored = EndpointRegistry::restore(&[], Some(registry_state(&original)), 1); + + let endpoint = restored + .endpoint(ENDPOINT_TWO) + .expect("dynamic endpoint must survive the round trip"); + assert_eq!(endpoint.auth_type, EndpointAuthType::Bearer); + assert_eq!( + endpoint + .auth_secret + .as_ref() + .map(|secret| secret.expose_secret()), + Some("whsec_dynamic"), + "a redacted secret would reject every request the sender signs" + ); + assert!(endpoint.submitted); + } + + #[test] + fn given_no_state_should_start_from_static_config_only() { + let restored = EndpointRegistry::restore(&[static_endpoint(ENDPOINT_ONE)], None, 1); + + assert_eq!(restored.len(), 1); + assert!(restored.endpoint(ENDPOINT_ONE).is_some()); + assert!(restored.endpoint(ENDPOINT_TWO).is_none()); + } + + #[test] + fn given_invalid_state_should_start_from_static_config_only() { + let invalid = ConnectorState(b"not valid msgpack".to_vec()); + + let restored = + EndpointRegistry::restore(&[static_endpoint(ENDPOINT_ONE)], Some(invalid), 1); + + assert_eq!(restored.len(), 1); + assert!(restored.endpoint(ENDPOINT_ONE).is_some()); + } + + #[test] + fn state_should_be_serializable_and_deserializable() { + let mut original = EndpointRegistry::default(); + assert!(original.insert(dynamic_endpoint(ENDPOINT_ONE))); + original.revoke(ENDPOINT_ONE, "compromised".to_string(), 42); + + let bytes = rmp_serde::to_vec(&original).expect("registry must serialize"); + let deserialized: EndpointRegistry = + rmp_serde::from_slice(&bytes).expect("registry must deserialize"); + + assert_eq!(original.len(), deserialized.len()); + assert_eq!( + deserialized + .endpoint(ENDPOINT_ONE) + .expect("endpoint must survive") + .state, + EndpointState::Revoked { + reason: "compromised".to_string(), + revoked_at: 42, + } + ); + } + + #[test] + fn serialize_state_helper_should_produce_valid_connector_state() { + let mut registry = EndpointRegistry::default(); + assert!(registry.insert(dynamic_endpoint(ENDPOINT_ONE))); + + let bytes = registry + .to_connector_state(1) + .expect("registry must serialize") + .0; + + let restored: EndpointRegistry = + rmp_serde::from_slice(&bytes).expect("registry must deserialize"); + assert!(restored.endpoint(ENDPOINT_ONE).is_some()); + } + + /// Mirrors `Endpoint`'s wire shape minus the trailing `state`, i.e. what a + /// writer from before that field existed would have emitted. Built as a + /// struct rather than by editing bytes, so it stays honest if the fixture + /// changes. + #[derive(Serialize)] + struct EndpointMissingState { + endpoint_id: String, + auth_type: String, + auth_secret: Option, + hmac_header: String, + hmac_prefix: String, + expires_at: Option, + origin: String, + } + + /// `Endpoint` plus one appended, defaulted field: what a FUTURE version + /// looks like reading today's bytes. + #[derive(Deserialize)] + #[allow(dead_code)] + struct EndpointWithAddedField { + endpoint_id: String, + auth_type: String, + auth_secret: Option, + hmac_header: String, + hmac_prefix: String, + expires_at: Option, + origin: String, + state: EndpointState, + #[serde(default)] + added_later: Option, + } + + /// The registry is a struct with one field, so under rmp's compact codec + /// it encodes as a one-element array wrapping the map, not a bare map. + #[derive(Serialize)] + struct RegistryMissingState { + endpoints: BTreeMap, + } + + #[derive(Deserialize)] + struct RegistryWithAddedField { + endpoints: BTreeMap, + } + + #[test] + fn given_a_field_appended_later_when_old_bytes_are_read_should_still_decode() { + // The forward-compat contract: append, and give the new field a + // default. Old state files must keep decoding. + let mut registry = EndpointRegistry::default(); + assert!(registry.insert(dynamic_endpoint(ENDPOINT_ONE))); + let bytes = rmp_serde::to_vec(®istry).expect("registry must serialize"); + + let decoded: RegistryWithAddedField = + rmp_serde::from_slice(&bytes).expect("appending a defaulted field must stay readable"); + + assert_eq!(decoded.endpoints.len(), 1); + assert!(decoded.endpoints[ENDPOINT_ONE].added_later.is_none()); + } + + #[test] + fn given_a_record_missing_its_state_when_restored_should_fail_closed() { + // The opposite direction, and the one that matters for security: a + // record that cannot supply `state` must NOT decode as Active. It + // previously did, because `state` carried `#[serde(default)]` and + // `EndpointState::default()` was `Active` - which silently put a + // revoked endpoint back into service. + let truncated = RegistryMissingState { + endpoints: BTreeMap::from([( + ENDPOINT_ONE.to_string(), + EndpointMissingState { + endpoint_id: ENDPOINT_ONE.to_string(), + auth_type: "bearer".to_string(), + auth_secret: Some("whsec_dynamic".to_string()), + hmac_header: crate::DEFAULT_HMAC_HEADER.to_string(), + hmac_prefix: crate::DEFAULT_HMAC_PREFIX.to_string(), + expires_at: None, + origin: "Dynamic".to_string(), + }, + )]), + }; + let bytes = rmp_serde::to_vec(&truncated).expect("the shadow must serialize"); + + assert!( + rmp_serde::from_slice::(&bytes).is_err(), + "a record that cannot supply its lifecycle state must fail the decode" + ); + + let restored = EndpointRegistry::restore(&[], Some(ConnectorState(bytes)), 1); + assert!( + restored.endpoint(ENDPOINT_ONE).is_none(), + "and restore must fall back to static-only rather than serve it" + ); + } + + #[test] + fn given_revoked_endpoint_in_state_when_restored_should_keep_tombstone_over_static_config() { + let mut persisted = EndpointRegistry::default(); + assert!(persisted.insert(dynamic_endpoint(ENDPOINT_ONE))); + persisted.revoke(ENDPOINT_ONE, "compromised".to_string(), 42); + + let restored = EndpointRegistry::restore( + &[static_endpoint(ENDPOINT_ONE)], + Some(registry_state(&persisted)), + 1, + ); + + let endpoint = restored.endpoint(ENDPOINT_ONE).expect("entry must exist"); + assert!( + !endpoint.is_active(), + "a stale TOML entry must not resurrect a revoked endpoint" + ); + } + + #[test] + fn given_active_endpoint_in_both_when_restored_should_prefer_static_config() { + let mut persisted = EndpointRegistry::default(); + assert!(persisted.insert(dynamic_endpoint(ENDPOINT_ONE))); + + let restored = EndpointRegistry::restore( + &[static_endpoint(ENDPOINT_ONE)], + Some(registry_state(&persisted)), + 1, + ); + + let endpoint = restored.endpoint(ENDPOINT_ONE).expect("entry must exist"); + assert_eq!( + endpoint.auth_type, + EndpointAuthType::HmacSha256, + "editing TOML and restarting is the documented way to change a static endpoint" + ); + } + + #[test] + fn given_active_endpoint_when_revoked_twice_should_reject_the_second() { + let mut registry = EndpointRegistry::default(); + assert!(registry.insert(dynamic_endpoint(ENDPOINT_ONE))); + + assert!(registry.revoke(ENDPOINT_ONE, "compromised".to_string(), 42)); + assert!(!registry.revoke(ENDPOINT_ONE, "again".to_string(), 43)); + assert!(!registry.revoke(ENDPOINT_TWO, "unknown".to_string(), 44)); + } + + #[test] + fn given_existing_endpoint_id_when_inserted_should_reject() { + let mut registry = EndpointRegistry::default(); + + assert!(registry.insert(dynamic_endpoint(ENDPOINT_ONE))); + assert!(!registry.insert(dynamic_endpoint(ENDPOINT_ONE))); + assert_eq!(registry.len(), 1); + } + + #[test] + fn given_mixed_registry_when_counted_should_exclude_tombstones() { + let mut registry = EndpointRegistry::default(); + assert!(registry.insert(dynamic_endpoint(ENDPOINT_ONE))); + assert!(registry.insert(dynamic_endpoint(ENDPOINT_TWO))); + registry.revoke(ENDPOINT_TWO, "compromised".to_string(), 42); + + assert_eq!(registry.len(), 2); + assert_eq!(registry.serving_count(0), 1); + } +} diff --git a/core/connectors/sources/http_source/src/types.rs b/core/connectors/sources/http_source/src/types.rs new file mode 100644 index 0000000000..ad7fe3b950 --- /dev/null +++ b/core/connectors/sources/http_source/src/types.rs @@ -0,0 +1,247 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use iggy_common::{HeaderKey, HeaderValue}; +use iggy_connector_sdk::ProducedMessage; +use serde::{Deserialize, Serialize}; +use std::borrow::Borrow; +use std::collections::BTreeMap; +use std::fmt::{self, Display, Formatter}; +use std::str::FromStr; +use std::time::{Instant, SystemTime, UNIX_EPOCH}; + +/// Iggy `HeaderValue` rejects values above this size, and forwarded HTTP +/// header values (e.g. `User-Agent`) routinely exceed it. +pub const MAX_HEADER_VALUE_BYTES: usize = 255; + +/// Validated secret-path endpoint identifier: exactly 32 lowercase hex +/// characters, giving the URL itself 128 bits of entropy. +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(try_from = "String", into = "String")] +pub struct EndpointId(String); + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum EndpointIdError { + InvalidLength(usize), + InvalidCharacter(char), +} + +impl EndpointId { + pub const LENGTH: usize = 32; + const LOG_PREFIX_LENGTH: usize = 8; + + pub fn as_str(&self) -> &str { + &self.0 + } + + /// A correlation handle for logs. The full id is the credential for a + /// secret-path endpoint, so it must never reach a log line. + pub fn log_prefix(&self) -> String { + Self::log_prefix_of(&self.0) + } + + pub fn log_prefix_of(endpoint_id: &str) -> String { + format!( + "{}...", + &endpoint_id[..Self::LOG_PREFIX_LENGTH.min(endpoint_id.len())] + ) + } +} + +impl FromStr for EndpointId { + type Err = EndpointIdError; + + fn from_str(value: &str) -> Result { + if value.len() != Self::LENGTH { + return Err(EndpointIdError::InvalidLength(value.len())); + } + if let Some(invalid) = value + .chars() + .find(|character| !matches!(character, '0'..='9' | 'a'..='f')) + { + return Err(EndpointIdError::InvalidCharacter(invalid)); + } + Ok(Self(value.to_string())) + } +} + +impl TryFrom for EndpointId { + type Error = EndpointIdError; + + fn try_from(value: String) -> Result { + value.parse() + } +} + +impl From for String { + fn from(endpoint_id: EndpointId) -> Self { + endpoint_id.0 + } +} + +/// Lets `HashMap` be probed with the raw path segment, so +/// request routing never allocates. Sound because the derived `Hash` on a +/// single-field newtype hashes exactly as the inner `String`, which in turn +/// hashes as `str`. +impl Borrow for EndpointId { + fn borrow(&self) -> &str { + &self.0 + } +} + +impl Display for EndpointId { + fn fmt(&self, formatter: &mut Formatter<'_>) -> fmt::Result { + write!(formatter, "{}", self.0) + } +} + +impl Display for EndpointIdError { + fn fmt(&self, formatter: &mut Formatter<'_>) -> fmt::Result { + match self { + Self::InvalidLength(length) => write!( + formatter, + "endpoint_id must be exactly {} characters, got {length}", + EndpointId::LENGTH + ), + Self::InvalidCharacter(character) => write!( + formatter, + "endpoint_id must be lowercase hex, found: {character}" + ), + } + } +} + +impl std::error::Error for EndpointIdError {} + +/// Message accepted by an HTTP handler, queued for `poll()` to drain. +#[derive(Debug)] +pub struct QueuedMessage { + /// Raw HTTP request body bytes, exactly as received. + pub payload: Vec, + /// Already filtered, clamped, and converted by the handler, so draining + /// the queue cannot fail on a malformed header. + pub headers: Option>, + /// Accept time, kept for a queue-latency metric that does not exist yet. + /// Never serialized into the message. + pub received_at: Instant, +} + +impl From for ProducedMessage { + fn from(message: QueuedMessage) -> Self { + ProducedMessage { + // Webhook bodies carry no identifier this connector can trust as a + // dedupe key, and `timestamp` / `checksum` are Iggy's to fill. + id: None, + checksum: None, + timestamp: None, + origin_timestamp: None, + headers: message.headers, + payload: message.payload, + } + } +} + +/// Wall-clock seconds since the Unix epoch, the unit `expires_at` and +/// `revoked_at` are expressed in. Endpoint expiry is evaluated against the +/// clock of the request that hits it, so no background sweeper is needed. +pub fn unix_now_seconds() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|elapsed| elapsed.as_secs()) + .unwrap_or_default() +} + +/// Clamps a forwarded header value to the Iggy `HeaderValue` limit on a +/// UTF-8 character boundary. Returns `None` for empty values, which Iggy +/// rejects outright. +pub fn clamp_header_value(value: &str) -> Option<&str> { + if value.is_empty() { + return None; + } + if value.len() <= MAX_HEADER_VALUE_BYTES { + return Some(value); + } + let mut boundary = MAX_HEADER_VALUE_BYTES; + while !value.is_char_boundary(boundary) { + boundary -= 1; + } + Some(&value[..boundary]) +} + +#[cfg(test)] +mod tests { + use super::*; + + const VALID_ID: &str = "a3f8c2e1b9d04f7a8e6c1d2b3a4f5e6d"; + + #[test] + fn given_valid_lowercase_hex_when_parsed_should_accept() { + let endpoint_id: EndpointId = VALID_ID.parse().expect("valid id must parse"); + assert_eq!(endpoint_id.as_str(), VALID_ID); + } + + #[test] + fn given_wrong_length_when_parsed_should_reject() { + let result = "a3f8".parse::(); + assert_eq!(result, Err(EndpointIdError::InvalidLength(4))); + } + + #[test] + fn given_uppercase_hex_when_parsed_should_reject() { + let uppercase = VALID_ID.to_uppercase(); + assert_eq!( + uppercase.parse::(), + Err(EndpointIdError::InvalidCharacter('A')) + ); + } + + #[test] + fn given_non_hex_character_when_parsed_should_reject() { + let with_invalid = format!("g{}", &VALID_ID[1..]); + assert_eq!( + with_invalid.parse::(), + Err(EndpointIdError::InvalidCharacter('g')) + ); + } + + #[test] + fn given_short_value_when_clamped_should_pass_through() { + assert_eq!(clamp_header_value("api-client/1.0"), Some("api-client/1.0")); + } + + #[test] + fn given_empty_value_when_clamped_should_drop() { + assert_eq!(clamp_header_value(""), None); + } + + #[test] + fn given_oversized_value_when_clamped_should_truncate_to_limit() { + let oversized = "a".repeat(MAX_HEADER_VALUE_BYTES + 100); + let clamped = clamp_header_value(&oversized).expect("non-empty stays present"); + assert_eq!(clamped.len(), MAX_HEADER_VALUE_BYTES); + } + + #[test] + fn given_multibyte_value_when_clamped_should_respect_char_boundary() { + // 128 two-byte characters = 256 bytes; the clamp must land on the + // 254-byte boundary, not split a character at 255. + let multibyte = "é".repeat(128); + let clamped = clamp_header_value(&multibyte).expect("non-empty stays present"); + assert_eq!(clamped.len(), 254); + assert!(clamped.chars().all(|character| character == 'é')); + } +} diff --git a/core/integration/Cargo.toml b/core/integration/Cargo.toml index d4d0bdbd53..a7cabd981f 100644 --- a/core/integration/Cargo.toml +++ b/core/integration/Cargo.toml @@ -55,6 +55,8 @@ dtor = { workspace = true } figment = { workspace = true } futures = { workspace = true } harness_derive = { workspace = true } +# Sign GitHub-style webhook payloads in the http_source integration tests. +hex = { workspace = true } humantime = { workspace = true } iggy = { workspace = true } iggy-cli = { workspace = true } @@ -78,6 +80,7 @@ rcgen = { workspace = true } reqwest = { workspace = true, features = ["http2", "json"] } reqwest-middleware = { workspace = true } reqwest-retry = { workspace = true } +ring = { workspace = true } rmcp = { workspace = true, features = [ "client", "reqwest", diff --git a/core/integration/tests/connectors/fixtures/http/mod.rs b/core/integration/tests/connectors/fixtures/http/mod.rs index 761942e895..cc41bc3d5b 100644 --- a/core/integration/tests/connectors/fixtures/http/mod.rs +++ b/core/integration/tests/connectors/fixtures/http/mod.rs @@ -17,8 +17,13 @@ mod container; mod sink; +mod source; pub use sink::{ HttpSinkIndividualFixture, HttpSinkJsonArrayFixture, HttpSinkMultiTopicFixture, HttpSinkNdjsonFixture, HttpSinkNoMetadataFixture, HttpSinkRawFixture, }; +pub use source::{ + GITHUB_ENDPOINT_ID, GITHUB_HMAC_HEADER, GITHUB_INSTANCE, HttpSourceFixture, MANAGEMENT_TOKEN, + PARTNER_BEARER_TOKEN, PARTNER_ENDPOINT_ID, PARTNER_INSTANCE, +}; diff --git a/core/integration/tests/connectors/fixtures/http/source.rs b/core/integration/tests/connectors/fixtures/http/source.rs new file mode 100644 index 0000000000..34fb2cb1ad --- /dev/null +++ b/core/integration/tests/connectors/fixtures/http/source.rs @@ -0,0 +1,110 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use async_trait::async_trait; +use integration::harness::{TestBinaryError, TestFixture}; +use ring::hmac; +use std::collections::HashMap; +use std::net::TcpListener; + +/// Both instances in `tests/connectors/http/source_config` share one listener, +/// so they must be handed the same pair of addresses. +const ENV_GITHUB_LISTEN_ADDR: &str = "IGGY_CONNECTORS_SOURCE_HTTP_GITHUB_PLUGIN_CONFIG_LISTEN_ADDR"; +const ENV_GITHUB_ADMIN_LISTEN_ADDR: &str = + "IGGY_CONNECTORS_SOURCE_HTTP_GITHUB_PLUGIN_CONFIG_ADMIN_LISTEN_ADDR"; +const ENV_PARTNER_LISTEN_ADDR: &str = + "IGGY_CONNECTORS_SOURCE_HTTP_PARTNER_PLUGIN_CONFIG_LISTEN_ADDR"; +const ENV_PARTNER_ADMIN_LISTEN_ADDR: &str = + "IGGY_CONNECTORS_SOURCE_HTTP_PARTNER_PLUGIN_CONFIG_ADMIN_LISTEN_ADDR"; + +/// Static endpoints declared in those configurations. +pub const GITHUB_ENDPOINT_ID: &str = "a3f8c2e1b9d04f7a8e6c1d2b3a4f5e6d"; +pub const GITHUB_HMAC_SECRET: &str = "whsec_github_test"; +pub const GITHUB_HMAC_HEADER: &str = "X-Hub-Signature-256"; +pub const PARTNER_ENDPOINT_ID: &str = "0b7d9e2f4a6c8e1d3b5f7a9c2e4d6f81"; +pub const PARTNER_BEARER_TOKEN: &str = "partner-token-test"; +pub const MANAGEMENT_TOKEN: &str = "mgmt-token-test"; +pub const GITHUB_INSTANCE: &str = "http_github"; +pub const PARTNER_INSTANCE: &str = "http_partner"; + +/// Boots the webhook gateway on ports nobody else in this test run holds. +/// +/// Unlike the container-backed fixtures there is no external system here: the +/// connector is itself the HTTP server, and the test's own client is the +/// webhook sender. +pub struct HttpSourceFixture { + public_addr: String, + admin_addr: String, +} + +impl HttpSourceFixture { + pub fn public_url(&self) -> String { + format!("http://{}", self.public_addr) + } + + pub fn admin_url(&self) -> String { + format!("http://{}", self.admin_addr) + } + + pub fn webhook_url(&self, endpoint_id: &str) -> String { + format!("{}/e/{endpoint_id}", self.public_url()) + } + + /// GitHub-style `sha256=` over the exact bytes that will be sent. + pub fn github_signature(&self, body: &[u8]) -> String { + let key = hmac::Key::new(hmac::HMAC_SHA256, GITHUB_HMAC_SECRET.as_bytes()); + format!("sha256={}", hex::encode(hmac::sign(&key, body).as_ref())) + } + + /// Reserves an ephemeral port and releases it so the connector can bind it. + /// A fixed port would collide as soon as two of these tests run at once. + fn reserve_port() -> Result { + let listener = TcpListener::bind("127.0.0.1:0").map_err(TestBinaryError::Io)?; + let port = listener.local_addr().map_err(TestBinaryError::Io)?.port(); + Ok(port) + } +} + +#[async_trait] +impl TestFixture for HttpSourceFixture { + async fn setup() -> Result { + let public_addr = format!("127.0.0.1:{}", Self::reserve_port()?); + let admin_addr = format!("127.0.0.1:{}", Self::reserve_port()?); + Ok(Self { + public_addr, + admin_addr, + }) + } + + fn connectors_runtime_envs(&self) -> HashMap { + HashMap::from([ + (ENV_GITHUB_LISTEN_ADDR.to_string(), self.public_addr.clone()), + ( + ENV_GITHUB_ADMIN_LISTEN_ADDR.to_string(), + self.admin_addr.clone(), + ), + ( + ENV_PARTNER_LISTEN_ADDR.to_string(), + self.public_addr.clone(), + ), + ( + ENV_PARTNER_ADMIN_LISTEN_ADDR.to_string(), + self.admin_addr.clone(), + ), + ]) + } +} diff --git a/core/integration/tests/connectors/fixtures/mod.rs b/core/integration/tests/connectors/fixtures/mod.rs index e4992d6785..a005a69402 100644 --- a/core/integration/tests/connectors/fixtures/mod.rs +++ b/core/integration/tests/connectors/fixtures/mod.rs @@ -58,8 +58,10 @@ pub use doris::{ }; pub use elasticsearch::{ElasticsearchSinkFixture, ElasticsearchSourcePreCreatedFixture}; pub use http::{ - HttpSinkIndividualFixture, HttpSinkJsonArrayFixture, HttpSinkMultiTopicFixture, - HttpSinkNdjsonFixture, HttpSinkNoMetadataFixture, HttpSinkRawFixture, + GITHUB_ENDPOINT_ID, GITHUB_HMAC_HEADER, GITHUB_INSTANCE, HttpSinkIndividualFixture, + HttpSinkJsonArrayFixture, HttpSinkMultiTopicFixture, HttpSinkNdjsonFixture, + HttpSinkNoMetadataFixture, HttpSinkRawFixture, HttpSourceFixture, MANAGEMENT_TOKEN, + PARTNER_BEARER_TOKEN, PARTNER_ENDPOINT_ID, PARTNER_INSTANCE, }; pub use iceberg::{ DEFAULT_NAMESPACE, DEFAULT_TABLE, IcebergEnvAuthFixture, IcebergOps, IcebergPreCreatedFixture, diff --git a/core/integration/tests/connectors/http/http_source.rs b/core/integration/tests/connectors/http/http_source.rs new file mode 100644 index 0000000000..c7e67c32d6 --- /dev/null +++ b/core/integration/tests/connectors/http/http_source.rs @@ -0,0 +1,496 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::{POLL_ATTEMPTS, POLL_INTERVAL_MS}; +use crate::connectors::fixtures::{ + GITHUB_ENDPOINT_ID, GITHUB_HMAC_HEADER, GITHUB_INSTANCE, HttpSourceFixture, MANAGEMENT_TOKEN, + PARTNER_BEARER_TOKEN, PARTNER_ENDPOINT_ID, PARTNER_INSTANCE, +}; +use iggy::prelude::IggyClient; +use iggy_common::MessageClient; +use iggy_common::{Consumer, Identifier, PollingStrategy}; +use iggy_connector_sdk::api::{ConnectorStatus, SourceInfoResponse}; +use integration::harness::seeds; +use integration::iggy_harness; +use reqwest::{Client, StatusCode}; +use serde_json::{Value, json}; +use std::time::Duration; +use tokio::time::sleep; + +const API_KEY: &str = "test-api-key"; +const RESTORED_TOKEN: &str = "token-that-must-survive-a-restart"; + +#[iggy_harness( + server(connectors_runtime(config_path = "tests/connectors/http/source.toml")), + seed = seeds::connector_multi_topic_stream +)] +async fn webhook_post_produces_message_to_iggy(harness: &TestHarness, fixture: HttpSourceFixture) { + let client = harness.root_client().await.unwrap(); + let http = webhook_client(); + wait_for_gateway(&http, &fixture).await; + + let body = r#"{"event":"push","repository":"apache/iggy"}"#; + let response = http + .post(fixture.webhook_url(GITHUB_ENDPOINT_ID)) + .header( + GITHUB_HMAC_HEADER, + fixture.github_signature(body.as_bytes()), + ) + .header("X-GitHub-Delivery", "72d3162e-cc78-11e3-81ab-4c9367dc0958") + .body(body) + .send() + .await + .expect("Failed to POST the webhook"); + + assert_eq!(response.status(), StatusCode::OK); + + let messages = poll_payloads(&client, seeds::names::TOPIC, "http_source_cg_1", 1).await; + assert_eq!( + String::from_utf8_lossy(&messages[0].0), + body, + "the raw request body must reach Iggy byte for byte" + ); + let headers = messages[0] + .1 + .as_ref() + .expect("HTTP metadata forwarding is enabled"); + assert_eq!( + header_value(headers, "iggy_source_instance").as_deref(), + Some(GITHUB_INSTANCE) + ); + assert_eq!( + header_value(headers, "X-GitHub-Delivery").as_deref(), + Some("72d3162e-cc78-11e3-81ab-4c9367dc0958"), + "the delivery id is the consumer's dedup key and must survive the FFI hop" + ); +} + +#[iggy_harness( + server(connectors_runtime(config_path = "tests/connectors/http/source.toml")), + seed = seeds::connector_multi_topic_stream +)] +async fn two_instances_share_one_listener(harness: &TestHarness, fixture: HttpSourceFixture) { + let client = harness.root_client().await.unwrap(); + let http = webhook_client(); + wait_for_gateway(&http, &fixture).await; + + let github_body = r#"{"from":"github"}"#; + let github = http + .post(fixture.webhook_url(GITHUB_ENDPOINT_ID)) + .header( + GITHUB_HMAC_HEADER, + fixture.github_signature(github_body.as_bytes()), + ) + .body(github_body) + .send() + .await + .expect("Failed to POST to the GitHub endpoint"); + let partner_body = r#"{"from":"partner"}"#; + let partner = http + .post(fixture.webhook_url(PARTNER_ENDPOINT_ID)) + .header("Authorization", format!("Bearer {PARTNER_BEARER_TOKEN}")) + .body(partner_body) + .send() + .await + .expect("Failed to POST to the partner endpoint"); + + assert_eq!(github.status(), StatusCode::OK); + assert_eq!( + partner.status(), + StatusCode::OK, + "the second instance must be reachable on the port the first one bound" + ); + + let first = poll_payloads(&client, seeds::names::TOPIC, "http_source_cg_2a", 1).await; + let second = poll_payloads(&client, seeds::names::TOPIC_2, "http_source_cg_2b", 1).await; + assert_eq!(String::from_utf8_lossy(&first[0].0), github_body); + assert_eq!( + String::from_utf8_lossy(&second[0].0), + partner_body, + "one listener, but each instance produces to its own topic" + ); + assert_eq!( + header_value( + second[0].1.as_ref().expect("metadata is on by default"), + "iggy_source_instance" + ) + .as_deref(), + Some(PARTNER_INSTANCE) + ); +} + +#[iggy_harness( + server(connectors_runtime(config_path = "tests/connectors/http/source.toml")), + seed = seeds::connector_multi_topic_stream +)] +async fn management_registered_endpoint_accepts_until_revoked( + harness: &TestHarness, + fixture: HttpSourceFixture, +) { + let client = harness.root_client().await.unwrap(); + let http = webhook_client(); + wait_for_gateway(&http, &fixture).await; + + let endpoint_id = register_endpoint(&http, &fixture, GITHUB_INSTANCE).await; + let body = r#"{"event":"dynamic"}"#; + let accepted = http + .post(fixture.webhook_url(&endpoint_id)) + .body(body) + .send() + .await + .expect("Failed to POST to the registered endpoint"); + assert_eq!( + accepted.status(), + StatusCode::OK, + "an endpoint registered through the API must serve without a restart" + ); + + let messages = poll_payloads(&client, seeds::names::TOPIC, "http_source_cg_3", 1).await; + assert_eq!(String::from_utf8_lossy(&messages[0].0), body); + + let revoked = http + .delete(format!( + "{}/admin/endpoints/{endpoint_id}", + fixture.admin_url() + )) + .header("Authorization", format!("Bearer {MANAGEMENT_TOKEN}")) + .json(&json!({"reason": "compromised"})) + .send() + .await + .expect("Failed to revoke the endpoint"); + assert_eq!(revoked.status(), StatusCode::NO_CONTENT); + + let after = http + .post(fixture.webhook_url(&endpoint_id)) + .body(body) + .send() + .await + .expect("Failed to POST to the revoked endpoint"); + assert_eq!( + after.status(), + StatusCode::NOT_FOUND, + "a revocation must take effect immediately, and must not leak that the URL once worked" + ); +} + +#[iggy_harness( + server(connectors_runtime(config_path = "tests/connectors/http/source.toml")), + seed = seeds::connector_multi_topic_stream +)] +async fn dynamic_endpoint_survives_connector_restart( + harness: &TestHarness, + fixture: HttpSourceFixture, +) { + let http = webhook_client(); + let api_url = harness + .connectors_runtime() + .expect("connector runtime should be available") + .http_url(); + wait_for_gateway(&http, &fixture).await; + + // Registered with a secret so the restart proves the credential survives + // the state round trip, not merely the endpoint id. + let endpoint_id = register_secured_endpoint(&http, &fixture, GITHUB_INSTANCE).await; + // The registry only reaches the runtime once a poll carrying it has been + // sent, so wait for the connector to report it submitted before pulling + // the rug out from under it. + wait_for_submitted(&http, &fixture, &endpoint_id).await; + + let restarted = http + .post(format!("{api_url}/sources/{GITHUB_INSTANCE}/restart")) + .header("api-key", API_KEY) + .send() + .await + .expect("Failed to restart the source connector"); + assert!( + restarted.status().is_success(), + "Restart request failed: {}", + restarted.status() + ); + wait_for_source_status(&http, &api_url, ConnectorStatus::Running).await; + wait_for_gateway(&http, &fixture).await; + + let authorized = http + .post(fixture.webhook_url(&endpoint_id)) + .header("Authorization", format!("Bearer {RESTORED_TOKEN}")) + .body(r#"{"event":"after-restart"}"#) + .send() + .await + .expect("Failed to POST after the restart"); + let unauthorized = http + .post(fixture.webhook_url(&endpoint_id)) + .body(r#"{"event":"after-restart"}"#) + .send() + .await + .expect("Failed to POST after the restart"); + + assert_eq!( + authorized.status(), + StatusCode::OK, + "an endpoint that only ever existed in ConnectorState must come back with the connector" + ); + assert_eq!( + unauthorized.status(), + StatusCode::UNAUTHORIZED, + "and its secret must come back with it, not be dropped to an open endpoint" + ); +} + +/// The README's strongest security claim: a revoked endpoint must not be +/// resurrected by a restart that re-reads the TOML still declaring it. +#[iggy_harness( + server(connectors_runtime(config_path = "tests/connectors/http/source.toml")), + seed = seeds::connector_multi_topic_stream +)] +async fn revoked_static_endpoint_stays_revoked_across_restart( + harness: &TestHarness, + fixture: HttpSourceFixture, +) { + let http = webhook_client(); + let api_url = harness + .connectors_runtime() + .expect("connector runtime should be available") + .http_url(); + wait_for_gateway(&http, &fixture).await; + + let body = r#"{"event":"push"}"#; + let before = http + .post(fixture.webhook_url(GITHUB_ENDPOINT_ID)) + .header( + GITHUB_HMAC_HEADER, + fixture.github_signature(body.as_bytes()), + ) + .body(body) + .send() + .await + .expect("Failed to POST before revocation"); + assert_eq!(before.status(), StatusCode::OK); + + let revoked = http + .delete(format!( + "{}/admin/endpoints/{GITHUB_ENDPOINT_ID}", + fixture.admin_url() + )) + .header("Authorization", format!("Bearer {MANAGEMENT_TOKEN}")) + .json(&json!({"reason": "compromised"})) + .send() + .await + .expect("Failed to revoke the static endpoint"); + assert_eq!(revoked.status(), StatusCode::NO_CONTENT); + wait_for_submitted(&http, &fixture, GITHUB_ENDPOINT_ID).await; + + let restarted = http + .post(format!("{api_url}/sources/{GITHUB_INSTANCE}/restart")) + .header("api-key", API_KEY) + .send() + .await + .expect("Failed to restart the source connector"); + assert!(restarted.status().is_success()); + wait_for_source_status(&http, &api_url, ConnectorStatus::Running).await; + wait_for_gateway(&http, &fixture).await; + + let after = http + .post(fixture.webhook_url(GITHUB_ENDPOINT_ID)) + .header( + GITHUB_HMAC_HEADER, + fixture.github_signature(body.as_bytes()), + ) + .body(body) + .send() + .await + .expect("Failed to POST after the restart"); + assert_eq!( + after.status(), + StatusCode::NOT_FOUND, + "the tombstone must outlive the restart even though http_github.toml still declares this endpoint" + ); +} + +/// Keep-alive is off on purpose. The gateway shuts down gracefully, so an idle +/// pooled socket holds the listener open until its shutdown timeout expires, +/// stalling harness teardown for seconds per test. +fn webhook_client() -> Client { + Client::builder() + .pool_max_idle_per_host(0) + .build() + .expect("Failed to build the webhook client") +} + +/// The listener binds during the connector's `open()`, which happens after the +/// runtime's own HTTP API is already answering. +async fn wait_for_gateway(http: &Client, fixture: &HttpSourceFixture) { + for _ in 0..POLL_ATTEMPTS { + if let Ok(response) = http + .get(format!("{}/health", fixture.public_url())) + .send() + .await + && response.status() == StatusCode::OK + { + return; + } + sleep(Duration::from_millis(POLL_INTERVAL_MS)).await; + } + panic!("The webhook gateway did not become healthy in time"); +} + +/// Registers a bearer-guarded endpoint, so a test can prove the secret and not +/// just the id survives whatever it does next. +async fn register_secured_endpoint( + http: &Client, + fixture: &HttpSourceFixture, + instance: &str, +) -> String { + let response = http + .post(format!("{}/admin/endpoints", fixture.admin_url())) + .header("Authorization", format!("Bearer {MANAGEMENT_TOKEN}")) + .json(&json!({ + "instance": instance, + "auth_type": "bearer", + "auth_secret": RESTORED_TOKEN, + })) + .send() + .await + .expect("Failed to register a secured endpoint"); + assert_eq!(response.status(), StatusCode::CREATED); + let body: Value = response + .json() + .await + .expect("Registration must return JSON"); + body["endpoint_id"] + .as_str() + .expect("Registration must return an endpoint id") + .to_string() +} + +async fn register_endpoint(http: &Client, fixture: &HttpSourceFixture, instance: &str) -> String { + let response = http + .post(format!("{}/admin/endpoints", fixture.admin_url())) + .header("Authorization", format!("Bearer {MANAGEMENT_TOKEN}")) + .json(&json!({"instance": instance})) + .send() + .await + .expect("Failed to register an endpoint"); + assert_eq!(response.status(), StatusCode::CREATED); + let body: Value = response + .json() + .await + .expect("Registration must return JSON"); + body["endpoint_id"] + .as_str() + .expect("Registration must return an endpoint id") + .to_string() +} + +async fn wait_for_submitted(http: &Client, fixture: &HttpSourceFixture, endpoint_id: &str) { + for _ in 0..POLL_ATTEMPTS { + if let Ok(response) = http + .get(format!( + "{}/admin/endpoints/{endpoint_id}", + fixture.admin_url() + )) + .header("Authorization", format!("Bearer {MANAGEMENT_TOKEN}")) + .send() + .await + && let Ok(body) = response.json::().await + && body["submitted"] == json!(true) + { + return; + } + sleep(Duration::from_millis(POLL_INTERVAL_MS)).await; + } + panic!("The registered endpoint was never handed to the runtime for persistence"); +} + +/// Compares the typed status rather than a string: `ConnectorStatus` +/// serializes lowercase, so a `{:?}` comparison would silently never match. +async fn wait_for_source_status(http: &Client, api_url: &str, expected: ConnectorStatus) { + for _ in 0..POLL_ATTEMPTS { + if let Ok(response) = http + .get(format!("{api_url}/sources/{GITHUB_INSTANCE}")) + .header("api-key", API_KEY) + .send() + .await + && let Ok(info) = response.json::().await + && info.status == expected + { + return; + } + sleep(Duration::from_millis(POLL_INTERVAL_MS)).await; + } + panic!("The source connector did not reach {expected:?} status in time"); +} + +type PolledMessage = (Vec, Option>); + +async fn poll_payloads( + client: &IggyClient, + topic: &str, + consumer: &str, + expected: usize, +) -> Vec { + let stream_id: Identifier = seeds::names::STREAM.try_into().unwrap(); + let topic_id: Identifier = topic.try_into().unwrap(); + let consumer_id: Identifier = consumer.try_into().unwrap(); + + let mut collected: Vec = Vec::new(); + for _ in 0..POLL_ATTEMPTS { + if let Ok(polled) = client + .poll_messages( + &stream_id, + &topic_id, + None, + &Consumer::new(consumer_id.clone()), + &PollingStrategy::next(), + 10, + true, + ) + .await + { + for message in polled.messages { + // `Display` on a header field renders ": ", so + // the bare name has to come from `as_str`. + let headers = message.user_headers_map().ok().flatten().map(|headers| { + headers + .into_iter() + .map(|(key, value)| { + ( + key.as_str().unwrap_or_default().to_string(), + value.as_str().unwrap_or_default().to_string(), + ) + }) + .collect() + }); + collected.push((message.payload.to_vec(), headers)); + } + if collected.len() >= expected { + return collected; + } + } + sleep(Duration::from_millis(POLL_INTERVAL_MS)).await; + } + panic!( + "Expected {expected} messages on {topic}, got {}", + collected.len() + ); +} + +fn header_value(headers: &[(String, String)], key: &str) -> Option { + headers + .iter() + .find(|(name, _)| name == key) + .map(|(_, value)| value.clone()) +} diff --git a/core/integration/tests/connectors/http/mod.rs b/core/integration/tests/connectors/http/mod.rs index 2fa0f39996..74f2700260 100644 --- a/core/integration/tests/connectors/http/mod.rs +++ b/core/integration/tests/connectors/http/mod.rs @@ -16,5 +16,8 @@ // under the License. mod http_sink; +mod http_source; const TEST_MESSAGE_COUNT: usize = 3; +const POLL_ATTEMPTS: usize = 100; +const POLL_INTERVAL_MS: u64 = 50; diff --git a/core/integration/tests/connectors/http/source.toml b/core/integration/tests/connectors/http/source.toml new file mode 100644 index 0000000000..b6a77659d9 --- /dev/null +++ b/core/integration/tests/connectors/http/source.toml @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[connectors] +config_type = "local" +config_dir = "tests/connectors/http/source_config" diff --git a/core/integration/tests/connectors/http/source_config/http_github.toml b/core/integration/tests/connectors/http/source_config/http_github.toml new file mode 100644 index 0000000000..42ab154f6b --- /dev/null +++ b/core/integration/tests/connectors/http/source_config/http_github.toml @@ -0,0 +1,53 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Instance 1 of a shared listener. The fixture supplies listen_addr and +# admin_listen_addr through env overrides so parallel test processes do not +# fight over a port. + +type = "source" +key = "http_github" +enabled = true +version = 0 +name = "HTTP source (GitHub)" +path = "../../target/debug/libiggy_connector_http_source" +plugin_config_format = "toml" + +[[streams]] +stream = "test_stream" +topic = "test_topic" +schema = "raw" +batch_length = 10 +linger_time = "5ms" + +[plugin_config] +listen_addr = "127.0.0.1:0" +admin_listen_addr = "127.0.0.1:0" +instance_name = "http_github" +topic_path = "test_topic" +management_token = "mgmt-token-test" +buffer_capacity = 128 +max_batch_size = 10 +include_http_metadata = true +forward_headers = ["X-GitHub-Delivery"] + +[[plugin_config.endpoints]] +endpoint_id = "a3f8c2e1b9d04f7a8e6c1d2b3a4f5e6d" +auth_type = "hmac-sha256" +auth_secret = "whsec_github_test" +hmac_header = "X-Hub-Signature-256" +hmac_prefix = "sha256=" diff --git a/core/integration/tests/connectors/http/source_config/http_partner.toml b/core/integration/tests/connectors/http/source_config/http_partner.toml new file mode 100644 index 0000000000..26e0bd4454 --- /dev/null +++ b/core/integration/tests/connectors/http/source_config/http_partner.toml @@ -0,0 +1,47 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Instance 2, joining the listener instance 1 binds. Everything the two must +# agree on comes from the same env overrides. + +type = "source" +key = "http_partner" +enabled = true +version = 0 +name = "HTTP source (partner)" +path = "../../target/debug/libiggy_connector_http_source" +plugin_config_format = "toml" + +[[streams]] +stream = "test_stream" +topic = "test_topic_2" +schema = "raw" +batch_length = 10 +linger_time = "5ms" + +[plugin_config] +listen_addr = "127.0.0.1:0" +admin_listen_addr = "127.0.0.1:0" +instance_name = "http_partner" +management_token = "mgmt-token-test" +buffer_capacity = 128 +max_batch_size = 10 + +[[plugin_config.endpoints]] +endpoint_id = "0b7d9e2f4a6c8e1d3b5f7a9c2e4d6f81" +auth_type = "bearer" +auth_secret = "partner-token-test" From 6d3bcac61915356f024e1c5e2465248c5261155d Mon Sep 17 00:00:00 2001 From: mlevkov Date: Sun, 2 Aug 2026 14:22:51 -0700 Subject: [PATCH 2/2] test(connectors): cover http_source auth guards and fail-closed paths The 94.5% patch coverage on this branch hid the gap that mattered: both management auth tests reached only GET /admin/endpoints, so the guard on each of the four mutating routes had never taken its rejection branch. Removing `denied()` from `revoke_endpoint` left the suite green, which a per-route table-driven test now catches at 204-instead-of-401. The rest closes the branches a reviewer would want pinned rather than every uncovered line. Chief among them, `republish_or_close` had no test proving it answers 500 instead of reporting a revoke that never reached the route table, and the endpoint-id route conflict rendered its message without any test asserting the id stays an 8-char prefix. `ServerState::new` widens to `pub(crate)` so the management tests can build a listener-less state and provoke the republish failure. Every new test was mutation-checked: each one was confirmed to fail against a deliberate break of the behaviour it claims to pin. Left uncovered on purpose: the shutdown abort branch needs a connection wedged past the 5s timeout, and roughly 30 of the remaining missed lines are tracing-macro arguments that only execute with a subscriber installed. --- .../connectors/sources/http_source/src/lib.rs | 31 ++++ .../sources/http_source/src/management.rs | 62 +++++++ .../sources/http_source/src/metrics.rs | 20 +++ .../sources/http_source/src/routes.rs | 12 ++ .../sources/http_source/src/server.rs | 157 +++++++++++++++++- .../sources/http_source/src/state.rs | 35 ++++ .../sources/http_source/src/types.rs | 15 ++ 7 files changed, 331 insertions(+), 1 deletion(-) diff --git a/core/connectors/sources/http_source/src/lib.rs b/core/connectors/sources/http_source/src/lib.rs index f8821a7e0b..daabd45398 100644 --- a/core/connectors/sources/http_source/src/lib.rs +++ b/core/connectors/sources/http_source/src/lib.rs @@ -797,6 +797,37 @@ mod tests { } } + #[test] + fn given_oversized_forward_header_when_validated_should_reject() { + // Valid as an HTTP name, too long to become a HeaderKey. Rejected here + // rather than dropped later, so the operator learns the header they + // asked to forward would never have ridden along. + let long_header = "x".repeat(256); + let config = parse(&format!( + r#"{{"listen_addr": "0.0.0.0:9090", "forward_headers": ["{long_header}"]}}"# + )); + assert!(matches!( + config.validate(), + Err(Error::InvalidConfigValue(message)) if message.contains("Iggy header key") + )); + } + + #[test] + fn given_auth_types_when_asked_for_an_algorithm_should_map_each_exactly_once() { + // A swapped arm would validate SHA-1 signatures with SHA-256 and + // reject every request the sender signs correctly. + assert_eq!( + EndpointAuthType::HmacSha256.hmac_algorithm(), + Some(HmacAlgorithm::HmacSha256) + ); + assert_eq!( + EndpointAuthType::HmacSha1.hmac_algorithm(), + Some(HmacAlgorithm::HmacSha1) + ); + assert_eq!(EndpointAuthType::Bearer.hmac_algorithm(), None); + assert_eq!(EndpointAuthType::None.hmac_algorithm(), None); + } + #[test] fn given_invalid_endpoint_id_when_deserialized_should_reject() { let result = serde_json::from_str::( diff --git a/core/connectors/sources/http_source/src/management.rs b/core/connectors/sources/http_source/src/management.rs index 2596dc9710..f11303710f 100644 --- a/core/connectors/sources/http_source/src/management.rs +++ b/core/connectors/sources/http_source/src/management.rs @@ -596,6 +596,68 @@ mod tests { fixture.close().await; } + #[tokio::test] + async fn given_wrong_token_when_each_route_called_should_answer_unauthorized() { + // Every route, not just the one route a single test happens to reach. + // Each handler carries its own guard, so a check dropped from any of + // the four mutating ones would still leave the suite green. + let fixture = Fixture::start(Some(TOKEN)).await; + let endpoints = format!("{}/admin/endpoints", fixture.admin); + let one = format!("{endpoints}/{ENDPOINT_ONE}"); + let http = client(); + let cases = vec![ + ( + "POST /admin/endpoints", + http.post(&endpoints) + .json(&json!({"instance": "http_github"})), + ), + ("GET /admin/endpoints", http.get(&endpoints)), + ("GET /admin/endpoints/{endpoint_id}", http.get(&one)), + ( + "PATCH /admin/endpoints/{endpoint_id}", + http.patch(&one) + .json(&json!({"auth_secret": "whsec_rotated"})), + ), + ("DELETE /admin/endpoints/{endpoint_id}", http.delete(&one)), + ]; + + for (route, request) in cases { + let response = request + .header(header::AUTHORIZATION, "Bearer not-the-token") + .send() + .await + .expect("the request must reach the admin listener"); + + assert_eq!( + response.status(), + StatusCode::UNAUTHORIZED, + "{route} must refuse a wrong token before it does anything else" + ); + } + fixture.close().await; + } + + #[tokio::test] + async fn given_no_bound_listener_when_republished_should_fail_closed() { + // A mutation that cannot reproject the route table must answer 500 + // rather than report success. For a revoke or a rotate the old table is + // still honouring the credential the operator believes is now dead, so + // the take-access-away variant drops the routes on its way out. + let mut config = crate::test_support::config(Some("github"), &[ENDPOINT_ONE]); + config.listen_addr = format!("127.0.0.1:{}", free_port()); + let state = Arc::new(ServerState::new(&config)); + + let failure = republish(&state) + .await + .expect("an unbound address cannot be reprojected"); + let fail_closed = republish_or_close(&state) + .await + .expect("and the take-access-away variant must report it too"); + + assert_eq!(failure.status(), StatusCode::INTERNAL_SERVER_ERROR); + assert_eq!(fail_closed.status(), StatusCode::INTERNAL_SERVER_ERROR); + } + #[tokio::test] async fn given_valid_request_when_endpoint_registered_should_generate_a_secret_path() { let fixture = Fixture::start(Some(TOKEN)).await; diff --git a/core/connectors/sources/http_source/src/metrics.rs b/core/connectors/sources/http_source/src/metrics.rs index b523bdbb91..d06fd012d0 100644 --- a/core/connectors/sources/http_source/src/metrics.rs +++ b/core/connectors/sources/http_source/src/metrics.rs @@ -381,6 +381,12 @@ mod tests { 401, Duration::from_micros(40), ); + metrics.record_request( + "http_github", + PathKind::Named, + 500, + Duration::from_micros(20), + ); metrics.record_rejected_full("http_github"); let encoded = metrics.encode(&[]); @@ -390,6 +396,13 @@ mod tests { assert!(encoded.contains( "http_source_requests_total{instance=\"http_github\",kind=\"named\",status=\"4xx\"} 1" )); + assert!( + encoded.contains( + "http_source_requests_total{instance=\"http_github\",kind=\"named\",status=\"5xx\"} 1" + ), + "every status class needs its label pinned: a wrong one silently \ + mislabels an alert rather than failing anything" + ); assert!( encoded.contains("http_source_rejected_full_total{instance=\"http_github\"} 1"), "a 429 needs its own series: it is backpressure, not a caller error" @@ -397,6 +410,13 @@ mod tests { assert!(encoded.contains("http_source_request_duration_seconds_bucket")); } + /// `Default` exists to satisfy `clippy::new_without_default`, so the only + /// thing it owes is agreeing with the constructor it stands in for. + #[test] + fn given_default_registry_when_encoded_should_match_a_new_one() { + assert_eq!(Metrics::default().encode(&[]), Metrics::new().encode(&[])); + } + #[test] fn given_header_losses_when_recorded_should_count_clamps_and_drops_apart() { let metrics = Metrics::new(); diff --git a/core/connectors/sources/http_source/src/routes.rs b/core/connectors/sources/http_source/src/routes.rs index fcdd52a047..119b0a2d70 100644 --- a/core/connectors/sources/http_source/src/routes.rs +++ b/core/connectors/sources/http_source/src/routes.rs @@ -336,6 +336,18 @@ mod tests { claimed_by: 7, } ); + // The rendered form reaches the operator's log through + // `Error::InvalidConfigValue`, and the id is the credential for a + // secret-path endpoint. + let message = conflict.to_string(); + assert!( + message.contains(&ENDPOINT_ONE[..8]), + "the message must identify which endpoint collided: {message}" + ); + assert!( + !message.contains(ENDPOINT_ONE), + "but never in full: {message}" + ); } #[test] diff --git a/core/connectors/sources/http_source/src/server.rs b/core/connectors/sources/http_source/src/server.rs index a4c2bb9a0a..6efca698ae 100644 --- a/core/connectors/sources/http_source/src/server.rs +++ b/core/connectors/sources/http_source/src/server.rs @@ -212,7 +212,7 @@ pub(crate) struct ServerState { } impl ServerState { - fn new(config: &HttpSourceConfig) -> Self { + pub(crate) fn new(config: &HttpSourceConfig) -> Self { ServerState { listen_addr: config.listen_addr.clone(), management_token: config.management_token.clone(), @@ -1498,6 +1498,161 @@ mod tests { ) } + #[test] + fn given_populated_routes_when_serving_nothing_should_drop_every_one() { + let state = ServerState::new(&config(free_port(), free_port(), &[ENDPOINT_ONE])); + state + .publish(vec![crate::test_support::instance( + 1, + Some("github"), + &[ENDPOINT_ONE], + )]) + .expect("a single instance cannot collide with itself"); + assert_eq!(state.routes.load().secret_path_count(), 1); + assert_eq!(state.routes.load().named_path_count(), 1); + + state.serve_nothing(); + + assert_eq!(state.routes.load().secret_path_count(), 0); + assert_eq!(state.routes.load().named_path_count(), 0); + assert_eq!( + state.instances().len(), + 1, + "the instances stay joined; only the routes projecting them stop being served" + ); + } + + #[tokio::test] + async fn given_no_bound_listener_when_routes_refreshed_should_name_the_address() { + let unbound = format!("127.0.0.1:{}", free_port()); + + let error = refresh_routes(&unbound) + .await + .expect_err("an address nothing is bound to cannot be reprojected"); + + assert!(matches!( + error, + Error::InitError(message) if message.contains(&unbound) + )); + } + + #[tokio::test] + async fn given_duplicate_instance_name_when_joined_should_reject() { + let public_port = free_port(); + let admin_port = free_port(); + let mut first_config = config(public_port, admin_port, &[ENDPOINT_ONE]); + first_config.instance_name = Some("http_github".to_string()); + let mut first = open(1, first_config).await; + + let mut second_config = config(public_port, admin_port, &[ENDPOINT_TWO]); + second_config.instance_name = Some("http_github".to_string()); + second_config.topic_path = Some("stripe".to_string()); + let mut second = HttpSource::new(2, second_config, None); + let error = second + .open() + .await + .expect_err("a duplicate name would address two instances at once"); + + assert!(matches!( + error, + Error::InvalidConfigValue(message) if message.contains("instance_name") + )); + assert_eq!( + post_signed(&base_url(&first), ENDPOINT_TWO, "{}") + .await + .status(), + StatusCode::NOT_FOUND, + "a rejected join must leave no routes behind" + ); + close(&mut first).await; + } + + #[tokio::test] + async fn given_unknown_named_path_when_posted_should_answer_not_found_as_unrouted() { + let mut config = config(free_port(), free_port(), &[]); + config.instance_name = Some("http_github".to_string()); + let admin = format!("http://{}", config.admin_listen_addr); + let mut source = open(1, config).await; + + let response = client() + .post(format!("{}/topics/unclaimed", base_url(&source))) + .body("{}") + .send() + .await + .expect("the request must reach the listener"); + + assert_eq!(response.status(), StatusCode::NOT_FOUND); + let scraped = client() + .get(format!("{admin}/admin/metrics")) + .send() + .await + .expect("the request must reach the admin listener") + .text() + .await + .expect("the scrape must have a body"); + assert!( + scraped.contains( + "http_source_requests_total{instance=\"unrouted\",kind=\"named\",status=\"4xx\"} 1" + ), + "a misconfigured sender posting to the wrong path is the thing an \ + operator needs to see, so it cannot go uncounted: {scraped}" + ); + close(&mut source).await; + } + + #[tokio::test] + async fn given_oversized_body_when_posted_to_a_named_path_should_answer_payload_too_large() { + let mut config = config(free_port(), free_port(), &[]); + config.max_body_size_bytes = 16; + let mut source = open(1, config).await; + + let response = client() + .post(format!("{}/topics/github", base_url(&source))) + .body("x".repeat(1024)) + .send() + .await + .expect("the request must reach the listener"); + + assert_eq!(response.status(), StatusCode::PAYLOAD_TOO_LARGE); + assert_eq!(source.shared.sender.len(), 0); + close(&mut source).await; + } + + #[test] + fn given_unrepresentable_forwarded_values_when_headers_built_should_drop_and_count_them() { + let mut config = config(free_port(), free_port(), &[]); + config.include_http_metadata = false; + config.forward_headers = vec!["x-binary".to_string(), "x-blank".to_string()]; + let source = HttpSource::new(1, config, None); + + let mut request_headers = HeaderMap::new(); + request_headers.insert( + axum::http::HeaderName::from_static("x-binary"), + axum::http::HeaderValue::from_bytes(&[0xff]) + .expect("an opaque byte is a legal HTTP header value"), + ); + request_headers.insert( + axum::http::HeaderName::from_static("x-blank"), + axum::http::HeaderValue::from_static(""), + ); + + let (headers, clamped, dropped) = message_headers( + &source.shared, + &request_headers, + "127.0.0.1:4444".parse().expect("a literal address parses"), + ); + + // Both are present but unrepresentable: one is not visible ASCII, the + // other clamps away to nothing. An absent header is not a loss, so a + // count above two would be silent over-reporting. + assert_eq!(dropped, 2); + assert_eq!(clamped, 0); + assert!( + headers.is_none(), + "with metadata off and both forwarded values dropped there is nothing to attach" + ); + } + async fn close(source: &mut HttpSource) { source.close().await.expect("close must succeed"); } diff --git a/core/connectors/sources/http_source/src/state.rs b/core/connectors/sources/http_source/src/state.rs index 73a04f4b5c..ceba3644f7 100644 --- a/core/connectors/sources/http_source/src/state.rs +++ b/core/connectors/sources/http_source/src/state.rs @@ -459,6 +459,41 @@ mod tests { ); } + #[test] + fn given_revoked_dynamic_endpoint_when_restored_should_keep_the_tombstone() { + // No static counterpart, which is the ordinary case: revoke a + // dynamically registered endpoint, restart, and the leaked URL must + // still be dead. + let mut persisted = EndpointRegistry::default(); + assert!(persisted.insert(dynamic_endpoint(ENDPOINT_TWO))); + persisted.revoke(ENDPOINT_TWO, "compromised".to_string(), 42); + + let restored = EndpointRegistry::restore(&[], Some(registry_state(&persisted)), 1); + + let endpoint = restored + .endpoint(ENDPOINT_TWO) + .expect("the tombstone must survive the restart, not vanish with it"); + assert!(!endpoint.is_active()); + assert_eq!(restored.serving_count(0), 0); + } + + #[test] + fn given_registered_endpoint_when_removed_should_drop_it_without_a_tombstone() { + let mut registry = EndpointRegistry::default(); + assert!(registry.insert(dynamic_endpoint(ENDPOINT_ONE))); + + assert!(registry.remove(ENDPOINT_ONE)); + assert!( + !registry.remove(ENDPOINT_ONE), + "removing twice must report that there was nothing left to undo" + ); + assert!(!registry.remove(ENDPOINT_TWO)); + // Outright, not tombstoned: this only undoes a registration that never + // became reachable, so there is no revocation to preserve. + assert!(registry.endpoint(ENDPOINT_ONE).is_none()); + assert!(registry.is_empty()); + } + #[test] fn given_active_endpoint_in_both_when_restored_should_prefer_static_config() { let mut persisted = EndpointRegistry::default(); diff --git a/core/connectors/sources/http_source/src/types.rs b/core/connectors/sources/http_source/src/types.rs index ad7fe3b950..7f85c39a8d 100644 --- a/core/connectors/sources/http_source/src/types.rs +++ b/core/connectors/sources/http_source/src/types.rs @@ -218,6 +218,21 @@ mod tests { ); } + #[test] + fn given_rejected_id_when_displayed_should_name_the_reason() { + assert_eq!( + EndpointIdError::InvalidLength(4).to_string(), + format!( + "endpoint_id must be exactly {} characters, got 4", + EndpointId::LENGTH + ) + ); + assert_eq!( + EndpointIdError::InvalidCharacter('g').to_string(), + "endpoint_id must be lowercase hex, found: g" + ); + } + #[test] fn given_short_value_when_clamped_should_pass_through() { assert_eq!(clamp_header_value("api-client/1.0"), Some("api-client/1.0"));