From dbca8120bca441b7b8f29b0de7e44c520e936e43 Mon Sep 17 00:00:00 2001 From: Zeying Zhu <50204836+zzylol@users.noreply.github.com> Date: Mon, 12 Oct 2026 17:53:02 +0000 Subject: [PATCH 1/6] Add asap-collector: the official OTAP engine with ASAP nodes linked in asap-collector --config pipeline.yaml runs the upstream controller (startup, admin /metrics and /readyz, signal handling) with the asap_sketches processor and the scenario generator registered, so every pipeline stage can run as a real collector process in its own container. --- asap-precompute-rs/Cargo.lock | 531 ++++++++++++++++++- asap-precompute-rs/Cargo.toml | 14 + asap-precompute-rs/src/bin/asap_collector.rs | 37 ++ 3 files changed, 563 insertions(+), 19 deletions(-) create mode 100644 asap-precompute-rs/src/bin/asap_collector.rs diff --git a/asap-precompute-rs/Cargo.lock b/asap-precompute-rs/Cargo.lock index 6d1167a..f9a50b4 100644 --- a/asap-precompute-rs/Cargo.lock +++ b/asap-precompute-rs/Cargo.lock @@ -63,6 +63,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "aligned-vec" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc890384c8602f339876ded803c97ad529f3842aba97f6392b3dba0dd171769b" +dependencies = [ + "equator", +] + [[package]] name = "allocator-api2" version = "0.2.21" @@ -350,6 +359,7 @@ dependencies = [ "libc", "linkme", "otel-arrow-dfe-config", + "otel-arrow-dfe-controller", "otel-arrow-dfe-core-nodes", "otel-arrow-dfe-engine", "otel-arrow-dfe-otap", @@ -488,11 +498,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" dependencies = [ "axum-core", + "base64 0.22.1", "bytes", + "form_urlencoded", "futures-util", "http", "http-body", "http-body-util", + "hyper", + "hyper-util", "itoa", "matchit", "memchr", @@ -500,10 +514,17 @@ dependencies = [ "percent-encoding", "pin-project-lite", "serde_core", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sha1", "sync_wrapper", + "tokio", + "tokio-tungstenite", "tower", "tower-layer", "tower-service", + "tracing", ] [[package]] @@ -522,6 +543,7 @@ dependencies = [ "sync_wrapper", "tower-layer", "tower-service", + "tracing", ] [[package]] @@ -1016,6 +1038,26 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "core_affinity" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a034b3a7b624016c6e13f5df875747cc25f884156aad2abd12b6c46797971342" +dependencies = [ + "libc", + "num_cpus", + "winapi", +] + +[[package]] +name = "cpp_demangle" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2bb79cb74d735044c972aae58ed0aaa9a837e85b01106a54c39e42e97f62253" +dependencies = [ + "cfg-if", +] + [[package]] name = "cpu-time" version = "1.0.0" @@ -1683,7 +1725,7 @@ dependencies = [ "itertools 0.14.0", "parking_lot", "paste", - "petgraph", + "petgraph 0.8.3", "tokio", ] @@ -1818,6 +1860,15 @@ dependencies = [ "sqlparser", ] +[[package]] +name = "debugid" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef552e6f588e446098f6ba40d89ac146c8c7b64aade83c051ee00bb5d2bc18d" +dependencies = [ + "uuid", +] + [[package]] name = "deflate64" version = "0.1.12" @@ -1927,7 +1978,17 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.60.2", + "windows-sys 0.61.2", +] + +[[package]] +name = "dispatch2" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" +dependencies = [ + "bitflags 2.13.0", + "objc2", ] [[package]] @@ -1977,6 +2038,26 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "equator" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4711b213838dfee0117e3be6ac926007d7f433d7bbe33595975d4190cb07e6fc" +dependencies = [ + "equator-macro", +] + +[[package]] +name = "equator-macro" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44f23cf4b44bfce11a86ace86f8a73ffdec849c9fd00a386a53d278bd9e81fb3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -2058,6 +2139,24 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "findshlibs" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40b9e59cd0f7e0806cca4be089683ecb6434e602038df21fe6bf6711b2f07f64" +dependencies = [ + "cc", + "lazy_static", + "libc", + "winapi", +] + +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + [[package]] name = "fixedbitset" version = "0.5.7" @@ -2104,7 +2203,7 @@ checksum = "5e139bc46ca777eb5efaf62df0ab8cc5fd400866427e56c68b22e414e53bd3be" dependencies = [ "futures-core", "futures-sink", - "spin", + "spin 0.9.9", ] [[package]] @@ -3246,6 +3345,12 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hermit-abi" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17592d60ebacc7d5e169f4663c5f84f9161cc90328abcfe8456f41e4dfcb284" + [[package]] name = "hex" version = "0.4.3" @@ -3659,9 +3764,9 @@ dependencies = [ [[package]] name = "itertools" -version = "0.13.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" dependencies = [ "either", ] @@ -4213,6 +4318,16 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "mime_guess" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" +dependencies = [ + "mime", + "unicase", +] + [[package]] name = "minijinja" version = "2.24.0" @@ -4281,6 +4396,17 @@ dependencies = [ "tempfile", ] +[[package]] +name = "nix" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b" +dependencies = [ + "bitflags 1.3.2", + "cfg-if", + "libc", +] + [[package]] name = "nix" version = "0.31.3" @@ -4335,6 +4461,15 @@ dependencies = [ "bitflags 2.13.0", ] +[[package]] +name = "ntapi" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3b335231dfd352ffb0f8017f3b6027a4917f7df785ea2143d8af2adc66980ae" +dependencies = [ + "winapi", +] + [[package]] name = "num" version = "0.4.3" @@ -4420,6 +4555,16 @@ dependencies = [ "libm", ] +[[package]] +name = "num_cpus" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +dependencies = [ + "hermit-abi", + "libc", +] + [[package]] name = "num_enum" version = "0.7.6" @@ -4442,6 +4587,63 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "objc2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08849bbd4767dfae9457696856ae1c84fe4e0281bbe4a7abff2d0e06fb7981f8" +dependencies = [ + "objc2-encode", +] + +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags 2.13.0", + "dispatch2", + "objc2", +] + +[[package]] +name = "objc2-encode" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" + +[[package]] +name = "objc2-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272" +dependencies = [ + "bitflags 2.13.0", + "objc2", +] + +[[package]] +name = "objc2-io-kit" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33fafba39597d6dc1fb709123dfa8289d39406734be322956a69f0931c73bb15" +dependencies = [ + "libc", + "objc2-core-foundation", +] + +[[package]] +name = "objc2-open-directory" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb82bed227edf5201dfedf072bba4015a33d3d4a98519837295a90f0a23f676d" +dependencies = [ + "objc2", + "objc2-core-foundation", + "objc2-foundation", +] + [[package]] name = "object" version = "0.37.3" @@ -4604,6 +4806,39 @@ dependencies = [ "num-traits", ] +[[package]] +name = "otel-arrow-dfe-admin" +version = "0.51.0" +source = "git+https://github.com/open-telemetry/otel-arrow?rev=3e85c3460361446ebfce99e9f35fffd2dd5ab740#3e85c3460361446ebfce99e9f35fffd2dd5ab740" +dependencies = [ + "axum", + "chrono", + "include_dir", + "mime_guess", + "otel-arrow-dfe-admin-types", + "otel-arrow-dfe-config", + "otel-arrow-dfe-engine", + "otel-arrow-dfe-state", + "otel-arrow-dfe-telemetry", + "pprof", + "serde", + "serde_json", + "thiserror 2.0.20", + "tokio", + "tokio-util", + "tower", +] + +[[package]] +name = "otel-arrow-dfe-admin-types" +version = "0.51.0" +source = "git+https://github.com/open-telemetry/otel-arrow?rev=3e85c3460361446ebfce99e9f35fffd2dd5ab740#3e85c3460361446ebfce99e9f35fffd2dd5ab740" +dependencies = [ + "otel-arrow-dfe-config", + "serde", + "serde_json", +] + [[package]] name = "otel-arrow-dfe-channel" version = "0.51.0" @@ -4639,6 +4874,38 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "otel-arrow-dfe-controller" +version = "0.51.0" +source = "git+https://github.com/open-telemetry/otel-arrow?rev=3e85c3460361446ebfce99e9f35fffd2dd5ab740#3e85c3460361446ebfce99e9f35fffd2dd5ab740" +dependencies = [ + "bytes", + "chrono", + "core_affinity", + "futures-util", + "humantime-serde", + "linkme", + "miette", + "otel-arrow-dfe-admin", + "otel-arrow-dfe-config", + "otel-arrow-dfe-engine", + "otel-arrow-dfe-state", + "otel-arrow-dfe-telemetry", + "otel-arrow-dfe-telemetry-macros", + "prost 0.14.4", + "serde", + "serde_json", + "serde_yaml", + "smallvec", + "sysinfo", + "thiserror 2.0.20", + "tokio", + "tokio-tungstenite", + "tokio-util", + "url", + "uuid", +] + [[package]] name = "otel-arrow-dfe-core-nodes" version = "0.51.0" @@ -4663,7 +4930,7 @@ dependencies = [ "humantime-serde", "libc", "linkme", - "nix", + "nix 0.31.3", "object_store", "otel-arrow-dfe-channel", "otel-arrow-dfe-config", @@ -4679,7 +4946,7 @@ dependencies = [ "parking_lot", "parquet", "prost 0.14.4", - "prost-types", + "prost-types 0.14.4", "rand 0.10.2", "regex", "reqwest", @@ -4719,7 +4986,7 @@ dependencies = [ "futures-core", "linkme", "memory-stats", - "nix", + "nix 0.31.3", "otel-arrow-dfe-channel", "otel-arrow-dfe-component-inventory-syntax", "otel-arrow-dfe-config", @@ -4787,7 +5054,7 @@ dependencies = [ "otel-arrow-dfe-telemetry-macros", "parking_lot", "prost 0.14.4", - "prost-types", + "prost-types 0.14.4", "reqwest", "rustls", "rustls-native-certs", @@ -4826,7 +5093,7 @@ dependencies = [ "datafusion", "hex", "itertools 0.15.0", - "nix", + "nix 0.31.3", "num_enum", "otel-arrow-dfe-config", "otel-arrow-dfe-pdata-otlp-macros", @@ -4923,7 +5190,7 @@ dependencies = [ "bytes", "crc32fast", "memmap2", - "nix", + "nix 0.31.3", "parking_lot", "thiserror 2.0.20", "tokio", @@ -5127,13 +5394,23 @@ dependencies = [ "pest", ] +[[package]] +name = "petgraph" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" +dependencies = [ + "fixedbitset 0.4.2", + "indexmap", +] + [[package]] name = "petgraph" version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" dependencies = [ - "fixedbitset", + "fixedbitset 0.5.7", "hashbrown 0.15.5", "indexmap", "serde", @@ -5225,6 +5502,31 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "196a7c80b9a7652aba7cc070827516c2abe4ccdf53d128e1944003cf5726cff1" +[[package]] +name = "pprof" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38a01da47675efa7673b032bf8efd8214f1917d89685e07e395ab125ea42b187" +dependencies = [ + "aligned-vec", + "backtrace", + "cfg-if", + "findshlibs", + "libc", + "log", + "nix 0.26.4", + "once_cell", + "prost 0.12.6", + "prost-build 0.12.6", + "prost-derive 0.12.6", + "sha2 0.10.9", + "smallvec", + "spin 0.10.1", + "symbolic-demangle", + "tempfile", + "thiserror 2.0.20", +] + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -5283,6 +5585,16 @@ dependencies = [ "parking_lot", ] +[[package]] +name = "prost" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "deb1435c188b76130da55f17a466d252ff7b1418b2ad3e037d127b94e3411f29" +dependencies = [ + "bytes", + "prost-derive 0.12.6", +] + [[package]] name = "prost" version = "0.13.5" @@ -5303,6 +5615,27 @@ dependencies = [ "prost-derive 0.14.4", ] +[[package]] +name = "prost-build" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22505a5c94da8e3b7c2996394d1c933236c4d743e81a410bcca4e6989fc066a4" +dependencies = [ + "bytes", + "heck", + "itertools 0.12.1", + "log", + "multimap", + "once_cell", + "petgraph 0.6.5", + "prettyplease", + "prost 0.12.6", + "prost-types 0.12.6", + "regex", + "syn 2.0.118", + "tempfile", +] + [[package]] name = "prost-build" version = "0.14.4" @@ -5310,13 +5643,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" dependencies = [ "heck", - "itertools 0.13.0", + "itertools 0.14.0", "log", "multimap", - "petgraph", + "petgraph 0.8.3", "prettyplease", "prost 0.14.4", - "prost-types", + "prost-types 0.14.4", "pulldown-cmark", "pulldown-cmark-to-cmark", "regex", @@ -5324,6 +5657,19 @@ dependencies = [ "tempfile", ] +[[package]] +name = "prost-derive" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1" +dependencies = [ + "anyhow", + "itertools 0.12.1", + "proc-macro2", + "quote", + "syn 2.0.118", +] + [[package]] name = "prost-derive" version = "0.13.5" @@ -5331,7 +5677,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.13.0", + "itertools 0.14.0", "proc-macro2", "quote", "syn 2.0.118", @@ -5344,12 +5690,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools 0.13.0", + "itertools 0.14.0", "proc-macro2", "quote", "syn 2.0.118", ] +[[package]] +name = "prost-types" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9091c90b0a32608e984ff2fa4091273cbdd755d54935c51d520887f4a1dbd5b0" +dependencies = [ + "prost 0.12.6", +] + [[package]] name = "prost-types" version = "0.14.4" @@ -6097,6 +6452,17 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -6276,6 +6642,15 @@ dependencies = [ "lock_api", ] +[[package]] +name = "spin" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" +dependencies = [ + "lock_api", +] + [[package]] name = "sqlparser" version = "0.61.0" @@ -6336,6 +6711,29 @@ version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7401a30af6cb5818bb64852270bb722533397edcfc7344954a38f420819ece2" +[[package]] +name = "symbolic-common" +version = "12.18.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "332615d90111d8eeaf86a84dc9bbe9f65d0d8c5cf11b4caccedc37754eb0dcfd" +dependencies = [ + "debugid", + "memmap2", + "stable_deref_trait", + "uuid", +] + +[[package]] +name = "symbolic-demangle" +version = "12.18.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "912017718eb4d21930546245af9a3475c9dccf15675a5c215664e76621afc471" +dependencies = [ + "cpp_demangle", + "rustc-demangle", + "symbolic-common", +] + [[package]] name = "syn" version = "1.0.109" @@ -6389,6 +6787,21 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "sysinfo" +version = "0.39.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2071df9448915b71c4fe6d25deaf1c22f12bd234f01540b77312bb8e41361e6" +dependencies = [ + "libc", + "memchr", + "ntapi", + "objc2-core-foundation", + "objc2-io-kit", + "objc2-open-directory", + "windows", +] + [[package]] name = "system-configuration" version = "0.7.0" @@ -6634,6 +7047,18 @@ dependencies = [ "tokio-util", ] +[[package]] +name = "tokio-tungstenite" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f72a05e828585856dacd553fba484c242c46e391fb0e58917c942ee9202915c" +dependencies = [ + "futures-util", + "log", + "tokio", + "tungstenite", +] + [[package]] name = "tokio-util" version = "0.7.19" @@ -6755,8 +7180,8 @@ checksum = "654e5643eff75d7f8c99197ce1440ed19a3474eada74c12bbac488b2cafdae27" dependencies = [ "prettyplease", "proc-macro2", - "prost-build", - "prost-types", + "prost-build 0.14.4", + "prost-types 0.14.4", "quote", "syn 2.0.118", "tempfile", @@ -6823,6 +7248,7 @@ version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ + "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -6869,6 +7295,22 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "tungstenite" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c01152af293afb9c7c2a57e4b559c5620b421f6d133261c60dd2d0cdb38e6b8" +dependencies = [ + "bytes", + "data-encoding", + "http", + "httparse", + "log", + "rand 0.9.4", + "sha1", + "thiserror 2.0.20", +] + [[package]] name = "twox-hash" version = "2.1.2" @@ -7401,6 +7843,27 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580" +dependencies = [ + "windows-collections", + "windows-core", + "windows-future", + "windows-numerics", +] + +[[package]] +name = "windows-collections" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610" +dependencies = [ + "windows-core", +] + [[package]] name = "windows-core" version = "0.62.2" @@ -7414,6 +7877,17 @@ dependencies = [ "windows-strings", ] +[[package]] +name = "windows-future" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb" +dependencies = [ + "windows-core", + "windows-link", + "windows-threading", +] + [[package]] name = "windows-implement" version = "0.60.2" @@ -7442,6 +7916,16 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-numerics" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26" +dependencies = [ + "windows-core", + "windows-link", +] + [[package]] name = "windows-registry" version = "0.6.1" @@ -7531,6 +8015,15 @@ dependencies = [ "windows_x86_64_msvc 0.53.1", ] +[[package]] +name = "windows-threading" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37" +dependencies = [ + "windows-link", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" diff --git a/asap-precompute-rs/Cargo.toml b/asap-precompute-rs/Cargo.toml index 274a537..99a8f55 100644 --- a/asap-precompute-rs/Cargo.toml +++ b/asap-precompute-rs/Cargo.toml @@ -43,6 +43,7 @@ otel-arrow-dfe-pdata-views = { git = "https://github.com/open-telemetry/otel-arr otel-arrow-dfe-state = { git = "https://github.com/open-telemetry/otel-arrow", rev = "3e85c3460361446ebfce99e9f35fffd2dd5ab740", package = "otel-arrow-dfe-state", optional = true } otel-arrow-dfe-telemetry = { git = "https://github.com/open-telemetry/otel-arrow", rev = "3e85c3460361446ebfce99e9f35fffd2dd5ab740", package = "otel-arrow-dfe-telemetry", optional = true } otel-arrow-dfe-core-nodes = { git = "https://github.com/open-telemetry/otel-arrow", rev = "3e85c3460361446ebfce99e9f35fffd2dd5ab740", package = "otel-arrow-dfe-core-nodes", optional = true } +otel-arrow-dfe-controller = { git = "https://github.com/open-telemetry/otel-arrow", rev = "3e85c3460361446ebfce99e9f35fffd2dd5ab740", package = "otel-arrow-dfe-controller", optional = true } linkme = { version = "0.3", optional = true } async-trait = { version = "0.1", optional = true } humantime-serde = { version = "1", optional = true } @@ -85,6 +86,14 @@ otap-engine = [ "dep:tokio", ] +# The deployable collector: the official OTAP engine controller with the +# ASAP nodes linked in. Picks a TLS provider, so it is not part of the library. +collector = [ + "otap-engine", + "dep:otel-arrow-dfe-controller", + "otel-arrow-dfe-otap/crypto-ring", +] + # Network benchmark only: do not impose a TLS provider on library consumers. stream-benchmark = [ "otap-engine", @@ -92,6 +101,11 @@ stream-benchmark = [ "otel-arrow-dfe-core-nodes/dev-tools", ] +[[bin]] +name = "asap-collector" +path = "src/bin/asap_collector.rs" +required-features = ["collector"] + [[bin]] name = "asap-stream-bench" path = "src/bin/asap_stream_bench.rs" diff --git a/asap-precompute-rs/src/bin/asap_collector.rs b/asap-precompute-rs/src/bin/asap_collector.rs new file mode 100644 index 0000000..dc0efa1 --- /dev/null +++ b/asap-precompute-rs/src/bin/asap_collector.rs @@ -0,0 +1,37 @@ +//! The official OTAP dataflow engine with the ASAP nodes linked in. +//! +//! `asap-collector --config pipeline.yaml` runs any pipeline the upstream +//! `df_engine` runs, plus `urn:asap:processor:asap_sketches` and +//! `urn:asap:receiver:scenario_generator`. Startup, admin endpoints +//! (`/metrics`, `/readyz`) and signal handling are the upstream controller's. +use asap_precompute_rs as _; +use otel_arrow_dfe_config::config_provider::{resolve_config, ConfigFormat}; +use otel_arrow_dfe_config::engine::OtelDataflowSpec; +use otel_arrow_dfe_controller::{startup, BuildInfo, Controller, ControllerRunOptions}; +use otel_arrow_dfe_core_nodes as _; +use otel_arrow_dfe_otap::OTAP_PIPELINE_FACTORY; + +fn main() -> Result<(), Box> { + otel_arrow_dfe_otap::crypto::install_crypto_provider()?; + let config = std::env::args() + .skip_while(|a| a != "--config" && a != "-c") + .nth(1) + .ok_or("usage: asap-collector --config ")?; + let resolved = resolve_config(Some(&config))?; + let spec = match resolved.format { + ConfigFormat::Json => OtelDataflowSpec::from_json(&resolved.content)?, + ConfigFormat::Yaml => OtelDataflowSpec::from_yaml(&resolved.content)?, + }; + let options = ControllerRunOptions { + build_info: BuildInfo { + service_name: Some(env!("CARGO_BIN_NAME").to_string()), + service_version: Some(env!("CARGO_PKG_VERSION").to_string()), + }, + handle_os_signals: true, + ..Default::default() + }; + startup::validate_engine_components(&spec, &OTAP_PIPELINE_FACTORY)?; + startup::validate_controller_extensions(&spec, &options.extensions)?; + Controller::new(&OTAP_PIPELINE_FACTORY).run_forever_with_options(spec, options)?; + Ok(()) +} From 88f180e1cbc9ccdc1acb5f3af50ce9fed950e470 Mon Sep 17 00:00:00 2001 From: Zeying Zhu <50204836+zzylol@users.noreply.github.com> Date: Mon, 12 Oct 2026 18:12:53 +0000 Subject: [PATCH 2/6] Add window_merge: merge sources' sketches by the window they describe Sketches with the same series and window are merged and emitted once, a grace period after the window ends; later arrivals are counted as late. The ASAPv1 decode used by the carrier check becomes a shared sketches::decode_snapshot. --- asap-precompute-rs/src/otap/codec.rs | 19 +- asap-precompute-rs/src/otap/mod.rs | 2 + asap-precompute-rs/src/otap/window_merge.rs | 291 ++++++++++++++++++++ asap-precompute-rs/src/sketches/mod.rs | 23 ++ 4 files changed, 320 insertions(+), 15 deletions(-) create mode 100644 asap-precompute-rs/src/otap/window_merge.rs diff --git a/asap-precompute-rs/src/otap/codec.rs b/asap-precompute-rs/src/otap/codec.rs index 6deafcb..720dfbe 100644 --- a/asap-precompute-rs/src/otap/codec.rs +++ b/asap-precompute-rs/src/otap/codec.rs @@ -69,10 +69,6 @@ use crate::config::{sketch_size_string, SketchParams}; use crate::envelope::{SketchEnvelope, SketchType}; use crate::observation::{KeyValue, Observation, ObservationValue}; -use crate::sketches::{ - CMSHeapWrapper, CMSWrapper, CountSketchWrapper, DDSketchWrapper, HLLWrapper, KLLWrapper, -}; - use super::schema::{ ATTR_AGG_ID, ATTR_ENVELOPE, ATTR_SCHEMA_VERSION, ATTR_SERIES_ID, ATTR_SKETCH_SIZE, ATTR_SKETCH_TYPE, ATTR_WINDOW_END_MS, ATTR_WINDOW_START_MS, @@ -184,17 +180,10 @@ fn validate_self_describing_payload(sketch_type: SketchType, payload: &[u8]) -> if payload.is_empty() { return Ok(()); } - let decoded = match sketch_type { - SketchType::DDSketch => DDSketchWrapper::from_snapshot(payload).map(drop), - SketchType::KLLSketch => KLLWrapper::from_snapshot(payload).map(drop), - SketchType::HLLSketch => HLLWrapper::from_snapshot(payload).map(drop), - SketchType::CountSketch => CountSketchWrapper::from_snapshot(payload).map(drop), - SketchType::CountMinSketch => CMSWrapper::from_snapshot(payload).map(drop), - SketchType::CountMinSketchWithHeap => CMSHeapWrapper::from_snapshot(payload).map(drop), - SketchType::Unspecified => { - return Err("sketch.envelope requires a sketch.type".into()); - } - }; + if sketch_type == SketchType::Unspecified { + return Err("sketch.envelope requires a sketch.type".into()); + } + let decoded = crate::sketches::decode_snapshot(sketch_type, payload).map(drop); decoded.map_err(|error| format!("invalid {} sketch.envelope: {error}", sketch_type.name())) } diff --git a/asap-precompute-rs/src/otap/mod.rs b/asap-precompute-rs/src/otap/mod.rs index 71f02b3..c2cef72 100644 --- a/asap-precompute-rs/src/otap/mod.rs +++ b/asap-precompute-rs/src/otap/mod.rs @@ -16,6 +16,8 @@ pub mod codec; pub mod processor; #[cfg(feature = "otap-engine")] pub mod scenario_generator; +#[cfg(feature = "otap-engine")] +pub mod window_merge; pub use config::{ConfigError, PluginConfig, SketchDispatch}; pub use schema::{ diff --git a/asap-precompute-rs/src/otap/window_merge.rs b/asap-precompute-rs/src/otap/window_merge.rs new file mode 100644 index 0000000..6a5534b --- /dev/null +++ b/asap-precompute-rs/src/otap/window_merge.rs @@ -0,0 +1,291 @@ +//! `urn:asap:processor:window_merge`: merges sketches from many sources by +//! the window they describe, not by when they arrive. +//! +//! Each input envelope names its series (labels, sketch type, agg id) and its +//! window. Envelopes with the same series and window are merged. A window is +//! emitted once, `grace` after its end on this host's clock; envelopes for a +//! window already emitted are counted as late and dropped. Scalar (non-sketch) +//! inputs pass through unchanged. +//! +//! ```yaml +//! type: urn:asap:processor:window_merge +//! config: {grace: 5s} +//! ``` +use std::collections::BTreeMap; +use std::sync::Arc; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use async_trait::async_trait; +use linkme::distributed_slice; +use otel_arrow_dfe_config::node::NodeUserConfig; +use otel_arrow_dfe_engine::config::ProcessorConfig; +use otel_arrow_dfe_engine::context::PipelineContext; +use otel_arrow_dfe_engine::control::NodeControlMsg; +use otel_arrow_dfe_engine::error::Error; +use otel_arrow_dfe_engine::local::processor as local; +use otel_arrow_dfe_engine::message::Message; +use otel_arrow_dfe_engine::node::NodeId; +use otel_arrow_dfe_engine::processor::ProcessorWrapper; +use otel_arrow_dfe_engine::{MessageSourceLocalEffectHandlerExtension, ProcessorFactory}; +use otel_arrow_dfe_otap::pdata::OtapPdata; +use otel_arrow_dfe_otap::OTAP_PROCESSOR_FACTORIES; +use serde::Deserialize; + +use super::codec::{decode_pdata_to_observations, OtapSketchEncoder}; +use crate::envelope::SketchEnvelope; +use crate::observation::ObservationValueKind; +use crate::precompute::{PrecomputeError, Sketch}; +use crate::sketches::decode_snapshot; + +/// Registered URN. +pub const WINDOW_MERGE_URN: &str = "urn:asap:processor:window_merge"; + +/// Processor configuration. +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct WindowMergeConfig { + /// How long after a window's end to wait for the last source. + #[serde(default = "default_grace", with = "humantime_serde")] + pub grace: Duration, +} + +fn default_grace() -> Duration { + Duration::from_secs(5) +} + +struct Pending { + envelope: SketchEnvelope, + sketch: Box, +} + +/// Windows waiting for their grace period, keyed by window end then series. +#[derive(Default)] +pub struct WindowMerger { + windows: BTreeMap>, + /// Every window ending at or before this has been emitted. + emitted_until: u64, + /// Envelopes that arrived after their window was emitted. + pub late: u64, +} + +fn series_key(e: &SketchEnvelope) -> String { + let mut labels: Vec<_> = e.labels.iter().map(|kv| (&kv.key, &kv.value)).collect(); + labels.sort(); + let mut resource: Vec<_> = e + .resource_labels + .iter() + .map(|kv| (&kv.key, &kv.value)) + .collect(); + resource.sort(); + format!( + "{}|{}|{}|{}|{:?}|{:?}", + e.metric_name, + e.sketch_type.name(), + e.agg_id, + e.window_start_ms, + resource, + labels + ) +} + +impl WindowMerger { + /// Adds one source's sketch for a window. + pub fn add(&mut self, envelope: SketchEnvelope) -> Result<(), PrecomputeError> { + if envelope.window_end_ms <= self.emitted_until { + self.late += 1; + return Ok(()); + } + let window = self.windows.entry(envelope.window_end_ms).or_default(); + match window.entry(series_key(&envelope)) { + std::collections::btree_map::Entry::Occupied(mut slot) => { + let pending = slot.get_mut(); + pending.sketch.merge_snapshot(&envelope.payload)?; + pending.envelope.count += envelope.count; + } + std::collections::btree_map::Entry::Vacant(slot) => { + let sketch = decode_snapshot(envelope.sketch_type, &envelope.payload)?; + slot.insert(Pending { envelope, sketch }); + } + } + Ok(()) + } + + /// Merged envelopes of every window that ended `grace` or more before `now_ms`. + pub fn due(&mut self, now_ms: u64, grace: Duration) -> Vec { + let cutoff = now_ms.saturating_sub(grace.as_millis() as u64); + let open = self.windows.split_off(&(cutoff + 1)); + let ready = std::mem::replace(&mut self.windows, open); + let mut out = Vec::new(); + for (end, series) in ready { + self.emitted_until = self.emitted_until.max(end); + for (_, mut p) in series { + if let Ok(payload) = p.sketch.snapshot() { + p.envelope.payload = payload; + out.push(p.envelope); + } + } + } + out + } +} + +struct WindowMergeProcessor { + config: WindowMergeConfig, + merger: WindowMerger, + encoder: OtapSketchEncoder, + timer_started: bool, +} + +fn now_ms() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0) +} + +impl WindowMergeProcessor { + async fn emit( + &mut self, + envs: Vec, + effects: &mut local::EffectHandler, + ) -> Result<(), Error> { + if envs.is_empty() { + return Ok(()); + } + match self.encoder.encode(&envs) { + Ok(pdata) => effects.send_message_with_source_node(pdata).await?, + Err(e) => { + effects + .info(&format!( + "window_merge: dropping window after encode failure: {e}" + )) + .await + } + } + Ok(()) + } +} + +#[async_trait(?Send)] +impl local::Processor for WindowMergeProcessor { + async fn process( + &mut self, + msg: Message, + effects: &mut local::EffectHandler, + ) -> Result<(), Error> { + if !self.timer_started { + self.timer_started = effects + .start_periodic_timer(Duration::from_secs(1)) + .await + .is_ok(); + } + match msg { + Message::PData(pdata) => { + let observations = match decode_pdata_to_observations(pdata) { + Ok(outcome) => outcome.observations, + Err(e) => { + effects + .info(&format!("window_merge: dropping undecodable input: {e}")) + .await; + return Ok(()); + } + }; + let mut passthrough = Vec::new(); + for obs in observations { + match (obs.value.kind, obs.value.envelope) { + (ObservationValueKind::Envelope, Some(env)) if !env.payload.is_empty() => { + if let Err(e) = self.merger.add(*env) { + effects.info(&format!("window_merge: {e}")).await; + } + } + (ObservationValueKind::Envelope, Some(env)) => passthrough.push(*env), + _ => {} + } + } + self.emit(passthrough, effects).await + } + Message::Control(NodeControlMsg::TimerTick { .. }) => { + let due = self.merger.due(now_ms(), self.config.grace); + self.emit(due, effects).await + } + Message::Control(NodeControlMsg::Shutdown { .. }) => { + let all = self.merger.due(u64::MAX, Duration::ZERO); + self.emit(all, effects).await + } + _ => Ok(()), + } + } +} + +#[allow(unsafe_code)] +#[distributed_slice(OTAP_PROCESSOR_FACTORIES)] +static WINDOW_MERGE: ProcessorFactory = ProcessorFactory { + name: WINDOW_MERGE_URN, + create: |_: PipelineContext, + node: NodeId, + node_config: Arc, + processor_config: &ProcessorConfig, + _: &otel_arrow_dfe_engine::capability::registry::Capabilities| { + let config: WindowMergeConfig = serde_json::from_value(node_config.config.clone()) + .map_err(|e| otel_arrow_dfe_config::error::Error::InvalidUserConfig { + error: format!("window_merge: {e}"), + })?; + Ok(ProcessorWrapper::local( + WindowMergeProcessor { + config, + merger: WindowMerger::default(), + encoder: OtapSketchEncoder::default(), + timer_started: false, + }, + node, + node_config, + processor_config, + )) + }, + wiring_contract: otel_arrow_dfe_engine::wiring_contract::WiringContract::UNRESTRICTED, + validate_config: otel_arrow_dfe_config::validation::validate_typed_config::, +}; + +#[cfg(test)] +mod tests { + use super::*; + use crate::envelope::SketchType; + use crate::observation::KeyValue; + use crate::precompute::CardinalitySketch; + use crate::sketches::HLLWrapper; + + fn hll(values: std::ops::Range, start: u64) -> SketchEnvelope { + let mut sk = HLLWrapper::new(); + values.for_each(|v| sk.update(&v.to_le_bytes())); + SketchEnvelope { + sketch_type: SketchType::HLLSketch, + labels: vec![KeyValue::new("attribute.key", "user.id")], + window_start_ms: start, + window_end_ms: start + 60_000, + payload: sk.snapshot().unwrap(), + metric_name: "asap.distinct".into(), + count: 1000, + ..Default::default() + } + } + + #[test] + fn merges_sources_per_window_after_grace() { + let mut m = WindowMerger::default(); + m.add(hll(0..1000, 0)).unwrap(); + m.add(hll(500..1500, 0)).unwrap(); + m.add(hll(0..10, 60_000)).unwrap(); + let grace = Duration::from_secs(5); + assert!(m.due(64_000, grace).is_empty()); + let out = m.due(65_000, grace); + assert_eq!(out.len(), 1); + assert_eq!(out[0].count, 2000); + let merged = HLLWrapper::from_snapshot(&out[0].payload).unwrap(); + let d = merged.estimate_cardinality(); + assert!((d - 1500.0).abs() < 1500.0 * 0.05, "{d}"); + // A source arriving after its window was emitted is late. + m.add(hll(0..5, 0)).unwrap(); + assert_eq!(m.late, 1); + assert_eq!(m.due(125_000, grace).len(), 1); + } +} diff --git a/asap-precompute-rs/src/sketches/mod.rs b/asap-precompute-rs/src/sketches/mod.rs index 023b070..74b8c23 100644 --- a/asap-precompute-rs/src/sketches/mod.rs +++ b/asap-precompute-rs/src/sketches/mod.rs @@ -19,3 +19,26 @@ pub use countsketch::{CountSketchObserver, CountSketchWrapper}; pub use ddsketch::{DDSketchObserver, DDSketchWrapper}; pub use hll::{HLLObserver, HLLWrapper}; pub use kll::{KLLObserver, KLLWrapper}; + +use crate::envelope::SketchType; +use crate::precompute::{PrecomputeError, Sketch}; + +/// Decodes an ASAPv1 snapshot into the wrapper for `sketch_type`. +pub fn decode_snapshot( + sketch_type: SketchType, + bytes: &[u8], +) -> Result, PrecomputeError> { + Ok(match sketch_type { + SketchType::DDSketch => Box::new(DDSketchWrapper::from_snapshot(bytes)?), + SketchType::KLLSketch => Box::new(KLLWrapper::from_snapshot(bytes)?), + SketchType::HLLSketch => Box::new(HLLWrapper::from_snapshot(bytes)?), + SketchType::CountSketch => Box::new(CountSketchWrapper::from_snapshot(bytes)?), + SketchType::CountMinSketch => Box::new(CMSWrapper::from_snapshot(bytes)?), + SketchType::CountMinSketchWithHeap => Box::new(CMSHeapWrapper::from_snapshot(bytes)?), + SketchType::Unspecified => { + return Err(PrecomputeError::Other( + "sketch.envelope requires a sketch.type".into(), + )) + } + }) +} From 29dc361b0788b9d2eaf7f35b102b678103347e18 Mon Sep 17 00:00:00 2001 From: Zeying Zhu <50204836+zzylol@users.noreply.github.com> Date: Mon, 12 Oct 2026 18:14:53 +0000 Subject: [PATCH 3/6] Add sketch_detector and exact_answers processors Both emit asap.quantile, asap.distinct, asap.topk and asap.points gauges in one shape, labelled method=sketch or method=exact, so dashboards can overlay them. Sketch answers carry lower/upper bounds from each sketch's configuration (HLL 95% standard error, KLL normalized rank error, Count-Min overestimate). exact_answers groups raw points into event-time windows and is both the ground truth and the exact baseline. --- asap-precompute-rs/src/answers.rs | 284 ++++++++++++++ asap-precompute-rs/src/lib.rs | 1 + asap-precompute-rs/src/otap/answer_nodes.rs | 414 ++++++++++++++++++++ asap-precompute-rs/src/otap/mod.rs | 2 + 4 files changed, 701 insertions(+) create mode 100644 asap-precompute-rs/src/answers.rs create mode 100644 asap-precompute-rs/src/otap/answer_nodes.rs diff --git a/asap-precompute-rs/src/answers.rs b/asap-precompute-rs/src/answers.rs new file mode 100644 index 0000000..65a3fab --- /dev/null +++ b/asap-precompute-rs/src/answers.rs @@ -0,0 +1,284 @@ +//! Answers to the three telemetry questions, from sketches or from exact data, +//! in one output shape so dashboards can overlay `method="sketch"` and +//! `method="exact"`. +//! +//! | metric | labels (besides the series labels) | value | +//! | --- | --- | --- | +//! | `asap.quantile` | `quantile`, `bound` | quantile of the window's values | +//! | `asap.distinct` | `bound` | distinct values of the series' attribute | +//! | `asap.points` | | data points in the window | +//! | `asap.topk` | `key`, `rank`, `bound` | count of a heavy-hitter value | +//! +//! Every answer also carries `metric` (the source metric) and `method`. +//! `bound` is `estimate`, `lower` or `upper`; exact answers only carry +//! `estimate`. Bounds come from each sketch's configuration: +//! +//! * HLL with 2^14 registers: standard error 1.04/sqrt(2^14) (0.81%), shown at +//! 95% (±1.96 standard errors). +//! * KLL: normalized rank error 2.296 / k^0.9723 (the DataSketches KLL +//! formula, 99% confidence; 1.3% at k = 200), turned into the quantiles at +//! q ± error. +//! * Count-Min: counts never underestimate; with width w and N points the +//! overestimate is at most e/w * N with probability 1 - e^-depth. +use std::collections::{HashMap, HashSet}; + +use crate::envelope::{SketchEnvelope, SketchType}; +use crate::observation::KeyValue; +use crate::precompute::{CardinalitySketch, FrequencySketch, PrecomputeError, QuantileSketch}; +use crate::sketches::{CMSHeapWrapper, HLLWrapper, KLLWrapper}; + +/// HLL standard error with 2^14 registers. +pub const HLL_RELATIVE_ERROR: f64 = 1.04 / 128.0; +/// Two-sided 95% normal quantile. +pub const Z95: f64 = 1.96; + +/// KLL normalized rank error for `k` (99% confidence). +pub fn kll_rank_error(k: usize) -> f64 { + 2.296 / (k as f64).powf(0.9723) +} + +/// One scalar answer for a window, ready for the OTAP encoder. +pub fn gauge( + template: &SketchEnvelope, + metric: &str, + method: &str, + extra: &[(&str, String)], + value: f64, +) -> SketchEnvelope { + let mut labels = template.labels.clone(); + labels.push(KeyValue::new("metric", template.metric_name.clone())); + labels.push(KeyValue::new("method", method)); + labels.extend(extra.iter().map(|(k, v)| KeyValue::new(*k, v.clone()))); + SketchEnvelope { + schema_version: 1, + sketch_type: SketchType::Unspecified, + agg_id: template.agg_id, + resource_labels: template.resource_labels.clone(), + labels, + window_start_ms: template.window_start_ms, + window_end_ms: template.window_end_ms, + payload: Vec::new(), + metric_name: metric.into(), + count: template.count, + aggregation_temporality: 0, + value, + } +} + +fn bounded( + template: &SketchEnvelope, + metric: &str, + extra: &[(&str, String)], + [estimate, lower, upper]: [f64; 3], +) -> Vec { + [("estimate", estimate), ("lower", lower), ("upper", upper)] + .into_iter() + .map(|(bound, v)| { + let mut labels = extra.to_vec(); + labels.push(("bound", bound.into())); + gauge(template, metric, "sketch", &labels, v) + }) + .collect() +} + +/// Answers from one merged sketch window. `quantiles` applies to KLL, `k` to +/// top-k. Other sketch types produce no answers. +pub fn sketch_answers( + env: &SketchEnvelope, + quantiles: &[f64], + k: usize, +) -> Result, PrecomputeError> { + let mut out = vec![gauge(env, "asap.points", "sketch", &[], env.count as f64)]; + match env.sketch_type { + SketchType::KLLSketch => { + let sk = KLLWrapper::from_snapshot(&env.payload)?; + let eps = kll_rank_error(sk.inner().k()); + for &q in quantiles { + let at = |p: f64| sk.quantile(p.clamp(0.0, 1.0)); + out.extend(bounded( + env, + "asap.quantile", + &[("quantile", q.to_string())], + [at(q), at(q - eps), at(q + eps)], + )); + } + } + SketchType::HLLSketch => { + let d = HLLWrapper::from_snapshot(&env.payload)?.estimate_cardinality(); + let e = Z95 * HLL_RELATIVE_ERROR * d; + out.extend(bounded(env, "asap.distinct", &[], [d, d - e, d + e])); + } + SketchType::CountMinSketchWithHeap => { + let sk = CMSHeapWrapper::from_snapshot(&env.payload)?; + let over = std::f64::consts::E / sk.inner().cols() as f64 * env.count as f64; + for (rank, entry) in sk.top_k(k).into_iter().enumerate() { + let key = String::from_utf8_lossy(&entry.key).into_owned(); + out.extend(bounded( + env, + "asap.topk", + &[("key", key), ("rank", (rank + 1).to_string())], + [entry.count, (entry.count - over).max(0.0), entry.count], + )); + } + } + _ => {} + } + Ok(out) +} + +/// Exact data for one series and window. +#[derive(Default)] +pub struct ExactWindow { + /// Every value, for quantiles. + pub values: Vec, + /// Distinct attribute values. + pub distinct: HashSet, + /// Occurrences per attribute value. + pub counts: HashMap, + /// Data points. + pub points: u64, +} + +/// Exact answers in the same shape as [`sketch_answers`]. +pub fn exact_answers( + template: &SketchEnvelope, + window: &mut ExactWindow, + quantiles: &[f64], + k: usize, +) -> Vec { + let e = |metric: &str, extra: &[(&str, String)], v: f64| { + let mut labels = extra.to_vec(); + labels.push(("bound", "estimate".into())); + gauge(template, metric, "exact", &labels, v) + }; + let mut out = vec![gauge( + template, + "asap.points", + "exact", + &[], + window.points as f64, + )]; + if !window.values.is_empty() { + window.values.sort_unstable_by(f64::total_cmp); + for &q in quantiles { + let i = ((window.values.len() - 1) as f64 * q).round() as usize; + out.push(e( + "asap.quantile", + &[("quantile", q.to_string())], + window.values[i], + )); + } + } + if !window.distinct.is_empty() { + out.push(e("asap.distinct", &[], window.distinct.len() as f64)); + } + let mut top: Vec<_> = window.counts.iter().collect(); + top.sort_unstable_by(|a, b| b.1.cmp(a.1).then(a.0.cmp(b.0))); + for (rank, (key, count)) in top.into_iter().take(k).enumerate() { + out.push(e( + "asap.topk", + &[("key", key.clone()), ("rank", (rank + 1).to_string())], + *count as f64, + )); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::precompute::Sketch; + + fn find<'a>( + out: &'a [SketchEnvelope], + metric: &str, + labels: &[(&str, &str)], + ) -> &'a SketchEnvelope { + out.iter() + .find(|e| { + e.metric_name == metric + && labels + .iter() + .all(|(k, v)| e.labels.iter().any(|kv| kv.key == *k && kv.value == *v)) + }) + .unwrap_or_else(|| panic!("missing {metric} {labels:?}")) + } + + #[test] + fn kll_bounds_contain_the_exact_quantile() { + let mut sk = KLLWrapper::new(200, Some(1)); + let mut exact = ExactWindow::default(); + for i in 0..100_000u64 { + let v = ((i * 7919) % 100_000) as f64; + sk.update(v); + exact.values.push(v); + } + let env = SketchEnvelope { + sketch_type: SketchType::KLLSketch, + payload: sk.snapshot().unwrap(), + count: 100_000, + ..Default::default() + }; + let out = sketch_answers(&env, &[0.5, 0.99], 10).unwrap(); + let truth = exact_answers(&env, &mut exact, &[0.5, 0.99], 10); + for q in ["0.5", "0.99"] { + let t = find(&truth, "asap.quantile", &[("quantile", q)]).value; + let lo = find( + &out, + "asap.quantile", + &[("quantile", q), ("bound", "lower")], + ) + .value; + let hi = find( + &out, + "asap.quantile", + &[("quantile", q), ("bound", "upper")], + ) + .value; + assert!(lo <= t && t <= hi, "{q}: {lo} <= {t} <= {hi}"); + } + } + + #[test] + fn hll_and_topk_answers_carry_bounds() { + let mut hll = HLLWrapper::new(); + let mut heap = CMSHeapWrapper::new(4, 2048, 3); + for i in 0..20_000u32 { + hll.update(&i.to_le_bytes()); + heap.update(format!("k{}", i % 7 % 3).as_bytes(), 1); + } + let base = SketchEnvelope { + count: 20_000, + ..Default::default() + }; + let hll_env = SketchEnvelope { + sketch_type: SketchType::HLLSketch, + payload: hll.snapshot().unwrap(), + ..base.clone() + }; + let out = sketch_answers(&hll_env, &[], 3).unwrap(); + let lo = find(&out, "asap.distinct", &[("bound", "lower")]).value; + let hi = find(&out, "asap.distinct", &[("bound", "upper")]).value; + assert!(lo < 20_000.0 && 20_000.0 < hi, "{lo} {hi}"); + + let heap_env = SketchEnvelope { + sketch_type: SketchType::CountMinSketchWithHeap, + payload: heap.snapshot().unwrap(), + ..base + }; + let out = sketch_answers(&heap_env, &[], 3).unwrap(); + assert_eq!( + find(&out, "asap.topk", &[("rank", "1"), ("bound", "estimate")]) + .labels + .iter() + .find(|kv| kv.key == "key") + .unwrap() + .value, + "k0" + ); + assert!(out.iter().all(|e| e + .labels + .iter() + .any(|kv| kv.key == "method" && kv.value == "sketch"))); + } +} diff --git a/asap-precompute-rs/src/lib.rs b/asap-precompute-rs/src/lib.rs index 2a931a9..e55c2fb 100644 --- a/asap-precompute-rs/src/lib.rs +++ b/asap-precompute-rs/src/lib.rs @@ -13,6 +13,7 @@ // rewrite behavior-sensitive comparisons. #![allow(clippy::neg_cmp_op_on_partial_ord)] +pub mod answers; pub mod config; pub mod envelope; pub mod kll_windows; diff --git a/asap-precompute-rs/src/otap/answer_nodes.rs b/asap-precompute-rs/src/otap/answer_nodes.rs new file mode 100644 index 0000000..aa6006a --- /dev/null +++ b/asap-precompute-rs/src/otap/answer_nodes.rs @@ -0,0 +1,414 @@ +//! Two processors that turn telemetry into [`crate::answers`] gauges: +//! +//! * `urn:asap:processor:sketch_detector` reads merged sketch windows (the +//! output of `window_merge`) and emits each window's answers with bounds. +//! * `urn:asap:processor:exact_answers` reads raw data points, groups them +//! into event-time windows, and emits the exact answers: the ground truth +//! and the exact baseline. +//! +//! ```yaml +//! detector: +//! type: urn:asap:processor:sketch_detector +//! config: {quantiles: [0.5, 0.99], k: 10} +//! exact: +//! type: urn:asap:processor:exact_answers +//! config: +//! window: 60s +//! group_by: [http.route] # quantile series, as aggregate_by +//! per_attribute: [user.id] # distinct values and top-k per key +//! ``` +use std::collections::BTreeMap; +use std::sync::Arc; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use async_trait::async_trait; +use linkme::distributed_slice; +use otel_arrow_dfe_config::node::NodeUserConfig; +use otel_arrow_dfe_engine::config::ProcessorConfig; +use otel_arrow_dfe_engine::context::PipelineContext; +use otel_arrow_dfe_engine::control::NodeControlMsg; +use otel_arrow_dfe_engine::error::Error; +use otel_arrow_dfe_engine::local::processor as local; +use otel_arrow_dfe_engine::message::Message; +use otel_arrow_dfe_engine::node::NodeId; +use otel_arrow_dfe_engine::processor::ProcessorWrapper; +use otel_arrow_dfe_engine::{MessageSourceLocalEffectHandlerExtension, ProcessorFactory}; +use otel_arrow_dfe_otap::pdata::OtapPdata; +use otel_arrow_dfe_otap::OTAP_PROCESSOR_FACTORIES; +use serde::Deserialize; + +use super::codec::{decode_pdata_to_observations, OtapSketchEncoder}; +use crate::answers::{exact_answers, sketch_answers, ExactWindow}; +use crate::envelope::SketchEnvelope; +use crate::observation::{KeyValue, Observation, ObservationValueKind}; + +/// Registered URN of the sketch detector. +pub const SKETCH_DETECTOR_URN: &str = "urn:asap:processor:sketch_detector"; +/// Registered URN of the exact baseline. +pub const EXACT_ANSWERS_URN: &str = "urn:asap:processor:exact_answers"; + +fn default_quantiles() -> Vec { + vec![0.5, 0.99] +} +fn default_k() -> usize { + 10 +} +fn default_window() -> Duration { + Duration::from_secs(60) +} +fn default_grace() -> Duration { + Duration::from_secs(5) +} + +/// `sketch_detector` configuration. +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DetectorConfig { + /// Quantiles answered from KLL windows. + #[serde(default = "default_quantiles")] + pub quantiles: Vec, + /// Heavy hitters answered from CMS + heap windows. + #[serde(default = "default_k")] + pub k: usize, +} + +/// `exact_answers` configuration. +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ExactConfig { + /// Event-time window size. + #[serde(default = "default_window", with = "humantime_serde")] + pub window: Duration, + /// Wait after a window's end before answering it. + #[serde(default = "default_grace", with = "humantime_serde")] + pub grace: Duration, + /// Labels identifying a quantile series; empty disables quantiles. + #[serde(default)] + pub group_by: Vec, + /// Attribute keys whose distinct values and heavy hitters are answered. + #[serde(default)] + pub per_attribute: Vec, + /// Quantiles to answer. + #[serde(default = "default_quantiles")] + pub quantiles: Vec, + /// Heavy hitters per attribute key. + #[serde(default = "default_k")] + pub k: usize, +} + +fn now_ms() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0) +} + +async fn emit( + encoder: &mut OtapSketchEncoder, + envs: &[SketchEnvelope], + effects: &mut local::EffectHandler, +) -> Result<(), Error> { + if envs.is_empty() { + return Ok(()); + } + match encoder.encode(envs) { + Ok(pdata) => effects.send_message_with_source_node(pdata).await?, + Err(e) => effects.info(&format!("answers: encode failed: {e}")).await, + } + Ok(()) +} + +async fn observations( + pdata: OtapPdata, + effects: &mut local::EffectHandler, +) -> Vec { + match decode_pdata_to_observations(pdata) { + Ok(outcome) => outcome.observations, + Err(e) => { + effects + .info(&format!("answers: undecodable input: {e}")) + .await; + Vec::new() + } + } +} + +struct SketchDetector { + config: DetectorConfig, + encoder: OtapSketchEncoder, +} + +#[async_trait(?Send)] +impl local::Processor for SketchDetector { + async fn process( + &mut self, + msg: Message, + effects: &mut local::EffectHandler, + ) -> Result<(), Error> { + let Message::PData(pdata) = msg else { + return Ok(()); + }; + let mut out = Vec::new(); + for obs in observations(pdata, effects).await { + let Some(env) = obs.value.envelope.filter(|e| !e.payload.is_empty()) else { + continue; + }; + match sketch_answers(&env, &self.config.quantiles, self.config.k) { + Ok(answers) => out.extend(answers), + Err(e) => effects.info(&format!("sketch_detector: {e}")).await, + } + } + emit(&mut self.encoder, &out, effects).await + } +} + +/// Exact state of one window: per quantile series and per attribute key. +#[derive(Default)] +struct ExactState { + series: BTreeMap, +} + +struct ExactAnswers { + config: ExactConfig, + /// Open windows by window end. + windows: BTreeMap, + emitted_until: u64, + encoder: OtapSketchEncoder, + timer_started: bool, +} + +impl ExactAnswers { + fn template(obs: &Observation, labels: Vec, start: u64, end: u64) -> SketchEnvelope { + SketchEnvelope { + labels, + window_start_ms: start, + window_end_ms: end, + metric_name: obs.metric.clone(), + ..Default::default() + } + } + + fn observe(&mut self, obs: &Observation) { + if obs.value.kind != ObservationValueKind::Float { + return; + } + let size = self.config.window.as_millis().max(1) as u64; + let start = obs.timestamp_ms / size * size; + let end = start + size; + if end <= self.emitted_until { + return; + } + let state = self.windows.entry(end).or_default(); + let label = |k: &str| { + obs.labels + .iter() + .find(|kv| kv.key == k) + .map(|kv| kv.value.clone()) + }; + if !self.config.group_by.is_empty() { + let labels: Vec = self + .config + .group_by + .iter() + .filter_map(|k| label(k).map(|v| KeyValue::new(k.clone(), v))) + .collect(); + let key = format!("q|{}|{labels:?}", obs.metric); + let (_, w) = state.series.entry(key).or_insert_with(|| { + ( + Self::template(obs, labels, start, end), + ExactWindow::default(), + ) + }); + w.values.push(obs.value.float); + w.points += 1; + } + for attr in &self.config.per_attribute { + let Some(value) = label(attr) else { continue }; + let key = format!("a|{}|{attr}", obs.metric); + let (_, w) = state.series.entry(key).or_insert_with(|| { + let labels = vec![KeyValue::new("attribute.key", attr.clone())]; + ( + Self::template(obs, labels, start, end), + ExactWindow::default(), + ) + }); + *w.counts.entry(value.clone()).or_default() += 1; + w.distinct.insert(value); + w.points += 1; + } + } + + fn due(&mut self, now: u64) -> Vec { + let cutoff = now.saturating_sub(self.config.grace.as_millis() as u64); + let open = self.windows.split_off(&(cutoff + 1)); + let ready = std::mem::replace(&mut self.windows, open); + let mut out = Vec::new(); + for (end, state) in ready { + self.emitted_until = self.emitted_until.max(end); + for (_, (mut template, mut w)) in state.series { + template.count = w.points; + out.extend(exact_answers( + &template, + &mut w, + &self.config.quantiles, + self.config.k, + )); + } + } + out + } +} + +#[async_trait(?Send)] +impl local::Processor for ExactAnswers { + async fn process( + &mut self, + msg: Message, + effects: &mut local::EffectHandler, + ) -> Result<(), Error> { + if !self.timer_started { + self.timer_started = effects + .start_periodic_timer(Duration::from_secs(1)) + .await + .is_ok(); + } + match msg { + Message::PData(pdata) => { + for obs in observations(pdata, effects).await { + self.observe(&obs); + } + Ok(()) + } + Message::Control(NodeControlMsg::TimerTick { .. }) => { + let out = self.due(now_ms()); + emit(&mut self.encoder, &out, effects).await + } + Message::Control(NodeControlMsg::Shutdown { .. }) => { + let out = self.due(u64::MAX); + emit(&mut self.encoder, &out, effects).await + } + _ => Ok(()), + } + } +} + +fn invalid(e: impl std::fmt::Display) -> otel_arrow_dfe_config::error::Error { + otel_arrow_dfe_config::error::Error::InvalidUserConfig { + error: e.to_string(), + } +} + +#[allow(unsafe_code)] +#[distributed_slice(OTAP_PROCESSOR_FACTORIES)] +static SKETCH_DETECTOR: ProcessorFactory = ProcessorFactory { + name: SKETCH_DETECTOR_URN, + create: |_: PipelineContext, + node: NodeId, + node_config: Arc, + processor_config: &ProcessorConfig, + _: &otel_arrow_dfe_engine::capability::registry::Capabilities| { + let config: DetectorConfig = + serde_json::from_value(node_config.config.clone()).map_err(invalid)?; + Ok(ProcessorWrapper::local( + SketchDetector { + config, + encoder: OtapSketchEncoder::default(), + }, + node, + node_config, + processor_config, + )) + }, + wiring_contract: otel_arrow_dfe_engine::wiring_contract::WiringContract::UNRESTRICTED, + validate_config: otel_arrow_dfe_config::validation::validate_typed_config::, +}; + +#[allow(unsafe_code)] +#[distributed_slice(OTAP_PROCESSOR_FACTORIES)] +static EXACT_ANSWERS: ProcessorFactory = ProcessorFactory { + name: EXACT_ANSWERS_URN, + create: |_: PipelineContext, + node: NodeId, + node_config: Arc, + processor_config: &ProcessorConfig, + _: &otel_arrow_dfe_engine::capability::registry::Capabilities| { + let config: ExactConfig = + serde_json::from_value(node_config.config.clone()).map_err(invalid)?; + Ok(ProcessorWrapper::local( + ExactAnswers { + config, + windows: BTreeMap::new(), + emitted_until: 0, + encoder: OtapSketchEncoder::default(), + timer_started: false, + }, + node, + node_config, + processor_config, + )) + }, + wiring_contract: otel_arrow_dfe_engine::wiring_contract::WiringContract::UNRESTRICTED, + validate_config: otel_arrow_dfe_config::validation::validate_typed_config::, +}; + +#[cfg(test)] +mod tests { + use super::*; + use crate::observation::ObservationValue; + + fn point(ts: u64, route: &str, user: &str, v: f64) -> Observation { + Observation { + timestamp_ms: ts, + metric: "latency".into(), + resource_labels: vec![], + labels: vec![ + KeyValue::new("http.route", route), + KeyValue::new("user.id", user), + ], + value: ObservationValue::float(v), + } + } + + #[test] + fn exact_windows_by_event_time_after_grace() { + let mut exact = ExactAnswers { + config: serde_json::from_value(serde_json::json!({ + "group_by": ["http.route"], "per_attribute": ["user.id"], "k": 2 + })) + .unwrap(), + windows: BTreeMap::new(), + emitted_until: 0, + encoder: OtapSketchEncoder::default(), + timer_started: false, + }; + for i in 0..100u64 { + exact.observe(&point(1_000 + i, "/a", &format!("u{}", i % 4), i as f64)); + } + exact.observe(&point(61_000, "/a", "u0", 1.0)); + assert!(exact.due(64_000).is_empty()); + let out = exact.due(65_000); + let get = |metric: &str, extra: (&str, &str)| { + out.iter() + .find(|e| { + e.metric_name == metric + && e.labels + .iter() + .any(|kv| kv.key == extra.0 && kv.value == extra.1) + }) + .map(|e| e.value) + }; + assert_eq!(get("asap.quantile", ("quantile", "0.5")), Some(50.0)); + assert_eq!( + get("asap.distinct", ("attribute.key", "user.id")), + Some(4.0) + ); + assert_eq!(get("asap.topk", ("rank", "1")), Some(25.0)); + assert!(out.iter().all(|e| e.window_end_ms == 60_000)); + // A late point for the emitted window is dropped. + exact.observe(&point(2_000, "/a", "u9", 1.0)); + assert!(exact + .due(125_000) + .iter() + .all(|e| e.window_end_ms == 120_000)); + } +} diff --git a/asap-precompute-rs/src/otap/mod.rs b/asap-precompute-rs/src/otap/mod.rs index c2cef72..141dcd2 100644 --- a/asap-precompute-rs/src/otap/mod.rs +++ b/asap-precompute-rs/src/otap/mod.rs @@ -10,6 +10,8 @@ mod schema; pub mod config; +#[cfg(feature = "otap-engine")] +pub mod answer_nodes; #[cfg(feature = "otap-engine")] pub mod codec; #[cfg(feature = "otap-engine")] From c936e123705b7400eb137068bab55b9451695a4d Mon Sep 17 00:00:00 2001 From: Zeying Zhu <50204836+zzylol@users.noreply.github.com> Date: Mon, 12 Oct 2026 18:32:40 +0000 Subject: [PATCH 4/6] Close tumbling windows by event time; expose aggregate_by A tumbling window used to close only on a timer started at the first message, so up to one window of later points landed in the previous window. Now a batch whose event time passes the open window's end closes it first, and the idle timer runs every second. aggregate_by selects the labels that identify a series (empty keeps every label). --- asap-precompute-rs/src/otap/config.rs | 5 +++- asap-precompute-rs/src/otap/processor.rs | 31 +++++++++++++++++++++++- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/asap-precompute-rs/src/otap/config.rs b/asap-precompute-rs/src/otap/config.rs index a48497a..d268d2e 100644 --- a/asap-precompute-rs/src/otap/config.rs +++ b/asap-precompute-rs/src/otap/config.rs @@ -114,6 +114,8 @@ pub struct PluginConfig { /// Quantiles emitted per series when `transmit_sketch = false` and the /// sketch is a quantile sketch (DDSketch / KLL). Ignored otherwise. pub quantiles: Vec, + /// Labels that identify a series; empty keys series by every label. + pub aggregate_by: Vec, } impl Default for PluginConfig { @@ -131,6 +133,7 @@ impl Default for PluginConfig { emit_window_stats: false, transmit_sketch: true, quantiles: Vec::new(), + aggregate_by: Vec::new(), } } } @@ -184,7 +187,7 @@ pub fn resolve(config: &PluginConfig) -> Result<(PrecomputeConfig, SketchDispatc ..Default::default() }, matchers: Vec::new(), - aggregate_by: Vec::new(), + aggregate_by: config.aggregate_by.clone(), transmit_sketch: config.transmit_sketch, quantiles: config.quantiles.clone(), sketch_params: config.sketch_params.clone(), diff --git a/asap-precompute-rs/src/otap/processor.rs b/asap-precompute-rs/src/otap/processor.rs index 5b899a9..52a107f 100644 --- a/asap-precompute-rs/src/otap/processor.rs +++ b/asap-precompute-rs/src/otap/processor.rs @@ -93,6 +93,7 @@ use otel_arrow_dfe_otap::OTAP_PROCESSOR_FACTORIES; use crate::config::PrecomputeConfigSet; use crate::envelope::SketchEnvelope; +use crate::observation::ObservationValueKind; use crate::otap::config::resolve as resolve_plugin_config; use crate::otap::PluginConfig; use crate::precompute::{Precompute, PrecomputeError, PrecomputeImpl}; @@ -151,6 +152,9 @@ pub struct AsapSketchesUserConfig { /// Quantiles emitted by DDSketch/KLL in estimate mode. #[serde(default)] pub quantiles: Vec, + /// Labels that identify a series; empty keys series by every label. + #[serde(default)] + pub aggregate_by: Vec, } impl AsapSketchesUserConfig { @@ -188,6 +192,7 @@ impl AsapSketchesUserConfig { emit_window_stats: self.emit_window_stats, transmit_sketch: self.transmit_sketch, quantiles: self.quantiles, + aggregate_by: self.aggregate_by, }; // Bounce the config through the resolver so config mistakes // (unsupported sketch_type, zero window) surface here rather @@ -203,9 +208,11 @@ const fn default_transmit_sketch() -> bool { true } +/// Tumbling windows close within a second of their end even when idle; +/// sliding windows emit once per slide. fn timer_period(config: &PluginConfig) -> Duration { if config.window_slide.is_zero() { - config.window_size + config.window_size.min(Duration::from_secs(1)) } else { config.window_slide } @@ -423,6 +430,18 @@ impl local::Processor for AsapSketchesProcessor { )) .await; } + // A tumbling window closes as soon as a point's event time + // passes its end, so no point lands in an earlier window. + if self.plugin_config.window_slide.is_zero() { + if let Some(first) = outcome + .observations + .iter() + .find(|o| o.value.kind != ObservationValueKind::Envelope) + { + let envs = self.precompute.tick(first.timestamp_ms); + self.emit_envelopes(&envs, effect_handler).await?; + } + } for obs in &outcome.observations { // Note: `observe` here handles both "genuine OTLP // metric" and "sketch shipped as binary inside an @@ -555,6 +574,16 @@ mod tests { assert!(parse("60s", "5s", true).is_err()); assert!(parse("5s", "5s", true).is_ok()); assert!(parse("5s", "0s", false).is_err()); + let tumbling = serde_json::from_value::( + serde_json::json!({"sketch_type":"kll","window_size":"60s","output_metric_name":"t"}), + ) + .unwrap() + .into_plugin_config() + .unwrap(); + assert_eq!( + super::timer_period(&tumbling), + std::time::Duration::from_secs(1) + ); assert!(parse("3ms", "1.5ms", false).is_err()); } // Unit tests confined to config-shape validation — full lifecycle From 501a8174ad41b7314b0ea8cc91d4c41d98050b7b Mon Sep 17 00:00:00 2001 From: Zeying Zhu <50204836+zzylol@users.noreply.github.com> Date: Mon, 12 Oct 2026 18:37:28 +0000 Subject: [PATCH 5/6] Run every pipeline stage as a pinned collector container deploy/telemetry-functions: generators, branches (sketch + raw copy via the official fanout), window merge, detector and the exact baseline each run asap-collector in their own container on their own CPU, over OTLP/HTTP. Prometheus receives sketch and exact answers over OTLP and scrapes every engine's own telemetry (CPU time, RSS, payload bytes per stage); Grafana overlays the sketch answer and its bound on the exact answer. Answers are written as plain OTLP gauges with labels on each point. --- asap-precompute-rs/src/otap/answer_nodes.rs | 71 ++- deploy/collector.Dockerfile | 4 + deploy/telemetry-functions/README.md | 38 ++ deploy/telemetry-functions/compose.yaml | 71 +++ .../grafana/dashboards/latency.json | 428 ++++++++++++++++++ .../grafana/provisioning/dashboards/asap.yaml | 3 + .../provisioning/datasources/datasources.yaml | 4 + .../telemetry-functions/pipelines/branch.yaml | 56 +++ .../pipelines/detector.yaml | 34 ++ .../telemetry-functions/pipelines/exact.yaml | 34 ++ .../pipelines/generator-a.yaml | 34 ++ .../pipelines/generator-b.yaml | 34 ++ .../telemetry-functions/pipelines/merge.yaml | 33 ++ deploy/telemetry-functions/prometheus.yml | 16 + deploy/telemetry-functions/run.sh | 17 + docs/images/latency-grafana.png | Bin 0 -> 105356 bytes 16 files changed, 863 insertions(+), 14 deletions(-) create mode 100644 deploy/collector.Dockerfile create mode 100644 deploy/telemetry-functions/README.md create mode 100644 deploy/telemetry-functions/compose.yaml create mode 100644 deploy/telemetry-functions/grafana/dashboards/latency.json create mode 100644 deploy/telemetry-functions/grafana/provisioning/dashboards/asap.yaml create mode 100644 deploy/telemetry-functions/grafana/provisioning/datasources/datasources.yaml create mode 100644 deploy/telemetry-functions/pipelines/branch.yaml create mode 100644 deploy/telemetry-functions/pipelines/detector.yaml create mode 100644 deploy/telemetry-functions/pipelines/exact.yaml create mode 100644 deploy/telemetry-functions/pipelines/generator-a.yaml create mode 100644 deploy/telemetry-functions/pipelines/generator-b.yaml create mode 100644 deploy/telemetry-functions/pipelines/merge.yaml create mode 100644 deploy/telemetry-functions/prometheus.yml create mode 100755 deploy/telemetry-functions/run.sh create mode 100644 docs/images/latency-grafana.png diff --git a/asap-precompute-rs/src/otap/answer_nodes.rs b/asap-precompute-rs/src/otap/answer_nodes.rs index aa6006a..c7a08c3 100644 --- a/asap-precompute-rs/src/otap/answer_nodes.rs +++ b/asap-precompute-rs/src/otap/answer_nodes.rs @@ -37,7 +37,17 @@ use otel_arrow_dfe_otap::pdata::OtapPdata; use otel_arrow_dfe_otap::OTAP_PROCESSOR_FACTORIES; use serde::Deserialize; -use super::codec::{decode_pdata_to_observations, OtapSketchEncoder}; +use otel_arrow_dfe_pdata::proto::opentelemetry::common::v1::{ + AnyValue, InstrumentationScope, KeyValue as OtlpKeyValue, +}; +use otel_arrow_dfe_pdata::proto::opentelemetry::metrics::v1::{ + Gauge, Metric, MetricsData, NumberDataPoint, ResourceMetrics, ScopeMetrics, +}; +use otel_arrow_dfe_pdata::proto::opentelemetry::resource::v1::Resource; +use otel_arrow_dfe_pdata::proto::OtlpProtoMessage; +use otel_arrow_dfe_pdata::OtapPayload; + +use super::codec::decode_pdata_to_observations; use crate::answers::{exact_answers, sketch_answers, ExactWindow}; use crate::envelope::SketchEnvelope; use crate::observation::{KeyValue, Observation, ObservationValueKind}; @@ -103,15 +113,55 @@ fn now_ms() -> u64 { .unwrap_or(0) } +/// Plain OTLP gauges with every label on the data point, so backends such +/// as Prometheus keep them as series labels. +fn gauges_pdata(envs: &[SketchEnvelope]) -> Result { + let mut by_metric: BTreeMap<&str, Vec> = BTreeMap::new(); + for e in envs { + let attributes: Vec = e + .labels + .iter() + .map(|kv| OtlpKeyValue::new(kv.key.clone(), AnyValue::new_string(kv.value.clone()))) + .collect(); + by_metric.entry(&e.metric_name).or_default().push( + NumberDataPoint::build() + .start_time_unix_nano(e.window_start_ms * 1_000_000) + .time_unix_nano(e.window_end_ms * 1_000_000) + .value_double(e.value) + .attributes(attributes) + .finish(), + ); + } + let metrics = by_metric + .into_iter() + .map(|(name, points)| { + Metric::build() + .name(name.to_string()) + .data_gauge(Gauge::new(points)) + .finish() + }) + .collect::>(); + let data = MetricsData::new(vec![ResourceMetrics::new( + Resource::build().finish(), + vec![ScopeMetrics::new( + InstrumentationScope::build().name("asap.answers").finish(), + metrics, + )], + )]); + let payload: OtapPayload = OtlpProtoMessage::Metrics(data) + .try_into() + .map_err(|e| format!("{e:?}"))?; + Ok(OtapPdata::new_todo_context(payload)) +} + async fn emit( - encoder: &mut OtapSketchEncoder, envs: &[SketchEnvelope], effects: &mut local::EffectHandler, ) -> Result<(), Error> { if envs.is_empty() { return Ok(()); } - match encoder.encode(envs) { + match gauges_pdata(envs) { Ok(pdata) => effects.send_message_with_source_node(pdata).await?, Err(e) => effects.info(&format!("answers: encode failed: {e}")).await, } @@ -135,7 +185,6 @@ async fn observations( struct SketchDetector { config: DetectorConfig, - encoder: OtapSketchEncoder, } #[async_trait(?Send)] @@ -158,7 +207,7 @@ impl local::Processor for SketchDetector { Err(e) => effects.info(&format!("sketch_detector: {e}")).await, } } - emit(&mut self.encoder, &out, effects).await + emit(&out, effects).await } } @@ -173,7 +222,6 @@ struct ExactAnswers { /// Open windows by window end. windows: BTreeMap, emitted_until: u64, - encoder: OtapSketchEncoder, timer_started: bool, } @@ -281,11 +329,11 @@ impl local::Processor for ExactAnswers { } Message::Control(NodeControlMsg::TimerTick { .. }) => { let out = self.due(now_ms()); - emit(&mut self.encoder, &out, effects).await + emit(&out, effects).await } Message::Control(NodeControlMsg::Shutdown { .. }) => { let out = self.due(u64::MAX); - emit(&mut self.encoder, &out, effects).await + emit(&out, effects).await } _ => Ok(()), } @@ -310,10 +358,7 @@ static SKETCH_DETECTOR: ProcessorFactory = ProcessorFactory { let config: DetectorConfig = serde_json::from_value(node_config.config.clone()).map_err(invalid)?; Ok(ProcessorWrapper::local( - SketchDetector { - config, - encoder: OtapSketchEncoder::default(), - }, + SketchDetector { config }, node, node_config, processor_config, @@ -339,7 +384,6 @@ static EXACT_ANSWERS: ProcessorFactory = ProcessorFactory { config, windows: BTreeMap::new(), emitted_until: 0, - encoder: OtapSketchEncoder::default(), timer_started: false, }, node, @@ -378,7 +422,6 @@ mod tests { .unwrap(), windows: BTreeMap::new(), emitted_until: 0, - encoder: OtapSketchEncoder::default(), timer_started: false, }; for i in 0..100u64 { diff --git a/deploy/collector.Dockerfile b/deploy/collector.Dockerfile new file mode 100644 index 0000000..e01b4e4 --- /dev/null +++ b/deploy/collector.Dockerfile @@ -0,0 +1,4 @@ +FROM ubuntu:24.04 +RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates && rm -rf /var/lib/apt/lists/* +COPY asap-collector /usr/local/bin/asap-collector +ENTRYPOINT ["/usr/local/bin/asap-collector"] diff --git a/deploy/telemetry-functions/README.md b/deploy/telemetry-functions/README.md new file mode 100644 index 0000000..1af874a --- /dev/null +++ b/deploy/telemetry-functions/README.md @@ -0,0 +1,38 @@ +# Telemetry functions on real collector containers + +Every stage is the official OTAP dataflow engine (`asap-collector`: the +upstream controller with the ASAP nodes linked in), one container per stage, +each pinned to its own CPU, connected by OTLP/HTTP over the compose network. + +```text +generator-a -> branch-a --sketch--> merge -> detector --> Prometheus -> Grafana +generator-b -> branch-b --sketch--/ + \--raw copy (fanout)--> exact ------/ +``` + +| Stage | Pipeline | What it does | +| --- | --- | --- | +| generator-a/b | `pipelines/generator-*.yaml` | `scenario_generator`: seeded latency points over 20 routes (temporary stand-in for the official traffic generator) | +| branch-a/b | `pipelines/branch.yaml` | `fanout` copies the input: `asap_sketches` builds one KLL per route per 1-minute window; the raw copy goes to `exact` | +| merge | `pipelines/merge.yaml` | `window_merge`: merges both sources' sketches per window, 5 s after the window ends | +| detector | `pipelines/detector.yaml` | `sketch_detector`: p50/p99 per route with lower/upper bounds | +| exact | `pipelines/exact.yaml` | `exact_answers`: exact p50/p99 per route from every raw point (ground truth and exact baseline) | + +Prometheus receives both answer streams over OTLP (`asap_quantile`, +`asap_points`, labelled `method="sketch"` or `"exact"`) and scrapes each +collector's engine telemetry (`/api/v1/metrics?format=prometheus`): CPU time +per pipeline (`cpu_time_total`), resident memory (`memory_rss`) and payload +bytes per receiver, labelled with the stage. + +## Run + +Linux, Docker with Compose, 7 free CPUs (0-6 by default; `CPU_BRANCH_A=10` +etc. move a stage). From the repository root: + +```sh +deploy/telemetry-functions/run.sh # DOCKER_COMMAND="sudo docker" if needed +``` + +Open . The first answers appear about +65 seconds after the first full minute. Stop with +`docker compose -f deploy/telemetry-functions/compose.yaml down`. diff --git a/deploy/telemetry-functions/compose.yaml b/deploy/telemetry-functions/compose.yaml new file mode 100644 index 0000000..672aa11 --- /dev/null +++ b/deploy/telemetry-functions/compose.yaml @@ -0,0 +1,71 @@ +# Every pipeline stage is the official OTAP engine (asap-collector) in its +# own container, pinned to one CPU, talking OTLP/HTTP over the compose +# network. Prometheus, Grafana and Loki are the observers. +# CPU_* variables move a stage to another core. +x-collector: &collector + image: asap-collector:local + restart: "no" + +services: + generator-a: + <<: *collector + command: ["--config", "/pipelines/generator-a.yaml"] + volumes: ["./pipelines:/pipelines:ro"] + cpuset: "${CPU_GENERATOR_A:-0}" + depends_on: [branch-a] + generator-b: + <<: *collector + command: ["--config", "/pipelines/generator-b.yaml"] + volumes: ["./pipelines:/pipelines:ro"] + cpuset: "${CPU_GENERATOR_B:-1}" + depends_on: [branch-b] + branch-a: + <<: *collector + command: ["--config", "/pipelines/branch.yaml"] + volumes: ["./pipelines:/pipelines:ro"] + cpuset: "${CPU_BRANCH_A:-2}" + depends_on: [merge, exact] + branch-b: + <<: *collector + command: ["--config", "/pipelines/branch.yaml"] + volumes: ["./pipelines:/pipelines:ro"] + cpuset: "${CPU_BRANCH_B:-3}" + depends_on: [merge, exact] + merge: + <<: *collector + command: ["--config", "/pipelines/merge.yaml"] + volumes: ["./pipelines:/pipelines:ro"] + cpuset: "${CPU_MERGE:-4}" + depends_on: [detector] + detector: + <<: *collector + command: ["--config", "/pipelines/detector.yaml"] + volumes: ["./pipelines:/pipelines:ro"] + cpuset: "${CPU_DETECTOR:-5}" + depends_on: [prometheus] + exact: + <<: *collector + command: ["--config", "/pipelines/exact.yaml"] + volumes: ["./pipelines:/pipelines:ro"] + cpuset: "${CPU_EXACT:-6}" + depends_on: [prometheus] + + prometheus: + image: prom/prometheus:v3.5.0 + command: + - --config.file=/etc/prometheus/prometheus.yml + - --web.enable-otlp-receiver + ports: ["127.0.0.1:${PROMETHEUS_PORT:-9090}:9090"] + volumes: ["./prometheus.yml:/etc/prometheus/prometheus.yml:ro"] + grafana: + image: grafana/grafana:12.1.0 + ports: ["127.0.0.1:${GRAFANA_PORT:-3000}:3000"] + environment: + GF_AUTH_ANONYMOUS_ENABLED: "true" + GF_AUTH_ANONYMOUS_ORG_ROLE: Viewer + volumes: + - ./grafana/provisioning:/etc/grafana/provisioning:ro + - ./grafana/dashboards:/var/lib/grafana/dashboards:ro + loki: + image: grafana/loki:3.5.0 + command: ["-config.file=/etc/loki/local-config.yaml"] diff --git a/deploy/telemetry-functions/grafana/dashboards/latency.json b/deploy/telemetry-functions/grafana/dashboards/latency.json new file mode 100644 index 0000000..cd493ed --- /dev/null +++ b/deploy/telemetry-functions/grafana/dashboards/latency.json @@ -0,0 +1,428 @@ +{ + "uid": "asap-latency", + "title": "Latency quantiles: sketch vs exact (real collector containers)", + "schemaVersion": 39, + "version": 1, + "refresh": "10s", + "time": { + "from": "now-30m", + "to": "now" + }, + "templating": { + "list": [ + { + "name": "route", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "asap-prometheus" + }, + "query": { + "query": "label_values(asap_quantile, http_route)", + "refId": "route" + }, + "refresh": 2, + "current": { + "text": "http.route-0", + "value": "http.route-0" + } + } + ] + }, + "panels": [ + { + "id": 1, + "type": "timeseries", + "title": "p99 latency for $route: sketch with bound vs exact", + "datasource": { + "type": "prometheus", + "uid": "asap-prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "expr": "asap_quantile{quantile=\"0.99\",http_route=\"$route\",method=\"sketch\",bound=\"estimate\"}", + "legendFormat": "sketch" + }, + { + "refId": "B", + "expr": "asap_quantile{quantile=\"0.99\",http_route=\"$route\",method=\"sketch\",bound=\"lower\"}", + "legendFormat": "sketch lower" + }, + { + "refId": "C", + "expr": "asap_quantile{quantile=\"0.99\",http_route=\"$route\",method=\"sketch\",bound=\"upper\"}", + "legendFormat": "sketch upper" + }, + { + "refId": "D", + "expr": "asap_quantile{quantile=\"0.99\",http_route=\"$route\",method=\"exact\"}", + "legendFormat": "exact" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "spanNulls": true + }, + "unit": "ms" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "sketch upper" + }, + "properties": [ + { + "id": "custom.fillBelowTo", + "value": "sketch lower" + }, + { + "id": "custom.lineWidth", + "value": 0 + }, + { + "id": "custom.fillOpacity", + "value": 20 + }, + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "blue" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "sketch lower" + }, + "properties": [ + { + "id": "custom.lineWidth", + "value": 0 + }, + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "blue" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "sketch" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "blue" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "exact" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "orange" + } + }, + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 6, + 4 + ] + } + } + ] + } + ] + } + }, + { + "id": 2, + "type": "timeseries", + "title": "p50 latency for $route: sketch with bound vs exact", + "datasource": { + "type": "prometheus", + "uid": "asap-prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "expr": "asap_quantile{quantile=\"0.5\",http_route=\"$route\",method=\"sketch\",bound=\"estimate\"}", + "legendFormat": "sketch" + }, + { + "refId": "B", + "expr": "asap_quantile{quantile=\"0.5\",http_route=\"$route\",method=\"sketch\",bound=\"lower\"}", + "legendFormat": "sketch lower" + }, + { + "refId": "C", + "expr": "asap_quantile{quantile=\"0.5\",http_route=\"$route\",method=\"sketch\",bound=\"upper\"}", + "legendFormat": "sketch upper" + }, + { + "refId": "D", + "expr": "asap_quantile{quantile=\"0.5\",http_route=\"$route\",method=\"exact\"}", + "legendFormat": "exact" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "spanNulls": true + }, + "unit": "ms" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "sketch upper" + }, + "properties": [ + { + "id": "custom.fillBelowTo", + "value": "sketch lower" + }, + { + "id": "custom.lineWidth", + "value": 0 + }, + { + "id": "custom.fillOpacity", + "value": 20 + }, + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "blue" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "sketch lower" + }, + "properties": [ + { + "id": "custom.lineWidth", + "value": 0 + }, + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "blue" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "sketch" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "blue" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "exact" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "orange" + } + }, + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 6, + 4 + ] + } + } + ] + } + ] + } + }, + { + "id": 3, + "type": "timeseries", + "title": "Relative error of the sketch p99 vs exact (routes with > 20,000 points per window)", + "datasource": { + "type": "prometheus", + "uid": "asap-prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "abs(asap_quantile{quantile=\"0.99\",method=\"sketch\",bound=\"estimate\"} - on(http_route) asap_quantile{quantile=\"0.99\",method=\"exact\"}) / on(http_route) asap_quantile{quantile=\"0.99\",method=\"exact\"} and on(http_route) asap_points{method=\"exact\"} > 20000", + "legendFormat": "{{http_route}}" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "spanNulls": true + }, + "unit": "percentunit" + }, + "overrides": [] + } + }, + { + "id": 4, + "type": "timeseries", + "title": "CPU cores per pinned stage", + "datasource": { + "type": "prometheus", + "uid": "asap-prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (stage) (rate(cpu_time_total{job=\"collectors\",otel_scope_pipeline_id=\"main\"}[1m]))", + "legendFormat": "{{stage}}" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "spanNulls": true + }, + "unit": "short" + }, + "overrides": [] + } + }, + { + "id": 5, + "type": "timeseries", + "title": "Bytes received per stage, log scale (sketches at merge, raw points at exact)", + "datasource": { + "type": "prometheus", + "uid": "asap-prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (stage) (rate(payload_size_bytes_total{job=\"collectors\",otel_scope_node_type=\"receiver\"}[1m])) * 8", + "legendFormat": "{{stage}}" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "spanNulls": true, + "scaleDistribution": { + "type": "log", + "log": 10 + } + }, + "unit": "bps" + }, + "overrides": [] + } + }, + { + "id": 6, + "type": "timeseries", + "title": "Resident memory per stage", + "datasource": { + "type": "prometheus", + "uid": "asap-prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "targets": [ + { + "refId": "A", + "expr": "max by (stage) (memory_rss{job=\"collectors\"})", + "legendFormat": "{{stage}}" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "spanNulls": true + }, + "unit": "bytes" + }, + "overrides": [] + } + } + ] +} diff --git a/deploy/telemetry-functions/grafana/provisioning/dashboards/asap.yaml b/deploy/telemetry-functions/grafana/provisioning/dashboards/asap.yaml new file mode 100644 index 0000000..bf894a9 --- /dev/null +++ b/deploy/telemetry-functions/grafana/provisioning/dashboards/asap.yaml @@ -0,0 +1,3 @@ +apiVersion: 1 +providers: + - {name: ASAP telemetry functions, folder: ASAP, type: file, options: {path: /var/lib/grafana/dashboards}} diff --git a/deploy/telemetry-functions/grafana/provisioning/datasources/datasources.yaml b/deploy/telemetry-functions/grafana/provisioning/datasources/datasources.yaml new file mode 100644 index 0000000..06fe817 --- /dev/null +++ b/deploy/telemetry-functions/grafana/provisioning/datasources/datasources.yaml @@ -0,0 +1,4 @@ +apiVersion: 1 +datasources: + - {name: Prometheus, uid: asap-prometheus, type: prometheus, access: proxy, url: http://prometheus:9090, isDefault: true, editable: false} + - {name: Loki, uid: asap-loki, type: loki, access: proxy, url: http://loki:3100, editable: false} diff --git a/deploy/telemetry-functions/pipelines/branch.yaml b/deploy/telemetry-functions/pipelines/branch.yaml new file mode 100644 index 0000000..4166399 --- /dev/null +++ b/deploy/telemetry-functions/pipelines/branch.yaml @@ -0,0 +1,56 @@ +# Stage 1 at each source: one KLL per route per 1-minute window goes to +# merge; the raw points are copied to the exact baseline. +version: otel_dataflow/v1 +engine: + http_admin: + bind_address: 0.0.0.0:8085 +policies: + resources: + core_allocation: + type: core_count + count: 1 +groups: + default: + pipelines: + main: + nodes: + receiver: + type: receiver:otlp + config: + protocols: + http: {listening_addr: "0.0.0.0:4318", wait_for_result: false} + fanout: + type: processor:fanout + outputs: [sketch, raw] + config: + destinations: + - {port: sketch} + - {port: raw} + sketch: + type: urn:asap:processor:asap_sketches + config: + sketch_type: kll + window_size: 60s + output_metric_name: http.server.request.duration + aggregate_by: [http.route] + omit_resource_attrs: true + sketch_params: {k: 200} + to_merge: + type: exporter:otlp_http + config: + endpoint: http://merge:4318 + client_pool_size: 1 + max_in_flight: 1 + http: {timeout: 30s, compression: none} + to_exact: + type: exporter:otlp_http + config: + endpoint: http://exact:4318 + client_pool_size: 1 + max_in_flight: 1 + http: {timeout: 30s, compression: none} + connections: + - {from: receiver, to: fanout} + - {from: 'fanout["sketch"]', to: sketch} + - {from: sketch, to: to_merge} + - {from: 'fanout["raw"]', to: to_exact} diff --git a/deploy/telemetry-functions/pipelines/detector.yaml b/deploy/telemetry-functions/pipelines/detector.yaml new file mode 100644 index 0000000..8c70015 --- /dev/null +++ b/deploy/telemetry-functions/pipelines/detector.yaml @@ -0,0 +1,34 @@ +# Answers (with bounds) from each merged window, written to Prometheus. +version: otel_dataflow/v1 +engine: + http_admin: + bind_address: 0.0.0.0:8085 +policies: + resources: + core_allocation: + type: core_count + count: 1 +groups: + default: + pipelines: + main: + nodes: + receiver: + type: receiver:otlp + config: + protocols: + http: {listening_addr: "0.0.0.0:4318", wait_for_result: false} + detector: + type: urn:asap:processor:sketch_detector + config: {quantiles: [0.5, 0.99]} + to_prometheus: + type: exporter:otlp_http + config: + endpoint: http://prometheus:9090 + metrics_endpoint: http://prometheus:9090/api/v1/otlp/v1/metrics + client_pool_size: 1 + max_in_flight: 1 + http: {timeout: 30s, compression: none} + connections: + - {from: receiver, to: detector} + - {from: detector, to: to_prometheus} diff --git a/deploy/telemetry-functions/pipelines/exact.yaml b/deploy/telemetry-functions/pipelines/exact.yaml new file mode 100644 index 0000000..1a9a648 --- /dev/null +++ b/deploy/telemetry-functions/pipelines/exact.yaml @@ -0,0 +1,34 @@ +# Exact baseline and ground truth: every raw point, exact answers per window. +version: otel_dataflow/v1 +engine: + http_admin: + bind_address: 0.0.0.0:8085 +policies: + resources: + core_allocation: + type: core_count + count: 1 +groups: + default: + pipelines: + main: + nodes: + receiver: + type: receiver:otlp + config: + protocols: + http: {listening_addr: "0.0.0.0:4318", wait_for_result: false} + exact: + type: urn:asap:processor:exact_answers + config: {window: 60s, group_by: [http.route], quantiles: [0.5, 0.99]} + to_prometheus: + type: exporter:otlp_http + config: + endpoint: http://prometheus:9090 + metrics_endpoint: http://prometheus:9090/api/v1/otlp/v1/metrics + client_pool_size: 1 + max_in_flight: 1 + http: {timeout: 30s, compression: none} + connections: + - {from: receiver, to: exact} + - {from: exact, to: to_prometheus} diff --git a/deploy/telemetry-functions/pipelines/generator-a.yaml b/deploy/telemetry-functions/pipelines/generator-a.yaml new file mode 100644 index 0000000..bfc3268 --- /dev/null +++ b/deploy/telemetry-functions/pipelines/generator-a.yaml @@ -0,0 +1,34 @@ +# Source a: seeded latency traffic over 20 routes. +version: otel_dataflow/v1 +engine: + http_admin: + bind_address: 0.0.0.0:8085 +policies: + resources: + core_allocation: + type: core_count + count: 1 +groups: + default: + pipelines: + main: + nodes: + generator: + type: urn:asap:receiver:scenario_generator + config: + seed: 1 + signals_per_second: 20000 + metric_name: http.server.request.duration + resource_attributes: {service.name: checkout, source: "a"} + attributes: + - {key: http.route, cardinality: 20, zipf: 1.1} + value: {lognormal: {median: 50.0, sigma: 0.6}} + export: + type: exporter:otlp_http + config: + endpoint: http://branch-a:4318 + client_pool_size: 1 + max_in_flight: 1 + http: {timeout: 30s, compression: none} + connections: + - {from: generator, to: export} diff --git a/deploy/telemetry-functions/pipelines/generator-b.yaml b/deploy/telemetry-functions/pipelines/generator-b.yaml new file mode 100644 index 0000000..1144ab5 --- /dev/null +++ b/deploy/telemetry-functions/pipelines/generator-b.yaml @@ -0,0 +1,34 @@ +# Source b: seeded latency traffic over 20 routes. +version: otel_dataflow/v1 +engine: + http_admin: + bind_address: 0.0.0.0:8085 +policies: + resources: + core_allocation: + type: core_count + count: 1 +groups: + default: + pipelines: + main: + nodes: + generator: + type: urn:asap:receiver:scenario_generator + config: + seed: 2 + signals_per_second: 20000 + metric_name: http.server.request.duration + resource_attributes: {service.name: checkout, source: "b"} + attributes: + - {key: http.route, cardinality: 20, zipf: 1.1} + value: {lognormal: {median: 50.0, sigma: 0.6}} + export: + type: exporter:otlp_http + config: + endpoint: http://branch-b:4318 + client_pool_size: 1 + max_in_flight: 1 + http: {timeout: 30s, compression: none} + connections: + - {from: generator, to: export} diff --git a/deploy/telemetry-functions/pipelines/merge.yaml b/deploy/telemetry-functions/pipelines/merge.yaml new file mode 100644 index 0000000..cb392f2 --- /dev/null +++ b/deploy/telemetry-functions/pipelines/merge.yaml @@ -0,0 +1,33 @@ +# Stage 2: merge every source's sketch for a window, 5 s after it ends. +version: otel_dataflow/v1 +engine: + http_admin: + bind_address: 0.0.0.0:8085 +policies: + resources: + core_allocation: + type: core_count + count: 1 +groups: + default: + pipelines: + main: + nodes: + receiver: + type: receiver:otlp + config: + protocols: + http: {listening_addr: "0.0.0.0:4318", wait_for_result: false} + merge: + type: urn:asap:processor:window_merge + config: {grace: 5s} + to_detector: + type: exporter:otlp_http + config: + endpoint: http://detector:4318 + client_pool_size: 1 + max_in_flight: 1 + http: {timeout: 30s, compression: none} + connections: + - {from: receiver, to: merge} + - {from: merge, to: to_detector} diff --git a/deploy/telemetry-functions/prometheus.yml b/deploy/telemetry-functions/prometheus.yml new file mode 100644 index 0000000..2325b68 --- /dev/null +++ b/deploy/telemetry-functions/prometheus.yml @@ -0,0 +1,16 @@ +global: + scrape_interval: 5s +scrape_configs: + # Each collector's own engine telemetry: CPU time per pipeline, RSS, and + # per-node message and payload counters. stage = the compose service. + - job_name: collectors + metrics_path: /api/v1/metrics + params: {format: [prometheus]} + static_configs: + - targets: [generator-a:8085, generator-b:8085, branch-a:8085, branch-b:8085, + merge:8085, detector:8085, exact:8085] + relabel_configs: + - source_labels: [__address__] + regex: (.+):8085 + target_label: stage +# Answers (sketch and exact) arrive through the OTLP receiver. diff --git a/deploy/telemetry-functions/run.sh b/deploy/telemetry-functions/run.sh new file mode 100755 index 0000000..f4cb2fc --- /dev/null +++ b/deploy/telemetry-functions/run.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# Build asap-collector, package it, and start every stage plus the observers. +# DOCKER_COMMAND="sudo -n docker" when Docker needs sudo. Stop with: +# $DOCKER_COMMAND compose -f deploy/telemetry-functions/compose.yaml down +set -euo pipefail +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) +cd "$repo_root" +target=${CARGO_TARGET_DIR:-"$repo_root/asap-precompute-rs/target"} +read -r -a docker <<< "${DOCKER_COMMAND:-docker}" +cargo build --manifest-path asap-precompute-rs/Cargo.toml --release --features collector --bin asap-collector +context=$(mktemp -d) +trap 'rm -rf "$context"' EXIT +cp "$target/release/asap-collector" "$context/" +strip --strip-debug "$context/asap-collector" +"${docker[@]}" build -q -f deploy/collector.Dockerfile -t asap-collector:local "$context" +"${docker[@]}" compose -f deploy/telemetry-functions/compose.yaml up -d +echo "Grafana: http://localhost:${GRAFANA_PORT:-3000}/d/asap-latency" diff --git a/docs/images/latency-grafana.png b/docs/images/latency-grafana.png new file mode 100644 index 0000000000000000000000000000000000000000..78e2026d43709db42500324973a82cec5f325a94 GIT binary patch literal 105356 zcmd43cQ{<{_cl5ri4-A;9zhV%iQb7Gz4tCgHR>o~^7(##*YA`+-uJxkxvn#RZT9SEKV>~@-RoZWGaoh874O}7bO!_i-BWrgrwsz# zd;tR8AiZ@B_|L6mrB5KxeUOrzw60g$&OCu137D>Rzd#MPxxlF-c&q32+gn|qBj0sB zOuojTa-V=9{OhfEAA&fhb?zr$V@rr+p}ua@m0zO1p*nNu-{Y{5B1rEN$A7N{#T?&z zFlV;MrNO0fc_Kb2g29B{0^SFFTJ=S6eZgIYK4`^t0t5v zap$9PY1GG$>Z(o5xEJ`Y|NgzXzP`S*%F!pMpNoGDf8G0Cx_XbmN9gE6&m?Dxy zVSArmSiOe9h>3`pa1#*}_Vz-nRI6A=N9TRAJ^v#EgQSfOd%V>cg&d;Ww*_%;A|xV; z3J-rszP7aVL`p668kOzU*J%>(kIH6|q|JLJA413JmHIN`@%@`~_(dam6b;$|m^p<{kN$9r+u5qj2>5Rt@fu`Fw4_s_b%h{@6x@z@+e$R7w25fL@~?Ca}OOPwgafG8;_ zG=NGrDz*d~5EG4Fa!O|5&&;ZPCyzT+*7EZ6Z{2m8t@ZDIm?!lZJHSrUP;EafcpfHm zw%_T|YrwH4Yb8XR2%2~mJ)U;hQLSG}ZN&M|^h1h}(|rAb6CNI(5JXN5>of9^L=k72 zLD}3aEM8OPF<@n>O5c;?V1ldD?LO9I!$Ru><_y0gkL@H{|LU_DygAwkL>h-^LyczFI6b_FVbpVWwZG4L&=ee*8d-ft7WCDQj`PKa~~j zgwCZ(t#khLX^zd3(`B}Ku71>sr(&4b)MIPxw$U-0UJ<&FZ&32eZ&W#GG0|OyO){yT!-b_-0vZ0?&SE3i2PnWB< zC(HbK*W55le|K`Yu%hDl5IySb>@@QHu$^jA6@!gN{5a|UxH;N#u~QX&adCkp^oi6g zk-c2l8%DUojm#PkcSiEU2-!z|4@{rp~Gah)KH#c08Nf$r1em7N`NyS+oRMut>= z$ijXCgrQ}X`wyccQTRKaO2(lB=lPqwJ~gk_YwH$#ULC9*C2^U(RBeL&SwfBGt6gra zocQ_sFZ$b6lPTxaB}FGCO^%f@52Oj`k%Q{*O~Nha5q9ea7HW^LFr8Lb4_xp05B}^I zerp{|yR#k~@m(?VUU{m2{O8PZTRlCAHH zqy|+|LC_51P^}y0*zlZr{C%S7B1rQVlkRBhNy!t*x`8X~e3pP7ra=0@-A>cx8ae`9YtBF5_rz z!Pd^^H?T|UBl0hP(ntk7wM+Dt^3i{_#`V6&QBq_rXP;WA)M@V9rmi=~@g9eMiDC5Q znIz`ts8Y9*@5!!lvvq}jYF6I_a@-S!VzS9hB!M#ZCS5+cs&$-wX;cthr%`5(&HRR~ zRw!Ml;DbuzJsOrcIINm)IEvrDeAO8ZEE#gGpCws%Gr|Fl%8!geyT;{vxaEr<8}{?i z1zuv_^B{bV#RdfO;mQMY)|-eqvbE(xtwTs8rNie`oO{!nBk0d7%kH4b+1ogH`pE9uX)I% zoS&bcA?c5v^_tAgJPvhumc^_HRgz$>jUY@B2JFj^$gdk^1@I0kHrtqccv}2 zZAXt9wDVU;mLEx6z0)toVv_o$F28fFJBuXs)H4D*qs*i(U~>e_Wh#6%s*xIa^{Q+? z$0{`&ZlR*A>=(GooEacG!{I+yWDq#l*=cY&qN*{uDjLiY&fR5y3F*J&%gh; zqMmu_HuSZQdd$_v@2*?3J8A(-E$Ga^2-9iFpd|6PTkVQ=8vEvAzrVC(Sx=R8#Tha( z&TfDn3n|wxWe&aP%gyrK(EHK-`==8oC}4N-;Sksh^LQ@ohfn4mKNqM@Hp6bv^*4C! z$L{Uz5fEM2`U#2zCnXuk$=#0O2HdgNW**V-aNRoh7Y~W86hhl^-`ZxbI*XPc&CQZ3 zNt{zvu!0M`xt#NDs(f?oHepclv}~jj+q`*1tKK{c4ZjdFZ=#9vg#@hJ>EtB|IjE zo9g@Kv11t;$sSJ7UC*c?VN?f02?6d0=ojZz51k@fF>^mBMoKNUHciS^{^P^E_yRP| zh38%+Ip@qBJ|!hJDlv$zv-2SVW9OGViLZpOyIOc3lPu9a>Fw^_6|)N*D?CW+lejvm zUuIIiqtf7Wgg$Z7XCbCq9(Lq|4pad<8US=5A@U!Z{`_IdzdJ|Cdb9Fd*R>o~8}6&Y zaVM5`SqqKxWr&Q-rE&~M>+Svhh$O$%`1p$%Cx7&?&)y7{SQq}u?+~q9&S?fYGPJo5 zqsne1@i$R_|Necq{o)_k%?uGYW6Ay@TX#R?c!$b5mGK``%V2j)XEYtcms?utHgWW4 zyKg^a&&|GjU0&Xi-4fS9)nR?*n)^~D)Ep#X=f|8z_VGXY?5<)1iD^W?QZGu_g@3xk zY0_-JJEI5nnS}4|L1;@&TB@zuiG8-|PB!h$9=vgn-!GZqv-9WVj_oS&J>Gc@^>hG2 z452*LvUB|=S8m%N~Us@-K5-_+1=h|zglQa{i+h9I5In{_(hfW zLa|tKqQ2KjCD8_m2*#gN8{VW+K%n9o5 zdO`uh=ZKqDOGbg=@Yh_u*S>4aIJ!8S%Ql{Q(Z9~7bHJrX6d4`mYRkc5qo z=vox!%O5N@FAY_Fpn#l#;o+xb$*e`CQRRy>d~a^%hL%WStuEb%zLkiDrJE6ACoK9gj7|%1hI4O zX3~~tot(exq0!9f(f9zw(2QU9#7&E&8{1`&-Cu3s1QsPfLe*XGiU*Fi!kQ|`X zCeaAn$S0-~8L9-QrX6e?>)M#PvNAF%h7n=mDltso)wYx5%)1i0H20falfW9{{%5(o z`@Mr#u7<^jLKk&Yn5Rt4teXqwlHLudV1*nb%=`#s-p6xu&P-BGUCF%JT5Eu-AD2;- zXmG1OVN?Txe@{mCxNT1h8*@TeQv6u_=cRH{kBkF0{SxQtl4mOD$r7%UFuLaLVeC=G ztCsIRM?G+}m}jB*r1U?sP|fFA7yaT+$NXm2Q0>^Tu&}+tXpM=ylb^oz7nT#)Z%WEN zXL*b{21bD>i+Y$xuY}w~YU&)-pFeFpEDzCWa&LotRS_!4kOv8gQOlRH9y#eK1$A`N z=INl+*=f$8)MEklIbuQov+0faYeI8XkMm6I0e5qPT4blIAJA#grJ8TFyv?gQTVhnK zSDeIjb$%{nfVPCpH*ETh1wQCB_rKF7Y?>sOF@6!&t$Acd3)%j5i zt*?0rQdytaO+~wU8NP>ZFt#!70edZ9y|=5&adLep)`LwqDxMy;XCoida&{K+{ktI( z6A$av+ugoC)fkHzRhQ!Xlvx$eQ8X<+2QDLkArbcTG}^G z$Zwl}xd7F}-IJ7si#iCH2L7-^g{|Eetd_(<71#Ne5d6Tj*-z_avhvZLo{wX?I8b*^bb1bsZ0nAmS(x0Nh6IHKbd@{mdhtTeog$jWG zrR)#GG8^jaBi_Y0jXKs03OPFZ)!K4b$5)v(`zfvV#%V}Yq3R?o`~|E!!aHD9k*W-P z1X(G9E}MIMlD-3nutjS0IqDo3V+HGWya7_N>A=d~{#0nuFK_ZA>gUH-$X){$ljF=E z{we$p@ZoP77s*$WLiwMi%H9QEQQ4}TjD(stdWt-ME~h{j4}jI#-Sbj@v_Jo+xXz25 z=mqTS!w-$#n+mV))c>e_e0EL{MI)qKHtTTN#W`2Oh^}AH5aR!Cn7QEkO;~1_p6Hyle=?I(bM!V5*R7DRpG0CeQPj1eF+8LjOZ_ zol+uuoe#(CHI%q(I3GV*WHTD>RwfIF}RKVfFxvU#pP(VQ5yprYh>6@mA|pH`~##D@(WBfiw{_gioYFw6Ngk~6%*$QXz|J`Ok< zHy86%7)a*pKOJWIuu~e)$CB-Jo2r%?2ZA}mrU9_@2N+LJ9ctSlE(%O7WOP&d?1=7Z zAfm@I(&!yHw99anuH|at4<;YTT}PcxNk~aAFzY%V8iClObrH9MyK$UmwXfUObu2cA z^r$*}0JD1IGQll>2f0Fi|NggB@~fzE@J7!1>S|hxTBTbm&WWwu((_qeRAVCZP+7+!nl+t@q~% zH#bvQE&MM5j>@d@_;ZUs0PlGHf7zJrr3kPi2hnd`iIS3j9|i0SI)52=RK3};DyuZz z;1M^H$4*bLZoYz=$@85Rqyg}kz{xBXnxeV-CcU3!hthk5P+_mOn}eXzx)!T^yp7=+`0g(AJKXI<=fi#ScSVEp+1G zxW~Dzwzd9tZ8#Y_=Jb?C1X`{TIaAg&;xS$XZnK_%W8n&s#&l7Sgf0)FS5{WsH_>)j zU(`KT6S(!svXY`;37%m-3#pOjhU3#4i$+%%?~k8f_vhczf2T)Fpetv1qvr{TMR$! zqytu?Tp&*+p+3oeV1V~gWWM{R_GFnNRi>H(!NQ|6^JHEy47Cujw1f)8dW>{;*UMnS zHN-LQ^P)%G<`=#rtAp|Mnfrf+?8ov{k^(G$cW;ZTB(Twtfk#A(bqk;6@R70WanWk= z3J4%KD%zbU;SGKm>pBD98fVOMgU4do1*9UT0}<|R(cs53$)O`@&r5VZ@L`yimbTH; zHA(AB%a!Ez&MGM-sdj}7GDG5gp%JF>>yvt#Fu`7?(%p3q`OwjL z|8L2lZ~1uFBuDb`aNdCvr*Sm^-m|zQ`1ssL^F7%0xdihLo>-a}sHb<;m~6-99q?7p zP+0h3*80EEh}GTh6%M@IS)wm%8LF1oFK!4`0~SQK;?3v`umsUpJ1Q==p)ZCN|0zR+ z6H;ZzzIlt9ECJYQRRGtm4j!@_a9cFH4@2wV1ZWowIs_o|w+?SxDHRs70-h))8rY19 zTzbFe+Yd&tzUi!V;qnnJ{R@rv!0_3`zn;XdsK_Gv3tTDztPmHJ5pn&}L z5Oha1Amx+IzI2re+C>1}OuY=)5(Bzxc9vQA>Cf?Ty(vnMM|klv02=1u8UFb3wjqhJLZRC5 z`u~(8SoHf;NY#EV!eoJ-{Zs4#m4x#A?=Ic^e^3jGo|6#w^YcqjPsgp*`yqS&+qd^w zf&H)F*o(c(t&SQ@|nYU-~Vr)2mO1N0?tDlRTXjiWpl!=10&L|L@+luAtv@l zwQa`pV{aW&8g=yn#?ZiWzs;G*?OMP|D6J;V?FW<*lM-hFmK$({ED7%hw;K>+0M`xcYgHR^C2ri{$(71(EwZ5VA2ZP2+i zk7ytPO-XLwC{N*sNM4l2GzVM)br1>9i>`xv)WkI6%WsV^wv082W(1Ryxq z9S>?+=wmmKAizYQl<_f3JvwOKi;0PW+ZZhay&>@^c$c$x_J>C1bmG$iF!{eaxzY6FN zUs@Vsc9a5GvNND2<1|^OU&+7_4kQsqFy!k+SKrVUrVPL3O}aYg=ds|`8erpydU_od zGH=JT;Jhh#C3-uN%jD}UA*RlwIB7LUZ&pD$yXmH0%jE%>juo~ym#mgLiJGrd5jp^n z(8}^kihxVjxb5Y4Zv-SYk0Pbke(I}IeI>F?(M;_1pXj{awUeFUviS%apW{tZ8X8W! zKZhK(&ig5u0hl*yC1q~3kvk*}oDS2b5T%uoJY~RNJDf*Z)H=_9OLaMzxCd5FVz+>| zE!CX?)P%_SOj^2`hKf<^=PZ7-3q9E z9MATJmJ18EaFkZzTcufsI!VA`b5Qyo@1=;m`td8$2{~b#c{N-#Zm~U4pxdooSXdaC zAujo9&7iwm$!FW$UDsdqI6^zGrwKdI`t93xoBfZjdJ2)=cKk(}bhw$1``%QkA}XqK zwd-!K3PqWS-)c}(8dp`k%s9fJb870#4gue!%X4H+{It!J)6Igdx$jgp{^zwju60>; zdX)Zlk%8JX)sPdv_WTaUa5_=H=OQ9J>&F}ZuK^E0!EIOr-p)62YZgRC>a3pGCU9Z9 z4C3s1^!hWz)FcxK3Y3wK*oBRN6Z3a5RqwN4e3P$;K#8wb@6HSka&vJBPUd~`lYivR zxux#IEjRzMLC13}9dHokZ`M#UQyUlxfRwfyOwSs&_c__(*63IEQC5b2TxN2w(kn8Q zzdSl2=Fb4j4ent@n@5h+4@;A)3d|cl^G4ar1fBm3Qt4o<&t0x)!7@bvYQb|Q{20~D zSgZ`oMm1gSDFXO!vQ&BPCP7GvY0aVevP6A<&&`*E6aJ8))ONEpvo$|4MlLS+D7)Eg z%VAHgiwVu+$BzX3q!X{bbN!bKNG<>}Sao%ElfW~cb@CpAq}i{k8@0I$RreM)q~zdM z3-65B6vKoTDcLNLSBhm;I!5bXgdI#}{F)YKRuy_&zKEexf9B>+`VxOLtbiU-tL#O8M-B9v52{A zhnk{dWq*w|9|U(r|8_+fWceX0RwOkYEfoYg*3DOqY2C%_H|B>iE7AUF*U}V?o)!V8 zi3!IVy<%NA_r5biN#nekMT&vmM_;^#V)*xKaz#D1>g(&f>%=(K(y_Y(C$nsXf!eFm z+q3IxMsVByWVp6SPXlxSd?YD+Z%atKP&Z~akUSsprtMEElgiE-C$%xZE6t-v`YZc0 zl|41~-F1~WpMA<;B1ELt)4GAmznQjt1?t+!B(wcxrUDJDC$jPgh2U$j`54C(k~+#;=b-(Z>>#kxtrBgxxaZDz4~)+e%5Z#bgcSK#69Z3&l<3u3%v{; z3&&Y$bMISs2=}IQTqkoVhh7 z8uU*}jjQKRX7{*(qFe#ed$GM;Hh+Vw<-%ia?9jD2oh~SSt$d%XGK9$l^)(Q7*)n zlryEK+5dcvfQ~3>8_2-5X&EqV4cmEPM08(x+?3(xzRQC-(?XSND>XF3G`gqx6ybXk|Ir#CY{8ARWPvT#Y-KIoEr>Js0Zc<>xc6CE zoD{anNWJ%joo63%JO}Vx-ZUEe($D;Oj)AS`b zm#)}c$6ThwY`ZX=jICS68z`U}HbQDx6-Rzf##(2iR?j@w&t1qurC6azD5IkPsb`9) zn)yz;qzAmIvY@VHfnYtkATxn=eE`cud-!XU(z}YBGx@}M{!GGroB9Fmo2|p{QYja0 zavo^n^Fzp*Paj@uHWd5rm~HK7GC2;HZNc*6s&o!)OFN@5*;lC&NU)UEA^XTJoE02u z!g`|qH3X?3k4scl&wFtipx9dtN%!d)1yl_;u3vW#ERAkDebR!z#+}Ak#`KJ7o-YIZ zzN9rIG*rp}<&7cZD!bcFO3}{4jbU2f+l#^O(E^oRi0HGm%wnOQZzc;@6IZL+=f|kw z$VXrOupdfXYb<<^Hz>%-Gt%O9^OS4I*eUMDflCkS=6ggFWDomBJ=`El>Y;)o?x71Q zrWVMzsVU6AqOZd9wkroqoP5&&47;hNnzZ+~1b~1|4iy8wM$Wu9Mf^=l(*h-CdvG!b zZaY}2mE#F`HK2%@h`7a zJi%xGS(uR#?e&3F%-#pBN0B~DOU(^bpVF85kC?W7QT2`ueTba5C=~k2>9^H_2V!{9Ho!br>5{>HIMt=R;#Ch{A&o) zQKUd!`Oxe6QEeubi1Q@adpy7!Yne;?RH&T;VgW&y5$mIq-YQ}|b^|~u(_yzQSr^cL zQu!sh(nr?1)8#y~`KDZwG$B!f||%x2|IyYxnXf1aRJ243A2 zJgdEvgDO$VBvtI3)8N1XFNBFJ=7*A!Qnw@i)%Kur!@dxg7^7=qTwKnDc+a>kiq7hS z!|HVYI;fZzdDl1~5@}Yvi+`rTH@P_a0%$)=Y0;wE%AdL`1~v|;kLh*T}k$yQD$P$ z2@#b8%!5lk?ep{4wSMYlX@Hh8sahl9;^OKV#lwb8ct6|LR+W)ynrliC$n-;Zi=r7+ zRaL8>C!5*AV{HjXzBXsE|5| z3`z`*>MqIqXS<3O6r+*m&sUG~5;9Ku{sa-^Lbeyko!mH$k z?OoK@g*N4w{OmHyUP3(seNlni?KnnD3OX!LM(dSj*^C!q-d_|`FR*A>#E6ciqr(9- z%q%^>X#rqVQqco~@>L_~SeM2lsjIdrjb(m7y?d%6*8rAX!YlJrJ~-ULpA3Yk0_F-Cdw z62lsP78aX6#Y`cGIa6AOz?LASw}ZUvhlR_v6zh%BWRX|W5wE;t|m%hb>~rmr|J_yZu7R03ZpeD(Zq_8IL(C5 zpdiQj%X%>w1e__I_zggZg%%ojh$Fn*T21nuzbu$jvJyX2R#q0rP8Jqq_^nOt=b$%- zUxgtMh@M2wz|#*u=<%V=(a+Ea2ZNWM+i(wH8HxSu+{mcv^_t(CruvCsqy3hr#6VmXGK_w8Gt)47>mUdnI{sud_dsW@?98KnM-J@U!=b}br=an>J*DPoI=vuXl z^ku`ER@T-;VGBYHSzHEwbvDQ%FdDi%^aBMjX>ZynUU{7C3&etdPL4ifE_fPADOu!_ znuKJ!7O9Xf0qcN8^d=NPDfa&%O^+@#{SDxPzA;AvBCleNGNI$WSplbkYJ%pUfEte3 zA7ixt;UN^Znis7m1)n?;Uqsjy7?%!obxk3B8@nO^+FCXF48sWS=}ECKjLpkeO@F;S z_$s^g@K|V)RMu4u4$I?dF05KpO(Oe{*gqRZDTq!AWOq}jWe)6=l9W6y{q0IC{QU}< z)Oh{+^=<(Ui;Aux!Cujc)7!5`v~_%sHpHw?y8y`Hcc{cwx_u@5Cv|Rromw!)VwHIA zIRf!qn^qOJGes{XbV9;h$wi%?m)9ej-1I4b?hUJf$f6>N3&ueQ;4we{U|hY?0BCTw zL;VHJuhE?jtU312=p9fD?7xO3aTn-22|{?MN-YN$(kq{>E-yFvcM39f{1j5B45-TR zsf;Rl&S_Mush ze)G+Xo-HRop3iUaDKs-ge%*_w!FfoZZ(;xv1UaV$1DyH*FOza&gUgd&+%Cp+tb;98 z>*^H81H)Q1e%-`qYHV+rB!&Es_Jau=&x*F29cNE+>(AcH1rQ_%f{67eA8g z@6rR}Tv?Vyvs(abK`VpVkg>m%o&&CNA3lcvA4qloxAQ&!y+JY;fH}MNJ-Q701D*uw?TxILQ^MB_9h4zYP*!<(LgAM{T=unTiYX@#!RuqTU&%In#Fr(F<*~aot9f? z?E&Ma&zz}*shQbLqD{B6!)_�ueD0;(%h*g1@iN(MD#%_S!-Vs?~M9e`le1H0193 zEMn|GT;|t}DqRX0K%WTIM8$2Us55A38{`6|0&7G$CCqZf5xzn{W@kHJHqa*YSS?%_ zL9N%}ySpg=>7w(RKWtZ!)8%;;g!)3%O{ahGtcIa};jbx~;GO^C_Js{RwZU)F6AFwtbRtxnTu z*5nnidxjcJ6Dh5`34_7J8cTy(S~79Vyqc-c{&;sQv4r&Ih9)JNQ-qb3iAf%quU0|a zJHHR#WlJcsJDE%GeaS^m}@bY7}tG2?F7!`!0v9 zKHT^bDJd^_M72iLWqb>)9LLVMncqn3;c<`{iHf4sE7DB{9BLn+ef{@2w?7BO-J{o- zlz~Jp;$JQxvB7^mG05fCEy7&aTcm$``c>AdJZ0(Y=VAk{^%VK}!g9_3wpk40VjAM; zJp~{<|1`Tp{J)~)Uwitu6|b)EHxjwmNT2pG?@t)TB-4WoZ{nKOloJ*uyUWGh$F{%b zfCmZ^5f*K97+1O$1Re^x5Ac=u*eGDYze}O+P!bSFD2`HRd89i)3$VE+dC+G;Hw^FN zJl>1p7XbHf*{eGMzZTSSAIDnDzp&M(kbyhngPfCcNbopX-)+bM#~H#wDHeDjXF48^ zRwz!Z9>nkhDT8}=xOWQ4~N0Mv%V2|G)q=V(HP?vQ>7?&%n39_ubis^l+(jXs2YP+j<% z-KmXNves`r>D6?6=$dcZ7?1}Wyp+8edHN>@2KKRkxlyxi0Myj8gaR8W#B>WyJ*GYY z@_Xr=1v;(WY{cZ5M>)eMk$oL3vUN74wJ$+UM&#=h4>qsu9=|)vfHat`K zjXO3}pg*N{ES3KMmDhYR+{poW|4uR9c;avD(ha&!GtGORXfnp3n%Qy_xEP?Pj9PIIMLoLFV>JDr@ zQRAC!AqTtWTP#2I@w%A=<0M~Qvfm$ytxBy^)D%f@;(5!^ zRvG-gPk3u5W&(1N$;|`mfkH8s? zFcRRj3>2=G2mK-xn3y(I@8sL)nZEiCkQVNBO>oZkd^|jKuTfap-AzwRtM)@$14bDx ziQ6&ZkBu#n!8ui7sT$vM;)=@|@>t1ptjWj{?^#A>{&|Cek^!NsYH`~&fEN<4Dspuq zg3*K~l^XK956$utkx|gCZ=GFfeE($5#Pkelnrd8n3FFmCje)CzbM1tMg+G*l5^d2* znF6j84LY3=3hoSx#>;a)JN50-AM^9`?W6ELn^hEwIp~~)5K}OIN3qdST-7QliiY$i zMVY3e0k;(oeDiI6J~J~@UT!X6d}wu~^Ek_=s85$(*CiP~D>i6VTz5{*yJnccu7@D; zXBQPco5NJ~8gS8Ke;6_$tb!FPDk`oH$@}6fFOw!W8swr8jU*j_txy^tg zqLF+;2h9dY)V7BU+VyVti89l*sT*H<|)s% z2^>V(81)S*$<*^bte7aBahdwAu&O2J^|8K+Jc-3z*~UXq*I_KgNErtkyN#1dG{eJ! zfnMsnP_q)79rn!j+O^r~1fF?U0bYn^0ww>2^)ZLgmI%6FNKxiBw4(SyGspNfr{lEE zU;+ff2dSW_*a_susu{fIN>^6%Tgx3PQ8aw}K&2O;H{qs7g9!>E5B^VKOs7P={=b44 zldLUp>Il%N7Xi*mKDnpEOq`!^zHqu@923#K0t94H#E8J;4aT+zjkM<;y%TgMg060t zC_dn*!bH)y3OBz)a=agE)|)Tk*Cqgu;wUKa@vnhPoqx2601Vb`P!N-dZRARTfniL9 z*f4u)4s>5tL+yS@=Wsv#mnQVzaQWXtWX}Engh-vSyTQc<->M40Vf(xWDyRR3OR)9) zCLo{445);J3eHbO2dKbE1$*J~}dR6LK|cfnZO79uAjFuOQAn z%Vz)h7bkK{9O%vq5uA9sxmh&WAD!OLOuO7(jc}PExPDzNmB&c@x-gXt4wEvlNyOcE zEi~WS+{zuBVs8td!|yFhGrfMNwR3W^NuGgGHV(Sp#3G*YgRI|IN|Dc#o?jcr`x9^r z_#n{gtnBzT_DL~;)bAWH$D1INl?~_dvga76T$%G{W#rD%hN)q@rHH7n@YE!t=RyaAjCHUm*`QGapp5-HmZ>x~r zCf57-iNDzM+4vhYp*t_EQ(Lyqm0Z6*nD1}Mi@4)Ir^DWRq~5=Ve!Wm=^jbxK^+n7J zPsm#s=VXYcveL_*mnx+COG{roP~s{J)hEBLjv2C))YTeIgzadfRk=I)`B3V=7n-LR z-faN5O;TJJOAzu~On~AsFs2}xH-V2~POp!@dSvZF-8pmbLZj4=Cu7sefe_wGF5%2s z4j2+V3(>pZS_uvjAWV#lKtvKg`F__* zs}rFMz@s1{0M*GiJogIK)ee1K$ubKhSbb4(^oN>HzjkKB=t+%UJt)%2z9J`I+tBW7 zZ+Rai@&(KFK78!-+u_)CD>HcWg8|4I-AM6kl*~&C82|vdJ=TNke=x@bS&FbRC#hQP zqj>E!_@YPdRq2o^ykcQavTob^3tR-(t+u5Au%VJjd$ae)aCdSl@r?L3aWHaeXU`H= zcaQ8Ze7FaF3gmPwXK5Vb;k9uD{bk`GFeQhTygJ-d7?k(~xCe3`YWKV=0}3*$JQUu; zIUD)638{%<^;33@;(ItO1RPqaFnlQw`1s4-Y_nT>5c8-gwzt_YkjoB;H*KYKYdS#NlAL|Zy zJ}xMins|Iu1?d=eLZ|z8WH*+VZ+J|ZumEggRZ~Cfhv66M5>#; z)9L5LrsA3AqLgYFqfyJg=+Hk%E zCB~&j2Qx-$g;Q=52IoO*2G5DvaE5Kkq)vMY?|8?dwGwZ|Q~FGv^r4DwKOU%@IPo+w z_sUW$moPY?aQv1E6k4$!GyM0FH@vN@TO?V~+A+)ZqPDx1MlQHBxZw2D-D!s?w}-tJ}m(Wqvf51hyrq{j8?E~95e&V;I|7X6O9V3X2NYU$fLHz=X>5rL z-M?09T9QRPmOf9Ckf82S@yjT=o>+&mB@`1;^H`3=@0*Dt^btPyRyIK(LuuOyFuRae zJXufVt-EjjcmYFXP^7x4^2NkeO>ZQX z-$By##|}_bW>$Y})?%ermCqSjur2qV!lKruzu3&T)UW}v^X0bWf+TSI#jwfpsE_Kv zY1~gHx!_HXEuS5SAp*LBQ$k>Xsmy2H>MxU{?}&Z;{RTZ?>E?#klqF&GK1>`Gtus}y zbwJwJ^)Z%7rN(syr(ZH$=q1+;1U;66G-2;1)nvs_x!Z5tj$V6sPzmP_ZjLfD ztEH{&tcstOHUm2N&GDQ-eR!(xRWp7|x=@vGnL41Fa6Qph1d8r4SNL||9ef88VS{eD= z^^!xR!fo={b61qpRNO!vHA6Gq05%bc$;g0D*f${%adB~pWu}1nJvWMJzLB+@N&Udq zF@#LyuT4DR=HMYr>>0l#XnmMS^nZtQx~6_<#oJeklrdUGxj&zZSh1Kz|Ig#MYjKyC z0XT-{bD(kQ#U}U8tq^)TQ~WlX$KifFuStFJ9C|Z=0%gLB-I#B0vNpuc(+Q7U+rT>By0yWZ&!jZz(*c+IW0R zO9)P_G*~PfM%)#aWnAY?tceg5zOwM&C{2cH!37CD;M=k24>Z~(In zHJ;)IayyNJ#_SChm6v0=Kew5T5BrujDS=4!EWA=~APt9A-sZst>s%+_bbWmQheLp4 zg_{#KlhbZ?|3m5i1GNN&QXn65dksB!=&6R(T?JOW#Gg@yAR*Yl#vE|8LH(7wcT1+- z@`-5=j{(MqhK44ZMvB+F$BW4yS(GO&O|SE;U$=NvztlXRzn(W1(y*;%AmloI3KY+H zZ9DuuRO_%C<_(^8r&bW~=Hwn_HY$cpzfi zLBuVZ$GeO^OzuJq)ib>!CUu<*z`W90;{pQl8c?g>iSOxv@2NZWo4+SwYrts;Ta|IH zmbPzry5Bs9jEzFbN>Nvz(2@t2q5~2&z;m`fJ)VisL^?AwerGT7W|T=F`AdjeGT~Z% zRxxS*-qY=_Ovre{{gJ@?z(W&qxZM9?$^cE zVZf~?#5;u)S;0Ni!&*$hU5MPqhAJ)`f(&sC%M02m9d4P%9VwQQk&x#)!Z!D60p|oV zy!&7G658@8+CERX-YuR9Fw(g1e~BGs(7 z%|Wymt)+OJ{OWG9HCvE4KxZUpf5t`Q)kq+lq`6*RDf&aC@wONfzN)cWcsV7(UpF+> z^ntmN*;K?UV4Jy3^`UD9c}N)7dr(kQMYQ>QHj48dsc+?nk+*2-aUL_s#qF5zkn`wg zx=v~c;wvy0PX>3sQe+{Hff#e9@II=#wKC}Q=W=hi#H|d-T(#%%+-A=qAW_IDHId@v zJPPwDo^Ul}1UgKc`!jC7Y}exH&Mn-iGp41vWzHdQ!J9lqy2rEUmG?tJ-aVzhgf5_0+ry0SZ zf^?q5b-v!(H&4%G{bzvkQQI39$nL(NK0C@gi?)E??jy&#}I#5wR0c2gZju`DSS}YjU3)Hs* z7vHWh`#}Mr1$V8(;$B|VsI4ksm#ylUB!4cZV`axVCJ~9|fVN(;OE=fdq)7~2z1z!$Z=py~6*- zt;cc0#koIo(|U`>8!nrYXFdhO<85>8SUB&2FP#1#IEO>j+g+fV@OzVWCjv(%S7yB= z{eUBrcgc?>B^TOw+0OGP?zT{@72ZA>dhhE9XBI(yKpT4xSaj1IPwjG%Sto1;!{(d_ z@>E41rSTbb$C8M+5ifXh|ITYp-*IQ|@ZL1KTH(3Ub#6-I+jG!uyH<2oFm_9o=*{GH z-&}%5CmHb(=Ep?yRLH#50IhZ7_78INeb8Qmw4*@*gXzNGzKqbdo^+@dIOD?`i&KLa zrpCYJ*fL|o zn||gqu=Nhp^PhfWH86*n%oJ`pA`#i$PD+W5Z9b3MGR{7)@$G;-Ws}bH0uMxcHuzQZ z@y}*UhPrv&vpaN?6|c_f;FS^Qh5Te@{(0p_^{Q3pr0BY12GP|OOCDfve{pAuOaenSA=97~Fic*@ho#aFO7aCCbg4-;(0U&V-@X9H{$g~lWg^mL3bjp2NWO( z`~BIY1!d28V*0cy=BTUr-Y$lf6W{Kzthp1B4h8;yOB@qn?P z5}dm$#_Dwm@dZgYJ8~~Zdz@X9Oek~*G{mq{mIX=FJITrZE)Ax3#8>3VYOy_i_kF_G z`?uK0b$?9r4Ry3jG5#O4y>(pGX}iX|3_uA{K~g}aTR=csN$KuxC8ax5Lb|&I326k0 zMN4;gN_TfH&a=>Yotby`yZ1SJpY_*#W`?zX`8;=A*Y_qZH<4;oEA6!B?|3y7K%5QXI$bMdaS-N@}YR>MrQ#DKV= zu0#&?7{b5z?BEG2J-F;rjgK}Xck0Ur8~vglav02RjF$rFhB|aL=`=L{7BX@SVRy62}rGh}P?z${Sh1#69K_cB_Xo4KAC0D8q zS*q#x>phiM>ChPpb3)X_Eln=mkLeT7*D^%S;^@mvxqhVIEO&Nx>72Tr?OYVJImzj; zw|r#jDD%PNQILPJ;pA{acN*)9T+$}nX8X-G9(|h-+dXP|_D}l@Tuhz`f2~kaT6HB= zm@MUKx=oe9>+yt}gv30~s$>Aq(S(2Dy-(D`XTE4lZ~J=U%V^oo=Ns4G3s&(g_IL9P zRR}z@d~g@%72j(L3e+B-FZ*J>rst?kB9%5|k7}nY4G}@W+g&T+`9EDVAK(zObACm1D zylBgBnBPOSRN^{27a0?5^c`P2%AAFLb@@n`Rr0nUgJPpFPA&(oP9~NTa1#T`D~$@H zqy)X@SNnBxjPU*7;GU6awQ3e%Ere}UDoaZbFX;X%_M>PCcwi?z*Itdxpn)xlPR!+Q`NJ#+*JZ32T%nB;c}tYO#GVkFnA2msYUmnSf`#mfIz$i zl18zVzP1j!SlvXX`NaEqKH3i}01;;0D$eZ%qas9yxm}({QIiF^ZJHg9Ry7zO1g zPnZWei5*XJ3{Om_s6{62K6l+eb4r#sIy~R(SPT-#8omC)u~KwsC@YK>uZqe#5?O1q zDr$T4j4xt^JjwpFspdu?B9gbG)0sp9WBb3VhSh0fBu|KPu|8c~&tF7i8(cq07>s?n zQic@1bBjVZ3T@PEYD?nv)br0215%%vm08>mzMm0Im)lr%l`>gVqTxUD4G6f%DAh_2 zKdABNYHVs6$dOisew8TB8vG2~u#6`E=dJjxPAa@7`RR)pEc>XJ6ORjZidOJU;~lzD z2?Jq)fkxUw_t~tjLpnroi-f(jayY{)6dUFw@)~F9*6$gRepVa`(r&%=b~l!kZ|F1&#;nTPrgsurD*C7#GeL6*e2pHe;zM*xh!UBc{!HZgqV>8xOT3l0pcG zH$jto6V&hd2?ac&j_vm|lKjuMSBu-JB1Aq9fx;u9PGF#f8EnCi4FRO4S^}Po;Z8A2 zh>7RgUkV`9%>_A6Z8>}?L=g(!U;i*(HS$v8yui`#JT5+dUu20zS#mg+PNQ;ugaN~R zy1JOwa;;5XG~#4|UrMSaT$pKa^&40cD8;|-&N~hag4TOAhmIU*o8)O!Fj@`CfqWNa zEH=t+6i2g?2W1xfXa3*7Rz2jet!%Y!i%EOO753OceRy97 zN%$zz4SLeZM4Fn&A1r)WJFQsP{^$w7Tj<-Jx8P!vudgFA`4g&Jd)-tGNoay<3rQ1K zZ5Er+Jv@~kd+B%1MPfY2Kw|u36;zzV0@ZrZ247C;KEJQ@^=pA|-vN^P1hcqOV&5H6 zC4WX+CvAaztuuHW3_e;C<;fGCo}Sz=%k^oefg>~I?1I;*?bXd@Lf?OJ^lCf8!T&-E zwf8)YayeY=-ST{(OySiyvSI%SWvYXd-0hEA0KC5rE`@j4120SZy@^kv9J##?QbGRO z5ymWI963gLP&@PZ4$YyH+@xEzPvJ=~_(V@9Foj%ufT#RgD&tdIXJ|?{sn8`Y@{R9g zN<>$gHYhRiw4*KLS<+%19=8v>YErm+Uv>JyA9r+H@^r6AC66C6OtERP|2haI_!q

LDHZULT2N10w_}{zB zBPf_WbXmmKR=6E?=y#?~1Ha58#PAeRpW1tGMI&XuRGjR`nnV0V&&mU zNpKrV{22WG`fU)+7qD7nu&SS$WE6Oz#5z50XGF!e z(PP$zS3;hC?W#VlWz`4l3iy#1mzXexK>wgNB$i!paILsB~25e50ewM%^ zUPL5YCw$zV#UK}(EIeoiI?edYvyRl(BmfTRLlIXYBb`38o$7#>Lmlu~jfSe#n`_-` zK_#0;>G_i505j-L8{MbdK+hdgKy*PSoz=ZP^ssueS~=xJezvn8{$8dfjY22a!bPGY&Fq zpR{OhHri)=)WMbX%GL~hS9C^yQLlVdyfhIp(Y;3gEk8KaC1`4NCX7k3REW*$vNMZb z<5F@Hp5`7*7pUYJ)7fu+=eV6MQLdyFw68gA>pm_h6lCS=32dH6#?zPqaax&X=SFj|Ie+=i2j%Aq=la{I!HzT3p z3>^gwqox^Z%JbV>K(iFtOBoo|fhw^xu9!|bWuVb-X*ll^)`5E=aScRGgo%lDfyZZO zAZWSmUMF=|CHL*OY}G=q@Z5;tw~Ck+dEBHQjH(??&kR#OQct~|PpB8|zn}N9?#JAm zDU#>+6q(Q63$ei45*`}*L=5X8rsYA~f?Cn8oZWj|gqw&>{6Z){G{I&I5*dFNf6O;g z-^Q#DCdXGYZ`V!4`yrHLp|ml#HxEZW(T>M$)_8i=7pE=Yw!fTHEj6R+r4AW{L%5eX+^5)f6K%_`WRZKr;fcGuj}GPJMV{J-lXfTjl2aigt6Sr!3^xcW(^s__pDUw8_F#KP0l!AtHA z&2%kPDcDYFP|dK+X-*GTn8yfz?+ScTZHz>6*vx0+DIWs0%RG;e5IGXLp^%UvQbgG2 zw$Yao(N4GS;0_8`I1w}P(bCWpALO+$y}-OLo^|4F*OL{dFV)f7+1$vXRILmO0Jf+! zN_bo+$$UWY!-cl|$#N&>ba%Kp>Imx}HZlgD8a>YeP!Yc?s zBM0jAmMsHDJ{mC;Ok4uloo}t>ZfA#Wg)V@di%)^v>$SK{e70gik#~LpJ$xr-p> zqjivP1O*=LsM>*2|8@K4`z{xqKs0vfm=bvc?tG?LCajSssD zqQGyz0$tw6Dq}yE1qPuVWC?t9Omhk9~PSeVnNfC;pwC8ksV?b ztS%t?5eHGAc-Z}HOP{-*{9zBNwF>tKc#*ZzrZ9#=W0EBT+Et0T@C`#WLcsIgk_~DZ z&jH8{z@-QtqE0$fGSSdPB?w*&_<71NVJ&jQa10`hSwa`YsY~gz?^jD#!XLo;*(l`sf8;6_1VFM5(CLHgLtW{0F zwXnF2dTVtgIor>QuJl47^WkD=(gO^PDyv~zp37Qry?}UmV~hY72ZzE6P!;~1?PhdF zgUBW_SvU~G^JN(CUN&cufi8uBpzp~R&#_c8T(|#XW^M5Ev==ah&*^e;!I%^pGv;_! zk51#-vGRe5OTEG@;;OJdXf!F_%v6lK(k3V9=+zfW&85zC(%ucnA7|>8`eAqHt6yHk zFrAj(wY0N4+TO=JY@DvSd+JpiDNTaRpZy^4aOm{=6M?$k-p=7MVngF%+`N^%l<4di z8qdq0L+vk@I(I>48yUVBtkl6QHIfN$zGAkQ&|iVRrH=7f9bcx&9IV62cdh$RTYzGW zjh$T~^L+Umu59(`DN4);RJT;t?SgW56L_-Pw3Pm?m$uXCVCQ67hX8dGjWmhcphTBh zWWa}IVbRLZ!l@=p-iGDgALiW3H^>`QjcCrra#>am<=j#-YG5s#q890UF^^{3xnv8N0f9F)@;}4z0yS3ipMg#_PA?LRvbWW)|%=Xt?1RuAMGwX4#?vkI@&dS z0VZ~}DqH=NHctlE8rKDPw)A?tRXJLlkFywr&?9$SwdDO z#{;Zt+$+~YB=!BSP1ii`MKPV|+?4E$<%l5ZjRR z{mO&fN59C<>euorC9?4G?PopG;}_PA@RD9xb>h!Q4g}3Rh?1nEN+J7Bph6f;N82Fv zz-U;7G0?|gJmgO&BkgMJ*Hnro8y%Z79Zm8daOyJCsHSpg z6Z%C$`KjNWSeXWF?NQi(=7d7rGN*v^x)MT$a1}L;Bp;;x;Z^FTjD@DYlG>m?$mm54 zfs{OE;tR`@xF4A+45el)zK$^Wh_Sz)=1d|jgOvT2u{C9}(^Zy)x#p1v3yBH2?*tdM zkAD&_G})tw)r=7>_r%vu%jh|8o{2=>-~^N5tx;0qwfz-unYibqa18VRqK`4f3C!<4GeiCH~7XLE< zMNDnd7)sB$?F3(jftD7C9+f;jH?u)HN>HE%5ly&no;3Ub#fNMovGKanKG8G$CU98M z{9g8YsMN&jwCf)y2%xfum{l@LhA-^01Gj<9M>?Fch6KX{`X+P^7jvSLk;XQvc3*A} z+ua)ro}OhgGCfmfps~p+K(Q2>-)u>^f7skt-vGLUKXn0hH_9=}!ENS`oyXBOi`6?t zkB#wvX7IX0v#~Ra#hB@4hsRa>w`G8=S;Kmdc5{7^)R$3wwQAtl*mmMj1Gi_gdAQ?) zcCYqbQ1|Lh&wvX8wDtcpH z#Ewl?%h9Io9Mg;s_j`hS0O#WnUOR_@{pO1dbhY5#jq5*a=d)=j02T!o&V!HrGN|)= z(B*Z3Fzx7U%juwxkC#_h*YZv1@IQEZGFkb+CaEtvR5?~W42;)JfiXYkHsGGdMihna zc!nBgvQ+pVcg6ooZ4UoA;2(0jdvtMj^tyNq$W-+WjF9$;#^#ELL3e6x zYbjqLBQ3)?+~H@w)C$e{tKO&;VPQXSG9*{EBm!e)WyL6ih-=cFulEI`Il?x43%Mae+9RQMN{&&-ct8**#h%h>Y;Ly%u= z4tyQ|Bfp60cQgh=7vP(lZNqfUm#KO}10n4eo?M0JDp_ns$)Di%u3+;&YOrA8f&V%% z#)3=)*01_hw(V52DeB>qR2CuLJkiQ*DSeccIt zZvug9PV1}5*1HvWr8w{}f=xQ0PLV( z-aDZ~J<+(!(q8HLj)^ZMa;xsC_4Y?V6JFU^aM?T`71ngcBOoxHeeb^UDfT9pT;wNj zDvW(8i$W%#r=cbG0u1t_st;cKJ^_BXjBm?AM}P%To=TB`MW;5Ea<0PKF&7kK97TVv zcal8jL!#(tx2tR~jIv7t!jZ@)Qb^rAlOLhf8N`%6eQQ9`x2uQ)LP>RiMw#x2XMfwW z24G@dsCe4o2*i{BdIUEMOg`plP5M+9y>6`p7}UsoCABPTT+%Axu*E&BjzHgX*VzVX z+y&F2Z)sxIHa2TVr!Vvou@#r-$%4X?#Ht*Drz*F3+=8=nW-Sx>ZpD3dvm;33$ zA>c_#P5P{PiM*+(8gY>m*!R7^bK^z_Tr@d z_~d(~=Sy%Q+IH1(4%a6gmbZc1L$6f^sC!^wY{>VRYXWG4B+JQ#;AX^p^7ak8&_<7ie|WT~nBCvk&nz-@)fVHnd_qdY?Ny24o$Kb zp)(qOOv=9m{`&U*JixeL3Y@fE_r;`O2j?cHc~`*G0sbUCp#{&jOGO~CtOZm47H9g? z+!~?D67J#CzwP*i@g@2La*lc&B^DBjr)6*gK?8brX-f^L^#KboG5OyOnBGATF)s=9 zHvipHV3=$8H|I;+!L6$o{IAcp_HWxvjQ#p?NULlU^-4Uhl!uh~dqOJF9=-^0SkM2S zK!Anmocg!666_v5x~V`%`K#Xo`9B+`1dqVspZ_mksB-VBKd)J=^MTUJ(dh(4JsHp; zgEs~f`AB&pUE7np2X)o+E4ehs3?7fuoj2jqOE_06?+eE$An8D~*egumEi5byrVonN zZH}zDDq}SFSUxX-NLfqUy?^|WuYG3Zy8DU=JFxCb7sW7Pqi!FHCkN&BkmZ8HnK$?}7 ze6d@7s6_k>9{eQ~m+5-vmE&59ciW8TxRF?yy#W5Xc|uPxMnKJh>uuY7Ok2FR_RMFY zfHT)Y12#j?-3Jl;Rea=8Z70wh5G8;GS(SjMZ0gYg_y4E6qh^pb*on(ut%T8BcEJA$ zhSw0eG0J3j21`ym>wW<;|M4x=U0t0Z^4ou9~jx}l*Vv8p1T z(}q_`Ch^Xvf^&v}kml7)@B#3<1v%wmw*~2crJ7Dp>cmUJmO*%g8+|BuD|?s>J(pS+g%3G4}vN zfr4M6j`2EmmDDMemFq?uP7FEa;%u5aQ_AHj931!ObnfJw?P!Lq?{e zK*PvXytmjL%9L|5Q(>W7b63Eq2DwaD*>Bj?-fc$-3X7PgT>)`zP~7mP%S@%Y#qpq7G6BZKrLOxryRUDcXwjM0WCfx!g9AIVFPBK*J{iizw{%=7 zMA*$50~7)ObnN976DUmd+hh$;rN}p@BKw1z`v#mbtGkGmXGNlZkE(gZsWF`-{2_SA0k_7?dS+D zMq2>(Jn1$Mm0SffKzT}#qmoWN%y#EUE43d8Fn;;+W!wJDXCz}zjrM7t?d~x0`xsk; zIm#7A5v$(lM9&K4f$~%Px-sDk?GxSTgd8XpGwsUQ0ktdaT2nuYJqb!z4`OP1`gx_P zsP|!lP@tn~)F9y5fUv)!!fC1sAH4Bos6_0L7ceQil+AEz__+f66A&15%z{$Y?qhP5 zlGhw2%T53ZNVn;hR<&c0AH;p0W7q9!Rh{ryb{1n+roT^Pt@ot{YA z`j&vJwflZ)!eP24T&K|&&2ewzCEpppfI7W$;Xvx`evbf`bZ}1aQ?nG9Y^4FT!WT1z z12{@iww^f#@#yrGxD<*K)*HNQ_baC*G&J+ zG}+&eQr33EcYWCI)452nqJUuakp&p>l$x6UMXqbNoliL~-tcHD!Nsu(Sf_Un4u~#S z=BZBb7JAd+cQ>N=IFTU#i13pPTkC^1KTDp=LV;HHw{I^2XQp+G&$=T)xA_i_N}7tG z$P!i)J*Sv=yAYN`w1< z1^xW=RyhknVPBI6*Qm)P<+pd@ch5sV<-^D~U!v6;pUZPWfgQuPwwB1Hq50fAEhwnF@PS|8L$2r` zYL=1`O@xU|a|4Nu(a+Q8(Du;K`#@$=veIlLQeK&3O@2C5Y^=Co4JX)hl=Wj!mwQU^uRzB9 zm&^t7znlvB`-lFgUr5`1jV3@4%%+2@5Z)#g75U{n!fi`~(|Ljub^s3p5a#aRpLKC| z&h!SwC<;9^F#ru-_>C353yQvX;bkiD8KHs~?BJjO<&eyHenO~|{qgZJI|m2c@o@_P zN5N6ZhK5;n{^{5p?r62*De3fao#?=Smbc|cI5_$&4{+Q51U;T_m%0JOMm-_&#KdSh zziG#}uFw`*FN6;gI_EH;)@#8xDoeAP*{bh}4QfBcTp zNosQl5~LQnWBU3U7}>;)w+Bn5=Lzr&+8Dh!0Kp-aL7JlanZe?~2P8M_!!H7RLkCGt zu#6-Y@%a4}K+~~GXzsH}dtWq$#qwst`B;_M<>DNhF4hTT0RV$V$Zqf90~H9|`Mf$i z+gGU^r(RJ4&_l@7)p?%FE={ymXkfg=+Gy1lXoX?ZCn$2J49wG~dwY0LDg%$>T$9*a z3)@6k=X{H%n^A9$ILK-(^S-y3oU}8d^h-vRxGD5f?(bcKU;~uqfEFYLg1iVW)+*pn zfF5Or8`Hv;-x63Z-<)S?wuM1MjaOnL)C(_8wePWyf{q7qya!Z3XqwIX7@h5_;Z|OL zP3+_kXj68L=3|;gP@?omK(4wha_9|pxK8%Ee>DwIX;AS%h)k?LL61{E?Z`v%^jKnl z18H$A254c+mA>i#lqHt!_qL=RvVgdaOP6jCzRiC--zn0X@?BbGmXgq+jV%Q#%QLNg z4lw^QSW;grTfV(yI#(l0A-gF5>dfvhYDHzV@lI!^rZ&iCz;gDis}`uSZa=C4)2?<$ z%e!0sB+7m*s$`s5A+g#4(l^Hx0vkHN3n*A?5z*wYJ>*C zF$$Dmt^}a7If9`GV^t%`hDj1|KFeftv)#^ndDRlLSmq^T49&3?lU;ch^GT_6Ybd}$ zq5=|A1b~92e>0(rGF2;=TM><-H=W*TkIcnKqm8?G5I_^*!aJ(W#TMi>zrMaZgQT0S z=)?(P&doYFovO@kluM=f6ksaDsg$XhnGd?dnhiV7so2;k&rkycvm~2cr2#OgbQ%}u zfz8FcX4)mJYQWO3M-*?1pmpO~9eek|jYgg2@||j;OB9$|@#OjQeV;K&^`P=W=eYO*F_y%C&k#rgWH2N;6sWRV*7_@V7@bPuqSfXVt$t2^e-?o+l^X~(6M$iET zn-_Ubrf@gaQy__N>(5Bca$4+oLt0B>>Njz!R^?TQ*_sqSI;^Eva<0E(nj9ke?&(H? zeEMJc+h}Y-1wNkB#|km7M)G~)G=xyBVYMSsqyF;AJ+wHiAPU#z?eFvR4eR(xYcH8B z@aGv;`k)n@>Ler$D33H_pBdS%4WW@brXAay(eFM{P;)lvin)ZHsnsI}2vp;o!kDYB zhizF^+xIB*&R_6rJ13AO?h{5EgG_!tCq;L;Wu9{sFCYCCees4=jqB%vo&ugb>yl!X zhHZ8XporcCzz$B1j%lgqvL7;kQlmo@?-_QTo=WT|fP&E%)3`%6?3K|f&1S{&DW7U) ztHS}~;XG@{g+VVg*d>Pg$Q9`eQZ*qWX}%M%q7`~m=Mq7* zxVGs?4DtZck3rorGFzjF=KDlXShaq`qVv=7lohn~D4`!LStESnIgICDw^V)f^Nag4 zF1nD?DTC=R1ZX$uBPVq?lfvHw=pKQ5{~R_zrY3l}zxWp0FbXa@8XsMW$lK6#Y&?!! zd$WIz0*1sDEp-8if0ffg)SwV47bz)}f9DciQ7%*G!O8G6Pfxg-XULQN+IpSy=Ph<_ zp!yD$jXP*)**o^8%3SK-=2da4?hB$+ChO|Y%YLFzEjeFk2T~#bOQLU=kuT;N2ZQ7D zjrF34>>FUmx=H7%*^6Pj=AE&{#VV$QRD%x*in?9}DP$fVdG3kquf7|9Bu34)g1a); zEFKwTG)Rao2(&^!R*T(S37Ad$#9)IL_it^Zmsv20ur4i7N6ke% z6f+vu)%F2MnF=ij`<#4^=JCQMJz6$y#mO88n9FNO=02%utZ#8`Ra^%sYp6B=Eh=U{ z)PE~g`Z4)~!7NKWo0AGGLiu`2e!=!^#wB}NRGIU_Y?A9#yp@4+k%)vamV@CkN8>)S z*vqby%bdYLZurN@7Ir6br3n-b z0hsFfkNO#-V#nsdmQg0`Vv{L_Ve2PLh3IM`n_3$xlB>3CicrD;P`^!pbw*jvwuJDZ zEarYWvEKlMpg7mNdbHd7n&c58d)+lA!eW)#?QI<$>tku&IV%#!Iwpr%B_Yg96THw5 zyDI>ZO=_o{=LTvc(yS@|=Fr6B<1)!+HxhE&ttq-7rT!*b7L)Ur+>>AT6qR+uhRGK^ z`$7F8k?XXLpWM(;pK=b0Of1mSqRp`I{NtJW%Oo(2vDw%KjCTVmIrJbyT% z5=Ey;rI=SYW|J*GfvBz0>TqKQ7N zqx3b^WO!t5>oRb6XNahF9KPwdwf1Dx?%N##T0tSvH@u+XP|7Ez#3mv$tkv=kT*bcU zSm?xOq^yFQ17FamC1f+eD#`A$I?J1kbBQsowz-=P!6`NA&NWMdNxf|OGfAsksaAW?HfieQQ^EGGoR0+ z8C0Itp7k%(b^7qL$p+>%gQ-%tATuSRwCdN+?FT?do5Ps=@N2ovDy1YgeGK;9yFC}h z+`ISFa{RhApD$I}36cYqN?)ICN^B{9xBKxe#LwEY`pZB2|Y)$)}wFWT^HL&N1c zRv-1dMJjti+tF}h+g~*8%~B08-B$P2D}a{_a&kENBA4~Sd-v&Xo)75~m^8b~Yd0#? zcCb8UcV5;CtQQxqog#c|sAyTk}fVT3)2-x`)(Pye_DWI z?Rz}lm@nKYrIO505PZNi<5{?#xaT=j#N^u7E;QK+e-ip7 zlbfh(7uq7ni_D`o7bGrh7-?u`-{Au|Qv!n<1RhP7!1i_DzHNy!N1eQEHs3j-K0}1Xdg6gc{ciHc9%6Posk%I%m*@+af0}&S~f4tmU53u&W4}My37ta5Vxx_#)8Sb!_q0(lT6| z2C8Bhb*U&EfNfvj+=hcoY-Q1l0B2ww4r)prPH=o)#$7H6VL(R`^r4)>_R*)ukF z*iuDz&r2qK{P_4ng^0-5C+_E) zNs7x7(~Hocp!`r*APNB8x0o-S6O)tXQwt}Qcck!6qs**k?#jB(~F3vFt3#!4=mp>YEbJA3y4 zhJ$pPjJ!TCn+_{%-XYWsi&~@$k42GvQF$`}p`ICnvXA1Ms<2?=AR_#~cm{tig?S;(vV1(nXOAYNlhW!McQ-{y)OU~NIG>?&`keCX)-ZK@- z3wBYlJFt7=UI<`W(1EZ+K|C%`73I%q3IIEmX z^$1#SlX&){Drg}HsS0;K&D;S^4e^nsOD_RONd%a!2rUEv16HDbjj@JN%VM2D8Rg!G zd%a6wYB|waD>lsa8YtnH^vI0j^5Bie6xxCDX~t26V3gZ9Kq7#KhSFh(C78t1x6pmQtdT!I0A6`a7$cVJXd~C_=Ex|P(lyt$K(MxC+OsLrCNluP@-$r<`Iu`)ZUbp zph3H)f`k`00s>Fk$4f$OjA@ZM)rcBdW3W#=29r1!$Xe0D;>f#ij5?v?vb|0>;cG-z z{=k*QWhV$ZIRFLFqxDHrVg++F7e!Yw#7c(wx!xr8%HyNe!S(?9vYlp|>S|XEp3AIJ zl-zkf3414p_69XWmjCcTH3j)k4ae*tZ%fvgfG%FJ6BRYnuK8iR971L_(8GUIdg^X- zvMb;~nJGMHaBrq@p}$j=RqS%HT`X4t#=YLZ769Z-ov}=`OEQ|0z&sfGOy~GZiK#~9 z@TX^Z#Z#!%)v9V>+GVMC04nULP0togHVW4~p`+^>w=?M{jXmxo>x#?VnWsIp!7EIHL5w*)$2rX$DoAbV_a z45c(48n44350p6FEdr=euR9JIwz(NB7iAi(UEw~iRc@*;pk;A!YoNft++?J_GZ_lv zl-8d5-zfM6Fa2>;UFhfn4R88GthweHFn6j2Z#7gOMc?!Qz$^i9(bjQ+dBI9MwHBXE zp=tUE%;^QwM-(VDrE#=Ue}GG&4ihmSt=#yiP3+dX!}MYc%nnxXFeFp z8_J+NqhIhsBVS>&=FP_yeGtfpr7I@o?O}Jwn89B2_mj~!T#VbaDuwO_^X!G^93PLA zdeu1m_yLR3+`NDHF3FM0iLZdj5Qy}egW}F>M|Yzl!jdIlewU=2tq~g8FYRklSU4Mq z4Vu+%i7n2aXMTWgcl7v!(3ip3fH>InaMSTiryndW-Rb_E8s*k6Aqd+i4Gc_%ZnX6a zH>MwIfB{0qYYJ6+u%2&70eauIN+*ee2v%lisd&_OXHqR;uREL=8HQufTr#3vrznou zra`ui`Ga3uF9E?&FWWy}-rAoCw<@1$?!gZLdI8x~Pw+NKOUZTDMv->uUmZLETUX=v zZXR@=>uGT16pJ$gVZj$LhfSZr0$^*ddpX$K(`l5kGg`KHjLK=}eg}m2!!z1;1sauz z=GP{$k;RN3Gv_9!_GdII+mj%YM?*9E;O&dbr&eXjHcYb$nvjM*-m4wNDA9SCRe;|L zHnbB^LI-x?C+%?A)gJonolkFp#MyLgpRXOubcr@Pw_*m9CPOZ>k1Ard_WQ+kblek4 zQAtV7icP0$kjR0R2t}~2IMg>0(WzI<6$$3T_f)%SiEDrd50FS&-dJ=n1@4#K5dav=ocw;@^ze0P5w zj>btsvHH}b@SU3UQ>|P2^@qPqx&Ln(0me5AxMABx^3*JPlDL3q(WbwK-FcY@>O18P zV1URTP*Q;WZ!nEhCSC0i_v#Knw(l0&L$ZOQl*l1f?J)eej+IOtcRH=#crbf7*oT|D zx_i+0ctH^Z$h>kp(q-7Hwt4-)YXBVC@DFDL4Pou-*=ZceEjoedIqlh-KIf_TqX5Dn z8ULJWP8J}S!+ij{=gL1Mk|txYHj>bq7aN`>R-k;SHnB{9V_Wk=yVd=~( zyn*fc$KCy(VB{y{IyxzClhvl6*6dp97~`x0IA>-vVD{`=6W`|HhgZ zib-VAwFELYuzYrbSWu%15k6b$=VAOvULv7MR}6B+XUPR40T1@Kdq{yg>L*ox8BUd- zdY`~vUGChd6T+Nx<|N5;&;d?@va_(z>X)*#d3stB0!=o-42NTXwyU!-QE_-jDL!4% z=rG}3o^h@35QRBGdXSIh=>kN%E}y4=#9zw^Kl9-7^$Em!VMc-9@&MuUt&?gCwaJ2gwW7l*>n!}(qa1t* zdUGb6FDWMAM{CW%mN!_UF020paIeE#rz`uQ0lU^eRi`*-D_?5eKLO;g3xh+vSqSJ6 z%je3NeotYu92;#_hJjf(wzefd1VKHYz?BX+;+(NA^5*S5HkhTgyRo2HtO)$-+2-EW z;REps+FzL{ktuW3`FToE{+Pf(sMY?+%+4HHzDkuEkeN1|R2Vd+v)w8xPngA7PbPKO zmqh%V(Ka@oNiI*$rl)oW(5@8eG~=JXiMMQy`$Ys>9Q`TU23}Wpg0F&*HQ+;4-=l5> zJc|Y5rT9G<$Lv}3S0eC?uD`E|V__NWxbamVpN#Jh6IbE8*??zv0C1!|#OmPA`CzB8q;iEsvPRk*49R zr91wE>ZIS=HOKB2`Lt2 z5nwQ~9ntg#OC@r)iAR|c;hw1L1nUCBbojH}xv*=k(5^!ve<4_*h~pRYH1&(Uj3T8cO z@aYV0PPbN1eq02Bg@bKr9Jf6D50&=?PWH1zjv?&z>#2PI2aHz4GdHS6;HscKvAGTDmxx7v1ix2NIaxVVZ4P#&YIEj zp|=Ydj03rq*3+S?%WENPQ9;CY9UHmG2=L7~X9o6Y)8LNQq=e0_Svn6}Fihxgphy5Y zB_POF$R#8HACWb()q=pIc#_tTs}dz+Ub+6HUMw!J?Dhx*zTqp?U%Z^?d{`?YmF>&+ zM*dC=Zj8ZZ{Kq;*(9#Ju2VM5MtN%3VEE)`%5DwOW{V83-2Eb%W~R~mCqH+Sz9 zTtOKN5Zx3_#bQtD#Y$xfK1_D9=ue!K+c|x?rL|f$5 zC7+Fd$fzzC)4H-)lOQ|vRLKn9@vp580eTmIVec<6jMu_CtBw-;}CM z;%>vy$S@Lse-AX%&l<|)damB3Mr8o*6;Ob*e}Ml7cV4X35QeI*$#d&9rm;DCg^gDe z$jDb$DZa*!g{AWeD{}a?oL%mf*_i@8lXKZYCd2_gDqh3j{4~_@yc+dr{g^bel^1 z8oDGToB!46U>p{k7X8N8IWxv-N|c zGe!K-XtzyC6>Gw(7hX&Gj9k+hk95|D(gYi6s;ZG8jPH1)B{n;GR6P0m)=Vm9 z2Sxm9s874VvHlvL0jCwO0KwVCa(#FB{=YLK04lPT(pWS!EKDxTjT;d}k|)|5M(_zL z*ul(*k>Kd;WC+$>adLDvRU&i^c%Rtl19zDW@l$FwQeENV>&&|=$*-4glqL*oS{Ct- zlj2H;1`)qkkSPd%tk^Fk~Zc{vJ4}3=o&!v9P!Z;b2?$*qe5S1LJfZA=5**jfYHFv&YB5U`K zhmSFo&IJ-IfC9sGxt166^$iwkwH`vjdKH3#Pq2SBE3F2YMDs|}%(?=jgya>Z0NHOC zp;_{{%o&QDZ@V$Nb->Dl%SuJZ-8DEU9z_k`IZr=#*{&Y1?mqAn7%JG@;<^}M8uhjr zw)n?zvfu;};qxo|26EEJ#B7hAN+}5zipUR=%bQ==DzH#R1O!0W^q{)YvX(M3aMuc? z7Luaq9`TEU2gc>4eE!@Grk8;Pu9wI^!UyQV{(V@*Xq4lwk0Gr3Egd)aCahrdpa^{} zQN7KjjpVR#n|2pl8E> zv5!na10GS}S-F@I+M16|!_Qamt~@C!ym1X;e_XkzUjw)i0L62<4L#Qvf5@VW z843ZLQx++bg`^FtBL$*G2B6!13cgo$9r<3>9$0`$AC0h5aDFoUQ0jT&T)kfP>FFg0 ztB!y+2wf`UUw?;03a$l5&Pa7A$9?(Q%_5?G4VP7yfQgBA#1{?Xj`ReaHl4~G7$GF8 z^~G&poJlCwOsMguH_}Jm`#&P@g&B#s_ z$<@83v!2S6P(p%-_b8Bk(wb}UA9ABA_}tx)N36ZC1KOO_QMvHx!eF-JQb(>@<|=Fd zl3R@tT#og;Dh``!PCtgqmTYiqV8A%P?~1ovRUgF6Jb z06~JgySoQ>Yg~i7yE_fRp>cP2_qRFce)qon-Vf+8dhcDUR@I!dX0>BgBdvS#kbljx zZbH6q)bGhreY|$@XMs^P4qjDLI{8o?(weqPxS7I}DHm7UgAP&w z;}1>C6_dRxVW3MRcMzEL%6^Zb4Q&kAVO7xn&h#8-+wcP@8Y9v^iaQuru;>c zvOs$gRm_bZR-_Z$T*mn`o&dFFob{inJzU%Nmp@42iN=WwP52n2CB1)#1J$ zV0yi5iveX0MGM3)E9o}~-@L$^k^_cL6o1r?iM~7)?q^%BIfmKnZ%xSYADSGl>b@1v zE_{q5q%U?;^}$fSDOLYxLF(U}BvXg^qN%~yJmrXph+oBuAT<8poq$Oc5rwC6t1!@< zOS}HN9jKz@O@*fp6wRvF;S#JmNR$xv)E7!8vZ8(!!^SGrK~1UlxbKH0=DDx_KJ?<2E?U} zIG(va_y@{sDsZc6qt9Q?xZYE71di?ur*rvB=TynN-W>b@D*Iejjt_+7YZOf0bse;T z)yLQuVcH7F=Z>OOIzRt6G{4E8=kw05j+h^a1Vo*<;`n@`KG${p{W|AA)=kNW@d`Hn zVXU$F%-eAR`=$ow^*sL(4pG%E%!Sjjbx+PN9*OuZ%!}xoJN~-uL&G+8AIv#CvYDKS z%>>U2Q@V&Z{_1H*1q;*hc8sdw3!vSl)!otWmR6L6qE1LiNl?-@G!bAPjEAYPWjUKs zyZ|=fG;xs@X1>Sh6(820Qxnfy!tZ@OY*2Ao)=)Hn@xd6*Ygz4$A-f{~p)i)c&bJtU zoCbp_GCP{i4|udsK`^XJ`*4;00)CA8?~g+TfNz0edc-G|+gE+G#616Y@OybjkZcFq!77$=9)^E*@LAQJ z5W{XbYh#rt?pXl2SX6GS>3i{HJB zBQrxEj)gX4h!wOhsO4tgQco&Q_>Xb$!9)@PUm?63cnhjxtzi8nPTp!mI%{id0Q6F? zwluH>1}vY4_AVCg9gsiR%A}nkvAfAJ+AXJAsn%X{In2Mzo(aZqENim3d*EUi0ezty z`L%(}ZUQ;Wo=p(Hq*i5@cxRaGikhTUm8qrv(uA@pDxaeMx@CcooJ5D${K+imLXVf& z7gG>kM&25oAB|y~p@?<U`7NsA{Y~2!Bk+- z-X344d6j9lGwUouIG8ZqZ=M7=?Y_V8U+v!yX)!NNPkYYAWPHi~;^PZlT2catoG%aN z(I@kU;@?$Z!TTOBxmhe-Qvg^YwqshYT6JK-Snn|Qa@l&2Go=?kW5%aF$?Gh4VNhyd zBF^l7=;U*F<19y&&v84sDJU&k&%BFS7w6OaFIg!5Bn@U%-n}S{=81PXf4p~1bo*&3 zoC0%{uWCE~YIA%J|Ly;}=k*WcEl2S5`xF$YF>=$EqPm(234{D&t_m&&hPBh^r}U{7 z(u-!tqei!rYd^o+sHlWDMl#pC*JNoX+COjgg25Y%ScK*Bf$%t<`zU+wyLMm3}P;dl$sp)Ea`zWb65mK-(JSQJI25g!;eykzFV7?8F(oI5Z}>Z zgKw8ZwD6_0=kk;Ppx;vee*N$_B_)S|NB_1&o*J`NTq2fC$o#LyM%Q=mVI{T${b1Z& zTqc8Sn6LTR%F8vcnse^rteyf)@EmOt1cBfUJ7pI^U}G5nfWkG}f6Q0;k&6O@izB>1 zUV>W`)U7>-gK2CIPb0?;k0Ge{daUQjdJBu;h$->FJlR0GF{>X2EGL3W4tP@mRRba~ zE7H(dY$ss9T+Q^bPd4aA{q$b<4BUpRqLLIJuNbo@XOt*11T0n2Y;=tV zg3r(dSYoLYjG+u9MxVtlJ`|xeIn>z*9C+=@L>C-f=unBR0~2#B8gLhevMIc_jU1z? zLVbv29}xXee@tG_5?3Kl*^oXC{nK%N@eExnysPv|o>9&uM{gmsYwz${ysG|gRCHkE zA%%Z;lh;C#!9h%S6DAV_0|U!sC+`4FJi?nO7>5n{tsOKE{715q^N{Fxl7fvmZL|Df zX-C)h@B0wD)-4)cxD4_P3f(4KhSYENkCX4bV^hL+1raBwBZQg1gnno_8usz`?>;TQ z%F*eqS~c3iSn-jA)r|k+N()wqNa|}Vnc(W!I=^!V-dF}9;3fPg4MNbN+NAFS`1eL= zc!@8E^nO94C>J4tNZTeT~}ID<3s3Vj8olN z7ZY=Y+K><_%%mWu$yq|R%EA-9`DQhW9@64kEVL{%GiD~lkDu2eut2aXvO`U?yyk1p zePACUA=}gt#C)LYJ(-bai@oEz-yXFhZFP0+$)sC{)qxTQMG`=ELN1n?tzgqwnZLRp zaz>6aF$>-3K|vu5jLw8g>NvB9>Y=jO9HZxM;m$L>JM&M&u)w#Nh?(N@;fliTL@htzqo3pVIGHu6k0=-L`D*+FS+I~lp9s!=hCMpMXDeBt`uyoJk}!t3HwKH*c{4g2mUQy429m%}uUM{r;`R6q$T(6T zBMl(dgOfj?BomX3^z-RIYR_?0qw2{8C+Fts&(5j=Mjhy%KOLX~{EYXIrDiV(|7 z<30q1w9&ZP8O0}c&-yL}?LR`yLYu>#R80d4bFIu73HSlr9>@Rn)Wzd?W&B$y(6E?H zY8~tqA3g#ilsNZ?$&g&&l3?a?=?tlK&;z*NDWU|15 z&sMXFK%Fw#93Pm6@cJrYxr>MIus6(`l6xDNrDj~eKq(%Vd@U2m9J z2#V!gj)*a#V0(zYt?BwGS)WtFo+VegN62kn#8vt3y~|rBNi-ly{Goi!Z7FN3|Gp-s zg0-?tC=ALmG^1^713-o3Ea`Qj!VK}OYSxEChAOQH8)js=_lA8}!J;@dB_J&MSpq82oKgMHi8y;R z<$GA?y#57{l%UOl;g-F32ruZ7tH;`FsCX}a1q$}Ea5PB#Z9yrWwiRoJd)tD*;MR9I zptOdv_-BvsW8n9Kg!s%I(+tdvqTsV?vhWJ?EAxpMKA2LPBvx^3DgtUMP==|+b>-Dh z33pZBZHJ)yIK0kQM`)|`G0MlS+Kqz%w5dJLZ6%T4dcM)zqzTjLzkpI8m{75Yjid;Xkf(b{cD^ty&|(k z6(SnsEQg%JrZE$=uEu^_f+hS8P6+QS{0`koxIDO^aUCbXmcIMD0%D&6on_+TeSz(A z*JU(Wo+YTz+IP%Alvp4;YxE*fd~CDZ4AmJlyKQ0l#*7@=lVglU)83aJ2|?~y*FW8D zr!kWkWour?P>zT%E93|kn#$rpnc7#Bkr~$JPzHbjD5=iZ;uRixy$yvA`*ex z9Noa`I5jnQeOn2YVQ+0X%&a9;vd*E3I0KY4xl@02K!m)R&bdKoeVqg09vPt<^isCa zvtV*oUReh?%8_O#JQ`0x(7JcT#Rfde&v&mafh{}l8>m$+LQVD3@fvI}^i5|jFjfS`WCfqycf_h4r*P|vs6T4wVkR;> z>G}syEX)`3iYZ+J?N5Bl(_~7g5YH1VSr!LjnE1|azf*dfl~t8X?+iAQswn~x31n^CK-u!@?uU>)J-{~WE8J@Ual8i5s@58P zc^%>TQ>$B;4v(!QnT14fCVdu+W%v+17GNpujDdxxx|C^C_LYK{C#(QcZpqTFy`PJT z8m%0c3}=ZjtwV_kjiAVlGiG1eaZ%2ZmMSO;r+J_1;lVnrqfCjTUJ1 zJWdarvt20b_%!NrRa1=Lf4Lh95m>1T2`O&SKchwC^-EoUMV8PYRUGZtILiqPh>uKE z6ymZ!nzx%dw)pt*>=NDjT0hcK3)muz&4A#g)*jO>PL1ASYj3}vQwcnm``gz%xAkhv z=I!;-S;geziL*#)QL`3V@)!M`g}b>NbO+*^Z*}6s5oASqZlyCSOeW*CD=$B;fVDC293RFgzB*WWa&}9JlyiB9;C9 zVCv0gD$Q9PQIrxU5GL{PQfX*Q*(_%bEe!0Q`r@DiuYs=g=TF|&%HIGDqWbTFGN14! zM{BpFV%a+z+>H0i^Z)85Lp=8f1`E>;Pb)nYc5pAp=bLt5#n zQKX_|CU@A0!|bZz=7Pm_w3US#>>&>4{1^9h9jT(EY{ z;9YCrN`HS+QJWuB6n-*&ds&rPJ|Q>U>(YYP798zT7L*}#9)y!BA!BK&jooigM9R!T zJlw3wl4V#o^fuqnmqhE(sS+|cu#Qq2?Dh~vecNWzEP^!3`;Cc>ZBY7)kcM7{D1}i7 zhx7cvsZ8x;RKneKgNrcH*hS5&=Wip zhqaMT21XpFaGB}MLB#{o-ohkA$8eoW-ty?@%=MpayuUno$7d)wMHPE*yF&3(_5MVl zkY*Njh;N{sHK_Au$Ut6g+M#Y|24_C9q+;pHS}GUGI7EIk%f?YNx=g>GxeFbIOi){l zPJO?};mYYPnNP}3K;U)cBHx7u&K{-o-%Ol{XjlJk?Of6vj)tnB`AB)Z$4c!a~_6D%wg(FMPobpKQff`b01j)!wgxxZ%?6H_2!pDolgLJYj9gw=gRMuyb>VO-e6JnZHD zM5d*;f*f@~xz%=hS2BO;f3N_jz5=EU4h1OMe{8}Y4|-E>2`tPA-CTypvS)fCWHV7< z(l?%?NQZHa*sWSF5!I2jg4Ea;ety~8Z*5ru_tJ0JJz_6X2c8Ta-1PdC)6{EXj1^$W z*1oGGk|ydWF`3T0rnE`k71-6*albk|7Qruu<#m>V98RTWtDwE?&ee97E)b{DPGNQY z9hs@7X(F`x=jMptR*Fp#d~WRzx-ZcQE<`dvDkWL@j2OBdOE1Bojk@G;HZccS*p@5T z4vkE=$(+1tFV&(<8r8>%#OJYj7#X&WudOr;$6@f?$a6r6Uh57BYjW6u0jg`^8hd2v zHZy*x@1d`e4fXQL6?A%(BOUox>v620bK!DqPwhGUr}->(7X4i4r&zIj&-k@$5P2YB z?Op-Kz2fw-+W;FS$w;L}u=={(XF)eFrwP{QKVi!PTCR5$D zoA_`2oQ73%TH&_DD#rwF^H|jsJk(Gx;dUv*{|}l)RfTzT9nT-6Pk~t!eA>9)GpM&#TuoCN4~w;R~GTT zcnK@iIdFR0uw*mNgD`#k6s2^Y>xl??ZcK;upt#FEcot9*k+U| zj@I3WbSWG@I2rdi6g>;s+r_>toeFcN^ZgL#!-Zmx;Qd>~CC!u2y&swaRP^nR&_H`* z>i(=P?noMvHdCa@wDS2G9~ShO`Zmip4})BQr`6~2ZG3GZC|6BRanbt@mO$(F;I?T^ z$U*f6=K^A6f#@Y=eh*8+WEZ;ckK{jbgz}0U+1A>A)(o27(%gq?dh>6!H}sSWOApE+ zH?ZcIHWUdV&)uqMWFl!?8N-ezJzI2o>@ruaf>94;;wHq;K`iL;m zY)cng0J*pkFU9%XcPMQeQ=DR3Ds?2kF?(I+n1s=a>MM;JClTp9ma{W-6WCLu zRlSS@7e&j>1L#=EsA{bO9VGZCQa(s)ScxY1m~uFn%e+j03X+@U+YHv7@B^Q2N4(?* z-}0$}yF`)qA3p#r=TDWz;d8$`S4wvvQ62r;7-4@og4=Ppu~0Nk^mC9BS6OC_gOswYeNut{|vkEXegb7?IB$M+DL_%s@~3NHuAa zHyg2Ks<6~gKE$Avb3ps>Wx47$%$Fv*9 zxY3b9OQZOhKkHP$y+bq(WFPkT2wYG_P%IJVx+Xfc+X>-HWtucBUjGeCdn^S^bq7XR zj{Q01kXKZk2D}UJpCVZR_lRhHy>IYMPAC4mfX1DCYb&(DCv5D-L~6g0yVZksa8okN z&{3N8kTfFtw+%o48=YX&*$2P;B4;R?{p)OQ!D2QGk5A1fWI^wxUDUNE_wQlC0%#w7 zED4B;;t5=|HX(|m#n%geyhN>U7z-ax3qt}!T`?ZV7HSaFj3*oBeLKE#qS!uE&DJ`y ziqc}PM_4+F+Z@i!;nRej(+p{i;&qXkt{%r?rkHm`xTWBr{T(0N$`9qnr}_4Jf(yhY zMb|d;m%PSky%B%v<;(?{+%5pkxb!@*zg%MEWWIK%Y-s^-M%fGP%XhVZU}R?>>-b|W zP}X&4Wy7eri})Fd=Q2&oxu*c6y4S+3J8;sZTs zQxwCwz2@EHM)p)r8q=qgW!$K)en#vjQ$q%H znxUcQsl}31DZo(ysDF<=oSvd19k@fWJz#c6w?Oq+^YMZ?U+e0vliJ{HO^K)pN@;4! zW`6L+Z8V$KF$tA{)z(~_qZV!}sUUT6PvYl+q4@A8>5IBgs26f?Z>unWu@BVbb+#e$ z-5fl=r|<<2tj}B`&p3{HJo7Gt2b*QYnI*expf9fCbzeBBM(Vu4t-hQ_580jgxV~N zb*B7rymM#4TuJm5hJz{nBx)^6qUBx0Kl!HQGR&TQvf-9CT+_x*BJvM|eSd$q=ryQI zXQ|y{s>)$NXDJkLu1ZW!LP_h^okNB)`LAw9X`{d^PFM=vHv#-HbiK|>B{Bn5Im5wO z52<(_#>U6}38Rj?@Gc`Ki7J8lIXFK%MKJwO$6pBRFD#v(9qyouq3Mx`g3e;3_kE5h zu64#KU1+2!j4H0IFvR8DEgDmmjX9}5alzN)dyX-2T&y)8r9SG4RGJ?tcmGUMo|&8E(v(_(cx~;7 zh$OR=8<^dMwdveR8P5MoNS5h%{v#wevS2oMlEath4F>28+xZ@V%B3^jBD%k>WO(CU z^c^5zE0ufJa#&0j4~4bY;tI-TeEoWEW3W}3t)7U#(2$U{8Y|x*wwGxzSo}S#dPDOs zok8Wf&#?;+L5PyrLeH*DD`=x+qiwt1r-MI_NgL*Mo~#;)@tXZ@E-0|M$nlfiL_9lH zCt`fM4r5&Btk$9>Mt&XF)(m!sTTcF$l|r@~9z||h%I*0^sZ`%xitWb4ddBRGL7|D= z@)RR)DsxQ9V!d=Yg=<1qxgiS6pA=)=nKQQPg_Ubb_0|Qywpv4`q`dcZ5dnJB+@L10 z!v)x5hxxC``gu-!%e~!>+=GB&4~)B$)6PsX2W4)n$NEHbeR^=~SVwBc03zJ2KY;QT z`25!7=fSl>h8_L*rT~X3v-zY~(uhJju}yc1kxT7?w-xrW&eOmq23T(*!?Oa6)Qy=4 z`gUD>_=hve(1(KB%bO8PTb6f4Uh|CkodHr_%MpUs^Yt-0TpUCC!j+I*UCy(7k+QY` zq?M;;v|kIie=nz@Q1wkr%@j!5zjo;V`q@#Ber>|U7nrc!e;(fyI{{q`&Tr*%<8m}r z<1wmF{=tOX`skOB@_1Z!Erc*M3~xt^FYs=9Np-aC{!}I}T&L8DHhZAflm|1p84}m@ zfo-icNfKWeWQ)3-=#Sqo&Bk2O@0DP1xo)8YSI&8_89=Ccj>^RL(h^iKvN*K?+L*Pv z=yC%Z0ZH*v=?{bRv7k<_)nDzo{+IFIk=0D9s>;;qvAk60D|LDtI=^8so&Yzcge%!%RBq2F)%@4<0KcGim2I1Vgiu_0!BlhpM zX7I4+XRfPpy7^YDxzY@R3`xMoIc{+<_xIaXoPQ2|&&_Yh6tE&Zz8tkPdgo)F1+#O! zD|P3ylr31!E+}h!Dtx0JADvtH`;8#Y2np3xPEx(eVsit#2n!QtdtK{_;bI{QWW2UJ z=hDe3o350kNN?fKj*`c{pPy}>O07D&BpfYRev~1}+IB_6jWw46yddQ8FGu0$n#Ft- zqB8s+EC5b+chW4P^a8-W>HF*Mi0NpU<>EZh*<2t;rY3H*)Kce)d~49%{ojfDE37ZV z8{3jzBT!7zZ4LAp5VsXDLa;iToL(GHJkxT?(IaOx{{mZWB-)-#dgXuw7e`FTl01w8 zIjwj?PKicw$rZ`Aq%S-2M0W@+i0#O6Rnl~mHguP8tYMjqNitE;CR>GT5aL<({MyeJ zaj>?qirT%Vy7~()QaUR(b%doz!{y<3fkLaua#-R`YQH_0w;1954UWD8;+G`&JSCs8 zFXba&3q!xz#+(nOeU@6cMOC|RYICz61&zyRJaiN#RXA$b2ChftMCq%w)88z|5YV_? zd?^9QKnc>^9!!|Hu%jcpNtcf(>y1+T_*_q4n3Z~0W~P2PvQcvZ7z{s+7#s8G=rod- zU3cd5>d)5%&RB}qi>sA{3RUoBj0^BO&k@@p6BQ1{ow0+93}^hyu9ZE&h|}?<$grbo zD<_9TMOieb#r$}hlZCgpLnC7s-JVDa4~wQSG-uN$R#JEOpc;HbtXTVHkU}8QUd4bYLoKf-=+i8$Mtr)JkEnG%oRiEc4nNniyzx*BnAn4c5T(& zeS-|>^AuvF(sR?&r%Dg{@Nu`e)LrHz+C5p@iCBbJx4T?d0RCDy7OyKf0id4a8~H+z zi^CRIc;KJpN@;~;W$)hKlg|O9p(>|QYs3bRxanW*K#0kr_gRooZ74aLm}!|OG&wC? zEx^mss58p>9#~(|F5=cWpn7;{Tt#-ZDDXW=aCL(pFL9lOG6f6`IafNT#$N&(eK81* znGZz^L%*^yK6QH|AdOm*0!st$O!`qdV$#Sy+YiFWEdH92v5q>VTI z-n|{{()7zjl17fJa?tho@WWa=$I%IG3~Kr~@W3t2)SAhn-e8Ak7*Hf~iOpLP(`i2hUggA-b7! zKRNQ43y1+N&(hgm_(U_U$%m%*}8rKyU9jBCPxE%Qe14o<#pxEvEas@xT7%$2U* zgHX_Xi#!+ z%jR~mM%{C~-5!xo<9iM2tA31VX|lc?|ET?k7(?N4I-2LTX+omWlC5s;b{H^%L61xo ztuWciO(%t%6kT1CX(485SpJBS;BkzVe1LXUVQ-NLw2h;K*!w#++r5H2-2ATgBnM^E z{s?V3S!@Qq<)5Wgf3{yt1{Q9-Yn^6xJKR?1iKRevbxn&#*;6f*YI3NJ4f~qK3EkX8 z*mOf4P2$MFyviev>42OM`(cqhnIdCrfnmyaYy2<&uJN8L!+BTCq~lIA+02cj^mA0a zK({ZI18rz)|D4&J*+w$Q`*vaFT4cuL`5mOzt94^AIZ(5IN8i<|(Ifm3R3#F|nlu^d z@d8~vYV0bdP0f`(lw-QANRSTiF5M>ftJ>2$BWw9 zwzbR0cjZ%wPVy%b!zMpFq^6kU*(v$JJx@jahQ?_*K^-)fVk0OxV!Bn>ikkbb1W{&_ zopd@o&y9B#HVJiR*yNaDM>xGb_10I!R0i`6hRf*}gI)NgVB8&WP3_;!3oLSS?2{Ll zyXWm?m*cWNxrC5V%!Worbg7W=jgw^$0R^+4eQDY5t*VGmhhk7VT3eVkbrpI=4ebL3#ypiJZyR1=+>j>#G)+R<;_U2r}9{YcyqVp}o?lW?A~ znhW*Ey#a6AIcJvnZYnB&kZu0@zH-8r#^a$82i@n^%xt1rJd<{)90{L2#9I49GlzYu zUdlBtUM&UBK$R>)BCaXcwle)87$&ol7ZYI-t(QN&j+b1-P-QP(L)D%=Dz>k>GNDNf zhOM(H50oX|=z6F14K{rQ!OXov=T;KWE3hr>&x!5sjxv?OUKDwaPY}QOuOa$;TAD*e%`>$k8vXUy-sKxqSI)GsQT0#r+ddNW>&%2 z1+I33jr4B!3qpFg-5yB{)(?ZWqv%bcxmuW1OBxK5eN2w`Nugd>^dAQ9dcU?hXn+H) zVwmmA5iq3j z%&!!CADM{ja_Sud9I#t^+ZTq+{RI7I;vd=Sg!mT~K4$ZznKr4k zceWd9n{af#Dd_A26Dea8h!~EU^vl`ui?_NmVyVnHF2CF^m7e zY3F57IcH^7hm_=z$VMuUsFh!qU$x$y5Q2Z00{T{BYwg+%%6uWSlr2kFz0ZTc3{LvV z>s<6SKv`K;)oIx9DAg3*M~24P^u?`~i#=^eH-VG007N_;Fr<$;x;SHkV7i}i>q*K* z->FQbuaaWR=q!|r3jK5-&7Ih0#Q((M@fNJ2Js(OL1E1eO`MipT`xCDBeQS-RZ0@@k`DpyU z>>+I1A<+2g`h%1=cytq!Mn_t{!RfRP4mVwHPye}lb0r_a#mzgh5(`0uk84uUvC9z^ z8qYc9&WBCw#h3z#E3YkN&!Dmmn~FiYi;_^fJIVi+WQr=()LSHaz!sl=2S7kDNtk=v zT2TYsS}3!K-P8P)#l|@gY=$&EwHSg-?I7d+vhziwk`MwT!3R|GcFF#c29zlGQIn*B zYQpIVD^F{FFG4oh!aKsYU=xiQ!QdR%vgxF2*_I7h5U#ytOq%(?KDF3Rv)i1;lM&LL z`tbioe<=nH0Vj2p=WbJ6vlKu&2Mkk34R?}`inSBPtBt>@2w;}H#x)a!`cw4}d&gM& zS)bbj{a34aGycK7(8u)UQ5jIW42>Il6^MLx9&WvqLxh#((Bo0-Fv~9Dg*WF2Li}QX z>P5&yd6r0pdu5;Kp?^=C@$2|1d8h>2k!?Tvj=EAaEi=?(o%=pfby~ZTExJj;^$-rX z7sR;Hol2MPg>M2$d0K=Fb@UeQ)j@-$IH^~`bJ(;7mrEd!Jw4NgM2kj> zVuFn5w{WQgf^d&%qLnOhyGe1gm&#huGS(5A)N1){?I3%?s!l4>4KKHsOCyY5xPeqf-x7HBT)H^Q2&FH3 zI*^S_J|0q4tN#X*|Ch#D(y*eYLiJ*>kI6^|9-rF>ID6`}N5YZv)bn1}%iN%aXc3hp zfwxd`2(N^9RAKnn2j`aY4Co)Ni;SQjG2%5;FQ&5`^Fn?fsxGnL%$ckm&cr1BetBHV zv~-R!Ld#B?fX!!{wu(wAx0XXy;TWB?6udkT9vP{9aL)6xMj9_4FL43yjzE^+{;d;?iUC{J4+Afb^E!Ua*4dmkLQgo-I@ zS9AvL%S7(5A#LZh-fWj6`K z{9(C^{1)f^v7AN#J_I4!waQtXc?hOq|-Wni6wPiPX?RjF0YqNJ{ZG}ax z376w}wA;|%K+$@?9$T>)xhR{SMh*3l+Uvfj?!k_9$H>4}zc1p2lmVpNk82Xv0rv z7u-qMSbPd%tE-!652uF~EBW0O zY9)Q$6DN+3bl*)qMWp4@?|6Das8{}b@|Vynkd!Pd%N>n+2Yu4!RT<21HpZkoE`*ou@Rnd4AO1M z?Ad_8^cSkEa_oEKl4m}(k>(^>S1^zoMEB>3re#QPdm57Q>c+3tk#o-!)E~S|G`R6U zzaobU$JGC$X*ULV`ah)dBI)@q%PE(i9EkBI$vp1qu4cEIaTNJvdKW&gZxy z;6A;Lx#irm;HHQi4EA(;y2%a}axmp!8C}btDLJyEc(etN)Y7!7DEYK6S9^0D)D;A1 zNoAXc=h7Jv#+l_h$7tvuXYUZdr;nF;uBtTHab%Nrli{?$VgMU=`@L&^-1!Hw-~V@l zBTJkzlf0fl#dDI^R6dz)CO=d3@IvWMS9-I1UyFux-(C`8TxRopebZ8Y*^02=Iv8Z4 z>D2|Wc#R*ehSm4KCdYzK1L7#Dtj)BK?aS}Kz2Gc>%fc$Z4T%x7o_>+71&eO0KW-^3 zH#;xz2e%FjWc%`rwj-G9&Mn97N$~OkhQ2?x+iBk$UxuV~lZE`FwJ+W!sV_ptfFq&y zwJ*TUpyJEeEj}M;RL#n!aEaDYjE?^Da5*S5H1H#+y5`qlLChZq>xo+-Zn}ig9W)04 zKojxj(*bLL`2zA@+IFlNx-bYn9ATwz_pF>xxt+I#Z zP`E|v{%)_UOR(9GDrVP2^s~#`^MMOSv);|tMrKDC*;;fBb4B-4j@qRne~tA~!6ghQ z>G8q(b8S@%@dHN=wfi^|)PzzYx$GW_HdoNl*zF}!OFZxLs?U;>33RA9}v-<%DhI5vB|_MTiTzcr_`KBM z69NfG(dZehPs+sbP&{=o`$yDRR8b5*A)Lecu0U0M@4YRHsnViPIC*^2%P$o9t}Xb* zMDeCur7OZAR}}BEH2j>&FN6X+d8%w9nwHbUSyy1`e&c1n;(8>sW4OE8?$bL<4TXi$ z$Y@Z*YVGR7e&$J5c}BYt_btKRT-kM>2?2s~8yf~dVof(nFVs?372 zHF{~GC&3{VmonL&+_?Q3wxS?3VMIFhn@4Yvfn(h@ux9es;z@!NG{8*2<+^^ImI5Hr z3d-`h%jXs6qShx<3w{*qEA6yIP=8onJNJSv#vi8oQ+t$vQssN3%Vn-+lFke*r|mhK z?d7Khvx>>aFO}@OPqls!nk+dBP_xl=g>kjJRLZr~^-v<%G>sNs{B{dIHQFq24pb;P=pK!*hyQ|+1WUn~YwB7i4i6ngI}BR`0wErmU_a%?5=@)ek@`ezzYL1+5$v}INl zKny=IYJMx8+gDln>!rzkv6bQ%;Gm&E&{4!{X&(_e$c9n=;58m++_TB0IGAF9*_N*I zh=$XUI+x<%ICH_&D96+;Ndc1(lqJ6I)ADZe%4#{|De+?9X#PsggTZZ%&#Uaxn5nQk z2Dwl&SCp^v%sf+Fj(V+B$g^as;zFb3e7I-}H_32&8L3~j3CwX92gzlZMo0tse3tra zYl?N$Z9F&Yy5hD$(6$KqVTI}tIwJFNb+ENfiF|;QqS7j(zdUd3WUFuX&2V#AK|K2X zVnZ#{VCVe>ytZu0v#N|1+0N-*3$UqDEbnBgeW_W?2tbG|a{`P}o0GA-%9(@+1X1`Wm3yXVIZbW{;|cZg9R1{26W*mq z)9|w5V3_f-sT|H0{39lcmEfTfou-?TC9`X;EYAGqTrOoc7to{KS?Z13H00SJ(r8Ar zQxvG8mtq}3X6al(-)rS(RoxSRk-IIkCiF>lNS1jZ2s7sC`eb0u z!ZsmkaWCwlbJ;>%qseG7l7O~VK^n?z`|{4> ziSxs|#{7|6*i;_MBo3<)wo=vmG|nTh!&~A3Jgah?wl;)NCZJ?)0@{rhmV;YPdUdfL zW=p_%Aw?8i#!z3@yV$~-pL_O)7&d4>5nJVuo3C1gEnK9W)Ul04YI7_7q|*i6P`UWG z+t3nw;T%C5a=oSQA7fOuX)4Y$q~?S597v?yGOC=@63)t1yP>;t26pDU^?*~TQG0c9 zpYb+|hxXEMe?rHIa5%jKwK{AiqQ#rIf%k=>e!x1J@1LbYX6pG?Iq1n4tkq|iV51Eq zlj@Uer5Fny1}nq)zRF72#&{|lKCJ1`5Wn>G(k*$+Bq&Ed_UYRF0cRElzLH>akEudj z*-~3h@Rt_)o4QF`UZfdN-tmQgP}APezWY;GN$ti>rKL|*WPHLR!tY>VRaI2#t!DOS zYN_+2l4w6_1A=P62YJ5fNCt>AM&=(f=7}Iu7D_-l-uk`3Iwn&|BtN1I->jA{M6GwZ zg?zP*HF~@1Pjcx;tacW({dl9{?Y7ixWY$OP*VRM52ND{7 zDW}sk4G~%5ESfJ^DIoqZ8i^5Z)Z695oQuSA9J*-PkAQl8+G(3Ix#k4JtC!vjfH7{l zYiv+u5Sc-Q9m#BzUMmi`VV>{*_)`&kPt}@@YBblAk@id~b!Bx>MjkdXQ4QrfUU;mw9 zgmT?o#3@Bsa}AdV#I=SoG(rr@Q|~YV5z9|?YS^{z99z$q zCO4d0PJ@UHSx+`Ws4;FSSYV(QnoTM4&+7Mqg@0RtiT@cDeW=4v*;QM)N0?+k+}sob zqNl2ry_MEtM?#Kes^aF%t1?uf2y9~&p^l?A(^v_C)Cbcau(QcA?@6r>deGOg&nY)l z45cY!gqy0m&vls$VTQjq0s57z1Gk%~CIvD5y7B~nAQzy_xs<_{KX-YaY1(%VU9Buh zBBrmHuZM(Ah*ryu)O*~Ng|e$XvWDtdFG4EQzZns+%?SqAdD?>H;B%OV-9D$R9ztRP zy7Oypc=jp3EypOKI=AS{DGwJwe2z&cBWhQgD+^?Pr@;_kHZ(9$RaF(w3neV?{=a1@ z;m>U(4{Ug}ua~$C`vXg%xtS#mxoK4FhzGVTvqSAEA%z=b9pVb3+HUiJu&JkIymEN} z5dAGQXHx?VzjZ;FeJ|KKj~WtU{QS==B;^ts5_>1qEcut=xicc$m%_~Jw)#U+kkr=t z9JX86|FlbKb8ckXE3aFtGv4q~j5le!NEOqRUUE6-j-*-%Lw%<<;q;D6j~>y-!EDbR zr?y0_S4htk6d~ZP%J__U@N7{WUn+@spg9EWe1!Y}Mi#2fOFnut|5sssfR>g$5Msl2 z#^^Ed1M!Ol>D>HW^r&xv_o2IB6M2AzF-u@Xd=dF!x5E5(N3s{)_UY4r`1y(lilS+|);UDZph z$4&TnrBQ_QEs^*1ggulb4TcT73E&;=hH@L$c6DVE6@{h6C>Xk3$|vM!pr3<@aIl=Y zS#eBu7K*G9+h>UOs+0Lnr4UB7S&)gqP!pY0qAvG;SbOWZsJgao7z-6eEE+^ax}-Y< z=^naOx?$+H0BLFI?rs=LX$I+%mL9r>u5S%`UDtiz&;7mM`}}_2{!5wJGkexvd#&R< zkK;T}S<4wIg2`M$VTEy^H=@SIN6Ss%Y1lO8yNWiJ^Kgtz>vt6@s8a%RtNmrZUu)H2 zEfR~1i~M_TU~?7D#KgS2+`!A<(_8dVGzJv3qK--V4Ky`>+}Ka#1VFj^HsU@Z_y3e| z`vU9SCtNOq%w4il?)b>Jd+bBXYCyC3`4L+S#~aCc6&@}w9ZJg zqf#JWO_HAl1TARdA7NJ@$#37$Wf6jNTAp8f-62WL(miiaL&l&X4P@@izrM686SVr@ zP`~`?cfZ`Q2SsawDHB(UL>)V)ptr7#pc(TJrC)w0T2kpe7Ff0@Yv+zZ(e!Wcz%WJ{ zCp%R^G{G8Rw($FH2s$vA&h&-Vv%_xQK1g68M@!nfeFk zPMzix#Js{RUXC?B2y)DWap>JLQFBb{9WGlgCcF z5$Qq?Y=7t44ve~@+a=^;S#00yFaEv%9m6nleXxiplU~k2q||?xb4BFhN_9Cx?fgI| zvB|>Hc`uR_{|1jQ{%8`mo)j$5bcuNV{f#qR!D(NXSjRUCL#k zMy!%?S(;+N2{@C4-W8Q@njS7p6Ht=O@9pdJ4Qr*4-=o&%c&u?hhbThzZXBzfcfUOu z56@~(*J>ia>nmK|MbF+`4STLTU`;?g;kx$PedvWrmka5YcQjY8(6Pbc-CDK!1^O_N zT@_#7ZMzTOUs&7j-NLvR{sZea=Kju3#-rooV_-)c8g0(bGp{T^ip|Z*sji`6HBn?S zouHR08bB#;@YfX#soeb#=6z2AW;s#gd2-}WY1i1)WDA^)tfuQII8A^-mXUBbni`8? z^h9uRPdCj|Gb#3qJJY8W^ji~g27O}3Cv1}!%S+CHHif}HArRzmwRkNi0sni%>L=S6GWCb zRI$%V*S133`}#tdnEF6vk=aPawf8yF%dq4|$r~Et%Ge)&r6t>fS(rOQmst6F3>4H; zhZ%wk0F`~Eq-JP&?>|aEw3Qb6C~_Sw=ic?ALCfXP1<6h_mFMh>0!jUg7Z5UC>bg*K`kD(@+laK=X^u zl@={9+!(FjAY`sQK3C3e_W3HdG4^>n^KSfl4Fd3#okCnze%Nqu1Y-;2%ScGnd(1IwW4+Ij0QaN!gZx5P?OSj=SitQ7p705L zW;5HVONbvyzrO7esz@!)r6qeIRJ95VdQL?KEG*We=#QwGX`UTkTw9NdZR_ZxX5e>N zK^6lJ`CM)oO>IEz^MP*p(@%0}Ns@h~Z@6g6-qrXRNtyzmMF5~LY>ONN0(~Q6l)jr~ z=6kM>mb=EzXNLeWcShe5@H|&}Ay-+z$5XL+I(cxXW2tSpn<21^0?i=F@>@s2^6G~! zNqf?~>Gt!@&D0o7Rx-2;uP|@#j%tU{8>sLF*y7-SeDiB@rmb|&`b+!0#ZnvnUgqfZ` zpC;k|?ZE-}sTJI(1&f||zC<)Zo{pf&SC|?TX8UW|_oZwLhDXUIKjwMLq+-@Pq} z&uQZJ>`7GTQm*6g!dLJUqt)nUt@~qGEh+Hjr$;hh3cYn}*|l5w8QnDFwx+EnC^32+ zRnCT2a$W!BnJ2a$4C2O8k7L5WH4gYqi_pEvzK=Q+=1jhGzp zqVsq{Xf_Jz45zdW0i zwOyH)?sl&8X<%Vr13$L+1=omdNzz}vSP}P7oA3NZz#zZ#7wb(fMes1;!|sFS931a9 zsuK0pplEc7=-%Cnqm;q9=15vqvr5<${rUvK63@`UBmYpMRf(>}@KcJ8VusKh$xtbH z?Dd;6=DCtlp-uk6ZH1?jG`CTSWz>B92pU~~G* zAa~;9VctD@0a8xxdAHZ|WLLSbXx}O)`TOM9f3{e7N^-77{2~q>-ZEvKth?)8R}1zw zzRh)D+k@hqP)Dzf|03bd>pb_LJ(XOZw((ZT?{0Ir3$}#Ozp=$6Qn@LOT`$iU;*B}2 z^CisXaUU((`aj?z3BOc&qYcl%=(g_%t^CQD0IqPCgPv7J*@tMo_l{p_R>P`}NVqgj zGt#|=+8@lpWpjrpEsa4E05n@z8@J8VoqqpH(JliCic~7BCRO)Kvx2)H6NhNK#1w{c z>ynnIrG&z%l-ZOAVfJnn=2Nzf7qgoUXA1~~^x&o~=l)W+4iI{Vgq`DhoEdzKk0%BX zZ?ApRSi03%{#Mh?8diMqR8_|PKwxMnm}_i>CPj1Sj(>e!0}W4paUhN9-{jptl^{&n zyvhYgL}3+MgLy41rD572VtEMe@vnPfnBrJj5 zXz?mU!fyHJ=6``655P#rvs-_WcpD^PTqK^H?!B*VqMJEh9Z38F!t<`8Zyyck?7cf! z$w?ZlcDJ%btr`b3;Tai*OV=!sD}9wtFqu^cWcN`*yboCCt2{Ti4ASDG!WVor&lk`xWi(dL89=%!lpkcZ0QG??FkYstT%DR6fWfA(~D zAKUI8XZAelgYf-^L*VI28fe{{S?B$(WE}E6Om4xD{0SGY7uvi1z7;f_Fcb5)rrcID zWuRq-Nth96@gN(`?%g)GzcF$&yHMIPfc0c5Jvu+%LWA9^?y!c7R`ud%vanIMdukqC zzAY5mDuegrm|Oq2jLR;>`A+5qY@f1$ow$B&NN$Bcd?HI`1xU zc(N`@bo{@E&aW96tx~9S==sR~g4{Hg6)xe|{l2Oko$cpokvinV5_`(x$0{u8G`T{i z5+dXmXYXRTestncD~Mzp8W=I?FoSB*6zAOLW~Y<~+-_@%2OHBK=d-utx3;#3K(B&v z$3j#xXv1_U$mH!;N8r78W0%9C*RlOD_07m|LiHX9^z+jSUQY07aIzi3A|w2(<9MTQ zbA6|rYU7m%jYtMn&GcvI(C#&jNj97$>8grCye&A1sTAJ*MMZ z7&PBEh&|u6h+inD6PG&O`t@V;O%FpmTz-f)fe1d-!T+Rsnss)#C{&fSiDhr3o|Okz zkB;_Z49)Yt4-jP`IjI^h*(X!2hR~f?^H2_o7A^SOPI!`hNNA4p*BV%rC!aL}uIoX^ zk*OWOF*MfmqBLSe#XlAY-+qa4f1I|28L?HYYyNN86SXF2|Ai-XLw@l2m^(1DPSqGu z0Twi)08OR3kJisHX>|U2(Wk~>Ie%cThKYB#nwIQYi2RQ52M}x&zZs3}3X3|}cC0Cn zka_v?$r&lBbL~F~ITpiu{&+AQX9>qgukt2BuX7itI6e+q6ofnuGaH|8u+ng+@&rv^lXINO6!&-m8;i+;# zB8ODju=a3jL9Ju$3ttqi)KF zWIq85ABD`y|Y~WGwo0`wuC2yee#v)!vAb1h04-7aYacgatTEjb7mgOF}sGinX zMS*&Eh3I`HYrrRK)E3+2-fC-8#sixLH?~lQPp=;MMNVbn5+9ZID>i;) zX~-n^_>w5r@AZW$1+L7?6`mDlrW@mB6sbr`u6@h*nG%H5*C9M7g`J}1)qLc8);q|* zasdPTGN`k}cYU0*^Rsf4K${>_L(>YEt{sfUVAw7$2R-D^FW2VYzj+Uj*=YC~%=Pa$ zAqea}2r33$&n^PzK3Z_&dFXCOf5ms$*&6KptaC-_!HVg7?A_-QveSa;AhV-dkT6|0 z*E{9T!=PI&&rAI9VR&pOmfO)%&*`G}9`nKS!&DtVBxA zm)n`4RSoqUvU7wU%0%SkJyT9e(E$s`GT|XLtLN9Kv1jqS%ju?nyuy|Sw;QRjg4>9B z^B6fcKar0aQ}1a*}f(ervBi6)zV$1SMj5b<6U)VVL$lLc=uF zL5FAgt|m_nYlVj8Z9!!p*cAiN%5QiDX8al%?^wvNp+|ZD-9zjC-v|`ZjrRu?qCu+J zNz_X{HmMIr3W6~2EE+vR%QF2NrD>}aUcgjd4Q{pWMDscj0G>%XEC}Vc3+hE)A%xV_ zd*@KN7VS3K9iL&nU}OvQj?L^iHcj&ZDA;l- z`~n|@23z#k5nzR8ogZ($mtr0O2^4pfC1fFZ`$4 z>^4Y0n9Cou;8$M!JUwT)f<}C)x7t!G}=vB{t+&Iz-^Jn@bF~+eIzBhyVIglT{L<{`k4qRt~gTr;AsI6cvwPs@q4jLJz(-9_zs zg&_6vaF4c9+&4pVZ=mv7ZbARh#`LP^n)Nq1fr5$(!v}i?*^PUh z&c?dz7L%`0s^+VS^`O5xvpe77U}ukJw`b=&b+jD6P;Pj6z3eB*PgqTT0u^EQeIL?) z^UdpzNYhAU%VkDG&YJVvX@)GdUD?gXphJZMvawtVJxen2dA>zSN-(=a+!4qz zx6MxbDo1i!C_*%QWtC;ITLX^}x6XPyZ>ZrsO{_ib_JBBfUwV`0sG~qEUTn86=iErS zz{ycL$W%SLLSM05xF;-zB;>Q)T7EY-Z`(Kry-emm#il`T{{TxqE8}~|=7vC|`dmFf zbdU8BV{;LUpCVyrW8)#QPE`%AZyB)As7$T=ez3+nSh)G^%r$s`utgX>kgA3z#%C&8 zm-C_q#6E5-N6dj=!FE4Ux%4wqrsk$QUmmC2_L)(KT}NO)X5W#hsOY0sxyjPi_JZ}B zV!?4qjE=^_zom8j9&WlS-W_3vr*)o2pp?Dusd9v=h+nvIkrFX+Jf}&-fQh6-d1MB0GM>3s=9ap0>}dr&O^uG4c&)m>#4cd zT)$@uPoiam>52oUYbHRzv94toIWxw%v{+ou51I93Z?1b1*ovv3Qw;oU9<-ZkG$Hc| zj%9Zq2JQVaqzOykoJFF1AP19v;kFDQpG}R8Wf<%Z`!~eu={IZDB2KhkT7Dbtzb2@r zuCX*f9}xS3mzD?Rqifs^Vuje)=&r;A;^4L#Yo??%{^yjGkrPyGdShD0n~he6Fc_>S zfk}XeN2)tt*5Lun6{2_(N%9zHKgb&$iA0Lus2V>RJlN1TFl3~Zqu|(6ZtnW26phKP zq?sf6m|8(I`|sp!*BE^9IFJ@8Mn@eiVSjB^MVgGzJC3&zSsNgB_55aW-rXJlaGE0( znRJZ`abFtAh;xL!?XI=sfmYd;t9Si*IbjsmlQAjyt^sB12VZYC-uVbqC+VqU?UXwl zyZQ#bM@_nK5x*2>W;TU&ZiMWpshBPHmNgWY*CyChjeSXm0)#mKo%%vP%K&%RL40)xZys!bNg8jCAwC7av^4 zBNCq2y_y3+3ScpPgzFQA^N0|%%6ea*?GlQSbmzF-gi-y)^Lc}B)r$0*6{M0(EG(NN z2dmny70PZQe}7%l7~~B3j68>S2smr;IoTZ?2n@S5oD!0dOc$q8M%0K`_vJQDmcHpV zhcZ(srKL$mj^;tp&+Npn)lJU!88qADy3HgM^4${PP|#IBRR>1RC+IZwRzw z_i}}x3(M8C{^`g&h3?HaA>!hjLQ)xXvwG%g6$ZQW3%<`E4Xd!< zxZ#YCPt-B~shQLD1bS7t9R_l^=_xcyPNN zDwf>^5ZcE0LV{o50EzD@B}IFg3X_)@#Qw%jT-NXbKw3XM0s^j|Jdy={lUjWJTtr;a zEX)IYdwU>aXJ+nS*2FccEK4a4Yj$=RA`pk^y7gJ~ z&8lPnNZdJGK>#uX~=b zJs0~`t7Ee!63=td>3f9CD|_RrD9DAD6q-f*ZjW^_DDHcHcx%rB3!XQiFS)q5$TKD0 z;D(-ZmEu}V?eGM55;K{v4Xu(SW;99SyUn4kEz~&NXc1XW zDf@(tr4o+~>ZW#B1C4SCr?Il8O<(>oBj;$pG9Rs4sLSmPZ%`k=;e#EVZ!eKw7d!*> zN;I48`;Q+LrS0Zp>}mZ8>AU~6UZLa~G30MfQgJ5+-q+`R5cILd3bc69_Qj(1mP@mn zly`ZV^-xKZ34myFz@$`{;JlraL`@O{*41%T6H8~88j5bG|jn`KKksQ_Y zqo!|*5kQesj~KLAU7VDVxD5ivY0L3qYFb*`lgNf;|8v9NflkKWiQBip*(G6O?xetS ztv-lymbfpv)E;mRWBjO-cWAN>vfLkqbyW1LY}lyxmX&mJu&E&_DG)xWc|&=`UWjm} zr*`>Fi@#9~Hh#HGx!v-XBNS@6I%0D1ekYM7$r5IBc<0lH^l8SOot&3Tg}ODqI?BCC zSlQ3PjVc^3f(h3!b|=(xrz751D=0hwBaJd_-<~@uU6h_r%j}A-7G(a%tUX>uir||? zZ<0-RIvsu9A3T-5$iv0m($WU(DRDdaoQLIB*!L-2yTyRRggF3h{x?Fn;QIBmt;<_&m2iJuVlt z0|Y>gr9sEAzk`EAw`>9HSIrfw)!}lZw*B~q6A#k&G_sex)b$*KziHWzQ$Oc+k3#Q) z#(2luC%U3RM=^l0LWdT_>u0}3wXFa$qf1wqwl*e)+qY44 zER7q&2z=-NmWtJUn7;E*)=!o|LpV242t*!$m>^ZfJt(|+Py0w}<{JKAu90p?F%^}J z`RGMK57)nP0bhT6t?i8LBKqo1Pft&y()J$z=3JvyWzbsf8Ehb5`C#l(`;f{|;3# zZ1-<^z+Be*JCl}>79fQcuNWuk$zp9=g{`H0=u-qtQAEVVTAN>yaF-NnkKenV!<@eA zjX{fTOscGmqMo|}^_-_u3RkUVS5kd8rJ2m#{vZ`IKM|DS*L9=>eT0UC9*9NH*$ zNp~UgH$zz%b+~zK8tp)nUZ8b4K|ZnG*P1D1UhdigK?+i{-Xo1P!d){1S&V z(w^%!*hClj2o9l^FXNY4P$eDI@`VqUuXP^wo`mwP!#-O(LwW08$C$a~yEgs3=)WJT zZ>dSBKu9uVm&#a+2ldbJXG>!ee?nc<>_W@GG&Z~ZCz>%Es($|WA(;DjU=K=5FUydA zzt9URRDMVGATCxJm_m!p^^d&>DNQ#7)*gXGOa*IY8t!m!uvU+jIQQJX@Nc--k4jBt z>AN*a#cjH=CgSQy!nN!b95BMe9_xYnS|-8CBqUrug>7~+$q`pC(Ze+zavFtDG|3dbjZ)1;KU4+Xn+m{ zN-Y|%)z#FH>u1WNCV>`83jR252Z!5d>b4Zn{(;3GT7ook5wx%bzGGI)LG#YxK$azG zct=M9i`j$h=YZGoWBV&e1e{MCQT*(zp6kIXowLPsb+l4wj@mZNetk60q*J-!oYP{_ zi9eCUzJ#LD6QW$*Pg?**g0@E?BUVBZb8b`9H({#Z$`iD`56<>AL9Bj((56+!R9cL#t}E+(ueA~u z7A|3Cp$tl7cU}Bo2^h4P=sJ%x$xtJ(MYhR3p-!l`-Bj5YUG{!E_ckCPpfxC()vWac zzpjUYg~g*CnyjQG!B(*%)3LG^uuTpeN@;1Wsp?{K+m9t?C7SxnRzp(2^=qRUTvA~Qm;O4yGP7@u*y@khLQglwC*Qo;oT_t8&CC>I6}d^1@bP03!QQfIpUnvNIxqdm#GAb=EXlA3;4|g6?-ey9_?I z8NbD%e#05MV~xba3dn1`+of+nMb$oCXG9p!>Adn2j@|H}Iw0UH__diM-|dcnCD#EV z#y!9$Dm{5GSY^?cC=wkwc5EN>!ugdbx=)h?kINcbou})}TQq7c?cPF}lfF6cUMUOM zYY^^%l``%gCxa)oi}Kmtw`5w+qY$Ncy%UPBr;Fa1kW&UHJPnH`&<}q^Py5l-e`BCC zkV?}|3WxE)Y#?Y#Rd=f^56UqkO5}z&73l&fBzB|a!O7#_{XLBDAstYct&-uhRc-` z-Vufu=H}8flv$}SO8%IJJ0+xFu?%5uuEwgNQD}K)h10{%#zIe1=j*vSP3}p{H$D>0 zlc8B}3n1Og6aOQc^4zA-!36<|C~ z75%R)7F0q8V7H$|=ADZjosT3a^774rHw8kueBMInZCqU)Up5*nskm68L&3_Qr#+m= zHfl=2c3@+TmGNVAHtc;j3QLXL1g_8w&|Jb9|mi7M<2b&E_FDN zglA#xXuwVr8|(GoVK$o7O&Xm;pBKlW(f~+UZRB|^pcnls(be}LFgzhegc1ka(x$C| zo^?a8A4AHj`^SI0FT+oa)%7O?Ot;(jZo6EZV`jhmD?4!ewQ|0nn11+FUi6)f$p+UQ zqym8uv4F!)R>&ij{DNL$GStXr34x%spI2Cz?53+&hd}`_0Oa-|qqG$@gw@yc*Nw!7 zgvd_AIBU|=i{ z^Q?$!Wsn>VPjr`|k5_|$o81MLl44Hz+kvUdg3RMO|I!WN3;G<%Xhu6XgBqe zrWWYX{$Q-GeZp^729m!?g<|28RN7nJ@q~&LJ!vBgNStlQX$BG=my|f-CznFF(G|8% z2`?takI3r_0s0iE?@u=Fb9bo@k)wtEL0KE#nVBP559X$Q>c|#uclo!|P#J}5p;*1o zG^(pv9@k(JPmEM}OYTz3>95dQI%CVBm@%Pz?BJ|qq%faQ(D?%cP6cTehY~EJnL>7U#bNg!iiFcJbe+~6hNsLe&iCSt{#dpJ4U$ZUa(GPV z4~w>TE_6@>E*OzofTQ-t+Yow;P80Zdpt-*>ZgT|pnqB_#C22y-<5#DV=UFu^IorZ; z{%T7qw3qfZNM4oXj+NNYRTV4lukki7EKbzApPhQK*k!2E9sgHuEoG!_yzv zPDpiq_?W25qsbJKz#01Nl3P>BS0v!!QMshm%4Z+2g->O(v`tR%s+^Xwlan7E2X+Fm z8@j_=hy5J&etm4>_>62*ZgYX+Sz@jV&zaNBAR_KA!&0lsdXGbd2P8qtaSFann8m6L z+T3RM1Ca(Qg55aHPHTK?r5%Q1{nGhGIx1IAT(_x3DDPuLdNde+KP3moEVNWqPRDC~ zo1S^3iL?YQ<3%1)A`I+hp0tTVcRF86VaS|Y0?F~-I?pe#_q=*3$*5NTOYZ|w4h_e; z!1t?441Y6qroW02evQcmfS&7@w@bX5en2hnA}QSGjuD>Vc5gqZn5lnLz+`O<_|V_J zUKBo#neMp5!o~IH!@>pJR4!ozkKQFj{uWhtjCt--B~MlByc&~6`ut3!%%Q9G+qb2B z&7-rTQ%zZyF~ATIvKohmbgS1`wtxVFwP{6}uwc4sBG+g-ZN7Q>;*gDkLb8NiPcIDQ zEXz$Mfz#&d_a>eedRmnFxw7XQBsv}H;6Po@mGWRr3 zcH_bJZ4V3X`K5))YS-Pdz)mn?WFm8h=R2!AH?cM&;nrJQQ>UkHkPIpy4^c6Fneo7O z4KF4+K5cJd`;nMDEgbLb zs~&87ZYp~4X5y!WNB+2xff2EQjqzAGS7?&xXz=3N{Il{Y$u8=P`&1`P0t@4L;d&PX zoie^Doj>4^bp?6qC5(;@ZJ66kXM%f^K&@uwdDK*0!^+g26a%B{$VPsTE^oeghuh~{ zg|CmKx+&58AR~VwtvAvcHo1`&_1fH43o9gJ73)76Hb+8_R3e8#Qf-dnZl*vqML;aU z@$t~%?IytR@EY{DEupJvX!Mid?c&J#2M5QqA5Ht*mX75fZ)|+KUrX(ITAiUeqO(5c z5N_VmJXmIKn4M-RJ6Tplq`w{t>*F&jR@=qxQmcGpKf*BSztr=UeWbq;I>W)jVW~4x zVBw=eKlV5{mfE*F9Nt^l^KBM8lFm=FM~SFgw?jYnZrtG zB!5!h_%)~N)I3NN1d^QCeEtZP5>4K@DgPm;?3eXL6JHn#BJaU_L{}J|F9Ut#*>mt2pudd=>0FU2x|9bpyaS?S# z=}r_0qJe@U!`b<*tE<^hIGq&fxk(kRj)NZ?mYtdSy4yOTGPX8$_kqnZAy%votmnCMChDdTvBkQNW?Z;w zF0DX!(#SoWtQQZ8jBwL3B=_BR$`tH;mREac+)IrZzWe}S&;waTilpQK#DevP5)d-W zCGgW#zaG7tc`c}u#_z(suTW6|{hsy~qWGGyPXNrC>D_7N)&gQK$`zf!Y z=eks>EmLa=fB9hibg5%oQ2xo{4;`m-Awi|VHnHRId~}EanZm2T6iYka*Miy?ET#Rb z!&@Tv9!PrF0ID+Tpe3T9vIMH~>;B=UrCVcVHlo-{HPf|xpXG;n-TeGahKT6o5lVwF zI0zQy6u*>{W~kz<)=Rbn;|t}cJ_eO1(4FV{t#S9pEx!lZDH{rdGOB@6Wmzap>pDVkPxY*^|9 z0$YRya4J{9ntQLrK_cj{Z^C#jv0ux1F4WZ9vKtsEa~d2+->q)?m0cZQA6q1w9>+*r zJfO6)GD$y_%d{cgnln z=8m%jy19j5ga@Q7U?pAuoei=3^hf$siL!s@_EDtySY3UThkwVYEB@=3`|3J2#LaUy{w%neaJV3i{Nr@yD$W|}F}LM`jxlFisa!^Gd-wg*ZIIb_+?W#5SGty+ zsPOdheaw`T*T%+z9Q3#Kt7@Ddj*{fh7pRe-%%59k@duW;d1`v6SIE2r>pbS5*>`W; z@Va1*ZUyA9E*75Xe2OD5ifiDrIc0Ts_bz8M+soP8TRD0tLRH4)3v_;Dv+!!c2s_g7 zu$SZTd5*!}U-qC|sGEq}JtBfOAK`E~G#cthm<^oGAF0Sfv3ufd= z%L5UNenU98Oy6kC(mbB9Xq?Q95MIWkaeFU7?jW!a?%n|ssM78j~#TX#YE&F{lYxfn+c z2M5R9)l=C)Lyu5-uw>JNoryiZ!MYWsjrMoV0G6PU|#uGG&ha z7a!#KbOFc%^=c>kMB^nU2T6rK*h|!w`{dCK)e10D?rDG;8xONx9f(KE&we379wGM( zB1r??j)tv3zJ|gY5Q!0>^Rz><1 z*QucX`+vC0|C;>%zq)Wk!N>YC)(a=88?1Bmbns>o8u$UPA~RZIqn-1^bg z<&I5oIx;Na#A}(o-fyo{>o%OL5g5Z}+Lge;MM0s_jZx#cI+U$9LNH|3O4-ckzCZsf znQ!OdeZlF((ZIN)CBsQ|US1wpryA^zPAiB#;TMnHPwpP3C~|P9RxU9nY&&s5q=GFG z%^hU-?#Xt=mIugvZg4+o1=6*aY^aLP{#=x%i$TK;GO|?CkD?XkeXWy|WE@eR8;#MR z+yJL=Dc9p8w;f__ZT@tAITh61Vt6=WBMszW-V8&$xgsKBlePBi{R_6(qn6c)3PLL* z^cPK$O5f#1ie4(h5>yL6&{nC2*YoQQ`Azfk)!}%sumCT(_Wbv0KS;@P2?z-Y+#Bpf zu1$;L<1gEhgO1!P^QB7jRAGw7kb@#T89tyU20Nul8v`hHJ)i3-k8eOslnbUP2qSCn0v1ga=fi6+iOsZJLJnulza>1O1uL4+kFZT(C5K-4+Hl z5)j%bTlvnMBk~@u&}C!*^2wyj!V)f+V)9T-11X(+-SoKGe3h<~;62UdjxSRkNmeH0 zd8#9hfKN~#jo!iH$Y!fvv+V6*cKeCv0`L0q<0%@9ocGk-=iwDPb+PC;0Yr<@-P>GI zL_3W3X$hvR*4~Bf1oLq({Cj3NtaLYbL>#km6VvzAyQ9@TsFi0Ia1>gwalz^}l+yZg z!fYsKyhLObFgyLrt0Rm9uT_?0C&?v%B!=n5iz>H;tMQSiQ(>$LQE5O;CEz|rq0@(Y}yn0074dn_FoUO*XHxbIa*s< zfKg(JVq5Ttw65KXq@ySu&c@H9sN!Oq69;1d5#8_5uvG2wpn;6;Z|w-0^xY+`}SUQAz~!qJg4_9(6Ijotcu<Tj z4-U=)aoX|VE;+`qb%bg=#3wjU?s&fA+OgFD1wEocCWf`MrzexAuu#_tlCRryApFWH z8!^-t(aD*$Fn9_<_&|#s(*(9kY)h62Rt5`H7w{oU7|Osk zpGayDkD5b_j+i@gUt5;8a#b1OutPeL;ppm#y8OIO`V>D}2C&i4i0#oAJ+F*-KH%$n z8_b{NN5wXMpu$w6($W!F@Ei{p+Sh#jng#5U?T$eO`TRxT61mjg#?a>XrW(*XN+i?d zMf>1jcsI2rmr&u0saniBP$w5(@3Ib&m5m&nbUL41*45+)uW4j--A|1@cUb04mKPsk zw>>`n)cgs0kgS&7GXzuCeeXvIug$TChK0}Vqftp<9T$PWOfsffk z5Hhw5+7+KS1SXHWEl%?D%;EGa7%M}ma*~q${hPSGR`bL8gh<7KO~lk0&rYRgNRPO_ z=OmQhTmt*TE+|O${reu8D%Be|ZjgBgr$^11d1!7guCGEXmG?VC_o~I;=!QncR-VdI zUcHvMIq9Azk!lDyI?^1_{rj=TC?6d;(*9PUeD7^aN?e!s{Lmjs0e8DsN4{&aZ{}-` z_+-#1H7V}eU=-LQsyZPUVSkX#sSUS6^ZVsm}G13lq#j!-|I8W4_^ngq? z5QI8uS=PDKiv?{&m7^bz2kSINxJd@dQ&A;=I*)f#Qd$iTigwb2fu03mbCT4S0G=%( zD&k>Jcwu6U@19$ckC-lO2unC_P7-h!LS~j`w}HYSMC*Bn?#I-y-LspOH!eI0an5B` z&}dJaTsy9J5fSb57s}LNZ0K|Z>q2fw1<&p}BO8m|YZ&S1=uC&`%q#TtjSp>Bhs-8i ztuvi2bQ>`39USt11x4g;5js1m?gWbS+pns4ELc{4NK_aep9IM3e;I?Jijf;!J z*ifEp?V6b6wVZ?pbV*i9 zCZ&(bg{8rB`aRH1Z48%>Y(Mf@cK0yBze7_c6e6F7;19Q(OQh8kl}SqU?^(j#Cplfp zNg*5>i8`W3u;8@}ye$?FnWX)~UD=o}hC{C=X?0b=eg8Q%wa3y%gG#=dDzxFVIW`>! z*Q;kX+el9VdoPz-_(cGu!LpiUUx4Ot8Nuj4M%IL}5m6c@rKAKJi0p&izt^?J&r+&; zi3al2EJThMPNQQCS4i{Dtra@Y?45V($9*ry0}SQP3DN{;5*nO<`vN4_keEercWrG= zHTgS4m3}EHQhRq{J;80|oMr*H@cny!zL|kXsc(FnUY;-NkIX~N7whq`oTyTt zy*Tkwb~})SWFg1OEL~UkF*}U2(+NIBx9#ZSS$c(j`dN^x0O*ix6StDA+{%XelUNh%(wQa3o!)6r4LewFoX z8Ol+11nT_7AF&!6i!?f6#KgqQ4Nijuh=AExPu9r3!XDM7?!-{uB*9??m2Ck8*bSAS z-bx=d7_lX@!41h{DTTV`zVD=mIO~!C{6r2pd%EG^urCu^mQ)C$k^XVX>grRT151fe z%W052a0whMU+EPXBFF9M?k-R+m942D2JUx9OR$>Qw!#K;1W2AX)L&~*uQLDjE|?7B z|0?AHD7|bjZ+Tbf4f5RCFvmcE7od(3-56LTJg+-Tw9DVRM=`<}Pd1ix_)dD`c5taz z0pkaJCD?LCV73Q|e&)#5z|MV#d$bzKsvY2%ssaF^T_mOY{G?OE;cO`E)f9xDNXL=0wj?n7pqnx2m|F+IrfD-FeBcq}m4>r0&{fW3L z8bkv%K!#-LwXT!m87LzMySUDmL;>65THm?MNl$B|@gSMdIArGe3Y)-fsBK$te68_! z$6^~Mf%8*~>YC?9UZkXqBWLISvAHJ`T+^&(V>_o_XdO>gpg7d4Q)lDI4^8duTZ{Rt z*rSM!fpa9r&|tHE_nL@MGGf9xx~t)W+9io}l0#2#v3pX}&*!nS@qu3lV;y9=F)?;{ z_i#ewPUPO|$sS1T`@U+NjRULP#%N7fN1p+ZcP(Qbf*fdeduE1%<#>b3g3A#PJ`qvX zo4LX%ouQ>48i?hY>Nj&SUsPZI`&0Th^S~9@t(XqwX@}H>I_)4y&pej-G+oFwHYz0D z)l2xJh!Q=*I~dC#u31!Grd3evl4s?`&k|Xw5Y<2uA};OsriK_v%CrN=RZ z?+Sc_f@*CK3`ZFFp11kurRL3ELf7z`$fq%UO;YK-w-tw5j3i3Bf56xC(HSa1HXpbr zB*8%@huUF#W@9EA@?KO9vb@T2kAzdiGwb5K?f+rzt;3@1y7u9rRX{+b1yn>-N~9zQ zECi&xK?J0Qp=(gYAf+V-X=$Z%5Co)S=#=j6n)&u%-1qZ7?{^%(FZj>ky7skW?X}Kz zu5;z#H^an#5>|DNRKs|?#J4lEO6fVYDl6@hulu~+i9nt?*;PQ4I?VqN$80`j&m6T$ zz0J=*jM>{@5ahBR4a$<1Q!IR%ks@$QGL~%d)^|`UJU(We-f@Y$1asH>dKaY4k0dGgSxG zum(AR*STZ9t=LCxZf7?b-cA6DP~oW~PV5B1jbd|j8#m{N?dA zO8}C6Qe5xIvD7I(R$5hNTJ0TJ?wLfV&=1>~9Jph+4zDJ9{dvQA=Fz+nh!JF6>+>Rb z{?h65@%V;($-#9c!S<^hRHFW7m#^ZE59lcS`zM3^{|LSNe+)VOAK`5O-*~M5enm_W zQ9e+K2x2;S`OlXrntT@bo>^Z0>bWG2VKD;&&O)BzfvhXP6iTR~6+ks)JXt9=Ij+3r zY$Kvr=@pZ{TYB0zSS0A=d%mv?^~^IIWbqukF5m zz5n4u6tE4u^Si0X2lbm<-vM&rxHgpKA`e85n8mylZw+Jj)iBIzt{9K+oVvYuU=RTT zGLceGcyHjae{4#sgY#(BOoK-7FSe`p+7EzXr}|=_|1?5 z(zuZ;)EoxeqHfax#xngGD8DOJYY`adiZ8L>A+sKKS{~oCZ1P~2Wv6ZdEsX_1=0}CC zW2DG^3E(L#S8Itp_+S5-anN7%z2wMWsj=U+xYMoN>JsNN0Ba$JHn#BjSWNrE=EGnB z4XG%H%9tA8rUe!|25&M~g|3fWmfumoz)jrim^sEHq5m6erc3U=S0M}t14 zSXA}&NAV#o`OaTSr5jX&Z`sYfc*K&V&idBsl;olEd&1JmL>w3GTQQ2Ccl;mJ0%go5 z2VIW<(iZhQD_~iEw#=p6_GZ<_9B~8q5#*{=pFP`O@TLS|-_4>6ze|1D=Nl)s4)Dco zGYBOVNfXaxzP#X~%XLmpW8$6n(+mpZzWvZI zfG*Yi4;JvKB?h`Td=u3WK7>_oqB`xcB=E9>ArpsO74OC?3y9){Dzo>@Y?og)L+Taj zByjm9W6iI6Kh*mrX~3?|Sx`{41~d&o+l6L?feaAA15{&BA39d;&H{zzsb*?&B=C5y zVs%8@x+&@!^eepHw>#L`@$>KpD!fZZ<#MgIz%_Tan%4jg8i=kwzHzJKETiSideAm? zfB6$VM3Ue8Y6hW<1Ti*I3CT&CgsA^f8ao8Qb8B`0MKWRcLNc0YWra~Nd-&GHbH~BV zp?qbbyvXbp4K5C?LxN(Cjh2C{d)vwhdE zbg%(o6i`C|SGZ1zHAlq-Fp!56rXs)R0>4*c36S16Fh)gzMQuO}%^w@R1u%$wc5Ere zmE@`l6=Hm$dk1O5h|uDtxqJ6+{R637t&*=UmkG{l(3Gs`>=6(Uah-S>H64hQFaou& zq$MERg%u^5EujjH3sn_I&@fV#?Uty8USGj}?a-r+H!~TXxT{PFYLm^T8%b-REY<)3 zirwSjfT-w;R!mdzYE35 zj(}nsCQ??Q`LTovX>#$5FLt*_-31qp#YD(PtG7COm!beE#=_$1;V2wWjgO;fBbf2P zbrO(!E;bg^^IMg_Kwkh*Zs=0FkYM)s)o7KRm_0zhdlDcglOxCWNrm&`eG}})z5FsJ zkq3LN-Tl3vl6B?~|3?+dgP)JiFQ>VlXC6dO&heXQVAYvUZI>nH+i0lJX!)+k_P!(l z{F>lEhc_wdYS01m@y1&NY^991diNkCGU+xU^*oa*-sKEJMX-ElV=m$O>%DG>n4Y}_ zI@zRRK=sb4@J<$1PVo+t_trdU<&k_{23Hf5;PK4?&yW>|<8x(|1<9eC%j3tlvE2Ml znKOo-umAl`KDXFJM1a_0C!lW?+h9=LwH7}TDIUFf!^&Iwj|NSVA!TrKN9GP^qXCfO3y(D1ffDc%*E^?f&JXZk_(p-lns`ToN0sn{gq&gwVQxHm z&EXl&d~n16IfiOM)zw~AlxpL2SiUM||#LGc2p_rt5EPs!_e z-c{)syjlgSbJ(PRNhD&Kp)+|`u3Qn{cKk6d%?rT$5Cnn^TJZMmoW==J6TK2P@<>=$}ON@517q%J$Vm`*jVas1Z$(uu3ReT+g z7M=h8c`EAhdvR}Ny=r@-rb%XQ195Kl`%OIgwb0imUk|aJp&^M8e^b`cpcp}-0;ueM z8f8iRGiUTa4cBNHC^`Ah_R9|uO5y*{b0T@F0|*}UKNF##aERH6=V3io&P0Ci2cL=$Xn{2f!5M$Cc6JUiLAfaACR0^kUlAp zD|EQ<*X)DeUZ)ct zv=c2aU}gEul0R-++Xm|@NBmPdlqlkttZ@OTh%XFhj$QnfTgQ$R(k2AnrOe2r6s+8p z^;E6uCN-hmZ>?TP=Z87G0R#QwXzgGrHZKp;+#!lSlZScN{9>NctvyE!mc)2B6ijC~ zHt}@~@{Ox)=D8;Ln^((rZGXBwHGaQU+C&1f#Pa9XUH|Uz%NBrr@lx4EH#bNufffv) z@Y?}X+Vq$*ED354|$%T+JU5A=!P+&os+k?1+yLyRiHH zmu=BM7t6+|5o_ZQ5bM`ibgH zRO1y<;w7Rui9)3JZ(rmpP`iKsU!AB@aW%05!A0&m_wI>$62yrO0%RThCAeE@UUrl_ zuM}Z^+BC%hT@i7&<3!JjAl3rU!L0G0d2fk*Yhz1t8O6(yF2Ez#tFU`74~pE4e|leW zDs}=@?CzK`3IsVAhqybuuxayl3&V@v>p}kCav-+?3kq5`nbn6ihe??FN~Vti`^0gn zJFMwhwdbB&UBmdDJ9k45fbq9o>X*FyL(H)Op)Vex9h>Dpyqj}0PtXC<9+7cK*f1zR zV=q4ow2VQa9>{B4X}T#rv*il#M@zU)1Tc1B4kRQ4SI*U`wJ|=BTY0d(0xO-2f^xB*Cxrj%I0dxw*1oog|8GHp8)ILlNvv&DyH45R+hS{x-L<2l zXVnp@RQ@J=tD_8n`20D{40Qf-Y~BAvK`{X?ihE!NY`3dBH*y!qLP>~p?{A+=yd$ms z*`UnrOOJNNr z)&7VFS!spZ4j%>9Ph>pis#~gd3{zfeboKI6 z&-gc~fHpDkStZIC{Z=MN>e>bB;15(Z%*J(JBc2@f+0yQB3f&|zeBXyb%XTn%zBx}v zMMvi){1dIcpxxQEv9#XHD6NUtQd|LE}jp-4Au-voc+}$ zYR8!7lVoZrFOZJ_sg-eqX?&jI<1 zrfonVS@6FY7|9P`Em27eyjgS>dr@=`Sm7eXfTN1ZAq-nwk12Q{)~?fK(1tJ0c9%hw zQ$T>jjVw=p^l+Rb!P>%9*mH3N3d2@}Q}O!q=Rr_5 z8@e!1I&T8?xDP(&C$g1vtFdRYf4789@$)U-|2>40X^Z?o&J2_=H$fl!wXq)^9qbXe zsQt~JASwXQ%dNlxD~y**6Ew6sIcf3K;uc-@Xm7pe0a=!p{()H49T0I}{gknM9gra% z$TD$0VAOqV7dsJok7V`T6IXiY2+i?UaNS22Rubmp5l1@V=$Rb$kCCSion`o*l!M zhCq62z7^z6t2~diL$8Qm*#&txe;UrM?QOq6|G{!= zpK4dJMsFqQ@esX07Q&z-f~^&mXI+0YcFvPJE0R@eaZ={m!oX1B#MEn(0t@ZL7b$Fv z#11E3u3TMrqP3!dbb7t*&d&A@oiCn+YUPgb%{(0UoB!o9ud)33{v=ho7d&lS#wP7m z)^C|eBINJmWV`R#?g7CCi|#aJUkkmOnD~|4eb;UveY%1|$_^elXkQ?%a5&IyG66z1 zE>wqWch6^DKDqV>3jkbq6io;qNJ{Jhck%$vf4AqBy+Qb>TCp0zuzr!1N@vQ;fpq%3 za=8hGd9nrdPfBSFYtc}zM5%Y##VGqBRrZh+W6P{*#>8IlI%$pP6eeMr)6UkGX8KO1 zGsQJYtAM_1vf0Y1gTBox$jooQ46?MWTkHdF@!|d6RTs&~I`R(5WWpKG0|0Sfl+(IZ zkLtO+1hJ|;&ApwQUg8b}puGWh))RA!;UXJBYL|6XLnacpHNl;M<5^qxOCL?XO|_%t0g!U5B}4e z$7l>|tM+S)PF1twa3-47*{eRsaVAq}da)q_3=o?L%A0Dj73GJWasCeZx>4NU#d&i(;IYKq zPb+YsA&DPmckdC`c|6D~B*Oh0DIm)7M(&h;zoxltWXYDdAUCC5$((+zis!S)p)-rT zi$K-F1MCA5U&}C`k^^sPbj@qb$0R=dC)V>Q&JNt1?UmWvC`kq^nBJfLgM&p&bk=^G zTNs|#-tu*Wr>~zBt~mGg;h};`;F;&7Shv%mvszxu7#$)wkdkaYX1irS7TtF;k!k+p zCg`Ay{XrKMQaBHJ65|fxgYc!(mmC<0Z)TQp8beu^S{~D7F!U%KSn+2)gHalK6GJo zW0+fw)UB{7Ui87Sjwz-EI(ls#To9+DeibRuzTix-hbcUdMW=;L4ANV91aqFH1}(JM z4mTL;xTR;cP*j};qc^M_IMF*U$@ok@-jP_jKnZ6K%Q)Dzk-JxQ4$C~7_AjaPXL)Xr zu3EdDQaK@s0jYuZF$u-2iJsW$G&X#>D#^b;!uNHy$gYWLr!>a;{goWQtZ&VqH=3>| zq_c>Q5qy8znbkqsPAF4zTD*k#bargnb4||v!6AMj+3#hQ&LBiYK#_(=@n4$ABhlUF z{Obm>8xo3Z-FnH+QFv!PB3}q#Kkh?bgin58dy8BW#0ugO;CUv_P zSrrSrk<&HYVRC4i@(B6?+!`Oi)djr8ZTE{?1{kD#L?eVC4Y~9|4H_LuTA5E%D>#^Q$w^vE!R8@=(cpGq$hOB4 z@g;~zR~yV21`&f15~u&<5EEveP+(r2HEpmZN`vxc|Lzi4^_pvdBOGYtmq?L1*rJj4UYQT35|)cP!|$h zVf@pH-r6dROWq(&VEo>xf&Hz-mf53OQF#G+2RUb^AB6U&p?3Tv_^_eJ808PvkDdse zMk}3q{`4FD1T1blWfH{M!B499-KmiU{JzlN-Wjb8ASX# zu#L@hm8sI6_vH21Ad0DX{}b8-`ntT9v$#Y==^y8R@Gq%fXJhM*2KfjkUi()S6>lGN z`1$!A!?xU-j-Tdo*?=NK;&bP94^L`Cu^A6T6O)Ji9na`H;;Wvw*|J{~V*TtP&Mkc1 ze#JIRSvd^Qz`gtKZ)J6mH`L%lIw_zEErXQ0O9w9nfl}(lUlALSQ-2DY9{x{>j=A~& zRadb&7nE3MXeCD`Cxg~@k4R%QyU##cnFjx3d}uHbyc-EiG+Ev5OMGc#d}PmYRKy`E)=f zq4wRh0EqBK#6(BLm$@|QHaUQnNf#)IjZtw9@-i};%h6k)gm|g%v!L7bYeC!lskvKS zQ%8&~|Jjg9wItIv zUbZ4(=k?i{!vAlIN+KPcq2T7*m>L0D*On`gM4ji^i%T!>lQG@8lzj^dwOuKv0_qD^ z8Hcq&(c}bC8eXcB^!wiVF=@b(#Sh_1oJ1)#zj6 zirCnr_E9)i>VD-7n28)2rtf-p=Iem#1rnlH`0sCAK3|U)9(n2eC*8l{w?mAN9GX#m zZcZnG9MZ}8AKF8qLjOq6?QCy11jmb7Ouq(%K#sC^OrLJL{RKkdo7+a|@_GO#4lVa$ z3TCGhNYJ+oWZ-oLJqDmT`?;!2Zlfj^X)4ZTS{w|xiue&YexL5BUh$_-?L(^e_XTC( zS-k27c^X6q+oEa{4o6cW*(LdyY=}A-@eu~@qjcxpeSk;HnjU@CKz=hIOYgT3wAB;w zaDZ;UrvmVAs^Dwy6yC7_z1&F)H!=UYn;^Stl-i_T9wcA`*Xn}Lae&@dD zAWI)B2LM7WuKVVeWcRU5jsdmBtG>)>uGh*$o;OlQBrnakHK~KH)fEn7&Are<>)|i( z@jTWopkAWMQ33drv<$IN;zePgEb!eK=Jj4+eiQG1OncKo_H>&aqQ~nsqROjM z0Ms35IwoePpFK+fZhJovl~d0t2f}Y;#Best4UmZJ?7tGm)>i4c_sGjD;n>>M#l@=^ z{^3~)C)fx%zFm3ZW)UN}velE4mw*yfZ<6a0n*y^8fYv+NYEw$f+$jC@hrjlnz;K~~%x4#|->gjv?^6E;)k1t1yGrwTrWjFqw z@e=T5@D9n6VDgOIRsH63lPk=K*T&S^eRa9tV}#B%wU{*HD1( z>ZNtU0!((NI_=3KZ+b6I_PO(umoJ?4m-_Kb@?9p)b4ifIL+h^LYfX;-CHEI$lbL8d zC6YMB^-ntKCOvKAr-7=Y5}+fbYhijzBz5`8Edee!Q0D%ic>nd?rhN5_Rm217@F?Xi zS^3AKAam^Pq}}sw&7&rvW=`93@R3s<$~3C!mGHy@@n9c0dt?|cYljLgC2+Dz~}C-+CQL6_kN56 zD>zuRiUXM2^*`2(U)EF-E-|?8tOM&Ohn8!W!IB9}CDYAo_-6*lCSbqV97q zf{05sp~CY%J#qGgc0V`mnZ&&E7lIk-=;$yb<<~(cHy{9KJ6;|63><+fmxYnZhb@50 z4+MvbZH6_SSburLzX5zOOz%wMwmk3dKka?eQwkCyb$~uA;OEhB5vtxst`^mR9yMn@ z#{Pl0Zw%r7k<$h`OiqU$14DyXQiv~{W>_U-LC;mL=RX)7chVLfrRWa3X3 z`i_h5eLDW{5j?N9mDgeG^0uTU!*LMwPQ4NElHgxi23+iY2FY+>t4M=B5xBQ~?%F?o z_3uo4(a@JC2QQxl)9g;+vxpo?BUT{-D1)!+W{zaNX0?)En2E2ac}I4v#M zhIWS>4tR-iFN0R)%z(&o8rGiX%CJu?l(P*U`Ml%e@aj$O^>NUL&Hf)KKZf`7`QSIX z2?eA`xGBXCfol+|zkOaH86jcj`v1V9G%+^oGbXXun0z3rwEx5^fAZcSM+Z8Mu~}GY z(B@jKUbe7cOR_~2kRsA(2_MiUH&NXn0^tHSk%x`B{|=6C!KL4U^3gZm;)%ffin~5XHbPvAdLOnpfNiZoU1T@&GW!uj81A4^|Q!h1Z~I zZb_9u{SA%8^Vs?0`FH*c%t@jR=r;_N;}K;xegf%%VEu>FpZ#k9wwXr)-0uQp=a4l!^mT-whKbZwBJp`?eIaz#E>?Qx#1Tj63oDVY>1#R;9 zkvSix!&5d7D(64`C_?7QNXua{s&gPCF+3v4cMwE{qsZ_(hi3Y}$V-Pk^8##-SbpoW z#eU}Wwo3}h@_=l?QDOC$T5>*QJL3-)@P}p+@&xscHc($W%~jN9%`1Mvva1AkhnLsf z&>y_}M#h);=N4j2Ng9bZzv|$g(-0FA0|mX3m4WZ_vM1yeUO#`11FfQ)(u>3HHM@-P zYWKZ2b<)FMcRIFc+Qaokp$f-W*R}R?tNi9V^ZSz<$*uuHfDKxzJyQGHty_~HF_NIe z9x!?m6!K}V6+>pb)vZ?{7QeVX-jVF5DcpYxSv>VA*9Z-(^k(DS-nPxqTP6r)|PStve5mGkBb=@dbmPQF$(BCXJCR+YfV-YLnq-H<( z%OP@ZPSFBNUS7cc_?Js`|5w|d$7z835OuOn3Ql!~_ZU+dl)puYrtXphX%t=J6|f1h z_xPEFkLUlU`Z8Xr=b-JHQ|R9ABBr-s3W$x8LXJzC4>&G#!JU_fK>u24dLf5r<8jf% zvIP0tX9DdOevwz4fl4l6zVIn|SpNZmbloBBAsQW<>X|S79^Rxw$V$u37b24)+%>1_kC#)dD$5O zu!hA&A^8539?cWX7bB<{4AAH1%+0fr>cB>9mhMl?yK*% zE(c>~x(eFwVm2C?7tG(b!wfz5yNIH6-Mk^@6X5=Uj{FAR&`lpnMRjd)<3qDi8+vWe^rDxD zGr~6rg>SsEacl*x%`EiuofOadT$dmJ%!cPqY-D26%PBbOwKhuTs0H09mEIdU__=~S zJYlL>8;#xh)TL&u?iqJ?$+&pLzSeyW$%r+YRRwC&TPz}+1D^2>dQ9B9?%nKHZ6LN+ zT07WFmqRU?T;|D+PS(BKm-dEW2(b1J-R6x{G*Jn#J_3Zt;YOi(UW3*+bF6J~J(y@Z z>=Uz4JSs; z{^eUVd7*;$&a4hQWg+mX(`ILZaO|5mhx^QZSX!s$`y82=;L((c7S=XFWV~@MSgw4$ zY+rwBQ&Y4^8>vd>Vd{=upY!9h%Jwj+)v}{8l-U!5Y7f^rxwbaBFWa~HV13)QNs|Mc zv$572i~OoLc9E6h@oFNk&8D||(EYc2WA5oKlr8P;?M-c@Z~T*zYQ2jR z?p#(REL0&p`#dAhpx8a<^MNO#I$OIoanA5l4|0JbwFhk0?ghKH)%d{9XD~f>y^ChPK-e>!D zBW(}*!q)0wMnnG`^40F(*X!{fvt5l$BQxKkiiqD(AWozcNuH;ZoC(K)taD4*^nllM z_ou|B#eXsZ)6t1wcAtM&7hExKHWz*|YPEPtKgz(>?s$g>tb>XQ>Nv(VN?+h$D9hG$ zWk&;j&<%YW3(A{AE!NFWcTHWUiZe!M^NSnUBoordwmgcy*cd_?TVmu+2g z&=;MKpwaP~{+7N-%`FUq&$=1+)e}8)^8+=F5nx_j);ygX-sskAF^649d0p#!JFMKb zd4~<*aQL2$QB6ezs;e@dzYWYOa&_^hKQ-qeG93{h=UvnrNjc`Zw=(dhz_eq3%a{yd zHzYLGm?vU9yB%v&ZQ5&Q2}T+Edz6rT0pjE=RI#qp%#zTN=t74!iP}xyhclkShRdZJ zUt)Kzt_GmmJ-2eqgV6`y^jD$Bj|@3A^KTojq+w@#C0r8sP#8BWyU?RyyQ6%RPxVmV zGI%odLI@X+-^=rLrL@9HS?p<~2+RcI-7#uHxwvW-g}xfj*FIBUTeufd=tqV;95?ZB zO(aB2z>ba$n$O%UNyQ=YtP`kNWQQ5@9N+l$j~^f~L!8?VjAcr7LB zZgvOV%liD3U!Q{mA<>``%At|h)Q&*8t@c!ESa##lrr`s08gl!ZTtfb{61%`T1;&FP z+u4ve{5XAcBlmu+C;yp?k$Yz(!quh=nF^%q(tJ4;T7Mh^gMh0^42C*KeeVsciNxt< zg&4XFWTqSp$kA8pc6)3M3x%c666E)hW2MO?dRByO4tQeuUK0be(o;ZylWlVPesaBk z@9v^#$P#R}=IOo?A%3&BQ}bn<*&5u-=W6FI8ovbEa$fqlyV}~=C!$r5HoN?5b76t}E zO?chrW^)j~j?D((bT1Fs8&7TCuJYp)w_mpF1PgEOgZtZ3M@Ofm zG55K(>rC(G@zDu)M$6i}%itsha|VO2#@?Dj(KyxsHAJb=Lof!IE&^;l+Pdd(dt6_Klzn3)i2(R7_Ucs293y#ML z*p7{;Ikz`8c>>$?w1wBy%cNae63Z{v}AmT-Mf(iLHaD zA8$5*yD3qibZd23A;!gmqegi7Lo&15H2R@G_4Y->RD7e|i0+=A?QLH8s8_|>8=OAp z>VlKCwW@;Q_`1emIXnY>eb@PKQgWg;-2^o8qb3L=0U5E|*F)ql;xYEA@?5GwA5RHp zlx_Hg8tZeCA~_=sa?Ho5F9@7T?k51}vrD<8)MjI{Jjhl>MOg(Ujv!=hO#v6{d9nQ; z10}(t`#rhtR{bkJqQ~(xy22%l<4LHdV68$^~MUWkH^R?8+xzg z9z5d<3lDfYjXwAr0V}RBY(E&0H>Zj&F7)(O4mf8^9KwqAgyWDWCkx5JF)mYq8goIH zTz>|Wxvn0VJzd%xuxuQtozLXa&~r7mI#w+)?MT?^8@9dVIQ`uWz1!nQm1EaI9l#)n zK5}2>HEYj?c;ghyI1p_6VQ1BT6@9{(E-!qvDzcUbT$#gYUObq>2is-Fx@j z@cShGvDqnW7x}%e&L`5&-|no>Un-rUd7^o^zTMiSvW0uKb+|V1J<56UJ%uKM!IUyQ z#x+QSu2UWuRG-%*wGUDt*`&p{%&VW*h7OXN|GwcM7g%c!N7iE#Rjr-?YgZB({>`?3 z^w`ohSzQk^8TxQEH`rQuJxuUbb-YPI-VD|5_H!|@m(>ZZwBEBWf)H;>^}p`_oP3*m z(RDG%p_Dg44MgU~hL+1uHd{hZCGo4q9;Ul*Q0Tt&RFEM~?0{ck^b(GWdv}Bhzv<%%enfZwDN3;(0|H=EsC-GSRr%Oy&7D_7(5$641xp zd0Z+4`S1YT9gSAq`Q}C~e-6pxgdPZl@R6&V=~KxkE{2HnpPzKZ4UfEiedD@l%2#f- zTMQvQwm&;AkeI@>4?Q9kT~uAH*1$pi^}vfd5qvj8VrKtf0SX7EqsbziaYcOA2sUC{ zPeC?FsKrMjO%ON$AKwsC6GJ|G6L!e-y>-ET5{0-TTQ!b+_Q1{Bpuv~b2|I9=m=GmY z_~S`RCMu@kqZ6WZk_$HHA=@b6yg|RBUin|=aGgg1{nG~;wel|0bbPQ5-V~*?JafhB=Zz^{H3sxY>qG@!*=~Y#ZsH2^H0&Db$ayZ4se4z9sUriG|GKvnQv{eiaX@p@02rlD%)2 zm)pMeO_nNN-E^5y>Y(2LAW4gFdAX+f#}njnv%i7IMj6+V^MKLZu$O1c@?gQ3%Ty5& z^Wl`d$Z9u=CC+P6C`#YM#KJU93(OrLl;YQiV*dLcP>3Thx8Tj+cJ|}U*gnTKaAmbc1ce? zEj5sq*KK48r5^}XV02J@dd^igWC)j$6hFVrCYpq*rw=Wy8BA8xzse8f?78zY$e3>acEfbdmPR6Y1qFEG)yGvlKNOUx_Iop@v?SDqsvOKm0U z*G)J%IYCr&aBx6My5l?wlY-PFW@sL4z}|br=o_#F$}B(Fs{PKEtPobpCV-n|Yj&0} zOUPv5z1KM-mGLN~h#d9fK24xCtf`YVMetzfh`rMkm4dtRFA3k~nlhn8CEP*N&r(_3+0uaEh z4DKbfxLHia;_v^6(6g_nuh_2Jnusu!Y6JED6{5VtdgclYbsp?jfqWMXd3+_v+Yh=c?s07!s)rBw9) zI93Z8iitT59W`MliXVlaN^8kF-ZL}MKJ=v`rsmY`>W+<<{c1l;&7qJT4I^$$W_O>Pe0Twa zB(fc4mn^?E&_sU^XywLyE`clJ5X!6K{@(?Ib2!LEm#atGicWXurO( z;KYS&(|SMd$S8Pc#rv?nj#j-7Ly-gTsZ;P7E=KeT^yWErbg^{PbG#L)O$C*$TsR(qb0xuvWmj zqOGJ|^rUiNlZ8xNt;D4ZhiH|NYaF(=CL`ODS3a4Rme%0DR>3Q9G3&m}hYROuYM(65 z-t0$sbT6aaztEkNsG$bK%}lP^ot(``*!0@zQ^7SdzNzsyqMrm9j~q@>5km184qjar1;oNIV7vaaW4dFQ7=sA7X<^r^#Vz8lV8dVRYdnF>RC(kWPVrxN{r`<=^O{FEsLOt8_ z(d)H{S_8A-@Sv52u1V3CdUFb7!SRPXYk9jFJK`=I#A}$X5(ZC}m$TarkwVOJZQM*c z27-B?Y-~=FcHv92E7gYbFyoUpeguK1^Zt*}bpI+y>@!_*pCL{2GWxCPg+2?NPhR$hHr~8TW!9u20j5@VGJs9O`cm*=y(zR z<{|J-g)*;d0XefK9sQ%FoUGinr8C)IyEo}j74Y6SGUMheO1jEdSFb(d>Aa#&V!&-# z?i5?U=N0Z*UpTRad9(e##8;9seXS?#%;>$3x>0&wN{C%STz?~JI`ZtYJ3rVvKu^JF zJw>#CUH{p`j6^G|M>322{g=jKTlJ%1@(T+vDb?~`r^mc}C!MKgS)1CpG^4fHZF>)W1 zfW&l_$Mg!Ap%#OcH~rA*R#o^hylxxEcARntI%_CLlt>3nc#!N<8050UPM?3{O!4Z@ zgNfF|Ldd%WBL>5r2%&shuV4FN!}WmqVt6liOl-}ulq@d_!_He$TBJb~)W&!aT%0yp z_`p;vv!nGK*;*T~E}LyncQO84jh}mD-)ucE&1(aM7y3l}p)qE25n(Vt`dUkm@7e5X zn2yU4rJ$o0a#5o4c%;_XeBnJmTOpsAb-a>q{isKPOaH7DG}GR~_yohLb7@;o-N{Rn zumtKdXtB3!1}ir?&*5Fv7{b86{B_LCxhmFcv+)3jTlWa)+fs9!$OdWB;t9~Duy z;ELeq@9Dd)$$;`{oupW(GFQ?hWaiRUtZ-23bgts|@e&~NF6skbp}}pVg*p^38D_5B zt)1NN*C>8lwt)6#X~q-Z*~20HhI+oh9~sTft1~aD=)m4&@t2y#_w0`gRS${ojin51 zr#mPC(@F6@U}FJ{;-z_erpaw#LEA?|-LKzraCS6bito(|u{5INn>D2|W9=TF_xx+* zT!sNwl5|$qR4`=)IA|kp3MZeT&*Koj1_;P60O=LK24U(90~cpT+}7?mr3-G0va+my zJ@`IcRG_=vk$#09tkfzEbiNPk4!gH7ZfBsQ5@~oOh^!~GUq2# z7PxAx4^z2aV@6jv8y@uI<9P6F51-(h;QXP{rzRgX-Ke%xNe&0&ZipOwM->lDA~&}- zq5JD6>|PrMF}rtdrve4Uj3#fIm{6ghN7G-~xpH$8PzN6Xz;9Ewu-l(rb%=r~tW=`= zV&OW9itBYJ&K<3-=a_f@8@$Nw4UDk!(ow)NTO&PJ5C5A=Ld zlT?$2J@kS}w|_k#9W>z3>|D8DrJ87kyYB4;u=PjmPFHS20x}B@ALgZM_;8B!*UtoYk z1e*Hka?urNsyOmwe0S!}pnt*P(7=9&b*FrIO`18}0~2@zCF#Te2YUnOAzS2Xa#~JN zzLoBf-{E(gp_Sit_w?l@=C=I7) zI#@_6KZ{bw#!7c=!>A2-szxbPT-gQfZbVEkp3_?896pECHL$P=SPK__ajgqCxTK5m z{|SAhhD6@I(PWch{v%gEjwfAU2Vt867dHOh$eae>_mU;w-akTC-c7D7{haws%T-!V zP1^FI^e`c7&P+Xf+FXhkXWkLAy7HsQQ~#dOhBuiFhwEAPt|#*iZ<-@otYNPjks~Wt zZVqynu-dBFGp}sP1)sR9+&}7HzU<=MZpol#Gx$Y3@5IvKXj3rsurO*)&_paDBwLQX zAKtTd1^a*ZiOS1&^AxS!wqDJ`gJ+jU1~{kU<~Pp7Wvs;tzUbnVq{m@iK4#k4LS`qC}^o9@u?LWBM%{BFPXx1oX z9;I}!o72x1;NuO(wFgHtWmhU~6u6!p%b+bB&-$oW?WHGYElY0epr%R4TR?w^iW>`; z#Kcl+$wqQi?;f%nc+DU{T4n8I6eJria81uqwdO^`k8Ev;qt?ogg8aFbXI1Dm3X3Z@ z7kk!L*Q-RloOP_)09<(t06O><63JLd)>XdP%wSxm_ErIU;xcoyz2X{#w%Vym;Fx)H z6_b4dT{dHY%#RUKw<#Fgb+;|4YlzTsbblP>ef_yM-7sJ4d2#dcp2M-oQKp^txc=EH z;j~;=J5L#yv0GJ@&^kjcs3DNPi*?OMV7i%tIX8sPUCs!O#G_=Dv{WXNH^npESHxRPjM}? zyzI+WF<4PWPUT!~s=;1Ro23nhfCAw0J5zOQ@HeTyJavzE5vlXuc3_xN{4$Kinmiunv;T zQwZ)8<0H)ALYrWETdbgSFHy&TL8F#cVggXZiniTo%nT{!a5)^VYv%}g;PVeeZ<$Nd zFh|uxttP-3k61sJ+KFZnc+$acqr(I*u@hn$mtW}TkR^@QGKOtVLTk%~2xPmJi4Hlw zdGn{s(s^8A&*S2s@6fvEuk5qqFJmfbT0h9f;ZqVDxi|Nq>ZVT$pP8D{M?Ch~?w0Ge z`RF%L@t>W^!J=ref#2t}wzeKF4AO{hOdID_q3xqiR9=NEg47qtSUVoAR5K82jvut| zB9VLTHq4dA=d%I+{y(?@;>thwSmHsx`MaM{Rc+w;!)9~Rr9iKr$42Xz;Sdp{l!=-*#+^Vx2`z`?77-26UpZuA(4q`PJ`W-m6&Jw zR%7F7Fb}j+wRjnH4q>eBut>wpu_#wLb)~~>s+%Vq0b0=n(b!<&-QHy*IuCF^J1x{+ zRyzG`wito2=cohzUXW*H_Si@3?Ty%tDR3_iY@-UUYiM-DZ?-T@UP7-Y?-@Angy+Gn zV{4V_xhwZQOdguV#b;&B9Zv*1>)k$pYmD+8A3lCL9_)}r+jomCJn(2kScjvPzUXq+ z3nQlqB&Ll%qTt!8n|Xw5PEIfl6^At=MRozkRBDx(Nm=QnePlDR!n1JN#y;trN8#Pr z%-Jg5idJF@(CLfad2BBVLkcb|YDkktbi#r1U}?D^g7gA^iF4Q^wd^>P&jx~@)$&BU zEX=f;k6m1mtIJDmTCcuY$eX)0RCV6gYN<~=&6ZLc=J*hWQANe_M~gCtybLt z8cU=ymY#I3(tFEsa#}t)jVtfHwmQ3g??oUU-7FFYDvndYiq0b(xBA& zw#{bWjr34Nu(H~127m3L01)vBA}*=8$fWM;=1{`h3{{gkKK{9jW-0i1u8}W+V|<4; zUSyM5QpHXiDc=(wp%x)_r@bIBLei}(V=~b|U58K__^nB|iTXg=7!XSiGufR-jVta- zwZ(`Puynhjnc>6M9V->tYn9}fPY_7b{a<4PDb82?N`zB)zGnKzDl03Sb?o1re_K|+ z6p*2FVUDS=YtR+86}g9eSwQ^AjPr9a?^aL7q@#Ftw8k*9VnW?L631G8?rj@0|L)XK z%+Sn^B-A6kDBJLfQJfh=<;B*FoL%|0nV zry8BWlMZ*z@bpTGO#52Oc8d|VlKLf&5W_*aI?3sMXzm&_t63{!m+kmC z9vY`JF-dlVA#OJOO`CLs*1avASQ(U?sFG{%M6l+>1p{qZK&graXl-mB$Pv;;~ev?b{|y}6T#V;i*wKLPf#Btl_DZg zsP_$X)Lei#Wu}Auc3=q0Nc!4tYKBG*VP;>~Hfoh*%#SXw9$lEQEUDzFxLbf^179F* zB~L@Tvs1S`^&SWEh7Z`_NWhHb&n<)}%C1R4yO$YbJ%d!?k6%HWQi07(?`x z6{+S9qSUSSS1T9OycF`rK7=Z3m&+@#HOsY}Sk-?;8#L?#RHjsjaojY@q~p1jw2WHo zH2R~#{ycVH6Q7Xp;B-*g%|Es<8jA}aD7I+*KhAyMn z5KvK3ks^oz5g3#rB4iY$h%^-$={*uk1XNU1Mp~pLh$2!1={0`{8{s`4B#km2>tvXP;eud;bk@?04;>1S*<3X@A;QTPV`i#l+Zc z%K1~YTO2n0c3x6W)paPG1G^)sR+iBs-lNdj`o`nj(4-M1Nvf@@yK3apsePK5(>u7l zT^%FwA+bwgt6f5uOnSyUk!+ioKHi>b!484B%lnCH%xl2D zuvqg-FQmQ-0scvmh>LHG`VTo3S;d*Rj~UDA?bwnj5;Fi!MY`Kx^DYj85zi|>4L7N) zMcjDa-E%2%)K@>I`j`&o%C0uVgR|$1UUhn!y?^Vq+^PDY-FWCuAziUOL;Eo$`)rrd z=H|x3f9#(qi@&B^K%~c3{Wa__oR*5!`^ogs8y9TdeGDCdA$Mf{7|3&&+3Bm6RaB%$ z*6TOPsm7@OAyE+Vw&wddO*{2Nj&|L-D|uW09Qo1H8!STY>YY$mDJXf)?){xS0tlog z`5__+A0IzB!_lMbqN+Co6JEl=nmf5J{PM!75IG7KWMw76+h)l%uveW$Fv@iMSzw(`EipKun|0~=?oWm z@_#Q6asjBXbf1VOioo@sXuL5UAuH`WY~1-qdi|>%S_pE6#;Gf(K1Em7M5(xSq=d=- zl0fAN$XYIZZ@S{I!tC`sXP`KeNoLB4f`aZRg12tn>c`*q2dJSJ9dH8-ARcPn8RU;c z@^uKx+xUl8pii2)uWx8*=$SKYVDCskR2+_|71>*T8R`r5wdUT_J8uwOgqEjVXJ#kw zsuF+Kr8-@9(0%^}*w?Jbf|}uhg{Q4)+I4dQ0ojvchvu_HVn%sd-nDh3)nUDJ9*eEo zaC{KsF|iG$Os>wVs5qQpckgCLlAYD1-8Q4aMT&MI3sVkGcY1@Tl$8bpGe_2Nv^tm$ zeTgH$T>_~Mzd#lsH&TX&9m)?K*)935&m5@0bl_kcUV<7z*()M{L{-%yq?iPv zGd*$C^b1XTU60S3<{Uaq{3$Y5&8Mep&jIUpBJn8SNXk>y9T=@_9Yw{5?bavvxwGD+ zk5l86<`uftl7&}ddsEOK_Ym8Oxi7GxgN05T8oC}$?k9-|0Q%@r*pB$kj%N@r#*)IX z7)95v56&_nqWQ2eogPwQ#y(1z)QWm14rf{5&t*cDc?Z~}jlLZw`51K`7@k#Pfo-R1C>zPLRH zwkq9C>}fa8)1|-ahMgk5Ng9er=I41@F@UeYvylOhB0^i$`Bnxtf_4t-b8EHe*I0qU2&gKxv}Z zFTVpy4x8iR6FQ%Btjm(W#;6JM;b4iru9C{H&-8gZOmm1k6h2j#!f@FhdqjO}etgY4Q)h5D5k{mDYh?WHi)r_8) zf=lCdSgq``VMNH!lLxNy3(B6|c%V$AT1w=CAgca@Ji4@L8#f+<;?_r!?U$o4An!_D z`T2qB$o`YFN5ANy9`2}lO=$cv_XVNYyd!kEX~s;0_+XQ;D*$88xlI8MsOUp;vf0Un za)rD~D(`@2D7*Jgyaa{80*JA`-X(AfONV^kKY`k+Ex(dODcSmT31+T>A!&RwB>g+;8}CZz zg1zR+U2q3}J9d6it2w{O!;LMbMGD5{d-6bODbMO{hP7G?Yhupmm%Q2N|9VazQS-`~(=}56AoW3&j;MmKS8sHBMFrcGJrHhsG z_k^>uvXVV21KpeeVlCx~#m@01#9yAAiLRW^PWI&XlEqJry}R>b4?%UdJ52woPum_a zlH*_`NaSK7%AW3AuHrGgUPBw=j?hNMs-;w@#BWA2UAOC;q2&{H{V3Bm_ye8b^Ls+t z#KhOFd%Ve?PTxd$U+5$%j}G=@wFDCG)#h6jj91KH++A(FhmXn2`h0#?TQSjhXYW2R zg7z>GN#ny2byfA7QC2%s{F^7;OF3k4pwK=H!7w=AeTi=tqyER-y>dXlna=i0jj_=+ zYm7ERnecW+k$yinQKSz@HPcpJ=&)uZsur>AqE4Me9d?lx8OyFMV_Vd1e&)4ve?h3# zM9gJ=upwR?C8(BD3qR;;P7OrsaK>ikqQ${%uYMzUDIoWWOlMONBi5QiK^w86zW7bT zE?n0f!2(Yxo@!FWRj>0e!BQ3$N@t(D=P7PgxwQW(zj3Z?U-^xAy|EF8Lndh=Jcm0b zCdP{pA&#$|M{8lr35LAM3IBC+j%V(uAKx7?d#pmU!VxkYIQ0jHo0FS61a6Q3oPU0# zj2&G9o$8F9HzvGEk2TX!4;JodYr`S&e3;O+tlt-v9j0&Z)5eo%ywy%pO+QvX)$2<6 zP1d?UZ{caWx0S)}y8F#6<-wZ%aL(l88o<=O2r!!VHV*FdFI*7%7T>;DL4#*xOj+7` zo12?Y)X+xgA9KtjxG8}iICrf7G;yXWK{jwHtFkFdOv8&DA5EzXx!(M^YBn6xnQY1C zDPYijbLA(c2p#FOjR$IY3c&y<1i%{VF5fjLSIrQ-rT}wMvyyFceZEFY<<+;l?H^<< zxOW5~5SEGcE&5|3#=puKF9=DC(g?{!16l%TvVW@J1VKpxu(XeZU;=nOI|vXo2xL#! zS3ne^p9Ijc|NED>jX+%edY=c8Ll5n6i11C|JQ(wT&}tCHJr1~=(BQfkdiVpuW1kBX zj&E@QmHk^^lH&%tKf@ov)E>%N_bW0y-@UsD6241upv^2Z4=>J8BJf2fl)OrW{H#pW zYRXJ(BvjUqoo(xo8O-Dp5;E+C_6YDP>nbBbxklH!TtZ5b(y-Hv_KM+j4S-hkf{unz zp48fTj;zAy9}+!lR+}G4$N}2BS#S$%#G9Fl4UzmqC}{Hk#3sBPmeiXk0S+EUa_= z3r7IS7?fALX+|zQvwjymFw&xCDeBU-;81INFlMKZ3gPAMtple#`R>}gW6H0x*u z1tiEe#ZtsnJSy}Zjgx80AU@SvJOy7QY5LxTZ9jGqA zb=E!HuNYI zWlQ{sU1!9(UwEZ8>n~cWSJ#pwrLkPaGCL&onS5ga=vgF=JR9mW2VnGejkPyhPRc9| zTsITxUfzQAq4xIMl}?D^nPm4`X)TigvyN;ncn+>xwD?E6NHn_ZXJJg>g$rL^bw8~g z&o+S%aH^!|qjlFNmlilZ-ST$6Y)Nj-LK&9R`!-rqmR_;^6Gswo{dmMm5!Re~tbnS3*Y<}w{4 zcXyjhMR2QI9TkL(BvX$QpU>yjjx&QeDjiLNkiKgSvttZ1X=>{IU-4NmZH&i!KuK!O zWgON;jY)CIdxl|i?90-~D5Z*ichefnjR=y+Cb{siV~aVG_Kf=bn=UWkhxfMIom>zn z!?-iU=zy~(CRLo~oCf&aZ@mPRlvJ_@+Q;1u6Q!)F3|up<%#xCf;$o70WGQ>a5_*qI zpTLA7mklu7P$4|JbcBJ6l=ZJmHxO>fFX~9VWMgA~3FIA2soT*s#m4VLbd6H zv961i=ch*+%-~aMt4UQpGC@jeJ5Kh$$OIi}`L_fNB07zm;_V+4!4^`Y(a5pwvgbpJ zac>KpLwgDY?!z=#m6SYOgRU#6=qOr+il6kuFZDNvZz>VjVP*1aA$08-Ml*btBLt2B zd7gNY)v~jUZpIW7e$Pjyvps3R!7$>o9F{QKb)SwjE9!={F;<1gCF>&nvviS)Gy>6D zyQC5)&&{!iCHizP$fw?(lZ)0_CuYjF1lW~`fId;5w`DufWrMT4D|MftkS15zRIR8JX<@$0B=BK>mQ0_du`Z#+K4SU;S)+2i9 zPRWpr)uG|N!-{6|xxf)ydnuKq>56c?;O$mXQc`gpwA_oUCNu=SXsO?TpAspA- ztgOSOp9ji=^{-rU3%sP~Xk+8LuJaS~#0inY%+xbHf1t(Z`Rf)<+ub<}VfWrkzaJU* zd%y3tGy-wXJGWF|n-Kg*_i71blQU8|JJ150)iSKJYjyeUL&aUd3K z3E`7;IADN47+T*7%*D1mmMQb94_DH9yjyeS?gUJmFewMHY;|BGDkxPcb-Yx}wt1eE zbkgOL<9b=#5)Pz741~8g4%SYX?l0&*aG@vfj3-b`540)c<1qucaX@!V&!eCxWc*rn zL{o3>08PcZ#q^Yce5A9Fv`|-JUhMwvSmjpbbqtxLvO2ZBLAPxffqSAS?)2FmoYY+r zhluS8&(S*jf>#RcXRfzAj`rPCH*}sidEWdfjJ-YZIrGj;if}(&Vw^mfEE_gjn2s@)eU1WCXC=k_plpdk9&?RFtC11jqpTE)f1yOF~m;^Qw9$jN+b zx|8`Z?NO)5fgVqw(?R9G?tm+IPoByJ4$il8H@Ld8vTnUJl(c0ODL@BrP|}ncewEK% z7eCT(PrE4jxa|l|kxmbgMo{7`R#|+tZ&E22SRvOM#v;l)S*;Dx;>i`uhRGyK1Y4_5 zmq{`=l2t#(x-9XU>5TbWh+tJ3Y<=YlVqN%krgXRrkP#nujg}dGid9}*EI~+OuW2tm zl$RA|jL+mxg<6#*{qR6ELLA}YUk2lVJ0tkkoN)iupr$ok2rbHfSY!5u2m(=|1y#ek zPMdLW(_KOb8}HW(Je8dMYE^#c#cftFmY!3lTB>Qw1YbNlY^~nVuWa zi4NTuv+8nisfbG~g7)->{siQvcvsuq^j!3@0uxjtCY*OlX|Ue&`8BE|N?B z(5o{Byyl?Uy+)kr(jXxrp~Y(Enx%d6kU4dfO(9MYPe*H`b~*%d%0>=F^9w*q`dfdNsfR^=WWWZBRAUj0fHFuqPJcby-Aoh6#!@ z_0Y&8*=Hu-6SJ~zFJtQ;3GPCl!|&r$#j8OrjF|T1to;bW`Svp|UJV&C`9&Zcd;2!R z0ycpQ^6Zu8@(OBSDtjP?1tTMoE#>T|I9|z}_ZD$-7SN;irA z9cm6|T}XwOFW$E&Rdq%y)sluJ-ztO#&E&uLaDoZ&L(arv%^zdU1wu0Kih>hO4$`r$ zJnyY^dPY0QB?xu%Ad9yG_LN0?L?!ecjR7>|T_xMXJe%J>2N}#}5IdVIpeokX0qcp2 zLQx-?esu%h6C*`Jxq4C_OHIBV*!8qCG(pI|m;Wee*hq~;!$4iXdfTdV$?sN-<~zu> zrub4tn zC_=oVz`XUQjJ5>zjgV)K8@K(YkI;GOJ<@lyAERUJurUD`-D$Nv3%myP^BwhHFsHxv z^D^H1pV;RA!mtk!dzS>d3w3vcO3U2 zlkM$QJaTA9hAcm1^Om)`_hJ`eHmj+i1633rK4;A1CwaaE%TFXZKy&|E4SaU0~k@_P?Xa-__7}HT3TW)xK+z@0#TQlO|y!fl0^Zy3h+U fz#sfP1CbB1>F9BYaDBfr&)hRMFw@UF?GXMq4X!5} literal 0 HcmV?d00001 From f8bc3203a6176aace04d5a08889b28c60ccaf02f Mon Sep 17 00:00:00 2001 From: Zeying Zhu <50204836+zzylol@users.noreply.github.com> Date: Mon, 19 Oct 2026 14:42:59 +0000 Subject: [PATCH 6/6] Give the exact baseline its own pipeline; add a raw pipeline The exact baseline no longer takes a fanout copy from the sketch branches. Three pipelines run side by side and share no processor; only the generated input is the same (each pipeline has its own generators with the same config and scenario start): - sketch: generator-a/b -> branch-a/b -> merge -> detector - exact: exact-generator-a/b -> exact-branch-a/b -> exact-merge -> exact-detector. exact_branch keeps every value per quantile series and a value -> count map per attribute key per event-time window (ExactWindow, sent as an "Exact" envelope); exact_merge adds the sources' windows; exact_detector computes the exact answers. - raw: raw-generator-a/b -> raw-branch-a/b -> raw-merge -> raw-sink. The branches and merge only forward points; raw_sink writes every point as a log record to loki-raw and watches its acknowledgements. Every pipeline reports its window completion delay as asap.answer.delay: sketch and exact when their detector emits a window's answers, raw when loki-raw acknowledged the window's last point. The latency dashboard draws the three pipelines and compares their delay, CPU, bytes and memory. --- asap-precompute-rs/src/answers.rs | 89 +- asap-precompute-rs/src/config.rs | 2 +- asap-precompute-rs/src/envelope.rs | 4 + asap-precompute-rs/src/otap/answer_nodes.rs | 317 +------ asap-precompute-rs/src/otap/baseline_nodes.rs | 881 ++++++++++++++++++ asap-precompute-rs/src/otap/codec.rs | 4 + asap-precompute-rs/src/otap/mod.rs | 2 + asap-precompute-rs/src/sketches/mod.rs | 5 + deploy/telemetry-functions/README.md | 51 +- .../common/exact-merge.yaml | 34 + .../common/raw-branch.yaml | 29 + .../telemetry-functions/common/raw-merge.yaml | 29 + .../telemetry-functions/common/raw-sink.yaml | 46 + deploy/telemetry-functions/compose.yaml | 131 ++- .../grafana/dashboards/latency.json | 286 +++++- deploy/telemetry-functions/loki-raw.yaml | 35 + .../telemetry-functions/pipelines/branch.yaml | 22 +- .../pipelines/exact-branch.yaml | 34 + .../{exact.yaml => exact-detector.yaml} | 15 +- .../pipelines/generator-a.yaml | 2 +- .../pipelines/generator-b.yaml | 2 +- deploy/telemetry-functions/prometheus.yml | 10 +- deploy/telemetry-functions/run.sh | 3 + 23 files changed, 1663 insertions(+), 370 deletions(-) create mode 100644 asap-precompute-rs/src/otap/baseline_nodes.rs create mode 100644 deploy/telemetry-functions/common/exact-merge.yaml create mode 100644 deploy/telemetry-functions/common/raw-branch.yaml create mode 100644 deploy/telemetry-functions/common/raw-merge.yaml create mode 100644 deploy/telemetry-functions/common/raw-sink.yaml create mode 100644 deploy/telemetry-functions/loki-raw.yaml create mode 100644 deploy/telemetry-functions/pipelines/exact-branch.yaml rename deploy/telemetry-functions/pipelines/{exact.yaml => exact-detector.yaml} (67%) diff --git a/asap-precompute-rs/src/answers.rs b/asap-precompute-rs/src/answers.rs index 65a3fab..a423908 100644 --- a/asap-precompute-rs/src/answers.rs +++ b/asap-precompute-rs/src/answers.rs @@ -20,7 +20,7 @@ //! q ± error. //! * Count-Min: counts never underestimate; with width w and N points the //! overestimate is at most e/w * N with probability 1 - e^-depth. -use std::collections::{HashMap, HashSet}; +use std::collections::HashMap; use crate::envelope::{SketchEnvelope, SketchType}; use crate::observation::KeyValue; @@ -126,19 +126,92 @@ pub fn sketch_answers( Ok(out) } -/// Exact data for one series and window. -#[derive(Default)] +/// Exact data for one series and window: what an exact branch builds and an +/// exact merge adds up. +#[derive(Debug, Default, PartialEq)] pub struct ExactWindow { /// Every value, for quantiles. pub values: Vec, - /// Distinct attribute values. - pub distinct: HashSet, - /// Occurrences per attribute value. + /// Occurrences per attribute value; its size is the distinct count. pub counts: HashMap, /// Data points. pub points: u64, } +const EXACT_MAGIC: &[u8; 4] = b"EXv1"; + +impl ExactWindow { + /// Adds another source's window. + pub fn merge(&mut self, other: ExactWindow) { + self.values.extend(other.values); + for (k, c) in other.counts { + *self.counts.entry(k).or_default() += c; + } + self.points += other.points; + } + + /// `EXv1`, points, values, then (key, count) pairs; little-endian. + pub fn encode(&self) -> Vec { + let keys: usize = self.counts.keys().map(|k| 12 + k.len()).sum(); + let mut out = Vec::with_capacity(28 + 8 * self.values.len() + keys); + out.extend_from_slice(EXACT_MAGIC); + out.extend_from_slice(&self.points.to_le_bytes()); + out.extend_from_slice(&(self.values.len() as u64).to_le_bytes()); + for v in &self.values { + out.extend_from_slice(&v.to_le_bytes()); + } + out.extend_from_slice(&(self.counts.len() as u64).to_le_bytes()); + for (k, c) in &self.counts { + out.extend_from_slice(&(k.len() as u32).to_le_bytes()); + out.extend_from_slice(k.as_bytes()); + out.extend_from_slice(&c.to_le_bytes()); + } + out + } + + /// Inverse of [`Self::encode`]. + pub fn decode(bytes: &[u8]) -> Result { + struct Cursor<'a>(&'a [u8]); + impl<'a> Cursor<'a> { + fn take(&mut self, n: usize) -> Result<&'a [u8], PrecomputeError> { + if self.0.len() < n { + return Err(PrecomputeError::Other( + "truncated exact window payload".into(), + )); + } + let (head, tail) = self.0.split_at(n); + self.0 = tail; + Ok(head) + } + fn u64(&mut self) -> Result { + Ok(u64::from_le_bytes(self.take(8)?.try_into().unwrap())) + } + } + let rest = bytes + .strip_prefix(EXACT_MAGIC) + .ok_or_else(|| PrecomputeError::Other("not an exact window payload".into()))?; + let mut c = Cursor(rest); + let points = c.u64()?; + let n = c.u64()? as usize; + let values = (0..n) + .map(|_| c.u64().map(f64::from_bits)) + .collect::, _>>()?; + let n = c.u64()? as usize; + let mut counts = HashMap::with_capacity(n.min(1 << 24)); + for _ in 0..n { + let len = u32::from_le_bytes(c.take(4)?.try_into().unwrap()) as usize; + let key = std::str::from_utf8(c.take(len)?) + .map_err(|_| PrecomputeError::Other("exact window key is not UTF-8".into()))?; + counts.insert(key.to_owned(), c.u64()?); + } + Ok(Self { + values, + counts, + points, + }) + } +} + /// Exact answers in the same shape as [`sketch_answers`]. pub fn exact_answers( template: &SketchEnvelope, @@ -169,8 +242,8 @@ pub fn exact_answers( )); } } - if !window.distinct.is_empty() { - out.push(e("asap.distinct", &[], window.distinct.len() as f64)); + if !window.counts.is_empty() { + out.push(e("asap.distinct", &[], window.counts.len() as f64)); } let mut top: Vec<_> = window.counts.iter().collect(); top.sort_unstable_by(|a, b| b.1.cmp(a.1).then(a.0.cmp(b.0))); diff --git a/asap-precompute-rs/src/config.rs b/asap-precompute-rs/src/config.rs index a4b7d92..dc45427 100644 --- a/asap-precompute-rs/src/config.rs +++ b/asap-precompute-rs/src/config.rs @@ -176,7 +176,7 @@ pub fn sketch_size_string(sketch_type: SketchType, params: &SketchParams) -> Opt _ => None, } } - SketchType::Unspecified => None, + SketchType::Unspecified | SketchType::Exact => None, } } diff --git a/asap-precompute-rs/src/envelope.rs b/asap-precompute-rs/src/envelope.rs index b09549a..0aed562 100644 --- a/asap-precompute-rs/src/envelope.rs +++ b/asap-precompute-rs/src/envelope.rs @@ -28,6 +28,8 @@ pub enum SketchType { CountMinSketch, /// Count-Min Sketch plus a top-k heavy-hitter heap (ASAPv1 kind `0x03 0x00`). CountMinSketchWithHeap, + /// Not a sketch: an exact baseline window ([`crate::answers::ExactWindow`]). + Exact, } impl SketchType { @@ -40,6 +42,7 @@ impl SketchType { Self::CountSketch => "CountSketch", Self::CountMinSketch => "CountMinSketch", Self::CountMinSketchWithHeap => "CountMinSketchWithHeap", + Self::Exact => "Exact", Self::Unspecified => "Unspecified", } } @@ -53,6 +56,7 @@ impl SketchType { Self::CountSketch, Self::CountMinSketch, Self::CountMinSketchWithHeap, + Self::Exact, Self::Unspecified, ] .into_iter() diff --git a/asap-precompute-rs/src/otap/answer_nodes.rs b/asap-precompute-rs/src/otap/answer_nodes.rs index c7a08c3..5cb7ace 100644 --- a/asap-precompute-rs/src/otap/answer_nodes.rs +++ b/asap-precompute-rs/src/otap/answer_nodes.rs @@ -1,21 +1,11 @@ -//! Two processors that turn telemetry into [`crate::answers`] gauges: -//! -//! * `urn:asap:processor:sketch_detector` reads merged sketch windows (the -//! output of `window_merge`) and emits each window's answers with bounds. -//! * `urn:asap:processor:exact_answers` reads raw data points, groups them -//! into event-time windows, and emits the exact answers: the ground truth -//! and the exact baseline. +//! `urn:asap:processor:sketch_detector` reads merged sketch windows (the +//! output of `window_merge`) and emits each window's answers with bounds. The +//! exact and raw baselines are in [`super::baseline_nodes`]. //! //! ```yaml //! detector: //! type: urn:asap:processor:sketch_detector //! config: {quantiles: [0.5, 0.99], k: 10} -//! exact: -//! type: urn:asap:processor:exact_answers -//! config: -//! window: 60s -//! group_by: [http.route] # quantile series, as aggregate_by -//! per_attribute: [user.id] # distinct values and top-k per key //! ``` use std::collections::BTreeMap; use std::sync::Arc; @@ -26,7 +16,6 @@ use linkme::distributed_slice; use otel_arrow_dfe_config::node::NodeUserConfig; use otel_arrow_dfe_engine::config::ProcessorConfig; use otel_arrow_dfe_engine::context::PipelineContext; -use otel_arrow_dfe_engine::control::NodeControlMsg; use otel_arrow_dfe_engine::error::Error; use otel_arrow_dfe_engine::local::processor as local; use otel_arrow_dfe_engine::message::Message; @@ -48,25 +37,23 @@ use otel_arrow_dfe_pdata::proto::OtlpProtoMessage; use otel_arrow_dfe_pdata::OtapPayload; use super::codec::decode_pdata_to_observations; -use crate::answers::{exact_answers, sketch_answers, ExactWindow}; +use crate::answers::sketch_answers; use crate::envelope::SketchEnvelope; -use crate::observation::{KeyValue, Observation, ObservationValueKind}; +use crate::observation::{KeyValue, Observation}; /// Registered URN of the sketch detector. pub const SKETCH_DETECTOR_URN: &str = "urn:asap:processor:sketch_detector"; -/// Registered URN of the exact baseline. -pub const EXACT_ANSWERS_URN: &str = "urn:asap:processor:exact_answers"; -fn default_quantiles() -> Vec { +pub(super) fn default_quantiles() -> Vec { vec![0.5, 0.99] } -fn default_k() -> usize { +pub(super) fn default_k() -> usize { 10 } -fn default_window() -> Duration { +pub(super) fn default_window() -> Duration { Duration::from_secs(60) } -fn default_grace() -> Duration { +pub(super) fn default_grace() -> Duration { Duration::from_secs(5) } @@ -82,40 +69,43 @@ pub struct DetectorConfig { pub k: usize, } -/// `exact_answers` configuration. -#[derive(Clone, Debug, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct ExactConfig { - /// Event-time window size. - #[serde(default = "default_window", with = "humantime_serde")] - pub window: Duration, - /// Wait after a window's end before answering it. - #[serde(default = "default_grace", with = "humantime_serde")] - pub grace: Duration, - /// Labels identifying a quantile series; empty disables quantiles. - #[serde(default)] - pub group_by: Vec, - /// Attribute keys whose distinct values and heavy hitters are answered. - #[serde(default)] - pub per_attribute: Vec, - /// Quantiles to answer. - #[serde(default = "default_quantiles")] - pub quantiles: Vec, - /// Heavy hitters per attribute key. - #[serde(default = "default_k")] - pub k: usize, -} - -fn now_ms() -> u64 { +pub(super) fn now_ms() -> u64 { SystemTime::now() .duration_since(UNIX_EPOCH) .map(|d| d.as_millis() as u64) .unwrap_or(0) } +/// Adds `asap.answer.delay` (seconds from a window's end to its answers) once +/// per window and method: the end-to-end latency of each answer path. +fn with_delay(envs: &[SketchEnvelope], now_ms: u64) -> Vec { + let mut out = envs.to_vec(); + let mut seen = std::collections::BTreeSet::new(); + for e in envs { + let method = e + .labels + .iter() + .find(|kv| kv.key == "method") + .map(|kv| kv.value.clone()); + if let Some(method) = method { + if seen.insert((e.window_end_ms, method.clone())) { + out.push(SketchEnvelope { + labels: vec![KeyValue::new("method", method)], + metric_name: "asap.answer.delay".into(), + window_start_ms: e.window_start_ms, + window_end_ms: e.window_end_ms, + value: now_ms.saturating_sub(e.window_end_ms) as f64 / 1000.0, + ..Default::default() + }); + } + } + } + out +} + /// Plain OTLP gauges with every label on the data point, so backends such /// as Prometheus keep them as series labels. -fn gauges_pdata(envs: &[SketchEnvelope]) -> Result { +pub(super) fn gauges_pdata(envs: &[SketchEnvelope]) -> Result { let mut by_metric: BTreeMap<&str, Vec> = BTreeMap::new(); for e in envs { let attributes: Vec = e @@ -154,21 +144,27 @@ fn gauges_pdata(envs: &[SketchEnvelope]) -> Result { Ok(OtapPdata::new_todo_context(payload)) } -async fn emit( +pub(super) async fn emit( envs: &[SketchEnvelope], effects: &mut local::EffectHandler, ) -> Result<(), Error> { if envs.is_empty() { return Ok(()); } - match gauges_pdata(envs) { + let envs = with_delay(envs, now_ms()); + match gauges_pdata(&envs) { Ok(pdata) => effects.send_message_with_source_node(pdata).await?, Err(e) => effects.info(&format!("answers: encode failed: {e}")).await, } Ok(()) } -async fn observations( +/// Identity of a series across windows: metric and labels. +pub(super) fn series_key(env: &SketchEnvelope) -> String { + format!("{}|{:?}", env.metric_name, env.labels) +} + +pub(super) async fn observations( pdata: OtapPdata, effects: &mut local::EffectHandler, ) -> Vec { @@ -211,136 +207,7 @@ impl local::Processor for SketchDetector { } } -/// Exact state of one window: per quantile series and per attribute key. -#[derive(Default)] -struct ExactState { - series: BTreeMap, -} - -struct ExactAnswers { - config: ExactConfig, - /// Open windows by window end. - windows: BTreeMap, - emitted_until: u64, - timer_started: bool, -} - -impl ExactAnswers { - fn template(obs: &Observation, labels: Vec, start: u64, end: u64) -> SketchEnvelope { - SketchEnvelope { - labels, - window_start_ms: start, - window_end_ms: end, - metric_name: obs.metric.clone(), - ..Default::default() - } - } - - fn observe(&mut self, obs: &Observation) { - if obs.value.kind != ObservationValueKind::Float { - return; - } - let size = self.config.window.as_millis().max(1) as u64; - let start = obs.timestamp_ms / size * size; - let end = start + size; - if end <= self.emitted_until { - return; - } - let state = self.windows.entry(end).or_default(); - let label = |k: &str| { - obs.labels - .iter() - .find(|kv| kv.key == k) - .map(|kv| kv.value.clone()) - }; - if !self.config.group_by.is_empty() { - let labels: Vec = self - .config - .group_by - .iter() - .filter_map(|k| label(k).map(|v| KeyValue::new(k.clone(), v))) - .collect(); - let key = format!("q|{}|{labels:?}", obs.metric); - let (_, w) = state.series.entry(key).or_insert_with(|| { - ( - Self::template(obs, labels, start, end), - ExactWindow::default(), - ) - }); - w.values.push(obs.value.float); - w.points += 1; - } - for attr in &self.config.per_attribute { - let Some(value) = label(attr) else { continue }; - let key = format!("a|{}|{attr}", obs.metric); - let (_, w) = state.series.entry(key).or_insert_with(|| { - let labels = vec![KeyValue::new("attribute.key", attr.clone())]; - ( - Self::template(obs, labels, start, end), - ExactWindow::default(), - ) - }); - *w.counts.entry(value.clone()).or_default() += 1; - w.distinct.insert(value); - w.points += 1; - } - } - - fn due(&mut self, now: u64) -> Vec { - let cutoff = now.saturating_sub(self.config.grace.as_millis() as u64); - let open = self.windows.split_off(&(cutoff + 1)); - let ready = std::mem::replace(&mut self.windows, open); - let mut out = Vec::new(); - for (end, state) in ready { - self.emitted_until = self.emitted_until.max(end); - for (_, (mut template, mut w)) in state.series { - template.count = w.points; - out.extend(exact_answers( - &template, - &mut w, - &self.config.quantiles, - self.config.k, - )); - } - } - out - } -} - -#[async_trait(?Send)] -impl local::Processor for ExactAnswers { - async fn process( - &mut self, - msg: Message, - effects: &mut local::EffectHandler, - ) -> Result<(), Error> { - if !self.timer_started { - self.timer_started = effects - .start_periodic_timer(Duration::from_secs(1)) - .await - .is_ok(); - } - match msg { - Message::PData(pdata) => { - for obs in observations(pdata, effects).await { - self.observe(&obs); - } - Ok(()) - } - Message::Control(NodeControlMsg::TimerTick { .. }) => { - let out = self.due(now_ms()); - emit(&out, effects).await - } - Message::Control(NodeControlMsg::Shutdown { .. }) => { - let out = self.due(u64::MAX); - emit(&out, effects).await - } - _ => Ok(()), - } - } -} - -fn invalid(e: impl std::fmt::Display) -> otel_arrow_dfe_config::error::Error { +pub(super) fn invalid(e: impl std::fmt::Display) -> otel_arrow_dfe_config::error::Error { otel_arrow_dfe_config::error::Error::InvalidUserConfig { error: e.to_string(), } @@ -367,91 +234,3 @@ static SKETCH_DETECTOR: ProcessorFactory = ProcessorFactory { wiring_contract: otel_arrow_dfe_engine::wiring_contract::WiringContract::UNRESTRICTED, validate_config: otel_arrow_dfe_config::validation::validate_typed_config::, }; - -#[allow(unsafe_code)] -#[distributed_slice(OTAP_PROCESSOR_FACTORIES)] -static EXACT_ANSWERS: ProcessorFactory = ProcessorFactory { - name: EXACT_ANSWERS_URN, - create: |_: PipelineContext, - node: NodeId, - node_config: Arc, - processor_config: &ProcessorConfig, - _: &otel_arrow_dfe_engine::capability::registry::Capabilities| { - let config: ExactConfig = - serde_json::from_value(node_config.config.clone()).map_err(invalid)?; - Ok(ProcessorWrapper::local( - ExactAnswers { - config, - windows: BTreeMap::new(), - emitted_until: 0, - timer_started: false, - }, - node, - node_config, - processor_config, - )) - }, - wiring_contract: otel_arrow_dfe_engine::wiring_contract::WiringContract::UNRESTRICTED, - validate_config: otel_arrow_dfe_config::validation::validate_typed_config::, -}; - -#[cfg(test)] -mod tests { - use super::*; - use crate::observation::ObservationValue; - - fn point(ts: u64, route: &str, user: &str, v: f64) -> Observation { - Observation { - timestamp_ms: ts, - metric: "latency".into(), - resource_labels: vec![], - labels: vec![ - KeyValue::new("http.route", route), - KeyValue::new("user.id", user), - ], - value: ObservationValue::float(v), - } - } - - #[test] - fn exact_windows_by_event_time_after_grace() { - let mut exact = ExactAnswers { - config: serde_json::from_value(serde_json::json!({ - "group_by": ["http.route"], "per_attribute": ["user.id"], "k": 2 - })) - .unwrap(), - windows: BTreeMap::new(), - emitted_until: 0, - timer_started: false, - }; - for i in 0..100u64 { - exact.observe(&point(1_000 + i, "/a", &format!("u{}", i % 4), i as f64)); - } - exact.observe(&point(61_000, "/a", "u0", 1.0)); - assert!(exact.due(64_000).is_empty()); - let out = exact.due(65_000); - let get = |metric: &str, extra: (&str, &str)| { - out.iter() - .find(|e| { - e.metric_name == metric - && e.labels - .iter() - .any(|kv| kv.key == extra.0 && kv.value == extra.1) - }) - .map(|e| e.value) - }; - assert_eq!(get("asap.quantile", ("quantile", "0.5")), Some(50.0)); - assert_eq!( - get("asap.distinct", ("attribute.key", "user.id")), - Some(4.0) - ); - assert_eq!(get("asap.topk", ("rank", "1")), Some(25.0)); - assert!(out.iter().all(|e| e.window_end_ms == 60_000)); - // A late point for the emitted window is dropped. - exact.observe(&point(2_000, "/a", "u9", 1.0)); - assert!(exact - .due(125_000) - .iter() - .all(|e| e.window_end_ms == 120_000)); - } -} diff --git a/asap-precompute-rs/src/otap/baseline_nodes.rs b/asap-precompute-rs/src/otap/baseline_nodes.rs new file mode 100644 index 0000000..a38d151 --- /dev/null +++ b/asap-precompute-rs/src/otap/baseline_nodes.rs @@ -0,0 +1,881 @@ +//! The exact baseline and the raw baseline, as separate pipelines that share +//! no processor with the sketch pipeline; only the generated input is the same. +//! +//! * `urn:asap:processor:exact_branch` (one per source) keeps every raw point +//! of each event-time window as an [`ExactWindow`]: the values of each +//! quantile series and a value -> count map per attribute key. It emits a +//! window when data crosses its end, or 5 s after its end if the stream idles. +//! * `urn:asap:processor:exact_merge` adds up the sources' windows, `grace` +//! after each window's end. +//! * `urn:asap:processor:exact_detector` computes each merged window's exact +//! answers and events, with `method="exact"`: the ground truth. +//! * `urn:asap:processor:raw_sink` ends the raw pipeline, which only forwards +//! raw points: it writes every point as one log record to a store through +//! its `logs` output and watches the store's acknowledgements. For each +//! event-time window it reports on its `latency` output when the store +//! acknowledged the window's last point (`asap.answer.delay`, +//! `method="raw"`), the per-point time from event to acknowledgement +//! (`asap.raw.transit`) and rejected points (`asap.raw.failed`). +//! +//! ```yaml +//! exact_branch: +//! type: urn:asap:processor:exact_branch +//! config: {window: 60s, group_by: [http.route], per_attribute: [user.id]} +//! exact_merge: +//! type: urn:asap:processor:exact_merge +//! config: {grace: 5s} +//! exact_detector: +//! type: urn:asap:processor:exact_detector +//! config: {quantiles: [0.5, 0.99], k: 10} +//! raw_sink: +//! type: urn:asap:processor:raw_sink +//! outputs: [logs, latency] +//! config: {window: 60s, grace: 5s} +//! ``` +use std::collections::{BTreeMap, HashMap}; +use std::sync::Arc; +use std::time::Duration; + +use async_trait::async_trait; +use linkme::distributed_slice; +use otel_arrow_dfe_config::node::NodeUserConfig; +use otel_arrow_dfe_engine::config::ProcessorConfig; +use otel_arrow_dfe_engine::context::PipelineContext; +use otel_arrow_dfe_engine::control::{AckMsg, CallData, Context8u8, NodeControlMsg}; +use otel_arrow_dfe_engine::error::Error; +use otel_arrow_dfe_engine::local::processor as local; +use otel_arrow_dfe_engine::message::Message; +use otel_arrow_dfe_engine::node::NodeId; +use otel_arrow_dfe_engine::processor::ProcessorWrapper; +use otel_arrow_dfe_engine::{ + ConsumerEffectHandlerExtension, Interests, ProducerEffectHandlerExtension, +}; +use otel_arrow_dfe_engine::{MessageSourceLocalEffectHandlerExtension, ProcessorFactory}; +use otel_arrow_dfe_otap::pdata::OtapPdata; +use otel_arrow_dfe_otap::OTAP_PROCESSOR_FACTORIES; +use otel_arrow_dfe_pdata::proto::opentelemetry::common::v1::{ + AnyValue, InstrumentationScope, KeyValue as OtlpKeyValue, +}; +use otel_arrow_dfe_pdata::proto::opentelemetry::logs::v1::{ + LogRecord, LogsData, ResourceLogs, ScopeLogs, +}; +use otel_arrow_dfe_pdata::proto::opentelemetry::resource::v1::Resource; +use otel_arrow_dfe_pdata::proto::OtlpProtoMessage; +use otel_arrow_dfe_pdata::OtapPayload; +use serde::Deserialize; + +use super::answer_nodes::{ + default_grace, default_k, default_quantiles, default_window, emit, gauges_pdata, invalid, + now_ms, observations, series_key, +}; +use super::codec::OtapSketchEncoder; +use crate::answers::{exact_answers, gauge, ExactWindow}; +use crate::envelope::{SketchEnvelope, SketchType}; + +use crate::observation::{KeyValue, Observation, ObservationValueKind}; + +/// Registered URN of the exact branch. +pub const EXACT_BRANCH_URN: &str = "urn:asap:processor:exact_branch"; +/// Registered URN of the exact merge. +pub const EXACT_MERGE_URN: &str = "urn:asap:processor:exact_merge"; +/// Registered URN of the exact detector. +pub const EXACT_DETECTOR_URN: &str = "urn:asap:processor:exact_detector"; +/// Registered URN of the raw pipeline's sink. +pub const RAW_SINK_URN: &str = "urn:asap:processor:raw_sink"; +/// `raw_sink` output with the log records for the store. +const LOGS_PORT: &str = "logs"; +/// `raw_sink` output with the latency gauges. +const LATENCY_PORT: &str = "latency"; + +/// Wait after a window's end before the idle timer closes it, as in +/// `asap_sketches`: data crossing the end closes it on time. +const IDLE_CLOSE_GRACE: Duration = Duration::from_secs(5); + +/// `exact_branch` configuration. +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ExactBranchConfig { + /// Event-time window size. + #[serde(default = "default_window", with = "humantime_serde")] + pub window: Duration, + /// Labels identifying a quantile series, as `aggregate_by`; empty + /// disables quantiles. + #[serde(default)] + pub group_by: Vec, + /// Attribute keys whose value counts are kept; `*` for every key. + #[serde(default)] + pub per_attribute: Vec, +} + +/// `exact_merge` configuration. +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ExactMergeConfig { + /// How long after a window's end to wait for the last source. + #[serde(default = "default_grace", with = "humantime_serde")] + pub grace: Duration, +} + +/// `exact_detector` configuration. +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ExactDetectorConfig { + /// Quantiles to answer. + #[serde(default = "default_quantiles")] + pub quantiles: Vec, + /// Heavy hitters per attribute key. + #[serde(default = "default_k")] + pub k: usize, +} + +/// `raw_sink` configuration. +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RawSinkConfig { + /// Event-time window size. + #[serde(default = "default_window", with = "humantime_serde")] + pub window: Duration, + /// Wait after a window's end before reporting it. + #[serde(default = "default_grace", with = "humantime_serde")] + pub grace: Duration, +} + +/// Open windows by window end, then series. +type Windows = BTreeMap>; + +/// Removes and encodes every window ending at or before `until`. +fn close(windows: &mut Windows, until: u64) -> Vec { + let open = windows.split_off(&(until.saturating_add(1))); + std::mem::replace(windows, open) + .into_values() + .flat_map(|series| series.into_values()) + .map(|(mut env, w)| { + env.count = w.points; + env.payload = w.encode(); + env + }) + .collect() +} + +async fn send( + encoder: &mut OtapSketchEncoder, + envs: &[SketchEnvelope], + effects: &mut local::EffectHandler, +) -> Result<(), Error> { + if envs.is_empty() { + return Ok(()); + } + match encoder.encode(envs) { + Ok(pdata) => effects.send_message_with_source_node(pdata).await?, + Err(e) => effects.info(&format!("exact: encode failed: {e}")).await, + } + Ok(()) +} + +struct ExactBranch { + config: ExactBranchConfig, + windows: Windows, + /// Every window ending at or before this has been emitted. + emitted_until: u64, + encoder: OtapSketchEncoder, + timer_started: bool, +} + +impl ExactBranch { + fn observe(&mut self, obs: &Observation) { + if obs.value.kind != ObservationValueKind::Float { + return; + } + let size = self.config.window.as_millis().max(1) as u64; + let start = obs.timestamp_ms / size * size; + let end = start + size; + if end <= self.emitted_until { + return; + } + let state = self.windows.entry(end).or_default(); + let template = |labels: Vec| SketchEnvelope { + schema_version: 1, + sketch_type: SketchType::Exact, + labels, + window_start_ms: start, + window_end_ms: end, + metric_name: obs.metric.clone(), + ..Default::default() + }; + let label = |k: &str| { + obs.labels + .iter() + .find(|kv| kv.key == k) + .map(|kv| kv.value.clone()) + }; + if !self.config.group_by.is_empty() { + let labels: Vec = self + .config + .group_by + .iter() + .filter_map(|k| label(k).map(|v| KeyValue::new(k.clone(), v))) + .collect(); + let (_, w) = state + .entry(format!("q|{labels:?}")) + .or_insert_with(|| (template(labels), ExactWindow::default())); + w.values.push(obs.value.float); + w.points += 1; + } + let all = self.config.per_attribute.iter().any(|k| k == "*"); + for kv in &obs.labels { + if !all && !self.config.per_attribute.contains(&kv.key) { + continue; + } + let (_, w) = state.entry(format!("a|{}", kv.key)).or_insert_with(|| { + let labels = vec![KeyValue::new("attribute.key", kv.key.clone())]; + (template(labels), ExactWindow::default()) + }); + *w.counts.entry(kv.value.clone()).or_default() += 1; + w.points += 1; + } + } + + fn close(&mut self, until: u64) -> Vec { + let out = close(&mut self.windows, until); + if let Some(end) = out.iter().map(|e| e.window_end_ms).max() { + self.emitted_until = self.emitted_until.max(end); + } + out + } +} + +#[async_trait(?Send)] +impl local::Processor for ExactBranch { + async fn process( + &mut self, + msg: Message, + effects: &mut local::EffectHandler, + ) -> Result<(), Error> { + if !self.timer_started { + self.timer_started = effects + .start_periodic_timer(Duration::from_secs(1)) + .await + .is_ok(); + } + let out = match msg { + Message::PData(pdata) => { + // Consumed into state here: ack so upstream can move on. + effects.notify_ack(AckMsg::new(pdata.clone())).await?; + let points = observations(pdata, effects).await; + // A window closes as soon as a point's event time passes its end. + let out = match points.first() { + Some(first) => self.close(first.timestamp_ms), + None => Vec::new(), + }; + points.iter().for_each(|obs| self.observe(obs)); + out + } + Message::Control(NodeControlMsg::TimerTick { .. }) => { + self.close(now_ms().saturating_sub(IDLE_CLOSE_GRACE.as_millis() as u64)) + } + Message::Control(NodeControlMsg::Shutdown { .. }) => self.close(u64::MAX), + _ => Vec::new(), + }; + send(&mut self.encoder, &out, effects).await + } +} + +struct ExactMerge { + config: ExactMergeConfig, + windows: Windows, + emitted_until: u64, + encoder: OtapSketchEncoder, + timer_started: bool, +} + +impl ExactMerge { + /// Adds one source's window; windows already emitted drop it as late. + fn add(&mut self, env: SketchEnvelope) -> Result<(), crate::precompute::PrecomputeError> { + if env.window_end_ms <= self.emitted_until { + return Ok(()); + } + let w = ExactWindow::decode(&env.payload)?; + let key = format!("{}|{}", series_key(&env), env.window_start_ms); + let window = self.windows.entry(env.window_end_ms).or_default(); + match window.get_mut(&key) { + Some((_, merged)) => merged.merge(w), + None => { + window.insert(key, (env, w)); + } + } + Ok(()) + } + + fn due(&mut self, now: u64) -> Vec { + let cutoff = now.saturating_sub(self.config.grace.as_millis() as u64); + let out = close(&mut self.windows, cutoff); + if let Some(end) = out.iter().map(|e| e.window_end_ms).max() { + self.emitted_until = self.emitted_until.max(end); + } + out + } +} + +#[async_trait(?Send)] +impl local::Processor for ExactMerge { + async fn process( + &mut self, + msg: Message, + effects: &mut local::EffectHandler, + ) -> Result<(), Error> { + if !self.timer_started { + self.timer_started = effects + .start_periodic_timer(Duration::from_secs(1)) + .await + .is_ok(); + } + let out = match msg { + Message::PData(pdata) => { + effects.notify_ack(AckMsg::new(pdata.clone())).await?; + for obs in observations(pdata, effects).await { + let Some(env) = obs.value.envelope else { + continue; + }; + if let Err(e) = self.add(*env) { + effects.info(&format!("exact_merge: {e}")).await; + } + } + Vec::new() + } + Message::Control(NodeControlMsg::TimerTick { .. }) => self.due(now_ms()), + Message::Control(NodeControlMsg::Shutdown { .. }) => self.due(u64::MAX), + _ => Vec::new(), + }; + send(&mut self.encoder, &out, effects).await + } +} + +struct ExactDetector { + config: ExactDetectorConfig, +} + +impl ExactDetector { + /// Answers of one merged window. + fn answer(&mut self, mut template: SketchEnvelope) -> Vec { + let mut out = Vec::new(); + let Ok(mut w) = ExactWindow::decode(&template.payload) else { + return out; + }; + template.payload = Vec::new(); + template.sketch_type = SketchType::Unspecified; + template.count = w.points; + out.extend(exact_answers( + &template, + &mut w, + &self.config.quantiles, + self.config.k, + )); + out + } +} + +#[async_trait(?Send)] +impl local::Processor for ExactDetector { + async fn process( + &mut self, + msg: Message, + effects: &mut local::EffectHandler, + ) -> Result<(), Error> { + let Message::PData(pdata) = msg else { + return Ok(()); + }; + effects.notify_ack(AckMsg::new(pdata.clone())).await?; + let mut out = Vec::new(); + for obs in observations(pdata, effects).await { + let Some(env) = obs.value.envelope.filter(|e| !e.payload.is_empty()) else { + continue; + }; + out.extend(self.answer(*env)); + } + emit(&out, effects).await + } +} + +/// What the raw pipeline stored for one event-time window. +#[derive(Default)] +struct RawWindow { + /// When the store acknowledged the window's last point (ms). + last_ack_ms: u64, + /// Store acknowledgement minus event time of every stored point (s). + transit_s: Vec, + /// Points the store rejected. + failed: u64, + /// Messages with points of this window still waiting for the store. + in_flight: usize, +} + +struct RawSink { + config: RawSinkConfig, + windows: BTreeMap, + /// Event times (ms) by window end of every message the store has not + /// acknowledged yet, by message id. + in_flight: HashMap)>>, + next_id: u64, + emitted_until: u64, + timer_started: bool, +} + +/// One log record per raw point: event time, the value as the body, the +/// point's attributes. +fn logs_pdata(points: &[Observation]) -> Result { + let mut by_resource: BTreeMap, Vec> = BTreeMap::new(); + for p in points { + let resource = p + .resource_labels + .iter() + .map(|kv| (kv.key.clone(), kv.value.clone())) + .collect(); + let attributes: Vec = p + .labels + .iter() + .map(|kv| OtlpKeyValue::new(kv.key.clone(), AnyValue::new_string(kv.value.clone()))) + .collect(); + by_resource.entry(resource).or_default().push( + LogRecord::build() + .time_unix_nano(p.timestamp_ms * 1_000_000) + .body(AnyValue::new_double(p.value.float)) + .attributes(attributes) + .finish(), + ); + } + let resource_logs: Vec = by_resource + .into_iter() + .map(|(resource, records)| { + let attributes: Vec = resource + .into_iter() + .map(|(k, v)| OtlpKeyValue::new(k, AnyValue::new_string(v))) + .collect(); + ResourceLogs::new( + Resource::build().attributes(attributes).finish(), + vec![ScopeLogs::new( + InstrumentationScope::build().name("asap.raw").finish(), + records, + )], + ) + }) + .collect(); + let payload: OtapPayload = OtlpProtoMessage::Logs(LogsData::new(resource_logs)) + .try_into() + .map_err(|e| format!("{e:?}"))?; + Ok(OtapPdata::new_todo_context(payload)) +} + +impl RawSink { + /// Registers a message about to go to the store; returns its id. + fn send(&mut self, points: &[Observation]) -> u64 { + let size = self.config.window.as_millis().max(1) as u64; + let mut by_window: BTreeMap> = BTreeMap::new(); + for p in points { + let end = p.timestamp_ms / size * size + size; + if end > self.emitted_until { + by_window.entry(end).or_default().push(p.timestamp_ms); + } + } + for end in by_window.keys() { + self.windows.entry(*end).or_default().in_flight += 1; + } + let id = self.next_id; + self.next_id += 1; + let _ = self.in_flight.insert(id, by_window.into_iter().collect()); + id + } + + /// The store answered message `id` at `now`. + fn answered(&mut self, id: u64, now: u64, stored: bool) { + for (end, times) in self.in_flight.remove(&id).unwrap_or_default() { + let Some(w) = self.windows.get_mut(&end) else { + continue; + }; + w.in_flight = w.in_flight.saturating_sub(1); + if stored { + w.last_ack_ms = w.last_ack_ms.max(now); + w.transit_s + .extend(times.iter().map(|t| now.saturating_sub(*t) as f64 / 1000.0)); + } else { + w.failed += times.len() as u64; + } + } + } + + /// Gauges of every window that ended `grace` or more before `now` and + /// has no message waiting for the store (or ended 10 graces ago). + fn due(&mut self, now: u64) -> Vec { + let grace = self.config.grace.as_millis() as u64; + let size = self.config.window.as_millis() as u64; + let ready: Vec = self + .windows + .iter() + .filter(|(end, w)| { + **end + grace <= now && (w.in_flight == 0 || **end + 10 * grace <= now) + }) + .map(|(end, _)| *end) + .collect(); + let mut out = Vec::new(); + for end in ready { + let mut w = self.windows.remove(&end).unwrap_or_default(); + self.emitted_until = self.emitted_until.max(end); + let template = SketchEnvelope { + window_start_ms: end - size, + window_end_ms: end, + count: w.transit_s.len() as u64, + ..Default::default() + }; + let metric = |name: &str, extra: &[(&str, String)], v: f64| { + let mut e = gauge(&template, name, "raw", extra, v); + // Window-level gauges: no source metric label. + e.labels.retain(|kv| kv.key != "metric"); + e + }; + // Complete once the last point is stored, and not before the + // window has ended. + out.push(metric( + "asap.answer.delay", + &[], + w.last_ack_ms.saturating_sub(end) as f64 / 1000.0, + )); + out.push(metric("asap.points", &[], w.transit_s.len() as f64)); + out.push(metric("asap.raw.failed", &[], w.failed as f64)); + if !w.transit_s.is_empty() { + w.transit_s.sort_unstable_by(f64::total_cmp); + let at = + |q: f64| w.transit_s[((w.transit_s.len() - 1) as f64 * q).round() as usize]; + for q in [0.5, 0.99] { + out.push(metric( + "asap.raw.transit", + &[("quantile", q.to_string())], + at(q), + )); + } + } + } + out + } +} + +#[async_trait(?Send)] +impl local::Processor for RawSink { + async fn process( + &mut self, + msg: Message, + effects: &mut local::EffectHandler, + ) -> Result<(), Error> { + if !self.timer_started { + self.timer_started = effects + .start_periodic_timer(Duration::from_secs(1)) + .await + .is_ok(); + } + let out = match msg { + Message::PData(pdata) => { + effects.notify_ack(AckMsg::new(pdata.clone())).await?; + let points: Vec = observations(pdata, effects) + .await + .into_iter() + .filter(|o| o.value.kind == ObservationValueKind::Float) + .collect(); + if points.is_empty() { + return Ok(()); + } + match logs_pdata(&points) { + Ok(mut logs) => { + let id = self.send(&points); + let calldata = CallData::from_vec(vec![Context8u8::from(id)]); + effects.subscribe_to(Interests::ACKS_OR_NACKS, calldata, &mut logs); + effects + .send_message_to(LOGS_PORT, logs) + .await + .map_err(Error::from)?; + } + Err(e) => effects.info(&format!("raw_sink: encode failed: {e}")).await, + } + return Ok(()); + } + Message::Control(NodeControlMsg::Ack(ack)) => { + if let Some(id) = ack.unwind.route.calldata.first() { + self.answered(u64::from(*id), now_ms(), true); + } + return Ok(()); + } + Message::Control(NodeControlMsg::Nack(nack)) => { + if let Some(id) = nack.unwind.route.calldata.first() { + self.answered(u64::from(*id), now_ms(), false); + } + return Ok(()); + } + Message::Control(NodeControlMsg::TimerTick { .. }) => self.due(now_ms()), + Message::Control(NodeControlMsg::Shutdown { .. }) => self.due(u64::MAX), + _ => return Ok(()), + }; + if out.is_empty() { + return Ok(()); + } + // Not through `emit`: the delay here is the last store + // acknowledgement, not the time this gauge is sent. + match gauges_pdata(&out) { + Ok(pdata) => effects + .send_message_to(LATENCY_PORT, pdata) + .await + .map_err(Error::from)?, + Err(e) => effects.info(&format!("raw_sink: encode failed: {e}")).await, + } + Ok(()) + } +} + +fn config( + node_config: &NodeUserConfig, +) -> Result { + serde_json::from_value(node_config.config.clone()).map_err(invalid) +} + +#[allow(unsafe_code)] +#[distributed_slice(OTAP_PROCESSOR_FACTORIES)] +static EXACT_BRANCH: ProcessorFactory = ProcessorFactory { + name: EXACT_BRANCH_URN, + create: |_: PipelineContext, + node: NodeId, + node_config: Arc, + processor_config: &ProcessorConfig, + _: &otel_arrow_dfe_engine::capability::registry::Capabilities| { + Ok(ProcessorWrapper::local( + ExactBranch { + config: config(&node_config)?, + windows: Windows::new(), + emitted_until: 0, + encoder: OtapSketchEncoder::default(), + timer_started: false, + }, + node, + node_config, + processor_config, + )) + }, + wiring_contract: otel_arrow_dfe_engine::wiring_contract::WiringContract::UNRESTRICTED, + validate_config: otel_arrow_dfe_config::validation::validate_typed_config::, +}; + +#[allow(unsafe_code)] +#[distributed_slice(OTAP_PROCESSOR_FACTORIES)] +static EXACT_MERGE: ProcessorFactory = ProcessorFactory { + name: EXACT_MERGE_URN, + create: |_: PipelineContext, + node: NodeId, + node_config: Arc, + processor_config: &ProcessorConfig, + _: &otel_arrow_dfe_engine::capability::registry::Capabilities| { + Ok(ProcessorWrapper::local( + ExactMerge { + config: config(&node_config)?, + windows: Windows::new(), + emitted_until: 0, + encoder: OtapSketchEncoder::default(), + timer_started: false, + }, + node, + node_config, + processor_config, + )) + }, + wiring_contract: otel_arrow_dfe_engine::wiring_contract::WiringContract::UNRESTRICTED, + validate_config: otel_arrow_dfe_config::validation::validate_typed_config::, +}; + +#[allow(unsafe_code)] +#[distributed_slice(OTAP_PROCESSOR_FACTORIES)] +static EXACT_DETECTOR: ProcessorFactory = ProcessorFactory { + name: EXACT_DETECTOR_URN, + create: |_: PipelineContext, + node: NodeId, + node_config: Arc, + processor_config: &ProcessorConfig, + _: &otel_arrow_dfe_engine::capability::registry::Capabilities| { + let config: ExactDetectorConfig = config(&node_config)?; + Ok(ProcessorWrapper::local( + ExactDetector { config }, + node, + node_config, + processor_config, + )) + }, + wiring_contract: otel_arrow_dfe_engine::wiring_contract::WiringContract::UNRESTRICTED, + validate_config: otel_arrow_dfe_config::validation::validate_typed_config::, +}; + +#[allow(unsafe_code)] +#[distributed_slice(OTAP_PROCESSOR_FACTORIES)] +static RAW_SINK: ProcessorFactory = ProcessorFactory { + name: RAW_SINK_URN, + create: |_: PipelineContext, + node: NodeId, + node_config: Arc, + processor_config: &ProcessorConfig, + _: &otel_arrow_dfe_engine::capability::registry::Capabilities| { + Ok(ProcessorWrapper::local( + RawSink { + config: config(&node_config)?, + windows: BTreeMap::new(), + in_flight: HashMap::new(), + next_id: 0, + emitted_until: 0, + timer_started: false, + }, + node, + node_config, + processor_config, + )) + }, + wiring_contract: otel_arrow_dfe_engine::wiring_contract::WiringContract::UNRESTRICTED, + validate_config: otel_arrow_dfe_config::validation::validate_typed_config::, +}; + +#[cfg(test)] +mod tests { + use super::*; + use crate::observation::ObservationValue; + + fn point(ts: u64, route: &str, user: &str, v: f64) -> Observation { + Observation { + timestamp_ms: ts, + metric: "latency".into(), + resource_labels: vec![], + labels: vec![ + KeyValue::new("http.route", route), + KeyValue::new("user.id", user), + ], + value: ObservationValue::float(v), + } + } + + fn branch() -> ExactBranch { + ExactBranch { + config: serde_json::from_value(serde_json::json!({ + "group_by": ["http.route"], "per_attribute": ["user.id"] + })) + .unwrap(), + windows: Windows::new(), + emitted_until: 0, + encoder: OtapSketchEncoder::default(), + timer_started: false, + } + } + + fn get(out: &[SketchEnvelope], metric: &str, extra: (&str, &str)) -> Option { + out.iter() + .find(|e| { + e.metric_name == metric + && e.labels + .iter() + .any(|kv| kv.key == extra.0 && kv.value == extra.1) + }) + .map(|e| e.value) + } + + #[test] + fn exact_window_round_trips() { + let mut w = ExactWindow { + values: vec![1.5, -2.0], + points: 3, + ..Default::default() + }; + w.counts.insert("a".into(), 2); + w.counts.insert("bé".into(), 1); + assert_eq!(ExactWindow::decode(&w.encode()).unwrap(), w); + assert!(ExactWindow::decode(b"EXv1\x01").is_err()); + } + + #[test] + fn branches_merge_and_answer_like_one_exact_window() { + // Two sources, 50 points each in window [0, 60 s); a later point + // closes the window in each branch. + let (mut a, mut b) = (branch(), branch()); + for i in 0..50u64 { + a.observe(&point(1_000 + i, "/a", &format!("u{}", i % 4), i as f64)); + b.observe(&point( + 1_000 + i, + "/a", + &format!("u{}", i % 4), + (50 + i) as f64, + )); + } + let mut sources = a.close(61_000); + sources.extend(b.close(61_000)); + assert_eq!( + sources.len(), + 4, + "a quantile and an attribute series per source" + ); + let mut merge = ExactMerge { + config: ExactMergeConfig { + grace: Duration::from_secs(5), + }, + windows: Windows::new(), + emitted_until: 0, + encoder: OtapSketchEncoder::default(), + timer_started: false, + }; + sources.into_iter().for_each(|e| merge.add(e).unwrap()); + assert!(merge.due(64_000).is_empty()); + let merged = merge.due(65_000); + assert_eq!(merged.len(), 2); + assert!(merged.iter().all(|e| e.count == 100)); + let config: ExactDetectorConfig = + serde_json::from_value(serde_json::json!({"k": 2})).unwrap(); + let mut detector = ExactDetector { config }; + let out: Vec<_> = merged + .into_iter() + .flat_map(|e| detector.answer(e)) + .collect(); + assert_eq!(get(&out, "asap.quantile", ("quantile", "0.5")), Some(50.0)); + assert_eq!( + get(&out, "asap.distinct", ("attribute.key", "user.id")), + Some(4.0) + ); + assert_eq!(get(&out, "asap.topk", ("rank", "1")), Some(26.0)); + assert!(out.iter().all(|e| e.window_end_ms == 60_000)); + // A late window is dropped. + merge.add(branch_window(2_000)).unwrap(); + assert!(merge.due(200_000).is_empty()); + } + + fn branch_window(ts: u64) -> SketchEnvelope { + let mut b = branch(); + b.observe(&point(ts, "/a", "u0", 1.0)); + b.close(u64::MAX).remove(0) + } + + #[test] + fn raw_sink_reports_the_last_acknowledgement_after_grace() { + let mut raw = RawSink { + config: serde_json::from_value(serde_json::json!({})).unwrap(), + windows: BTreeMap::new(), + in_flight: HashMap::new(), + next_id: 0, + emitted_until: 0, + timer_started: false, + }; + let first = raw.send(&[point(10_000, "/a", "u0", 1.0)]); + let last = raw.send(&[ + point(59_900, "/a", "u1", 1.0), + point(59_950, "/b", "u2", 1.0), + ]); + let lost = raw.send(&[point(30_000, "/a", "u3", 1.0)]); + raw.answered(first, 10_500, true); + raw.answered(lost, 30_100, false); + // Past the grace, but one message is still waiting for the store. + assert!(raw.due(65_000).is_empty()); + raw.answered(last, 61_200, true); + let out = raw.due(65_000); + assert_eq!(get(&out, "asap.answer.delay", ("method", "raw")), Some(1.2)); + assert_eq!(get(&out, "asap.points", ("method", "raw")), Some(3.0)); + assert_eq!(get(&out, "asap.raw.failed", ("method", "raw")), Some(1.0)); + assert_eq!( + get(&out, "asap.raw.transit", ("quantile", "0.99")), + Some(1.3) + ); + } +} diff --git a/asap-precompute-rs/src/otap/codec.rs b/asap-precompute-rs/src/otap/codec.rs index 720dfbe..61dbbaf 100644 --- a/asap-precompute-rs/src/otap/codec.rs +++ b/asap-precompute-rs/src/otap/codec.rs @@ -183,6 +183,10 @@ fn validate_self_describing_payload(sketch_type: SketchType, payload: &[u8]) -> if sketch_type == SketchType::Unspecified { return Err("sketch.envelope requires a sketch.type".into()); } + if sketch_type == SketchType::Exact { + // Exact windows can be large; their consumer decodes them once. + return Ok(()); + } let decoded = crate::sketches::decode_snapshot(sketch_type, payload).map(drop); decoded.map_err(|error| format!("invalid {} sketch.envelope: {error}", sketch_type.name())) } diff --git a/asap-precompute-rs/src/otap/mod.rs b/asap-precompute-rs/src/otap/mod.rs index 141dcd2..838e0ba 100644 --- a/asap-precompute-rs/src/otap/mod.rs +++ b/asap-precompute-rs/src/otap/mod.rs @@ -13,6 +13,8 @@ pub mod config; #[cfg(feature = "otap-engine")] pub mod answer_nodes; #[cfg(feature = "otap-engine")] +pub mod baseline_nodes; +#[cfg(feature = "otap-engine")] pub mod codec; #[cfg(feature = "otap-engine")] pub mod processor; diff --git a/asap-precompute-rs/src/sketches/mod.rs b/asap-precompute-rs/src/sketches/mod.rs index 74b8c23..8a77948 100644 --- a/asap-precompute-rs/src/sketches/mod.rs +++ b/asap-precompute-rs/src/sketches/mod.rs @@ -40,5 +40,10 @@ pub fn decode_snapshot( "sketch.envelope requires a sketch.type".into(), )) } + SketchType::Exact => { + return Err(PrecomputeError::Other( + "an exact window is not a sketch".into(), + )) + } }) } diff --git a/deploy/telemetry-functions/README.md b/deploy/telemetry-functions/README.md index 1af874a..7c84c72 100644 --- a/deploy/telemetry-functions/README.md +++ b/deploy/telemetry-functions/README.md @@ -4,30 +4,49 @@ Every stage is the official OTAP dataflow engine (`asap-collector`: the upstream controller with the ASAP nodes linked in), one container per stage, each pinned to its own CPU, connected by OTLP/HTTP over the compose network. +Three pipelines run side by side over the same input and share no processor: + ```text -generator-a -> branch-a --sketch--> merge -> detector --> Prometheus -> Grafana -generator-b -> branch-b --sketch--/ - \--raw copy (fanout)--> exact ------/ +sketch: generator-a/b -> branch-a/b (sketch per window) -> merge -> detector -> Prometheus +exact: exact-generator-a/b -> exact-branch-a/b (exact window) -> exact-merge -> exact-detector -> Prometheus +raw: raw-generator-a/b -> raw-branch-a/b (forward) -> raw-merge (forward) -> raw-sink -> loki-raw ``` -| Stage | Pipeline | What it does | +The two generators of each pipeline (sources a and b) run the same configs +with the same scenario start (`SCENARIO_START`), and every point is a function +of scenario time, so all three pipelines receive identical points. + +| Stage | Config | What it runs | | --- | --- | --- | -| generator-a/b | `pipelines/generator-*.yaml` | `scenario_generator`: seeded latency points over 20 routes (temporary stand-in for the official traffic generator) | -| branch-a/b | `pipelines/branch.yaml` | `fanout` copies the input: `asap_sketches` builds one KLL per route per 1-minute window; the raw copy goes to `exact` | +| generator-a/b, exact-generator-a/b, raw-generator-a/b | `pipelines/generator-*.yaml` | `scenario_generator`: seeded latency points over 20 routes (temporary stand-in for the official traffic generator) | +| branch-a/b | `pipelines/branch.yaml` | `asap_sketches`: one KLL per route per 1-minute window | | merge | `pipelines/merge.yaml` | `window_merge`: merges both sources' sketches per window, 5 s after the window ends | -| detector | `pipelines/detector.yaml` | `sketch_detector`: p50/p99 per route with lower/upper bounds | -| exact | `pipelines/exact.yaml` | `exact_answers`: exact p50/p99 per route from every raw point (ground truth and exact baseline) | - -Prometheus receives both answer streams over OTLP (`asap_quantile`, -`asap_points`, labelled `method="sketch"` or `"exact"`) and scrapes each -collector's engine telemetry (`/api/v1/metrics?format=prometheus`): CPU time -per pipeline (`cpu_time_total`), resident memory (`memory_rss`) and payload -bytes per receiver, labelled with the stage. +| detector | `pipelines/detector.yaml` | `sketch_detector`: p50/p99 per route with lower/upper bounds (`method="sketch"`) | +| exact-branch-a/b | `pipelines/exact-branch.yaml` | `exact_branch`: every value per route per 1-minute window | +| exact-merge | `common/exact-merge.yaml` | `exact_merge`: adds both sources' exact windows, 5 s after the window ends | +| exact-detector | `pipelines/exact-detector.yaml` | `exact_detector`: exact p50/p99 per route (`method="exact"`): the ground truth | +| raw-branch-a/b, raw-merge | `common/raw-*.yaml` | forward raw points (OTLP receiver -> exporter) | +| raw-sink | `common/raw-sink.yaml` | `raw_sink`: writes every raw point as one log record to `loki-raw` and reports when `loki-raw` acknowledged each window's last point | + +Prometheus receives the answers over OTLP (`asap_quantile`, `asap_points`, +labelled `method="sketch"` or `"exact"`) and scrapes each collector's engine +telemetry (`/api/v1/metrics?format=prometheus`): CPU time per pipeline +(`cpu_time_total`), resident memory (`memory_rss`) and payload bytes per +receiver, labelled with the stage. + +Every pipeline reports its window completion delay as `asap_answer_delay` +(seconds from a window's end until its result is complete at the end of the +pipeline): sketch and exact when their detector emits the window's answers, +raw when `loki-raw` has acknowledged the window's last point. raw-sink also +reports each point's time from its event time to that acknowledgement +(`asap_raw_transit`, p50 and p99) and points `loki-raw` rejected +(`asap_raw_failed`). ## Run -Linux, Docker with Compose, 7 free CPUs (0-6 by default; `CPU_BRANCH_A=10` -etc. move a stage). From the repository root: +Linux, Docker with Compose, 18 free CPUs (0-17 by default; `CPU_BRANCH_A=20` +etc. move a stage) and memory for `loki-raw` (capped at `LOKI_RAW_MEMORY`, +default 16g). From the repository root: ```sh deploy/telemetry-functions/run.sh # DOCKER_COMMAND="sudo docker" if needed diff --git a/deploy/telemetry-functions/common/exact-merge.yaml b/deploy/telemetry-functions/common/exact-merge.yaml new file mode 100644 index 0000000..e8f44de --- /dev/null +++ b/deploy/telemetry-functions/common/exact-merge.yaml @@ -0,0 +1,34 @@ +# Exact pipeline, stage 2: add up both sources' exact windows, 5 s after +# each window ends. +version: otel_dataflow/v1 +engine: + http_admin: + bind_address: 0.0.0.0:8085 +policies: + resources: + core_allocation: + type: core_count + count: 1 +groups: + default: + pipelines: + main: + nodes: + receiver: + type: receiver:otlp + config: + protocols: + http: {listening_addr: "0.0.0.0:4318", wait_for_result: false, max_request_body_size: 1GiB} + merge: + type: urn:asap:processor:exact_merge + config: {grace: 5s} + to_detector: + type: exporter:otlp_http + config: + endpoint: http://exact-detector:4318 + client_pool_size: 1 + max_in_flight: 1 + http: {timeout: 30s, compression: none} + connections: + - {from: receiver, to: merge} + - {from: merge, to: to_detector} diff --git a/deploy/telemetry-functions/common/raw-branch.yaml b/deploy/telemetry-functions/common/raw-branch.yaml new file mode 100644 index 0000000..ea1b866 --- /dev/null +++ b/deploy/telemetry-functions/common/raw-branch.yaml @@ -0,0 +1,29 @@ +# Raw pipeline, stage 1 at each source: forward every raw point. +version: otel_dataflow/v1 +engine: + http_admin: + bind_address: 0.0.0.0:8085 +policies: + resources: + core_allocation: + type: core_count + count: 1 +groups: + default: + pipelines: + main: + nodes: + receiver: + type: receiver:otlp + config: + protocols: + http: {listening_addr: "0.0.0.0:4318", wait_for_result: false} + to_merge: + type: exporter:otlp_http + config: + endpoint: http://raw-merge:4318 + client_pool_size: 1 + max_in_flight: 1 + http: {timeout: 30s, compression: none} + connections: + - {from: receiver, to: to_merge} diff --git a/deploy/telemetry-functions/common/raw-merge.yaml b/deploy/telemetry-functions/common/raw-merge.yaml new file mode 100644 index 0000000..e9d3664 --- /dev/null +++ b/deploy/telemetry-functions/common/raw-merge.yaml @@ -0,0 +1,29 @@ +# Raw pipeline, stage 2: forward both sources' raw points to one place. +version: otel_dataflow/v1 +engine: + http_admin: + bind_address: 0.0.0.0:8085 +policies: + resources: + core_allocation: + type: core_count + count: 1 +groups: + default: + pipelines: + main: + nodes: + receiver: + type: receiver:otlp + config: + protocols: + http: {listening_addr: "0.0.0.0:4318", wait_for_result: false} + to_sink: + type: exporter:otlp_http + config: + endpoint: http://raw-sink:4318 + client_pool_size: 1 + max_in_flight: 1 + http: {timeout: 30s, compression: none} + connections: + - {from: receiver, to: to_sink} diff --git a/deploy/telemetry-functions/common/raw-sink.yaml b/deploy/telemetry-functions/common/raw-sink.yaml new file mode 100644 index 0000000..fa34ce2 --- /dev/null +++ b/deploy/telemetry-functions/common/raw-sink.yaml @@ -0,0 +1,46 @@ +# Raw pipeline, end: write every raw point as a log record to loki-raw, and +# report per window when loki-raw acknowledged its last point +# (asap.answer.delay, method="raw") and each point's time to acknowledgement. +version: otel_dataflow/v1 +engine: + http_admin: + bind_address: 0.0.0.0:8085 +policies: + resources: + core_allocation: + type: core_count + count: 1 +groups: + default: + pipelines: + main: + nodes: + receiver: + type: receiver:otlp + config: + protocols: + http: {listening_addr: "0.0.0.0:4318", wait_for_result: false} + sink: + type: urn:asap:processor:raw_sink + outputs: [logs, latency] + config: {window: 60s, grace: 5s} + to_loki_raw: + type: exporter:otlp_http + config: + endpoint: http://loki-raw:3100 + logs_endpoint: http://loki-raw:3100/otlp/v1/logs + client_pool_size: 1 + max_in_flight: 1 + http: {timeout: 30s, compression: none} + to_prometheus: + type: exporter:otlp_http + config: + endpoint: http://prometheus:9090 + metrics_endpoint: http://prometheus:9090/api/v1/otlp/v1/metrics + client_pool_size: 1 + max_in_flight: 1 + http: {timeout: 30s, compression: none} + connections: + - {from: receiver, to: sink} + - {from: 'sink["logs"]', to: to_loki_raw} + - {from: 'sink["latency"]', to: to_prometheus} diff --git a/deploy/telemetry-functions/compose.yaml b/deploy/telemetry-functions/compose.yaml index 672aa11..8931593 100644 --- a/deploy/telemetry-functions/compose.yaml +++ b/deploy/telemetry-functions/compose.yaml @@ -1,54 +1,142 @@ -# Every pipeline stage is the official OTAP engine (asap-collector) in its -# own container, pinned to one CPU, talking OTLP/HTTP over the compose -# network. Prometheus, Grafana and Loki are the observers. -# CPU_* variables move a stage to another core. +# Three pipelines over the same generated input, sharing no processor: +# sketch: generator-a/b -> branch-a/b (sketch) -> merge -> detector +# exact: exact-generator-a/b -> exact-branch-a/b (exact window) -> +# exact-merge -> exact-detector +# raw: raw-generator-a/b -> raw-branch-a/b -> raw-merge -> raw-sink +# (every raw point into loki-raw) +# Each stage is the official OTAP engine (asap-collector) in its own container, +# pinned to one CPU, talking OTLP/HTTP over the compose network. The generators +# of a source share one config (`downstream` is their pipeline's first stage) +# and SCENARIO_START, so all three pipelines get identical points. +# ./pipelines holds the use case's stage configs, ./common the stages that do +# not depend on it; CPU_* variables move a stage to another core. Prometheus, +# Grafana and Loki are the observers. x-collector: &collector image: asap-collector:local restart: "no" +x-generator: &generator + <<: *collector + environment: + ASAP_SCENARIO_START: "${SCENARIO_START:-}" services: + # sketch pipeline generator-a: - <<: *collector + <<: *generator command: ["--config", "/pipelines/generator-a.yaml"] - volumes: ["./pipelines:/pipelines:ro"] + volumes: ["./pipelines:/pipelines:ro", "./common:/common:ro"] cpuset: "${CPU_GENERATOR_A:-0}" + links: ["branch-a:downstream"] depends_on: [branch-a] generator-b: - <<: *collector + <<: *generator command: ["--config", "/pipelines/generator-b.yaml"] - volumes: ["./pipelines:/pipelines:ro"] + volumes: ["./pipelines:/pipelines:ro", "./common:/common:ro"] cpuset: "${CPU_GENERATOR_B:-1}" + links: ["branch-b:downstream"] depends_on: [branch-b] branch-a: <<: *collector command: ["--config", "/pipelines/branch.yaml"] - volumes: ["./pipelines:/pipelines:ro"] + volumes: ["./pipelines:/pipelines:ro", "./common:/common:ro"] cpuset: "${CPU_BRANCH_A:-2}" - depends_on: [merge, exact] + depends_on: [merge] branch-b: <<: *collector command: ["--config", "/pipelines/branch.yaml"] - volumes: ["./pipelines:/pipelines:ro"] + volumes: ["./pipelines:/pipelines:ro", "./common:/common:ro"] cpuset: "${CPU_BRANCH_B:-3}" - depends_on: [merge, exact] + depends_on: [merge] merge: <<: *collector command: ["--config", "/pipelines/merge.yaml"] - volumes: ["./pipelines:/pipelines:ro"] + volumes: ["./pipelines:/pipelines:ro", "./common:/common:ro"] cpuset: "${CPU_MERGE:-4}" depends_on: [detector] detector: <<: *collector command: ["--config", "/pipelines/detector.yaml"] - volumes: ["./pipelines:/pipelines:ro"] + volumes: ["./pipelines:/pipelines:ro", "./common:/common:ro"] cpuset: "${CPU_DETECTOR:-5}" depends_on: [prometheus] - exact: + # exact pipeline + exact-generator-a: + <<: *generator + command: ["--config", "/pipelines/generator-a.yaml"] + volumes: ["./pipelines:/pipelines:ro", "./common:/common:ro"] + cpuset: "${CPU_EXACT_GENERATOR_A:-6}" + links: ["exact-branch-a:downstream"] + depends_on: [exact-branch-a] + exact-generator-b: + <<: *generator + command: ["--config", "/pipelines/generator-b.yaml"] + volumes: ["./pipelines:/pipelines:ro", "./common:/common:ro"] + cpuset: "${CPU_EXACT_GENERATOR_B:-7}" + links: ["exact-branch-b:downstream"] + depends_on: [exact-branch-b] + exact-branch-a: + <<: *collector + command: ["--config", "/pipelines/exact-branch.yaml"] + volumes: ["./pipelines:/pipelines:ro", "./common:/common:ro"] + cpuset: "${CPU_EXACT_BRANCH_A:-8}" + depends_on: [exact-merge] + exact-branch-b: + <<: *collector + command: ["--config", "/pipelines/exact-branch.yaml"] + volumes: ["./pipelines:/pipelines:ro", "./common:/common:ro"] + cpuset: "${CPU_EXACT_BRANCH_B:-9}" + depends_on: [exact-merge] + exact-merge: + <<: *collector + command: ["--config", "/common/exact-merge.yaml"] + volumes: ["./pipelines:/pipelines:ro", "./common:/common:ro"] + cpuset: "${CPU_EXACT_MERGE:-10}" + depends_on: [exact-detector] + exact-detector: <<: *collector - command: ["--config", "/pipelines/exact.yaml"] - volumes: ["./pipelines:/pipelines:ro"] - cpuset: "${CPU_EXACT:-6}" + command: ["--config", "/pipelines/exact-detector.yaml"] + volumes: ["./pipelines:/pipelines:ro", "./common:/common:ro"] + cpuset: "${CPU_EXACT_DETECTOR:-11}" depends_on: [prometheus] + # raw pipeline + raw-generator-a: + <<: *generator + command: ["--config", "/pipelines/generator-a.yaml"] + volumes: ["./pipelines:/pipelines:ro", "./common:/common:ro"] + cpuset: "${CPU_RAW_GENERATOR_A:-12}" + links: ["raw-branch-a:downstream"] + depends_on: [raw-branch-a] + raw-generator-b: + <<: *generator + command: ["--config", "/pipelines/generator-b.yaml"] + volumes: ["./pipelines:/pipelines:ro", "./common:/common:ro"] + cpuset: "${CPU_RAW_GENERATOR_B:-13}" + links: ["raw-branch-b:downstream"] + depends_on: [raw-branch-b] + raw-branch-a: + <<: *collector + command: ["--config", "/common/raw-branch.yaml"] + volumes: ["./pipelines:/pipelines:ro", "./common:/common:ro"] + cpuset: "${CPU_RAW_BRANCH_A:-14}" + depends_on: [raw-merge] + raw-branch-b: + <<: *collector + command: ["--config", "/common/raw-branch.yaml"] + volumes: ["./pipelines:/pipelines:ro", "./common:/common:ro"] + cpuset: "${CPU_RAW_BRANCH_B:-15}" + depends_on: [raw-merge] + raw-merge: + <<: *collector + command: ["--config", "/common/raw-merge.yaml"] + volumes: ["./pipelines:/pipelines:ro", "./common:/common:ro"] + cpuset: "${CPU_RAW_MERGE:-16}" + depends_on: [raw-sink] + raw-sink: + <<: *collector + command: ["--config", "/common/raw-sink.yaml"] + volumes: ["./pipelines:/pipelines:ro", "./common:/common:ro"] + cpuset: "${CPU_RAW_SINK:-17}" + depends_on: [prometheus, loki-raw] prometheus: image: prom/prometheus:v3.5.0 @@ -57,6 +145,13 @@ services: - --web.enable-otlp-receiver ports: ["127.0.0.1:${PROMETHEUS_PORT:-9090}:9090"] volumes: ["./prometheus.yml:/etc/prometheus/prometheus.yml:ro"] + # The raw pipeline's store, apart from the Loki with the events. + loki-raw: + image: grafana/loki:3.5.0 + command: ["-config.file=/etc/loki/loki-raw.yaml"] + ports: ["127.0.0.1:${LOKI_RAW_PORT:-3101}:3100"] + volumes: ["./loki-raw.yaml:/etc/loki/loki-raw.yaml:ro"] + mem_limit: "${LOKI_RAW_MEMORY:-16g}" grafana: image: grafana/grafana:12.1.0 ports: ["127.0.0.1:${GRAFANA_PORT:-3000}:3000"] diff --git a/deploy/telemetry-functions/grafana/dashboards/latency.json b/deploy/telemetry-functions/grafana/dashboards/latency.json index cd493ed..ddbd86e 100644 --- a/deploy/telemetry-functions/grafana/dashboards/latency.json +++ b/deploy/telemetry-functions/grafana/dashboards/latency.json @@ -31,7 +31,23 @@ }, "panels": [ { - "id": 1, + "type": "text", + "title": "Pipeline", + "description": "Three pipelines that share no processor, fed identical points by their own generators (same seed and scenario start). Each box is one collector container pinned to its own CPU; arrows are OTLP/HTTP between containers.", + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 0 + }, + "options": { + "mode": "html", + "content": "
generator-a/b
sources A and B
→
branch-a/b
Stage 1 per source: KLL (k = 200) per route per 1-min window
→
merge
Stage 2: merges both sources' sketches per window
→
detector
p50/p99 with bounds
→
Prometheus + Loki
method="sketch"
exact-generator-a/b
same points
→
exact-branch-a/b
Stage 1 per source: every value per route per 1-min window
→
exact-merge
Stage 2: adds both sources' exact windows
→
exact-detector
exact p50/p99
→
Prometheus + Loki
method="exact" (ground truth)
raw-generator-a/b
same points
→
raw-branch-a/b
Stage 1 per source: forwards every raw point
→
raw-merge
Stage 2: forwards both sources' raw points
→
raw-sink
writes every raw point; measures when the window's last point is stored
→
loki-raw
one log record per raw point
" + }, + "id": 1 + }, + { + "id": 2, "type": "timeseries", "title": "p99 latency for $route: sketch with bound vs exact", "datasource": { @@ -42,7 +58,7 @@ "h": 8, "w": 12, "x": 0, - "y": 0 + "y": 10 }, "targets": [ { @@ -164,7 +180,7 @@ } }, { - "id": 2, + "id": 3, "type": "timeseries", "title": "p50 latency for $route: sketch with bound vs exact", "datasource": { @@ -175,7 +191,7 @@ "h": 8, "w": 12, "x": 12, - "y": 0 + "y": 10 }, "targets": [ { @@ -297,7 +313,7 @@ } }, { - "id": 3, + "id": 4, "type": "timeseries", "title": "Relative error of the sketch p99 vs exact (routes with > 20,000 points per window)", "datasource": { @@ -308,7 +324,7 @@ "h": 8, "w": 12, "x": 0, - "y": 8 + "y": 18 }, "targets": [ { @@ -328,9 +344,86 @@ } }, { - "id": 4, "type": "timeseries", - "title": "CPU cores per pinned stage", + "title": "Window completion delay: seconds from a window's end until its result is complete", + "datasource": { + "type": "prometheus", + "uid": "asap-prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 26 + }, + "targets": [ + { + "refId": "A", + "expr": "avg by (method) (asap_answer_delay)", + "legendFormat": "{{method}}" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "spanNulls": true + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "sketch" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#5794f2" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "exact" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#ff9830" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "raw" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#8e8e8e" + } + } + ] + } + ] + }, + "description": "sketch and exact: the detector has emitted the window's answers. raw: loki-raw has acknowledged the window's last raw point.", + "id": 5 + }, + { + "type": "timeseries", + "title": "CPU cores per pipeline (all stages but the generators)", "datasource": { "type": "prometheus", "uid": "asap-prometheus" @@ -339,13 +432,13 @@ "h": 8, "w": 12, "x": 12, - "y": 8 + "y": 26 }, "targets": [ { "refId": "A", - "expr": "sum by (stage) (rate(cpu_time_total{job=\"collectors\",otel_scope_pipeline_id=\"main\"}[1m]))", - "legendFormat": "{{stage}}" + "expr": "sum by (pipeline) (label_replace(label_replace(rate(cpu_time_total{job=\"collectors\",stage!~\".*generator.*\",otel_scope_pipeline_id=\"main\"}[1m]), \"pipeline\", \"sketch\", \"stage\", \".*\"), \"pipeline\", \"$1\", \"stage\", \"(exact|raw)-.*\"))", + "legendFormat": "{{pipeline}}" } ], "fieldConfig": { @@ -355,13 +448,59 @@ }, "unit": "short" }, - "overrides": [] - } + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "sketch" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#5794f2" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "exact" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#ff9830" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "raw" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#8e8e8e" + } + } + ] + } + ] + }, + "id": 6 }, { - "id": 5, "type": "timeseries", - "title": "Bytes received per stage, log scale (sketches at merge, raw points at exact)", + "title": "Bytes received per pipeline (all stages but the generators), log scale", "datasource": { "type": "prometheus", "uid": "asap-prometheus" @@ -370,13 +509,13 @@ "h": 8, "w": 12, "x": 0, - "y": 16 + "y": 34 }, "targets": [ { "refId": "A", - "expr": "sum by (stage) (rate(payload_size_bytes_total{job=\"collectors\",otel_scope_node_type=\"receiver\"}[1m])) * 8", - "legendFormat": "{{stage}}" + "expr": "sum by (pipeline) (label_replace(label_replace(rate(payload_size_bytes_total{job=\"collectors\",stage!~\".*generator.*\",otel_scope_node_type=\"receiver\"}[1m]), \"pipeline\", \"sketch\", \"stage\", \".*\"), \"pipeline\", \"$1\", \"stage\", \"(exact|raw)-.*\")) * 8", + "legendFormat": "{{pipeline}}" } ], "fieldConfig": { @@ -390,13 +529,59 @@ }, "unit": "bps" }, - "overrides": [] - } + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "sketch" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#5794f2" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "exact" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#ff9830" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "raw" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#8e8e8e" + } + } + ] + } + ] + }, + "id": 7 }, { - "id": 6, "type": "timeseries", - "title": "Resident memory per stage", + "title": "Resident memory per pipeline (all stages but the generators)", "datasource": { "type": "prometheus", "uid": "asap-prometheus" @@ -405,13 +590,13 @@ "h": 8, "w": 12, "x": 12, - "y": 16 + "y": 34 }, "targets": [ { "refId": "A", - "expr": "max by (stage) (memory_rss{job=\"collectors\"})", - "legendFormat": "{{stage}}" + "expr": "sum by (pipeline) (label_replace(label_replace(max by (stage) (memory_rss{job=\"collectors\",stage!~\".*generator.*\"}), \"pipeline\", \"sketch\", \"stage\", \".*\"), \"pipeline\", \"$1\", \"stage\", \"(exact|raw)-.*\"))", + "legendFormat": "{{pipeline}}" } ], "fieldConfig": { @@ -421,8 +606,55 @@ }, "unit": "bytes" }, - "overrides": [] - } + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "sketch" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#5794f2" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "exact" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#ff9830" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "raw" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#8e8e8e" + } + } + ] + } + ] + }, + "id": 8 } ] } diff --git a/deploy/telemetry-functions/loki-raw.yaml b/deploy/telemetry-functions/loki-raw.yaml new file mode 100644 index 0000000..04ff483 --- /dev/null +++ b/deploy/telemetry-functions/loki-raw.yaml @@ -0,0 +1,35 @@ +# Store of the raw pipeline: one log record per raw point. Limits are raised +# so that ingestion speed, not a rate limit, decides what the pipeline sees. +auth_enabled: false +server: + http_listen_port: 3100 + grpc_server_max_recv_msg_size: 1073741824 +common: + instance_addr: 127.0.0.1 + path_prefix: /loki + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory +schema_config: + configs: + - from: 2020-10-24 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h +limits_config: + ingestion_rate_mb: 1024 + ingestion_burst_size_mb: 2048 + per_stream_rate_limit: 1GB + per_stream_rate_limit_burst: 2GB + reject_old_samples: false + max_structured_metadata_entries_count: 512 +analytics: + reporting_enabled: false diff --git a/deploy/telemetry-functions/pipelines/branch.yaml b/deploy/telemetry-functions/pipelines/branch.yaml index 4166399..8d2d148 100644 --- a/deploy/telemetry-functions/pipelines/branch.yaml +++ b/deploy/telemetry-functions/pipelines/branch.yaml @@ -1,5 +1,5 @@ -# Stage 1 at each source: one KLL per route per 1-minute window goes to -# merge; the raw points are copied to the exact baseline. +# Sketch pipeline, stage 1 at each source: one KLL per route per 1-minute +# window goes to merge. version: otel_dataflow/v1 engine: http_admin: @@ -19,13 +19,6 @@ groups: config: protocols: http: {listening_addr: "0.0.0.0:4318", wait_for_result: false} - fanout: - type: processor:fanout - outputs: [sketch, raw] - config: - destinations: - - {port: sketch} - - {port: raw} sketch: type: urn:asap:processor:asap_sketches config: @@ -42,15 +35,6 @@ groups: client_pool_size: 1 max_in_flight: 1 http: {timeout: 30s, compression: none} - to_exact: - type: exporter:otlp_http - config: - endpoint: http://exact:4318 - client_pool_size: 1 - max_in_flight: 1 - http: {timeout: 30s, compression: none} connections: - - {from: receiver, to: fanout} - - {from: 'fanout["sketch"]', to: sketch} + - {from: receiver, to: sketch} - {from: sketch, to: to_merge} - - {from: 'fanout["raw"]', to: to_exact} diff --git a/deploy/telemetry-functions/pipelines/exact-branch.yaml b/deploy/telemetry-functions/pipelines/exact-branch.yaml new file mode 100644 index 0000000..a061d73 --- /dev/null +++ b/deploy/telemetry-functions/pipelines/exact-branch.yaml @@ -0,0 +1,34 @@ +# Exact pipeline, stage 1 at each source: every value per route per 1-minute window +# goes to exact-merge. +version: otel_dataflow/v1 +engine: + http_admin: + bind_address: 0.0.0.0:8085 +policies: + resources: + core_allocation: + type: core_count + count: 1 +groups: + default: + pipelines: + main: + nodes: + receiver: + type: receiver:otlp + config: + protocols: + http: {listening_addr: "0.0.0.0:4318", wait_for_result: false} + branch: + type: urn:asap:processor:exact_branch + config: {window: 60s, group_by: [http.route]} + to_merge: + type: exporter:otlp_http + config: + endpoint: http://exact-merge:4318 + client_pool_size: 1 + max_in_flight: 1 + http: {timeout: 30s, compression: none} + connections: + - {from: receiver, to: branch} + - {from: branch, to: to_merge} diff --git a/deploy/telemetry-functions/pipelines/exact.yaml b/deploy/telemetry-functions/pipelines/exact-detector.yaml similarity index 67% rename from deploy/telemetry-functions/pipelines/exact.yaml rename to deploy/telemetry-functions/pipelines/exact-detector.yaml index 1a9a648..740c6b2 100644 --- a/deploy/telemetry-functions/pipelines/exact.yaml +++ b/deploy/telemetry-functions/pipelines/exact-detector.yaml @@ -1,4 +1,5 @@ -# Exact baseline and ground truth: every raw point, exact answers per window. +# Exact pipeline, end: exact answers of each merged window, the +# ground truth (method="exact"). version: otel_dataflow/v1 engine: http_admin: @@ -17,10 +18,10 @@ groups: type: receiver:otlp config: protocols: - http: {listening_addr: "0.0.0.0:4318", wait_for_result: false} - exact: - type: urn:asap:processor:exact_answers - config: {window: 60s, group_by: [http.route], quantiles: [0.5, 0.99]} + http: {listening_addr: "0.0.0.0:4318", wait_for_result: false, max_request_body_size: 1GiB} + detector: + type: urn:asap:processor:exact_detector + config: {quantiles: [0.5, 0.99]} to_prometheus: type: exporter:otlp_http config: @@ -30,5 +31,5 @@ groups: max_in_flight: 1 http: {timeout: 30s, compression: none} connections: - - {from: receiver, to: exact} - - {from: exact, to: to_prometheus} + - {from: receiver, to: detector} + - {from: detector, to: to_prometheus} diff --git a/deploy/telemetry-functions/pipelines/generator-a.yaml b/deploy/telemetry-functions/pipelines/generator-a.yaml index bfc3268..d63b7bd 100644 --- a/deploy/telemetry-functions/pipelines/generator-a.yaml +++ b/deploy/telemetry-functions/pipelines/generator-a.yaml @@ -26,7 +26,7 @@ groups: export: type: exporter:otlp_http config: - endpoint: http://branch-a:4318 + endpoint: http://downstream:4318 client_pool_size: 1 max_in_flight: 1 http: {timeout: 30s, compression: none} diff --git a/deploy/telemetry-functions/pipelines/generator-b.yaml b/deploy/telemetry-functions/pipelines/generator-b.yaml index 1144ab5..081b113 100644 --- a/deploy/telemetry-functions/pipelines/generator-b.yaml +++ b/deploy/telemetry-functions/pipelines/generator-b.yaml @@ -26,7 +26,7 @@ groups: export: type: exporter:otlp_http config: - endpoint: http://branch-b:4318 + endpoint: http://downstream:4318 client_pool_size: 1 max_in_flight: 1 http: {timeout: 30s, compression: none} diff --git a/deploy/telemetry-functions/prometheus.yml b/deploy/telemetry-functions/prometheus.yml index 2325b68..b2eff75 100644 --- a/deploy/telemetry-functions/prometheus.yml +++ b/deploy/telemetry-functions/prometheus.yml @@ -7,10 +7,14 @@ scrape_configs: metrics_path: /api/v1/metrics params: {format: [prometheus]} static_configs: - - targets: [generator-a:8085, generator-b:8085, branch-a:8085, branch-b:8085, - merge:8085, detector:8085, exact:8085] + - targets: [generator-a:8085, generator-b:8085, branch-a:8085, branch-b:8085, merge:8085, + detector:8085, exact-generator-a:8085, exact-generator-b:8085, + exact-branch-a:8085, exact-branch-b:8085, exact-merge:8085, + exact-detector:8085, raw-generator-a:8085, raw-generator-b:8085, + raw-branch-a:8085, raw-branch-b:8085, raw-merge:8085, raw-sink:8085] relabel_configs: - source_labels: [__address__] regex: (.+):8085 target_label: stage -# Answers (sketch and exact) arrive through the OTLP receiver. +# Answers (sketch, exact) and the raw pipeline's latency arrive through the +# OTLP receiver. diff --git a/deploy/telemetry-functions/run.sh b/deploy/telemetry-functions/run.sh index f4cb2fc..5f27c09 100755 --- a/deploy/telemetry-functions/run.sh +++ b/deploy/telemetry-functions/run.sh @@ -13,5 +13,8 @@ trap 'rm -rf "$context"' EXIT cp "$target/release/asap-collector" "$context/" strip --strip-debug "$context/asap-collector" "${docker[@]}" build -q -f deploy/collector.Dockerfile -t asap-collector:local "$context" +# Every generator counts scenario time from this second, so the three +# pipelines get identical points. +export SCENARIO_START=${SCENARIO_START:-$(($(date +%s) + 10))} "${docker[@]}" compose -f deploy/telemetry-functions/compose.yaml up -d echo "Grafana: http://localhost:${GRAFANA_PORT:-3000}/d/asap-latency"