From 86397a844790ff71561f9b1dd25352ff4556da44 Mon Sep 17 00:00:00 2001 From: Dom Del Nano Date: Sun, 12 Apr 2026 18:57:46 -0700 Subject: [PATCH 01/10] Changes needed to get clickhouse e2e test working with external clickhouse Signed-off-by: Dom Del Nano --- .../exec/clickhouse_export_sink_node.cc | 37 +++++++++++++++++-- src/carnot/funcs/metadata/metadata_ops.cc | 1 + src/carnot/funcs/metadata/metadata_ops.h | 27 ++++++++++++++ src/vizier/funcs/md_udtfs/md_udtfs_impl.h | 37 ++++++++++++++----- 4 files changed, 90 insertions(+), 12 deletions(-) diff --git a/src/carnot/exec/clickhouse_export_sink_node.cc b/src/carnot/exec/clickhouse_export_sink_node.cc index 6a11a42d37a..c7000ab99d4 100644 --- a/src/carnot/exec/clickhouse_export_sink_node.cc +++ b/src/carnot/exec/clickhouse_export_sink_node.cc @@ -35,6 +35,9 @@ namespace px { namespace carnot { namespace exec { +// TODO(ddelnano): Defend against columns that don't exist. These should be +// ignored by the Node. + using table_store::schema::RowBatch; using table_store::schema::RowDescriptor; @@ -148,12 +151,12 @@ Status ClickHouseExportSinkNode::ConsumeNextImpl(ExecState* /*exec_state*/, cons break; } case types::UINT128: { - // UINT128 is exported as STRING (UUID format) + // UINT128 is exported as STRING in "high:low" format to match + // the ClickHouseSourceNode's parsing in clickhouse_source_node.cc auto col = std::make_shared(); for (int64_t i = 0; i < num_rows; ++i) { auto val = types::GetValueFromArrowArray(arrow_col.get(), i); - std::string uuid_str = sole::rebuild(absl::Uint128High64(val), absl::Uint128Low64(val)).str(); - col->Append(uuid_str); + col->Append(absl::Substitute("$0:$1", absl::Uint128High64(val), absl::Uint128Low64(val))); } block.AppendColumn(mapping.clickhouse_column_name(), col); break; @@ -164,6 +167,34 @@ Status ClickHouseExportSinkNode::ConsumeNextImpl(ExecState* /*exec_state*/, cons } } + // Auto-derive event_time from time_ if time_ is present but event_time is not. + // The ClickHouse table schema uses event_time (DateTime64(3), milliseconds) for + // partitioning and ordering, but the Pixie table has time_ (TIME64NS, nanoseconds). + bool has_time_ = false; + bool has_event_time = false; + int time_col_index = -1; + for (const auto& mapping : plan_node_->column_mappings()) { + if (mapping.clickhouse_column_name() == "time_") { + has_time_ = true; + time_col_index = mapping.input_column_index(); + } + if (mapping.clickhouse_column_name() == "event_time") { + has_event_time = true; + } + } + + if (has_time_ && !has_event_time && time_col_index >= 0) { + auto arrow_col = rb.ColumnAt(time_col_index); + int64_t num_rows = arrow_col->length(); + auto event_time_col = std::make_shared(3); + for (int64_t i = 0; i < num_rows; ++i) { + int64_t ns_val = types::GetValueFromArrowArray(arrow_col.get(), i); + // Convert nanoseconds to milliseconds for DateTime64(3) + event_time_col->Append(ns_val / 1000000LL); + } + block.AppendColumn("event_time", event_time_col); + } + // Insert the block into ClickHouse clickhouse_client_->Insert(plan_node_->table_name(), block); diff --git a/src/carnot/funcs/metadata/metadata_ops.cc b/src/carnot/funcs/metadata/metadata_ops.cc index 3fe4e21692d..d6409e6f456 100644 --- a/src/carnot/funcs/metadata/metadata_ops.cc +++ b/src/carnot/funcs/metadata/metadata_ops.cc @@ -127,6 +127,7 @@ void RegisterMetadataOpsOrDie(px::carnot::udf::Registry* registry) { registry->RegisterOrDie("upid_to_deployment_id"); registry->RegisterOrDie("upid_to_string"); registry->RegisterOrDie("_exec_hostname"); + registry->RegisterOrDie("_pem_hostname"); registry->RegisterOrDie("_exec_host_num_cpus"); registry->RegisterOrDie("vizier_id"); registry->RegisterOrDie("vizier_name"); diff --git a/src/carnot/funcs/metadata/metadata_ops.h b/src/carnot/funcs/metadata/metadata_ops.h index 241079858a4..af82f9738f8 100644 --- a/src/carnot/funcs/metadata/metadata_ops.h +++ b/src/carnot/funcs/metadata/metadata_ops.h @@ -2926,6 +2926,33 @@ class HostnameUDF : public ScalarUDF { } }; +class PEMHostnameUDF : public ScalarUDF { + public: + /** + * @brief Gets the hostname of the PEM agent's machine. + * Unlike _exec_hostname (UDF_ALL), this is restricted to UDF_PEM so the + * distributed planner is forced to execute it on the PEM before data is + * shipped to Kelvin. Use this when the hostname must reflect the agent + * that collected the data rather than the agent that exports it. + */ + StringValue Exec(FunctionContext* ctx) { + auto md = GetMetadataState(ctx); + return md->hostname(); + } + + static udf::ScalarUDFDocBuilder Doc() { + return udf::ScalarUDFDocBuilder("Get the hostname of the PEM agent.") + .Details( + "Get the hostname of the PEM agent that collected the data. " + "This UDF is restricted to PEM execution, so the distributed planner " + "will always run it on the PEM even when the downstream sink is on Kelvin.") + .Example("df.hostname = px._pem_hostname()") + .Returns("The hostname of the PEM agent."); + } + + static udfspb::UDFSourceExecutor Executor() { return udfspb::UDFSourceExecutor::UDF_PEM; } +}; + class HostNumCPUsUDF : public ScalarUDF { public: /** diff --git a/src/vizier/funcs/md_udtfs/md_udtfs_impl.h b/src/vizier/funcs/md_udtfs/md_udtfs_impl.h index ff5fdcbe6c2..9b1d9936df3 100644 --- a/src/vizier/funcs/md_udtfs/md_udtfs_impl.h +++ b/src/vizier/funcs/md_udtfs/md_udtfs_impl.h @@ -1145,12 +1145,17 @@ class CreateClickHouseSchemas final : public carnot::udf::UDTF("database", "ClickHouse database", "'default'"), UDTFArg::Make( - "use_if_not_exists", "Whether to use IF NOT EXISTS in CREATE TABLE statements", true)); + "use_if_not_exists", "Whether to use IF NOT EXISTS in CREATE TABLE statements", true), + UDTFArg::Make( + "cluster_name", + "ClickHouse cluster name for ON CLUSTER DDL and ReplicatedMergeTree engine. " + "Empty string disables cluster mode.", + "''")); } Status Init(FunctionContext*, types::StringValue host, types::Int64Value port, types::StringValue username, types::StringValue password, types::StringValue database, - types::BoolValue use_if_not_exists) { + types::BoolValue use_if_not_exists, types::StringValue cluster_name) { // Store ClickHouse connection parameters host_ = std::string(host); port_ = port.val; @@ -1158,6 +1163,7 @@ class CreateClickHouseSchemas final : public carnot::udf::UDTFExecute(absl::Substitute("DROP TABLE IF EXISTS $0", table_name)); + std::string drop_cluster_clause = + cluster_name_.empty() ? "" : absl::Substitute(" ON CLUSTER '$0'", cluster_name_); + clickhouse_client_->Execute( + absl::Substitute("DROP TABLE IF EXISTS $0$1", table_name, drop_cluster_clause)); } // Create new table @@ -1276,7 +1286,8 @@ class CreateClickHouseSchemas final : public carnot::udf::UDTF column_defs; // Add columns from schema @@ -1301,14 +1312,21 @@ class CreateClickHouseSchemas final : public carnot::udf::UDTF= 22.x). + std::string engine = cluster_name.empty() ? "MergeTree()" : "ReplicatedMergeTree()"; std::string create_sql = absl::Substitute(R"( - CREATE TABLE $0$1 ( - $2 - ) ENGINE = MergeTree() + CREATE TABLE $0$1$2 ( + $3 + ) ENGINE = $4 PARTITION BY toYYYYMM(event_time) ORDER BY (hostname, event_time) )", - if_not_exists_clause, table_name, columns_str); + if_not_exists_clause, table_name, on_cluster_clause, + columns_str, engine); return create_sql; } @@ -1326,6 +1344,7 @@ class CreateClickHouseSchemas final : public carnot::udf::UDTF Date: Wed, 15 Apr 2026 20:12:37 -0700 Subject: [PATCH 02/10] Implement parquet export format Signed-off-by: Dom Del Nano --- go.mod | 11 +- go.sum | 18 +- go_deps.bzl | 27 +- src/e2e_test/perf_tool/cmd/BUILD.bazel | 1 + src/e2e_test/perf_tool/cmd/run.go | 60 ++- .../perf_tool/pkg/exporter/BUILD.bazel | 50 ++ .../{run/row.go => exporter/bq_exporter.go} | 65 ++- .../perf_tool/pkg/exporter/exporter.go | 37 ++ .../pkg/exporter/parquet_exporter.go | 285 ++++++++++ .../pkg/exporter/parquet_exporter_test.go | 500 ++++++++++++++++++ src/e2e_test/perf_tool/pkg/run/BUILD.bazel | 3 +- src/e2e_test/perf_tool/pkg/run/run.go | 55 +- 12 files changed, 1027 insertions(+), 85 deletions(-) create mode 100644 src/e2e_test/perf_tool/pkg/exporter/BUILD.bazel rename src/e2e_test/perf_tool/pkg/{run/row.go => exporter/bq_exporter.go} (58%) create mode 100644 src/e2e_test/perf_tool/pkg/exporter/exporter.go create mode 100644 src/e2e_test/perf_tool/pkg/exporter/parquet_exporter.go create mode 100644 src/e2e_test/perf_tool/pkg/exporter/parquet_exporter_test.go diff --git a/go.mod b/go.mod index 4224503b9c1..10f19e7657b 100644 --- a/go.mod +++ b/go.mod @@ -52,6 +52,7 @@ require ( github.com/ory/dockertest/v3 v3.8.1 github.com/ory/hydra-client-go v1.9.2 github.com/ory/kratos-client-go v0.10.1 + github.com/parquet-go/parquet-go v0.25.1 github.com/phayes/freeport v0.0.0-20171002181615-b8543db493a5 github.com/prometheus/client_golang v1.14.0 github.com/prometheus/client_model v0.3.0 @@ -115,6 +116,7 @@ require ( github.com/VividCortex/ewma v1.1.1 // indirect github.com/a8m/envsubst v1.3.0 // indirect github.com/alecthomas/participle/v2 v2.0.0-beta.5 // indirect + github.com/andybalholm/brotli v1.1.0 // indirect github.com/andybalholm/cascadia v1.1.0 // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/beorn7/perks v1.0.1 // indirect @@ -171,7 +173,7 @@ require ( github.com/google/go-querystring v1.1.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect - github.com/google/uuid v1.3.0 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/googleapis/gax-go/v2 v2.7.0 // indirect github.com/gorilla/securecookie v1.1.1 // indirect github.com/gorilla/websocket v1.5.0 // indirect @@ -191,7 +193,7 @@ require ( github.com/json-iterator/go v1.1.12 // indirect github.com/jstemmer/go-junit-report v0.9.1 // indirect github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd // indirect - github.com/klauspost/compress v1.17.2 // indirect + github.com/klauspost/compress v1.17.9 // indirect github.com/kr/pretty v0.2.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect @@ -232,6 +234,7 @@ require ( github.com/patrickmn/go-cache v2.1.0+incompatible // indirect github.com/pelletier/go-toml v1.9.3 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect + github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/procfs v0.9.0 // indirect @@ -276,7 +279,7 @@ require ( golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/protobuf v1.29.1 // indirect + google.golang.org/protobuf v1.34.2 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/launchdarkly/go-jsonstream.v1 v1.0.1 // indirect @@ -317,3 +320,5 @@ replace ( google.golang.org/grpc => google.golang.org/grpc v1.43.0 gopkg.in/yaml.v2 => gopkg.in/yaml.v2 v2.4.0 ) + +replace google.golang.org/protobuf => google.golang.org/protobuf v1.29.1 diff --git a/go.sum b/go.sum index b8697cb4add..533a9f3f9b6 100644 --- a/go.sum +++ b/go.sum @@ -87,6 +87,8 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= +github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo= github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA= @@ -447,8 +449,8 @@ github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaU github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.7.0 h1:IcsPKeInNvYi7eqSaDjiZqDDKu5rsmunY0Y1YupQSSQ= github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8= @@ -579,8 +581,8 @@ github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0 github.com/klauspost/compress v1.9.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4= -github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -775,6 +777,8 @@ github.com/ory/hydra-client-go v1.9.2 h1:sbp+8zwEJvhqSxcY8HiOkXeY2FspsfSOJ5ajJ07 github.com/ory/hydra-client-go v1.9.2/go.mod h1:TTg4Gt0SDC8+XoGtj5qzdtqxapfFW+Vmm41PFuC6n/E= github.com/ory/kratos-client-go v0.10.1 h1:kSRk+0leCJ1nPMS+FPho8b9WMzrKNpgszvta0Xo32QU= github.com/ory/kratos-client-go v0.10.1/go.mod h1:dOQIsar76K07wMPJD/6aMhrWyY+sFGEagLDLso1CpsA= +github.com/parquet-go/parquet-go v0.25.1 h1:l7jJwNM0xrk0cnIIptWMtnSnuxRkwq53S+Po3KG8Xgo= +github.com/parquet-go/parquet-go v0.25.1/go.mod h1:AXBuotO1XiBtcqJb/FKFyjBG4aqa3aQAAWF3ZPzCanY= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= @@ -788,6 +792,8 @@ github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+v github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/phayes/freeport v0.0.0-20171002181615-b8543db493a5 h1:rZQtoozkfsiNs36c7Tdv/gyGNzD1X1XWKO8rptVNZuM= github.com/phayes/freeport v0.0.0-20171002181615-b8543db493a5/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE= +github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= +github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e h1:aoZm08cpOy4WuID//EZDgcC4zIxODThtZNPirFr42+A= @@ -1327,10 +1333,6 @@ google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa h1:I0YcKz0I7OAhddo google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/grpc v1.43.0 h1:Eeu7bZtDZ2DpRCsLhUlcrLnvYaMK1Gz86a+hMVvELmM= google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.29.1 h1:7QBf+IK2gx70Ap/hDsOmam3GE0v9HicjfEdAxE62UoM= google.golang.org/protobuf v1.29.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= diff --git a/go_deps.bzl b/go_deps.bzl index 6590dff5052..8ff37dbcbf6 100644 --- a/go_deps.bzl +++ b/go_deps.bzl @@ -156,8 +156,8 @@ def pl_go_dependencies(): name = "com_github_andybalholm_brotli", build_directives = ["gazelle:map_kind go_binary pl_go_binary @px//bazel:pl_build_system.bzl", "gazelle:map_kind go_test pl_go_test @px//bazel:pl_build_system.bzl"], importpath = "github.com/andybalholm/brotli", - sum = "h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=", - version = "v1.0.5", + sum = "h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=", + version = "v1.1.0", ) go_repository( name = "com_github_andybalholm_cascadia", @@ -1628,8 +1628,8 @@ def pl_go_dependencies(): name = "com_github_google_uuid", build_directives = ["gazelle:map_kind go_binary pl_go_binary @px//bazel:pl_build_system.bzl", "gazelle:map_kind go_test pl_go_test @px//bazel:pl_build_system.bzl"], importpath = "github.com/google/uuid", - sum = "h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=", - version = "v1.3.0", + sum = "h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=", + version = "v1.6.0", ) go_repository( name = "com_github_googleapis_enterprise_certificate_proxy", @@ -2282,8 +2282,8 @@ def pl_go_dependencies(): name = "com_github_klauspost_compress", build_directives = ["gazelle:map_kind go_binary pl_go_binary @px//bazel:pl_build_system.bzl", "gazelle:map_kind go_test pl_go_test @px//bazel:pl_build_system.bzl"], importpath = "github.com/klauspost/compress", - sum = "h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4=", - version = "v1.17.2", + sum = "h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=", + version = "v1.17.9", ) go_repository( name = "com_github_klauspost_cpuid", @@ -2992,6 +2992,13 @@ def pl_go_dependencies(): sum = "h1:mvZaddk4E4kLcXhzb+cxBsMPYp2pHqiQpWYkInsuZPQ=", version = "v1.3.0", ) + go_repository( + name = "com_github_parquet_go_parquet_go", + build_directives = ["gazelle:map_kind go_binary pl_go_binary @px//bazel:pl_build_system.bzl", "gazelle:map_kind go_test pl_go_test @px//bazel:pl_build_system.bzl"], + importpath = "github.com/parquet-go/parquet-go", + sum = "h1:l7jJwNM0xrk0cnIIptWMtnSnuxRkwq53S+Po3KG8Xgo=", + version = "v0.25.1", + ) go_repository( name = "com_github_pascaldekloe_goe", build_directives = ["gazelle:map_kind go_binary pl_go_binary @px//bazel:pl_build_system.bzl", "gazelle:map_kind go_test pl_go_test @px//bazel:pl_build_system.bzl"], @@ -3041,6 +3048,13 @@ def pl_go_dependencies(): sum = "h1:rZQtoozkfsiNs36c7Tdv/gyGNzD1X1XWKO8rptVNZuM=", version = "v0.0.0-20171002181615-b8543db493a5", ) + go_repository( + name = "com_github_pierrec_lz4_v4", + build_directives = ["gazelle:map_kind go_binary pl_go_binary @px//bazel:pl_build_system.bzl", "gazelle:map_kind go_test pl_go_test @px//bazel:pl_build_system.bzl"], + importpath = "github.com/pierrec/lz4/v4", + sum = "h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ=", + version = "v4.1.21", + ) go_repository( name = "com_github_pingcap_errors", build_directives = ["gazelle:map_kind go_binary pl_go_binary @px//bazel:pl_build_system.bzl", "gazelle:map_kind go_test pl_go_test @px//bazel:pl_build_system.bzl"], @@ -4427,6 +4441,7 @@ def pl_go_dependencies(): name = "org_golang_google_protobuf", build_directives = ["gazelle:map_kind go_binary pl_go_binary @px//bazel:pl_build_system.bzl", "gazelle:map_kind go_test pl_go_test @px//bazel:pl_build_system.bzl"], importpath = "google.golang.org/protobuf", + replace = "google.golang.org/protobuf", sum = "h1:7QBf+IK2gx70Ap/hDsOmam3GE0v9HicjfEdAxE62UoM=", version = "v1.29.1", ) diff --git a/src/e2e_test/perf_tool/cmd/BUILD.bazel b/src/e2e_test/perf_tool/cmd/BUILD.bazel index 012fd3488b0..23540786c4b 100644 --- a/src/e2e_test/perf_tool/cmd/BUILD.bazel +++ b/src/e2e_test/perf_tool/cmd/BUILD.bazel @@ -33,6 +33,7 @@ go_library( "//src/e2e_test/perf_tool/pkg/cluster", "//src/e2e_test/perf_tool/pkg/cluster/gke", "//src/e2e_test/perf_tool/pkg/cluster/local", + "//src/e2e_test/perf_tool/pkg/exporter", "//src/e2e_test/perf_tool/pkg/pixie", "//src/e2e_test/perf_tool/pkg/run", "//src/e2e_test/perf_tool/pkg/suites", diff --git a/src/e2e_test/perf_tool/cmd/run.go b/src/e2e_test/perf_tool/cmd/run.go index 5d8a89a9f7a..71e7859fc93 100644 --- a/src/e2e_test/perf_tool/cmd/run.go +++ b/src/e2e_test/perf_tool/cmd/run.go @@ -45,6 +45,7 @@ import ( "px.dev/pixie/src/e2e_test/perf_tool/pkg/cluster" "px.dev/pixie/src/e2e_test/perf_tool/pkg/cluster/gke" "px.dev/pixie/src/e2e_test/perf_tool/pkg/cluster/local" + "px.dev/pixie/src/e2e_test/perf_tool/pkg/exporter" "px.dev/pixie/src/e2e_test/perf_tool/pkg/pixie" "px.dev/pixie/src/e2e_test/perf_tool/pkg/run" "px.dev/pixie/src/e2e_test/perf_tool/pkg/suites" @@ -74,9 +75,13 @@ func init() { RunCmd.Flags().String("api_key", "", "The Pixie API key to use for deploying pixie") RunCmd.Flags().String("cloud_addr", "withpixie.ai:443", "The Pixie Cloud address to use for deploying pixie") + RunCmd.Flags().String("export_backend", "bq", "Export backend: 'bq' or 'parquet-gcs'") RunCmd.Flags().String("bq_project", "pl-pixies", "The gcloud project to put bigquery results/specs in") RunCmd.Flags().String("bq_dataset", "px_perf", "The name of the bigquery dataset to put results/specs in") RunCmd.Flags().String("bq_dataset_loc", "us-west1", "The gcloud region for the bigquery dataset") + RunCmd.Flags().String("gcs_bucket", "", "GCS bucket for parquet export (required when export_backend=parquet-gcs)") + RunCmd.Flags().String("gcs_prefix", "", "Path prefix within the GCS bucket for parquet export") + RunCmd.Flags().Int("parquet_batch_size", 10000, "Number of rows per parquet file when using parquet-gcs backend") RunCmd.Flags().String("gke_project", "pl-pixies", "The gcloud project to use for GKE clusters") RunCmd.Flags().String("gke_zone", "us-west1-a", "The gcloud zone to use for GKE clusters") @@ -162,16 +167,12 @@ func runCmd(ctx context.Context, cmd *cobra.Command) error { } } - resultTable, err := createResultTable() + metricsExporter, err := createExporter(ctx) if err != nil { - log.WithError(err).Error("failed to create results table") - return err - } - specTable, err := createSpecTable() - if err != nil { - log.WithError(err).Error("failed to create spec table") + log.WithError(err).Error("failed to create exporter") return err } + defer metricsExporter.Close() containerRegistryRepo := viper.GetString("container_repo") maxRetries := viper.GetInt("max_retries") @@ -189,7 +190,7 @@ func runCmd(ctx context.Context, cmd *cobra.Command) error { s := spec n := name eg.Go(func() error { - expID, err := runExperiment(ctx, s, c, pxAPIKey, pxCloudAddr, resultTable, specTable, containerRegistryRepo, maxRetries) + expID, err := runExperiment(ctx, s, c, pxAPIKey, pxCloudAddr, metricsExporter, containerRegistryRepo, maxRetries) if err != nil { log.WithError(err).Error("failed to run experiment") return err @@ -257,8 +258,7 @@ func runExperiment( c cluster.Provider, pxAPIKey string, pxCloudAddr string, - resultTable *bq.Table, - specTable *bq.Table, + metricsExporter exporter.Exporter, containerRegistryRepo string, maxRetries int, ) (uuid.UUID, error) { @@ -268,7 +268,7 @@ func runExperiment( } op := func() error { pxCtx := pixie.NewContext(pxAPIKey, pxCloudAddr) - r := run.NewRunner(c, pxCtx, resultTable, specTable, containerRegistryRepo) + r := run.NewRunner(c, pxCtx, metricsExporter, containerRegistryRepo) var err error expID, err = uuid.NewV4() if err != nil { @@ -335,7 +335,24 @@ func getExperimentSpecs() (map[string]*experimentpb.ExperimentSpec, error) { return nil, errors.New("must specify one of --experiment_proto or --suite") } -func createResultTable() (*bq.Table, error) { +func createExporter(ctx context.Context) (exporter.Exporter, error) { + switch viper.GetString("export_backend") { + case "bq": + return createBQExporter() + case "parquet-gcs": + bucket := viper.GetString("gcs_bucket") + if bucket == "" { + return nil, errors.New("--gcs_bucket is required when using parquet-gcs backend") + } + prefix := viper.GetString("gcs_prefix") + batchSize := viper.GetInt("parquet_batch_size") + return exporter.NewParquetGCSExporter(ctx, bucket, prefix, batchSize) + default: + return nil, fmt.Errorf("unknown export backend: %s", viper.GetString("export_backend")) + } +} + +func createBQExporter() (*exporter.BQExporter, error) { bqProject := viper.GetString("bq_project") bqDataset := viper.GetString("bq_dataset") bqDatasetLoc := viper.GetString("bq_dataset_loc") @@ -343,15 +360,16 @@ func createResultTable() (*bq.Table, error) { Type: bigquery.DayPartitioningType, Field: "timestamp", } - return bq.NewTableForStruct(bqProject, bqDataset, bqDatasetLoc, "results", timePartitioning, run.ResultRow{}) -} - -func createSpecTable() (*bq.Table, error) { - bqProject := viper.GetString("bq_project") - bqDataset := viper.GetString("bq_dataset") - bqDatasetLoc := viper.GetString("bq_dataset_loc") - var timePartitioning *bigquery.TimePartitioning - return bq.NewTableForStruct(bqProject, bqDataset, bqDatasetLoc, "specs", timePartitioning, run.SpecRow{}) + resultTable, err := bq.NewTableForStruct(bqProject, bqDataset, bqDatasetLoc, "results", timePartitioning, exporter.ResultRow{}) + if err != nil { + return nil, err + } + var specTimePartitioning *bigquery.TimePartitioning + specTable, err := bq.NewTableForStruct(bqProject, bqDataset, bqDatasetLoc, "specs", specTimePartitioning, exporter.SpecRow{}) + if err != nil { + return nil, err + } + return exporter.NewBQExporter(resultTable, specTable), nil } func getNumNodesInCluster(ctx context.Context, c cluster.Provider) (int, error) { diff --git a/src/e2e_test/perf_tool/pkg/exporter/BUILD.bazel b/src/e2e_test/perf_tool/pkg/exporter/BUILD.bazel new file mode 100644 index 00000000000..022cd797f39 --- /dev/null +++ b/src/e2e_test/perf_tool/pkg/exporter/BUILD.bazel @@ -0,0 +1,50 @@ +# Copyright 2018- The Pixie Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +load("//bazel:pl_build_system.bzl", "pl_go_test") +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "exporter", + srcs = [ + "bq_exporter.go", + "exporter.go", + "parquet_exporter.go", + ], + importpath = "px.dev/pixie/src/e2e_test/perf_tool/pkg/exporter", + visibility = ["//visibility:public"], + deps = [ + "//src/e2e_test/perf_tool/pkg/metrics", + "//src/shared/bq", + "@com_github_gofrs_uuid//:uuid", + "@com_github_parquet_go_parquet_go//:parquet-go", + "@com_github_sirupsen_logrus//:logrus", + "@com_google_cloud_go_storage//:storage", + ], +) + +pl_go_test( + name = "exporter_test", + srcs = ["parquet_exporter_test.go"], + embed = [":exporter"], + deps = [ + "//src/e2e_test/perf_tool/pkg/metrics", + "@com_github_gofrs_uuid//:uuid", + "@com_github_parquet_go_parquet_go//:parquet-go", + "@com_github_stretchr_testify//assert", + "@com_github_stretchr_testify//require", + ], +) diff --git a/src/e2e_test/perf_tool/pkg/run/row.go b/src/e2e_test/perf_tool/pkg/exporter/bq_exporter.go similarity index 58% rename from src/e2e_test/perf_tool/pkg/run/row.go rename to src/e2e_test/perf_tool/pkg/exporter/bq_exporter.go index 17959d97d78..023db03c4f4 100644 --- a/src/e2e_test/perf_tool/pkg/run/row.go +++ b/src/e2e_test/perf_tool/pkg/exporter/bq_exporter.go @@ -16,15 +16,18 @@ * SPDX-License-Identifier: Apache-2.0 */ -package run +package exporter import ( + "context" "encoding/json" "time" "github.com/gofrs/uuid" + log "github.com/sirupsen/logrus" "px.dev/pixie/src/e2e_test/perf_tool/pkg/metrics" + "px.dev/pixie/src/shared/bq" ) // ResultRow represents a single datapoint for a single metric, to be stored in bigquery. @@ -51,7 +54,7 @@ type SpecRow struct { CommitTopoOrder int `bigquery:"commit_topo_order"` } -// MetricsRowToResultRow converts a `metrics.ResultRow` into a `bq.ResultRow`. +// MetricsRowToResultRow converts a `metrics.ResultRow` into a `ResultRow`. func MetricsRowToResultRow(expID uuid.UUID, row *metrics.ResultRow) (*ResultRow, error) { encodedTags, err := json.Marshal(row.Tags) if err != nil { @@ -65,3 +68,61 @@ func MetricsRowToResultRow(expID uuid.UUID, row *metrics.ResultRow) (*ResultRow, Tags: string(encodedTags), }, nil } + +// BQExporter exports experiment results and specs to BigQuery. +type BQExporter struct { + resultTable *bq.Table + specTable *bq.Table +} + +// NewBQExporter creates a new BigQuery exporter. +func NewBQExporter(resultTable, specTable *bq.Table) *BQExporter { + return &BQExporter{ + resultTable: resultTable, + specTable: specTable, + } +} + +// ExportResults consumes metrics from resultCh and inserts them into BigQuery in batches. +func (e *BQExporter) ExportResults(ctx context.Context, expID uuid.UUID, resultCh <-chan *metrics.ResultRow) error { + bqCh := make(chan interface{}) + defer close(bqCh) + + inserter := &bq.BatchInserter{ + Table: e.resultTable, + BatchSize: 512, + PushTimeout: 2 * time.Minute, + } + go inserter.Run(bqCh) + + for row := range resultCh { + bqRow, err := MetricsRowToResultRow(expID, row) + if err != nil { + log.WithError(err).Error("Failed to convert result row") + continue + } + bqCh <- bqRow + } + return nil +} + +// ExportSpec writes the experiment spec to BigQuery on experiment success. +func (e *BQExporter) ExportSpec(ctx context.Context, expID uuid.UUID, encodedSpec string, commitTopoOrder int) error { + specRow := &SpecRow{ + ExperimentID: expID.String(), + Spec: encodedSpec, + CommitTopoOrder: commitTopoOrder, + } + + inserter := e.specTable.Inserter() + inserter.SkipInvalidRows = false + + putCtx, cancel := context.WithTimeout(ctx, 5*time.Minute) + defer cancel() + return inserter.Put(putCtx, specRow) +} + +// Close is a no-op for the BigQuery exporter. +func (e *BQExporter) Close() error { + return nil +} diff --git a/src/e2e_test/perf_tool/pkg/exporter/exporter.go b/src/e2e_test/perf_tool/pkg/exporter/exporter.go new file mode 100644 index 00000000000..c89d6898032 --- /dev/null +++ b/src/e2e_test/perf_tool/pkg/exporter/exporter.go @@ -0,0 +1,37 @@ +/* + * Copyright 2018- The Pixie Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package exporter + +import ( + "context" + + "github.com/gofrs/uuid" + + "px.dev/pixie/src/e2e_test/perf_tool/pkg/metrics" +) + +// Exporter handles exporting experiment results and specs to a storage backend. +type Exporter interface { + // ExportResults consumes metrics from resultCh until it closes, then flushes. + ExportResults(ctx context.Context, expID uuid.UUID, resultCh <-chan *metrics.ResultRow) error + // ExportSpec writes the experiment spec for a successful experiment. + ExportSpec(ctx context.Context, expID uuid.UUID, encodedSpec string, commitTopoOrder int) error + // Close releases any resources held by the exporter. + Close() error +} diff --git a/src/e2e_test/perf_tool/pkg/exporter/parquet_exporter.go b/src/e2e_test/perf_tool/pkg/exporter/parquet_exporter.go new file mode 100644 index 00000000000..6c289b25887 --- /dev/null +++ b/src/e2e_test/perf_tool/pkg/exporter/parquet_exporter.go @@ -0,0 +1,285 @@ +/* + * Copyright 2018- The Pixie Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package exporter + +import ( + "context" + "fmt" + "io" + "os" + "sort" + "time" + + "cloud.google.com/go/storage" + "github.com/gofrs/uuid" + "github.com/parquet-go/parquet-go" + log "github.com/sirupsen/logrus" + + "px.dev/pixie/src/e2e_test/perf_tool/pkg/metrics" +) + +type bufferedRow struct { + ExperimentID string + Timestamp time.Time + Name string + Value float64 + Tags map[string]string +} + +// uploadFunc is the signature for uploading a local file to a remote path. +type uploadFunc func(ctx context.Context, objectPath string, localPath string) error + +// ParquetGCSExporter exports experiment results as parquet files to GCS. +type ParquetGCSExporter struct { + bucket string + prefix string + batchSize int + gcsClient *storage.Client + upload uploadFunc +} + +// NewParquetGCSExporter creates a new Parquet+GCS exporter. +func NewParquetGCSExporter(ctx context.Context, bucket, prefix string, batchSize int) (*ParquetGCSExporter, error) { + client, err := storage.NewClient(ctx) + if err != nil { + return nil, fmt.Errorf("failed to create GCS client: %w", err) + } + e := &ParquetGCSExporter{ + bucket: bucket, + prefix: prefix, + batchSize: batchSize, + gcsClient: client, + } + e.upload = e.uploadToGCS + return e, nil +} + +// ExportResults consumes metrics from resultCh and writes them as batched parquet files to GCS. +func (e *ParquetGCSExporter) ExportResults(ctx context.Context, expID uuid.UUID, resultCh <-chan *metrics.ResultRow) error { + now := time.Now() + basePath := e.gcsPath(now, expID) + seqNum := 0 + batch := make([]bufferedRow, 0, e.batchSize) + + for row := range resultCh { + batch = append(batch, bufferedRow{ + ExperimentID: expID.String(), + Timestamp: row.Timestamp, + Name: row.Name, + Value: row.Value, + Tags: row.Tags, + }) + if len(batch) >= e.batchSize { + if err := e.flushBatch(ctx, basePath, seqNum, batch); err != nil { + return err + } + seqNum++ + batch = batch[:0] + } + } + + if len(batch) > 0 { + if err := e.flushBatch(ctx, basePath, seqNum, batch); err != nil { + return err + } + } + return nil +} + +// ExportSpec writes the experiment spec as a parquet file to GCS. +func (e *ParquetGCSExporter) ExportSpec(ctx context.Context, expID uuid.UUID, encodedSpec string, commitTopoOrder int) error { + type specRow struct { + ExperimentID string `parquet:"experiment_id"` + Spec string `parquet:"spec"` + CommitTopoOrder int64 `parquet:"commit_topo_order"` + } + + tmpFile, err := os.CreateTemp("", "spec-*.parquet") + if err != nil { + return fmt.Errorf("failed to create temp file for spec parquet: %w", err) + } + tmpPath := tmpFile.Name() + defer os.Remove(tmpPath) + + writer := parquet.NewGenericWriter[specRow](tmpFile) + _, err = writer.Write([]specRow{{ + ExperimentID: expID.String(), + Spec: encodedSpec, + CommitTopoOrder: int64(commitTopoOrder), + }}) + if err != nil { + tmpFile.Close() + return fmt.Errorf("failed to write spec parquet: %w", err) + } + if err := writer.Close(); err != nil { + tmpFile.Close() + return fmt.Errorf("failed to close spec parquet writer: %w", err) + } + tmpFile.Close() + + now := time.Now() + gcsPath := fmt.Sprintf("%s/spec.parquet", e.gcsPath(now, expID)) + return e.upload(ctx, gcsPath, tmpPath) +} + +// Close releases resources held by the exporter. +func (e *ParquetGCSExporter) Close() error { + return e.gcsClient.Close() +} + +func (e *ParquetGCSExporter) gcsPath(t time.Time, expID uuid.UUID) string { + datePath := t.Format("2006/01/02") + if e.prefix != "" { + return fmt.Sprintf("%s/%s/%s", e.prefix, datePath, expID.String()) + } + return fmt.Sprintf("%s/%s", datePath, expID.String()) +} + +func (e *ParquetGCSExporter) flushBatch(ctx context.Context, basePath string, seqNum int, rows []bufferedRow) error { + tagKeys := collectTagKeys(rows) + schema := buildResultSchema(tagKeys) + + tmpFile, err := os.CreateTemp("", "results-*.parquet") + if err != nil { + return fmt.Errorf("failed to create temp file for parquet: %w", err) + } + tmpPath := tmpFile.Name() + defer os.Remove(tmpPath) + + writer := parquet.NewWriter(tmpFile, schema) + + for _, row := range rows { + parquetRow := buildResultRow(row, tagKeys) + if _, err := writer.WriteRows([]parquet.Row{parquetRow}); err != nil { + tmpFile.Close() + return fmt.Errorf("failed to write parquet row: %w", err) + } + } + + if err := writer.Close(); err != nil { + tmpFile.Close() + return fmt.Errorf("failed to close parquet writer: %w", err) + } + tmpFile.Close() + + gcsPath := fmt.Sprintf("%s/results_%04d.parquet", basePath, seqNum) + log.WithField("gcs_path", gcsPath).WithField("rows", len(rows)).Info("Uploading parquet batch") + return e.upload(ctx, gcsPath, tmpPath) +} + +func (e *ParquetGCSExporter) uploadToGCS(ctx context.Context, objectPath string, localPath string) error { + f, err := os.Open(localPath) + if err != nil { + return fmt.Errorf("failed to open temp file for upload: %w", err) + } + defer f.Close() + + obj := e.gcsClient.Bucket(e.bucket).Object(objectPath) + wc := obj.NewWriter(ctx) + if _, err := io.Copy(wc, f); err != nil { + wc.Close() + return fmt.Errorf("failed to upload to GCS: %w", err) + } + if err := wc.Close(); err != nil { + return fmt.Errorf("failed to finalize GCS upload: %w", err) + } + return nil +} + +// collectTagKeys returns a sorted list of unique tag keys across all rows. +func collectTagKeys(rows []bufferedRow) []string { + keySet := make(map[string]struct{}) + for _, row := range rows { + for k := range row.Tags { + keySet[k] = struct{}{} + } + } + keys := make([]string, 0, len(keySet)) + for k := range keySet { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} + +// buildResultSchema creates a parquet schema with fixed columns plus dynamic tag columns. +func buildResultSchema(tagKeys []string) *parquet.Schema { + group := parquet.Group{ + "experiment_id": parquet.String(), + "timestamp": parquet.Timestamp(parquet.Millisecond), + "name": parquet.String(), + "value": parquet.Leaf(parquet.DoubleType), + } + for _, key := range tagKeys { + group["tag_"+key] = parquet.Optional(parquet.String()) + } + return parquet.NewSchema("result", group) +} + +// buildResultRow constructs a parquet.Row from a bufferedRow with the given tag key ordering. +// Column ordering matches the schema's sorted field order (alphabetical by field name). +func buildResultRow(row bufferedRow, tagKeys []string) parquet.Row { + // parquet.Group sorts fields alphabetically. We must produce values in that order. + // Build named values, sort them, then assign column indices. + + type colEntry struct { + name string + val parquet.Value + optional bool + } + + entries := []colEntry{ + {"experiment_id", parquet.ValueOf(row.ExperimentID), false}, + {"name", parquet.ValueOf(row.Name), false}, + {"timestamp", parquet.ValueOf(row.Timestamp), false}, + {"value", parquet.ValueOf(row.Value), false}, + } + + for _, key := range tagKeys { + colName := "tag_" + key + if v, ok := row.Tags[key]; ok { + entries = append(entries, colEntry{colName, parquet.ValueOf(v), true}) + } else { + // Null value for missing optional tag. + entries = append(entries, colEntry{colName, parquet.Value{}, true}) + } + } + + // Sort by column name to match schema field order. + sort.Slice(entries, func(i, j int) bool { + return entries[i].name < entries[j].name + }) + + parquetRow := make(parquet.Row, len(entries)) + for i, e := range entries { + if e.optional { + if e.val.IsNull() { + // Null optional: definitionLevel=0 + parquetRow[i] = parquet.Value{}.Level(0, 0, i) + } else { + // Present optional: definitionLevel=1 + parquetRow[i] = e.val.Level(0, 1, i) + } + } else { + // Required: definitionLevel=0 + parquetRow[i] = e.val.Level(0, 0, i) + } + } + return parquetRow +} diff --git a/src/e2e_test/perf_tool/pkg/exporter/parquet_exporter_test.go b/src/e2e_test/perf_tool/pkg/exporter/parquet_exporter_test.go new file mode 100644 index 00000000000..e20816bfd5b --- /dev/null +++ b/src/e2e_test/perf_tool/pkg/exporter/parquet_exporter_test.go @@ -0,0 +1,500 @@ +/* + * Copyright 2018- The Pixie Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package exporter + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + "testing" + "time" + + "github.com/gofrs/uuid" + "github.com/parquet-go/parquet-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "px.dev/pixie/src/e2e_test/perf_tool/pkg/metrics" +) + +func TestCollectTagKeys(t *testing.T) { + rows := []bufferedRow{ + {Tags: map[string]string{"pod": "pod-1", "node_name": "node-1"}}, + {Tags: map[string]string{"pod": "pod-2", "instance": "inst-1"}}, + {Tags: map[string]string{}}, + } + + keys := collectTagKeys(rows) + + assert.Equal(t, []string{"instance", "node_name", "pod"}, keys) +} + +func TestCollectTagKeys_Empty(t *testing.T) { + rows := []bufferedRow{ + {Tags: map[string]string{}}, + } + + keys := collectTagKeys(rows) + + assert.Empty(t, keys) +} + +func TestBuildResultSchema(t *testing.T) { + tagKeys := []string{"node_name", "pod"} + + schema := buildResultSchema(tagKeys) + + fields := schema.Fields() + fieldNames := make([]string, len(fields)) + for i, f := range fields { + fieldNames[i] = f.Name() + } + sort.Strings(fieldNames) + + assert.Equal(t, []string{ + "experiment_id", + "name", + "tag_node_name", + "tag_pod", + "timestamp", + "value", + }, fieldNames) +} + +func TestBuildResultRow_AllTagsPresent(t *testing.T) { + ts := time.Date(2026, 4, 15, 10, 30, 0, 0, time.UTC) + row := bufferedRow{ + ExperimentID: "test-id", + Timestamp: ts, + Name: "cpu_usage", + Value: 42.5, + Tags: map[string]string{"pod": "pod-1", "node_name": "node-1"}, + } + tagKeys := []string{"node_name", "pod"} + + parquetRow := buildResultRow(row, tagKeys) + + // Schema sorts fields alphabetically: + // experiment_id, name, tag_node_name, tag_pod, timestamp, value + assert.Equal(t, 6, len(parquetRow)) + + // Verify column indices are sequential. + for i, v := range parquetRow { + assert.Equal(t, i, v.Column(), "column index mismatch at position %d", i) + } +} + +func TestBuildResultRow_MissingTag(t *testing.T) { + ts := time.Date(2026, 4, 15, 10, 30, 0, 0, time.UTC) + row := bufferedRow{ + ExperimentID: "test-id", + Timestamp: ts, + Name: "rss", + Value: 1024.0, + Tags: map[string]string{"pod": "pod-1"}, + } + tagKeys := []string{"node_name", "pod"} + + parquetRow := buildResultRow(row, tagKeys) + + assert.Equal(t, 6, len(parquetRow)) + + // Find the tag_node_name column (should be null). + // Alphabetical order: experiment_id(0), name(1), tag_node_name(2), tag_pod(3), timestamp(4), value(5) + tagNodeNameVal := parquetRow[2] + assert.True(t, tagNodeNameVal.IsNull(), "missing tag should produce a null value") + assert.Equal(t, 0, tagNodeNameVal.DefinitionLevel(), "null optional field should have definitionLevel=0") + + // tag_pod should be present. + tagPodVal := parquetRow[3] + assert.False(t, tagPodVal.IsNull()) + assert.Equal(t, 1, tagPodVal.DefinitionLevel(), "present optional field should have definitionLevel=1") +} + +func TestFlushBatch_WritesValidParquet(t *testing.T) { + tmpDir := t.TempDir() + var uploadedPath string + + e := &ParquetGCSExporter{ + batchSize: 100, + upload: func(ctx context.Context, objectPath string, localPath string) error { + // Copy the parquet file to our temp dir before it gets cleaned up. + dest := filepath.Join(tmpDir, filepath.Base(objectPath)) + src, err := os.Open(localPath) + if err != nil { + return err + } + defer src.Close() + dst, err := os.Create(dest) + if err != nil { + return err + } + defer dst.Close() + if _, err := io.Copy(dst, src); err != nil { + return err + } + uploadedPath = dest + return nil + }, + } + + ts := time.Date(2026, 4, 15, 12, 0, 0, 0, time.UTC) + rows := []bufferedRow{ + { + ExperimentID: "exp-1", + Timestamp: ts, + Name: "cpu_usage", + Value: 0.85, + Tags: map[string]string{"pod": "kelvin-abc", "node_name": "node-1"}, + }, + { + ExperimentID: "exp-1", + Timestamp: ts.Add(30 * time.Second), + Name: "rss", + Value: 1048576, + Tags: map[string]string{"pod": "kelvin-abc"}, + }, + } + + err := e.flushBatch(context.Background(), "test/path", 0, rows) + require.NoError(t, err) + require.NotEmpty(t, uploadedPath) + + // Read back the parquet file and verify contents. + f, err := os.Open(uploadedPath) + require.NoError(t, err) + defer f.Close() + + stat, err := f.Stat() + require.NoError(t, err) + + pf, err := parquet.OpenFile(f, stat.Size()) + require.NoError(t, err) + + schema := pf.Schema() + assert.Equal(t, int64(2), pf.NumRows()) + + // Verify schema has expected columns. + fields := schema.Fields() + fieldNames := make([]string, len(fields)) + for i, f := range fields { + fieldNames[i] = f.Name() + } + sort.Strings(fieldNames) + assert.Equal(t, []string{ + "experiment_id", + "name", + "tag_node_name", + "tag_pod", + "timestamp", + "value", + }, fieldNames) + + // Re-open the file for the reader (the File consumed the initial handle). + f2, err := os.Open(uploadedPath) + require.NoError(t, err) + defer f2.Close() + + reader := parquet.NewReader(f2) + defer reader.Close() + + parquetRows := make([]parquet.Row, 2) + n, err := reader.ReadRows(parquetRows) + // ReadRows returns io.EOF when it reaches the end, even if it read rows. + if err != nil && !errors.Is(err, io.EOF) { + require.NoError(t, err) + } + assert.Equal(t, 2, n) + + // First row should have all tags present. + // Second row should have tag_node_name as null. + // Column order (alphabetical): experiment_id(0), name(1), tag_node_name(2), tag_pod(3), timestamp(4), value(5) + row0NodeName := parquetRows[0][2] + assert.False(t, row0NodeName.IsNull(), "first row tag_node_name should be present") + + row1NodeName := parquetRows[1][2] + assert.True(t, row1NodeName.IsNull(), "second row tag_node_name should be null") +} + +func TestExportResults_SingleBatch(t *testing.T) { + tmpDir := t.TempDir() + uploadedFiles := make(map[string]string) + + expID := uuid.Must(uuid.NewV4()) + e := &ParquetGCSExporter{ + prefix: "perf-results", + batchSize: 100, + upload: func(ctx context.Context, objectPath string, localPath string) error { + dest := filepath.Join(tmpDir, strings.ReplaceAll(objectPath, "/", "_")) + src, err := os.Open(localPath) + if err != nil { + return err + } + defer src.Close() + dst, err := os.Create(dest) + if err != nil { + return err + } + defer dst.Close() + if _, err := io.Copy(dst, src); err != nil { + return err + } + uploadedFiles[objectPath] = dest + return nil + }, + } + + resultCh := make(chan *metrics.ResultRow, 3) + ts := time.Date(2026, 4, 15, 14, 0, 0, 0, time.UTC) + resultCh <- &metrics.ResultRow{ + Timestamp: ts, + Name: "cpu_seconds_counter", + Value: 100.5, + Tags: map[string]string{"pod": "server-abc"}, + } + resultCh <- &metrics.ResultRow{ + Timestamp: ts.Add(30 * time.Second), + Name: "rss", + Value: 2097152, + Tags: map[string]string{"pod": "server-abc", "node_name": "node-0"}, + } + resultCh <- &metrics.ResultRow{ + Timestamp: ts.Add(60 * time.Second), + Name: "vsize", + Value: 4194304, + Tags: map[string]string{"pod": "server-abc", "node_name": "node-0"}, + } + close(resultCh) + + err := e.ExportResults(context.Background(), expID, resultCh) + require.NoError(t, err) + + // Should have produced exactly one batch file. + assert.Equal(t, 1, len(uploadedFiles), "expected exactly one parquet file") + + // Verify the GCS path includes the date and experiment ID. + for objectPath := range uploadedFiles { + assert.Contains(t, objectPath, expID.String()) + assert.Contains(t, objectPath, "perf-results/") + assert.Contains(t, objectPath, "results_0000.parquet") + } + + // Read the parquet file and verify row count. + for _, localPath := range uploadedFiles { + f, err := os.Open(localPath) + require.NoError(t, err) + defer f.Close() + + stat, err := f.Stat() + require.NoError(t, err) + + pf, err := parquet.OpenFile(f, stat.Size()) + require.NoError(t, err) + assert.Equal(t, int64(3), pf.NumRows()) + + // Verify schema has tag columns from the union of all rows. + fields := pf.Schema().Fields() + fieldNames := make([]string, len(fields)) + for i, f := range fields { + fieldNames[i] = f.Name() + } + sort.Strings(fieldNames) + assert.Equal(t, []string{ + "experiment_id", + "name", + "tag_node_name", + "tag_pod", + "timestamp", + "value", + }, fieldNames) + } +} + +func TestExportResults_MultipleBatches(t *testing.T) { + tmpDir := t.TempDir() + uploadedFiles := make(map[string]string) + + expID := uuid.Must(uuid.NewV4()) + e := &ParquetGCSExporter{ + batchSize: 2, // Small batch size to force multiple files. + upload: func(ctx context.Context, objectPath string, localPath string) error { + dest := filepath.Join(tmpDir, strings.ReplaceAll(objectPath, "/", "_")) + src, err := os.Open(localPath) + if err != nil { + return err + } + defer src.Close() + dst, err := os.Create(dest) + if err != nil { + return err + } + defer dst.Close() + if _, err := io.Copy(dst, src); err != nil { + return err + } + uploadedFiles[objectPath] = dest + return nil + }, + } + + resultCh := make(chan *metrics.ResultRow, 5) + ts := time.Date(2026, 4, 15, 14, 0, 0, 0, time.UTC) + for i := 0; i < 5; i++ { + resultCh <- &metrics.ResultRow{ + Timestamp: ts.Add(time.Duration(i) * 30 * time.Second), + Name: "cpu_usage", + Value: float64(i) * 0.1, + Tags: map[string]string{"pod": "test-pod"}, + } + } + close(resultCh) + + err := e.ExportResults(context.Background(), expID, resultCh) + require.NoError(t, err) + + // 5 rows with batch size 2 should produce 3 files: [2, 2, 1]. + assert.Equal(t, 3, len(uploadedFiles), "expected 3 parquet files for 5 rows with batch size 2") + + // Verify file naming. + hasFile0, hasFile1, hasFile2 := false, false, false + for objectPath := range uploadedFiles { + if strings.Contains(objectPath, "results_0000.parquet") { + hasFile0 = true + } + if strings.Contains(objectPath, "results_0001.parquet") { + hasFile1 = true + } + if strings.Contains(objectPath, "results_0002.parquet") { + hasFile2 = true + } + } + assert.True(t, hasFile0, "missing results_0000.parquet") + assert.True(t, hasFile1, "missing results_0001.parquet") + assert.True(t, hasFile2, "missing results_0002.parquet") + + // Verify total row count across all files. + totalRows := int64(0) + for _, localPath := range uploadedFiles { + f, err := os.Open(localPath) + require.NoError(t, err) + defer f.Close() + stat, err := f.Stat() + require.NoError(t, err) + pf, err := parquet.OpenFile(f, stat.Size()) + require.NoError(t, err) + totalRows += pf.NumRows() + } + assert.Equal(t, int64(5), totalRows) +} + +func TestExportResults_EmptyChannel(t *testing.T) { + uploadCalled := false + e := &ParquetGCSExporter{ + batchSize: 100, + upload: func(ctx context.Context, objectPath string, localPath string) error { + uploadCalled = true + return nil + }, + } + + resultCh := make(chan *metrics.ResultRow) + close(resultCh) + + expID := uuid.Must(uuid.NewV4()) + err := e.ExportResults(context.Background(), expID, resultCh) + require.NoError(t, err) + assert.False(t, uploadCalled, "no files should be uploaded for empty channel") +} + +// --- Benchmarks --- + +// makeBenchRows generates n buffered rows with the specified number of tag keys. +func makeBenchRows(n int, numTags int) []bufferedRow { + ts := time.Date(2026, 4, 15, 12, 0, 0, 0, time.UTC) + rows := make([]bufferedRow, n) + for i := range rows { + tags := make(map[string]string, numTags) + for j := 0; j < numTags; j++ { + tags[fmt.Sprintf("tag_key_%d", j)] = fmt.Sprintf("value_%d_%d", i, j) + } + rows[i] = bufferedRow{ + ExperimentID: "bench-exp-id", + Timestamp: ts.Add(time.Duration(i) * 30 * time.Second), + Name: "cpu_usage", + Value: float64(i) * 0.01, + Tags: tags, + } + } + return rows +} + +func BenchmarkBuildResultRow(b *testing.B) { + for _, numTags := range []int{2, 5, 10} { + b.Run(fmt.Sprintf("tags=%d", numTags), func(b *testing.B) { + rows := makeBenchRows(1, numTags) + tagKeys := collectTagKeys(rows) + row := rows[0] + b.ResetTimer() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + buildResultRow(row, tagKeys) + } + }) + } +} + +func BenchmarkCollectTagKeys(b *testing.B) { + for _, numRows := range []int{100, 1000, 10000} { + b.Run(fmt.Sprintf("rows=%d", numRows), func(b *testing.B) { + rows := makeBenchRows(numRows, 3) + b.ResetTimer() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + collectTagKeys(rows) + } + }) + } +} + +func BenchmarkFlushBatch(b *testing.B) { + for _, numRows := range []int{100, 1000, 10000} { + b.Run(fmt.Sprintf("rows=%d", numRows), func(b *testing.B) { + rows := makeBenchRows(numRows, 3) + e := &ParquetGCSExporter{ + batchSize: numRows, + upload: func(ctx context.Context, objectPath string, localPath string) error { + // No-op upload: measures only in-memory conversion + parquet write to disk. + return nil + }, + } + b.ResetTimer() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + if err := e.flushBatch(context.Background(), "bench/path", 0, rows); err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/src/e2e_test/perf_tool/pkg/run/BUILD.bazel b/src/e2e_test/perf_tool/pkg/run/BUILD.bazel index 55b3fdc18a9..a7f1b9b1580 100644 --- a/src/e2e_test/perf_tool/pkg/run/BUILD.bazel +++ b/src/e2e_test/perf_tool/pkg/run/BUILD.bazel @@ -19,7 +19,6 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library") go_library( name = "run", srcs = [ - "row.go", "run.go", ], importpath = "px.dev/pixie/src/e2e_test/perf_tool/pkg/run", @@ -28,9 +27,9 @@ go_library( "//src/e2e_test/perf_tool/experimentpb:experiment_pl_go_proto", "//src/e2e_test/perf_tool/pkg/cluster", "//src/e2e_test/perf_tool/pkg/deploy", + "//src/e2e_test/perf_tool/pkg/exporter", "//src/e2e_test/perf_tool/pkg/metrics", "//src/e2e_test/perf_tool/pkg/pixie", - "//src/shared/bq", "@com_github_cenkalti_backoff_v4//:backoff", "@com_github_gofrs_uuid//:uuid", "@com_github_gogo_protobuf//jsonpb", diff --git a/src/e2e_test/perf_tool/pkg/run/run.go b/src/e2e_test/perf_tool/pkg/run/run.go index b02b15219c2..a7ae2ca9c52 100644 --- a/src/e2e_test/perf_tool/pkg/run/run.go +++ b/src/e2e_test/perf_tool/pkg/run/run.go @@ -39,17 +39,16 @@ import ( "px.dev/pixie/src/e2e_test/perf_tool/experimentpb" "px.dev/pixie/src/e2e_test/perf_tool/pkg/cluster" "px.dev/pixie/src/e2e_test/perf_tool/pkg/deploy" + "px.dev/pixie/src/e2e_test/perf_tool/pkg/exporter" "px.dev/pixie/src/e2e_test/perf_tool/pkg/metrics" "px.dev/pixie/src/e2e_test/perf_tool/pkg/pixie" - "px.dev/pixie/src/shared/bq" ) // Runner is responsible for running experiments using the ClusterProvider to get a cluster for the experiment. type Runner struct { c cluster.Provider pxCtx *pixie.Context - resultTable *bq.Table - specTable *bq.Table + exporter exporter.Exporter containerRegistryRepo string clusterCtx *cluster.Context @@ -66,12 +65,11 @@ type Runner struct { } // NewRunner creates a new Runner for the given contexts. -func NewRunner(c cluster.Provider, pxCtx *pixie.Context, resultTable *bq.Table, specTable *bq.Table, containerRegistryRepo string) *Runner { +func NewRunner(c cluster.Provider, pxCtx *pixie.Context, exp exporter.Exporter, containerRegistryRepo string) *Runner { return &Runner{ c: c, pxCtx: pxCtx, - resultTable: resultTable, - specTable: specTable, + exporter: exp, containerRegistryRepo: containerRegistryRepo, } } @@ -98,7 +96,12 @@ func (r *Runner) RunExperiment(ctx context.Context, expID uuid.UUID, spec *exper defer metricsChCloseOnce.Do(func() { close(r.metricsResultCh) }) r.wg.Add(1) - go r.runBQInserter(expID) + go func() { + defer r.wg.Done() + if err := r.exporter.ExportResults(ctx, expID, r.metricsResultCh); err != nil { + log.WithError(err).Error("Failed to export results") + } + }() if err := eg.Wait(); err != nil { if r.clusterCleanup != nil { @@ -126,23 +129,12 @@ func (r *Runner) RunExperiment(ctx context.Context, expID uuid.UUID, spec *exper return err } - // The experiment succeeded so we write the spec to bigquery. + // The experiment succeeded so we write the spec to the exporter. encodedSpec, err := (&jsonpb.Marshaler{}).MarshalToString(spec) if err != nil { return err } - specRow := &SpecRow{ - ExperimentID: expID.String(), - Spec: encodedSpec, - CommitTopoOrder: commitTopoOrder, - } - - inserter := r.specTable.Inserter() - inserter.SkipInvalidRows = false - - putCtx, cancel := context.WithTimeout(ctx, 5*time.Minute) - defer cancel() - if err := inserter.Put(putCtx, specRow); err != nil { + if err := r.exporter.ExportSpec(ctx, expID, encodedSpec, commitTopoOrder); err != nil { return err } @@ -368,29 +360,6 @@ func (r *Runner) prepareWorkloads(ctx context.Context, spec *experimentpb.Experi return nil } -func (r *Runner) runBQInserter(expID uuid.UUID) { - defer r.wg.Done() - - bqCh := make(chan interface{}) - defer close(bqCh) - - inserter := &bq.BatchInserter{ - Table: r.resultTable, - BatchSize: 512, - PushTimeout: 2 * time.Minute, - } - go inserter.Run(bqCh) - - for row := range r.metricsResultCh { - bqRow, err := MetricsRowToResultRow(expID, row) - if err != nil { - log.WithError(err).Error("Failed to convert result row") - continue - } - bqCh <- bqRow - } -} - func getTopoOrder() (int, error) { cmd := exec.Command("git", "rev-list", "--count", "HEAD") var stdout bytes.Buffer From 3510794c40cece03a1aed0df5b697ab7f5dabddd Mon Sep 17 00:00:00 2001 From: Dom Del Nano Date: Wed, 15 Apr 2026 20:26:22 -0700 Subject: [PATCH 03/10] Allow prometheus recorders to specifiy different kubeconfig or kubecontext Signed-off-by: Dom Del Nano --- .../perf_tool/experimentpb/experiment.pb.go | 342 ++++++++++++------ .../perf_tool/experimentpb/experiment.proto | 6 + src/e2e_test/perf_tool/pkg/cluster/context.go | 30 ++ .../pkg/metrics/prometheus_recorder.go | 12 +- .../perf_tool/pkg/metrics/recorder.go | 29 +- src/e2e_test/perf_tool/pkg/run/run.go | 6 +- 6 files changed, 298 insertions(+), 127 deletions(-) diff --git a/src/e2e_test/perf_tool/experimentpb/experiment.pb.go b/src/e2e_test/perf_tool/experimentpb/experiment.pb.go index dc43e5d79be..40be13d29e8 100755 --- a/src/e2e_test/perf_tool/experimentpb/experiment.pb.go +++ b/src/e2e_test/perf_tool/experimentpb/experiment.pb.go @@ -1254,6 +1254,8 @@ type PrometheusScrapeSpec struct { Port int32 `protobuf:"varint,4,opt,name=port,proto3" json:"port,omitempty"` ScrapePeriod *types.Duration `protobuf:"bytes,5,opt,name=scrape_period,json=scrapePeriod,proto3" json:"scrape_period,omitempty"` MetricNames map[string]string `protobuf:"bytes,6,rep,name=metric_names,json=metricNames,proto3" json:"metric_names,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + KubeconfigPath string `protobuf:"bytes,7,opt,name=kubeconfig_path,json=kubeconfigPath,proto3" json:"kubeconfig_path,omitempty"` + KubeContext string `protobuf:"bytes,8,opt,name=kube_context,json=kubeContext,proto3" json:"kube_context,omitempty"` } func (m *PrometheusScrapeSpec) Reset() { *m = PrometheusScrapeSpec{} } @@ -1330,6 +1332,20 @@ func (m *PrometheusScrapeSpec) GetMetricNames() map[string]string { return nil } +func (m *PrometheusScrapeSpec) GetKubeconfigPath() string { + if m != nil { + return m.KubeconfigPath + } + return "" +} + +func (m *PrometheusScrapeSpec) GetKubeContext() string { + if m != nil { + return m.KubeContext + } + return "" +} + type ClusterSpec struct { NumNodes int32 `protobuf:"varint,1,opt,name=num_nodes,json=numNodes,proto3" json:"num_nodes,omitempty"` Node *NodeSpec `protobuf:"bytes,2,opt,name=node,proto3" json:"node,omitempty"` @@ -1560,119 +1576,121 @@ func init() { } var fileDescriptor_96d7e52dda1e6fe3 = []byte{ - // 1786 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0xcd, 0x73, 0x1b, 0x49, - 0x15, 0xd7, 0x48, 0xb2, 0x25, 0x3d, 0xc9, 0xb2, 0xdc, 0xf9, 0x40, 0xf1, 0xa6, 0xe4, 0xec, 0x6c, - 0x01, 0x21, 0xec, 0x5a, 0x24, 0xcb, 0x87, 0xd9, 0x2c, 0x5b, 0x25, 0xc9, 0x06, 0x2b, 0x71, 0x6c, - 0xd1, 0xf2, 0x7a, 0x61, 0x8b, 0xaa, 0xa9, 0xf6, 0x4c, 0x47, 0x9a, 0xf2, 0x7c, 0x65, 0xba, 0x95, - 0xb5, 0x39, 0x71, 0xa1, 0x38, 0x51, 0xc5, 0x01, 0xfe, 0x03, 0x0e, 0xfc, 0x09, 0xdc, 0x39, 0x00, - 0xb7, 0x1c, 0xf7, 0xe4, 0x22, 0xca, 0x85, 0xe3, 0x1e, 0xb8, 0x43, 0xf5, 0xc7, 0x8c, 0x46, 0xb2, - 0x92, 0x40, 0x15, 0xb7, 0x9e, 0x5f, 0xff, 0xde, 0xeb, 0xd7, 0xaf, 0xfb, 0xf7, 0x5e, 0x4b, 0xf0, - 0x5d, 0x16, 0xdb, 0x6d, 0xfa, 0x80, 0x5a, 0x9c, 0x32, 0xde, 0x8e, 0x68, 0xfc, 0xd4, 0xe2, 0x61, - 0xe8, 0xb5, 0xe9, 0x79, 0x44, 0x63, 0xd7, 0xa7, 0x01, 0x8f, 0x4e, 0x33, 0x1f, 0xdb, 0x51, 0x1c, - 0xf2, 0x10, 0xd5, 0xa2, 0xf3, 0xed, 0x94, 0xbb, 0xd9, 0x1a, 0x85, 0xe1, 0xc8, 0xa3, 0x6d, 0x39, - 0x77, 0x3a, 0x79, 0xda, 0x76, 0x26, 0x31, 0xe1, 0x6e, 0x18, 0x28, 0xf6, 0xe6, 0xf5, 0x51, 0x38, - 0x0a, 0xe5, 0xb0, 0x2d, 0x46, 0x0a, 0x35, 0xff, 0x9d, 0x87, 0xfa, 0x5e, 0xea, 0x78, 0x18, 0x51, - 0x1b, 0x3d, 0x84, 0xea, 0x73, 0xf7, 0x97, 0x2e, 0x8d, 0x2d, 0x16, 0x51, 0xbb, 0x69, 0xdc, 0x31, - 0xee, 0x56, 0x1f, 0x6c, 0x6e, 0x67, 0x17, 0xdb, 0xfe, 0x2c, 0x8c, 0xcf, 0xbc, 0x90, 0x38, 0xc2, - 0x00, 0x83, 0xa2, 0x4b, 0xe3, 0x0e, 0xd4, 0xbf, 0xd0, 0x73, 0xd2, 0x9c, 0x35, 0xf3, 0x77, 0x0a, - 0x6f, 0xb1, 0x5f, 0xfb, 0x22, 0xf3, 0xc5, 0xd0, 0x43, 0xa8, 0xf9, 0x94, 0xc7, 0xae, 0xad, 0x1d, - 0x14, 0xa4, 0x83, 0xe6, 0xbc, 0x83, 0x27, 0x92, 0x21, 0xcd, 0xab, 0x7e, 0x3a, 0x66, 0xe8, 0x63, - 0xa8, 0xd9, 0xde, 0x84, 0xf1, 0x24, 0xfa, 0xa2, 0x8c, 0xfe, 0xd6, 0xbc, 0x71, 0x4f, 0x31, 0x94, - 0xb5, 0x3d, 0xfb, 0x40, 0xdf, 0x81, 0x72, 0x3c, 0x09, 0x94, 0xe5, 0x8a, 0xb4, 0xbc, 0x31, 0x6f, - 0x89, 0x27, 0x81, 0xb4, 0x2a, 0xc5, 0x6a, 0x80, 0xde, 0x07, 0xb0, 0x43, 0xdf, 0x77, 0xb9, 0xc5, - 0xc6, 0xa4, 0xb9, 0x7a, 0xc7, 0xb8, 0x5b, 0xe9, 0xae, 0x4d, 0x2f, 0xb7, 0x2a, 0x3d, 0x89, 0x0e, - 0xf7, 0x3b, 0xb8, 0xa2, 0x08, 0xc3, 0x31, 0x41, 0x08, 0x8a, 0x9c, 0x8c, 0x58, 0xb3, 0x74, 0xa7, - 0x70, 0xb7, 0x82, 0xe5, 0xd8, 0xfc, 0xab, 0x01, 0xb5, 0x6c, 0x3a, 0x04, 0x29, 0x20, 0x3e, 0x95, - 0x89, 0xaf, 0x60, 0x39, 0x16, 0x39, 0x71, 0x68, 0xe4, 0x85, 0x17, 0x16, 0xe3, 0x34, 0x4a, 0x92, - 0xba, 0x90, 0x93, 0x5d, 0xc9, 0x18, 0x72, 0x1a, 0xe1, 0xaa, 0x93, 0x8e, 0x19, 0xfa, 0x11, 0xd4, - 0xc6, 0x94, 0x78, 0x7c, 0x6c, 0x8f, 0xa9, 0x7d, 0x96, 0x24, 0x74, 0x21, 0x27, 0xfb, 0x92, 0xd1, - 0x13, 0x0c, 0x3c, 0x47, 0x47, 0xdf, 0x84, 0x75, 0x62, 0x8b, 0x8b, 0x64, 0x31, 0xea, 0x51, 0x9b, - 0x87, 0xb1, 0xcc, 0x6a, 0x05, 0xd7, 0x15, 0x3c, 0xd4, 0xa8, 0xf9, 0x77, 0x03, 0x60, 0x16, 0x03, - 0xea, 0x41, 0x35, 0x8a, 0x69, 0x4c, 0x03, 0x87, 0xc6, 0xd4, 0xd1, 0xf7, 0x68, 0x6b, 0x7e, 0xd5, - 0xc1, 0x8c, 0xa0, 0x2c, 0xf7, 0x73, 0x38, 0x6b, 0x85, 0x3e, 0x82, 0x32, 0x3b, 0x23, 0x4f, 0x9f, - 0x86, 0x9e, 0xd3, 0xcc, 0x4b, 0x0f, 0xb7, 0xe7, 0x3d, 0x0c, 0xf5, 0x6c, 0x6a, 0x9e, 0xf2, 0xd1, - 0xb7, 0x21, 0x1f, 0x9d, 0x37, 0x0b, 0xcb, 0x6e, 0xc0, 0xe0, 0xbc, 0x77, 0xd0, 0x4f, 0x4d, 0xf2, - 0xd1, 0x79, 0x77, 0x0d, 0x74, 0xce, 0x2c, 0x7e, 0x11, 0x51, 0xf3, 0xf7, 0x06, 0x54, 0x33, 0x29, - 0x41, 0x1f, 0x43, 0xe1, 0x6c, 0x87, 0x2d, 0xdf, 0xc4, 0xe3, 0x9d, 0xe1, 0x20, 0x74, 0x18, 0xa6, - 0xc4, 0xb9, 0x90, 0xec, 0x6e, 0x69, 0x7a, 0xb9, 0x55, 0x78, 0xbc, 0x33, 0xdc, 0xcf, 0x61, 0x61, - 0x86, 0x7e, 0x08, 0x85, 0xe8, 0xdc, 0x5b, 0xbe, 0x81, 0xc1, 0xf9, 0x41, 0x66, 0x21, 0x65, 0x2a, - 0xb0, 0x1c, 0x16, 0x36, 0xdd, 0x1a, 0x80, 0x3c, 0x07, 0x15, 0xd6, 0x7d, 0xd8, 0xb8, 0xb2, 0x1a, - 0xba, 0x0d, 0x15, 0x71, 0x49, 0x58, 0x44, 0xec, 0xe4, 0xd6, 0xcc, 0x00, 0xf3, 0x08, 0xea, 0xf3, - 0x4b, 0xa0, 0x9b, 0xb0, 0xca, 0xec, 0xd8, 0x8d, 0xb8, 0x26, 0xeb, 0x2f, 0xf4, 0x75, 0xa8, 0xb3, - 0x89, 0x6d, 0x53, 0xc6, 0x2c, 0x3b, 0xf4, 0x26, 0x7e, 0x20, 0x03, 0xae, 0xe0, 0x35, 0x8d, 0xf6, - 0x24, 0x68, 0xfe, 0x02, 0x2a, 0x03, 0xc2, 0xed, 0xb1, 0xbc, 0xac, 0xb7, 0xa1, 0x78, 0x41, 0x7c, - 0x4f, 0x79, 0xea, 0x96, 0xa7, 0x97, 0x5b, 0xc5, 0x9f, 0x77, 0x9e, 0x1c, 0x60, 0x89, 0xa2, 0xfb, - 0xb0, 0xca, 0x49, 0x3c, 0xa2, 0x5c, 0x6f, 0x7d, 0xf1, 0x14, 0x84, 0x9b, 0x63, 0x49, 0xc0, 0x9a, - 0x68, 0xfe, 0x26, 0x0f, 0xd5, 0x0c, 0x8e, 0xbe, 0x05, 0x15, 0x12, 0xb9, 0xd6, 0x28, 0x0e, 0x27, - 0x91, 0x5e, 0xa5, 0x36, 0xbd, 0xdc, 0x2a, 0x77, 0x06, 0xfd, 0x9f, 0x08, 0x0c, 0x97, 0x49, 0xe4, - 0xca, 0x11, 0x6a, 0x43, 0x55, 0x50, 0x9f, 0xd3, 0x98, 0xb9, 0xa1, 0x0e, 0xbe, 0x5b, 0x9f, 0x5e, - 0x6e, 0x41, 0x67, 0xd0, 0x3f, 0x51, 0x28, 0x06, 0x12, 0xb9, 0x7a, 0x2c, 0x94, 0x76, 0xe6, 0x06, - 0x8e, 0xbc, 0x22, 0x15, 0x2c, 0xc7, 0xa9, 0xfa, 0x8a, 0x19, 0xf5, 0xcd, 0x25, 0x78, 0x65, 0x21, - 0xc1, 0x22, 0x6d, 0x1e, 0x39, 0xa5, 0xde, 0x4c, 0x1e, 0xab, 0x2a, 0x6d, 0x12, 0x4d, 0xd4, 0x81, - 0xda, 0x70, 0x8d, 0x04, 0x41, 0xc8, 0xc9, 0xbc, 0x94, 0x4a, 0x92, 0x8b, 0x66, 0x53, 0xa9, 0x9c, - 0x38, 0x6c, 0x5c, 0x91, 0x87, 0xa8, 0x37, 0x22, 0xb3, 0x56, 0x44, 0xf8, 0x58, 0x5c, 0xc7, 0x42, - 0x52, 0x6f, 0x44, 0xd6, 0x07, 0x02, 0xc4, 0x15, 0x41, 0x90, 0x43, 0x74, 0x1f, 0x4a, 0x91, 0xc8, - 0x25, 0x4d, 0x2a, 0xc6, 0xd7, 0x96, 0x1c, 0x80, 0x2a, 0x68, 0x9a, 0x67, 0xfe, 0xd6, 0x80, 0xfa, - 0xbc, 0xa6, 0xd0, 0x7b, 0xb0, 0x96, 0x68, 0x4a, 0xae, 0xab, 0xaf, 0x4d, 0x2d, 0x01, 0xc5, 0x5a, - 0x73, 0x24, 0x12, 0x8f, 0xd4, 0x82, 0x19, 0x52, 0x27, 0x1e, 0xcd, 0xc5, 0x53, 0xf8, 0x2f, 0xe3, - 0xb9, 0x80, 0x6a, 0x46, 0xac, 0xe2, 0x78, 0xa4, 0x77, 0x43, 0x55, 0x50, 0x31, 0x46, 0x2d, 0x80, - 0xf4, 0x34, 0x92, 0x75, 0x33, 0x08, 0xfa, 0x3e, 0xd4, 0x19, 0xe5, 0x56, 0xd2, 0x17, 0x5c, 0x75, - 0xe0, 0xe5, 0x6e, 0x63, 0x7a, 0xb9, 0x55, 0x1b, 0x52, 0xae, 0xdb, 0x41, 0x7f, 0x17, 0xd7, 0xd8, - 0xec, 0xcb, 0x31, 0xff, 0x6c, 0x00, 0xcc, 0xfa, 0x0c, 0xda, 0x51, 0x22, 0x56, 0x25, 0xe0, 0x9d, - 0x2b, 0x22, 0x1e, 0x4a, 0x11, 0x09, 0xe6, 0xa2, 0x86, 0xd1, 0x0e, 0x14, 0xa3, 0x38, 0xf4, 0xb5, - 0x08, 0xcc, 0xc5, 0x12, 0x18, 0xfa, 0x94, 0x8f, 0xe9, 0x84, 0x0d, 0xed, 0x98, 0x44, 0x54, 0x78, - 0xd8, 0xcf, 0x61, 0x69, 0xb1, 0xac, 0xf6, 0x3a, 0xcb, 0x6a, 0xaf, 0x28, 0x5f, 0xba, 0x69, 0xca, - 0x3a, 0x31, 0x2d, 0xc0, 0xda, 0x5c, 0x4c, 0xaf, 0x15, 0xfd, 0x6d, 0xa8, 0x30, 0x1e, 0x53, 0xe2, - 0xbb, 0xc1, 0x48, 0x06, 0x58, 0xc6, 0x33, 0x00, 0xfd, 0x18, 0x36, 0xec, 0xd0, 0x13, 0x6b, 0x88, - 0x18, 0xc4, 0x33, 0x21, 0x74, 0xd2, 0x8a, 0xaa, 0x1e, 0x1c, 0xdb, 0xc9, 0x83, 0x63, 0x7b, 0x57, - 0x3f, 0x38, 0x70, 0x63, 0x66, 0x33, 0x90, 0x26, 0xe8, 0x67, 0xb0, 0xce, 0xa9, 0x1f, 0x79, 0x84, - 0x53, 0xeb, 0x39, 0xf1, 0x26, 0x94, 0x35, 0x8b, 0xf2, 0x02, 0xb4, 0xdf, 0x90, 0xc7, 0xed, 0x63, - 0x6d, 0x72, 0x22, 0x2d, 0xf6, 0x02, 0x1e, 0x5f, 0xe0, 0x3a, 0x9f, 0x03, 0x11, 0x86, 0x35, 0x4e, - 0x4e, 0x3d, 0x6a, 0x85, 0x13, 0x1e, 0x4d, 0x38, 0x6b, 0xae, 0x48, 0xbf, 0x1f, 0xbc, 0xd1, 0xaf, - 0x30, 0x38, 0x52, 0x7c, 0xe5, 0xb5, 0xc6, 0x33, 0xd0, 0x66, 0x07, 0xae, 0x2d, 0x59, 0x1a, 0x35, - 0xa0, 0x70, 0x46, 0x2f, 0x74, 0xfe, 0xc4, 0x10, 0x5d, 0x87, 0x15, 0xb9, 0x1b, 0x5d, 0x28, 0xd5, - 0xc7, 0x47, 0xf9, 0x1d, 0x63, 0xf3, 0x14, 0x36, 0xae, 0xac, 0xb2, 0xc4, 0xc1, 0x0f, 0xb2, 0x0e, - 0xaa, 0x0f, 0xde, 0x7d, 0x4d, 0xd4, 0xca, 0xcb, 0x81, 0xcb, 0x78, 0x66, 0x0d, 0x13, 0xc3, 0xb5, - 0x25, 0x0c, 0xf4, 0x10, 0x4a, 0x49, 0x2e, 0x0c, 0x99, 0x8b, 0x37, 0x7b, 0x55, 0x72, 0xd3, 0x16, - 0xe6, 0x5f, 0x8c, 0x2b, 0x4e, 0xe5, 0xf5, 0x79, 0x04, 0x6b, 0xcc, 0x0d, 0x46, 0x1e, 0xb5, 0xd4, - 0x35, 0xd3, 0x32, 0x78, 0x6f, 0xa1, 0x19, 0x4b, 0x8a, 0xd2, 0xcc, 0xe0, 0xfc, 0x40, 0xd9, 0xef, - 0xe7, 0x70, 0x8d, 0x65, 0x26, 0xd0, 0x4f, 0x61, 0xc3, 0x21, 0x9c, 0x58, 0x5e, 0x28, 0x3b, 0xcd, - 0x24, 0xe0, 0x34, 0xd6, 0x09, 0x58, 0xf0, 0xb7, 0x4b, 0x38, 0x39, 0x08, 0x45, 0xe7, 0x91, 0xa4, - 0xd4, 0xdf, 0xba, 0x33, 0x3f, 0x21, 0xae, 0xbf, 0xda, 0x81, 0x7c, 0xbb, 0x99, 0x7f, 0x30, 0xe0, - 0xc6, 0xd2, 0x58, 0x44, 0x99, 0xe2, 0xae, 0x4f, 0x19, 0x27, 0x7e, 0x24, 0xba, 0x5c, 0x52, 0xcb, - 0x52, 0xb0, 0x17, 0x7a, 0x68, 0x2b, 0x15, 0x93, 0x6c, 0x05, 0xea, 0x70, 0x41, 0x41, 0x87, 0xa2, - 0x21, 0xbc, 0x03, 0x15, 0x79, 0x0c, 0xd2, 0x83, 0xea, 0x1e, 0x65, 0x09, 0x08, 0xeb, 0x5b, 0x50, - 0xe6, 0x64, 0x24, 0xa6, 0xd4, 0x25, 0xaf, 0xe0, 0x12, 0x27, 0xa3, 0x5e, 0xe8, 0x31, 0xf1, 0x42, - 0xba, 0xb1, 0x74, 0x4f, 0xff, 0xa7, 0xb8, 0xee, 0x01, 0x30, 0xfa, 0xcc, 0x72, 0x9d, 0x59, 0x60, - 0xaa, 0x5b, 0x0e, 0xe9, 0xb3, 0xfe, 0x6e, 0x2f, 0xf4, 0x70, 0x99, 0xd1, 0x67, 0x7d, 0x47, 0x38, - 0xfb, 0x04, 0xd6, 0x74, 0xca, 0xb4, 0xac, 0x8b, 0x6f, 0x93, 0x75, 0x4d, 0xf1, 0x95, 0xa4, 0xcd, - 0x7f, 0xe5, 0xe1, 0xfa, 0xb2, 0xda, 0xf5, 0xe6, 0xe7, 0x08, 0xfa, 0x06, 0xac, 0xfb, 0xa2, 0xb4, - 0x5b, 0xaa, 0x67, 0x0a, 0x3d, 0xe8, 0x57, 0x86, 0x84, 0x0f, 0x04, 0xfa, 0x98, 0x5e, 0xa0, 0x7b, - 0xb0, 0x91, 0xe5, 0x29, 0x95, 0xa8, 0x54, 0xaf, 0xcf, 0x98, 0x52, 0x9e, 0xa2, 0x29, 0x44, 0x61, - 0xcc, 0xe5, 0x0e, 0x56, 0xb0, 0x1c, 0x8b, 0xed, 0x31, 0x19, 0x53, 0xb2, 0xbd, 0x95, 0xb7, 0x6e, - 0x4f, 0xf1, 0x75, 0xc5, 0x3a, 0x49, 0x7f, 0x85, 0xc8, 0xd8, 0x9b, 0xab, 0x52, 0x4a, 0x1f, 0xbe, - 0xbd, 0x76, 0xeb, 0x9f, 0x26, 0xe2, 0x3c, 0x74, 0x71, 0xa9, 0xce, 0x4e, 0x88, 0x6d, 0x7e, 0x02, - 0x8d, 0x45, 0xc2, 0xff, 0x52, 0x58, 0xcc, 0x13, 0xa8, 0x66, 0x7e, 0xbe, 0x88, 0x9b, 0x18, 0x4c, - 0x7c, 0x2b, 0x08, 0x1d, 0xaa, 0x5e, 0xa7, 0x2b, 0xb8, 0x1c, 0x4c, 0xfc, 0x43, 0xf1, 0x8d, 0xee, - 0x41, 0x51, 0x4c, 0x68, 0x6d, 0xdd, 0x9c, 0x8f, 0x5d, 0x50, 0xa4, 0xf6, 0x25, 0xc7, 0xfc, 0x00, - 0xca, 0x09, 0x82, 0xde, 0x85, 0x9a, 0x4f, 0xec, 0xb1, 0x1b, 0x50, 0xd9, 0x4d, 0x74, 0x60, 0x55, - 0x8d, 0x1d, 0x8b, 0x06, 0xd3, 0x87, 0x92, 0xfe, 0x2d, 0x84, 0x1e, 0x40, 0x49, 0x35, 0xa3, 0xd7, - 0xfc, 0x54, 0xeb, 0xa8, 0x4e, 0x25, 0xcb, 0x8c, 0x26, 0x3e, 0x2a, 0x96, 0x8d, 0x46, 0xfe, 0x51, - 0xb1, 0x9c, 0x6f, 0x14, 0xcc, 0x5f, 0x1b, 0x00, 0x33, 0x0e, 0x7a, 0x1f, 0x8a, 0xe9, 0xa2, 0xf5, - 0xe5, 0xbe, 0x44, 0x04, 0x58, 0xb2, 0xd0, 0xf7, 0xa0, 0x9c, 0xfc, 0xce, 0x4d, 0xdf, 0x98, 0xaf, - 0x3d, 0xe1, 0x94, 0x9a, 0xbe, 0xf2, 0x0a, 0xb3, 0x57, 0xde, 0xbd, 0x3f, 0xa6, 0x71, 0x08, 0xff, - 0xa8, 0x01, 0xb5, 0xe1, 0x71, 0x07, 0x1f, 0x5b, 0x27, 0xfd, 0xcf, 0xfb, 0x7b, 0xb8, 0x91, 0x43, - 0xd7, 0x60, 0x5d, 0x21, 0x9f, 0x1d, 0xe1, 0xc7, 0x07, 0x47, 0x9d, 0xdd, 0x61, 0xc3, 0x40, 0x9b, - 0x70, 0x53, 0x81, 0x4f, 0xf6, 0x8e, 0x71, 0xbf, 0x67, 0xe1, 0xbd, 0xde, 0x11, 0xde, 0xdd, 0xc3, - 0xc3, 0x46, 0x1e, 0xad, 0x43, 0x75, 0x78, 0x7c, 0x34, 0x48, 0x3c, 0x14, 0x10, 0x82, 0xba, 0x04, - 0x66, 0x0e, 0x8a, 0xe8, 0x16, 0xdc, 0x90, 0xd8, 0x15, 0xfb, 0x15, 0x54, 0x82, 0x02, 0xfe, 0xf4, - 0xb0, 0xb1, 0x8a, 0x00, 0x56, 0xbb, 0x9f, 0xe2, 0xc3, 0xfe, 0x61, 0xa3, 0xd4, 0xed, 0xbe, 0x78, - 0xd9, 0xca, 0x7d, 0xf9, 0xb2, 0x95, 0xfb, 0xea, 0x65, 0xcb, 0xf8, 0xd5, 0xb4, 0x65, 0xfc, 0x69, - 0xda, 0x32, 0xfe, 0x36, 0x6d, 0x19, 0x2f, 0xa6, 0x2d, 0xe3, 0x1f, 0xd3, 0x96, 0xf1, 0xcf, 0x69, - 0x2b, 0xf7, 0xd5, 0xb4, 0x65, 0xfc, 0xee, 0x55, 0x2b, 0xf7, 0xe2, 0x55, 0x2b, 0xf7, 0xe5, 0xab, - 0x56, 0xee, 0xf3, 0x5a, 0xf6, 0xaf, 0x84, 0xd3, 0x55, 0x99, 0x9b, 0x0f, 0xff, 0x13, 0x00, 0x00, - 0xff, 0xff, 0x11, 0xaf, 0xeb, 0x55, 0x78, 0x10, 0x00, 0x00, + // 1822 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0xcb, 0x73, 0x1b, 0x49, + 0x19, 0xd7, 0x48, 0xb2, 0x25, 0x7d, 0x7a, 0x58, 0xee, 0x3c, 0x50, 0xbc, 0x29, 0x39, 0xab, 0x2d, + 0x20, 0x84, 0x5d, 0x8b, 0x64, 0x79, 0x98, 0xcd, 0xb2, 0x55, 0x92, 0x6c, 0xb0, 0x12, 0x27, 0x16, + 0x2d, 0xaf, 0x17, 0xb6, 0xa8, 0x9a, 0x1a, 0xcf, 0xb4, 0xa5, 0x29, 0xcf, 0x2b, 0x33, 0xad, 0xac, + 0xcc, 0x89, 0x0b, 0xc5, 0x89, 0x2a, 0x0e, 0xf0, 0x1f, 0x70, 0xe0, 0x4f, 0xe0, 0x48, 0x15, 0x07, + 0xe0, 0x96, 0xe3, 0x9e, 0x5c, 0x44, 0xb9, 0x70, 0xdc, 0xff, 0x00, 0xaa, 0xbf, 0xee, 0x19, 0x8d, + 0x64, 0x25, 0x81, 0x2a, 0x6e, 0x3d, 0xbf, 0xfe, 0x7d, 0x8f, 0xfe, 0xfa, 0x7b, 0xb4, 0x04, 0xdf, + 0x8d, 0x42, 0xb3, 0xcd, 0x1e, 0x30, 0x9d, 0xb3, 0x88, 0xb7, 0x03, 0x16, 0x9e, 0xe9, 0xdc, 0xf7, + 0x9d, 0x36, 0x9b, 0x06, 0x2c, 0xb4, 0x5d, 0xe6, 0xf1, 0xe0, 0x34, 0xf5, 0xb1, 0x13, 0x84, 0x3e, + 0xf7, 0x49, 0x25, 0x98, 0xee, 0x24, 0xdc, 0xad, 0xe6, 0xc8, 0xf7, 0x47, 0x0e, 0x6b, 0xe3, 0xde, + 0xe9, 0xe4, 0xac, 0x6d, 0x4d, 0x42, 0x83, 0xdb, 0xbe, 0x27, 0xd9, 0x5b, 0xd7, 0x47, 0xfe, 0xc8, + 0xc7, 0x65, 0x5b, 0xac, 0x24, 0xda, 0xfa, 0x77, 0x16, 0x6a, 0xfb, 0x89, 0xe2, 0x61, 0xc0, 0x4c, + 0xf2, 0x10, 0xca, 0xcf, 0xed, 0x5f, 0xda, 0x2c, 0xd4, 0xa3, 0x80, 0x99, 0x0d, 0xed, 0x8e, 0x76, + 0xb7, 0xfc, 0x60, 0x6b, 0x27, 0x6d, 0x6c, 0xe7, 0x33, 0x3f, 0x3c, 0x77, 0x7c, 0xc3, 0x12, 0x02, + 0x14, 0x24, 0x1d, 0x85, 0x3b, 0x50, 0xfb, 0x42, 0xed, 0xa1, 0x78, 0xd4, 0xc8, 0xde, 0xc9, 0xbd, + 0x45, 0xbe, 0xfa, 0x45, 0xea, 0x2b, 0x22, 0x0f, 0xa1, 0xe2, 0x32, 0x1e, 0xda, 0xa6, 0x52, 0x90, + 0x43, 0x05, 0x8d, 0x45, 0x05, 0x4f, 0x90, 0x81, 0xe2, 0x65, 0x37, 0x59, 0x47, 0xe4, 0x63, 0xa8, + 0x98, 0xce, 0x24, 0xe2, 0xb1, 0xf7, 0x79, 0xf4, 0xfe, 0xd6, 0xa2, 0x70, 0x4f, 0x32, 0xa4, 0xb4, + 0x39, 0xff, 0x20, 0xdf, 0x81, 0x62, 0x38, 0xf1, 0xa4, 0xe4, 0x1a, 0x4a, 0xde, 0x58, 0x94, 0xa4, + 0x13, 0x0f, 0xa5, 0x0a, 0xa1, 0x5c, 0x90, 0xf7, 0x01, 0x4c, 0xdf, 0x75, 0x6d, 0xae, 0x47, 0x63, + 0xa3, 0xb1, 0x7e, 0x47, 0xbb, 0x5b, 0xea, 0x56, 0x67, 0x97, 0xdb, 0xa5, 0x1e, 0xa2, 0xc3, 0x83, + 0x0e, 0x2d, 0x49, 0xc2, 0x70, 0x6c, 0x10, 0x02, 0x79, 0x6e, 0x8c, 0xa2, 0x46, 0xe1, 0x4e, 0xee, + 0x6e, 0x89, 0xe2, 0xba, 0xf5, 0x37, 0x0d, 0x2a, 0xe9, 0x70, 0x08, 0x92, 0x67, 0xb8, 0x0c, 0x03, + 0x5f, 0xa2, 0xb8, 0x16, 0x31, 0xb1, 0x58, 0xe0, 0xf8, 0x17, 0x7a, 0xc4, 0x59, 0x10, 0x07, 0x75, + 0x29, 0x26, 0x7b, 0xc8, 0x18, 0x72, 0x16, 0xd0, 0xb2, 0x95, 0xac, 0x23, 0xf2, 0x23, 0xa8, 0x8c, + 0x99, 0xe1, 0xf0, 0xb1, 0x39, 0x66, 0xe6, 0x79, 0x1c, 0xd0, 0xa5, 0x98, 0x1c, 0x20, 0xa3, 0x27, + 0x18, 0x74, 0x81, 0x4e, 0xbe, 0x09, 0x1b, 0x86, 0x29, 0x12, 0x49, 0x8f, 0x98, 0xc3, 0x4c, 0xee, + 0x87, 0x18, 0xd5, 0x12, 0xad, 0x49, 0x78, 0xa8, 0xd0, 0xd6, 0x3f, 0x34, 0x80, 0xb9, 0x0f, 0xa4, + 0x07, 0xe5, 0x20, 0x64, 0x21, 0xf3, 0x2c, 0x16, 0x32, 0x4b, 0xe5, 0xd1, 0xf6, 0xa2, 0xd5, 0xc1, + 0x9c, 0x20, 0x25, 0x0f, 0x32, 0x34, 0x2d, 0x45, 0x3e, 0x82, 0x62, 0x74, 0x6e, 0x9c, 0x9d, 0xf9, + 0x8e, 0xd5, 0xc8, 0xa2, 0x86, 0xdb, 0x8b, 0x1a, 0x86, 0x6a, 0x37, 0x11, 0x4f, 0xf8, 0xe4, 0xdb, + 0x90, 0x0d, 0xa6, 0x8d, 0xdc, 0xaa, 0x0c, 0x18, 0x4c, 0x7b, 0x87, 0xfd, 0x44, 0x24, 0x1b, 0x4c, + 0xbb, 0x55, 0x50, 0x31, 0xd3, 0xf9, 0x45, 0xc0, 0x5a, 0xbf, 0xd7, 0xa0, 0x9c, 0x0a, 0x09, 0xf9, + 0x18, 0x72, 0xe7, 0xbb, 0xd1, 0xea, 0x43, 0x3c, 0xde, 0x1d, 0x0e, 0x7c, 0x2b, 0xa2, 0xcc, 0xb0, + 0x2e, 0x90, 0xdd, 0x2d, 0xcc, 0x2e, 0xb7, 0x73, 0x8f, 0x77, 0x87, 0x07, 0x19, 0x2a, 0xc4, 0xc8, + 0x0f, 0x21, 0x17, 0x4c, 0x9d, 0xd5, 0x07, 0x18, 0x4c, 0x0f, 0x53, 0x86, 0xa4, 0xa8, 0xc0, 0x32, + 0x54, 0xc8, 0x74, 0x2b, 0x00, 0x78, 0x0f, 0xd2, 0xad, 0xfb, 0xb0, 0x79, 0xc5, 0x1a, 0xb9, 0x0d, + 0x25, 0x91, 0x24, 0x51, 0x60, 0x98, 0x71, 0xd6, 0xcc, 0x81, 0xd6, 0x11, 0xd4, 0x16, 0x4d, 0x90, + 0x9b, 0xb0, 0x1e, 0x99, 0xa1, 0x1d, 0x70, 0x45, 0x56, 0x5f, 0xe4, 0xeb, 0x50, 0x8b, 0x26, 0xa6, + 0xc9, 0xa2, 0x48, 0x37, 0x7d, 0x67, 0xe2, 0x7a, 0xe8, 0x70, 0x89, 0x56, 0x15, 0xda, 0x43, 0xb0, + 0xf5, 0x0b, 0x28, 0x0d, 0x0c, 0x6e, 0x8e, 0x31, 0x59, 0x6f, 0x43, 0xfe, 0xc2, 0x70, 0x1d, 0xa9, + 0xa9, 0x5b, 0x9c, 0x5d, 0x6e, 0xe7, 0x7f, 0xde, 0x79, 0x72, 0x48, 0x11, 0x25, 0xf7, 0x61, 0x9d, + 0x1b, 0xe1, 0x88, 0x71, 0x75, 0xf4, 0xe5, 0x5b, 0x10, 0x6a, 0x8e, 0x91, 0x40, 0x15, 0xb1, 0xf5, + 0x9b, 0x2c, 0x94, 0x53, 0x38, 0xf9, 0x16, 0x94, 0x8c, 0xc0, 0xd6, 0x47, 0xa1, 0x3f, 0x09, 0x94, + 0x95, 0xca, 0xec, 0x72, 0xbb, 0xd8, 0x19, 0xf4, 0x7f, 0x22, 0x30, 0x5a, 0x34, 0x02, 0x1b, 0x57, + 0xa4, 0x0d, 0x65, 0x41, 0x7d, 0xce, 0xc2, 0xc8, 0xf6, 0x95, 0xf3, 0xdd, 0xda, 0xec, 0x72, 0x1b, + 0x3a, 0x83, 0xfe, 0x89, 0x44, 0x29, 0x18, 0x81, 0xad, 0xd6, 0xa2, 0xd2, 0xce, 0x6d, 0xcf, 0xc2, + 0x14, 0x29, 0x51, 0x5c, 0x27, 0xd5, 0x97, 0x4f, 0x55, 0xdf, 0x42, 0x80, 0xd7, 0x96, 0x02, 0x2c, + 0xc2, 0xe6, 0x18, 0xa7, 0xcc, 0x99, 0x97, 0xc7, 0xba, 0x0c, 0x1b, 0xa2, 0x71, 0x75, 0x90, 0x36, + 0x5c, 0x33, 0x3c, 0xcf, 0xe7, 0xc6, 0x62, 0x29, 0x15, 0x90, 0x4b, 0xe6, 0x5b, 0x49, 0x39, 0x71, + 0xd8, 0xbc, 0x52, 0x1e, 0xa2, 0xdf, 0x88, 0xc8, 0xea, 0x81, 0xc1, 0xc7, 0x22, 0x1d, 0x73, 0x71, + 0xbf, 0x11, 0x51, 0x1f, 0x08, 0x90, 0x96, 0x04, 0x01, 0x97, 0xe4, 0x3e, 0x14, 0x02, 0x11, 0x4b, + 0x16, 0x77, 0x8c, 0xaf, 0xad, 0xb8, 0x00, 0xd9, 0xd0, 0x14, 0xaf, 0xf5, 0x5b, 0x0d, 0x6a, 0x8b, + 0x35, 0x45, 0xde, 0x83, 0x6a, 0x5c, 0x53, 0x68, 0x57, 0xa5, 0x4d, 0x25, 0x06, 0x85, 0xad, 0x05, + 0x92, 0x11, 0x8e, 0xa4, 0xc1, 0x14, 0xa9, 0x13, 0x8e, 0x16, 0xfc, 0xc9, 0xfd, 0x97, 0xfe, 0x5c, + 0x40, 0x39, 0x55, 0xac, 0xe2, 0x7a, 0x50, 0xbb, 0x26, 0x3b, 0xa8, 0x58, 0x93, 0x26, 0x40, 0x72, + 0x1b, 0xb1, 0xdd, 0x14, 0x42, 0xbe, 0x0f, 0xb5, 0x88, 0x71, 0x3d, 0x9e, 0x0b, 0xb6, 0xbc, 0xf0, + 0x62, 0xb7, 0x3e, 0xbb, 0xdc, 0xae, 0x0c, 0x19, 0x57, 0xe3, 0xa0, 0xbf, 0x47, 0x2b, 0xd1, 0xfc, + 0xcb, 0x6a, 0xfd, 0x59, 0x03, 0x98, 0xcf, 0x19, 0xb2, 0x2b, 0x8b, 0x58, 0xb6, 0x80, 0x77, 0xae, + 0x14, 0xf1, 0x10, 0x8b, 0x48, 0x30, 0x97, 0x6b, 0x98, 0xec, 0x42, 0x3e, 0x08, 0x7d, 0x57, 0x15, + 0x41, 0x6b, 0xb9, 0x05, 0xfa, 0x2e, 0xe3, 0x63, 0x36, 0x89, 0x86, 0x66, 0x68, 0x04, 0x4c, 0x68, + 0x38, 0xc8, 0x50, 0x94, 0x58, 0xd5, 0x7b, 0xad, 0x55, 0xbd, 0x57, 0xb4, 0x2f, 0x35, 0x34, 0xb1, + 0x4f, 0xcc, 0x72, 0x50, 0x5d, 0xf0, 0xe9, 0xb5, 0x45, 0x7f, 0x1b, 0x4a, 0x11, 0x0f, 0x99, 0xe1, + 0xda, 0xde, 0x08, 0x1d, 0x2c, 0xd2, 0x39, 0x40, 0x7e, 0x0c, 0x9b, 0xa6, 0xef, 0x08, 0x1b, 0xc2, + 0x07, 0xf1, 0x4c, 0xf0, 0xad, 0xa4, 0xa3, 0xca, 0x07, 0xc7, 0x4e, 0xfc, 0xe0, 0xd8, 0xd9, 0x53, + 0x0f, 0x0e, 0x5a, 0x9f, 0xcb, 0x0c, 0x50, 0x84, 0xfc, 0x0c, 0x36, 0x38, 0x73, 0x03, 0xc7, 0xe0, + 0x4c, 0x7f, 0x6e, 0x38, 0x13, 0x16, 0x35, 0xf2, 0x98, 0x00, 0xed, 0x37, 0xc4, 0x71, 0xe7, 0x58, + 0x89, 0x9c, 0xa0, 0xc4, 0xbe, 0xc7, 0xc3, 0x0b, 0x5a, 0xe3, 0x0b, 0x20, 0xa1, 0x50, 0xe5, 0xc6, + 0xa9, 0xc3, 0x74, 0x7f, 0xc2, 0x83, 0x09, 0x8f, 0x1a, 0x6b, 0xa8, 0xf7, 0x83, 0x37, 0xea, 0x15, + 0x02, 0x47, 0x92, 0x2f, 0xb5, 0x56, 0x78, 0x0a, 0xda, 0xea, 0xc0, 0xb5, 0x15, 0xa6, 0x49, 0x1d, + 0x72, 0xe7, 0xec, 0x42, 0xc5, 0x4f, 0x2c, 0xc9, 0x75, 0x58, 0xc3, 0xd3, 0xa8, 0x46, 0x29, 0x3f, + 0x3e, 0xca, 0xee, 0x6a, 0x5b, 0xa7, 0xb0, 0x79, 0xc5, 0xca, 0x0a, 0x05, 0x3f, 0x48, 0x2b, 0x28, + 0x3f, 0x78, 0xf7, 0x35, 0x5e, 0x4b, 0x2d, 0x87, 0x76, 0xc4, 0x53, 0x36, 0x5a, 0x14, 0xae, 0xad, + 0x60, 0x90, 0x87, 0x50, 0x88, 0x63, 0xa1, 0x61, 0x2c, 0xde, 0xac, 0x55, 0x96, 0x9b, 0x92, 0x68, + 0xfd, 0x55, 0xbb, 0xa2, 0x14, 0xd3, 0xe7, 0x11, 0x54, 0x23, 0xdb, 0x1b, 0x39, 0x4c, 0x97, 0x69, + 0xa6, 0xca, 0xe0, 0xbd, 0xa5, 0x61, 0x8c, 0x14, 0x59, 0x33, 0x83, 0xe9, 0xa1, 0x94, 0x3f, 0xc8, + 0xd0, 0x4a, 0x94, 0xda, 0x20, 0x3f, 0x85, 0x4d, 0xcb, 0xe0, 0x86, 0xee, 0xf8, 0x38, 0x69, 0x26, + 0x1e, 0x67, 0xa1, 0x0a, 0xc0, 0x92, 0xbe, 0x3d, 0x83, 0x1b, 0x87, 0xbe, 0x98, 0x3c, 0x48, 0x4a, + 0xf4, 0x6d, 0x58, 0x8b, 0x1b, 0x22, 0xfd, 0xe5, 0x09, 0xf0, 0xed, 0xd6, 0xfa, 0x83, 0x06, 0x37, + 0x56, 0xfa, 0x22, 0xda, 0x14, 0xb7, 0x5d, 0x16, 0x71, 0xc3, 0x0d, 0xc4, 0x94, 0x8b, 0x7b, 0x59, + 0x02, 0xf6, 0x7c, 0x87, 0x6c, 0x27, 0xc5, 0x84, 0xa3, 0x40, 0x5e, 0x2e, 0x48, 0xe8, 0xa9, 0x18, + 0x08, 0xef, 0x40, 0x09, 0xaf, 0x01, 0x35, 0xc8, 0xe9, 0x51, 0x44, 0x40, 0x48, 0xdf, 0x82, 0x22, + 0x37, 0x46, 0x62, 0x4b, 0x26, 0x79, 0x89, 0x16, 0xb8, 0x31, 0xea, 0xf9, 0x4e, 0x24, 0x5e, 0x48, + 0x37, 0x56, 0x9e, 0xe9, 0xff, 0xe4, 0xd7, 0x3d, 0x80, 0x88, 0x3d, 0xd3, 0x6d, 0x6b, 0xee, 0x98, + 0x9c, 0x96, 0x43, 0xf6, 0xac, 0xbf, 0xd7, 0xf3, 0x1d, 0x5a, 0x8c, 0xd8, 0xb3, 0xbe, 0x25, 0x94, + 0x7d, 0x02, 0x55, 0x15, 0x32, 0x55, 0xd6, 0xf9, 0xb7, 0x95, 0x75, 0x45, 0xf2, 0x65, 0x49, 0xb7, + 0xfe, 0x92, 0x83, 0xeb, 0xab, 0x7a, 0xd7, 0x9b, 0x9f, 0x23, 0xe4, 0x1b, 0xb0, 0xe1, 0x8a, 0xd6, + 0xae, 0xcb, 0x99, 0x29, 0xea, 0x41, 0xbd, 0x32, 0x10, 0x3e, 0x14, 0xe8, 0x63, 0x76, 0x41, 0xee, + 0xc1, 0x66, 0x9a, 0x27, 0xab, 0x44, 0x86, 0x7a, 0x63, 0xce, 0xc4, 0xf2, 0x14, 0x43, 0x21, 0xf0, + 0x43, 0x8e, 0x27, 0x58, 0xa3, 0xb8, 0x16, 0xc7, 0x8b, 0xd0, 0xa7, 0xf8, 0x78, 0x6b, 0x6f, 0x3d, + 0x9e, 0xe4, 0xab, 0x8e, 0x75, 0x92, 0xfc, 0x0a, 0x41, 0xdf, 0x1b, 0xeb, 0x58, 0x4a, 0x1f, 0xbe, + 0xbd, 0x77, 0xab, 0x9f, 0x26, 0xe2, 0x3e, 0x54, 0x73, 0x29, 0xcf, 0x6f, 0x08, 0x5f, 0xd3, 0xe7, + 0x93, 0x53, 0x66, 0xfa, 0xde, 0x99, 0x3d, 0x92, 0xe3, 0x54, 0x3e, 0x01, 0x6a, 0x73, 0x18, 0x07, + 0xea, 0xbb, 0x50, 0x11, 0x88, 0x6e, 0xfa, 0x1e, 0x67, 0x53, 0xde, 0x28, 0x22, 0xab, 0x2c, 0xb0, + 0x9e, 0x84, 0xb6, 0x3e, 0x81, 0xfa, 0xb2, 0xb1, 0xff, 0xa5, 0x49, 0xb5, 0x4e, 0xa0, 0x9c, 0xfa, + 0x29, 0x24, 0xb2, 0xda, 0x9b, 0xb8, 0xba, 0xe7, 0x5b, 0x4c, 0xbe, 0x74, 0xd7, 0x68, 0xd1, 0x9b, + 0xb8, 0x4f, 0xc5, 0x37, 0xb9, 0x07, 0x79, 0xb1, 0xa1, 0xea, 0xf4, 0xe6, 0x62, 0x1c, 0x04, 0x05, + 0xfb, 0x08, 0x72, 0x5a, 0x1f, 0x40, 0x31, 0x46, 0xc4, 0x31, 0x5c, 0xc3, 0x1c, 0xdb, 0x1e, 0xc3, + 0xc9, 0xa4, 0x1c, 0x2b, 0x2b, 0xec, 0x58, 0x0c, 0xab, 0x3e, 0x14, 0xd4, 0xef, 0x2a, 0xf2, 0x00, + 0x0a, 0x72, 0xb0, 0xbd, 0xe6, 0x67, 0x5f, 0x47, 0x4e, 0x3d, 0x6c, 0x59, 0x8a, 0xf8, 0x28, 0x5f, + 0xd4, 0xea, 0xd9, 0x47, 0xf9, 0x62, 0xb6, 0x9e, 0x6b, 0xfd, 0x5a, 0x03, 0x98, 0x73, 0xc8, 0xfb, + 0x90, 0x4f, 0x8c, 0xd6, 0x56, 0xeb, 0x12, 0x1e, 0x50, 0x64, 0x91, 0xef, 0x41, 0x31, 0xfe, 0xcd, + 0x9c, 0xbc, 0x57, 0x5f, 0x9b, 0x2d, 0x09, 0x35, 0x79, 0x31, 0xe6, 0xe6, 0x2f, 0xc6, 0x7b, 0x7f, + 0x4c, 0xfc, 0x10, 0xfa, 0x49, 0x1d, 0x2a, 0xc3, 0xe3, 0x0e, 0x3d, 0xd6, 0x4f, 0xfa, 0x9f, 0xf7, + 0xf7, 0x69, 0x3d, 0x43, 0xae, 0xc1, 0x86, 0x44, 0x3e, 0x3b, 0xa2, 0x8f, 0x0f, 0x8f, 0x3a, 0x7b, + 0xc3, 0xba, 0x46, 0xb6, 0xe0, 0xa6, 0x04, 0x9f, 0xec, 0x1f, 0xd3, 0x7e, 0x4f, 0xa7, 0xfb, 0xbd, + 0x23, 0xba, 0xb7, 0x4f, 0x87, 0xf5, 0x2c, 0xd9, 0x80, 0xf2, 0xf0, 0xf8, 0x68, 0x10, 0x6b, 0xc8, + 0x11, 0x02, 0x35, 0x04, 0xe6, 0x0a, 0xf2, 0xe4, 0x16, 0xdc, 0x40, 0xec, 0x8a, 0xfc, 0x1a, 0x29, + 0x40, 0x8e, 0x7e, 0xfa, 0xb4, 0xbe, 0x4e, 0x00, 0xd6, 0xbb, 0x9f, 0xd2, 0xa7, 0xfd, 0xa7, 0xf5, + 0x42, 0xb7, 0xfb, 0xe2, 0x65, 0x33, 0xf3, 0xe5, 0xcb, 0x66, 0xe6, 0xab, 0x97, 0x4d, 0xed, 0x57, + 0xb3, 0xa6, 0xf6, 0xa7, 0x59, 0x53, 0xfb, 0xfb, 0xac, 0xa9, 0xbd, 0x98, 0x35, 0xb5, 0x7f, 0xce, + 0x9a, 0xda, 0xbf, 0x66, 0xcd, 0xcc, 0x57, 0xb3, 0xa6, 0xf6, 0xbb, 0x57, 0xcd, 0xcc, 0x8b, 0x57, + 0xcd, 0xcc, 0x97, 0xaf, 0x9a, 0x99, 0xcf, 0x2b, 0xe9, 0xbf, 0x25, 0x4e, 0xd7, 0x31, 0x36, 0x1f, + 0xfe, 0x27, 0x00, 0x00, 0xff, 0xff, 0xf1, 0xcb, 0x88, 0xa6, 0xc4, 0x10, 0x00, 0x00, } func (x ActionType) String() string { @@ -2546,6 +2564,12 @@ func (this *PrometheusScrapeSpec) Equal(that interface{}) bool { return false } } + if this.KubeconfigPath != that1.KubeconfigPath { + return false + } + if this.KubeContext != that1.KubeContext { + return false + } return true } func (this *ClusterSpec) Equal(that interface{}) bool { @@ -2995,7 +3019,7 @@ func (this *PrometheusScrapeSpec) GoString() string { if this == nil { return "nil" } - s := make([]string, 0, 10) + s := make([]string, 0, 12) s = append(s, "&experimentpb.PrometheusScrapeSpec{") s = append(s, "Namespace: "+fmt.Sprintf("%#v", this.Namespace)+",\n") s = append(s, "MatchLabelKey: "+fmt.Sprintf("%#v", this.MatchLabelKey)+",\n") @@ -3017,6 +3041,8 @@ func (this *PrometheusScrapeSpec) GoString() string { if this.MetricNames != nil { s = append(s, "MetricNames: "+mapStringForMetricNames+",\n") } + s = append(s, "KubeconfigPath: "+fmt.Sprintf("%#v", this.KubeconfigPath)+",\n") + s = append(s, "KubeContext: "+fmt.Sprintf("%#v", this.KubeContext)+",\n") s = append(s, "}") return strings.Join(s, "") } @@ -4165,6 +4191,20 @@ func (m *PrometheusScrapeSpec) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if len(m.KubeContext) > 0 { + i -= len(m.KubeContext) + copy(dAtA[i:], m.KubeContext) + i = encodeVarintExperiment(dAtA, i, uint64(len(m.KubeContext))) + i-- + dAtA[i] = 0x42 + } + if len(m.KubeconfigPath) > 0 { + i -= len(m.KubeconfigPath) + copy(dAtA[i:], m.KubeconfigPath) + i = encodeVarintExperiment(dAtA, i, uint64(len(m.KubeconfigPath))) + i-- + dAtA[i] = 0x3a + } if len(m.MetricNames) > 0 { for k := range m.MetricNames { v := m.MetricNames[k] @@ -4917,6 +4957,14 @@ func (m *PrometheusScrapeSpec) Size() (n int) { n += mapEntrySize + 1 + sovExperiment(uint64(mapEntrySize)) } } + l = len(m.KubeconfigPath) + if l > 0 { + n += 1 + l + sovExperiment(uint64(l)) + } + l = len(m.KubeContext) + if l > 0 { + n += 1 + l + sovExperiment(uint64(l)) + } return n } @@ -5359,6 +5407,8 @@ func (this *PrometheusScrapeSpec) String() string { `Port:` + fmt.Sprintf("%v", this.Port) + `,`, `ScrapePeriod:` + strings.Replace(fmt.Sprintf("%v", this.ScrapePeriod), "Duration", "types.Duration", 1) + `,`, `MetricNames:` + mapStringForMetricNames + `,`, + `KubeconfigPath:` + fmt.Sprintf("%v", this.KubeconfigPath) + `,`, + `KubeContext:` + fmt.Sprintf("%v", this.KubeContext) + `,`, `}`, }, "") return s @@ -8569,6 +8619,70 @@ func (m *PrometheusScrapeSpec) Unmarshal(dAtA []byte) error { } m.MetricNames[mapkey] = mapvalue iNdEx = postIndex + case 7: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field KubeconfigPath", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowExperiment + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthExperiment + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthExperiment + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.KubeconfigPath = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 8: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field KubeContext", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowExperiment + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthExperiment + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthExperiment + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.KubeContext = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipExperiment(dAtA[iNdEx:]) diff --git a/src/e2e_test/perf_tool/experimentpb/experiment.proto b/src/e2e_test/perf_tool/experimentpb/experiment.proto index d5482d5d249..4d3a908d51d 100644 --- a/src/e2e_test/perf_tool/experimentpb/experiment.proto +++ b/src/e2e_test/perf_tool/experimentpb/experiment.proto @@ -220,6 +220,12 @@ message PrometheusScrapeSpec { // How often to scrape the matched pods. google.protobuf.Duration scrape_period = 5; map metric_names = 6; + // Optional path to a kubeconfig file for connecting to a different cluster. + // If empty, the experiment's default cluster context is used. + string kubeconfig_path = 7; + // Optional kubectl context name to use within the kubeconfig. + // If empty, the current-context from the kubeconfig is used. + string kube_context = 8; } // ClusterSpec specifies the type and size of cluster an experiment should run on. diff --git a/src/e2e_test/perf_tool/pkg/cluster/context.go b/src/e2e_test/perf_tool/pkg/cluster/context.go index bd79bf433f3..c274a6726b0 100644 --- a/src/e2e_test/perf_tool/pkg/cluster/context.go +++ b/src/e2e_test/perf_tool/pkg/cluster/context.go @@ -53,6 +53,36 @@ func NewContextFromPath(kubeconfigPath string) (*Context, error) { }, nil } +// NewContextFromOptions creates a new Context using the specified kubeconfig path and/or context name. +// If kubeconfigPath is empty, the default kubeconfig path is used. +// If kubeContext is empty, the current-context from the kubeconfig is used. +func NewContextFromOptions(kubeconfigPath string, kubeContext string) (*Context, error) { + loadingRules := &clientcmd.ClientConfigLoadingRules{} + if kubeconfigPath != "" { + loadingRules.ExplicitPath = kubeconfigPath + } else { + loadingRules = clientcmd.NewDefaultClientConfigLoadingRules() + } + overrides := &clientcmd.ConfigOverrides{} + if kubeContext != "" { + overrides.CurrentContext = kubeContext + } + config := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, overrides) + restConfig, err := config.ClientConfig() + if err != nil { + return nil, err + } + if kubeconfigPath == "" { + kubeconfigPath = clientcmd.RecommendedHomeFile + } + clientset := k8s.GetClientset(restConfig) + return &Context{ + configPath: kubeconfigPath, + restConfig: restConfig, + clientset: clientset, + }, nil +} + // NewContextFromConfig writes the given kubeconfig to a file, and the returns NewContextFromPath for that file. func NewContextFromConfig(kubeconfig []byte) (*Context, error) { tmpFile, err := os.CreateTemp("", "*") diff --git a/src/e2e_test/perf_tool/pkg/metrics/prometheus_recorder.go b/src/e2e_test/perf_tool/pkg/metrics/prometheus_recorder.go index 19d08b1b0a9..8e5c1768e24 100644 --- a/src/e2e_test/perf_tool/pkg/metrics/prometheus_recorder.go +++ b/src/e2e_test/perf_tool/pkg/metrics/prometheus_recorder.go @@ -43,10 +43,11 @@ import ( ) type prometheusRecorderImpl struct { - clusterCtx *cluster.Context - spec *experimentpb.PrometheusScrapeSpec - eg *errgroup.Group - resultCh chan<- *ResultRow + clusterCtx *cluster.Context + ownsClusterCtx bool + spec *experimentpb.PrometheusScrapeSpec + eg *errgroup.Group + resultCh chan<- *ResultRow wg sync.WaitGroup stopCh chan struct{} @@ -79,6 +80,9 @@ func (r *prometheusRecorderImpl) Close() { for _, fw := range r.fws { fw.Close() } + if r.ownsClusterCtx { + r.clusterCtx.Close() + } } func (r *prometheusRecorderImpl) run() error { diff --git a/src/e2e_test/perf_tool/pkg/metrics/recorder.go b/src/e2e_test/perf_tool/pkg/metrics/recorder.go index 7e7e44e06e2..12bdf8fd502 100644 --- a/src/e2e_test/perf_tool/pkg/metrics/recorder.go +++ b/src/e2e_test/perf_tool/pkg/metrics/recorder.go @@ -20,6 +20,7 @@ package metrics import ( "context" + "fmt" "golang.org/x/sync/errgroup" @@ -35,7 +36,7 @@ type Recorder interface { } // NewMetricsRecorder creates a new Recorder for the given MetricSpec. -func NewMetricsRecorder(pxCtx *pixie.Context, clusterCtx *cluster.Context, spec *experimentpb.MetricSpec, eg *errgroup.Group, resultCh chan<- *ResultRow) Recorder { +func NewMetricsRecorder(pxCtx *pixie.Context, clusterCtx *cluster.Context, spec *experimentpb.MetricSpec, eg *errgroup.Group, resultCh chan<- *ResultRow) (Recorder, error) { switch spec.MetricType.(type) { case *experimentpb.MetricSpec_PxL: return &pxlScriptRecorderImpl{ @@ -44,14 +45,26 @@ func NewMetricsRecorder(pxCtx *pixie.Context, clusterCtx *cluster.Context, spec eg: eg, resultCh: resultCh, - } + }, nil case *experimentpb.MetricSpec_Prom: - return &prometheusRecorderImpl{ - clusterCtx: clusterCtx, - spec: spec.GetProm(), - eg: eg, - resultCh: resultCh, + promSpec := spec.GetProm() + recorderCtx := clusterCtx + ownsCtx := false + if promSpec.KubeconfigPath != "" || promSpec.KubeContext != "" { + var err error + recorderCtx, err = cluster.NewContextFromOptions(promSpec.KubeconfigPath, promSpec.KubeContext) + if err != nil { + return nil, fmt.Errorf("failed to create cluster context for prometheus recorder: %w", err) + } + ownsCtx = true } + return &prometheusRecorderImpl{ + clusterCtx: recorderCtx, + ownsClusterCtx: ownsCtx, + spec: promSpec, + eg: eg, + resultCh: resultCh, + }, nil } - return nil + return nil, nil } diff --git a/src/e2e_test/perf_tool/pkg/run/run.go b/src/e2e_test/perf_tool/pkg/run/run.go index a7ae2ca9c52..293a04e0da1 100644 --- a/src/e2e_test/perf_tool/pkg/run/run.go +++ b/src/e2e_test/perf_tool/pkg/run/run.go @@ -225,7 +225,11 @@ func (r *Runner) startMetricRecorders(ctx context.Context, spec *experimentpb.Ex continue } - recorder := metrics.NewMetricsRecorder(r.pxCtx, r.clusterCtx, ms, r.eg, r.metricsResultCh) + recorder, err := metrics.NewMetricsRecorder(r.pxCtx, r.clusterCtx, ms, r.eg, r.metricsResultCh) + if err != nil { + _ = r.stopMetricRecorders(selector) + return noCleanup, fmt.Errorf("failed to create metrics recorder: %w", err) + } r.metricsBySelector[selector] = append(r.metricsBySelector[selector], recorder) if err := recorder.Start(ctx); err != nil { _ = r.stopMetricRecorders(selector) From 5a8fb65ce97b66d4150411ddc23266bdfbff469e Mon Sep 17 00:00:00 2001 From: Dom Del Nano Date: Wed, 15 Apr 2026 23:11:13 -0700 Subject: [PATCH 04/10] Fix parquet file overflow bug Signed-off-by: Dom Del Nano --- src/e2e_test/perf_tool/pkg/exporter/parquet_exporter.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/e2e_test/perf_tool/pkg/exporter/parquet_exporter.go b/src/e2e_test/perf_tool/pkg/exporter/parquet_exporter.go index 6c289b25887..c5fe259e93a 100644 --- a/src/e2e_test/perf_tool/pkg/exporter/parquet_exporter.go +++ b/src/e2e_test/perf_tool/pkg/exporter/parquet_exporter.go @@ -247,7 +247,7 @@ func buildResultRow(row bufferedRow, tagKeys []string) parquet.Row { entries := []colEntry{ {"experiment_id", parquet.ValueOf(row.ExperimentID), false}, {"name", parquet.ValueOf(row.Name), false}, - {"timestamp", parquet.ValueOf(row.Timestamp), false}, + {"timestamp", parquet.Int64Value(row.Timestamp.UnixMilli()), false}, {"value", parquet.ValueOf(row.Value), false}, } From 17188d5919d288971f41f559e1f6f4beee146536 Mon Sep 17 00:00:00 2001 From: Dom Del Nano Date: Wed, 15 Apr 2026 23:15:48 -0700 Subject: [PATCH 05/10] Add duck db wasm visualization file Signed-off-by: Dom Del Nano --- src/e2e_test/perf_tool/ui/index.html | 1006 ++++++++++++++++++++++++++ 1 file changed, 1006 insertions(+) create mode 100644 src/e2e_test/perf_tool/ui/index.html diff --git a/src/e2e_test/perf_tool/ui/index.html b/src/e2e_test/perf_tool/ui/index.html new file mode 100644 index 00000000000..070a20eb2d1 --- /dev/null +++ b/src/e2e_test/perf_tool/ui/index.html @@ -0,0 +1,1006 @@ + + + + + + Pixie Perf Tool Dashboard + + + + +
+

Pixie Perf Tool Dashboard

+ DuckDB WASM + Parquet +
+ +
Initializing DuckDB...
+ +
+ +
+

Data Source

+
+
+
+

Drop parquet files here or click to browse

+

results_*.parquet and spec.parquet files

+ +
+
+
OR
+
+
+ + + + + + + +

+ Bucket must be publicly readable or have CORS configured. +

+
+
+
+
+
+ + + +
+ + + + From 63f7d5fbf52c623deff550b93e95fe9db84e8c35 Mon Sep 17 00:00:00 2001 From: Dom Del Nano Date: Thu, 16 Apr 2026 18:04:29 -0700 Subject: [PATCH 06/10] Temporary changes to make load testing easier Signed-off-by: Dom Del Nano --- src/e2e_test/perf_tool/pkg/run/run.go | 23 +++++----------- .../perf_tool/pkg/suites/experiments.go | 2 +- src/e2e_test/perf_tool/pkg/suites/suites.go | 2 +- .../perf_tool/pkg/suites/workloads.go | 26 +++++++++++++++++++ 4 files changed, 34 insertions(+), 19 deletions(-) diff --git a/src/e2e_test/perf_tool/pkg/run/run.go b/src/e2e_test/perf_tool/pkg/run/run.go index 293a04e0da1..e82db72c2c9 100644 --- a/src/e2e_test/perf_tool/pkg/run/run.go +++ b/src/e2e_test/perf_tool/pkg/run/run.go @@ -81,14 +81,12 @@ func (r *Runner) RunExperiment(ctx context.Context, expID uuid.UUID, spec *exper return err } - eg := errgroup.Group{} - eg.Go(func() error { return r.getCluster(ctx, spec.ClusterSpec) }) - eg.Go(func() error { - if err := r.prepareWorkloads(ctx, spec); err != nil { - return backoff.Permanent(err) - } - return nil - }) + if err := r.getCluster(ctx, spec.ClusterSpec); err != nil { + return err + } + if err := r.prepareWorkloads(ctx, spec); err != nil { + return err + } r.metricsBySelector = make(map[string][]metrics.Recorder) r.metricsResultCh = make(chan *metrics.ResultRow) @@ -103,15 +101,6 @@ func (r *Runner) RunExperiment(ctx context.Context, expID uuid.UUID, spec *exper } }() - if err := eg.Wait(); err != nil { - if r.clusterCleanup != nil { - r.clusterCleanup() - } - if r.clusterCtx != nil { - r.clusterCtx.Close() - } - return err - } defer r.clusterCleanup() defer r.clusterCtx.Close() diff --git a/src/e2e_test/perf_tool/pkg/suites/experiments.go b/src/e2e_test/perf_tool/pkg/suites/experiments.go index 998b31c7197..9ae11fd92ff 100644 --- a/src/e2e_test/perf_tool/pkg/suites/experiments.go +++ b/src/e2e_test/perf_tool/pkg/suites/experiments.go @@ -36,7 +36,7 @@ func HTTPLoadTestExperiment( dur time.Duration, ) *experimentpb.ExperimentSpec { e := &experimentpb.ExperimentSpec{ - VizierSpec: VizierWorkload(), + VizierSpec: VizierReleaseWorkload(), WorkloadSpecs: []*experimentpb.WorkloadSpec{ HTTPLoadTestWorkload(numConnections, targetRPS, true), }, diff --git a/src/e2e_test/perf_tool/pkg/suites/suites.go b/src/e2e_test/perf_tool/pkg/suites/suites.go index 4d5597ddf04..0c5e4b9a404 100644 --- a/src/e2e_test/perf_tool/pkg/suites/suites.go +++ b/src/e2e_test/perf_tool/pkg/suites/suites.go @@ -38,7 +38,7 @@ var ExperimentSuiteRegistry = map[string]ExperimentSuite{ func nightlyExperimentSuite() map[string]*pb.ExperimentSpec { defaultMetricPeriod := 30 * time.Second preDur := 5 * time.Minute - dur := 40 * time.Minute + dur := 5 * time.Minute httpNumConns := 100 exps := map[string]*pb.ExperimentSpec{ "http-loadtest/100/100": HTTPLoadTestExperiment(httpNumConns, 100, defaultMetricPeriod, preDur, dur), diff --git a/src/e2e_test/perf_tool/pkg/suites/workloads.go b/src/e2e_test/perf_tool/pkg/suites/workloads.go index e0679e5cfb8..c819b794649 100644 --- a/src/e2e_test/perf_tool/pkg/suites/workloads.go +++ b/src/e2e_test/perf_tool/pkg/suites/workloads.go @@ -30,6 +30,32 @@ import ( pb "px.dev/pixie/src/e2e_test/perf_tool/experimentpb" ) +// VizierReleaseWorkload returns the workload spec to deploy a released version of Vizier via `px deploy`. +// This skips the skaffold build step, using pre-built images from the Pixie release. +func VizierReleaseWorkload() *pb.WorkloadSpec { + return &pb.WorkloadSpec{ + Name: "vizier", + DeploySteps: []*pb.DeployStep{ + { + DeployType: &pb.DeployStep_Px{ + Px: &pb.PxCLIDeploy{ + Args: []string{ + "deploy", + }, + SetClusterID: true, + Namespaces: []string{ + "pl", + "px-operator", + "olm", + }, + }, + }, + }, + }, + Healthchecks: VizierHealthChecks(), + } +} + // VizierWorkload returns the workload spec to deploy Vizier. func VizierWorkload() *pb.WorkloadSpec { return &pb.WorkloadSpec{ From 839af021938e2e251fa4593a71b7fde0153f4f82 Mon Sep 17 00:00:00 2001 From: Dom Del Nano Date: Sun, 19 Apr 2026 20:56:33 -0700 Subject: [PATCH 07/10] Add clickhouse perf_tool suite, ability to query cross kubeconfig/kube context prometheus backends and test out the write clickhouse experiment Signed-off-by: Dom Del Nano --- src/e2e_test/perf_tool/cmd/run.go | 70 ++++- .../perf_tool/experimentpb/experiment.pb.go | 287 +++++++++++------- .../perf_tool/experimentpb/experiment.proto | 3 + src/e2e_test/perf_tool/pkg/run/run.go | 43 ++- src/e2e_test/perf_tool/pkg/suites/BUILD.bazel | 2 + .../perf_tool/pkg/suites/experiments.go | 126 ++++++++ src/e2e_test/perf_tool/pkg/suites/metrics.go | 103 +++++++ .../pkg/suites/scripts/clickhouse_export.pxl | 39 +++ .../pkg/suites/scripts/clickhouse_read.pxl | 37 +++ src/e2e_test/perf_tool/pkg/suites/suites.go | 56 +++- .../perf_tool/pkg/suites/workloads.go | 30 ++ src/e2e_test/perf_tool/ui/index.html | 163 ++++++++-- 12 files changed, 817 insertions(+), 142 deletions(-) create mode 100644 src/e2e_test/perf_tool/pkg/suites/scripts/clickhouse_export.pxl create mode 100644 src/e2e_test/perf_tool/pkg/suites/scripts/clickhouse_read.pxl diff --git a/src/e2e_test/perf_tool/cmd/run.go b/src/e2e_test/perf_tool/cmd/run.go index 71e7859fc93..f8c0a509111 100644 --- a/src/e2e_test/perf_tool/cmd/run.go +++ b/src/e2e_test/perf_tool/cmd/run.go @@ -100,6 +100,9 @@ func init() { RunCmd.Flags().String("ds_experiment_page_id", "p_g7fj6pf4yc", "The unique ID of the datastudio experiment page, used to print links to datastudio views") RunCmd.Flags().Bool("pretty", false, "Pretty print output json") + RunCmd.Flags().StringSlice("prom_recorder_override", []string{}, "Override kubeconfig/kube_context for a named prometheus recorder. Format: name=kubeconfig_path:kube_context (either side may be empty). Repeatable.") + RunCmd.Flags().Bool("keep_on_failure", false, "If the experiment fails, skip teardown (stop vizier/workloads/recorders and cluster cleanup) so the cluster state can be inspected. Implies --max_retries=1.") + RootCmd.AddCommand(RunCmd) } @@ -136,6 +139,15 @@ func runCmd(ctx context.Context, cmd *cobra.Command) error { return err } + promOverrides, err := parsePromRecorderOverrides(viper.GetStringSlice("prom_recorder_override")) + if err != nil { + log.WithError(err).Error("failed to parse --prom_recorder_override flags") + return err + } + for _, spec := range specs { + applyPromRecorderOverrides(spec, promOverrides) + } + var c cluster.Provider if viper.GetBool("use_local_cluster") { c = &local.ClusterProvider{} @@ -177,6 +189,13 @@ func runCmd(ctx context.Context, cmd *cobra.Command) error { containerRegistryRepo := viper.GetString("container_repo") maxRetries := viper.GetInt("max_retries") numRuns := viper.GetInt("num_runs") + keepOnFailure := viper.GetBool("keep_on_failure") + if keepOnFailure { + if maxRetries > 1 { + log.Warn("--keep_on_failure is set; forcing --max_retries=1 to avoid retries racing with preserved cluster state") + } + maxRetries = 1 + } eg := errgroup.Group{} experiments := make(chan *exp, len(specs)*numRuns) @@ -190,7 +209,7 @@ func runCmd(ctx context.Context, cmd *cobra.Command) error { s := spec n := name eg.Go(func() error { - expID, err := runExperiment(ctx, s, c, pxAPIKey, pxCloudAddr, metricsExporter, containerRegistryRepo, maxRetries) + expID, err := runExperiment(ctx, s, c, pxAPIKey, pxCloudAddr, metricsExporter, containerRegistryRepo, maxRetries, keepOnFailure) if err != nil { log.WithError(err).Error("failed to run experiment") return err @@ -261,6 +280,7 @@ func runExperiment( metricsExporter exporter.Exporter, containerRegistryRepo string, maxRetries int, + keepOnFailure bool, ) (uuid.UUID, error) { var expID uuid.UUID bo := &maxRetryBackoff{ @@ -269,6 +289,7 @@ func runExperiment( op := func() error { pxCtx := pixie.NewContext(pxAPIKey, pxCloudAddr) r := run.NewRunner(c, pxCtx, metricsExporter, containerRegistryRepo) + r.SetKeepOnFailure(keepOnFailure) var err error expID, err = uuid.NewV4() if err != nil { @@ -406,3 +427,50 @@ func datastudioLink(dsReportID string, dsExperimentPageID string, expID uuid.UUI encodedParams := url.QueryEscape(params) return fmt.Sprintf("https://datastudio.google.com/reporting/%s/page/%s?params=%s", dsReportID, dsExperimentPageID, encodedParams) } + +type promRecorderOverride struct { + KubeconfigPath string + KubeContext string +} + +func parsePromRecorderOverrides(raw []string) (map[string]promRecorderOverride, error) { + out := make(map[string]promRecorderOverride, len(raw)) + for _, s := range raw { + nameAndVal := strings.SplitN(s, "=", 2) + if len(nameAndVal) != 2 || nameAndVal[0] == "" { + return nil, fmt.Errorf("invalid --prom_recorder_override %q: expected name=kubeconfig:context", s) + } + parts := strings.SplitN(nameAndVal[1], ":", 2) + ov := promRecorderOverride{KubeconfigPath: parts[0]} + if len(parts) == 2 { + ov.KubeContext = parts[1] + } + if ov.KubeconfigPath == "" && ov.KubeContext == "" { + return nil, fmt.Errorf("invalid --prom_recorder_override %q: at least one of kubeconfig or context must be set", s) + } + out[nameAndVal[0]] = ov + } + return out, nil +} + +func applyPromRecorderOverrides(spec *experimentpb.ExperimentSpec, overrides map[string]promRecorderOverride) { + if len(overrides) == 0 { + return + } + for _, m := range spec.MetricSpecs { + prom := m.GetProm() + if prom == nil || prom.Name == "" { + continue + } + ov, ok := overrides[prom.Name] + if !ok { + continue + } + if ov.KubeconfigPath != "" { + prom.KubeconfigPath = ov.KubeconfigPath + } + if ov.KubeContext != "" { + prom.KubeContext = ov.KubeContext + } + } +} diff --git a/src/e2e_test/perf_tool/experimentpb/experiment.pb.go b/src/e2e_test/perf_tool/experimentpb/experiment.pb.go index 40be13d29e8..a58d54c4cd0 100755 --- a/src/e2e_test/perf_tool/experimentpb/experiment.pb.go +++ b/src/e2e_test/perf_tool/experimentpb/experiment.pb.go @@ -1256,6 +1256,7 @@ type PrometheusScrapeSpec struct { MetricNames map[string]string `protobuf:"bytes,6,rep,name=metric_names,json=metricNames,proto3" json:"metric_names,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` KubeconfigPath string `protobuf:"bytes,7,opt,name=kubeconfig_path,json=kubeconfigPath,proto3" json:"kubeconfig_path,omitempty"` KubeContext string `protobuf:"bytes,8,opt,name=kube_context,json=kubeContext,proto3" json:"kube_context,omitempty"` + Name string `protobuf:"bytes,9,opt,name=name,proto3" json:"name,omitempty"` } func (m *PrometheusScrapeSpec) Reset() { *m = PrometheusScrapeSpec{} } @@ -1346,6 +1347,13 @@ func (m *PrometheusScrapeSpec) GetKubeContext() string { return "" } +func (m *PrometheusScrapeSpec) GetName() string { + if m != nil { + return m.Name + } + return "" +} + type ClusterSpec struct { NumNodes int32 `protobuf:"varint,1,opt,name=num_nodes,json=numNodes,proto3" json:"num_nodes,omitempty"` Node *NodeSpec `protobuf:"bytes,2,opt,name=node,proto3" json:"node,omitempty"` @@ -1576,121 +1584,122 @@ func init() { } var fileDescriptor_96d7e52dda1e6fe3 = []byte{ - // 1822 bytes of a gzipped FileDescriptorProto + // 1828 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0xcb, 0x73, 0x1b, 0x49, - 0x19, 0xd7, 0x48, 0xb2, 0x25, 0x7d, 0x7a, 0x58, 0xee, 0x3c, 0x50, 0xbc, 0x29, 0x39, 0xab, 0x2d, - 0x20, 0x84, 0x5d, 0x8b, 0x64, 0x79, 0x98, 0xcd, 0xb2, 0x55, 0x92, 0x6c, 0xb0, 0x12, 0x27, 0x16, - 0x2d, 0xaf, 0x17, 0xb6, 0xa8, 0x9a, 0x1a, 0xcf, 0xb4, 0xa5, 0x29, 0xcf, 0x2b, 0x33, 0xad, 0xac, - 0xcc, 0x89, 0x0b, 0xc5, 0x89, 0x2a, 0x0e, 0xf0, 0x1f, 0x70, 0xe0, 0x4f, 0xe0, 0x48, 0x15, 0x07, - 0xe0, 0x96, 0xe3, 0x9e, 0x5c, 0x44, 0xb9, 0x70, 0xdc, 0xff, 0x00, 0xaa, 0xbf, 0xee, 0x19, 0x8d, - 0x64, 0x25, 0x81, 0x2a, 0x6e, 0x3d, 0xbf, 0xfe, 0x7d, 0x8f, 0xfe, 0xfa, 0x7b, 0xb4, 0x04, 0xdf, - 0x8d, 0x42, 0xb3, 0xcd, 0x1e, 0x30, 0x9d, 0xb3, 0x88, 0xb7, 0x03, 0x16, 0x9e, 0xe9, 0xdc, 0xf7, - 0x9d, 0x36, 0x9b, 0x06, 0x2c, 0xb4, 0x5d, 0xe6, 0xf1, 0xe0, 0x34, 0xf5, 0xb1, 0x13, 0x84, 0x3e, - 0xf7, 0x49, 0x25, 0x98, 0xee, 0x24, 0xdc, 0xad, 0xe6, 0xc8, 0xf7, 0x47, 0x0e, 0x6b, 0xe3, 0xde, - 0xe9, 0xe4, 0xac, 0x6d, 0x4d, 0x42, 0x83, 0xdb, 0xbe, 0x27, 0xd9, 0x5b, 0xd7, 0x47, 0xfe, 0xc8, - 0xc7, 0x65, 0x5b, 0xac, 0x24, 0xda, 0xfa, 0x77, 0x16, 0x6a, 0xfb, 0x89, 0xe2, 0x61, 0xc0, 0x4c, - 0xf2, 0x10, 0xca, 0xcf, 0xed, 0x5f, 0xda, 0x2c, 0xd4, 0xa3, 0x80, 0x99, 0x0d, 0xed, 0x8e, 0x76, - 0xb7, 0xfc, 0x60, 0x6b, 0x27, 0x6d, 0x6c, 0xe7, 0x33, 0x3f, 0x3c, 0x77, 0x7c, 0xc3, 0x12, 0x02, - 0x14, 0x24, 0x1d, 0x85, 0x3b, 0x50, 0xfb, 0x42, 0xed, 0xa1, 0x78, 0xd4, 0xc8, 0xde, 0xc9, 0xbd, - 0x45, 0xbe, 0xfa, 0x45, 0xea, 0x2b, 0x22, 0x0f, 0xa1, 0xe2, 0x32, 0x1e, 0xda, 0xa6, 0x52, 0x90, - 0x43, 0x05, 0x8d, 0x45, 0x05, 0x4f, 0x90, 0x81, 0xe2, 0x65, 0x37, 0x59, 0x47, 0xe4, 0x63, 0xa8, - 0x98, 0xce, 0x24, 0xe2, 0xb1, 0xf7, 0x79, 0xf4, 0xfe, 0xd6, 0xa2, 0x70, 0x4f, 0x32, 0xa4, 0xb4, - 0x39, 0xff, 0x20, 0xdf, 0x81, 0x62, 0x38, 0xf1, 0xa4, 0xe4, 0x1a, 0x4a, 0xde, 0x58, 0x94, 0xa4, - 0x13, 0x0f, 0xa5, 0x0a, 0xa1, 0x5c, 0x90, 0xf7, 0x01, 0x4c, 0xdf, 0x75, 0x6d, 0xae, 0x47, 0x63, - 0xa3, 0xb1, 0x7e, 0x47, 0xbb, 0x5b, 0xea, 0x56, 0x67, 0x97, 0xdb, 0xa5, 0x1e, 0xa2, 0xc3, 0x83, - 0x0e, 0x2d, 0x49, 0xc2, 0x70, 0x6c, 0x10, 0x02, 0x79, 0x6e, 0x8c, 0xa2, 0x46, 0xe1, 0x4e, 0xee, - 0x6e, 0x89, 0xe2, 0xba, 0xf5, 0x37, 0x0d, 0x2a, 0xe9, 0x70, 0x08, 0x92, 0x67, 0xb8, 0x0c, 0x03, - 0x5f, 0xa2, 0xb8, 0x16, 0x31, 0xb1, 0x58, 0xe0, 0xf8, 0x17, 0x7a, 0xc4, 0x59, 0x10, 0x07, 0x75, - 0x29, 0x26, 0x7b, 0xc8, 0x18, 0x72, 0x16, 0xd0, 0xb2, 0x95, 0xac, 0x23, 0xf2, 0x23, 0xa8, 0x8c, - 0x99, 0xe1, 0xf0, 0xb1, 0x39, 0x66, 0xe6, 0x79, 0x1c, 0xd0, 0xa5, 0x98, 0x1c, 0x20, 0xa3, 0x27, - 0x18, 0x74, 0x81, 0x4e, 0xbe, 0x09, 0x1b, 0x86, 0x29, 0x12, 0x49, 0x8f, 0x98, 0xc3, 0x4c, 0xee, - 0x87, 0x18, 0xd5, 0x12, 0xad, 0x49, 0x78, 0xa8, 0xd0, 0xd6, 0x3f, 0x34, 0x80, 0xb9, 0x0f, 0xa4, - 0x07, 0xe5, 0x20, 0x64, 0x21, 0xf3, 0x2c, 0x16, 0x32, 0x4b, 0xe5, 0xd1, 0xf6, 0xa2, 0xd5, 0xc1, - 0x9c, 0x20, 0x25, 0x0f, 0x32, 0x34, 0x2d, 0x45, 0x3e, 0x82, 0x62, 0x74, 0x6e, 0x9c, 0x9d, 0xf9, - 0x8e, 0xd5, 0xc8, 0xa2, 0x86, 0xdb, 0x8b, 0x1a, 0x86, 0x6a, 0x37, 0x11, 0x4f, 0xf8, 0xe4, 0xdb, - 0x90, 0x0d, 0xa6, 0x8d, 0xdc, 0xaa, 0x0c, 0x18, 0x4c, 0x7b, 0x87, 0xfd, 0x44, 0x24, 0x1b, 0x4c, - 0xbb, 0x55, 0x50, 0x31, 0xd3, 0xf9, 0x45, 0xc0, 0x5a, 0xbf, 0xd7, 0xa0, 0x9c, 0x0a, 0x09, 0xf9, - 0x18, 0x72, 0xe7, 0xbb, 0xd1, 0xea, 0x43, 0x3c, 0xde, 0x1d, 0x0e, 0x7c, 0x2b, 0xa2, 0xcc, 0xb0, - 0x2e, 0x90, 0xdd, 0x2d, 0xcc, 0x2e, 0xb7, 0x73, 0x8f, 0x77, 0x87, 0x07, 0x19, 0x2a, 0xc4, 0xc8, - 0x0f, 0x21, 0x17, 0x4c, 0x9d, 0xd5, 0x07, 0x18, 0x4c, 0x0f, 0x53, 0x86, 0xa4, 0xa8, 0xc0, 0x32, - 0x54, 0xc8, 0x74, 0x2b, 0x00, 0x78, 0x0f, 0xd2, 0xad, 0xfb, 0xb0, 0x79, 0xc5, 0x1a, 0xb9, 0x0d, - 0x25, 0x91, 0x24, 0x51, 0x60, 0x98, 0x71, 0xd6, 0xcc, 0x81, 0xd6, 0x11, 0xd4, 0x16, 0x4d, 0x90, - 0x9b, 0xb0, 0x1e, 0x99, 0xa1, 0x1d, 0x70, 0x45, 0x56, 0x5f, 0xe4, 0xeb, 0x50, 0x8b, 0x26, 0xa6, - 0xc9, 0xa2, 0x48, 0x37, 0x7d, 0x67, 0xe2, 0x7a, 0xe8, 0x70, 0x89, 0x56, 0x15, 0xda, 0x43, 0xb0, - 0xf5, 0x0b, 0x28, 0x0d, 0x0c, 0x6e, 0x8e, 0x31, 0x59, 0x6f, 0x43, 0xfe, 0xc2, 0x70, 0x1d, 0xa9, - 0xa9, 0x5b, 0x9c, 0x5d, 0x6e, 0xe7, 0x7f, 0xde, 0x79, 0x72, 0x48, 0x11, 0x25, 0xf7, 0x61, 0x9d, - 0x1b, 0xe1, 0x88, 0x71, 0x75, 0xf4, 0xe5, 0x5b, 0x10, 0x6a, 0x8e, 0x91, 0x40, 0x15, 0xb1, 0xf5, - 0x9b, 0x2c, 0x94, 0x53, 0x38, 0xf9, 0x16, 0x94, 0x8c, 0xc0, 0xd6, 0x47, 0xa1, 0x3f, 0x09, 0x94, - 0x95, 0xca, 0xec, 0x72, 0xbb, 0xd8, 0x19, 0xf4, 0x7f, 0x22, 0x30, 0x5a, 0x34, 0x02, 0x1b, 0x57, - 0xa4, 0x0d, 0x65, 0x41, 0x7d, 0xce, 0xc2, 0xc8, 0xf6, 0x95, 0xf3, 0xdd, 0xda, 0xec, 0x72, 0x1b, - 0x3a, 0x83, 0xfe, 0x89, 0x44, 0x29, 0x18, 0x81, 0xad, 0xd6, 0xa2, 0xd2, 0xce, 0x6d, 0xcf, 0xc2, - 0x14, 0x29, 0x51, 0x5c, 0x27, 0xd5, 0x97, 0x4f, 0x55, 0xdf, 0x42, 0x80, 0xd7, 0x96, 0x02, 0x2c, - 0xc2, 0xe6, 0x18, 0xa7, 0xcc, 0x99, 0x97, 0xc7, 0xba, 0x0c, 0x1b, 0xa2, 0x71, 0x75, 0x90, 0x36, - 0x5c, 0x33, 0x3c, 0xcf, 0xe7, 0xc6, 0x62, 0x29, 0x15, 0x90, 0x4b, 0xe6, 0x5b, 0x49, 0x39, 0x71, - 0xd8, 0xbc, 0x52, 0x1e, 0xa2, 0xdf, 0x88, 0xc8, 0xea, 0x81, 0xc1, 0xc7, 0x22, 0x1d, 0x73, 0x71, - 0xbf, 0x11, 0x51, 0x1f, 0x08, 0x90, 0x96, 0x04, 0x01, 0x97, 0xe4, 0x3e, 0x14, 0x02, 0x11, 0x4b, - 0x16, 0x77, 0x8c, 0xaf, 0xad, 0xb8, 0x00, 0xd9, 0xd0, 0x14, 0xaf, 0xf5, 0x5b, 0x0d, 0x6a, 0x8b, - 0x35, 0x45, 0xde, 0x83, 0x6a, 0x5c, 0x53, 0x68, 0x57, 0xa5, 0x4d, 0x25, 0x06, 0x85, 0xad, 0x05, - 0x92, 0x11, 0x8e, 0xa4, 0xc1, 0x14, 0xa9, 0x13, 0x8e, 0x16, 0xfc, 0xc9, 0xfd, 0x97, 0xfe, 0x5c, - 0x40, 0x39, 0x55, 0xac, 0xe2, 0x7a, 0x50, 0xbb, 0x26, 0x3b, 0xa8, 0x58, 0x93, 0x26, 0x40, 0x72, - 0x1b, 0xb1, 0xdd, 0x14, 0x42, 0xbe, 0x0f, 0xb5, 0x88, 0x71, 0x3d, 0x9e, 0x0b, 0xb6, 0xbc, 0xf0, - 0x62, 0xb7, 0x3e, 0xbb, 0xdc, 0xae, 0x0c, 0x19, 0x57, 0xe3, 0xa0, 0xbf, 0x47, 0x2b, 0xd1, 0xfc, - 0xcb, 0x6a, 0xfd, 0x59, 0x03, 0x98, 0xcf, 0x19, 0xb2, 0x2b, 0x8b, 0x58, 0xb6, 0x80, 0x77, 0xae, - 0x14, 0xf1, 0x10, 0x8b, 0x48, 0x30, 0x97, 0x6b, 0x98, 0xec, 0x42, 0x3e, 0x08, 0x7d, 0x57, 0x15, - 0x41, 0x6b, 0xb9, 0x05, 0xfa, 0x2e, 0xe3, 0x63, 0x36, 0x89, 0x86, 0x66, 0x68, 0x04, 0x4c, 0x68, - 0x38, 0xc8, 0x50, 0x94, 0x58, 0xd5, 0x7b, 0xad, 0x55, 0xbd, 0x57, 0xb4, 0x2f, 0x35, 0x34, 0xb1, - 0x4f, 0xcc, 0x72, 0x50, 0x5d, 0xf0, 0xe9, 0xb5, 0x45, 0x7f, 0x1b, 0x4a, 0x11, 0x0f, 0x99, 0xe1, - 0xda, 0xde, 0x08, 0x1d, 0x2c, 0xd2, 0x39, 0x40, 0x7e, 0x0c, 0x9b, 0xa6, 0xef, 0x08, 0x1b, 0xc2, - 0x07, 0xf1, 0x4c, 0xf0, 0xad, 0xa4, 0xa3, 0xca, 0x07, 0xc7, 0x4e, 0xfc, 0xe0, 0xd8, 0xd9, 0x53, - 0x0f, 0x0e, 0x5a, 0x9f, 0xcb, 0x0c, 0x50, 0x84, 0xfc, 0x0c, 0x36, 0x38, 0x73, 0x03, 0xc7, 0xe0, - 0x4c, 0x7f, 0x6e, 0x38, 0x13, 0x16, 0x35, 0xf2, 0x98, 0x00, 0xed, 0x37, 0xc4, 0x71, 0xe7, 0x58, - 0x89, 0x9c, 0xa0, 0xc4, 0xbe, 0xc7, 0xc3, 0x0b, 0x5a, 0xe3, 0x0b, 0x20, 0xa1, 0x50, 0xe5, 0xc6, - 0xa9, 0xc3, 0x74, 0x7f, 0xc2, 0x83, 0x09, 0x8f, 0x1a, 0x6b, 0xa8, 0xf7, 0x83, 0x37, 0xea, 0x15, - 0x02, 0x47, 0x92, 0x2f, 0xb5, 0x56, 0x78, 0x0a, 0xda, 0xea, 0xc0, 0xb5, 0x15, 0xa6, 0x49, 0x1d, - 0x72, 0xe7, 0xec, 0x42, 0xc5, 0x4f, 0x2c, 0xc9, 0x75, 0x58, 0xc3, 0xd3, 0xa8, 0x46, 0x29, 0x3f, - 0x3e, 0xca, 0xee, 0x6a, 0x5b, 0xa7, 0xb0, 0x79, 0xc5, 0xca, 0x0a, 0x05, 0x3f, 0x48, 0x2b, 0x28, - 0x3f, 0x78, 0xf7, 0x35, 0x5e, 0x4b, 0x2d, 0x87, 0x76, 0xc4, 0x53, 0x36, 0x5a, 0x14, 0xae, 0xad, - 0x60, 0x90, 0x87, 0x50, 0x88, 0x63, 0xa1, 0x61, 0x2c, 0xde, 0xac, 0x55, 0x96, 0x9b, 0x92, 0x68, - 0xfd, 0x55, 0xbb, 0xa2, 0x14, 0xd3, 0xe7, 0x11, 0x54, 0x23, 0xdb, 0x1b, 0x39, 0x4c, 0x97, 0x69, - 0xa6, 0xca, 0xe0, 0xbd, 0xa5, 0x61, 0x8c, 0x14, 0x59, 0x33, 0x83, 0xe9, 0xa1, 0x94, 0x3f, 0xc8, - 0xd0, 0x4a, 0x94, 0xda, 0x20, 0x3f, 0x85, 0x4d, 0xcb, 0xe0, 0x86, 0xee, 0xf8, 0x38, 0x69, 0x26, - 0x1e, 0x67, 0xa1, 0x0a, 0xc0, 0x92, 0xbe, 0x3d, 0x83, 0x1b, 0x87, 0xbe, 0x98, 0x3c, 0x48, 0x4a, - 0xf4, 0x6d, 0x58, 0x8b, 0x1b, 0x22, 0xfd, 0xe5, 0x09, 0xf0, 0xed, 0xd6, 0xfa, 0x83, 0x06, 0x37, - 0x56, 0xfa, 0x22, 0xda, 0x14, 0xb7, 0x5d, 0x16, 0x71, 0xc3, 0x0d, 0xc4, 0x94, 0x8b, 0x7b, 0x59, - 0x02, 0xf6, 0x7c, 0x87, 0x6c, 0x27, 0xc5, 0x84, 0xa3, 0x40, 0x5e, 0x2e, 0x48, 0xe8, 0xa9, 0x18, - 0x08, 0xef, 0x40, 0x09, 0xaf, 0x01, 0x35, 0xc8, 0xe9, 0x51, 0x44, 0x40, 0x48, 0xdf, 0x82, 0x22, - 0x37, 0x46, 0x62, 0x4b, 0x26, 0x79, 0x89, 0x16, 0xb8, 0x31, 0xea, 0xf9, 0x4e, 0x24, 0x5e, 0x48, - 0x37, 0x56, 0x9e, 0xe9, 0xff, 0xe4, 0xd7, 0x3d, 0x80, 0x88, 0x3d, 0xd3, 0x6d, 0x6b, 0xee, 0x98, - 0x9c, 0x96, 0x43, 0xf6, 0xac, 0xbf, 0xd7, 0xf3, 0x1d, 0x5a, 0x8c, 0xd8, 0xb3, 0xbe, 0x25, 0x94, - 0x7d, 0x02, 0x55, 0x15, 0x32, 0x55, 0xd6, 0xf9, 0xb7, 0x95, 0x75, 0x45, 0xf2, 0x65, 0x49, 0xb7, - 0xfe, 0x92, 0x83, 0xeb, 0xab, 0x7a, 0xd7, 0x9b, 0x9f, 0x23, 0xe4, 0x1b, 0xb0, 0xe1, 0x8a, 0xd6, - 0xae, 0xcb, 0x99, 0x29, 0xea, 0x41, 0xbd, 0x32, 0x10, 0x3e, 0x14, 0xe8, 0x63, 0x76, 0x41, 0xee, - 0xc1, 0x66, 0x9a, 0x27, 0xab, 0x44, 0x86, 0x7a, 0x63, 0xce, 0xc4, 0xf2, 0x14, 0x43, 0x21, 0xf0, - 0x43, 0x8e, 0x27, 0x58, 0xa3, 0xb8, 0x16, 0xc7, 0x8b, 0xd0, 0xa7, 0xf8, 0x78, 0x6b, 0x6f, 0x3d, - 0x9e, 0xe4, 0xab, 0x8e, 0x75, 0x92, 0xfc, 0x0a, 0x41, 0xdf, 0x1b, 0xeb, 0x58, 0x4a, 0x1f, 0xbe, - 0xbd, 0x77, 0xab, 0x9f, 0x26, 0xe2, 0x3e, 0x54, 0x73, 0x29, 0xcf, 0x6f, 0x08, 0x5f, 0xd3, 0xe7, - 0x93, 0x53, 0x66, 0xfa, 0xde, 0x99, 0x3d, 0x92, 0xe3, 0x54, 0x3e, 0x01, 0x6a, 0x73, 0x18, 0x07, - 0xea, 0xbb, 0x50, 0x11, 0x88, 0x6e, 0xfa, 0x1e, 0x67, 0x53, 0xde, 0x28, 0x22, 0xab, 0x2c, 0xb0, - 0x9e, 0x84, 0xb6, 0x3e, 0x81, 0xfa, 0xb2, 0xb1, 0xff, 0xa5, 0x49, 0xb5, 0x4e, 0xa0, 0x9c, 0xfa, - 0x29, 0x24, 0xb2, 0xda, 0x9b, 0xb8, 0xba, 0xe7, 0x5b, 0x4c, 0xbe, 0x74, 0xd7, 0x68, 0xd1, 0x9b, - 0xb8, 0x4f, 0xc5, 0x37, 0xb9, 0x07, 0x79, 0xb1, 0xa1, 0xea, 0xf4, 0xe6, 0x62, 0x1c, 0x04, 0x05, - 0xfb, 0x08, 0x72, 0x5a, 0x1f, 0x40, 0x31, 0x46, 0xc4, 0x31, 0x5c, 0xc3, 0x1c, 0xdb, 0x1e, 0xc3, - 0xc9, 0xa4, 0x1c, 0x2b, 0x2b, 0xec, 0x58, 0x0c, 0xab, 0x3e, 0x14, 0xd4, 0xef, 0x2a, 0xf2, 0x00, - 0x0a, 0x72, 0xb0, 0xbd, 0xe6, 0x67, 0x5f, 0x47, 0x4e, 0x3d, 0x6c, 0x59, 0x8a, 0xf8, 0x28, 0x5f, - 0xd4, 0xea, 0xd9, 0x47, 0xf9, 0x62, 0xb6, 0x9e, 0x6b, 0xfd, 0x5a, 0x03, 0x98, 0x73, 0xc8, 0xfb, - 0x90, 0x4f, 0x8c, 0xd6, 0x56, 0xeb, 0x12, 0x1e, 0x50, 0x64, 0x91, 0xef, 0x41, 0x31, 0xfe, 0xcd, - 0x9c, 0xbc, 0x57, 0x5f, 0x9b, 0x2d, 0x09, 0x35, 0x79, 0x31, 0xe6, 0xe6, 0x2f, 0xc6, 0x7b, 0x7f, - 0x4c, 0xfc, 0x10, 0xfa, 0x49, 0x1d, 0x2a, 0xc3, 0xe3, 0x0e, 0x3d, 0xd6, 0x4f, 0xfa, 0x9f, 0xf7, - 0xf7, 0x69, 0x3d, 0x43, 0xae, 0xc1, 0x86, 0x44, 0x3e, 0x3b, 0xa2, 0x8f, 0x0f, 0x8f, 0x3a, 0x7b, - 0xc3, 0xba, 0x46, 0xb6, 0xe0, 0xa6, 0x04, 0x9f, 0xec, 0x1f, 0xd3, 0x7e, 0x4f, 0xa7, 0xfb, 0xbd, - 0x23, 0xba, 0xb7, 0x4f, 0x87, 0xf5, 0x2c, 0xd9, 0x80, 0xf2, 0xf0, 0xf8, 0x68, 0x10, 0x6b, 0xc8, - 0x11, 0x02, 0x35, 0x04, 0xe6, 0x0a, 0xf2, 0xe4, 0x16, 0xdc, 0x40, 0xec, 0x8a, 0xfc, 0x1a, 0x29, - 0x40, 0x8e, 0x7e, 0xfa, 0xb4, 0xbe, 0x4e, 0x00, 0xd6, 0xbb, 0x9f, 0xd2, 0xa7, 0xfd, 0xa7, 0xf5, - 0x42, 0xb7, 0xfb, 0xe2, 0x65, 0x33, 0xf3, 0xe5, 0xcb, 0x66, 0xe6, 0xab, 0x97, 0x4d, 0xed, 0x57, - 0xb3, 0xa6, 0xf6, 0xa7, 0x59, 0x53, 0xfb, 0xfb, 0xac, 0xa9, 0xbd, 0x98, 0x35, 0xb5, 0x7f, 0xce, - 0x9a, 0xda, 0xbf, 0x66, 0xcd, 0xcc, 0x57, 0xb3, 0xa6, 0xf6, 0xbb, 0x57, 0xcd, 0xcc, 0x8b, 0x57, - 0xcd, 0xcc, 0x97, 0xaf, 0x9a, 0x99, 0xcf, 0x2b, 0xe9, 0xbf, 0x25, 0x4e, 0xd7, 0x31, 0x36, 0x1f, - 0xfe, 0x27, 0x00, 0x00, 0xff, 0xff, 0xf1, 0xcb, 0x88, 0xa6, 0xc4, 0x10, 0x00, 0x00, + 0x19, 0xd7, 0x48, 0xb2, 0x25, 0x7d, 0x92, 0x65, 0xb9, 0xf3, 0x40, 0xf1, 0xa6, 0xe4, 0xac, 0xb6, + 0x80, 0x10, 0x76, 0x6d, 0x92, 0xe5, 0x61, 0x36, 0xcb, 0x56, 0x49, 0xb2, 0xc1, 0x4a, 0x9c, 0x58, + 0xb4, 0xbc, 0x5e, 0xd8, 0xa2, 0x6a, 0x6a, 0x3c, 0xd3, 0x96, 0xa6, 0x3c, 0xaf, 0xcc, 0xb4, 0xb2, + 0x32, 0x27, 0x2e, 0x14, 0x27, 0xaa, 0x38, 0xc0, 0x7f, 0xc0, 0x81, 0x3f, 0x81, 0x3b, 0x07, 0xe0, + 0x96, 0x03, 0x87, 0x3d, 0xb9, 0x88, 0x72, 0xe1, 0xb8, 0xff, 0x01, 0x54, 0x7f, 0xdd, 0xf3, 0x90, + 0xac, 0x24, 0x50, 0xc5, 0xad, 0xe7, 0xd7, 0xbf, 0xef, 0xd1, 0x5f, 0x7f, 0x8f, 0x96, 0xe0, 0xbb, + 0x51, 0x68, 0xee, 0xb0, 0x07, 0x4c, 0xe7, 0x2c, 0xe2, 0x3b, 0x01, 0x0b, 0xcf, 0x74, 0xee, 0xfb, + 0xce, 0x0e, 0x9b, 0x06, 0x2c, 0xb4, 0x5d, 0xe6, 0xf1, 0xe0, 0x34, 0xf3, 0xb1, 0x1d, 0x84, 0x3e, + 0xf7, 0x49, 0x2d, 0x98, 0x6e, 0x27, 0xdc, 0xcd, 0xd6, 0xc8, 0xf7, 0x47, 0x0e, 0xdb, 0xc1, 0xbd, + 0xd3, 0xc9, 0xd9, 0x8e, 0x35, 0x09, 0x0d, 0x6e, 0xfb, 0x9e, 0x64, 0x6f, 0x5e, 0x1f, 0xf9, 0x23, + 0x1f, 0x97, 0x3b, 0x62, 0x25, 0xd1, 0xf6, 0xbf, 0xf3, 0x50, 0xdf, 0x4f, 0x14, 0x0f, 0x03, 0x66, + 0x92, 0x87, 0x50, 0x7d, 0x6e, 0xff, 0xd2, 0x66, 0xa1, 0x1e, 0x05, 0xcc, 0x6c, 0x6a, 0x77, 0xb4, + 0xbb, 0xd5, 0x07, 0x9b, 0xdb, 0x59, 0x63, 0xdb, 0x9f, 0xf9, 0xe1, 0xb9, 0xe3, 0x1b, 0x96, 0x10, + 0xa0, 0x20, 0xe9, 0x28, 0xdc, 0x81, 0xfa, 0x17, 0x6a, 0x0f, 0xc5, 0xa3, 0x66, 0xfe, 0x4e, 0xe1, + 0x2d, 0xf2, 0x6b, 0x5f, 0x64, 0xbe, 0x22, 0xf2, 0x10, 0x6a, 0x2e, 0xe3, 0xa1, 0x6d, 0x2a, 0x05, + 0x05, 0x54, 0xd0, 0x9c, 0x57, 0xf0, 0x04, 0x19, 0x28, 0x5e, 0x75, 0x93, 0x75, 0x44, 0x3e, 0x86, + 0x9a, 0xe9, 0x4c, 0x22, 0x1e, 0x7b, 0x5f, 0x44, 0xef, 0x6f, 0xcd, 0x0b, 0xf7, 0x24, 0x43, 0x4a, + 0x9b, 0xe9, 0x07, 0xf9, 0x0e, 0x94, 0xc3, 0x89, 0x27, 0x25, 0x57, 0x50, 0xf2, 0xc6, 0xbc, 0x24, + 0x9d, 0x78, 0x28, 0x55, 0x0a, 0xe5, 0x82, 0xbc, 0x0f, 0x60, 0xfa, 0xae, 0x6b, 0x73, 0x3d, 0x1a, + 0x1b, 0xcd, 0xd5, 0x3b, 0xda, 0xdd, 0x4a, 0x77, 0x6d, 0x76, 0xb9, 0x55, 0xe9, 0x21, 0x3a, 0x3c, + 0xe8, 0xd0, 0x8a, 0x24, 0x0c, 0xc7, 0x06, 0x21, 0x50, 0xe4, 0xc6, 0x28, 0x6a, 0x96, 0xee, 0x14, + 0xee, 0x56, 0x28, 0xae, 0xdb, 0x7f, 0xd5, 0xa0, 0x96, 0x0d, 0x87, 0x20, 0x79, 0x86, 0xcb, 0x30, + 0xf0, 0x15, 0x8a, 0x6b, 0x11, 0x13, 0x8b, 0x05, 0x8e, 0x7f, 0xa1, 0x47, 0x9c, 0x05, 0x71, 0x50, + 0x17, 0x62, 0xb2, 0x87, 0x8c, 0x21, 0x67, 0x01, 0xad, 0x5a, 0xc9, 0x3a, 0x22, 0x3f, 0x82, 0xda, + 0x98, 0x19, 0x0e, 0x1f, 0x9b, 0x63, 0x66, 0x9e, 0xc7, 0x01, 0x5d, 0x88, 0xc9, 0x01, 0x32, 0x7a, + 0x82, 0x41, 0xe7, 0xe8, 0xe4, 0x9b, 0xb0, 0x6e, 0x98, 0x22, 0x91, 0xf4, 0x88, 0x39, 0xcc, 0xe4, + 0x7e, 0x88, 0x51, 0xad, 0xd0, 0xba, 0x84, 0x87, 0x0a, 0x6d, 0xff, 0x5d, 0x03, 0x48, 0x7d, 0x20, + 0x3d, 0xa8, 0x06, 0x21, 0x0b, 0x99, 0x67, 0xb1, 0x90, 0x59, 0x2a, 0x8f, 0xb6, 0xe6, 0xad, 0x0e, + 0x52, 0x82, 0x94, 0x3c, 0xc8, 0xd1, 0xac, 0x14, 0xf9, 0x08, 0xca, 0xd1, 0xb9, 0x71, 0x76, 0xe6, + 0x3b, 0x56, 0x33, 0x8f, 0x1a, 0x6e, 0xcf, 0x6b, 0x18, 0xaa, 0xdd, 0x44, 0x3c, 0xe1, 0x93, 0x6f, + 0x43, 0x3e, 0x98, 0x36, 0x0b, 0xcb, 0x32, 0x60, 0x30, 0xed, 0x1d, 0xf6, 0x13, 0x91, 0x7c, 0x30, + 0xed, 0xae, 0x81, 0x8a, 0x99, 0xce, 0x2f, 0x02, 0xd6, 0xfe, 0xbd, 0x06, 0xd5, 0x4c, 0x48, 0xc8, + 0xc7, 0x50, 0x38, 0xdf, 0x8d, 0x96, 0x1f, 0xe2, 0xf1, 0xee, 0x70, 0xe0, 0x5b, 0x11, 0x65, 0x86, + 0x75, 0x81, 0xec, 0x6e, 0x69, 0x76, 0xb9, 0x55, 0x78, 0xbc, 0x3b, 0x3c, 0xc8, 0x51, 0x21, 0x46, + 0x7e, 0x08, 0x85, 0x60, 0xea, 0x2c, 0x3f, 0xc0, 0x60, 0x7a, 0x98, 0x31, 0x24, 0x45, 0x05, 0x96, + 0xa3, 0x42, 0xa6, 0x5b, 0x03, 0xc0, 0x7b, 0x90, 0x6e, 0xdd, 0x87, 0x8d, 0x2b, 0xd6, 0xc8, 0x6d, + 0xa8, 0x88, 0x24, 0x89, 0x02, 0xc3, 0x8c, 0xb3, 0x26, 0x05, 0xda, 0x47, 0x50, 0x9f, 0x37, 0x41, + 0x6e, 0xc2, 0x6a, 0x64, 0x86, 0x76, 0xc0, 0x15, 0x59, 0x7d, 0x91, 0xaf, 0x43, 0x3d, 0x9a, 0x98, + 0x26, 0x8b, 0x22, 0xdd, 0xf4, 0x9d, 0x89, 0xeb, 0xa1, 0xc3, 0x15, 0xba, 0xa6, 0xd0, 0x1e, 0x82, + 0xed, 0x5f, 0x40, 0x65, 0x60, 0x70, 0x73, 0x8c, 0xc9, 0x7a, 0x1b, 0x8a, 0x17, 0x86, 0xeb, 0x48, + 0x4d, 0xdd, 0xf2, 0xec, 0x72, 0xab, 0xf8, 0xf3, 0xce, 0x93, 0x43, 0x8a, 0x28, 0xb9, 0x0f, 0xab, + 0xdc, 0x08, 0x47, 0x8c, 0xab, 0xa3, 0x2f, 0xde, 0x82, 0x50, 0x73, 0x8c, 0x04, 0xaa, 0x88, 0xed, + 0xdf, 0xe4, 0xa1, 0x9a, 0xc1, 0xc9, 0xb7, 0xa0, 0x62, 0x04, 0xb6, 0x3e, 0x0a, 0xfd, 0x49, 0xa0, + 0xac, 0xd4, 0x66, 0x97, 0x5b, 0xe5, 0xce, 0xa0, 0xff, 0x13, 0x81, 0xd1, 0xb2, 0x11, 0xd8, 0xb8, + 0x22, 0x3b, 0x50, 0x15, 0xd4, 0xe7, 0x2c, 0x8c, 0x6c, 0x5f, 0x39, 0xdf, 0xad, 0xcf, 0x2e, 0xb7, + 0xa0, 0x33, 0xe8, 0x9f, 0x48, 0x94, 0x82, 0x11, 0xd8, 0x6a, 0x2d, 0x2a, 0xed, 0xdc, 0xf6, 0x2c, + 0x4c, 0x91, 0x0a, 0xc5, 0x75, 0x52, 0x7d, 0xc5, 0x4c, 0xf5, 0xcd, 0x05, 0x78, 0x65, 0x21, 0xc0, + 0x22, 0x6c, 0x8e, 0x71, 0xca, 0x9c, 0xb4, 0x3c, 0x56, 0x65, 0xd8, 0x10, 0x8d, 0xab, 0x83, 0xec, + 0xc0, 0x35, 0xc3, 0xf3, 0x7c, 0x6e, 0xcc, 0x97, 0x52, 0x09, 0xb9, 0x24, 0xdd, 0x4a, 0xca, 0x89, + 0xc3, 0xc6, 0x95, 0xf2, 0x10, 0xfd, 0x46, 0x44, 0x56, 0x0f, 0x0c, 0x3e, 0x16, 0xe9, 0x58, 0x88, + 0xfb, 0x8d, 0x88, 0xfa, 0x40, 0x80, 0xb4, 0x22, 0x08, 0xb8, 0x24, 0xf7, 0xa1, 0x14, 0x88, 0x58, + 0xb2, 0xb8, 0x63, 0x7c, 0x6d, 0xc9, 0x05, 0xc8, 0x86, 0xa6, 0x78, 0xed, 0xdf, 0x6a, 0x50, 0x9f, + 0xaf, 0x29, 0xf2, 0x1e, 0xac, 0xc5, 0x35, 0x85, 0x76, 0x55, 0xda, 0xd4, 0x62, 0x50, 0xd8, 0x9a, + 0x23, 0x19, 0xe1, 0x48, 0x1a, 0xcc, 0x90, 0x3a, 0xe1, 0x68, 0xce, 0x9f, 0xc2, 0x7f, 0xe9, 0xcf, + 0x05, 0x54, 0x33, 0xc5, 0x2a, 0xae, 0x07, 0xb5, 0x6b, 0xb2, 0x83, 0x8a, 0x35, 0x69, 0x01, 0x24, + 0xb7, 0x11, 0xdb, 0xcd, 0x20, 0xe4, 0xfb, 0x50, 0x8f, 0x18, 0xd7, 0xe3, 0xb9, 0x60, 0xcb, 0x0b, + 0x2f, 0x77, 0x1b, 0xb3, 0xcb, 0xad, 0xda, 0x90, 0x71, 0x35, 0x0e, 0xfa, 0x7b, 0xb4, 0x16, 0xa5, + 0x5f, 0x56, 0xfb, 0xcf, 0x1a, 0x40, 0x3a, 0x67, 0xc8, 0xae, 0x2c, 0x62, 0xd9, 0x02, 0xde, 0xb9, + 0x52, 0xc4, 0x43, 0x2c, 0x22, 0xc1, 0x5c, 0xac, 0x61, 0xb2, 0x0b, 0xc5, 0x20, 0xf4, 0x5d, 0x55, + 0x04, 0xed, 0xc5, 0x16, 0xe8, 0xbb, 0x8c, 0x8f, 0xd9, 0x24, 0x1a, 0x9a, 0xa1, 0x11, 0x30, 0xa1, + 0xe1, 0x20, 0x47, 0x51, 0x62, 0x59, 0xef, 0xb5, 0x96, 0xf5, 0x5e, 0xd1, 0xbe, 0xd4, 0xd0, 0xc4, + 0x3e, 0x31, 0x2b, 0xc0, 0xda, 0x9c, 0x4f, 0xaf, 0x2d, 0xfa, 0xdb, 0x50, 0x89, 0x78, 0xc8, 0x0c, + 0xd7, 0xf6, 0x46, 0xe8, 0x60, 0x99, 0xa6, 0x00, 0xf9, 0x31, 0x6c, 0x98, 0xbe, 0x23, 0x6c, 0x08, + 0x1f, 0xc4, 0x33, 0xc1, 0xb7, 0x92, 0x8e, 0x2a, 0x1f, 0x1c, 0xdb, 0xf1, 0x83, 0x63, 0x7b, 0x4f, + 0x3d, 0x38, 0x68, 0x23, 0x95, 0x19, 0xa0, 0x08, 0xf9, 0x19, 0xac, 0x73, 0xe6, 0x06, 0x8e, 0xc1, + 0x99, 0xfe, 0xdc, 0x70, 0x26, 0x2c, 0x6a, 0x16, 0x31, 0x01, 0x76, 0xde, 0x10, 0xc7, 0xed, 0x63, + 0x25, 0x72, 0x82, 0x12, 0xfb, 0x1e, 0x0f, 0x2f, 0x68, 0x9d, 0xcf, 0x81, 0x84, 0xc2, 0x1a, 0x37, + 0x4e, 0x1d, 0xa6, 0xfb, 0x13, 0x1e, 0x4c, 0x78, 0xd4, 0x5c, 0x41, 0xbd, 0x1f, 0xbc, 0x51, 0xaf, + 0x10, 0x38, 0x92, 0x7c, 0xa9, 0xb5, 0xc6, 0x33, 0xd0, 0x66, 0x07, 0xae, 0x2d, 0x31, 0x4d, 0x1a, + 0x50, 0x38, 0x67, 0x17, 0x2a, 0x7e, 0x62, 0x49, 0xae, 0xc3, 0x0a, 0x9e, 0x46, 0x35, 0x4a, 0xf9, + 0xf1, 0x51, 0x7e, 0x57, 0xdb, 0x3c, 0x85, 0x8d, 0x2b, 0x56, 0x96, 0x28, 0xf8, 0x41, 0x56, 0x41, + 0xf5, 0xc1, 0xbb, 0xaf, 0xf1, 0x5a, 0x6a, 0x39, 0xb4, 0x23, 0x9e, 0xb1, 0xd1, 0xa6, 0x70, 0x6d, + 0x09, 0x83, 0x3c, 0x84, 0x52, 0x1c, 0x0b, 0x0d, 0x63, 0xf1, 0x66, 0xad, 0xb2, 0xdc, 0x94, 0x44, + 0xfb, 0x2f, 0xda, 0x15, 0xa5, 0x98, 0x3e, 0x8f, 0x60, 0x2d, 0xb2, 0xbd, 0x91, 0xc3, 0x74, 0x99, + 0x66, 0xaa, 0x0c, 0xde, 0x5b, 0x18, 0xc6, 0x48, 0x91, 0x35, 0x33, 0x98, 0x1e, 0x4a, 0xf9, 0x83, + 0x1c, 0xad, 0x45, 0x99, 0x0d, 0xf2, 0x53, 0xd8, 0xb0, 0x0c, 0x6e, 0xe8, 0x8e, 0x8f, 0x93, 0x66, + 0xe2, 0x71, 0x16, 0xaa, 0x00, 0x2c, 0xe8, 0xdb, 0x33, 0xb8, 0x71, 0xe8, 0x8b, 0xc9, 0x83, 0xa4, + 0x44, 0xdf, 0xba, 0x35, 0xbf, 0x21, 0xd2, 0x5f, 0x9e, 0x00, 0xdf, 0x6e, 0xed, 0x3f, 0x68, 0x70, + 0x63, 0xa9, 0x2f, 0xa2, 0x4d, 0x71, 0xdb, 0x65, 0x11, 0x37, 0xdc, 0x40, 0x4c, 0xb9, 0xb8, 0x97, + 0x25, 0x60, 0xcf, 0x77, 0xc8, 0x56, 0x52, 0x4c, 0x38, 0x0a, 0xe4, 0xe5, 0x82, 0x84, 0x9e, 0x8a, + 0x81, 0xf0, 0x0e, 0x54, 0xf0, 0x1a, 0x50, 0x83, 0x9c, 0x1e, 0x65, 0x04, 0x84, 0xf4, 0x2d, 0x28, + 0x73, 0x63, 0x24, 0xb6, 0x64, 0x92, 0x57, 0x68, 0x89, 0x1b, 0xa3, 0x9e, 0xef, 0x44, 0xe2, 0x85, + 0x74, 0x63, 0xe9, 0x99, 0xfe, 0x4f, 0x7e, 0xdd, 0x03, 0x88, 0xd8, 0x33, 0xdd, 0xb6, 0x52, 0xc7, + 0xe4, 0xb4, 0x1c, 0xb2, 0x67, 0xfd, 0xbd, 0x9e, 0xef, 0xd0, 0x72, 0xc4, 0x9e, 0xf5, 0x2d, 0xa1, + 0xec, 0x13, 0x58, 0x53, 0x21, 0x53, 0x65, 0x5d, 0x7c, 0x5b, 0x59, 0xd7, 0x24, 0x5f, 0x96, 0x74, + 0xfb, 0x1f, 0x05, 0xb8, 0xbe, 0xac, 0x77, 0xbd, 0xf9, 0x39, 0x42, 0xbe, 0x01, 0xeb, 0xae, 0x68, + 0xed, 0xba, 0x9c, 0x99, 0xa2, 0x1e, 0xd4, 0x2b, 0x03, 0xe1, 0x43, 0x81, 0x3e, 0x66, 0x17, 0xe4, + 0x1e, 0x6c, 0x64, 0x79, 0xb2, 0x4a, 0x64, 0xa8, 0xd7, 0x53, 0x26, 0x96, 0xa7, 0x18, 0x0a, 0x81, + 0x1f, 0x72, 0x3c, 0xc1, 0x0a, 0xc5, 0xb5, 0x38, 0x5e, 0x84, 0x3e, 0xc5, 0xc7, 0x5b, 0x79, 0xeb, + 0xf1, 0x24, 0x5f, 0x75, 0xac, 0x93, 0xe4, 0x57, 0x08, 0xfa, 0xde, 0x5c, 0xc5, 0x52, 0xfa, 0xf0, + 0xed, 0xbd, 0x5b, 0xfd, 0x34, 0x11, 0xf7, 0xa1, 0x9a, 0x4b, 0x35, 0xbd, 0x21, 0x7c, 0x4d, 0x9f, + 0x4f, 0x4e, 0x99, 0xe9, 0x7b, 0x67, 0xf6, 0x48, 0x8e, 0x53, 0xf9, 0x04, 0xa8, 0xa7, 0x30, 0x0e, + 0xd4, 0x77, 0xa1, 0x26, 0x10, 0xdd, 0xf4, 0x3d, 0xce, 0xa6, 0xbc, 0x59, 0x46, 0x56, 0x55, 0x60, + 0x3d, 0x09, 0x25, 0x6f, 0x95, 0x4a, 0xfa, 0x56, 0xd9, 0xfc, 0x04, 0x1a, 0x8b, 0x0e, 0xfc, 0x2f, + 0x8d, 0xab, 0x7d, 0x02, 0xd5, 0xcc, 0xcf, 0x23, 0x91, 0xe9, 0xde, 0xc4, 0xd5, 0x3d, 0xdf, 0x62, + 0xf2, 0xf5, 0xbb, 0x42, 0xcb, 0xde, 0xc4, 0x7d, 0x2a, 0xbe, 0xc9, 0x3d, 0x28, 0x8a, 0x0d, 0x55, + 0xbb, 0x37, 0xe7, 0x63, 0x23, 0x28, 0xd8, 0x5b, 0x90, 0xd3, 0xfe, 0x00, 0xca, 0x31, 0x22, 0x8e, + 0xe6, 0x1a, 0xe6, 0xd8, 0xf6, 0x18, 0x4e, 0x2b, 0xe5, 0x58, 0x55, 0x61, 0xc7, 0x62, 0x80, 0xf5, + 0xa1, 0xa4, 0x7e, 0x6b, 0x91, 0x07, 0x50, 0x92, 0xc3, 0xee, 0x35, 0x3f, 0x05, 0x3b, 0x72, 0x12, + 0x62, 0x1b, 0x53, 0xc4, 0x47, 0xc5, 0xb2, 0xd6, 0xc8, 0x3f, 0x2a, 0x96, 0xf3, 0x8d, 0x42, 0xfb, + 0xd7, 0x1a, 0x40, 0xca, 0x21, 0xef, 0x43, 0x31, 0x31, 0x5a, 0x5f, 0xae, 0x4b, 0x78, 0x40, 0x91, + 0x45, 0xbe, 0x07, 0xe5, 0xf8, 0x77, 0x74, 0xf2, 0x86, 0x7d, 0x6d, 0x06, 0x25, 0xd4, 0xe4, 0x66, + 0x0a, 0xe9, 0xcd, 0xdc, 0xfb, 0x63, 0xe2, 0x87, 0xd0, 0x4f, 0x1a, 0x50, 0x1b, 0x1e, 0x77, 0xe8, + 0xb1, 0x7e, 0xd2, 0xff, 0xbc, 0xbf, 0x4f, 0x1b, 0x39, 0x72, 0x0d, 0xd6, 0x25, 0xf2, 0xd9, 0x11, + 0x7d, 0x7c, 0x78, 0xd4, 0xd9, 0x1b, 0x36, 0x34, 0xb2, 0x09, 0x37, 0x25, 0xf8, 0x64, 0xff, 0x98, + 0xf6, 0x7b, 0x3a, 0xdd, 0xef, 0x1d, 0xd1, 0xbd, 0x7d, 0x3a, 0x6c, 0xe4, 0xc9, 0x3a, 0x54, 0x87, + 0xc7, 0x47, 0x83, 0x58, 0x43, 0x81, 0x10, 0xa8, 0x23, 0x90, 0x2a, 0x28, 0x92, 0x5b, 0x70, 0x03, + 0xb1, 0x2b, 0xf2, 0x2b, 0xa4, 0x04, 0x05, 0xfa, 0xe9, 0xd3, 0xc6, 0x2a, 0x01, 0x58, 0xed, 0x7e, + 0x4a, 0x9f, 0xf6, 0x9f, 0x36, 0x4a, 0xdd, 0xee, 0x8b, 0x97, 0xad, 0xdc, 0x97, 0x2f, 0x5b, 0xb9, + 0xaf, 0x5e, 0xb6, 0xb4, 0x5f, 0xcd, 0x5a, 0xda, 0x9f, 0x66, 0x2d, 0xed, 0x6f, 0xb3, 0x96, 0xf6, + 0x62, 0xd6, 0xd2, 0xfe, 0x39, 0x6b, 0x69, 0xff, 0x9a, 0xb5, 0x72, 0x5f, 0xcd, 0x5a, 0xda, 0xef, + 0x5e, 0xb5, 0x72, 0x2f, 0x5e, 0xb5, 0x72, 0x5f, 0xbe, 0x6a, 0xe5, 0x3e, 0xaf, 0x65, 0xff, 0xaa, + 0x38, 0x5d, 0xc5, 0xd8, 0x7c, 0xf8, 0x9f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x0b, 0x28, 0x5f, 0x2b, + 0xd8, 0x10, 0x00, 0x00, } func (x ActionType) String() string { @@ -2570,6 +2579,9 @@ func (this *PrometheusScrapeSpec) Equal(that interface{}) bool { if this.KubeContext != that1.KubeContext { return false } + if this.Name != that1.Name { + return false + } return true } func (this *ClusterSpec) Equal(that interface{}) bool { @@ -3019,7 +3031,7 @@ func (this *PrometheusScrapeSpec) GoString() string { if this == nil { return "nil" } - s := make([]string, 0, 12) + s := make([]string, 0, 13) s = append(s, "&experimentpb.PrometheusScrapeSpec{") s = append(s, "Namespace: "+fmt.Sprintf("%#v", this.Namespace)+",\n") s = append(s, "MatchLabelKey: "+fmt.Sprintf("%#v", this.MatchLabelKey)+",\n") @@ -3043,6 +3055,7 @@ func (this *PrometheusScrapeSpec) GoString() string { } s = append(s, "KubeconfigPath: "+fmt.Sprintf("%#v", this.KubeconfigPath)+",\n") s = append(s, "KubeContext: "+fmt.Sprintf("%#v", this.KubeContext)+",\n") + s = append(s, "Name: "+fmt.Sprintf("%#v", this.Name)+",\n") s = append(s, "}") return strings.Join(s, "") } @@ -4191,6 +4204,13 @@ func (m *PrometheusScrapeSpec) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if len(m.Name) > 0 { + i -= len(m.Name) + copy(dAtA[i:], m.Name) + i = encodeVarintExperiment(dAtA, i, uint64(len(m.Name))) + i-- + dAtA[i] = 0x4a + } if len(m.KubeContext) > 0 { i -= len(m.KubeContext) copy(dAtA[i:], m.KubeContext) @@ -4965,6 +4985,10 @@ func (m *PrometheusScrapeSpec) Size() (n int) { if l > 0 { n += 1 + l + sovExperiment(uint64(l)) } + l = len(m.Name) + if l > 0 { + n += 1 + l + sovExperiment(uint64(l)) + } return n } @@ -5409,6 +5433,7 @@ func (this *PrometheusScrapeSpec) String() string { `MetricNames:` + mapStringForMetricNames + `,`, `KubeconfigPath:` + fmt.Sprintf("%v", this.KubeconfigPath) + `,`, `KubeContext:` + fmt.Sprintf("%v", this.KubeContext) + `,`, + `Name:` + fmt.Sprintf("%v", this.Name) + `,`, `}`, }, "") return s @@ -8683,6 +8708,38 @@ func (m *PrometheusScrapeSpec) Unmarshal(dAtA []byte) error { } m.KubeContext = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 9: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowExperiment + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthExperiment + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthExperiment + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Name = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipExperiment(dAtA[iNdEx:]) diff --git a/src/e2e_test/perf_tool/experimentpb/experiment.proto b/src/e2e_test/perf_tool/experimentpb/experiment.proto index 4d3a908d51d..0b276fa5b09 100644 --- a/src/e2e_test/perf_tool/experimentpb/experiment.proto +++ b/src/e2e_test/perf_tool/experimentpb/experiment.proto @@ -226,6 +226,9 @@ message PrometheusScrapeSpec { // Optional kubectl context name to use within the kubeconfig. // If empty, the current-context from the kubeconfig is used. string kube_context = 8; + // Identifier for this prometheus recorder, used by the CLI to target + // recorders with kubeconfig/kube_context overrides at runtime. + string name = 9; } // ClusterSpec specifies the type and size of cluster an experiment should run on. diff --git a/src/e2e_test/perf_tool/pkg/run/run.go b/src/e2e_test/perf_tool/pkg/run/run.go index e82db72c2c9..5561dd99f7e 100644 --- a/src/e2e_test/perf_tool/pkg/run/run.go +++ b/src/e2e_test/perf_tool/pkg/run/run.go @@ -50,6 +50,10 @@ type Runner struct { pxCtx *pixie.Context exporter exporter.Exporter containerRegistryRepo string + // KeepOnFailure, when true, skips teardown (stop vizier/workloads/recorders + // and cluster cleanup) if the experiment errors, so the cluster state can + // be inspected after the fact. Successful runs still tear down normally. + keepOnFailure bool clusterCtx *cluster.Context clusterCleanup func() @@ -74,6 +78,11 @@ func NewRunner(c cluster.Provider, pxCtx *pixie.Context, exp exporter.Exporter, } } +// SetKeepOnFailure toggles whether teardown is skipped on experiment failure. +func (r *Runner) SetKeepOnFailure(v bool) { + r.keepOnFailure = v +} + // RunExperiment runs an experiment according to the given ExperimentSpec. func (r *Runner) RunExperiment(ctx context.Context, expID uuid.UUID, spec *experimentpb.ExperimentSpec) error { commitTopoOrder, err := getTopoOrder() @@ -101,8 +110,16 @@ func (r *Runner) RunExperiment(ctx context.Context, expID uuid.UUID, spec *exper } }() - defer r.clusterCleanup() - defer r.clusterCtx.Close() + var runErr error + defer func() { + if r.keepOnFailure && runErr != nil { + log.WithError(runErr).Warn("Experiment failed; --keep_on_failure is set, leaving cluster state intact. " + + "Inspect with kubectl; you are responsible for manual cleanup (e.g. `px delete`, delete workload namespaces).") + return + } + r.clusterCleanup() + r.clusterCtx.Close() + }() var egCtx context.Context r.eg, egCtx = errgroup.WithContext(ctx) @@ -115,6 +132,7 @@ func (r *Runner) RunExperiment(ctx context.Context, expID uuid.UUID, spec *exper }) if err := r.eg.Wait(); err != nil { + runErr = err return err } @@ -133,8 +151,21 @@ func (r *Runner) RunExperiment(ctx context.Context, expID uuid.UUID, spec *exper return nil } -func (r *Runner) runActions(ctx context.Context, spec *experimentpb.ExperimentSpec) error { +func (r *Runner) runActions(ctx context.Context, spec *experimentpb.ExperimentSpec) (retErr error) { canceledErr := backoff.Permanent(context.Canceled) + // Collect start-action cleanups explicitly so we can skip them when + // --keep_on_failure is set and the experiment errors. + var cleanups []func() + defer func() { + failed := retErr != nil || ctx.Err() != nil + if r.keepOnFailure && failed { + log.Warn("Skipping per-action teardown due to --keep_on_failure") + return + } + for i := len(cleanups) - 1; i >= 0; i-- { + cleanups[i]() + } + }() for _, a := range spec.RunSpec.Actions { log.Tracef("started action %s", experimentpb.ActionType_name[int32(a.Type)]) if canceled := r.sendActionTimestamp(ctx, a, "begin"); canceled { @@ -146,19 +177,19 @@ func (r *Runner) runActions(ctx context.Context, spec *experimentpb.ExperimentSp if err != nil { return err } - defer cleanup() + cleanups = append(cleanups, cleanup) case experimentpb.START_WORKLOADS: cleanup, err := r.startWorkloads(ctx, spec, a.Name) if err != nil { return err } - defer cleanup() + cleanups = append(cleanups, cleanup) case experimentpb.START_METRIC_RECORDERS: cleanup, err := r.startMetricRecorders(ctx, spec, a.Name) if err != nil { return err } - defer cleanup() + cleanups = append(cleanups, cleanup) case experimentpb.STOP_VIZIER: if err := r.stopVizier(); err != nil { return err diff --git a/src/e2e_test/perf_tool/pkg/suites/BUILD.bazel b/src/e2e_test/perf_tool/pkg/suites/BUILD.bazel index 57b8a9fe368..39a35a7709e 100644 --- a/src/e2e_test/perf_tool/pkg/suites/BUILD.bazel +++ b/src/e2e_test/perf_tool/pkg/suites/BUILD.bazel @@ -26,6 +26,8 @@ go_library( "workloads.go", ], embedsrcs = [ + "scripts/clickhouse_export.pxl", + "scripts/clickhouse_read.pxl", "scripts/healthcheck/http_data_in_namespace.pxl", "scripts/healthcheck/vizier.pxl", "scripts/heap_size.pxl", diff --git a/src/e2e_test/perf_tool/pkg/suites/experiments.go b/src/e2e_test/perf_tool/pkg/suites/experiments.go index 9ae11fd92ff..af0671886f0 100644 --- a/src/e2e_test/perf_tool/pkg/suites/experiments.go +++ b/src/e2e_test/perf_tool/pkg/suites/experiments.go @@ -347,6 +347,132 @@ func HTTPLoadApplicationOverheadExperiment( return e } +// ClickHouseExportExperiment drives load against Pixie's ClickHouse export +// path. An HTTP loadtest populates http_events on the PEMs, and the +// clickhouse_export PxL script runs on a tight period to continuously export +// a windowed slice of http_events to ClickHouse. +func ClickHouseExportExperiment( + numConnections int, + targetRPS int, + metricPeriod time.Duration, + exportPeriod time.Duration, + exportWindow time.Duration, + clickhouseDSN string, + clickhouseTable string, + predeployDur time.Duration, + dur time.Duration, +) *experimentpb.ExperimentSpec { + e := &experimentpb.ExperimentSpec{ + VizierSpec: VizierWorkload(), + WorkloadSpecs: []*experimentpb.WorkloadSpec{ + HTTPLoadTestWorkload(numConnections, targetRPS, true), + }, + MetricSpecs: []*experimentpb.MetricSpec{ + ProcessStatsMetrics(metricPeriod), + // Stagger the second query a little bit because of query stability issues. + HeapMetrics(metricPeriod + (2 * time.Second)), + ClickHouseExportLoadMetric(exportPeriod, clickhouseDSN, clickhouseTable, exportWindow), + ClickHouseOperatorMetrics(metricPeriod), + }, + RunSpec: &experimentpb.RunSpec{ + Actions: []*experimentpb.ActionSpec{ + { + Type: experimentpb.START_VIZIER, + }, + { + Type: experimentpb.START_METRIC_RECORDERS, + }, + { + Type: experimentpb.BURNIN, + Duration: types.DurationProto(predeployDur), + }, + { + Type: experimentpb.START_WORKLOADS, + }, + { + Type: experimentpb.RUN, + Duration: types.DurationProto(dur), + }, + { + Type: experimentpb.STOP_METRIC_RECORDERS, + }, + }, + }, + ClusterSpec: DefaultCluster, + } + e = addTags(e, + "workload/clickhouse-export", + fmt.Sprintf("parameter/num_conns/%d", numConnections), + fmt.Sprintf("parameter/target_rps/%d", targetRPS), + fmt.Sprintf("parameter/export_window/%s", exportWindow), + ) + return e +} + +// ClickHouseReadExperiment drives load against Pixie's ClickHouse read path. +// HTTP loadtest populates http_events; a (placeholder) read-load workload +// drives sustained pressure against ClickHouse; the clickhouse_read PxL +// script periodically queries the ClickHouse source from Pixie so we can +// observe Pixie-side read performance as well. +func ClickHouseReadExperiment( + numConnections int, + targetRPS int, + metricPeriod time.Duration, + readPeriod time.Duration, + readWindow time.Duration, + clickhouseDSN string, + clickhouseTable string, + predeployDur time.Duration, + dur time.Duration, +) *experimentpb.ExperimentSpec { + e := &experimentpb.ExperimentSpec{ + VizierSpec: VizierWorkload(), + WorkloadSpecs: []*experimentpb.WorkloadSpec{ + HTTPLoadTestWorkload(numConnections, targetRPS, true), + ClickHouseReadLoadWorkload(), + }, + MetricSpecs: []*experimentpb.MetricSpec{ + ProcessStatsMetrics(metricPeriod), + // Stagger the second query a little bit because of query stability issues. + HeapMetrics(metricPeriod + (2 * time.Second)), + ClickHouseReadLoadMetric(readPeriod, clickhouseDSN, clickhouseTable, readWindow), + ClickHouseOperatorMetrics(metricPeriod), + }, + RunSpec: &experimentpb.RunSpec{ + Actions: []*experimentpb.ActionSpec{ + { + Type: experimentpb.START_VIZIER, + }, + { + Type: experimentpb.START_METRIC_RECORDERS, + }, + { + Type: experimentpb.BURNIN, + Duration: types.DurationProto(predeployDur), + }, + { + Type: experimentpb.START_WORKLOADS, + }, + { + Type: experimentpb.RUN, + Duration: types.DurationProto(dur), + }, + { + Type: experimentpb.STOP_METRIC_RECORDERS, + }, + }, + }, + ClusterSpec: DefaultCluster, + } + e = addTags(e, + "workload/clickhouse-read", + fmt.Sprintf("parameter/num_conns/%d", numConnections), + fmt.Sprintf("parameter/target_rps/%d", targetRPS), + fmt.Sprintf("parameter/read_window/%s", readWindow), + ) + return e +} + func addTags(e *experimentpb.ExperimentSpec, tags ...string) *experimentpb.ExperimentSpec { if e.Tags == nil { e.Tags = []string{} diff --git a/src/e2e_test/perf_tool/pkg/suites/metrics.go b/src/e2e_test/perf_tool/pkg/suites/metrics.go index aaa7d75bbd0..9142f11d4dd 100644 --- a/src/e2e_test/perf_tool/pkg/suites/metrics.go +++ b/src/e2e_test/perf_tool/pkg/suites/metrics.go @@ -37,6 +37,17 @@ var heapSizeScript string //go:embed scripts/http_data_loss.pxl var httpDataLossScript string +//go:embed scripts/clickhouse_export.pxl +var clickhouseExportScript string + +//go:embed scripts/clickhouse_read.pxl +var clickhouseReadScript string + +// ClickHouseOperatorPromRecorderName is the canonical name used by the CLI's +// --prom_recorder_override flag to retarget the ClickHouse operator scraper at +// a different cluster (kubeconfig/kube_context). +const ClickHouseOperatorPromRecorderName = "clickhouse-operator" + // ProcessStatsMetrics adds a metric spec that collects process stats such as rss,vsize, and cpu_usage. func ProcessStatsMetrics(period time.Duration) *pb.MetricSpec { return &pb.MetricSpec{ @@ -133,6 +144,98 @@ func ProtocolLoadtestPromMetrics(scrapePeriod time.Duration) *pb.MetricSpec { } } +// ClickHouseExportLoadMetric runs the clickhouse export PxL script on a tight +// period to drive load against the ClickHouse write path, and reports the +// row count of each export as a metric. +func ClickHouseExportLoadMetric(period time.Duration, dsn string, table string, window time.Duration) *pb.MetricSpec { + return &pb.MetricSpec{ + MetricType: &pb.MetricSpec_PxL{ + PxL: &pb.PxLScriptSpec{ + Script: clickhouseExportScript, + Streaming: false, + CollectionPeriod: types.DurationProto(period), + TemplateValues: map[string]string{ + "dsn": dsn, + "table": table, + "window": window.String(), + }, + TableOutputs: map[string]*pb.PxLScriptOutputList{ + "*": { + Outputs: []*pb.PxLScriptOutputSpec{ + singleMetricOutputWithPodNodeName("row_count", "clickhouse_export_rows"), + }, + }, + }, + }, + }, + } +} + +// ClickHouseReadLoadMetric runs the clickhouse read PxL script on a tight +// period to drive load against the ClickHouse read path, and reports the +// row count of each readback as a metric. +func ClickHouseReadLoadMetric(period time.Duration, dsn string, table string, window time.Duration) *pb.MetricSpec { + return &pb.MetricSpec{ + MetricType: &pb.MetricSpec_PxL{ + PxL: &pb.PxLScriptSpec{ + Script: clickhouseReadScript, + Streaming: false, + CollectionPeriod: types.DurationProto(period), + TemplateValues: map[string]string{ + "dsn": dsn, + "table": table, + "window": window.String(), + }, + TableOutputs: map[string]*pb.PxLScriptOutputList{ + "*": { + Outputs: []*pb.PxLScriptOutputSpec{ + singleMetricOutputWithPodNodeName("row_count", "clickhouse_read_rows"), + }, + }, + }, + }, + }, + } +} + +// ClickHouseOperatorMetrics scrapes the Altinity clickhouse-operator's +// metrics-exporter sidecar (`ch-metrics` port 8888), which proxies per-shard +// ClickHouse server metrics. Named so the --prom_recorder_override CLI flag +// can point it at a different cluster via kubeconfig/kube_context. +func ClickHouseOperatorMetrics(scrapePeriod time.Duration) *pb.MetricSpec { + return &pb.MetricSpec{ + MetricType: &pb.MetricSpec_Prom{ + Prom: &pb.PrometheusScrapeSpec{ + Name: ClickHouseOperatorPromRecorderName, + Namespace: "clickhouse", + MatchLabelKey: "app.kubernetes.io/name", + MatchLabelValue: "altinity-clickhouse-operator", + Port: 8888, + ScrapePeriod: types.DurationProto(scrapePeriod), + MetricNames: map[string]string{ + // Gauges: in-flight load on CH servers. + "chi_clickhouse_metric_Query": "clickhouse_active_queries", + "chi_clickhouse_metric_TCPConnection": "clickhouse_tcp_connections", + "chi_clickhouse_metric_HTTPConnection": "clickhouse_http_connections", + "chi_clickhouse_metric_MemoryTracking": "clickhouse_memory_tracking_bytes", + "chi_clickhouse_metric_BackgroundMergesAndMutationsPoolTask": "clickhouse_background_merge_tasks", + "chi_clickhouse_metric_PartsActive": "clickhouse_parts_active", + // Counters: throughput and errors. + "chi_clickhouse_event_Query": "clickhouse_queries_total", + "chi_clickhouse_event_InsertedRows": "clickhouse_inserted_rows_total", + "chi_clickhouse_event_SelectedRows": "clickhouse_selected_rows_total", + "chi_clickhouse_event_FailedQuery": "clickhouse_failed_queries_total", + "chi_clickhouse_event_NetworkSendBytes": "clickhouse_network_send_bytes_total", + "chi_clickhouse_event_NetworkReceiveBytes": "clickhouse_network_receive_bytes_total", + // Per-table gauges: storage-side pressure. + "chi_clickhouse_table_parts_rows": "clickhouse_table_parts_rows", + "chi_clickhouse_table_parts_bytes": "clickhouse_table_parts_bytes", + }, + }, + }, + } +} + func singleMetricOutputWithPodNodeName(col string, newName ...string) *pb.PxLScriptOutputSpec { metricName := col if len(newName) > 0 { diff --git a/src/e2e_test/perf_tool/pkg/suites/scripts/clickhouse_export.pxl b/src/e2e_test/perf_tool/pkg/suites/scripts/clickhouse_export.pxl new file mode 100644 index 00000000000..dfcc77a9ad1 --- /dev/null +++ b/src/e2e_test/perf_tool/pkg/suites/scripts/clickhouse_export.pxl @@ -0,0 +1,39 @@ +# Copyright 2018- The Pixie Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +# Exports a windowed slice of http_events to ClickHouse on every invocation, +# producing sustained load on the export path. px._pem_hostname() ensures the +# Map runs on the PEM so each row carries the correct hostname. + +import px + +df = px.DataFrame('http_events', start_time='-{{.TemplateValues.window}}') +df.hostname = px._pem_hostname() +px.export(df, px.otel.ClickHouseRows( + table='{{.TemplateValues.table}}', + endpoint=px.otel.Endpoint( + url='{{.TemplateValues.dsn}}', + ), +)) + +# Emit one metric row per invocation so we can chart export cadence and row +# counts. The metric recorder will pick up row_count as a single metric. +metric_df = df.groupby([]).agg(row_count=('time_', px.count)) +metric_df.timestamp = px.now() +metric_df.node_name = px._exec_hostname() +metric_df.pod = 'clickhouse-export-driver' +metric_df = metric_df[['timestamp', 'node_name', 'pod', 'row_count']] +px.display(metric_df, 'export_stats') diff --git a/src/e2e_test/perf_tool/pkg/suites/scripts/clickhouse_read.pxl b/src/e2e_test/perf_tool/pkg/suites/scripts/clickhouse_read.pxl new file mode 100644 index 00000000000..8975e21e879 --- /dev/null +++ b/src/e2e_test/perf_tool/pkg/suites/scripts/clickhouse_read.pxl @@ -0,0 +1,37 @@ +# Copyright 2018- The Pixie Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +# Reads a windowed slice of http_events back from ClickHouse on every +# invocation, exercising the ClickHouse read path and Pixie's ClickHouse +# source plan. Emits a metric row reporting the number of rows returned so +# we can track read throughput. + +import px + +df = px.DataFrame( + '{{.TemplateValues.table}}', + clickhouse_dsn='{{.TemplateValues.dsn}}', + start_time='-{{.TemplateValues.window}}', +) + +# A light-weight aggregation ensures ClickHouse actually has to scan the +# window rather than just serving the first page of rows. +metric_df = df.groupby([]).agg(row_count=('time_', px.count)) +metric_df.timestamp = px.now() +metric_df.node_name = px._exec_hostname() +metric_df.pod = 'clickhouse-read-driver' +metric_df = metric_df[['timestamp', 'node_name', 'pod', 'row_count']] +px.display(metric_df, 'read_stats') diff --git a/src/e2e_test/perf_tool/pkg/suites/suites.go b/src/e2e_test/perf_tool/pkg/suites/suites.go index 0c5e4b9a404..66a1ca432c7 100644 --- a/src/e2e_test/perf_tool/pkg/suites/suites.go +++ b/src/e2e_test/perf_tool/pkg/suites/suites.go @@ -30,9 +30,10 @@ type ExperimentSuite func() map[string]*pb.ExperimentSpec // ExperimentSuiteRegistry contains all the ExperimentSuite, keyed by name. var ExperimentSuiteRegistry = map[string]ExperimentSuite{ - "nightly": nightlyExperimentSuite, - "http-grid": httpGridSuite, - "k8ssandra": k8ssandraExperimentSuite, + "nightly": nightlyExperimentSuite, + "http-grid": httpGridSuite, + "k8ssandra": k8ssandraExperimentSuite, + "clickhouse-exec": clickhouseExecSuite, } func nightlyExperimentSuite() map[string]*pb.ExperimentSpec { @@ -73,6 +74,55 @@ func k8ssandraExperimentSuite() map[string]*pb.ExperimentSpec { return exps } +// clickhouseExecSuite covers the two sides of Pixie's ClickHouse integration +// under load: the write/export path and the read/query path. Both experiments +// share the same metric shape (process/heap/clickhouse-operator) so results +// can be compared directly. +// +// The ClickHouse operator metrics are scraped via the prometheus recorder +// named "clickhouse-operator" -- point the CLI at the correct cluster with: +// +// --prom_recorder_override clickhouse-operator=/path/to/kubeconfig:my-ctx +func clickhouseExecSuite() map[string]*pb.ExperimentSpec { + defaultMetricPeriod := 30 * time.Second + preDur := 5 * time.Minute + // preDur := 2 * time.Minute + dur := 30 * time.Minute + // dur := 5 * time.Minute + httpNumConns := 100 + httpTargetRPS := 3000 + + // Tight cadence on the export/read scripts to apply real pressure. + exportPeriod := 5 * time.Second + exportWindow := 30 * time.Second + readPeriod := 5 * time.Second + readWindow := 5 * time.Minute + + clickhouseDSN := "pixie:pixie_password@clickhouse.forensic.austrianopencloudcommunity.org:9000/default" + clickhouseTable := "http_events" + + exps := map[string]*pb.ExperimentSpec{ + "clickhouse-export": ClickHouseExportExperiment( + httpNumConns, httpTargetRPS, + defaultMetricPeriod, + exportPeriod, exportWindow, + clickhouseDSN, clickhouseTable, + preDur, dur, + ), + "clickhouse-read": ClickHouseReadExperiment( + httpNumConns, httpTargetRPS, + defaultMetricPeriod, + readPeriod, readWindow, + clickhouseDSN, clickhouseTable, + preDur, dur, + ), + } + for _, e := range exps { + addTags(e, "suite/clickhouse-exec") + } + return exps +} + func httpGridSuite() map[string]*pb.ExperimentSpec { defaultMetricPeriod := 30 * time.Second preDur := 5 * time.Minute diff --git a/src/e2e_test/perf_tool/pkg/suites/workloads.go b/src/e2e_test/perf_tool/pkg/suites/workloads.go index c819b794649..dd91bc02715 100644 --- a/src/e2e_test/perf_tool/pkg/suites/workloads.go +++ b/src/e2e_test/perf_tool/pkg/suites/workloads.go @@ -215,6 +215,36 @@ func OnlineBoutiqueWorkload() *pb.WorkloadSpec { } } +// ClickHouseReadLoadWorkload deploys the (future) skaffold application that +// generates sustained ClickHouse read traffic alongside the Pixie read +// experiment. The skaffold path below is a placeholder; wire up the real +// application once it exists in the tree. +func ClickHouseReadLoadWorkload() *pb.WorkloadSpec { + return &pb.WorkloadSpec{ + Name: "clickhouse-read-load", + DeploySteps: []*pb.DeployStep{ + { + DeployType: &pb.DeployStep_Skaffold{ + Skaffold: &pb.SkaffoldDeploy{ + // TODO(ddelnano): replace with the real skaffold path once + // the ClickHouse read-load generator app lands. + SkaffoldPath: "src/e2e_test/clickhouse_read_load/skaffold.yaml", + }, + }, + }, + }, + Healthchecks: []*pb.HealthCheck{ + { + CheckType: &pb.HealthCheck_K8S{ + K8S: &pb.K8SPodsReadyCheck{ + Namespace: "px-clickhouse-read-load", + }, + }, + }, + }, + } +} + // KafkaWorkload returns the WorkloadSpec to deploy the kafka demo. func KafkaWorkload() *pb.WorkloadSpec { return &pb.WorkloadSpec{ diff --git a/src/e2e_test/perf_tool/ui/index.html b/src/e2e_test/perf_tool/ui/index.html index 070a20eb2d1..42de8865a5f 100644 --- a/src/e2e_test/perf_tool/ui/index.html +++ b/src/e2e_test/perf_tool/ui/index.html @@ -155,6 +155,21 @@

Select Experiments

+ +

ClickHouse Metrics

+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -516,13 +531,28 @@

Select Experiments

{ id: 'exp-heap', title: 'Heap (ignoring table store)', names: ['heap_size_bytes', 'table_size'], unit: 'bytes', computed: true }, { id: 'exp-rss', title: 'RSS', names: ['rss'], unit: 'bytes' }, { id: 'exp-http-data-loss', title: 'HTTP Data Loss', names: ['http_data_loss'], unit: 'percent' }, + // ClickHouse: PxL-driven export/read row counts per invocation. + { id: 'exp-ch-pxl-rows', title: 'ClickHouse Export/Read Rows (PxL)', names: ['clickhouse_export_rows', 'clickhouse_read_rows'], unit: 'count' }, + // ClickHouse server gauges (Altinity operator metrics-exporter). + { id: 'exp-ch-active-queries', title: 'ClickHouse Active Queries', names: ['clickhouse_active_queries'], unit: 'count' }, + { id: 'exp-ch-connections', title: 'ClickHouse Connections', names: ['clickhouse_tcp_connections', 'clickhouse_http_connections'], unit: 'count' }, + { id: 'exp-ch-memory', title: 'ClickHouse Memory Tracking', names: ['clickhouse_memory_tracking_bytes'], unit: 'bytes' }, + { id: 'exp-ch-parts-active', title: 'ClickHouse Active Parts', names: ['clickhouse_parts_active'], unit: 'count' }, + { id: 'exp-ch-bg-merges', title: 'ClickHouse Background Merge Tasks', names: ['clickhouse_background_merge_tasks'], unit: 'count' }, + { id: 'exp-ch-table-rows', title: 'ClickHouse Table Parts Rows', names: ['clickhouse_table_parts_rows'], unit: 'count' }, + { id: 'exp-ch-table-bytes', title: 'ClickHouse Table Parts Bytes', names: ['clickhouse_table_parts_bytes'], unit: 'bytes' }, + // ClickHouse server counters, charted as rate-of-change per second. + { id: 'exp-ch-query-rate', title: 'ClickHouse Query Rate', names: ['clickhouse_queries_total', 'clickhouse_failed_queries_total'], unit: 'rate', counter: true }, + { id: 'exp-ch-row-rate', title: 'ClickHouse Row I/O Rate', names: ['clickhouse_inserted_rows_total', 'clickhouse_selected_rows_total'], unit: 'rate', counter: true, + secondaryAxisNames: ['clickhouse_selected_rows_total'], + primaryAxisTitle: 'Inserted rows/s', secondaryAxisTitle: 'Selected rows/s' }, + { id: 'exp-ch-network-rate', title: 'ClickHouse Network Rate', names: ['clickhouse_network_send_bytes_total', 'clickhouse_network_receive_bytes_total'], unit: 'bytes-rate', counter: true }, ]; function buildExperimentQuery(experimentIds, metricConfig) { const idList = experimentIds.map(id => `'${id}'`).join(','); const nameList = metricConfig.names.map(n => `'${n}'`).join(','); - let valueExpr; if (metricConfig.computed && metricConfig.names.includes('heap_size_bytes')) { // Self-join to subtract table_size from heap_size_bytes return ` @@ -535,9 +565,10 @@

Select Experiments

heap_data AS ( SELECT r.experiment_id, + 'heap_size_bytes' as name, date_diff('second', mt.min_ts, r.timestamp) as elapsed_s, r.value - COALESCE(t.value, 0) as metric_value, - rtrim(regexp_extract(COALESCE(r.tag_pod, ''), '(?:[a-z\\-]+/)?((?:[a-z]+-)+)', 1), '-') as pod + regexp_replace(COALESCE(r.tag_pod, ''), '^[^/]+/', '') as pod FROM results r JOIN min_times mt USING (experiment_id) LEFT JOIN results t @@ -548,12 +579,55 @@

Select Experiments

WHERE r.experiment_id IN (${idList}) AND r.name = 'heap_size_bytes' ) - SELECT experiment_id, elapsed_s, metric_value, pod + SELECT experiment_id, name, elapsed_s, metric_value, pod FROM heap_data ORDER BY experiment_id, elapsed_s `; } + if (metricConfig.counter) { + // Counter: compute per-second rate via LAG over (experiment, name, pod). + // Drop negative deltas (counter resets on restarts). + return ` + WITH min_times AS ( + SELECT experiment_id, min(timestamp) as min_ts + FROM results + WHERE experiment_id IN (${idList}) + GROUP BY experiment_id + ), + base AS ( + SELECT + r.experiment_id, + r.name, + r.timestamp, + mt.min_ts, + r.value, + COALESCE(r.tag_pod, '') as tag_pod_key, + regexp_replace(COALESCE(r.tag_pod, ''), '^[^/]+/', '') as pod + FROM results r + JOIN min_times mt USING (experiment_id) + WHERE r.experiment_id IN (${idList}) + AND r.name IN (${nameList}) + ), + with_prev AS ( + SELECT + *, + lag(value) OVER (PARTITION BY experiment_id, name, tag_pod_key ORDER BY timestamp) as prev_value, + lag(timestamp) OVER (PARTITION BY experiment_id, name, tag_pod_key ORDER BY timestamp) as prev_ts + FROM base + ) + SELECT + experiment_id, + name, + date_diff('second', min_ts, timestamp) as elapsed_s, + (value - prev_value) / NULLIF(date_diff('millisecond', prev_ts, timestamp) / 1000.0, 0) as metric_value, + pod + FROM with_prev + WHERE prev_value IS NOT NULL AND value >= prev_value + ORDER BY experiment_id, elapsed_s + `; + } + return ` WITH min_times AS ( SELECT experiment_id, min(timestamp) as min_ts @@ -563,13 +637,14 @@

Select Experiments

) SELECT r.experiment_id, + r.name, date_diff('second', mt.min_ts, r.timestamp) as elapsed_s, r.value as metric_value, - rtrim(regexp_extract(COALESCE(r.tag_pod, ''), '(?:[a-z\\-]+/)?((?:[a-z]+-)+)', 1), '-') as pod + regexp_replace(COALESCE(r.tag_pod, ''), '^[^/]+/', '') as pod FROM results r JOIN min_times mt USING (experiment_id) WHERE r.experiment_id IN (${idList}) - AND r.name = '${metricConfig.names[0]}' + AND r.name IN (${nameList}) ORDER BY r.experiment_id, elapsed_s `; } @@ -672,24 +747,34 @@

Select Experiments

return; } - const isBytesUnit = config.unit === 'bytes'; + const isBytesUnit = config.unit === 'bytes' || config.unit === 'bytes-rate'; + const multiName = (config.names && config.names.length > 1) && !config.computed; + const secondaryAxisSet = new Set(config.secondaryAxisNames || []); - // Group by experiment_id + pod + // Group by experiment_id + pod (+ name when the config plots multiple names). const groups = {}; for (const row of data) { - const key = `${row.experiment_id.slice(0, 8)}...${row.pod ? ' (' + row.pod + ')' : ''}`; + const idShort = row.experiment_id.slice(0, 8); + const podPart = row.pod ? ` (${row.pod})` : ''; + const namePart = multiName && row.name ? `${row.name} — ` : ''; + const key = `${namePart}${idShort}${podPart}`; if (!groups[key]) groups[key] = []; groups[key].push(row); } - const traces = Object.entries(groups).map(([key, rows]) => ({ - x: rows.map(r => r.elapsed_s), - y: rows.map(r => isBytesUnit ? r.metric_value / 1024 / 1024 : r.metric_value), - name: key, - mode: 'lines', - type: 'scatter', - hovertemplate: `${key}
Time: %{x}s
Value: %{y:.4f}`, - })); + const traces = Object.entries(groups).map(([key, rows]) => { + const metricName = rows[0] ? rows[0].name : null; + const onSecondary = metricName && secondaryAxisSet.has(metricName); + return { + x: rows.map(r => r.elapsed_s), + y: rows.map(r => isBytesUnit ? r.metric_value / 1024 / 1024 : r.metric_value), + name: key, + mode: 'lines', + type: 'scatter', + yaxis: onSecondary ? 'y2' : 'y', + hovertemplate: `${key}
Time: %{x}s
Value: %{y:.4f}`, + }; + }); // Add action markers as vertical lines const runActions = actions.filter(a => a.action_type === 'run'); @@ -706,10 +791,18 @@

Select Experiments

yanchor: 'bottom', })); + let yaxisTitle; + if (config.unit === 'bytes-rate') yaxisTitle = 'MiB/s'; + else if (config.unit === 'bytes') yaxisTitle = 'MiB'; + else if (config.unit === 'percent') yaxisTitle = 'Ratio'; + else if (config.unit === 'rate') yaxisTitle = 'per Second'; + else if (config.unit === 'count') yaxisTitle = 'Count'; + else yaxisTitle = 'Value'; + const layout = { title: { text: config.title, font: { size: 14 } }, xaxis: { title: 'Time Elapsed (s)' }, - yaxis: { title: isBytesUnit ? 'MiB' : (config.unit === 'percent' ? 'Ratio' : 'Value') }, + yaxis: { title: config.primaryAxisTitle || yaxisTitle }, margin: { t: 40, b: 60, l: 60, r: 20 }, hovermode: 'closest', showlegend: true, @@ -717,6 +810,42 @@

Select Experiments

shapes, annotations, }; + if (secondaryAxisSet.size > 0) { + // Compute per-axis ranges from the data so each axis scales only to + // the traces assigned to it (Plotly's auto-range occasionally syncs + // overlaid axes otherwise). + let minP = Infinity, maxP = -Infinity; + let minS = Infinity, maxS = -Infinity; + for (const row of data) { + if (row.metric_value == null) continue; + const v = isBytesUnit ? row.metric_value / 1024 / 1024 : row.metric_value; + if (secondaryAxisSet.has(row.name)) { + if (v < minS) minS = v; + if (v > maxS) maxS = v; + } else { + if (v < minP) minP = v; + if (v > maxP) maxP = v; + } + } + layout.yaxis2 = { + title: config.secondaryAxisTitle || yaxisTitle, + overlaying: 'y', + side: 'right', + anchor: 'x', + }; + if (maxP > -Infinity) { + const lo = Math.min(0, minP); + const hi = maxP > 0 ? maxP * 1.1 : maxP + 1; + layout.yaxis.range = [lo, hi]; + layout.yaxis.autorange = false; + } + if (maxS > -Infinity) { + const lo = Math.min(0, minS); + const hi = maxS > 0 ? maxS * 1.1 : maxS + 1; + layout.yaxis2.range = [lo, hi]; + layout.yaxis2.autorange = false; + } + } Plotly.newPlot(containerId, traces, layout, { responsive: true }); } From 06a8d3a592d04c7bb809915ea671cdf31fdbdef1 Mon Sep 17 00:00:00 2001 From: Dom Del Nano Date: Sun, 19 Apr 2026 20:57:06 -0700 Subject: [PATCH 08/10] Ensure px delete works with external k8s ApiService Signed-off-by: Dom Del Nano --- src/utils/shared/k8s/delete.go | 48 +++++++++++++++++++++++++++++++--- 1 file changed, 45 insertions(+), 3 deletions(-) diff --git a/src/utils/shared/k8s/delete.go b/src/utils/shared/k8s/delete.go index 3adb2c8b986..689e0f8be54 100644 --- a/src/utils/shared/k8s/delete.go +++ b/src/utils/shared/k8s/delete.go @@ -29,7 +29,9 @@ import ( "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/cli-runtime/pkg/genericclioptions" "k8s.io/cli-runtime/pkg/printers" @@ -44,6 +46,12 @@ import ( cmdwait "k8s.io/kubectl/pkg/cmd/wait" ) +var apiServiceGVR = schema.GroupVersionResource{ + Group: "apiregistration.k8s.io", + Version: "v1", + Resource: "apiservices", +} + // ObjectDeleter has methods to delete K8s objects and wait for them. This code is adopted from `kubectl delete`. type ObjectDeleter struct { Namespace string @@ -110,6 +118,32 @@ func (o *ObjectDeleter) DeleteNamespace() error { return err } +// getAggregatedGroupVersions returns the set of group/versions that are served +// by an aggregated APIService (spec.service is non-nil). Resources in those +// groups are skipped during cluster-wide deletion sweeps because aggregated +// servers frequently advertise the delete verb on read-only virtual resources +// and fail the call with "operation not supported". +func (o *ObjectDeleter) getAggregatedGroupVersions() (sets.String, error) { + out := sets.NewString() + list, err := o.dynamicClient.Resource(apiServiceGVR).List(context.TODO(), metav1.ListOptions{}) + if err != nil { + if errors.IsNotFound(err) || meta.IsNoMatchError(err) { + return out, nil + } + return nil, err + } + for _, item := range list.Items { + svc, found, err := unstructured.NestedMap(item.Object, "spec", "service") + if err != nil || !found || svc == nil { + continue + } + group, _, _ := unstructured.NestedString(item.Object, "spec", "group") + version, _, _ := unstructured.NestedString(item.Object, "spec", "version") + out.Insert(schema.GroupVersion{Group: group, Version: version}.String()) + } + return out, nil +} + func (o *ObjectDeleter) getDeletableResourceTypes() ([]string, error) { discoveryClient, err := o.rcg.ToDiscoveryClient() if err != nil { @@ -121,11 +155,19 @@ func (o *ObjectDeleter) getDeletableResourceTypes() ([]string, error) { return nil, err } + aggregated, err := o.getAggregatedGroupVersions() + if err != nil { + return nil, err + } + resources := []string{} for _, list := range lists { if len(list.APIResources) == 0 { continue } + if aggregated.Has(list.GroupVersion) { + continue + } for _, resource := range list.APIResources { if len(resource.Verbs) == 0 { @@ -145,6 +187,9 @@ func (o *ObjectDeleter) DeleteByLabel(selector string, resourceKinds ...string) if err := o.initRestClientGetter(); err != nil { return 0, err } + if err := o.initDynamicClient(); err != nil { + return 0, err + } b := resource.NewBuilder(o.rcg) if len(resourceKinds) == 0 { @@ -169,9 +214,6 @@ func (o *ObjectDeleter) DeleteByLabel(selector string, resourceKinds ...string) if err != nil { return 0, err } - if err := o.initDynamicClient(); err != nil { - return 0, err - } return o.runDelete(r) } From af340a5fff054f3c9c8d822e4da1cefd8996dcf3 Mon Sep 17 00:00:00 2001 From: entlein Date: Tue, 21 Apr 2026 00:00:00 +0200 Subject: [PATCH 09/10] experiment with the adaptive feature Signed-off-by: entlein --- .../bootstrap/adaptive_export_deployment.yaml | 23 ++ .../services/adaptive_export/cmd/main.go | 163 ++++++++++---- .../adaptive_export/internal/config/config.go | 205 +++++++++++++----- .../adaptive_export/internal/pixie/pixie.go | 12 + 4 files changed, 306 insertions(+), 97 deletions(-) diff --git a/k8s/vizier/bootstrap/adaptive_export_deployment.yaml b/k8s/vizier/bootstrap/adaptive_export_deployment.yaml index dcb9305bbb4..6a100a9b191 100644 --- a/k8s/vizier/bootstrap/adaptive_export_deployment.yaml +++ b/k8s/vizier/bootstrap/adaptive_export_deployment.yaml @@ -57,6 +57,29 @@ spec: value: "10" - name: DETECTION_LOOKBACK_SEC value: "30" + # EXPORT_MODE controls the reconcile behaviour: + # auto - detection drives on/off (default) + # always - plugin always enabled (bypass detection) + # never - plugin always disabled and ch-* scripts purged + - name: EXPORT_MODE + value: "auto" + # Number of consecutive empty detection ticks before auto-disable fires. + - name: EXPORT_QUIET_TICKS + value: "6" + # Optional overrides for the ClickHouse PxL scripts. When unset they are + # parsed from CLICKHOUSE_DSN. Individual fields win over the parsed DSN. + - name: KUBESCAPE_TABLE + value: "kubescape_logs" + # - name: CLICKHOUSE_HOST + # value: "clickhouse-forensic-soc-db.clickhouse.svc.cluster.local" + # - name: CLICKHOUSE_PORT + # value: "9000" + # - name: CLICKHOUSE_USER + # value: "otelcollector" + # - name: CLICKHOUSE_PASSWORD + # value: "changeme" + # - name: CLICKHOUSE_DATABASE + # value: "default" securityContext: allowPrivilegeEscalation: false capabilities: diff --git a/src/vizier/services/adaptive_export/cmd/main.go b/src/vizier/services/adaptive_export/cmd/main.go index b283fe8083b..824cbf8a579 100644 --- a/src/vizier/services/adaptive_export/cmd/main.go +++ b/src/vizier/services/adaptive_export/cmd/main.go @@ -42,21 +42,20 @@ const ( ) const ( - // TODO(ddelnano): Clickhouse configuration should come from plugin config. - schemaCreationScript = ` + schemaCreationScriptTmpl = ` import px px.display(px.CreateClickHouseSchemas( - host="hyperdx-hdx-oss-v2-clickhouse.click.svc.cluster.local", - port=9000, - username="otelcollector", - password="otelcollectorpass", - database="default" + host="%s", + port=%s, + username="%s", + password="%s", + database="%s" )) ` - detectionScript = ` + detectionScriptTmpl = ` import px -df = px.DataFrame('kubescape_logs', clickhouse_dsn='otelcollector:otelcollectorpass@hyperdx-hdx-oss-v2-clickhouse.click.svc.cluster.local:9000/default', start_time='-%ds') +df = px.DataFrame('%s', clickhouse_dsn='%s', start_time='-%ds') df.alert = df.message df.namespace = px.pluck(df.RuntimeK8sDetails, "podNamespace") df.podName = px.pluck(df.RuntimeK8sDetails, "podName") @@ -66,6 +65,15 @@ px.display(df) ` ) +func renderSchemaScript(cfg config.ClickHouse) string { + return fmt.Sprintf(schemaCreationScriptTmpl, + cfg.Host(), cfg.Port(), cfg.User(), cfg.Password(), cfg.Database()) +} + +func renderDetectionScript(cfg config.ClickHouse, lookback int64) string { + return fmt.Sprintf(detectionScriptTmpl, cfg.Table(), cfg.DSN(), lookback) +} + func main() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -95,9 +103,9 @@ func main() { } // Start schema creation background task - go runSchemaCreationTask(ctx, pxClient, clusterId) + go runSchemaCreationTask(ctx, pxClient, clusterId, cfg.ClickHouse()) - // Start detection script that monitors for when to enable persistence + // Start detection + reconcile loop that turns the retention plugin on/off go runDetectionTask(ctx, pxClient, pluginClient, cfg, clusterId, clusterName) // Wait for signal to shutdown @@ -110,34 +118,29 @@ func main() { time.Sleep(1 * time.Second) } -func runSchemaCreationTask(ctx context.Context, client *pxapi.Client, clusterID string) { +func runSchemaCreationTask(ctx context.Context, client *pxapi.Client, clusterID string, chCfg config.ClickHouse) { ticker := time.NewTicker(schemaCreationInterval) defer ticker.Stop() - // Run immediately on startup - log.Info("Running schema creation script") - execCtx, cancel := context.WithTimeout(ctx, scriptExecutionTimeout) - if _, err := pxl.ExecuteScript(execCtx, client, clusterID, schemaCreationScript); err != nil { - log.WithError(err).Error("failed to execute schema creation script") - } else { + runOnce := func() { + log.Info("Running schema creation script") + execCtx, cancel := context.WithTimeout(ctx, scriptExecutionTimeout) + defer cancel() + if _, err := pxl.ExecuteScript(execCtx, client, clusterID, renderSchemaScript(chCfg)); err != nil { + log.WithError(err).Error("failed to execute schema creation script") + return + } log.Info("Schema creation script completed successfully") } - cancel() + runOnce() for { select { case <-ctx.Done(): log.Info("Schema creation task shutting down") return case <-ticker.C: - log.Info("Running schema creation script") - execCtx, cancel := context.WithTimeout(ctx, scriptExecutionTimeout) - if _, err := pxl.ExecuteScript(execCtx, client, clusterID, schemaCreationScript); err != nil { - log.WithError(err).Error("failed to execute schema creation script") - } else { - log.Info("Schema creation script completed successfully") - } - cancel() + runOnce() } } } @@ -145,11 +148,47 @@ func runSchemaCreationTask(ctx context.Context, client *pxapi.Client, clusterID func runDetectionTask(ctx context.Context, pxClient *pxapi.Client, pluginClient *pixie.Client, cfg config.Config, clusterID string, clusterName string) { detectionInterval := time.Duration(cfg.Worker().DetectionInterval()) * time.Second detectionLookback := cfg.Worker().DetectionLookback() + quietTicks := cfg.Worker().ExportQuietTicks() + mode := cfg.Worker().ExportMode() ticker := time.NewTicker(detectionInterval) defer ticker.Stop() - pluginEnabled := false + // pluginEnabled tracks our last-known retention-plugin state. A nil value means + // we haven't reconciled yet; we always query on the first tick. + var pluginEnabled *bool + quietStreak := int64(0) + + reconcile := func(want bool) { + if pluginEnabled != nil && *pluginEnabled == want { + log.Debugf("export already in desired state (enabled=%v), no action taken", want) + return + } + pluginCtx, pluginCancel := context.WithTimeout(ctx, 2*time.Minute) + defer pluginCancel() + if want { + log.Info("Enabling forensic export") + if err := enableClickHousePlugin(pluginCtx, pluginClient, cfg, clusterID, clusterName); err != nil { + log.WithError(err).Error("failed to enable forensic export") + return + } + v := true + pluginEnabled = &v + log.Info("Forensic export enabled successfully") + } else { + log.Info("Disabling forensic export") + if err := disableClickHousePlugin(pluginCtx, pluginClient, cfg, clusterID, clusterName); err != nil { + log.WithError(err).Error("failed to disable forensic export") + return + } + v := false + pluginEnabled = &v + quietStreak = 0 + log.Info("Forensic export disabled successfully") + } + } + + log.Infof("Detection task starting (mode=%s, quietTicks=%d)", mode, quietTicks) for { select { @@ -157,38 +196,70 @@ func runDetectionTask(ctx context.Context, pxClient *pxapi.Client, pluginClient log.Info("Detection task shutting down") return case <-ticker.C: - log.Info("Running detection script") - // Run detection script with lookback period - detectionPxl := fmt.Sprintf(detectionScript, detectionLookback) + switch mode { + case config.ExportModeAlways: + reconcile(true) + continue + case config.ExportModeNever: + reconcile(false) + continue + } + + // auto mode: detection drives the state. + log.Debug("Running detection script") execCtx, cancel := context.WithTimeout(ctx, scriptExecutionTimeout) - recordCount, err := pxl.ExecuteScript(execCtx, pxClient, clusterID, detectionPxl) + recordCount, err := pxl.ExecuteScript(execCtx, pxClient, clusterID, renderDetectionScript(cfg.ClickHouse(), detectionLookback)) cancel() - if err != nil { log.WithError(err).Error("failed to execute detection script") continue } - log.Debugf("Detection script returned %d records", recordCount) - // If we have records and plugin is not enabled, enable it - if recordCount > 0 && !pluginEnabled { - log.Info("Detection script returned records - enabling forensic export") - pluginCtx, pluginCancel := context.WithTimeout(ctx, 2*time.Minute) - if err := enableClickHousePlugin(pluginCtx, pluginClient, cfg, clusterID, clusterName); err != nil { - log.WithError(err).Error("failed to enable forensic export") - } else { - pluginEnabled = true - log.Info("Forensic export enabled successfully") + if recordCount > 0 { + quietStreak = 0 + reconcile(true) + } else { + quietStreak++ + if quietStreak >= quietTicks { + reconcile(false) } - pluginCancel() - } else if recordCount > 0 && pluginEnabled { - log.Info("Detection script returned records but forensic export already enabled, no action taken") } } } } +func disableClickHousePlugin(ctx context.Context, client *pixie.Client, cfg config.Config, clusterID string, clusterName string) error { + plugin, err := client.GetClickHousePlugin() + if err != nil { + return fmt.Errorf("getting data retention plugins failed: %w", err) + } + if !plugin.RetentionEnabled { + log.Info("ClickHouse plugin already disabled; removing any lingering ch-* scripts") + } else { + if err := client.DisableClickHousePlugin(plugin.LatestVersion); err != nil { + return fmt.Errorf("failed to disable ClickHouse plugin: %w", err) + } + } + + // Tear down the per-cluster ch-* retention scripts so the demo can be re-run cleanly. + current, err := client.GetClusterScripts(clusterID, clusterName) + if err != nil { + return fmt.Errorf("failed to list retention scripts: %w", err) + } + var errs []error + for _, s := range current { + log.Infof("Deleting retention script %s", s.Name) + if err := client.DeleteDataRetentionScript(s.ScriptId); err != nil { + errs = append(errs, err) + } + } + if len(errs) > 0 { + return fmt.Errorf("errors while deleting retention scripts: %v", errs) + } + return nil +} + func enableClickHousePlugin(ctx context.Context, client *pixie.Client, cfg config.Config, clusterID string, clusterName string) error { log.Info("Checking the current ClickHouse plugin configuration") plugin, err := client.GetClickHousePlugin() diff --git a/src/vizier/services/adaptive_export/internal/config/config.go b/src/vizier/services/adaptive_export/internal/config/config.go index fc500359dfe..3d98e7897be 100644 --- a/src/vizier/services/adaptive_export/internal/config/config.go +++ b/src/vizier/services/adaptive_export/internal/config/config.go @@ -33,20 +33,39 @@ import ( ) const ( - envVerbose = "VERBOSE" - envClickHouseDSN = "CLICKHOUSE_DSN" - envPixieClusterID = "PIXIE_CLUSTER_ID" - envPixieEndpoint = "PIXIE_ENDPOINT" - envPixieAPIKey = "PIXIE_API_KEY" - envClusterName = "CLUSTER_NAME" - envCollectInterval = "COLLECT_INTERVAL_SEC" - envDetectionInterval = "DETECTION_INTERVAL_SEC" - envDetectionLookback = "DETECTION_LOOKBACK_SEC" - defPixieHostname = "work.withpixie.ai:443" - boolTrue = "true" - defCollectInterval = 30 - defDetectionInterval = 10 - defDetectionLookback = 15 + envVerbose = "VERBOSE" + envClickHouseDSN = "CLICKHOUSE_DSN" + envClickHouseHost = "CLICKHOUSE_HOST" + envClickHousePort = "CLICKHOUSE_PORT" + envClickHouseUser = "CLICKHOUSE_USER" + envClickHousePass = "CLICKHOUSE_PASSWORD" + envClickHouseDB = "CLICKHOUSE_DATABASE" + envKubescapeTable = "KUBESCAPE_TABLE" + envPixieClusterID = "PIXIE_CLUSTER_ID" + envPixieEndpoint = "PIXIE_ENDPOINT" + envPixieAPIKey = "PIXIE_API_KEY" + envClusterName = "CLUSTER_NAME" + envCollectInterval = "COLLECT_INTERVAL_SEC" + envDetectionInterval = "DETECTION_INTERVAL_SEC" + envDetectionLookback = "DETECTION_LOOKBACK_SEC" + envExportMode = "EXPORT_MODE" + envExportQuietTicks = "EXPORT_QUIET_TICKS" + defPixieHostname = "work.pixie.austrianopencloudcommunity.org:443" + defClickHousePort = "9000" + defKubescapeTable = "kubescape_logs" + defExportMode = "auto" + defExportQuietTicks = 6 + boolTrue = "true" + defCollectInterval = 30 + defDetectionInterval = 10 + defDetectionLookback = 15 +) + +// ExportMode values. +const ( + ExportModeAuto = "auto" + ExportModeAlways = "always" + ExportModeNever = "never" ) var ( @@ -206,6 +225,32 @@ func setUpConfig() error { return err } + exportQuietTicks, err := getIntEnvWithDefault(envExportQuietTicks, defExportQuietTicks) + if err != nil { + return err + } + + exportMode := strings.ToLower(getEnvWithDefault(envExportMode, defExportMode)) + switch exportMode { + case ExportModeAuto, ExportModeAlways, ExportModeNever: + default: + return fmt.Errorf("invalid %s=%q (must be auto|always|never)", envExportMode, exportMode) + } + + // Parse the DSN into its parts; individual env vars override the parsed values. + dsnHost, dsnPort, dsnUser, dsnPass, dsnDB := parseDSN(clickhouseDSN) + chHost := getEnvWithDefault(envClickHouseHost, dsnHost) + chPort := getEnvWithDefault(envClickHousePort, firstNonEmpty(dsnPort, defClickHousePort)) + chUser := getEnvWithDefault(envClickHouseUser, dsnUser) + chPass := getEnvWithDefault(envClickHousePass, dsnPass) + chDB := getEnvWithDefault(envClickHouseDB, dsnDB) + chTable := getEnvWithDefault(envKubescapeTable, defKubescapeTable) + + // If individual fields were provided but CLICKHOUSE_DSN was not, build one. + if clickhouseDSN == "" && chHost != "" && chUser != "" { + clickhouseDSN = fmt.Sprintf("%s:%s@%s:%s/%s", chUser, chPass, chHost, chPort, chDB) + } + instance = &config{ settings: &settings{ buildDate: buildDate, @@ -213,14 +258,22 @@ func setUpConfig() error { version: integrationVersion, }, worker: &worker{ - clusterName: clusterName, - pixieClusterID: pixieClusterID, - collectInterval: collectInterval, - detectionInterval: detectionInterval, - detectionLookback: detectionLookback, + clusterName: clusterName, + pixieClusterID: pixieClusterID, + collectInterval: collectInterval, + detectionInterval: detectionInterval, + detectionLookback: detectionLookback, + exportMode: exportMode, + exportQuietTicks: exportQuietTicks, }, clickhouse: &clickhouse{ dsn: clickhouseDSN, + host: chHost, + port: chPort, + user: chUser, + password: chPass, + database: chDB, + table: chTable, userAgent: "pixie-clickhouse/" + integrationVersion, }, pixie: &pixie{ @@ -232,6 +285,47 @@ func setUpConfig() error { return instance.validate() } +// parseDSN best-effort splits `user:pass@host:port/db`. Missing parts come back empty. +func parseDSN(dsn string) (host, port, user, pass, db string) { + if dsn == "" { + return + } + at := strings.LastIndex(dsn, "@") + if at < 0 { + return + } + creds := dsn[:at] + rest := dsn[at+1:] + + if i := strings.Index(creds, ":"); i >= 0 { + user = creds[:i] + pass = creds[i+1:] + } else { + user = creds + } + + if i := strings.Index(rest, "/"); i >= 0 { + db = rest[i+1:] + rest = rest[:i] + } + if i := strings.Index(rest, ":"); i >= 0 { + host = rest[:i] + port = rest[i+1:] + } else { + host = rest + } + return +} + +func firstNonEmpty(vals ...string) string { + for _, v := range vals { + if v != "" { + return v + } + } + return "" +} + func getEnvWithDefault(key, defaultValue string) string { value := os.Getenv(key) if value == "" { @@ -325,29 +419,46 @@ func (s *settings) BuildDate() string { type ClickHouse interface { DSN() string + Host() string + Port() string + User() string + Password() string + Database() string + Table() string UserAgent() string validate() error } type clickhouse struct { dsn string + host string + port string + user string + password string + database string + table string userAgent string } func (c *clickhouse) validate() error { if c.dsn == "" { - return fmt.Errorf("missing required env variable '%s'", envClickHouseDSN) + return fmt.Errorf("missing required env variable '%s' (or provide %s/%s/%s/%s/%s)", + envClickHouseDSN, envClickHouseHost, envClickHousePort, envClickHouseUser, envClickHousePass, envClickHouseDB) + } + if c.host == "" || c.user == "" || c.database == "" { + return fmt.Errorf("ClickHouse host/user/database could not be derived from %s=%q", envClickHouseDSN, c.dsn) } return nil } -func (c *clickhouse) DSN() string { - return c.dsn -} - -func (c *clickhouse) UserAgent() string { - return c.userAgent -} +func (c *clickhouse) DSN() string { return c.dsn } +func (c *clickhouse) Host() string { return c.host } +func (c *clickhouse) Port() string { return c.port } +func (c *clickhouse) User() string { return c.user } +func (c *clickhouse) Password() string { return c.password } +func (c *clickhouse) Database() string { return c.database } +func (c *clickhouse) Table() string { return c.table } +func (c *clickhouse) UserAgent() string { return c.userAgent } type Pixie interface { APIKey() string @@ -390,15 +501,19 @@ type Worker interface { CollectInterval() int64 DetectionInterval() int64 DetectionLookback() int64 + ExportMode() string + ExportQuietTicks() int64 validate() error } type worker struct { - clusterName string - pixieClusterID string - collectInterval int64 - detectionInterval int64 - detectionLookback int64 + clusterName string + pixieClusterID string + collectInterval int64 + detectionInterval int64 + detectionLookback int64 + exportMode string + exportQuietTicks int64 } func (a *worker) validate() error { @@ -408,22 +523,10 @@ func (a *worker) validate() error { return nil } -func (a *worker) ClusterName() string { - return a.clusterName -} - -func (a *worker) PixieClusterID() string { - return a.pixieClusterID -} - -func (a *worker) CollectInterval() int64 { - return a.collectInterval -} - -func (a *worker) DetectionInterval() int64 { - return a.detectionInterval -} - -func (a *worker) DetectionLookback() int64 { - return a.detectionLookback -} +func (a *worker) ClusterName() string { return a.clusterName } +func (a *worker) PixieClusterID() string { return a.pixieClusterID } +func (a *worker) CollectInterval() int64 { return a.collectInterval } +func (a *worker) DetectionInterval() int64 { return a.detectionInterval } +func (a *worker) DetectionLookback() int64 { return a.detectionLookback } +func (a *worker) ExportMode() string { return a.exportMode } +func (a *worker) ExportQuietTicks() int64 { return a.exportQuietTicks } diff --git a/src/vizier/services/adaptive_export/internal/pixie/pixie.go b/src/vizier/services/adaptive_export/internal/pixie/pixie.go index 97e5bb8ae23..8cf8176ddc3 100644 --- a/src/vizier/services/adaptive_export/internal/pixie/pixie.go +++ b/src/vizier/services/adaptive_export/internal/pixie/pixie.go @@ -146,6 +146,18 @@ func (c *Client) EnableClickHousePlugin(config *ClickHousePluginConfig, version return err } +// DisableClickHousePlugin flips the retention plugin off without touching scripts. +// Scripts are expected to be removed separately via DeleteDataRetentionScript. +func (c *Client) DisableClickHousePlugin(version string) error { + req := &cloudpb.UpdateRetentionPluginConfigRequest{ + PluginId: clickhousePluginId, + Enabled: &types.BoolValue{Value: false}, + Version: &types.StringValue{Value: version}, + } + _, err := c.pluginClient.UpdateRetentionPluginConfig(c.ctx, req) + return err +} + func (c *Client) GetPresetScripts() ([]*script.ScriptDefinition, error) { resp, err := c.pluginClient.GetRetentionScripts(c.ctx, &cloudpb.GetRetentionScriptsRequest{}) if err != nil { From 6ab28357a69d72478970e5b92a85fa3ca6c4e3c3 Mon Sep 17 00:00:00 2001 From: Entlein Date: Fri, 1 May 2026 16:15:07 +0200 Subject: [PATCH 10/10] bazel: pin opentelemetry-proto v1.10.0 and add profiles targets Bumped from v1.3.2 to bring in opentelemetry/proto/profiles/v1development/profiles.proto (profiles signal first appeared upstream from v1.4.0; v1development path is alpha). Added profiles_proto, profiles_service_proto, profiles_service_grpc_cc targets in bazel/external/opentelemetry.BUILD matching the existing trace/metric/log pattern. Required for the abstracted-OTel-profiles SBOB PoC. Tarball sha256 verified. Also seeds .claude/RESUME.md as session-handoff state for the PoC branch. --- .claude/RESUME.md | 36 ++++++++++++++++++++++++++ bazel/external/opentelemetry.BUILD | 41 ++++++++++++++++++++++++++++++ bazel/repository_locations.bzl | 6 ++--- 3 files changed, 80 insertions(+), 3 deletions(-) create mode 100644 .claude/RESUME.md diff --git a/.claude/RESUME.md b/.claude/RESUME.md new file mode 100644 index 00000000000..5d6eb431f41 --- /dev/null +++ b/.claude/RESUME.md @@ -0,0 +1,36 @@ +# Resume — poc/otel-profiles-sbob + +## Active branch +`poc/otel-profiles-sbob` off `demo/adaptive-export-config` (fork: `k8sstormcenter/pixie`). + +## PoC goal +Feasibility: extend SBOB with abstracted OTel Profiles emitted by Pixie. Target = redis under bobctl test vs bobctl attack (bmlv-demo). Deadline Mon 2026-05-04. + +## Docs / fixtures live separately +`~/biz/PoC/OTel/` — `state.yaml` is the authoritative status file. Markdown, mermaid PNGs, slides go there. Pixie tree only carries code. + +## Task tracker +9 tasks total. #1 done. Sequence + parallelism in `~/biz/PoC/OTel/state.yaml` under `day_status`. Tracks run in 3 swimlanes Fri–Sun: +- A (research): #2 capture → #3 userspace prototype (Q1 gate) → #4 UDAs +- B (plumbing): #5 Profiles sink ‖ #6 planner builder → #7 PxL script → #8 e2e validation +- C (write-up): #9 rolling draft Fri–Sun, finalize Mon AM + +## Staged but uncommitted +- `bazel/repository_locations.bzl` — OTel proto pin bumped v1.3.2 → v1.10.0 + - tarball sha256: `52c85df79badc45da7e6a8735e8090b05a961b0208756187e1492a40db2d1f5f` + - tag SHA: `ca839c51f706f5d53bfb46f06c3e90c3af3a52c6` +- `bazel/external/opentelemetry.BUILD` — added `profiles_proto`, `profiles_service_proto`, `profiles_service_grpc_cc` targets matching existing trace/metric/log pattern. + +Bump rationale: profiles signal proto only present from v1.4.0 onward; v1development path is alpha. Risk: if existing trace/metric/log code references removed/renamed fields, build breaks. Mitigation if it bites: dual-pin with separate `com_github_opentelemetry_proto_dev` repo for profiles only. + +## Build verification not yet run +Have not invoked `bazel build` against the bumped pin. First build attempt will surface any breakage. + +## Pending decisions awaiting user +1. Commit checkpoints — user has a standing no-commit-without-ask rule. Currently nothing on this branch beyond the base. +2. bmlv-demo cluster state — task #2 needs redis + pixie + bobctl up on local k3s. Status unknown. + +## Hard rules in force +- No Co-Authored-By Claude / "Generated with Claude Code" on any commit, patch, or PR body. +- No fabricated metrics in any user-facing artifact — empirical numbers from actual bmlv runs only, otherwise placeholders. +- No upstream PRs — fork strategy. This branch lives on `k8sstormcenter/pixie`. diff --git a/bazel/external/opentelemetry.BUILD b/bazel/external/opentelemetry.BUILD index 888a797107e..5f3dba93a7c 100644 --- a/bazel/external/opentelemetry.BUILD +++ b/bazel/external/opentelemetry.BUILD @@ -162,3 +162,44 @@ cc_grpc_library( grpc_only = True, deps = [":logs_service_proto_cc"], ) + +# Profiles signal — v1development (alpha). Pinned via opentelemetry-proto v1.10.0. +# Schema is explicitly experimental; expect to re-pin on upstream changes. +proto_library( + name = "profiles_proto", + srcs = [ + "opentelemetry/proto/profiles/v1development/profiles.proto", + ], + deps = [ + ":common_proto", + ":resource_proto", + ], +) + +cc_proto_library( + name = "profiles_proto_cc", + deps = [":profiles_proto"], +) + +proto_library( + name = "profiles_service_proto", + srcs = [ + "opentelemetry/proto/collector/profiles/v1development/profiles_service.proto", + ], + deps = [ + ":profiles_proto", + ], +) + +cc_proto_library( + name = "profiles_service_proto_cc", + deps = [":profiles_service_proto"], +) + +cc_grpc_library( + name = "profiles_service_grpc_cc", + srcs = [":profiles_service_proto"], + generate_mocks = True, + grpc_only = True, + deps = [":profiles_service_proto_cc"], +) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 4584d725f9b..5731a3f93a9 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -185,9 +185,9 @@ REPOSITORY_LOCATIONS = dict( urls = ["https://github.com/nlohmann/json/releases/download/v3.7.3/include.zip"], ), com_github_opentelemetry_proto = dict( - urls = ["https://github.com/open-telemetry/opentelemetry-proto/archive/refs/tags/v1.3.2.tar.gz"], - strip_prefix = "opentelemetry-proto-1.3.2", - sha256 = "c069c0d96137cf005d34411fa67dd3b6f1f8c64af1e7fb2fe0089a41c425acd7", + urls = ["https://github.com/open-telemetry/opentelemetry-proto/archive/refs/tags/v1.10.0.tar.gz"], + strip_prefix = "opentelemetry-proto-1.10.0", + sha256 = "52c85df79badc45da7e6a8735e8090b05a961b0208756187e1492a40db2d1f5f", ), com_github_packetzero_dnsparser = dict( sha256 = "bdf6c7f56f33725c1c32e672a4779576fb639dd2df565115778eb6be48296431",