diff --git a/include/fluent-bit/flb_mp.h b/include/fluent-bit/flb_mp.h index 5c0a6db3550..60cfa3635e5 100644 --- a/include/fluent-bit/flb_mp.h +++ b/include/fluent-bit/flb_mp.h @@ -27,6 +27,11 @@ #define FLB_MP_ARRAY MSGPACK_OBJECT_ARRAY int flb_mp_count(const void *data, size_t bytes); +int flb_mp_count_log_records(const void *data, size_t bytes); +int flb_mp_normalize_log_buffer_groups_msgpack(const void *in_buf, size_t in_size, + char **out_buf, size_t *out_size); +int flb_mp_normalize_log_buffer_groups(const void *in_buf, size_t in_size, + char **out_buf, size_t *out_size); int flb_mp_count_remaining(const void *data, size_t bytes, size_t *remaining_bytes); int flb_mp_validate_log_chunk(const void *data, size_t bytes, int *out_records, size_t *processed_bytes); diff --git a/include/fluent-bit/flb_mp_chunk.h b/include/fluent-bit/flb_mp_chunk.h index 899e7512442..7f3ff751141 100644 --- a/include/fluent-bit/flb_mp_chunk.h +++ b/include/fluent-bit/flb_mp_chunk.h @@ -71,6 +71,8 @@ struct flb_mp_chunk_cobj *flb_mp_chunk_cobj_create(struct flb_log_event_encoder int flb_mp_chunk_cobj_destroy(struct flb_mp_chunk_cobj *chunk_cobj); int flb_mp_chunk_cobj_encode(struct flb_mp_chunk_cobj *chunk_cobj, char **out_buf, size_t *out_size); +int flb_mp_chunk_cobj_count_log_records(struct flb_mp_chunk_cobj *chunk_cobj); +int flb_mp_chunk_cobj_normalize_groups(struct flb_mp_chunk_cobj *chunk_cobj); diff --git a/include/fluent-bit/flb_output.h b/include/fluent-bit/flb_output.h index 267b6ff0dbd..1d3d7955701 100644 --- a/include/fluent-bit/flb_output.h +++ b/include/fluent-bit/flb_output.h @@ -48,6 +48,7 @@ #include #include #include +#include #include #include @@ -777,6 +778,9 @@ struct flb_output_flush *flb_output_flush_create(struct flb_task *task, if (flb_processor_is_active(o_ins->processor)) { if (evc->type == FLB_EVENT_TYPE_LOGS) { + char *normalized_buf; + size_t normalized_size; + /* run the processor */ ret = flb_processor_run(o_ins->processor, 0, @@ -790,7 +794,22 @@ struct flb_output_flush *flb_output_flush_create(struct flb_task *task, return NULL; } - records = flb_mp_count(p_buf, p_size); + normalized_buf = NULL; + normalized_size = 0; + + ret = flb_mp_normalize_log_buffer_groups_msgpack(p_buf, p_size, + &normalized_buf, + &normalized_size); + if (ret == 0) { + if (p_buf != evc->data) { + flb_free(p_buf); + } + + p_buf = normalized_buf; + p_size = normalized_size; + } + + records = flb_mp_count_log_records(p_buf, p_size); tmp = flb_event_chunk_create(evc->type, records, evc->tag, flb_sds_len(evc->tag), p_buf, p_size); if (!tmp) { flb_coro_destroy(coro); @@ -1190,10 +1209,13 @@ struct flb_output_flush *flb_output_flush_create(struct flb_task *task, */ static inline void flb_output_return(int ret, struct flb_coro *co) { int n; + int records; int pipe_fd; uint32_t set; uint64_t val; + size_t bytes; struct flb_task *task; + struct flb_event_chunk *counted_event_chunk; struct flb_output_flush *out_flush; struct flb_output_instance *o_ins; struct flb_out_thread_instance *th_ins = NULL; @@ -1202,10 +1224,22 @@ static inline void flb_output_return(int ret, struct flb_coro *co) { o_ins = out_flush->o_ins; task = out_flush->task; - flb_task_acquire_lock(task); + if (out_flush->processed_event_chunk) { + counted_event_chunk = out_flush->processed_event_chunk; + } + else { + counted_event_chunk = task->event_chunk; + } - flb_task_deactivate_route(task, o_ins); + records = task->event_chunk->total_events; + if (counted_event_chunk->type == FLB_EVENT_TYPE_LOGS) { + records = counted_event_chunk->total_events; + } + bytes = counted_event_chunk->size; + flb_task_acquire_lock(task); + flb_task_set_route_data(task, o_ins, records, bytes); + flb_task_deactivate_route(task, o_ins); flb_task_release_lock(task); #ifdef FLB_HAVE_CHUNK_TRACE diff --git a/include/fluent-bit/flb_processor.h b/include/fluent-bit/flb_processor.h index 3373186a2de..772874df276 100644 --- a/include/fluent-bit/flb_processor.h +++ b/include/fluent-bit/flb_processor.h @@ -130,6 +130,19 @@ struct flb_processor { flb_pipefd_t notification_channel; + /* + * Processor chain accounting metrics + * ---------------------------------- + * These counters are registered in the owner context metrics (input/output) + * and updated per processor unit execution. + */ + struct cmt_counter *cmt_invocations; + struct cmt_counter *cmt_errors; + struct cmt_counter *cmt_items_in; + struct cmt_counter *cmt_items_out; + struct cmt_counter *cmt_items_drop; + struct cmt_counter *cmt_items_add; + /* Fluent Bit context */ struct flb_config *config; }; diff --git a/include/fluent-bit/flb_task.h b/include/fluent-bit/flb_task.h index 6d93ba1535e..de7720d19b8 100644 --- a/include/fluent-bit/flb_task.h +++ b/include/fluent-bit/flb_task.h @@ -57,6 +57,8 @@ struct flb_task_route { int status; + int records; + size_t bytes; struct flb_output_instance *out; struct mk_list _head; }; @@ -257,6 +259,52 @@ static FLB_INLINE void flb_task_set_route_status( } } +static FLB_INLINE void flb_task_set_route_data( + struct flb_task *task, + struct flb_output_instance *o_ins, + int records, + size_t bytes) +{ + struct mk_list *iterator; + struct flb_task_route *route; + + mk_list_foreach(iterator, &task->routes) { + route = mk_list_entry(iterator, struct flb_task_route, _head); + + if (route->out == o_ins) { + route->records = records; + route->bytes = bytes; + break; + } + } +} + +static FLB_INLINE int flb_task_get_route_data( + struct flb_task *task, + struct flb_output_instance *o_ins, + int *records, + size_t *bytes) +{ + struct mk_list *iterator; + struct flb_task_route *route; + + if (records == NULL || bytes == NULL) { + return -1; + } + + mk_list_foreach(iterator, &task->routes) { + route = mk_list_entry(iterator, struct flb_task_route, _head); + + if (route->out == o_ins) { + *records = route->records; + *bytes = route->bytes; + return 0; + } + } + + return -1; +} + static FLB_INLINE void flb_task_activate_route( struct flb_task *task, diff --git a/plugins/filter_alter_size/alter_size.c b/plugins/filter_alter_size/alter_size.c index 297414d2a3c..2537247b26f 100644 --- a/plugins/filter_alter_size/alter_size.c +++ b/plugins/filter_alter_size/alter_size.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -142,7 +143,7 @@ static int cb_alter_size_filter(const void *data, size_t bytes, count = 0; /* Count number of current items */ - total = flb_mp_count(data, bytes); + total = flb_mp_count_log_records(data, bytes); total -= ctx->remove; if (total <= 0) { /* zero records */ diff --git a/plugins/out_azure/azure.c b/plugins/out_azure/azure.c index 2106b80ecb4..459563d910a 100644 --- a/plugins/out_azure/azure.c +++ b/plugins/out_azure/azure.c @@ -76,7 +76,7 @@ static int azure_format(const void *in_buf, size_t in_bytes, int ret; /* Count number of items */ - array_size = flb_mp_count(in_buf, in_bytes); + array_size = flb_mp_count_log_records(in_buf, in_bytes); ret = flb_log_event_decoder_init(&log_decoder, (char *) in_buf, in_bytes); diff --git a/plugins/out_azure_kusto/azure_kusto.c b/plugins/out_azure_kusto/azure_kusto.c index cd3f1c3966b..adb9561048d 100644 --- a/plugins/out_azure_kusto/azure_kusto.c +++ b/plugins/out_azure_kusto/azure_kusto.c @@ -997,7 +997,7 @@ static int azure_kusto_format(struct flb_azure_kusto *ctx, const char *tag, int flb_sds_t out_buf; /* Create array for all records */ - records = flb_mp_count(data, bytes); + records = flb_mp_count_log_records(data, bytes); if (records <= 0) { flb_plg_error(ctx->ins, "error counting msgpack entries"); return -1; diff --git a/plugins/out_azure_logs_ingestion/azure_logs_ingestion.c b/plugins/out_azure_logs_ingestion/azure_logs_ingestion.c index c72559f9ad6..3aee4ac182e 100644 --- a/plugins/out_azure_logs_ingestion/azure_logs_ingestion.c +++ b/plugins/out_azure_logs_ingestion/azure_logs_ingestion.c @@ -25,8 +25,10 @@ #include #include #include +#include #include #include +#include #include #include "azure_logs_ingestion.h" @@ -58,14 +60,13 @@ static int az_li_format(const void *in_buf, size_t in_bytes, struct flb_config *config) { int i; + int ret; int array_size = 0; int map_size; - size_t off = 0; double t; struct flb_time tm; - msgpack_unpacked result; - msgpack_object root; - msgpack_object *obj; + struct flb_log_event_decoder log_decoder; + struct flb_log_event log_event; msgpack_object map; msgpack_object k; msgpack_object v; @@ -80,26 +81,28 @@ static int az_li_format(const void *in_buf, size_t in_bytes, int len; /* Count number of items */ - array_size = flb_mp_count(in_buf, in_bytes); - msgpack_unpacked_init(&result); + array_size = flb_mp_count_log_records(in_buf, in_bytes); /* Create temporary msgpack buffer */ msgpack_sbuffer_init(&mp_sbuf); msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); msgpack_pack_array(&mp_pck, array_size); - off = 0; - while (msgpack_unpack_next(&result, in_buf, in_bytes, &off) == MSGPACK_UNPACK_SUCCESS) { - root = result.data; + ret = flb_log_event_decoder_init(&log_decoder, (char *) in_buf, in_bytes); + if (ret != FLB_EVENT_DECODER_SUCCESS) { + msgpack_sbuffer_destroy(&mp_sbuf); + return -1; + } - /* Get timestamp */ - flb_time_pop_from_msgpack(&tm, &result, &obj); + while ((ret = flb_log_event_decoder_next(&log_decoder, &log_event)) == + FLB_EVENT_DECODER_SUCCESS) { + flb_time_copy(&tm, &log_event.timestamp); /* Create temporary msgpack buffer */ msgpack_sbuffer_init(&tmp_sbuf); msgpack_packer_init(&tmp_pck, &tmp_sbuf, msgpack_sbuffer_write); - map = root.via.array.ptr[1]; + map = *log_event.body; map_size = map.via.map.size; msgpack_pack_map(&mp_pck, map_size + 1); @@ -147,12 +150,12 @@ static int az_li_format(const void *in_buf, size_t in_bytes, if (!record) { flb_errno(); msgpack_sbuffer_destroy(&mp_sbuf); - msgpack_unpacked_destroy(&result); + flb_log_event_decoder_destroy(&log_decoder); return -1; } msgpack_sbuffer_destroy(&mp_sbuf); - msgpack_unpacked_destroy(&result); + flb_log_event_decoder_destroy(&log_decoder); *out_buf = record; *out_size = flb_sds_len(record); diff --git a/plugins/out_bigquery/bigquery.c b/plugins/out_bigquery/bigquery.c index b919912fab0..39ece9e9f00 100644 --- a/plugins/out_bigquery/bigquery.c +++ b/plugins/out_bigquery/bigquery.c @@ -867,7 +867,7 @@ static int bigquery_format(const void *data, size_t bytes, return -1; } - array_size = flb_mp_count(data, bytes); + array_size = flb_mp_count_log_records(data, bytes); /* Create temporary msgpack buffer */ msgpack_sbuffer_init(&mp_sbuf); diff --git a/plugins/out_counter/counter.c b/plugins/out_counter/counter.c index 62f11daacbd..70eb9c9c09e 100644 --- a/plugins/out_counter/counter.c +++ b/plugins/out_counter/counter.c @@ -25,10 +25,6 @@ #include #include -struct flb_counter_ctx { - uint64_t total; -}; - static int cb_counter_init(struct flb_output_instance *ins, struct flb_config *config, void *data) @@ -36,20 +32,6 @@ static int cb_counter_init(struct flb_output_instance *ins, (void) ins; (void) config; (void) data; - struct flb_counter_ctx *ctx; - - ctx = flb_malloc(sizeof(struct flb_counter_ctx)); - if (!ctx) { - flb_errno(); - return -1; - } - ctx->total = 0; - flb_output_set_context(ins, ctx); - if (flb_output_config_map_set(ins, (void *)ctx) == -1) { - flb_plg_error(ins, "unable to load configuration"); - flb_free(ctx); - return -1; - } return 0; } @@ -61,32 +43,40 @@ static void cb_counter_flush(struct flb_event_chunk *event_chunk, struct flb_config *config) { (void) i_ins; + (void) out_flush; (void) out_context; (void) config; - size_t cnt; - struct flb_counter_ctx *ctx = out_context; + size_t serialized_events; + size_t log_records; + size_t total; struct flb_time tm; - /* Count number of parent items */ - cnt = flb_mp_count(event_chunk->data, event_chunk->size); - ctx->total += cnt; + /* Count number of serialized msgpack root objects */ + serialized_events = flb_mp_count(event_chunk->data, event_chunk->size); + + /* Count number of logical log records (group markers excluded) */ + log_records = 0; + if (event_chunk->type == FLB_EVENT_TYPE_LOGS) { + log_records = flb_mp_count_log_records(event_chunk->data, + event_chunk->size); + } + total = serialized_events; flb_time_get(&tm); - printf("%f,%lu (total = %"PRIu64")\n", flb_time_to_double(&tm), cnt, - ctx->total); + printf("{\"ts\":%.6f,\"serialized_events\":%zu,\"log_records\":%zu," + "\"total\":%zu}\n", + flb_time_to_double(&tm), + serialized_events, + log_records, + total); FLB_OUTPUT_RETURN(FLB_OK); } static int cb_counter_exit(void *data, struct flb_config *config) { - struct flb_counter_ctx *ctx = data; - - if (!ctx) { - return 0; - } - - flb_free(ctx); + (void) config; + (void) data; return 0; } diff --git a/plugins/out_datadog/datadog.c b/plugins/out_datadog/datadog.c index 8dbec8ec416..0aa88a1bd89 100644 --- a/plugins/out_datadog/datadog.c +++ b/plugins/out_datadog/datadog.c @@ -26,6 +26,7 @@ #include #include #include +#include #include #include @@ -125,7 +126,7 @@ static int datadog_format(struct flb_config *config, event_chunk = flush_ctx; array_size = event_chunk->total_events; } else { - array_size = flb_mp_count(data, bytes); + array_size = flb_mp_count_log_records(data, bytes); } ret = flb_log_event_decoder_init(&log_decoder, (char *) data, bytes); diff --git a/plugins/out_forward/forward.c b/plugins/out_forward/forward.c index a73b95cfe60..671c395eec6 100644 --- a/plugins/out_forward/forward.c +++ b/plugins/out_forward/forward.c @@ -29,6 +29,7 @@ #include #include #include +#include #include #include "forward.h" @@ -1256,6 +1257,20 @@ static int pack_metricses_payload(msgpack_packer *mp_pck, const void *data, size return 0; } +static inline int forward_event_type_is_non_log(int event_type) +{ + return event_type == FLB_EVENT_TYPE_METRICS || + event_type == FLB_EVENT_TYPE_TRACES || + event_type == FLB_EVENT_TYPE_PROFILES || + event_type == FLB_EVENT_TYPE_BLOBS; +} + +static inline int forward_event_type_supports_fluentd_compat(int event_type) +{ + return event_type == FLB_EVENT_TYPE_METRICS || + event_type == FLB_EVENT_TYPE_TRACES; +} + #include /* * Forward Mode: this is the generic mechanism used in Fluent Bit, it takes @@ -1296,7 +1311,7 @@ static int flush_forward_mode(struct flb_forward *ctx, msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); send_options = fc->send_options; - if (event_type == FLB_EVENT_TYPE_METRICS || event_type == FLB_EVENT_TYPE_TRACES) { + if (forward_event_type_is_non_log(event_type)) { send_options = FLB_TRUE; } msgpack_pack_array(&mp_pck, send_options ? 3 : 2); @@ -1357,12 +1372,12 @@ static int flush_forward_mode(struct flb_forward *ctx, if (event_type == FLB_EVENT_TYPE_LOGS) { /* for log events we create an array for the serialized messages */ - entries = flb_mp_count(data, bytes); + entries = flb_mp_count(final_data, final_bytes); msgpack_pack_array(&mp_pck, entries); } else { - /* FLB_EVENT_TYPE_METRICS and FLB_EVENT_TYPE_TRACES */ - if (fc->fluentd_compat) { + if (fc->fluentd_compat && + forward_event_type_supports_fluentd_compat(event_type)) { pack_metricses_payload(&mp_pck, data, bytes); } else { diff --git a/plugins/out_forward/forward_format.c b/plugins/out_forward/forward_format.c index 47248b229bc..3a333b01511 100644 --- a/plugins/out_forward/forward_format.c +++ b/plugins/out_forward/forward_format.c @@ -28,6 +28,14 @@ #include "forward.h" +static inline int forward_event_type_is_non_log(int event_type) +{ + return event_type == FLB_EVENT_TYPE_METRICS || + event_type == FLB_EVENT_TYPE_TRACES || + event_type == FLB_EVENT_TYPE_PROFILES || + event_type == FLB_EVENT_TYPE_BLOBS; +} + void flb_forward_format_bin_to_hex(uint8_t *buf, size_t len, char *out) { int i; @@ -150,10 +158,9 @@ static int append_options(struct flb_forward *ctx, msgpack_pack_str_body(mp_pck, "gzip", 4); } else if (fc->compress == COMPRESS_GZIP && - /* for metrics or traces, we're also able to send as + /* for non-log signals, we're also able to send as * gzipped payloads */ - (event_type == FLB_EVENT_TYPE_METRICS || - event_type == FLB_EVENT_TYPE_TRACES)) { + forward_event_type_is_non_log(event_type)) { flb_mp_map_header_append(&mh); msgpack_pack_str(mp_pck, 10); msgpack_pack_str_body(mp_pck, "compressed", 10); @@ -161,7 +168,7 @@ static int append_options(struct flb_forward *ctx, msgpack_pack_str_body(mp_pck, "gzip", 4); } - /* event type (FLB_EVENT_TYPE_LOGS, FLB_EVENT_TYPE_METRICS, FLB_EVENT_TYPE_TRACES) */ + /* event type (FLB_EVENT_TYPE_*) */ flb_mp_map_header_append(&mh); msgpack_pack_str(mp_pck, 13); msgpack_pack_str_body(mp_pck, "fluent_signal", 13); @@ -220,8 +227,6 @@ static int flb_forward_format_message_mode(struct flb_forward *ctx, void **out_buf, size_t *out_size) { int entries = 0; - size_t pre = 0; - size_t off = 0; size_t record_size; char *chunk; char chunk_buf[33]; @@ -284,7 +289,7 @@ static int flb_forward_format_message_mode(struct flb_forward *ctx, /* Pack records */ msgpack_pack_object(&mp_pck, *log_event.body); - record_size = off - pre; + record_size = log_decoder.record_length; if (ff) { chunk = ff->checksum_hex; @@ -294,11 +299,10 @@ static int flb_forward_format_message_mode(struct flb_forward *ctx, } append_options(ctx, fc, FLB_EVENT_TYPE_LOGS, &mp_pck, 0, - (char *) data + pre, record_size, + (void *) log_decoder.record_base, record_size, log_event.metadata, chunk); - pre = off; entries++; } @@ -423,7 +427,7 @@ static int flb_forward_format_forward_mode(struct flb_forward *ctx, chunk = chunk_buf; } - if (fc->send_options == FLB_TRUE || (event_type == FLB_EVENT_TYPE_METRICS || event_type == FLB_EVENT_TYPE_TRACES)) { + if (fc->send_options == FLB_TRUE || forward_event_type_is_non_log(event_type)) { if (event_type == FLB_EVENT_TYPE_LOGS) { entries = flb_mp_count(data, bytes); } @@ -439,6 +443,7 @@ static int flb_forward_format_forward_mode(struct flb_forward *ctx, &transcoded_length); if (result == 0) { + entries = flb_mp_count(transcoded_buffer, transcoded_length); append_options(ctx, fc, event_type, &mp_pck, entries, transcoded_buffer, transcoded_length, @@ -507,7 +512,7 @@ static int flb_forward_format_forward_compat_mode(struct flb_forward *ctx, NULL, tag, tag_len); /* Entries */ - entries = flb_mp_count(data, bytes); + entries = flb_mp_count_log_records(data, bytes); msgpack_pack_array(&mp_pck, entries); while ((ret = flb_log_event_decoder_next( @@ -572,11 +577,7 @@ int flb_forward_format(struct flb_config *config, return -1; } - if (event_type == FLB_EVENT_TYPE_METRICS) { - mode = MODE_FORWARD; - goto do_formatting; - } - else if (event_type == FLB_EVENT_TYPE_TRACES) { + if (forward_event_type_is_non_log(event_type)) { mode = MODE_FORWARD; goto do_formatting; } diff --git a/plugins/out_kafka_rest/kafka.c b/plugins/out_kafka_rest/kafka.c index 10248295651..f1e5efc2d19 100644 --- a/plugins/out_kafka_rest/kafka.c +++ b/plugins/out_kafka_rest/kafka.c @@ -128,7 +128,7 @@ static flb_sds_t kafka_rest_format(const void *data, size_t bytes, msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); /* Count number of entries */ - arr_size = flb_mp_count(data, bytes); + arr_size = flb_mp_count_log_records(data, bytes); /* Root map */ msgpack_pack_map(&mp_pck, 1); diff --git a/plugins/out_logdna/logdna.c b/plugins/out_logdna/logdna.c index d594dbc7234..ffff06b829a 100644 --- a/plugins/out_logdna/logdna.c +++ b/plugins/out_logdna/logdna.c @@ -156,7 +156,7 @@ static flb_sds_t logdna_compose_payload(struct flb_logdna *ctx, } /* Count number of records */ - total_lines = flb_mp_count(data, bytes); + total_lines = flb_mp_count_log_records(data, bytes); /* Initialize msgpack buffers */ msgpack_sbuffer_init(&mp_sbuf); diff --git a/plugins/out_loki/loki.c b/plugins/out_loki/loki.c index 8f9ed6bf515..d7d91c44ade 100644 --- a/plugins/out_loki/loki.c +++ b/plugins/out_loki/loki.c @@ -2244,7 +2244,7 @@ static int cb_loki_format_test(struct flb_config *config, dynamic_tenant_id = NULL; /* Count number of records */ - total_records = flb_mp_count(data, bytes); + total_records = flb_mp_count_log_records(data, bytes); payload = loki_compose_payload(ctx, total_records, (char *) tag, tag_len, data, bytes, diff --git a/plugins/out_nats/nats.c b/plugins/out_nats/nats.c index e8d1611d042..589b57061a4 100644 --- a/plugins/out_nats/nats.c +++ b/plugins/out_nats/nats.c @@ -106,7 +106,7 @@ static int msgpack_to_json(struct flb_out_nats_config *ctx, return -1; } - array_size = flb_mp_count(data, bytes); + array_size = flb_mp_count_log_records(data, bytes); /* Convert MsgPack to JSON */ msgpack_sbuffer_init(&mp_sbuf); diff --git a/plugins/out_nrlogs/newrelic.c b/plugins/out_nrlogs/newrelic.c index 702799f7b82..6e46ca7309a 100644 --- a/plugins/out_nrlogs/newrelic.c +++ b/plugins/out_nrlogs/newrelic.c @@ -180,7 +180,7 @@ static flb_sds_t newrelic_compose_payload(struct flb_newrelic *ctx, } /* Count number of records */ - total_records = flb_mp_count(data, bytes); + total_records = flb_mp_count_log_records(data, bytes); /* Initialize msgpack buffers */ msgpack_sbuffer_init(&mp_sbuf); diff --git a/plugins/out_oracle_log_analytics/oci_logan.c b/plugins/out_oracle_log_analytics/oci_logan.c index 5cedd871648..40b9beb7d87 100644 --- a/plugins/out_oracle_log_analytics/oci_logan.c +++ b/plugins/out_oracle_log_analytics/oci_logan.c @@ -1103,7 +1103,7 @@ static int total_flush(struct flb_event_chunk *event_chunk, /* pack oci fields */ /* pack_oci_fields(&mp_pck, ctx); */ - num_records = flb_mp_count(event_chunk->data, event_chunk->size); + num_records = flb_mp_count_log_records(event_chunk->data, event_chunk->size); while ((ret = flb_log_event_decoder_next( &log_decoder, diff --git a/plugins/out_s3/s3.c b/plugins/out_s3/s3.c index 97c1b1d0fdb..ef2aca16a44 100644 --- a/plugins/out_s3/s3.c +++ b/plugins/out_s3/s3.c @@ -3385,7 +3385,7 @@ static flb_sds_t flb_pack_msgpack_extract_log_key(void *out_context, const char struct flb_log_event log_event; /* Iterate the original buffer and perform adjustments */ - records = flb_mp_count(data, bytes); + records = flb_mp_count_log_records(data, bytes); if (records <= 0) { return NULL; } diff --git a/plugins/out_skywalking/skywalking.c b/plugins/out_skywalking/skywalking.c index 1c266aae5c7..a646f6e6b4a 100644 --- a/plugins/out_skywalking/skywalking.c +++ b/plugins/out_skywalking/skywalking.c @@ -252,7 +252,7 @@ static int sw_format(struct flb_output_sw* ctx, const void *data, size_t bytes, msgpack_sbuffer_init(&sbuf); msgpack_packer_init(&pk, &sbuf, msgpack_sbuffer_write); - chunk_size = flb_mp_count(data, bytes); + chunk_size = flb_mp_count_log_records(data, bytes); flb_plg_debug(ctx->ins, "%i messages flushed", chunk_size); msgpack_pack_array(&pk, chunk_size); diff --git a/plugins/out_stackdriver/stackdriver.c b/plugins/out_stackdriver/stackdriver.c index 1375ba2c463..38fefd7a8fd 100644 --- a/plugins/out_stackdriver/stackdriver.c +++ b/plugins/out_stackdriver/stackdriver.c @@ -1772,7 +1772,8 @@ static flb_sds_t stackdriver_format(struct flb_stackdriver *ctx, int total_records, const char *tag, int tag_len, const void *data, size_t bytes, - struct flb_config *config) + struct flb_config *config, + int *formatted_records) { int len; int ret; @@ -1851,6 +1852,10 @@ static flb_sds_t stackdriver_format(struct flb_stackdriver *ctx, /* Count number of records */ array_size = total_records; + if (formatted_records != NULL) { + *formatted_records = 0; + } + /* Parameters for labels */ msgpack_object *payload_labels_ptr; int labels_size = 0; @@ -1893,6 +1898,10 @@ static flb_sds_t stackdriver_format(struct flb_stackdriver *ctx, return NULL; } + if (formatted_records != NULL) { + *formatted_records = array_size; + } + /* Create temporal msgpack buffer */ msgpack_sbuffer_init(&mp_sbuf); msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); @@ -2677,10 +2686,10 @@ static int stackdriver_format_test(struct flb_config *config, struct flb_stackdriver *ctx = plugin_context; /* Count number of records */ - total_records = flb_mp_count(data, bytes); + total_records = flb_mp_count_log_records(data, bytes); payload = stackdriver_format(ctx, total_records, - (char *) tag, tag_len, data, bytes, config); + (char *) tag, tag_len, data, bytes, config, NULL); if (payload == NULL) { return -1; } @@ -2729,7 +2738,7 @@ static void update_http_metrics(struct flb_stackdriver* ctx, } static void update_retry_metric(struct flb_stackdriver *ctx, - struct flb_event_chunk *event_chunk, + int retried_records, uint64_t ts, int http_status) { @@ -2739,7 +2748,7 @@ static void update_retry_metric(struct flb_stackdriver *ctx, /* convert status to string format */ snprintf(tmp, sizeof(tmp) - 1, "%i", http_status); cmt_counter_add(ctx->cmt_retried_records_total, - ts, event_chunk->total_events, 2, (char *[]) {tmp, name}); + ts, retried_records, 2, (char *[]) {tmp, name}); } #endif @@ -2747,7 +2756,6 @@ static void update_retry_metric(struct flb_stackdriver *ctx, static int parse_partial_success_response(struct flb_http_client* c, struct flb_stackdriver* ctx, uint64_t ts, - int total_events, int* grpc_status_codes) { int ret; @@ -2931,6 +2939,7 @@ static void cb_stackdriver_flush(struct flb_event_chunk *event_chunk, int code; int ret_partial_success; int ret_code = FLB_RETRY; + int formatted_records = 0; int grpc_status_counts[GRPC_STATUS_CODES_SIZE] = {0}; size_t b_sent; flb_sds_t token; @@ -2954,7 +2963,8 @@ static void cb_stackdriver_flush(struct flb_event_chunk *event_chunk, event_chunk->total_events, event_chunk->tag, flb_sds_len(event_chunk->tag), event_chunk->data, event_chunk->size, - config); + config, + &formatted_records); if (!payload_buf) { #ifdef FLB_HAVE_METRICS cmt_counter_inc(ctx->cmt_failed_requests, @@ -2982,7 +2992,7 @@ static void cb_stackdriver_flush(struct flb_event_chunk *event_chunk, /* OLD api */ flb_metrics_sum(FLB_STACKDRIVER_FAILED_REQUESTS, 1, ctx->ins->metrics); - update_retry_metric(ctx, event_chunk, ts, STACKDRIVER_NET_ERROR); + update_retry_metric(ctx, formatted_records, ts, STACKDRIVER_NET_ERROR); #endif flb_sds_destroy(payload_buf); FLB_OUTPUT_RETURN(FLB_RETRY); @@ -3065,7 +3075,6 @@ static void cb_stackdriver_flush(struct flb_event_chunk *event_chunk, parse_partial_success_response(c, ctx, ts, - (int) event_chunk->total_events, grpc_status_counts); int failed_records = 0; @@ -3078,16 +3087,16 @@ static void cb_stackdriver_flush(struct flb_event_chunk *event_chunk, cmt_counter_add(ctx->ins->cmt_dropped_records, ts, failed_records, 1, (char* []) {name}); int successful_records = - (int) event_chunk->total_events - failed_records; + formatted_records - failed_records; if (successful_records != 0) { add_record_metrics(ctx, ts, successful_records, 200, 0); } } else { - add_record_metrics(ctx, ts, (int) event_chunk->total_events, + add_record_metrics(ctx, ts, formatted_records, c->resp.status, -1); cmt_counter_add(ctx->ins->cmt_dropped_records, ts, - (int) event_chunk->total_events, 1, + formatted_records, 1, (char* []) {name}); } #endif @@ -3118,7 +3127,7 @@ static void cb_stackdriver_flush(struct flb_event_chunk *event_chunk, if (write_entries_latency > 0.0) { cmt_histogram_observe(ctx->cmt_write_entries_latency, ts, write_entries_latency, 1, (char *[]) {name}); } - add_record_metrics(ctx, ts, (int) event_chunk->total_events, 200, 0); + add_record_metrics(ctx, ts, formatted_records, 200, 0); /* OLD api */ flb_metrics_sum(FLB_STACKDRIVER_SUCCESSFUL_REQUESTS, 1, ctx->ins->metrics); @@ -3131,7 +3140,7 @@ static void cb_stackdriver_flush(struct flb_event_chunk *event_chunk, } if (ret_code == FLB_RETRY) { - update_retry_metric(ctx, event_chunk, ts, c->resp.status); + update_retry_metric(ctx, formatted_records, ts, c->resp.status); } /* Update metrics counter by using labels/http status code */ diff --git a/src/flb_engine.c b/src/flb_engine.c index 9d998b1b702..090bfc3c72f 100644 --- a/src/flb_engine.c +++ b/src/flb_engine.c @@ -286,11 +286,13 @@ static inline int handle_output_event(uint64_t ts, int ret; int task_id; int out_id; + int effective_records = 0; int retries; int retry_seconds; uint32_t type; uint32_t key; double latency_seconds; + size_t effective_bytes = 0; char *in_name; char *out_name; struct flb_task *task; @@ -340,6 +342,14 @@ static inline int handle_output_event(uint64_t ts, } in_name = (char *) flb_input_name(task->i_ins); out_name = (char *) flb_output_name(ins); + flb_task_acquire_lock(task); + if (flb_task_get_route_data(task, ins, + &effective_records, + &effective_bytes) != 0) { + effective_records = task->event_chunk->total_events; + effective_bytes = task->event_chunk->size; + } + flb_task_release_lock(task); /* If we are in synchronous mode, flush the next waiting task */ if (ins->flags & FLB_OUTPUT_SYNCHRONOUS) { @@ -351,19 +361,19 @@ static inline int handle_output_event(uint64_t ts, /* A task has finished, delete it */ if (ret == FLB_OK) { /* cmetrics */ - cmt_counter_add(ins->cmt_proc_records, ts, task->event_chunk->total_events, + cmt_counter_add(ins->cmt_proc_records, ts, effective_records, 1, (char *[]) {out_name}); - cmt_counter_add(ins->cmt_proc_bytes, ts, task->event_chunk->size, + cmt_counter_add(ins->cmt_proc_bytes, ts, effective_bytes, 1, (char *[]) {out_name}); if (config->router && task->event_chunk->type == FLB_EVENT_TYPE_LOGS) { cmt_counter_add(config->router->logs_records_total, ts, - task->event_chunk->total_events, + effective_records, 2, (char *[]) {in_name, out_name}); cmt_counter_add(config->router->logs_bytes_total, ts, - task->event_chunk->size, + effective_bytes, 2, (char *[]) {in_name, out_name}); } @@ -378,9 +388,9 @@ static inline int handle_output_event(uint64_t ts, #ifdef FLB_HAVE_METRICS if (ins->metrics) { flb_metrics_sum(FLB_METRIC_OUT_OK_RECORDS, - task->event_chunk->total_events, ins->metrics); + effective_records, ins->metrics); flb_metrics_sum(FLB_METRIC_OUT_OK_BYTES, - task->event_chunk->size, ins->metrics); + effective_bytes, ins->metrics); } #endif /* Inform the user if a 'retry' succedeed */ @@ -416,17 +426,17 @@ static inline int handle_output_event(uint64_t ts, handle_dlq_if_available(config, task, ins, 0); /* cmetrics: output_dropped_records_total */ - cmt_counter_add(ins->cmt_dropped_records, ts, task->records, + cmt_counter_add(ins->cmt_dropped_records, ts, effective_records, 1, (char *[]) {out_name}); if (config->router && task->event_chunk && task->event_chunk->type == FLB_EVENT_TYPE_LOGS) { cmt_counter_add(config->router->logs_drop_records_total, ts, - task->records, + effective_records, 2, (char *[]) {in_name, out_name}); cmt_counter_add(config->router->logs_drop_bytes_total, ts, - task->event_chunk->size, + effective_bytes, 2, (char *[]) {in_name, out_name}); } @@ -436,7 +446,7 @@ static inline int handle_output_event(uint64_t ts, /* OLD metrics API */ #ifdef FLB_HAVE_METRICS - flb_metrics_sum(FLB_METRIC_OUT_DROPPED_RECORDS, task->records, ins->metrics); + flb_metrics_sum(FLB_METRIC_OUT_DROPPED_RECORDS, effective_records, ins->metrics); #endif flb_info("[engine] chunk '%s' is not retried (no retry config): " "task_id=%i, input=%s > output=%s (out_id=%i)", @@ -465,17 +475,17 @@ static inline int handle_output_event(uint64_t ts, /* cmetrics */ cmt_counter_inc(ins->cmt_retries_failed, ts, 1, (char *[]) {out_name}); - cmt_counter_add(ins->cmt_dropped_records, ts, task->records, + cmt_counter_add(ins->cmt_dropped_records, ts, effective_records, 1, (char *[]) {out_name}); if (config->router && task->event_chunk && task->event_chunk->type == FLB_EVENT_TYPE_LOGS) { cmt_counter_add(config->router->logs_drop_records_total, ts, - task->records, + effective_records, 2, (char *[]) {in_name, out_name}); cmt_counter_add(config->router->logs_drop_bytes_total, ts, - task->event_chunk->size, + effective_bytes, 2, (char *[]) {in_name, out_name}); } @@ -486,7 +496,7 @@ static inline int handle_output_event(uint64_t ts, /* OLD metrics API */ #ifdef FLB_HAVE_METRICS flb_metrics_sum(FLB_METRIC_OUT_RETRY_FAILED, 1, ins->metrics); - flb_metrics_sum(FLB_METRIC_OUT_DROPPED_RECORDS, task->records, ins->metrics); + flb_metrics_sum(FLB_METRIC_OUT_DROPPED_RECORDS, effective_records, ins->metrics); #endif /* Notify about this failed retry */ flb_error("[engine] chunk '%s' cannot be retried: " @@ -538,7 +548,7 @@ static inline int handle_output_event(uint64_t ts, /* cmetrics */ cmt_counter_inc(ins->cmt_retries, ts, 1, (char *[]) {out_name}); - cmt_counter_add(ins->cmt_retried_records, ts, task->records, + cmt_counter_add(ins->cmt_retried_records, ts, effective_records, 1, (char *[]) {out_name}); cmt_gauge_set(ins->cmt_chunk_available_capacity_percent, ts, @@ -548,7 +558,7 @@ static inline int handle_output_event(uint64_t ts, /* OLD metrics API: update the metrics since a new retry is coming */ #ifdef FLB_HAVE_METRICS flb_metrics_sum(FLB_METRIC_OUT_RETRY, 1, ins->metrics); - flb_metrics_sum(FLB_METRIC_OUT_RETRIED_RECORDS, task->records, ins->metrics); + flb_metrics_sum(FLB_METRIC_OUT_RETRIED_RECORDS, effective_records, ins->metrics); #endif } } @@ -556,17 +566,17 @@ static inline int handle_output_event(uint64_t ts, handle_dlq_if_available(config, task, ins, 0); /* cmetrics */ cmt_counter_inc(ins->cmt_errors, ts, 1, (char *[]) {out_name}); - cmt_counter_add(ins->cmt_dropped_records, ts, task->records, + cmt_counter_add(ins->cmt_dropped_records, ts, effective_records, 1, (char *[]) {out_name}); if (config->router && task->event_chunk && task->event_chunk->type == FLB_EVENT_TYPE_LOGS) { cmt_counter_add(config->router->logs_drop_records_total, ts, - task->records, + effective_records, 2, (char *[]) {in_name, out_name}); cmt_counter_add(config->router->logs_drop_bytes_total, ts, - task->event_chunk->size, + effective_bytes, 2, (char *[]) {in_name, out_name}); } @@ -577,7 +587,7 @@ static inline int handle_output_event(uint64_t ts, /* OLD API */ #ifdef FLB_HAVE_METRICS flb_metrics_sum(FLB_METRIC_OUT_ERROR, 1, ins->metrics); - flb_metrics_sum(FLB_METRIC_OUT_DROPPED_RECORDS, task->records, ins->metrics); + flb_metrics_sum(FLB_METRIC_OUT_DROPPED_RECORDS, effective_records, ins->metrics); #endif flb_task_retry_clean(task, ins); diff --git a/src/flb_filter.c b/src/flb_filter.c index b4bae5b41f8..0daa5e92195 100644 --- a/src/flb_filter.c +++ b/src/flb_filter.c @@ -267,7 +267,7 @@ void flb_filter_do(struct flb_input_chunk *ic, break; } else { - out_records = flb_mp_count(out_buf, out_size); + out_records = flb_mp_count_log_records(out_buf, out_size); #ifdef FLB_HAVE_METRICS if (out_records > in_records) { diff --git a/src/flb_input_chunk.c b/src/flb_input_chunk.c index 58d98779782..31b371f2ef9 100644 --- a/src/flb_input_chunk.c +++ b/src/flb_input_chunk.c @@ -245,7 +245,8 @@ static int flb_input_chunk_is_task_safe_delete(struct flb_task *task); static int flb_input_chunk_drop_task_route( struct flb_task *task, struct flb_output_instance *o_ins, - ssize_t *dropped_record_count); + ssize_t *dropped_record_count, + ssize_t *dropped_byte_count); static ssize_t get_input_chunk_record_count(struct flb_input_chunk *input_chunk) @@ -274,7 +275,12 @@ static ssize_t get_input_chunk_record_count(struct flb_input_chunk *input_chunk) &chunk_size); if (ret == CIO_OK) { - record_count = flb_mp_count(chunk_buffer, chunk_size); + if (input_chunk->event_type == FLB_INPUT_LOGS) { + record_count = flb_mp_count_log_records(chunk_buffer, chunk_size); + } + else { + record_count = flb_mp_count(chunk_buffer, chunk_size); + } } else { record_count = -1; @@ -298,6 +304,7 @@ static int flb_input_chunk_release_space( struct mk_list *input_chunk_iterator; struct flb_router *router; ssize_t dropped_record_count; + ssize_t dropped_byte_count; int chunk_destroy_flag; struct flb_input_chunk *old_input_chunk; ssize_t released_space; @@ -325,7 +332,8 @@ static int flb_input_chunk_release_space( if (flb_input_chunk_drop_task_route(old_input_chunk->task, output_plugin, - &dropped_record_count) == FLB_FALSE) { + &dropped_record_count, + &dropped_byte_count) == FLB_FALSE) { continue; } @@ -352,42 +360,46 @@ static int flb_input_chunk_release_space( } #ifdef FLB_HAVE_METRICS - if (dropped_record_count == 0) { + if (dropped_record_count < 0) { dropped_record_count = get_input_chunk_record_count(old_input_chunk); + } - if (dropped_record_count == -1) { - flb_debug("[task] error getting chunk record count : %s", - old_input_chunk->in->name); - } - else { - cmt_counter_add(output_plugin->cmt_dropped_records, + if (dropped_byte_count < 0) { + dropped_byte_count = chunk_size; + } + + if (dropped_record_count == -1) { + flb_debug("[task] error getting chunk record count : %s", + old_input_chunk->in->name); + } + else if (dropped_record_count > 0) { + cmt_counter_add(output_plugin->cmt_dropped_records, + cfl_time_now(), + dropped_record_count, + 1, (char *[]) {(char *) flb_output_name(output_plugin)}); + + if (input_plugin->config && input_plugin->config->router && + old_input_chunk->event_type == FLB_INPUT_LOGS) { + router = input_plugin->config->router; + + cmt_counter_add(router->logs_drop_records_total, cfl_time_now(), - dropped_record_count, - 1, (char *[]) {(char *) flb_output_name(output_plugin)}); - - if (input_plugin->config && input_plugin->config->router && - old_input_chunk->event_type == FLB_INPUT_LOGS) { - router = input_plugin->config->router; - - cmt_counter_add(router->logs_drop_records_total, - cfl_time_now(), - (double) dropped_record_count, - 2, - (char *[]){(char *) flb_input_name(old_input_chunk->in), - (char *) flb_output_name(output_plugin)}); - - cmt_counter_add(router->logs_drop_bytes_total, - cfl_time_now(), - (double) chunk_size, - 2, - (char *[]){(char *) flb_input_name(old_input_chunk->in), - (char *) flb_output_name(output_plugin)}); - } + (double) dropped_record_count, + 2, + (char *[]){(char *) flb_input_name(old_input_chunk->in), + (char *) flb_output_name(output_plugin)}); - flb_metrics_sum(FLB_METRIC_OUT_DROPPED_RECORDS, - dropped_record_count, - output_plugin->metrics); + cmt_counter_add(router->logs_drop_bytes_total, + cfl_time_now(), + (double) dropped_byte_count, + 2, + (char *[]){(char *) flb_input_name(old_input_chunk->in), + (char *) flb_output_name(output_plugin)}); } + + flb_metrics_sum(FLB_METRIC_OUT_DROPPED_RECORDS, + dropped_record_count, + output_plugin->metrics); } #endif @@ -509,12 +521,18 @@ int flb_input_chunk_write_at(void *data, off_t offset, static int flb_input_chunk_drop_task_route( struct flb_task *task, struct flb_output_instance *output_plugin, - ssize_t *dropped_record_count) + ssize_t *dropped_record_count, + ssize_t *dropped_byte_count) { int route_status; + int route_records; int result; + size_t route_bytes; - *dropped_record_count = 0; + *dropped_record_count = -1; + *dropped_byte_count = -1; + route_records = 0; + route_bytes = 0; if (task == NULL) { return FLB_TRUE; @@ -535,7 +553,12 @@ static int flb_input_chunk_drop_task_route( output_plugin, FLB_TASK_ROUTE_DROPPED); - *dropped_record_count = (ssize_t) task->records; + if (flb_task_get_route_data(task, output_plugin, + &route_records, + &route_bytes) == 0) { + *dropped_record_count = (ssize_t) route_records; + *dropped_byte_count = (ssize_t) route_bytes; + } result = FLB_TRUE; } @@ -1070,9 +1093,15 @@ struct flb_input_chunk *flb_input_chunk_map(struct flb_input_instance *in, cio_chunk_write_at(chunk, offset, NULL, 0); } + if (ic->event_type == FLB_INPUT_LOGS) { + ic->total_records = flb_mp_count_log_records(buf_data, offset); + } + else { + ic->total_records = records; + } + /* Update metrics */ #ifdef FLB_HAVE_METRICS - ic->total_records = records; if (ic->total_records > 0) { /* timestamp */ ts = cfl_time_now(); @@ -2734,10 +2763,14 @@ static int input_chunk_append_raw(struct flb_input_instance *in, } /* - * Set the total_records based on the records that n_records - * says we should be writing. These values may be overwritten - * flb_filter_do, where a filter may add/remove records. + * Set total_records based on the caller-provided count. For log chunks, + * recover from callers that pass 0 by deriving the logical record count + * from the payload (group markers excluded). */ + if (event_type == FLB_INPUT_LOGS && n_records == 0) { + n_records = flb_mp_count_log_records(buf, buf_size); + } + total_records_start = ic->total_records; ic->added_records = n_records; ic->total_records += n_records; diff --git a/src/flb_input_log.c b/src/flb_input_log.c index 11ffaba77ba..fcf4b594a06 100644 --- a/src/flb_input_log.c +++ b/src/flb_input_log.c @@ -1351,7 +1351,7 @@ static int input_log_append(struct flb_input_instance *ins, if (buf != out_buf) { /* a new buffer was created, re-count the number of records */ - records = flb_mp_count(out_buf, out_size); + records = flb_mp_count_log_records(out_buf, out_size); } } @@ -1417,7 +1417,7 @@ int flb_input_log_append(struct flb_input_instance *ins, int ret; size_t records; - records = flb_mp_count(buf, buf_size); + records = flb_mp_count_log_records(buf, buf_size); ret = input_log_append(ins, 0, records, tag, tag_len, buf, buf_size); return ret; } @@ -1432,7 +1432,7 @@ int flb_input_log_append_skip_processor_stages(struct flb_input_instance *ins, { return input_log_append(ins, processor_starting_stage, - flb_mp_count(buf, buf_size), + flb_mp_count_log_records(buf, buf_size), tag, tag_len, buf, @@ -1451,4 +1451,3 @@ int flb_input_log_append_records(struct flb_input_instance *ins, return ret; } - diff --git a/src/flb_lib.c b/src/flb_lib.c index 859cc11d310..7257480b063 100644 --- a/src/flb_lib.c +++ b/src/flb_lib.c @@ -30,6 +30,7 @@ #include #include #include +#include #include #include #include @@ -339,7 +340,7 @@ int flb_input_set_processor(flb_ctx_t *ctx, int ffd, struct flb_processor *proc) struct flb_input_instance *i_ins; i_ins = in_instance_get(ctx, ffd); - if (!i_ins) { + if (!i_ins || proc == NULL) { return -1; } @@ -347,6 +348,8 @@ int flb_input_set_processor(flb_ctx_t *ctx, int ffd, struct flb_processor *proc) flb_processor_destroy(i_ins->processor); } + proc->data = i_ins; + proc->source_plugin_type = FLB_PLUGIN_INPUT; i_ins->processor = proc; return 0; @@ -555,7 +558,7 @@ int flb_output_set_processor(flb_ctx_t *ctx, int ffd, struct flb_processor *proc struct flb_output_instance *o_ins; o_ins = out_instance_get(ctx, ffd); - if (!o_ins) { + if (!o_ins || proc == NULL) { return -1; } @@ -563,6 +566,8 @@ int flb_output_set_processor(flb_ctx_t *ctx, int ffd, struct flb_processor *proc flb_processor_destroy(o_ins->processor); } + proc->data = o_ins; + proc->source_plugin_type = FLB_PLUGIN_OUTPUT; o_ins->processor = proc; return 0; diff --git a/src/flb_mp.c b/src/flb_mp.c index 641616372cd..ac3f2831ece 100644 --- a/src/flb_mp.c +++ b/src/flb_mp.c @@ -45,6 +45,272 @@ int flb_mp_count(const void *data, size_t bytes) return flb_mp_count_remaining(data, bytes, NULL); } +/* Return the number of log events excluding group markers */ +int flb_mp_count_log_records(const void *data, size_t bytes) +{ + int count; + int ret; + struct flb_log_event log_event; + struct flb_log_event_decoder decoder; + + count = 0; + + ret = flb_log_event_decoder_init(&decoder, (char *) data, bytes); + if (ret != FLB_EVENT_DECODER_SUCCESS) { + return 0; + } + + while ((ret = flb_log_event_decoder_next(&decoder, + &log_event)) == FLB_EVENT_DECODER_SUCCESS) { + count++; + } + + flb_log_event_decoder_destroy(&decoder); + + return count; +} + +struct flb_mp_record_span { + const char *base; + size_t length; + int type; + int keep; +}; + +int flb_mp_normalize_log_buffer_groups_msgpack(const void *in_buf, size_t in_size, + char **out_buf, size_t *out_size) +{ + int i; + int ret; + int entry_count; + int stack_count; + int *group_stack; + int *group_has_content; + struct flb_log_event log_event; + struct flb_log_event_decoder decoder; + struct flb_mp_record_span *entries; + msgpack_sbuffer mp_sbuf; + + *out_buf = NULL; + *out_size = 0; + + entry_count = flb_mp_count(in_buf, in_size); + if (entry_count <= 0) { + return 0; + } + + entries = flb_calloc(entry_count, sizeof(struct flb_mp_record_span)); + if (entries == NULL) { + flb_errno(); + return -1; + } + + group_stack = flb_calloc(entry_count, sizeof(int)); + if (group_stack == NULL) { + flb_errno(); + flb_free(entries); + return -1; + } + + group_has_content = flb_calloc(entry_count, sizeof(int)); + if (group_has_content == NULL) { + flb_errno(); + flb_free(group_stack); + flb_free(entries); + return -1; + } + + ret = flb_log_event_decoder_init(&decoder, (char *) in_buf, in_size); + if (ret != FLB_EVENT_DECODER_SUCCESS) { + flb_free(group_has_content); + flb_free(group_stack); + flb_free(entries); + return -1; + } + + ret = flb_log_event_decoder_read_groups(&decoder, FLB_TRUE); + if (ret != FLB_EVENT_DECODER_SUCCESS) { + flb_log_event_decoder_destroy(&decoder); + flb_free(group_has_content); + flb_free(group_stack); + flb_free(entries); + return -1; + } + + i = 0; + while (i < entry_count && + (ret = flb_log_event_decoder_next(&decoder, &log_event)) == + FLB_EVENT_DECODER_SUCCESS) { + entries[i].base = decoder.record_base; + entries[i].length = decoder.record_length; + entries[i].keep = FLB_TRUE; + + ret = flb_log_event_decoder_get_record_type(&log_event, + &entries[i].type); + if (ret != FLB_EVENT_DECODER_SUCCESS) { + entries[i].type = FLB_LOG_EVENT_NORMAL; + } + + i++; + } + + flb_log_event_decoder_destroy(&decoder); + + entry_count = i; + if (entry_count == 0) { + flb_free(group_has_content); + flb_free(group_stack); + flb_free(entries); + return 0; + } + + stack_count = 0; + + for (i = 0; i < entry_count; i++) { + if (entries[i].type == FLB_LOG_EVENT_GROUP_START) { + group_stack[stack_count] = i; + group_has_content[stack_count] = FLB_FALSE; + stack_count++; + continue; + } + + if (entries[i].type == FLB_LOG_EVENT_GROUP_END) { + if (stack_count == 0) { + entries[i].keep = FLB_FALSE; + continue; + } + + stack_count--; + + if (group_has_content[stack_count] == FLB_FALSE) { + entries[group_stack[stack_count]].keep = FLB_FALSE; + entries[i].keep = FLB_FALSE; + } + else if (stack_count > 0) { + group_has_content[stack_count - 1] = FLB_TRUE; + } + + continue; + } + + if (stack_count > 0) { + group_has_content[stack_count - 1] = FLB_TRUE; + } + } + + while (stack_count > 0) { + stack_count--; + entries[group_stack[stack_count]].keep = FLB_FALSE; + } + + msgpack_sbuffer_init(&mp_sbuf); + + for (i = 0; i < entry_count; i++) { + if (entries[i].keep != FLB_TRUE) { + continue; + } + + msgpack_sbuffer_write(&mp_sbuf, entries[i].base, entries[i].length); + } + + if (mp_sbuf.size > 0) { + *out_buf = mp_sbuf.data; + *out_size = mp_sbuf.size; + } + else { + msgpack_sbuffer_destroy(&mp_sbuf); + } + + flb_free(group_has_content); + flb_free(group_stack); + flb_free(entries); + + return 0; +} + +int flb_mp_normalize_log_buffer_groups(const void *in_buf, size_t in_size, + char **out_buf, size_t *out_size) +{ + int ret; + char *encoded_buf; + size_t encoded_size; + struct flb_log_event_decoder *log_decoder; + struct flb_log_event_encoder *log_encoder; + struct flb_mp_chunk_cobj *chunk_cobj; + struct flb_mp_chunk_record *record; + + *out_buf = NULL; + *out_size = 0; + + log_decoder = flb_log_event_decoder_create((char *) in_buf, in_size); + if (log_decoder == NULL) { + return -1; + } + + ret = flb_log_event_decoder_read_groups(log_decoder, FLB_TRUE); + if (ret != FLB_EVENT_DECODER_SUCCESS) { + flb_log_event_decoder_destroy(log_decoder); + return -1; + } + + log_encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); + if (log_encoder == NULL) { + flb_log_event_decoder_destroy(log_decoder); + return -1; + } + + chunk_cobj = flb_mp_chunk_cobj_create(log_encoder, log_decoder); + if (chunk_cobj == NULL) { + flb_log_event_encoder_destroy(log_encoder); + flb_log_event_decoder_destroy(log_decoder); + return -1; + } + + while ((ret = flb_mp_chunk_cobj_record_next(chunk_cobj, &record)) == + FLB_MP_CHUNK_RECORD_OK) { + (void) record; + } + + if (ret != FLB_MP_CHUNK_RECORD_EOF) { + flb_mp_chunk_cobj_destroy(chunk_cobj); + flb_log_event_encoder_destroy(log_encoder); + flb_log_event_decoder_destroy(log_decoder); + return -1; + } + + ret = flb_mp_chunk_cobj_normalize_groups(chunk_cobj); + if (ret != 0) { + flb_mp_chunk_cobj_destroy(chunk_cobj); + flb_log_event_encoder_destroy(log_encoder); + flb_log_event_decoder_destroy(log_decoder); + return -1; + } + + if (cfl_list_size(&chunk_cobj->records) == 0) { + flb_mp_chunk_cobj_destroy(chunk_cobj); + flb_log_event_encoder_destroy(log_encoder); + flb_log_event_decoder_destroy(log_decoder); + return 0; + } + + ret = flb_mp_chunk_cobj_encode(chunk_cobj, &encoded_buf, &encoded_size); + if (ret != 0) { + flb_mp_chunk_cobj_destroy(chunk_cobj); + flb_log_event_encoder_destroy(log_encoder); + flb_log_event_decoder_destroy(log_decoder); + return -1; + } + + *out_buf = encoded_buf; + *out_size = encoded_size; + + flb_mp_chunk_cobj_destroy(chunk_cobj); + flb_log_event_encoder_destroy(log_encoder); + flb_log_event_decoder_destroy(log_decoder); + + return 0; +} + int flb_mp_count_remaining(const void *data, size_t bytes, size_t *remaining_bytes) { size_t remaining; @@ -191,7 +457,14 @@ int flb_mp_validate_log_chunk(const void *data, size_t bytes, if (ts.type != MSGPACK_OBJECT_POSITIVE_INTEGER && ts.type != MSGPACK_OBJECT_FLOAT && - ts.type != MSGPACK_OBJECT_EXT) { + ts.type != MSGPACK_OBJECT_EXT && + ts.type != MSGPACK_OBJECT_NEGATIVE_INTEGER) { + goto error; + } + + if (ts.type == MSGPACK_OBJECT_NEGATIVE_INTEGER && + ts.via.i64 != FLB_LOG_EVENT_GROUP_START && + ts.via.i64 != FLB_LOG_EVENT_GROUP_END) { goto error; } @@ -1206,6 +1479,169 @@ int flb_mp_chunk_cobj_encode(struct flb_mp_chunk_cobj *chunk_cobj, char **out_bu return 0; } +int flb_mp_chunk_cobj_count_log_records(struct flb_mp_chunk_cobj *chunk_cobj) +{ + int count; + int record_type; + int ret; + struct cfl_list *head; + struct flb_mp_chunk_record *record; + + if (chunk_cobj == NULL) { + return 0; + } + + count = 0; + + cfl_list_foreach(head, &chunk_cobj->records) { + record = cfl_list_entry(head, struct flb_mp_chunk_record, _head); + + ret = flb_log_event_decoder_get_record_type(&record->event, &record_type); + if (ret != FLB_EVENT_DECODER_SUCCESS) { + continue; + } + + if (record_type == FLB_LOG_EVENT_NORMAL) { + count++; + } + } + + return count; +} + +static int cobj_remove_list_add(struct flb_mp_chunk_record **remove_list, + size_t remove_list_size, + size_t *remove_list_count, + struct flb_mp_chunk_record *record) +{ + size_t index; + + if (record == NULL) { + return -1; + } + + for (index = 0; index < *remove_list_count; index++) { + if (remove_list[index] == record) { + return 0; + } + } + + if (*remove_list_count >= remove_list_size) { + return -1; + } + + remove_list[*remove_list_count] = record; + (*remove_list_count)++; + + return 0; +} + +int flb_mp_chunk_cobj_normalize_groups(struct flb_mp_chunk_cobj *chunk_cobj) +{ + int ret; + int record_type; + size_t index; + size_t stack_size; + size_t stack_count; + size_t remove_count; + struct cfl_list *head; + struct flb_mp_chunk_record *record; + struct flb_mp_chunk_record **remove_list; + struct flb_mp_chunk_record **group_start_stack; + int *group_has_content; + + if (chunk_cobj == NULL) { + return -1; + } + + stack_size = cfl_list_size(&chunk_cobj->records); + if (stack_size == 0) { + return 0; + } + + remove_list = flb_calloc(stack_size, sizeof(struct flb_mp_chunk_record *)); + if (remove_list == NULL) { + flb_errno(); + return -1; + } + + group_start_stack = flb_calloc(stack_size, sizeof(struct flb_mp_chunk_record *)); + if (group_start_stack == NULL) { + flb_errno(); + flb_free(remove_list); + return -1; + } + + group_has_content = flb_calloc(stack_size, sizeof(int)); + if (group_has_content == NULL) { + flb_errno(); + flb_free(group_start_stack); + flb_free(remove_list); + return -1; + } + + stack_count = 0; + remove_count = 0; + + cfl_list_foreach(head, &chunk_cobj->records) { + record = cfl_list_entry(head, struct flb_mp_chunk_record, _head); + + ret = flb_log_event_decoder_get_record_type(&record->event, &record_type); + if (ret != FLB_EVENT_DECODER_SUCCESS) { + continue; + } + + if (record_type == FLB_LOG_EVENT_GROUP_START) { + if (stack_count < stack_size) { + group_start_stack[stack_count] = record; + group_has_content[stack_count] = FLB_FALSE; + stack_count++; + } + continue; + } + + if (record_type == FLB_LOG_EVENT_GROUP_END) { + if (stack_count == 0) { + cobj_remove_list_add(remove_list, stack_size, &remove_count, record); + continue; + } + + stack_count--; + + if (group_has_content[stack_count] == FLB_FALSE) { + cobj_remove_list_add(remove_list, stack_size, &remove_count, + group_start_stack[stack_count]); + cobj_remove_list_add(remove_list, stack_size, &remove_count, record); + } + else if (stack_count > 0) { + group_has_content[stack_count - 1] = FLB_TRUE; + } + + continue; + } + + for (index = 0; index < stack_count; index++) { + group_has_content[index] = FLB_TRUE; + } + } + + while (stack_count > 0) { + stack_count--; + cobj_remove_list_add(remove_list, stack_size, &remove_count, + group_start_stack[stack_count]); + } + + for (index = 0; index < remove_count; index++) { + flb_mp_chunk_cobj_record_destroy(chunk_cobj, remove_list[index]); + } + + flb_free(group_has_content); + flb_free(group_start_stack); + flb_free(remove_list); + + return 0; +} + int flb_mp_chunk_cobj_destroy(struct flb_mp_chunk_cobj *chunk_cobj) { struct cfl_list *tmp; diff --git a/src/flb_processor.c b/src/flb_processor.c index c0787b3eff0..2f03e842b63 100644 --- a/src/flb_processor.c +++ b/src/flb_processor.c @@ -31,6 +31,10 @@ #include #include #include +#include +#include +#include +#include #include struct flb_config_map processor_global_properties[] = { @@ -51,6 +55,224 @@ struct flb_config_map processor_global_properties[] = { {0} }; +static const char *processor_metrics_scope(struct flb_processor *proc) +{ + if (proc->source_plugin_type == FLB_PLUGIN_INPUT) { + return "input"; + } + else if (proc->source_plugin_type == FLB_PLUGIN_OUTPUT) { + return "output"; + } + + return "unknown"; +} + +static const char *processor_metrics_owner(struct flb_processor *proc) +{ + struct flb_input_instance *in; + struct flb_output_instance *out; + + if (proc->source_plugin_type == FLB_PLUGIN_INPUT) { + in = proc->data; + if (in != NULL) { + return flb_input_name(in); + } + } + else if (proc->source_plugin_type == FLB_PLUGIN_OUTPUT) { + out = proc->data; + if (out != NULL) { + return flb_output_name(out); + } + } + + return "unknown"; +} + +static struct cmt *processor_metrics_context(struct flb_processor *proc) +{ + struct flb_input_instance *in; + struct flb_output_instance *out; + + if (proc->source_plugin_type == FLB_PLUGIN_INPUT) { + in = proc->data; + if (in != NULL) { + return in->cmt; + } + } + else if (proc->source_plugin_type == FLB_PLUGIN_OUTPUT) { + out = proc->data; + if (out != NULL) { + return out->cmt; + } + } + + return NULL; +} + +static const char *processor_signal_type_to_string(int type) +{ + if (type == FLB_PROCESSOR_LOGS) { + return "logs"; + } + else if (type == FLB_PROCESSOR_METRICS) { + return "metrics"; + } + else if (type == FLB_PROCESSOR_TRACES) { + return "traces"; + } + else if (type == FLB_PROCESSOR_PROFILES) { + return "profiles"; + } + + return "unknown"; +} + +static int processor_metrics_register(struct flb_processor *proc) +{ + struct cmt *cmt; + + if (proc == NULL) { + return -1; + } + + if (proc->cmt_invocations != NULL) { + return 0; + } + + cmt = processor_metrics_context(proc); + if (cmt == NULL) { + return 0; + } + + proc->cmt_invocations = cmt_counter_create(cmt, + "fluentbit", + "processor", + "invocations_total", + "Total number of processor unit invocations.", + 5, + (char *[]) {"scope", "owner", + "processor", "stage", + "signal"}); + if (proc->cmt_invocations == NULL) { + return -1; + } + + proc->cmt_errors = cmt_counter_create(cmt, + "fluentbit", + "processor", + "errors_total", + "Total number of processor unit errors.", + 5, + (char *[]) {"scope", "owner", + "processor", "stage", + "signal"}); + if (proc->cmt_errors == NULL) { + return -1; + } + + proc->cmt_items_in = cmt_counter_create(cmt, + "fluentbit", + "processor", + "items_in_total", + "Total number of items seen by processor units.", + 5, + (char *[]) {"scope", "owner", + "processor", "stage", + "signal"}); + if (proc->cmt_items_in == NULL) { + return -1; + } + + proc->cmt_items_out = cmt_counter_create(cmt, + "fluentbit", + "processor", + "items_out_total", + "Total number of items produced by processor units.", + 5, + (char *[]) {"scope", "owner", + "processor", "stage", + "signal"}); + if (proc->cmt_items_out == NULL) { + return -1; + } + + proc->cmt_items_drop = cmt_counter_create(cmt, + "fluentbit", + "processor", + "items_drop_total", + "Total number of items dropped by processor units.", + 5, + (char *[]) {"scope", "owner", + "processor", "stage", + "signal"}); + if (proc->cmt_items_drop == NULL) { + return -1; + } + + proc->cmt_items_add = cmt_counter_create(cmt, + "fluentbit", + "processor", + "items_add_total", + "Total number of items added by processor units.", + 5, + (char *[]) {"scope", "owner", + "processor", "stage", + "signal"}); + if (proc->cmt_items_add == NULL) { + return -1; + } + + return 0; +} + +static void processor_metrics_update(struct flb_processor *proc, + struct flb_processor_unit *pu, + const char *scope, + const char *owner, + const char *signal, + int in_items, + int out_items, + int item_metrics_available, + int error) +{ + char stage_label[32]; + char *labels[5]; + + if (proc == NULL || pu == NULL || proc->cmt_invocations == NULL) { + return; + } + snprintf(stage_label, sizeof(stage_label) - 1, "%zu", pu->stage); + stage_label[sizeof(stage_label) - 1] = '\0'; + + labels[0] = (char *) scope; + labels[1] = (char *) owner; + labels[2] = (char *) pu->name; + labels[3] = (char *) stage_label; + labels[4] = (char *) signal; + + cmt_counter_inc(proc->cmt_invocations, cfl_time_now(), 5, labels); + + if (error == FLB_TRUE) { + cmt_counter_inc(proc->cmt_errors, cfl_time_now(), 5, labels); + } + + if (item_metrics_available == FLB_TRUE && in_items >= 0 && out_items >= 0) { + cmt_counter_add(proc->cmt_items_in, cfl_time_now(), in_items, 5, labels); + cmt_counter_add(proc->cmt_items_out, cfl_time_now(), out_items, 5, labels); + + if (in_items > out_items) { + cmt_counter_add(proc->cmt_items_drop, cfl_time_now(), + (in_items - out_items), + 5, labels); + } + else if (out_items > in_items) { + cmt_counter_add(proc->cmt_items_add, cfl_time_now(), + (out_items - in_items), + 5, labels); + } + } +} + static int append_raw_record_to_encoder(struct flb_log_event_encoder *encoder, char *record_base, size_t record_length) @@ -967,6 +1189,12 @@ int flb_processor_init(struct flb_processor *proc) struct mk_list *head; struct flb_processor_unit *pu; + ret = processor_metrics_register(proc); + if (ret != 0) { + flb_error("[processor] could not register processor metrics"); + return -1; + } + /* Go through every unit and initialize it */ mk_list_foreach(head, &proc->logs) { pu = mk_list_entry(head, struct flb_processor_unit, _head); @@ -1043,6 +1271,10 @@ int flb_processor_run(struct flb_processor *proc, void **out_buf, size_t *out_size) { int ret; + int pu_error; + int unit_in_items; + int unit_out_items; + int item_metrics_available; int finalize; void *cur_buf = NULL; size_t cur_size; @@ -1055,6 +1287,9 @@ int flb_processor_run(struct flb_processor *proc, struct flb_filter_instance *f_ins; struct flb_processor_instance *p_ins; struct flb_mp_chunk_cobj *chunk_cobj = NULL; + const char *metrics_scope; + const char *metrics_owner; + const char *metrics_signal; #ifdef FLB_HAVE_METRICS int in_records = 0; int out_records = 0; @@ -1076,6 +1311,10 @@ int flb_processor_run(struct flb_processor *proc, list = &proc->profiles; } + metrics_scope = processor_metrics_scope(proc); + metrics_owner = processor_metrics_owner(proc); + metrics_signal = processor_signal_type_to_string(type); + #ifdef FLB_HAVE_METRICS /* timestamp */ ts = cfl_time_now(); @@ -1099,6 +1338,16 @@ int flb_processor_run(struct flb_processor *proc, tmp_buf = NULL; tmp_size = 0; + pu_error = FLB_FALSE; + unit_in_items = -1; + unit_out_items = -1; + item_metrics_available = FLB_FALSE; + + if (type == FLB_PROCESSOR_LOGS) { + unit_in_items = flb_mp_count_log_records(cur_buf, cur_size); + unit_out_items = unit_in_items; + item_metrics_available = FLB_TRUE; + } ret = acquire_lock(&pu->lock, FLB_PROCESSOR_LOCK_RETRY_LIMIT, @@ -1136,7 +1385,7 @@ int flb_processor_run(struct flb_processor *proc, } #ifdef FLB_HAVE_METRICS name = (char *) (flb_filter_name(f_ins)); - in_records = flb_mp_count(cur_buf, cur_size); + in_records = flb_mp_count_log_records(cur_buf, cur_size); cmt_counter_add(f_ins->cmt_records, ts, in_records, 1, (char *[]) {name}); cmt_counter_add(f_ins->cmt_bytes, ts, tmp_size, @@ -1170,6 +1419,7 @@ int flb_processor_run(struct flb_processor *proc, if (tmp_size == 0) { *out_buf = NULL; *out_size = 0; + unit_out_items = 0; #ifdef FLB_HAVE_METRICS /* cmetrics */ @@ -1180,6 +1430,14 @@ int flb_processor_run(struct flb_processor *proc, flb_metrics_sum(FLB_METRIC_N_DROPPED, in_records, f_ins->metrics); #endif + processor_metrics_update(proc, pu, + metrics_scope, + metrics_owner, + metrics_signal, + unit_in_items, + unit_out_items, + item_metrics_available, + pu_error); release_lock(&pu->lock, FLB_PROCESSOR_LOCK_RETRY_LIMIT, FLB_PROCESSOR_LOCK_RETRY_DELAY); @@ -1190,7 +1448,8 @@ int flb_processor_run(struct flb_processor *proc, /* set new buffer */ cur_buf = tmp_buf; cur_size = tmp_size; - out_records = flb_mp_count(tmp_buf, tmp_size); + out_records = flb_mp_count_log_records(tmp_buf, tmp_size); + unit_out_items = out_records; #ifdef FLB_HAVE_METRICS if (out_records > in_records) { diff = (out_records - in_records); @@ -1219,6 +1478,7 @@ int flb_processor_run(struct flb_processor *proc, } else if (ret == FLB_FILTER_NOTOUCH) { /* keep original data, do nothing */ + unit_out_items = unit_in_items; } } else { @@ -1245,6 +1505,15 @@ int flb_processor_run(struct flb_processor *proc, flb_free(cur_buf); } + pu_error = FLB_TRUE; + processor_metrics_update(proc, pu, + metrics_scope, + metrics_owner, + metrics_signal, + unit_in_items, + unit_out_items, + item_metrics_available, + pu_error); release_lock(&pu->lock, FLB_PROCESSOR_LOCK_RETRY_LIMIT, FLB_PROCESSOR_LOCK_RETRY_DELAY); @@ -1265,6 +1534,7 @@ int flb_processor_run(struct flb_processor *proc, /* Invoke processor plugin callback */ ret = p_ins->p->cb_process_logs(p_ins, chunk_cobj, tag, tag_len); if (ret != FLB_PROCESSOR_SUCCESS) { + pu_error = FLB_TRUE; flb_warn("[processor] failed to process chunk"); } chunk_cobj->record_pos = NULL; @@ -1289,13 +1559,24 @@ int flb_processor_run(struct flb_processor *proc, } if (finalize == FLB_TRUE) { + flb_mp_chunk_cobj_normalize_groups(chunk_cobj); + if (cfl_list_size(&chunk_cobj->records) == 0) { flb_log_event_encoder_reset(p_ins->log_encoder); flb_mp_chunk_cobj_destroy(chunk_cobj); *out_buf = NULL; *out_size = 0; - + unit_out_items = 0; + + processor_metrics_update(proc, pu, + metrics_scope, + metrics_owner, + metrics_signal, + unit_in_items, + unit_out_items, + item_metrics_available, + pu_error); release_lock(&pu->lock, FLB_PROCESSOR_LOCK_RETRY_LIMIT, FLB_PROCESSOR_LOCK_RETRY_DELAY); @@ -1315,6 +1596,15 @@ int flb_processor_run(struct flb_processor *proc, FLB_PROCESSOR_LOCK_RETRY_LIMIT, FLB_PROCESSOR_LOCK_RETRY_DELAY); + pu_error = FLB_TRUE; + processor_metrics_update(proc, pu, + metrics_scope, + metrics_owner, + metrics_signal, + unit_in_items, + unit_out_items, + item_metrics_available, + pu_error); return -1; } @@ -1331,6 +1621,13 @@ int flb_processor_run(struct flb_processor *proc, flb_mp_chunk_cobj_destroy(chunk_cobj); chunk_cobj = NULL; } + + if (chunk_cobj != NULL) { + unit_out_items = flb_mp_chunk_cobj_count_log_records(chunk_cobj); + } + else if (cur_buf != NULL) { + unit_out_items = flb_mp_count_log_records(cur_buf, cur_size); + } } } else if (type == FLB_PROCESSOR_METRICS) { @@ -1343,6 +1640,15 @@ int flb_processor_run(struct flb_processor *proc, tag_len); if (ret != FLB_PROCESSOR_SUCCESS) { + pu_error = FLB_TRUE; + processor_metrics_update(proc, pu, + metrics_scope, + metrics_owner, + metrics_signal, + unit_in_items, + unit_out_items, + item_metrics_available, + pu_error); release_lock(&pu->lock, FLB_PROCESSOR_LOCK_RETRY_LIMIT, FLB_PROCESSOR_LOCK_RETRY_DELAY); @@ -1371,6 +1677,15 @@ int flb_processor_run(struct flb_processor *proc, tag, tag_len); if (ret == FLB_PROCESSOR_FAILURE) { + pu_error = FLB_TRUE; + processor_metrics_update(proc, pu, + metrics_scope, + metrics_owner, + metrics_signal, + unit_in_items, + unit_out_items, + item_metrics_available, + pu_error); release_lock(&pu->lock, FLB_PROCESSOR_LOCK_RETRY_LIMIT, FLB_PROCESSOR_LOCK_RETRY_DELAY); @@ -1385,6 +1700,14 @@ int flb_processor_run(struct flb_processor *proc, * will enqueue the trace through a different mechanism, * we just return saying nothing else is needed. */ + processor_metrics_update(proc, pu, + metrics_scope, + metrics_owner, + metrics_signal, + unit_in_items, + unit_out_items, + item_metrics_available, + pu_error); release_lock(&pu->lock, FLB_PROCESSOR_LOCK_RETRY_LIMIT, FLB_PROCESSOR_LOCK_RETRY_DELAY); @@ -1402,6 +1725,15 @@ int flb_processor_run(struct flb_processor *proc, tag_len); if (ret != FLB_PROCESSOR_SUCCESS) { + pu_error = FLB_TRUE; + processor_metrics_update(proc, pu, + metrics_scope, + metrics_owner, + metrics_signal, + unit_in_items, + unit_out_items, + item_metrics_available, + pu_error); release_lock(&pu->lock, FLB_PROCESSOR_LOCK_RETRY_LIMIT, FLB_PROCESSOR_LOCK_RETRY_DELAY); @@ -1412,12 +1744,52 @@ int flb_processor_run(struct flb_processor *proc, } } + processor_metrics_update(proc, pu, + metrics_scope, + metrics_owner, + metrics_signal, + unit_in_items, + unit_out_items, + item_metrics_available, + pu_error); release_lock(&pu->lock, FLB_PROCESSOR_LOCK_RETRY_LIMIT, FLB_PROCESSOR_LOCK_RETRY_DELAY); } /* set output buffer */ + if (type == FLB_PROCESSOR_LOGS && + cur_buf != NULL) { + ret = flb_mp_normalize_log_buffer_groups_msgpack(cur_buf, cur_size, + (char **) &tmp_buf, + &tmp_size); + if (ret == 0) { + if (cur_buf != data) { + flb_free(cur_buf); + } + + cur_buf = tmp_buf; + cur_size = tmp_size; + } + } + + if (type == FLB_PROCESSOR_LOGS && + (cur_buf == NULL || flb_mp_count_log_records(cur_buf, cur_size) == 0)) { + if (cur_buf != data) { + flb_free(cur_buf); + } + + if (out_buf != NULL) { + *out_buf = NULL; + } + + if (out_size != NULL) { + *out_size = 0; + } + + return 0; + } + if (out_buf != NULL) { *out_buf = cur_buf; } diff --git a/src/flb_task.c b/src/flb_task.c index 5e671f5ee40..928f75d5bb0 100644 --- a/src/flb_task.c +++ b/src/flb_task.c @@ -708,6 +708,8 @@ struct flb_task *flb_task_create(uint64_t ref_id, } route->status = FLB_TASK_ROUTE_INACTIVE; + route->records = evc->total_events; + route->bytes = evc->size; route->out = stored_matches[stored_match_index]; mk_list_add(&route->_head, &task->routes); direct_count++; @@ -810,6 +812,8 @@ struct flb_task *flb_task_create(uint64_t ref_id, } route->status = FLB_TASK_ROUTE_INACTIVE; + route->records = evc->total_events; + route->bytes = evc->size; route->out = o_ins; mk_list_add(&route->_head, &task->routes); direct_count++; @@ -856,6 +860,8 @@ struct flb_task *flb_task_create(uint64_t ref_id, } route->status = FLB_TASK_ROUTE_INACTIVE; + route->records = evc->total_events; + route->bytes = evc->size; route->out = o_ins; mk_list_add(&route->_head, &task->routes); count++; diff --git a/tests/internal/input_chunk.c b/tests/internal/input_chunk.c index 99f8826635a..4dc916b1256 100644 --- a/tests/internal/input_chunk.c +++ b/tests/internal/input_chunk.c @@ -9,6 +9,8 @@ #include #include #include +#include +#include #include "flb_tests_internal.h" #include "chunkio/chunkio.h" #include "data/input_chunk/log/test_buffer_drop_chunks.h" @@ -341,6 +343,100 @@ static int gen_buf(msgpack_sbuffer *mp_sbuf, char *buf, size_t buf_size) return 0; } +static int build_grouped_log_payload(char **out_buf, size_t *out_size) +{ + int ret; + char *copied_buffer; + struct flb_time ts; + struct flb_log_event_encoder *encoder; + + *out_buf = NULL; + *out_size = 0; + + encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); + if (encoder == NULL) { + return -1; + } + + ret = flb_log_event_encoder_group_init(encoder); + if (ret != 0) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + ret = flb_log_event_encoder_append_metadata_values( + encoder, + FLB_LOG_EVENT_STRING_VALUE("group", 5), + FLB_LOG_EVENT_CSTRING_VALUE("g1")); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + ret = flb_log_event_encoder_append_body_values( + encoder, + FLB_LOG_EVENT_STRING_VALUE("resource", 8), + FLB_LOG_EVENT_CSTRING_VALUE("test")); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + ret = flb_log_event_encoder_group_header_end(encoder); + if (ret != 0) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + ret = flb_log_event_encoder_begin_record(encoder); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + flb_time_set(&ts, 1700000000, 0); + ret = flb_log_event_encoder_set_timestamp(encoder, &ts); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + ret = flb_log_event_encoder_append_body_values( + encoder, + FLB_LOG_EVENT_STRING_VALUE("message", 7), + FLB_LOG_EVENT_CSTRING_VALUE("hello")); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + ret = flb_log_event_encoder_commit_record(encoder); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + ret = flb_log_event_encoder_group_end(encoder); + if (ret != 0) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + copied_buffer = flb_malloc(encoder->output_length); + if (copied_buffer == NULL) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + memcpy(copied_buffer, encoder->output_buffer, encoder->output_length); + + *out_buf = copied_buffer; + *out_size = encoder->output_length; + + flb_log_event_encoder_destroy(encoder); + return 0; +} + static int log_cb(struct cio_ctx *data, int level, const char *file, int line, char *str) { @@ -360,6 +456,30 @@ static int log_cb(struct cio_ctx *data, int level, const char *file, int line, return 0; } +static int get_counter_value_1(struct cmt_counter *counter, + char *label_value_0, + double *value) +{ + char *labels[1]; + + labels[0] = label_value_0; + + return cmt_counter_get_val(counter, 1, labels, value); +} + +static int get_counter_value_2(struct cmt_counter *counter, + char *label_value_0, + char *label_value_1, + double *value) +{ + char *labels[2]; + + labels[0] = label_value_0; + labels[1] = label_value_1; + + return cmt_counter_get_val(counter, 2, labels, value); +} + /* This tests uses the subsystems of the engine directly * to avoid threading issues when submitting chunks. */ @@ -575,6 +695,231 @@ void flb_test_input_chunk_correct_total_records(void) flb_free(root_path); } +void flb_test_input_chunk_grouped_auto_records(void) +{ + int ret; + int in_ffd; + int out_ffd; + char *payload; + size_t payload_size; + flb_ctx_t *ctx; + struct flb_input_instance *i_ins; + struct mk_list *head; + struct flb_input_chunk *ic; + + payload = NULL; + payload_size = 0; + + ret = build_grouped_log_payload(&payload, &payload_size); + TEST_CHECK(ret == 0); + if (ret != 0) { + return; + } + + ctx = flb_create(); + TEST_CHECK(ctx != NULL); + if (!ctx) { + flb_free(payload); + return; + } + + ret = flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + TEST_CHECK(ret == 0); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "null", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + if (ret != 0) { + flb_destroy(ctx); + flb_free(payload); + return; + } + + i_ins = mk_list_entry_first(&ctx->config->inputs, + struct flb_input_instance, + _head); + TEST_CHECK(i_ins != NULL); + if (!i_ins) { + flb_stop(ctx); + flb_destroy(ctx); + flb_free(payload); + return; + } + + ret = flb_input_chunk_append_raw(i_ins, + FLB_INPUT_LOGS, + 0, + "test", + 4, + payload, + payload_size); + TEST_CHECK(ret == 0); + + flb_time_msleep(200); + + ic = NULL; + mk_list_foreach(head, &i_ins->chunks) { + ic = mk_list_entry(head, struct flb_input_chunk, _head); + break; + } + + TEST_CHECK(ic != NULL); + if (ic != NULL) { + TEST_CHECK(ic->total_records == 1); + } + + flb_stop(ctx); + flb_destroy(ctx); + flb_free(payload); +} + +void flb_test_input_chunk_grouped_release_space_drop_counters(void) +{ + int i; + int ret; + int in_ffd; + int out_ffd; + int append_count; + char *payload; + size_t payload_size; + double output_dropped_records; + double router_dropped_records; + flb_ctx_t *ctx; + char *storage_path; + struct flb_input_instance *i_ins; + struct flb_output_instance *o_ins; + + payload = NULL; + payload_size = 0; + output_dropped_records = 0.0; + router_dropped_records = 0.0; + append_count = 8; + + storage_path = flb_test_tmpdir_cat("/input-chunk-grouped-release-space/"); + TEST_CHECK(storage_path != NULL); + if (!storage_path) { + return; + } + + ret = build_grouped_log_payload(&payload, &payload_size); + TEST_CHECK(ret == 0); + if (ret != 0) { + flb_free(storage_path); + return; + } + + ctx = flb_create(); + TEST_CHECK(ctx != NULL); + if (!ctx) { + flb_free(payload); + flb_free(storage_path); + return; + } + + ret = flb_service_set(ctx, + "flush", "0.2", + "grace", "1", + "storage.path", storage_path, + "Log_Level", "error", + NULL); + TEST_CHECK(ret == 0); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, + "tag", "test", + "storage.type", "filesystem", + NULL); + + out_ffd = flb_output(ctx, (char *) "http", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "match", "test", + "Host", "127.0.0.1", + "Port", "1", + "retry_limit", "no_retries", + "storage.total_limit_size", "1K", + NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + if (ret != 0) { + flb_destroy(ctx); + flb_free(payload); + flb_free(storage_path); + return; + } + + i_ins = mk_list_entry_first(&ctx->config->inputs, + struct flb_input_instance, + _head); + TEST_CHECK(i_ins != NULL); + if (!i_ins) { + flb_stop(ctx); + flb_destroy(ctx); + flb_free(payload); + flb_free(storage_path); + return; + } + + o_ins = mk_list_entry_first(&ctx->config->outputs, + struct flb_output_instance, + _head); + TEST_CHECK(o_ins != NULL); + if (!o_ins) { + flb_stop(ctx); + flb_destroy(ctx); + flb_free(payload); + flb_free(storage_path); + return; + } + + for (i = 0; i < append_count; i++) { + ret = flb_input_chunk_append_raw(i_ins, + FLB_INPUT_LOGS, + 0, + "test", + 4, + payload, + payload_size); + TEST_CHECK(ret == 0); + flb_time_msleep(250); + } + + flb_time_msleep(1500); + + ret = get_counter_value_1(o_ins->cmt_dropped_records, + (char *) flb_output_name(o_ins), + &output_dropped_records); + TEST_CHECK(ret == 0); + + ret = get_counter_value_2(ctx->config->router->logs_drop_records_total, + (char *) flb_input_name(i_ins), + (char *) flb_output_name(o_ins), + &router_dropped_records); + TEST_CHECK(ret == 0); + + /* + * Each appended grouped payload contains one logical log record. + * Dropped record counters must never exceed ingested logical records. + */ + TEST_CHECK(output_dropped_records <= append_count); + TEST_CHECK(router_dropped_records <= append_count); + TEST_CHECK(output_dropped_records == router_dropped_records); + + flb_stop(ctx); + flb_destroy(ctx); + flb_free(payload); + flb_free(storage_path); +} + /* Test list */ TEST_LIST = { @@ -583,5 +928,8 @@ TEST_LIST = { {"input_chunk_dropping_chunks", flb_test_input_chunk_dropping_chunks}, {"input_chunk_fs_chunk_size_real", flb_test_input_chunk_fs_chunks_size_real}, {"input_chunk_correct_total_records", flb_test_input_chunk_correct_total_records}, + {"input_chunk_grouped_auto_records", flb_test_input_chunk_grouped_auto_records}, + {"input_chunk_grouped_release_space_drop_counters", + flb_test_input_chunk_grouped_release_space_drop_counters}, {NULL, NULL} }; diff --git a/tests/internal/input_chunk_routes.c b/tests/internal/input_chunk_routes.c index 7c26ba1e808..bef509b38cc 100644 --- a/tests/internal/input_chunk_routes.c +++ b/tests/internal/input_chunk_routes.c @@ -18,6 +18,8 @@ #undef flb_plg_trace #include +#include +#include #include #include #include @@ -61,6 +63,99 @@ static int write_test_log_payload(struct cio_chunk *chunk) return ret; } +static int write_test_grouped_log_payload(struct cio_chunk *chunk) +{ + int ret; + char *buf; + size_t size; + struct flb_time ts; + struct flb_log_event_encoder *encoder; + + encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); + if (encoder == NULL) { + return -1; + } + + ret = flb_log_event_encoder_group_init(encoder); + if (ret != 0) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + ret = flb_log_event_encoder_append_metadata_values( + encoder, + FLB_LOG_EVENT_STRING_VALUE("group", 5), + FLB_LOG_EVENT_CSTRING_VALUE("g1")); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + ret = flb_log_event_encoder_append_body_values( + encoder, + FLB_LOG_EVENT_STRING_VALUE("resource", 8), + FLB_LOG_EVENT_CSTRING_VALUE("test")); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + ret = flb_log_event_encoder_group_header_end(encoder); + if (ret != 0) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + ret = flb_log_event_encoder_begin_record(encoder); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + flb_time_set(&ts, 1700000000, 0); + ret = flb_log_event_encoder_set_timestamp(encoder, &ts); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + ret = flb_log_event_encoder_append_body_values( + encoder, + FLB_LOG_EVENT_STRING_VALUE("message", 7), + FLB_LOG_EVENT_CSTRING_VALUE("hello")); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + ret = flb_log_event_encoder_commit_record(encoder); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + ret = flb_log_event_encoder_group_end(encoder); + if (ret != 0) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + size = encoder->output_length; + buf = flb_malloc(size); + if (buf == NULL) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + memcpy(buf, encoder->output_buffer, size); + flb_log_event_encoder_destroy(encoder); + + ret = cio_chunk_write(chunk, buf, size); + flb_free(buf); + + return ret; +} + static int init_test_config(struct flb_config *config, struct flb_input_instance *in, struct flb_input_plugin *plugin) @@ -832,9 +927,139 @@ static void test_chunk_restore_alias_plugin_null_matches_all() flb_free(stream_path); } +static void test_chunk_map_grouped_logs_total_records() +{ + struct cio_options opts; + struct cio_ctx *ctx; + struct cio_stream *stream; + struct cio_chunk *chunk; + struct flb_input_chunk *ic; + struct flb_config config; + struct flb_input_instance in; + struct flb_input_plugin input_plugin; + struct flb_output_instance stdout_out; + struct flb_output_instance stdout_dummy_one; + struct flb_output_instance stdout_dummy_two; + struct flb_output_plugin stdout_plugin; + char *stream_path; + const char *tag_string; + int tag_len; + int ret; + int err; + int config_ready; + + stream_path = flb_test_tmpdir_cat("/flb-chunk-grouped-total-records"); + TEST_CHECK(stream_path != NULL); + if (!stream_path) { + return; + } + + ctx = NULL; + stream = NULL; + chunk = NULL; + ic = NULL; + config_ready = FLB_FALSE; + tag_string = "test.tag"; + tag_len = (int) strlen(tag_string); + + cio_utils_recursive_delete(stream_path); + memset(&opts, 0, sizeof(opts)); + cio_options_init(&opts); + opts.root_path = stream_path; + opts.flags = CIO_OPEN; + + ctx = cio_create(&opts); + TEST_CHECK(ctx != NULL); + if (!ctx) { + flb_free(stream_path); + return; + } + + stream = cio_stream_create(ctx, "direct", CIO_STORE_FS); + TEST_CHECK(stream != NULL); + if (!stream) { + goto cleanup; + } + + chunk = cio_chunk_open(ctx, stream, "meta", CIO_OPEN, 1024, &err); + TEST_CHECK(chunk != NULL); + if (!chunk) { + goto cleanup; + } + + ret = cio_chunk_is_up(chunk); + if (ret == CIO_FALSE) { + ret = cio_chunk_up_force(chunk); + TEST_CHECK(ret == CIO_OK); + if (ret != CIO_OK) { + goto cleanup; + } + } + + ret = write_legacy_chunk_metadata(chunk, FLB_INPUT_LOGS, + tag_string, tag_len); + TEST_CHECK(ret == 0); + if (ret != 0) { + goto cleanup; + } + + ret = write_test_grouped_log_payload(chunk); + TEST_CHECK(ret == 0); + if (ret != 0) { + goto cleanup; + } + + memset(&input_plugin, 0, sizeof(input_plugin)); + input_plugin.name = (char *) "dummy"; + memset(&stdout_plugin, 0, sizeof(stdout_plugin)); + stdout_plugin.name = (char *) "stdout"; + memset(&stdout_dummy_one, 0, sizeof(stdout_dummy_one)); + memset(&stdout_dummy_two, 0, sizeof(stdout_dummy_two)); + + ret = init_test_config(&config, &in, &input_plugin); + TEST_CHECK(ret == 0); + if (ret != 0) { + goto cleanup; + } + config_ready = FLB_TRUE; + +#ifdef FLB_HAVE_METRICS + cmt_initialize(); +#endif + + ret = flb_input_instance_init(&in, &config); + TEST_CHECK(ret == 0); + if (ret != 0) { + goto cleanup; + } + + ret = add_test_output(&config, &stdout_out, &stdout_plugin, 11, "stdout"); + TEST_CHECK(ret == 0); + if (ret != 0) { + goto cleanup; + } + + ic = flb_input_chunk_map(&in, FLB_INPUT_LOGS, chunk); + TEST_CHECK(ic != NULL); + if (!ic) { + goto cleanup; + } + + chunk = NULL; + + TEST_CHECK(ic->total_records == 1); + +cleanup: + cleanup_test_routing_scenario(ic, &stdout_out, &stdout_dummy_one, &stdout_dummy_two, + &in, &config, chunk, ctx, config_ready, + stream_path); + flb_free(stream_path); +} + TEST_LIST = { { "chunk_metadata_direct_routes", test_chunk_metadata_direct_routes }, { "chunk_restore_alias_plugin_match_multiple", test_chunk_restore_alias_plugin_match_multiple }, { "chunk_restore_alias_plugin_null_matches_all", test_chunk_restore_alias_plugin_null_matches_all }, + { "chunk_map_grouped_logs_total_records", test_chunk_map_grouped_logs_total_records }, { 0 } }; diff --git a/tests/internal/log_event_decoder.c b/tests/internal/log_event_decoder.c index 531fce41d86..d47b8087ff3 100644 --- a/tests/internal/log_event_decoder.c +++ b/tests/internal/log_event_decoder.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -1420,6 +1421,49 @@ void decoder_group_end_start_sequence() msgpack_sbuffer_destroy(&sbuf); } +void mp_count_log_records_excludes_groups() +{ + int all_records; + int log_records; + struct flb_time tm1; + struct flb_time tm2; + msgpack_sbuffer sbuf; + msgpack_packer pck; + + msgpack_sbuffer_init(&sbuf); + msgpack_packer_init(&pck, &sbuf, msgpack_sbuffer_write); + + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_START); + + flb_time_set(&tm1, 100, 1); + msgpack_pack_array(&pck, 2); + pack_event_time(&pck, &tm1); + msgpack_pack_map(&pck, 1); + msgpack_pack_str(&pck, 3); + msgpack_pack_str_body(&pck, "log", 3); + msgpack_pack_str(&pck, 2); + msgpack_pack_str_body(&pck, "a1", 2); + + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_END); + + flb_time_set(&tm2, 101, 2); + msgpack_pack_array(&pck, 2); + pack_event_time(&pck, &tm2); + msgpack_pack_map(&pck, 1); + msgpack_pack_str(&pck, 3); + msgpack_pack_str_body(&pck, "log", 3); + msgpack_pack_str(&pck, 2); + msgpack_pack_str_body(&pck, "b2", 2); + + all_records = flb_mp_count(sbuf.data, sbuf.size); + log_records = flb_mp_count_log_records(sbuf.data, sbuf.size); + + TEST_CHECK(all_records == 4); + TEST_CHECK(log_records == 2); + + msgpack_sbuffer_destroy(&sbuf); +} + TEST_LIST = { @@ -1435,5 +1479,6 @@ TEST_LIST = { { "decoder_corrupted_group_timestamps", decoder_corrupted_group_timestamps }, { "decoder_invalid_marker_preserves_group_state", decoder_invalid_marker_preserves_group_state }, { "decoder_group_end_start_sequence", decoder_group_end_start_sequence }, + { "mp_count_log_records_excludes_groups", mp_count_log_records_excludes_groups }, { 0 } }; diff --git a/tests/internal/processor.c b/tests/internal/processor.c index 77947aeec98..e2ebcd7bf86 100644 --- a/tests/internal/processor.c +++ b/tests/internal/processor.c @@ -29,6 +29,10 @@ #include #include #include +#include +#include +#include +#include #include "flb_tests_internal.h" @@ -63,6 +67,135 @@ static int create_msgpack_records(char **out_buf, size_t *out_size) return 0; } +static int create_grouped_msgpack_records(char **out_buf, size_t *out_size) +{ + int ret; + struct flb_time ts; + char *copied_buffer; + struct flb_log_event_encoder *encoder; + + *out_buf = NULL; + *out_size = 0; + + encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); + if (encoder == NULL) { + return -1; + } + + ret = flb_log_event_encoder_group_init(encoder); + if (ret != 0) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + ret = flb_log_event_encoder_append_metadata_values( + encoder, + FLB_LOG_EVENT_STRING_VALUE("group", 5), + FLB_LOG_EVENT_CSTRING_VALUE("g1")); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + ret = flb_log_event_encoder_group_header_end(encoder); + if (ret != 0) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + ret = flb_log_event_encoder_begin_record(encoder); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + flb_time_set(&ts, 1700000000, 0); + ret = flb_log_event_encoder_set_timestamp(encoder, &ts); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + ret = flb_log_event_encoder_append_body_values( + encoder, + FLB_LOG_EVENT_STRING_VALUE("message", 7), + FLB_LOG_EVENT_CSTRING_VALUE("hello")); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + ret = flb_log_event_encoder_commit_record(encoder); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + ret = flb_log_event_encoder_group_end(encoder); + if (ret != 0) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + copied_buffer = flb_malloc(encoder->output_length); + if (copied_buffer == NULL) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + memcpy(copied_buffer, encoder->output_buffer, encoder->output_length); + + *out_buf = copied_buffer; + *out_size = encoder->output_length; + + flb_log_event_encoder_destroy(encoder); + return 0; +} + +static int get_counter_value_5(struct cmt_counter *counter, + char *label_value_0, + char *label_value_1, + char *label_value_2, + char *label_value_3, + char *label_value_4, + double *value) +{ + char *labels[5]; + + labels[0] = label_value_0; + labels[1] = label_value_1; + labels[2] = label_value_2; + labels[3] = label_value_3; + labels[4] = label_value_4; + + return cmt_counter_get_val(counter, 5, labels, value); +} + +static int get_counter_value_5_or_zero(struct cmt_counter *counter, + char *label_value_0, + char *label_value_1, + char *label_value_2, + char *label_value_3, + char *label_value_4, + double *value) +{ + int ret; + + ret = get_counter_value_5(counter, + label_value_0, + label_value_1, + label_value_2, + label_value_3, + label_value_4, + value); + if (ret != 0) { + *value = 0.0; + return 0; + } + + return ret; +} + static void processor() { int ret; @@ -125,6 +258,117 @@ static void processor() flb_sds_destroy(hostname_prop_key); } +static void processor_grouped_filter_counters() +{ + int ret; + int vret; + double records; + double dropped; + double added; + struct flb_processor *proc; + struct flb_processor_unit *pu; + struct flb_config *config; + struct flb_filter_instance *f_ins; + char *mp_buf; + size_t mp_size; + void *out_buf; + size_t out_size; + int init_ok; + flb_sds_t regex_prop_key; + struct cfl_variant var = { + .type = CFL_VARIANT_STRING, + .data.as_string = NULL, + }; + char *labels[1]; + + records = 0; + dropped = 0; + added = 0; + init_ok = FLB_FALSE; + mp_buf = NULL; + out_buf = NULL; + out_size = 0; + + flb_init_env(); + + regex_prop_key = flb_sds_create("message ^doesnotmatch$"); + TEST_CHECK(regex_prop_key != NULL); + var.data.as_string = regex_prop_key; + + config = flb_config_init(); + TEST_CHECK(config != NULL); + if (config == NULL) { + flb_sds_destroy(regex_prop_key); + return; + } + + proc = flb_processor_create(config, "unit_test", NULL, 0); + TEST_CHECK(proc != NULL); + if (proc == NULL) { + flb_config_exit(config); + flb_sds_destroy(regex_prop_key); + return; + } + + pu = flb_processor_unit_create(proc, FLB_PROCESSOR_LOGS, "grep"); + TEST_CHECK(pu != NULL); + if (pu == NULL) { + flb_processor_destroy(proc); + flb_config_exit(config); + flb_sds_destroy(regex_prop_key); + return; + } + + ret = flb_processor_unit_set_property(pu, "regex", &var); + TEST_CHECK(ret == 0); + + ret = flb_processor_init(proc); + TEST_CHECK(ret == 0); + if (ret == 0) { + init_ok = FLB_TRUE; + } + + ret = create_grouped_msgpack_records(&mp_buf, &mp_size); + TEST_CHECK(ret == 0); + if (ret == 0 && init_ok == FLB_TRUE) { + ret = flb_processor_run(proc, 0, FLB_PROCESSOR_LOGS, + "TEST", 4, mp_buf, mp_size, + &out_buf, &out_size); + TEST_CHECK(ret == 0); + TEST_CHECK(out_size == 0); + } + + if (out_buf != NULL && out_buf != mp_buf) { + flb_free(out_buf); + out_buf = NULL; + } + + if (mp_buf != NULL) { + flb_free(mp_buf); + mp_buf = NULL; + } + + if (ret == 0 && init_ok == FLB_TRUE) { + f_ins = pu->ctx; + labels[0] = f_ins->name; + + vret = cmt_counter_get_val(f_ins->cmt_records, 1, labels, &records); + TEST_CHECK(vret == 0); + vret = cmt_counter_get_val(f_ins->cmt_drop_records, 1, labels, &dropped); + TEST_CHECK(vret == 0); + vret = cmt_counter_get_val(f_ins->cmt_add_records, 1, labels, &added); + TEST_CHECK(vret == 0); + + TEST_CHECK(records == 1.0); + TEST_CHECK(dropped == 1.0); + TEST_CHECK(added == 0.0); + } + + flb_processor_destroy(proc); + flb_config_exit(config); + flb_sds_destroy(regex_prop_key); +} + static void processor_private_inputs_use_main_loop() { int ret; @@ -196,8 +440,195 @@ static void processor_private_inputs_use_main_loop() #endif } +static void processor_metrics_counters() +{ + int ret; + int vret; + double invocations; + double errors; + double items_in; + double items_out; + double items_drop; + double items_add; + char stage_label[32]; + struct flb_processor *proc; + struct flb_processor_unit *pu; + struct flb_config *config; + struct flb_input_instance owner_input; + char *mp_buf; + size_t mp_size; + void *out_buf; + size_t out_size; + flb_sds_t regex_prop_key; + struct cfl_variant var = { + .type = CFL_VARIANT_STRING, + .data.as_string = NULL, + }; + + invocations = 0.0; + errors = 0.0; + items_in = 0.0; + items_out = 0.0; + items_drop = 0.0; + items_add = 0.0; + out_buf = NULL; + out_size = 0; + + flb_init_env(); + + regex_prop_key = flb_sds_create("message ^doesnotmatch$"); + TEST_CHECK(regex_prop_key != NULL); + var.data.as_string = regex_prop_key; + + config = flb_config_init(); + TEST_CHECK(config != NULL); + if (config == NULL) { + flb_sds_destroy(regex_prop_key); + return; + } + + proc = flb_processor_create(config, "unit_test", NULL, 0); + TEST_CHECK(proc != NULL); + if (proc == NULL) { + flb_config_exit(config); + flb_sds_destroy(regex_prop_key); + return; + } + + memset(&owner_input, 0, sizeof(owner_input)); + snprintf(owner_input.name, sizeof(owner_input.name) - 1, "%s", + "unit_input.0"); + owner_input.name[sizeof(owner_input.name) - 1] = '\0'; + owner_input.cmt = cmt_create(); + TEST_CHECK(owner_input.cmt != NULL); + if (owner_input.cmt == NULL) { + flb_processor_destroy(proc); + flb_config_exit(config); + flb_sds_destroy(regex_prop_key); + return; + } + + proc->data = &owner_input; + proc->source_plugin_type = FLB_PLUGIN_INPUT; + + pu = flb_processor_unit_create(proc, FLB_PROCESSOR_LOGS, "grep"); + TEST_CHECK(pu != NULL); + if (pu == NULL) { + cmt_destroy(owner_input.cmt); + flb_processor_destroy(proc); + flb_config_exit(config); + flb_sds_destroy(regex_prop_key); + return; + } + + ret = flb_processor_unit_set_property(pu, "regex", &var); + TEST_CHECK(ret == 0); + + ret = flb_processor_init(proc); + TEST_CHECK(ret == 0); + if (ret != 0) { + cmt_destroy(owner_input.cmt); + flb_processor_destroy(proc); + flb_config_exit(config); + flb_sds_destroy(regex_prop_key); + return; + } + + ret = create_grouped_msgpack_records(&mp_buf, &mp_size); + TEST_CHECK(ret == 0); + if (ret != 0) { + cmt_destroy(owner_input.cmt); + flb_processor_destroy(proc); + flb_config_exit(config); + flb_sds_destroy(regex_prop_key); + return; + } + + ret = flb_processor_run(proc, 0, FLB_PROCESSOR_LOGS, + "TEST", 4, mp_buf, mp_size, + &out_buf, &out_size); + TEST_CHECK(ret == 0); + TEST_CHECK(out_size == 0); + if (out_buf != NULL && out_buf != mp_buf) { + flb_free(out_buf); + out_buf = NULL; + } + flb_free(mp_buf); + + snprintf(stage_label, sizeof(stage_label) - 1, "%zu", pu->stage); + stage_label[sizeof(stage_label) - 1] = '\0'; + + vret = get_counter_value_5(proc->cmt_invocations, + "input", + (char *) flb_input_name(&owner_input), + (char *) pu->name, + stage_label, + "logs", + &invocations); + TEST_CHECK(vret == 0); + + vret = get_counter_value_5_or_zero(proc->cmt_errors, + "input", + (char *) flb_input_name(&owner_input), + (char *) pu->name, + stage_label, + "logs", + &errors); + TEST_CHECK(vret == 0); + + vret = get_counter_value_5(proc->cmt_items_in, + "input", + (char *) flb_input_name(&owner_input), + (char *) pu->name, + stage_label, + "logs", + &items_in); + TEST_CHECK(vret == 0); + + vret = get_counter_value_5(proc->cmt_items_out, + "input", + (char *) flb_input_name(&owner_input), + (char *) pu->name, + stage_label, + "logs", + &items_out); + TEST_CHECK(vret == 0); + + vret = get_counter_value_5(proc->cmt_items_drop, + "input", + (char *) flb_input_name(&owner_input), + (char *) pu->name, + stage_label, + "logs", + &items_drop); + TEST_CHECK(vret == 0); + + vret = get_counter_value_5_or_zero(proc->cmt_items_add, + "input", + (char *) flb_input_name(&owner_input), + (char *) pu->name, + stage_label, + "logs", + &items_add); + TEST_CHECK(vret == 0); + + TEST_CHECK(invocations == 1.0); + TEST_CHECK(errors == 0.0); + TEST_CHECK(items_in == 1.0); + TEST_CHECK(items_out == 0.0); + TEST_CHECK(items_drop == 1.0); + TEST_CHECK(items_add == 0.0); + + cmt_destroy(owner_input.cmt); + flb_processor_destroy(proc); + flb_config_exit(config); + flb_sds_destroy(regex_prop_key); +} + TEST_LIST = { { "processor_private_inputs_use_main_loop", processor_private_inputs_use_main_loop }, { "processor", processor }, + { "processor_grouped_filter_counters", processor_grouped_filter_counters }, + { "processor_metrics_counters", processor_metrics_counters }, { 0 } }; diff --git a/tests/internal/task_map.c b/tests/internal/task_map.c index f1ba353128a..ba08cbdc114 100644 --- a/tests/internal/task_map.c +++ b/tests/internal/task_map.c @@ -5,6 +5,9 @@ #include #include #include +#include +#include +#include #include "flb_tests_internal.h" @@ -12,6 +15,7 @@ struct test_ctx { struct flb_config *config; + struct mk_event_loop *evl; }; struct test_ctx* test_ctx_create() @@ -32,6 +36,22 @@ struct test_ctx* test_ctx_create() return NULL; } + ret_ctx->evl = mk_event_loop_create(8); + if(!TEST_CHECK(ret_ctx->evl != NULL)) { + flb_config_exit(ret_ctx->config); + flb_free(ret_ctx); + return NULL; + } + + ret_ctx->config->evl = ret_ctx->evl; + ret_ctx->config->sched = flb_sched_create(ret_ctx->config, ret_ctx->evl); + if(!TEST_CHECK(ret_ctx->config->sched != NULL)) { + mk_event_loop_destroy(ret_ctx->evl); + flb_config_exit(ret_ctx->config); + flb_free(ret_ctx); + return NULL; + } + return ret_ctx; } @@ -88,7 +108,91 @@ void test_task_map_limit() test_ctx_destroy(ctx); } +void test_task_route_data_preserved_across_retry() +{ + int ret; + int records; + size_t bytes; + struct test_ctx *ctx; + struct flb_task *task; + struct flb_output_instance out_a; + struct flb_output_instance out_b; + struct flb_task_route *route_a; + struct flb_task_route *route_b; + struct flb_task_retry *retry; + + ctx = test_ctx_create(); + if (!TEST_CHECK(ctx != NULL)) { + return; + } + + task = task_alloc(ctx->config); + if (!TEST_CHECK(task != NULL)) { + test_ctx_destroy(ctx); + return; + } + + /* Avoid input chunk up/down side effects in retry creation. */ + task->users = 2; + + memset(&out_a, 0, sizeof(out_a)); + memset(&out_b, 0, sizeof(out_b)); + out_a.retry_limit = 5; + out_b.retry_limit = 5; + + route_a = flb_calloc(1, sizeof(struct flb_task_route)); + route_b = flb_calloc(1, sizeof(struct flb_task_route)); + TEST_CHECK(route_a != NULL); + TEST_CHECK(route_b != NULL); + if (!route_a || !route_b) { + flb_free(route_a); + flb_free(route_b); + flb_task_destroy(task, FLB_TRUE); + test_ctx_destroy(ctx); + return; + } + + route_a->status = FLB_TASK_ROUTE_ACTIVE; + route_a->out = &out_a; + route_a->records = 3; + route_a->bytes = 300; + route_b->status = FLB_TASK_ROUTE_ACTIVE; + route_b->out = &out_b; + route_b->records = 7; + route_b->bytes = 700; + mk_list_add(&route_a->_head, &task->routes); + mk_list_add(&route_b->_head, &task->routes); + + flb_task_set_route_data(task, &out_a, 1, 111); + ret = flb_task_get_route_data(task, &out_a, &records, &bytes); + TEST_CHECK(ret == 0); + TEST_CHECK(records == 1); + TEST_CHECK(bytes == 111); + + retry = flb_task_retry_create(task, &out_a); + TEST_CHECK(retry != NULL); + if (retry != NULL) { + TEST_CHECK(retry->attempts == 1); + } + + retry = flb_task_retry_create(task, &out_a); + TEST_CHECK(retry != NULL); + if (retry != NULL) { + TEST_CHECK(retry->attempts == 2); + } + + ret = flb_task_get_route_data(task, &out_a, &records, &bytes); + TEST_CHECK(ret == 0); + TEST_CHECK(records == 1); + TEST_CHECK(bytes == 111); + + flb_task_retry_clean(task, &out_a); + flb_task_destroy(task, FLB_TRUE); + test_ctx_destroy(ctx); +} + TEST_LIST = { { "task_map_limit" , test_task_map_limit}, + { "task_route_data_preserved_across_retry", test_task_route_data_preserved_across_retry}, { 0 } }; diff --git a/tests/runtime/CMakeLists.txt b/tests/runtime/CMakeLists.txt index dd76c16faee..ab93ea8d07f 100644 --- a/tests/runtime/CMakeLists.txt +++ b/tests/runtime/CMakeLists.txt @@ -194,6 +194,13 @@ if(FLB_IN_LIB AND FLB_OUT_LIB) endif () FLB_RT_TEST(FLB_FILTER_ECS "filter_ecs.c") FLB_RT_TEST(FLB_FILTER_LOG_TO_METRICS "filter_log_to_metrics.c") + if(FLB_IN_LIB) + FLB_RT_TEST(FLB_FILTER_GREP "filter_counter_semantics.c") + endif() +endif() + +if(FLB_IN_LIB AND FLB_FILTER_GREP AND FLB_OUT_NULL AND FLB_OUT_HTTP) + FLB_RT_TEST(FLB_IN_LIB "counter_parity_e2e.c") endif() @@ -231,6 +238,11 @@ if(FLB_IN_LIB) FLB_RT_TEST(FLB_OUT_CLOUDWATCH_LOGS "out_cloudwatch.c") FLB_RT_TEST(FLB_OUT_KINESIS_FIREHOSE "out_firehose.c") FLB_RT_TEST(FLB_OUT_KINESIS_STREAMS "out_kinesis.c") + if(FLB_OUT_FORWARD AND FLB_OUT_LOKI) + if (FLB_RECORD_ACCESSOR AND FLB_OUT_STACKDRIVER) + FLB_RT_TEST(FLB_OUT_LIB "group_counter_semantics.c") + endif() + endif() # These plugins work only on Linux if(NOT FLB_SYSTEM_WINDOWS) FLB_RT_TEST(FLB_OUT_FILE "out_file.c") diff --git a/tests/runtime/counter_parity_e2e.c b/tests/runtime/counter_parity_e2e.c new file mode 100644 index 00000000000..1c8a0e791b2 --- /dev/null +++ b/tests/runtime/counter_parity_e2e.c @@ -0,0 +1,878 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "flb_tests_runtime.h" + +static struct flb_input_instance *get_input_instance_by_name(flb_ctx_t *ctx, + const char *name) +{ + struct mk_list *head; + struct flb_input_instance *ins; + + mk_list_foreach(head, &ctx->config->inputs) { + ins = mk_list_entry(head, struct flb_input_instance, _head); + if (ins->p && strcmp(ins->p->name, name) == 0) { + return ins; + } + } + + return NULL; +} + +static struct flb_output_instance *get_output_instance_by_name(flb_ctx_t *ctx, + const char *name) +{ + struct mk_list *head; + struct flb_output_instance *ins; + + mk_list_foreach(head, &ctx->config->outputs) { + ins = mk_list_entry(head, struct flb_output_instance, _head); + if (ins->p && strcmp(ins->p->name, name) == 0) { + return ins; + } + } + + return NULL; +} + +static struct flb_filter_instance *get_filter_instance_by_name(flb_ctx_t *ctx, + const char *name) +{ + struct mk_list *head; + struct flb_filter_instance *ins; + + mk_list_foreach(head, &ctx->config->filters) { + ins = mk_list_entry(head, struct flb_filter_instance, _head); + if (ins->p && strcmp(ins->p->name, name) == 0) { + return ins; + } + } + + return NULL; +} + +static int get_counter_value_1(struct cmt_counter *counter, + char *label_value_0, + double *value) +{ + char *labels[1]; + + labels[0] = label_value_0; + + return cmt_counter_get_val(counter, 1, labels, value); +} + +static int get_counter_value_1_or_zero(struct cmt_counter *counter, + char *label_value_0, + double *value) +{ + int ret; + + ret = get_counter_value_1(counter, label_value_0, value); + if (ret == -1) { + *value = 0.0; + return 0; + } + + return ret; +} + +static int get_counter_value_2(struct cmt_counter *counter, + char *label_value_0, + char *label_value_1, + double *value) +{ + char *labels[2]; + + labels[0] = label_value_0; + labels[1] = label_value_1; + + return cmt_counter_get_val(counter, 2, labels, value); +} + +static int get_counter_value_2_or_zero(struct cmt_counter *counter, + char *label_value_0, + char *label_value_1, + double *value) +{ + int ret; + + ret = get_counter_value_2(counter, label_value_0, label_value_1, value); + if (ret == -1) { + *value = 0.0; + return 0; + } + + return ret; +} + +static int get_counter_value_5(struct cmt_counter *counter, + char *label_value_0, + char *label_value_1, + char *label_value_2, + char *label_value_3, + char *label_value_4, + double *value) +{ + char *labels[5]; + + labels[0] = label_value_0; + labels[1] = label_value_1; + labels[2] = label_value_2; + labels[3] = label_value_3; + labels[4] = label_value_4; + + return cmt_counter_get_val(counter, 5, labels, value); +} + +static int get_counter_value_5_or_zero(struct cmt_counter *counter, + char *label_value_0, + char *label_value_1, + char *label_value_2, + char *label_value_3, + char *label_value_4, + double *value) +{ + int ret; + + ret = get_counter_value_5(counter, + label_value_0, + label_value_1, + label_value_2, + label_value_3, + label_value_4, + value); + if (ret == -1) { + *value = 0.0; + return 0; + } + + return ret; +} + +static int get_processor_counter_value(struct cmt_counter *counter, + const char *scope, + const char *owner, + struct flb_processor_unit *pu, + const char *signal, + double *value) +{ + char stage_label[32]; + + snprintf(stage_label, sizeof(stage_label) - 1, "%zu", pu->stage); + + return get_counter_value_5(counter, + (char *) scope, + (char *) owner, + (char *) pu->name, + stage_label, + (char *) signal, + value); +} + +static int get_processor_counter_value_or_zero(struct cmt_counter *counter, + const char *scope, + const char *owner, + struct flb_processor_unit *pu, + const char *signal, + double *value) +{ + char stage_label[32]; + + snprintf(stage_label, sizeof(stage_label) - 1, "%zu", pu->stage); + + return get_counter_value_5_or_zero(counter, + (char *) scope, + (char *) owner, + (char *) pu->name, + stage_label, + (char *) signal, + value); +} + +static int build_grouped_log_payload(char **out_buf, size_t *out_size) +{ + int ret; + struct flb_time ts; + char *copied_buffer; + struct flb_log_event_encoder *encoder; + + *out_buf = NULL; + *out_size = 0; + + encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); + if (encoder == NULL) { + return -1; + } + + ret = flb_log_event_encoder_group_init(encoder); + if (ret != 0) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + ret = flb_log_event_encoder_append_metadata_values( + encoder, + FLB_LOG_EVENT_STRING_VALUE("group", 5), + FLB_LOG_EVENT_CSTRING_VALUE("g1")); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + ret = flb_log_event_encoder_group_header_end(encoder); + if (ret != 0) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + ret = flb_log_event_encoder_begin_record(encoder); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + flb_time_set(&ts, 1700000000, 0); + ret = flb_log_event_encoder_set_timestamp(encoder, &ts); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + ret = flb_log_event_encoder_append_body_values( + encoder, + FLB_LOG_EVENT_STRING_VALUE("message", 7), + FLB_LOG_EVENT_CSTRING_VALUE("hello")); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + ret = flb_log_event_encoder_commit_record(encoder); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + ret = flb_log_event_encoder_group_end(encoder); + if (ret != 0) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + copied_buffer = flb_malloc(encoder->output_length); + if (copied_buffer == NULL) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + memcpy(copied_buffer, encoder->output_buffer, encoder->output_length); + *out_buf = copied_buffer; + *out_size = encoder->output_length; + + flb_log_event_encoder_destroy(encoder); + + return 0; +} + +static int inject_grouped_log_chunk(flb_ctx_t *ctx, const char *tag) +{ + int ret; + char *payload; + size_t payload_size; + struct flb_input_instance *ins; + + payload = NULL; + payload_size = 0; + + ins = get_input_instance_by_name(ctx, "lib"); + if (ins == NULL) { + return -1; + } + + ret = build_grouped_log_payload(&payload, &payload_size); + if (ret != 0) { + return -1; + } + + ret = flb_input_chunk_append_raw(ins, + FLB_INPUT_LOGS, + 0, + tag, + strlen(tag), + payload, + payload_size); + + flb_free(payload); + return ret; +} + +static void flb_test_mixed_input_processor_filter_parity(void) +{ + int ret; + int in_ffd; + int out_ffd; + int f_ffd; + double output_proc_records; + double router_records; + double router_drop_records; + double filter_records; + double filter_drop_records; + flb_ctx_t *ctx; + struct flb_processor *proc; + struct flb_processor_unit *pu; + struct flb_filter_instance *f_ins; + struct flb_input_instance *i_ins; + struct flb_output_instance *o_ins; + + output_proc_records = 0.0; + router_records = 0.0; + router_drop_records = 0.0; + filter_records = 0.0; + filter_drop_records = 0.0; + + ctx = flb_create(); + TEST_CHECK(ctx != NULL); + if (!ctx) { + return; + } + + ret = flb_service_set(ctx, "Flush", "0.2", "Grace", "1", "Log_Level", "error", NULL); + TEST_CHECK(ret == 0); + + proc = flb_processor_create(ctx->config, "unit_test", NULL, 0); + TEST_CHECK(proc != NULL); + if (!proc) { + flb_destroy(ctx); + return; + } + + pu = flb_processor_unit_create(proc, FLB_PROCESSOR_LOGS, "grep"); + TEST_CHECK(pu != NULL); + if (!pu) { + flb_destroy(ctx); + return; + } + + ret = flb_processor_unit_set_property_str(pu, "regex", "message ^hello$"); + TEST_CHECK(ret == 0); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + ret = flb_input_set(ctx, in_ffd, "tag", "test", NULL); + TEST_CHECK(ret == 0); + + ret = flb_input_set_processor(ctx, in_ffd, proc); + TEST_CHECK(ret == 0); + + f_ffd = flb_filter(ctx, (char *) "grep", NULL); + TEST_CHECK(f_ffd >= 0); + ret = flb_filter_set(ctx, f_ffd, + "match", "test", + "regex", "message ^hello$", + NULL); + TEST_CHECK(ret == 0); + + out_ffd = flb_output(ctx, (char *) "null", NULL); + TEST_CHECK(out_ffd >= 0); + ret = flb_output_set(ctx, out_ffd, "match", "test", NULL); + TEST_CHECK(ret == 0); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + if (ret != 0) { + flb_destroy(ctx); + return; + } + + ret = inject_grouped_log_chunk(ctx, "test"); + TEST_CHECK(ret == 0); + + flb_time_msleep(1500); + + i_ins = get_input_instance_by_name(ctx, "lib"); + o_ins = get_output_instance_by_name(ctx, "null"); + f_ins = get_filter_instance_by_name(ctx, "grep"); + + TEST_CHECK(i_ins != NULL); + TEST_CHECK(o_ins != NULL); + TEST_CHECK(f_ins != NULL); + + if (i_ins && o_ins && f_ins) { + ret = get_counter_value_1(o_ins->cmt_proc_records, + (char *) flb_output_name(o_ins), + &output_proc_records); + TEST_CHECK(ret == 0); + + ret = get_counter_value_2(ctx->config->router->logs_records_total, + (char *) flb_input_name(i_ins), + (char *) flb_output_name(o_ins), + &router_records); + TEST_CHECK(ret == 0); + + ret = get_counter_value_2_or_zero(ctx->config->router->logs_drop_records_total, + (char *) flb_input_name(i_ins), + (char *) flb_output_name(o_ins), + &router_drop_records); + TEST_CHECK(ret == 0); + + ret = get_counter_value_1(f_ins->cmt_records, (char *) f_ins->name, + &filter_records); + TEST_CHECK(ret == 0); + + ret = get_counter_value_1(f_ins->cmt_drop_records, (char *) f_ins->name, + &filter_drop_records); + TEST_CHECK(ret == 0); + + TEST_CHECK(output_proc_records == 1.0); + TEST_CHECK(router_records == 1.0); + TEST_CHECK(router_drop_records == 0.0); + TEST_CHECK(filter_records == 1.0); + TEST_CHECK(filter_drop_records == 0.0); + } + + flb_stop(ctx); + flb_destroy(ctx); +} + +static void flb_test_output_processor_drop_parity(void) +{ + int ret; + int in_ffd; + int out_ffd; + double output_proc_records; + double router_records; + double router_drop_records; + double processor_invocations; + double processor_errors; + double processor_items_in; + double processor_items_out; + double processor_items_drop; + flb_ctx_t *ctx; + struct flb_processor *proc; + struct flb_processor_unit *pu; + struct flb_input_instance *i_ins; + struct flb_output_instance *o_ins; + + output_proc_records = 0.0; + router_records = 0.0; + router_drop_records = 0.0; + processor_invocations = 0.0; + processor_errors = 0.0; + processor_items_in = 0.0; + processor_items_out = 0.0; + processor_items_drop = 0.0; + + ctx = flb_create(); + TEST_CHECK(ctx != NULL); + if (!ctx) { + return; + } + + ret = flb_service_set(ctx, "Flush", "0.2", "Grace", "1", "Log_Level", "error", NULL); + TEST_CHECK(ret == 0); + + proc = flb_processor_create(ctx->config, "unit_test", NULL, 0); + TEST_CHECK(proc != NULL); + if (!proc) { + flb_destroy(ctx); + return; + } + + pu = flb_processor_unit_create(proc, FLB_PROCESSOR_LOGS, "grep"); + TEST_CHECK(pu != NULL); + if (!pu) { + flb_destroy(ctx); + return; + } + + ret = flb_processor_unit_set_property_str(pu, "regex", "message ^doesnotmatch$"); + TEST_CHECK(ret == 0); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + ret = flb_input_set(ctx, in_ffd, "tag", "test", NULL); + TEST_CHECK(ret == 0); + + out_ffd = flb_output(ctx, (char *) "null", NULL); + TEST_CHECK(out_ffd >= 0); + ret = flb_output_set(ctx, out_ffd, "match", "test", NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set_processor(ctx, out_ffd, proc); + TEST_CHECK(ret == 0); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + if (ret != 0) { + flb_destroy(ctx); + return; + } + + ret = inject_grouped_log_chunk(ctx, "test"); + TEST_CHECK(ret == 0); + + flb_time_msleep(1500); + + i_ins = get_input_instance_by_name(ctx, "lib"); + o_ins = get_output_instance_by_name(ctx, "null"); + + TEST_CHECK(i_ins != NULL); + TEST_CHECK(o_ins != NULL); + + if (i_ins && o_ins) { + ret = get_counter_value_1(o_ins->cmt_proc_records, + (char *) flb_output_name(o_ins), + &output_proc_records); + TEST_CHECK(ret == 0); + + ret = get_counter_value_2_or_zero(ctx->config->router->logs_records_total, + (char *) flb_input_name(i_ins), + (char *) flb_output_name(o_ins), + &router_records); + TEST_CHECK(ret == 0); + + ret = get_counter_value_2_or_zero(ctx->config->router->logs_drop_records_total, + (char *) flb_input_name(i_ins), + (char *) flb_output_name(o_ins), + &router_drop_records); + TEST_CHECK(ret == 0); + + ret = get_processor_counter_value(o_ins->processor->cmt_invocations, + "output", + flb_output_name(o_ins), + pu, + "logs", + &processor_invocations); + TEST_CHECK(ret == 0); + + ret = get_processor_counter_value_or_zero(o_ins->processor->cmt_errors, + "output", + flb_output_name(o_ins), + pu, + "logs", + &processor_errors); + TEST_CHECK(ret == 0); + + ret = get_processor_counter_value(o_ins->processor->cmt_items_in, + "output", + flb_output_name(o_ins), + pu, + "logs", + &processor_items_in); + TEST_CHECK(ret == 0); + + ret = get_processor_counter_value(o_ins->processor->cmt_items_out, + "output", + flb_output_name(o_ins), + pu, + "logs", + &processor_items_out); + TEST_CHECK(ret == 0); + + ret = get_processor_counter_value(o_ins->processor->cmt_items_drop, + "output", + flb_output_name(o_ins), + pu, + "logs", + &processor_items_drop); + TEST_CHECK(ret == 0); + + TEST_CHECK(output_proc_records == 0.0); + TEST_CHECK(router_records == 0.0); + TEST_CHECK(router_drop_records == 0.0); + TEST_CHECK(processor_invocations >= 1.0); + TEST_CHECK(processor_errors == 0.0); + TEST_CHECK(processor_items_in >= 1.0); + TEST_CHECK(processor_items_out == 0.0); + TEST_CHECK(processor_items_drop >= 1.0); + } + + flb_stop(ctx); + flb_destroy(ctx); +} + +static void flb_test_retry_drop_route_parity_grouped(void) +{ + int ret; + int in_ffd; + int out_ffd; + double output_proc_records; + double output_retries; + double output_retried_records; + double output_dropped_records; + double output_retries_failed; + double router_records; + double router_drop_records; + flb_ctx_t *ctx; + struct flb_input_instance *i_ins; + struct flb_output_instance *o_ins; + + output_proc_records = 0.0; + output_retries = 0.0; + output_retried_records = 0.0; + output_dropped_records = 0.0; + output_retries_failed = 0.0; + router_records = 0.0; + router_drop_records = 0.0; + + ctx = flb_create(); + TEST_CHECK(ctx != NULL); + if (!ctx) { + return; + } + + ret = flb_service_set(ctx, "Flush", "0.2", "Grace", "2", "Log_Level", "error", NULL); + TEST_CHECK(ret == 0); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + ret = flb_input_set(ctx, in_ffd, "tag", "test", NULL); + TEST_CHECK(ret == 0); + + out_ffd = flb_output(ctx, (char *) "http", NULL); + TEST_CHECK(out_ffd >= 0); + ret = flb_output_set(ctx, out_ffd, + "match", "test", + "host", "127.0.0.1", + "port", "1", + "uri", "/", + "retry_limit", "no_retries", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + if (ret != 0) { + flb_destroy(ctx); + return; + } + + ret = inject_grouped_log_chunk(ctx, "test"); + TEST_CHECK(ret == 0); + + i_ins = get_input_instance_by_name(ctx, "lib"); + o_ins = get_output_instance_by_name(ctx, "http"); + + TEST_CHECK(i_ins != NULL); + TEST_CHECK(o_ins != NULL); + + if (i_ins && o_ins) { + flb_time_msleep(2000); + + ret = get_counter_value_1(o_ins->cmt_proc_records, + (char *) flb_output_name(o_ins), + &output_proc_records); + TEST_CHECK(ret == 0); + + ret = get_counter_value_1_or_zero(o_ins->cmt_retries, + (char *) flb_output_name(o_ins), + &output_retries); + TEST_CHECK(ret == 0); + + ret = get_counter_value_1_or_zero(o_ins->cmt_retried_records, + (char *) flb_output_name(o_ins), + &output_retried_records); + TEST_CHECK(ret == 0); + + ret = get_counter_value_1_or_zero(o_ins->cmt_dropped_records, + (char *) flb_output_name(o_ins), + &output_dropped_records); + TEST_CHECK(ret == 0); + + ret = get_counter_value_1_or_zero(o_ins->cmt_retries_failed, + (char *) flb_output_name(o_ins), + &output_retries_failed); + TEST_CHECK(ret == 0); + + ret = get_counter_value_2_or_zero(ctx->config->router->logs_records_total, + (char *) flb_input_name(i_ins), + (char *) flb_output_name(o_ins), + &router_records); + TEST_CHECK(ret == 0); + + ret = get_counter_value_2_or_zero(ctx->config->router->logs_drop_records_total, + (char *) flb_input_name(i_ins), + (char *) flb_output_name(o_ins), + &router_drop_records); + TEST_CHECK(ret == 0); + + TEST_CHECK(output_proc_records == 0.0); + TEST_CHECK(output_retries == 0.0); + TEST_CHECK(output_retried_records == 0.0); + TEST_CHECK(output_dropped_records == 1.0); + TEST_CHECK(output_retries_failed == 0.0); + TEST_CHECK(router_records == 0.0); + TEST_CHECK(router_drop_records == 1.0); + } + + flb_stop(ctx); + flb_destroy(ctx); +} + +static int poll_retry_scheduled_route_parity_grouped( + struct flb_input_instance *i_ins, + struct flb_output_instance *o_ins, + flb_ctx_t *ctx, + double *output_proc_records, + double *output_retries, + double *output_retried_records, + double *router_records) +{ + int ret; + int attempts; + + for (attempts = 0; attempts < 50; attempts++) { + ret = get_counter_value_1(o_ins->cmt_proc_records, + (char *) flb_output_name(o_ins), + output_proc_records); + if (ret != 0) { + return ret; + } + + ret = get_counter_value_1_or_zero(o_ins->cmt_retries, + (char *) flb_output_name(o_ins), + output_retries); + if (ret != 0) { + return ret; + } + + ret = get_counter_value_1_or_zero(o_ins->cmt_retried_records, + (char *) flb_output_name(o_ins), + output_retried_records); + if (ret != 0) { + return ret; + } + + ret = get_counter_value_2_or_zero(ctx->config->router->logs_records_total, + (char *) flb_input_name(i_ins), + (char *) flb_output_name(o_ins), + router_records); + if (ret != 0) { + return ret; + } + + if (*output_proc_records == 0.0 && + *output_retries >= 1.0 && + *output_retried_records >= 1.0 && + *router_records == 0.0) { + return 0; + } + + flb_time_msleep(100); + } + + return -1; +} + +static void flb_test_retry_scheduled_route_parity_grouped(void) +{ + int ret; + int in_ffd; + int out_ffd; + double output_proc_records; + double output_retries; + double output_retried_records; + double router_records; + flb_ctx_t *ctx; + struct flb_input_instance *i_ins; + struct flb_output_instance *o_ins; + + output_proc_records = 0.0; + output_retries = 0.0; + output_retried_records = 0.0; + router_records = 0.0; + + ctx = flb_create(); + TEST_CHECK(ctx != NULL); + if (!ctx) { + return; + } + + ret = flb_service_set(ctx, "Flush", "0.2", "Grace", "2", "Log_Level", "error", NULL); + TEST_CHECK(ret == 0); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + ret = flb_input_set(ctx, in_ffd, "tag", "test", NULL); + TEST_CHECK(ret == 0); + + out_ffd = flb_output(ctx, (char *) "http", NULL); + TEST_CHECK(out_ffd >= 0); + ret = flb_output_set(ctx, out_ffd, + "match", "test", + "host", "127.0.0.1", + "port", "1", + "uri", "/", + "retry_limit", "2", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + if (ret != 0) { + flb_destroy(ctx); + return; + } + + ret = inject_grouped_log_chunk(ctx, "test"); + TEST_CHECK(ret == 0); + + i_ins = get_input_instance_by_name(ctx, "lib"); + o_ins = get_output_instance_by_name(ctx, "http"); + + TEST_CHECK(i_ins != NULL); + TEST_CHECK(o_ins != NULL); + + if (i_ins && o_ins) { + ret = poll_retry_scheduled_route_parity_grouped(i_ins, o_ins, ctx, + &output_proc_records, + &output_retries, + &output_retried_records, + &router_records); + TEST_CHECK(ret == 0); + + TEST_CHECK(output_proc_records == 0.0); + TEST_CHECK(output_retries >= 1.0); + TEST_CHECK(output_retried_records >= 1.0); + TEST_CHECK(router_records == 0.0); + } + + flb_stop(ctx); + flb_destroy(ctx); +} + +TEST_LIST = { + {"mixed_input_processor_filter_parity", flb_test_mixed_input_processor_filter_parity}, + {"output_processor_drop_parity", flb_test_output_processor_drop_parity}, + {"retry_drop_route_parity_grouped", flb_test_retry_drop_route_parity_grouped}, + {"retry_scheduled_route_parity_grouped", flb_test_retry_scheduled_route_parity_grouped}, + {NULL, NULL} +}; diff --git a/tests/runtime/filter_counter_semantics.c b/tests/runtime/filter_counter_semantics.c new file mode 100644 index 00000000000..4d3afeda26b --- /dev/null +++ b/tests/runtime/filter_counter_semantics.c @@ -0,0 +1,274 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "flb_tests_runtime.h" + +static struct flb_input_instance *get_input_instance_by_name(flb_ctx_t *ctx, + const char *name) +{ + struct mk_list *head; + struct flb_input_instance *ins; + + mk_list_foreach(head, &ctx->config->inputs) { + ins = mk_list_entry(head, struct flb_input_instance, _head); + if (ins->p && strcmp(ins->p->name, name) == 0) { + return ins; + } + } + + return NULL; +} + +static struct flb_filter_instance *get_filter_instance_by_name(flb_ctx_t *ctx, + const char *name) +{ + struct mk_list *head; + struct flb_filter_instance *ins; + + mk_list_foreach(head, &ctx->config->filters) { + ins = mk_list_entry(head, struct flb_filter_instance, _head); + if (ins->p && strcmp(ins->p->name, name) == 0) { + return ins; + } + } + + return NULL; +} + +static int get_counter_value_1(struct cmt_counter *counter, + char *label_value_0, + double *value) +{ + char *labels[1]; + + labels[0] = label_value_0; + + return cmt_counter_get_val(counter, 1, labels, value); +} + +static int build_grouped_log_payload(char **out_buf, size_t *out_size) +{ + int ret; + struct flb_time ts; + char *copied_buffer; + struct flb_log_event_encoder *encoder; + + *out_buf = NULL; + *out_size = 0; + + encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); + if (encoder == NULL) { + return -1; + } + + ret = flb_log_event_encoder_group_init(encoder); + if (ret != 0) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + ret = flb_log_event_encoder_append_metadata_values( + encoder, + FLB_LOG_EVENT_STRING_VALUE("group", 5), + FLB_LOG_EVENT_CSTRING_VALUE("g1")); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + ret = flb_log_event_encoder_append_body_values( + encoder, + FLB_LOG_EVENT_STRING_VALUE("resource", 8), + FLB_LOG_EVENT_CSTRING_VALUE("test")); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + ret = flb_log_event_encoder_group_header_end(encoder); + if (ret != 0) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + ret = flb_log_event_encoder_begin_record(encoder); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + flb_time_set(&ts, 1700000000, 0); + ret = flb_log_event_encoder_set_timestamp(encoder, &ts); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + ret = flb_log_event_encoder_append_body_values( + encoder, + FLB_LOG_EVENT_STRING_VALUE("message", 7), + FLB_LOG_EVENT_CSTRING_VALUE("hello")); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + ret = flb_log_event_encoder_commit_record(encoder); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + ret = flb_log_event_encoder_group_end(encoder); + if (ret != 0) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + copied_buffer = flb_malloc(encoder->output_length); + if (copied_buffer == NULL) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + memcpy(copied_buffer, encoder->output_buffer, encoder->output_length); + *out_buf = copied_buffer; + *out_size = encoder->output_length; + + flb_log_event_encoder_destroy(encoder); + + return 0; +} + +static int inject_grouped_log_chunk(flb_ctx_t *ctx, const char *tag) +{ + int ret; + char *payload; + size_t payload_size; + struct flb_input_instance *ins; + + ins = get_input_instance_by_name(ctx, "lib"); + if (ins == NULL) { + return -1; + } + + ret = build_grouped_log_payload(&payload, &payload_size); + if (ret != 0) { + return -1; + } + + ret = flb_input_chunk_append_raw(ins, + FLB_INPUT_LOGS, + 0, + tag, + strlen(tag), + payload, + payload_size); + + flb_free(payload); + + return ret; +} + +static void flb_test_grouped_log_filter_counters(void) +{ + int ret; + int in_ffd; + int out_ffd; + int f_ffd; + double records; + double dropped; + double added; + flb_ctx_t *ctx; + struct flb_filter_instance *f_ins; + + records = 0; + dropped = 0; + added = 0; + + ctx = flb_create(); + TEST_CHECK(ctx != NULL); + if (!ctx) { + return; + } + + ret = flb_service_set(ctx, "Flush", "0.2", "Grace", "1", "Log_Level", "error", NULL); + TEST_CHECK(ret == 0); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + f_ffd = flb_filter(ctx, (char *) "grep", NULL); + TEST_CHECK(f_ffd >= 0); + flb_filter_set(ctx, f_ffd, + "match", "test", + "regex", "message ^doesnotmatch$", + NULL); + + out_ffd = flb_output(ctx, (char *) "null", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + if (ret != 0) { + flb_destroy(ctx); + return; + } + + ret = inject_grouped_log_chunk(ctx, "test"); + TEST_CHECK(ret == 0); + + flb_time_msleep(1500); + + f_ins = get_filter_instance_by_name(ctx, "grep"); + TEST_CHECK(f_ins != NULL); + if (f_ins != NULL) { + ret = get_counter_value_1(f_ins->cmt_records, (char *) f_ins->name, &records); + TEST_CHECK(ret == 0); + + ret = get_counter_value_1(f_ins->cmt_drop_records, (char *) f_ins->name, &dropped); + TEST_CHECK(ret == 0); + + ret = get_counter_value_1(f_ins->cmt_add_records, (char *) f_ins->name, &added); + TEST_CHECK(ret == 0); + + TEST_CHECK(records == 1.0); + TEST_CHECK(dropped == 1.0); + TEST_CHECK(added == 0.0); + } + + flb_stop(ctx); + flb_destroy(ctx); +} + +TEST_LIST = { + {"grouped_log_filter_counters", flb_test_grouped_log_filter_counters}, + {NULL, NULL} +}; + diff --git a/tests/runtime/group_counter_semantics.c b/tests/runtime/group_counter_semantics.c new file mode 100644 index 00000000000..e2e23a13862 --- /dev/null +++ b/tests/runtime/group_counter_semantics.c @@ -0,0 +1,846 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "flb_tests_runtime.h" +#include "../../plugins/out_forward/forward.h" + +#include +#include +#include + +#define SERVICE_CREDENTIALS \ + FLB_TESTS_DATA_PATH "/data/stackdriver/stackdriver-credentials.json" + +static pthread_mutex_t g_result_mutex = PTHREAD_MUTEX_INITIALIZER; +static int g_forward_size = -1; +static int g_loki_values = -1; +static int g_stackdriver_entries = -1; + +static void reset_results() +{ + pthread_mutex_lock(&g_result_mutex); + g_forward_size = -1; + g_loki_values = -1; + g_stackdriver_entries = -1; + pthread_mutex_unlock(&g_result_mutex); +} + +static void set_forward_size(int value) +{ + pthread_mutex_lock(&g_result_mutex); + g_forward_size = value; + pthread_mutex_unlock(&g_result_mutex); +} + +static void set_loki_values(int value) +{ + pthread_mutex_lock(&g_result_mutex); + g_loki_values = value; + pthread_mutex_unlock(&g_result_mutex); +} + +static void set_stackdriver_entries(int value) +{ + pthread_mutex_lock(&g_result_mutex); + g_stackdriver_entries = value; + pthread_mutex_unlock(&g_result_mutex); +} + +static int get_forward_size() +{ + int value; + + pthread_mutex_lock(&g_result_mutex); + value = g_forward_size; + pthread_mutex_unlock(&g_result_mutex); + + return value; +} + +static int get_loki_values() +{ + int value; + + pthread_mutex_lock(&g_result_mutex); + value = g_loki_values; + pthread_mutex_unlock(&g_result_mutex); + + return value; +} + +static int get_stackdriver_entries() +{ + int value; + + pthread_mutex_lock(&g_result_mutex); + value = g_stackdriver_entries; + pthread_mutex_unlock(&g_result_mutex); + + return value; +} + +static struct flb_input_instance *get_input_instance_by_name(flb_ctx_t *ctx, + const char *name) +{ + struct mk_list *head; + struct flb_input_instance *ins; + + mk_list_foreach(head, &ctx->config->inputs) { + ins = mk_list_entry(head, struct flb_input_instance, _head); + if (ins->p && strcmp(ins->p->name, name) == 0) { + return ins; + } + } + + return NULL; +} + +static int build_grouped_log_payload(char **out_buf, size_t *out_size) +{ + int ret; + struct flb_time ts; + char *copied_buffer; + struct flb_log_event_encoder *encoder; + + *out_buf = NULL; + *out_size = 0; + + encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); + if (encoder == NULL) { + return -1; + } + + ret = flb_log_event_encoder_group_init(encoder); + if (ret != 0) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + ret = flb_log_event_encoder_append_metadata_values( + encoder, + FLB_LOG_EVENT_STRING_VALUE("group", 5), + FLB_LOG_EVENT_CSTRING_VALUE("g1")); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + ret = flb_log_event_encoder_append_body_values( + encoder, + FLB_LOG_EVENT_STRING_VALUE("resource", 8), + FLB_LOG_EVENT_CSTRING_VALUE("test")); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + ret = flb_log_event_encoder_group_header_end(encoder); + if (ret != 0) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + ret = flb_log_event_encoder_begin_record(encoder); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + flb_time_set(&ts, 1700000000, 0); + ret = flb_log_event_encoder_set_timestamp(encoder, &ts); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + ret = flb_log_event_encoder_append_body_values( + encoder, + FLB_LOG_EVENT_STRING_VALUE("message", 7), + FLB_LOG_EVENT_CSTRING_VALUE("hello")); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + ret = flb_log_event_encoder_commit_record(encoder); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + ret = flb_log_event_encoder_group_end(encoder); + if (ret != 0) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + copied_buffer = flb_malloc(encoder->output_length); + if (copied_buffer == NULL) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + memcpy(copied_buffer, encoder->output_buffer, encoder->output_length); + + *out_buf = copied_buffer; + *out_size = encoder->output_length; + + flb_log_event_encoder_destroy(encoder); + + return 0; +} + +static int append_log_record(struct flb_log_event_encoder *encoder, + int64_t seconds, + const char *message) +{ + int ret; + struct flb_time ts; + + ret = flb_log_event_encoder_begin_record(encoder); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + return -1; + } + + flb_time_set(&ts, seconds, 0); + ret = flb_log_event_encoder_set_timestamp(encoder, &ts); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + return -1; + } + + ret = flb_log_event_encoder_append_body_values( + encoder, + FLB_LOG_EVENT_STRING_VALUE("message", 7), + FLB_LOG_EVENT_CSTRING_VALUE(message)); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + return -1; + } + + ret = flb_log_event_encoder_commit_record(encoder); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + return -1; + } + + return 0; +} + +static int build_mixed_with_empty_group_payload(char **out_buf, size_t *out_size) +{ + int ret; + char *copied_buffer; + struct flb_log_event_encoder *encoder; + + *out_buf = NULL; + *out_size = 0; + + encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); + if (encoder == NULL) { + return -1; + } + + ret = append_log_record(encoder, 1700000000, "r1"); + if (ret != 0) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + ret = append_log_record(encoder, 1700000001, "r2"); + if (ret != 0) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + ret = flb_log_event_encoder_group_init(encoder); + if (ret != 0) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + ret = flb_log_event_encoder_append_metadata_values( + encoder, + FLB_LOG_EVENT_STRING_VALUE("group", 5), + FLB_LOG_EVENT_CSTRING_VALUE("empty")); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + ret = flb_log_event_encoder_group_header_end(encoder); + if (ret != 0) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + ret = flb_log_event_encoder_group_end(encoder); + if (ret != 0) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + ret = append_log_record(encoder, 1700000002, "r3"); + if (ret != 0) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + ret = append_log_record(encoder, 1700000003, "r4"); + if (ret != 0) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + copied_buffer = flb_malloc(encoder->output_length); + if (copied_buffer == NULL) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + memcpy(copied_buffer, encoder->output_buffer, encoder->output_length); + *out_buf = copied_buffer; + *out_size = encoder->output_length; + + flb_log_event_encoder_destroy(encoder); + + return 0; +} + +static int inject_grouped_log_chunk(flb_ctx_t *ctx, const char *tag) +{ + int ret; + char *payload; + size_t payload_size; + struct flb_input_instance *ins; + + ins = get_input_instance_by_name(ctx, "lib"); + if (ins == NULL) { + return -1; + } + + payload = NULL; + payload_size = 0; + + ret = build_grouped_log_payload(&payload, &payload_size); + if (ret != 0) { + return -1; + } + + ret = flb_input_chunk_append_raw(ins, + FLB_INPUT_LOGS, + 0, + tag, + strlen(tag), + payload, + payload_size); + + flb_free(payload); + return ret; +} + +static int inject_mixed_with_empty_group_chunk(flb_ctx_t *ctx, const char *tag) +{ + int ret; + char *payload; + size_t payload_size; + struct flb_input_instance *ins; + + ins = get_input_instance_by_name(ctx, "lib"); + if (ins == NULL) { + return -1; + } + + payload = NULL; + payload_size = 0; + + ret = build_mixed_with_empty_group_payload(&payload, &payload_size); + if (ret != 0) { + return -1; + } + + ret = flb_input_chunk_append_raw(ins, + FLB_INPUT_LOGS, + 0, + tag, + strlen(tag), + payload, + payload_size); + + flb_free(payload); + return ret; +} + +static int map_find_key(msgpack_object map, const char *key, msgpack_object **out) +{ + size_t i; + size_t key_len; + msgpack_object k; + + if (map.type != MSGPACK_OBJECT_MAP) { + return -1; + } + + key_len = strlen(key); + + for (i = 0; i < map.via.map.size; i++) { + k = map.via.map.ptr[i].key; + + if (k.type != MSGPACK_OBJECT_STR) { + continue; + } + + if (k.via.str.size != key_len) { + continue; + } + + if (strncmp(k.via.str.ptr, key, key_len) == 0) { + *out = &map.via.map.ptr[i].val; + return 0; + } + } + + return -1; +} + +static void cb_forward_size_check(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + size_t off; + msgpack_object *size_obj; + msgpack_object root; + msgpack_unpacked result; + + (void) ctx; + (void) ffd; + (void) res_size; + (void) data; + + TEST_CHECK(res_ret == MODE_FORWARD); + + off = 0; + msgpack_unpacked_init(&result); + ret = msgpack_unpack_next(&result, res_data, res_size, &off); + TEST_CHECK(ret == MSGPACK_UNPACK_SUCCESS); + if (ret == MSGPACK_UNPACK_SUCCESS) { + root = result.data; + if (map_find_key(root, "size", &size_obj) == 0 && + size_obj->type == MSGPACK_OBJECT_POSITIVE_INTEGER) { + set_forward_size((int) size_obj->via.u64); + } + } + + msgpack_unpacked_destroy(&result); + flb_free(res_data); +} + +static int extract_json_array_size(char *json_data, size_t json_len, + const char *array_key, int *size_out) +{ + int ret; + int type; + size_t off; + char *mp_buf; + size_t mp_size; + msgpack_object *array_obj; + msgpack_object root; + msgpack_unpacked result; + + *size_out = -1; + mp_buf = NULL; + mp_size = 0; + + ret = flb_pack_json((const char *) json_data, + json_len, + &mp_buf, + &mp_size, + &type, + NULL); + if (ret != 0) { + return -1; + } + + off = 0; + msgpack_unpacked_init(&result); + ret = msgpack_unpack_next(&result, mp_buf, mp_size, &off); + if (ret != MSGPACK_UNPACK_SUCCESS) { + msgpack_unpacked_destroy(&result); + flb_free(mp_buf); + return -1; + } + + root = result.data; + if (map_find_key(root, array_key, &array_obj) == 0 && + array_obj->type == MSGPACK_OBJECT_ARRAY) { + *size_out = (int) array_obj->via.array.size; + } + + msgpack_unpacked_destroy(&result); + flb_free(mp_buf); + + if (*size_out < 0) { + return -1; + } + + return 0; +} + +static void cb_loki_values_check(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + int streams_size; + int values_size; + size_t i; + msgpack_object *streams_obj; + msgpack_object stream_obj; + msgpack_object *values_obj; + msgpack_object root; + msgpack_unpacked result; + size_t off; + int type; + char *mp_buf; + size_t mp_size; + + (void) ctx; + (void) ffd; + (void) res_ret; + (void) data; + + streams_size = -1; + values_size = -1; + mp_buf = NULL; + mp_size = 0; + + ret = flb_pack_json((const char *) res_data, + res_size, + &mp_buf, + &mp_size, + &type, + NULL); + TEST_CHECK(ret == 0); + if (ret != 0) { + flb_sds_destroy((flb_sds_t) res_data); + return; + } + + off = 0; + msgpack_unpacked_init(&result); + ret = msgpack_unpack_next(&result, mp_buf, mp_size, &off); + TEST_CHECK(ret == MSGPACK_UNPACK_SUCCESS); + if (ret == MSGPACK_UNPACK_SUCCESS) { + root = result.data; + if (map_find_key(root, "streams", &streams_obj) == 0 && + streams_obj->type == MSGPACK_OBJECT_ARRAY) { + streams_size = (int) streams_obj->via.array.size; + for (i = 0; i < streams_obj->via.array.size; i++) { + stream_obj = streams_obj->via.array.ptr[i]; + if (stream_obj.type != MSGPACK_OBJECT_MAP) { + continue; + } + + if (map_find_key(stream_obj, "values", &values_obj) == 0 && + values_obj->type == MSGPACK_OBJECT_ARRAY) { + values_size = (int) values_obj->via.array.size; + break; + } + } + } + } + + msgpack_unpacked_destroy(&result); + flb_free(mp_buf); + flb_sds_destroy((flb_sds_t) res_data); + + TEST_CHECK(streams_size > 0); + TEST_CHECK(values_size >= 0); + if (values_size >= 0) { + set_loki_values(values_size); + } +} + +static void cb_stackdriver_entries_check(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int entries_size; + int ret; + + (void) ctx; + (void) ffd; + (void) res_ret; + (void) data; + + entries_size = -1; + + ret = extract_json_array_size((char *) res_data, + res_size, + "entries", + &entries_size); + + TEST_CHECK(ret == 0); + if (ret == 0) { + set_stackdriver_entries(entries_size); + } + + flb_sds_destroy((flb_sds_t) res_data); +} + +static void run_group_count_test(flb_ctx_t *ctx) +{ + int ret; + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + if (ret != 0) { + return; + } + + ret = inject_grouped_log_chunk(ctx, "test"); + TEST_CHECK(ret == 0); + + flb_time_msleep(1500); + flb_stop(ctx); +} + +static void flb_test_forward_group_size_default() +{ + int out_ffd; + int in_ffd; + int ret; + flb_ctx_t *ctx; + + reset_results(); + + ctx = flb_create(); + flb_service_set(ctx, "Flush", "0.2", "Grace", "1", "Log_Level", "error", NULL); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "forward", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "match", "test", + "tag", "new.tag", + "send_options", "true", + NULL); + + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_forward_size_check, + NULL, NULL); + TEST_CHECK(ret == 0); + + run_group_count_test(ctx); + TEST_CHECK(get_forward_size() == 1); + + flb_destroy(ctx); +} + +static void flb_test_forward_group_size_retain_metadata() +{ + int out_ffd; + int in_ffd; + int ret; + flb_ctx_t *ctx; + + reset_results(); + + ctx = flb_create(); + flb_service_set(ctx, "Flush", "0.2", "Grace", "1", "Log_Level", "error", NULL); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "forward", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "match", "test", + "tag", "new.tag", + "send_options", "true", + "retain_metadata_in_forward_mode", "true", + NULL); + + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_forward_size_check, + NULL, NULL); + TEST_CHECK(ret == 0); + + run_group_count_test(ctx); + TEST_CHECK(get_forward_size() == 3); + + flb_destroy(ctx); +} + +static void flb_test_loki_group_values_count() +{ + int out_ffd; + int in_ffd; + int ret; + flb_ctx_t *ctx; + + reset_results(); + + ctx = flb_create(); + flb_service_set(ctx, "Flush", "0.2", "Grace", "1", "Log_Level", "error", NULL); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "loki", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_loki_values_check, + NULL, NULL); + TEST_CHECK(ret == 0); + + run_group_count_test(ctx); + TEST_CHECK(get_loki_values() == 1); + + flb_destroy(ctx); +} + +static void flb_test_stackdriver_group_entries_count() +{ + int out_ffd; + int in_ffd; + int ret; + flb_ctx_t *ctx; + + reset_results(); + + ctx = flb_create(); + flb_service_set(ctx, "Flush", "0.2", "Grace", "1", "Log_Level", "error", NULL); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "match", "test", + "google_service_credentials", SERVICE_CREDENTIALS, + "resource", "global", + NULL); + + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_stackdriver_entries_check, + NULL, NULL); + TEST_CHECK(ret == 0); + + run_group_count_test(ctx); + TEST_CHECK(get_stackdriver_entries() == 1); + + flb_destroy(ctx); +} + +static void flb_test_forward_output_processor_mixed_payload_smoke() +{ + int ret; + int out_ffd; + int in_ffd; + flb_ctx_t *ctx; + struct flb_processor *proc; + struct flb_processor_unit *pu; + + reset_results(); + + ctx = flb_create(); + flb_service_set(ctx, "Flush", "0.2", "Grace", "1", "Log_Level", "error", NULL); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "forward", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "match", "test", + "tag", "new.tag", + "send_options", "true", + "retain_metadata_in_forward_mode", "true", + NULL); + + proc = flb_processor_create(ctx->config, "unit_test", NULL, 0); + TEST_CHECK(proc != NULL); + if (proc == NULL) { + flb_destroy(ctx); + return; + } + + pu = flb_processor_unit_create(proc, FLB_PROCESSOR_LOGS, "grep"); + TEST_CHECK(pu != NULL); + if (pu == NULL) { + flb_destroy(ctx); + return; + } + + ret = flb_processor_unit_set_property_str(pu, "regex", "message ^r[1-4]$"); + TEST_CHECK(ret == 0); + + ret = flb_output_set_processor(ctx, out_ffd, proc); + TEST_CHECK(ret == 0); + + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_forward_size_check, + NULL, NULL); + TEST_CHECK(ret == 0); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + if (ret != 0) { + flb_destroy(ctx); + return; + } + + ret = inject_mixed_with_empty_group_chunk(ctx, "test"); + TEST_CHECK(ret == 0); + + flb_time_msleep(1500); + flb_stop(ctx); + + TEST_CHECK(get_forward_size() > 0); + + flb_destroy(ctx); +} + +TEST_LIST = { + {"forward_group_size_default", flb_test_forward_group_size_default}, + {"forward_group_size_retain_metadata", flb_test_forward_group_size_retain_metadata}, + {"loki_group_values_count", flb_test_loki_group_values_count}, + {"stackdriver_group_entries_count", flb_test_stackdriver_group_entries_count}, + {"forward_output_processor_mixed_payload_smoke", + flb_test_forward_output_processor_mixed_payload_smoke}, + {NULL, NULL} +}; diff --git a/tests/runtime/out_counter.c b/tests/runtime/out_counter.c index 525b78f4790..72c8f08ad71 100644 --- a/tests/runtime/out_counter.c +++ b/tests/runtime/out_counter.c @@ -3,11 +3,118 @@ #include #include "flb_tests_runtime.h" +#include + +#ifdef _WIN32 +#include +#define flb_test_close _close +#define flb_test_dup _dup +#define flb_test_dup2 _dup2 +#define flb_test_fileno _fileno +#else +#include +#define flb_test_close close +#define flb_test_dup dup +#define flb_test_dup2 dup2 +#define flb_test_fileno fileno +#endif + /* Test data */ #include "data/common/json_invalid.h" /* JSON_INVALID */ #include "data/common/json_long.h" /* JSON_LONG */ #include "data/common/json_small.h" /* JSON_SMALL */ +static int run_counter_and_capture(const char *payload, size_t payload_size, + char *buffer, size_t buffer_size) +{ + int i; + int ret; + int bytes; + int in_ffd; + int out_ffd; + int stdout_fd; + FILE *capture; + flb_ctx_t *ctx; + size_t bytes_read; + + capture = tmpfile(); + if (capture == NULL) { + return -1; + } + + fflush(stdout); + + stdout_fd = flb_test_dup(flb_test_fileno(stdout)); + if (stdout_fd == -1) { + fclose(capture); + return -1; + } + + if (flb_test_dup2(flb_test_fileno(capture), flb_test_fileno(stdout)) == -1) { + flb_test_close(stdout_fd); + fclose(capture); + return -1; + } + + ctx = flb_create(); + if (ctx == NULL) { + ret = -1; + goto restore_stdout; + } + + flb_service_set(ctx, "Flush", "1", "Grace", "1", "Log_Level", "error", NULL); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + if (in_ffd < 0) { + ret = -1; + goto destroy_ctx; + } + + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "counter", NULL); + if (out_ffd < 0) { + ret = -1; + goto destroy_ctx; + } + + flb_output_set(ctx, out_ffd, "match", "test", NULL); + + ret = flb_start(ctx); + if (ret != 0) { + goto destroy_ctx; + } + + for (i = 0; i < (int) payload_size; i++) { + bytes = flb_lib_push(ctx, in_ffd, (char *) payload + i, 1); + if (bytes != 1) { + ret = -1; + goto stop_ctx; + } + } + + sleep(1); + ret = 0; + +stop_ctx: + flb_stop(ctx); + +destroy_ctx: + flb_destroy(ctx); + +restore_stdout: + fflush(stdout); + flb_test_dup2(stdout_fd, flb_test_fileno(stdout)); + flb_test_close(stdout_fd); + + rewind(capture); + bytes_read = fread(buffer, 1, buffer_size - 1, capture); + buffer[bytes_read] = '\0'; + + fclose(capture); + return ret; +} + /* Test functions */ void flb_test_counter_json_invalid(void); void flb_test_counter_json_long(void); @@ -92,35 +199,30 @@ void flb_test_counter_json_long(void) void flb_test_counter_json_small(void) { - int i; int ret; - int bytes; - char *p = (char *) JSON_SMALL; - flb_ctx_t *ctx; - int in_ffd; - int out_ffd; - - ctx = flb_create(); - flb_service_set(ctx, "Flush", "1", "Grace", "1", "Log_Level", "error", NULL); - - in_ffd = flb_input(ctx, (char *) "lib", NULL); - TEST_CHECK(in_ffd >= 0); - flb_input_set(ctx, in_ffd, "tag", "test", NULL); - - out_ffd = flb_output(ctx, (char *) "counter", NULL); - TEST_CHECK(out_ffd >= 0); - flb_output_set(ctx, out_ffd, "match", "test", NULL); - - ret = flb_start(ctx); + int parsed; + unsigned long serialized_events; + unsigned long log_records; + unsigned long total; + double timestamp; + char output[256]; + + ret = run_counter_and_capture((const char *) JSON_SMALL, + sizeof(JSON_SMALL) - 1, + output, sizeof(output)); TEST_CHECK(ret == 0); - - for (i = 0; i < (int) sizeof(JSON_SMALL) - 1; i++) { - bytes = flb_lib_push(ctx, in_ffd, p + i, 1); - TEST_CHECK(bytes == 1); - } - - sleep(1); /* waiting flush */ - - flb_stop(ctx); - flb_destroy(ctx); + TEST_CHECK(output[0] == '{'); + + parsed = sscanf(output, + "{\"ts\":%lf,\"serialized_events\":%lu," + "\"log_records\":%lu,\"total\":%lu}", + ×tamp, + &serialized_events, + &log_records, + &total); + TEST_CHECK(parsed == 4); + TEST_CHECK(timestamp > 0.0); + TEST_CHECK(serialized_events == 1); + TEST_CHECK(log_records == 1); + TEST_CHECK(total == 1); } diff --git a/tests/runtime/out_forward.c b/tests/runtime/out_forward.c index 7e3b90aec84..f1fcd0ebcdb 100644 --- a/tests/runtime/out_forward.c +++ b/tests/runtime/out_forward.c @@ -20,12 +20,182 @@ #include #include #include +#include +#include +#include #include "flb_tests_runtime.h" /* Include plugin header to get the flush_ctx structure definition */ #include "../../plugins/out_forward/forward.h" +static struct flb_input_instance *get_input_instance_by_name(flb_ctx_t *ctx, + const char *name) +{ + struct mk_list *head; + struct flb_input_instance *ins; + + mk_list_foreach(head, &ctx->config->inputs) { + ins = mk_list_entry(head, struct flb_input_instance, _head); + if (ins->p && strcmp(ins->p->name, name) == 0) { + return ins; + } + } + + return NULL; +} + +static struct flb_output_instance *get_output_instance_by_name(flb_ctx_t *ctx, + const char *name) +{ + struct mk_list *head; + struct flb_output_instance *ins; + + mk_list_foreach(head, &ctx->config->outputs) { + ins = mk_list_entry(head, struct flb_output_instance, _head); + if (ins->p && strcmp(ins->p->name, name) == 0) { + return ins; + } + } + + return NULL; +} + +static int run_forward_formatter_non_log(flb_ctx_t *ctx, + int event_type, + const char *tag, + const void *data, + size_t size, + void **out_buf, + size_t *out_size) +{ + int ret; + struct flb_input_instance *i_ins; + struct flb_output_instance *o_ins; + + i_ins = get_input_instance_by_name(ctx, "lib"); + if (i_ins == NULL) { + return -1; + } + + o_ins = get_output_instance_by_name(ctx, "forward"); + if (o_ins == NULL || + o_ins->p == NULL || + o_ins->p->test_formatter.callback == NULL || + o_ins->context == NULL) { + return -1; + } + + ret = o_ins->p->test_formatter.callback(ctx->config, + i_ins, + o_ins->context, + NULL, + event_type, + tag, + strlen(tag), + data, + size, + out_buf, + out_size); + + return ret; +} + +static void verify_non_log_options_map(void *res_data, + size_t res_size, + int expected_signal) +{ + int i; + int ret; + int have_size; + int have_signal; + size_t off; + msgpack_object key; + msgpack_object val; + msgpack_object root; + msgpack_unpacked result; + + have_size = FLB_FALSE; + have_signal = FLB_FALSE; + off = 0; + + msgpack_unpacked_init(&result); + ret = msgpack_unpack_next(&result, res_data, res_size, &off); + TEST_CHECK(ret == MSGPACK_UNPACK_SUCCESS); + if (ret != MSGPACK_UNPACK_SUCCESS) { + msgpack_unpacked_destroy(&result); + return; + } + + root = result.data; + TEST_CHECK(root.type == MSGPACK_OBJECT_MAP); + + for (i = 0; i < root.via.map.size; i++) { + key = root.via.map.ptr[i].key; + val = root.via.map.ptr[i].val; + + if (key.type != MSGPACK_OBJECT_STR) { + continue; + } + + if (key.via.str.size == 4 && + strncmp(key.via.str.ptr, "size", 4) == 0) { + have_size = FLB_TRUE; + } + else if (key.via.str.size == 13 && + strncmp(key.via.str.ptr, "fluent_signal", 13) == 0) { + TEST_CHECK(val.type == MSGPACK_OBJECT_POSITIVE_INTEGER || + val.type == MSGPACK_OBJECT_NEGATIVE_INTEGER); + if (val.type == MSGPACK_OBJECT_POSITIVE_INTEGER) { + TEST_CHECK((int) val.via.u64 == expected_signal); + } + else if (val.type == MSGPACK_OBJECT_NEGATIVE_INTEGER) { + TEST_CHECK((int) val.via.i64 == expected_signal); + } + have_signal = FLB_TRUE; + } + } + + TEST_CHECK(have_size == FLB_FALSE); + TEST_CHECK(have_signal == FLB_TRUE); + + msgpack_unpacked_destroy(&result); +} + +static int inject_raw_non_log_chunk(flb_ctx_t *ctx, int input_type, const char *tag) +{ + int ret; + struct flb_input_instance *ins; + msgpack_sbuffer mp_sbuf; + msgpack_packer mp_pck; + + ins = get_input_instance_by_name(ctx, "lib"); + if (ins == NULL) { + return -1; + } + + msgpack_sbuffer_init(&mp_sbuf); + msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); + + msgpack_pack_map(&mp_pck, 1); + msgpack_pack_str(&mp_pck, 1); + msgpack_pack_str_body(&mp_pck, "k", 1); + msgpack_pack_str(&mp_pck, 1); + msgpack_pack_str_body(&mp_pck, "v", 1); + + ret = flb_input_chunk_append_raw(ins, + input_type, + 1, + tag, + strlen(tag), + mp_sbuf.data, + mp_sbuf.size); + + msgpack_sbuffer_destroy(&mp_sbuf); + + return ret; +} + static void cb_check_message_mode(void *ctx, int ffd, int res_ret, void *res_data, size_t res_size, void *data) @@ -155,6 +325,226 @@ static void cb_check_forward_mode(void *ctx, int ffd, flb_free(res_data); } +static void cb_check_forward_mode_ack_options(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int i; + int ret; + int have_chunk; + int have_size; + int have_signal; + size_t off; + msgpack_object key; + msgpack_object val; + msgpack_object root; + msgpack_unpacked result; + + (void) ctx; + (void) ffd; + (void) data; + + TEST_CHECK(res_ret == MODE_FORWARD); + + have_chunk = FLB_FALSE; + have_size = FLB_FALSE; + have_signal = FLB_FALSE; + off = 0; + + msgpack_unpacked_init(&result); + ret = msgpack_unpack_next(&result, res_data, res_size, &off); + TEST_CHECK(ret == MSGPACK_UNPACK_SUCCESS); + if (ret != MSGPACK_UNPACK_SUCCESS) { + msgpack_unpacked_destroy(&result); + flb_free(res_data); + return; + } + + root = result.data; + TEST_CHECK(root.type == MSGPACK_OBJECT_MAP); + + for (i = 0; i < root.via.map.size; i++) { + key = root.via.map.ptr[i].key; + val = root.via.map.ptr[i].val; + + if (key.type != MSGPACK_OBJECT_STR) { + continue; + } + + if (key.via.str.size == 5 && + strncmp(key.via.str.ptr, "chunk", 5) == 0) { + TEST_CHECK(val.type == MSGPACK_OBJECT_STR); + if (val.type == MSGPACK_OBJECT_STR) { + TEST_CHECK(val.via.str.size == 32); + } + have_chunk = FLB_TRUE; + } + else if (key.via.str.size == 4 && + strncmp(key.via.str.ptr, "size", 4) == 0) { + TEST_CHECK(val.type == MSGPACK_OBJECT_POSITIVE_INTEGER); + if (val.type == MSGPACK_OBJECT_POSITIVE_INTEGER) { + TEST_CHECK(val.via.u64 == 1); + } + have_size = FLB_TRUE; + } + else if (key.via.str.size == 13 && + strncmp(key.via.str.ptr, "fluent_signal", 13) == 0) { + TEST_CHECK(val.type == MSGPACK_OBJECT_POSITIVE_INTEGER); + if (val.type == MSGPACK_OBJECT_POSITIVE_INTEGER) { + TEST_CHECK(val.via.u64 == 0); + } + have_signal = FLB_TRUE; + } + } + + TEST_CHECK(have_chunk == FLB_TRUE); + TEST_CHECK(have_size == FLB_TRUE); + TEST_CHECK(have_signal == FLB_TRUE); + + msgpack_unpacked_destroy(&result); + flb_free(res_data); +} + +#ifdef FLB_HAVE_METRICS +static void cb_check_forward_mode_metrics_options(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int i; + int ret; + int have_signal; + size_t off; + msgpack_object key; + msgpack_object val; + msgpack_object root; + msgpack_unpacked result; + + (void) ctx; + (void) ffd; + (void) data; + + TEST_CHECK(res_ret == MODE_FORWARD); + + have_signal = FLB_FALSE; + off = 0; + + msgpack_unpacked_init(&result); + ret = msgpack_unpack_next(&result, res_data, res_size, &off); + TEST_CHECK(ret == MSGPACK_UNPACK_SUCCESS); + if (ret != MSGPACK_UNPACK_SUCCESS) { + msgpack_unpacked_destroy(&result); + flb_free(res_data); + return; + } + + root = result.data; + TEST_CHECK(root.type == MSGPACK_OBJECT_MAP); + + for (i = 0; i < root.via.map.size; i++) { + key = root.via.map.ptr[i].key; + val = root.via.map.ptr[i].val; + + if (key.type != MSGPACK_OBJECT_STR) { + continue; + } + + if (key.via.str.size == 4 && + strncmp(key.via.str.ptr, "size", 4) == 0) { + TEST_CHECK(val.type == MSGPACK_OBJECT_POSITIVE_INTEGER); + if (val.type == MSGPACK_OBJECT_POSITIVE_INTEGER) { + TEST_CHECK(val.via.u64 >= 0); + } + } + else if (key.via.str.size == 13 && + strncmp(key.via.str.ptr, "fluent_signal", 13) == 0) { + TEST_CHECK(val.type == MSGPACK_OBJECT_POSITIVE_INTEGER || + val.type == MSGPACK_OBJECT_NEGATIVE_INTEGER); + if (val.type == MSGPACK_OBJECT_POSITIVE_INTEGER) { + TEST_CHECK(val.via.u64 == FLB_EVENT_TYPE_METRICS); + } + else if (val.type == MSGPACK_OBJECT_NEGATIVE_INTEGER) { + TEST_CHECK(val.via.i64 == FLB_EVENT_TYPE_METRICS); + } + have_signal = FLB_TRUE; + } + } + + TEST_CHECK(have_signal == FLB_TRUE); + + msgpack_unpacked_destroy(&result); + flb_free(res_data); +} +#endif + +static void cb_check_forward_mode_traces_options(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int i; + int ret; + int have_size; + int have_signal; + size_t off; + msgpack_object key; + msgpack_object val; + msgpack_object root; + msgpack_unpacked result; + + (void) ctx; + (void) ffd; + (void) data; + + TEST_CHECK(res_ret == MODE_FORWARD); + + have_size = FLB_FALSE; + have_signal = FLB_FALSE; + off = 0; + + msgpack_unpacked_init(&result); + ret = msgpack_unpack_next(&result, res_data, res_size, &off); + TEST_CHECK(ret == MSGPACK_UNPACK_SUCCESS); + if (ret != MSGPACK_UNPACK_SUCCESS) { + msgpack_unpacked_destroy(&result); + flb_free(res_data); + return; + } + + root = result.data; + TEST_CHECK(root.type == MSGPACK_OBJECT_MAP); + + for (i = 0; i < root.via.map.size; i++) { + key = root.via.map.ptr[i].key; + val = root.via.map.ptr[i].val; + + if (key.type != MSGPACK_OBJECT_STR) { + continue; + } + + if (key.via.str.size == 4 && + strncmp(key.via.str.ptr, "size", 4) == 0) { + have_size = FLB_TRUE; + } + else if (key.via.str.size == 13 && + strncmp(key.via.str.ptr, "fluent_signal", 13) == 0) { + TEST_CHECK(val.type == MSGPACK_OBJECT_POSITIVE_INTEGER || + val.type == MSGPACK_OBJECT_NEGATIVE_INTEGER); + if (val.type == MSGPACK_OBJECT_POSITIVE_INTEGER) { + TEST_CHECK(val.via.u64 == FLB_EVENT_TYPE_TRACES); + } + else if (val.type == MSGPACK_OBJECT_NEGATIVE_INTEGER) { + TEST_CHECK(val.via.i64 == FLB_EVENT_TYPE_TRACES); + } + have_signal = FLB_TRUE; + } + } + + TEST_CHECK(have_size == FLB_FALSE); + TEST_CHECK(have_signal == FLB_TRUE); + + msgpack_unpacked_destroy(&result); + flb_free(res_data); +} + static void cb_check_forward_compat_mode(void *ctx, int ffd, int res_ret, void *res_data, size_t res_size, void *data) @@ -310,6 +700,42 @@ void flb_test_forward_mode() flb_destroy(ctx); } +void flb_test_forward_mode_ack_options() +{ + int ret; + int in_ffd; + int out_ffd; + flb_ctx_t *ctx; + + ctx = flb_create(); + flb_service_set(ctx, "flush", "2", "grace", "1", NULL); + + in_ffd = flb_input(ctx, (char *) "dummy", NULL); + flb_input_set(ctx, in_ffd, + "tag", "test", + "samples", "1", + "dummy", "{\"key1\": 123, \"key2\": {\"s1\": \"fluent\"}}", + NULL); + + out_ffd = flb_output(ctx, (char *) "forward", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "tag", "new.tag", + "require_ack_response", "true", + NULL); + + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_forward_mode_ack_options, + NULL, NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + void flb_test_forward_compat_mode() { int ret; @@ -352,6 +778,204 @@ void flb_test_forward_compat_mode() flb_destroy(ctx); } +void flb_test_forward_mode_traces_options() +{ + int ret; + int in_ffd; + int out_ffd; + flb_ctx_t *ctx; + + ctx = flb_create(); + flb_service_set(ctx, "flush", "0.2", "grace", "1", NULL); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "forward", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "tag", "new.tag.traces", + NULL); + + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_forward_mode_traces_options, + NULL, NULL); + TEST_CHECK(ret == 0); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + ret = inject_raw_non_log_chunk(ctx, FLB_INPUT_TRACES, "test"); + TEST_CHECK(ret == 0); + + flb_time_msleep(1500); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_forward_mode_profiles_no_crash() +{ + int ret; + int in_ffd; + int out_ffd; + flb_ctx_t *ctx; + void *res_data; + size_t res_size; + msgpack_sbuffer mp_sbuf; + msgpack_packer mp_pck; + + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "forward", NULL); + flb_output_set(ctx, out_ffd, + "match", "*", + "tag", "new.tag", + "time_as_integer", "true", + NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + msgpack_sbuffer_init(&mp_sbuf); + msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); + msgpack_pack_map(&mp_pck, 1); + msgpack_pack_str(&mp_pck, 1); + msgpack_pack_str_body(&mp_pck, "k", 1); + msgpack_pack_str(&mp_pck, 1); + msgpack_pack_str_body(&mp_pck, "v", 1); + + res_data = NULL; + res_size = 0; + ret = run_forward_formatter_non_log(ctx, + FLB_EVENT_TYPE_PROFILES, + "test", + mp_sbuf.data, + mp_sbuf.size, + &res_data, + &res_size); + + TEST_CHECK(ret == MODE_FORWARD); + TEST_CHECK(res_data != NULL); + TEST_CHECK(res_size > 0); + + if (ret == MODE_FORWARD && res_data != NULL && res_size > 0) { + verify_non_log_options_map(res_data, res_size, FLB_EVENT_TYPE_PROFILES); + } + + if (res_data != NULL) { + flb_free(res_data); + } + + msgpack_sbuffer_destroy(&mp_sbuf); + + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_forward_mode_blobs_no_crash() +{ + int ret; + int in_ffd; + int out_ffd; + flb_ctx_t *ctx; + void *res_data; + size_t res_size; + msgpack_sbuffer mp_sbuf; + msgpack_packer mp_pck; + + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "forward", NULL); + flb_output_set(ctx, out_ffd, + "match", "*", + "tag", "new.tag", + "time_as_integer", "true", + NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + msgpack_sbuffer_init(&mp_sbuf); + msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); + msgpack_pack_map(&mp_pck, 1); + msgpack_pack_str(&mp_pck, 1); + msgpack_pack_str_body(&mp_pck, "k", 1); + msgpack_pack_str(&mp_pck, 1); + msgpack_pack_str_body(&mp_pck, "v", 1); + + res_data = NULL; + res_size = 0; + ret = run_forward_formatter_non_log(ctx, + FLB_EVENT_TYPE_BLOBS, + "test", + mp_sbuf.data, + mp_sbuf.size, + &res_data, + &res_size); + + TEST_CHECK(ret == MODE_FORWARD); + TEST_CHECK(res_data != NULL); + TEST_CHECK(res_size > 0); + + if (ret == MODE_FORWARD && res_data != NULL && res_size > 0) { + verify_non_log_options_map(res_data, res_size, FLB_EVENT_TYPE_BLOBS); + } + + if (res_data != NULL) { + flb_free(res_data); + } + + msgpack_sbuffer_destroy(&mp_sbuf); + + flb_stop(ctx); + flb_destroy(ctx); +} + +#ifdef FLB_HAVE_METRICS +void flb_test_forward_mode_metrics_options() +{ + int ret; + int in_ffd; + int out_ffd; + flb_ctx_t *ctx; + + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + in_ffd = flb_input(ctx, (char *) "fluentbit_metrics", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "scrape_interval", "1", NULL); + + out_ffd = flb_output(ctx, (char *) "forward", NULL); + flb_output_set(ctx, out_ffd, + "match", "*", + "tag", "new.tag.metrics", + NULL); + + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_forward_mode_metrics_options, + NULL, NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} +#endif + /* Test list */ TEST_LIST = { #ifdef FLB_HAVE_RECORD_ACCESSOR @@ -359,6 +983,13 @@ TEST_LIST = { {"message_compat_mode", flb_test_message_compat_mode }, #endif {"forward_mode" , flb_test_forward_mode }, + {"forward_mode_ack_options", flb_test_forward_mode_ack_options }, {"forward_compat_mode", flb_test_forward_compat_mode }, + {"forward_mode_traces_options", flb_test_forward_mode_traces_options }, + {"forward_mode_profiles_no_crash", flb_test_forward_mode_profiles_no_crash }, + {"forward_mode_blobs_no_crash", flb_test_forward_mode_blobs_no_crash }, +#ifdef FLB_HAVE_METRICS + {"forward_mode_metrics_options", flb_test_forward_mode_metrics_options }, +#endif {NULL, NULL} };