Skip to content

filter_type_converter: support map_key #8200

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
nokute78 wants to merge 3 commits into
base: master
Choose a base branch
from
Open

filter_type_converter: support map_key #8200

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
6e430af
tests: internal: typecast: add test from map to json_str
nokute78 Nov 18, 2023
868c684
typecast: support casting from map to json_str
nokute78 Nov 18, 2023
ed3d2c8
filter_type_converter: support map_key
nokute78 Nov 18, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion include/fluent-bit/flb_typecast.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@ typedef enum {
FLB_TYPECAST_TYPE_BOOL,
FLB_TYPECAST_TYPE_STR,
FLB_TYPECAST_TYPE_HEX,
FLB_TYPECAST_TYPE_JSON_STR,
FLB_TYPECAST_TYPE_MAP,
FLB_TYPECAST_TYPE_ERROR,
} flb_typecast_type_t;

Expand All @@ -48,7 +50,7 @@ struct flb_typecast_value {
int64_t i_num; /* int */
uint64_t ui_num; /* uint, hex */
double d_num; /* float */
flb_sds_t str; /* string */
flb_sds_t str; /* string, json_str */
} val;
};

Expand Down
9 changes: 9 additions & 0 deletions plugins/filter_type_converter/type_converter.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,9 @@ static int configure(struct type_converter_ctx *ctx,
flb_config_map_foreach(head, mv, ctx->float_keys) {
config_rule(ctx, "float", mv);
}
flb_config_map_foreach(head, mv, ctx->map_keys) {
config_rule(ctx, "map", mv);
}

if (mk_list_size(&ctx->conv_entries) == 0) {
flb_plg_error(ctx->ins, "no rules");
Expand Down Expand Up @@ -384,6 +387,12 @@ static struct flb_config_map config_map[] = {
FLB_CONFIG_MAP_MULT, FLB_TRUE, offsetof(struct type_converter_ctx, str_keys),
"Convert string to other type. e.g. str_key id id_val integer"
},
{
FLB_CONFIG_MAP_SLIST_3, "map_key", NULL,
FLB_CONFIG_MAP_MULT, FLB_TRUE, offsetof(struct type_converter_ctx, map_keys),
"Convert map to other type. e.g. map_key obj obj_str json_string"
},

{0}
};

Expand Down
1 change: 1 addition & 0 deletions plugins/filter_type_converter/type_converter.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ struct type_converter_ctx {
struct mk_list *uint_keys;
struct mk_list *float_keys;
struct mk_list *str_keys;
struct mk_list *map_keys;
};

#endif
80 changes: 78 additions & 2 deletions src/flb_typecast.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
#include <fluent-bit/flb_mem.h>
#include <fluent-bit/flb_utils.h>
#include <fluent-bit/flb_typecast.h>
#include <fluent-bit/flb_pack.h>
#include <string.h>
#include <inttypes.h>
#include <msgpack.h>
Expand All @@ -44,6 +45,13 @@ flb_typecast_type_t flb_typecast_str_to_type_t(char *type_str, int type_len)
else if(!strncasecmp(type_str, "bool", type_len)) {
return FLB_TYPECAST_TYPE_BOOL;
}
else if(!strncasecmp(type_str, "json_string", type_len)) {
return FLB_TYPECAST_TYPE_JSON_STR;
}
else if(!strncasecmp(type_str, "map", type_len)) {
return FLB_TYPECAST_TYPE_MAP;
}


return FLB_TYPECAST_TYPE_ERROR;
}
Expand All @@ -63,12 +71,67 @@ const char * flb_typecast_type_t_to_str(flb_typecast_type_t type)
return "string";
case FLB_TYPECAST_TYPE_BOOL:
return "bool";
case FLB_TYPECAST_TYPE_JSON_STR:
return "json_string";
case FLB_TYPECAST_TYPE_MAP:
return "map";
default:
return "unknown type";
}

}

static int flb_typecast_to_json(msgpack_object *obj,
struct flb_typecast_rule *rule,
msgpack_packer *pck,
struct flb_typecast_value *output)
{
flb_sds_t json_str = flb_sds_create_size(4096);
int ret;

switch(rule->to_type) {
case FLB_TYPECAST_TYPE_JSON_STR:
ret = flb_msgpack_to_json(json_str, flb_sds_len(json_str), obj);
if (ret < 0) {
flb_error("%s: flb_msgpack_to_json failed. ret=%d", __FUNCTION__, ret);
flb_sds_destroy(json_str);
return -1;
}
output->val.str = flb_sds_create(json_str);
if (pck != NULL) {
msgpack_pack_str(pck, flb_sds_len(output->val.str));
msgpack_pack_str_body(pck, output->val.str, flb_sds_len(output->val.str));
}
ret = 0;

break;

default:
flb_error("%s: unknown type %d", __FUNCTION__, rule->to_type);
ret = -1;
}
flb_sds_destroy(json_str);

return ret;
}

static int flb_typecast_conv_map(msgpack_object *obj,
struct flb_typecast_rule *rule,
msgpack_packer *pck,
struct flb_typecast_value *output)
{
if(obj == NULL || rule == NULL || output == NULL) {
return -1;
}
else if (rule->to_type != FLB_TYPECAST_TYPE_JSON_STR) {
flb_error("%s: type %s is not supported",__FUNCTION__,
flb_typecast_type_t_to_str(rule->to_type));
return -1;
}

return flb_typecast_to_json(obj, rule, pck, output);
}

static int flb_typecast_conv_str(const char *input, int input_len,
struct flb_typecast_rule *rule,
msgpack_packer *pck,
Expand Down Expand Up @@ -159,6 +222,7 @@ static int flb_typecast_conv_str(const char *input, int input_len,

break;
case FLB_TYPECAST_TYPE_STR:
case FLB_TYPECAST_TYPE_JSON_STR:
flb_error("%s: str to str. nothing to do.", __FUNCTION__);
return -1;
break;
Expand Down Expand Up @@ -221,6 +285,7 @@ static int flb_typecast_conv_int(int64_t input,

switch(rule->to_type) {
case FLB_TYPECAST_TYPE_STR:
case FLB_TYPECAST_TYPE_JSON_STR:
i = snprintf(temp, sizeof(temp) -1, "%"PRId64, input);
output->val.str = flb_sds_create_len(temp, i);
if(pck != NULL) {
Expand Down Expand Up @@ -264,6 +329,7 @@ static int flb_typecast_conv_uint(uint64_t input,

switch(rule->to_type) {
case FLB_TYPECAST_TYPE_STR:
case FLB_TYPECAST_TYPE_JSON_STR:
i = snprintf(temp, sizeof(temp) -1, "%"PRIu64, input);
output->val.str = flb_sds_create_len(temp, i);
if(pck != NULL) {
Expand Down Expand Up @@ -307,6 +373,7 @@ static int flb_typecast_conv_float(double input,

switch(rule->to_type) {
case FLB_TYPECAST_TYPE_STR:
case FLB_TYPECAST_TYPE_JSON_STR:
if (input == (double)(long long int)input) {
i = snprintf(temp, sizeof(temp)-1, "%.1f", input);
}
Expand Down Expand Up @@ -406,6 +473,7 @@ static int flb_typecast_value_conv(msgpack_object input,

switch(rule->from_type) {
case FLB_TYPECAST_TYPE_STR:
case FLB_TYPECAST_TYPE_JSON_STR:
if (input.type != MSGPACK_OBJECT_STR) {
flb_error("%s: src type is not str", __FUNCTION__);
return -1;
Expand Down Expand Up @@ -449,6 +517,13 @@ static int flb_typecast_value_conv(msgpack_object input,
ret = flb_typecast_conv_float(input.via.f64, rule, pck, output);

break;
case FLB_TYPECAST_TYPE_MAP:
if (input.type != MSGPACK_OBJECT_MAP) {
flb_error("%s: src type is not map", __FUNCTION__);
return -1;
}
ret = flb_typecast_conv_map(&input, rule, pck, output);
break;

default:
flb_error("%s: unknown type %d", __FUNCTION__, rule->from_type);
Expand All @@ -461,7 +536,7 @@ int flb_typecast_value_destroy(struct flb_typecast_value* val)
if (val == NULL) {
return 0;
}
if (val->type == FLB_TYPECAST_TYPE_STR) {
if (val->type == FLB_TYPECAST_TYPE_STR || val->type == FLB_TYPECAST_TYPE_JSON_STR) {
flb_sds_destroy(val->val.str);
}
flb_free(val);
Expand Down Expand Up @@ -517,7 +592,8 @@ int flb_typecast_pack(msgpack_object input,

ret = flb_typecast_value_conv(input, rule, pck, &val);

if (ret == 0 && rule->to_type == FLB_TYPECAST_TYPE_STR) {
if (ret == 0 &&
(rule->to_type == FLB_TYPECAST_TYPE_STR || rule->to_type == FLB_TYPECAST_TYPE_JSON_STR)) {
flb_sds_destroy(val.val.str);
}

Expand Down
50 changes: 50 additions & 0 deletions tests/internal/typecast.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -347,6 +347,55 @@ void float_to_str()
msgpack_unpacked_destroy(&result);
}

void map_to_json_str()
{
msgpack_sbuffer sbuf;
msgpack_packer pck;
msgpack_unpacked result;
size_t off = 0;

struct flb_typecast_rule *rule = NULL;
struct flb_typecast_value *val = NULL;

/* create input object */
msgpack_sbuffer_init(&sbuf);
msgpack_packer_init(&pck, &sbuf, msgpack_sbuffer_write);

/* map {"k":"v"}*/
msgpack_pack_map(&pck, 1);
msgpack_pack_str(&pck, 1);
msgpack_pack_str_body(&pck, "k", 1);
msgpack_pack_str(&pck, 1);
msgpack_pack_str_body(&pck, "v", 1);

msgpack_unpacked_init(&result);
msgpack_unpack_next(&result, sbuf.data, sbuf.size, &off);

/* create rule */
rule = flb_typecast_rule_create("map", 3, "json_string", 11);
if (!TEST_CHECK(rule != NULL)) {
TEST_MSG("failed to create rule");
exit(EXIT_FAILURE);
}

val = flb_typecast_value_create(result.data, rule);
if(!TEST_CHECK(val != NULL)){
TEST_MSG("failed to create value");
exit(EXIT_FAILURE);
}

TEST_CHECK(val->type == FLB_TYPECAST_TYPE_JSON_STR);
if(!TEST_CHECK(strstr(val->val.str, "\"k\":\"v\"") != NULL)) {
TEST_MSG("got %s. expect \"k\":\"v\"", val->val.str);
}

flb_typecast_rule_destroy(rule);
flb_typecast_value_destroy(val);

msgpack_sbuffer_destroy(&sbuf);
msgpack_unpacked_destroy(&result);
}

TEST_LIST = {
{"str_to_int", str_to_int},
{"int_to_str", int_to_str},
Expand All @@ -355,5 +404,6 @@ TEST_LIST = {
{"bool_to_str", bool_to_str},
{"str_to_bool", str_to_bool},
{"str_to_hex", str_to_hex},
{"map_to_json_str",map_to_json_str},
{NULL, NULL}
};