diff --git a/include/fluent-bit/flb_typecast.h b/include/fluent-bit/flb_typecast.h index 9ae66cedf82..add05f4d245 100644 --- a/include/fluent-bit/flb_typecast.h +++ b/include/fluent-bit/flb_typecast.h @@ -33,6 +33,8 @@ typedef enum { FLB_TYPECAST_TYPE_BOOL, FLB_TYPECAST_TYPE_STR, FLB_TYPECAST_TYPE_HEX, + FLB_TYPECAST_TYPE_JSON_STR, + FLB_TYPECAST_TYPE_MAP, FLB_TYPECAST_TYPE_ERROR, } flb_typecast_type_t; @@ -48,7 +50,7 @@ struct flb_typecast_value { int64_t i_num; /* int */ uint64_t ui_num; /* uint, hex */ double d_num; /* float */ - flb_sds_t str; /* string */ + flb_sds_t str; /* string, json_str */ } val; }; diff --git a/plugins/filter_type_converter/type_converter.c b/plugins/filter_type_converter/type_converter.c index 36422002cbc..62f53cfb7e0 100644 --- a/plugins/filter_type_converter/type_converter.c +++ b/plugins/filter_type_converter/type_converter.c @@ -130,6 +130,9 @@ static int configure(struct type_converter_ctx *ctx, flb_config_map_foreach(head, mv, ctx->float_keys) { config_rule(ctx, "float", mv); } + flb_config_map_foreach(head, mv, ctx->map_keys) { + config_rule(ctx, "map", mv); + } if (mk_list_size(&ctx->conv_entries) == 0) { flb_plg_error(ctx->ins, "no rules"); @@ -384,6 +387,12 @@ static struct flb_config_map config_map[] = { FLB_CONFIG_MAP_MULT, FLB_TRUE, offsetof(struct type_converter_ctx, str_keys), "Convert string to other type. e.g. str_key id id_val integer" }, + { + FLB_CONFIG_MAP_SLIST_3, "map_key", NULL, + FLB_CONFIG_MAP_MULT, FLB_TRUE, offsetof(struct type_converter_ctx, map_keys), + "Convert map to other type. e.g. map_key obj obj_str json_string" + }, + {0} }; diff --git a/plugins/filter_type_converter/type_converter.h b/plugins/filter_type_converter/type_converter.h index cd75403510a..e80c2871b48 100644 --- a/plugins/filter_type_converter/type_converter.h +++ b/plugins/filter_type_converter/type_converter.h @@ -41,6 +41,7 @@ struct type_converter_ctx { struct mk_list *uint_keys; struct mk_list *float_keys; struct mk_list *str_keys; + struct mk_list *map_keys; }; #endif diff --git a/src/flb_typecast.c b/src/flb_typecast.c index 4c2d10536f9..e1a6ca74d38 100644 --- a/src/flb_typecast.c +++ b/src/flb_typecast.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -44,6 +45,13 @@ flb_typecast_type_t flb_typecast_str_to_type_t(char *type_str, int type_len) else if(!strncasecmp(type_str, "bool", type_len)) { return FLB_TYPECAST_TYPE_BOOL; } + else if(!strncasecmp(type_str, "json_string", type_len)) { + return FLB_TYPECAST_TYPE_JSON_STR; + } + else if(!strncasecmp(type_str, "map", type_len)) { + return FLB_TYPECAST_TYPE_MAP; + } + return FLB_TYPECAST_TYPE_ERROR; } @@ -63,12 +71,67 @@ const char * flb_typecast_type_t_to_str(flb_typecast_type_t type) return "string"; case FLB_TYPECAST_TYPE_BOOL: return "bool"; + case FLB_TYPECAST_TYPE_JSON_STR: + return "json_string"; + case FLB_TYPECAST_TYPE_MAP: + return "map"; default: return "unknown type"; } } +static int flb_typecast_to_json(msgpack_object *obj, + struct flb_typecast_rule *rule, + msgpack_packer *pck, + struct flb_typecast_value *output) +{ + flb_sds_t json_str = flb_sds_create_size(4096); + int ret; + + switch(rule->to_type) { + case FLB_TYPECAST_TYPE_JSON_STR: + ret = flb_msgpack_to_json(json_str, flb_sds_len(json_str), obj); + if (ret < 0) { + flb_error("%s: flb_msgpack_to_json failed. ret=%d", __FUNCTION__, ret); + flb_sds_destroy(json_str); + return -1; + } + output->val.str = flb_sds_create(json_str); + if (pck != NULL) { + msgpack_pack_str(pck, flb_sds_len(output->val.str)); + msgpack_pack_str_body(pck, output->val.str, flb_sds_len(output->val.str)); + } + ret = 0; + + break; + + default: + flb_error("%s: unknown type %d", __FUNCTION__, rule->to_type); + ret = -1; + } + flb_sds_destroy(json_str); + + return ret; +} + +static int flb_typecast_conv_map(msgpack_object *obj, + struct flb_typecast_rule *rule, + msgpack_packer *pck, + struct flb_typecast_value *output) +{ + if(obj == NULL || rule == NULL || output == NULL) { + return -1; + } + else if (rule->to_type != FLB_TYPECAST_TYPE_JSON_STR) { + flb_error("%s: type %s is not supported",__FUNCTION__, + flb_typecast_type_t_to_str(rule->to_type)); + return -1; + } + + return flb_typecast_to_json(obj, rule, pck, output); +} + static int flb_typecast_conv_str(const char *input, int input_len, struct flb_typecast_rule *rule, msgpack_packer *pck, @@ -159,6 +222,7 @@ static int flb_typecast_conv_str(const char *input, int input_len, break; case FLB_TYPECAST_TYPE_STR: + case FLB_TYPECAST_TYPE_JSON_STR: flb_error("%s: str to str. nothing to do.", __FUNCTION__); return -1; break; @@ -221,6 +285,7 @@ static int flb_typecast_conv_int(int64_t input, switch(rule->to_type) { case FLB_TYPECAST_TYPE_STR: + case FLB_TYPECAST_TYPE_JSON_STR: i = snprintf(temp, sizeof(temp) -1, "%"PRId64, input); output->val.str = flb_sds_create_len(temp, i); if(pck != NULL) { @@ -264,6 +329,7 @@ static int flb_typecast_conv_uint(uint64_t input, switch(rule->to_type) { case FLB_TYPECAST_TYPE_STR: + case FLB_TYPECAST_TYPE_JSON_STR: i = snprintf(temp, sizeof(temp) -1, "%"PRIu64, input); output->val.str = flb_sds_create_len(temp, i); if(pck != NULL) { @@ -307,6 +373,7 @@ static int flb_typecast_conv_float(double input, switch(rule->to_type) { case FLB_TYPECAST_TYPE_STR: + case FLB_TYPECAST_TYPE_JSON_STR: if (input == (double)(long long int)input) { i = snprintf(temp, sizeof(temp)-1, "%.1f", input); } @@ -406,6 +473,7 @@ static int flb_typecast_value_conv(msgpack_object input, switch(rule->from_type) { case FLB_TYPECAST_TYPE_STR: + case FLB_TYPECAST_TYPE_JSON_STR: if (input.type != MSGPACK_OBJECT_STR) { flb_error("%s: src type is not str", __FUNCTION__); return -1; @@ -449,6 +517,13 @@ static int flb_typecast_value_conv(msgpack_object input, ret = flb_typecast_conv_float(input.via.f64, rule, pck, output); break; + case FLB_TYPECAST_TYPE_MAP: + if (input.type != MSGPACK_OBJECT_MAP) { + flb_error("%s: src type is not map", __FUNCTION__); + return -1; + } + ret = flb_typecast_conv_map(&input, rule, pck, output); + break; default: flb_error("%s: unknown type %d", __FUNCTION__, rule->from_type); @@ -461,7 +536,7 @@ int flb_typecast_value_destroy(struct flb_typecast_value* val) if (val == NULL) { return 0; } - if (val->type == FLB_TYPECAST_TYPE_STR) { + if (val->type == FLB_TYPECAST_TYPE_STR || val->type == FLB_TYPECAST_TYPE_JSON_STR) { flb_sds_destroy(val->val.str); } flb_free(val); @@ -517,7 +592,8 @@ int flb_typecast_pack(msgpack_object input, ret = flb_typecast_value_conv(input, rule, pck, &val); - if (ret == 0 && rule->to_type == FLB_TYPECAST_TYPE_STR) { + if (ret == 0 && + (rule->to_type == FLB_TYPECAST_TYPE_STR || rule->to_type == FLB_TYPECAST_TYPE_JSON_STR)) { flb_sds_destroy(val.val.str); } diff --git a/tests/internal/typecast.c b/tests/internal/typecast.c index 511d8cf595a..38a9abcbec9 100644 --- a/tests/internal/typecast.c +++ b/tests/internal/typecast.c @@ -347,6 +347,55 @@ void float_to_str() msgpack_unpacked_destroy(&result); } +void map_to_json_str() +{ + msgpack_sbuffer sbuf; + msgpack_packer pck; + msgpack_unpacked result; + size_t off = 0; + + struct flb_typecast_rule *rule = NULL; + struct flb_typecast_value *val = NULL; + + /* create input object */ + msgpack_sbuffer_init(&sbuf); + msgpack_packer_init(&pck, &sbuf, msgpack_sbuffer_write); + + /* map {"k":"v"}*/ + msgpack_pack_map(&pck, 1); + msgpack_pack_str(&pck, 1); + msgpack_pack_str_body(&pck, "k", 1); + msgpack_pack_str(&pck, 1); + msgpack_pack_str_body(&pck, "v", 1); + + msgpack_unpacked_init(&result); + msgpack_unpack_next(&result, sbuf.data, sbuf.size, &off); + + /* create rule */ + rule = flb_typecast_rule_create("map", 3, "json_string", 11); + if (!TEST_CHECK(rule != NULL)) { + TEST_MSG("failed to create rule"); + exit(EXIT_FAILURE); + } + + val = flb_typecast_value_create(result.data, rule); + if(!TEST_CHECK(val != NULL)){ + TEST_MSG("failed to create value"); + exit(EXIT_FAILURE); + } + + TEST_CHECK(val->type == FLB_TYPECAST_TYPE_JSON_STR); + if(!TEST_CHECK(strstr(val->val.str, "\"k\":\"v\"") != NULL)) { + TEST_MSG("got %s. expect \"k\":\"v\"", val->val.str); + } + + flb_typecast_rule_destroy(rule); + flb_typecast_value_destroy(val); + + msgpack_sbuffer_destroy(&sbuf); + msgpack_unpacked_destroy(&result); +} + TEST_LIST = { {"str_to_int", str_to_int}, {"int_to_str", int_to_str}, @@ -355,5 +404,6 @@ TEST_LIST = { {"bool_to_str", bool_to_str}, {"str_to_bool", str_to_bool}, {"str_to_hex", str_to_hex}, + {"map_to_json_str",map_to_json_str}, {NULL, NULL} };