X-Git-Url: https://git.tokkee.org/?p=collectd.git;a=blobdiff_plain;f=src%2Fwrite_kafka.c;h=6d826c25ba213ad9acf6ae1f8a56a16e62fceeab;hp=eaaf171262bd0b06ef07e0ddb8e32056bb9d873b;hb=1326af38b3ef25c41c994cd76c043202636b3d70;hpb=68d6aff96c029c9a455b7a0146c788af08f9ff22 diff --git a/src/write_kafka.c b/src/write_kafka.c index eaaf1712..6d826c25 100644 --- a/src/write_kafka.c +++ b/src/write_kafka.c @@ -28,37 +28,31 @@ #include "plugin.h" #include "common.h" -#include "configfile.h" -#include "utils_cache.h" #include "utils_cmd_putval.h" #include "utils_format_graphite.h" #include "utils_format_json.h" -#include "utils_crc32.h" #include #include -#include -#include #include struct kafka_topic_context { #define KAFKA_FORMAT_JSON 0 #define KAFKA_FORMAT_COMMAND 1 #define KAFKA_FORMAT_GRAPHITE 2 - uint8_t format; + uint8_t format; unsigned int graphite_flags; _Bool store_rates; rd_kafka_topic_conf_t *conf; rd_kafka_topic_t *topic; rd_kafka_conf_t *kafka_conf; rd_kafka_t *kafka; - int has_key; - uint32_t key; + char *key; char *prefix; char *postfix; char escape_char; char *topic_name; - pthread_mutex_t lock; + pthread_mutex_t lock; }; static int kafka_handle(struct kafka_topic_context *); @@ -83,13 +77,30 @@ static void kafka_log(const rd_kafka_t *rkt, int level, } #endif +static uint32_t kafka_hash(const char *keydata, size_t keylen) +{ + uint32_t hash = 5381; + for (; keylen > 0; keylen--) + hash = ((hash << 5) + hash) + keydata[keylen - 1]; + return hash; +} + +/* 31 bit -> 4 byte -> 8 byte hex string + null byte */ +#define KAFKA_RANDOM_KEY_SIZE 9 +#define KAFKA_RANDOM_KEY_BUFFER (char[KAFKA_RANDOM_KEY_SIZE]) {""} +static char *kafka_random_key(char buffer[static KAFKA_RANDOM_KEY_SIZE]) +{ + ssnprintf(buffer, KAFKA_RANDOM_KEY_SIZE, "%08lX", (unsigned long) mrand48()); + return buffer; +} + static int32_t kafka_partition(const rd_kafka_topic_t *rkt, const void *keydata, size_t keylen, int32_t partition_cnt, void *p, void *m) { - uint32_t key = *((uint32_t *)keydata ); + uint32_t key = kafka_hash(keydata, keylen); uint32_t target = key % partition_cnt; - int32_t i = partition_cnt; + int32_t i = partition_cnt; while (--i > 0 && !rd_kafka_topic_partition_available(rkt, target)) { target = (target + 1) % partition_cnt; @@ -114,37 +125,37 @@ static int kafka_handle(struct kafka_topic_context *ctx) /* {{{ */ if ((ctx->kafka = rd_kafka_new(RD_KAFKA_PRODUCER, conf, errbuf, sizeof(errbuf))) == NULL) { - ERROR("write_kafka plugin: cannot create kafka handle."); - return 1; + ERROR("write_kafka plugin: cannot create kafka handle."); + return 1; } - rd_kafka_conf_destroy(ctx->kafka_conf); - ctx->kafka_conf = NULL; + rd_kafka_conf_destroy(ctx->kafka_conf); + ctx->kafka_conf = NULL; - INFO ("write_kafka plugin: created KAFKA handle : %s", rd_kafka_name(ctx->kafka)); + INFO ("write_kafka plugin: created KAFKA handle : %s", rd_kafka_name(ctx->kafka)); -#ifdef HAVE_LIBRDKAFKA_LOGGER - rd_kafka_set_logger(ctx->kafka, kafka_log); +#if defined(HAVE_LIBRDKAFKA_LOGGER) && !defined(HAVE_LIBRDKAFKA_LOG_CB) + rd_kafka_set_logger(ctx->kafka, kafka_log); #endif } if (ctx->topic == NULL ) { - if ((topic_conf = rd_kafka_topic_conf_dup(ctx->conf)) == NULL) { + if ((topic_conf = rd_kafka_topic_conf_dup(ctx->conf)) == NULL) { ERROR("write_kafka plugin: cannot duplicate kafka topic config"); return 1; - } + } - if ((ctx->topic = rd_kafka_topic_new(ctx->kafka, ctx->topic_name, - topic_conf)) == NULL) { - ERROR("write_kafka plugin: cannot create topic : %s\n", - rd_kafka_err2str(rd_kafka_errno2err(errno))); - return errno; - } + if ((ctx->topic = rd_kafka_topic_new(ctx->kafka, ctx->topic_name, + topic_conf)) == NULL) { + ERROR("write_kafka plugin: cannot create topic : %s\n", + rd_kafka_err2str(rd_kafka_errno2err(errno))); + return errno; + } - rd_kafka_topic_conf_destroy(ctx->conf); - ctx->conf = NULL; + rd_kafka_topic_conf_destroy(ctx->conf); + ctx->conf = NULL; - INFO ("write_kafka plugin: handle created for topic : %s", rd_kafka_topic_name(ctx->topic)); + INFO ("write_kafka plugin: handle created for topic : %s", rd_kafka_topic_name(ctx->topic)); } return(0); @@ -152,16 +163,17 @@ static int kafka_handle(struct kafka_topic_context *ctx) /* {{{ */ } /* }}} int kafka_handle */ static int kafka_write(const data_set_t *ds, /* {{{ */ - const value_list_t *vl, - user_data_t *ud) + const value_list_t *vl, + user_data_t *ud) { - int status = 0; - uint32_t key; - char buffer[8192]; - size_t bfree = sizeof(buffer); - size_t bfill = 0; - size_t blen = 0; - struct kafka_topic_context *ctx = ud->data; + int status = 0; + void *key; + size_t keylen = 0; + char buffer[8192]; + size_t bfree = sizeof(buffer); + size_t bfill = 0; + size_t blen = 0; + struct kafka_topic_context *ctx = ud->data; if ((ds == NULL) || (vl == NULL) || (ctx == NULL)) return EINVAL; @@ -185,7 +197,6 @@ static int kafka_write(const data_set_t *ds, /* {{{ */ blen = strlen(buffer); break; case KAFKA_FORMAT_JSON: - format_json_initialize(buffer, &bfill, &bfree); format_json_value_list(buffer, &bfill, &bfree, ds, vl, ctx->store_rates); @@ -208,27 +219,24 @@ static int kafka_write(const data_set_t *ds, /* {{{ */ return -1; } - /* - * We partition our stream by metric name - */ - if (ctx->has_key) - key = ctx->key; - else - key = rand(); + key = (ctx->key != NULL) + ? ctx->key + : kafka_random_key(KAFKA_RANDOM_KEY_BUFFER); + keylen = strlen (key); rd_kafka_produce(ctx->topic, RD_KAFKA_PARTITION_UA, RD_KAFKA_MSG_F_COPY, buffer, blen, - &key, sizeof(key), NULL); + key, keylen, NULL); - return status; + return status; } /* }}} int kafka_write */ static void kafka_topic_context_free(void *p) /* {{{ */ { - struct kafka_topic_context *ctx = p; + struct kafka_topic_context *ctx = p; - if (ctx == NULL) - return; + if (ctx == NULL) + return; if (ctx->topic_name != NULL) sfree(ctx->topic_name); @@ -247,24 +255,23 @@ static void kafka_topic_context_free(void *p) /* {{{ */ static void kafka_config_topic(rd_kafka_conf_t *conf, oconfig_item_t *ci) /* {{{ */ { int status; - int i; struct kafka_topic_context *tctx; char *key = NULL; char *val; char callback_name[DATA_MAX_NAME_LEN]; char errbuf[1024]; - user_data_t ud; - oconfig_item_t *child; + oconfig_item_t *child; rd_kafka_conf_res_t ret; - if ((tctx = calloc(1, sizeof (*tctx))) == NULL) { - ERROR ("write_kafka plugin: calloc failed."); + if ((tctx = calloc(1, sizeof (*tctx))) == NULL) { + ERROR ("write_kafka plugin: calloc failed."); return; - } + } tctx->escape_char = '.'; tctx->store_rates = 1; tctx->format = KAFKA_FORMAT_JSON; + tctx->key = NULL; if ((tctx->kafka_conf = rd_kafka_conf_dup(conf)) == NULL) { sfree(tctx); @@ -298,48 +305,41 @@ static void kafka_config_topic(rd_kafka_conf_t *conf, oconfig_item_t *ci) /* {{{ goto errout; } - for (i = 0; i < ci->children_num; i++) { - /* - * The code here could be simplified but makes room - * for easy adding of new options later on. - */ - child = &ci->children[i]; - status = 0; - - if (strcasecmp ("Property", child->key) == 0) { - if (child->values_num != 2) { - WARNING("kafka properties need both a key and a value."); + for (int i = 0; i < ci->children_num; i++) { + /* + * The code here could be simplified but makes room + * for easy adding of new options later on. + */ + child = &ci->children[i]; + status = 0; + + if (strcasecmp ("Property", child->key) == 0) { + if (child->values_num != 2) { + WARNING("kafka properties need both a key and a value."); goto errout; - } - if (child->values[0].type != OCONFIG_TYPE_STRING || - child->values[1].type != OCONFIG_TYPE_STRING) { - WARNING("kafka properties needs string arguments."); + } + if (child->values[0].type != OCONFIG_TYPE_STRING || + child->values[1].type != OCONFIG_TYPE_STRING) { + WARNING("kafka properties needs string arguments."); goto errout; - } + } key = child->values[0].value.string; val = child->values[1].value.string; ret = rd_kafka_topic_conf_set(tctx->conf,key, val, errbuf, sizeof(errbuf)); if (ret != RD_KAFKA_CONF_OK) { - WARNING("cannot set kafka topic property %s to %s: %s.", + WARNING("cannot set kafka topic property %s to %s: %s.", key, val, errbuf); goto errout; - } - - } else if (strcasecmp ("Key", child->key) == 0) { - char *tmp_buf = NULL; - status = cf_util_get_string(child, &tmp_buf); - if (status != 0) { - WARNING("write_kafka plugin: invalid key supplied"); - break; } - if (strcasecmp(tmp_buf, "Random") != 0) { - tctx->has_key = 1; - tctx->key = crc32_buffer((u_char *)tmp_buf, strlen(tmp_buf)); + } else if (strcasecmp ("Key", child->key) == 0) { + if (cf_util_get_string (child, &tctx->key) != 0) + continue; + if (strcasecmp ("Random", tctx->key) == 0) { + sfree(tctx->key); + tctx->key = strdup (kafka_random_key (KAFKA_RANDOM_KEY_BUFFER)); } - sfree(tmp_buf); - } else if (strcasecmp ("Format", child->key) == 0) { status = cf_util_get_string(child, &key); if (status != 0) @@ -402,14 +402,16 @@ static void kafka_config_topic(rd_kafka_conf_t *conf, oconfig_item_t *ci) /* {{{ ssnprintf(callback_name, sizeof(callback_name), "write_kafka/%s", tctx->topic_name); - ud.data = tctx; - ud.free_func = kafka_topic_context_free; + user_data_t ud = { + .data = tctx, + .free_func = kafka_topic_context_free + }; - status = plugin_register_write (callback_name, kafka_write, &ud); - if (status != 0) { - WARNING ("write_kafka plugin: plugin_register_write (\"%s\") " - "failed with status %i.", - callback_name, status); + status = plugin_register_write (callback_name, kafka_write, &ud); + if (status != 0) { + WARNING ("write_kafka plugin: plugin_register_write (\"%s\") " + "failed with status %i.", + callback_name, status); goto errout; } @@ -422,14 +424,13 @@ static void kafka_config_topic(rd_kafka_conf_t *conf, oconfig_item_t *ci) /* {{{ if (tctx->conf != NULL) rd_kafka_topic_conf_destroy(tctx->conf); if (tctx->kafka_conf != NULL) - rd_kafka_conf_destroy(tctx->kafka_conf); + rd_kafka_conf_destroy(tctx->kafka_conf); sfree(tctx); } /* }}} int kafka_config_topic */ static int kafka_config(oconfig_item_t *ci) /* {{{ */ { - int i; - oconfig_item_t *child; + oconfig_item_t *child; rd_kafka_conf_t *conf; rd_kafka_conf_res_t ret; char errbuf[1024]; @@ -438,46 +439,49 @@ static int kafka_config(oconfig_item_t *ci) /* {{{ */ WARNING("cannot allocate kafka configuration."); return -1; } - for (i = 0; i < ci->children_num; i++) { - child = &ci->children[i]; + for (int i = 0; i < ci->children_num; i++) { + child = &ci->children[i]; - if (strcasecmp("Topic", child->key) == 0) { - kafka_config_topic (conf, child); - } else if (strcasecmp(child->key, "Property") == 0) { - char *key = NULL; - char *val = NULL; + if (strcasecmp("Topic", child->key) == 0) { + kafka_config_topic (conf, child); + } else if (strcasecmp(child->key, "Property") == 0) { + char *key = NULL; + char *val = NULL; - if (child->values_num != 2) { - WARNING("kafka properties need both a key and a value."); + if (child->values_num != 2) { + WARNING("kafka properties need both a key and a value."); goto errout; - } - if (child->values[0].type != OCONFIG_TYPE_STRING || - child->values[1].type != OCONFIG_TYPE_STRING) { - WARNING("kafka properties needs string arguments."); + } + if (child->values[0].type != OCONFIG_TYPE_STRING || + child->values[1].type != OCONFIG_TYPE_STRING) { + WARNING("kafka properties needs string arguments."); goto errout; - } - if ((key = strdup(child->values[0].value.string)) == NULL) { - WARNING("cannot allocate memory for attribute key."); + } + if ((key = strdup(child->values[0].value.string)) == NULL) { + WARNING("cannot allocate memory for attribute key."); goto errout; - } - if ((val = strdup(child->values[1].value.string)) == NULL) { - WARNING("cannot allocate memory for attribute value."); + } + if ((val = strdup(child->values[1].value.string)) == NULL) { + WARNING("cannot allocate memory for attribute value."); + sfree(key); goto errout; - } + } ret = rd_kafka_conf_set(conf, key, val, errbuf, sizeof(errbuf)); if (ret != RD_KAFKA_CONF_OK) { WARNING("cannot set kafka property %s to %s: %s", key, val, errbuf); + sfree(key); + sfree(val); goto errout; } - sfree(key); - sfree(val); - } else { - WARNING ("write_kafka plugin: Ignoring unknown " - "configuration option \"%s\" at top level.", - child->key); - } - } + sfree(key); + sfree(val); + } else { + WARNING ("write_kafka plugin: Ignoring unknown " + "configuration option \"%s\" at top level.", + child->key); + } + } if (conf != NULL) rd_kafka_conf_destroy(conf); return (0); @@ -489,7 +493,5 @@ static int kafka_config(oconfig_item_t *ci) /* {{{ */ void module_register(void) { - plugin_register_complex_config ("write_kafka", kafka_config); + plugin_register_complex_config ("write_kafka", kafka_config); } - -/* vim: set sw=8 sts=8 ts=8 noet : */