From f2a35cefd56bdf4b26cc82637a29445abe1744fb Mon Sep 17 00:00:00 2001 From: Florian Forster Date: Thu, 8 Oct 2009 11:34:20 +0200 Subject: [PATCH] hashed match: Add a match to simplify load balancing. --- configure.in | 2 + src/Makefile.am | 8 ++ src/collectd.conf.pod | 71 ++++++++++++++++ src/match_hashed.c | 184 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 265 insertions(+) create mode 100644 src/match_hashed.c diff --git a/configure.in b/configure.in index f091d025..d21df3cd 100644 --- a/configure.in +++ b/configure.in @@ -3869,6 +3869,7 @@ AC_PLUGIN([load], [$plugin_load], [System load]) AC_PLUGIN([logfile], [yes], [File logging plugin]) AC_PLUGIN([madwifi], [$have_linux_wireless_h], [Madwifi wireless statistics]) AC_PLUGIN([match_empty_counter], [yes], [The empty counter match]) +AC_PLUGIN([match_hashed], [yes], [The hashed match]) AC_PLUGIN([match_regex], [yes], [The regex match]) AC_PLUGIN([match_timediff], [yes], [The timediff match]) AC_PLUGIN([match_value], [yes], [The value match]) @@ -4173,6 +4174,7 @@ Configuration: logfile . . . . . . . $enable_logfile madwifi . . . . . . . $enable_madwifi match_empty_counter . $enable_match_empty_counter + match_hashed . . . . $enable_match_hashed match_regex . . . . . $enable_match_regex match_timediff . . . $enable_match_timediff match_value . . . . . $enable_match_value diff --git a/src/Makefile.am b/src/Makefile.am index 9bae902b..d0cd99d6 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -478,6 +478,14 @@ collectd_LDADD += "-dlopen" match_empty_counter.la collectd_DEPENDENCIES += match_empty_counter.la endif +if BUILD_PLUGIN_MATCH_HASHED +pkglib_LTLIBRARIES += match_hashed.la +match_hashed_la_SOURCES = match_hashed.c +match_hashed_la_LDFLAGS = -module -avoid-version +collectd_LDADD += "-dlopen" match_hashed.la +collectd_DEPENDENCIES += match_hashed.la +endif + if BUILD_PLUGIN_MATCH_REGEX pkglib_LTLIBRARIES += match_regex.la match_regex_la_SOURCES = match_regex.c diff --git a/src/collectd.conf.pod b/src/collectd.conf.pod index ae56da3d..2458462a 100644 --- a/src/collectd.conf.pod +++ b/src/collectd.conf.pod @@ -4632,6 +4632,77 @@ time. If the counter is reset for some reason (machine or service restarted, usually), the graph will be empty (NAN) for a long time. People may not understand why. +=item B + +Calculates a hash value of the host name and matches values according to that +hash value. This makes it possible to divide all hosts into groups and match +only values that are in a specific group. The intended use is in load +balancing, where you want to handle only part of all data and leave the rest +for other servers. + +The hashing function used tries to distribute the hosts evenly. First, it +calculates a 32Ebit hash value using the characters of the hostname: + + hash_value = 0; + for (i = 0; host[i] != 0; i++) + hash_value = (hash_value * 251) + host[i]; + +The constant 251 is a prime number which is supposed to make this hash value +more random. The code then checks the group for this host according to the +I and I arguments: + + if ((hash_value % Total) == Match) + matches; + else + does not match; + +Please note that when you set I to two (i.Ee. you have only two +groups), then the least significant bit of the hash value will be the XOR of +all least significant bits in the host name. One consequence is that when you +have two hosts, "server0.example.com" and "server1.example.com", where the host +name differs in one digit only and the digits differ by one, those hosts will +never end up in the same group. + +Available options: + +=over 4 + +=item B I I + +Divide the data into I groups and match all hosts in group I as +described above. The groups are numbered from zero, i.Ee. I must +be smaller than I. I must be at least one, although only values +greater than one really do make any sense. + +You can repeat this option to match multiple groups, for example: + + Match 3 7 + Match 5 7 + +The above config will divide the data into seven groups and match groups three +and five. One use would be to keep every value on two hosts so that if one +fails the missing data can later be reconstructed from the second host. + +=back + +Example: + + # Operate on the pre-cache chain, so that ignored values are not even in the + # global cache. + + + + # Divide all received hosts in seven groups and accept all hosts in + # group three. + Match 3 7 + + # If matched: Return and continue. + Target "return" + + # If not matched: Return and stop. + Target "stop" + + =back =head2 Available targets diff --git a/src/match_hashed.c b/src/match_hashed.c new file mode 100644 index 00000000..062a7a72 --- /dev/null +++ b/src/match_hashed.c @@ -0,0 +1,184 @@ +/** + * collectd - src/match_hashed.c + * Copyright (C) 2009 Florian Forster + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; only version 2 of the License is applicable. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * Authors: + * Florian Forster + **/ + +#include "collectd.h" +#include "common.h" +#include "utils_cache.h" +#include "filter_chain.h" + +/* + * private data types + */ +struct mh_hash_match_s +{ + uint32_t match; + uint32_t total; +}; +typedef struct mh_hash_match_s mh_hash_match_t; + +struct mh_match_s; +typedef struct mh_match_s mh_match_t; +struct mh_match_s +{ + mh_hash_match_t *matches; + size_t matches_num; +}; + +/* + * internal helper functions + */ +static int mh_config_match (const oconfig_item_t *ci, /* {{{ */ + mh_match_t *m) +{ + mh_hash_match_t *tmp; + + if ((ci->values_num != 2) + || (ci->values[0].type != OCONFIG_TYPE_NUMBER) + || (ci->values[1].type != OCONFIG_TYPE_NUMBER)) + { + ERROR ("hashed match: The `Match' option requires " + "exactly two numeric arguments."); + return (-1); + } + + if ((ci->values[0].value.number < 0) + || (ci->values[1].value.number < 0)) + { + ERROR ("hashed match: The arguments of the `Match' " + "option must be positive."); + return (-1); + } + + tmp = realloc (m->matches, sizeof (*tmp) * (m->matches_num + 1)); + if (tmp == NULL) + { + ERROR ("hashed match: realloc failed."); + return (-1); + } + m->matches = tmp; + tmp = m->matches + m->matches_num; + + tmp->match = (uint32_t) (ci->values[0].value.number + .5); + tmp->total = (uint32_t) (ci->values[1].value.number + .5); + + if (tmp->match >= tmp->total) + { + ERROR ("hashed match: The first argument of the `Match' option " + "must be smaller than the second argument."); + return (-1); + } + assert (tmp->total != 0); + + m->matches_num++; + return (0); +} /* }}} int mh_config_match */ + +static int mh_create (const oconfig_item_t *ci, void **user_data) /* {{{ */ +{ + mh_match_t *m; + int i; + + m = (mh_match_t *) malloc (sizeof (*m)); + if (m == NULL) + { + ERROR ("mh_create: malloc failed."); + return (-ENOMEM); + } + memset (m, 0, sizeof (*m)); + + for (i = 0; i < ci->children_num; i++) + { + oconfig_item_t *child = ci->children + i; + + if (strcasecmp ("Match", child->key) == 0) + mh_config_match (child, m); + else + ERROR ("hashed match: No such config option: %s", child->key); + } + + if (m->matches_num == 0) + { + sfree (m->matches); + sfree (m); + ERROR ("hashed match: No matches were configured. Not creating match."); + return (-1); + } + + *user_data = m; + return (0); +} /* }}} int mh_create */ + +static int mh_destroy (void **user_data) /* {{{ */ +{ + mh_match_t *mh; + + if ((user_data == NULL) || (*user_data == NULL)) + return (0); + + mh = *user_data; + sfree (mh->matches); + sfree (mh); + + return (0); +} /* }}} int mh_destroy */ + +static int mh_match (const data_set_t __attribute__((unused)) *ds, /* {{{ */ + const value_list_t *vl, + notification_meta_t __attribute__((unused)) **meta, void **user_data) +{ + mh_match_t *m; + uint32_t hash_val; + const char *host_ptr; + size_t i; + + if ((user_data == NULL) || (*user_data == NULL)) + return (-1); + + m = *user_data; + + hash_val = 0; + + for (host_ptr = vl->host; *host_ptr != 0; host_ptr++) + { + /* 251 is the largest prime smaller than 256. */ + hash_val = (hash_val * 251) + ((uint32_t) *host_ptr); + } + DEBUG ("hashed match: host = %s; hash_val = %"PRIu32";", vl->host, hash_val); + + for (i = 0; i < m->matches_num; i++) + if ((hash_val % m->matches[i].total) == m->matches[i].match) + return (FC_MATCH_MATCHES); + + return (FC_MATCH_NO_MATCH); +} /* }}} int mh_match */ + +void module_register (void) +{ + match_proc_t mproc; + + memset (&mproc, 0, sizeof (mproc)); + mproc.create = mh_create; + mproc.destroy = mh_destroy; + mproc.match = mh_match; + fc_register_match ("hashed", mproc); +} /* module_register */ + +/* vim: set sw=2 sts=2 tw=78 et fdm=marker : */ -- 2.30.2