From 31247addc8e371fbfcacce2ad58e98ae43556eb5 Mon Sep 17 00:00:00 2001 From: Denes Matetelki Date: Fri, 22 Sep 2017 17:25:04 +0200 Subject: [PATCH] Updating the varnish plugin --- src/collectd.conf.in | 7 +- src/collectd.conf.pod | 36 ++++- src/types.db | 1 + src/varnish.c | 297 +++++++++++++++++++++++++++++++++++++++++- 4 files changed, 334 insertions(+), 7 deletions(-) diff --git a/src/collectd.conf.in b/src/collectd.conf.in index fa6c96c1..6e24c937 100644 --- a/src/collectd.conf.in +++ b/src/collectd.conf.in @@ -1523,7 +1523,7 @@ # CollectPurge false # Varnish 2 only # CollectSession false # CollectSHM true -# CollectSMA false # Varnish 2 only +# CollectSMA false # Varnish 2 & 4 only # CollectSMS false # CollectSM false # Varnish 2 only # CollectStruct false @@ -1532,6 +1532,11 @@ # CollectVCL false # CollectVSM false # Varnish 4 only # CollectWorkers false +# CollectLock false # Varnish 4 only +# CollectMempool false # Varnish 4 only +# CollectManagement false # Varnish 4 only +# CollectSMF false # Varnish 4 only +# CollectVBE false # Varnish 4 only # # diff --git a/src/collectd.conf.pod b/src/collectd.conf.pod index 379e0838..8b9ef543 100644 --- a/src/collectd.conf.pod +++ b/src/collectd.conf.pod @@ -8300,6 +8300,11 @@ Synopsis: CollectVCL false CollectVSM false CollectWorkers false + CollectLock false + CollectMempool false + CollectManagement false + CollectSMF false + CollectVBE false @@ -8375,7 +8380,10 @@ log messages which is flushed to disk when full. True by default. =item B B|B malloc or umem (umem_alloc(3MALLOC) based) storage statistics. The umem storage -component is Solaris specific. Only available with Varnish 2.x. False by +component is Solaris specific. +Note: SMA and SMF share counters, enable only the one used by the Varnish +instance. +Only available with Varnish 2.x. False by default. =item B B|B @@ -8385,7 +8393,8 @@ component is used internally only. False by default. =item B B|B -file (memory mapped file) storage statistics. Only available with Varnish 2.x. +file (memory mapped file) storage statistics. Only available with Varnish 2.x., +in varnish 4.x. use CollectSMF. False by default. =item B B|B @@ -8416,6 +8425,29 @@ statistics subsystems). Only available with Varnish 4.x. False by default. Collect statistics about worker threads. False by default. +=item B B|B + +Backend counters. Only available with Varnish 4.x. False by default. + +=item B B|B + +file (memory mapped file) storage statistics. Only available with Varnish 4.x. +Note: SMA and SMF share counters, enable only the one used by the Varnish +instance. +Used to be called SM in Varnish 2.x. False by default. + +=item B B|B + +Management process counters. Only available with Varnish 4.x. False by default. + +=item B B|B + +Lock counters. Only available with Varnish 4.x. False by default. + +=item B B|B + +Memory pool counters. Only available with Varnish 4.x. False by default. + =back =head2 Plugin C diff --git a/src/types.db b/src/types.db index 87a82ee0..577d28f1 100644 --- a/src/types.db +++ b/src/types.db @@ -191,6 +191,7 @@ ping value:GAUGE:0:65535 ping_droprate value:GAUGE:0:100 ping_stddev value:GAUGE:0:65535 players value:GAUGE:0:1000000 +pools value:GAUGE:0:U power value:GAUGE:U:U pressure value:GAUGE:0:U protocol_counter value:DERIVE:0:U diff --git a/src/varnish.c b/src/varnish.c index 7c5a9ef0..910d1e77 100644 --- a/src/varnish.c +++ b/src/varnish.c @@ -21,6 +21,7 @@ * Jérôme Renard * Marc Fournier * Florian octo Forster + * Denes Matetelki **/ #include "collectd.h" @@ -69,6 +70,8 @@ struct user_config_s { _Bool collect_sms; #if HAVE_VARNISH_V2 _Bool collect_sm; +#endif +#if HAVE_VARNISH_V2 || HAVE_VARNISH_V4 _Bool collect_sma; #endif _Bool collect_struct; @@ -80,6 +83,11 @@ struct user_config_s { _Bool collect_workers; #if HAVE_VARNISH_V4 _Bool collect_vsm; + _Bool collect_lck; + _Bool collect_mempool; + _Bool collect_mgt; + _Bool collect_smf; + _Bool collect_vbe; #endif }; typedef struct user_config_s user_config_t; /* }}} */ @@ -182,6 +190,14 @@ static int varnish_monitor(void *priv, else if (strcmp(name, "client_req") == 0) return varnish_submit_derive(conf->instance, "connections", "connections", "received", val); +#ifdef HAVE_VARNISH_V4 + else if (strcmp(name, "client_req_400") == 0) + return varnish_submit_derive(conf->instance, "connections", "connections", + "error_400", val); + else if (strcmp(name, "client_req_417") == 0) + return varnish_submit_derive(conf->instance, "connections", "connections", + "error_417", val); +#endif } #ifdef HAVE_VARNISH_V3 @@ -286,6 +302,20 @@ static int varnish_monitor(void *priv, else if (strcmp(name, "fetch_304") == 0) return varnish_submit_derive(conf->instance, "fetch", "http_requests", "no_body_304", val); +#if HAVE_VARNISH_V4 + else if (strcmp(name, "fetch_no_thread") == 0) + return varnish_submit_derive(conf->instance, "fetch", "http_requests", + "no_thread", val); + else if (strcmp(name, "fetch_none") == 0) + return varnish_submit_derive(conf->instance, "fetch", "http_requests", + "none", val); + else if (strcmp(name, "busy_sleep") == 0) + return varnish_submit_derive(conf->instance, "fetch", "http_requests", + "busy_sleep", val); + else if (strcmp(name, "busy_wakeup") == 0) + return varnish_submit_derive(conf->instance, "fetch", "http_requests", + "busy_wakeup", val); +#endif } if (conf->collect_hcb) { @@ -331,6 +361,14 @@ static int varnish_monitor(void *priv, else if (strcmp(name, "n_objoverflow") == 0) return varnish_submit_derive(conf->instance, "objects", "total_objects", "workspace_overflow", val); +#if HAVE_VARNISH_V4 + else if (strcmp(name, "exp_mailed") == 0) + return varnish_submit_derive(conf->instance, "struct", "objects", + "exp_mailed", val); + else if (strcmp(name, "exp_received") == 0) + return varnish_submit_derive(conf->instance, "struct", "objects", + "exp_received", val); +#endif } #if HAVE_VARNISH_V3 @@ -381,6 +419,34 @@ static int varnish_monitor(void *priv, else if (strcmp(name, "bans_dups") == 0) return varnish_submit_derive(conf->instance, "ban", "total_operations", "duplicate", val); + else if (strcmp(name, "bans_tested") == 0) + return varnish_submit_derive(conf->instance, "ban", "total_operations", + "tested", val); + else if (strcmp(name, "bans_lurker_contention") == 0) + return varnish_submit_derive(conf->instance, "ban", "total_operations", + "lurker_contention", val); + else if (strcmp(name, "bans_lurker_obj_killed") == 0) + return varnish_submit_derive(conf->instance, "ban", "total_operations", + "lurker_obj_killed", val); + else if (strcmp(name, "bans_lurker_tested") == 0) + return varnish_submit_derive(conf->instance, "ban", "total_operations", + "lurker_tested", val); + else if (strcmp(name, "bans_lurker_tests_tested") == 0) + return varnish_submit_derive(conf->instance, "ban", "total_operations", + "lurker_tests_tested", val); + else if (strcmp(name, "bans_obj_killed") == 0) + return varnish_submit_derive(conf->instance, "ban", "total_operations", + "obj_killed", val); + else if (strcmp(name, "bans_persisted_bytes") == 0) + return varnish_submit_gauge(conf->instance, "ban", "total_bytes", + "persisted_bytes", val); + else if (strcmp(name, "bans_persisted_fragmentation") == 0) + return varnish_submit_gauge(conf->instance, "ban", "total_bytes", + "persisted_fragmentation", val); + else if (strcmp(name, "bans_tests_tested") == 0) + return varnish_submit_derive(conf->instance, "ban", "total_operations", + "tests_tested", val); + } #endif @@ -415,6 +481,14 @@ static int varnish_monitor(void *priv, else if (strcmp(name, "sess_herd") == 0) return varnish_submit_derive(conf->instance, "session", "total_operations", "herd", val); +#if HAVE_VARNISH_V4 + else if (strcmp(name, "sess_closed_err") == 0) + return varnish_submit_derive(conf->instance, "session", + "total_operations", "closed_err", val); + else if (strcmp(name, "sess_dropped") == 0) + return varnish_submit_derive(conf->instance, "session", + "total_operations", "dropped_for_thread", val); +#endif } if (conf->collect_shm) { @@ -613,6 +687,15 @@ static int varnish_monitor(void *priv, else if (strcmp(name, "n_wrk_lqueue") == 0) return varnish_submit_derive(conf->instance, "workers", "total_requests", "queue_length", val); +#if HAVE_VARNISH_V4 + else if (strcmp(name, "pools") == 0) + return varnish_submit_gauge(conf->instance, "workers", "pools", + "pools", val); + else if (strcmp(name, "busy_killed") == 0) + return varnish_submit_derive(conf->instance, "workers", "http_requests", + "busy_killed", val); + +#endif } #if HAVE_VARNISH_V4 @@ -631,6 +714,159 @@ static int varnish_monitor(void *priv, return varnish_submit_derive(conf->instance, "vsm", "total_bytes", "overflowed", val); } + + if (conf->collect_vbe) { + /* @TODO figure out the collectd type for bitmap + if (strcmp(name, "happy") == 0) + return varnish_submit_derive(conf->instance, "vbe", + "bitmap", "happy_hprobes", val); + */ + if (strcmp(name, "bereq_hdrbytes") == 0) + return varnish_submit_derive(conf->instance, "vbe", + "total_bytes", "bereq_hdrbytes", val); + else if (strcmp(name, "bereq_bodybytes") == 0) + return varnish_submit_derive(conf->instance, "vbe", + "total_bytes", "bereq_bodybytes", val); + else if (strcmp(name, "beresp_hdrbytes") == 0) + return varnish_submit_derive(conf->instance, "vbe", + "total_bytes", "beresp_hdrbytes", val); + else if (strcmp(name, "beresp_bodybytes") == 0) + return varnish_submit_derive(conf->instance, "vbe", + "total_bytes", "beresp_bodybytes", val); + else if (strcmp(name, "pipe_hdrbytes") == 0) + return varnish_submit_derive(conf->instance, "vbe", + "total_bytes", "pipe_hdrbytes", val); + else if (strcmp(name, "pipe_out") == 0) + return varnish_submit_derive(conf->instance, "vbe", + "total_bytes", "pipe_out", val); + else if (strcmp(name, "pipe_in") == 0) + return varnish_submit_derive(conf->instance, "vbe", + "total_bytes", "pipe_in", val); + else if (strcmp(name, "conn") == 0) + return varnish_submit_derive(conf->instance, "vbe", "connections", + "c_conns", val); + else if (strcmp(name, "req") == 0) + return varnish_submit_gauge(conf->instance, "vbe", "http_requests", + "b_reqs", val); + } + + /* All Stevedores support these counters */ + if (conf->collect_sma || conf->collect_smf) { + + char category[4]; + if (conf->collect_sma) + strncpy(category, "sma", 4); + else + strncpy(category, "smf", 4); + + if (strcmp(name, "c_req") == 0) + return varnish_submit_derive(conf->instance, category, + "total_operations", "alloc_req", val); + else if (strcmp(name, "c_fail") == 0) + return varnish_submit_derive(conf->instance, category, + "total_operations", "alloc_fail", val); + else if (strcmp(name, "c_bytes") == 0) + return varnish_submit_derive(conf->instance, category, + "total_bytes", "bytes_allocated", val); + else if (strcmp(name, "c_freed") == 0) + return varnish_submit_derive(conf->instance, category, + "total_bytes", "bytes_freed", val); + else if (strcmp(name, "g_alloc") == 0) + return varnish_submit_gauge(conf->instance, category, + "total_operations", "alloc_outstanding", val); + else if (strcmp(name, "g_bytes") == 0) + return varnish_submit_gauge(conf->instance, category, "bytes", + "bytes_outstanding", val); + else if (strcmp(name, "g_space") == 0) + return varnish_submit_gauge(conf->instance, category, "bytes", + "bytes_available", val); + } + + /* No SMA specific counters */ + + if (conf->collect_smf) { + if (strcmp(name, "g_smf") == 0) + return varnish_submit_gauge(conf->instance, "smf", "objects", + "n_struct_smf", val); + else if (strcmp(name, "g_smf_frag") == 0) + return varnish_submit_gauge(conf->instance, "smf", "objects", + "n_small_free_smf", val); + else if (strcmp(name, "g_smf_large") == 0) + return varnish_submit_gauge(conf->instance, "smf", "objects", + "n_large_free_smf", val); + } + + if (conf->collect_mgt) { + if (strcmp(name, "uptime") == 0) + return varnish_submit_gauge(conf->instance, "mgt", "uptime", + "mgt_proc_uptime", val); + else if (strcmp(name, "child_start") == 0) + return varnish_submit_derive(conf->instance, "mgt", + "total_operations", "child_start", val); + else if (strcmp(name, "child_exit") == 0) + return varnish_submit_derive(conf->instance, "mgt", + "total_operations", "child_exit", val); + else if (strcmp(name, "child_stop") == 0) + return varnish_submit_derive(conf->instance, "mgt", + "total_operations", "child_stop", val); + else if (strcmp(name, "child_died") == 0) + return varnish_submit_derive(conf->instance, "mgt", + "total_operations", "child_died", val); + else if (strcmp(name, "child_dump") == 0) + return varnish_submit_derive(conf->instance, "mgt", + "total_operations", "child_dump", val); + else if (strcmp(name, "child_panic") == 0) + return varnish_submit_derive(conf->instance, "mgt", + "total_operations", "child_panic", val); + } + + if (conf->collect_lck) { + if (strcmp(name, "creat") == 0) + return varnish_submit_derive(conf->instance, "lck", "objects", + "created", val); + else if (strcmp(name, "destroy") == 0) + return varnish_submit_derive(conf->instance, "lck", "objects", + "destroyed", val); + else if (strcmp(name, "locks") == 0) + return varnish_submit_derive(conf->instance, "lck", "total_operations", + "lock_ops", val); + } + + if (conf->collect_mempool) { + if (strcmp(name, "live") == 0) + return varnish_submit_gauge(conf->instance, "mempool", "objects", + "in_use", val); + else if (strcmp(name, "pool") == 0) + return varnish_submit_gauge(conf->instance, "mempool", "objects", + "in_pool", val); + else if (strcmp(name, "sz_wanted") == 0) + return varnish_submit_gauge(conf->instance, "mempool", "bytes", + "size_requested", val); + else if (strcmp(name, "sz_actual") == 0) + return varnish_submit_gauge(conf->instance, "mempool", "bytes", + "size_allocated", val); + else if (strcmp(name, "allocs") == 0) + return varnish_submit_derive(conf->instance, "mempool", + "total_operations", "allocations", val); + else if (strcmp(name, "frees") == 0) + return varnish_submit_derive(conf->instance, "mempool", + "total_operations", "frees", val); + else if (strcmp(name, "recycle") == 0) + return varnish_submit_derive(conf->instance, "mempool", + "objects", "recycled", val); + else if (strcmp(name, "timeout") == 0) + return varnish_submit_derive(conf->instance, "mempool", + "objects", "timed_out", val); + else if (strcmp(name, "toosmall") == 0) + return varnish_submit_derive(conf->instance, "mempool", + "objects", "too_small", val); + else if (strcmp(name, "surplus") == 0) + return varnish_submit_derive(conf->instance, "mempool", + "objects", "surplus", val); + else if (strcmp(name, "randry") == 0) + return varnish_submit_derive(conf->instance, "mempool", + "objects", "ran_dry", val); + } #endif return 0; @@ -1100,6 +1336,8 @@ static int varnish_config_apply_default(user_config_t *conf) /* {{{ */ conf->collect_shm = 1; #if HAVE_VARNISH_V2 conf->collect_sm = 0; +#endif +#if HAVE_VARNISH_V2 || HAVE_VARNISH_V4 conf->collect_sma = 0; #endif conf->collect_sms = 0; @@ -1112,6 +1350,11 @@ static int varnish_config_apply_default(user_config_t *conf) /* {{{ */ conf->collect_workers = 0; #if HAVE_VARNISH_V4 conf->collect_vsm = 0; + conf->collect_lck = 0; + conf->collect_mempool = 0; + conf->collect_mgt = 0; + conf->collect_smf = 0; + conf->collect_vbe = 0; #endif return 0; @@ -1223,11 +1466,11 @@ static int varnish_config_instance(const oconfig_item_t *ci) /* {{{ */ else if (strcasecmp("CollectSMS", child->key) == 0) cf_util_get_boolean(child, &conf->collect_sms); else if (strcasecmp("CollectSMA", child->key) == 0) -#if HAVE_VARNISH_V2 +#if HAVE_VARNISH_V2 || HAVE_VARNISH_V4 cf_util_get_boolean(child, &conf->collect_sma); #else WARNING("Varnish plugin: \"%s\" is available for Varnish %s only.", - child->key, "v2"); + child->key, "v2 and v4"); #endif else if (strcasecmp("CollectSM", child->key) == 0) #if HAVE_VARNISH_V2 @@ -1257,6 +1500,48 @@ static int varnish_config_instance(const oconfig_item_t *ci) /* {{{ */ #else WARNING("Varnish plugin: \"%s\" is available for Varnish %s only.", child->key, "v4"); +#endif + else if (strcasecmp("CollectLock", child->key) == 0) +#if HAVE_VARNISH_V4 + cf_util_get_boolean(child, &conf->collect_lck); +#else + WARNING("Varnish plugin: \"%s\" is available for Varnish %s only.", + child->key, "v4"); +#endif + else if (strcasecmp("CollectMempool", child->key) == 0) +#if HAVE_VARNISH_V4 + cf_util_get_boolean(child, &conf->collect_mempool); +#else + WARNING("Varnish plugin: \"%s\" is available for Varnish %s only.", + child->key, "v4"); +#endif + else if (strcasecmp("CollectManagement", child->key) == 0) +#if HAVE_VARNISH_V4 + cf_util_get_boolean(child, &conf->collect_mgt); +#else + WARNING("Varnish plugin: \"%s\" is available for Varnish %s only.", + child->key, "v4"); +#endif + else if (strcasecmp("CollectSMF", child->key) == 0) +#if HAVE_VARNISH_V4 + cf_util_get_boolean(child, &conf->collect_smf); +#else + WARNING("Varnish plugin: \"%s\" is available for Varnish %s only.", + child->key, "v4"); +#endif + else if (strcasecmp("CollectSMF", child->key) == 0) +#if HAVE_VARNISH_V4 + cf_util_get_boolean(child, &conf->collect_smf); +#else + WARNING("Varnish plugin: \"%s\" is available for Varnish %s only.", + child->key, "v4"); +#endif + else if (strcasecmp("CollectVBE", child->key) == 0) +#if HAVE_VARNISH_V4 + cf_util_get_boolean(child, &conf->collect_vbe); +#else + WARNING("Varnish plugin: \"%s\" is available for Varnish %s only.", + child->key, "v4"); #endif else { WARNING("Varnish plugin: Ignoring unknown " @@ -1280,7 +1565,10 @@ static int varnish_config_instance(const oconfig_item_t *ci) /* {{{ */ #endif && !conf->collect_session && !conf->collect_shm && !conf->collect_sms #if HAVE_VARNISH_V2 - && !conf->collect_sma && !conf->collect_sm + && !conf->collect_sm +#endif +#if HAVE_VARNISH_V2 || HAVE_VARNISH_V4 + && !conf->collect_sma #endif && !conf->collect_struct && !conf->collect_totals #if HAVE_VARNISH_V3 || HAVE_VARNISH_V4 @@ -1288,7 +1576,8 @@ static int varnish_config_instance(const oconfig_item_t *ci) /* {{{ */ #endif && !conf->collect_vcl && !conf->collect_workers #if HAVE_VARNISH_V4 - && !conf->collect_vsm + && !conf->collect_vsm && !conf->collect_vbe && !conf->collect_smf + && !conf->collect_mgt && !conf->collect_lck && !conf->collect_mempool #endif ) { WARNING("Varnish plugin: No metric has been configured for " -- 2.30.2