src/varnish.c

   1 /**
   2  * collectd - src/varnish.c
   3  * Copyright (C) 2010 Jerome Renard
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License as published by the
   7  * Free Software Foundation; only version 2 of the License is applicable.
   8  *
   9  * This program is distributed in the hope that it will be useful, but
  10  * WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  12  * General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License along
  15  * with this program; if not, write to the Free Software Foundation, Inc.,
  16  * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
  17  *
  18  * Authors:
  19  *   Florian octo Forster <octo at verplant.org>
  20  *   Jerome Renard <jerome.renard@gmail.com>
  21  **/
  22
  23 /**
  24  * Current list of what is monitored and what is not monitored (yet)
  25  * {{{
  26  * Field name           Description                           Monitored
  27  * ----------           -----------                           ---------
  28  * uptime               Child uptime                              N
  29  * client_conn          Client connections accepted               Y
  30  * client_drop          Connection dropped, no sess               Y
  31  * client_req           Client requests received                  Y
  32  * cache_hit            Cache hits                                Y
  33  * cache_hitpass        Cache hits for pass                       Y
  34  * cache_miss           Cache misses                              Y
  35  * backend_conn         Backend conn. success                     Y
  36  * backend_unhealthy    Backend conn. not attempted               Y
  37  * backend_busy         Backend conn. too many                    Y
  38  * backend_fail         Backend conn. failures                    Y
  39  * backend_reuse        Backend conn. reuses                      Y
  40  * backend_toolate      Backend conn. was closed                  Y
  41  * backend_recycle      Backend conn. recycles                    Y
  42  * backend_unused       Backend conn. unused                      Y
  43  * fetch_head           Fetch head                                Y
  44  * fetch_length         Fetch with Length                         Y
  45  * fetch_chunked        Fetch chunked                             Y
  46  * fetch_eof            Fetch EOF                                 Y
  47  * fetch_bad            Fetch had bad headers                     Y
  48  * fetch_close          Fetch wanted close                        Y
  49  * fetch_oldhttp        Fetch pre HTTP/1.1 closed                 Y
  50  * fetch_zero           Fetch zero len                            Y
  51  * fetch_failed         Fetch failed                              Y
  52  * n_sess_mem           N struct sess_mem                         N
  53  * n_sess               N struct sess                             N
  54  * n_object             N struct object                           N
  55  * n_vampireobject      N unresurrected objects                   N
  56  * n_objectcore         N struct objectcore                       N
  57  * n_objecthead         N struct objecthead                       N
  58  * n_smf                N struct smf                              N
  59  * n_smf_frag           N small free smf                          N
  60  * n_smf_large          N large free smf                          N
  61  * n_vbe_conn           N struct vbe_conn                         N
  62  * n_wrk                N worker threads                          N
  63  * n_wrk_create         N worker threads created                  N
  64  * n_wrk_failed         N worker threads not created              N
  65  * n_wrk_max            N worker threads limited                  N
  66  * n_wrk_queue          N queued work requests                    N
  67  * n_wrk_overflow       N overflowed work requests                N
  68  * n_wrk_drop           N dropped work requests                   N
  69  * n_backend            N backends                                N
  70  * n_expired            N expired objects                         N
  71  * n_lru_nuked          N LRU nuked objects                       N
  72  * n_lru_saved          N LRU saved objects                       N
  73  * n_lru_moved          N LRU moved objects                       N
  74  * n_deathrow           N objects on deathrow                     N
  75  * losthdr              HTTP header overflows                     N
  76  * n_objsendfile        Objects sent with sendfile                N
  77  * n_objwrite           Objects sent with write                   N
  78  * n_objoverflow        Objects overflowing workspace             N
  79  * s_sess               Total Sessions                            N
  80  * s_req                Total Requests                            N
  81  * s_pipe               Total pipe                                N
  82  * s_pass               Total pass                                N
  83  * s_fetch              Total fetch                               N
  84  * s_hdrbytes           Total header bytes                        N
  85  * s_bodybytes          Total body bytes                          N
  86  * sess_closed          Session Closed                            N
  87  * sess_pipeline        Session Pipeline                          N
  88  * sess_readahead       Session Read Ahead                        N
  89  * sess_linger          Session Linger                            N
  90  * sess_herd            Session herd                              N
  91  * shm_records          SHM records                               Y
  92  * shm_writes           SHM writes                                Y
  93  * shm_flushes          SHM flushes due to overflow               Y
  94  * shm_cont             SHM MTX contention                        Y
  95  * shm_cycles           SHM cycles through buffer                 Y
  96  * sm_nreq              allocator requests                        Y
  97  * sm_nobj              outstanding allocations                   Y
  98  * sm_balloc            bytes allocated                           Y
  99  * sm_bfree             bytes free                                Y
 100  * sma_nreq             SMA allocator requests                    Y
 101  * sma_nobj             SMA outstanding allocations               Y
 102  * sma_nbytes           SMA outstanding bytes                     Y
 103  * sma_balloc           SMA bytes allocated                       Y
 104  * sma_bfree            SMA bytes free                            Y
 105  * sms_nreq             SMS allocator requests                    Y
 106  * sms_nobj             SMS outstanding allocations               Y
 107  * sms_nbytes           SMS outstanding bytes                     Y
 108  * sms_balloc           SMS bytes allocated                       Y
 109  * sms_bfree            SMS bytes freed                           Y
 110  * backend_req          Backend requests made                     N
 111  * n_vcl                N vcl total                               N
 112  * n_vcl_avail          N vcl available                           N
 113  * n_vcl_discard        N vcl discarded                           N
 114  * n_purge              N total active purges                     N
 115  * n_purge_add          N new purges added                        N
 116  * n_purge_retire       N old purges deleted                      N
 117  * n_purge_obj_test     N objects tested                          N
 118  * n_purge_re_test      N regexps tested against                  N
 119  * n_purge_dups         N duplicate purges removed                N
 120  * hcb_nolock           HCB Lookups without lock                  Y
 121  * hcb_lock             HCB Lookups with lock                     Y
 122  * hcb_insert           HCB Inserts                               Y
 123  * esi_parse            Objects ESI parsed (unlock)               Y
 124  * esi_errors           ESI parse errors (unlock)                 Y
 125  * }}}
 126  */
 127 #include "collectd.h"
 128 #include "common.h"
 129 #include "plugin.h"
 130 #include "configfile.h"
 131
 132 #include <varnish/varnishapi.h>
 133
 134 /* {{{ user_config_s */
 135 struct user_config_s {
 136         char *instance;
 137
 138         _Bool collect_cache;
 139         _Bool collect_connections;
 140         _Bool collect_esi;
 141         _Bool collect_backend;
 142         _Bool collect_fetch;
 143         _Bool collect_hcb;
 144         _Bool collect_shm;
 145         _Bool collect_sma;
 146         _Bool collect_sms;
 147         _Bool collect_sm;
 148 };
 149 typedef struct user_config_s user_config_t; /* }}} */
 150
 151 static _Bool have_instance = 0;
 152
 153 static int varnish_submit (const char *plugin_instance, /* {{{ */
 154                 const char *type, const char *type_instance, value_t value)
 155 {
 156         value_list_t vl = VALUE_LIST_INIT;
 157
 158         vl.values = &value;
 159         vl.values_len = 1;
 160
 161         sstrncpy(vl.host, hostname_g, sizeof(vl.host));
 162
 163         sstrncpy(vl.plugin, "varnish", sizeof(vl.plugin));
 164         if (plugin_instance != NULL)
 165                 sstrncpy (vl.plugin_instance, plugin_instance,
 166                                 sizeof (vl.plugin_instance));
 167
 168         sstrncpy(vl.type, type, sizeof(vl.type));
 169         if (type_instance != NULL)
 170                 sstrncpy(vl.type_instance, type_instance,
 171                                 sizeof(vl.type_instance));
 172
 173         return (plugin_dispatch_values(&vl));
 174 } /* }}} int varnish_submit */
 175
 176 static int varnish_submit_gauge (const char *plugin_instance, /* {{{ */
 177                 const char *type, const char *type_instance,
 178                 uint64_t gauge_value)
 179 {
 180         value_t value;
 181
 182         value.gauge = (gauge_t) gauge_value;
 183
 184         return (varnish_submit (plugin_instance, type, type_instance, value));
 185 } /* }}} int varnish_submit_gauge */
 186
 187 static int varnish_submit_derive (const char *plugin_instance, /* {{{ */
 188                 const char *type, const char *type_instance,
 189                 uint64_t derive_value)
 190 {
 191         value_t value;
 192
 193         value.derive = (derive_t) derive_value;
 194
 195         return (varnish_submit (plugin_instance, type, type_instance, value));
 196 } /* }}} int varnish_submit_derive */
 197
 198 static void varnish_monitor(const user_config_t *conf, struct varnish_stats *VSL_stats) /* {{{ */
 199 {
 200         if(conf->collect_cache)
 201         {
 202                 /* Cache hits */
 203                 varnish_submit_derive (conf->instance, "cache_result", "hit"    , VSL_stats->cache_hit);
 204                 /* Cache misses */
 205                 varnish_submit_derive (conf->instance, "cache_result", "miss"   , VSL_stats->cache_miss);
 206                 /* Cache hits for pass */
 207                 varnish_submit_derive (conf->instance, "cache_result", "hitpass", VSL_stats->cache_hitpass);
 208         }
 209
 210         if(conf->collect_connections)
 211         {
 212                 /* Client connections accepted */
 213                 varnish_submit_derive (conf->instance, "connections", "client-accepted", VSL_stats->client_conn);
 214                 /* Connection dropped, no sess */
 215                 varnish_submit_derive (conf->instance, "connections", "client-dropped" , VSL_stats->client_drop);
 216                 /* Client requests received    */
 217                 varnish_submit_derive (conf->instance, "connections", "client-received", VSL_stats->client_req);
 218         }
 219
 220         if(conf->collect_esi)
 221         {
 222                 /* Objects ESI parsed (unlock) */
 223                 varnish_submit_derive (conf->instance, "total_operations", "esi-parsed", VSL_stats->esi_parse);
 224                 /* ESI parse errors (unlock)   */
 225                 varnish_submit_derive (conf->instance, "total_operations", "esi-error", VSL_stats->esi_errors);
 226         }
 227
 228         if(conf->collect_backend)
 229         {
 230                 /* Backend conn. success       */
 231                 varnish_submit_derive (conf->instance, "connections", "backend-success"      , VSL_stats->backend_conn);
 232                 /* Backend conn. not attempted */
 233                 varnish_submit_derive (conf->instance, "connections", "backend-not-attempted", VSL_stats->backend_unhealthy);
 234                 /* Backend conn. too many      */
 235                 varnish_submit_derive (conf->instance, "connections", "backend-too-many"     , VSL_stats->backend_busy);
 236                 /* Backend conn. failures      */
 237                 varnish_submit_derive (conf->instance, "connections", "backend-failures"     , VSL_stats->backend_fail);
 238                 /* Backend conn. reuses        */
 239                 varnish_submit_derive (conf->instance, "connections", "backend-reuses"       , VSL_stats->backend_reuse);
 240                 /* Backend conn. was closed    */
 241                 varnish_submit_derive (conf->instance, "connections", "backend-was-closed"   , VSL_stats->backend_toolate);
 242                 /* Backend conn. recycles      */
 243                 varnish_submit_derive (conf->instance, "connections", "backend-recycled"     , VSL_stats->backend_recycle);
 244                 /* Backend conn. unused        */
 245                 varnish_submit_derive (conf->instance, "connections", "backend-unused"       , VSL_stats->backend_unused);
 246         }
 247
 248         if(conf->collect_fetch)
 249         {
 250                 /* Fetch head                */
 251                 varnish_submit_derive (conf->instance, "http_requests", "fetch-head"       , VSL_stats->fetch_head);
 252                 /* Fetch with length         */
 253                 varnish_submit_derive (conf->instance, "http_requests", "fetch-length"     , VSL_stats->fetch_length);
 254                 /* Fetch chunked             */
 255                 varnish_submit_derive (conf->instance, "http_requests", "fetch-chunked"    , VSL_stats->fetch_chunked);
 256                 /* Fetch EOF                 */
 257                 varnish_submit_derive (conf->instance, "http_requests", "fetch-eof"        , VSL_stats->fetch_eof);
 258                 /* Fetch bad headers         */
 259                 varnish_submit_derive (conf->instance, "http_requests", "fetch-bad_headers", VSL_stats->fetch_bad);
 260                 /* Fetch wanted close        */
 261                 varnish_submit_derive (conf->instance, "http_requests", "fetch-close"      , VSL_stats->fetch_close);
 262                 /* Fetch pre HTTP/1.1 closed */
 263                 varnish_submit_derive (conf->instance, "http_requests", "fetch-oldhttp"    , VSL_stats->fetch_oldhttp);
 264                 /* Fetch zero len            */
 265                 varnish_submit_derive (conf->instance, "http_requests", "fetch-zero"       , VSL_stats->fetch_zero);
 266                 /* Fetch failed              */
 267                 varnish_submit_derive (conf->instance, "http_requests", "fetch-failed"     , VSL_stats->fetch_failed);
 268         }
 269
 270         if(conf->collect_hcb)
 271         {
 272                 /* HCB Lookups without lock */
 273                 varnish_submit_derive (conf->instance, "cache_operation", "lookup_nolock", VSL_stats->hcb_nolock);
 274                 /* HCB Lookups with lock    */
 275                 varnish_submit_derive (conf->instance, "cache_operation", "lookup_lock",   VSL_stats->hcb_lock);
 276                 /* HCB Inserts              */
 277                 varnish_submit_derive (conf->instance, "cache_operation", "insert",        VSL_stats->hcb_insert);
 278         }
 279
 280         if(conf->collect_shm)
 281         {
 282                 /* SHM records                 */
 283                 varnish_submit_derive (conf->instance, "total_operations", "shmlog-records"   , VSL_stats->shm_records);
 284                 /* SHM writes                  */
 285                 varnish_submit_derive (conf->instance, "total_operations", "shmlog-writes"    , VSL_stats->shm_writes);
 286                 /* SHM flushes due to overflow */
 287                 varnish_submit_derive (conf->instance, "total_operations", "shmlog-flushes"   , VSL_stats->shm_flushes);
 288                 /* SHM MTX contention          */
 289                 varnish_submit_derive (conf->instance, "total_operations", "shmlog-contention", VSL_stats->shm_cont);
 290                 /* SHM cycles through buffer   */
 291                 varnish_submit_derive (conf->instance, "total_operations", "shmlog-cycles"    , VSL_stats->shm_cycles);
 292         }
 293
 294         if(conf->collect_sm)
 295         {
 296                 /* allocator requests */
 297                 varnish_submit_derive (conf->instance, "total_requests", "storage-file",      VSL_stats->sm_nreq);
 298                 /* outstanding allocations */
 299                 varnish_submit_gauge (conf->instance, "requests", "storage-file-outstanding", VSL_stats->sm_nobj);
 300                 /* bytes allocated */
 301                 varnish_submit_gauge (conf->instance, "bytes", "storage-file-allocated",      VSL_stats->sm_balloc);
 302                 /* bytes free */
 303                 varnish_submit_gauge (conf->instance, "bytes", "storage-file-free",           VSL_stats->sm_bfree);
 304         }
 305
 306         if(conf->collect_sma)
 307         {
 308                 /* SMA allocator requests */
 309                 varnish_submit_derive (conf->instance, "total_requests", "storage-mem",      VSL_stats->sma_nreq);
 310                 /* SMA outstanding allocations */
 311                 varnish_submit_gauge (conf->instance, "requests", "storage-mem-outstanding", VSL_stats->sma_nobj);
 312                 /* SMA outstanding bytes */
 313                 varnish_submit_gauge (conf->instance, "bytes", "storage-mem-outstanding",    VSL_stats->sma_nbytes);
 314                 /* SMA bytes allocated */
 315                 varnish_submit_gauge (conf->instance, "bytes", "storage-mem-allocated",      VSL_stats->sma_balloc);
 316                 /* SMA bytes free */
 317                 varnish_submit_gauge (conf->instance, "bytes", "storage-mem-free" ,          VSL_stats->sma_bfree);
 318         }
 319
 320         if(conf->collect_sms)
 321         {
 322                 /* SMS allocator requests */
 323                 varnish_submit_derive (conf->instance, "total_requests", "storage-synth",      VSL_stats->sms_nreq);
 324                 /* SMS outstanding allocations */
 325                 varnish_submit_gauge (conf->instance, "requests", "storage-synth-outstanding", VSL_stats->sms_nobj);
 326                 /* SMS outstanding bytes */
 327                 varnish_submit_gauge (conf->instance, "bytes", "storage-synth-outstanding",    VSL_stats->sms_nbytes);
 328                 /* SMS bytes allocated */
 329                 varnish_submit_gauge (conf->instance, "bytes", "storage-synth-allocated",      VSL_stats->sms_balloc);
 330                 /* SMS bytes freed */
 331                 varnish_submit_gauge (conf->instance, "bytes", "storage-synth-free",           VSL_stats->sms_bfree);
 332         }
 333 } /* }}} void varnish_monitor */
 334
 335 static int varnish_read(user_data_t *ud) /* {{{ */
 336 {
 337         struct varnish_stats *VSL_stats;
 338         user_config_t *conf;
 339
 340         if ((ud == NULL) || (ud->data == NULL))
 341                 return (EINVAL);
 342
 343         conf = ud->data;
 344
 345         VSL_stats = VSL_OpenStats(conf->instance);
 346         if (VSL_stats == NULL)
 347         {
 348                 ERROR("Varnish plugin : unable to load statistics");
 349
 350                 return (-1);
 351         }
 352
 353         varnish_monitor(conf, VSL_stats);
 354
 355     return (0);
 356 } /* }}} */
 357
 358 static void varnish_config_free (void *ptr) /* {{{ */
 359 {
 360         user_config_t *conf = ptr;
 361
 362         if (conf == NULL)
 363                 return;
 364
 365         sfree (conf->instance);
 366         sfree (conf);
 367 } /* }}} */
 368
 369 static int varnish_init (void) /* {{{ */
 370 {
 371         user_config_t *conf;
 372         user_data_t ud;
 373
 374         if (have_instance)
 375                 return (0);
 376
 377         conf = malloc (sizeof (*conf));
 378         if (conf == NULL)
 379                 return (ENOMEM);
 380         memset (conf, 0, sizeof (*conf));
 381
 382         /* Default settings: */
 383         conf->instance = NULL;
 384         conf->collect_cache = 1;
 385         conf->collect_backend = 1;
 386         conf->collect_connections = 1;
 387         conf->collect_shm = 1;
 388
 389         ud.data = conf;
 390         ud.free_func = varnish_config_free;
 391
 392         plugin_register_complex_read (/* group = */ "varnish",
 393                         /* name      = */ "varnish/localhost",
 394                         /* callback  = */ varnish_read,
 395                         /* interval  = */ NULL,
 396                         /* user data = */ &ud);
 397
 398         return (0);
 399 } /* }}} int varnish_init */
 400
 401 static int varnish_config_instance (const oconfig_item_t *ci) /* {{{ */
 402 {
 403         user_config_t *conf;
 404         user_data_t ud;
 405         char callback_name[DATA_MAX_NAME_LEN];
 406         int i;
 407
 408         conf = malloc (sizeof (*conf));
 409         if (conf == NULL)
 410                 return (ENOMEM);
 411         memset (conf, 0, sizeof (*conf));
 412         conf->instance = NULL;
 413
 414         if (ci->values_num == 1)
 415         {
 416                 int status;
 417
 418                 status = cf_util_get_string (ci, &conf->instance);
 419                 if (status != 0)
 420                 {
 421                         sfree (conf);
 422                         return (status);
 423                 }
 424                 assert (conf->instance != NULL);
 425
 426                 if (strcmp ("localhost", conf->instance) == 0)
 427                 {
 428                         sfree (conf->instance);
 429                         conf->instance = NULL;
 430                 }
 431         }
 432         else if (ci->values_num > 1)
 433         {
 434                 WARNING ("Varnish plugin: \"Instance\" blocks accept only "
 435                                 "one argument.");
 436                 return (EINVAL);
 437         }
 438
 439         for (i = 0; i < ci->children_num; i++)
 440         {
 441                 oconfig_item_t *child = ci->children + i;
 442
 443                 if (strcasecmp ("CollectCache", child->key) == 0)
 444                         cf_util_get_boolean (child, &conf->collect_cache);
 445                 else if (strcasecmp ("CollectConnections", child->key) == 0)
 446                         cf_util_get_boolean (child, &conf->collect_connections);
 447                 else if (strcasecmp ("CollectESI", child->key) == 0)
 448                         cf_util_get_boolean (child, &conf->collect_esi);
 449                 else if (strcasecmp ("CollectBackend", child->key) == 0)
 450                         cf_util_get_boolean (child, &conf->collect_backend);
 451                 else if (strcasecmp ("CollectFetch", child->key) == 0)
 452                         cf_util_get_boolean (child, &conf->collect_fetch);
 453                 else if (strcasecmp ("CollectHCB", child->key) == 0)
 454                         cf_util_get_boolean (child, &conf->collect_hcb);
 455                 else if (strcasecmp ("CollectSHM", child->key) == 0)
 456                         cf_util_get_boolean (child, &conf->collect_shm);
 457                 else if (strcasecmp ("CollectSMA", child->key) == 0)
 458                         cf_util_get_boolean (child, &conf->collect_sma);
 459                 else if (strcasecmp ("CollectSMS", child->key) == 0)
 460                         cf_util_get_boolean (child, &conf->collect_sms);
 461                 else if (strcasecmp ("CollectSM", child->key) == 0)
 462                         cf_util_get_boolean (child, &conf->collect_sm);
 463                 else
 464                 {
 465                         WARNING ("Varnish plugin: Ignoring unknown "
 466                                         "configuration option: \"%s\"",
 467                                         child->key);
 468                 }
 469         }
 470
 471         if (!conf->collect_cache
 472                         && !conf->collect_connections
 473                         && !conf->collect_esi
 474                         && !conf->collect_backend
 475                         && !conf->collect_fetch
 476                         && !conf->collect_hcb
 477                         && !conf->collect_shm
 478                         && !conf->collect_sma
 479                         && !conf->collect_sms
 480                         && !conf->collect_sm)
 481         {
 482                 WARNING ("Varnish plugin: No metric has been configured for "
 483                                 "instance \"%s\". Disabling this instance.",
 484                                 (conf->instance == NULL) ? "localhost" : conf->instance);
 485                 return (EINVAL);
 486         }
 487
 488         ssnprintf (callback_name, sizeof (callback_name), "varnish/%s",
 489                         (conf->instance == NULL) ? "localhost" : conf->instance);
 490
 491         ud.data = conf;
 492         ud.free_func = varnish_config_free;
 493
 494         plugin_register_complex_read (/* group = */ "varnish",
 495                         /* name      = */ callback_name,
 496                         /* callback  = */ varnish_read,
 497                         /* interval  = */ NULL,
 498                         /* user data = */ &ud);
 499
 500         have_instance = 1;
 501
 502         return (0);
 503 } /* }}} int varnish_config_instance */
 504
 505 static int varnish_config (oconfig_item_t *ci) /* {{{ */
 506 {
 507         int i;
 508
 509         for (i = 0; i < ci->children_num; i++)
 510         {
 511                 oconfig_item_t *child = ci->children + i;
 512
 513                 if (strcasecmp ("Instance", child->key) == 0)
 514                         varnish_config_instance (child);
 515                 else
 516                 {
 517                         WARNING ("Varnish plugin: Ignoring unknown "
 518                                         "configuration option: \"%s\"",
 519                                         child->key);
 520                 }
 521         }
 522
 523         return (0);
 524 } /* }}} int varnish_config */
 525
 526 void module_register (void) /* {{{ */
 527 {
 528         plugin_register_complex_config("varnish", varnish_config);
 529         plugin_register_init ("varnish", varnish_init);
 530 } /* }}} */
 531
 532 /* vim: set sw=8 noet fdm=marker : */