Code

Introduce the DERIVE and ABSOLUTE data source types.
authorMariusz Gronczewski <xani666@gmail.com>
Wed, 1 Jul 2009 07:14:29 +0000 (09:14 +0200)
committerFlorian Forster <octo@noris.net>
Wed, 1 Jul 2009 07:17:27 +0000 (09:17 +0200)
Hi,

i've updated my patch to 4.7.0, most of "data input" plugins (curl, java, exec,
perl, tail, couchdb) should work with derive. In case of couchdb and curl, if u
use absolute DS you can only "Set", no "Inc" or "Add" coz obviously that
wouldn't make much sense with it. Other plugins can be "enabled" globally to
use derive by changing "COUNTER" to "DERIVE" in types.db but that way is ugly
(but makes sense in some cases, like when u have lot of tunnels or ppp
interfaces) and either needs converting or recreating rrd files.

Regards
Mariusz

---

Hi,

ive been running my patch with 4.7.1, found a minor bug, but after repairing
that i didnt had any problems with it on my servers, im including patch
(against 4.7.1 from webpage),

Regards,
XANi

16 files changed:
src/common.c
src/csv.c
src/curl.c
src/java.c
src/libcollectdclient/client.c
src/libcollectdclient/client.h
src/perl.c
src/plugin.h
src/rrdcached.c
src/rrdtool.c
src/tail.c
src/types.db
src/types_list.c
src/utils_match.c
src/utils_match.h
src/utils_rrdcreate.c

index d42453e444f1e5f484ef459a9c6d72b031e0a3a2..8de871bd70a362b4066a2a865019eeb76900a0f1 100644 (file)
@@ -845,6 +845,10 @@ int parse_value (const char *value, value_t *ret_value, const data_source_t ds)
                ret_value->counter = (counter_t)strtoll (value, &endptr, 0);
        else if (DS_TYPE_GAUGE == ds.type)
                ret_value->gauge = (gauge_t)strtod (value, &endptr);
+       else if (DS_TYPE_DERIVE == ds.type)
+               ret_value->counter = (derive_t)strtoll (value, &endptr, 0);
+       else if (DS_TYPE_ABSOLUTE == ds.type)
+               ret_value->counter = (absolute_t)strtoll (value, &endptr, 0);
        else {
                ERROR ("parse_value: Invalid data source \"%s\" "
                                "(type = %i).", ds.name, ds.type);
index 5c43b8a87421b312216281aa4562fe47b11d7703..f38599dc7b32be95dbad169a98822fabbeff286e 100644 (file)
--- a/src/csv.c
+++ b/src/csv.c
@@ -61,7 +61,9 @@ static int value_list_to_string (char *buffer, int buffer_len,
        for (i = 0; i < ds->ds_num; i++)
        {
                if ((ds->ds[i].type != DS_TYPE_COUNTER)
-                               && (ds->ds[i].type != DS_TYPE_GAUGE))
+                               && (ds->ds[i].type != DS_TYPE_GAUGE)
+                               && (ds->ds[i].type != DS_TYPE_DERIVE)
+                               && (ds->ds[i].type != DS_TYPE_ABSOLUTE))
                        return (-1);
 
                if (ds->ds[i].type == DS_TYPE_COUNTER)
@@ -88,10 +90,58 @@ static int value_list_to_string (char *buffer, int buffer_len,
                                                ",%lf", rates[i]);
                        }
                }
-               else /* if (ds->ds[i].type == DS_TYPE_GAUGE) */
+               else  if (ds->ds[i].type == DS_TYPE_GAUGE) 
                {
                        status = ssnprintf (buffer + offset, buffer_len - offset,
                                        ",%lf", vl->values[i].gauge);
+               } 
+               else if (ds->ds[i].type == DS_TYPE_DERIVE)
+               {
+                       if (store_rates == 0)
+                       {
+                               status = ssnprintf (buffer + offset,
+                                               buffer_len - offset,
+                                               ",%llu",
+                                               vl->values[i].derive);
+                       }
+                       else /* if (store_rates == 1) */
+                       {
+                               if (rates == NULL)
+                                       rates = uc_get_rate (ds, vl);
+                               if (rates == NULL)
+                               {
+                                       WARNING ("csv plugin: "
+                                                       "uc_get_rate failed.");
+                                       return (-1);
+                               }
+                               status = ssnprintf (buffer + offset,
+                                               buffer_len - offset,
+                                               ",%lf", rates[i]);
+                       }
+               }
+               else if (ds->ds[i].type == DS_TYPE_ABSOLUTE)
+               {
+                       if (store_rates == 0)
+                       {
+                               status = ssnprintf (buffer + offset,
+                                               buffer_len - offset,
+                                               ",%llu",
+                                               vl->values[i].absolute);
+                       }
+                       else /* if (store_rates == 1) */
+                       {
+                               if (rates == NULL)
+                                       rates = uc_get_rate (ds, vl);
+                               if (rates == NULL)
+                               {
+                                       WARNING ("csv plugin: "
+                                                       "uc_get_rate failed.");
+                                       return (-1);
+                               }
+                               status = ssnprintf (buffer + offset,
+                                               buffer_len - offset,
+                                               ",%lf", rates[i]);
+                       }
                }
 
                if ((status < 1) || (status >= (buffer_len - offset)))
index 88b5496d4a4b5f317fc3a0ead52cdaf3aec9c5f3..41f11f614610fe32842bd45f051e2ae4a664cab4 100644 (file)
@@ -222,6 +222,29 @@ static int cc_config_add_match_dstype (int *dstype_ret, /* {{{ */
     else
       dstype = 0;
   }
+else if (strncasecmp ("Derive", ci->values[0].value.string,
+        strlen ("Derive")) == 0)
+  {
+    dstype = UTILS_MATCH_DS_TYPE_DERIVE;
+    if (strcasecmp ("DeriveSet", ci->values[0].value.string) == 0)
+      dstype |= UTILS_MATCH_CF_DERIVE_SET;
+    else if (strcasecmp ("DeriveAdd", ci->values[0].value.string) == 0)
+      dstype |= UTILS_MATCH_CF_DERIVE_ADD;
+    else if (strcasecmp ("DeriveInc", ci->values[0].value.string) == 0)
+      dstype |= UTILS_MATCH_CF_DERIVE_INC;
+    else
+      dstype = 0;
+  }
+else if (strncasecmp ("Absolute", ci->values[0].value.string,
+        strlen ("Absolute")) == 0)
+  {
+    dstype = UTILS_MATCH_DS_TYPE_ABSOLUTE;
+    if (strcasecmp ("AbsoluteSet", ci->values[0].value.string) == 0) /* Absolute DS is reset-on-read so no sense doin anything else but set */
+      dstype |= UTILS_MATCH_CF_ABSOLUTE_SET;
+    else
+      dstype = 0;
+  }
+
   else
   {
     dstype = 0;
index 20523f90b464f674b27b59944ac97edd81fa34af..64e51d319a596a678e165724d638dff80595fd4f 100644 (file)
@@ -284,6 +284,10 @@ static jobject ctoj_value_to_number (JNIEnv *jvm_env, /* {{{ */
     return (ctoj_jlong_to_number (jvm_env, (jlong) value.counter));
   else if (ds_type == DS_TYPE_GAUGE)
     return (ctoj_jdouble_to_number (jvm_env, (jdouble) value.gauge));
+  if (ds_type == DS_TYPE_DERIVE)
+    return (ctoj_jlong_to_number (jvm_env, (jlong) value.derive));
+  if (ds_type == DS_TYPE_ABSOLUTE)
+    return (ctoj_jlong_to_number (jvm_env, (jlong) value.absolute));
   else
     return (NULL);
 } /* }}} jobject ctoj_value_to_number */
index 501c729c143e53227a6256ace249295103136650..bb7ee95628506b99470fc99885e7009433978703 100644 (file)
@@ -792,6 +792,11 @@ int lcc_putval (lcc_connection_t *c, const lcc_value_list_t *vl) /* {{{ */
       else
         SSTRCATF (command, ":%g", vl->values[i].gauge);
     }
+    else if (vl->values_types[i] == LCC_TYPE_DERIVE)
+       SSTRCATF (command, ":%"PRIu64, vl->values[i].derive);
+    else if (vl->values_types[i] == LCC_TYPE_ABSOLUTE)
+       SSTRCATF (command, ":%"PRIu64, vl->values[i].absolute);
+
   } /* for (i = 0; i < vl->values_len; i++) */
 
   status = lcc_sendreceive (c, command, &res);
index d5371fb2a20f8f6a1c80b3401fefb0b69f02aeab..b0d092d0d8c9ab3534249242279a1c43f43f94fd 100644 (file)
  */
 #define LCC_TYPE_COUNTER 0
 #define LCC_TYPE_GAUGE   1
+#define LCC_TYPE_DERIVE   2
+#define LCC_TYPE_ABSOLUTE   3
 
 LCC_BEGIN_DECLS
 
 typedef uint64_t counter_t;
 typedef double gauge_t;
+typedef uint64_t derive_t;
+typedef uint64_t absolute_t;
 
 union value_u
 {
   counter_t counter;
   gauge_t   gauge;
+  derive_t  derive;
+  absolute_t absolute;
 };
 typedef union value_u value_t;
 
index b6e7b22d27bcf724603d7f85f9a6a86da6ed99d4..dd82ed9487d3531afe131463a6ba7f6074704e60 100644 (file)
@@ -194,6 +194,8 @@ struct {
        { "Collectd::TYPE_DATASET",       PLUGIN_DATASET },
        { "Collectd::DS_TYPE_COUNTER",    DS_TYPE_COUNTER },
        { "Collectd::DS_TYPE_GAUGE",      DS_TYPE_GAUGE },
+       { "Collectd::DS_TYPE_DERIVE",     DS_TYPE_DERIVE },
+       { "Collectd::DS_TYPE_ABSOLUTE",   DS_TYPE_ABSOLUTE },
        { "Collectd::LOG_ERR",            LOG_ERR },
        { "Collectd::LOG_WARNING",        LOG_WARNING },
        { "Collectd::LOG_NOTICE",         LOG_NOTICE },
@@ -267,7 +269,7 @@ static int hv2data_source (pTHX_ HV *hash, data_source_t *ds)
        if (NULL != (tmp = hv_fetch (hash, "type", 4, 0))) {
                ds->type = SvIV (*tmp);
 
-               if ((DS_TYPE_COUNTER != ds->type) && (DS_TYPE_GAUGE != ds->type)) {
+               if ((DS_TYPE_COUNTER != ds->type) && (DS_TYPE_GAUGE != ds->type) && (DS_TYPE_DERIVE != ds->type) && (DS_TYPE_ABSOLUTE != ds->type)) {
                        log_err ("hv2data_source: Invalid DS type.");
                        return -1;
                }
@@ -320,8 +322,12 @@ static int av2value (pTHX_ char *name, AV *array, value_t *value, int len)
                if (NULL != tmp) {
                        if (DS_TYPE_COUNTER == ds->ds[i].type)
                                value[i].counter = SvIV (*tmp);
-                       else
+                       else if (DS_TYPE_GAUGE == ds->ds[i].type)
                                value[i].gauge = SvNV (*tmp);
+                       else if (DS_TYPE_DERIVE == ds->ds[i].type)
+                               value[i].derive = SvIV (*tmp);
+                       else if (DS_TYPE_ABSOLUTE == ds->ds[i].type)
+                               value[i].absolute = SvIV (*tmp);
                }
                else {
                        return -1;
@@ -637,8 +643,12 @@ static int value_list2hv (pTHX_ value_list_t *vl, data_set_t *ds, HV *hash)
 
                if (DS_TYPE_COUNTER == ds->ds[i].type)
                        val = newSViv (vl->values[i].counter);
-               else
+               else if (DS_TYPE_GAUGE == ds->ds[i].type)
                        val = newSVnv (vl->values[i].gauge);
+               else if (DS_TYPE_DERIVE == ds->ds[i].type)
+                       val = newSVnv (vl->values[i].derive);
+               else if (DS_TYPE_ABSOLUTE == ds->ds[i].type)
+                       val = newSVnv (vl->values[i].absolute);
 
                if (NULL == av_store (values, i, val)) {
                        av_undef (values);
index e8e49567147e70becc9209692cfd93e8d54f47ba..fd741501d8cc9724f7889a75ebda9c931486377b 100644 (file)
@@ -30,6 +30,8 @@
 
 #define DS_TYPE_COUNTER 0
 #define DS_TYPE_GAUGE   1
+#define DS_TYPE_DERIVE   2
+#define DS_TYPE_ABSOLUTE   3
 
 #ifndef LOG_ERR
 # define LOG_ERR 3
  */
 typedef unsigned long long counter_t;
 typedef double gauge_t;
+typedef unsigned long long derive_t;
+typedef unsigned long long absolute_t;
 
 union value_u
 {
        counter_t counter;
        gauge_t   gauge;
+       derive_t   derive;
+       absolute_t   absolute;
 };
 typedef union value_u value_t;
 
index 326a8898622179c1ca741ba3d0f710f3a828d161..6295a53f79d39b67670de86066495790766a08fe 100644 (file)
@@ -75,7 +75,9 @@ static int value_list_to_string (char *buffer, int buffer_len,
   for (i = 0; i < ds->ds_num; i++)
   {
     if ((ds->ds[i].type != DS_TYPE_COUNTER)
-        && (ds->ds[i].type != DS_TYPE_GAUGE))
+        && (ds->ds[i].type != DS_TYPE_GAUGE)
+       && (ds->ds[i].type != DS_TYPE_DERIVE)
+       && (ds->ds[i].type != DS_TYPE_ABSOLUTE)
       return (-1);
 
     if (ds->ds[i].type == DS_TYPE_COUNTER)
@@ -83,11 +85,20 @@ static int value_list_to_string (char *buffer, int buffer_len,
       status = ssnprintf (buffer + offset, buffer_len - offset,
           ":%llu", vl->values[i].counter);
     }
-    else /* if (ds->ds[i].type == DS_TYPE_GAUGE) */
+    else if (ds->ds[i].type == DS_TYPE_GAUGE) 
     {
       status = ssnprintf (buffer + offset, buffer_len - offset,
           ":%lf", vl->values[i].gauge);
     }
+    else if (ds->ds[i].type == DS_TYPE_DERIVE) {
+      status = ssnprintf (buffer + offset, buffer_len - offset,
+         ":%llu", vl->values[i].derive);
+    }
+    else /* if (ds->ds[i].type == DS_TYPE_ABSOLUTE) */ {
+      status = ssnprintf (buffer + offset, buffer_len - offset,
+         ":%llu", vl->values[i].absolute);
+    }
 
     if ((status < 1) || (status >= (buffer_len - offset)))
       return (-1);
index 243a8c8ff90feff941e5d20e179f63ef2e15f815..a361501d0ff1a8e596143d2326684b3d409d86fb 100644 (file)
@@ -204,15 +204,23 @@ static int value_list_to_string (char *buffer, int buffer_len,
        for (i = 0; i < ds->ds_num; i++)
        {
                if ((ds->ds[i].type != DS_TYPE_COUNTER)
-                               && (ds->ds[i].type != DS_TYPE_GAUGE))
+                               && (ds->ds[i].type != DS_TYPE_GAUGE)
+                               && (ds->ds[i].type != DS_TYPE_DERIVE)
+                               && (ds->ds[i].type != DS_TYPE_ABSOLUTE))
                        return (-1);
 
                if (ds->ds[i].type == DS_TYPE_COUNTER)
                        status = ssnprintf (buffer + offset, buffer_len - offset,
                                        ":%llu", vl->values[i].counter);
-               else
+               else if (ds->ds[i].type == DS_TYPE_GAUGE)
                        status = ssnprintf (buffer + offset, buffer_len - offset,
                                        ":%lf", vl->values[i].gauge);
+               else if (ds->ds[i].type == DS_TYPE_DERIVE)
+                       status = ssnprintf (buffer + offset, buffer_len - offset,
+                                       ":%llu", vl->values[i].derive);
+               else /*if (ds->ds[i].type == DS_TYPE_ABSOLUTE) */
+                       status = ssnprintf (buffer + offset, buffer_len - offset,
+                                       ":%llu", vl->values[i].absolute);
 
                if ((status < 1) || (status >= (buffer_len - offset)))
                        return (-1);
index 02afd7910af42ed75229020b5c5c0ed0b85b8473..8becc05bc4a86cfcdda73195775ed7a1d5bdc76e 100644 (file)
@@ -101,6 +101,26 @@ static int ctail_config_add_match_dstype (ctail_config_match_t *cm,
     else
       cm->flags = 0;
   }
+  else if (strncasecmp ("Derive", ci->values[0].value.string, strlen ("Derive")) == 0)
+  {
+    cm->flags = UTILS_MATCH_DS_TYPE_DERIVE;
+    if (strcasecmp ("DeriveSet", ci->values[0].value.string) == 0)
+      cm->flags |= UTILS_MATCH_CF_DERIVE_SET;
+    else if (strcasecmp ("DeriveAdd", ci->values[0].value.string) == 0)
+      cm->flags |= UTILS_MATCH_CF_DERIVE_ADD;
+    else if (strcasecmp ("DeriveInc", ci->values[0].value.string) == 0)
+      cm->flags |= UTILS_MATCH_CF_DERIVE_INC;
+    else
+      cm->flags = 0;
+  }
+  else if (strncasecmp ("Absolute", ci->values[0].value.string, strlen ("Absolute")) == 0)
+  {
+    cm->flags = UTILS_MATCH_DS_TYPE_ABSOLUTE;
+    if (strcasecmp ("AbsoluteSet", ci->values[0].value.string) == 0)
+      cm->flags |= UTILS_MATCH_CF_ABSOLUTE_SET;
+    else
+      cm->flags = 0;
+  }
   else
   {
     cm->flags = 0;
index 7028fe7fa9187ae38db440dd4a9af3518e70569d..83e7007dd90627f501b670d4809c1afd1a5ed701 100644 (file)
@@ -1,3 +1,4 @@
+absolute               count:ABSOLUTE:0:U
 apache_bytes           count:COUNTER:0:134217728
 apache_connections     count:GAUGE:0:65535
 apache_requests                count:COUNTER:0:134217728
@@ -15,6 +16,7 @@ cpufreq                       value:GAUGE:0:U
 cpu                    value:COUNTER:0:4294967295
 current                        value:GAUGE:U:U
 delay                  seconds:GAUGE:-1000000:1000000
+derive                 value:DERIVE:0:U
 df                     used:GAUGE:0:1125899906842623, free:GAUGE:0:1125899906842623
 disk_merged            read:COUNTER:0:4294967295, write:COUNTER:0:4294967295
 disk_octets            read:COUNTER:0:17179869183, write:COUNTER:0:17179869183
index ed832fee0baf7721a1de9ab5610741d283882332..49714176374a2979a24e5c161ac6b303d184708f 100644 (file)
@@ -68,9 +68,13 @@ static int parse_ds (data_source_t *dsrc, char *buf, size_t buf_len)
     dsrc->type = DS_TYPE_GAUGE;
   else if (strcasecmp (fields[1], "COUNTER") == 0)
     dsrc->type = DS_TYPE_COUNTER;
+  else if (strcasecmp (fields[1], "DERIVE") == 0)
+    dsrc->type = DS_TYPE_DERIVE;
+  else if (strcasecmp (fields[1], "ABSOLUTE") == 0)
+    dsrc->type = DS_TYPE_ABSOLUTE;
   else
   {
-    ERROR ("(fields[1] = %s) != (GAUGE || COUNTER)", fields[1]);
+    ERROR ("(fields[1] = %s) != (GAUGE || COUNTER || DERIVE || ABSOLUTE)", fields[1]);
     return (-1);
   }
 
index c19c5ffc3671d7fdd45d581aea4032d949f182cb..bdbad3f213155a544b8d06009aa90c7322e7522c 100644 (file)
@@ -145,6 +145,59 @@ static int default_callback (const char __attribute__((unused)) *str,
 
     data->values_num++;
   }
+  else if (data->ds_type & UTILS_MATCH_DS_TYPE_DERIVE)
+  {
+    derive_t value;
+    char *endptr = NULL;
+
+    if (data->ds_type & UTILS_MATCH_CF_DERIVE_INC)
+    {
+      data->value.counter++;
+      data->values_num++;
+      return (0);
+    }
+
+    if (matches_num < 2)
+      return (-1);
+
+    value = strtoll (matches[1], &endptr, 0);
+    if (matches[1] == endptr)
+      return (-1);
+
+    if (data->ds_type & UTILS_MATCH_CF_DERIVE_SET)
+      data->value.derive = value;
+    else if (data->ds_type & UTILS_MATCH_CF_DERIVE_ADD)
+      data->value.derive += value;
+    else
+    {
+      ERROR ("utils_match: default_callback: obj->ds_type is invalid!");
+      return (-1);
+    }
+
+    data->values_num++;
+  }
+  else if (data->ds_type & UTILS_MATCH_DS_TYPE_ABSOLUTE)
+  {
+    absolute_t value;
+    char *endptr = NULL;
+
+    if (matches_num < 2)
+      return (-1);
+
+    value = strtoll (matches[1], &endptr, 0);
+    if (matches[1] == endptr)
+      return (-1);
+
+    if (data->ds_type & UTILS_MATCH_CF_ABSOLUTE_SET)
+      data->value.absolute = value;
+    else
+    {
+      ERROR ("utils_match: default_callback: obj->ds_type is invalid!");
+      return (-1);
+    }
+
+    data->values_num++;
+  }
   else
   {
     ERROR ("utils_match: default_callback: obj->ds_type is invalid!");
index a32a6fe065656f7b04fe2007f92bcbe5e53f9371..5a0c0337fe6b14591a93ecfc256d61ecf81a26c5 100644 (file)
@@ -30,6 +30,8 @@
  */
 #define UTILS_MATCH_DS_TYPE_GAUGE   0x10
 #define UTILS_MATCH_DS_TYPE_COUNTER 0x20
+#define UTILS_MATCH_DS_TYPE_DERIVE 0x30
+#define UTILS_MATCH_DS_TYPE_ABSOLUTE 0x40
 
 #define UTILS_MATCH_CF_GAUGE_AVERAGE 0x01
 #define UTILS_MATCH_CF_GAUGE_MIN     0x02
 #define UTILS_MATCH_CF_COUNTER_ADD   0x02
 #define UTILS_MATCH_CF_COUNTER_INC   0x04
 
+#define UTILS_MATCH_CF_DERIVE_SET   0x01
+#define UTILS_MATCH_CF_DERIVE_ADD   0x02
+#define UTILS_MATCH_CF_DERIVE_INC   0x04
+
+#define UTILS_MATCH_CF_ABSOLUTE_SET   0x01
+#define UTILS_MATCH_CF_ABSOLUTE_ADD   0x02
+#define UTILS_MATCH_CF_ABSOLUTE_INC   0x04
+
 /*
  * Data types
  */
index c4e9d8ba9bdeb30778fcba3cecd98b4a63335e25..0abbb40b299ad795aa0a59c4a03e5ef9885a4376 100644 (file)
@@ -213,6 +213,10 @@ static int ds_get (char ***ret, /* {{{ */
       type = "COUNTER";
     else if (d->type == DS_TYPE_GAUGE)
       type = "GAUGE";
+    else if (d->type == DS_TYPE_DERIVE)
+      type = "DERIVE";
+    else if (d->type == DS_TYPE_ABSOLUTE)
+      type = "ABSOLUTE";
     else
     {
       ERROR ("rrdtool plugin: Unknown DS type: %i",