src/utils_rrdcreate.[ch]: Implement the srrd_create_async() function.

This new functionality, which is also exported by the rrdtool and
rrdcached plguins, allows to create new RRD files in the background.
This avoid blocking write threads on the RRD creation, when hundreds of
files need to be created.

See also Github issues #243 and #244.

diff --git a/src/collectd.conf.in b/src/collectd.conf.in
index 8744b21afcaf437020d045d0eb26ca95bc69fca3..9d8e688fc15681d17c66baf0ab676f849019b3c9 100644 (file)
--- a/src/collectd.conf.in
+++ b/src/collectd.conf.in
@@ -854,13 +854,16 @@
 #      DaemonAddress "unix:/tmp/rrdcached.sock"
 #      DataDir "@localstatedir@/lib/@PACKAGE_NAME@/rrd"
 #      CreateFiles true
+#      CreateFilesAsync false
 #      CollectStatistics true
 #</Plugin>
 
 #<Plugin rrdtool>
 #      DataDir "@localstatedir@/lib/@PACKAGE_NAME@/rrd"
+#      CreateFilesAsync false
 #      CacheTimeout 120
 #      CacheFlush   900
+#      WritesPerSecond 50
 #</Plugin>
 
 #<Plugin sensors>

diff --git a/src/collectd.conf.pod b/src/collectd.conf.pod
index 70bc997d650c7993d8c3526d82c73bdc93544eb2..ec527d7a433d1fe7933ff427dc0487bc116948f4 100644 (file)
--- a/src/collectd.conf.pod
+++ b/src/collectd.conf.pod
@@ -4644,6 +4644,15 @@ Enables or disables the creation of RRD files. If the daemon is not running
 locally, or B<DataDir> is set to a relative path, this will not work as
 expected. Default is B<true>.
 
+=item B<CreateFilesAsync> B<false>|B<true>
+
+When enabled, new RRD files are enabled asynchronously, using a separate thread
+that runs in the background. This prevents writes to block, which is a problem
+especially when many hundreds of files need to be created at once. However,
+this may lead to transient errors reported by code that assumes that the files
+are already existing. When disabled (the default) files are created
+synchronously, blocking for a short while, while the file is being written.
+
 =item B<StepSize> I<Seconds>
 
 B<Force> the stepsize of newly created RRD-files. Ideally (and per default)
@@ -4701,8 +4710,17 @@ can safely ignore these settings.
 
 =item B<DataDir> I<Directory>
 
-Set the directory to store RRD-files under. Per default RRD-files are generated
-beneath the daemon's working directory, i.E<nbsp>e. the B<BaseDir>.
+Set the directory to store RRD files under. By default RRD files are generated
+beneath the daemon's working directory, i.e. the B<BaseDir>.
+
+=item B<CreateFilesAsync> B<false>|B<true>
+
+When enabled, new RRD files are enabled asynchronously, using a separate thread
+that runs in the background. This prevents writes to block, which is a problem
+especially when many hundreds of files need to be created at once. However,
+this may lead to transient errors reported by code that assumes that the files
+are already existing. When disabled (the default) files are created
+synchronously, blocking for a short while, while the file is being written.
 
 =item B<StepSize> I<Seconds>
 
@@ -4723,7 +4741,7 @@ a very good reason to do so.
 
 The C<rrdtool plugin> calculates the number of PDPs per CDP based on the
 B<StepSize>, this setting and a timespan. This plugin creates RRD-files with
-three times five RRAs, i. e. five RRAs with the CFs B<MIN>, B<AVERAGE>, and
+three times five RRAs, i.e. five RRAs with the CFs B<MIN>, B<AVERAGE>, and
 B<MAX>. The five RRAs are optimized for graphs covering one hour, one day, one
 week, one month, and one year.
 

diff --git a/src/rrdcached.c b/src/rrdcached.c
index 45553b7f250ee708ec620f8058a6f18691711f34..3c5656530706c726dea0ca2c4530e4d7bf330423 100644 (file)
--- a/src/rrdcached.c
+++ b/src/rrdcached.c
@@ -46,7 +46,9 @@ static rrdcreate_config_t rrdcreate_config =
        /* timespans_num = */ 0,
 
        /* consolidation_functions = */ NULL,
-       /* consolidation_functions_num = */ 0
+       /* consolidation_functions_num = */ 0,
+
+        /* async = */ 0
 };
 
 /*
@@ -250,6 +252,8 @@ static int rc_config (oconfig_item_t *ci)
       status = cf_util_get_string (child, &daemon_address);
     else if (strcasecmp ("CreateFiles", key) == 0)
       status = cf_util_get_boolean (child, &config_create_files);
+    else if (strcasecmp ("CreateFilesAsync", key) == 0)
+      status = cf_util_get_boolean (child, &rrdcreate_config.async);
     else if (strcasecmp ("CollectStatistics", key) == 0)
       status = cf_util_get_boolean (child, &config_collect_stats);
     else if (strcasecmp ("StepSize", key) == 0)

diff --git a/src/rrdtool.c b/src/rrdtool.c
index b1d13ee3c19ee279c836741739b521359872c63b..da4fcb8c51b2703c50aee78dc0c2298ce4f14096 100644 (file)
--- a/src/rrdtool.c
+++ b/src/rrdtool.c
@@ -75,6 +75,7 @@ static const char *config_keys[] =
 {
        "CacheTimeout",
        "CacheFlush",
+       "CreateFilesAsync",
        "DataDir",
        "StepSize",
        "HeartBeat",
@@ -102,7 +103,9 @@ static rrdcreate_config_t rrdcreate_config =
        /* timespans_num = */ 0,
 
        /* consolidation_functions = */ NULL,
-       /* consolidation_functions_num = */ 0
+       /* consolidation_functions_num = */ 0,
+
+        /* async = */ 0
 };
 
 /* XXX: If you need to lock both, cache_lock and queue_lock, at the same time,
@@ -1008,6 +1011,13 @@ static int rrd_config (const char *key, const char *value)
                if (temp > 0)
                        rrdcreate_config.heartbeat = temp;
        }
+       else if (strcasecmp ("CreateFilesAsync", key) == 0)
+       {
+               if (IS_TRUE (value))
+                       rrdcreate_config.async = 1;
+               else
+                       rrdcreate_config.async = 0;
+       }
        else if (strcasecmp ("RRARows", key) == 0)
        {
                int tmp = atoi (value);

diff --git a/src/utils_rrdcreate.c b/src/utils_rrdcreate.c
index 091b5fa6dce1d2cc3bc48bb093298f39e5d0af46..2228660ea750b417171bd830efd5bce649a5875d 100644 (file)
--- a/src/utils_rrdcreate.c
+++ b/src/utils_rrdcreate.c
@@ -1,6 +1,6 @@
 /**
  * collectd - src/utils_rrdcreate.c
- * Copyright (C) 2006-2008  Florian octo Forster
+ * Copyright (C) 2006-2013  Florian octo Forster
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the
@@ -16,7 +16,7 @@
  * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
  *
  * Authors:
- *   Florian octo Forster <octo at verplant.org>
+ *   Florian octo Forster <octo at collectd.org>
  **/
 
 #include "collectd.h"
@@ -26,6 +26,16 @@
 #include <pthread.h>
 #include <rrd.h>
 
+struct srrd_create_args_s
+{
+  char *filename;
+  unsigned long pdp_step;
+  time_t last_up;
+  int argc;
+  char **argv;
+};
+typedef struct srrd_create_args_s srrd_create_args_t;
+
 /*
  * Private variables
  */
@@ -65,6 +75,71 @@ static void rra_free (int rra_num, char **rra_def) /* {{{ */
   sfree (rra_def);
 } /* }}} void rra_free */
 
+static void srrd_create_args_destroy (srrd_create_args_t *args)
+{
+  if (args == NULL)
+    return;
+
+  sfree (args->filename);
+  if (args->argv != NULL)
+  {
+    int i;
+    for (i = 0; i < args->argc; i++)
+      sfree (args->argv[i]);
+    sfree (args->argv);
+  }
+} /* void srrd_create_args_destroy */
+
+static srrd_create_args_t *srrd_create_args_create (const char *filename,
+    unsigned long pdp_step, time_t last_up,
+    int argc, const char **argv)
+{
+  srrd_create_args_t *args;
+
+  args = malloc (sizeof (*args));
+  if (args == NULL)
+  {
+    ERROR ("srrd_create_args_create: malloc failed.");
+    return (NULL);
+  }
+  memset (args, 0, sizeof (*args));
+  args->filename = NULL;
+  args->pdp_step = pdp_step;
+  args->last_up = last_up;
+  args->argv = NULL;
+
+  args->filename = strdup (filename);
+  if (args->filename == NULL)
+  {
+    ERROR ("srrd_create_args_create: strdup failed.");
+    srrd_create_args_destroy (args);
+    return (NULL);
+  }
+
+  args->argv = calloc ((size_t) (argc + 1), sizeof (*args->argv));
+  if (args->argv == NULL)
+  {
+    ERROR ("srrd_create_args_create: calloc failed.");
+    srrd_create_args_destroy (args);
+    return (NULL);
+  }
+
+  for (args->argc = 0; args->argc < argc; args->argc++)
+  {
+    args->argv[args->argc] = strdup (argv[args->argc]);
+    if (args->argv[args->argc] == NULL)
+    {
+      ERROR ("srrd_create_args_create: strdup failed.");
+      srrd_create_args_destroy (args);
+      return (NULL);
+    }
+  }
+  assert (args->argc == argc);
+  args->argv[args->argc] = NULL;
+
+  return (args);
+} /* srrd_create_args_t *srrd_create_args_create */
+
 /* * * * * * * * * *
  * WARNING:  Magic *
  * * * * * * * * * */
@@ -359,6 +434,65 @@ static int srrd_create (const char *filename, /* {{{ */
 } /* }}} int srrd_create */
 #endif /* !HAVE_THREADSAFE_LIBRRD */
 
+static void *srrd_create_thread (void *targs)
+{
+  srrd_create_args_t *args = targs;
+  int status;
+
+  status = srrd_create (args->filename, args->pdp_step, args->last_up,
+      args->argc, (void *) args->argv);
+  if (status != 0)
+  {
+    WARNING ("srrd_create_thread: srrd_create (%s) returned status %i.",
+        args->filename, status);
+  }
+  else
+  {
+    DEBUG ("srrd_create_thread: Successfully created RRD file \"%s\".",
+        args->filename);
+  }
+
+  srrd_create_args_destroy (args);
+
+  return (0);
+} /* void *srrd_create_thread */
+
+static int srrd_create_async (const char *filename,
+    unsigned long pdp_step, time_t last_up,
+    int argc, const char **argv)
+{
+  srrd_create_args_t *args;
+  pthread_t thread;
+  pthread_attr_t attr;
+  int status;
+
+  args = srrd_create_args_create (filename, pdp_step, last_up, argc, argv);
+  if (args == NULL)
+    return (-1);
+
+  status = pthread_attr_init (&attr);
+  if (status != 0)
+    return (-1);
+
+  status = pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
+  if (status != 0)
+    return (-1);
+
+  status = pthread_create (&thread, &attr, srrd_create_thread, args);
+  if (status != 0)
+  {
+    char errbuf[1024];
+    ERROR ("srrd_create_async: pthread_create failed: %s",
+        sstrerror (status, errbuf, sizeof (errbuf)));
+    pthread_attr_destroy (&attr);
+    srrd_create_args_destroy (args);
+    return (status);
+  }
+
+  pthread_attr_destroy (&attr);
+  return (0);
+}
+
 /*
  * Public functions
  */
@@ -415,24 +549,36 @@ int cu_rrd_create_file (const char *filename, /* {{{ */
   else
     stepsize = (unsigned long) CDTIME_T_TO_TIME_T (vl->interval);
 
-  status = srrd_create (filename, stepsize, last_up,
-      argc, (const char **) argv);
-
-  free (argv);
-  ds_free (ds_num, ds_def);
-  rra_free (rra_num, rra_def);
-
-  if (status != 0)
+  if (cfg->async)
   {
-    WARNING ("cu_rrd_create_file: srrd_create (%s) returned status %i.",
-        filename, status);
+    status = srrd_create_async (filename, stepsize, last_up,
+        argc, (const char **) argv);
+    if (status != 0)
+      WARNING ("cu_rrd_create_file: srrd_create_async (%s) "
+          "returned status %i.",
+          filename, status);
   }
-  else
+  else /* synchronous */
   {
-    DEBUG ("cu_rrd_create_file: Successfully created RRD file \"%s\".",
-        filename);
+    status = srrd_create (filename, stepsize, last_up,
+        argc, (const char **) argv);
+
+    if (status != 0)
+    {
+      WARNING ("cu_rrd_create_file: srrd_create (%s) returned status %i.",
+          filename, status);
+    }
+    else
+    {
+      DEBUG ("cu_rrd_create_file: Successfully created RRD file \"%s\".",
+          filename);
+    }
   }
 
+  free (argv);
+  ds_free (ds_num, ds_def);
+  rra_free (rra_num, rra_def);
+
   return (status);
 } /* }}} int cu_rrd_create_file */
 

diff --git a/src/utils_rrdcreate.h b/src/utils_rrdcreate.h
index 103ca57070ad3b99fa72183966eefdbd69a1fedc..fdfd6ecb9727f51a6589e551bda125db72018836 100644 (file)
--- a/src/utils_rrdcreate.h
+++ b/src/utils_rrdcreate.h
@@ -1,6 +1,6 @@
 /**
  * collectd - src/utils_rrdcreate.h
- * Copyright (C) 2008  Florian octo Forster
+ * Copyright (C) 2008-2013  Florian octo Forster
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the
@@ -16,7 +16,7 @@
  * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
  *
  * Authors:
- *   Florian octo Forster <octo at verplant.org>
+ *   Florian octo Forster <octo at collectd.org>
  **/
 
 #ifndef UTILS_RRDCREATE_H
@@ -38,6 +38,8 @@ struct rrdcreate_config_s
 
   char **consolidation_functions;
   size_t consolidation_functions_num;
+
+  _Bool async;
 };
 typedef struct rrdcreate_config_s rrdcreate_config_t;