1 #!/usr/bin/perl
3 # collectd - contrib/rrd_filter.px
4 # Copyright (C) 2007-2008 Florian octo Forster
5 #
6 # This program is free software; you can redistribute it and/or modify it
7 # under the terms of the GNU General Public License as published by the
8 # Free Software Foundation; only version 2 of the License is applicable.
9 #
10 # This program is distributed in the hope that it will be useful, but
11 # WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 # General Public License for more details.
14 #
15 # You should have received a copy of the GNU General Public License along
16 # with this program; if not, write to the Free Software Foundation, Inc.,
17 # 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
18 #
19 # Authors:
20 # Florian octo Forster <octo at verplant.org>
22 use strict;
23 use warnings;
25 =head1 NAME
27 rrd_filter.px - Perform same advanced non-standard operations on an RRD file.
29 =head1 SYNOPSYS
31 rrd_filter.px -i input.rrd -o output.rrd [options]
33 =head1 DEPENDENCIES
35 extractDS.px requires the RRDTool binary, Perl and the included L<Getopt::Long>
36 module.
38 =cut
40 use Getopt::Long ('GetOptions');
41 use Data::Dumper ();
43 our $InFile;
44 our $InDS = [];
45 our $OutFile;
46 our $OutDS = [];
48 our $NewRRAs = [];
50 our $Step = 0;
52 =head1 OPTIONS
54 The following options can be passed on the command line:
56 =over 4
58 =item B<--infile> I<file>
60 =item B<-i> I<file>
62 Reads from I<file>. If I<file> ends in C<.rrd>, then C<rrdtool dump> is invoked
63 to create an XML dump of the RRD file. Otherwise the XML dump is expected
64 directly. The special filename C<-> can be used to read from STDIN.
66 =item B<--outfile> I<file>
68 =item B<-o> I<file>
70 Writes output to I<file>. If I<file> ends in C<.rrd>, then C<rrdtool restore>
71 is invoked to create a binary RRD file. Otherwise an XML output is written. The
72 special filename C<-> can be used to write to STDOUT.
74 =item B<--map> I<in_ds>:I<out_ds>
76 =item B<-m> I<in_ds>:I<out_ds>
78 Writes the datasource I<in_ds> to the output and renames it to I<out_ds>. This
79 is useful to extract one DS from an RRD file.
81 =item B<--step> I<seconds>
83 =item B<-s> I<seconds>
85 Changes the step of the output RRD file to be I<seconds>. The new stepsize must
86 be a multiple of the old stepsize of the other way around. When increasing the
87 stepsize the number of PDPs in each RRA must be dividable by the factor by
88 which the stepsize is increased. The length of CDPs and the absolute length of
89 RRAs (and thus the data itself) is not altered.
91 Examples:
93 step = 10, rra_steps = 12 => step = 60, rra_steps = 2
94 step = 300, rra_steps = 1 => step = 10, rra_steps = 30
96 =item B<--rra> B<RRA>:I<CF>:I<XFF>:I<steps>:I<rows>
98 =item B<-a> B<RRA>:I<CF>:I<XFF>:I<steps>:I<rows>
100 Inserts a new RRA in the generated RRD file. This is done B<after> the step has
101 been adjusted, take that into account when specifying I<steps> and I<rows>. For
102 an explanation of the format please see L<rrdcreate(1)>.
104 =back
106 =cut
108 GetOptions ("infile|i=s" => \$InFile,
109 "outfile|o=s" => \$OutFile,
110 'map|m=s' => sub
111 {
112 my ($in_ds, $out_ds) = split (':', $_[1]);
113 if (!defined ($in_ds) || !defined ($out_ds))
114 {
115 print STDERR "Argument for `map' incorrect! The format is `--map in_ds:out_ds'\n";
116 exit (1);
117 }
118 push (@$InDS, $in_ds);
119 push (@$OutDS, $out_ds);
120 },
121 'step|s=i' => \$Step,
122 'rra|a=s' => sub
123 {
124 my ($rra, $cf, $xff, $steps, $rows) = split (':', $_[1]);
125 if (($rra ne 'RRA') || !defined ($rows))
126 {
127 print STDERR "Please use the standard RRDTool syntax when adding RRAs. I. e. RRA:<cf><xff>:<steps>:<rows>.\n";
128 exit (1);
129 }
130 push (@$NewRRAs, {cf => $cf, xff => $xff, steps => $steps, rows => $rows});
131 }
132 ) or exit (1);
134 if (!$InFile || !$OutFile)
135 {
136 print STDERR "Usage: $0 -i <infile> -m <in_ds>:<out_ds> -s <step>\n";
137 exit (1);
138 }
139 if ((1 + @$InDS) != (1 + @$OutDS))
140 {
141 print STDERR "You need the same amount of in- and out-DSes\n";
142 exit (1);
143 }
145 main ($InFile, $OutFile);
146 exit (0);
148 {
149 my $ds_index;
150 my $current_index;
151 # state 0 == searching for DS index
152 # state 1 == parse RRA header
153 # state 2 == parse values
154 my $state;
155 my $out_cache;
156 sub handle_line_dsmap
157 {
158 my $line = shift;
159 my $index = shift;
160 my $ret = '';
162 if ((@$InDS == 0) || (@$OutDS == 0))
163 {
164 post_line ($line, $index + 1);
165 return;
166 }
168 if (!defined ($state))
169 {
170 $current_index = -1;
171 $state = 0;
172 $out_cache = [];
174 # $ds_index->[new_index] = old_index
175 $ds_index = [];
176 for (my $i = 0; $i < @$InDS; $i++)
177 {
178 $ds_index->[$i] = -1;
179 }
180 }
182 if ($state == 0)
183 {
184 if ($line =~ m/<ds>/)
185 {
186 $current_index++;
187 $out_cache->[$current_index] = $line;
188 }
189 elsif ($line =~ m#<name>\s*([^<\s]+)\s*</name>#)
190 {
191 # old_index == $current_index
192 # new_index == $i
193 for (my $i = 0; $i < @$InDS; $i++)
194 {
195 next if ($ds_index->[$i] >= 0);
197 if ($1 eq $InDS->[$i])
198 {
199 $line =~ s#<name>\s*([^<\s]+)\s*</name>#<name> $OutDS->[$i] </name>#;
200 $ds_index->[$i] = $current_index;
201 last;
202 }
203 }
205 $out_cache->[$current_index] .= $line;
206 }
207 elsif ($line =~ m#</ds>#)
208 {
209 $out_cache->[$current_index] .= $line;
210 }
211 elsif ($line =~ m#<rra>#)
212 {
213 # Print out all the DS definitions we need
214 for (my $new_index = 0; $new_index < @$InDS; $new_index++)
215 {
216 my $old_index = $ds_index->[$new_index];
217 while ($out_cache->[$old_index] =~ m/^(.*)$/gm)
218 {
219 post_line ("$1\n", $index + 1);
220 }
221 }
223 # Clear the cache - it's used in state1, too.
224 for (my $i = 0; $i <= $current_index; $i++)
225 {
226 $out_cache->[$i] = '';
227 }
229 $ret .= $line;
230 $current_index = -1;
231 $state = 1;
232 }
233 elsif ($current_index == -1)
234 {
235 # Print all the lines before the first DS definition
236 $ret .= $line;
237 }
238 else
239 {
240 # Something belonging to a DS-definition
241 $out_cache->[$current_index] .= $line;
242 }
243 }
244 elsif ($state == 1)
245 {
246 if ($line =~ m#<ds>#)
247 {
248 $current_index++;
249 $out_cache->[$current_index] .= $line;
250 }
251 elsif ($line =~ m#</cdp_prep>#)
252 {
253 # Print out all the DS definitions we need
254 for (my $new_index = 0; $new_index < @$InDS; $new_index++)
255 {
256 my $old_index = $ds_index->[$new_index];
257 while ($out_cache->[$old_index] =~ m/^(.*)$/gm)
258 {
259 post_line ("$1\n", $index + 1);
260 }
261 }
263 # Clear the cache
264 for (my $i = 0; $i <= $current_index; $i++)
265 {
266 $out_cache->[$i] = '';
267 }
269 $ret .= $line;
270 $current_index = -1;
271 }
272 elsif ($line =~ m#<database>#)
273 {
274 $ret .= $line;
275 $state = 2;
276 }
277 elsif ($current_index == -1)
278 {
279 # Print all the lines before the first DS definition
280 # and after cdp_prep
281 $ret .= $line;
282 }
283 else
284 {
285 # Something belonging to a DS-definition
286 $out_cache->[$current_index] .= $line;
287 }
288 }
289 elsif ($state == 2)
290 {
291 if ($line =~ m#</database>#)
292 {
293 $ret .= $line;
294 $current_index = -1;
295 $state = 1;
296 }
297 else
298 {
299 my @values = ();
300 my $i;
302 $ret .= "\t\t";
304 if ($line =~ m#(<!-- .*? -->)#)
305 {
306 $ret .= "$1 ";
307 }
308 $ret .= "<row> ";
310 $i = 0;
311 while ($line =~ m#<v>\s*([^<\s]+)\s*</v>#g)
312 {
313 $values[$i] = $1;
314 $i++;
315 }
317 for (my $new_index = 0; $new_index < @$InDS; $new_index++)
318 {
319 my $old_index = $ds_index->[$new_index];
320 $ret .= '<v> ' . $values[$old_index] . ' </v> ';
321 }
322 $ret .= "</row>\n";
323 }
324 }
325 else
326 {
327 die;
328 }
330 if ($ret)
331 {
332 post_line ($ret, $index + 1);
333 }
334 }} # handle_line_dsmap
336 #
337 # The _step_ handler
338 #
339 {
340 my $step_factor_up;
341 my $step_factor_down;
342 sub handle_line_step
343 {
344 my $line = shift;
345 my $index = shift;
347 if (!$Step)
348 {
349 post_line ($line, $index + 1);
350 return;
351 }
353 $step_factor_up ||= 0;
354 $step_factor_down ||= 0;
356 if (($step_factor_up == 0) && ($step_factor_down == 0))
357 {
358 if ($line =~ m#<step>\s*(\d+)\s*</step>#i)
359 {
360 my $old_step = 0 + $1;
361 if ($Step < $old_step)
362 {
363 $step_factor_down = int ($old_step / $Step);
364 if (($step_factor_down * $Step) != $old_step)
365 {
366 print STDERR "The old step ($old_step seconds) "
367 . "is not a multiple of the new step "
368 . "($Step seconds).\n";
369 exit (1);
370 }
371 $line = "<step> $Step </step>\n";
372 }
373 elsif ($Step > $old_step)
374 {
375 $step_factor_up = int ($Step / $old_step);
376 if (($step_factor_up * $old_step) != $Step)
377 {
378 print STDERR "The new step ($Step seconds) "
379 . "is not a multiple of the old step "
380 . "($old_step seconds).\n";
381 exit (1);
382 }
383 $line = "<step> $Step </step>\n";
384 }
385 else
386 {
387 $Step = 0;
388 }
389 }
390 }
391 elsif ($line =~ m#<pdp_per_row>\s*(\d+)\s*</pdp_per_row>#i)
392 {
393 my $old_val = 0 + $1;
394 my $new_val;
395 if ($step_factor_up)
396 {
397 $new_val = int ($old_val / $step_factor_up);
398 if (($new_val * $step_factor_up) != $old_val)
399 {
400 print STDERR "Can't divide number of PDPs per row ($old_val) by step-factor ($step_factor_up).\n";
401 exit (1);
402 }
403 }
404 else
405 {
406 $new_val = $step_factor_down * $old_val;
407 }
408 $line = "<pdp_per_row> $new_val </pdp_per_row>\n";
409 }
411 post_line ($line, $index + 1);
412 }} # handle_line_step
414 #
415 # The _add RRA_ handler
416 #
417 {
418 my $add_rra_done;
419 my $num_ds;
420 sub handle_line_add_rra
421 {
422 my $line = shift;
423 my $index = shift;
425 my $post = sub { for (@_) { post_line ($_, $index + 1); } };
427 $num_ds ||= 0;
429 if (!@$NewRRAs || $add_rra_done)
430 {
431 $post->($line);
432 return;
433 }
435 if ($line =~ m#<ds>#i)
436 {
437 $num_ds++;
438 }
439 elsif ($line =~ m#<rra>#i)
440 {
441 for (my $i = 0; $i < @$NewRRAs; $i++)
442 {
443 my $rra = $NewRRAs->[$i];
444 my $temp;
445 $post->("\t<rra>\n",
446 "\t\t<cf> $rra->{'cf'} </cf>\n",
447 "\t\t<pdp_per_row> $rra->{'steps'} </pdp_per_row>\n",
448 "\t\t<params>\n",
449 "\t\t\t<xff> $rra->{'xff'} </xff>\n",
450 "\t\t</params>\n",
451 "\t\t<cdp_prep>\n");
453 for (my $j = 0; $j < $num_ds; $j++)
454 {
455 $post->("\t\t\t<ds>\n",
456 "\t\t\t\t<primary_value> NaN </primary_value>\n",
457 "\t\t\t\t<secondary_value> NaN </secondary_value>\n",
458 "\t\t\t\t<value> NaN </value>\n",
459 "\t\t\t\t<unknown_datapoints> 0 </unknown_datapoints>\n",
460 "\t\t\t</ds>\n");
461 }
463 $post->("\t\t</cdp_prep>\n", "\t\t<database>\n");
464 $temp = "\t\t\t<row>" . join ('', map { "<v> NaN </v>" } (1 .. $num_ds)) . "</row>\n";
465 for (my $j = 0; $j < $rra->{'rows'}; $j++)
466 {
467 $post->($temp);
468 }
469 $post->("\t\t</database>\n");
470 }
471 }
473 $post->($line);
474 }} # handle_line_add_rra
476 #
477 # The _output_ handler
478 #
479 {
480 my $fh;
481 sub set_output
482 {
483 $fh = shift;
484 }
486 sub handle_line_output
487 {
488 my $line = shift;
489 my $index = shift;
491 if (!defined ($fh))
492 {
493 post_line ($line, $index + 1);
494 return;
495 }
497 print $fh $line;
498 }} # handle_line_output
500 #
501 # Dispatching logic
502 #
503 {
504 my @handlers = ();
505 sub add_handler
506 {
507 my $handler = shift;
509 die unless (ref ($handler) eq 'CODE');
510 push (@handlers, $handler);
511 } # add_handler
513 sub post_line
514 {
515 my $line = shift;
516 my $index = shift;
518 if (0)
519 {
520 my $copy = $line;
521 chomp ($copy);
522 print "DEBUG: post_line ($copy, $index);\n";
523 }
525 if ($index > $#handlers)
526 {
527 return;
528 }
529 $handlers[$index]->($line, $index);
530 }} # post_line
532 sub handle_fh
533 {
534 my $in_fh = shift;
535 my $out_fh = shift;
537 set_output ($out_fh);
539 if (@$InDS)
540 {
541 add_handler (\&handle_line_dsmap);
542 }
544 if ($Step)
545 {
546 add_handler (\&handle_line_step);
547 }
549 if (@$NewRRAs)
550 {
551 add_handler (\&handle_line_add_rra);
552 }
554 add_handler (\&handle_line_output);
556 while (my $line = <$in_fh>)
557 {
558 post_line ($line, 0);
559 }
560 } # handle_fh
562 sub main
563 {
564 my $in_file = shift;
565 my $out_file = shift;
567 my $in_fh;
568 my $out_fh;
570 my $in_needs_close = 1;
571 my $out_needs_close = 1;
573 if ($in_file =~ m/\.rrd$/i)
574 {
575 open ($in_fh, '-|', 'rrdtool', 'dump', $in_file) or die ("open (rrdtool): $!");
576 }
577 elsif ($in_file eq '-')
578 {
579 $in_fh = \*STDIN;
580 $in_needs_close = 0;
581 }
582 else
583 {
584 open ($in_fh, '<', $in_file) or die ("open ($in_file): $!");
585 }
587 if ($out_file =~ m/\.rrd$/i)
588 {
589 open ($out_fh, '|-', 'rrdtool', 'restore', '-', $out_file) or die ("open (rrdtool): $!");
590 }
591 elsif ($out_file eq '-')
592 {
593 $out_fh = \*STDOUT;
594 $out_needs_close = 0;
595 }
596 else
597 {
598 open ($out_fh, '>', $out_file) or die ("open ($out_file): $!");
599 }
601 handle_fh ($in_fh, $out_fh);
603 if ($in_needs_close)
604 {
605 close ($in_fh);
606 }
607 if ($out_needs_close)
608 {
609 close ($out_fh);
610 }
611 } # main
613 =head1 LICENSE
615 This script is licensed under the GNU general public license, versionE<nbsp>2
616 (GPLv2).
618 =head1 AUTHOR
620 Florian octo Forster E<lt>octo at verplant.orgE<gt>