1 #!/usr/bin/perl -w
2 require 5.008;
3 use lib qw(/scratch/rrd-1.4.3-test2/lib/perl);
4 use RRDs;
5 use strict;
6 use Getopt::Long 2.25 qw(:config posix_default no_ignore_case);
7 use Pod::Usage 1.14;
8 use Data::Dumper;
10 '$Revision$ ' =~ /Revision: (\S*)/;
11 my $Revision = $1;
13 # main loop
14 my %opt = ();
15 sub main()
16 {
17 # parse options
18 GetOptions(\%opt, 'help|h', 'man', 'version', 'noaction|no-action|n',
19 'verbose|v','src-tmpl=s','dst-tmpl=s') or exit(1);
20 if($opt{help}) { pod2usage(1) }
21 if($opt{man}) { pod2usage(-exitstatus => 0, -verbose => 2) }
22 if($opt{version}) { print "rrdjig $Revision\n"; exit(0) }
23 my $src = shift @ARGV or pod2usage(1);
24 if (not -r $src) { pod2usage("Reading $src: $!") }
25 my $dst = shift @ARGV or pod2usage(1);
26 if (not -w $dst) { pod2usage("Accessing $dst: $!") }
28 rrdjig($src,$opt{'src-tmpl'},$dst,$opt{'dst-tmpl'});
29 }
31 main;
33 sub rrd_err_check(){
34 my $err = RRDs::error();
35 if ($err){
36 die "RRD Error: $err\n";
37 }
38 }
40 # how should the data be fetched from the source
41 # to provide the best approximation of the original data
43 sub step_sync ($$){
44 my $value = shift;
45 my $step = shift;
46 return ($value - ($value % $step));
47 }
49 sub get_rra_size_map($){
50 my $info = shift;
51 my $map = {};
52 my $min_start;
53 for (my $i=0;;$i++){
54 my $cf = $info->{"rra[$i].cf"};
55 last if not $cf;
56 next if $cf !~ /AVERAGE|MIN|MAX/;
57 my $pdp_per_row = $info->{"rra[$i].pdp_per_row"};
58 next if $cf =~ /MIN|MAX/ and $pdp_per_row == 1;
59 my $rows = $info->{"rra[$i].rows"};
60 my $step = $pdp_per_row*$info->{step};
61 my $start = step_sync($info->{last_update},$step) - $step*$rows;
62 if (not defined $min_start or $start < $min_start) {
63 $min_start = $start;
64 }
65 if ( $map->{$cf}{$pdp_per_row}{rows} || 0 < $rows
66 or $map->{$cf}{$pdp_per_row}{start} || 0 < $start ){
67 $map->{$cf}{$pdp_per_row} = {
68 id => $i,
69 rows => $rows,
70 step => $step,
71 start => $start
72 };
73 }
74 }
75 return ($min_start,$map);
76 }
79 sub prep_fetch_tasks ($$){
80 my $src_info = shift;
81 my $dst_info = shift;
82 my ($min_start,$src_size) = get_rra_size_map($src_info);
83 my $now = step_sync($src_info->{last_update}, $src_info->{step});
84 my $first = step_sync($dst_info->{last_update} , $dst_info->{step});
85 if ($min_start > $first ) {
86 $first = $min_start;
87 }
88 print "Search $first to $now\n" if $opt{verbose};
89 my $task = {};
90 for my $cf (qw(AVERAGE MIN MAX)){
91 my $x = $src_size->{$cf};
92 my $pointer = $now;
93 $task->{$cf} = [];
94 for my $pdp_per_row (sort {$a <=> $b} keys %$x){
95 my $step = $x->{$pdp_per_row}{step};
96 my $new_pointer = $x->{$pdp_per_row}{start};
97 print "look $cf $pdp_per_row * $step - $new_pointer\n" if $opt{verbose};
98 if ($new_pointer <= $first){
99 $new_pointer = step_sync($first,$step);
100 }
101 if ($new_pointer <= $pointer){
102 unshift @{$task->{$cf}}, {
103 start => $new_pointer,
104 end => step_sync($pointer,$step),
105 step => $step
106 };
107 $pointer = $new_pointer;
108 }
109 last if $pointer <= $first;
110 }
111 }
112 return ($first,$task);
113 }
115 sub fetch_data($$$){
116 my $src = shift;
117 my $first = shift;
118 my $tasks = shift;
119 my %data;
120 my @tmpl;
121 if ($opt{'src-tmpl'}){
122 @tmpl = split /:/, $opt{'src-tmpl'};
123 }
124 my %map;
125 for my $cf (keys %$tasks){
126 print STDERR "FETCH #### CF $cf #####################################\n"
127 if $opt{verbose};
128 for my $t (@{$tasks->{$cf}}){
129 my ($start,$step,$names,$array) = RRDs::fetch(
130 $src,$cf,'--resolution',$t->{step},
131 '--start',$t->{start},'--end',$t->{end}
132 );
133 my $id = 0;
134 if (@tmpl and not %map){
135 %map = ( map { ($_,$id++) } @$names );
136 for my $key (@tmpl){
137 die "ERROR: src key '$key' is not known in $src. Pick from ".join(':',@$names)."\n"
138 if not exists $map{$key};
139 }
140 }
141 rrd_err_check();
142 print STDERR "FETCH: want setp $t->{step} -> got step $step / want start $t->{start} -> got start $start\n" if $opt{verbose};
143 my $now = $start;
144 while (my $row = shift @$array){
145 if (@tmpl){
146 push @{$data{$cf}} , [ $now, $step, [ @$row[@map{@tmpl}] ] ];
147 }
148 else {
149 push @{$data{$cf}} , [ $now, $step, $row ];
150 }
151 $now+=$step;
152 }
153 }
154 }
155 die "ERROR: no AVERAGE RRA found in src rrd. Enhance me to be able to deal with this!\n"
156 if not $data{AVERAGE};
157 # if older data is required, generate a fake average entry.
158 my $start = $data{AVERAGE}[0][0] - $data{AVERAGE}[0][1];
159 if ($start > $first ) {
160 my $step = $start - $first;
161 unshift @{$data{AVERAGE}}, [ $start, $step, [ map {undef} @{$data{AVERAGE}[0][2]} ] ];
162 }
163 return \%data;
164 }
166 sub reupdate($$$){
167 my $step = shift;
168 my $dst = shift;
169 my $data = shift;
170 my @hidden_rows;
171 my @fake_rows;
172 my @up;
173 while (my $av = shift @{$data->{AVERAGE}}){
174 my $end = $av->[0];
175 my $start = $end - $av->[1];
176 if (my $av_nx = $data->{AVERAGE}[0]){
177 my $start_nx = $av_nx->[0] - $av_nx->[1];
178 if ($end > $start_nx){
179 $end = $start_nx;
180 }
181 }
182 STEP:
183 for (my $t = $start+$step;$t<=$end;$t+=$step){
184 my @out = @{$av->[2]};
185 # lets see if we a usable a MIN or MAX entry pending
186 if (@hidden_rows < 2 and $av->[1] > $step) {
187 for my $cf (qw(MIN MAX)){
188 my $m = $data->{$cf}[0];
189 # drop any MIN/MAX entries which we could not use
190 while ($m->[0] <= $start) {
191 print STDERR "# DROP $cf $m->[0], $m->[1]\n" if $opt{verbose};
192 shift @{$data->{$cf}};
193 $m = $data->{$cf}[0];
194 }
195 my $cend = $m->[0];
196 my $cstep = $m->[1];
197 my $crow = $m->[2];
198 if ($cend >= $t and $cend- $cstep <= $t - $step){
199 my $row = "$t:".join(':',map {defined $_ ? $_ : 'U'} @{$crow});
200 print STDERR ($cf eq 'MIN' ? 'm' : 'M' ) ,$row,"\n" if $opt{verbose};
201 push @up, $row;
202 push @hidden_rows, $av->[2];
203 push @fake_rows, $crow;
204 shift @{$data->{$cf}};
205 next STEP;
206 }
207 }
208 }
209 # compensate for the AVERAGE data NOT shown
210 while (my $row = shift @hidden_rows){
211 for (my $i = 0; $i <@$row; $i++){
212 if (not defined $row->[$i] or not defined $out[$i]){
213 $out[$i] = undef;
214 } else {
215 $out[$i] += $row->[$i];
216 }
217 }
218 }
219 # compensate for the MIN/MAX data shown INSTEAD
220 while (my $row = shift @fake_rows){
221 for (my $i = 0; $i <@$row; $i++){
222 if (not defined $row->[$i] or not defined $out[$i]){
223 $out[$i] = undef;
224 } else {
225 $out[$i] -= $row->[$i];
226 }
227 }
228 }
229 # show the result;
230 my $row = "$t:".join(':',map {defined $_ ? $_ : 'U'} @out);
231 print STDERR " ",$row,"\n" if $opt{verbose};
232 push @up, $row;
233 }
234 }
235 pop @up; # the last update is most likely one too many ...
236 if (@up == 0) {
237 warn "WARNING: src has no entries new enough to fill dst\n";
238 } else {
239 RRDs::update($dst,
240 $opt{'dst-tmpl'} ? '--template='.$opt{'dst-tmpl'} : (),
241 @up);
242 rrd_err_check();
243 }
244 }
246 sub set_gauge($$){
247 my $dst = shift;
248 my $info = shift;
249 my @tasks;
250 for my $key (keys %$info) {
251 if ($key =~ m/^ds\[(.+)\]\.type$/
252 and $info->{$key} ne 'GAUGE'){
253 print STDERR "DS $1 -> GAUGE\n" if $opt{verbose};
254 push @tasks, "--data-source-type=${1}:GAUGE";
255 }
256 if (@tasks) {
257 RRDs::tune($dst,@tasks);
258 rrd_err_check();
259 }
260 }
261 }
263 sub unset_gauge($$){
264 my $dst = shift;
265 my $info = shift;
266 my @tasks;
267 for my $key (keys %$info) {
268 if ($key =~ m/^ds\[(.+)\]\.type$/
269 and $info->{$key} ne 'GAUGE'){
270 print STDERR "DS $1 -> $info->{$key}\n" if $opt{verbose};
271 push @tasks, "--data-source-type=${1}:$info->{$key}";
272 }
273 if (@tasks) {
274 RRDs::tune($dst,@tasks);
275 rrd_err_check();
276 }
277 }
278 }
280 sub rrdjig($$$$){
281 my $src = shift;
282 my $src_tmpl = shift;
283 my $dst = shift;
284 my $dst_tmpl = shift;
285 my $dst_info = RRDs::info($dst);
286 rrd_err_check();
287 my $src_info = RRDs::info($src);
288 rrd_err_check();
289 my ($first,$fetch_tasks) = prep_fetch_tasks($src_info,$dst_info);
290 my $updates = fetch_data($src,$first,$fetch_tasks);
291 set_gauge($dst,$dst_info);
292 reupdate($src_info->{step},$dst,$updates);
293 unset_gauge($dst,$dst_info);
294 }
297 __END__
299 =head1 NAME
301 rrdjig - use data from an existing rrd file to populate a new one
303 =head1 SYNOPSIS
305 B<rrdjig> [I<options>...] I<src.rrd> I<dest.rrd>
307 --man show man-page and exit
308 -h, --help display this help and exit
309 --version show version information and exit
310 --verbose talk while you work
311 --noaction just talk don't act
312 --src-tmpl=tmpl output template for the source rrd
313 --dst-tmpl=tmpl input template for the destination rrd
315 =head1 DESCRIPTION
317 In rrdtool, data gets processed immediately upon arrival. This means that
318 the original data is never stored and it is thus not easily possible to
319 restructure data at a later stage. In the rrdtool core there are no
320 functions to modify the base step size nor the number and types of RRAs in a
321 graceful manner.
323 The rrdjig tool tries to rebild the original data as closely as possible
324 based on the data found in the rrd file. It takes AVERAGE, MIN and MAX RRAs
325 into account and rebilds the original data stream such that it can be
326 re-enterd into a fresh rrd file. Depending on the configuration of the new
327 rrd file the resulting data closely matches the data in the original rrd
328 file.
330 If the DS configuration of the new RRD file differs from the original
331 one the B<--src-tmp> and B<--dest-tmp> options can be used to override
332 the default order of DS entries.
334 =head1 BEWARE
336 There are two warnings you should keep in mind:
338 =over
340 =item *
342 This is NEW CODE, so there may be hidden problem. This this first on your real data before doing any major conversions.
344 =item *
346 In my testing there were differences between source and destination which I attribute to
347 quantization issues especially when switching from one consolidation level to the next one.
349 =back
351 =head1 EXAMPLE
353 F<legacy.rrd> has data for the last two years and F<new.rrd> is still empty
354 but created with a start data two years in the past. F<legacy.rrd> contains
355 4 Date Sources (in,out,error,drop) and F<new.rrd> contains 3 datasources
356 (myout,myin,overrun). We want to transfer the old 'in' to 'myin' and 'out'
357 to 'myout' while dropping 'error' and 'drop'.
359 rrdjiig --src-tmpl=in:out --dst-tmpl=myin:myout legacy.rrd new.rrd
361 =head1 COPYRIGHT
363 Copyright (c) 2010 by OETIKER+PARTNER AG. All rights reserved.
365 =head1 LICENSE
367 This program is free software; you can redistribute it and/or modify
368 it under the terms of the GNU General Public License as published by
369 the Free Software Foundation; either version 3 of the License, or
370 (at your option) any later version.
372 This program is distributed in the hope that it will be useful,
373 but WITHOUT ANY WARRANTY; without even the implied warranty of
374 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
375 GNU General Public License for more details.
377 You should have received a copy of the GNU General Public License
378 along with this program; if not, write to the Free Software
379 Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
381 =head1 AUTHOR
383 S<Tobi Oetiker E<lt>tobi@oetiker.chE<gt>>
385 The development of this tool has been sponsored by L<www.init7.net|http://www.init7.net>.
387 =head1 HISTORY
389 2010-02-25 to Initial Version
391 =cut
393 # Emacs Configuration
394 #
395 # Local Variables:
396 # mode: cperl
397 # eval: (cperl-set-style "PerlStyle")
398 # mode: flyspell
399 # mode: flyspell-prog
400 # End:
401 #
402 # vi: sw=4 et