1 /*****************************************************************************
2 * RRDtool 1.3.2 Copyright by Tobi Oetiker, 1997-2008
3 *****************************************************************************
4 * rrd_format.h RRD Database Format header
5 *****************************************************************************/
7 #ifndef _RRD_FORMAT_H
8 #define _RRD_FORMAT_H
10 /*
11 * _RRD_TOOL_H
12 * We're building RRDTool itself.
13 *
14 * RRD_EXPORT_DEPRECATED
15 * User is requesting internal function which need this struct. They have
16 * been told that this will change and have agreed to adapt their programs.
17 */
18 #if !defined(_RRD_TOOL_H) && !defined(RRD_EXPORT_DEPRECATED)
19 # error "Do not include rrd_format.h directly. Include rrd.h instead!"
20 #endif
22 #include "rrd.h"
24 /*****************************************************************************
25 * put this in your /usr/lib/magic file (/etc/magic on HPUX)
26 *
27 * # rrd database format
28 * 0 string RRD\0 rrd file
29 * >5 string >\0 version '%s'
30 *
31 *****************************************************************************/
33 #define RRD_COOKIE "RRD"
34 /* #define RRD_VERSION "0002" */
35 /* changed because microsecond precision requires another field */
36 #define RRD_VERSION "0004"
37 #define RRD_VERSION3 "0003"
38 #define FLOAT_COOKIE 8.642135E130
40 typedef union unival {
41 unsigned long u_cnt;
42 rrd_value_t u_val;
43 } unival;
46 /****************************************************************************
47 * The RRD Database Structure
48 * ---------------------------
49 *
50 * In oder to properly describe the database structure lets define a few
51 * new words:
52 *
53 * ds - Data Source (ds) providing input to the database. A Data Source (ds)
54 * can be a traffic counter, a temperature, the number of users logged
55 * into a system. The rrd database format can handle the input of
56 * several Data Sources (ds) in a singe database.
57 *
58 * dst - Data Source Type (dst). The Data Source Type (dst) defines the rules
59 * applied to Build Primary Data Points from the input provided by the
60 * data sources (ds).
61 *
62 * pdp - Primary Data Point (pdp). After the database has accepted the
63 * input from the data sources (ds). It starts building Primary
64 * Data Points (pdp) from the data. Primary Data Points (pdp)
65 * are evenly spaced along the time axis (pdp_step). The values
66 * of the Primary Data Points are calculated from the values of
67 * the data source (ds) and the exact time these values were
68 * provided by the data source (ds).
69 *
70 * pdp_st - PDP Start (pdp_st). The moments (pdp_st) in time where
71 * these steps occur are defined by the moments where the
72 * number of seconds since 1970-jan-1 modulo pdp_step equals
73 * zero (pdp_st).
74 *
75 * cf - Consolidation Function (cf). An arbitrary Consolidation Function (cf)
76 * (averaging, min, max) is applied to the primary data points (pdp) to
77 * calculate the consolidated data point.
78 *
79 * cdp - Consolidated Data Point (cdp) is the long term storage format for data
80 * in the rrd database. Consolidated Data Points represent one or
81 * several primary data points collected along the time axis. The
82 * Consolidated Data Points (cdp) are stored in Round Robin Archives
83 * (rra).
84 *
85 * rra - Round Robin Archive (rra). This is the place where the
86 * consolidated data points (cdp) get stored. The data is
87 * organized in rows (row) and columns (col). The Round Robin
88 * Archive got its name from the method data is stored in
89 * there. An RRD database can contain several Round Robin
90 * Archives. Each Round Robin Archive can have a different row
91 * spacing along the time axis (pdp_cnt) and a different
92 * consolidation function (cf) used to build its consolidated
93 * data points (cdp).
94 *
95 * rra_st - RRA Start (rra_st). The moments (rra_st) in time where
96 * Consolidated Data Points (cdp) are added to an rra are
97 * defined by the moments where the number of seconds since
98 * 1970-jan-1 modulo pdp_cnt*pdp_step equals zero (rra_st).
99 *
100 * row - Row (row). A row represent all consolidated data points (cdp)
101 * in a round robin archive who are of the same age.
102 *
103 * col - Column (col). A column (col) represent all consolidated
104 * data points (cdp) in a round robin archive (rra) who
105 * originated from the same data source (ds).
106 *
107 */
109 /****************************************************************************
110 * POS 1: stat_head_t static header of the database
111 ****************************************************************************/
113 typedef struct stat_head_t {
115 /* Data Base Identification Section ** */
116 char cookie[4]; /* RRD */
117 char version[5]; /* version of the format */
118 double float_cookie; /* is it the correct double
119 * representation ? */
121 /* Data Base Structure Definition **** */
122 unsigned long ds_cnt; /* how many different ds provide
123 * input to the rrd */
124 unsigned long rra_cnt; /* how many rras will be maintained
125 * in the rrd */
126 unsigned long pdp_step; /* pdp interval in seconds */
128 unival par[10]; /* global parameters ... unused
129 at the moment */
130 } stat_head_t;
133 /****************************************************************************
134 * POS 2: ds_def_t (* ds_cnt) Data Source definitions
135 ****************************************************************************/
137 enum dst_en { DST_COUNTER = 0, /* data source types available */
138 DST_ABSOLUTE,
139 DST_GAUGE,
140 DST_DERIVE,
141 DST_CDEF
142 };
144 enum ds_param_en { DS_mrhb_cnt = 0, /* minimum required heartbeat. A
145 * data source must provide input at
146 * least every ds_mrhb seconds,
147 * otherwise it is regarded dead and
148 * will be set to UNKNOWN */
149 DS_min_val, /* the processed input of a ds must */
150 DS_max_val, /* be between max_val and min_val
151 * both can be set to UNKNOWN if you
152 * do not care. Data outside the limits
153 * set to UNKNOWN */
154 DS_cdef = DS_mrhb_cnt
155 }; /* pointer to encoded rpn
156 * expression only applies to DST_CDEF */
158 /* The magic number here is one less than DS_NAM_SIZE */
159 #define DS_NAM_FMT "%19[a-zA-Z0-9_-]"
160 #define DS_NAM_SIZE 20
162 #define DST_FMT "%19[A-Z]"
163 #define DST_SIZE 20
165 typedef struct ds_def_t {
166 char ds_nam[DS_NAM_SIZE]; /* Name of the data source (null terminated) */
167 char dst[DST_SIZE]; /* Type of data source (null terminated) */
168 unival par[10]; /* index of this array see ds_param_en */
169 } ds_def_t;
171 /****************************************************************************
172 * POS 3: rra_def_t ( * rra_cnt) one for each store to be maintained
173 ****************************************************************************/
174 enum cf_en { CF_AVERAGE = 0, /* data consolidation functions */
175 CF_MINIMUM,
176 CF_MAXIMUM,
177 CF_LAST,
178 CF_HWPREDICT,
179 /* An array of predictions using the seasonal
180 * Holt-Winters algorithm. Requires an RRA of type
181 * CF_SEASONAL for this data source. */
182 CF_SEASONAL,
183 /* An array of seasonal effects. Requires an RRA of
184 * type CF_HWPREDICT for this data source. */
185 CF_DEVPREDICT,
186 /* An array of deviation predictions based upon
187 * smoothed seasonal deviations. Requires an RRA of
188 * type CF_DEVSEASONAL for this data source. */
189 CF_DEVSEASONAL,
190 /* An array of smoothed seasonal deviations. Requires
191 * an RRA of type CF_HWPREDICT for this data source.
192 * */
193 CF_FAILURES,
194 /* HWPREDICT that follows a moving baseline */
195 CF_MHWPREDICT
196 /* new entries must come last !!! */
197 };
199 /* A binary array of failure indicators: 1 indicates
200 * that the number of violations in the prescribed
201 * window exceeded the prescribed threshold. */
203 #define MAX_RRA_PAR_EN 10
204 enum rra_par_en { RRA_cdp_xff_val = 0, /* what part of the consolidated
205 * datapoint must be known, to produce a
206 * valid entry in the rra */
207 /* CF_HWPREDICT: */
208 RRA_hw_alpha = 1,
209 /* exponential smoothing parameter for the intercept in
210 * the Holt-Winters prediction algorithm. */
211 RRA_hw_beta = 2,
212 /* exponential smoothing parameter for the slope in
213 * the Holt-Winters prediction algorithm. */
215 RRA_dependent_rra_idx = 3,
216 /* For CF_HWPREDICT: index of the RRA with the seasonal
217 * effects of the Holt-Winters algorithm (of type
218 * CF_SEASONAL).
219 * For CF_DEVPREDICT: index of the RRA with the seasonal
220 * deviation predictions (of type CF_DEVSEASONAL).
221 * For CF_SEASONAL: index of the RRA with the Holt-Winters
222 * intercept and slope coefficient (of type CF_HWPREDICT).
223 * For CF_DEVSEASONAL: index of the RRA with the
224 * Holt-Winters prediction (of type CF_HWPREDICT).
225 * For CF_FAILURES: index of the CF_DEVSEASONAL array.
226 * */
228 /* CF_SEASONAL and CF_DEVSEASONAL: */
229 RRA_seasonal_gamma = 1,
230 /* exponential smoothing parameter for seasonal effects. */
232 RRA_seasonal_smoothing_window = 2,
233 /* fraction of the season to include in the running average
234 * smoother */
236 /* RRA_dependent_rra_idx = 3, */
238 RRA_seasonal_smooth_idx = 4,
239 /* an integer between 0 and row_count - 1 which
240 * is index in the seasonal cycle for applying
241 * the period smoother. */
243 /* CF_FAILURES: */
244 RRA_delta_pos = 1, /* confidence bound scaling parameters */
245 RRA_delta_neg = 2,
246 /* RRA_dependent_rra_idx = 3, */
247 RRA_window_len = 4,
248 RRA_failure_threshold = 5,
249 /* For CF_FAILURES, number of violations within the last
250 * window required to mark a failure. */
251 };
253 /* For CF_FAILURES, the length of the window for measuring
254 * failures. */
256 #define CF_NAM_FMT "%19[A-Z]"
257 #define CF_NAM_SIZE 20
259 typedef struct rra_def_t {
260 char cf_nam[CF_NAM_SIZE]; /* consolidation function (null term) */
261 unsigned long row_cnt; /* number of entries in the store */
262 unsigned long pdp_cnt; /* how many primary data points are
263 * required for a consolidated data
264 * point?*/
265 unival par[MAX_RRA_PAR_EN]; /* index see rra_param_en */
267 } rra_def_t;
270 /****************************************************************************
271 ****************************************************************************
272 ****************************************************************************
273 * LIVE PART OF THE HEADER. THIS WILL BE WRITTEN ON EVERY UPDATE *
274 ****************************************************************************
275 ****************************************************************************
276 ****************************************************************************/
277 /****************************************************************************
278 * POS 4: live_head_t
279 ****************************************************************************/
281 typedef struct live_head_t {
282 time_t last_up; /* when was rrd last updated */
283 long last_up_usec; /* micro seconds part of the
284 update timestamp. Always >= 0 */
285 } live_head_t;
288 /****************************************************************************
289 * POS 5: pdp_prep_t (* ds_cnt) here we prepare the pdps
290 ****************************************************************************/
291 #define LAST_DS_LEN 30 /* DO NOT CHANGE THIS ... */
293 enum pdp_par_en { PDP_unkn_sec_cnt = 0, /* how many seconds of the current
294 * pdp value is unknown data? */
296 PDP_val
297 }; /* current value of the pdp.
298 this depends on dst */
300 typedef struct pdp_prep_t {
301 char last_ds[LAST_DS_LEN]; /* the last reading from the data
302 * source. this is stored in ASCII
303 * to cater for very large counters
304 * we might encounter in connection
305 * with SNMP. */
306 unival scratch[10]; /* contents according to pdp_par_en */
307 } pdp_prep_t;
309 /* data is passed from pdp to cdp when seconds since epoch modulo pdp_step == 0
310 obviously the updates do not occur at these times only. Especially does the
311 format allow for updates to occur at different times for each data source.
312 The rules which makes this work is as follows:
314 * DS updates may only occur at ever increasing points in time
315 * When any DS update arrives after a cdp update time, the *previous*
316 update cycle gets executed. All pdps are transfered to cdps and the
317 cdps feed the rras where necessary. Only then the new DS value
318 is loaded into the PDP. */
321 /****************************************************************************
322 * POS 6: cdp_prep_t (* rra_cnt * ds_cnt ) data prep area for cdp values
323 ****************************************************************************/
324 #define MAX_CDP_PAR_EN 10
325 #define MAX_CDP_FAILURES_IDX 8
326 /* max CDP scratch entries avail to record violations for a FAILURES RRA */
327 #define MAX_FAILURES_WINDOW_LEN 28
328 enum cdp_par_en { CDP_val = 0,
329 /* the base_interval is always an
330 * average */
331 CDP_unkn_pdp_cnt,
332 /* how many unknown pdp were
333 * integrated. This and the cdp_xff
334 * will decide if this is going to
335 * be a UNKNOWN or a valid value */
336 CDP_hw_intercept,
337 /* Current intercept coefficient for the Holt-Winters
338 * prediction algorithm. */
339 CDP_hw_last_intercept,
340 /* Last iteration intercept coefficient for the Holt-Winters
341 * prediction algorihtm. */
342 CDP_hw_slope,
343 /* Current slope coefficient for the Holt-Winters
344 * prediction algorithm. */
345 CDP_hw_last_slope,
346 /* Last iteration slope coeffient. */
347 CDP_null_count,
348 /* Number of sequential Unknown (DNAN) values + 1 preceding
349 * the current prediction.
350 * */
351 CDP_last_null_count,
352 /* Last iteration count of Unknown (DNAN) values. */
353 CDP_primary_val = 8,
354 /* optimization for bulk updates: the value of the first CDP
355 * value to be written in the bulk update. */
356 CDP_secondary_val = 9,
357 /* optimization for bulk updates: the value of subsequent
358 * CDP values to be written in the bulk update. */
359 CDP_hw_seasonal = CDP_hw_intercept,
360 /* Current seasonal coefficient for the Holt-Winters
361 * prediction algorithm. This is stored in CDP prep to avoid
362 * redundant seek operations. */
363 CDP_hw_last_seasonal = CDP_hw_last_intercept,
364 /* Last iteration seasonal coeffient. */
365 CDP_seasonal_deviation = CDP_hw_intercept,
366 CDP_last_seasonal_deviation = CDP_hw_last_intercept,
367 CDP_init_seasonal = CDP_null_count
368 };
370 /* init_seasonal is a flag which when > 0, forces smoothing updates
371 * to occur when rra_ptr.cur_row == 0 */
373 typedef struct cdp_prep_t {
374 unival scratch[MAX_CDP_PAR_EN];
375 /* contents according to cdp_par_en *
376 * init state should be NAN */
378 } cdp_prep_t;
380 /****************************************************************************
381 * POS 7: rra_ptr_t (* rra_cnt) pointers to the current row in each rra
382 ****************************************************************************/
384 typedef struct rra_ptr_t {
385 unsigned long cur_row; /* current row in the rra */
386 } rra_ptr_t;
389 /****************************************************************************
390 ****************************************************************************
391 * One single struct to hold all the others. For convenience.
392 ****************************************************************************
393 ****************************************************************************/
394 typedef struct rrd_t {
395 stat_head_t *stat_head; /* the static header */
396 ds_def_t *ds_def; /* list of data source definitions */
397 rra_def_t *rra_def; /* list of round robin archive def */
398 live_head_t *live_head; /* rrd v >= 3 last_up with us */
399 time_t *legacy_last_up; /* rrd v < 3 last_up time */
400 pdp_prep_t *pdp_prep; /* pdp data prep area */
401 cdp_prep_t *cdp_prep; /* cdp prep area */
402 rra_ptr_t *rra_ptr; /* list of rra pointers */
403 rrd_value_t *rrd_value; /* list of rrd values */
404 } rrd_t;
406 /****************************************************************************
407 ****************************************************************************
408 * AFTER the header section we have the DATA STORAGE AREA it is made up from
409 * Consolidated Data Points organized in Round Robin Archives.
410 ****************************************************************************
411 ****************************************************************************
413 *RRA 0
414 (0,0) .................... ( ds_cnt -1 , 0)
415 .
416 .
417 .
418 (0, row_cnt -1) ... (ds_cnt -1, row_cnt -1)
420 *RRA 1
421 *RRA 2
423 *RRA rra_cnt -1
425 ****************************************************************************/
428 #endif