1 /*****************************************************************************
2 * RRDtool 1.0.33 Copyright Tobias Oetiker, 1997, 1998, 1999
3 *****************************************************************************
4 * rrd_format.h RRD Database Format header
5 *****************************************************************************/
7 #ifndef _RRD_FORMAT_H
8 #define _RRD_FORMAT_H
10 #include "rrd.h"
12 /*****************************************************************************
13 * put this in your /usr/lib/magic file (/etc/magic on HPUX)
14 *
15 * # rrd database format
16 * 0 string RRD\0 rrd file
17 * >5 string >\0 version '%s'
18 *
19 *****************************************************************************/
21 #define RRD_COOKIE "RRD"
22 #define RRD_VERSION "0002"
23 #define FLOAT_COOKIE 8.642135E130
25 #include "rrd_nan_inf.h"
27 typedef union unival {
28 unsigned long u_cnt;
29 rrd_value_t u_val;
30 } unival;
33 /****************************************************************************
34 * The RRD Database Structure
35 * ---------------------------
36 *
37 * In oder to properly describe the database structure lets define a few
38 * new words:
39 *
40 * ds - Data Source (ds) providing input to the database. A Data Source (ds)
41 * can be a traffic counter, a temperature, the number of users logged
42 * into a system. The rrd database format can handle the input of
43 * several Data Sources (ds) in a singe database.
44 *
45 * dst - Data Source Type (dst). The Data Source Type (dst) defines the rules
46 * applied to Build Primary Data Points from the input provided by the
47 * data sources (ds).
48 *
49 * pdp - Primary Data Point (pdp). After the database has accepted the
50 * input from the data sources (ds). It starts building Primary
51 * Data Points (pdp) from the data. Primary Data Points (pdp)
52 * are evenly spaced along the time axis (pdp_step). The values
53 * of the Primary Data Points are calculated from the values of
54 * the data source (ds) and the exact time these values were
55 * provided by the data source (ds).
56 *
57 * pdp_st - PDP Start (pdp_st). The moments (pdp_st) in time where
58 * these steps occur are defined by the moments where the
59 * number of seconds since 1970-jan-1 modulo pdp_step equals
60 * zero (pdp_st).
61 *
62 * cf - Consolidation Function (cf). An arbitrary Consolidation Function (cf)
63 * (averaging, min, max) is applied to the primary data points (pdp) to
64 * calculate the consolidated data point.
65 *
66 * cdp - Consolidated Data Point (cdp) is the long term storage format for data
67 * in the rrd database. Consolidated Data Points represent one or
68 * several primary data points collected along the time axis. The
69 * Consolidated Data Points (cdp) are stored in Round Robin Archives
70 * (rra).
71 *
72 * rra - Round Robin Archive (rra). This is the place where the
73 * consolidated data points (cdp) get stored. The data is
74 * organized in rows (row) and columns (col). The Round Robin
75 * Archive got its name from the method data is stored in
76 * there. An RRD database can contain several Round Robin
77 * Archives. Each Round Robin Archive can have a different row
78 * spacing along the time axis (pdp_cnt) and a different
79 * consolidation function (cf) used to build its consolidated
80 * data points (cdp).
81 *
82 * rra_st - RRA Start (rra_st). The moments (rra_st) in time where
83 * Consolidated Data Points (cdp) are added to an rra are
84 * defined by the moments where the number of seconds since
85 * 1970-jan-1 modulo pdp_cnt*pdp_step equals zero (rra_st).
86 *
87 * row - Row (row). A row represent all consolidated data points (cdp)
88 * in a round robin archive who are of the same age.
89 *
90 * col - Column (col). A column (col) represent all consolidated
91 * data points (cdp) in a round robin archive (rra) who
92 * originated from the same data source (ds).
93 *
94 */
96 /****************************************************************************
97 * POS 1: stat_head_t static header of the database
98 ****************************************************************************/
100 typedef struct stat_head_t {
102 /* Data Base Identification Section ***/
103 char cookie[4]; /* RRD */
104 char version[5]; /* version of the format */
105 double float_cookie; /* is it the correct double
106 * representation ? */
108 /* Data Base Structure Definition *****/
109 unsigned long ds_cnt; /* how many different ds provide
110 * input to the rrd */
111 unsigned long rra_cnt; /* how many rras will be maintained
112 * in the rrd */
113 unsigned long pdp_step; /* pdp interval in seconds */
115 unival par[10]; /* global parameters ... unused
116 at the moment */
117 } stat_head_t;
120 /****************************************************************************
121 * POS 2: ds_def_t (* ds_cnt) Data Source definitions
122 ****************************************************************************/
124 enum dst_en { DST_COUNTER=0, /* data source types available */
125 DST_ABSOLUTE,
126 DST_GAUGE,
127 DST_DERIVE,
128 DST_CDEF};
130 enum ds_param_en { DS_mrhb_cnt=0, /* minimum required heartbeat. A
131 * data source must provide input at
132 * least every ds_mrhb seconds,
133 * otherwise it is regarded dead and
134 * will be set to UNKNOWN */
135 DS_min_val, /* the processed input of a ds must */
136 DS_max_val, /* be between max_val and min_val
137 * both can be set to UNKNOWN if you
138 * do not care. Data outside the limits
139 * set to UNKNOWN */
140 DS_cdef = DS_mrhb_cnt}; /* pointer to encoded rpn
141 * expression only applies to DST_CDEF */
143 /* The magic number here is one less than DS_NAM_SIZE */
144 #define DS_NAM_FMT "%19[a-zA-Z0-9_-]"
145 #define DS_NAM_SIZE 20
147 #define DST_FMT "%19[A-Z]"
148 #define DST_SIZE 20
150 typedef struct ds_def_t {
151 char ds_nam[DS_NAM_SIZE]; /* Name of the data source (null terminated)*/
152 char dst[DST_SIZE]; /* Type of data source (null terminated)*/
153 unival par[10]; /* index of this array see ds_param_en */
154 } ds_def_t;
156 /****************************************************************************
157 * POS 3: rra_def_t ( * rra_cnt) one for each store to be maintained
158 ****************************************************************************/
159 enum cf_en { CF_AVERAGE=0, /* data consolidation functions */
160 CF_MINIMUM,
161 CF_MAXIMUM,
162 CF_LAST,
163 CF_HWPREDICT,
164 /* An array of predictions using the seasonal
165 * Holt-Winters algorithm. Requires an RRA of type
166 * CF_SEASONAL for this data source. */
167 CF_SEASONAL,
168 /* An array of seasonal effects. Requires an RRA of
169 * type CF_HWPREDICT for this data source. */
170 CF_DEVPREDICT,
171 /* An array of deviation predictions based upon
172 * smoothed seasonal deviations. Requires an RRA of
173 * type CF_DEVSEASONAL for this data source. */
174 CF_DEVSEASONAL,
175 /* An array of smoothed seasonal deviations. Requires
176 * an RRA of type CF_HWPREDICT for this data source.
177 * */
178 CF_FAILURES};
179 /* A binary array of failure indicators: 1 indicates
180 * that the number of violations in the prescribed
181 * window exceeded the prescribed threshold. */
183 #define MAX_RRA_PAR_EN 10
184 enum rra_par_en { RRA_cdp_xff_val=0, /* what part of the consolidated
185 * datapoint must be known, to produce a
186 * valid entry in the rra */
187 RRA_hw_alpha,
188 /* exponential smoothing parameter for the intercept in
189 * the Holt-Winters prediction algorithm. */
190 RRA_hw_beta,
191 /* exponential smoothing parameter for the slope in
192 * the Holt-Winters prediction algorithm. */
193 RRA_dependent_rra_idx,
194 /* For CF_HWPREDICT: index of the RRA with the seasonal
195 * effects of the Holt-Winters algorithm (of type
196 * CF_SEASONAL).
197 * For CF_DEVPREDICT: index of the RRA with the seasonal
198 * deviation predictions (of type CF_DEVSEASONAL).
199 * For CF_SEASONAL: index of the RRA with the Holt-Winters
200 * intercept and slope coefficient (of type CF_HWPREDICT).
201 * For CF_DEVSEASONAL: index of the RRA with the
202 * Holt-Winters prediction (of type CF_HWPREDICT).
203 * For CF_FAILURES: index of the CF_DEVSEASONAL array.
204 * */
205 RRA_seasonal_smooth_idx,
206 /* For CF_SEASONAL and CF_DEVSEASONAL:
207 * an integer between 0 and row_count - 1 which
208 * is index in the seasonal cycle for applying
209 * the period smoother. */
210 RRA_failure_threshold,
211 /* For CF_FAILURES, number of violations within the last
212 * window required to mark a failure. */
213 RRA_seasonal_gamma = RRA_hw_alpha,
214 /* exponential smoothing parameter for seasonal effects.
215 * */
216 RRA_delta_pos = RRA_hw_alpha,
217 RRA_delta_neg = RRA_hw_beta,
218 /* confidence bound scaling parameters for the
219 * the FAILURES RRA. */
220 RRA_window_len = RRA_seasonal_smooth_idx};
221 /* For CF_FAILURES, the length of the window for measuring
222 * failures. */
224 #define CF_NAM_FMT "%19[A-Z]"
225 #define CF_NAM_SIZE 20
227 typedef struct rra_def_t {
228 char cf_nam[CF_NAM_SIZE];/* consolidation function (null term) */
229 unsigned long row_cnt; /* number of entries in the store */
230 unsigned long pdp_cnt; /* how many primary data points are
231 * required for a consolidated data
232 * point?*/
233 unival par[MAX_RRA_PAR_EN]; /* index see rra_param_en */
235 } rra_def_t;
238 /****************************************************************************
239 ****************************************************************************
240 ****************************************************************************
241 * LIVE PART OF THE HEADER. THIS WILL BE WRITTEN ON EVERY UPDATE *
242 ****************************************************************************
243 ****************************************************************************
244 ****************************************************************************/
245 /****************************************************************************
246 * POS 4: live_head_t
247 ****************************************************************************/
249 typedef struct live_head_t {
250 time_t last_up; /* when was rrd last updated */
251 } live_head_t;
254 /****************************************************************************
255 * POS 5: pdp_prep_t (* ds_cnt) here we prepare the pdps
256 ****************************************************************************/
257 #define LAST_DS_LEN 30 /* DO NOT CHANGE THIS ... */
259 enum pdp_par_en { PDP_unkn_sec_cnt=0, /* how many seconds of the current
260 * pdp value is unknown data? */
262 PDP_val}; /* current value of the pdp.
263 this depends on dst */
265 typedef struct pdp_prep_t{
266 char last_ds[LAST_DS_LEN]; /* the last reading from the data
267 * source. this is stored in ASCII
268 * to cater for very large counters
269 * we might encounter in connection
270 * with SNMP. */
271 unival scratch[10]; /* contents according to pdp_par_en */
272 } pdp_prep_t;
274 /* data is passed from pdp to cdp when seconds since epoch modulo pdp_step == 0
275 obviously the updates do not occur at these times only. Especially does the
276 format allow for updates to occur at different times for each data source.
277 The rules which makes this work is as follows:
279 * DS updates may only occur at ever increasing points in time
280 * When any DS update arrives after a cdp update time, the *previous*
281 update cycle gets executed. All pdps are transfered to cdps and the
282 cdps feed the rras where necessary. Only then the new DS value
283 is loaded into the PDP. */
286 /****************************************************************************
287 * POS 6: cdp_prep_t (* rra_cnt * ds_cnt ) data prep area for cdp values
288 ****************************************************************************/
289 #define MAX_CDP_PAR_EN 10
290 #define MAX_CDP_FAILURES_IDX 8
291 /* max CDP scratch entries avail to record violations for a FAILURES RRA */
292 #define MAX_FAILURES_WINDOW_LEN 28
293 enum cdp_par_en { CDP_val=0,
294 /* the base_interval is always an
295 * average */
296 CDP_unkn_pdp_cnt,
297 /* how many unknown pdp were
298 * integrated. This and the cdp_xff
299 * will decide if this is going to
300 * be a UNKNOWN or a valid value */
301 CDP_hw_intercept,
302 /* Current intercept coefficient for the Holt-Winters
303 * prediction algorithm. */
304 CDP_hw_last_intercept,
305 /* Last iteration intercept coefficient for the Holt-Winters
306 * prediction algorihtm. */
307 CDP_hw_slope,
308 /* Current slope coefficient for the Holt-Winters
309 * prediction algorithm. */
310 CDP_hw_last_slope,
311 /* Last iteration slope coeffient. */
312 CDP_null_count,
313 /* Number of sequential Unknown (DNAN) values + 1 preceding
314 * the current prediction.
315 * */
316 CDP_last_null_count,
317 /* Last iteration count of Unknown (DNAN) values. */
318 CDP_primary_val = 8,
319 /* optimization for bulk updates: the value of the first CDP
320 * value to be written in the bulk update. */
321 CDP_secondary_val = 9,
322 /* optimization for bulk updates: the value of subsequent
323 * CDP values to be written in the bulk update. */
324 CDP_hw_seasonal = CDP_hw_intercept,
325 /* Current seasonal coefficient for the Holt-Winters
326 * prediction algorithm. This is stored in CDP prep to avoid
327 * redundant seek operations. */
328 CDP_hw_last_seasonal = CDP_hw_last_intercept,
329 /* Last iteration seasonal coeffient. */
330 CDP_seasonal_deviation = CDP_hw_intercept,
331 CDP_last_seasonal_deviation = CDP_hw_last_intercept,
332 CDP_init_seasonal = CDP_null_count};
333 /* init_seasonal is a flag which when > 0, forces smoothing updates
334 * to occur when rra_ptr.cur_row == 0 */
336 typedef struct cdp_prep_t{
337 unival scratch[MAX_CDP_PAR_EN];
338 /* contents according to cdp_par_en *
339 * init state should be NAN */
341 } cdp_prep_t;
343 /****************************************************************************
344 * POS 7: rra_ptr_t (* rra_cnt) pointers to the current row in each rra
345 ****************************************************************************/
347 typedef struct rra_ptr_t {
348 unsigned long cur_row; /* current row in the rra*/
349 } rra_ptr_t;
352 /****************************************************************************
353 ****************************************************************************
354 * One single struct to hold all the others. For convenience.
355 ****************************************************************************
356 ****************************************************************************/
357 typedef struct rrd_t {
358 stat_head_t *stat_head; /* the static header */
359 ds_def_t *ds_def; /* list of data source definitions */
360 rra_def_t *rra_def; /* list of round robin archive def */
361 live_head_t *live_head;
362 pdp_prep_t *pdp_prep; /* pdp data prep area */
363 cdp_prep_t *cdp_prep; /* cdp prep area */
364 rra_ptr_t *rra_ptr; /* list of rra pointers */
365 rrd_value_t *rrd_value; /* list of rrd values */
366 } rrd_t;
368 /****************************************************************************
369 ****************************************************************************
370 * AFTER the header section we have the DATA STORAGE AREA it is made up from
371 * Consolidated Data Points organized in Round Robin Archives.
372 ****************************************************************************
373 ****************************************************************************
375 *RRA 0
376 (0,0) .................... ( ds_cnt -1 , 0)
377 .
378 .
379 .
380 (0, row_cnt -1) ... (ds_cnt -1, row_cnt -1)
382 *RRA 1
383 *RRA 2
385 *RRA rra_cnt -1
387 ****************************************************************************/
390 #endif