1 /*****************************************************************************
2 * RRDtool 1.2.23 Copyright by Tobi Oetiker, 1997-2007
3 *****************************************************************************
4 * rrd_open.c Open an RRD File
5 *****************************************************************************
6 * $Id$
7 * $Log$
8 * Revision 1.10 2004/05/26 22:11:12 oetiker
9 * reduce compiler warnings. Many small fixes. -- Mike Slifcak <slif@bellsouth.net>
10 *
11 * Revision 1.9 2003/04/29 21:56:49 oetiker
12 * readline in rrd_open.c reads the file in 8 KB blocks, and calls realloc for
13 * each block. realloc is very slow in Mac OS X for huge blocks, e.g. when
14 * restoring databases from huge xml files. This patch finds the size of the
15 * file, and starts out with malloc'ing the full size.
16 * -- Peter Speck <speck@ruc.dk>
17 *
18 * Revision 1.8 2003/04/11 19:43:44 oetiker
19 * New special value COUNT which allows calculations based on the position of a
20 * value within a data set. Bug fix in rrd_rpncalc.c. PREV returned erroneus
21 * value for the second value. Bug fix in rrd_restore.c. Bug causing seek error
22 * when accesing an RRD restored from an xml that holds an RRD version <3.
23 * -- Ruben Justo <ruben@ainek.com>
24 *
25 * Revision 1.7 2003/03/31 21:22:12 oetiker
26 * enables RRDtool updates with microsecond or in case of windows millisecond
27 * precision. This is needed to reduce time measurement error when archive step
28 * is small. (<30s) -- Sasha Mikheev <sasha@avalon-net.co.il>
29 *
30 * Revision 1.6 2003/02/13 07:05:27 oetiker
31 * Find attached the patch I promised to send to you. Please note that there
32 * are three new source files (src/rrd_is_thread_safe.h, src/rrd_thread_safe.c
33 * and src/rrd_not_thread_safe.c) and the introduction of librrd_th. This
34 * library is identical to librrd, but it contains support code for per-thread
35 * global variables currently used for error information only. This is similar
36 * to how errno per-thread variables are implemented. librrd_th must be linked
37 * alongside of libpthred
38 *
39 * There is also a new file "THREADS", holding some documentation.
40 *
41 * -- Peter Stamfest <peter@stamfest.at>
42 *
43 * Revision 1.5 2002/06/20 00:21:03 jake
44 * More Win32 build changes; thanks to Kerry Calvert.
45 *
46 * Revision 1.4 2002/02/01 20:34:49 oetiker
47 * fixed version number and date/time
48 *
49 * Revision 1.3 2001/03/04 13:01:55 oetiker
50 * Aberrant Behavior Detection support. A brief overview added to rrdtool.pod.
51 * Major updates to rrd_update.c, rrd_create.c. Minor update to other core files.
52 * This is backwards compatible! But new files using the Aberrant stuff are not readable
53 * by old rrdtool versions. See http://cricket.sourceforge.net/aberrant/rrd_hw.htm
54 * -- Jake Brutlag <jakeb@corp.webtv.net>
55 *
56 * Revision 1.2 2001/03/04 10:29:20 oetiker
57 * fixed filedescriptor leak
58 * -- Mike Franusich <mike@franusich.com>
59 *
60 * Revision 1.1.1.1 2001/02/25 22:25:05 oetiker
61 * checkin
62 *
63 *****************************************************************************/
65 #include "rrd_tool.h"
66 #include "unused.h"
67 #define MEMBLK 8192
69 /* DEBUG 2 prints information obtained via mincore(2) */
70 // #define DEBUG 2
71 /* do not calculate exact madvise hints but assume 1 page for headers and
72 * set DONTNEED for the rest, which is assumed to be data */
73 //#define ONE_PAGE 1
74 /* Avoid calling madvise on areas that were already hinted. May be benefical if
75 * your syscalls are very slow */
76 #define CHECK_MADVISE_OVERLAPS 1
78 #ifdef HAVE_MMAP
79 #define __rrd_read(dst, dst_t, cnt) \
80 (dst) = (dst_t*) (data + offset); \
81 offset += sizeof(dst_t) * (cnt)
82 #else
83 #define __rrd_read(dst, dst_t, cnt) \
84 if ((dst = malloc(sizeof(dst_t)*(cnt))) == NULL) { \
85 rrd_set_error(#dst " malloc"); \
86 goto out_nullify_head; \
87 } \
88 offset += read (rrd_file->fd, dst, sizeof(dst_t)*(cnt))
89 #endif
91 /* next page-aligned (i.e. page-align up) */
92 #ifndef PAGE_ALIGN
93 #define PAGE_ALIGN(addr) (((addr)+_page_size-1)&(~(_page_size-1)))
94 #endif
95 /* previous page-aligned (i.e. page-align down) */
96 #ifndef PAGE_ALIGN_DOWN
97 #define PAGE_ALIGN_DOWN(addr) (((addr)+_page_size-1)&(~(_page_size-1)))
98 #endif
100 #ifdef HAVE_MMAP
101 /* vector of last madvise hint */
102 typedef struct _madvise_vec_t {
103 void *start;
104 ssize_t length;
105 } _madvise_vec_t;
106 _madvise_vec_t _madv_vec = { NULL, 0 };
107 #endif
109 #if defined CHECK_MADVISE_OVERLAPS
110 #define _madvise(_start, _off, _hint) \
111 if ((_start) != _madv_vec.start && (ssize_t)(_off) != _madv_vec.length) { \
112 _madv_vec.start = (_start) ; _madv_vec.length = (_off); \
113 madvise((_start), (_off), (_hint)); \
114 }
115 #else
116 #define _madvise(_start, _off, _hint) \
117 madvise((_start), (_off), (_hint))
118 #endif
120 /* Open a database file, return its header and an open filehandle,
121 * positioned to the first cdp in the first rra.
122 * In the error path of rrd_open, only rrd_free(&rrd) has to be called
123 * before returning an error. Do not call rrd_close upon failure of rrd_open.
124 */
126 rrd_file_t *rrd_open(
127 const char *const file_name,
128 rrd_t *rrd,
129 unsigned rdwr)
130 {
131 int flags = 0;
132 mode_t mode = S_IRUSR;
133 int version;
135 #ifdef HAVE_MMAP
136 ssize_t _page_size = sysconf(_SC_PAGESIZE);
137 int mm_prot = PROT_READ, mm_flags = 0;
138 char *data;
139 #endif
140 off_t offset = 0;
141 struct stat statb;
142 rrd_file_t *rrd_file = NULL;
144 rrd_init(rrd);
145 rrd_file = malloc(sizeof(rrd_file_t));
146 if (rrd_file == NULL) {
147 rrd_set_error("allocating rrd_file descriptor for '%s'", file_name);
148 return NULL;
149 }
150 memset(rrd_file, 0, sizeof(rrd_file_t));
152 #ifdef DEBUG
153 if ((rdwr & (RRD_READONLY | RRD_READWRITE)) ==
154 (RRD_READONLY | RRD_READWRITE)) {
155 /* Both READONLY and READWRITE were given, which is invalid. */
156 rrd_set_error("in read/write request mask");
157 exit(-1);
158 }
159 #endif
160 if (rdwr & RRD_READONLY) {
161 flags |= O_RDONLY;
162 #ifdef HAVE_MMAP
163 mm_flags = MAP_PRIVATE;
164 # ifdef MAP_NORESERVE
165 mm_flags |= MAP_NORESERVE; /* readonly, so no swap backing needed */
166 # endif
167 #endif
168 } else {
169 if (rdwr & RRD_READWRITE) {
170 mode |= S_IWUSR;
171 flags |= O_RDWR;
172 #ifdef HAVE_MMAP
173 mm_flags = MAP_SHARED;
174 mm_prot |= PROT_WRITE;
175 #endif
176 }
177 if (rdwr & RRD_CREAT) {
178 flags |= (O_CREAT | O_TRUNC);
179 }
180 }
181 if (rdwr & RRD_READAHEAD) {
182 #ifdef MAP_POPULATE
183 mm_flags |= MAP_POPULATE; /* populate ptes and data */
184 #endif
185 #if defined MAP_NONBLOCK
186 mm_flags |= MAP_NONBLOCK; /* just populate ptes */
187 #endif
188 #ifdef USE_DIRECT_IO
189 } else {
190 flags |= O_DIRECT;
191 #endif
192 }
193 #ifdef O_NONBLOCK
194 flags |= O_NONBLOCK;
195 #endif
197 if ((rrd_file->fd = open(file_name, flags, mode)) < 0) {
198 rrd_set_error("opening '%s': %s", file_name, rrd_strerror(errno));
199 goto out_free;
200 }
202 /* Better try to avoid seeks as much as possible. stat may be heavy but
203 * many concurrent seeks are even worse. */
204 if ((fstat(rrd_file->fd, &statb)) < 0) {
205 rrd_set_error("fstat '%s': %s", file_name, rrd_strerror(errno));
206 goto out_close;
207 }
208 rrd_file->file_len = statb.st_size;
210 #ifdef HAVE_POSIX_FADVISE
211 /* In general we need no read-ahead when dealing with rrd_files.
212 When we stop reading, it is highly unlikely that we start up again.
213 In this manner we actually save time and diskaccess (and buffer cache).
214 Thanks to Dave Plonka for the Idea of using POSIX_FADV_RANDOM here. */
215 if (0 != posix_fadvise(rrd_file->fd, 0, 0, POSIX_FADV_RANDOM)) {
216 rrd_set_error("setting POSIX_FADV_RANDOM on '%s': %s", file_name,
217 rrd_strerror(errno));
218 goto out_close;
219 }
220 #endif
222 /*
223 if (rdwr & RRD_READWRITE)
224 {
225 if (setvbuf((rrd_file->fd),NULL,_IONBF,2)) {
226 rrd_set_error("failed to disable the stream buffer\n");
227 return (-1);
228 }
229 }
230 */
231 #ifdef HAVE_MMAP
232 data = mmap(0, rrd_file->file_len, mm_prot, mm_flags,
233 rrd_file->fd, offset);
235 /* lets see if the first read worked */
236 if (data == MAP_FAILED) {
237 rrd_set_error("mmaping file '%s': %s", file_name,
238 rrd_strerror(errno));
239 goto out_close;
240 }
241 rrd_file->file_start = data;
242 #endif
243 #ifdef USE_MADVISE
244 if (rdwr & RRD_COPY) {
245 /* We will read everything in a moment (copying) */
246 _madvise(data, rrd_file->file_len, MADV_WILLNEED | MADV_SEQUENTIAL);
247 goto out_done;
248 }
249 /* We do not need to read anything in for the moment */
250 #ifndef ONE_PAGE
251 _madvise(data, rrd_file->file_len, MADV_DONTNEED);
252 // _madvise(data, rrd_file->file_len, MADV_RANDOM);
253 #else
254 /* alternatively: keep 2 pages worth of data, likely headers,
255 * don't need the rest. */
256 _madvise(data, _page_size, MADV_WILLNEED | MADV_SEQUENTIAL);
257 _madvise(data + _page_size, (rrd_file->file_len >= _page_size)
258 ? rrd_file->file_len - _page_size : 0, MADV_DONTNEED);
259 #endif
260 #endif
262 #if defined USE_MADVISE && !defined ONE_PAGE
263 /* the stat_head will be needed soonish, so hint accordingly */
264 // too finegrained to calc the individual sizes, just keep 2 pages worth of hdr
265 _madvise(data + PAGE_ALIGN_DOWN(offset), PAGE_ALIGN(sizeof(stat_head_t)),
266 MADV_WILLNEED);
268 #endif
270 __rrd_read(rrd->stat_head, stat_head_t,
271 1);
273 /* lets do some test if we are on track ... */
274 if (memcmp(rrd->stat_head->cookie, RRD_COOKIE, sizeof(RRD_COOKIE)) != 0) {
275 rrd_set_error("'%s' is not an RRD file", file_name);
276 goto out_nullify_head;
277 }
279 if (rrd->stat_head->float_cookie != FLOAT_COOKIE) {
280 rrd_set_error("This RRD was created on another architecture");
281 goto out_nullify_head;
282 }
284 version = atoi(rrd->stat_head->version);
286 if (version > atoi(RRD_VERSION)) {
287 rrd_set_error("can't handle RRD file version %s",
288 rrd->stat_head->version);
289 goto out_nullify_head;
290 }
291 #if defined USE_MADVISE && !defined ONE_PAGE
292 /* the ds_def will be needed soonish, so hint accordingly */
293 _madvise(data + PAGE_ALIGN_DOWN(offset),
294 PAGE_ALIGN(sizeof(ds_def_t) * rrd->stat_head->ds_cnt),
295 MADV_WILLNEED);
296 #endif
297 __rrd_read(rrd->ds_def, ds_def_t,
298 rrd->stat_head->ds_cnt);
300 #if defined USE_MADVISE && !defined ONE_PAGE
301 /* the rra_def will be needed soonish, so hint accordingly */
302 _madvise(data + PAGE_ALIGN_DOWN(offset),
303 PAGE_ALIGN(sizeof(rra_def_t) * rrd->stat_head->rra_cnt),
304 MADV_WILLNEED);
305 #endif
306 __rrd_read(rrd->rra_def, rra_def_t,
307 rrd->stat_head->rra_cnt);
309 /* handle different format for the live_head */
310 if (version < 3) {
311 rrd->live_head = (live_head_t *) malloc(sizeof(live_head_t));
312 if (rrd->live_head == NULL) {
313 rrd_set_error("live_head_t malloc");
314 goto out_close;
315 }
316 #ifdef HAVE_MMAP
317 memmove(&rrd->live_head->last_up, data + offset, sizeof(long));
318 offset += sizeof(long);
319 #else
320 offset += read(rrd_file->fd, &rrd->live_head->last_up, sizeof(long));
321 #endif
322 rrd->live_head->last_up_usec = 0;
323 } else {
324 #if defined USE_MADVISE && !defined ONE_PAGE
325 /* the live_head will be needed soonish, so hint accordingly */
326 _madvise(data + PAGE_ALIGN_DOWN(offset),
327 PAGE_ALIGN(sizeof(live_head_t)), MADV_WILLNEED);
328 #endif
329 __rrd_read(rrd->live_head, live_head_t,
330 1);
331 }
332 //XXX: This doesn't look like it needs madvise
333 __rrd_read(rrd->pdp_prep, pdp_prep_t,
334 rrd->stat_head->ds_cnt);
336 //XXX: This could benefit from madvise()ing
337 __rrd_read(rrd->cdp_prep, cdp_prep_t,
338 rrd->stat_head->rra_cnt * rrd->stat_head->ds_cnt);
340 //XXX: This could benefit from madvise()ing
341 __rrd_read(rrd->rra_ptr, rra_ptr_t,
342 rrd->stat_head->rra_cnt);
344 #ifdef USE_MADVISE
345 out_done:
346 #endif
347 rrd_file->header_len = offset;
348 rrd_file->pos = offset;
350 return (rrd_file);
351 out_nullify_head:
352 rrd->stat_head = NULL;
353 out_close:
354 close(rrd_file->fd);
355 out_free:
356 free(rrd_file);
357 return NULL;
358 }
361 /* Close a reference to an rrd_file. */
363 int rrd_close(
364 rrd_file_t *rrd_file)
365 {
366 int ret;
368 #if defined HAVE_MMAP || defined DEBUG
369 ssize_t _page_size = sysconf(_SC_PAGESIZE);
370 #endif
371 #if defined DEBUG && DEBUG > 1
372 /* pretty print blocks in core */
373 off_t off;
374 unsigned char *vec;
376 off = rrd_file->file_len +
377 ((rrd_file->file_len + _page_size - 1) / _page_size);
378 vec = malloc(off);
379 if (vec != NULL) {
380 memset(vec, 0, off);
381 if (mincore(rrd_file->file_start, rrd_file->file_len, vec) == 0) {
382 int prev;
383 unsigned is_in = 0, was_in = 0;
385 for (off = 0, prev = 0; off < rrd_file->file_len; ++off) {
386 is_in = vec[off] & 1; /* if lsb set then is core resident */
387 if (off == 0)
388 was_in = is_in;
389 if (was_in != is_in) {
390 fprintf(stderr, "%sin core: %p len %ld\n",
391 was_in ? "" : "not ", vec + prev, off - prev);
392 was_in = is_in;
393 prev = off;
394 }
395 }
396 fprintf(stderr,
397 "%sin core: %p len %ld\n",
398 was_in ? "" : "not ", vec + prev, off - prev);
399 } else
400 fprintf(stderr, "mincore: %s", rrd_strerror(errno));
401 }
402 #endif /* DEBUG */
404 #ifdef USE_MADVISE
405 # ifdef ONE_PAGE
406 /* Keep headers around, round up to next page boundary. */
407 ret =
408 PAGE_ALIGN(rrd_file->header_len % _page_size + rrd_file->header_len);
409 if (rrd_file->file_len > ret)
410 _madvise(rrd_file->file_start + ret,
411 rrd_file->file_len - ret, MADV_DONTNEED);
412 # else
413 /* ignoring errors from RRDs that are smaller then the file_len+rounding */
414 _madvise(rrd_file->file_start + PAGE_ALIGN_DOWN(rrd_file->header_len),
415 rrd_file->file_len - PAGE_ALIGN(rrd_file->header_len),
416 MADV_DONTNEED);
417 # endif
418 #endif
419 #ifdef HAVE_MMAP
420 ret = munmap(rrd_file->file_start, rrd_file->file_len);
421 if (ret != 0)
422 rrd_set_error("munmap rrd_file: %s", rrd_strerror(errno));
423 #endif
424 ret = close(rrd_file->fd);
425 if (ret != 0)
426 rrd_set_error("closing file: %s", rrd_strerror(errno));
427 free(rrd_file);
428 rrd_file = NULL;
429 return ret;
430 }
433 /* Set position of rrd_file. */
435 off_t rrd_seek(
436 rrd_file_t *rrd_file,
437 off_t off,
438 int whence)
439 {
440 off_t ret = 0;
442 #ifdef HAVE_MMAP
443 if (whence == SEEK_SET)
444 rrd_file->pos = off;
445 else if (whence == SEEK_CUR)
446 rrd_file->pos += off;
447 else if (whence == SEEK_END)
448 rrd_file->pos = rrd_file->file_len + off;
449 #else
450 ret = lseek(rrd_file->fd, off, whence);
451 if (ret < 0)
452 rrd_set_error("lseek: %s", rrd_strerror(errno));
453 rrd_file->pos = ret;
454 #endif
455 //XXX: mimic fseek, which returns 0 upon success
456 return ret == -1; //XXX: or just ret to mimic lseek
457 }
460 /* Get current position in rrd_file. */
462 inline off_t rrd_tell(
463 rrd_file_t *rrd_file)
464 {
465 return rrd_file->pos;
466 }
469 /* read count bytes into buffer buf, starting at rrd_file->pos.
470 * Returns the number of bytes read. */
472 inline ssize_t rrd_read(
473 rrd_file_t *rrd_file,
474 void *buf,
475 size_t count)
476 {
477 #ifdef HAVE_MMAP
478 buf = memcpy(buf, rrd_file->file_start + rrd_file->pos, count);
479 rrd_file->pos += count; /* mimmic read() semantics */
480 return count;
481 #else
482 ssize_t ret;
484 ret = read(rrd_file->fd, buf, count);
485 //XXX: eventually add generic rrd_set_error(""); here
486 rrd_file->pos += count; /* mimmic read() semantics */
487 return ret;
488 #endif
489 }
492 /* write count bytes from buffer buf to the current position
493 * rrd_file->pos of rrd_file->fd.
494 * Returns the number of bytes written. */
496 inline ssize_t rrd_write(
497 rrd_file_t *rrd_file,
498 const void *buf,
499 size_t count)
500 {
501 #ifdef HAVE_MMAP
502 memcpy(rrd_file->file_start + rrd_file->pos, buf, count);
503 rrd_file->pos += count;
504 return count; /* mimmic write() semantics */
505 #else
506 ssize_t _sz = write(rrd_file->fd, buf, count);
507 if (_sz > 0)
508 rrd_file->pos += _sz;
509 return _sz;
510 #endif
511 }
514 /* flush all data pending to be written to FD. */
516 inline void rrd_flush(
517 rrd_file_t *rrd_file)
518 {
519 if (fdatasync(rrd_file->fd) != 0) {
520 rrd_set_error("flushing fd %d: %s", rrd_file->fd,
521 rrd_strerror(errno));
522 }
523 }
526 /* Initialize RRD header. */
528 void rrd_init(
529 rrd_t *rrd)
530 {
531 rrd->stat_head = NULL;
532 rrd->ds_def = NULL;
533 rrd->rra_def = NULL;
534 rrd->live_head = NULL;
535 rrd->rra_ptr = NULL;
536 rrd->pdp_prep = NULL;
537 rrd->cdp_prep = NULL;
538 rrd->rrd_value = NULL;
539 }
542 /* free RRD header data. */
544 #ifdef HAVE_MMAP
545 inline void rrd_free(
546 rrd_t UNUSED(*rrd))
547 {
548 }
549 #else
550 void rrd_free(
551 rrd_t *rrd)
552 {
553 if (atoi(rrd->stat_head->version) < 3)
554 free(rrd->live_head);
555 free(rrd->stat_head);
556 free(rrd->ds_def);
557 free(rrd->rra_def);
558 free(rrd->rra_ptr);
559 free(rrd->pdp_prep);
560 free(rrd->cdp_prep);
561 free(rrd->rrd_value);
562 }
563 #endif
566 /* routine used by external libraries to free memory allocated by
567 * rrd library */
569 void rrd_freemem(
570 void *mem)
571 {
572 free(mem);
573 }
576 /* XXX: FIXME: missing documentation. */
577 /*XXX: FIXME should be renamed to rrd_readfile or _rrd_readfile */
579 int /*_rrd_*/ readfile(
580 const char *file_name,
581 char **buffer,
582 int skipfirst)
583 {
584 long writecnt = 0, totalcnt = MEMBLK;
585 long offset = 0;
586 FILE *input = NULL;
587 char c;
589 if ((strcmp("-", file_name) == 0)) {
590 input = stdin;
591 } else {
592 if ((input = fopen(file_name, "rb")) == NULL) {
593 rrd_set_error("opening '%s': %s", file_name, rrd_strerror(errno));
594 return (-1);
595 }
596 }
597 if (skipfirst) {
598 do {
599 c = getc(input);
600 offset++;
601 } while (c != '\n' && !feof(input));
602 }
603 if (strcmp("-", file_name)) {
604 fseek(input, 0, SEEK_END);
605 /* have extra space for detecting EOF without realloc */
606 totalcnt = (ftell(input) + 1) / sizeof(char) - offset;
607 if (totalcnt < MEMBLK)
608 totalcnt = MEMBLK; /* sanitize */
609 fseek(input, offset * sizeof(char), SEEK_SET);
610 }
611 if (((*buffer) = (char *) malloc((totalcnt + 4) * sizeof(char))) == NULL) {
612 perror("Allocate Buffer:");
613 exit(1);
614 };
615 do {
616 writecnt +=
617 fread((*buffer) + writecnt, 1,
618 (totalcnt - writecnt) * sizeof(char), input);
619 if (writecnt >= totalcnt) {
620 totalcnt += MEMBLK;
621 if (((*buffer) =
622 rrd_realloc((*buffer),
623 (totalcnt + 4) * sizeof(char))) == NULL) {
624 perror("Realloc Buffer:");
625 exit(1);
626 };
627 }
628 } while (!feof(input));
629 (*buffer)[writecnt] = '\0';
630 if (strcmp("-", file_name) != 0) {
631 fclose(input);
632 };
633 return writecnt;
634 }