1 /*****************************************************************************
2 * RRDtool 1.2.23 Copyright by Tobi Oetiker, 1997-2007
3 *****************************************************************************
4 * rrd_open.c Open an RRD File
5 *****************************************************************************
6 * $Id$
7 * $Log$
8 * Revision 1.10 2004/05/26 22:11:12 oetiker
9 * reduce compiler warnings. Many small fixes. -- Mike Slifcak <slif@bellsouth.net>
10 *
11 * Revision 1.9 2003/04/29 21:56:49 oetiker
12 * readline in rrd_open.c reads the file in 8 KB blocks, and calls realloc for
13 * each block. realloc is very slow in Mac OS X for huge blocks, e.g. when
14 * restoring databases from huge xml files. This patch finds the size of the
15 * file, and starts out with malloc'ing the full size.
16 * -- Peter Speck <speck@ruc.dk>
17 *
18 * Revision 1.8 2003/04/11 19:43:44 oetiker
19 * New special value COUNT which allows calculations based on the position of a
20 * value within a data set. Bug fix in rrd_rpncalc.c. PREV returned erroneus
21 * value for the second value. Bug fix in rrd_restore.c. Bug causing seek error
22 * when accesing an RRD restored from an xml that holds an RRD version <3.
23 * -- Ruben Justo <ruben@ainek.com>
24 *
25 * Revision 1.7 2003/03/31 21:22:12 oetiker
26 * enables RRDtool updates with microsecond or in case of windows millisecond
27 * precision. This is needed to reduce time measurement error when archive step
28 * is small. (<30s) -- Sasha Mikheev <sasha@avalon-net.co.il>
29 *
30 * Revision 1.6 2003/02/13 07:05:27 oetiker
31 * Find attached the patch I promised to send to you. Please note that there
32 * are three new source files (src/rrd_is_thread_safe.h, src/rrd_thread_safe.c
33 * and src/rrd_not_thread_safe.c) and the introduction of librrd_th. This
34 * library is identical to librrd, but it contains support code for per-thread
35 * global variables currently used for error information only. This is similar
36 * to how errno per-thread variables are implemented. librrd_th must be linked
37 * alongside of libpthred
38 *
39 * There is also a new file "THREADS", holding some documentation.
40 *
41 * -- Peter Stamfest <peter@stamfest.at>
42 *
43 * Revision 1.5 2002/06/20 00:21:03 jake
44 * More Win32 build changes; thanks to Kerry Calvert.
45 *
46 * Revision 1.4 2002/02/01 20:34:49 oetiker
47 * fixed version number and date/time
48 *
49 * Revision 1.3 2001/03/04 13:01:55 oetiker
50 * Aberrant Behavior Detection support. A brief overview added to rrdtool.pod.
51 * Major updates to rrd_update.c, rrd_create.c. Minor update to other core files.
52 * This is backwards compatible! But new files using the Aberrant stuff are not readable
53 * by old rrdtool versions. See http://cricket.sourceforge.net/aberrant/rrd_hw.htm
54 * -- Jake Brutlag <jakeb@corp.webtv.net>
55 *
56 * Revision 1.2 2001/03/04 10:29:20 oetiker
57 * fixed filedescriptor leak
58 * -- Mike Franusich <mike@franusich.com>
59 *
60 * Revision 1.1.1.1 2001/02/25 22:25:05 oetiker
61 * checkin
62 *
63 *****************************************************************************/
65 #include "rrd_tool.h"
66 #include "unused.h"
67 #define MEMBLK 8192
69 /* DEBUG 2 prints information obtained via mincore(2) */
70 // #define DEBUG 2
71 /* do not calculate exact madvise hints but assume 1 page for headers and
72 * set DONTNEED for the rest, which is assumed to be data */
73 //#define ONE_PAGE 1
74 /* Avoid calling madvise on areas that were already hinted. May be benefical if
75 * your syscalls are very slow */
76 #define CHECK_MADVISE_OVERLAPS 1
78 #ifdef HAVE_MMAP
79 /* the cast to void* is there to avoid this warning seen on ia64 with certain
80 versions of gcc: 'cast increases required alignment of target type'
81 */
82 #define __rrd_read(dst, dst_t, cnt) \
83 (dst) = (dst_t*)(void*) (data + offset); \
84 offset += sizeof(dst_t) * (cnt)
85 #else
86 #define __rrd_read(dst, dst_t, cnt) \
87 if ((dst = malloc(sizeof(dst_t)*(cnt))) == NULL) { \
88 rrd_set_error(#dst " malloc"); \
89 goto out_nullify_head; \
90 } \
91 offset += read (rrd_file->fd, dst, sizeof(dst_t)*(cnt))
92 #endif
94 /* next page-aligned (i.e. page-align up) */
95 #ifndef PAGE_ALIGN
96 #define PAGE_ALIGN(addr) (((addr)+_page_size-1)&(~(_page_size-1)))
97 #endif
98 /* previous page-aligned (i.e. page-align down) */
99 #ifndef PAGE_ALIGN_DOWN
100 #define PAGE_ALIGN_DOWN(addr) (((addr)+_page_size-1)&(~(_page_size-1)))
101 #endif
103 #ifdef HAVE_MMAP
104 /* vector of last madvise hint */
105 typedef struct _madvise_vec_t {
106 void *start;
107 ssize_t length;
108 } _madvise_vec_t;
109 _madvise_vec_t _madv_vec = { NULL, 0 };
110 #endif
112 #if defined CHECK_MADVISE_OVERLAPS
113 #define _madvise(_start, _off, _hint) \
114 if ((_start) != _madv_vec.start && (ssize_t)(_off) != _madv_vec.length) { \
115 _madv_vec.start = (_start) ; _madv_vec.length = (_off); \
116 madvise((_start), (_off), (_hint)); \
117 }
118 #else
119 #define _madvise(_start, _off, _hint) \
120 madvise((_start), (_off), (_hint))
121 #endif
123 /* Open a database file, return its header and an open filehandle,
124 * positioned to the first cdp in the first rra.
125 * In the error path of rrd_open, only rrd_free(&rrd) has to be called
126 * before returning an error. Do not call rrd_close upon failure of rrd_open.
127 */
129 rrd_file_t *rrd_open(
130 const char *const file_name,
131 rrd_t *rrd,
132 unsigned rdwr)
133 {
134 int flags = 0;
135 mode_t mode = S_IRUSR;
136 int version;
138 #ifdef HAVE_MMAP
139 ssize_t _page_size = sysconf(_SC_PAGESIZE);
140 int mm_prot = PROT_READ, mm_flags = 0;
141 char *data;
142 #endif
143 off_t offset = 0;
144 struct stat statb;
145 rrd_file_t *rrd_file = NULL;
147 rrd_init(rrd);
148 rrd_file = malloc(sizeof(rrd_file_t));
149 if (rrd_file == NULL) {
150 rrd_set_error("allocating rrd_file descriptor for '%s'", file_name);
151 return NULL;
152 }
153 memset(rrd_file, 0, sizeof(rrd_file_t));
155 #ifdef DEBUG
156 if ((rdwr & (RRD_READONLY | RRD_READWRITE)) ==
157 (RRD_READONLY | RRD_READWRITE)) {
158 /* Both READONLY and READWRITE were given, which is invalid. */
159 rrd_set_error("in read/write request mask");
160 exit(-1);
161 }
162 #endif
163 if (rdwr & RRD_READONLY) {
164 flags |= O_RDONLY;
165 #ifdef HAVE_MMAP
166 mm_flags = MAP_PRIVATE;
167 # ifdef MAP_NORESERVE
168 mm_flags |= MAP_NORESERVE; /* readonly, so no swap backing needed */
169 # endif
170 #endif
171 } else {
172 if (rdwr & RRD_READWRITE) {
173 mode |= S_IWUSR;
174 flags |= O_RDWR;
175 #ifdef HAVE_MMAP
176 mm_flags = MAP_SHARED;
177 mm_prot |= PROT_WRITE;
178 #endif
179 }
180 if (rdwr & RRD_CREAT) {
181 flags |= (O_CREAT | O_TRUNC);
182 }
183 }
184 if (rdwr & RRD_READAHEAD) {
185 #ifdef MAP_POPULATE
186 mm_flags |= MAP_POPULATE; /* populate ptes and data */
187 #endif
188 #if defined MAP_NONBLOCK
189 mm_flags |= MAP_NONBLOCK; /* just populate ptes */
190 #endif
191 #ifdef USE_DIRECT_IO
192 } else {
193 flags |= O_DIRECT;
194 #endif
195 }
196 #ifdef O_NONBLOCK
197 flags |= O_NONBLOCK;
198 #endif
200 if ((rrd_file->fd = open(file_name, flags, mode)) < 0) {
201 rrd_set_error("opening '%s': %s", file_name, rrd_strerror(errno));
202 goto out_free;
203 }
205 /* Better try to avoid seeks as much as possible. stat may be heavy but
206 * many concurrent seeks are even worse. */
207 if ((fstat(rrd_file->fd, &statb)) < 0) {
208 rrd_set_error("fstat '%s': %s", file_name, rrd_strerror(errno));
209 goto out_close;
210 }
211 rrd_file->file_len = statb.st_size;
213 #ifdef HAVE_POSIX_FADVISE
214 /* In general we need no read-ahead when dealing with rrd_files.
215 When we stop reading, it is highly unlikely that we start up again.
216 In this manner we actually save time and diskaccess (and buffer cache).
217 Thanks to Dave Plonka for the Idea of using POSIX_FADV_RANDOM here. */
218 if (0 != posix_fadvise(rrd_file->fd, 0, 0, POSIX_FADV_RANDOM)) {
219 rrd_set_error("setting POSIX_FADV_RANDOM on '%s': %s", file_name,
220 rrd_strerror(errno));
221 goto out_close;
222 }
223 #endif
225 /*
226 if (rdwr & RRD_READWRITE)
227 {
228 if (setvbuf((rrd_file->fd),NULL,_IONBF,2)) {
229 rrd_set_error("failed to disable the stream buffer\n");
230 return (-1);
231 }
232 }
233 */
234 #ifdef HAVE_MMAP
235 data = mmap(0, rrd_file->file_len, mm_prot, mm_flags,
236 rrd_file->fd, offset);
238 /* lets see if the first read worked */
239 if (data == MAP_FAILED) {
240 rrd_set_error("mmaping file '%s': %s", file_name,
241 rrd_strerror(errno));
242 goto out_close;
243 }
244 rrd_file->file_start = data;
245 #endif
246 #ifdef USE_MADVISE
247 if (rdwr & RRD_COPY) {
248 /* We will read everything in a moment (copying) */
249 _madvise(data, rrd_file->file_len, MADV_WILLNEED | MADV_SEQUENTIAL);
250 goto out_done;
251 }
252 # ifndef ONE_PAGE
253 /* We do not need to read anything in for the moment */
254 _madvise(data, rrd_file->file_len, MADV_DONTNEED);
255 # else
256 /* alternatively: keep 1 page worth of data, likely headers,
257 * don't need the rest. */
258 _madvise(data, _page_size, MADV_WILLNEED | MADV_SEQUENTIAL);
259 _madvise(data + _page_size, (rrd_file->file_len >= _page_size)
260 ? rrd_file->file_len - _page_size : 0, MADV_DONTNEED);
261 # endif
262 #endif
264 #if defined USE_MADVISE && !defined ONE_PAGE
265 /* the stat_head will be needed soonish, so hint accordingly */
266 _madvise(data + PAGE_ALIGN_DOWN(offset), PAGE_ALIGN(sizeof(stat_head_t)),
267 MADV_WILLNEED | MADV_RANDOM);
269 #endif
271 __rrd_read(rrd->stat_head, stat_head_t,
272 1);
274 /* lets do some test if we are on track ... */
275 if (memcmp(rrd->stat_head->cookie, RRD_COOKIE, sizeof(RRD_COOKIE)) != 0) {
276 rrd_set_error("'%s' is not an RRD file", file_name);
277 goto out_nullify_head;
278 }
280 if (rrd->stat_head->float_cookie != FLOAT_COOKIE) {
281 rrd_set_error("This RRD was created on another architecture");
282 goto out_nullify_head;
283 }
285 version = atoi(rrd->stat_head->version);
287 if (version > atoi(RRD_VERSION)) {
288 rrd_set_error("can't handle RRD file version %s",
289 rrd->stat_head->version);
290 goto out_nullify_head;
291 }
292 #if defined USE_MADVISE && !defined ONE_PAGE
293 /* the ds_def will be needed soonish, so hint accordingly */
294 _madvise(data + PAGE_ALIGN_DOWN(offset),
295 PAGE_ALIGN(sizeof(ds_def_t) * rrd->stat_head->ds_cnt),
296 MADV_WILLNEED);
297 #endif
298 __rrd_read(rrd->ds_def, ds_def_t,
299 rrd->stat_head->ds_cnt);
301 #if defined USE_MADVISE && !defined ONE_PAGE
302 /* the rra_def will be needed soonish, so hint accordingly */
303 _madvise(data + PAGE_ALIGN_DOWN(offset),
304 PAGE_ALIGN(sizeof(rra_def_t) * rrd->stat_head->rra_cnt),
305 MADV_WILLNEED);
306 #endif
307 __rrd_read(rrd->rra_def, rra_def_t,
308 rrd->stat_head->rra_cnt);
310 /* handle different format for the live_head */
311 if (version < 3) {
312 rrd->live_head = (live_head_t *) malloc(sizeof(live_head_t));
313 if (rrd->live_head == NULL) {
314 rrd_set_error("live_head_t malloc");
315 goto out_close;
316 }
317 #ifdef HAVE_MMAP
318 memmove(&rrd->live_head->last_up, data + offset, sizeof(long));
319 offset += sizeof(long);
320 #else
321 offset += read(rrd_file->fd, &rrd->live_head->last_up, sizeof(long));
322 #endif
323 rrd->live_head->last_up_usec = 0;
324 } else {
325 #if defined USE_MADVISE && !defined ONE_PAGE
326 /* the live_head will be needed soonish, so hint accordingly */
327 _madvise(data + PAGE_ALIGN_DOWN(offset),
328 PAGE_ALIGN(sizeof(live_head_t)), MADV_WILLNEED);
329 #endif
330 __rrd_read(rrd->live_head, live_head_t,
331 1);
332 }
333 //XXX: This doesn't look like it needs madvise
334 __rrd_read(rrd->pdp_prep, pdp_prep_t,
335 rrd->stat_head->ds_cnt);
337 //XXX: This could benefit from madvise()ing
338 __rrd_read(rrd->cdp_prep, cdp_prep_t,
339 rrd->stat_head->rra_cnt * rrd->stat_head->ds_cnt);
341 //XXX: This could benefit from madvise()ing
342 __rrd_read(rrd->rra_ptr, rra_ptr_t,
343 rrd->stat_head->rra_cnt);
345 #ifdef USE_MADVISE
346 out_done:
347 #endif
348 rrd_file->header_len = offset;
349 rrd_file->pos = offset;
351 return (rrd_file);
352 out_nullify_head:
353 rrd->stat_head = NULL;
354 out_close:
355 close(rrd_file->fd);
356 out_free:
357 free(rrd_file);
358 return NULL;
359 }
362 /* Close a reference to an rrd_file. */
364 int rrd_close(
365 rrd_file_t *rrd_file)
366 {
367 int ret;
369 #if defined HAVE_MMAP || defined DEBUG
370 ssize_t _page_size = sysconf(_SC_PAGESIZE);
371 #endif
372 #if defined DEBUG && DEBUG > 1
373 /* pretty print blocks in core */
374 off_t off;
375 unsigned char *vec;
377 off = rrd_file->file_len +
378 ((rrd_file->file_len + _page_size - 1) / _page_size);
379 vec = malloc(off);
380 if (vec != NULL) {
381 memset(vec, 0, off);
382 if (mincore(rrd_file->file_start, rrd_file->file_len, vec) == 0) {
383 int prev;
384 unsigned is_in = 0, was_in = 0;
386 for (off = 0, prev = 0; off < rrd_file->file_len; ++off) {
387 is_in = vec[off] & 1; /* if lsb set then is core resident */
388 if (off == 0)
389 was_in = is_in;
390 if (was_in != is_in) {
391 fprintf(stderr, "%sin core: %p len %ld\n",
392 was_in ? "" : "not ", vec + prev, off - prev);
393 was_in = is_in;
394 prev = off;
395 }
396 }
397 fprintf(stderr,
398 "%sin core: %p len %ld\n",
399 was_in ? "" : "not ", vec + prev, off - prev);
400 } else
401 fprintf(stderr, "mincore: %s", rrd_strerror(errno));
402 }
403 #endif /* DEBUG */
405 #ifdef USE_MADVISE
406 # ifdef ONE_PAGE
407 /* Keep headers around, round up to next page boundary. */
408 ret =
409 PAGE_ALIGN(rrd_file->header_len % _page_size + rrd_file->header_len);
410 if (rrd_file->file_len > ret)
411 _madvise(rrd_file->file_start + ret,
412 rrd_file->file_len - ret, MADV_DONTNEED);
413 # else
414 /* ignoring errors from RRDs that are smaller then the file_len+rounding */
415 _madvise(rrd_file->file_start + PAGE_ALIGN_DOWN(rrd_file->header_len),
416 rrd_file->file_len - PAGE_ALIGN(rrd_file->header_len),
417 MADV_DONTNEED);
418 # endif
419 #endif
420 #ifdef HAVE_MMAP
421 ret = munmap(rrd_file->file_start, rrd_file->file_len);
422 if (ret != 0)
423 rrd_set_error("munmap rrd_file: %s", rrd_strerror(errno));
424 #endif
425 ret = close(rrd_file->fd);
426 if (ret != 0)
427 rrd_set_error("closing file: %s", rrd_strerror(errno));
428 free(rrd_file);
429 rrd_file = NULL;
430 return ret;
431 }
434 /* Set position of rrd_file. */
436 off_t rrd_seek(
437 rrd_file_t *rrd_file,
438 off_t off,
439 int whence)
440 {
441 off_t ret = 0;
443 #ifdef HAVE_MMAP
444 if (whence == SEEK_SET)
445 rrd_file->pos = off;
446 else if (whence == SEEK_CUR)
447 rrd_file->pos += off;
448 else if (whence == SEEK_END)
449 rrd_file->pos = rrd_file->file_len + off;
450 #else
451 ret = lseek(rrd_file->fd, off, whence);
452 if (ret < 0)
453 rrd_set_error("lseek: %s", rrd_strerror(errno));
454 rrd_file->pos = ret;
455 #endif
456 //XXX: mimic fseek, which returns 0 upon success
457 return ret == -1; //XXX: or just ret to mimic lseek
458 }
461 /* Get current position in rrd_file. */
463 inline off_t rrd_tell(
464 rrd_file_t *rrd_file)
465 {
466 return rrd_file->pos;
467 }
470 /* read count bytes into buffer buf, starting at rrd_file->pos.
471 * Returns the number of bytes read. */
473 inline ssize_t rrd_read(
474 rrd_file_t *rrd_file,
475 void *buf,
476 size_t count)
477 {
478 #ifdef HAVE_MMAP
479 buf = memcpy(buf, rrd_file->file_start + rrd_file->pos, count);
480 rrd_file->pos += count; /* mimmic read() semantics */
481 return count;
482 #else
483 ssize_t ret;
485 ret = read(rrd_file->fd, buf, count);
486 //XXX: eventually add generic rrd_set_error(""); here
487 rrd_file->pos += count; /* mimmic read() semantics */
488 return ret;
489 #endif
490 }
493 /* write count bytes from buffer buf to the current position
494 * rrd_file->pos of rrd_file->fd.
495 * Returns the number of bytes written. */
497 inline ssize_t rrd_write(
498 rrd_file_t *rrd_file,
499 const void *buf,
500 size_t count)
501 {
502 #ifdef HAVE_MMAP
503 memcpy(rrd_file->file_start + rrd_file->pos, buf, count);
504 rrd_file->pos += count;
505 return count; /* mimmic write() semantics */
506 #else
507 ssize_t _sz = write(rrd_file->fd, buf, count);
508 if (_sz > 0)
509 rrd_file->pos += _sz;
510 return _sz;
511 #endif
512 }
515 /* flush all data pending to be written to FD. */
517 inline void rrd_flush(
518 rrd_file_t *rrd_file)
519 {
520 if (fdatasync(rrd_file->fd) != 0) {
521 rrd_set_error("flushing fd %d: %s", rrd_file->fd,
522 rrd_strerror(errno));
523 }
524 }
527 /* Initialize RRD header. */
529 void rrd_init(
530 rrd_t *rrd)
531 {
532 rrd->stat_head = NULL;
533 rrd->ds_def = NULL;
534 rrd->rra_def = NULL;
535 rrd->live_head = NULL;
536 rrd->rra_ptr = NULL;
537 rrd->pdp_prep = NULL;
538 rrd->cdp_prep = NULL;
539 rrd->rrd_value = NULL;
540 }
543 /* free RRD header data. */
545 #ifdef HAVE_MMAP
546 inline void rrd_free(
547 rrd_t UNUSED(*rrd))
548 {
549 }
550 #else
551 void rrd_free(
552 rrd_t *rrd)
553 {
554 if (atoi(rrd->stat_head->version) < 3)
555 free(rrd->live_head);
556 free(rrd->stat_head);
557 free(rrd->ds_def);
558 free(rrd->rra_def);
559 free(rrd->rra_ptr);
560 free(rrd->pdp_prep);
561 free(rrd->cdp_prep);
562 free(rrd->rrd_value);
563 }
564 #endif
567 /* routine used by external libraries to free memory allocated by
568 * rrd library */
570 void rrd_freemem(
571 void *mem)
572 {
573 free(mem);
574 }
577 /* XXX: FIXME: missing documentation. */
578 /*XXX: FIXME should be renamed to rrd_readfile or _rrd_readfile */
580 int /*_rrd_*/ readfile(
581 const char *file_name,
582 char **buffer,
583 int skipfirst)
584 {
585 long writecnt = 0, totalcnt = MEMBLK;
586 long offset = 0;
587 FILE *input = NULL;
588 char c;
590 if ((strcmp("-", file_name) == 0)) {
591 input = stdin;
592 } else {
593 if ((input = fopen(file_name, "rb")) == NULL) {
594 rrd_set_error("opening '%s': %s", file_name, rrd_strerror(errno));
595 return (-1);
596 }
597 }
598 if (skipfirst) {
599 do {
600 c = getc(input);
601 offset++;
602 } while (c != '\n' && !feof(input));
603 }
604 if (strcmp("-", file_name)) {
605 fseek(input, 0, SEEK_END);
606 /* have extra space for detecting EOF without realloc */
607 totalcnt = (ftell(input) + 1) / sizeof(char) - offset;
608 if (totalcnt < MEMBLK)
609 totalcnt = MEMBLK; /* sanitize */
610 fseek(input, offset * sizeof(char), SEEK_SET);
611 }
612 if (((*buffer) = (char *) malloc((totalcnt + 4) * sizeof(char))) == NULL) {
613 perror("Allocate Buffer:");
614 exit(1);
615 };
616 do {
617 writecnt +=
618 fread((*buffer) + writecnt, 1,
619 (totalcnt - writecnt) * sizeof(char), input);
620 if (writecnt >= totalcnt) {
621 totalcnt += MEMBLK;
622 if (((*buffer) =
623 rrd_realloc((*buffer),
624 (totalcnt + 4) * sizeof(char))) == NULL) {
625 perror("Realloc Buffer:");
626 exit(1);
627 };
628 }
629 } while (!feof(input));
630 (*buffer)[writecnt] = '\0';
631 if (strcmp("-", file_name) != 0) {
632 fclose(input);
633 };
634 return writecnt;
635 }