1 /*****************************************************************************
2 * RRDtool 1.2.23 Copyright by Tobi Oetiker, 1997-2007
3 *****************************************************************************
4 * rrd_open.c Open an RRD File
5 *****************************************************************************
6 * $Id$
7 *****************************************************************************/
9 #include "rrd_tool.h"
10 #include "unused.h"
11 #define MEMBLK 8192
13 /* DEBUG 2 prints information obtained via mincore(2) */
14 // #define DEBUG 2
15 /* do not calculate exact madvise hints but assume 1 page for headers and
16 * set DONTNEED for the rest, which is assumed to be data */
17 //#define ONE_PAGE 1
18 /* Avoid calling madvise on areas that were already hinted. May be benefical if
19 * your syscalls are very slow */
20 #define CHECK_MADVISE_OVERLAPS 1
22 #ifdef HAVE_MMAP
23 /* the cast to void* is there to avoid this warning seen on ia64 with certain
24 versions of gcc: 'cast increases required alignment of target type'
25 */
26 #define __rrd_read(dst, dst_t, cnt) \
27 (dst) = (dst_t*)(void*) (data + offset); \
28 offset += sizeof(dst_t) * (cnt)
29 #else
30 #define __rrd_read(dst, dst_t, cnt) \
31 if ((dst = malloc(sizeof(dst_t)*(cnt))) == NULL) { \
32 rrd_set_error(#dst " malloc"); \
33 goto out_nullify_head; \
34 } \
35 offset += read (rrd_file->fd, dst, sizeof(dst_t)*(cnt))
36 #endif
38 /* next page-aligned (i.e. page-align up) */
39 #ifndef PAGE_ALIGN
40 #define PAGE_ALIGN(addr) (((addr)+_page_size-1)&(~(_page_size-1)))
41 #endif
42 /* previous page-aligned (i.e. page-align down) */
43 #ifndef PAGE_ALIGN_DOWN
44 #define PAGE_ALIGN_DOWN(addr) (((addr)+_page_size-1)&(~(_page_size-1)))
45 #endif
47 #ifdef HAVE_MMAP
48 /* vector of last madvise hint */
49 typedef struct _madvise_vec_t {
50 void *start;
51 ssize_t length;
52 } _madvise_vec_t;
53 _madvise_vec_t _madv_vec = { NULL, 0 };
54 #endif
56 #if defined CHECK_MADVISE_OVERLAPS
57 #define _madvise(_start, _off, _hint) \
58 if ((_start) != _madv_vec.start && (ssize_t)(_off) != _madv_vec.length) { \
59 _madv_vec.start = (_start) ; _madv_vec.length = (_off); \
60 madvise((_start), (_off), (_hint)); \
61 }
62 #else
63 #define _madvise(_start, _off, _hint) \
64 madvise((_start), (_off), (_hint))
65 #endif
67 /* Open a database file, return its header and an open filehandle,
68 * positioned to the first cdp in the first rra.
69 * In the error path of rrd_open, only rrd_free(&rrd) has to be called
70 * before returning an error. Do not call rrd_close upon failure of rrd_open.
71 */
73 rrd_file_t *rrd_open(
74 const char *const file_name,
75 rrd_t *rrd,
76 unsigned rdwr)
77 {
78 int flags = 0;
79 mode_t mode = S_IRUSR;
80 int version;
82 #ifdef HAVE_MMAP
83 ssize_t _page_size = sysconf(_SC_PAGESIZE);
84 int mm_prot = PROT_READ, mm_flags = 0;
85 char *data;
86 #endif
87 off_t offset = 0;
88 struct stat statb;
89 rrd_file_t *rrd_file = NULL;
90 off_t newfile_size = 0;
92 if (rdwr & RRD_CREAT) {
93 /* yes bad inline signaling alert, we are using the
94 floatcookie to pass the size in ... only used in resize */
95 newfile_size = (off_t) rrd->stat_head->float_cookie;
96 free(rrd->stat_head);
97 }
98 rrd_init(rrd);
99 rrd_file = malloc(sizeof(rrd_file_t));
100 if (rrd_file == NULL) {
101 rrd_set_error("allocating rrd_file descriptor for '%s'", file_name);
102 return NULL;
103 }
104 memset(rrd_file, 0, sizeof(rrd_file_t));
106 #ifdef DEBUG
107 if ((rdwr & (RRD_READONLY | RRD_READWRITE)) ==
108 (RRD_READONLY | RRD_READWRITE)) {
109 /* Both READONLY and READWRITE were given, which is invalid. */
110 rrd_set_error("in read/write request mask");
111 exit(-1);
112 }
113 #endif
114 if (rdwr & RRD_READONLY) {
115 flags |= O_RDONLY;
116 #ifdef HAVE_MMAP
117 mm_flags = MAP_PRIVATE;
118 # ifdef MAP_NORESERVE
119 mm_flags |= MAP_NORESERVE; /* readonly, so no swap backing needed */
120 # endif
121 #endif
122 } else {
123 if (rdwr & RRD_READWRITE) {
124 mode |= S_IWUSR;
125 flags |= O_RDWR;
126 #ifdef HAVE_MMAP
127 mm_flags = MAP_SHARED;
128 mm_prot |= PROT_WRITE;
129 #endif
130 }
131 if (rdwr & RRD_CREAT) {
132 flags |= (O_CREAT | O_TRUNC);
133 }
134 }
135 if (rdwr & RRD_READAHEAD) {
136 #ifdef MAP_POPULATE
137 mm_flags |= MAP_POPULATE; /* populate ptes and data */
138 #endif
139 #if defined MAP_NONBLOCK
140 mm_flags |= MAP_NONBLOCK; /* just populate ptes */
141 #endif
142 #ifdef USE_DIRECT_IO
143 } else {
144 flags |= O_DIRECT;
145 #endif
146 }
147 #ifdef O_NONBLOCK
148 flags |= O_NONBLOCK;
149 #endif
151 if ((rrd_file->fd = open(file_name, flags, mode)) < 0) {
152 rrd_set_error("opening '%s': %s", file_name, rrd_strerror(errno));
153 goto out_free;
154 }
156 /* Better try to avoid seeks as much as possible. stat may be heavy but
157 * many concurrent seeks are even worse. */
158 if (newfile_size == 0 && ((fstat(rrd_file->fd, &statb)) < 0)) {
159 rrd_set_error("fstat '%s': %s", file_name, rrd_strerror(errno));
160 goto out_close;
161 }
162 if (newfile_size == 0) {
163 rrd_file->file_len = statb.st_size;
164 } else {
165 rrd_file->file_len = newfile_size;
166 lseek(rrd_file->fd, newfile_size - 1, SEEK_SET);
167 write(rrd_file->fd, "\0", 1); /* poke */
168 lseek(rrd_file->fd, 0, SEEK_SET);
169 }
170 #ifdef HAVE_POSIX_FADVISE
171 /* In general we need no read-ahead when dealing with rrd_files.
172 When we stop reading, it is highly unlikely that we start up again.
173 In this manner we actually save time and diskaccess (and buffer cache).
174 Thanks to Dave Plonka for the Idea of using POSIX_FADV_RANDOM here. */
175 if (0 != posix_fadvise(rrd_file->fd, 0, 0, POSIX_FADV_RANDOM)) {
176 rrd_set_error("setting POSIX_FADV_RANDOM on '%s': %s", file_name,
177 rrd_strerror(errno));
178 goto out_close;
179 }
180 #endif
182 /*
183 if (rdwr & RRD_READWRITE)
184 {
185 if (setvbuf((rrd_file->fd),NULL,_IONBF,2)) {
186 rrd_set_error("failed to disable the stream buffer\n");
187 return (-1);
188 }
189 }
190 */
191 #ifdef HAVE_MMAP
192 data = mmap(0, rrd_file->file_len, mm_prot, mm_flags,
193 rrd_file->fd, offset);
195 /* lets see if the first read worked */
196 if (data == MAP_FAILED) {
197 rrd_set_error("mmaping file '%s': %s", file_name,
198 rrd_strerror(errno));
199 goto out_close;
200 }
201 rrd_file->file_start = data;
202 if (rdwr & RRD_CREAT) {
203 memset(data, DNAN, newfile_size - 1);
204 goto out_done;
205 }
206 #endif
207 if (rdwr & RRD_CREAT)
208 goto out_done;
209 #ifdef USE_MADVISE
210 if (rdwr & RRD_COPY) {
211 /* We will read everything in a moment (copying) */
212 _madvise(data, rrd_file->file_len, MADV_WILLNEED | MADV_SEQUENTIAL);
213 } else {
214 # ifndef ONE_PAGE
215 /* We do not need to read anything in for the moment */
216 _madvise(data, rrd_file->file_len, MADV_DONTNEED);
217 /* the stat_head will be needed soonish, so hint accordingly */
218 _madvise(data + PAGE_ALIGN_DOWN(offset),
219 PAGE_ALIGN(sizeof(stat_head_t)),
220 MADV_WILLNEED | MADV_RANDOM);
222 # else
223 /* alternatively: keep 1 page worth of data, likely headers,
224 * don't need the rest. */
225 _madvise(data, _page_size, MADV_WILLNEED | MADV_SEQUENTIAL);
226 _madvise(data + _page_size, (rrd_file->file_len >= _page_size)
227 ? rrd_file->file_len - _page_size : 0, MADV_DONTNEED);
228 # endif
229 }
230 #endif
232 __rrd_read(rrd->stat_head, stat_head_t,
233 1);
235 /* lets do some test if we are on track ... */
236 if (memcmp(rrd->stat_head->cookie, RRD_COOKIE, sizeof(RRD_COOKIE)) != 0) {
237 rrd_set_error("'%s' is not an RRD file", file_name);
238 goto out_nullify_head;
239 }
241 if (rrd->stat_head->float_cookie != FLOAT_COOKIE) {
242 rrd_set_error("This RRD was created on another architecture");
243 goto out_nullify_head;
244 }
246 version = atoi(rrd->stat_head->version);
248 if (version > atoi(RRD_VERSION)) {
249 rrd_set_error("can't handle RRD file version %s",
250 rrd->stat_head->version);
251 goto out_nullify_head;
252 }
253 #if defined USE_MADVISE && !defined ONE_PAGE
254 /* the ds_def will be needed soonish, so hint accordingly */
255 _madvise(data + PAGE_ALIGN_DOWN(offset),
256 PAGE_ALIGN(sizeof(ds_def_t) * rrd->stat_head->ds_cnt),
257 MADV_WILLNEED);
258 #endif
259 __rrd_read(rrd->ds_def, ds_def_t,
260 rrd->stat_head->ds_cnt);
262 #if defined USE_MADVISE && !defined ONE_PAGE
263 /* the rra_def will be needed soonish, so hint accordingly */
264 _madvise(data + PAGE_ALIGN_DOWN(offset),
265 PAGE_ALIGN(sizeof(rra_def_t) * rrd->stat_head->rra_cnt),
266 MADV_WILLNEED);
267 #endif
268 __rrd_read(rrd->rra_def, rra_def_t,
269 rrd->stat_head->rra_cnt);
271 /* handle different format for the live_head */
272 if (version < 3) {
273 rrd->live_head = (live_head_t *) malloc(sizeof(live_head_t));
274 if (rrd->live_head == NULL) {
275 rrd_set_error("live_head_t malloc");
276 goto out_close;
277 }
278 #ifdef HAVE_MMAP
279 memmove(&rrd->live_head->last_up, data + offset, sizeof(long));
280 offset += sizeof(long);
281 #else
282 offset += read(rrd_file->fd, &rrd->live_head->last_up, sizeof(long));
283 #endif
284 rrd->live_head->last_up_usec = 0;
285 } else {
286 #if defined USE_MADVISE && !defined ONE_PAGE
287 /* the live_head will be needed soonish, so hint accordingly */
288 _madvise(data + PAGE_ALIGN_DOWN(offset),
289 PAGE_ALIGN(sizeof(live_head_t)), MADV_WILLNEED);
290 #endif
291 __rrd_read(rrd->live_head, live_head_t,
292 1);
293 }
294 //XXX: This doesn't look like it needs madvise
295 __rrd_read(rrd->pdp_prep, pdp_prep_t,
296 rrd->stat_head->ds_cnt);
298 //XXX: This could benefit from madvise()ing
299 __rrd_read(rrd->cdp_prep, cdp_prep_t,
300 rrd->stat_head->rra_cnt * rrd->stat_head->ds_cnt);
302 //XXX: This could benefit from madvise()ing
303 __rrd_read(rrd->rra_ptr, rra_ptr_t,
304 rrd->stat_head->rra_cnt);
306 rrd_file->header_len = offset;
307 rrd_file->pos = offset;
308 out_done:
309 return (rrd_file);
310 out_nullify_head:
311 rrd->stat_head = NULL;
312 out_close:
313 close(rrd_file->fd);
314 out_free:
315 free(rrd_file);
316 return NULL;
317 }
320 /* Close a reference to an rrd_file. */
322 int rrd_close(
323 rrd_file_t *rrd_file)
324 {
325 int ret;
327 #if defined HAVE_MMAP || defined DEBUG
328 ssize_t _page_size = sysconf(_SC_PAGESIZE);
329 #endif
330 #if defined DEBUG && DEBUG > 1
331 /* pretty print blocks in core */
332 off_t off;
333 unsigned char *vec;
335 off = rrd_file->file_len +
336 ((rrd_file->file_len + _page_size - 1) / _page_size);
337 vec = malloc(off);
338 if (vec != NULL) {
339 memset(vec, 0, off);
340 if (mincore(rrd_file->file_start, rrd_file->file_len, vec) == 0) {
341 int prev;
342 unsigned is_in = 0, was_in = 0;
344 for (off = 0, prev = 0; off < rrd_file->file_len; ++off) {
345 is_in = vec[off] & 1; /* if lsb set then is core resident */
346 if (off == 0)
347 was_in = is_in;
348 if (was_in != is_in) {
349 fprintf(stderr, "%sin core: %p len %ld\n",
350 was_in ? "" : "not ", vec + prev, off - prev);
351 was_in = is_in;
352 prev = off;
353 }
354 }
355 fprintf(stderr,
356 "%sin core: %p len %ld\n",
357 was_in ? "" : "not ", vec + prev, off - prev);
358 } else
359 fprintf(stderr, "mincore: %s", rrd_strerror(errno));
360 }
361 #endif /* DEBUG */
363 #ifdef USE_MADVISE
364 # ifdef ONE_PAGE
365 /* Keep headers around, round up to next page boundary. */
366 ret =
367 PAGE_ALIGN(rrd_file->header_len % _page_size + rrd_file->header_len);
368 if (rrd_file->file_len > ret)
369 _madvise(rrd_file->file_start + ret,
370 rrd_file->file_len - ret, MADV_DONTNEED);
371 # else
372 /* ignoring errors from RRDs that are smaller then the file_len+rounding */
373 _madvise(rrd_file->file_start + PAGE_ALIGN_DOWN(rrd_file->header_len),
374 rrd_file->file_len - PAGE_ALIGN(rrd_file->header_len),
375 MADV_DONTNEED);
376 # endif
377 #endif
378 #ifdef HAVE_MMAP
379 ret = munmap(rrd_file->file_start, rrd_file->file_len);
380 if (ret != 0)
381 rrd_set_error("munmap rrd_file: %s", rrd_strerror(errno));
382 #endif
383 ret = close(rrd_file->fd);
384 if (ret != 0)
385 rrd_set_error("closing file: %s", rrd_strerror(errno));
386 free(rrd_file);
387 rrd_file = NULL;
388 return ret;
389 }
392 /* Set position of rrd_file. */
394 off_t rrd_seek(
395 rrd_file_t *rrd_file,
396 off_t off,
397 int whence)
398 {
399 off_t ret = 0;
401 #ifdef HAVE_MMAP
402 if (whence == SEEK_SET)
403 rrd_file->pos = off;
404 else if (whence == SEEK_CUR)
405 rrd_file->pos += off;
406 else if (whence == SEEK_END)
407 rrd_file->pos = rrd_file->file_len + off;
408 #else
409 ret = lseek(rrd_file->fd, off, whence);
410 if (ret < 0)
411 rrd_set_error("lseek: %s", rrd_strerror(errno));
412 rrd_file->pos = ret;
413 #endif
414 //XXX: mimic fseek, which returns 0 upon success
415 return ret == -1; //XXX: or just ret to mimic lseek
416 }
419 /* Get current position in rrd_file. */
421 inline off_t rrd_tell(
422 rrd_file_t *rrd_file)
423 {
424 return rrd_file->pos;
425 }
428 /* read count bytes into buffer buf, starting at rrd_file->pos.
429 * Returns the number of bytes read or <0 on error. */
431 inline ssize_t rrd_read(
432 rrd_file_t *rrd_file,
433 void *buf,
434 size_t count)
435 {
436 #ifdef HAVE_MMAP
437 size_t _cnt = count;
438 ssize_t _surplus = rrd_file->pos + _cnt - rrd_file->file_len;
440 if (_surplus > 0) { /* short read */
441 _cnt -= _surplus;
442 }
443 if (_cnt == 0)
444 return 0; /* EOF */
445 buf = memcpy(buf, rrd_file->file_start + rrd_file->pos, _cnt);
447 rrd_file->pos += _cnt; /* mimmic read() semantics */
448 return _cnt;
449 #else
450 ssize_t ret;
452 ret = read(rrd_file->fd, buf, count);
453 if (ret > 0)
454 rrd_file->pos += ret; /* mimmic read() semantics */
455 return ret;
456 #endif
457 }
460 /* write count bytes from buffer buf to the current position
461 * rrd_file->pos of rrd_file->fd.
462 * Returns the number of bytes written. */
464 inline ssize_t rrd_write(
465 rrd_file_t *rrd_file,
466 const void *buf,
467 size_t count)
468 {
469 #ifdef HAVE_MMAP
470 memcpy(rrd_file->file_start + rrd_file->pos, buf, count);
471 rrd_file->pos += count;
472 return count; /* mimmic write() semantics */
473 #else
474 ssize_t _sz = write(rrd_file->fd, buf, count);
476 if (_sz > 0)
477 rrd_file->pos += _sz;
478 return _sz;
479 #endif
480 }
483 /* flush all data pending to be written to FD. */
485 inline void rrd_flush(
486 rrd_file_t *rrd_file)
487 {
488 if (fdatasync(rrd_file->fd) != 0) {
489 rrd_set_error("flushing fd %d: %s", rrd_file->fd,
490 rrd_strerror(errno));
491 }
492 }
495 /* Initialize RRD header. */
497 void rrd_init(
498 rrd_t *rrd)
499 {
500 rrd->stat_head = NULL;
501 rrd->ds_def = NULL;
502 rrd->rra_def = NULL;
503 rrd->live_head = NULL;
504 rrd->rra_ptr = NULL;
505 rrd->pdp_prep = NULL;
506 rrd->cdp_prep = NULL;
507 rrd->rrd_value = NULL;
508 }
511 /* free RRD header data. */
513 #ifdef HAVE_MMAP
514 inline void rrd_free(
515 rrd_t UNUSED(*rrd))
516 {
517 }
518 #else
519 void rrd_free(
520 rrd_t *rrd)
521 {
522 free(rrd->live_head);
523 free(rrd->stat_head);
524 free(rrd->ds_def);
525 free(rrd->rra_def);
526 free(rrd->rra_ptr);
527 free(rrd->pdp_prep);
528 free(rrd->cdp_prep);
529 free(rrd->rrd_value);
530 }
531 #endif
534 /* routine used by external libraries to free memory allocated by
535 * rrd library */
537 void rrd_freemem(
538 void *mem)
539 {
540 free(mem);
541 }
544 /* XXX: FIXME: missing documentation. */
545 /*XXX: FIXME should be renamed to rrd_readfile or _rrd_readfile */
547 int /*_rrd_*/ readfile(
548 const char *file_name,
549 char **buffer,
550 int skipfirst)
551 {
552 long writecnt = 0, totalcnt = MEMBLK;
553 long offset = 0;
554 FILE *input = NULL;
555 char c;
557 if ((strcmp("-", file_name) == 0)) {
558 input = stdin;
559 } else {
560 if ((input = fopen(file_name, "rb")) == NULL) {
561 rrd_set_error("opening '%s': %s", file_name, rrd_strerror(errno));
562 return (-1);
563 }
564 }
565 if (skipfirst) {
566 do {
567 c = getc(input);
568 offset++;
569 } while (c != '\n' && !feof(input));
570 }
571 if (strcmp("-", file_name)) {
572 fseek(input, 0, SEEK_END);
573 /* have extra space for detecting EOF without realloc */
574 totalcnt = (ftell(input) + 1) / sizeof(char) - offset;
575 if (totalcnt < MEMBLK)
576 totalcnt = MEMBLK; /* sanitize */
577 fseek(input, offset * sizeof(char), SEEK_SET);
578 }
579 if (((*buffer) = (char *) malloc((totalcnt + 4) * sizeof(char))) == NULL) {
580 perror("Allocate Buffer:");
581 exit(1);
582 };
583 do {
584 writecnt +=
585 fread((*buffer) + writecnt, 1,
586 (totalcnt - writecnt) * sizeof(char), input);
587 if (writecnt >= totalcnt) {
588 totalcnt += MEMBLK;
589 if (((*buffer) =
590 rrd_realloc((*buffer),
591 (totalcnt + 4) * sizeof(char))) == NULL) {
592 perror("Realloc Buffer:");
593 exit(1);
594 };
595 }
596 } while (!feof(input));
597 (*buffer)[writecnt] = '\0';
598 if (strcmp("-", file_name) != 0) {
599 fclose(input);
600 };
601 return writecnt;
602 }