4b9a0f9220d126b9bc94b4ced97806a2fac25a65
1 /*****************************************************************************
2 * RRDtool 1.2.23 Copyright by Tobi Oetiker, 1997-2007
3 *****************************************************************************
4 * rrd_open.c Open an RRD File
5 *****************************************************************************
6 * $Id$
7 *****************************************************************************/
9 #include "rrd_tool.h"
10 #include "unused.h"
11 #define MEMBLK 8192
13 /* DEBUG 2 prints information obtained via mincore(2) */
14 // #define DEBUG 2
15 /* do not calculate exact madvise hints but assume 1 page for headers and
16 * set DONTNEED for the rest, which is assumed to be data */
17 //#define ONE_PAGE 1
18 /* Avoid calling madvise on areas that were already hinted. May be benefical if
19 * your syscalls are very slow */
20 #define CHECK_MADVISE_OVERLAPS 1
22 #ifdef HAVE_MMAP
23 /* the cast to void* is there to avoid this warning seen on ia64 with certain
24 versions of gcc: 'cast increases required alignment of target type'
25 */
26 #define __rrd_read(dst, dst_t, cnt) \
27 (dst) = (dst_t*)(void*) (data + offset); \
28 offset += sizeof(dst_t) * (cnt)
29 #else
30 #define __rrd_read(dst, dst_t, cnt) \
31 if ((dst = malloc(sizeof(dst_t)*(cnt))) == NULL) { \
32 rrd_set_error(#dst " malloc"); \
33 goto out_nullify_head; \
34 } \
35 offset += read (rrd_file->fd, dst, sizeof(dst_t)*(cnt))
36 #endif
38 /* next page-aligned (i.e. page-align up) */
39 #ifndef PAGE_ALIGN
40 #define PAGE_ALIGN(addr) (((addr)+_page_size-1)&(~(_page_size-1)))
41 #endif
42 /* previous page-aligned (i.e. page-align down) */
43 #ifndef PAGE_ALIGN_DOWN
44 #define PAGE_ALIGN_DOWN(addr) (((addr)+_page_size-1)&(~(_page_size-1)))
45 #endif
47 #ifdef HAVE_MMAP
48 /* vector of last madvise hint */
49 typedef struct _madvise_vec_t {
50 void *start;
51 ssize_t length;
52 } _madvise_vec_t;
53 _madvise_vec_t _madv_vec = { NULL, 0 };
54 #endif
56 #if defined CHECK_MADVISE_OVERLAPS
57 #define _madvise(_start, _off, _hint) \
58 if ((_start) != _madv_vec.start && (ssize_t)(_off) != _madv_vec.length) { \
59 _madv_vec.start = (_start) ; _madv_vec.length = (_off); \
60 madvise((_start), (_off), (_hint)); \
61 }
62 #else
63 #define _madvise(_start, _off, _hint) \
64 madvise((_start), (_off), (_hint))
65 #endif
67 /* Open a database file, return its header and an open filehandle,
68 * positioned to the first cdp in the first rra.
69 * In the error path of rrd_open, only rrd_free(&rrd) has to be called
70 * before returning an error. Do not call rrd_close upon failure of rrd_open.
71 */
73 rrd_file_t *rrd_open(
74 const char *const file_name,
75 rrd_t *rrd,
76 unsigned rdwr)
77 {
78 int flags = 0;
79 mode_t mode = S_IRUSR;
80 int version;
82 #ifdef HAVE_MMAP
83 ssize_t _page_size = sysconf(_SC_PAGESIZE);
84 int mm_prot = PROT_READ, mm_flags = 0;
85 char *data;
86 #endif
87 off_t offset = 0;
88 struct stat statb;
89 rrd_file_t *rrd_file = NULL;
90 off_t newfile_size = 0;
92 if (rdwr & RRD_CREAT) {
93 newfile_size = (off_t) rrd->stat_head->float_cookie;
94 free(rrd->stat_head);
95 }
96 rrd_init(rrd);
97 rrd_file = malloc(sizeof(rrd_file_t));
98 if (rrd_file == NULL) {
99 rrd_set_error("allocating rrd_file descriptor for '%s'", file_name);
100 return NULL;
101 }
102 memset(rrd_file, 0, sizeof(rrd_file_t));
104 #ifdef DEBUG
105 if ((rdwr & (RRD_READONLY | RRD_READWRITE)) ==
106 (RRD_READONLY | RRD_READWRITE)) {
107 /* Both READONLY and READWRITE were given, which is invalid. */
108 rrd_set_error("in read/write request mask");
109 exit(-1);
110 }
111 #endif
112 if (rdwr & RRD_READONLY) {
113 flags |= O_RDONLY;
114 #ifdef HAVE_MMAP
115 mm_flags = MAP_PRIVATE;
116 # ifdef MAP_NORESERVE
117 mm_flags |= MAP_NORESERVE; /* readonly, so no swap backing needed */
118 # endif
119 #endif
120 } else {
121 if (rdwr & RRD_READWRITE) {
122 mode |= S_IWUSR;
123 flags |= O_RDWR;
124 #ifdef HAVE_MMAP
125 mm_flags = MAP_SHARED;
126 mm_prot |= PROT_WRITE;
127 #endif
128 }
129 if (rdwr & RRD_CREAT) {
130 flags |= (O_CREAT | O_TRUNC);
131 }
132 }
133 if (rdwr & RRD_READAHEAD) {
134 #ifdef MAP_POPULATE
135 mm_flags |= MAP_POPULATE; /* populate ptes and data */
136 #endif
137 #if defined MAP_NONBLOCK
138 mm_flags |= MAP_NONBLOCK; /* just populate ptes */
139 #endif
140 #ifdef USE_DIRECT_IO
141 } else {
142 flags |= O_DIRECT;
143 #endif
144 }
145 #ifdef O_NONBLOCK
146 flags |= O_NONBLOCK;
147 #endif
149 if ((rrd_file->fd = open(file_name, flags, mode)) < 0) {
150 rrd_set_error("opening '%s': %s", file_name, rrd_strerror(errno));
151 goto out_free;
152 }
154 /* Better try to avoid seeks as much as possible. stat may be heavy but
155 * many concurrent seeks are even worse. */
156 if (newfile_size == 0 && ((fstat(rrd_file->fd, &statb)) < 0)) {
157 rrd_set_error("fstat '%s': %s", file_name, rrd_strerror(errno));
158 goto out_close;
159 }
160 if (newfile_size == 0) {
161 rrd_file->file_len = statb.st_size;
162 } else {
163 rrd_file->file_len = newfile_size;
164 lseek(rrd_file->fd, newfile_size - 1, SEEK_SET);
165 write(rrd_file->fd, "\0", 1); /* poke */
166 lseek(rrd_file->fd, 0, SEEK_SET);
167 }
168 #ifdef HAVE_POSIX_FADVISE
169 /* In general we need no read-ahead when dealing with rrd_files.
170 When we stop reading, it is highly unlikely that we start up again.
171 In this manner we actually save time and diskaccess (and buffer cache).
172 Thanks to Dave Plonka for the Idea of using POSIX_FADV_RANDOM here. */
173 if (0 != posix_fadvise(rrd_file->fd, 0, 0, POSIX_FADV_RANDOM)) {
174 rrd_set_error("setting POSIX_FADV_RANDOM on '%s': %s", file_name,
175 rrd_strerror(errno));
176 goto out_close;
177 }
178 #endif
180 /*
181 if (rdwr & RRD_READWRITE)
182 {
183 if (setvbuf((rrd_file->fd),NULL,_IONBF,2)) {
184 rrd_set_error("failed to disable the stream buffer\n");
185 return (-1);
186 }
187 }
188 */
189 #ifdef HAVE_MMAP
190 data = mmap(0, rrd_file->file_len, mm_prot, mm_flags,
191 rrd_file->fd, offset);
193 /* lets see if the first read worked */
194 if (data == MAP_FAILED) {
195 rrd_set_error("mmaping file '%s': %s", file_name,
196 rrd_strerror(errno));
197 goto out_close;
198 }
199 rrd_file->file_start = data;
200 if (rdwr & RRD_CREAT) {
201 memset(data, DNAN, newfile_size - 1);
202 goto out_done;
203 }
204 #endif
205 if (rdwr & RRD_CREAT)
206 goto out_done;
207 #ifdef USE_MADVISE
208 if (rdwr & RRD_COPY) {
209 /* We will read everything in a moment (copying) */
210 _madvise(data, rrd_file->file_len, MADV_WILLNEED | MADV_SEQUENTIAL);
211 } else {
212 # ifndef ONE_PAGE
213 /* We do not need to read anything in for the moment */
214 _madvise(data, rrd_file->file_len, MADV_DONTNEED);
215 /* the stat_head will be needed soonish, so hint accordingly */
216 _madvise(data + PAGE_ALIGN_DOWN(offset),
217 PAGE_ALIGN(sizeof(stat_head_t)),
218 MADV_WILLNEED | MADV_RANDOM);
220 # else
221 /* alternatively: keep 1 page worth of data, likely headers,
222 * don't need the rest. */
223 _madvise(data, _page_size, MADV_WILLNEED | MADV_SEQUENTIAL);
224 _madvise(data + _page_size, (rrd_file->file_len >= _page_size)
225 ? rrd_file->file_len - _page_size : 0, MADV_DONTNEED);
226 # endif
227 }
228 #endif
230 __rrd_read(rrd->stat_head, stat_head_t,
231 1);
233 /* lets do some test if we are on track ... */
234 if (memcmp(rrd->stat_head->cookie, RRD_COOKIE, sizeof(RRD_COOKIE)) != 0) {
235 rrd_set_error("'%s' is not an RRD file", file_name);
236 goto out_nullify_head;
237 }
239 if (rrd->stat_head->float_cookie != FLOAT_COOKIE) {
240 rrd_set_error("This RRD was created on another architecture");
241 goto out_nullify_head;
242 }
244 version = atoi(rrd->stat_head->version);
246 if (version > atoi(RRD_VERSION)) {
247 rrd_set_error("can't handle RRD file version %s",
248 rrd->stat_head->version);
249 goto out_nullify_head;
250 }
251 #if defined USE_MADVISE && !defined ONE_PAGE
252 /* the ds_def will be needed soonish, so hint accordingly */
253 _madvise(data + PAGE_ALIGN_DOWN(offset),
254 PAGE_ALIGN(sizeof(ds_def_t) * rrd->stat_head->ds_cnt),
255 MADV_WILLNEED);
256 #endif
257 __rrd_read(rrd->ds_def, ds_def_t,
258 rrd->stat_head->ds_cnt);
260 #if defined USE_MADVISE && !defined ONE_PAGE
261 /* the rra_def will be needed soonish, so hint accordingly */
262 _madvise(data + PAGE_ALIGN_DOWN(offset),
263 PAGE_ALIGN(sizeof(rra_def_t) * rrd->stat_head->rra_cnt),
264 MADV_WILLNEED);
265 #endif
266 __rrd_read(rrd->rra_def, rra_def_t,
267 rrd->stat_head->rra_cnt);
269 /* handle different format for the live_head */
270 if (version < 3) {
271 rrd->live_head = (live_head_t *) malloc(sizeof(live_head_t));
272 if (rrd->live_head == NULL) {
273 rrd_set_error("live_head_t malloc");
274 goto out_close;
275 }
276 #ifdef HAVE_MMAP
277 memmove(&rrd->live_head->last_up, data + offset, sizeof(long));
278 offset += sizeof(long);
279 #else
280 offset += read(rrd_file->fd, &rrd->live_head->last_up, sizeof(long));
281 #endif
282 rrd->live_head->last_up_usec = 0;
283 } else {
284 #if defined USE_MADVISE && !defined ONE_PAGE
285 /* the live_head will be needed soonish, so hint accordingly */
286 _madvise(data + PAGE_ALIGN_DOWN(offset),
287 PAGE_ALIGN(sizeof(live_head_t)), MADV_WILLNEED);
288 #endif
289 __rrd_read(rrd->live_head, live_head_t,
290 1);
291 }
292 //XXX: This doesn't look like it needs madvise
293 __rrd_read(rrd->pdp_prep, pdp_prep_t,
294 rrd->stat_head->ds_cnt);
296 //XXX: This could benefit from madvise()ing
297 __rrd_read(rrd->cdp_prep, cdp_prep_t,
298 rrd->stat_head->rra_cnt * rrd->stat_head->ds_cnt);
300 //XXX: This could benefit from madvise()ing
301 __rrd_read(rrd->rra_ptr, rra_ptr_t,
302 rrd->stat_head->rra_cnt);
304 rrd_file->header_len = offset;
305 rrd_file->pos = offset;
306 out_done:
307 return (rrd_file);
308 out_nullify_head:
309 rrd->stat_head = NULL;
310 out_close:
311 close(rrd_file->fd);
312 out_free:
313 free(rrd_file);
314 return NULL;
315 }
318 /* Close a reference to an rrd_file. */
320 int rrd_close(
321 rrd_file_t *rrd_file)
322 {
323 int ret;
325 #if defined HAVE_MMAP || defined DEBUG
326 ssize_t _page_size = sysconf(_SC_PAGESIZE);
327 #endif
328 #if defined DEBUG && DEBUG > 1
329 /* pretty print blocks in core */
330 off_t off;
331 unsigned char *vec;
333 off = rrd_file->file_len +
334 ((rrd_file->file_len + _page_size - 1) / _page_size);
335 vec = malloc(off);
336 if (vec != NULL) {
337 memset(vec, 0, off);
338 if (mincore(rrd_file->file_start, rrd_file->file_len, vec) == 0) {
339 int prev;
340 unsigned is_in = 0, was_in = 0;
342 for (off = 0, prev = 0; off < rrd_file->file_len; ++off) {
343 is_in = vec[off] & 1; /* if lsb set then is core resident */
344 if (off == 0)
345 was_in = is_in;
346 if (was_in != is_in) {
347 fprintf(stderr, "%sin core: %p len %ld\n",
348 was_in ? "" : "not ", vec + prev, off - prev);
349 was_in = is_in;
350 prev = off;
351 }
352 }
353 fprintf(stderr,
354 "%sin core: %p len %ld\n",
355 was_in ? "" : "not ", vec + prev, off - prev);
356 } else
357 fprintf(stderr, "mincore: %s", rrd_strerror(errno));
358 }
359 #endif /* DEBUG */
361 #ifdef USE_MADVISE
362 # ifdef ONE_PAGE
363 /* Keep headers around, round up to next page boundary. */
364 ret =
365 PAGE_ALIGN(rrd_file->header_len % _page_size + rrd_file->header_len);
366 if (rrd_file->file_len > ret)
367 _madvise(rrd_file->file_start + ret,
368 rrd_file->file_len - ret, MADV_DONTNEED);
369 # else
370 /* ignoring errors from RRDs that are smaller then the file_len+rounding */
371 _madvise(rrd_file->file_start + PAGE_ALIGN_DOWN(rrd_file->header_len),
372 rrd_file->file_len - PAGE_ALIGN(rrd_file->header_len),
373 MADV_DONTNEED);
374 # endif
375 #endif
376 #ifdef HAVE_MMAP
377 ret = munmap(rrd_file->file_start, rrd_file->file_len);
378 if (ret != 0)
379 rrd_set_error("munmap rrd_file: %s", rrd_strerror(errno));
380 #endif
381 ret = close(rrd_file->fd);
382 if (ret != 0)
383 rrd_set_error("closing file: %s", rrd_strerror(errno));
384 free(rrd_file);
385 rrd_file = NULL;
386 return ret;
387 }
390 /* Set position of rrd_file. */
392 off_t rrd_seek(
393 rrd_file_t *rrd_file,
394 off_t off,
395 int whence)
396 {
397 off_t ret = 0;
399 #ifdef HAVE_MMAP
400 if (whence == SEEK_SET)
401 rrd_file->pos = off;
402 else if (whence == SEEK_CUR)
403 rrd_file->pos += off;
404 else if (whence == SEEK_END)
405 rrd_file->pos = rrd_file->file_len + off;
406 #else
407 ret = lseek(rrd_file->fd, off, whence);
408 if (ret < 0)
409 rrd_set_error("lseek: %s", rrd_strerror(errno));
410 rrd_file->pos = ret;
411 #endif
412 //XXX: mimic fseek, which returns 0 upon success
413 return ret == -1; //XXX: or just ret to mimic lseek
414 }
417 /* Get current position in rrd_file. */
419 inline off_t rrd_tell(
420 rrd_file_t *rrd_file)
421 {
422 return rrd_file->pos;
423 }
426 /* read count bytes into buffer buf, starting at rrd_file->pos.
427 * Returns the number of bytes read or <0 on error. */
429 inline ssize_t rrd_read(
430 rrd_file_t *rrd_file,
431 void *buf,
432 size_t count)
433 {
434 #ifdef HAVE_MMAP
435 size_t _cnt = count;
436 ssize_t _surplus = rrd_file->pos + _cnt - rrd_file->file_len;
438 if (_surplus > 0) { /* short read */
439 _cnt -= _surplus;
440 }
441 if (_cnt == 0)
442 return 0; /* EOF */
443 buf = memcpy(buf, rrd_file->file_start + rrd_file->pos, _cnt);
445 rrd_file->pos += _cnt; /* mimmic read() semantics */
446 return _cnt;
447 #else
448 ssize_t ret;
450 ret = read(rrd_file->fd, buf, count);
451 if (ret > 0)
452 rrd_file->pos += ret; /* mimmic read() semantics */
453 return ret;
454 #endif
455 }
458 /* write count bytes from buffer buf to the current position
459 * rrd_file->pos of rrd_file->fd.
460 * Returns the number of bytes written. */
462 inline ssize_t rrd_write(
463 rrd_file_t *rrd_file,
464 const void *buf,
465 size_t count)
466 {
467 #ifdef HAVE_MMAP
468 memcpy(rrd_file->file_start + rrd_file->pos, buf, count);
469 rrd_file->pos += count;
470 return count; /* mimmic write() semantics */
471 #else
472 ssize_t _sz = write(rrd_file->fd, buf, count);
474 if (_sz > 0)
475 rrd_file->pos += _sz;
476 return _sz;
477 #endif
478 }
481 /* flush all data pending to be written to FD. */
483 inline void rrd_flush(
484 rrd_file_t *rrd_file)
485 {
486 if (fdatasync(rrd_file->fd) != 0) {
487 rrd_set_error("flushing fd %d: %s", rrd_file->fd,
488 rrd_strerror(errno));
489 }
490 }
493 /* Initialize RRD header. */
495 void rrd_init(
496 rrd_t *rrd)
497 {
498 rrd->stat_head = NULL;
499 rrd->ds_def = NULL;
500 rrd->rra_def = NULL;
501 rrd->live_head = NULL;
502 rrd->rra_ptr = NULL;
503 rrd->pdp_prep = NULL;
504 rrd->cdp_prep = NULL;
505 rrd->rrd_value = NULL;
506 }
509 /* free RRD header data. */
511 #ifdef HAVE_MMAP
512 inline void rrd_free(
513 rrd_t UNUSED(*rrd))
514 {
515 }
516 #else
517 void rrd_free(
518 rrd_t *rrd)
519 {
520 free(rrd->live_head);
521 free(rrd->stat_head);
522 free(rrd->ds_def);
523 free(rrd->rra_def);
524 free(rrd->rra_ptr);
525 free(rrd->pdp_prep);
526 free(rrd->cdp_prep);
527 free(rrd->rrd_value);
528 }
529 #endif
532 /* routine used by external libraries to free memory allocated by
533 * rrd library */
535 void rrd_freemem(
536 void *mem)
537 {
538 free(mem);
539 }
542 /* XXX: FIXME: missing documentation. */
543 /*XXX: FIXME should be renamed to rrd_readfile or _rrd_readfile */
545 int /*_rrd_*/ readfile(
546 const char *file_name,
547 char **buffer,
548 int skipfirst)
549 {
550 long writecnt = 0, totalcnt = MEMBLK;
551 long offset = 0;
552 FILE *input = NULL;
553 char c;
555 if ((strcmp("-", file_name) == 0)) {
556 input = stdin;
557 } else {
558 if ((input = fopen(file_name, "rb")) == NULL) {
559 rrd_set_error("opening '%s': %s", file_name, rrd_strerror(errno));
560 return (-1);
561 }
562 }
563 if (skipfirst) {
564 do {
565 c = getc(input);
566 offset++;
567 } while (c != '\n' && !feof(input));
568 }
569 if (strcmp("-", file_name)) {
570 fseek(input, 0, SEEK_END);
571 /* have extra space for detecting EOF without realloc */
572 totalcnt = (ftell(input) + 1) / sizeof(char) - offset;
573 if (totalcnt < MEMBLK)
574 totalcnt = MEMBLK; /* sanitize */
575 fseek(input, offset * sizeof(char), SEEK_SET);
576 }
577 if (((*buffer) = (char *) malloc((totalcnt + 4) * sizeof(char))) == NULL) {
578 perror("Allocate Buffer:");
579 exit(1);
580 };
581 do {
582 writecnt +=
583 fread((*buffer) + writecnt, 1,
584 (totalcnt - writecnt) * sizeof(char), input);
585 if (writecnt >= totalcnt) {
586 totalcnt += MEMBLK;
587 if (((*buffer) =
588 rrd_realloc((*buffer),
589 (totalcnt + 4) * sizeof(char))) == NULL) {
590 perror("Realloc Buffer:");
591 exit(1);
592 };
593 }
594 } while (!feof(input));
595 (*buffer)[writecnt] = '\0';
596 if (strcmp("-", file_name) != 0) {
597 fclose(input);
598 };
599 return writecnt;
600 }