80f9a04d09f1e63f270bfadc8adac015f321dae7
1 /*****************************************************************************
2 * RRDtool 1.2.23 Copyright by Tobi Oetiker, 1997-2007
3 *****************************************************************************
4 * rrd_open.c Open an RRD File
5 *****************************************************************************
6 * $Id$
7 *****************************************************************************/
9 #include "rrd_tool.h"
10 #include "unused.h"
11 #define MEMBLK 8192
13 /* DEBUG 2 prints information obtained via mincore(2) */
14 #define DEBUG 1
15 /* do not calculate exact madvise hints but assume 1 page for headers and
16 * set DONTNEED for the rest, which is assumed to be data */
17 /* Avoid calling madvise on areas that were already hinted. May be benefical if
18 * your syscalls are very slow */
20 #ifdef HAVE_MMAP
21 /* the cast to void* is there to avoid this warning seen on ia64 with certain
22 versions of gcc: 'cast increases required alignment of target type'
23 */
24 #define __rrd_read(dst, dst_t, cnt) \
25 (dst) = (dst_t*)(void*) (data + offset); \
26 offset += sizeof(dst_t) * (cnt)
27 #else
28 #define __rrd_read(dst, dst_t, cnt) \
29 if ((dst = malloc(sizeof(dst_t)*(cnt))) == NULL) { \
30 rrd_set_error(#dst " malloc"); \
31 goto out_nullify_head; \
32 } \
33 offset += read (rrd_file->fd, dst, sizeof(dst_t)*(cnt))
34 #endif
36 /* get the address of the start of this page */
37 #ifndef PAGE_START
38 #define PAGE_START(addr) ((addr)&(~(_page_size-1)))
39 #endif
42 /* Open a database file, return its header and an open filehandle,
43 * positioned to the first cdp in the first rra.
44 * In the error path of rrd_open, only rrd_free(&rrd) has to be called
45 * before returning an error. Do not call rrd_close upon failure of rrd_open.
46 */
48 rrd_file_t *rrd_open(
49 const char *const file_name,
50 rrd_t *rrd,
51 unsigned rdwr)
52 {
53 int flags = 0;
54 mode_t mode = S_IRUSR;
55 int version;
57 #ifdef HAVE_MMAP
58 ssize_t _page_size = sysconf(_SC_PAGESIZE);
59 int mm_prot = PROT_READ, mm_flags = 0;
60 char *data;
61 #endif
62 off_t offset = 0;
63 struct stat statb;
64 rrd_file_t *rrd_file = NULL;
65 off_t newfile_size = 0;
67 if (rdwr & RRD_CREAT) {
68 /* yes bad inline signaling alert, we are using the
69 floatcookie to pass the size in ... only used in resize */
70 newfile_size = (off_t) rrd->stat_head->float_cookie;
71 free(rrd->stat_head);
72 }
73 rrd_init(rrd);
74 rrd_file = malloc(sizeof(rrd_file_t));
75 if (rrd_file == NULL) {
76 rrd_set_error("allocating rrd_file descriptor for '%s'", file_name);
77 return NULL;
78 }
79 memset(rrd_file, 0, sizeof(rrd_file_t));
81 #ifdef DEBUG
82 if ((rdwr & (RRD_READONLY | RRD_READWRITE)) ==
83 (RRD_READONLY | RRD_READWRITE)) {
84 /* Both READONLY and READWRITE were given, which is invalid. */
85 rrd_set_error("in read/write request mask");
86 exit(-1);
87 }
88 #endif
89 if (rdwr & RRD_READONLY) {
90 flags |= O_RDONLY;
91 #ifdef HAVE_MMAP
92 mm_flags = MAP_PRIVATE;
93 # ifdef MAP_NORESERVE
94 mm_flags |= MAP_NORESERVE; /* readonly, so no swap backing needed */
95 # endif
96 #endif
97 } else {
98 if (rdwr & RRD_READWRITE) {
99 mode |= S_IWUSR;
100 flags |= O_RDWR;
101 #ifdef HAVE_MMAP
102 mm_flags = MAP_SHARED;
103 mm_prot |= PROT_WRITE;
104 #endif
105 }
106 if (rdwr & RRD_CREAT) {
107 flags |= (O_CREAT | O_TRUNC);
108 }
109 }
110 if (rdwr & RRD_READAHEAD) {
111 #ifdef MAP_POPULATE
112 mm_flags |= MAP_POPULATE; /* populate ptes and data */
113 #endif
114 #if defined MAP_NONBLOCK
115 mm_flags |= MAP_NONBLOCK; /* just populate ptes */
116 #endif
117 }
119 if ((rrd_file->fd = open(file_name, flags, mode)) < 0) {
120 rrd_set_error("opening '%s': %s", file_name, rrd_strerror(errno));
121 goto out_free;
122 }
124 /* Better try to avoid seeks as much as possible. stat may be heavy but
125 * many concurrent seeks are even worse. */
126 if (newfile_size == 0 && ((fstat(rrd_file->fd, &statb)) < 0)) {
127 rrd_set_error("fstat '%s': %s", file_name, rrd_strerror(errno));
128 goto out_close;
129 }
130 if (newfile_size == 0) {
131 rrd_file->file_len = statb.st_size;
132 } else {
133 rrd_file->file_len = newfile_size;
134 lseek(rrd_file->fd, newfile_size - 1, SEEK_SET);
135 write(rrd_file->fd, "\0", 1); /* poke */
136 lseek(rrd_file->fd, 0, SEEK_SET);
137 }
138 #ifdef HAVE_POSIX_FADVISE
139 /* In general we need no read-ahead when dealing with rrd_files.
140 When we stop reading, it is highly unlikely that we start up again.
141 In this manner we actually save time and diskaccess (and buffer cache).
142 Thanks to Dave Plonka for the Idea of using POSIX_FADV_RANDOM here. */
143 if (0 != posix_fadvise(rrd_file->fd, 0, 0, POSIX_FADV_RANDOM)) {
144 rrd_set_error("setting POSIX_FADV_RANDOM on '%s': %s", file_name,
145 rrd_strerror(errno));
146 goto out_close;
147 }
148 #endif
150 /*
151 if (rdwr & RRD_READWRITE)
152 {
153 if (setvbuf((rrd_file->fd),NULL,_IONBF,2)) {
154 rrd_set_error("failed to disable the stream buffer\n");
155 return (-1);
156 }
157 }
158 */
159 #ifdef HAVE_MMAP
160 data = mmap(0, rrd_file->file_len, mm_prot, mm_flags,
161 rrd_file->fd, offset);
163 /* lets see if the first read worked */
164 if (data == MAP_FAILED) {
165 rrd_set_error("mmaping file '%s': %s", file_name,
166 rrd_strerror(errno));
167 goto out_close;
168 }
169 rrd_file->file_start = data;
170 if (rdwr & RRD_CREAT) {
171 memset(data, DNAN, newfile_size - 1);
172 goto out_done;
173 }
174 #endif
175 if (rdwr & RRD_CREAT)
176 goto out_done;
177 #ifdef USE_MADVISE
178 if (rdwr & RRD_COPY) {
179 /* We will read everything in a moment (copying) */
180 madvise(data, rrd_file->file_len, MADV_WILLNEED | MADV_SEQUENTIAL);
181 } else {
182 /* We do not need to read anything in for the moment */
183 madvise(data, rrd_file->file_len, MADV_RANDOM);
184 /* the stat_head will be needed soonish, so hint accordingly */
185 madvise(data, sizeof(stat_head_t), MADV_WILLNEED | MADV_RANDOM);
186 }
187 #endif
189 __rrd_read(rrd->stat_head, stat_head_t,
190 1);
192 /* lets do some test if we are on track ... */
193 if (memcmp(rrd->stat_head->cookie, RRD_COOKIE, sizeof(RRD_COOKIE)) != 0) {
194 rrd_set_error("'%s' is not an RRD file", file_name);
195 goto out_nullify_head;
196 }
198 if (rrd->stat_head->float_cookie != FLOAT_COOKIE) {
199 rrd_set_error("This RRD was created on another architecture");
200 goto out_nullify_head;
201 }
203 version = atoi(rrd->stat_head->version);
205 if (version > atoi(RRD_VERSION)) {
206 rrd_set_error("can't handle RRD file version %s",
207 rrd->stat_head->version);
208 goto out_nullify_head;
209 }
210 #if defined USE_MADVISE
211 /* the ds_def will be needed soonish, so hint accordingly */
212 madvise(data + PAGE_START(offset),
213 sizeof(ds_def_t) * rrd->stat_head->ds_cnt, MADV_WILLNEED);
214 #endif
215 __rrd_read(rrd->ds_def, ds_def_t,
216 rrd->stat_head->ds_cnt);
218 #if defined USE_MADVISE
219 /* the rra_def will be needed soonish, so hint accordingly */
220 madvise(data + PAGE_START(offset),
221 sizeof(rra_def_t) * rrd->stat_head->rra_cnt, MADV_WILLNEED);
222 #endif
223 __rrd_read(rrd->rra_def, rra_def_t,
224 rrd->stat_head->rra_cnt);
226 /* handle different format for the live_head */
227 if (version < 3) {
228 rrd->live_head = (live_head_t *) malloc(sizeof(live_head_t));
229 if (rrd->live_head == NULL) {
230 rrd_set_error("live_head_t malloc");
231 goto out_close;
232 }
233 #ifdef HAVE_MMAP
234 memmove(&rrd->live_head->last_up, data + offset, sizeof(long));
235 offset += sizeof(long);
236 #else
237 offset += read(rrd_file->fd, &rrd->live_head->last_up, sizeof(long));
238 #endif
239 rrd->live_head->last_up_usec = 0;
240 } else {
241 #if defined USE_MADVISE
242 /* the live_head will be needed soonish, so hint accordingly */
243 madvise(data + PAGE_START(offset),
244 sizeof(live_head_t), MADV_WILLNEED);
245 #endif
246 __rrd_read(rrd->live_head, live_head_t,
247 1);
248 }
249 //XXX: This doesn't look like it needs madvise
250 __rrd_read(rrd->pdp_prep, pdp_prep_t,
251 rrd->stat_head->ds_cnt);
253 //XXX: This could benefit from madvise()ing
254 __rrd_read(rrd->cdp_prep, cdp_prep_t,
255 rrd->stat_head->rra_cnt * rrd->stat_head->ds_cnt);
257 //XXX: This could benefit from madvise()ing
258 __rrd_read(rrd->rra_ptr, rra_ptr_t,
259 rrd->stat_head->rra_cnt);
261 rrd_file->header_len = offset;
262 rrd_file->pos = offset;
263 out_done:
264 return (rrd_file);
265 out_nullify_head:
266 rrd->stat_head = NULL;
267 out_close:
268 close(rrd_file->fd);
269 out_free:
270 free(rrd_file);
271 return NULL;
272 }
275 /* Close a reference to an rrd_file. */
276 static
277 void mincore_print(
278 rrd_file_t *rrd_file,
279 char *mark)
280 {
281 #ifdef HAVE_MMAP
282 /* pretty print blocks in core */
283 off_t off;
284 unsigned char *vec;
285 ssize_t _page_size = sysconf(_SC_PAGESIZE);
287 off = rrd_file->file_len +
288 ((rrd_file->file_len + _page_size - 1) / _page_size);
289 vec = malloc(off);
290 if (vec != NULL) {
291 memset(vec, 0, off);
292 if (mincore(rrd_file->file_start, rrd_file->file_len, vec) == 0) {
293 int prev;
294 unsigned is_in = 0, was_in = 0;
296 for (off = 0, prev = 0; off < rrd_file->file_len; ++off) {
297 is_in = vec[off] & 1; /* if lsb set then is core resident */
298 if (off == 0)
299 was_in = is_in;
300 if (was_in != is_in) {
301 fprintf(stderr, "%s: %sin core: %p len %ld\n", mark,
302 was_in ? "" : "not ", vec + prev, off - prev);
303 was_in = is_in;
304 prev = off;
305 }
306 }
307 fprintf(stderr,
308 "%s: %sin core: %p len %ld\n", mark,
309 was_in ? "" : "not ", vec + prev, off - prev);
310 } else
311 fprintf(stderr, "mincore: %s", rrd_strerror(errno));
312 }
313 #else
314 fprintf(stderr, "sorry mincore only works with mmap");
315 #endif
316 }
319 /* drop cache except for the header and the active pages */
320 void rrd_dontneed(
321 rrd_file_t *rrd_file,
322 rrd_t *rrd)
323 {
324 unsigned long dontneed_start;
325 unsigned long rra_start;
326 unsigned long active_block;
327 unsigned long i;
328 ssize_t _page_size = sysconf(_SC_PAGESIZE);
330 #if defined DEBUG && DEBUG > 1
331 mincore_print(rrd_file, "before");
332 #endif
334 /* ignoring errors from RRDs that are smaller then the file_len+rounding */
335 rra_start = rrd_file->header_len;
336 dontneed_start = PAGE_START(rra_start) + _page_size;
337 for (i = 0; i < rrd->stat_head->rra_cnt; ++i) {
338 active_block =
339 PAGE_START(rra_start
340 + rrd->rra_ptr[i].cur_row
341 * rrd->stat_head->ds_cnt * sizeof(rrd_value_t));
342 if (active_block > dontneed_start) {
343 #ifdef USE_MADVISE
344 madvise(rrd_file->file_start + dontneed_start,
345 active_block - dontneed_start - 1, MADV_DONTNEED);
346 #endif
347 /* in linux at least only fadvise DONTNEED seems to purge pages from cache */
348 #ifdef HAVE_POSIX_FADVISE
349 posix_fadvise(rrd_file->fd, dontneed_start,
350 active_block - dontneed_start - 1,
351 POSIX_FADV_DONTNEED);
352 #endif
353 }
354 dontneed_start = active_block;
355 /* do not relase 'hot' block if update for this RAA will occure within 10 minutes */
356 if (rrd->stat_head->pdp_step * rrd->rra_def[i].pdp_cnt -
357 rrd->live_head->last_up % (rrd->stat_head->pdp_step *
358 rrd->rra_def[i].pdp_cnt) < 10 * 60) {
359 dontneed_start += _page_size;
360 }
361 rra_start +=
362 rrd->rra_def[i].row_cnt * rrd->stat_head->ds_cnt *
363 sizeof(rrd_value_t);
364 }
365 #ifdef USE_MADVISE
366 madvise(rrd_file->file_start + dontneed_start,
367 rrd_file->file_len - dontneed_start, MADV_DONTNEED);
368 #endif
369 #ifdef HAVE_POSIX_FADVISE
370 posix_fadvise(rrd_file->fd, dontneed_start,
371 rrd_file->file_len - dontneed_start, POSIX_FADV_DONTNEED);
372 #endif
373 #if defined DEBUG && DEBUG > 1
374 mincore_print(rrd_file, "after");
375 #endif
376 }
378 int rrd_close(
379 rrd_file_t *rrd_file)
380 {
381 int ret;
383 #ifdef HAVE_MMAP
384 ret = munmap(rrd_file->file_start, rrd_file->file_len);
385 if (ret != 0)
386 rrd_set_error("munmap rrd_file: %s", rrd_strerror(errno));
387 #endif
388 ret = close(rrd_file->fd);
389 if (ret != 0)
390 rrd_set_error("closing file: %s", rrd_strerror(errno));
391 free(rrd_file);
392 rrd_file = NULL;
393 return ret;
394 }
397 /* Set position of rrd_file. */
399 off_t rrd_seek(
400 rrd_file_t *rrd_file,
401 off_t off,
402 int whence)
403 {
404 off_t ret = 0;
406 #ifdef HAVE_MMAP
407 if (whence == SEEK_SET)
408 rrd_file->pos = off;
409 else if (whence == SEEK_CUR)
410 rrd_file->pos += off;
411 else if (whence == SEEK_END)
412 rrd_file->pos = rrd_file->file_len + off;
413 #else
414 ret = lseek(rrd_file->fd, off, whence);
415 if (ret < 0)
416 rrd_set_error("lseek: %s", rrd_strerror(errno));
417 rrd_file->pos = ret;
418 #endif
419 //XXX: mimic fseek, which returns 0 upon success
420 return ret == -1; //XXX: or just ret to mimic lseek
421 }
424 /* Get current position in rrd_file. */
426 inline off_t rrd_tell(
427 rrd_file_t *rrd_file)
428 {
429 return rrd_file->pos;
430 }
433 /* read count bytes into buffer buf, starting at rrd_file->pos.
434 * Returns the number of bytes read or <0 on error. */
436 inline ssize_t rrd_read(
437 rrd_file_t *rrd_file,
438 void *buf,
439 size_t count)
440 {
441 #ifdef HAVE_MMAP
442 size_t _cnt = count;
443 ssize_t _surplus = rrd_file->pos + _cnt - rrd_file->file_len;
445 if (_surplus > 0) { /* short read */
446 _cnt -= _surplus;
447 }
448 if (_cnt == 0)
449 return 0; /* EOF */
450 buf = memcpy(buf, rrd_file->file_start + rrd_file->pos, _cnt);
452 rrd_file->pos += _cnt; /* mimmic read() semantics */
453 return _cnt;
454 #else
455 ssize_t ret;
457 ret = read(rrd_file->fd, buf, count);
458 if (ret > 0)
459 rrd_file->pos += ret; /* mimmic read() semantics */
460 return ret;
461 #endif
462 }
465 /* write count bytes from buffer buf to the current position
466 * rrd_file->pos of rrd_file->fd.
467 * Returns the number of bytes written. */
469 inline ssize_t rrd_write(
470 rrd_file_t *rrd_file,
471 const void *buf,
472 size_t count)
473 {
474 #ifdef HAVE_MMAP
475 memcpy(rrd_file->file_start + rrd_file->pos, buf, count);
476 rrd_file->pos += count;
477 return count; /* mimmic write() semantics */
478 #else
479 ssize_t _sz = write(rrd_file->fd, buf, count);
481 if (_sz > 0)
482 rrd_file->pos += _sz;
483 return _sz;
484 #endif
485 }
488 /* flush all data pending to be written to FD. */
490 inline void rrd_flush(
491 rrd_file_t *rrd_file)
492 {
493 if (fdatasync(rrd_file->fd) != 0) {
494 rrd_set_error("flushing fd %d: %s", rrd_file->fd,
495 rrd_strerror(errno));
496 }
497 }
500 /* Initialize RRD header. */
502 void rrd_init(
503 rrd_t *rrd)
504 {
505 rrd->stat_head = NULL;
506 rrd->ds_def = NULL;
507 rrd->rra_def = NULL;
508 rrd->live_head = NULL;
509 rrd->rra_ptr = NULL;
510 rrd->pdp_prep = NULL;
511 rrd->cdp_prep = NULL;
512 rrd->rrd_value = NULL;
513 }
516 /* free RRD header data. */
518 #ifdef HAVE_MMAP
519 inline void rrd_free(
520 rrd_t UNUSED(*rrd))
521 {
522 }
523 #else
524 void rrd_free(
525 rrd_t *rrd)
526 {
527 free(rrd->live_head);
528 free(rrd->stat_head);
529 free(rrd->ds_def);
530 free(rrd->rra_def);
531 free(rrd->rra_ptr);
532 free(rrd->pdp_prep);
533 free(rrd->cdp_prep);
534 free(rrd->rrd_value);
535 }
536 #endif
539 /* routine used by external libraries to free memory allocated by
540 * rrd library */
542 void rrd_freemem(
543 void *mem)
544 {
545 free(mem);
546 }
549 /* XXX: FIXME: missing documentation. */
550 /*XXX: FIXME should be renamed to rrd_readfile or _rrd_readfile */
552 int /*_rrd_*/ readfile(
553 const char *file_name,
554 char **buffer,
555 int skipfirst)
556 {
557 long writecnt = 0, totalcnt = MEMBLK;
558 long offset = 0;
559 FILE *input = NULL;
560 char c;
562 if ((strcmp("-", file_name) == 0)) {
563 input = stdin;
564 } else {
565 if ((input = fopen(file_name, "rb")) == NULL) {
566 rrd_set_error("opening '%s': %s", file_name, rrd_strerror(errno));
567 return (-1);
568 }
569 }
570 if (skipfirst) {
571 do {
572 c = getc(input);
573 offset++;
574 } while (c != '\n' && !feof(input));
575 }
576 if (strcmp("-", file_name)) {
577 fseek(input, 0, SEEK_END);
578 /* have extra space for detecting EOF without realloc */
579 totalcnt = (ftell(input) + 1) / sizeof(char) - offset;
580 if (totalcnt < MEMBLK)
581 totalcnt = MEMBLK; /* sanitize */
582 fseek(input, offset * sizeof(char), SEEK_SET);
583 }
584 if (((*buffer) = (char *) malloc((totalcnt + 4) * sizeof(char))) == NULL) {
585 perror("Allocate Buffer:");
586 exit(1);
587 };
588 do {
589 writecnt +=
590 fread((*buffer) + writecnt, 1,
591 (totalcnt - writecnt) * sizeof(char), input);
592 if (writecnt >= totalcnt) {
593 totalcnt += MEMBLK;
594 if (((*buffer) =
595 rrd_realloc((*buffer),
596 (totalcnt + 4) * sizeof(char))) == NULL) {
597 perror("Realloc Buffer:");
598 exit(1);
599 };
600 }
601 } while (!feof(input));
602 (*buffer)[writecnt] = '\0';
603 if (strcmp("-", file_name) != 0) {
604 fclose(input);
605 };
606 return writecnt;
607 }