baeb00355308de85f3a83e6a54c25947ff08fde9
1 /*****************************************************************************
2 * RRDtool 1.2.23 Copyright by Tobi Oetiker, 1997-2007
3 *****************************************************************************
4 * rrd_open.c Open an RRD File
5 *****************************************************************************
6 * $Id$
7 * $Log$
8 * Revision 1.10 2004/05/26 22:11:12 oetiker
9 * reduce compiler warnings. Many small fixes. -- Mike Slifcak <slif@bellsouth.net>
10 *
11 * Revision 1.9 2003/04/29 21:56:49 oetiker
12 * readline in rrd_open.c reads the file in 8 KB blocks, and calls realloc for
13 * each block. realloc is very slow in Mac OS X for huge blocks, e.g. when
14 * restoring databases from huge xml files. This patch finds the size of the
15 * file, and starts out with malloc'ing the full size.
16 * -- Peter Speck <speck@ruc.dk>
17 *
18 * Revision 1.8 2003/04/11 19:43:44 oetiker
19 * New special value COUNT which allows calculations based on the position of a
20 * value within a data set. Bug fix in rrd_rpncalc.c. PREV returned erroneus
21 * value for the second value. Bug fix in rrd_restore.c. Bug causing seek error
22 * when accesing an RRD restored from an xml that holds an RRD version <3.
23 * -- Ruben Justo <ruben@ainek.com>
24 *
25 * Revision 1.7 2003/03/31 21:22:12 oetiker
26 * enables RRDtool updates with microsecond or in case of windows millisecond
27 * precision. This is needed to reduce time measurement error when archive step
28 * is small. (<30s) -- Sasha Mikheev <sasha@avalon-net.co.il>
29 *
30 * Revision 1.6 2003/02/13 07:05:27 oetiker
31 * Find attached the patch I promised to send to you. Please note that there
32 * are three new source files (src/rrd_is_thread_safe.h, src/rrd_thread_safe.c
33 * and src/rrd_not_thread_safe.c) and the introduction of librrd_th. This
34 * library is identical to librrd, but it contains support code for per-thread
35 * global variables currently used for error information only. This is similar
36 * to how errno per-thread variables are implemented. librrd_th must be linked
37 * alongside of libpthred
38 *
39 * There is also a new file "THREADS", holding some documentation.
40 *
41 * -- Peter Stamfest <peter@stamfest.at>
42 *
43 * Revision 1.5 2002/06/20 00:21:03 jake
44 * More Win32 build changes; thanks to Kerry Calvert.
45 *
46 * Revision 1.4 2002/02/01 20:34:49 oetiker
47 * fixed version number and date/time
48 *
49 * Revision 1.3 2001/03/04 13:01:55 oetiker
50 * Aberrant Behavior Detection support. A brief overview added to rrdtool.pod.
51 * Major updates to rrd_update.c, rrd_create.c. Minor update to other core files.
52 * This is backwards compatible! But new files using the Aberrant stuff are not readable
53 * by old rrdtool versions. See http://cricket.sourceforge.net/aberrant/rrd_hw.htm
54 * -- Jake Brutlag <jakeb@corp.webtv.net>
55 *
56 * Revision 1.2 2001/03/04 10:29:20 oetiker
57 * fixed filedescriptor leak
58 * -- Mike Franusich <mike@franusich.com>
59 *
60 * Revision 1.1.1.1 2001/02/25 22:25:05 oetiker
61 * checkin
62 *
63 *****************************************************************************/
65 #include "rrd_tool.h"
66 #include "unused.h"
67 #define MEMBLK 8192
69 #ifdef HAVE_MMAP
70 #define __rrd_read(dst, dst_t, cnt) \
71 (dst) = (dst_t*) (data + offset); \
72 offset += sizeof(dst_t) * (cnt)
73 #else
74 #define __rrd_read(dst, dst_t, cnt) \
75 if ((dst = malloc(sizeof(dst_t)*(cnt))) == NULL) { \
76 rrd_set_error(#dst " malloc"); \
77 goto out_nullify_head; \
78 } \
79 offset += read (rrd_file->fd, dst, sizeof(dst_t)*(cnt))
80 #endif
82 /* open a database file, return its header and an open filehandle */
83 /* positioned to the first cdp in the first rra */
85 rrd_file_t *rrd_open(
86 const char *const file_name,
87 rrd_t *rrd,
88 unsigned rdwr)
89 {
90 int flags = 0;
91 mode_t mode = S_IRUSR;
92 int version;
94 #ifdef HAVE_MMAP
95 int mm_prot = PROT_READ, mm_flags = 0;
96 char *data;
97 #endif
98 off_t offset = 0;
99 struct stat statb;
100 rrd_file_t *rrd_file = malloc(sizeof(rrd_file_t));
102 if (rrd_file == NULL) {
103 rrd_set_error("allocating rrd_file descriptor for '%s'", file_name);
104 return NULL;
105 }
106 memset(rrd_file, 0, sizeof(rrd_file_t));
107 rrd_init(rrd);
108 #ifdef DEBUG
109 if ((rdwr & (RRD_READONLY | RRD_READWRITE)) ==
110 (RRD_READONLY | RRD_READWRITE)) {
111 /* Both READONLY and READWRITE were given, which is invalid. */
112 rrd_set_error("in read/write request mask");
113 exit(-1);
114 }
115 #endif
116 if (rdwr & RRD_READONLY) {
117 flags |= O_RDONLY;
118 #ifdef HAVE_MMAP
119 mm_flags = MAP_PRIVATE;
120 # ifdef MAP_NORESERVE
121 mm_flags |= MAP_NORESERVE;
122 # endif
123 mm_flags |= MAP_PRIVATE;
124 #endif
125 } else {
126 if (rdwr & RRD_READWRITE) {
127 mode |= S_IWUSR;
128 flags |= O_RDWR;
129 #ifdef HAVE_MMAP
130 mm_flags = MAP_SHARED;
131 mm_prot |= PROT_WRITE;
132 #endif
133 }
134 if (rdwr & RRD_CREAT) {
135 flags |= (O_CREAT | O_TRUNC);
136 }
137 }
138 if (rdwr & RRD_READAHEAD) {
139 #ifdef MAP_POPULATE
140 mm_flags |= MAP_POPULATE;
141 #endif
142 #if defined MAP_NONBLOCK && !defined USE_DIRECT_IO
143 mm_flags |= MAP_NONBLOCK; /* just populage ptes */
144 #endif
145 } else {
146 #ifdef USE_DIRECT_IO
147 flags |= O_DIRECT;
148 #endif
149 #if 0 //def O_NONBLOCK
150 flags |= O_NONBLOCK;
151 #endif
152 }
154 if ((rrd_file->fd = open(file_name, flags, mode)) < 0) {
155 rrd_set_error("opening '%s': %s", file_name, rrd_strerror(errno));
156 return NULL;
157 }
159 /* Better try to avoid seeks as much as possible. stat may be heavy but
160 * many concurrent seeks are even worse. */
161 if ((fstat(rrd_file->fd, &statb)) < 0) {
162 rrd_set_error("fstat '%s': %s", file_name, rrd_strerror(errno));
163 goto out_close;
164 }
165 rrd_file->file_len = statb.st_size;
167 #ifdef HAVE_POSIX_FADVISE
168 /* In general we need no read-ahead when dealing with rrd_files.
169 When we stop reading, it is highly unlikely that we start up again.
170 In this manner we actually save time and diskaccess (and buffer cache).
171 Thanks to Dave Plonka for the Idea of using POSIX_FADV_RANDOM here. */
172 if (0 != posix_fadvise(rrd_file->fd, 0, 0, POSIX_FADV_RANDOM)) {
173 rrd_set_error("setting POSIX_FADV_RANDOM on '%s': %s", file_name,
174 rrd_strerror(errno));
175 goto out_close;
176 }
177 #endif
179 /*
180 if (rdwr & RRD_READWRITE)
181 {
182 if (setvbuf((rrd_file->fd),NULL,_IONBF,2)) {
183 rrd_set_error("failed to disable the stream buffer\n");
184 return (-1);
185 }
186 }
187 */
188 #ifdef HAVE_MMAP
189 data = mmap(0, rrd_file->file_len, mm_prot, mm_flags,
190 rrd_file->fd, offset);
192 /* lets see if the first read worked */
193 if (data == MAP_FAILED) {
194 rrd_set_error("error mmaping file '%s': %s", file_name,
195 rrd_strerror(errno));
196 goto out_close;
197 }
198 rrd_file->file_start = data;
199 #else
200 #endif
201 #ifdef USE_MADVISE
202 if (rdwr & RRD_COPY) { /*XXX: currently not used! */
203 /* We will read everything in a moment (copying) */
204 madvise(data, rrd_file->file_len, MADV_WILLNEED | MADV_SEQUENTIAL);
205 goto out_done;
206 }
207 /* We do not need to read anything in for the moment */
208 madvise(data, rrd_file->file_len, MADV_DONTNEED);
209 #endif
211 #ifdef USE_MADVISE
212 /* the stat_head will be needed soonish, so hint accordingly */
213 madvise(data + offset, sizeof(stat_head_t), MADV_WILLNEED);
214 #endif
216 __rrd_read(rrd->stat_head, stat_head_t,
217 1);
219 /* lets do some test if we are on track ... */
220 if (memcmp(rrd->stat_head->cookie, RRD_COOKIE, sizeof(RRD_COOKIE)) != 0) {
221 rrd_set_error("'%s' is not an RRD file", file_name);
222 goto out_nullify_head;
223 }
225 if (rrd->stat_head->float_cookie != FLOAT_COOKIE) {
226 rrd_set_error("This RRD was created on other architecture");
227 goto out_nullify_head;
228 }
230 version = atoi(rrd->stat_head->version);
232 if (version > atoi(RRD_VERSION)) {
233 rrd_set_error("can't handle RRD file version %s",
234 rrd->stat_head->version);
235 goto out_nullify_head;
236 }
237 #ifdef USE_MADVISE
238 /* the ds_def will be needed soonish, so hint accordingly */
239 madvise(data + offset, sizeof(ds_def_t) * rrd->stat_head->ds_cnt,
240 MADV_WILLNEED);
241 #endif
242 __rrd_read(rrd->ds_def, ds_def_t,
243 rrd->stat_head->ds_cnt);
245 #ifdef USE_MADVISE
246 /* the rra_def will be needed soonish, so hint accordingly */
247 madvise(data + offset, sizeof(rra_def_t) * rrd->stat_head->rra_cnt,
248 MADV_WILLNEED);
249 #endif
250 __rrd_read(rrd->rra_def, rra_def_t,
251 rrd->stat_head->rra_cnt);
253 /* handle different format for the live_head */
254 if (version < 3) {
255 rrd->live_head = (live_head_t *) malloc(sizeof(live_head_t));
256 if (rrd->live_head == NULL) {
257 rrd_set_error("live_head_t malloc");
258 goto out_close;
259 }
260 #ifdef HAVE_MMAP
261 memmove(&rrd->live_head->last_up, data + offset, sizeof(long));
262 offset += sizeof(long);
263 #else
264 offset += read(rrd_file->fd, &rrd->live_head->last_up, sizeof(long));
265 #endif
266 rrd->live_head->last_up_usec = 0;
267 } else {
268 #ifdef USE_MADVISE
269 /* the live_head will be needed soonish, so hint accordingly */
270 madvise(data + offset, sizeof(live_head_t), MADV_WILLNEED);
271 #endif
272 __rrd_read(rrd->live_head, live_head_t,
273 1);
274 }
275 //XXX: This doesn't look like it needs madvise
276 __rrd_read(rrd->pdp_prep, pdp_prep_t,
277 rrd->stat_head->ds_cnt);
279 //XXX: This could benefit from madvise()ing
280 __rrd_read(rrd->cdp_prep, cdp_prep_t,
281 rrd->stat_head->rra_cnt * rrd->stat_head->ds_cnt);
283 //XXX: This could benefit from madvise()ing
284 __rrd_read(rrd->rra_ptr, rra_ptr_t,
285 rrd->stat_head->rra_cnt);
287 #ifdef USE_MADVISE
288 out_done:
289 #endif
290 rrd_file->header_len = offset;
291 rrd_file->pos = offset;
292 /* we could close(rrd_file->fd); here, the mapping is still valid anyway */
293 return (rrd_file);
294 out_nullify_head:
295 rrd->stat_head = NULL;
296 out_close:
297 close(rrd_file->fd);
298 return NULL;
299 }
301 /* Close a reference to an rrd_file. */
302 int rrd_close(
303 rrd_file_t *rrd_file)
304 {
305 int ret;
307 #ifdef HAVE_MMAP
308 ret = munmap(rrd_file->file_start, rrd_file->file_len);
309 if (ret != 0)
310 rrd_set_error("munmap rrd_file: %s", rrd_strerror(errno));
311 #endif
312 ret = close(rrd_file->fd);
313 if (ret != 0)
314 rrd_set_error("closing file: %s", rrd_strerror(errno));
315 free(rrd_file);
316 rrd_file = NULL;
317 return ret;
318 }
320 /* Set position of rrd_file. */
321 off_t rrd_seek(
322 rrd_file_t *rrd_file,
323 off_t off,
324 int whence)
325 {
326 off_t ret = 0;
328 #ifdef HAVE_MMAP
329 if (whence == SEEK_SET)
330 rrd_file->pos = off;
331 else if (whence == SEEK_CUR)
332 rrd_file->pos += off;
333 else if (whence == SEEK_END)
334 rrd_file->pos = rrd_file->file_len + off;
335 #else
336 ret = lseek(rrd_file->fd, off, whence);
337 if (ret < 0)
338 rrd_set_error("lseek: %s", rrd_strerror(errno));
339 rrd_file->pos = ret;
340 #endif
341 //XXX: mimic fseek, which returns 0 upon success
342 return ret == -1; //XXX: or just ret to mimic lseek
343 }
345 /* Get current position in rrd_file. */
346 inline off_t rrd_tell(
347 rrd_file_t *rrd_file)
348 {
349 return rrd_file->pos;
350 }
352 /* read count bytes into buffer buf, starting at rrd_file->pos.
353 * Returns the number of bytes read. */
354 ssize_t rrd_read(
355 rrd_file_t *rrd_file,
356 void *buf,
357 size_t count)
358 {
359 #ifdef HAVE_MMAP
360 char *pos = rrd_file->file_start + rrd_file->pos;
362 buf = memmove(buf, pos, count);
363 rrd_file->pos += count; /* mimmic read() semantics */
364 return count;
365 #else
366 ssize_t ret;
368 ret = read(rrd_file->fd, buf, count);
369 //XXX: eventually add generic rrd_set_error(""); here
370 rrd_file->pos += count; /* mimmic read() semantics */
371 return ret;
372 #endif
373 }
375 /* write count bytes from buffer buf to the current position
376 * rrd_file->pos of rrd_file->fd.
377 * Returns the number of bytes written. */
378 ssize_t rrd_write(
379 rrd_file_t *rrd_file,
380 const void *buf,
381 size_t count)
382 {
383 #ifdef HAVE_MMAP
384 memmove(rrd_file->file_start + rrd_file->pos, buf, count);
385 return count; /* mimmic write() semantics */
386 #else
387 return write(rrd_file->fd, buf, count);
388 #endif
389 }
391 /* flush all data pending to be written to FD. */
392 inline void rrd_flush(
393 rrd_file_t *rrd_file)
394 {
395 if (fdatasync(rrd_file->fd) != 0) {
396 rrd_set_error("flushing fd %d: %s", rrd_file->fd,
397 rrd_strerror(errno));
398 }
399 }
401 void rrd_init(
402 rrd_t *rrd)
403 {
404 rrd->stat_head = NULL;
405 rrd->ds_def = NULL;
406 rrd->rra_def = NULL;
407 rrd->live_head = NULL;
408 rrd->rra_ptr = NULL;
409 rrd->pdp_prep = NULL;
410 rrd->cdp_prep = NULL;
411 rrd->rrd_value = NULL;
412 }
414 void rrd_free(
415 rrd_t UNUSED(*rrd))
416 {
417 #ifndef HAVE_MMAP
418 if (atoi(rrd->stat_head->version) < 3)
419 free(rrd->live_head);
420 free(rrd->stat_head);
421 free(rrd->ds_def);
422 free(rrd->rra_def);
423 free(rrd->rra_ptr);
424 free(rrd->pdp_prep);
425 free(rrd->cdp_prep);
426 free(rrd->rrd_value);
427 //XXX: ? rrd_init(rrd);
428 #endif
429 }
431 /* routine used by external libraries to free memory allocated by
432 * rrd library */
433 void rrd_freemem(
434 void *mem)
435 {
436 free(mem);
437 }
439 int readfile(
440 const char *file_name,
441 char **buffer,
442 int skipfirst)
443 {
444 long writecnt = 0, totalcnt = MEMBLK;
445 long offset = 0;
446 FILE *input = NULL;
447 char c;
449 if ((strcmp("-", file_name) == 0)) {
450 input = stdin;
451 } else {
452 if ((input = fopen(file_name, "rb")) == NULL) {
453 rrd_set_error("opening '%s': %s", file_name, rrd_strerror(errno));
454 return (-1);
455 }
456 }
457 if (skipfirst) {
458 do {
459 c = getc(input);
460 offset++;
461 } while (c != '\n' && !feof(input));
462 }
463 if (strcmp("-", file_name)) {
464 fseek(input, 0, SEEK_END);
465 /* have extra space for detecting EOF without realloc */
466 totalcnt = (ftell(input) + 1) / sizeof(char) - offset;
467 if (totalcnt < MEMBLK)
468 totalcnt = MEMBLK; /* sanitize */
469 fseek(input, offset * sizeof(char), SEEK_SET);
470 }
471 if (((*buffer) = (char *) malloc((totalcnt + 4) * sizeof(char))) == NULL) {
472 perror("Allocate Buffer:");
473 exit(1);
474 };
475 do {
476 writecnt +=
477 fread((*buffer) + writecnt, 1,
478 (totalcnt - writecnt) * sizeof(char), input);
479 if (writecnt >= totalcnt) {
480 totalcnt += MEMBLK;
481 if (((*buffer) =
482 rrd_realloc((*buffer),
483 (totalcnt + 4) * sizeof(char))) == NULL) {
484 perror("Realloc Buffer:");
485 exit(1);
486 };
487 }
488 } while (!feof(input));
489 (*buffer)[writecnt] = '\0';
490 if (strcmp("-", file_name) != 0) {
491 fclose(input);
492 };
493 return writecnt;
494 }