Code

parser: Let the TIMESERIES command accept optional data-source names.
[sysdb.git] / src / frontend / query.c
1 /*
2  * SysDB - src/frontend/query.c
3  * Copyright (C) 2013-2014 Sebastian 'tokkee' Harl <sh@tokkee.org>
4  * All rights reserved.
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions
8  * are met:
9  * 1. Redistributions of source code must retain the above copyright
10  *    notice, this list of conditions and the following disclaimer.
11  * 2. Redistributions in binary form must reproduce the above copyright
12  *    notice, this list of conditions and the following disclaimer in the
13  *    documentation and/or other materials provided with the distribution.
14  *
15  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
16  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
17  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
18  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR
19  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
20  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
21  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
22  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
23  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
24  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
25  * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26  */
28 #ifdef HAVE_CONFIG_H
29 #       include "config.h"
30 #endif
32 #include "sysdb.h"
34 #include "core/plugin.h"
35 #include "frontend/connection-private.h"
36 #include "parser/ast.h"
37 #include "parser/parser.h"
38 #include "utils/error.h"
39 #include "utils/proto.h"
40 #include "utils/strbuf.h"
42 #include <errno.h>
43 #include <ctype.h>
44 #include <string.h>
46 /*
47  * metric fetcher:
48  * Implements the callbacks necessary to read a metric object.
49  */
51 typedef struct {
52         char *type;
53         char *id;
54         sdb_time_t last_update;
55 } metric_store_t;
57 static int
58 metric_fetcher_host(sdb_store_host_t __attribute__((unused)) *host,
59                 sdb_object_t __attribute__((unused)) *user_data)
60 {
61         return 0;
62 } /* metric_fetcher_host */
64 static int
65 metric_fetcher_metric(sdb_store_metric_t *metric, sdb_object_t *user_data)
66 {
67         metric_store_t *st = SDB_OBJ_WRAPPER(user_data)->data;
68         sdb_time_t last_update = 0;
69         size_t idx = 0, i;
71         if (! metric->stores_num)
72                 return -1;
74         /* Find the most up to date data store.
75          * TODO: Consider merging multiple results? */
76         for (i = 0; i < metric->stores_num; ++i) {
77                 if (metric->stores[i].last_update > last_update) {
78                         last_update = metric->stores[i].last_update;
79                         idx = i;
80                 }
81         }
82         st->type = strdup(metric->stores[idx].type);
83         st->id = strdup(metric->stores[idx].id);
84         st->last_update = metric->stores[idx].last_update;
85         return 0;
86 } /* metric_fetcher_metric */
88 static sdb_store_writer_t metric_fetcher = {
89         metric_fetcher_host, NULL, metric_fetcher_metric, NULL,
90 };
92 /*
93  * private helper functions
94  */
96 static char *
97 sstrdup(const char *s)
98 {
99         return s ? strdup(s) : NULL;
100 } /* sstrdup */
102 static size_t
103 sstrlen(const char *s)
105         return s ? strlen(s) : 0;
106 } /* sstrlen */
108 static int
109 exec_query(sdb_ast_node_t *ast, sdb_strbuf_t *buf, sdb_strbuf_t *errbuf)
111         sdb_store_json_formatter_t *f;
112         int type = 0, flags = 0;
113         uint32_t res_type = 0;
114         int status;
116         switch (ast->type) {
117         case SDB_AST_TYPE_FETCH:
118                 type = SDB_AST_FETCH(ast)->obj_type;
119                 res_type = htonl(SDB_CONNECTION_FETCH);
120                 break;
121         case SDB_AST_TYPE_LIST:
122                 type = SDB_AST_LIST(ast)->obj_type;
123                 flags = SDB_WANT_ARRAY;
124                 res_type = htonl(SDB_CONNECTION_LIST);
125                 break;
126         case SDB_AST_TYPE_LOOKUP:
127                 type = SDB_AST_LOOKUP(ast)->obj_type;
128                 flags = SDB_WANT_ARRAY;
129                 res_type = htonl(SDB_CONNECTION_LOOKUP);
130                 break;
131         default:
132                 sdb_strbuf_sprintf(errbuf, "invalid command %s (%#x)",
133                                 SDB_AST_TYPE_TO_STRING(ast), ast->type);
134                 return -1;
135         }
137         f = sdb_store_json_formatter(buf, type, flags);
138         sdb_strbuf_memcpy(buf, &res_type, sizeof(res_type));
139         status = sdb_plugin_query(ast, &sdb_store_json_writer, SDB_OBJ(f),
140                         &(sdb_query_opts_t){ true }, errbuf);
141         if (status < 0)
142                 sdb_strbuf_clear(buf);
143         sdb_store_json_finish(f);
144         sdb_object_deref(SDB_OBJ(f));
145         return status;
146 } /* exec_query */
148 static int
149 exec_store(sdb_ast_store_t *st, sdb_strbuf_t *buf, sdb_strbuf_t *errbuf)
151         char name[sstrlen(st->hostname) + sstrlen(st->parent) + sstrlen(st->name) + 3];
152         sdb_metric_store_t metric_store;
153         int type = st->obj_type, status = -1;
155         switch (st->obj_type) {
156         case SDB_HOST:
157                 strncpy(name, st->name, sizeof(name));
158                 status = sdb_plugin_store_host(st->name, st->last_update);
159                 break;
161         case SDB_SERVICE:
162                 snprintf(name, sizeof(name), "%s.%s", st->hostname, st->name);
163                 status = sdb_plugin_store_service(st->hostname, st->name, st->last_update);
164                 break;
166         case SDB_METRIC:
167                 snprintf(name, sizeof(name), "%s.%s", st->hostname, st->name);
168                 metric_store.type = st->store_type;
169                 metric_store.id = st->store_id;
170                 metric_store.last_update = st->store_last_update;
171                 status = sdb_plugin_store_metric(st->hostname, st->name,
172                                 &metric_store, st->last_update);
173                 break;
175         case SDB_ATTRIBUTE:
176                 type |= st->parent_type;
178                 if (st->parent)
179                         snprintf(name, sizeof(name), "%s.%s.%s",
180                                         st->hostname, st->parent, st->name);
181                 else
182                         snprintf(name, sizeof(name), "%s.%s", st->hostname, st->name);
184                 switch (st->parent_type) {
185                 case 0:
186                         type |= SDB_HOST;
187                         status = sdb_plugin_store_attribute(st->hostname,
188                                         st->name, &st->value, st->last_update);
189                         break;
191                 case SDB_SERVICE:
192                         status = sdb_plugin_store_service_attribute(st->hostname, st->parent,
193                                         st->name, &st->value, st->last_update);
194                         break;
196                 case SDB_METRIC:
197                         status = sdb_plugin_store_metric_attribute(st->hostname, st->parent,
198                                         st->name, &st->value, st->last_update);
199                         break;
201                 default:
202                         sdb_log(SDB_LOG_ERR, "store: Invalid parent type in STORE: %s",
203                                         SDB_STORE_TYPE_TO_NAME(st->parent_type));
204                         return -1;
205                 }
206                 break;
208         default:
209                 sdb_log(SDB_LOG_ERR, "store: Invalid object type in STORE: %s",
210                                 SDB_STORE_TYPE_TO_NAME(st->obj_type));
211                 return -1;
212         }
214         if (status < 0) {
215                 sdb_strbuf_sprintf(errbuf, "STORE: Failed to store %s object",
216                                 SDB_STORE_TYPE_TO_NAME(type));
217                 return -1;
218         }
220         if (! status) {
221                 sdb_strbuf_sprintf(buf, "Successfully stored %s %s",
222                                 SDB_STORE_TYPE_TO_NAME(type), name);
223         }
224         else {
225                 char type_str[32];
226                 strncpy(type_str, SDB_STORE_TYPE_TO_NAME(type), sizeof(type_str));
227                 type_str[0] = (char)toupper((int)type_str[0]);
228                 sdb_strbuf_sprintf(buf, "%s %s already up to date", type_str, name);
229         }
231         return SDB_CONNECTION_OK;
232 } /* exec_store */
234 static int
235 exec_timeseries(sdb_ast_timeseries_t *ts, sdb_strbuf_t *buf, sdb_strbuf_t *errbuf)
237         metric_store_t st = { NULL, NULL, 0 };
238         sdb_object_wrapper_t obj = SDB_OBJECT_WRAPPER_STATIC(&st);
239         sdb_ast_fetch_t fetch = SDB_AST_FETCH_INIT;
240         sdb_timeseries_opts_t opts = { 0, 0, NULL, 0 };
241         sdb_timeseries_t *series = NULL;
242         int status;
244         if ((! ts) || (! ts->hostname) || (! ts->metric))
245                 return -1;
247         fetch.obj_type = SDB_METRIC;
248         fetch.hostname = strdup(ts->hostname);
249         fetch.name = strdup(ts->metric);
250         opts.data_names = (const char * const *)ts->data_names;
251         opts.data_names_len = ts->data_names_len;
252         opts.start = ts->start;
253         opts.end = ts->end;
255         status = sdb_plugin_query(SDB_AST_NODE(&fetch),
256                         &metric_fetcher, SDB_OBJ(&obj),
257                         &(sdb_query_opts_t){ true }, errbuf);
258         if ((status < 0) || (! st.type) || (! st.id)) {
259                 sdb_log(SDB_LOG_ERR, "frontend: Failed to fetch time-series '%s/%s' "
260                                 "- no data-store configured for the stored metric",
261                                 ts->hostname, ts->metric);
262                 status = -1;
263         }
264         if (status >= 0) {
265                 series = sdb_plugin_fetch_timeseries(st.type, st.id, &opts);
266                 if (series) {
267                         uint32_t res_type = htonl(SDB_CONNECTION_TIMESERIES);
268                         sdb_strbuf_memcpy(buf, &res_type, sizeof(res_type));
269                         sdb_timeseries_tojson(series, buf);
270                         sdb_timeseries_destroy(series);
271                 }
272                 else {
273                         sdb_log(SDB_LOG_ERR, "frontend: Failed to fetch time-series '%s/%s' "
274                                         "- %s fetcher callback returned no data for '%s'",
275                                         ts->hostname, ts->metric, st.type, st.id);
276                         status = -1;
277                 }
278         }
280         free(fetch.hostname);
281         free(fetch.name);
282         if (st.type)
283                 free(st.type);
284         if (st.id)
285                 free(st.id);
286         return status;
287 } /* exec_timeseries */
289 static int
290 exec_cmd(sdb_conn_t *conn, sdb_ast_node_t *ast)
292         sdb_strbuf_t *buf;
293         int status;
295         if (! ast) {
296                 sdb_strbuf_sprintf(conn->errbuf, "out of memory");
297                 return -1;
298         }
300         buf = sdb_strbuf_create(1024);
301         if (! buf) {
302                 sdb_strbuf_sprintf(conn->errbuf, "Out of memory");
303                 return -1;
304         }
306         if (ast->type == SDB_AST_TYPE_STORE)
307                 status = exec_store(SDB_AST_STORE(ast), buf, conn->errbuf);
308         else if (ast->type == SDB_AST_TYPE_TIMESERIES)
309                 status = exec_timeseries(SDB_AST_TIMESERIES(ast), buf, conn->errbuf);
310         else
311                 status = exec_query(ast, buf, conn->errbuf);
313         if (status < 0) {
314                 char query[conn->cmd_len + 1];
315                 strncpy(query, sdb_strbuf_string(conn->buf), conn->cmd_len);
316                 query[sizeof(query) - 1] = '\0';
317                 sdb_log(SDB_LOG_ERR, "frontend: failed to execute query '%s'", query);
318         }
319         else
320                 sdb_connection_send(conn, status,
321                                 (uint32_t)sdb_strbuf_len(buf), sdb_strbuf_string(buf));
323         sdb_strbuf_destroy(buf);
324         return status < 0 ? status : 0;
325 } /* exec_cmd */
327 /*
328  * public API
329  */
331 int
332 sdb_conn_query(sdb_conn_t *conn)
334         sdb_llist_t *parsetree;
335         sdb_ast_node_t *ast = NULL;
336         int status = 0;
338         if ((! conn) || (conn->cmd != SDB_CONNECTION_QUERY))
339                 return -1;
341         parsetree = sdb_parser_parse(sdb_strbuf_string(conn->buf),
342                         (int)conn->cmd_len, conn->errbuf);
343         if (! parsetree) {
344                 char query[conn->cmd_len + 1];
345                 strncpy(query, sdb_strbuf_string(conn->buf), conn->cmd_len);
346                 query[sizeof(query) - 1] = '\0';
347                 sdb_log(SDB_LOG_ERR, "frontend: Failed to parse query '%s': %s",
348                                 query, sdb_strbuf_string(conn->errbuf));
349                 return -1;
350         }
352         switch (sdb_llist_len(parsetree)) {
353                 case 0:
354                         /* skipping empty command; send back an empty reply */
355                         sdb_connection_send(conn, SDB_CONNECTION_DATA, 0, NULL);
356                         break;
357                 case 1:
358                         ast = SDB_AST_NODE(sdb_llist_get(parsetree, 0));
359                         break;
361                 default:
362                         {
363                                 char query[conn->cmd_len + 1];
364                                 strncpy(query, sdb_strbuf_string(conn->buf), conn->cmd_len);
365                                 query[sizeof(query) - 1] = '\0';
366                                 sdb_log(SDB_LOG_WARNING, "frontend: Ignoring %zu command%s "
367                                                 "in multi-statement query '%s'",
368                                                 sdb_llist_len(parsetree) - 1,
369                                                 sdb_llist_len(parsetree) == 2 ? "" : "s",
370                                                 query);
371                                 ast = SDB_AST_NODE(sdb_llist_get(parsetree, 0));
372                         }
373         }
375         if (ast) {
376                 status = exec_cmd(conn, ast);
377                 sdb_object_deref(SDB_OBJ(ast));
378         }
379         sdb_llist_destroy(parsetree);
380         return status;
381 } /* sdb_conn_query */
383 int
384 sdb_conn_fetch(sdb_conn_t *conn)
386         sdb_ast_node_t *ast;
387         char hostname[conn->cmd_len + 1];
388         char name[conn->cmd_len + 1];
389         uint32_t type;
390         int status;
392         if ((! conn) || (conn->cmd != SDB_CONNECTION_FETCH))
393                 return -1;
395         if (conn->cmd_len < sizeof(uint32_t)) {
396                 sdb_log(SDB_LOG_ERR, "frontend: Invalid command length %d for "
397                                 "FETCH command", conn->cmd_len);
398                 sdb_strbuf_sprintf(conn->errbuf, "FETCH: Invalid command length %d",
399                                 conn->cmd_len);
400                 return -1;
401         }
403         /* TODO: support other types besides hosts */
404         hostname[0] = '\0';
406         sdb_proto_unmarshal_int32(SDB_STRBUF_STR(conn->buf), &type);
407         strncpy(name, sdb_strbuf_string(conn->buf) + sizeof(uint32_t),
408                         conn->cmd_len - sizeof(uint32_t));
409         name[sizeof(name) - 1] = '\0';
411         ast = sdb_ast_fetch_create((int)type,
412                         hostname[0] ? strdup(hostname) : NULL,
413                         -1, NULL,
414                         name[0] ? strdup(name) : NULL,
415                         /* full */ 1, /* filter = */ NULL);
416         status = exec_cmd(conn, ast);
417         sdb_object_deref(SDB_OBJ(ast));
418         return status;
419 } /* sdb_conn_fetch */
421 int
422 sdb_conn_list(sdb_conn_t *conn)
424         sdb_ast_node_t *ast;
425         uint32_t type = SDB_HOST;
426         int status;
428         if ((! conn) || (conn->cmd != SDB_CONNECTION_LIST))
429                 return -1;
431         if (conn->cmd_len == sizeof(uint32_t))
432                 sdb_proto_unmarshal_int32(SDB_STRBUF_STR(conn->buf), &type);
433         else if (conn->cmd_len) {
434                 sdb_log(SDB_LOG_ERR, "frontend: Invalid command length %d for "
435                                 "LIST command", conn->cmd_len);
436                 sdb_strbuf_sprintf(conn->errbuf, "LIST: Invalid command length %d",
437                                 conn->cmd_len);
438                 return -1;
439         }
441         ast = sdb_ast_list_create((int)type, /* filter = */ NULL);
442         status = exec_cmd(conn, ast);
443         sdb_object_deref(SDB_OBJ(ast));
444         return status;
445 } /* sdb_conn_list */
447 int
448 sdb_conn_lookup(sdb_conn_t *conn)
450         sdb_ast_node_t *ast, *m;
451         const char *matcher;
452         size_t matcher_len;
454         uint32_t type;
455         int status;
457         if ((! conn) || (conn->cmd != SDB_CONNECTION_LOOKUP))
458                 return -1;
460         if (conn->cmd_len < sizeof(uint32_t)) {
461                 sdb_log(SDB_LOG_ERR, "frontend: Invalid command length %d for "
462                                 "LOOKUP command", conn->cmd_len);
463                 sdb_strbuf_sprintf(conn->errbuf, "LOOKUP: Invalid command length %d",
464                                 conn->cmd_len);
465                 return -1;
466         }
467         sdb_proto_unmarshal_int32(SDB_STRBUF_STR(conn->buf), &type);
469         matcher = sdb_strbuf_string(conn->buf) + sizeof(uint32_t);
470         matcher_len = conn->cmd_len - sizeof(uint32_t);
471         m = sdb_parser_parse_conditional((int)type,
472                         matcher, (int)matcher_len, conn->errbuf);
473         if (! m) {
474                 char expr[matcher_len + 1];
475                 char err[sdb_strbuf_len(conn->errbuf) + sizeof(expr) + 64];
476                 strncpy(expr, matcher, sizeof(expr));
477                 expr[sizeof(expr) - 1] = '\0';
478                 snprintf(err, sizeof(err), "Failed to parse lookup condition '%s': %s",
479                                 expr, sdb_strbuf_string(conn->errbuf));
480                 sdb_log(SDB_LOG_ERR, "frontend: %s", err);
481                 sdb_strbuf_sprintf(conn->errbuf, "%s", err);
482                 return -1;
483         }
485         ast = sdb_ast_lookup_create((int)type, m, /* filter = */ NULL);
486         status = exec_cmd(conn, ast);
487         if (! ast)
488                 sdb_object_deref(SDB_OBJ(m));
489         sdb_object_deref(SDB_OBJ(ast));
490         return status;
491 } /* sdb_conn_lookup */
493 int
494 sdb_conn_store(sdb_conn_t *conn)
496         sdb_ast_node_t *ast = NULL;
497         const char *buf = sdb_strbuf_string(conn->buf);
498         size_t len = conn->cmd_len;
499         uint32_t type;
500         ssize_t n;
501         int status;
503         if ((! conn) || (conn->cmd != SDB_CONNECTION_STORE))
504                 return -1;
506         if ((n = sdb_proto_unmarshal_int32(buf, len, &type)) < 0) {
507                 sdb_log(SDB_LOG_ERR, "frontend: Invalid command length %zu for "
508                                 "STORE command", len);
509                 sdb_strbuf_sprintf(conn->errbuf,
510                                 "STORE: Invalid command length %zu", len);
511                 return -1;
512         }
514         switch (type) {
515                 case SDB_HOST:
516                 {
517                         sdb_proto_host_t host;
518                         if (sdb_proto_unmarshal_host(buf, len, &host) < 0) {
519                                 sdb_strbuf_sprintf(conn->errbuf,
520                                                 "STORE: Failed to unmarshal host object");
521                                 return -1;
522                         }
523                         ast = sdb_ast_store_create(SDB_HOST, /* host */ NULL,
524                                         /* parent */ 0, NULL, sstrdup(host.name), host.last_update,
525                                         /* metric store */ NULL, NULL, 0, SDB_DATA_NULL);
526                 }
527                 break;
529                 case SDB_SERVICE:
530                 {
531                         sdb_proto_service_t svc;
532                         if (sdb_proto_unmarshal_service(buf, len, &svc) < 0) {
533                                 sdb_strbuf_sprintf(conn->errbuf,
534                                                 "STORE: Failed to unmarshal service object");
535                                 return -1;
536                         }
537                         ast = sdb_ast_store_create(SDB_SERVICE, sstrdup(svc.hostname),
538                                         /* parent */ 0, NULL, sstrdup(svc.name), svc.last_update,
539                                         /* metric store */ NULL, NULL, 0, SDB_DATA_NULL);
540                 }
541                 break;
543                 case SDB_METRIC:
544                 {
545                         sdb_proto_metric_t metric;
546                         if (sdb_proto_unmarshal_metric(buf, len, &metric) < 0) {
547                                 sdb_strbuf_sprintf(conn->errbuf,
548                                                 "STORE: Failed to unmarshal metric object");
549                                 return -1;
550                         }
551                         ast = sdb_ast_store_create(SDB_METRIC, sstrdup(metric.hostname),
552                                         /* parent */ 0, NULL, sstrdup(metric.name), metric.last_update,
553                                         sstrdup(metric.store_type), sstrdup(metric.store_id),
554                                         metric.store_last_update, SDB_DATA_NULL);
555                 }
556                 break;
557         }
559         if (type & SDB_ATTRIBUTE) {
560                 sdb_proto_attribute_t attr;
561                 const char *hostname, *parent;
562                 int parent_type;
563                 if (sdb_proto_unmarshal_attribute(buf, len, &attr) < 0) {
564                         sdb_strbuf_sprintf(conn->errbuf,
565                                         "STORE: Failed to unmarshal attribute object");
566                         return -1;
567                 }
568                 if (attr.parent_type == SDB_HOST) {
569                         hostname = attr.parent;
570                         parent_type = 0;
571                         parent = NULL;
572                 }
573                 else {
574                         hostname = attr.hostname;
575                         parent_type = attr.parent_type;
576                         parent = attr.parent;
577                 }
578                 ast = sdb_ast_store_create(SDB_ATTRIBUTE, sstrdup(hostname),
579                                 parent_type, sstrdup(parent), sstrdup(attr.key),
580                                 attr.last_update, /* metric store */ NULL, NULL, 0,
581                                 attr.value);
582         }
584         if (! ast) {
585                 sdb_log(SDB_LOG_ERR, "frontend: Invalid object type %d for "
586                                 "STORE COMMAND", type);
587                 sdb_strbuf_sprintf(conn->errbuf, "STORE: Invalid object type %d", type);
588                 return -1;
589         }
591         status = sdb_parser_analyze(ast, conn->errbuf);
592         if (! status)
593                 status = exec_cmd(conn, ast);
594         sdb_object_deref(SDB_OBJ(ast));
595         return status;
596 } /* sdb_conn_store */
598 /* vim: set tw=78 sw=4 ts=4 noexpandtab : */