From: Sebastian Harl Date: Mon, 5 Sep 2016 01:18:00 +0000 (-0400) Subject: Let the network protocol and SysQL support last_update for metric stores. X-Git-Url: https://git.tokkee.org/?p=sysdb.git;a=commitdiff_plain;h=5f7e0ee1cf668eaaec097d8f2fa0753b0c089946 Let the network protocol and SysQL support last_update for metric stores. --- diff --git a/doc/sysdbql.7.txt b/doc/sysdbql.7.txt index ceb5834..587cd6c 100644 --- a/doc/sysdbql.7.txt +++ b/doc/sysdbql.7.txt @@ -263,7 +263,7 @@ storing the different data types: provided as well. If omitted, the current time on the server will be used instead. -*STORE* metric ''.'' STORE '' '' [*LAST UPDATE* '']:: +*STORE* metric ''.'' STORE '' '' [''] [*LAST UPDATE* '']:: Store a metric and provide information about the metric store associated with it. A metric store describes how to access a metric's data and can be used to retrieve time-series information associated with the metric. See diff --git a/src/frontend/query.c b/src/frontend/query.c index 5d00636..3f33a99 100644 --- a/src/frontend/query.c +++ b/src/frontend/query.c @@ -158,7 +158,7 @@ exec_store(sdb_ast_store_t *st, sdb_strbuf_t *buf, sdb_strbuf_t *errbuf) snprintf(name, sizeof(name), "%s.%s", st->hostname, st->name); metric_store.type = st->store_type; metric_store.id = st->store_id; - metric_store.last_update = st->last_update; + metric_store.last_update = st->store_last_update; status = sdb_plugin_store_metric(st->hostname, st->name, &metric_store, st->last_update); break; @@ -510,7 +510,7 @@ sdb_conn_store(sdb_conn_t *conn) } ast = sdb_ast_store_create(SDB_HOST, /* host */ NULL, /* parent */ 0, NULL, sstrdup(host.name), host.last_update, - /* metric store */ NULL, NULL, SDB_DATA_NULL); + /* metric store */ NULL, NULL, 0, SDB_DATA_NULL); } break; @@ -524,7 +524,7 @@ sdb_conn_store(sdb_conn_t *conn) } ast = sdb_ast_store_create(SDB_SERVICE, sstrdup(svc.hostname), /* parent */ 0, NULL, sstrdup(svc.name), svc.last_update, - /* metric store */ NULL, NULL, SDB_DATA_NULL); + /* metric store */ NULL, NULL, 0, SDB_DATA_NULL); } break; @@ -539,7 +539,7 @@ sdb_conn_store(sdb_conn_t *conn) ast = sdb_ast_store_create(SDB_METRIC, sstrdup(metric.hostname), /* parent */ 0, NULL, sstrdup(metric.name), metric.last_update, sstrdup(metric.store_type), sstrdup(metric.store_id), - SDB_DATA_NULL); + metric.store_last_update, SDB_DATA_NULL); } break; } @@ -565,7 +565,7 @@ sdb_conn_store(sdb_conn_t *conn) } ast = sdb_ast_store_create(SDB_ATTRIBUTE, sstrdup(hostname), parent_type, sstrdup(parent), sstrdup(attr.key), - attr.last_update, /* metric store */ NULL, NULL, + attr.last_update, /* metric store */ NULL, NULL, 0, attr.value); } diff --git a/src/include/parser/ast.h b/src/include/parser/ast.h index 01446ab..4272ec0 100644 --- a/src/include/parser/ast.h +++ b/src/include/parser/ast.h @@ -309,6 +309,7 @@ typedef struct { /* metric specific */ char *store_type; char *store_id; + sdb_time_t store_last_update; /* attribute specific */ sdb_data_t value; @@ -316,7 +317,7 @@ typedef struct { #define SDB_AST_STORE(obj) ((sdb_ast_store_t *)(obj)) #define SDB_AST_STORE_INIT \ { { SDB_OBJECT_INIT, SDB_AST_TYPE_STORE, -1 }, \ - -1, NULL, -1, NULL, NULL, 0, NULL, NULL, SDB_DATA_INIT } + -1, NULL, -1, NULL, NULL, 0, NULL, NULL, 0, SDB_DATA_INIT } /* * sdb_ast_timeseries_t represents a TIMESERIES command. @@ -418,7 +419,8 @@ sdb_ast_lookup_create(int obj_type, sdb_ast_node_t *matcher, sdb_ast_node_t * sdb_ast_store_create(int obj_type, char *hostname, int parent_type, char *parent, char *name, sdb_time_t last_update, - char *store_type, char *store_id, sdb_data_t value); + char *store_type, char *store_id, sdb_time_t store_last_update, + sdb_data_t value); /* * sdb_ast_timeseries_create: diff --git a/src/include/utils/proto.h b/src/include/utils/proto.h index 5671563..a9ea1e3 100644 --- a/src/include/utils/proto.h +++ b/src/include/utils/proto.h @@ -61,8 +61,9 @@ typedef struct { const char *name; const char *store_type; /* optional */ const char *store_id; /* optional */ + sdb_time_t store_last_update; /* optional */ } sdb_proto_metric_t; -#define SDB_PROTO_METRIC_INIT { 0, NULL, NULL, NULL, NULL } +#define SDB_PROTO_METRIC_INIT { 0, NULL, NULL, NULL, NULL, 0 } typedef struct { sdb_time_t last_update; diff --git a/src/parser/ast.c b/src/parser/ast.c index dcfe786..96c05ff 100644 --- a/src/parser/ast.c +++ b/src/parser/ast.c @@ -347,7 +347,8 @@ sdb_ast_lookup_create(int obj_type, sdb_ast_node_t *matcher, sdb_ast_node_t * sdb_ast_store_create(int obj_type, char *hostname, int parent_type, char *parent, char *name, sdb_time_t last_update, - char *store_type, char *store_id, sdb_data_t value) + char *store_type, char *store_id, sdb_time_t store_last_update, + sdb_data_t value) { sdb_ast_store_t *store; store = SDB_AST_STORE(sdb_object_create("STORE", st_type)); @@ -364,6 +365,7 @@ sdb_ast_store_create(int obj_type, char *hostname, store->last_update = last_update; store->store_type = store_type; store->store_id = store_id; + store->store_last_update = store_last_update; store->value = value; return SDB_AST_NODE(store); } /* sdb_ast_store_create */ diff --git a/src/parser/grammar.y b/src/parser/grammar.y index c350d54..6654780 100644 --- a/src/parser/grammar.y +++ b/src/parser/grammar.y @@ -104,7 +104,7 @@ sdb_parser_yyerrorf(YYLTYPE *lval, sdb_parser_yyscan_t scanner, const char *fmt, sdb_llist_t *list; sdb_ast_node_t *node; - struct { char *type; char *id; } metric_store; + struct { char *type; char *id; sdb_time_t last_update; } metric_store; } %start statements @@ -339,42 +339,42 @@ store_statement: STORE HOST_T STRING last_update_clause { $$ = sdb_ast_store_create(SDB_HOST, NULL, 0, NULL, - $3, $4, NULL, NULL, SDB_DATA_NULL); + $3, $4, NULL, NULL, 0, SDB_DATA_NULL); CK_OOM($$); } | STORE SERVICE_T STRING '.' STRING last_update_clause { $$ = sdb_ast_store_create(SDB_SERVICE, $3, 0, NULL, - $5, $6, NULL, NULL, SDB_DATA_NULL); + $5, $6, NULL, NULL, 0, SDB_DATA_NULL); CK_OOM($$); } | STORE METRIC_T STRING '.' STRING metric_store_clause last_update_clause { $$ = sdb_ast_store_create(SDB_METRIC, $3, 0, NULL, - $5, $7, $6.type, $6.id, SDB_DATA_NULL); + $5, $7, $6.type, $6.id, $6.last_update, SDB_DATA_NULL); CK_OOM($$); } | STORE HOST_T ATTRIBUTE_T STRING '.' STRING data last_update_clause { $$ = sdb_ast_store_create(SDB_ATTRIBUTE, $4, 0, NULL, - $6, $8, NULL, NULL, $7); + $6, $8, NULL, NULL, 0, $7); CK_OOM($$); } | STORE SERVICE_T ATTRIBUTE_T STRING '.' STRING '.' STRING data last_update_clause { $$ = sdb_ast_store_create(SDB_ATTRIBUTE, $4, SDB_SERVICE, $6, - $8, $10, NULL, NULL, $9); + $8, $10, NULL, NULL, 0, $9); CK_OOM($$); } | STORE METRIC_T ATTRIBUTE_T STRING '.' STRING '.' STRING data last_update_clause { $$ = sdb_ast_store_create(SDB_ATTRIBUTE, $4, SDB_METRIC, $6, - $8, $10, NULL, NULL, $9); + $8, $10, NULL, NULL, 0, $9); CK_OOM($$); } ; @@ -385,9 +385,11 @@ last_update_clause: /* empty */ { $$ = sdb_gettime(); } metric_store_clause: - STORE STRING STRING { $$.type = $2; $$.id = $3; } + STORE STRING STRING datetime { $$.type = $2; $$.id = $3; $$.last_update = $4; } | - /* empty */ { $$.type = $$.id = NULL; } + STORE STRING STRING { $$.type = $2; $$.id = $3; $$.last_update = 0; } + | + /* empty */ { $$.type = $$.id = NULL; $$.last_update = 0; } /* * TIMESERIES . [START ] [END ]; diff --git a/src/plugins/store/network.c b/src/plugins/store/network.c index 8223d38..77012b9 100644 --- a/src/plugins/store/network.c +++ b/src/plugins/store/network.c @@ -149,7 +149,7 @@ store_metric(sdb_store_metric_t *metric, sdb_object_t *user_data) { sdb_proto_metric_t m = { metric->last_update, metric->hostname, metric->name, - metric->store.type, metric->store.id, + metric->store.type, metric->store.id, metric->store.last_update, }; size_t len = sdb_proto_marshal_metric(NULL, 0, &m); char buf[len]; diff --git a/src/utils/proto.c b/src/utils/proto.c index f8265d6..6167626 100644 --- a/src/utils/proto.c +++ b/src/utils/proto.c @@ -466,8 +466,11 @@ sdb_proto_marshal_metric(char *buf, size_t buf_len, return -1; len = OBJ_HEADER_LEN + strlen(metric->hostname) + strlen(metric->name) + 2; - if (metric->store_type && metric->store_id) + if (metric->store_type && metric->store_id) { len += strlen(metric->store_type) + strlen(metric->store_id) + 2; + if (metric->store_last_update > 0) + len += sizeof(sdb_time_t); + } if (buf_len < len) return len; @@ -480,7 +483,12 @@ sdb_proto_marshal_metric(char *buf, size_t buf_len, if (metric->store_type && metric->store_id) { n = marshal_string(buf, buf_len, metric->store_type); buf += n; buf_len -= n; - marshal_string(buf, buf_len, metric->store_id); + n = marshal_string(buf, buf_len, metric->store_id); + buf += n; buf_len -= n; + if (metric->store_last_update > 0) { + n = marshal_datetime(buf, buf_len, metric->store_last_update); + buf += n; buf_len -= n; + } } return len; } /* sdb_proto_marshal_metric */ @@ -771,6 +779,8 @@ sdb_proto_unmarshal_metric(const char *buf, size_t len, buf += n; len -= n; l += n; if ((n = unmarshal_string(buf, len, metric ? &metric->name : NULL)) < 0) return n; + + metric->store_last_update = 0; if (len > (size_t)n) { buf += n; len -= n; l += n; if ((n = unmarshal_string(buf, len, @@ -780,6 +790,12 @@ sdb_proto_unmarshal_metric(const char *buf, size_t len, if ((n = unmarshal_string(buf, len, metric ? &metric->store_id : NULL)) < 0) return n; + if (len >= sizeof(sdb_time_t)) { + buf += n; len -= n; l += n; + if ((n = unmarshal_datetime(buf, len, + metric ? &metric->store_last_update : NULL)) < 0) + return n; + } } else if (metric) { metric->store_type = NULL; diff --git a/t/unit/core/store_test.c b/t/unit/core/store_test.c index 96ac0c1..d6a44e0 100644 --- a/t/unit/core/store_test.c +++ b/t/unit/core/store_test.c @@ -280,16 +280,6 @@ START_TEST(test_store_metric) golden_data[i].host, golden_data[i].metric, golden_data[i].store, golden_data[i].last_update, status, golden_data[i].expected); - - if (status < 0) - continue; - - if (golden_data[i].store != NULL) - fail_unless(golden_data[i].store->last_update > 0, - "sdb_memstore_metric(%s, %s, %p, %d, 0) did not update " - "store->last_update", - golden_data[i].host, golden_data[i].metric, - golden_data[i].store, golden_data[i].last_update); } } END_TEST diff --git a/t/unit/parser/parser_test.c b/t/unit/parser/parser_test.c index f6601f1..14977ff 100644 --- a/t/unit/parser/parser_test.c +++ b/t/unit/parser/parser_test.c @@ -220,6 +220,11 @@ struct { "STORE 'typ' 'id' " "LAST UPDATE " "2015-02-01", -1, 1, SDB_AST_TYPE_STORE, SDB_METRIC }, + { "STORE metric " + "'host'.'metric' " + "STORE 'typ' 'id' 2015-02-01 13:37 " + "LAST UPDATE " + "2015-02-01", -1, 1, SDB_AST_TYPE_STORE, SDB_METRIC }, { "STORE metric attribute " "'host'.'metric'.'key' " "123", -1, 1, SDB_AST_TYPE_STORE, SDB_ATTRIBUTE }, diff --git a/t/unit/utils/proto_test.c b/t/unit/utils/proto_test.c index 7e2e9b9..bc5f910 100644 --- a/t/unit/utils/proto_test.c +++ b/t/unit/utils/proto_test.c @@ -376,28 +376,36 @@ START_TEST(test_marshal_metric) char *expected; } golden_data[] = { { - { 4711, "hostA", "metricX", NULL, NULL }, + { 4711, "hostA", "metricX", NULL, NULL, 0 }, 26, METRIC_TYPE "\0\0\0\0\0\0\x12\x67" "hostA\0metricX\0" }, { - { 0, "hostA", "metricX", NULL, NULL }, + { 0, "hostA", "metricX", NULL, NULL, 0 }, 26, METRIC_TYPE "\0\0\0\0\0\0\0\0" "hostA\0metricX\0" }, { - { 0, "hostA", "metricX", "type", NULL }, + { 0, "hostA", "metricX", "type", NULL, 0 }, 26, METRIC_TYPE "\0\0\0\0\0\0\0\0" "hostA\0metricX\0" }, { - { 0, "hostA", "metricX", NULL, "id" }, + { 0, "hostA", "metricX", NULL, "id", 0 }, 26, METRIC_TYPE "\0\0\0\0\0\0\0\0" "hostA\0metricX\0" }, { - { 4711, "hostA", "metricX", "type", "id" }, + { 0, "hostA", "metricX", NULL, NULL, 4711 }, + 26, METRIC_TYPE "\0\0\0\0\0\0\0\0" "hostA\0metricX\0" + }, + { + { 4711, "hostA", "metricX", "type", "id", 0 }, 34, METRIC_TYPE "\0\0\0\0\0\0\x12\x67" "hostA\0metricX\0type\0id\0" }, - { { 4711, "hostA", NULL, NULL, NULL }, -1, NULL }, - { { 4711, NULL, "metricX", NULL, NULL }, -1, NULL }, - { { 4711, NULL, NULL, NULL, NULL }, -1, NULL }, + { + { 4711, "hostA", "metricX", "type", "id", 4711 }, + 42, METRIC_TYPE "\0\0\0\0\0\0\x12\x67" "hostA\0metricX\0type\0id\0" "\0\0\0\0\0\0\x12\x67" + }, + { { 4711, "hostA", NULL, NULL, NULL, 0 }, -1, NULL }, + { { 4711, NULL, "metricX", NULL, NULL, 0 }, -1, NULL }, + { { 4711, NULL, NULL, NULL, NULL, 0 }, -1, NULL }, }; size_t i; @@ -434,9 +442,10 @@ START_TEST(test_marshal_metric) if ((! golden_data[i].metric.store_type) || (! golden_data[i].metric.store_id)) { - /* if any of these is NULL, we expect both to be NULL */ + /* if any of these is NULL, we expect all to be NULL */ golden_data[i].metric.store_type = NULL; golden_data[i].metric.store_id = NULL; + golden_data[i].metric.store_last_update = 0; } check = sdb_proto_unmarshal_metric(buf, len, &metric); @@ -447,15 +456,18 @@ START_TEST(test_marshal_metric) && streq(metric.hostname, golden_data[i].metric.hostname) && streq(metric.name, golden_data[i].metric.name) && streq(metric.store_type, golden_data[i].metric.store_type) - && streq(metric.store_id, golden_data[i].metric.store_id), + && streq(metric.store_id, golden_data[i].metric.store_id) + && metric.store_last_update == golden_data[i].metric.store_last_update, "<%zu> sdb_proto_unmarshal_metric(buf<%s>) = " - "{ %"PRIsdbTIME", %s, %s, %s, %s }; expected: " - "{ %"PRIsdbTIME", %s, %s, %s, %s }", i, golden_data[i].metric.name, + "{ %"PRIsdbTIME", %s, %s, %s, %s, %"PRIsdbTIME" }; expected: " + "{ %"PRIsdbTIME", %s, %s, %s, %s, %"PRIsdbTIME" }", + i, golden_data[i].metric.name, metric.last_update, metric.hostname, metric.name, - metric.store_type, metric.store_id, + metric.store_type, metric.store_id, metric.store_last_update, golden_data[i].metric.last_update, golden_data[i].metric.hostname, golden_data[i].metric.name, - golden_data[i].metric.store_type, golden_data[i].metric.store_id); + golden_data[i].metric.store_type, golden_data[i].metric.store_id, + golden_data[i].metric.store_last_update); } } END_TEST