X-Git-Url: https://arthur.barton.de/gitweb/?a=blobdiff_plain;f=src%2Frrd.c;h=2dce02e669741df8de4d1a31d4a923fe13a15b0d;hb=d7d1f5f7addff7e57954936f81b60c843d4958af;hp=ac96ea6d8125fe8cc067508a061cfd90b557dfcc;hpb=52aad6d81f0a0a8e14ac3de60a59cafc40d86faa;p=netdata.git diff --git a/src/rrd.c b/src/rrd.c old mode 100755 new mode 100644 index ac96ea6d..2dce02e6 --- a/src/rrd.c +++ b/src/rrd.c @@ -53,7 +53,11 @@ static int rrdset_compare(void* a, void* b) { avl_tree rrdset_root_index = { NULL, rrdset_compare, +#ifdef AVL_LOCK_WITH_MUTEX + PTHREAD_MUTEX_INITIALIZER +#else PTHREAD_RWLOCK_INITIALIZER +#endif }; #define rrdset_index_add(st) avl_insert(&rrdset_root_index, (avl *)(st)) @@ -90,7 +94,11 @@ static int rrdset_compare_name(void* a, void* b) { avl_tree rrdset_root_index_name = { NULL, rrdset_compare_name, +#ifdef AVL_LOCK_WITH_MUTEX + PTHREAD_MUTEX_INITIALIZER +#else PTHREAD_RWLOCK_INITIALIZER +#endif }; int rrdset_index_add_name(RRDSET *st) { @@ -327,8 +335,13 @@ void rrdset_reset(RRDSET *st) } } -RRDSET *rrdset_create(const char *type, const char *id, const char *name, const char *family, const char *title, const char *units, long priority, int update_every, int chart_type) +RRDSET *rrdset_create(const char *type, const char *id, const char *name, const char *family, const char *context, const char *title, const char *units, long priority, int update_every, int chart_type) { + if(!type || !type[0]) { + fatal("Cannot create rrd stats without a type."); + return NULL; + } + if(!id || !id[0]) { fatal("Cannot create rrd stats without an id."); return NULL; @@ -340,6 +353,12 @@ RRDSET *rrdset_create(const char *type, const char *id, const char *name, const snprintf(fullid, RRD_ID_LENGTH_MAX, "%s.%s", type, id); + st = rrdset_find(fullid); + if(st) { + error("Cannot create rrd stats for '%s', it already exists.", fullid); + return st; + } + long entries = config_get_number(fullid, "history", rrd_default_history_entries); if(entries < 5) entries = config_set_number(fullid, "history", 5); if(entries > RRD_HISTORY_ENTRIES_MAX) entries = config_set_number(fullid, "history", RRD_HISTORY_ENTRIES_MAX); @@ -388,6 +407,7 @@ RRDSET *rrdset_create(const char *type, const char *id, const char *name, const st->name = NULL; st->type = NULL; st->family = NULL; + st->context = NULL; st->title = NULL; st->units = NULL; st->dimensions = NULL; @@ -414,10 +434,11 @@ RRDSET *rrdset_create(const char *type, const char *id, const char *name, const st->cache_dir = cache_dir; - st->family = config_get(st->id, "family", family?family:st->id); - st->units = config_get(st->id, "units", units?units:""); - st->type = config_get(st->id, "type", type); st->chart_type = rrdset_type_id(config_get(st->id, "chart type", rrdset_type_name(chart_type))); + st->type = config_get(st->id, "type", type); + st->family = config_get(st->id, "family", family?family:st->type); + st->context = config_get(st->id, "context", context?context:st->id); + st->units = config_get(st->id, "units", units?units:""); st->priority = config_get_number(st->id, "priority", priority); st->enabled = enabled; @@ -429,7 +450,8 @@ RRDSET *rrdset_create(const char *type, const char *id, const char *name, const st->last_collected_time.tv_usec = 0; st->counter_done = 0; - st->gap_when_lost_iterations_above = config_get_number(st->id, "gap when lost iterations above", RRD_DEFAULT_GAP_INTERPOLATIONS) + 2; + st->gap_when_lost_iterations_above = (int) ( + config_get_number(st->id, "gap when lost iterations above", RRD_DEFAULT_GAP_INTERPOLATIONS) + 2); avl_init(&st->dimensions_index, rrddim_compare); @@ -562,6 +584,7 @@ RRDDIM *rrddim_add(RRDSET *st, const char *id, const char *name, long multiplier rd->counter = 0; // append this dimension + pthread_rwlock_wrlock(&st->rwlock); if(!st->dimensions) st->dimensions = rd; else { @@ -569,6 +592,7 @@ RRDDIM *rrddim_add(RRDSET *st, const char *id, const char *name, long multiplier for(; td->next; td = td->next) ; td->next = rd; } + pthread_rwlock_unlock(&st->rwlock); rrddim_index_add(st, rd); @@ -581,14 +605,14 @@ void rrddim_set_name(RRDSET *st, RRDDIM *rd, const char *name) char varname[CONFIG_MAX_NAME + 1]; snprintf(varname, CONFIG_MAX_NAME, "dim %s name", rd->id); - config_get(st->id, varname, name); + config_set_default(st->id, varname, name); } void rrddim_free(RRDSET *st, RRDDIM *rd) { debug(D_RRD_CALLS, "rrddim_free() %s.%s", st->name, rd->name); - RRDDIM *i = st->dimensions, *last = NULL; + RRDDIM *i, *last = NULL; for(i = st->dimensions; i && i != rd ; i = i->next) last = i; if(!i) { @@ -596,8 +620,9 @@ void rrddim_free(RRDSET *st, RRDDIM *rd) return; } - if(last) last = i->next; - else st->dimensions = i->next; + if(last) last->next = rd->next; + else st->dimensions = rd->next; + rd->next = NULL; rrddim_index_del(st, rd); @@ -686,10 +711,7 @@ RRDSET *rrdset_find(const char *id) { debug(D_RRD_CALLS, "rrdset_find() for chart %s", id); - pthread_rwlock_rdlock(&rrdset_root_rwlock); RRDSET *st = rrdset_index_find(id, 0); - pthread_rwlock_unlock(&rrdset_root_rwlock); - return(st); } @@ -702,8 +724,8 @@ RRDSET *rrdset_find_bytype(const char *type, const char *id) strncpy(buf, type, RRD_ID_LENGTH_MAX - 1); buf[RRD_ID_LENGTH_MAX - 1] = '\0'; strcat(buf, "."); - int len = strlen(buf); - strncpy(&buf[len], id, RRD_ID_LENGTH_MAX - len); + int len = (int) strlen(buf); + strncpy(&buf[len], id, (size_t) (RRD_ID_LENGTH_MAX - len)); buf[RRD_ID_LENGTH_MAX] = '\0'; return(rrdset_find(buf)); @@ -713,10 +735,7 @@ RRDSET *rrdset_find_byname(const char *name) { debug(D_RRD_CALLS, "rrdset_find_byname() for chart %s", name); - pthread_rwlock_rdlock(&rrdset_root_rwlock); RRDSET *st = rrdset_index_find_name(name, 0); - pthread_rwlock_unlock(&rrdset_root_rwlock); - return(st); } @@ -798,6 +817,8 @@ void rrdset_next(RRDSET *st) gettimeofday(&now, NULL); microseconds = usecdiff(&now, &st->last_collected_time); } + // prevent infinite loop + else microseconds = st->update_every * 1000000ULL; rrdset_next_usec(st, microseconds); } @@ -812,7 +833,8 @@ unsigned long long rrdset_done(RRDSET *st) debug(D_RRD_CALLS, "rrdset_done() for chart %s", st->name); RRDDIM *rd, *last; - int oldstate, store_this_entry = 1; + int oldstate, store_this_entry = 1, first_entry = 0; + unsigned long long last_ut, now_ut, next_ut, stored_entries = 0; if(unlikely(pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &oldstate) != 0)) error("Cannot set pthread cancel state to DISABLE."); @@ -823,11 +845,12 @@ unsigned long long rrdset_done(RRDSET *st) // enable the chart, if it was disabled st->enabled = 1; - // check if the chart has a long time to be refreshed + // check if the chart has a long time to be updated if(unlikely(st->usec_since_last_update > st->entries * st->update_every * 1000000ULL)) { info("%s: took too long to be updated (%0.3Lf secs). Reseting it.", st->name, (long double)(st->usec_since_last_update / 1000000.0)); rrdset_reset(st); st->usec_since_last_update = st->update_every * 1000000ULL; + first_entry = 1; } if(unlikely(st->debug)) debug(D_RRD_STATS, "%s: microseconds since last update: %llu", st->name, st->usec_since_last_update); @@ -839,15 +862,16 @@ unsigned long long rrdset_done(RRDSET *st) // the first entry should not be stored store_this_entry = 0; + first_entry = 1; - if(unlikely(st->debug)) debug(D_RRD_STATS, "%s: initializing last_collected to now. Will not store the next entry.", st->name); + if(unlikely(st->debug)) debug(D_RRD_STATS, "%s: has not set last_collected_time. Setting it now. Will not store the next entry.", st->name); } else { // it is not the first entry // calculate the proper last_collected_time, using usec_since_last_update unsigned long long ut = st->last_collected_time.tv_sec * 1000000ULL + st->last_collected_time.tv_usec + st->usec_since_last_update; - st->last_collected_time.tv_sec = ut / 1000000ULL; - st->last_collected_time.tv_usec = ut % 1000000ULL; + st->last_collected_time.tv_sec = (time_t) (ut / 1000000ULL); + st->last_collected_time.tv_usec = (useconds_t) (ut % 1000000ULL); } // if this set has not been updated in the past @@ -856,11 +880,12 @@ unsigned long long rrdset_done(RRDSET *st) // it has never been updated before // set a fake last_updated, in the past using usec_since_last_update unsigned long long ut = st->last_collected_time.tv_sec * 1000000ULL + st->last_collected_time.tv_usec - st->usec_since_last_update; - st->last_updated.tv_sec = ut / 1000000ULL; - st->last_updated.tv_usec = ut % 1000000ULL; + st->last_updated.tv_sec = (time_t) (ut / 1000000ULL); + st->last_updated.tv_usec = (useconds_t) (ut % 1000000ULL); // the first entry should not be stored store_this_entry = 0; + first_entry = 1; if(unlikely(st->debug)) debug(D_RRD_STATS, "%s: initializing last_updated to now - %llu microseconds (%0.3Lf). Will not store the next entry.", st->name, st->usec_since_last_update, (long double)ut/1000000.0); } @@ -875,20 +900,25 @@ unsigned long long rrdset_done(RRDSET *st) gettimeofday(&st->last_collected_time, NULL); unsigned long long ut = st->last_collected_time.tv_sec * 1000000ULL + st->last_collected_time.tv_usec - st->usec_since_last_update; - st->last_updated.tv_sec = ut / 1000000ULL; - st->last_updated.tv_usec = ut % 1000000ULL; + st->last_updated.tv_sec = (time_t) (ut / 1000000ULL); + st->last_updated.tv_usec = (useconds_t) (ut % 1000000ULL); // the first entry should not be stored store_this_entry = 0; + first_entry = 1; } // these are the 3 variables that will help us in interpolation // last_ut = the last time we added a value to the storage // now_ut = the time the current value is taken at // next_ut = the time of the next interpolation point - unsigned long long last_ut = st->last_updated.tv_sec * 1000000ULL + st->last_updated.tv_usec; - unsigned long long now_ut = st->last_collected_time.tv_sec * 1000000ULL + st->last_collected_time.tv_usec; - unsigned long long next_ut = (st->last_updated.tv_sec + st->update_every) * 1000000ULL; + last_ut = st->last_updated.tv_sec * 1000000ULL + st->last_updated.tv_usec; + now_ut = st->last_collected_time.tv_sec * 1000000ULL + st->last_collected_time.tv_usec; + next_ut = (st->last_updated.tv_sec + st->update_every) * 1000000ULL; + + if(unlikely(!first_entry && now_ut < next_ut)) { + if(unlikely(st->debug)) debug(D_RRD_STATS, "%s: THIS IS IN THE SAME INTERPOLATION POINT", st->name); + } if(unlikely(st->debug)) { debug(D_RRD_STATS, "%s: last ut = %0.3Lf (last updated time)", st->name, (long double)last_ut/1000000.0); @@ -915,12 +945,7 @@ unsigned long long rrdset_done(RRDSET *st) // at this stage we do not interpolate anything for( rd = st->dimensions ; likely(rd) ; rd = rd->next ) { - if(unlikely(!rd->updated || rd->counter <= 1)) { - rd->calculated_value = 0; - continue; - } - - if(unlikely(st->debug)) debug(D_RRD_STATS, "%s/%s: " + if(unlikely(st->debug)) debug(D_RRD_STATS, "%s/%s: START " " last_collected_value = " COLLECTED_NUMBER_FORMAT " collected_value = " COLLECTED_NUMBER_FORMAT " last_calculated_value = " CALCULATED_NUMBER_FORMAT @@ -933,26 +958,24 @@ unsigned long long rrdset_done(RRDSET *st) ); switch(rd->algorithm) { - case RRDDIM_PCENT_OVER_DIFF_TOTAL: - // the percentage of the current increment - // over the increment of all dimensions together - if(unlikely(st->collected_total == st->last_collected_total)) rd->calculated_value = rd->last_calculated_value; - else rd->calculated_value = - (calculated_number)100 - * (calculated_number)(rd->collected_value - rd->last_collected_value) - / (calculated_number)(st->collected_total - st->last_collected_total); - - if(unlikely(st->debug)) - debug(D_RRD_STATS, "%s/%s: CALC PCENT-DIFF " - CALCULATED_NUMBER_FORMAT " = 100" - " * (" COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT ")" - " / (" COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT ")" - , st->id, rd->name - , rd->calculated_value - , rd->collected_value, rd->last_collected_value - , st->collected_total, st->last_collected_total - ); - break; + case RRDDIM_ABSOLUTE: + rd->calculated_value = (calculated_number)rd->collected_value + * (calculated_number)rd->multiplier + / (calculated_number)rd->divisor; + + if(unlikely(st->debug)) + debug(D_RRD_STATS, "%s/%s: CALC ABS/ABS-NO-IN " + CALCULATED_NUMBER_FORMAT " = " + COLLECTED_NUMBER_FORMAT + " * " CALCULATED_NUMBER_FORMAT + " / " CALCULATED_NUMBER_FORMAT + , st->id, rd->name + , rd->calculated_value + , rd->collected_value + , (calculated_number)rd->multiplier + , (calculated_number)rd->divisor + ); + break; case RRDDIM_PCENT_OVER_ROW_TOTAL: if(unlikely(!st->collected_total)) rd->calculated_value = 0; @@ -977,6 +1000,11 @@ unsigned long long rrdset_done(RRDSET *st) break; case RRDDIM_INCREMENTAL: + if(unlikely(!rd->updated || rd->counter <= 1)) { + rd->calculated_value = 0; + continue; + } + // if the new is smaller than the old (an overflow, or reset), set the old equal to the new // to reset the calculation (it will give zero as the calculation for this second) if(unlikely(rd->last_collected_value > rd->collected_value)) { @@ -988,17 +1016,17 @@ unsigned long long rrdset_done(RRDSET *st) rd->last_collected_value = rd->collected_value; } - rd->calculated_value += (calculated_number)(rd->collected_value - rd->last_collected_value) + rd->calculated_value = (calculated_number)(rd->collected_value - rd->last_collected_value) * (calculated_number)rd->multiplier / (calculated_number)rd->divisor; if(unlikely(st->debug)) - debug(D_RRD_STATS, "%s/%s: CALC INC " - CALCULATED_NUMBER_FORMAT " += (" + debug(D_RRD_STATS, "%s/%s: CALC INC PRE " + CALCULATED_NUMBER_FORMAT " = (" COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT - " * %ld" - " / %ld" ")" + " * " CALCULATED_NUMBER_FORMAT + " / " CALCULATED_NUMBER_FORMAT , st->id, rd->name , rd->calculated_value , rd->collected_value, rd->last_collected_value @@ -1007,22 +1035,29 @@ unsigned long long rrdset_done(RRDSET *st) ); break; - case RRDDIM_ABSOLUTE: - rd->calculated_value = (calculated_number)rd->collected_value - * (calculated_number)rd->multiplier - / (calculated_number)rd->divisor; + case RRDDIM_PCENT_OVER_DIFF_TOTAL: + if(unlikely(!rd->updated || rd->counter <= 1)) { + rd->calculated_value = 0; + continue; + } + + // the percentage of the current increment + // over the increment of all dimensions together + if(unlikely(st->collected_total == st->last_collected_total)) rd->calculated_value = rd->last_calculated_value; + else rd->calculated_value = + (calculated_number)100 + * (calculated_number)(rd->collected_value - rd->last_collected_value) + / (calculated_number)(st->collected_total - st->last_collected_total); if(unlikely(st->debug)) - debug(D_RRD_STATS, "%s/%s: CALC ABS/ABS-NO-IN " - CALCULATED_NUMBER_FORMAT " = " - COLLECTED_NUMBER_FORMAT - " * %ld" - " / %ld" + debug(D_RRD_STATS, "%s/%s: CALC PCENT-DIFF " + CALCULATED_NUMBER_FORMAT " = 100" + " * (" COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT ")" + " / (" COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT ")" , st->id, rd->name , rd->calculated_value - , rd->collected_value - , (calculated_number)rd->multiplier - , (calculated_number)rd->divisor + , rd->collected_value, rd->last_collected_value + , st->collected_total, st->last_collected_total ); break; @@ -1039,23 +1074,39 @@ unsigned long long rrdset_done(RRDSET *st) ); break; } + + if(unlikely(st->debug)) debug(D_RRD_STATS, "%s/%s: PHASE2 " + " last_collected_value = " COLLECTED_NUMBER_FORMAT + " collected_value = " COLLECTED_NUMBER_FORMAT + " last_calculated_value = " CALCULATED_NUMBER_FORMAT + " calculated_value = " CALCULATED_NUMBER_FORMAT + , st->id, rd->name + , rd->last_collected_value + , rd->collected_value + , rd->last_calculated_value + , rd->calculated_value + ); + } // at this point we have all the calculated values ready // it is now time to interpolate values on a second boundary unsigned long long first_ut = last_ut; - int iterations = (now_ut - last_ut) / (st->update_every * 1000000ULL); + long long iterations = (now_ut - last_ut) / (st->update_every * 1000000ULL); + if((now_ut % (st->update_every * 1000000ULL)) == 0) iterations++; for( ; likely(next_ut <= now_ut) ; next_ut += st->update_every * 1000000ULL, iterations-- ) { - if(iterations <= 0) error("iterations calculation wrapped!"); +#ifdef NETDATA_INTERNAL_CHECKS + if(iterations < 0) { error("%s: iterations calculation wrapped! first_ut = %llu, last_ut = %llu, next_ut = %llu, now_ut = %llu", st->name, first_ut, last_ut, next_ut, now_ut); } +#endif if(unlikely(st->debug)) { debug(D_RRD_STATS, "%s: last ut = %0.3Lf (last updated time)", st->name, (long double)last_ut/1000000.0); debug(D_RRD_STATS, "%s: next ut = %0.3Lf (next interpolation point)", st->name, (long double)next_ut/1000000.0); } - st->last_updated.tv_sec = next_ut / 1000000ULL; + st->last_updated.tv_sec = (time_t) (next_ut / 1000000ULL); st->last_updated.tv_usec = 0; for( rd = st->dimensions ; likely(rd) ; rd = rd->next ) { @@ -1078,11 +1129,13 @@ unsigned long long rrdset_done(RRDSET *st) , st->id, rd->name , new_value , rd->calculated_value - , (unsigned long long)(next_ut - last_ut) - , (unsigned long long)(now_ut - last_ut) + , (next_ut - last_ut) + , (now_ut - last_ut) ); rd->calculated_value -= new_value; + new_value += rd->last_calculated_value; + rd->last_calculated_value = 0; new_value /= (calculated_number)st->update_every; break; @@ -1154,6 +1207,8 @@ unsigned long long rrdset_done(RRDSET *st) rd->values[st->current_entry] = pack_storage_number(0, SN_NOT_EXISTS); } + stored_entries++; + if(unlikely(st->debug)) { calculated_number t1 = new_value * (calculated_number)rd->multiplier / (calculated_number)rd->divisor; calculated_number t2 = unpack_storage_number(rd->values[st->current_entry]); @@ -1190,23 +1245,38 @@ unsigned long long rrdset_done(RRDSET *st) last_ut = next_ut; } + // align next interpolation to last collection point + if(likely(stored_entries || !store_this_entry)) { + st->last_updated.tv_sec = st->last_collected_time.tv_sec; + st->last_updated.tv_usec = st->last_collected_time.tv_usec; + } + for( rd = st->dimensions; likely(rd) ; rd = rd->next ) { if(unlikely(!rd->updated)) continue; - rd->last_collected_value = rd->collected_value; - rd->last_calculated_value = rd->calculated_value; + + if(likely(stored_entries || !store_this_entry)) { + if(unlikely(st->debug)) debug(D_RRD_STATS, "%s/%s: setting last_collected_value (old: " COLLECTED_NUMBER_FORMAT ") to last_collected_value (new: " COLLECTED_NUMBER_FORMAT ")", st->id, rd->name, rd->last_collected_value, rd->collected_value); + rd->last_collected_value = rd->collected_value; + + if(unlikely(st->debug)) debug(D_RRD_STATS, "%s/%s: setting last_calculated_value (old: " CALCULATED_NUMBER_FORMAT ") to last_calculated_value (new: " CALCULATED_NUMBER_FORMAT ")", st->id, rd->name, rd->last_calculated_value, rd->calculated_value); + rd->last_calculated_value = rd->calculated_value; + } + + rd->calculated_value = 0; rd->collected_value = 0; rd->updated = 0; - // if this is the first entry of incremental dimensions - // we have to set the first calculated_value to zero - // to eliminate the first spike - if(unlikely(st->counter_done == 1)) switch(rd->algorithm) { - case RRDDIM_PCENT_OVER_DIFF_TOTAL: - case RRDDIM_INCREMENTAL: - rd->calculated_value = 0; - // the next time, a new incremental total will be calculated - break; - } + if(unlikely(st->debug)) debug(D_RRD_STATS, "%s/%s: END " + " last_collected_value = " COLLECTED_NUMBER_FORMAT + " collected_value = " COLLECTED_NUMBER_FORMAT + " last_calculated_value = " CALCULATED_NUMBER_FORMAT + " calculated_value = " CALCULATED_NUMBER_FORMAT + , st->id, rd->name + , rd->last_collected_value + , rd->collected_value + , rd->last_calculated_value + , rd->calculated_value + ); } st->last_collected_total = st->collected_total;