X-Git-Url: https://arthur.barton.de/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=plugins.d%2Fcharts.d.plugin;h=00206f95f96fe9844585b51c8d72b89d4e78396a;hb=HEAD;hp=eefa31a508a2bf54b6c77bc3b5a3f4b389a7bad4;hpb=6b21dd924c3f9fa908551e92db26dcd781894281;p=netdata.git diff --git a/plugins.d/charts.d.plugin b/plugins.d/charts.d.plugin index eefa31a5..00206f95 100755 --- a/plugins.d/charts.d.plugin +++ b/plugins.d/charts.d.plugin @@ -1,18 +1,130 @@ -#!/bin/sh +#!/usr/bin/env bash + +# netdata +# real-time performance and health monitoring, done right! +# (C) 2016 Costa Tsaousis +# GPL v3+ +# +# charts.d.plugin allows easy development of BASH plugins +# +# if you need to run parallel charts.d processes, link this file to a different name +# in the same directory, with a .plugin suffix and netdata will start both of them, +# each will have a different config file and modules configuration directory. +# + +export PATH="${PATH}:/sbin:/usr/sbin:/usr/local/bin:/usr/local/sbin" + +PROGRAM_FILE="$0" +PROGRAM_NAME="$(basename $0)" +PROGRAM_NAME="${PROGRAM_NAME/.plugin}" +MODULE_NAME="main" # ----------------------------------------------------------------------------- -# insternal defaults +# create temp dir -pause_method="sleep" # use either "suspend" or "sleep" - # DO NOT USE SUSPEND - LINUX WILL SUSPEND NETDATA TOO - # THE WHOLE PROCESS GROUP - NOT JUST THE SHELL +debug=0 +TMP_DIR= +chartsd_cleanup() { + if [ ! -z "$TMP_DIR" -a -d "$TMP_DIR" ] + then + [ $debug -eq 1 ] && echo >&2 "$PROGRAM_NAME: cleaning up temporary directory $TMP_DIR ..." + rm -rf "$TMP_DIR" + fi + exit 0 +} +trap chartsd_cleanup EXIT +trap chartsd_cleanup SIGHUP +trap chartsd_cleanup INT + +if [ $UID = "0" ] +then + TMP_DIR="$( mktemp -d /var/run/netdata-${PROGRAM_NAME}-XXXXXXXXXX )" +else + TMP_DIR="$( mktemp -d /tmp/.netdata-${PROGRAM_NAME}-XXXXXXXXXX )" +fi + +logdate() { + date "+%Y-%m-%d %H:%M:%S" +} + +log() { + local status="${1}" + shift + + echo >&2 "$(logdate): ${PROGRAM_NAME}: ${status}: ${MODULE_NAME}: ${*}" + +} + +warning() { + log WARNING "${@}" +} + +error() { + log ERROR "${@}" +} + +info() { + log INFO "${@}" +} + +fatal() { + log FATAL "${@}" + echo "DISABLE" + exit 1 +} + +debug() { + [ $debug -eq 1 ] && log DEBUG "${@}" +} + +# ----------------------------------------------------------------------------- +# check a few commands + +require_cmd() { + local x=$(which "${1}" 2>/dev/null || command -v "${1}" 2>/dev/null) + if [ -z "${x}" -o ! -x "${x}" ] + then + warning "command '${1}' is not found in ${PATH}." + eval "${1^^}_CMD=\"\"" + return 1 + fi + + eval "${1^^}_CMD=\"${x}\"" + return 0 +} + +require_cmd date || exit 1 +require_cmd sed || exit 1 +require_cmd basename || exit 1 +require_cmd dirname || exit 1 +require_cmd cat || exit 1 +require_cmd grep || exit 1 +require_cmd egrep || exit 1 +require_cmd mktemp || exit 1 +require_cmd awk || exit 1 +require_cmd timeout || exit 1 +require_cmd curl || exit 1 + +# ----------------------------------------------------------------------------- + +[ $(( ${BASH_VERSINFO[0]} )) -lt 4 ] && fatal "BASH version 4 or later is required, but found version: ${BASH_VERSION}. Please upgrade." + +info "started from '$PROGRAM_FILE' with options: $*" + +# ----------------------------------------------------------------------------- +# internal defaults +# netdata exposes a few environment variables for us -pluginsd="plugins.d" -confd="conf.d" -chartsd="charts.d" -myconfig="$confd/charts.d.conf" -minimum_update_frequency=1 -update_every=1 # this is overwritten by the command line +pluginsd="${NETDATA_PLUGINS_DIR}" +[ -z "$pluginsd" ] && pluginsd="$( dirname $PROGRAM_FILE )" + +confd="${NETDATA_CONFIG_DIR-/etc/netdata}" +chartsd="$pluginsd/../charts.d" + +myconfig="$confd/$PROGRAM_NAME.conf" + +minimum_update_frequency="${NETDATA_UPDATE_EVERY-1}" +update_every=${minimum_update_frequency} # this will be overwritten by the command line # work around for non BASH shells charts_create="_create" @@ -20,210 +132,390 @@ charts_update="_update" charts_check="_check" charts_undescore="_" +# when making iterations, charts.d can loop more frequently +# to prevent plugins missing iterations. +# this is a percentage relative to update_every to align its +# iterations. +# The minimum is 10%, the maximum 100%. +# So, if update_every is 1 second and time_divisor is 50, +# charts.d will iterate every 500ms. +# Charts will be called to collect data only if the time +# passed since the last time the collected data is equal or +# above their update_every. +time_divisor=50 + +# number of seconds to run without restart +# after this time, charts.d.plugin will exit +# netdata will restart it +restart_timeout=$((3600 * 4)) + +# check if the charts.d plugins are using global variables +# they should not. +# It does not currently support BASH v4 arrays, so it is +# disabled +dryrunner=0 + +# check for timeout command +check_for_timeout=1 + +# the default enable/disable value for all charts +enable_all_charts="yes" + # ----------------------------------------------------------------------------- # parse parameters -debug=0 check=0 chart_only= while [ ! -z "$1" ] do - if [ "$1" = "check" ] - then - check=1 - shift - continue - fi - - if [ "$1" = "debug" -o "$1" = "all" ] - then - debug=1 - shift - continue - fi - - if [ -f "$chartsd/$1.chart.sh" ] - then - debug=1 - chart_only="`echo $1.chart.sh | sed "s/\.chart\.sh$//g"`" - shift - continue - fi - - if [ -f "$chartsd/$1" ] - then - debug=1 - chart_only="`echo $1 | sed "s/\.chart\.sh$//g"`" - shift - continue - fi - - if [ -f "$1" ] - then - debug=1 - chart_only="`basename "$1" | sed "s/\.chart\.sh$//g"`" - shift - continue - fi - - # number check - n="$1" - x=$((n + 1 - 1)) - if [ "$x" = "$n" ] - then - update_every=$x - shift - continue - fi - - echo >&2 "Cannot understand parameter $1. Aborting." - echo "DISABLE" - exit 1 + if [ "$1" = "check" ] + then + check=1 + shift + continue + fi + + if [ "$1" = "debug" -o "$1" = "all" ] + then + debug=1 + shift + continue + fi + + if [ -f "$chartsd/$1.chart.sh" ] + then + debug=1 + chart_only="$( echo $1.chart.sh | sed "s/\.chart\.sh$//g" )" + shift + continue + fi + + if [ -f "$chartsd/$1" ] + then + debug=1 + chart_only="$( echo $1 | sed "s/\.chart\.sh$//g" )" + shift + continue + fi + + # number check + n="$1" + x=$(( n )) + if [ "$x" = "$n" ] + then + shift + update_every=$x + [ $update_every -lt $minimum_update_frequency ] && update_every=$minimum_update_frequency + continue + fi + + fatal "Cannot understand parameter $1. Aborting." done +# ----------------------------------------------------------------------------- +# loop control + +# default sleep function +LOOPSLEEPMS_HIGHRES=0 +now_ms= +current_time_ms_default() { + now_ms="$(date +'%s')000" +} +current_time_ms="current_time_ms_default" +current_time_ms_accuracy=1 +mysleep="sleep" + +# if found and included, this file overwrites loopsleepms() +# and current_time_ms() with a high resolution timer function +# for precise looping. +. "$pluginsd/loopsleepms.sh.inc" + # ----------------------------------------------------------------------------- # load my configuration if [ -f "$myconfig" ] - then - . "$myconfig" - if [ $? -ne 0 ] - then - echo >&2 "charts.d: cannot load $myconfig" - echo "DISABLE" - exit 1 - fi + then + . "$myconfig" + [ $? -ne 0 ] && fatal "cannot load $myconfig" + + time_divisor=$((time_divisor)) + [ $time_divisor -lt 10 ] && time_divisor=10 + [ $time_divisor -gt 100 ] && time_divisor=100 +else + info "configuration file '$myconfig' not found. Using defaults." fi -if [ "$pause_method" = "suspend" ] -then - # enable bash job control - # this is required for suspend to work - set -m +# we check for the timeout command, after we load our +# configuration, so that the user may overwrite the +# timeout command we use, providing a function that +# can emulate the timeout command we need: +# > timeout SECONDS command ... +if [ $check_for_timeout -eq 1 ] + then + require_cmd timeout || exit 1 fi # ----------------------------------------------------------------------------- # internal checks # netdata passes the requested update frequency as the first argument -update_every=$(( update_every + 1 - 1)) # makes sure it is a number +update_every=$(( update_every + 1 - 1)) # makes sure it is a number test $update_every -eq 0 && update_every=1 # if it is zero, make it 1 # check the charts.d directory -if [ ! -d "$chartsd" ] - then - echo >&2 "charts.d: cannot find charts directory '$chartsd'" - echo "DISABLE" -fi - +[ ! -d "$chartsd" ] && fatal "cannot find charts directory '$chartsd'" # ----------------------------------------------------------------------------- -# loop control +# library functions -# default sleep function -loopsleepms() { - sleep $1 +fixid() { + echo "$*" |\ + tr -c "[A-Z][a-z][0-9]" "_" |\ + sed -e "s|^_\+||g" -e "s|_\+$||g" -e "s|_\+|_|g" |\ + tr "[A-Z]" "[a-z]" +} + +run() { + local ret pid="${BASHPID}" t + + if [ "z${1}" = "z-t" -a "${2}" != "0" ] + then + t="${2}" + shift 2 + timeout ${t} "${@}" 2>"${TMP_DIR}/run.${pid}" + ret=$? + else + "${@}" 2>"${TMP_DIR}/run.${pid}" + ret=$? + fi + + if [ ${ret} -ne 0 ] + then + { + printf "$(logdate): ${PROGRAM_NAME}: ${status}: ${MODULE_NAME}: command '" + printf "%q " "${@}" + printf "' failed:\n --- BEGIN TRACE ---\n" + cat "${TMP_DIR}/run.${pid}" + printf " --- END TRACE ---\n" + } >&2 + fi + rm "${TMP_DIR}/run.${pid}" + + return ${ret} +} + +# convert any floating point number +# to integer, give a multiplier +# the result is stored in ${FLOAT2INT_RESULT} +# so that no fork is necessary +# the multiplier must be a power of 10 +float2int() { + local f m="$2" a b l v=($1) + f=${v[0]} + + # the length of the multiplier - 1 + l=$(( ${#m} - 1 )) + + # check if the number is in scientific notation + if [[ ${f} =~ ^[[:space:]]*(-)?[0-9.]+(e|E)(\+|-)[0-9]+ ]] + then + # convert it to decimal + # unfortunately, this fork cannot be avoided + # if you know of a way to avoid it, please let me know + f=$(printf "%0.${l}f" ${f}) + fi + + # split the floating point number + # in integer (a) and decimal (b) + a=${f/.*/} + b=${f/*./} + + # if the integer part is missing + # set it to zero + [ -z "${a}" ] && a="0" + + # strip leading zeros from the integer part + # base 10 convertion + a=$((10#$a)) + + # check the length of the decimal part + # against the length of the multiplier + if [ ${#b} -gt ${l} ] + then + # too many digits - take the most significant + b=${b:0:${l}} + + elif [ ${#b} -lt ${l} ] + then + # too few digits - pad with zero on the right + local z="00000000000000000000000" r=$((l - ${#b})) + b="${b}${z:0:${r}}" + fi + + # strip leading zeros from the decimal part + # base 10 convertion + b=$((10#$b)) + + # store the result + FLOAT2INT_RESULT=$(( (a * m) + b )) } -# if found and included, this file overwrites loopsleepms() -# with a high resolution timer function for precise looping. -. "`dirname $0`/loopsleepms.sh.inc" # ----------------------------------------------------------------------------- # charts check functions all_charts() { - cd "$chartsd" - ls *.chart.sh | sed "s/\.chart\.sh$//g" + cd "$chartsd" + [ $? -ne 0 ] && error "cannot cd to $chartsd" && return 1 + + ls *.chart.sh | sed "s/\.chart\.sh$//g" } +declare -A charts_enable_keyword=( + ['apache']="force" + ['cpu_apps']="force" + ['cpufreq']="force" + ['example']="force" + ['exim']="force" + ['hddtemp']="force" + ['load_average']="force" + ['mem_apps']="force" + ['mysql']="force" + ['nginx']="force" + ['phpfpm']="force" + ['postfix']="force" + ['sensors']="force" + ['squid']="force" + ['tomcat']="force" + ) + all_enabled_charts() { - local charts= - - # find all enabled charts - - for chart in `all_charts` - do - eval "enabled=\$$chart" - if [ "$enabled" = "yes" ] - then - local charts="$charts $chart" - else - echo >&2 "charts.d: chart '$chart' is NOT enabled. Add a line with $chart=yes in $myconfig to enable it." - fi - done - - local charts2= - for chart in $charts - do - # check the enabled charts - local check=`cat "$chartsd/$chart.chart.sh" | sed "s/^ \+//g" | grep "^$chart$charts_check()"` - if [ -z "$check" ] - then - echo >&2 "charts.d: chart '$chart' does not seem to have a $chart$charts_check() function. Disabling it." - continue - fi - - local create=`cat "$chartsd/$chart.chart.sh" | sed "s/^ \+//g" | grep "^$chart$charts_create()"` - if [ -z "$create" ] - then - echo >&2 "charts.d: chart '$chart' does not seem to have a $chart$charts_create() function. Disabling it." - continue - fi - - local update=`cat "$chartsd/$chart.chart.sh" | sed "s/^ \+//g" | grep "^$chart$charts_update()"` - if [ -z "$update" ] - then - echo >&2 "charts.d: chart '$chart' does not seem to have a $chart$charts_update() function. Disabling it." - continue - fi - - # check its config - if [ -f "$confd/$chart.conf" ] - then - if [ ! -z "`cat "$confd/$chart.conf" | sed "s/^ \+//g" | grep -v "^$" | grep -v "^#" | grep -v "^$chart$charts_undescore"`" ] - then - echo >&2 "charts.d: chart's $chart config $confd/$chart.conf should only have lines starting with $chart$charts_undescore . Disabling it." - continue - fi - fi - - "$pluginsd/charts.d.dryrun-helper.sh" "$chart" "$chartsd/$chart.chart.sh" "$confd/$chart.conf" >/dev/null - if [ $? -ne 0 ] - then - echo >&2 "charts.d: chart's $chart did not pass the dry run check. This means it uses global variables not starting with $chart. Disabling it." - continue - fi - - local charts2="$charts2 $chart" - done - - echo $charts2 + local charts= enabled= required= + + # find all enabled charts + + for chart in $( all_charts ) + do + MODULE_NAME="${chart}" + + eval "enabled=\$$chart" + if [ -z "${enabled}" ] + then + enabled="${enable_all_charts}" + fi + + required="${charts_enable_keyword[${chart}]}" + [ -z "${required}" ] && required="yes" + + if [ ! "${enabled}" = "${required}" ] + then + info "is disabled. Add a line with $chart=$required in $myconfig to enable it (or remove the line that disables it)." + else + debug "is enabled for auto-detection." + local charts="$charts $chart" + fi + done + MODULE_NAME="main" + + local charts2= + for chart in $charts + do + MODULE_NAME="${chart}" + + # check the enabled charts + local check="$( cat "$chartsd/$chart.chart.sh" | sed "s/^ \+//g" | grep "^$chart$charts_check()" )" + if [ -z "$check" ] + then + error "module '$chart' does not seem to have a $chart$charts_check() function. Disabling it." + continue + fi + + local create="$( cat "$chartsd/$chart.chart.sh" | sed "s/^ \+//g" | grep "^$chart$charts_create()" )" + if [ -z "$create" ] + then + error "module '$chart' does not seem to have a $chart$charts_create() function. Disabling it." + continue + fi + + local update="$( cat "$chartsd/$chart.chart.sh" | sed "s/^ \+//g" | grep "^$chart$charts_update()" )" + if [ -z "$update" ] + then + error "module '$chart' does not seem to have a $chart$charts_update() function. Disabling it." + continue + fi + + # check its config + #if [ -f "$confd/$chart.conf" ] + #then + # if [ ! -z "$( cat "$confd/$chart.conf" | sed "s/^ \+//g" | grep -v "^$" | grep -v "^#" | grep -v "^$chart$charts_undescore" )" ] + # then + # error "module's $chart config $confd/$chart.conf should only have lines starting with $chart$charts_undescore . Disabling it." + # continue + # fi + #fi + + #if [ $dryrunner -eq 1 ] + # then + # "$pluginsd/charts.d.dryrun-helper.sh" "$chart" "$chartsd/$chart.chart.sh" "$confd/$chart.conf" >/dev/null + # if [ $? -ne 0 ] + # then + # error "module's $chart did not pass the dry run check. This means it uses global variables not starting with $chart. Disabling it." + # continue + # fi + #fi + + local charts2="$charts2 $chart" + done + MODULE_NAME="main" + + echo $charts2 + debug "enabled charts: $charts2" } - # ----------------------------------------------------------------------------- # load the charts +suffix_update_every="_update_every" active_charts= -for chart in `all_enabled_charts` +for chart in $( all_enabled_charts ) do - . "$chartsd/$chart.chart.sh" - - if [ -f "$confd/$chart.conf" ] - then - . "$confd/$chart.conf" - fi - - $chart$charts_check - if [ $? -eq 0 ] - then - active_charts="$active_charts $chart" - else - echo >&2 "charts.d: chart '$chart' check() function reports failure." - fi + MODULE_NAME="${chart}" + + debug "loading module: '$chartsd/$chart.chart.sh'" + + . "$chartsd/$chart.chart.sh" + + if [ -f "$confd/$PROGRAM_NAME/$chart.conf" ] + then + debug "loading module configuration: '$confd/$PROGRAM_NAME/$chart.conf'" + . "$confd/$PROGRAM_NAME/$chart.conf" + elif [ -f "$confd/$chart.conf" ] + then + debug "loading module configuration: '$confd/$chart.conf'" + . "$confd/$chart.conf" + else + warning "configuration file '$confd/$PROGRAM_NAME/$chart.conf' not found. Using defaults." + fi + + eval "dt=\$$chart$suffix_update_every" + dt=$(( dt + 1 - 1 )) # make sure it is a number + if [ $dt -lt $update_every ] + then + eval "$chart$suffix_update_every=$update_every" + fi + + $chart$charts_check + if [ $? -eq 0 ] + then + debug "module '$chart' activated" + active_charts="$active_charts $chart" + else + error "module's '$chart' check() function reports failure." + fi done +MODULE_NAME="main" +debug "activated modules: $active_charts" # ----------------------------------------------------------------------------- @@ -234,28 +526,33 @@ debug_time= test $debug -eq 1 && debug_time=tellwork # if we only need a specific chart, remove all the others -if [ ! -z "$chart_only" ] +if [ ! -z "${chart_only}" ] then - check_charts= - for chart in $active_charts - do - if [ "$chart" = "$chart_only" ] - then - check_charts="$chart" - break - fi - done - active_charts="$check_charts" + debug "requested to run only for: '${chart_only}'" + check_charts= + for chart in $active_charts + do + if [ "$chart" = "$chart_only" ] + then + check_charts="$chart" + break + fi + done + active_charts="$check_charts" fi +debug "activated charts: $active_charts" # stop if we just need a pre-check if [ $check -eq 1 ] then - echo "CHECK RESULT" - echo "Will run the charts: $active_charts" - exit 0 + info "CHECK RESULT" + info "Will run the charts: $active_charts" + exit 0 fi +# ----------------------------------------------------------------------------- + +cd "${TMP_DIR}" || exit 1 # ----------------------------------------------------------------------------- # create charts @@ -263,56 +560,148 @@ fi run_charts= for chart in $active_charts do - $chart$charts_create - if [ $? -eq 0 ] - then - run_charts="$run_charts $chart" - else - echo >&2 "charts.d: chart '$chart' create() function reports failure." - fi + MODULE_NAME="${chart}" + + debug "calling '$chart$charts_create()'..." + $chart$charts_create + if [ $? -eq 0 ] + then + run_charts="$run_charts $chart" + debug "'$chart' initialized." + else + error "module's '$chart' function '$chart$charts_create()' reports failure." + fi done +MODULE_NAME="main" +debug "run_charts='$run_charts'" # ----------------------------------------------------------------------------- # update dimensions + +[ -z "$run_charts" ] && fatal "No charts to collect data from." + +declare -A charts_last_update=() charts_update_every=() charts_next_update=() charts_run_counter=() charts_serial_failures=() global_update() { - while [ 1 ] - do - local now_charts=$run_charts - local run_charts= - - local chart= - for chart in $now_charts - do - local d=`date +'%s.%N'` - local s=`echo $d | cut -d '.' -f 1` - local m=`echo $d | cut -d '.' -f 2 | cut -b 1-3` - local now="$s$m" # milliseconds since epoch (1-1-1970) - - eval "local last=\$last_update_$chart" - test -z "$last" && local last=$((now - 1000)) - - local dt=$(( (now - last) * 1000 )) - eval "last_update_$chart=$now" - - $chart$charts_update $dt - if [ $? -eq 0 ] - then - run_charts="$run_charts $chart" - else - echo >&2 "charts.d: chart '$chart' update() function reports failure. Disabling it." - fi - done - - if [ "$pause_method" = "suspend" ] - then - echo "STOPPING_WAKE_ME_UP_PLEASE" - suspend || ( echo >&2 "suspend returned error $?, falling back to sleep."; loopsleepms $debug_time $update_every ) - else - # wait the time you are required to - loopsleepms $debug_time $update_every - fi - done + local exit_at \ + c=0 dt ret last_ms exec_start_ms exec_end_ms \ + chart now_charts=() next_charts=($run_charts) \ + next_ms x seconds millis + + # return the current time in ms in $now_ms + ${current_time_ms} + + exit_at=$(( now_ms + (restart_timeout * 1000) )) + + for chart in $run_charts + do + eval "charts_update_every[$chart]=\$$chart$suffix_update_every" + test -z "${charts_update_every[$chart]}" && charts_update_every[$charts]=$update_every + charts_last_update[$chart]=$((now_ms - (now_ms % (charts_update_every[$chart] * 1000) ) )) + charts_next_update[$chart]=$(( charts_last_update[$chart] + (charts_update_every[$chart] * 1000) )) + charts_run_counter[$chart]=0 + charts_serial_failures[$chart]=0 + + echo "CHART netdata.plugin_chartsd_$chart '' 'Execution time for $chart plugin' 'milliseconds / run' charts.d netdata.plugin_charts area 145000 ${charts_update_every[$chart]}" + echo "DIMENSION run_time 'run time' absolute 1 1" + done + + # the main loop + while [ "${#next_charts[@]}" -gt 0 ] + do + c=$((c + 1)) + now_charts=("${next_charts[@]}") + next_charts=() + + # return the current time in ms in $now_ms + ${current_time_ms} + + for chart in "${now_charts[@]}" + do + MODULE_NAME="${chart}" + + if [ ${now_ms} -ge ${charts_next_update[$chart]} ] + then + last_ms=${charts_last_update[$chart]} + dt=$(( (now_ms - last_ms) )) + + charts_last_update[$chart]=${now_ms} + + while [ ${charts_next_update[$chart]} -lt ${now_ms} ] + do + charts_next_update[$chart]=$(( charts_next_update[$chart] + (charts_update_every[$chart] * 1000) )) + done + + # the first call should not give a duration + # so that netdata calibrates to current time + dt=$(( dt * 1000 )) + charts_run_counter[$chart]=$(( charts_run_counter[$chart] + 1 )) + if [ ${charts_run_counter[$chart]} -eq 1 ] + then + dt= + fi + + exec_start_ms=$now_ms + $chart$charts_update $dt + ret=$? + + # return the current time in ms in $now_ms + ${current_time_ms}; exec_end_ms=$now_ms + + echo "BEGIN netdata.plugin_chartsd_$chart $dt" + echo "SET run_time = $(( exec_end_ms - exec_start_ms ))" + echo "END" + + if [ $ret -eq 0 ] + then + charts_serial_failures[$chart]=0 + next_charts+=($chart) + else + charts_serial_failures[$chart]=$(( charts_serial_failures[$chart] + 1 )) + + if [ ${charts_serial_failures[$chart]} -gt 10 ] + then + error "module's '$chart' update() function reported failure ${charts_serial_failures[$chart]} times. Disabling it." + else + error "module's '$chart' update() function reports failure. Will keep trying for a while." + next_charts+=($chart) + fi + fi + else + next_charts+=($chart) + fi + done + MODULE_NAME="${chart}" + + # wait the time you are required to + next_ms=$((now_ms + (update_every * 1000 * 100) )) + for x in "${charts_next_update[@]}"; do [ ${x} -lt ${next_ms} ] && next_ms=${x}; done + next_ms=$((next_ms - now_ms)) + + if [ ${LOOPSLEEPMS_HIGHRES} -eq 1 -a ${next_ms} -gt 0 ] + then + next_ms=$(( next_ms + current_time_ms_accuracy )) + seconds=$(( next_ms / 1000 )) + millis=$(( next_ms % 1000 )) + if [ ${millis} -lt 10 ] + then + millis="00${millis}" + elif [ ${millis} -lt 100 ] + then + millis="0${millis}" + fi + + debug "sleeping for ${seconds}.${millis} seconds." + ${mysleep} ${seconds}.${millis} + else + debug "sleeping for ${update_every} seconds." + ${mysleep} $update_every + fi + + test ${now_ms} -ge ${exit_at} && exit 0 + done + + fatal "nothing left to do, exiting..." } global_update