4 # real-time performance and health monitoring, done right!
5 # (C) 2016 Costa Tsaousis <costa@tsaousis.gr>
8 # charts.d.plugin allows easy development of BASH plugins
10 # if you need to run parallel charts.d processes, link this file to a different name
11 # in the same directory, with a .plugin suffix and netdata will start both of them,
12 # each will have a different config file and modules configuration directory.
15 export PATH="${PATH}:/sbin:/usr/sbin:/usr/local/bin:/usr/local/sbin"
18 PROGRAM_NAME="$(basename $0)"
19 PROGRAM_NAME="${PROGRAM_NAME/.plugin}"
22 # -----------------------------------------------------------------------------
28 if [ ! -z "$TMP_DIR" -a -d "$TMP_DIR" ]
30 [ $debug -eq 1 ] && echo >&2 "$PROGRAM_NAME: cleaning up temporary directory $TMP_DIR ..."
35 trap chartsd_cleanup EXIT
36 trap chartsd_cleanup SIGHUP
37 trap chartsd_cleanup INT
41 TMP_DIR="$( mktemp -d /var/run/netdata-${PROGRAM_NAME}-XXXXXXXXXX )"
43 TMP_DIR="$( mktemp -d /tmp/.netdata-${PROGRAM_NAME}-XXXXXXXXXX )"
47 date "+%Y-%m-%d %H:%M:%S"
54 echo >&2 "$(logdate): ${PROGRAM_NAME}: ${status}: ${MODULE_NAME}: ${*}"
77 [ $debug -eq 1 ] && log DEBUG "${@}"
80 # -----------------------------------------------------------------------------
81 # check a few commands
84 local x=$(which "${1}" 2>/dev/null || command -v "${1}" 2>/dev/null)
85 if [ -z "${x}" -o ! -x "${x}" ]
87 warning "command '${1}' is not found in ${PATH}."
88 eval "${1^^}_CMD=\"\""
92 eval "${1^^}_CMD=\"${x}\""
96 require_cmd date || exit 1
97 require_cmd sed || exit 1
98 require_cmd basename || exit 1
99 require_cmd dirname || exit 1
100 require_cmd cat || exit 1
101 require_cmd grep || exit 1
102 require_cmd egrep || exit 1
103 require_cmd mktemp || exit 1
104 require_cmd awk || exit 1
105 require_cmd timeout || exit 1
106 require_cmd curl || exit 1
108 # -----------------------------------------------------------------------------
110 [ $(( ${BASH_VERSINFO[0]} )) -lt 4 ] && fatal "BASH version 4 or later is required, but found version: ${BASH_VERSION}. Please upgrade."
112 info "started from '$PROGRAM_FILE' with options: $*"
114 # -----------------------------------------------------------------------------
116 # netdata exposes a few environment variables for us
118 pluginsd="${NETDATA_PLUGINS_DIR}"
119 [ -z "$pluginsd" ] && pluginsd="$( dirname $PROGRAM_FILE )"
121 confd="${NETDATA_CONFIG_DIR-/etc/netdata}"
122 chartsd="$pluginsd/../charts.d"
124 myconfig="$confd/$PROGRAM_NAME.conf"
126 minimum_update_frequency="${NETDATA_UPDATE_EVERY-1}"
127 update_every=${minimum_update_frequency} # this will be overwritten by the command line
129 # work around for non BASH shells
130 charts_create="_create"
131 charts_update="_update"
132 charts_check="_check"
135 # when making iterations, charts.d can loop more frequently
136 # to prevent plugins missing iterations.
137 # this is a percentage relative to update_every to align its
139 # The minimum is 10%, the maximum 100%.
140 # So, if update_every is 1 second and time_divisor is 50,
141 # charts.d will iterate every 500ms.
142 # Charts will be called to collect data only if the time
143 # passed since the last time the collected data is equal or
144 # above their update_every.
147 # number of seconds to run without restart
148 # after this time, charts.d.plugin will exit
149 # netdata will restart it
150 restart_timeout=$((3600 * 4))
152 # check if the charts.d plugins are using global variables
154 # It does not currently support BASH v4 arrays, so it is
158 # check for timeout command
161 # the default enable/disable value for all charts
162 enable_all_charts="yes"
164 # -----------------------------------------------------------------------------
171 if [ "$1" = "check" ]
178 if [ "$1" = "debug" -o "$1" = "all" ]
185 if [ -f "$chartsd/$1.chart.sh" ]
188 chart_only="$( echo $1.chart.sh | sed "s/\.chart\.sh$//g" )"
193 if [ -f "$chartsd/$1" ]
196 chart_only="$( echo $1 | sed "s/\.chart\.sh$//g" )"
208 [ $update_every -lt $minimum_update_frequency ] && update_every=$minimum_update_frequency
212 fatal "Cannot understand parameter $1. Aborting."
216 # -----------------------------------------------------------------------------
219 # default sleep function
220 LOOPSLEEPMS_HIGHRES=0
222 current_time_ms_default() {
223 now_ms="$(date +'%s')000"
225 current_time_ms="current_time_ms_default"
226 current_time_ms_accuracy=1
229 # if found and included, this file overwrites loopsleepms()
230 # and current_time_ms() with a high resolution timer function
231 # for precise looping.
232 . "$pluginsd/loopsleepms.sh.inc"
234 # -----------------------------------------------------------------------------
235 # load my configuration
237 if [ -f "$myconfig" ]
240 [ $? -ne 0 ] && fatal "cannot load $myconfig"
242 time_divisor=$((time_divisor))
243 [ $time_divisor -lt 10 ] && time_divisor=10
244 [ $time_divisor -gt 100 ] && time_divisor=100
246 info "configuration file '$myconfig' not found. Using defaults."
249 # we check for the timeout command, after we load our
250 # configuration, so that the user may overwrite the
251 # timeout command we use, providing a function that
252 # can emulate the timeout command we need:
253 # > timeout SECONDS command ...
254 if [ $check_for_timeout -eq 1 ]
256 require_cmd timeout || exit 1
259 # -----------------------------------------------------------------------------
262 # netdata passes the requested update frequency as the first argument
263 update_every=$(( update_every + 1 - 1)) # makes sure it is a number
264 test $update_every -eq 0 && update_every=1 # if it is zero, make it 1
266 # check the charts.d directory
267 [ ! -d "$chartsd" ] && fatal "cannot find charts directory '$chartsd'"
269 # -----------------------------------------------------------------------------
274 tr -c "[A-Z][a-z][0-9]" "_" |\
275 sed -e "s|^_\+||g" -e "s|_\+$||g" -e "s|_\+|_|g" |\
280 local ret pid="${BASHPID}" t
282 if [ "z${1}" = "z-t" -a "${2}" != "0" ]
286 timeout ${t} "${@}" 2>"${TMP_DIR}/run.${pid}"
289 "${@}" 2>"${TMP_DIR}/run.${pid}"
296 printf "$(logdate): ${PROGRAM_NAME}: ${status}: ${MODULE_NAME}: command '"
298 printf "' failed:\n --- BEGIN TRACE ---\n"
299 cat "${TMP_DIR}/run.${pid}"
300 printf " --- END TRACE ---\n"
303 rm "${TMP_DIR}/run.${pid}"
308 # convert any floating point number
309 # to integer, give a multiplier
310 # the result is stored in ${FLOAT2INT_RESULT}
311 # so that no fork is necessary
312 # the multiplier must be a power of 10
314 local f m="$2" a b l v=($1)
317 # the length of the multiplier - 1
320 # check if the number is in scientific notation
321 if [[ ${f} =~ ^[[:space:]]*(-)?[0-9.]+(e|E)(\+|-)[0-9]+ ]]
323 # convert it to decimal
324 # unfortunately, this fork cannot be avoided
325 # if you know of a way to avoid it, please let me know
326 f=$(printf "%0.${l}f" ${f})
329 # split the floating point number
330 # in integer (a) and decimal (b)
334 # if the integer part is missing
336 [ -z "${a}" ] && a="0"
338 # strip leading zeros from the integer part
342 # check the length of the decimal part
343 # against the length of the multiplier
344 if [ ${#b} -gt ${l} ]
346 # too many digits - take the most significant
349 elif [ ${#b} -lt ${l} ]
351 # too few digits - pad with zero on the right
352 local z="00000000000000000000000" r=$((l - ${#b}))
356 # strip leading zeros from the decimal part
361 FLOAT2INT_RESULT=$(( (a * m) + b ))
365 # -----------------------------------------------------------------------------
366 # charts check functions
370 [ $? -ne 0 ] && error "cannot cd to $chartsd" && return 1
372 ls *.chart.sh | sed "s/\.chart\.sh$//g"
375 declare -A charts_enable_keyword=(
382 ['load_average']="force"
393 all_enabled_charts() {
394 local charts= enabled= required=
396 # find all enabled charts
398 for chart in $( all_charts )
400 MODULE_NAME="${chart}"
402 eval "enabled=\$$chart"
403 if [ -z "${enabled}" ]
405 enabled="${enable_all_charts}"
408 required="${charts_enable_keyword[${chart}]}"
409 [ -z "${required}" ] && required="yes"
411 if [ ! "${enabled}" = "${required}" ]
413 info "is disabled. Add a line with $chart=$required in $myconfig to enable it (or remove the line that disables it)."
415 debug "is enabled for auto-detection."
416 local charts="$charts $chart"
424 MODULE_NAME="${chart}"
426 # check the enabled charts
427 local check="$( cat "$chartsd/$chart.chart.sh" | sed "s/^ \+//g" | grep "^$chart$charts_check()" )"
430 error "module '$chart' does not seem to have a $chart$charts_check() function. Disabling it."
434 local create="$( cat "$chartsd/$chart.chart.sh" | sed "s/^ \+//g" | grep "^$chart$charts_create()" )"
437 error "module '$chart' does not seem to have a $chart$charts_create() function. Disabling it."
441 local update="$( cat "$chartsd/$chart.chart.sh" | sed "s/^ \+//g" | grep "^$chart$charts_update()" )"
444 error "module '$chart' does not seem to have a $chart$charts_update() function. Disabling it."
449 #if [ -f "$confd/$chart.conf" ]
451 # if [ ! -z "$( cat "$confd/$chart.conf" | sed "s/^ \+//g" | grep -v "^$" | grep -v "^#" | grep -v "^$chart$charts_undescore" )" ]
453 # error "module's $chart config $confd/$chart.conf should only have lines starting with $chart$charts_undescore . Disabling it."
458 #if [ $dryrunner -eq 1 ]
460 # "$pluginsd/charts.d.dryrun-helper.sh" "$chart" "$chartsd/$chart.chart.sh" "$confd/$chart.conf" >/dev/null
463 # error "module's $chart did not pass the dry run check. This means it uses global variables not starting with $chart. Disabling it."
468 local charts2="$charts2 $chart"
473 debug "enabled charts: $charts2"
476 # -----------------------------------------------------------------------------
479 suffix_update_every="_update_every"
481 for chart in $( all_enabled_charts )
483 MODULE_NAME="${chart}"
485 debug "loading module: '$chartsd/$chart.chart.sh'"
487 . "$chartsd/$chart.chart.sh"
489 if [ -f "$confd/$PROGRAM_NAME/$chart.conf" ]
491 debug "loading module configuration: '$confd/$PROGRAM_NAME/$chart.conf'"
492 . "$confd/$PROGRAM_NAME/$chart.conf"
493 elif [ -f "$confd/$chart.conf" ]
495 debug "loading module configuration: '$confd/$chart.conf'"
496 . "$confd/$chart.conf"
498 warning "configuration file '$confd/$PROGRAM_NAME/$chart.conf' not found. Using defaults."
501 eval "dt=\$$chart$suffix_update_every"
502 dt=$(( dt + 1 - 1 )) # make sure it is a number
503 if [ $dt -lt $update_every ]
505 eval "$chart$suffix_update_every=$update_every"
511 debug "module '$chart' activated"
512 active_charts="$active_charts $chart"
514 error "module's '$chart' check() function reports failure."
518 debug "activated modules: $active_charts"
521 # -----------------------------------------------------------------------------
524 # enable work time reporting
526 test $debug -eq 1 && debug_time=tellwork
528 # if we only need a specific chart, remove all the others
529 if [ ! -z "${chart_only}" ]
531 debug "requested to run only for: '${chart_only}'"
533 for chart in $active_charts
535 if [ "$chart" = "$chart_only" ]
537 check_charts="$chart"
541 active_charts="$check_charts"
543 debug "activated charts: $active_charts"
545 # stop if we just need a pre-check
549 info "Will run the charts: $active_charts"
553 # -----------------------------------------------------------------------------
555 cd "${TMP_DIR}" || exit 1
557 # -----------------------------------------------------------------------------
561 for chart in $active_charts
563 MODULE_NAME="${chart}"
565 debug "calling '$chart$charts_create()'..."
569 run_charts="$run_charts $chart"
570 debug "'$chart' initialized."
572 error "module's '$chart' function '$chart$charts_create()' reports failure."
576 debug "run_charts='$run_charts'"
579 # -----------------------------------------------------------------------------
582 [ -z "$run_charts" ] && fatal "No charts to collect data from."
584 declare -A charts_last_update=() charts_update_every=() charts_next_update=() charts_run_counter=() charts_serial_failures=()
587 c=0 dt ret last_ms exec_start_ms exec_end_ms \
588 chart now_charts=() next_charts=($run_charts) \
589 next_ms x seconds millis
591 # return the current time in ms in $now_ms
594 exit_at=$(( now_ms + (restart_timeout * 1000) ))
596 for chart in $run_charts
598 eval "charts_update_every[$chart]=\$$chart$suffix_update_every"
599 test -z "${charts_update_every[$chart]}" && charts_update_every[$charts]=$update_every
600 charts_last_update[$chart]=$((now_ms - (now_ms % (charts_update_every[$chart] * 1000) ) ))
601 charts_next_update[$chart]=$(( charts_last_update[$chart] + (charts_update_every[$chart] * 1000) ))
602 charts_run_counter[$chart]=0
603 charts_serial_failures[$chart]=0
605 echo "CHART netdata.plugin_chartsd_$chart '' 'Execution time for $chart plugin' 'milliseconds / run' charts.d netdata.plugin_charts area 145000 ${charts_update_every[$chart]}"
606 echo "DIMENSION run_time 'run time' absolute 1 1"
610 while [ "${#next_charts[@]}" -gt 0 ]
613 now_charts=("${next_charts[@]}")
616 # return the current time in ms in $now_ms
619 for chart in "${now_charts[@]}"
621 MODULE_NAME="${chart}"
623 if [ ${now_ms} -ge ${charts_next_update[$chart]} ]
625 last_ms=${charts_last_update[$chart]}
626 dt=$(( (now_ms - last_ms) ))
628 charts_last_update[$chart]=${now_ms}
630 while [ ${charts_next_update[$chart]} -lt ${now_ms} ]
632 charts_next_update[$chart]=$(( charts_next_update[$chart] + (charts_update_every[$chart] * 1000) ))
635 # the first call should not give a duration
636 # so that netdata calibrates to current time
638 charts_run_counter[$chart]=$(( charts_run_counter[$chart] + 1 ))
639 if [ ${charts_run_counter[$chart]} -eq 1 ]
644 exec_start_ms=$now_ms
645 $chart$charts_update $dt
648 # return the current time in ms in $now_ms
649 ${current_time_ms}; exec_end_ms=$now_ms
651 echo "BEGIN netdata.plugin_chartsd_$chart $dt"
652 echo "SET run_time = $(( exec_end_ms - exec_start_ms ))"
657 charts_serial_failures[$chart]=0
658 next_charts+=($chart)
660 charts_serial_failures[$chart]=$(( charts_serial_failures[$chart] + 1 ))
662 if [ ${charts_serial_failures[$chart]} -gt 10 ]
664 error "module's '$chart' update() function reported failure ${charts_serial_failures[$chart]} times. Disabling it."
666 error "module's '$chart' update() function reports failure. Will keep trying for a while."
667 next_charts+=($chart)
671 next_charts+=($chart)
674 MODULE_NAME="${chart}"
676 # wait the time you are required to
677 next_ms=$((now_ms + (update_every * 1000 * 100) ))
678 for x in "${charts_next_update[@]}"; do [ ${x} -lt ${next_ms} ] && next_ms=${x}; done
679 next_ms=$((next_ms - now_ms))
681 if [ ${LOOPSLEEPMS_HIGHRES} -eq 1 -a ${next_ms} -gt 0 ]
683 next_ms=$(( next_ms + current_time_ms_accuracy ))
684 seconds=$(( next_ms / 1000 ))
685 millis=$(( next_ms % 1000 ))
686 if [ ${millis} -lt 10 ]
689 elif [ ${millis} -lt 100 ]
694 debug "sleeping for ${seconds}.${millis} seconds."
695 ${mysleep} ${seconds}.${millis}
697 debug "sleeping for ${update_every} seconds."
698 ${mysleep} $update_every
701 test ${now_ms} -ge ${exit_at} && exit 0
704 fatal "nothing left to do, exiting..."