X-Git-Url: https://arthur.barton.de/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=plugins.d%2Falarm-notify.sh;h=a41c55bc70c520896bb0e9e0c059da51d1f3c6df;hb=0b006a56cf575d7fdb05fb2614db8411a1542ca7;hp=f1cfce225c7fc830f8a161cfb1260aa5adee81ce;hpb=e57e106052221ad1892d72d65f3419c951df24e2;p=netdata.git diff --git a/plugins.d/alarm-notify.sh b/plugins.d/alarm-notify.sh index f1cfce22..a41c55bc 100755 --- a/plugins.d/alarm-notify.sh +++ b/plugins.d/alarm-notify.sh @@ -1,22 +1,76 @@ #!/usr/bin/env bash -# (C) Costa Tsaousis -# pushover support by Jan Arnold +# netdata +# real-time performance and health monitoring, done right! +# (C) 2016 Costa Tsaousis +# GPL v3+ +# +# Script to send alarm notifications for netdata +# +# Features: +# - multiple notification methods +# - multiple roles per alarm +# - multiple recipients per role +# - severity filtering per recipient +# +# Supported notification methods: +# - emails +# - pushover.net notifications +# - pushbullet.com push notifications +# - slack.com notifications +# - telegram.org notifications +# + +export PATH="${PATH}:/sbin:/usr/sbin:/usr/local/sbin" +export LC_ALL=C -me="${0}" +# ----------------------------------------------------------------------------- -if [ $(( ${BASH_VERSINFO[0]} )) -lt 4 ] -then - echo >&2 - echo >&2 "$me: ERROR" - echo >&2 "BASH version 4 or later is required." - echo >&2 "You are running version: ${BASH_VERSION}" - echo >&2 "Please upgrade." - echo >&2 +PROGRAM_NAME="$(basename "${0}")" + +logdate() { + date "+%Y-%m-%d %H:%M:%S" +} + +log() { + local status="${1}" + shift + + echo >&2 "$(logdate): ${PROGRAM_NAME}: ${status}: ${*}" + +} + +warning() { + log WARNING "${@}" +} + +error() { + log ERROR "${@}" +} + +info() { + log INFO "${@}" +} + +fatal() { + log FATAL "${@}" exit 1 -fi +} + +debug=0 +debug() { + [ $debug -eq 1 ] && log DEBUG "${@}" +} + +# ----------------------------------------------------------------------------- + +# check for BASH v4+ (required for associative arrays) +[ $(( ${BASH_VERSINFO[0]} )) -lt 4 ] && \ + fatal "BASH version 4 or later is required (this is ${BASH_VERSION})." +# ----------------------------------------------------------------------------- # defaults to allow running this script by hand + NETDATA_CONFIG_DIR="${NETDATA_CONFIG_DIR-/etc/netdata}" NETDATA_CACHE_DIR="${NETDATA_CACHE_DIR-/var/cache/netdata}" [ -z "${NETDATA_REGISTRY_URL}" ] && NETDATA_REGISTRY_URL="https://registry.my-netdata.io" @@ -26,22 +80,22 @@ NETDATA_CACHE_DIR="${NETDATA_CACHE_DIR-/var/cache/netdata}" # ----------------------------------------------------------------------------- # parse command line parameters -recipient="${1}" # the recepient of the email -host="${2}" # the host this event refers to +roles="${1}" # the roles that should be notified for this event +host="${2}" # the host generated this event unique_id="${3}" # the unique id of this event alarm_id="${4}" # the unique id of the alarm that generated this event -event_id="${5}" # the incremental id of the event, for this alarm -when="${6}" # the timestamp this event occured +event_id="${5}" # the incremental id of the event, for this alarm id +when="${6}" # the timestamp this event occurred name="${7}" # the name of the alarm, as given in netdata health.d entries chart="${8}" # the name of the chart (type.id) family="${9}" # the family of the chart status="${10}" # the current status : REMOVED, UNITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL old_status="${11}" # the previous status: REMOVED, UNITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL -value="${12}" # the current value -old_value="${13}" # the previous value +value="${12}" # the current value of the alarm +old_value="${13}" # the previous value of the alarm src="${14}" # the line number and file the alarm has been configured -duration="${15}" # the duration in seconds the previous state took -non_clear_duration="${16}" # the total duration in seconds this is non-clear +duration="${15}" # the duration in seconds of the previous alarm state +non_clear_duration="${16}" # the total duration in seconds this is/was non-clear units="${17}" # the units of the value info="${18}" # a short description of the alarm @@ -51,23 +105,24 @@ info="${18}" # a short description of the alarm # don't do anything if this is not WARNING, CRITICAL or CLEAR if [ "${status}" != "WARNING" -a "${status}" != "CRITICAL" -a "${status}" != "CLEAR" ] then - echo >&2 "${me}: not sending notification for ${status} on '${chart}.${name}'" + info "not sending notification for ${status} on '${chart}.${name}'" exit 1 fi # don't do anything if this is CLEAR, but it was not WARNING or CRITICAL if [ "${old_status}" != "WARNING" -a "${old_status}" != "CRITICAL" -a "${status}" = "CLEAR" ] then - echo >&2 "${me}: not sending notification for ${status} on '${chart}.${name}' (last status was ${old_status})" + info "not sending notification for ${status} on '${chart}.${name}' (last status was ${old_status})" exit 1 fi # ----------------------------------------------------------------------------- # load configuration -# this is defined here so that private registries -# can setup their own -# images_base_url="${NETDATA_REGISTRY_URL}" +# By default fetch images from the global public registry. +# This is required by default, since all notification methods need to download +# images via the Internet, and private registries might not be reachable. +# This can be overwritten at the configuration file. images_base_url="https://registry.my-netdata.io" # needed commands @@ -78,7 +133,9 @@ sendmail= # enable / disable features SEND_SLACK="YES" SEND_PUSHOVER="YES" +SEND_TELEGRAM="YES" SEND_EMAIL="YES" +SEND_PUSHBULLET="YES" # slack configs SLACK_WEBHOOK_URL= @@ -90,6 +147,16 @@ PUSHOVER_APP_TOKEN= DEFAULT_RECIPIENT_PUSHOVER= declare -A role_recipients_pushover=() +# pushbullet configs +PUSHBULLET_ACCESS_TOKEN= +DEFAULT_RECIPIENT_PUSHBULLET= +declare -A role_recipients_pushbullet=() + +# telegram configs +TELEGRAM_BOT_TOKEN= +DEFAULT_RECIPIENT_TELEGRAM= +declare -A role_recipients_telegram=() + # email configs DEFAULT_RECIPIENT_EMAIL="root" declare -A role_recipients_email=() @@ -102,14 +169,14 @@ if [ -f "${NETDATA_CONFIG_DIR}/health_alarm_notify.conf" ] fi # ----------------------------------------------------------------------------- -# filter recipients based on the criticality of each +# filter a recipient based on alarm event severity filter_recipient_by_criticality() { local method="${1}" x="${2}" r s shift - r="${x/|*/}" - s="${x/*|/}" + r="${x/|*/}" # the recipient + s="${x/*|/}" # the severity required for notifying this recipient # no severity filtering for this person [ "${r}" = "${s}" ] && return 0 @@ -142,18 +209,24 @@ filter_recipient_by_criticality() { } # ----------------------------------------------------------------------------- -# find the recipient's addresses per method +# find the recipients' addresses per method declare -A arr_slack=() declare -A arr_pushover=() +declare -A arr_pushbullet=() +declare -A arr_telegram=() declare -A arr_email=() -# netdata may call us with multiple recipients -# so, here we find the unique ones -for x in ${recipient//,/ } +# netdata may call us with multiple roles, and roles may have multiple but +# overlapping recipients - so, here we find the unique recipients. +for x in ${roles//,/ } do + # the roles 'silent' and 'disabled' mean: + # don't send a notification for this role + [ "${x}" = "silent" -o "${x}" = "disabled" ] && continue + # email - a="${role_recipients_email[${recipient}]}" + a="${role_recipients_email[${x}]}" [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_EMAIL}" for r in ${a//,/ } do @@ -161,15 +234,31 @@ do done # pushover - a="${role_recipients_pushover[${recipient}]}" + a="${role_recipients_pushover[${x}]}" [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_PUSHOVER}" for r in ${a//,/ } do [ "${r}" != "disabled" ] && filter_recipient_by_criticality pushover "${r}" && arr_pushover[${r/|*/}]="1" done + # pushbullet + a="${role_recipients_pushbullet[${x}]}" + [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_PUSHBULLET}" + for r in ${a//,/ } + do + [ "${r}" != "disabled" ] && filter_recipient_by_criticality pushbullet "${r}" && arr_pushbullet[${r/|*/}]="1" + done + + # telegram + a="${role_recipients_telegram[${x}]}" + [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_TELEGRAM}" + for r in ${a//,/ } + do + [ "${r}" != "disabled" ] && filter_recipient_by_criticality telegram "${r}" && arr_telegram[${r/|*/}]="1" + done + # slack - a="${role_recipients_slack[${recipient}]}" + a="${role_recipients_slack[${x}]}" [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_SLACK}" for r in ${a//,/ } do @@ -185,6 +274,14 @@ to_slack="${!arr_slack[*]}" to_pushover="${!arr_pushover[*]}" [ -z "${to_pushover}" ] && SEND_PUSHOVER="NO" +# build the list of pushbulet recipients (user tokens) +to_pushbullet="${!arr_pushbullet[*]}" +[ -z "${to_pushbullet}" ] && SEND_PUSHBULLET="NO" + +# check array of telegram recipients (chat ids) +to_telegram="${!arr_telegram[*]}" +[ -z "${to_telegram}" ] && SEND_TELEGRAM="NO" + # build the list of email recipients (email addresses) to_email= for x in "${!arr_email[@]}" @@ -204,12 +301,20 @@ done # check pushover [ -z "${PUSHOVER_APP_TOKEN}" ] && SEND_PUSHOVER="NO" -if [ \( "${SEND_PUSHOVER}" = "YES" -o "${SEND_SLACK}" = "YES" \) -a -z "${curl}" ] +# check pushbullet +[ -z "${DEFAULT_RECIPIENT_PUSHBULLET}" ] && SEND_PUSHBULLET="NO" + +# check telegram +[ -z "${TELEGRAM_BOT_TOKEN}" ] && SEND_TELEGRAM="NO" + +if [ \( "${SEND_PUSHOVER}" = "YES" -o "${SEND_SLACK}" = "YES" -o "${SEND_TELEGRAM}" = "YES" -o "${SEND_PUSHBULLET}" = "YES" \) -a -z "${curl}" ] then curl="$(which curl 2>/dev/null || command -v curl 2>/dev/null)" if [ -z "${curl}" ] then SEND_PUSHOVER="NO" + SEND_PUSHBULLET="NO" + SEND_TELEGRAM="NO" SEND_SLACK="NO" fi fi @@ -221,10 +326,9 @@ if [ "${SEND_EMAIL}" = "YES" -a -z "${sendmail}" ] fi # check that we have at least a method enabled -if [ "${SEND_EMAIL}" != "YES" -a "${SEND_PUSHOVER}" != "YES" -a "${SEND_SLACK}" != "YES" ] +if [ "${SEND_EMAIL}" != "YES" -a "${SEND_PUSHOVER}" != "YES" -a "${SEND_TELEGRAM}" != "YES" -a "${SEND_SLACK}" != "YES" -a "${SEND_PUSHBULLET}" != "YES" ] then - echo >&2 "All notification methods are disabled. Not sending a notification." - exit 1 + fatal "All notification methods are disabled. Not sending a notification." fi # ----------------------------------------------------------------------------- @@ -333,10 +437,10 @@ send_email() { if [ $ret -eq 0 ] then - echo >&2 "${me}: Sent email notification for: ${host} ${chart}.${name} is ${status} to '${to_email}'" + info "sent email notification for: ${host} ${chart}.${name} is ${status} to '${to_email}'" return 0 else - echo >&2 "${me}: Failed to send email notification for: ${host} ${chart}.${name} is ${status} to '${to_email}' with error code ${ret}." + error "failed to send email notification for: ${host} ${chart}.${name} is ${status} to '${to_email}' with error code ${ret}." return 1 fi fi @@ -378,10 +482,83 @@ send_pushover() { if [ "${httpcode}" == "200" ] then - echo >&2 "${me}: Sent pushover notification for: ${host} ${chart}.${name} is ${status} to '${user}'" + info "sent pushover notification for: ${host} ${chart}.${name} is ${status} to '${user}'" + sent=$((sent + 1)) + else + error "failed to send pushover notification for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP error code ${httpcode}." + fi + done + + [ ${sent} -gt 0 ] && return 0 + fi + + return 1 +} + +# ----------------------------------------------------------------------------- +# pushbullet sender + +send_pushbullet() { + local userapikey="${1}" recipients="${2}" message="${3}" title="${4}" httpcode sent=0 user + if [ "${SEND_PUSHBULLET}" = "YES" -a ! -z "${userapikey}" -a ! -z "${recipients}" -a ! -z "${message}" -a ! -z "${title}" ] + then + #https://docs.pushbullet.com/#create-push + for user in ${recipients} + do + httpcode=$(${curl} --write-out %{http_code} --silent --output /dev/null \ + --header 'Access-Token: '$userapikey'' \ + --header 'Content-Type: application/json' \ + --data-binary @<(cat <&2 "${me}: Sent pushbullet notification for: ${host} ${chart}.${name} is ${status} to '${user}'" sent=$((sent + 1)) else - echo >&2 "${me}: Failed to send pushover notification for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP error code ${httpcode}." + echo >&2 "${me}: Failed to send pushbullet notification for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP error code ${httpcode}." + fi + done + + [ ${sent} -gt 0 ] && return 0 + fi + + return 1 +} + +# ----------------------------------------------------------------------------- +# telegram sender + +send_telegram() { + local bottoken="${1}" chatids="${2}" message="${3}" httpcode sent=0 chatid disableNotification="" + + if [ "${status}" = "CLEAR" ]; then disableNotification="--data-urlencode disable_notification=true"; fi + + if [ "${SEND_TELEGRAM}" = "YES" -a ! -z "${bottoken}" -a ! -z "${chatids}" -a ! -z "${message}" ]; + then + for chatid in ${chatids} + do + # https://core.telegram.org/bots/api#sendmessage + httpcode=$(${curl} --write-out %{http_code} --silent --output /dev/null ${disableNotification} \ + --data-urlencode "parse_mode=HTML" \ + --data-urlencode "disable_web_page_preview=true" \ + --data-urlencode "text=$message" \ + "https://api.telegram.org/bot${bottoken}/sendMessage?chat_id=$chatid") + + if [ "${httpcode}" == "200" ] + then + info "sent telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}'" + sent=$((sent + 1)) + elif [ "${httpcode}" == "401" ] + then + error "failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}': Wrong bot token." + else + error "failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}' with HTTP error code ${httpcode}." fi done @@ -443,10 +620,10 @@ EOF httpcode=$(${curl} --write-out %{http_code} --silent --output /dev/null -X POST --data-urlencode "payload=${payload}" "${webhook}") if [ "${httpcode}" == "200" ] then - echo >&2 "${me}: Sent slack notification for: ${host} ${chart}.${name} is ${status} to '${channel}'" + info "sent slack notification for: ${host} ${chart}.${name} is ${status} to '${channel}'" sent=$((sent + 1)) else - echo >&2 "${me}: Failed to send slack notification for: ${host} ${chart}.${name} is ${status} to '${channel}', with HTTP error code ${httpcode}." + error "failed to send slack notification for: ${host} ${chart}.${name} is ${status} to '${channel}', with HTTP error code ${httpcode}." fi done @@ -570,6 +747,31 @@ send_pushover "${PUSHOVER_APP_TOKEN}" "${to_pushover}" "${when}" "${goto_url}" " SENT_PUSHOVER=$? +# ----------------------------------------------------------------------------- +# send the pushbullet notification + +pushbullet_message="${alarm} \n +Severity: ${severity} \n +Chart: ${chart} \n +Family: ${family} \n +To View Netdata go to: ${goto_url} \n +The source of this alarm is line ${src}" +pushbullet_title="${status} at ${host} ${status_message} - ${name//_/ } - ${chart}}" +send_pushbullet "${PUSHBULLET_ACCESS_TOKEN}" "${to_pushbullet}" "$pushbullet_message" "$pushbullet_title" + +SENT_PUSHBULLET=$? + +# ----------------------------------------------------------------------------- +# send the telegram.org message + +# https://core.telegram.org/bots/api#formatting-options +send_telegram "${TELEGRAM_BOT_TOKEN}" "${to_telegram}" "${host} ${status_message} - ${name//_/ } +${chart} (${family}) +${alarm} +${info}" + +SENT_TELEGRAM=$? + # ----------------------------------------------------------------------------- # send the email @@ -668,8 +870,8 @@ SENT_EMAIL=$? # ----------------------------------------------------------------------------- # let netdata know -# we did send somehting -[ ${SENT_EMAIL} -eq 0 -o ${SENT_PUSHOVER} -eq 0 -o ${SENT_SLACK} -eq 0 ] && exit 0 +# we did send something +[ ${SENT_EMAIL} -eq 0 -o ${SENT_PUSHOVER} -eq 0 -o ${SENT_TELEGRAM} -eq 0 -o ${SENT_SLACK} -eq 0 -o ${SENT_PUSHBULLET} -eq 0 ] && exit 0 # we did not send anything exit 1