plugins.d/alarm-notify.sh

   1 #!/usr/bin/env bash
   2
   3 # netdata
   4 # real-time performance and health monitoring, done right!
   5 # (C) 2016 Costa Tsaousis <costa@tsaousis.gr>
   6 # GPL v3+
   7 #
   8 # Script to send alarm notifications for netdata
   9 #
  10 # Features:
  11 #  - multiple notification methods
  12 #  - multiple roles per alarm
  13 #  - multiple recipients per role
  14 #  - severity filtering per recipient
  15 #
  16 # Supported notification methods:
  17 #  - emails
  18 #  - slack.com notifications
  19 #  - pushover.net notifications
  20 #  - pushbullet.com push notifications by Tiago Peralta @tperalta82 PR #1070
  21 #  - telegram.org notifications by @hashworks PR #1002
  22 #
  23
  24 # -----------------------------------------------------------------------------
  25 # testing notifications
  26
  27 if [ \( "${1}" = "test" -o "${2}" = "test" \) -a "${#}" -le 2 ]
  28 then
  29     if [ "${2}" = "test" ]
  30     then
  31         recipient="${1}"
  32     else
  33         recipient="${2}"
  34     fi
  35
  36     [ -z "${recipient}" ] && recipient="sysadmin"
  37
  38     echo >&2 ">> SENDING TEST ALARM TO ROLE: ${recipient} <<"
  39
  40     "${0}" "${recipient}" 'test' '1' '1' '1' '1' 'test_alarm' 'test_alarm' 'test' 'CRITICAL' 'CLEAR' '1' '1' "${0}" '1' '1' 'number' 'test alarm to verify notifications work'
  41     ret=$?
  42
  43     [ ${ret} -ne 0 ] && echo >&2 ">> FAILED <<" && exit ${ret}
  44     echo >&2 ">> OK <<"
  45     exit $?
  46 fi
  47
  48 export PATH="${PATH}:/sbin:/usr/sbin:/usr/local/sbin"
  49 export LC_ALL=C
  50
  51 # -----------------------------------------------------------------------------
  52
  53 PROGRAM_NAME="$(basename "${0}")"
  54
  55 logdate() {
  56     date "+%Y-%m-%d %H:%M:%S"
  57 }
  58
  59 log() {
  60     local status="${1}"
  61     shift
  62
  63     echo >&2 "$(logdate): ${PROGRAM_NAME}: ${status}: ${*}"
  64
  65 }
  66
  67 warning() {
  68     log WARNING "${@}"
  69 }
  70
  71 error() {
  72     log ERROR "${@}"
  73 }
  74
  75 info() {
  76     log INFO "${@}"
  77 }
  78
  79 fatal() {
  80     log FATAL "${@}"
  81     exit 1
  82 }
  83
  84 debug=0
  85 debug() {
  86     [ $debug -eq 1 ] && log DEBUG "${@}"
  87 }
  88
  89 # -----------------------------------------------------------------------------
  90
  91 # check for BASH v4+ (required for associative arrays)
  92 [ $(( ${BASH_VERSINFO[0]} )) -lt 4 ] && \
  93     fatal "BASH version 4 or later is required (this is ${BASH_VERSION})."
  94
  95 # -----------------------------------------------------------------------------
  96 # defaults to allow running this script by hand
  97
  98 NETDATA_CONFIG_DIR="${NETDATA_CONFIG_DIR-/etc/netdata}"
  99 NETDATA_CACHE_DIR="${NETDATA_CACHE_DIR-/var/cache/netdata}"
 100 [ -z "${NETDATA_REGISTRY_URL}" ] && NETDATA_REGISTRY_URL="https://registry.my-netdata.io"
 101 [ -z "${NETDATA_HOSTNAME}" ] && NETDATA_HOSTNAME="$(hostname)"
 102 [ -z "${NETDATA_REGISTRY_HOSTNAME}" ] && NETDATA_REGISTRY_HOSTNAME="${NETDATA_HOSTNAME}"
 103
 104 # -----------------------------------------------------------------------------
 105 # parse command line parameters
 106
 107 roles="${1}"       # the roles that should be notified for this event
 108 host="${2}"        # the host generated this event
 109 unique_id="${3}"   # the unique id of this event
 110 alarm_id="${4}"    # the unique id of the alarm that generated this event
 111 event_id="${5}"    # the incremental id of the event, for this alarm id
 112 when="${6}"        # the timestamp this event occurred
 113 name="${7}"        # the name of the alarm, as given in netdata health.d entries
 114 chart="${8}"       # the name of the chart (type.id)
 115 family="${9}"      # the family of the chart
 116 status="${10}"     # the current status : REMOVED, UNITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
 117 old_status="${11}" # the previous status: REMOVED, UNITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
 118 value="${12}"      # the current value of the alarm
 119 old_value="${13}"  # the previous value of the alarm
 120 src="${14}"        # the line number and file the alarm has been configured
 121 duration="${15}"   # the duration in seconds of the previous alarm state
 122 non_clear_duration="${16}" # the total duration in seconds this is/was non-clear
 123 units="${17}"      # the units of the value
 124 info="${18}"       # a short description of the alarm
 125
 126 # -----------------------------------------------------------------------------
 127 # screen statuses we don't need to send a notification
 128
 129 # don't do anything if this is not WARNING, CRITICAL or CLEAR
 130 if [ "${status}" != "WARNING" -a "${status}" != "CRITICAL" -a "${status}" != "CLEAR" ]
 131 then
 132     info "not sending notification for ${status} on '${chart}.${name}'"
 133     exit 1
 134 fi
 135
 136 # don't do anything if this is CLEAR, but it was not WARNING or CRITICAL
 137 if [ "${old_status}" != "WARNING" -a "${old_status}" != "CRITICAL" -a "${status}" = "CLEAR" ]
 138 then
 139     info "not sending notification for ${status} on '${chart}.${name}' (last status was ${old_status})"
 140     exit 1
 141 fi
 142
 143 # -----------------------------------------------------------------------------
 144 # load configuration
 145
 146 # By default fetch images from the global public registry.
 147 # This is required by default, since all notification methods need to download
 148 # images via the Internet, and private registries might not be reachable.
 149 # This can be overwritten at the configuration file.
 150 images_base_url="https://registry.my-netdata.io"
 151
 152 # needed commands
 153 # if empty they will be searched in the system path
 154 curl=
 155 sendmail=
 156
 157 # enable / disable features
 158 SEND_SLACK="YES"
 159 SEND_PUSHOVER="YES"
 160 SEND_TELEGRAM="YES"
 161 SEND_EMAIL="YES"
 162 SEND_PUSHBULLET="YES"
 163
 164 # slack configs
 165 SLACK_WEBHOOK_URL=
 166 DEFAULT_RECIPIENT_SLACK=
 167 declare -A role_recipients_slack=()
 168
 169 # pushover configs
 170 PUSHOVER_APP_TOKEN=
 171 DEFAULT_RECIPIENT_PUSHOVER=
 172 declare -A role_recipients_pushover=()
 173
 174 # pushbullet configs
 175 PUSHBULLET_ACCESS_TOKEN=
 176 DEFAULT_RECIPIENT_PUSHBULLET=
 177 declare -A role_recipients_pushbullet=()
 178
 179 # telegram configs
 180 TELEGRAM_BOT_TOKEN=
 181 DEFAULT_RECIPIENT_TELEGRAM=
 182 declare -A role_recipients_telegram=()
 183
 184 # email configs
 185 DEFAULT_RECIPIENT_EMAIL="root"
 186 declare -A role_recipients_email=()
 187
 188 # load the user configuration
 189 # this will overwrite the variables above
 190 if [ -f "${NETDATA_CONFIG_DIR}/health_alarm_notify.conf" ]
 191     then
 192     source "${NETDATA_CONFIG_DIR}/health_alarm_notify.conf"
 193 fi
 194
 195 # -----------------------------------------------------------------------------
 196 # filter a recipient based on alarm event severity
 197
 198 filter_recipient_by_criticality() {
 199     local method="${1}" x="${2}" r s
 200     shift
 201
 202     r="${x/|*/}" # the recipient
 203     s="${x/*|/}" # the severity required for notifying this recipient
 204
 205     # no severity filtering for this person
 206     [ "${r}" = "${s}" ] && return 0
 207
 208     # the severity is invalid
 209     s="${s^^}"
 210     [ "${s}" != "CRITICAL" ] && return 0
 211
 212     # the new or the old status matches the severity
 213     if [ "${s}" = "${status}" -o "${s}" = "${old_status}" ]
 214         then
 215         [ ! -d "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}" ] && \
 216             mkdir -p "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}"
 217
 218         # we need to keep track of the notifications we sent
 219         # so that the same user will receive the recovery
 220         # even if old_status does not match the required severity
 221         touch "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}"
 222         return 0
 223     fi
 224
 225     # it is a cleared alarm we have sent notification for
 226     if [ "${status}" != "WARNING" -a "${status}" != "CRITICAL" -a -f "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}" ]
 227         then
 228         rm "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}"
 229         return 0
 230     fi
 231
 232     return 1
 233 }
 234
 235 # -----------------------------------------------------------------------------
 236 # find the recipients' addresses per method
 237
 238 declare -A arr_slack=()
 239 declare -A arr_pushover=()
 240 declare -A arr_pushbullet=()
 241 declare -A arr_telegram=()
 242 declare -A arr_email=()
 243
 244 # netdata may call us with multiple roles, and roles may have multiple but
 245 # overlapping recipients - so, here we find the unique recipients.
 246 for x in ${roles//,/ }
 247 do
 248     # the roles 'silent' and 'disabled' mean:
 249     # don't send a notification for this role
 250     [ "${x}" = "silent" -o "${x}" = "disabled" ] && continue
 251
 252     # email
 253     a="${role_recipients_email[${x}]}"
 254     [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_EMAIL}"
 255     for r in ${a//,/ }
 256     do
 257         [ "${r}" != "disabled" ] && filter_recipient_by_criticality email "${r}" && arr_email[${r/|*/}]="1"
 258     done
 259
 260     # pushover
 261     a="${role_recipients_pushover[${x}]}"
 262     [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_PUSHOVER}"
 263     for r in ${a//,/ }
 264     do
 265         [ "${r}" != "disabled" ] && filter_recipient_by_criticality pushover "${r}" && arr_pushover[${r/|*/}]="1"
 266     done
 267
 268     # pushbullet
 269     a="${role_recipients_pushbullet[${x}]}"
 270     [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_PUSHBULLET}"
 271     for r in ${a//,/ }
 272     do
 273         [ "${r}" != "disabled" ] && filter_recipient_by_criticality pushbullet "${r}" && arr_pushbullet[${r/|*/}]="1"
 274     done
 275
 276     # telegram
 277     a="${role_recipients_telegram[${x}]}"
 278     [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_TELEGRAM}"
 279     for r in ${a//,/ }
 280     do
 281         [ "${r}" != "disabled" ] && filter_recipient_by_criticality telegram "${r}" && arr_telegram[${r/|*/}]="1"
 282     done
 283
 284     # slack
 285     a="${role_recipients_slack[${x}]}"
 286     [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_SLACK}"
 287     for r in ${a//,/ }
 288     do
 289         [ "${r}" != "disabled" ] && filter_recipient_by_criticality slack "${r}" && arr_slack[${r/|*/}]="1"
 290     done
 291 done
 292
 293 # build the list of slack recipients (channels)
 294 to_slack="${!arr_slack[*]}"
 295 [ -z "${to_slack}" ] && SEND_SLACK="NO"
 296
 297 # build the list of pushover recipients (user tokens)
 298 to_pushover="${!arr_pushover[*]}"
 299 [ -z "${to_pushover}" ] && SEND_PUSHOVER="NO"
 300
 301 # build the list of pushbulet recipients (user tokens)
 302 to_pushbullet="${!arr_pushbullet[*]}"
 303 [ -z "${to_pushbullet}" ] && SEND_PUSHBULLET="NO"
 304
 305 # check array of telegram recipients (chat ids)
 306 to_telegram="${!arr_telegram[*]}"
 307 [ -z "${to_telegram}" ] && SEND_TELEGRAM="NO"
 308
 309 # build the list of email recipients (email addresses)
 310 to_email=
 311 for x in "${!arr_email[@]}"
 312 do
 313     [ ! -z "${to_email}" ] && to_email="${to_email}, "
 314     to_email="${to_email}${x}"
 315 done
 316 [ -z "${to_email}" ] && SEND_EMAIL="NO"
 317
 318
 319 # -----------------------------------------------------------------------------
 320 # verify the delivery methods supported
 321
 322 # check slack
 323 [ -z "${SLACK_WEBHOOK_URL}" ] && SEND_SLACK="NO"
 324
 325 # check pushover
 326 [ -z "${PUSHOVER_APP_TOKEN}" ] && SEND_PUSHOVER="NO"
 327
 328 # check pushbullet
 329 [ -z "${DEFAULT_RECIPIENT_PUSHBULLET}" ] && SEND_PUSHBULLET="NO"
 330
 331 # check telegram
 332 [ -z "${TELEGRAM_BOT_TOKEN}" ] && SEND_TELEGRAM="NO"
 333
 334 if [ \( "${SEND_PUSHOVER}" = "YES" -o "${SEND_SLACK}" = "YES" -o "${SEND_TELEGRAM}" = "YES" -o "${SEND_PUSHBULLET}" = "YES" \) -a -z "${curl}" ]
 335     then
 336     curl="$(which curl 2>/dev/null || command -v curl 2>/dev/null)"
 337     if [ -z "${curl}" ]
 338         then
 339         SEND_PUSHOVER="NO"
 340         SEND_PUSHBULLET="NO"
 341         SEND_TELEGRAM="NO"
 342         SEND_SLACK="NO"
 343     fi
 344 fi
 345
 346 if [ "${SEND_EMAIL}" = "YES" -a -z "${sendmail}" ]
 347     then
 348     sendmail="$(which sendmail 2>/dev/null || command -v sendmail 2>/dev/null)"
 349     [ -z "${sendmail}" ] && SEND_EMAIL="NO"
 350 fi
 351
 352 # check that we have at least a method enabled
 353 if [ "${SEND_EMAIL}" != "YES" -a "${SEND_PUSHOVER}" != "YES" -a "${SEND_TELEGRAM}" != "YES" -a "${SEND_SLACK}" != "YES" -a "${SEND_PUSHBULLET}" != "YES" ]
 354     then
 355     fatal "All notification methods are disabled. Not sending a notification."
 356 fi
 357
 358 # -----------------------------------------------------------------------------
 359 # get the system hostname
 360
 361 [ -z "${host}" ] && host="${NETDATA_HOSTNAME}"
 362 [ -z "${host}" ] && host="${NETDATA_REGISTRY_HOSTNAME}"
 363 [ -z "${host}" ] && host="$(hostname 2>/dev/null)"
 364
 365 # -----------------------------------------------------------------------------
 366 # get the date the alarm happened
 367
 368 date="$(date --date=@${when} 2>/dev/null)"
 369 [ -z "${date}" ] && date="$(date 2>/dev/null)"
 370
 371 # -----------------------------------------------------------------------------
 372 # URL encode a string
 373
 374 urlencode() {
 375     local string="${1}" strlen encoded pos c o
 376
 377     strlen=${#string}
 378     for (( pos=0 ; pos<strlen ; pos++ ))
 379     do
 380         c=${string:$pos:1}
 381         case "$c" in
 382             [-_.~a-zA-Z0-9])
 383                 o="${c}"
 384                 ;;
 385
 386             *)
 387                 printf -v o '%%%02x' "'$c"
 388                 ;;
 389         esac
 390         encoded+="${o}"
 391     done
 392
 393     REPLY="${encoded}"
 394     echo "${REPLY}"
 395 }
 396
 397 # -----------------------------------------------------------------------------
 398 # convert a duration in seconds, to a human readable duration
 399 # using DAYS, MINUTES, SECONDS
 400
 401 duration4human() {
 402     local s="${1}" d=0 h=0 m=0 ds="day" hs="hour" ms="minute" ss="second" ret
 403     d=$(( s / 86400 ))
 404     s=$(( s - (d * 86400) ))
 405     h=$(( s / 3600 ))
 406     s=$(( s - (h * 3600) ))
 407     m=$(( s / 60 ))
 408     s=$(( s - (m * 60) ))
 409
 410     if [ ${d} -gt 0 ]
 411     then
 412         [ ${m} -ge 30 ] && h=$(( h + 1 ))
 413         [ ${d} -gt 1 ] && ds="days"
 414         [ ${h} -gt 1 ] && hs="hours"
 415         if [ ${h} -gt 0 ]
 416         then
 417             ret="${d} ${ds} and ${h} ${hs}"
 418         else
 419             ret="${d} ${ds}"
 420         fi
 421     elif [ ${h} -gt 0 ]
 422     then
 423         [ ${s} -ge 30 ] && m=$(( m + 1 ))
 424         [ ${h} -gt 1 ] && hs="hours"
 425         [ ${m} -gt 1 ] && ms="minutes"
 426         if [ ${m} -gt 0 ]
 427         then
 428             ret="${h} ${hs} and ${m} ${ms}"
 429         else
 430             ret="${h} ${hs}"
 431         fi
 432     elif [ ${m} -gt 0 ]
 433     then
 434         [ ${m} -gt 1 ] && ms="minutes"
 435         [ ${s} -gt 1 ] && ss="seconds"
 436         if [ ${s} -gt 0 ]
 437         then
 438             ret="${m} ${ms} and ${s} ${ss}"
 439         else
 440             ret="${m} ${ms}"
 441         fi
 442     else
 443         [ ${s} -gt 1 ] && ss="seconds"
 444         ret="${s} ${ss}"
 445     fi
 446
 447     REPLY="${ret}"
 448     echo "${REPLY}"
 449 }
 450
 451 # -----------------------------------------------------------------------------
 452 # email sender
 453
 454 send_email() {
 455     local ret=
 456     if [ "${SEND_EMAIL}" = "YES" ]
 457         then
 458
 459         "${sendmail}" -t
 460         ret=$?
 461
 462         if [ $ret -eq 0 ]
 463         then
 464             info "sent email notification for: ${host} ${chart}.${name} is ${status} to '${to_email}'"
 465             return 0
 466         else
 467             error "failed to send email notification for: ${host} ${chart}.${name} is ${status} to '${to_email}' with error code ${ret}."
 468             return 1
 469         fi
 470     fi
 471
 472     return 1
 473 }
 474
 475 # -----------------------------------------------------------------------------
 476 # pushover sender
 477
 478 send_pushover() {
 479     local apptoken="${1}" usertokens="${2}" when="${3}" url="${4}" status="${5}" title="${6}" message="${7}" httpcode sent=0 user priority
 480
 481     if [ "${SEND_PUSHOVER}" = "YES" -a ! -z "${apptoken}" -a ! -z "${usertokens}" -a ! -z "${title}" -a ! -z "${message}" ]
 482         then
 483
 484         # https://pushover.net/api
 485         priority=-2
 486         case "${status}" in
 487             CLEAR) priority=-1;;   # low priority: no sound or vibration
 488             WARNING) priotity=0;;  # normal priority: respect quiet hours
 489             CRITICAL) priority=1;; # high priority: bypass quiet hours
 490             *) priority=-2;;       # lowest priority: no notification at all
 491         esac
 492
 493         for user in ${usertokens}
 494         do
 495             httpcode=$(${curl} --write-out %{http_code} --silent --output /dev/null \
 496                 --form-string "token=${apptoken}" \
 497                 --form-string "user=${user}" \
 498                 --form-string "html=1" \
 499                 --form-string "title=${title}" \
 500                 --form-string "message=${message}" \
 501                 --form-string "timestamp=${when}" \
 502                 --form-string "url=${url}" \
 503                 --form-string "url_title=Open netdata dashboard to view the alarm" \
 504                 --form-string "priority=${priority}" \
 505                 https://api.pushover.net/1/messages.json)
 506
 507             if [ "${httpcode}" == "200" ]
 508             then
 509                 info "sent pushover notification for: ${host} ${chart}.${name} is ${status} to '${user}'"
 510                 sent=$((sent + 1))
 511             else
 512                 error "failed to send pushover notification for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP error code ${httpcode}."
 513             fi
 514         done
 515
 516         [ ${sent} -gt 0 ] && return 0
 517     fi
 518
 519     return 1
 520 }
 521
 522 # -----------------------------------------------------------------------------
 523 # pushbullet sender
 524
 525 send_pushbullet() {
 526     local userapikey="${1}" recipients="${2}"  title="${3}" message="${4}" httpcode sent=0 user
 527     if [ "${SEND_PUSHBULLET}" = "YES" -a ! -z "${userapikey}" -a ! -z "${recipients}" -a ! -z "${message}" -a ! -z "${title}" ]
 528         then
 529         #https://docs.pushbullet.com/#create-push
 530         for user in ${recipients}
 531         do
 532             httpcode=$(${curl} --write-out %{http_code} --silent --output /dev/null \
 533               --header 'Access-Token: '$userapikey'' \
 534               --header 'Content-Type: application/json' \
 535               --data-binary  @<(cat <<EOF
 536                               {"title": "${title}",
 537                               "type": "note",
 538                               "email": "${user}",
 539                               "body": "$( echo -n ${message})"}
 540 EOF
 541                ) "https://api.pushbullet.com/v2/pushes" -X POST)
 542
 543             if [ "${httpcode}" == "200" ]
 544             then
 545                 info "sent pushbullet notification for: ${host} ${chart}.${name} is ${status} to '${user}'"
 546                 sent=$((sent + 1))
 547             else
 548                 error "failed to send pushbullet notification for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP error code ${httpcode}."
 549             fi
 550         done
 551
 552         [ ${sent} -gt 0 ] && return 0
 553     fi
 554
 555     return 1
 556 }
 557
 558 # -----------------------------------------------------------------------------
 559 # telegram sender
 560
 561 send_telegram() {
 562     local bottoken="${1}" chatids="${2}" message="${3}" httpcode sent=0 chatid disableNotification=""
 563
 564     if [ "${status}" = "CLEAR" ]; then disableNotification="--data-urlencode disable_notification=true"; fi
 565
 566     if [ "${SEND_TELEGRAM}" = "YES" -a ! -z "${bottoken}" -a ! -z "${chatids}" -a ! -z "${message}" ];
 567     then
 568         for chatid in ${chatids}
 569         do
 570             # https://core.telegram.org/bots/api#sendmessage
 571             httpcode=$(${curl} --write-out %{http_code} --silent --output /dev/null ${disableNotification} \
 572                 --data-urlencode "parse_mode=HTML" \
 573                 --data-urlencode "disable_web_page_preview=true" \
 574                 --data-urlencode "text=$message" \
 575                 "https://api.telegram.org/bot${bottoken}/sendMessage?chat_id=$chatid")
 576
 577             if [ "${httpcode}" == "200" ]
 578             then
 579                 info "sent telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}'"
 580                 sent=$((sent + 1))
 581             elif [ "${httpcode}" == "401" ]
 582             then
 583                 error "failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}': Wrong bot token."
 584             else
 585                 error "failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}' with HTTP error code ${httpcode}."
 586             fi
 587         done
 588
 589         [ ${sent} -gt 0 ] && return 0
 590     fi
 591
 592     return 1
 593 }
 594
 595 # -----------------------------------------------------------------------------
 596 # slack sender
 597
 598 send_slack() {
 599     local webhook="${1}" channels="${2}" httpcode sent=0 channel color payload
 600
 601     [ "${SEND_SLACK}" != "YES" ] && return 1
 602
 603     case "${status}" in
 604         WARNING) color="warning" ;;
 605         CRITICAL) color="danger" ;;
 606         CLEAR) color="good" ;;
 607         *) color="#777777" ;;
 608     esac
 609
 610     for channel in ${channels}
 611     do
 612         payload="$(cat <<EOF
 613         {
 614             "channel": "#${channel}",
 615             "username": "netdata on ${host}",
 616             "icon_url": "${images_base_url}/images/seo-performance-128.png",
 617             "text": "${host} ${status_message}, \`${chart}\` (_${family}_), *${alarm}*",
 618             "attachments": [
 619                 {
 620                     "fallback": "${alarm} - ${chart} (${family}) - ${info}",
 621                     "color": "${color}",
 622                     "title": "${alarm}",
 623                     "title_link": "${goto_url}",
 624                     "text": "${info}",
 625                     "fields": [
 626                         {
 627                             "title": "${chart}",
 628                             "short": true
 629                         },
 630                         {
 631                             "title": "${family}",
 632                             "short": true
 633                         }
 634                     ],
 635                     "thumb_url": "${image}",
 636                     "footer": "<${goto_url}|${host}>",
 637                     "ts": ${when}
 638                 }
 639             ]
 640         }
 641 EOF
 642         )"
 643
 644         httpcode=$(${curl} --write-out %{http_code} --silent --output /dev/null -X POST --data-urlencode "payload=${payload}" "${webhook}")
 645         if [ "${httpcode}" == "200" ]
 646         then
 647             info "sent slack notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
 648             sent=$((sent + 1))
 649         else
 650             error "failed to send slack notification for: ${host} ${chart}.${name} is ${status} to '${channel}', with HTTP error code ${httpcode}."
 651         fi
 652     done
 653
 654     [ ${sent} -gt 0 ] && return 0
 655
 656     return 1
 657 }
 658
 659
 660 # -----------------------------------------------------------------------------
 661 # prepare the content of the notification
 662
 663 # the url to send the user on click
 664 urlencode "${NETDATA_REGISTRY_HOSTNAME}" >/dev/null; url_host="${REPLY}"
 665 urlencode "${chart}" >/dev/null; url_chart="${REPLY}"
 666 urlencode "${family}" >/dev/null; url_family="${REPLY}"
 667 urlencode "${name}" >/dev/null; url_name="${REPLY}"
 668 goto_url="${NETDATA_REGISTRY_URL}/goto-host-from-alarm.html?host=${url_host}&chart=${url_chart}&family=${url_family}&alarm=${url_name}&alarm_unique_id=${unique_id}&alarm_id=${alarm_id}&alarm_event_id=${event_id}"
 669
 670 # the severity of the alarm
 671 severity="${status}"
 672
 673 # the time the alarm was raised
 674 duration4human ${duration} >/dev/null; duration_txt="${REPLY}"
 675 duration4human ${non_clear_duration} >/dev/null; non_clear_duration_txt="${REPLY}"
 676 raised_for="(was ${old_status,,} for ${duration_txt})"
 677
 678 # the key status message
 679 status_message="status unknown"
 680
 681 # the color of the alarm
 682 color="grey"
 683
 684 # the alarm value
 685 alarm="${name//_/ } = ${value} ${units}"
 686
 687 # the image of the alarm
 688 image="${images_base_url}/images/seo-performance-128.png"
 689
 690 # prepare the title based on status
 691 case "${status}" in
 692         CRITICAL)
 693         image="${images_base_url}/images/alert-128-red.png"
 694         status_message="is critical"
 695         color="#ca414b"
 696         ;;
 697
 698     WARNING)
 699         image="${images_base_url}/images/alert-128-orange.png"
 700         status_message="needs attention"
 701         color="#caca4b"
 702                 ;;
 703
 704         CLEAR)
 705         image="${images_base_url}/images/check-mark-2-128-green.png"
 706         status_message="recovered"
 707                 color="#77ca6d"
 708
 709                 # don't show the value when the status is CLEAR
 710                 # for certain alarms, this value might not have any meaning
 711                 alarm="${name//_/ } ${raised_for}"
 712                 ;;
 713 esac
 714
 715 if [ "${status}" = "CLEAR" ]
 716 then
 717     severity="Recovered from ${old_status}"
 718     if [ $non_clear_duration -gt $duration ]
 719     then
 720         raised_for="(alarm was raised for ${non_clear_duration_txt})"
 721     fi
 722
 723 elif [ "${old_status}" = "WARNING" -a "${status}" = "CRITICAL" ]
 724 then
 725     severity="Escalated to ${status}"
 726     if [ $non_clear_duration -gt $duration ]
 727     then
 728         raised_for="(alarm is raised for ${non_clear_duration_txt})"
 729     fi
 730
 731 elif [ "${old_status}" = "CRITICAL" -a "${status}" = "WARNING" ]
 732 then
 733     severity="Demoted to ${status}"
 734     if [ $non_clear_duration -gt $duration ]
 735     then
 736         raised_for="(alarm is raised for ${non_clear_duration_txt})"
 737     fi
 738
 739 else
 740     raised_for=
 741 fi
 742
 743 # prepare HTML versions of elements
 744 info_html=
 745 [ ! -z "${info}" ] && info_html=" <small><br/>${info}</small>"
 746
 747 raised_for_html=
 748 [ ! -z "${raised_for}" ] && raised_for_html="<br/><small>${raised_for}</small>"
 749
 750 # -----------------------------------------------------------------------------
 751 # send the slack notification
 752
 753 # slack aggregates posts from the same username
 754 # so we use "${host} ${status}" as the bot username, to make them diff
 755
 756 send_slack "${SLACK_WEBHOOK_URL}" "${to_slack}"
 757 SENT_SLACK=$?
 758
 759 # -----------------------------------------------------------------------------
 760 # send the pushover notification
 761
 762 send_pushover "${PUSHOVER_APP_TOKEN}" "${to_pushover}" "${when}" "${goto_url}" "${status}" "${host} ${status_message} - ${name//_/ } - ${chart}" "
 763 <font color=\"${color}\"><b>${alarm}</b></font>${info_html}<br/>&nbsp;
 764 <small><b>${chart}</b><br/>Chart<br/>&nbsp;</small>
 765 <small><b>${family}</b><br/>Family<br/>&nbsp;</small>
 766 <small><b>${severity}</b><br/>Severity<br/>&nbsp;</small>
 767 <small><b>${date}${raised_for_html}</b><br/>Time<br/>&nbsp;</small>
 768 <a href=\"${goto_url}\">View Netdata</a><br/>&nbsp;
 769 <small><small>The source of this alarm is line ${src}</small></small>
 770 "
 771
 772 SENT_PUSHOVER=$?
 773
 774 # -----------------------------------------------------------------------------
 775 # send the pushbullet notification
 776
 777 send_pushbullet "${PUSHBULLET_ACCESS_TOKEN}" "${to_pushbullet}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm}\n
 778 Severity: ${severity}\n
 779 Chart: ${chart}\n
 780 Family: ${family}\n
 781 To View Netdata go to: ${goto_url}\n
 782 The source of this alarm is line ${src}"
 783
 784 SENT_PUSHBULLET=$?
 785
 786 # -----------------------------------------------------------------------------
 787 # send the telegram.org message
 788
 789 # https://core.telegram.org/bots/api#formatting-options
 790 send_telegram "${TELEGRAM_BOT_TOKEN}" "${to_telegram}" "${host} ${status_message} - <b>${name//_/ }</b>
 791 ${chart} (${family})
 792 <a href=\"${goto_url}\">${alarm}</a>
 793 <i>${info}</i>"
 794
 795 SENT_TELEGRAM=$?
 796
 797 # -----------------------------------------------------------------------------
 798 # send the email
 799
 800 send_email <<EOF
 801 To: ${to_email}
 802 Subject: ${host} ${status_message} - ${name//_/ } - ${chart}
 803 Content-Type: text/html
 804
 805 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
 806 <html xmlns="http://www.w3.org/1999/xhtml" style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0; padding: 0;">
 807 <body style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 14px; width: 100% !important; min-height: 100%; line-height: 1.6; background: #f6f6f6; margin:0; padding: 0;">
 808 <table>
 809     <tbody>
 810     <tr>
 811         <td style="vertical-align: top;" valign="top"></td>
 812         <td width="700" style="vertical-align: top; display: block !important; max-width: 700px !important; clear: both !important; margin: 0 auto; padding: 0;" valign="top">
 813             <div style="max-width: 700px; display: block; margin: 0 auto; padding: 20px;">
 814                 <table width="100%" cellpadding="0" cellspacing="0" style="background: #fff; border: 1px solid #e9e9e9;">
 815                     <tbody>
 816                     <tr>
 817                         <td bgcolor="#eee" style="padding: 5px 20px 5px 20px; background-color: #eee;">
 818                             <div style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 20px; color: #777; font-weight: bold;">netdata notification</div>
 819                         </td>
 820                     </tr>
 821                     <tr>
 822                         <td bgcolor="${color}" style="font-size: 16px; vertical-align: top; font-weight: 400; text-align: center; margin: 0; padding: 10px; color: #ffffff; background: ${color} !important; border: 1px solid ${color}; border-top-color: ${color};" align="center" valign="top">
 823                             <h1 style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-weight: 400; margin: 0;">${host} ${status_message}</h1>
 824                         </td>
 825                     </tr>
 826                     <tr>
 827                         <td style="vertical-align: top;" valign="top">
 828                             <div style="margin: 0; padding: 20px; max-width: 700px;">
 829                                 <table width="100%" cellpadding="0" cellspacing="0" style="max-width:700px">
 830                                     <tbody>
 831                                     <tr>
 832                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding:0 0 20px;" align="left" valign="top">
 833                                             <span>${chart}</span>
 834                                             <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Chart</span>
 835                                         </td>
 836                                     </tr>
 837                                     <tr style="margin: 0; padding: 0;">
 838                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
 839                                             <span><b>${alarm}</b>${info_html}</span>
 840                                             <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Alarm</span>
 841                                         </td>
 842                                     </tr>
 843                                     <tr>
 844                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
 845                                             <span>${family}</span>
 846                                             <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Family</span>
 847                                         </td>
 848                                     </tr>
 849                                     <tr style="margin: 0; padding: 0;">
 850                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
 851                                             <span>${severity}</span>
 852                                             <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Severity</span>
 853                                         </td>
 854                                     </tr>
 855                                     <tr style="margin: 0; padding: 0;">
 856                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top"><span>${date}</span>
 857                                             <span>${raised_for_html}</span> <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Time</span>
 858                                         </td>
 859                                     </tr>
 860                                     <tr style="margin: 0; padding: 0;">
 861                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;">
 862                                             <a href="${goto_url}" style="font-size: 14px; color: #ffffff; text-decoration: none; line-height: 1.5; font-weight: bold; text-align: center; display: inline-block; text-transform: capitalize; background: #35568d; border-width: 1px; border-style: solid; border-color: #2b4c86; margin: 0; padding: 10px 15px;" target="_blank">View Netdata</a>
 863                                         </td>
 864                                     </tr>
 865                                     <tr style="text-align: center; margin: 0; padding: 0;">
 866                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 11px; vertical-align: top; margin: 0; padding: 10px 0 0 0; color: #666666;" align="center" valign="bottom">The source of this alarm is line <code>${src}</code>
 867                                         </td>
 868                                     </tr>
 869                                     <tr style="text-align: center; margin: 0; padding: 0;">
 870                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 12px; vertical-align: top; margin:0; padding: 20px 0 0 0; color: #666666; border-top: 1px solid #f0f0f0;" align="center" valign="bottom">Sent by
 871                                             <a href="https://mynetdata.io/" target="_blank">netdata</a>, the real-time performance monitoring.
 872                                         </td>
 873                                     </tr>
 874                                     </tbody>
 875                                 </table>
 876                             </div>
 877                         </td>
 878                     </tr>
 879                     </tbody>
 880                 </table>
 881             </div>
 882         </td>
 883     </tr>
 884     </tbody>
 885 </table>
 886 </body>
 887 </html>
 888 EOF
 889
 890 SENT_EMAIL=$?
 891
 892 # -----------------------------------------------------------------------------
 893 # let netdata know
 894
 895 # we did send something
 896 [ ${SENT_EMAIL} -eq 0 -o ${SENT_PUSHOVER} -eq 0 -o ${SENT_TELEGRAM} -eq 0 -o ${SENT_SLACK} -eq 0 -o ${SENT_PUSHBULLET} -eq 0 ] && exit 0
 897
 898 # we did not send anything
 899 exit 1