plugins.d/alarm-notify.sh

   1 #!/usr/bin/env bash
   2
   3 # netdata
   4 # real-time performance and health monitoring, done right!
   5 # (C) 2016 Costa Tsaousis <costa@tsaousis.gr>
   6 # GPL v3+
   7 #
   8 # Script to send alarm notifications for netdata
   9 #
  10 # Features:
  11 #  - multiple notification methods
  12 #  - multiple roles per alarm
  13 #  - multiple recipients per role
  14 #  - severity filtering per recipient
  15 #
  16 # Supported notification methods:
  17 #  - emails
  18 #  - pushover.net notifications
  19 #  - slack.com notifications
  20 #  - telegram.org notifications
  21 #
  22
  23 export PATH="${PATH}:/sbin:/usr/sbin:/usr/local/sbin"
  24 export LC_ALL=C
  25
  26 # -----------------------------------------------------------------------------
  27
  28 PROGRAM_NAME="$(basename "${0}")"
  29
  30 logdate() {
  31     date "+%Y-%m-%d %H:%M:%S"
  32 }
  33
  34 log() {
  35     local status="${1}"
  36     shift
  37
  38     echo >&2 "$(logdate): ${PROGRAM_NAME}: ${status}: ${*}"
  39
  40 }
  41
  42 warning() {
  43     log WARNING "${@}"
  44 }
  45
  46 error() {
  47     log ERROR "${@}"
  48 }
  49
  50 info() {
  51     log INFO "${@}"
  52 }
  53
  54 fatal() {
  55     log FATAL "${@}"
  56     exit 1
  57 }
  58
  59 debug=0
  60 debug() {
  61     [ $debug -eq 1 ] && log DEBUG "${@}"
  62 }
  63
  64 # -----------------------------------------------------------------------------
  65
  66 # check for BASH v4+ (required for associative arrays)
  67 [ $(( ${BASH_VERSINFO[0]} )) -lt 4 ] && \
  68     fatal "BASH version 4 or later is required (this is ${BASH_VERSION})."
  69
  70 # -----------------------------------------------------------------------------
  71 # defaults to allow running this script by hand
  72
  73 NETDATA_CONFIG_DIR="${NETDATA_CONFIG_DIR-/etc/netdata}"
  74 NETDATA_CACHE_DIR="${NETDATA_CACHE_DIR-/var/cache/netdata}"
  75 [ -z "${NETDATA_REGISTRY_URL}" ] && NETDATA_REGISTRY_URL="https://registry.my-netdata.io"
  76 [ -z "${NETDATA_HOSTNAME}" ] && NETDATA_HOSTNAME="$(hostname)"
  77 [ -z "${NETDATA_REGISTRY_HOSTNAME}" ] && NETDATA_REGISTRY_HOSTNAME="${NETDATA_HOSTNAME}"
  78
  79 # -----------------------------------------------------------------------------
  80 # parse command line parameters
  81
  82 roles="${1}"       # the roles that should be notified for this event
  83 host="${2}"        # the host generated this event
  84 unique_id="${3}"   # the unique id of this event
  85 alarm_id="${4}"    # the unique id of the alarm that generated this event
  86 event_id="${5}"    # the incremental id of the event, for this alarm id
  87 when="${6}"        # the timestamp this event occurred
  88 name="${7}"        # the name of the alarm, as given in netdata health.d entries
  89 chart="${8}"       # the name of the chart (type.id)
  90 family="${9}"      # the family of the chart
  91 status="${10}"     # the current status : REMOVED, UNITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
  92 old_status="${11}" # the previous status: REMOVED, UNITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
  93 value="${12}"      # the current value of the alarm
  94 old_value="${13}"  # the previous value of the alarm
  95 src="${14}"        # the line number and file the alarm has been configured
  96 duration="${15}"   # the duration in seconds of the previous alarm state
  97 non_clear_duration="${16}" # the total duration in seconds this is/was non-clear
  98 units="${17}"      # the units of the value
  99 info="${18}"       # a short description of the alarm
 100
 101 # -----------------------------------------------------------------------------
 102 # screen statuses we don't need to send a notification
 103
 104 # don't do anything if this is not WARNING, CRITICAL or CLEAR
 105 if [ "${status}" != "WARNING" -a "${status}" != "CRITICAL" -a "${status}" != "CLEAR" ]
 106 then
 107     info "not sending notification for ${status} on '${chart}.${name}'"
 108     exit 1
 109 fi
 110
 111 # don't do anything if this is CLEAR, but it was not WARNING or CRITICAL
 112 if [ "${old_status}" != "WARNING" -a "${old_status}" != "CRITICAL" -a "${status}" = "CLEAR" ]
 113 then
 114     info "not sending notification for ${status} on '${chart}.${name}' (last status was ${old_status})"
 115     exit 1
 116 fi
 117
 118 # -----------------------------------------------------------------------------
 119 # load configuration
 120
 121 # By default fetch images from the global public registry.
 122 # This is required by default, since all notification methods need to download
 123 # images via the Internet, and private registries might not be reachable.
 124 # This can be overwritten at the configuration file.
 125 images_base_url="https://registry.my-netdata.io"
 126
 127 # needed commands
 128 # if empty they will be searched in the system path
 129 curl=
 130 sendmail=
 131
 132 # enable / disable features
 133 SEND_SLACK="YES"
 134 SEND_PUSHOVER="YES"
 135 SEND_TELEGRAM="YES"
 136 SEND_EMAIL="YES"
 137
 138 # slack configs
 139 SLACK_WEBHOOK_URL=
 140 DEFAULT_RECIPIENT_SLACK=
 141 declare -A role_recipients_slack=()
 142
 143 # pushover configs
 144 PUSHOVER_APP_TOKEN=
 145 DEFAULT_RECIPIENT_PUSHOVER=
 146 declare -A role_recipients_pushover=()
 147
 148 # telegram configs
 149 TELEGRAM_BOT_TOKEN=
 150 DEFAULT_RECIPIENT_TELEGRAM=
 151 declare -A role_recipients_telegram=()
 152
 153 # email configs
 154 DEFAULT_RECIPIENT_EMAIL="root"
 155 declare -A role_recipients_email=()
 156
 157 # load the user configuration
 158 # this will overwrite the variables above
 159 if [ -f "${NETDATA_CONFIG_DIR}/health_alarm_notify.conf" ]
 160     then
 161     source "${NETDATA_CONFIG_DIR}/health_alarm_notify.conf"
 162 fi
 163
 164 # -----------------------------------------------------------------------------
 165 # filter a recipient based on alarm event severity
 166
 167 filter_recipient_by_criticality() {
 168     local method="${1}" x="${2}" r s
 169     shift
 170
 171     r="${x/|*/}" # the recipient
 172     s="${x/*|/}" # the severity required for notifying this recipient
 173
 174     # no severity filtering for this person
 175     [ "${r}" = "${s}" ] && return 0
 176
 177     # the severity is invalid
 178     s="${s^^}"
 179     [ "${s}" != "CRITICAL" ] && return 0
 180
 181     # the new or the old status matches the severity
 182     if [ "${s}" = "${status}" -o "${s}" = "${old_status}" ]
 183         then
 184         [ ! -d "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}" ] && \
 185             mkdir -p "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}"
 186
 187         # we need to keep track of the notifications we sent
 188         # so that the same user will receive the recovery
 189         # even if old_status does not match the required severity
 190         touch "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}"
 191         return 0
 192     fi
 193
 194     # it is a cleared alarm we have sent notification for
 195     if [ "${status}" != "WARNING" -a "${status}" != "CRITICAL" -a -f "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}" ]
 196         then
 197         rm "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}"
 198         return 0
 199     fi
 200
 201     return 1
 202 }
 203
 204 # -----------------------------------------------------------------------------
 205 # find the recipients' addresses per method
 206
 207 declare -A arr_slack=()
 208 declare -A arr_pushover=()
 209 declare -A arr_telegram=()
 210 declare -A arr_email=()
 211
 212 # netdata may call us with multiple roles, and roles may have multiple but
 213 # overlapping recipients - so, here we find the unique recipients.
 214 for x in ${roles//,/ }
 215 do
 216     # the roles 'silent' and 'disabled' mean:
 217     # don't send a notification for this role
 218     [ "${x}" = "silent" -o "${x}" = "disabled" ] && continue
 219
 220     # email
 221     a="${role_recipients_email[${x}]}"
 222     [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_EMAIL}"
 223     for r in ${a//,/ }
 224     do
 225         [ "${r}" != "disabled" ] && filter_recipient_by_criticality email "${r}" && arr_email[${r/|*/}]="1"
 226     done
 227
 228     # pushover
 229     a="${role_recipients_pushover[${x}]}"
 230     [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_PUSHOVER}"
 231     for r in ${a//,/ }
 232     do
 233         [ "${r}" != "disabled" ] && filter_recipient_by_criticality pushover "${r}" && arr_pushover[${r/|*/}]="1"
 234     done
 235
 236     # telegram
 237     a="${role_recipients_telegram[${x}]}"
 238     [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_TELEGRAM}"
 239     for r in ${a//,/ }
 240     do
 241         [ "${r}" != "disabled" ] && filter_recipient_by_criticality telegram "${r}" && arr_telegram[${r/|*/}]="1"
 242     done
 243
 244     # slack
 245     a="${role_recipients_slack[${x}]}"
 246     [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_SLACK}"
 247     for r in ${a//,/ }
 248     do
 249         [ "${r}" != "disabled" ] && filter_recipient_by_criticality slack "${r}" && arr_slack[${r/|*/}]="1"
 250     done
 251 done
 252
 253 # build the list of slack recipients (channels)
 254 to_slack="${!arr_slack[*]}"
 255 [ -z "${to_slack}" ] && SEND_SLACK="NO"
 256
 257 # build the list of pushover recipients (user tokens)
 258 to_pushover="${!arr_pushover[*]}"
 259 [ -z "${to_pushover}" ] && SEND_PUSHOVER="NO"
 260
 261 # check array of telegram recipients (chat ids)
 262 to_telegram="${!arr_telegram[*]}"
 263 [ -z "${to_telegram}" ] && SEND_TELEGRAM="NO"
 264
 265 # build the list of email recipients (email addresses)
 266 to_email=
 267 for x in "${!arr_email[@]}"
 268 do
 269     [ ! -z "${to_email}" ] && to_email="${to_email}, "
 270     to_email="${to_email}${x}"
 271 done
 272 [ -z "${to_email}" ] && SEND_EMAIL="NO"
 273
 274
 275 # -----------------------------------------------------------------------------
 276 # verify the delivery methods supported
 277
 278 # check slack
 279 [ -z "${SLACK_WEBHOOK_URL}" ] && SEND_SLACK="NO"
 280
 281 # check pushover
 282 [ -z "${PUSHOVER_APP_TOKEN}" ] && SEND_PUSHOVER="NO"
 283
 284 # check telegram
 285 [ -z "${TELEGRAM_BOT_TOKEN}" ] && SEND_TELEGRAM="NO"
 286
 287 if [ \( "${SEND_PUSHOVER}" = "YES" -o "${SEND_SLACK}" = "YES" -o "${SEND_TELEGRAM}" = "YES" \) -a -z "${curl}" ]
 288     then
 289     curl="$(which curl 2>/dev/null || command -v curl 2>/dev/null)"
 290     if [ -z "${curl}" ]
 291         then
 292         SEND_PUSHOVER="NO"
 293         SEND_TELEGRAM="NO"
 294         SEND_SLACK="NO"
 295     fi
 296 fi
 297
 298 if [ "${SEND_EMAIL}" = "YES" -a -z "${sendmail}" ]
 299     then
 300     sendmail="$(which sendmail 2>/dev/null || command -v sendmail 2>/dev/null)"
 301     [ -z "${sendmail}" ] && SEND_EMAIL="NO"
 302 fi
 303
 304 # check that we have at least a method enabled
 305 if [ "${SEND_EMAIL}" != "YES" -a "${SEND_PUSHOVER}" != "YES" -a "${SEND_TELEGRAM}" != "YES" -a "${SEND_SLACK}" != "YES" ]
 306     then
 307     fatal "All notification methods are disabled. Not sending a notification."
 308 fi
 309
 310 # -----------------------------------------------------------------------------
 311 # get the system hostname
 312
 313 [ -z "${host}" ] && host="${NETDATA_HOSTNAME}"
 314 [ -z "${host}" ] && host="${NETDATA_REGISTRY_HOSTNAME}"
 315 [ -z "${host}" ] && host="$(hostname 2>/dev/null)"
 316
 317 # -----------------------------------------------------------------------------
 318 # get the date the alarm happened
 319
 320 date="$(date --date=@${when} 2>/dev/null)"
 321 [ -z "${date}" ] && date="$(date 2>/dev/null)"
 322
 323 # -----------------------------------------------------------------------------
 324 # URL encode a string
 325
 326 urlencode() {
 327     local string="${1}" strlen encoded pos c o
 328
 329     strlen=${#string}
 330     for (( pos=0 ; pos<strlen ; pos++ ))
 331     do
 332         c=${string:$pos:1}
 333         case "$c" in
 334             [-_.~a-zA-Z0-9])
 335                 o="${c}"
 336                 ;;
 337
 338             *)
 339                 printf -v o '%%%02x' "'$c"
 340                 ;;
 341         esac
 342         encoded+="${o}"
 343     done
 344
 345     REPLY="${encoded}"
 346     echo "${REPLY}"
 347 }
 348
 349 # -----------------------------------------------------------------------------
 350 # convert a duration in seconds, to a human readable duration
 351 # using DAYS, MINUTES, SECONDS
 352
 353 duration4human() {
 354     local s="${1}" d=0 h=0 m=0 ds="day" hs="hour" ms="minute" ss="second" ret
 355     d=$(( s / 86400 ))
 356     s=$(( s - (d * 86400) ))
 357     h=$(( s / 3600 ))
 358     s=$(( s - (h * 3600) ))
 359     m=$(( s / 60 ))
 360     s=$(( s - (m * 60) ))
 361
 362     if [ ${d} -gt 0 ]
 363     then
 364         [ ${m} -ge 30 ] && h=$(( h + 1 ))
 365         [ ${d} -gt 1 ] && ds="days"
 366         [ ${h} -gt 1 ] && hs="hours"
 367         if [ ${h} -gt 0 ]
 368         then
 369             ret="${d} ${ds} and ${h} ${hs}"
 370         else
 371             ret="${d} ${ds}"
 372         fi
 373     elif [ ${h} -gt 0 ]
 374     then
 375         [ ${s} -ge 30 ] && m=$(( m + 1 ))
 376         [ ${h} -gt 1 ] && hs="hours"
 377         [ ${m} -gt 1 ] && ms="minutes"
 378         if [ ${m} -gt 0 ]
 379         then
 380             ret="${h} ${hs} and ${m} ${ms}"
 381         else
 382             ret="${h} ${hs}"
 383         fi
 384     elif [ ${m} -gt 0 ]
 385     then
 386         [ ${m} -gt 1 ] && ms="minutes"
 387         [ ${s} -gt 1 ] && ss="seconds"
 388         if [ ${s} -gt 0 ]
 389         then
 390             ret="${m} ${ms} and ${s} ${ss}"
 391         else
 392             ret="${m} ${ms}"
 393         fi
 394     else
 395         [ ${s} -gt 1 ] && ss="seconds"
 396         ret="${s} ${ss}"
 397     fi
 398
 399     REPLY="${ret}"
 400     echo "${REPLY}"
 401 }
 402
 403 # -----------------------------------------------------------------------------
 404 # email sender
 405
 406 send_email() {
 407     local ret=
 408     if [ "${SEND_EMAIL}" = "YES" ]
 409         then
 410
 411         "${sendmail}" -t
 412         ret=$?
 413
 414         if [ $ret -eq 0 ]
 415         then
 416             info "sent email notification for: ${host} ${chart}.${name} is ${status} to '${to_email}'"
 417             return 0
 418         else
 419             error "failed to send email notification for: ${host} ${chart}.${name} is ${status} to '${to_email}' with error code ${ret}."
 420             return 1
 421         fi
 422     fi
 423
 424     return 1
 425 }
 426
 427 # -----------------------------------------------------------------------------
 428 # pushover sender
 429
 430 send_pushover() {
 431     local apptoken="${1}" usertokens="${2}" when="${3}" url="${4}" status="${5}" title="${6}" message="${7}" httpcode sent=0 user priority
 432
 433     if [ "${SEND_PUSHOVER}" = "YES" -a ! -z "${apptoken}" -a ! -z "${usertokens}" -a ! -z "${title}" -a ! -z "${message}" ]
 434         then
 435
 436         # https://pushover.net/api
 437         priority=-2
 438         case "${status}" in
 439             CLEAR) priority=-1;;   # low priority: no sound or vibration
 440             WARNING) priotity=0;;  # normal priority: respect quiet hours
 441             CRITICAL) priority=1;; # high priority: bypass quiet hours
 442             *) priority=-2;;       # lowest priority: no notification at all
 443         esac
 444
 445         for user in ${usertokens}
 446         do
 447             httpcode=$(${curl} --write-out %{http_code} --silent --output /dev/null \
 448                 --form-string "token=${apptoken}" \
 449                 --form-string "user=${user}" \
 450                 --form-string "html=1" \
 451                 --form-string "title=${title}" \
 452                 --form-string "message=${message}" \
 453                 --form-string "timestamp=${when}" \
 454                 --form-string "url=${url}" \
 455                 --form-string "url_title=Open netdata dashboard to view the alarm" \
 456                 --form-string "priority=${priority}" \
 457                 https://api.pushover.net/1/messages.json)
 458
 459             if [ "${httpcode}" == "200" ]
 460             then
 461                 info "sent pushover notification for: ${host} ${chart}.${name} is ${status} to '${user}'"
 462                 sent=$((sent + 1))
 463             else
 464                 error "failed to send pushover notification for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP error code ${httpcode}."
 465             fi
 466         done
 467
 468         [ ${sent} -gt 0 ] && return 0
 469     fi
 470
 471     return 1
 472 }
 473
 474
 475 # -----------------------------------------------------------------------------
 476 # telegram sender
 477
 478 send_telegram() {
 479     local bottoken="${1}" chatids="${2}" message="${3}" httpcode sent=0 chatid disableNotification=""
 480
 481     if [ "${status}" = "CLEAR" ]; then disableNotification="--data-urlencode disable_notification=true"; fi
 482
 483     if [ "${SEND_TELEGRAM}" = "YES" -a ! -z "${bottoken}" -a ! -z "${chatids}" -a ! -z "${message}" ];
 484     then
 485         for chatid in ${chatids}
 486         do
 487             # https://core.telegram.org/bots/api#sendmessage
 488             httpcode=$(${curl} --write-out %{http_code} --silent --output /dev/null ${disableNotification} \
 489                 --data-urlencode "parse_mode=HTML" \
 490                 --data-urlencode "disable_web_page_preview=true" \
 491                 --data-urlencode "text=$message" \
 492                 "https://api.telegram.org/bot${bottoken}/sendMessage?chat_id=$chatid")
 493
 494             if [ "${httpcode}" == "200" ]
 495             then
 496                 info "sent telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}'"
 497                 sent=$((sent + 1))
 498             elif [ "${httpcode}" == "401" ]
 499             then
 500                 error "failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}': Wrong bot token."
 501             else
 502                 error "failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}' with HTTP error code ${httpcode}."
 503             fi
 504         done
 505
 506         [ ${sent} -gt 0 ] && return 0
 507     fi
 508
 509     return 1
 510 }
 511
 512 # -----------------------------------------------------------------------------
 513 # slack sender
 514
 515 send_slack() {
 516     local webhook="${1}" channels="${2}" httpcode sent=0 channel color payload
 517
 518     [ "${SEND_SLACK}" != "YES" ] && return 1
 519
 520     case "${status}" in
 521         WARNING) color="warning" ;;
 522         CRITICAL) color="danger" ;;
 523         CLEAR) color="good" ;;
 524         *) color="#777777" ;;
 525     esac
 526
 527     for channel in ${channels}
 528     do
 529         payload="$(cat <<EOF
 530         {
 531             "channel": "#${channel}",
 532             "username": "netdata on ${host}",
 533             "icon_url": "${images_base_url}/images/seo-performance-128.png",
 534             "text": "${host} ${status_message}, \`${chart}\` (_${family}_), *${alarm}*",
 535             "attachments": [
 536                 {
 537                     "fallback": "${alarm} - ${chart} (${family}) - ${info}",
 538                     "color": "${color}",
 539                     "title": "${alarm}",
 540                     "title_link": "${goto_url}",
 541                     "text": "${info}",
 542                     "fields": [
 543                         {
 544                             "title": "${chart}",
 545                             "short": true
 546                         },
 547                         {
 548                             "title": "${family}",
 549                             "short": true
 550                         }
 551                     ],
 552                     "thumb_url": "${image}",
 553                     "footer": "<${goto_url}|${host}>",
 554                     "ts": ${when}
 555                 }
 556             ]
 557         }
 558 EOF
 559         )"
 560
 561         httpcode=$(${curl} --write-out %{http_code} --silent --output /dev/null -X POST --data-urlencode "payload=${payload}" "${webhook}")
 562         if [ "${httpcode}" == "200" ]
 563         then
 564             info "sent slack notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
 565             sent=$((sent + 1))
 566         else
 567             error "failed to send slack notification for: ${host} ${chart}.${name} is ${status} to '${channel}', with HTTP error code ${httpcode}."
 568         fi
 569     done
 570
 571     [ ${sent} -gt 0 ] && return 0
 572
 573     return 1
 574 }
 575
 576
 577 # -----------------------------------------------------------------------------
 578 # prepare the content of the notification
 579
 580 # the url to send the user on click
 581 urlencode "${NETDATA_REGISTRY_HOSTNAME}" >/dev/null; url_host="${REPLY}"
 582 urlencode "${chart}" >/dev/null; url_chart="${REPLY}"
 583 urlencode "${family}" >/dev/null; url_family="${REPLY}"
 584 urlencode "${name}" >/dev/null; url_name="${REPLY}"
 585 goto_url="${NETDATA_REGISTRY_URL}/goto-host-from-alarm.html?host=${url_host}&chart=${url_chart}&family=${url_family}&alarm=${url_name}&alarm_unique_id=${unique_id}&alarm_id=${alarm_id}&alarm_event_id=${event_id}"
 586
 587 # the severity of the alarm
 588 severity="${status}"
 589
 590 # the time the alarm was raised
 591 duration4human ${duration} >/dev/null; duration_txt="${REPLY}"
 592 duration4human ${non_clear_duration} >/dev/null; non_clear_duration_txt="${REPLY}"
 593 raised_for="(was ${old_status,,} for ${duration_txt})"
 594
 595 # the key status message
 596 status_message="status unknown"
 597
 598 # the color of the alarm
 599 color="grey"
 600
 601 # the alarm value
 602 alarm="${name//_/ } = ${value} ${units}"
 603
 604 # the image of the alarm
 605 image="${images_base_url}/images/seo-performance-128.png"
 606
 607 # prepare the title based on status
 608 case "${status}" in
 609         CRITICAL)
 610         image="${images_base_url}/images/alert-128-red.png"
 611         status_message="is critical"
 612         color="#ca414b"
 613         ;;
 614
 615     WARNING)
 616         image="${images_base_url}/images/alert-128-orange.png"
 617         status_message="needs attention"
 618         color="#caca4b"
 619                 ;;
 620
 621         CLEAR)
 622         image="${images_base_url}/images/check-mark-2-128-green.png"
 623         status_message="recovered"
 624                 color="#77ca6d"
 625
 626                 # don't show the value when the status is CLEAR
 627                 # for certain alarms, this value might not have any meaning
 628                 alarm="${name//_/ } ${raised_for}"
 629                 ;;
 630 esac
 631
 632 if [ "${status}" = "CLEAR" ]
 633 then
 634     severity="Recovered from ${old_status}"
 635     if [ $non_clear_duration -gt $duration ]
 636     then
 637         raised_for="(alarm was raised for ${non_clear_duration_txt})"
 638     fi
 639
 640 elif [ "${old_status}" = "WARNING" -a "${status}" = "CRITICAL" ]
 641 then
 642     severity="Escalated to ${status}"
 643     if [ $non_clear_duration -gt $duration ]
 644     then
 645         raised_for="(alarm is raised for ${non_clear_duration_txt})"
 646     fi
 647
 648 elif [ "${old_status}" = "CRITICAL" -a "${status}" = "WARNING" ]
 649 then
 650     severity="Demoted to ${status}"
 651     if [ $non_clear_duration -gt $duration ]
 652     then
 653         raised_for="(alarm is raised for ${non_clear_duration_txt})"
 654     fi
 655
 656 else
 657     raised_for=
 658 fi
 659
 660 # prepare HTML versions of elements
 661 info_html=
 662 [ ! -z "${info}" ] && info_html=" <small><br/>${info}</small>"
 663
 664 raised_for_html=
 665 [ ! -z "${raised_for}" ] && raised_for_html="<br/><small>${raised_for}</small>"
 666
 667 # -----------------------------------------------------------------------------
 668 # send the slack notification
 669
 670 # slack aggregates posts from the same username
 671 # so we use "${host} ${status}" as the bot username, to make them diff
 672
 673 send_slack "${SLACK_WEBHOOK_URL}" "${to_slack}"
 674 SENT_SLACK=$?
 675
 676 # -----------------------------------------------------------------------------
 677 # send the pushover notification
 678
 679 send_pushover "${PUSHOVER_APP_TOKEN}" "${to_pushover}" "${when}" "${goto_url}" "${status}" "${host} ${status_message} - ${name//_/ } - ${chart}" "
 680 <font color=\"${color}\"><b>${alarm}</b></font>${info_html}<br/>&nbsp;
 681 <small><b>${chart}</b><br/>Chart<br/>&nbsp;</small>
 682 <small><b>${family}</b><br/>Family<br/>&nbsp;</small>
 683 <small><b>${severity}</b><br/>Severity<br/>&nbsp;</small>
 684 <small><b>${date}${raised_for_html}</b><br/>Time<br/>&nbsp;</small>
 685 <a href=\"${goto_url}\">View Netdata</a><br/>&nbsp;
 686 <small><small>The source of this alarm is line ${src}</small></small>
 687 "
 688
 689 SENT_PUSHOVER=$?
 690
 691 # -----------------------------------------------------------------------------
 692 # send the telegram.org message
 693
 694 # https://core.telegram.org/bots/api#formatting-options
 695 send_telegram "${TELEGRAM_BOT_TOKEN}" "${to_telegram}" "${host} ${status_message} - <b>${name//_/ }</b>
 696 ${chart} (${family})
 697 <a href=\"${goto_url}\">${alarm}</a>
 698 <i>${info}</i>"
 699
 700 SENT_TELEGRAM=$?
 701
 702 # -----------------------------------------------------------------------------
 703 # send the email
 704
 705 send_email <<EOF
 706 To: ${to_email}
 707 Subject: ${host} ${status_message} - ${name//_/ } - ${chart}
 708 Content-Type: text/html
 709
 710 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
 711 <html xmlns="http://www.w3.org/1999/xhtml" style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0; padding: 0;">
 712 <body style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 14px; width: 100% !important; min-height: 100%; line-height: 1.6; background: #f6f6f6; margin:0; padding: 0;">
 713 <table>
 714     <tbody>
 715     <tr>
 716         <td style="vertical-align: top;" valign="top"></td>
 717         <td width="700" style="vertical-align: top; display: block !important; max-width: 700px !important; clear: both !important; margin: 0 auto; padding: 0;" valign="top">
 718             <div style="max-width: 700px; display: block; margin: 0 auto; padding: 20px;">
 719                 <table width="100%" cellpadding="0" cellspacing="0" style="background: #fff; border: 1px solid #e9e9e9;">
 720                     <tbody>
 721                     <tr>
 722                         <td bgcolor="#eee" style="padding: 5px 20px 5px 20px; background-color: #eee;">
 723                             <div style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 20px; color: #777; font-weight: bold;">netdata notification</div>
 724                         </td>
 725                     </tr>
 726                     <tr>
 727                         <td bgcolor="${color}" style="font-size: 16px; vertical-align: top; font-weight: 400; text-align: center; margin: 0; padding: 10px; color: #ffffff; background: ${color} !important; border: 1px solid ${color}; border-top-color: ${color};" align="center" valign="top">
 728                             <h1 style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-weight: 400; margin: 0;">${host} ${status_message}</h1>
 729                         </td>
 730                     </tr>
 731                     <tr>
 732                         <td style="vertical-align: top;" valign="top">
 733                             <div style="margin: 0; padding: 20px; max-width: 700px;">
 734                                 <table width="100%" cellpadding="0" cellspacing="0" style="max-width:700px">
 735                                     <tbody>
 736                                     <tr>
 737                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding:0 0 20px;" align="left" valign="top">
 738                                             <span>${chart}</span>
 739                                             <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Chart</span>
 740                                         </td>
 741                                     </tr>
 742                                     <tr style="margin: 0; padding: 0;">
 743                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
 744                                             <span><b>${alarm}</b>${info_html}</span>
 745                                             <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Alarm</span>
 746                                         </td>
 747                                     </tr>
 748                                     <tr>
 749                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
 750                                             <span>${family}</span>
 751                                             <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Family</span>
 752                                         </td>
 753                                     </tr>
 754                                     <tr style="margin: 0; padding: 0;">
 755                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
 756                                             <span>${severity}</span>
 757                                             <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Severity</span>
 758                                         </td>
 759                                     </tr>
 760                                     <tr style="margin: 0; padding: 0;">
 761                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top"><span>${date}</span>
 762                                             <span>${raised_for_html}</span> <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Time</span>
 763                                         </td>
 764                                     </tr>
 765                                     <tr style="margin: 0; padding: 0;">
 766                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;">
 767                                             <a href="${goto_url}" style="font-size: 14px; color: #ffffff; text-decoration: none; line-height: 1.5; font-weight: bold; text-align: center; display: inline-block; text-transform: capitalize; background: #35568d; border-width: 1px; border-style: solid; border-color: #2b4c86; margin: 0; padding: 10px 15px;" target="_blank">View Netdata</a>
 768                                         </td>
 769                                     </tr>
 770                                     <tr style="text-align: center; margin: 0; padding: 0;">
 771                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 11px; vertical-align: top; margin: 0; padding: 10px 0 0 0; color: #666666;" align="center" valign="bottom">The source of this alarm is line <code>${src}</code>
 772                                         </td>
 773                                     </tr>
 774                                     <tr style="text-align: center; margin: 0; padding: 0;">
 775                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 12px; vertical-align: top; margin:0; padding: 20px 0 0 0; color: #666666; border-top: 1px solid #f0f0f0;" align="center" valign="bottom">Sent by
 776                                             <a href="https://mynetdata.io/" target="_blank">netdata</a>, the real-time performance monitoring.
 777                                         </td>
 778                                     </tr>
 779                                     </tbody>
 780                                 </table>
 781                             </div>
 782                         </td>
 783                     </tr>
 784                     </tbody>
 785                 </table>
 786             </div>
 787         </td>
 788     </tr>
 789     </tbody>
 790 </table>
 791 </body>
 792 </html>
 793 EOF
 794
 795 SENT_EMAIL=$?
 796
 797 # -----------------------------------------------------------------------------
 798 # let netdata know
 799
 800 # we did send something
 801 [ ${SENT_EMAIL} -eq 0 -o ${SENT_PUSHOVER} -eq 0 -o ${SENT_TELEGRAM} -eq 0 -o ${SENT_SLACK} -eq 0 ] && exit 0
 802
 803 # we did not send anything
 804 exit 1