plugins.d/alarm-notify.sh

   1 #!/usr/bin/env bash
   2
   3 # netdata
   4 # real-time performance and health monitoring, done right!
   5 # (C) 2016 Costa Tsaousis <costa@tsaousis.gr>
   6 # GPL v3+
   7 #
   8 # Script the send alarm notifications for netdata
   9 #
  10 # Features:
  11 #  - multiple notification methods
  12 #  - multiple roles per alarm
  13 #  - multiple recipients per role
  14 #  - severity filtering per recipient
  15 #
  16 # Supported notification methods:
  17 #  - emails
  18 #  - pushover.net notifications
  19 #  - slack.com notifications
  20 #  - telegram.org notifications
  21 #
  22
  23 me="${0}"
  24
  25 # check for BASH v4+ (required for associative arrays)
  26 [ $(( ${BASH_VERSINFO[0]} )) -lt 4 ] && \
  27     echo >&2 "${me}: BASH version 4 or later is required (this is ${BASH_VERSION})." && \
  28     exit 1
  29
  30 # defaults to allow running this script by hand
  31 NETDATA_CONFIG_DIR="${NETDATA_CONFIG_DIR-/etc/netdata}"
  32 NETDATA_CACHE_DIR="${NETDATA_CACHE_DIR-/var/cache/netdata}"
  33 [ -z "${NETDATA_REGISTRY_URL}" ] && NETDATA_REGISTRY_URL="https://registry.my-netdata.io"
  34 [ -z "${NETDATA_HOSTNAME}" ] && NETDATA_HOSTNAME="$(hostname)"
  35 [ -z "${NETDATA_REGISTRY_HOSTNAME}" ] && NETDATA_REGISTRY_HOSTNAME="${NETDATA_HOSTNAME}"
  36
  37 # -----------------------------------------------------------------------------
  38 # parse command line parameters
  39
  40 roles="${1}"       # the roles that should be notified for this event
  41 host="${2}"        # the host generated this event
  42 unique_id="${3}"   # the unique id of this event
  43 alarm_id="${4}"    # the unique id of the alarm that generated this event
  44 event_id="${5}"    # the incremental id of the event, for this alarm id
  45 when="${6}"        # the timestamp this event occurred
  46 name="${7}"        # the name of the alarm, as given in netdata health.d entries
  47 chart="${8}"       # the name of the chart (type.id)
  48 family="${9}"      # the family of the chart
  49 status="${10}"     # the current status : REMOVED, UNITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
  50 old_status="${11}" # the previous status: REMOVED, UNITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
  51 value="${12}"      # the current value of the alarm
  52 old_value="${13}"  # the previous value of the alarm
  53 src="${14}"        # the line number and file the alarm has been configured
  54 duration="${15}"   # the duration in seconds of the previous alarm state
  55 non_clear_duration="${16}" # the total duration in seconds this is/was non-clear
  56 units="${17}"      # the units of the value
  57 info="${18}"       # a short description of the alarm
  58
  59 # -----------------------------------------------------------------------------
  60 # screen statuses we don't need to send a notification
  61
  62 # don't do anything if this is not WARNING, CRITICAL or CLEAR
  63 if [ "${status}" != "WARNING" -a "${status}" != "CRITICAL" -a "${status}" != "CLEAR" ]
  64 then
  65     echo >&2 "${me}: not sending notification for ${status} on '${chart}.${name}'"
  66     exit 1
  67 fi
  68
  69 # don't do anything if this is CLEAR, but it was not WARNING or CRITICAL
  70 if [ "${old_status}" != "WARNING" -a "${old_status}" != "CRITICAL" -a "${status}" = "CLEAR" ]
  71 then
  72     echo >&2 "${me}: not sending notification for ${status} on '${chart}.${name}' (last status was ${old_status})"
  73     exit 1
  74 fi
  75
  76 # -----------------------------------------------------------------------------
  77 # load configuration
  78
  79 # By default fetch images from the global public registry.
  80 # This is required by default, since all notification methods need to download
  81 # images via the Internet, and private registries might not be reachable.
  82 # This can be overwritten at the configuration file.
  83 images_base_url="https://registry.my-netdata.io"
  84
  85 # needed commands
  86 # if empty they will be searched in the system path
  87 curl=
  88 sendmail=
  89
  90 # enable / disable features
  91 SEND_SLACK="YES"
  92 SEND_PUSHOVER="YES"
  93 SEND_TELEGRAM="YES"
  94 SEND_EMAIL="YES"
  95
  96 # slack configs
  97 SLACK_WEBHOOK_URL=
  98 DEFAULT_RECIPIENT_SLACK=
  99 declare -A role_recipients_slack=()
 100
 101 # pushover configs
 102 PUSHOVER_APP_TOKEN=
 103 DEFAULT_RECIPIENT_PUSHOVER=
 104 declare -A role_recipients_pushover=()
 105
 106 # telegram configs
 107 TELEGRAM_BOT_TOKEN=
 108 DEFAULT_RECIPIENT_TELEGRAM=
 109 declare -A role_recipients_telegram=()
 110
 111 # email configs
 112 DEFAULT_RECIPIENT_EMAIL="root"
 113 declare -A role_recipients_email=()
 114
 115 # load the user configuration
 116 # this will overwrite the variables above
 117 if [ -f "${NETDATA_CONFIG_DIR}/health_alarm_notify.conf" ]
 118     then
 119     source "${NETDATA_CONFIG_DIR}/health_alarm_notify.conf"
 120 fi
 121
 122 # -----------------------------------------------------------------------------
 123 # filter a recipient based on alarm event severity
 124
 125 filter_recipient_by_criticality() {
 126     local method="${1}" x="${2}" r s
 127     shift
 128
 129     r="${x/|*/}" # the recipient
 130     s="${x/*|/}" # the severity required for notifying this recipient
 131
 132     # no severity filtering for this person
 133     [ "${r}" = "${s}" ] && return 0
 134
 135     # the severity is invalid
 136     s="${s^^}"
 137     [ "${s}" != "CRITICAL" ] && return 0
 138
 139     # the new or the old status matches the severity
 140     if [ "${s}" = "${status}" -o "${s}" = "${old_status}" ]
 141         then
 142         [ ! -d "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}" ] && \
 143             mkdir -p "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}"
 144
 145         # we need to keep track of the notifications we sent
 146         # so that the same user will receive the recovery
 147         # even if old_status does not match the required severity
 148         touch "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}"
 149         return 0
 150     fi
 151
 152     # it is a cleared alarm we have sent notification for
 153     if [ "${status}" != "WARNING" -a "${status}" != "CRITICAL" -a -f "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}" ]
 154         then
 155         rm "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}"
 156         return 0
 157     fi
 158
 159     return 1
 160 }
 161
 162 # -----------------------------------------------------------------------------
 163 # find the recipients' addresses per method
 164
 165 declare -A arr_slack=()
 166 declare -A arr_pushover=()
 167 declare -A arr_telegram=()
 168 declare -A arr_email=()
 169
 170 # netdata may call us with multiple roles, and roles may have multiple but
 171 # overlapping recipients - so, here we find the unique recipients.
 172 for x in ${roles//,/ }
 173 do
 174     # the roles 'silent' and 'disabled' mean:
 175     # don't send a notification for this role
 176     [ "${x}" = "silent" -o "${x}" = "disabled" ] && continue
 177
 178     # email
 179     a="${role_recipients_email[${x}]}"
 180     [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_EMAIL}"
 181     for r in ${a//,/ }
 182     do
 183         [ "${r}" != "disabled" ] && filter_recipient_by_criticality email "${r}" && arr_email[${r/|*/}]="1"
 184     done
 185
 186     # pushover
 187     a="${role_recipients_pushover[${x}]}"
 188     [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_PUSHOVER}"
 189     for r in ${a//,/ }
 190     do
 191         [ "${r}" != "disabled" ] && filter_recipient_by_criticality pushover "${r}" && arr_pushover[${r/|*/}]="1"
 192     done
 193
 194     # telegram
 195     a="${role_recipients_telegram[${x}]}"
 196     [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_TELEGRAM}"
 197     for r in ${a//,/ }
 198     do
 199         [ "${r}" != "disabled" ] && filter_recipient_by_criticality telegram "${r}" && arr_telegram[${r/|*/}]="1"
 200     done
 201
 202     # slack
 203     a="${role_recipients_slack[${x}]}"
 204     [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_SLACK}"
 205     for r in ${a//,/ }
 206     do
 207         [ "${r}" != "disabled" ] && filter_recipient_by_criticality slack "${r}" && arr_slack[${r/|*/}]="1"
 208     done
 209 done
 210
 211 # build the list of slack recipients (channels)
 212 to_slack="${!arr_slack[*]}"
 213 [ -z "${to_slack}" ] && SEND_SLACK="NO"
 214
 215 # build the list of pushover recipients (user tokens)
 216 to_pushover="${!arr_pushover[*]}"
 217 [ -z "${to_pushover}" ] && SEND_PUSHOVER="NO"
 218
 219 # check array of telegram recipients (chat ids)
 220 to_telegram="${!arr_telegram[*]}"
 221 [ -z "${to_telegram}" ] && SEND_TELEGRAM="NO"
 222
 223 # build the list of email recipients (email addresses)
 224 to_email=
 225 for x in "${!arr_email[@]}"
 226 do
 227     [ ! -z "${to_email}" ] && to_email="${to_email}, "
 228     to_email="${to_email}${x}"
 229 done
 230 [ -z "${to_email}" ] && SEND_EMAIL="NO"
 231
 232
 233 # -----------------------------------------------------------------------------
 234 # verify the delivery methods supported
 235
 236 # check slack
 237 [ -z "${SLACK_WEBHOOK_URL}" ] && SEND_SLACK="NO"
 238
 239 # check pushover
 240 [ -z "${PUSHOVER_APP_TOKEN}" ] && SEND_PUSHOVER="NO"
 241
 242 # check telegram
 243 [ -z "${TELEGRAM_BOT_TOKEN}" ] && SEND_TELEGRAM="NO"
 244
 245 if [ \( "${SEND_PUSHOVER}" = "YES" -o "${SEND_SLACK}" = "YES" -o "${SEND_TELEGRAM}" = "YES" \) -a -z "${curl}" ]
 246     then
 247     curl="$(which curl 2>/dev/null || command -v curl 2>/dev/null)"
 248     if [ -z "${curl}" ]
 249         then
 250         SEND_PUSHOVER="NO"
 251         SEND_TELEGRAM="NO"
 252         SEND_SLACK="NO"
 253     fi
 254 fi
 255
 256 if [ "${SEND_EMAIL}" = "YES" -a -z "${sendmail}" ]
 257     then
 258     sendmail="$(which sendmail 2>/dev/null || command -v sendmail 2>/dev/null)"
 259     [ -z "${sendmail}" ] && SEND_EMAIL="NO"
 260 fi
 261
 262 # check that we have at least a method enabled
 263 if [ "${SEND_EMAIL}" != "YES" -a "${SEND_PUSHOVER}" != "YES" -a "${SEND_TELEGRAM}" != "YES" -a "${SEND_SLACK}" != "YES" ]
 264     then
 265     echo >&2 "All notification methods are disabled. Not sending a notification."
 266     exit 1
 267 fi
 268
 269 # -----------------------------------------------------------------------------
 270 # get the system hostname
 271
 272 [ -z "${host}" ] && host="${NETDATA_HOSTNAME}"
 273 [ -z "${host}" ] && host="${NETDATA_REGISTRY_HOSTNAME}"
 274 [ -z "${host}" ] && host="$(hostname 2>/dev/null)"
 275
 276 # -----------------------------------------------------------------------------
 277 # get the date the alarm happened
 278
 279 date="$(date --date=@${when} 2>/dev/null)"
 280 [ -z "${date}" ] && date="$(date 2>/dev/null)"
 281
 282 # -----------------------------------------------------------------------------
 283 # URL encode a string
 284
 285 urlencode() {
 286     local string="${1}" strlen encoded pos c o
 287
 288     strlen=${#string}
 289     for (( pos=0 ; pos<strlen ; pos++ ))
 290     do
 291         c=${string:$pos:1}
 292         case "$c" in
 293             [-_.~a-zA-Z0-9])
 294                 o="${c}"
 295                 ;;
 296
 297             *)
 298                 printf -v o '%%%02x' "'$c"
 299                 ;;
 300         esac
 301         encoded+="${o}"
 302     done
 303
 304     REPLY="${encoded}"
 305     echo "${REPLY}"
 306 }
 307
 308 # -----------------------------------------------------------------------------
 309 # convert a duration in seconds, to a human readable duration
 310 # using DAYS, MINUTES, SECONDS
 311
 312 duration4human() {
 313     local s="${1}" d=0 h=0 m=0 ds="day" hs="hour" ms="minute" ss="second" ret
 314     d=$(( s / 86400 ))
 315     s=$(( s - (d * 86400) ))
 316     h=$(( s / 3600 ))
 317     s=$(( s - (h * 3600) ))
 318     m=$(( s / 60 ))
 319     s=$(( s - (m * 60) ))
 320
 321     if [ ${d} -gt 0 ]
 322     then
 323         [ ${m} -ge 30 ] && h=$(( h + 1 ))
 324         [ ${d} -gt 1 ] && ds="days"
 325         [ ${h} -gt 1 ] && hs="hours"
 326         if [ ${h} -gt 0 ]
 327         then
 328             ret="${d} ${ds} and ${h} ${hs}"
 329         else
 330             ret="${d} ${ds}"
 331         fi
 332     elif [ ${h} -gt 0 ]
 333     then
 334         [ ${s} -ge 30 ] && m=$(( m + 1 ))
 335         [ ${h} -gt 1 ] && hs="hours"
 336         [ ${m} -gt 1 ] && ms="minutes"
 337         if [ ${m} -gt 0 ]
 338         then
 339             ret="${h} ${hs} and ${m} ${ms}"
 340         else
 341             ret="${h} ${hs}"
 342         fi
 343     elif [ ${m} -gt 0 ]
 344     then
 345         [ ${m} -gt 1 ] && ms="minutes"
 346         [ ${s} -gt 1 ] && ss="seconds"
 347         if [ ${s} -gt 0 ]
 348         then
 349             ret="${m} ${ms} and ${s} ${ss}"
 350         else
 351             ret="${m} ${ms}"
 352         fi
 353     else
 354         [ ${s} -gt 1 ] && ss="seconds"
 355         ret="${s} ${ss}"
 356     fi
 357
 358     REPLY="${ret}"
 359     echo "${REPLY}"
 360 }
 361
 362 # -----------------------------------------------------------------------------
 363 # email sender
 364
 365 send_email() {
 366     local ret=
 367     if [ "${SEND_EMAIL}" = "YES" ]
 368         then
 369
 370         "${sendmail}" -t
 371         ret=$?
 372
 373         if [ $ret -eq 0 ]
 374         then
 375             echo >&2 "${me}: Sent email notification for: ${host} ${chart}.${name} is ${status} to '${to_email}'"
 376             return 0
 377         else
 378             echo >&2 "${me}: Failed to send email notification for: ${host} ${chart}.${name} is ${status} to '${to_email}' with error code ${ret}."
 379             return 1
 380         fi
 381     fi
 382
 383     return 1
 384 }
 385
 386 # -----------------------------------------------------------------------------
 387 # pushover sender
 388
 389 send_pushover() {
 390     local apptoken="${1}" usertokens="${2}" when="${3}" url="${4}" status="${5}" title="${6}" message="${7}" httpcode sent=0 user priority
 391
 392     if [ "${SEND_PUSHOVER}" = "YES" -a ! -z "${apptoken}" -a ! -z "${usertokens}" -a ! -z "${title}" -a ! -z "${message}" ]
 393         then
 394
 395         # https://pushover.net/api
 396         priority=-2
 397         case "${status}" in
 398             CLEAR) priority=-1;;   # low priority: no sound or vibration
 399             WARNING) priotity=0;;  # normal priority: respect quiet hours
 400             CRITICAL) priority=1;; # high priority: bypass quiet hours
 401             *) priority=-2;;       # lowest priority: no notification at all
 402         esac
 403
 404         for user in ${usertokens}
 405         do
 406             httpcode=$(${curl} --write-out %{http_code} --silent --output /dev/null \
 407                 --form-string "token=${apptoken}" \
 408                 --form-string "user=${user}" \
 409                 --form-string "html=1" \
 410                 --form-string "title=${title}" \
 411                 --form-string "message=${message}" \
 412                 --form-string "timestamp=${when}" \
 413                 --form-string "url=${url}" \
 414                 --form-string "url_title=Open netdata dashboard to view the alarm" \
 415                 --form-string "priority=${priority}" \
 416                 https://api.pushover.net/1/messages.json)
 417
 418             if [ "${httpcode}" == "200" ]
 419             then
 420                 echo >&2 "${me}: Sent pushover notification for: ${host} ${chart}.${name} is ${status} to '${user}'"
 421                 sent=$((sent + 1))
 422             else
 423                 echo >&2 "${me}: Failed to send pushover notification for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP error code ${httpcode}."
 424             fi
 425         done
 426
 427         [ ${sent} -gt 0 ] && return 0
 428     fi
 429
 430     return 1
 431 }
 432
 433
 434 # -----------------------------------------------------------------------------
 435 # telegram sender
 436
 437 send_telegram() {
 438     local bottoken="${1}" chatids="${2}" message="${3}" httpcode sent=0 chatid disableNotification=""
 439
 440     if [ "${status}" = "CLEAR" ]; then disableNotification="--data-urlencode disable_notification=true"; fi
 441
 442     if [ "${SEND_TELEGRAM}" = "YES" -a ! -z "${bottoken}" -a ! -z "${chatids}" -a ! -z "${message}" ];
 443     then
 444         for chatid in ${chatids}
 445         do
 446             # https://core.telegram.org/bots/api#sendmessage
 447             httpcode=$(${curl} --write-out %{http_code} --silent --output /dev/null ${disableNotification} \
 448                 --data-urlencode "parse_mode=HTML" \
 449                 --data-urlencode "disable_web_page_preview=true" \
 450                 --data-urlencode "text=$message" \
 451                 "https://api.telegram.org/bot${bottoken}/sendMessage?chat_id=$chatid")
 452
 453             if [ "${httpcode}" == "200" ]
 454             then
 455                 echo >&2 "${me}: Sent telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}'"
 456                 sent=$((sent + 1))
 457             elif [ "${httpcode}" == "401" ]
 458             then
 459                 echo >&2 "${me}: Failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}': Wrong bot token."
 460             else
 461                 echo >&2 "${me}: Failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}' with HTTP error code ${httpcode}."
 462             fi
 463         done
 464
 465         [ ${sent} -gt 0 ] && return 0
 466     fi
 467
 468     return 1
 469 }
 470
 471 # -----------------------------------------------------------------------------
 472 # slack sender
 473
 474 send_slack() {
 475     local webhook="${1}" channels="${2}" httpcode sent=0 channel color payload
 476
 477     [ "${SEND_SLACK}" != "YES" ] && return 1
 478
 479     case "${status}" in
 480         WARNING) color="warning" ;;
 481         CRITICAL) color="danger" ;;
 482         CLEAR) color="good" ;;
 483         *) color="#777777" ;;
 484     esac
 485
 486     for channel in ${channels}
 487     do
 488         payload="$(cat <<EOF
 489         {
 490             "channel": "#${channel}",
 491             "username": "netdata on ${host}",
 492             "icon_url": "${images_base_url}/images/seo-performance-128.png",
 493             "text": "${host} ${status_message}, \`${chart}\` (_${family}_), *${alarm}*",
 494             "attachments": [
 495                 {
 496                     "fallback": "${alarm} - ${chart} (${family}) - ${info}",
 497                     "color": "${color}",
 498                     "title": "${alarm}",
 499                     "title_link": "${goto_url}",
 500                     "text": "${info}",
 501                     "fields": [
 502                         {
 503                             "title": "${chart}",
 504                             "short": true
 505                         },
 506                         {
 507                             "title": "${family}",
 508                             "short": true
 509                         }
 510                     ],
 511                     "thumb_url": "${image}",
 512                     "footer": "<${goto_url}|${host}>",
 513                     "ts": ${when}
 514                 }
 515             ]
 516         }
 517 EOF
 518         )"
 519
 520         httpcode=$(${curl} --write-out %{http_code} --silent --output /dev/null -X POST --data-urlencode "payload=${payload}" "${webhook}")
 521         if [ "${httpcode}" == "200" ]
 522         then
 523             echo >&2 "${me}: Sent slack notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
 524             sent=$((sent + 1))
 525         else
 526             echo >&2 "${me}: Failed to send slack notification for: ${host} ${chart}.${name} is ${status} to '${channel}', with HTTP error code ${httpcode}."
 527         fi
 528     done
 529
 530     [ ${sent} -gt 0 ] && return 0
 531
 532     return 1
 533 }
 534
 535
 536 # -----------------------------------------------------------------------------
 537 # prepare the content of the notification
 538
 539 # the url to send the user on click
 540 urlencode "${NETDATA_REGISTRY_HOSTNAME}" >/dev/null; url_host="${REPLY}"
 541 urlencode "${chart}" >/dev/null; url_chart="${REPLY}"
 542 urlencode "${family}" >/dev/null; url_family="${REPLY}"
 543 urlencode "${name}" >/dev/null; url_name="${REPLY}"
 544 goto_url="${NETDATA_REGISTRY_URL}/goto-host-from-alarm.html?host=${url_host}&chart=${url_chart}&family=${url_family}&alarm=${url_name}&alarm_unique_id=${unique_id}&alarm_id=${alarm_id}&alarm_event_id=${event_id}"
 545
 546 # the severity of the alarm
 547 severity="${status}"
 548
 549 # the time the alarm was raised
 550 duration4human ${duration} >/dev/null; duration_txt="${REPLY}"
 551 duration4human ${non_clear_duration} >/dev/null; non_clear_duration_txt="${REPLY}"
 552 raised_for="(was ${old_status,,} for ${duration_txt})"
 553
 554 # the key status message
 555 status_message="status unknown"
 556
 557 # the color of the alarm
 558 color="grey"
 559
 560 # the alarm value
 561 alarm="${name//_/ } = ${value} ${units}"
 562
 563 # the image of the alarm
 564 image="${images_base_url}/images/seo-performance-128.png"
 565
 566 # prepare the title based on status
 567 case "${status}" in
 568         CRITICAL)
 569         image="${images_base_url}/images/alert-128-red.png"
 570         status_message="is critical"
 571         color="#ca414b"
 572         ;;
 573
 574     WARNING)
 575         image="${images_base_url}/images/alert-128-orange.png"
 576         status_message="needs attention"
 577         color="#caca4b"
 578                 ;;
 579
 580         CLEAR)
 581         image="${images_base_url}/images/check-mark-2-128-green.png"
 582         status_message="recovered"
 583                 color="#77ca6d"
 584
 585                 # don't show the value when the status is CLEAR
 586                 # for certain alarms, this value might not have any meaning
 587                 alarm="${name//_/ } ${raised_for}"
 588                 ;;
 589 esac
 590
 591 if [ "${status}" = "CLEAR" ]
 592 then
 593     severity="Recovered from ${old_status}"
 594     if [ $non_clear_duration -gt $duration ]
 595     then
 596         raised_for="(alarm was raised for ${non_clear_duration_txt})"
 597     fi
 598
 599 elif [ "${old_status}" = "WARNING" -a "${status}" = "CRITICAL" ]
 600 then
 601     severity="Escalated to ${status}"
 602     if [ $non_clear_duration -gt $duration ]
 603     then
 604         raised_for="(alarm is raised for ${non_clear_duration_txt})"
 605     fi
 606
 607 elif [ "${old_status}" = "CRITICAL" -a "${status}" = "WARNING" ]
 608 then
 609     severity="Demoted to ${status}"
 610     if [ $non_clear_duration -gt $duration ]
 611     then
 612         raised_for="(alarm is raised for ${non_clear_duration_txt})"
 613     fi
 614
 615 else
 616     raised_for=
 617 fi
 618
 619 # prepare HTML versions of elements
 620 info_html=
 621 [ ! -z "${info}" ] && info_html=" <small><br/>${info}</small>"
 622
 623 raised_for_html=
 624 [ ! -z "${raised_for}" ] && raised_for_html="<br/><small>${raised_for}</small>"
 625
 626 # -----------------------------------------------------------------------------
 627 # send the slack notification
 628
 629 # slack aggregates posts from the same username
 630 # so we use "${host} ${status}" as the bot username, to make them diff
 631
 632 send_slack "${SLACK_WEBHOOK_URL}" "${to_slack}"
 633 SENT_SLACK=$?
 634
 635 # -----------------------------------------------------------------------------
 636 # send the pushover notification
 637
 638 send_pushover "${PUSHOVER_APP_TOKEN}" "${to_pushover}" "${when}" "${goto_url}" "${status}" "${host} ${status_message} - ${name//_/ } - ${chart}" "
 639 <font color=\"${color}\"><b>${alarm}</b></font>${info_html}<br/>&nbsp;
 640 <small><b>${chart}</b><br/>Chart<br/>&nbsp;</small>
 641 <small><b>${family}</b><br/>Family<br/>&nbsp;</small>
 642 <small><b>${severity}</b><br/>Severity<br/>&nbsp;</small>
 643 <small><b>${date}${raised_for_html}</b><br/>Time<br/>&nbsp;</small>
 644 <a href=\"${goto_url}\">View Netdata</a><br/>&nbsp;
 645 <small><small>The source of this alarm is line ${src}</small></small>
 646 "
 647
 648 SENT_PUSHOVER=$?
 649
 650 # -----------------------------------------------------------------------------
 651 # send the telegram.org message
 652
 653 # https://core.telegram.org/bots/api#formatting-options
 654 send_telegram "${TELEGRAM_BOT_TOKEN}" "${to_telegram}" "${host} ${status_message} - <b>${name//_/ }</b>
 655 ${chart} (${family})
 656 <a href=\"${goto_url}\">${alarm}</a>
 657 <i>${info}</i>"
 658
 659 SENT_TELEGRAM=$?
 660
 661 # -----------------------------------------------------------------------------
 662 # send the email
 663
 664 send_email <<EOF
 665 To: ${to_email}
 666 Subject: ${host} ${status_message} - ${name//_/ } - ${chart}
 667 Content-Type: text/html
 668
 669 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
 670 <html xmlns="http://www.w3.org/1999/xhtml" style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0; padding: 0;">
 671 <body style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 14px; width: 100% !important; min-height: 100%; line-height: 1.6; background: #f6f6f6; margin:0; padding: 0;">
 672 <table>
 673     <tbody>
 674     <tr>
 675         <td style="vertical-align: top;" valign="top"></td>
 676         <td width="700" style="vertical-align: top; display: block !important; max-width: 700px !important; clear: both !important; margin: 0 auto; padding: 0;" valign="top">
 677             <div style="max-width: 700px; display: block; margin: 0 auto; padding: 20px;">
 678                 <table width="100%" cellpadding="0" cellspacing="0" style="background: #fff; border: 1px solid #e9e9e9;">
 679                     <tbody>
 680                     <tr>
 681                         <td bgcolor="#eee" style="padding: 5px 20px 5px 20px; background-color: #eee;">
 682                             <div style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 20px; color: #777; font-weight: bold;">netdata notification</div>
 683                         </td>
 684                     </tr>
 685                     <tr>
 686                         <td bgcolor="${color}" style="font-size: 16px; vertical-align: top; font-weight: 400; text-align: center; margin: 0; padding: 10px; color: #ffffff; background: ${color} !important; border: 1px solid ${color}; border-top-color: ${color};" align="center" valign="top">
 687                             <h1 style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-weight: 400; margin: 0;">${host} ${status_message}</h1>
 688                         </td>
 689                     </tr>
 690                     <tr>
 691                         <td style="vertical-align: top;" valign="top">
 692                             <div style="margin: 0; padding: 20px; max-width: 700px;">
 693                                 <table width="100%" cellpadding="0" cellspacing="0" style="max-width:700px">
 694                                     <tbody>
 695                                     <tr>
 696                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding:0 0 20px;" align="left" valign="top">
 697                                             <span>${chart}</span>
 698                                             <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Chart</span>
 699                                         </td>
 700                                     </tr>
 701                                     <tr style="margin: 0; padding: 0;">
 702                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
 703                                             <span><b>${alarm}</b>${info_html}</span>
 704                                             <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Alarm</span>
 705                                         </td>
 706                                     </tr>
 707                                     <tr>
 708                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
 709                                             <span>${family}</span>
 710                                             <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Family</span>
 711                                         </td>
 712                                     </tr>
 713                                     <tr style="margin: 0; padding: 0;">
 714                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
 715                                             <span>${severity}</span>
 716                                             <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Severity</span>
 717                                         </td>
 718                                     </tr>
 719                                     <tr style="margin: 0; padding: 0;">
 720                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top"><span>${date}</span>
 721                                             <span>${raised_for_html}</span> <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Time</span>
 722                                         </td>
 723                                     </tr>
 724                                     <tr style="margin: 0; padding: 0;">
 725                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;">
 726                                             <a href="${goto_url}" style="font-size: 14px; color: #ffffff; text-decoration: none; line-height: 1.5; font-weight: bold; text-align: center; display: inline-block; text-transform: capitalize; background: #35568d; border-width: 1px; border-style: solid; border-color: #2b4c86; margin: 0; padding: 10px 15px;" target="_blank">View Netdata</a>
 727                                         </td>
 728                                     </tr>
 729                                     <tr style="text-align: center; margin: 0; padding: 0;">
 730                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 11px; vertical-align: top; margin: 0; padding: 10px 0 0 0; color: #666666;" align="center" valign="bottom">The source of this alarm is line <code>${src}</code>
 731                                         </td>
 732                                     </tr>
 733                                     <tr style="text-align: center; margin: 0; padding: 0;">
 734                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 12px; vertical-align: top; margin:0; padding: 20px 0 0 0; color: #666666; border-top: 1px solid #f0f0f0;" align="center" valign="bottom">Sent by
 735                                             <a href="https://mynetdata.io/" target="_blank">netdata</a>, the real-time performance monitoring.
 736                                         </td>
 737                                     </tr>
 738                                     </tbody>
 739                                 </table>
 740                             </div>
 741                         </td>
 742                     </tr>
 743                     </tbody>
 744                 </table>
 745             </div>
 746         </td>
 747     </tr>
 748     </tbody>
 749 </table>
 750 </body>
 751 </html>
 752 EOF
 753
 754 SENT_EMAIL=$?
 755
 756 # -----------------------------------------------------------------------------
 757 # let netdata know
 758
 759 # we did send something
 760 [ ${SENT_EMAIL} -eq 0 -o ${SENT_PUSHOVER} -eq 0 -o ${SENT_TELEGRAM} -eq 0 -o ${SENT_SLACK} -eq 0 ] && exit 0
 761
 762 # we did not send anything
 763 exit 1