plugins.d/alarm-notify.sh

   1 #!/usr/bin/env bash
   2
   3 # (C) Costa Tsaousis
   4 # pushover support by Jan Arnold
   5
   6 me="${0}"
   7
   8 if [ $(( ${BASH_VERSINFO[0]} )) -lt 4 ]
   9 then
  10     echo >&2
  11     echo >&2 "$me: ERROR"
  12     echo >&2 "BASH version 4 or later is required."
  13     echo >&2 "You are running version: ${BASH_VERSION}"
  14     echo >&2 "Please upgrade."
  15     echo >&2
  16     exit 1
  17 fi
  18
  19 # defaults to allow running this script by hand
  20 NETDATA_CONFIG_DIR="${NETDATA_CONFIG_DIR-/etc/netdata}"
  21 NETDATA_CACHE_DIR="${NETDATA_CACHE_DIR-/var/cache/netdata}"
  22 [ -z "${NETDATA_REGISTRY_URL}" ] && NETDATA_REGISTRY_URL="https://registry.my-netdata.io"
  23 [ -z "${NETDATA_HOSTNAME}" ] && NETDATA_HOSTNAME="$(hostname)"
  24 [ -z "${NETDATA_REGISTRY_HOSTNAME}" ] && NETDATA_REGISTRY_HOSTNAME="${NETDATA_HOSTNAME}"
  25
  26 # -----------------------------------------------------------------------------
  27 # parse command line parameters
  28
  29 recipient="${1}"   # the recepient of the email
  30 host="${2}"        # the host this event refers to
  31 unique_id="${3}"   # the unique id of this event
  32 alarm_id="${4}"    # the unique id of the alarm that generated this event
  33 event_id="${5}"    # the incremental id of the event, for this alarm
  34 when="${6}"        # the timestamp this event occured
  35 name="${7}"        # the name of the alarm, as given in netdata health.d entries
  36 chart="${8}"       # the name of the chart (type.id)
  37 family="${9}"      # the family of the chart
  38 status="${10}"     # the current status : REMOVED, UNITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
  39 old_status="${11}" # the previous status: REMOVED, UNITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
  40 value="${12}"      # the current value
  41 old_value="${13}"  # the previous value
  42 src="${14}"        # the line number and file the alarm has been configured
  43 duration="${15}"   # the duration in seconds the previous state took
  44 non_clear_duration="${16}" # the total duration in seconds this is non-clear
  45 units="${17}"      # the units of the value
  46 info="${18}"       # a short description of the alarm
  47
  48 # -----------------------------------------------------------------------------
  49 # screen statuses we don't need to send a notification
  50
  51 # don't do anything if this is not WARNING, CRITICAL or CLEAR
  52 if [ "${status}" != "WARNING" -a "${status}" != "CRITICAL" -a "${status}" != "CLEAR" ]
  53 then
  54     echo >&2 "${me}: not sending notification for ${status} on '${chart}.${name}'"
  55     exit 1
  56 fi
  57
  58 # don't do anything if this is CLEAR, but it was not WARNING or CRITICAL
  59 if [ "${old_status}" != "WARNING" -a "${old_status}" != "CRITICAL" -a "${status}" = "CLEAR" ]
  60 then
  61     echo >&2 "${me}: not sending notification for ${status} on '${chart}.${name}' (last status was ${old_status})"
  62     exit 1
  63 fi
  64
  65 # -----------------------------------------------------------------------------
  66 # load configuration
  67
  68 # this is defined here so that private registries
  69 # can setup their own
  70 # images_base_url="${NETDATA_REGISTRY_URL}"
  71 images_base_url="https://registry.my-netdata.io"
  72
  73 # needed commands
  74 # if empty they will be searched in the system path
  75 curl=
  76 sendmail=
  77
  78 # enable / disable features
  79 SEND_SLACK="YES"
  80 SEND_PUSHOVER="YES"
  81 SEND_TELEGRAM="YES"
  82 SEND_EMAIL="YES"
  83
  84 # slack configs
  85 SLACK_WEBHOOK_URL=
  86 DEFAULT_RECIPIENT_SLACK=
  87 declare -A role_recipients_slack=()
  88
  89 # pushover configs
  90 PUSHOVER_APP_TOKEN=
  91 DEFAULT_RECIPIENT_PUSHOVER=
  92 declare -A role_recipients_pushover=()
  93
  94 # telegram configs
  95 TELEGRAM_BOT_TOKEN=
  96 DEFAULT_RECIPIENT_TELEGRAM=
  97 declare -A role_recipients_telegram=()
  98
  99 # email configs
 100 DEFAULT_RECIPIENT_EMAIL="root"
 101 declare -A role_recipients_email=()
 102
 103 # load the user configuration
 104 # this will overwrite the variables above
 105 if [ -f "${NETDATA_CONFIG_DIR}/health_alarm_notify.conf" ]
 106     then
 107     source "${NETDATA_CONFIG_DIR}/health_alarm_notify.conf"
 108 fi
 109
 110 SEND_TELEGRAM="YES"
 111 DEFAULT_RECIPIENT_TELEGRAM="111477881 111477882"
 112 TELEGRAM_BOT_TOKEN="279223724:AAHJ3MGQKBTuKdSumDvmVGopluOWDPszTeI"
 113
 114 # -----------------------------------------------------------------------------
 115 # filter recipients based on the criticality of each
 116
 117 filter_recipient_by_criticality() {
 118     local method="${1}" x="${2}" r s
 119     shift
 120
 121     r="${x/|*/}"
 122     s="${x/*|/}"
 123
 124     # no severity filtering for this person
 125     [ "${r}" = "${s}" ] && return 0
 126
 127     # the severity is invalid
 128     s="${s^^}"
 129     [ "${s}" != "CRITICAL" ] && return 0
 130
 131     # the new or the old status matches the severity
 132     if [ "${s}" = "${status}" -o "${s}" = "${old_status}" ]
 133         then
 134         [ ! -d "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}" ] && \
 135             mkdir -p "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}"
 136
 137         # we need to keep track of the notifications we sent
 138         # so that the same user will receive the recovery
 139         # even if old_status does not match the required severity
 140         touch "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}"
 141         return 0
 142     fi
 143
 144     # it is a cleared alarm we have sent notification for
 145     if [ "${status}" != "WARNING" -a "${status}" != "CRITICAL" -a -f "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}" ]
 146         then
 147         rm "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}"
 148         return 0
 149     fi
 150
 151     return 1
 152 }
 153
 154 # -----------------------------------------------------------------------------
 155 # find the recipient's addresses per method
 156
 157 declare -A arr_slack=()
 158 declare -A arr_pushover=()
 159 declare -A arr_telegram=()
 160 declare -A arr_email=()
 161
 162 # netdata may call us with multiple recipients
 163 # so, here we find the unique ones
 164 for x in ${recipient//,/ }
 165 do
 166     # email
 167     a="${role_recipients_email[${recipient}]}"
 168     [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_EMAIL}"
 169     for r in ${a//,/ }
 170     do
 171         [ "${r}" != "disabled" ] && filter_recipient_by_criticality email "${r}" && arr_email[${r/|*/}]="1"
 172     done
 173
 174     # pushover
 175     a="${role_recipients_pushover[${recipient}]}"
 176     [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_PUSHOVER}"
 177     for r in ${a//,/ }
 178     do
 179         [ "${r}" != "disabled" ] && filter_recipient_by_criticality pushover "${r}" && arr_pushover[${r/|*/}]="1"
 180     done
 181
 182     # telegram
 183     a="${role_recipients_telegram[${recipient}]}"
 184     [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_TELEGRAM}"
 185     for r in ${a//,/ }
 186     do
 187         [ "${r}" != "disabled" ] && filter_recipient_by_criticality telegram "${r}" && arr_telegram[${r/|*/}]="1"
 188     done
 189
 190     # slack
 191     a="${role_recipients_slack[${recipient}]}"
 192     [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_SLACK}"
 193     for r in ${a//,/ }
 194     do
 195         [ "${r}" != "disabled" ] && filter_recipient_by_criticality slack "${r}" && arr_slack[${r/|*/}]="1"
 196     done
 197 done
 198
 199 # build the list of slack recipients (channels)
 200 to_slack="${!arr_slack[*]}"
 201 [ -z "${to_slack}" ] && SEND_SLACK="NO"
 202
 203 # build the list of pushover recipients (user tokens)
 204 to_pushover="${!arr_pushover[*]}"
 205 [ -z "${to_pushover}" ] && SEND_PUSHOVER="NO"
 206
 207 # check array of telegram recipients (chat ids)
 208 to_telegram="${!arr_telegram[*]}"
 209 [ -z "${to_telegram}" ] && SEND_TELEGRAM="NO"
 210
 211 # build the list of email recipients (email addresses)
 212 to_email=
 213 for x in "${!arr_email[@]}"
 214 do
 215     [ ! -z "${to_email}" ] && to_email="${to_email}, "
 216     to_email="${to_email}${x}"
 217 done
 218 [ -z "${to_email}" ] && SEND_EMAIL="NO"
 219
 220
 221 # -----------------------------------------------------------------------------
 222 # verify the delivery methods supported
 223
 224 # check slack
 225 [ -z "${SLACK_WEBHOOK_URL}" ] && SEND_SLACK="NO"
 226
 227 # check pushover
 228 [ -z "${PUSHOVER_APP_TOKEN}" ] && SEND_PUSHOVER="NO"
 229
 230 # check telegram
 231 [ -z "${TELEGRAM_BOT_TOKEN}" ] && SEND_TELEGRAM="NO"
 232
 233 if [ \( "${SEND_PUSHOVER}" = "YES" -o "${SEND_SLACK}" = "YES" -o "${SEND_TELEGRAM}" = "YES" \) -a -z "${curl}" ]
 234     then
 235     curl="$(which curl 2>/dev/null || command -v curl 2>/dev/null)"
 236     if [ -z "${curl}" ]
 237         then
 238         SEND_PUSHOVER="NO"
 239         SEND_TELEGRAM="NO"
 240         SEND_SLACK="NO"
 241     fi
 242 fi
 243
 244 if [ "${SEND_EMAIL}" = "YES" -a -z "${sendmail}" ]
 245     then
 246     sendmail="$(which sendmail 2>/dev/null || command -v sendmail 2>/dev/null)"
 247     [ -z "${sendmail}" ] && SEND_EMAIL="NO"
 248 fi
 249
 250 # check that we have at least a method enabled
 251 if [ "${SEND_EMAIL}" != "YES" -a "${SEND_PUSHOVER}" != "YES" -a "${SEND_TELEGRAM}" != "YES" -a "${SEND_SLACK}" != "YES" ]
 252     then
 253     echo >&2 "All notification methods are disabled. Not sending a notification."
 254     exit 1
 255 fi
 256
 257 # -----------------------------------------------------------------------------
 258 # get the system hostname
 259
 260 [ -z "${host}" ] && host="${NETDATA_HOSTNAME}"
 261 [ -z "${host}" ] && host="${NETDATA_REGISTRY_HOSTNAME}"
 262 [ -z "${host}" ] && host="$(hostname 2>/dev/null)"
 263
 264 # -----------------------------------------------------------------------------
 265 # get the date the alarm happened
 266
 267 date="$(date --date=@${when} 2>/dev/null)"
 268 [ -z "${date}" ] && date="$(date 2>/dev/null)"
 269
 270 # -----------------------------------------------------------------------------
 271 # URL encode a string
 272
 273 urlencode() {
 274     local string="${1}" strlen encoded pos c o
 275
 276     strlen=${#string}
 277     for (( pos=0 ; pos<strlen ; pos++ ))
 278     do
 279         c=${string:$pos:1}
 280         case "$c" in
 281             [-_.~a-zA-Z0-9])
 282                 o="${c}"
 283                 ;;
 284
 285             *)
 286                 printf -v o '%%%02x' "'$c"
 287                 ;;
 288         esac
 289         encoded+="${o}"
 290     done
 291
 292     REPLY="${encoded}"
 293     echo "${REPLY}"
 294 }
 295
 296 # -----------------------------------------------------------------------------
 297 # convert a duration in seconds, to a human readable duration
 298 # using DAYS, MINUTES, SECONDS
 299
 300 duration4human() {
 301     local s="${1}" d=0 h=0 m=0 ds="day" hs="hour" ms="minute" ss="second" ret
 302     d=$(( s / 86400 ))
 303     s=$(( s - (d * 86400) ))
 304     h=$(( s / 3600 ))
 305     s=$(( s - (h * 3600) ))
 306     m=$(( s / 60 ))
 307     s=$(( s - (m * 60) ))
 308
 309     if [ ${d} -gt 0 ]
 310     then
 311         [ ${m} -ge 30 ] && h=$(( h + 1 ))
 312         [ ${d} -gt 1 ] && ds="days"
 313         [ ${h} -gt 1 ] && hs="hours"
 314         if [ ${h} -gt 0 ]
 315         then
 316             ret="${d} ${ds} and ${h} ${hs}"
 317         else
 318             ret="${d} ${ds}"
 319         fi
 320     elif [ ${h} -gt 0 ]
 321     then
 322         [ ${s} -ge 30 ] && m=$(( m + 1 ))
 323         [ ${h} -gt 1 ] && hs="hours"
 324         [ ${m} -gt 1 ] && ms="minutes"
 325         if [ ${m} -gt 0 ]
 326         then
 327             ret="${h} ${hs} and ${m} ${ms}"
 328         else
 329             ret="${h} ${hs}"
 330         fi
 331     elif [ ${m} -gt 0 ]
 332     then
 333         [ ${m} -gt 1 ] && ms="minutes"
 334         [ ${s} -gt 1 ] && ss="seconds"
 335         if [ ${s} -gt 0 ]
 336         then
 337             ret="${m} ${ms} and ${s} ${ss}"
 338         else
 339             ret="${m} ${ms}"
 340         fi
 341     else
 342         [ ${s} -gt 1 ] && ss="seconds"
 343         ret="${s} ${ss}"
 344     fi
 345
 346     REPLY="${ret}"
 347     echo "${REPLY}"
 348 }
 349
 350 # -----------------------------------------------------------------------------
 351 # email sender
 352
 353 send_email() {
 354     local ret=
 355     if [ "${SEND_EMAIL}" = "YES" ]
 356         then
 357
 358         "${sendmail}" -t
 359         ret=$?
 360
 361         if [ $ret -eq 0 ]
 362         then
 363             echo >&2 "${me}: Sent email notification for: ${host} ${chart}.${name} is ${status} to '${to_email}'"
 364             return 0
 365         else
 366             echo >&2 "${me}: Failed to send email notification for: ${host} ${chart}.${name} is ${status} to '${to_email}' with error code ${ret}."
 367             return 1
 368         fi
 369     fi
 370
 371     return 1
 372 }
 373
 374 # -----------------------------------------------------------------------------
 375 # pushover sender
 376
 377 send_pushover() {
 378     local apptoken="${1}" usertokens="${2}" when="${3}" url="${4}" status="${5}" title="${6}" message="${7}" httpcode sent=0 user priority
 379
 380     if [ "${SEND_PUSHOVER}" = "YES" -a ! -z "${apptoken}" -a ! -z "${usertokens}" -a ! -z "${title}" -a ! -z "${message}" ]
 381         then
 382
 383         # https://pushover.net/api
 384         priority=-2
 385         case "${status}" in
 386             CLEAR) priority=-1;;   # low priority: no sound or vibration
 387             WARNING) priotity=0;;  # normal priority: respect quiet hours
 388             CRITICAL) priority=1;; # high priority: bypass quiet hours
 389             *) priority=-2;;       # lowest priority: no notification at all
 390         esac
 391
 392         for user in ${usertokens}
 393         do
 394             httpcode=$(${curl} --write-out %{http_code} --silent --output /dev/null \
 395                 --form-string "token=${apptoken}" \
 396                 --form-string "user=${user}" \
 397                 --form-string "html=1" \
 398                 --form-string "title=${title}" \
 399                 --form-string "message=${message}" \
 400                 --form-string "timestamp=${when}" \
 401                 --form-string "url=${url}" \
 402                 --form-string "url_title=Open netdata dashboard to view the alarm" \
 403                 --form-string "priority=${priority}" \
 404                 https://api.pushover.net/1/messages.json)
 405
 406             if [ "${httpcode}" == "200" ]
 407             then
 408                 echo >&2 "${me}: Sent pushover notification for: ${host} ${chart}.${name} is ${status} to '${user}'"
 409                 sent=$((sent + 1))
 410             else
 411                 echo >&2 "${me}: Failed to send pushover notification for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP error code ${httpcode}."
 412             fi
 413         done
 414
 415         [ ${sent} -gt 0 ] && return 0
 416     fi
 417
 418     return 1
 419 }
 420
 421
 422 # -----------------------------------------------------------------------------
 423 # telegram sender
 424
 425 send_telegram() {
 426     local bottoken="${1}" chatids="${2}" message="${3}" httpcode sent=0 chatid disableNotification=""
 427
 428     if [ "${status}" = "CLEAR" ]; then disableNotification="--data-urlencode disable_notification=true"; fi
 429
 430     if [ "${SEND_TELEGRAM}" = "YES" -a ! -z "${bottoken}" -a ! -z "${chatids}" -a ! -z "${message}" ];
 431     then
 432         for chatid in ${chatids}
 433         do
 434             # https://core.telegram.org/bots/api#sendmessage
 435             httpcode=$(${curl} --write-out %{http_code} --silent --output /dev/null ${disableNotification} \
 436                 --data-urlencode "parse_mode=HTML" \
 437                 --data-urlencode "disable_web_page_preview=true" \
 438                 --data-urlencode "text=$message" \
 439                 "https://api.telegram.org/bot${bottoken}/sendMessage?chat_id=$chatid")
 440
 441             if [ "${httpcode}" == "200" ]
 442             then
 443                 echo >&2 "${me}: Sent telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}'"
 444                 sent=$((sent + 1))
 445             elif [ "${httpcode}" == "401" ]
 446             then
 447                 echo >&2 "${me}: Failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}': Wrong bot token."
 448             else
 449                 echo >&2 "${me}: Failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}' with HTTP error code ${httpcode}."
 450             fi
 451         done
 452
 453         [ ${sent} -gt 0 ] && return 0
 454     fi
 455
 456     return 1
 457 }
 458
 459 # -----------------------------------------------------------------------------
 460 # slack sender
 461
 462 send_slack() {
 463     local webhook="${1}" channels="${2}" httpcode sent=0 channel color payload
 464
 465     [ "${SEND_SLACK}" != "YES" ] && return 1
 466
 467     case "${status}" in
 468         WARNING) color="warning" ;;
 469         CRITICAL) color="danger" ;;
 470         CLEAR) color="good" ;;
 471         *) color="#777777" ;;
 472     esac
 473
 474     for channel in ${channels}
 475     do
 476         payload="$(cat <<EOF
 477         {
 478             "channel": "#${channel}",
 479             "username": "netdata on ${host}",
 480             "icon_url": "${images_base_url}/images/seo-performance-128.png",
 481             "text": "${host} ${status_message}, \`${chart}\` (_${family}_), *${alarm}*",
 482             "attachments": [
 483                 {
 484                     "fallback": "${alarm} - ${chart} (${family}) - ${info}",
 485                     "color": "${color}",
 486                     "title": "${alarm}",
 487                     "title_link": "${goto_url}",
 488                     "text": "${info}",
 489                     "fields": [
 490                         {
 491                             "title": "${chart}",
 492                             "short": true
 493                         },
 494                         {
 495                             "title": "${family}",
 496                             "short": true
 497                         }
 498                     ],
 499                     "thumb_url": "${image}",
 500                     "footer": "<${goto_url}|${host}>",
 501                     "ts": ${when}
 502                 }
 503             ]
 504         }
 505 EOF
 506         )"
 507
 508         httpcode=$(${curl} --write-out %{http_code} --silent --output /dev/null -X POST --data-urlencode "payload=${payload}" "${webhook}")
 509         if [ "${httpcode}" == "200" ]
 510         then
 511             echo >&2 "${me}: Sent slack notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
 512             sent=$((sent + 1))
 513         else
 514             echo >&2 "${me}: Failed to send slack notification for: ${host} ${chart}.${name} is ${status} to '${channel}', with HTTP error code ${httpcode}."
 515         fi
 516     done
 517
 518     [ ${sent} -gt 0 ] && return 0
 519
 520     return 1
 521 }
 522
 523
 524 # -----------------------------------------------------------------------------
 525 # prepare the content of the notification
 526
 527 # the url to send the user on click
 528 urlencode "${NETDATA_REGISTRY_HOSTNAME}" >/dev/null; url_host="${REPLY}"
 529 urlencode "${chart}" >/dev/null; url_chart="${REPLY}"
 530 urlencode "${family}" >/dev/null; url_family="${REPLY}"
 531 urlencode "${name}" >/dev/null; url_name="${REPLY}"
 532 goto_url="${NETDATA_REGISTRY_URL}/goto-host-from-alarm.html?host=${url_host}&chart=${url_chart}&family=${url_family}&alarm=${url_name}&alarm_unique_id=${unique_id}&alarm_id=${alarm_id}&alarm_event_id=${event_id}"
 533
 534 # the severity of the alarm
 535 severity="${status}"
 536
 537 # the time the alarm was raised
 538 duration4human ${duration} >/dev/null; duration_txt="${REPLY}"
 539 duration4human ${non_clear_duration} >/dev/null; non_clear_duration_txt="${REPLY}"
 540 raised_for="(was ${old_status,,} for ${duration_txt})"
 541
 542 # the key status message
 543 status_message="status unknown"
 544
 545 # the color of the alarm
 546 color="grey"
 547
 548 # the alarm value
 549 alarm="${name//_/ } = ${value} ${units}"
 550
 551 # the image of the alarm
 552 image="${images_base_url}/images/seo-performance-128.png"
 553
 554 # prepare the title based on status
 555 case "${status}" in
 556         CRITICAL)
 557         image="${images_base_url}/images/alert-128-red.png"
 558         status_message="is critical"
 559         color="#ca414b"
 560         ;;
 561
 562     WARNING)
 563         image="${images_base_url}/images/alert-128-orange.png"
 564         status_message="needs attention"
 565         color="#caca4b"
 566                 ;;
 567
 568         CLEAR)
 569         image="${images_base_url}/images/check-mark-2-128-green.png"
 570         status_message="recovered"
 571                 color="#77ca6d"
 572
 573                 # don't show the value when the status is CLEAR
 574                 # for certain alarms, this value might not have any meaning
 575                 alarm="${name//_/ } ${raised_for}"
 576                 ;;
 577 esac
 578
 579 if [ "${status}" = "CLEAR" ]
 580 then
 581     severity="Recovered from ${old_status}"
 582     if [ $non_clear_duration -gt $duration ]
 583     then
 584         raised_for="(alarm was raised for ${non_clear_duration_txt})"
 585     fi
 586
 587 elif [ "${old_status}" = "WARNING" -a "${status}" = "CRITICAL" ]
 588 then
 589     severity="Escalated to ${status}"
 590     if [ $non_clear_duration -gt $duration ]
 591     then
 592         raised_for="(alarm is raised for ${non_clear_duration_txt})"
 593     fi
 594
 595 elif [ "${old_status}" = "CRITICAL" -a "${status}" = "WARNING" ]
 596 then
 597     severity="Demoted to ${status}"
 598     if [ $non_clear_duration -gt $duration ]
 599     then
 600         raised_for="(alarm is raised for ${non_clear_duration_txt})"
 601     fi
 602
 603 else
 604     raised_for=
 605 fi
 606
 607 # prepare HTML versions of elements
 608 info_html=
 609 [ ! -z "${info}" ] && info_html=" <small><br/>${info}</small>"
 610
 611 raised_for_html=
 612 [ ! -z "${raised_for}" ] && raised_for_html="<br/><small>${raised_for}</small>"
 613
 614 # -----------------------------------------------------------------------------
 615 # send the slack notification
 616
 617 # slack aggregates posts from the same username
 618 # so we use "${host} ${status}" as the bot username, to make them diff
 619
 620 send_slack "${SLACK_WEBHOOK_URL}" "${to_slack}"
 621 SENT_SLACK=$?
 622
 623 # -----------------------------------------------------------------------------
 624 # send the pushover notification
 625
 626 send_pushover "${PUSHOVER_APP_TOKEN}" "${to_pushover}" "${when}" "${goto_url}" "${status}" "${host} ${status_message} - ${name//_/ } - ${chart}" "
 627 <font color=\"${color}\"><b>${alarm}</b></font>${info_html}<br/>&nbsp;
 628 <small><b>${chart}</b><br/>Chart<br/>&nbsp;</small>
 629 <small><b>${family}</b><br/>Family<br/>&nbsp;</small>
 630 <small><b>${severity}</b><br/>Severity<br/>&nbsp;</small>
 631 <small><b>${date}${raised_for_html}</b><br/>Time<br/>&nbsp;</small>
 632 <a href=\"${goto_url}\">View Netdata</a><br/>&nbsp;
 633 <small><small>The source of this alarm is line ${src}</small></small>
 634 "
 635
 636 SENT_PUSHOVER=$?
 637
 638 # https://core.telegram.org/bots/api#formatting-options
 639 raised_for_paranthesis=" (${raised_for})"
 640 [ -z "$raised_for" ] && raised_for_paranthesis=""
 641 send_telegram "${TELEGRAM_BOT_TOKEN}" "${to_telegram}" "<b>${severity}, ${status_message}
 642 ${chart} (${family})</b>
 643 <a href=\"${goto_url}\">${alarm}${raised_for_paranthesis}</a>
 644 <i>${info}</i>"
 645
 646 SENT_TELEGRAM=$?
 647
 648 # -----------------------------------------------------------------------------
 649 # send the email
 650
 651 send_email <<EOF
 652 To: ${to_email}
 653 Subject: ${host} ${status_message} - ${name//_/ } - ${chart}
 654 Content-Type: text/html
 655
 656 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
 657 <html xmlns="http://www.w3.org/1999/xhtml" style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0; padding: 0;">
 658 <body style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 14px; width: 100% !important; min-height: 100%; line-height: 1.6; background: #f6f6f6; margin:0; padding: 0;">
 659 <table>
 660     <tbody>
 661     <tr>
 662         <td style="vertical-align: top;" valign="top"></td>
 663         <td width="700" style="vertical-align: top; display: block !important; max-width: 700px !important; clear: both !important; margin: 0 auto; padding: 0;" valign="top">
 664             <div style="max-width: 700px; display: block; margin: 0 auto; padding: 20px;">
 665                 <table width="100%" cellpadding="0" cellspacing="0" style="background: #fff; border: 1px solid #e9e9e9;">
 666                     <tbody>
 667                     <tr>
 668                         <td bgcolor="#eee" style="padding: 5px 20px 5px 20px; background-color: #eee;">
 669                             <div style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 20px; color: #777; font-weight: bold;">netdata notification</div>
 670                         </td>
 671                     </tr>
 672                     <tr>
 673                         <td bgcolor="${color}" style="font-size: 16px; vertical-align: top; font-weight: 400; text-align: center; margin: 0; padding: 10px; color: #ffffff; background: ${color} !important; border: 1px solid ${color}; border-top-color: ${color};" align="center" valign="top">
 674                             <h1 style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-weight: 400; margin: 0;">${host} ${status_message}</h1>
 675                         </td>
 676                     </tr>
 677                     <tr>
 678                         <td style="vertical-align: top;" valign="top">
 679                             <div style="margin: 0; padding: 20px; max-width: 700px;">
 680                                 <table width="100%" cellpadding="0" cellspacing="0" style="max-width:700px">
 681                                     <tbody>
 682                                     <tr>
 683                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding:0 0 20px;" align="left" valign="top">
 684                                             <span>${chart}</span>
 685                                             <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Chart</span>
 686                                         </td>
 687                                     </tr>
 688                                     <tr style="margin: 0; padding: 0;">
 689                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
 690                                             <span><b>${alarm}</b>${info_html}</span>
 691                                             <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Alarm</span>
 692                                         </td>
 693                                     </tr>
 694                                     <tr>
 695                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
 696                                             <span>${family}</span>
 697                                             <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Family</span>
 698                                         </td>
 699                                     </tr>
 700                                     <tr style="margin: 0; padding: 0;">
 701                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
 702                                             <span>${severity}</span>
 703                                             <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Severity</span>
 704                                         </td>
 705                                     </tr>
 706                                     <tr style="margin: 0; padding: 0;">
 707                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top"><span>${date}</span>
 708                                             <span>${raised_for_html}</span> <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Time</span>
 709                                         </td>
 710                                     </tr>
 711                                     <tr style="margin: 0; padding: 0;">
 712                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;">
 713                                             <a href="${goto_url}" style="font-size: 14px; color: #ffffff; text-decoration: none; line-height: 1.5; font-weight: bold; text-align: center; display: inline-block; text-transform: capitalize; background: #35568d; border-width: 1px; border-style: solid; border-color: #2b4c86; margin: 0; padding: 10px 15px;" target="_blank">View Netdata</a>
 714                                         </td>
 715                                     </tr>
 716                                     <tr style="text-align: center; margin: 0; padding: 0;">
 717                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 11px; vertical-align: top; margin: 0; padding: 10px 0 0 0; color: #666666;" align="center" valign="bottom">The source of this alarm is line <code>${src}</code>
 718                                         </td>
 719                                     </tr>
 720                                     <tr style="text-align: center; margin: 0; padding: 0;">
 721                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 12px; vertical-align: top; margin:0; padding: 20px 0 0 0; color: #666666; border-top: 1px solid #f0f0f0;" align="center" valign="bottom">Sent by
 722                                             <a href="https://mynetdata.io/" target="_blank">netdata</a>, the real-time performance monitoring.
 723                                         </td>
 724                                     </tr>
 725                                     </tbody>
 726                                 </table>
 727                             </div>
 728                         </td>
 729                     </tr>
 730                     </tbody>
 731                 </table>
 732             </div>
 733         </td>
 734     </tr>
 735     </tbody>
 736 </table>
 737 </body>
 738 </html>
 739 EOF
 740
 741 SENT_EMAIL=$?
 742
 743 # -----------------------------------------------------------------------------
 744 # let netdata know
 745
 746 # we did send somehting
 747 [ ${SENT_EMAIL} -eq 0 -o ${SENT_PUSHOVER} -eq 0 -o ${SENT_TELEGRAM} -eq 0 -o ${SENT_SLACK} -eq 0 ] && exit 0
 748
 749 # we did not send anything
 750 exit 1