plugins.d/alarm-notify.sh

   1 #!/usr/bin/env bash
   2
   3 # netdata
   4 # real-time performance and health monitoring, done right!
   5 # (C) 2016 Costa Tsaousis <costa@tsaousis.gr>
   6 # GPL v3+
   7 #
   8 # Script the send alarm notifications for netdata
   9 #
  10 # Features:
  11 #  - multiple notification methods
  12 #  - multiple roles per alarm
  13 #  - multiple recipients per role
  14 #  - severity filtering per recipient
  15 #
  16 # Supported notification methods:
  17 #  - emails
  18 #  - pushover.net notifications
  19 #  - pushbullet.com push notifications
  20 #  - slack.com notifications
  21 #  - telegram.org notifications
  22 #
  23
  24 me="${0}"
  25
  26 # check for BASH v4+ (required for associative arrays)
  27 [ $(( ${BASH_VERSINFO[0]} )) -lt 4 ] && \
  28     echo >&2 "${me}: BASH version 4 or later is required (this is ${BASH_VERSION})." && \
  29     exit 1
  30
  31 # defaults to allow running this script by hand
  32 NETDATA_CONFIG_DIR="${NETDATA_CONFIG_DIR-/etc/netdata}"
  33 NETDATA_CACHE_DIR="${NETDATA_CACHE_DIR-/var/cache/netdata}"
  34 [ -z "${NETDATA_REGISTRY_URL}" ] && NETDATA_REGISTRY_URL="https://registry.my-netdata.io"
  35 [ -z "${NETDATA_HOSTNAME}" ] && NETDATA_HOSTNAME="$(hostname)"
  36 [ -z "${NETDATA_REGISTRY_HOSTNAME}" ] && NETDATA_REGISTRY_HOSTNAME="${NETDATA_HOSTNAME}"
  37
  38 # -----------------------------------------------------------------------------
  39 # parse command line parameters
  40
  41 roles="${1}"       # the roles that should be notified for this event
  42 host="${2}"        # the host generated this event
  43 unique_id="${3}"   # the unique id of this event
  44 alarm_id="${4}"    # the unique id of the alarm that generated this event
  45 event_id="${5}"    # the incremental id of the event, for this alarm id
  46 when="${6}"        # the timestamp this event occurred
  47 name="${7}"        # the name of the alarm, as given in netdata health.d entries
  48 chart="${8}"       # the name of the chart (type.id)
  49 family="${9}"      # the family of the chart
  50 status="${10}"     # the current status : REMOVED, UNITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
  51 old_status="${11}" # the previous status: REMOVED, UNITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
  52 value="${12}"      # the current value of the alarm
  53 old_value="${13}"  # the previous value of the alarm
  54 src="${14}"        # the line number and file the alarm has been configured
  55 duration="${15}"   # the duration in seconds of the previous alarm state
  56 non_clear_duration="${16}" # the total duration in seconds this is/was non-clear
  57 units="${17}"      # the units of the value
  58 info="${18}"       # a short description of the alarm
  59
  60 # -----------------------------------------------------------------------------
  61 # screen statuses we don't need to send a notification
  62
  63 # don't do anything if this is not WARNING, CRITICAL or CLEAR
  64 if [ "${status}" != "WARNING" -a "${status}" != "CRITICAL" -a "${status}" != "CLEAR" ]
  65 then
  66     echo >&2 "${me}: not sending notification for ${status} on '${chart}.${name}'"
  67     exit 1
  68 fi
  69
  70 # don't do anything if this is CLEAR, but it was not WARNING or CRITICAL
  71 if [ "${old_status}" != "WARNING" -a "${old_status}" != "CRITICAL" -a "${status}" = "CLEAR" ]
  72 then
  73     echo >&2 "${me}: not sending notification for ${status} on '${chart}.${name}' (last status was ${old_status})"
  74     exit 1
  75 fi
  76
  77 # -----------------------------------------------------------------------------
  78 # load configuration
  79
  80 # By default fetch images from the global public registry.
  81 # This is required by default, since all notification methods need to download
  82 # images via the Internet, and private registries might not be reachable.
  83 # This can be overwritten at the configuration file.
  84 images_base_url="https://registry.my-netdata.io"
  85
  86 # needed commands
  87 # if empty they will be searched in the system path
  88 curl=
  89 sendmail=
  90
  91 # enable / disable features
  92 SEND_SLACK="YES"
  93 SEND_PUSHOVER="YES"
  94 SEND_TELEGRAM="YES"
  95 SEND_EMAIL="YES"
  96 SEND_PUSHBULLET="YES"
  97
  98 # slack configs
  99 SLACK_WEBHOOK_URL=
 100 DEFAULT_RECIPIENT_SLACK=
 101 declare -A role_recipients_slack=()
 102
 103 # pushover configs
 104 PUSHOVER_APP_TOKEN=
 105 DEFAULT_RECIPIENT_PUSHOVER=
 106 declare -A role_recipients_pushover=()
 107
 108 # pushbullet configs
 109 PUSHBULLET_ACCESS_TOKEN=
 110 DEFAULT_RECIPIENT_PUSHBULLET=
 111 declare -A role_recipients_pushbullet=()
 112
 113 # telegram configs
 114 TELEGRAM_BOT_TOKEN=
 115 DEFAULT_RECIPIENT_TELEGRAM=
 116 declare -A role_recipients_telegram=()
 117
 118 # email configs
 119 DEFAULT_RECIPIENT_EMAIL="root"
 120 declare -A role_recipients_email=()
 121
 122 # load the user configuration
 123 # this will overwrite the variables above
 124 if [ -f "${NETDATA_CONFIG_DIR}/health_alarm_notify.conf" ]
 125     then
 126     source "${NETDATA_CONFIG_DIR}/health_alarm_notify.conf"
 127 fi
 128
 129 # -----------------------------------------------------------------------------
 130 # filter a recipient based on alarm event severity
 131
 132 filter_recipient_by_criticality() {
 133     local method="${1}" x="${2}" r s
 134     shift
 135
 136     r="${x/|*/}" # the recipient
 137     s="${x/*|/}" # the severity required for notifying this recipient
 138
 139     # no severity filtering for this person
 140     [ "${r}" = "${s}" ] && return 0
 141
 142     # the severity is invalid
 143     s="${s^^}"
 144     [ "${s}" != "CRITICAL" ] && return 0
 145
 146     # the new or the old status matches the severity
 147     if [ "${s}" = "${status}" -o "${s}" = "${old_status}" ]
 148         then
 149         [ ! -d "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}" ] && \
 150             mkdir -p "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}"
 151
 152         # we need to keep track of the notifications we sent
 153         # so that the same user will receive the recovery
 154         # even if old_status does not match the required severity
 155         touch "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}"
 156         return 0
 157     fi
 158
 159     # it is a cleared alarm we have sent notification for
 160     if [ "${status}" != "WARNING" -a "${status}" != "CRITICAL" -a -f "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}" ]
 161         then
 162         rm "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}"
 163         return 0
 164     fi
 165
 166     return 1
 167 }
 168
 169 # -----------------------------------------------------------------------------
 170 # find the recipients' addresses per method
 171
 172 declare -A arr_slack=()
 173 declare -A arr_pushover=()
 174 declare -A arr_pushbullet=()
 175 declare -A arr_telegram=()
 176 declare -A arr_email=()
 177
 178 # netdata may call us with multiple roles, and roles may have multiple but
 179 # overlapping recipients - so, here we find the unique recipients.
 180 for x in ${roles//,/ }
 181 do
 182     # the roles 'silent' and 'disabled' mean:
 183     # don't send a notification for this role
 184     [ "${x}" = "silent" -o "${x}" = "disabled" ] && continue
 185
 186     # email
 187     a="${role_recipients_email[${x}]}"
 188     [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_EMAIL}"
 189     for r in ${a//,/ }
 190     do
 191         [ "${r}" != "disabled" ] && filter_recipient_by_criticality email "${r}" && arr_email[${r/|*/}]="1"
 192     done
 193
 194     # pushover
 195     a="${role_recipients_pushover[${x}]}"
 196     [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_PUSHOVER}"
 197     for r in ${a//,/ }
 198     do
 199         [ "${r}" != "disabled" ] && filter_recipient_by_criticality pushover "${r}" && arr_pushover[${r/|*/}]="1"
 200     done
 201
 202     # pushbullet
 203     a="${role_recipients_pushbullet[${x}]}"
 204     [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_PUSHBULLET}"
 205     for r in ${a//,/ }
 206     do
 207         [ "${r}" != "disabled" ] && filter_recipient_by_criticality pushbullet "${r}" && arr_pushbullet[${r/|*/}]="1"
 208     done
 209
 210     # telegram
 211     a="${role_recipients_telegram[${x}]}"
 212     [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_TELEGRAM}"
 213     for r in ${a//,/ }
 214     do
 215         [ "${r}" != "disabled" ] && filter_recipient_by_criticality telegram "${r}" && arr_telegram[${r/|*/}]="1"
 216     done
 217
 218     # slack
 219     a="${role_recipients_slack[${x}]}"
 220     [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_SLACK}"
 221     for r in ${a//,/ }
 222     do
 223         [ "${r}" != "disabled" ] && filter_recipient_by_criticality slack "${r}" && arr_slack[${r/|*/}]="1"
 224     done
 225 done
 226
 227 # build the list of slack recipients (channels)
 228 to_slack="${!arr_slack[*]}"
 229 [ -z "${to_slack}" ] && SEND_SLACK="NO"
 230
 231 # build the list of pushover recipients (user tokens)
 232 to_pushover="${!arr_pushover[*]}"
 233 [ -z "${to_pushover}" ] && SEND_PUSHOVER="NO"
 234
 235 # build the list of pushbulet recipients (user tokens)
 236 to_pushbullet="${!arr_pushbullet[*]}"
 237 [ -z "${to_pushbullet}" ] && SEND_PUSHBULLET="NO"
 238
 239 # check array of telegram recipients (chat ids)
 240 to_telegram="${!arr_telegram[*]}"
 241 [ -z "${to_telegram}" ] && SEND_TELEGRAM="NO"
 242
 243 # build the list of email recipients (email addresses)
 244 to_email=
 245 for x in "${!arr_email[@]}"
 246 do
 247     [ ! -z "${to_email}" ] && to_email="${to_email}, "
 248     to_email="${to_email}${x}"
 249 done
 250 [ -z "${to_email}" ] && SEND_EMAIL="NO"
 251
 252
 253 # -----------------------------------------------------------------------------
 254 # verify the delivery methods supported
 255
 256 # check slack
 257 [ -z "${SLACK_WEBHOOK_URL}" ] && SEND_SLACK="NO"
 258
 259 # check pushover
 260 [ -z "${PUSHOVER_APP_TOKEN}" ] && SEND_PUSHOVER="NO"
 261
 262 # check pushbullet
 263 [ -z "${DEFAULT_RECIPIENT_PUSHBULLET}" ] && SEND_PUSHBULLET="NO"
 264
 265 # check telegram
 266 [ -z "${TELEGRAM_BOT_TOKEN}" ] && SEND_TELEGRAM="NO"
 267
 268 if [ \( "${SEND_PUSHOVER}" = "YES" -o "${SEND_SLACK}" = "YES" -o "${SEND_TELEGRAM}" = "YES" -o "${SEND_PUSHBULLET}" = "YES" \) -a -z "${curl}" ]
 269     then
 270     curl="$(which curl 2>/dev/null || command -v curl 2>/dev/null)"
 271     if [ -z "${curl}" ]
 272         then
 273         SEND_PUSHOVER="NO"
 274         SEND_PUSHBULLET="NO"
 275         SEND_TELEGRAM="NO"
 276         SEND_SLACK="NO"
 277     fi
 278 fi
 279
 280 if [ "${SEND_EMAIL}" = "YES" -a -z "${sendmail}" ]
 281     then
 282     sendmail="$(which sendmail 2>/dev/null || command -v sendmail 2>/dev/null)"
 283     [ -z "${sendmail}" ] && SEND_EMAIL="NO"
 284 fi
 285
 286 # check that we have at least a method enabled
 287 if [ "${SEND_EMAIL}" != "YES" -a "${SEND_PUSHOVER}" != "YES" -a "${SEND_TELEGRAM}" != "YES" -a "${SEND_SLACK}" != "YES" -a "${SEND_PUSHBULLET}" != "YES" ]
 288     then
 289     echo >&2 "All notification methods are disabled. Not sending a notification."
 290     exit 1
 291 fi
 292
 293 # -----------------------------------------------------------------------------
 294 # get the system hostname
 295
 296 [ -z "${host}" ] && host="${NETDATA_HOSTNAME}"
 297 [ -z "${host}" ] && host="${NETDATA_REGISTRY_HOSTNAME}"
 298 [ -z "${host}" ] && host="$(hostname 2>/dev/null)"
 299
 300 # -----------------------------------------------------------------------------
 301 # get the date the alarm happened
 302
 303 date="$(date --date=@${when} 2>/dev/null)"
 304 [ -z "${date}" ] && date="$(date 2>/dev/null)"
 305
 306 # -----------------------------------------------------------------------------
 307 # URL encode a string
 308
 309 urlencode() {
 310     local string="${1}" strlen encoded pos c o
 311
 312     strlen=${#string}
 313     for (( pos=0 ; pos<strlen ; pos++ ))
 314     do
 315         c=${string:$pos:1}
 316         case "$c" in
 317             [-_.~a-zA-Z0-9])
 318                 o="${c}"
 319                 ;;
 320
 321             *)
 322                 printf -v o '%%%02x' "'$c"
 323                 ;;
 324         esac
 325         encoded+="${o}"
 326     done
 327
 328     REPLY="${encoded}"
 329     echo "${REPLY}"
 330 }
 331
 332 # -----------------------------------------------------------------------------
 333 # convert a duration in seconds, to a human readable duration
 334 # using DAYS, MINUTES, SECONDS
 335
 336 duration4human() {
 337     local s="${1}" d=0 h=0 m=0 ds="day" hs="hour" ms="minute" ss="second" ret
 338     d=$(( s / 86400 ))
 339     s=$(( s - (d * 86400) ))
 340     h=$(( s / 3600 ))
 341     s=$(( s - (h * 3600) ))
 342     m=$(( s / 60 ))
 343     s=$(( s - (m * 60) ))
 344
 345     if [ ${d} -gt 0 ]
 346     then
 347         [ ${m} -ge 30 ] && h=$(( h + 1 ))
 348         [ ${d} -gt 1 ] && ds="days"
 349         [ ${h} -gt 1 ] && hs="hours"
 350         if [ ${h} -gt 0 ]
 351         then
 352             ret="${d} ${ds} and ${h} ${hs}"
 353         else
 354             ret="${d} ${ds}"
 355         fi
 356     elif [ ${h} -gt 0 ]
 357     then
 358         [ ${s} -ge 30 ] && m=$(( m + 1 ))
 359         [ ${h} -gt 1 ] && hs="hours"
 360         [ ${m} -gt 1 ] && ms="minutes"
 361         if [ ${m} -gt 0 ]
 362         then
 363             ret="${h} ${hs} and ${m} ${ms}"
 364         else
 365             ret="${h} ${hs}"
 366         fi
 367     elif [ ${m} -gt 0 ]
 368     then
 369         [ ${m} -gt 1 ] && ms="minutes"
 370         [ ${s} -gt 1 ] && ss="seconds"
 371         if [ ${s} -gt 0 ]
 372         then
 373             ret="${m} ${ms} and ${s} ${ss}"
 374         else
 375             ret="${m} ${ms}"
 376         fi
 377     else
 378         [ ${s} -gt 1 ] && ss="seconds"
 379         ret="${s} ${ss}"
 380     fi
 381
 382     REPLY="${ret}"
 383     echo "${REPLY}"
 384 }
 385
 386 # -----------------------------------------------------------------------------
 387 # email sender
 388
 389 send_email() {
 390     local ret=
 391     if [ "${SEND_EMAIL}" = "YES" ]
 392         then
 393
 394         "${sendmail}" -t
 395         ret=$?
 396
 397         if [ $ret -eq 0 ]
 398         then
 399             echo >&2 "${me}: Sent email notification for: ${host} ${chart}.${name} is ${status} to '${to_email}'"
 400             return 0
 401         else
 402             echo >&2 "${me}: Failed to send email notification for: ${host} ${chart}.${name} is ${status} to '${to_email}' with error code ${ret}."
 403             return 1
 404         fi
 405     fi
 406
 407     return 1
 408 }
 409
 410 # -----------------------------------------------------------------------------
 411 # pushover sender
 412
 413 send_pushover() {
 414     local apptoken="${1}" usertokens="${2}" when="${3}" url="${4}" status="${5}" title="${6}" message="${7}" httpcode sent=0 user priority
 415
 416     if [ "${SEND_PUSHOVER}" = "YES" -a ! -z "${apptoken}" -a ! -z "${usertokens}" -a ! -z "${title}" -a ! -z "${message}" ]
 417         then
 418
 419         # https://pushover.net/api
 420         priority=-2
 421         case "${status}" in
 422             CLEAR) priority=-1;;   # low priority: no sound or vibration
 423             WARNING) priotity=0;;  # normal priority: respect quiet hours
 424             CRITICAL) priority=1;; # high priority: bypass quiet hours
 425             *) priority=-2;;       # lowest priority: no notification at all
 426         esac
 427
 428         for user in ${usertokens}
 429         do
 430             httpcode=$(${curl} --write-out %{http_code} --silent --output /dev/null \
 431                 --form-string "token=${apptoken}" \
 432                 --form-string "user=${user}" \
 433                 --form-string "html=1" \
 434                 --form-string "title=${title}" \
 435                 --form-string "message=${message}" \
 436                 --form-string "timestamp=${when}" \
 437                 --form-string "url=${url}" \
 438                 --form-string "url_title=Open netdata dashboard to view the alarm" \
 439                 --form-string "priority=${priority}" \
 440                 https://api.pushover.net/1/messages.json)
 441
 442             if [ "${httpcode}" == "200" ]
 443             then
 444                 echo >&2 "${me}: Sent pushover notification for: ${host} ${chart}.${name} is ${status} to '${user}'"
 445                 sent=$((sent + 1))
 446             else
 447                 echo >&2 "${me}: Failed to send pushover notification for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP error code ${httpcode}."
 448             fi
 449         done
 450
 451         [ ${sent} -gt 0 ] && return 0
 452     fi
 453
 454     return 1
 455 }
 456
 457 # -----------------------------------------------------------------------------
 458 # pushbullet sender
 459
 460 send_pushbullet() {
 461     local userapikey="${1}" recipients="${2}" message="${3}" title="${4}" httpcode sent=0 user
 462
 463     if [ "${SEND_PUSHBULLET}" = "YES" -a ! -z "${userapikey}" -a ! -z "${recipients}" -a ! -z "${message}" -a ! -z "${title}" ]
 464         then
 465         #https://docs.pushbullet.com/#create-push
 466         for user in ${recipients}
 467         do
 468             httpcode=$(${curl} --write-out %{http_code} --silent --output /dev/null \
 469                  -u "$userapikey": \
 470                  -d type="note" \
 471                  --data-urlencode email="${user}" \
 472                  --data-urlencode body="${message}" \
 473                  --data-urlencode title="${title}" \
 474                  "https://api.pushbullet.com/v2/pushes"
 475                 )
 476
 477             if [ "${httpcode}" == "200" ]
 478             then
 479                 echo >&2 "${me}: Sent pushbullet notification for: ${host} ${chart}.${name} is ${status} to '${user}'"
 480                 sent=$((sent + 1))
 481             else
 482                 echo >&2 "${me}: Failed to send pushbullet notification for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP error code ${httpcode}."
 483             fi
 484         done
 485
 486         [ ${sent} -gt 0 ] && return 0
 487     fi
 488
 489     return 1
 490 }
 491
 492
 493
 494 # -----------------------------------------------------------------------------
 495 # telegram sender
 496
 497 send_telegram() {
 498     local bottoken="${1}" chatids="${2}" message="${3}" httpcode sent=0 chatid disableNotification=""
 499
 500     if [ "${status}" = "CLEAR" ]; then disableNotification="--data-urlencode disable_notification=true"; fi
 501
 502     if [ "${SEND_TELEGRAM}" = "YES" -a ! -z "${bottoken}" -a ! -z "${chatids}" -a ! -z "${message}" ];
 503     then
 504         for chatid in ${chatids}
 505         do
 506             # https://core.telegram.org/bots/api#sendmessage
 507             httpcode=$(${curl} --write-out %{http_code} --silent --output /dev/null ${disableNotification} \
 508                 --data-urlencode "parse_mode=HTML" \
 509                 --data-urlencode "disable_web_page_preview=true" \
 510                 --data-urlencode "text=$message" \
 511                 "https://api.telegram.org/bot${bottoken}/sendMessage?chat_id=$chatid")
 512
 513             if [ "${httpcode}" == "200" ]
 514             then
 515                 echo >&2 "${me}: Sent telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}'"
 516                 sent=$((sent + 1))
 517             elif [ "${httpcode}" == "401" ]
 518             then
 519                 echo >&2 "${me}: Failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}': Wrong bot token."
 520             else
 521                 echo >&2 "${me}: Failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}' with HTTP error code ${httpcode}."
 522             fi
 523         done
 524
 525         [ ${sent} -gt 0 ] && return 0
 526     fi
 527
 528     return 1
 529 }
 530
 531 # -----------------------------------------------------------------------------
 532 # slack sender
 533
 534 send_slack() {
 535     local webhook="${1}" channels="${2}" httpcode sent=0 channel color payload
 536
 537     [ "${SEND_SLACK}" != "YES" ] && return 1
 538
 539     case "${status}" in
 540         WARNING) color="warning" ;;
 541         CRITICAL) color="danger" ;;
 542         CLEAR) color="good" ;;
 543         *) color="#777777" ;;
 544     esac
 545
 546     for channel in ${channels}
 547     do
 548         payload="$(cat <<EOF
 549         {
 550             "channel": "#${channel}",
 551             "username": "netdata on ${host}",
 552             "icon_url": "${images_base_url}/images/seo-performance-128.png",
 553             "text": "${host} ${status_message}, \`${chart}\` (_${family}_), *${alarm}*",
 554             "attachments": [
 555                 {
 556                     "fallback": "${alarm} - ${chart} (${family}) - ${info}",
 557                     "color": "${color}",
 558                     "title": "${alarm}",
 559                     "title_link": "${goto_url}",
 560                     "text": "${info}",
 561                     "fields": [
 562                         {
 563                             "title": "${chart}",
 564                             "short": true
 565                         },
 566                         {
 567                             "title": "${family}",
 568                             "short": true
 569                         }
 570                     ],
 571                     "thumb_url": "${image}",
 572                     "footer": "<${goto_url}|${host}>",
 573                     "ts": ${when}
 574                 }
 575             ]
 576         }
 577 EOF
 578         )"
 579
 580         httpcode=$(${curl} --write-out %{http_code} --silent --output /dev/null -X POST --data-urlencode "payload=${payload}" "${webhook}")
 581         if [ "${httpcode}" == "200" ]
 582         then
 583             echo >&2 "${me}: Sent slack notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
 584             sent=$((sent + 1))
 585         else
 586             echo >&2 "${me}: Failed to send slack notification for: ${host} ${chart}.${name} is ${status} to '${channel}', with HTTP error code ${httpcode}."
 587         fi
 588     done
 589
 590     [ ${sent} -gt 0 ] && return 0
 591
 592     return 1
 593 }
 594
 595
 596 # -----------------------------------------------------------------------------
 597 # prepare the content of the notification
 598
 599 # the url to send the user on click
 600 urlencode "${NETDATA_REGISTRY_HOSTNAME}" >/dev/null; url_host="${REPLY}"
 601 urlencode "${chart}" >/dev/null; url_chart="${REPLY}"
 602 urlencode "${family}" >/dev/null; url_family="${REPLY}"
 603 urlencode "${name}" >/dev/null; url_name="${REPLY}"
 604 goto_url="${NETDATA_REGISTRY_URL}/goto-host-from-alarm.html?host=${url_host}&chart=${url_chart}&family=${url_family}&alarm=${url_name}&alarm_unique_id=${unique_id}&alarm_id=${alarm_id}&alarm_event_id=${event_id}"
 605
 606 # the severity of the alarm
 607 severity="${status}"
 608
 609 # the time the alarm was raised
 610 duration4human ${duration} >/dev/null; duration_txt="${REPLY}"
 611 duration4human ${non_clear_duration} >/dev/null; non_clear_duration_txt="${REPLY}"
 612 raised_for="(was ${old_status,,} for ${duration_txt})"
 613
 614 # the key status message
 615 status_message="status unknown"
 616
 617 # the color of the alarm
 618 color="grey"
 619
 620 # the alarm value
 621 alarm="${name//_/ } = ${value} ${units}"
 622
 623 # the image of the alarm
 624 image="${images_base_url}/images/seo-performance-128.png"
 625
 626 # prepare the title based on status
 627 case "${status}" in
 628         CRITICAL)
 629         image="${images_base_url}/images/alert-128-red.png"
 630         status_message="is critical"
 631         color="#ca414b"
 632         ;;
 633
 634     WARNING)
 635         image="${images_base_url}/images/alert-128-orange.png"
 636         status_message="needs attention"
 637         color="#caca4b"
 638                 ;;
 639
 640         CLEAR)
 641         image="${images_base_url}/images/check-mark-2-128-green.png"
 642         status_message="recovered"
 643                 color="#77ca6d"
 644
 645                 # don't show the value when the status is CLEAR
 646                 # for certain alarms, this value might not have any meaning
 647                 alarm="${name//_/ } ${raised_for}"
 648                 ;;
 649 esac
 650
 651 if [ "${status}" = "CLEAR" ]
 652 then
 653     severity="Recovered from ${old_status}"
 654     if [ $non_clear_duration -gt $duration ]
 655     then
 656         raised_for="(alarm was raised for ${non_clear_duration_txt})"
 657     fi
 658
 659 elif [ "${old_status}" = "WARNING" -a "${status}" = "CRITICAL" ]
 660 then
 661     severity="Escalated to ${status}"
 662     if [ $non_clear_duration -gt $duration ]
 663     then
 664         raised_for="(alarm is raised for ${non_clear_duration_txt})"
 665     fi
 666
 667 elif [ "${old_status}" = "CRITICAL" -a "${status}" = "WARNING" ]
 668 then
 669     severity="Demoted to ${status}"
 670     if [ $non_clear_duration -gt $duration ]
 671     then
 672         raised_for="(alarm is raised for ${non_clear_duration_txt})"
 673     fi
 674
 675 else
 676     raised_for=
 677 fi
 678
 679 # prepare HTML versions of elements
 680 info_html=
 681 [ ! -z "${info}" ] && info_html=" <small><br/>${info}</small>"
 682
 683 raised_for_html=
 684 [ ! -z "${raised_for}" ] && raised_for_html="<br/><small>${raised_for}</small>"
 685
 686 # -----------------------------------------------------------------------------
 687 # send the slack notification
 688
 689 # slack aggregates posts from the same username
 690 # so we use "${host} ${status}" as the bot username, to make them diff
 691
 692 send_slack "${SLACK_WEBHOOK_URL}" "${to_slack}"
 693 SENT_SLACK=$?
 694
 695 # -----------------------------------------------------------------------------
 696 # send the pushover notification
 697
 698 send_pushover "${PUSHOVER_APP_TOKEN}" "${to_pushover}" "${when}" "${goto_url}" "${status}" "${host} ${status_message} - ${name//_/ } - ${chart}" "
 699 <font color=\"${color}\"><b>${alarm}</b></font>${info_html}<br/>&nbsp;
 700 <small><b>${chart}</b><br/>Chart<br/>&nbsp;</small>
 701 <small><b>${family}</b><br/>Family<br/>&nbsp;</small>
 702 <small><b>${severity}</b><br/>Severity<br/>&nbsp;</small>
 703 <small><b>${date}${raised_for_html}</b><br/>Time<br/>&nbsp;</small>
 704 <a href=\"${goto_url}\">View Netdata</a><br/>&nbsp;
 705 <small><small>The source of this alarm is line ${src}</small></small>
 706 "
 707
 708 SENT_PUSHOVER=$?
 709
 710 # -----------------------------------------------------------------------------
 711 # send the pushbullet notification
 712
 713 pushbullet_message="
 714 ${alarm}
 715 Severity: ${severity}
 716 Chart: ${chart}
 717 Family: ${family}
 718 To View Netdata go to: ${goto_url}
 719 The source of this alarm is line ${src}"
 720 pushbullet_title="${status} at ${host} ${status_message} - ${name//_/ } - ${chart}}"
 721 send_pushbullet "${PUSHBULLET_ACCESS_TOKEN}" "${to_pushbullet}" "$pushbullet_message" "$pushbullet_title"
 722
 723 SENT_PUSHBULLET=$?
 724
 725 # -----------------------------------------------------------------------------
 726 # send the telegram.org message
 727
 728 # https://core.telegram.org/bots/api#formatting-options
 729 telegram_message="<b>${severity}"
 730 [ "${status_message}" != "recovered" ] && telegram_message="${telegram_message}, ${status_message}"
 731 telegram_message="${telegram_message}
 732 ${chart} (${family})</b>
 733 <a href=\"${goto_url}\">${alarm}</a>
 734 <i>${info}</i>"
 735
 736 send_telegram "${TELEGRAM_BOT_TOKEN}" "${to_telegram}" "${telegram_message}"
 737
 738 SENT_TELEGRAM=$?
 739
 740 # -----------------------------------------------------------------------------
 741 # send the email
 742
 743 send_email <<EOF
 744 To: ${to_email}
 745 Subject: ${host} ${status_message} - ${name//_/ } - ${chart}
 746 Content-Type: text/html
 747
 748 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
 749 <html xmlns="http://www.w3.org/1999/xhtml" style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0; padding: 0;">
 750 <body style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 14px; width: 100% !important; min-height: 100%; line-height: 1.6; background: #f6f6f6; margin:0; padding: 0;">
 751 <table>
 752     <tbody>
 753     <tr>
 754         <td style="vertical-align: top;" valign="top"></td>
 755         <td width="700" style="vertical-align: top; display: block !important; max-width: 700px !important; clear: both !important; margin: 0 auto; padding: 0;" valign="top">
 756             <div style="max-width: 700px; display: block; margin: 0 auto; padding: 20px;">
 757                 <table width="100%" cellpadding="0" cellspacing="0" style="background: #fff; border: 1px solid #e9e9e9;">
 758                     <tbody>
 759                     <tr>
 760                         <td bgcolor="#eee" style="padding: 5px 20px 5px 20px; background-color: #eee;">
 761                             <div style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 20px; color: #777; font-weight: bold;">netdata notification</div>
 762                         </td>
 763                     </tr>
 764                     <tr>
 765                         <td bgcolor="${color}" style="font-size: 16px; vertical-align: top; font-weight: 400; text-align: center; margin: 0; padding: 10px; color: #ffffff; background: ${color} !important; border: 1px solid ${color}; border-top-color: ${color};" align="center" valign="top">
 766                             <h1 style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-weight: 400; margin: 0;">${host} ${status_message}</h1>
 767                         </td>
 768                     </tr>
 769                     <tr>
 770                         <td style="vertical-align: top;" valign="top">
 771                             <div style="margin: 0; padding: 20px; max-width: 700px;">
 772                                 <table width="100%" cellpadding="0" cellspacing="0" style="max-width:700px">
 773                                     <tbody>
 774                                     <tr>
 775                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding:0 0 20px;" align="left" valign="top">
 776                                             <span>${chart}</span>
 777                                             <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Chart</span>
 778                                         </td>
 779                                     </tr>
 780                                     <tr style="margin: 0; padding: 0;">
 781                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
 782                                             <span><b>${alarm}</b>${info_html}</span>
 783                                             <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Alarm</span>
 784                                         </td>
 785                                     </tr>
 786                                     <tr>
 787                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
 788                                             <span>${family}</span>
 789                                             <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Family</span>
 790                                         </td>
 791                                     </tr>
 792                                     <tr style="margin: 0; padding: 0;">
 793                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
 794                                             <span>${severity}</span>
 795                                             <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Severity</span>
 796                                         </td>
 797                                     </tr>
 798                                     <tr style="margin: 0; padding: 0;">
 799                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top"><span>${date}</span>
 800                                             <span>${raised_for_html}</span> <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Time</span>
 801                                         </td>
 802                                     </tr>
 803                                     <tr style="margin: 0; padding: 0;">
 804                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;">
 805                                             <a href="${goto_url}" style="font-size: 14px; color: #ffffff; text-decoration: none; line-height: 1.5; font-weight: bold; text-align: center; display: inline-block; text-transform: capitalize; background: #35568d; border-width: 1px; border-style: solid; border-color: #2b4c86; margin: 0; padding: 10px 15px;" target="_blank">View Netdata</a>
 806                                         </td>
 807                                     </tr>
 808                                     <tr style="text-align: center; margin: 0; padding: 0;">
 809                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 11px; vertical-align: top; margin: 0; padding: 10px 0 0 0; color: #666666;" align="center" valign="bottom">The source of this alarm is line <code>${src}</code>
 810                                         </td>
 811                                     </tr>
 812                                     <tr style="text-align: center; margin: 0; padding: 0;">
 813                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 12px; vertical-align: top; margin:0; padding: 20px 0 0 0; color: #666666; border-top: 1px solid #f0f0f0;" align="center" valign="bottom">Sent by
 814                                             <a href="https://mynetdata.io/" target="_blank">netdata</a>, the real-time performance monitoring.
 815                                         </td>
 816                                     </tr>
 817                                     </tbody>
 818                                 </table>
 819                             </div>
 820                         </td>
 821                     </tr>
 822                     </tbody>
 823                 </table>
 824             </div>
 825         </td>
 826     </tr>
 827     </tbody>
 828 </table>
 829 </body>
 830 </html>
 831 EOF
 832
 833 SENT_EMAIL=$?
 834
 835 # -----------------------------------------------------------------------------
 836 # let netdata know
 837
 838 # we did send something
 839 [ ${SENT_EMAIL} -eq 0 -o ${SENT_PUSHOVER} -eq 0 -o ${SENT_TELEGRAM} -eq 0 -o ${SENT_SLACK} -eq 0 -o ${SENT_PUSHBULLET} -eq 0 ] && exit 0
 840
 841 # we did not send anything
 842 exit 1