plugins.d/alarm-notify.sh

   1 #!/usr/bin/env bash
   2
   3 # netdata
   4 # real-time performance and health monitoring, done right!
   5 # (C) 2016 Costa Tsaousis <costa@tsaousis.gr>
   6 # GPL v3+
   7 #
   8 # Script the send alarm notifications for netdata
   9 #
  10 # Features:
  11 #  - multiple notification methods
  12 #  - multiple roles per alarm
  13 #  - multiple recipients per role
  14 #  - severity filtering per recipient
  15 #
  16 # Supported notification methods:
  17 #  - emails
  18 #  - pushover.net notifications
  19 #  - pushbullet.com push notifications
  20 #  - slack.com notifications
  21 #  - telegram.org notifications
  22 #
  23
  24 me="${0}"
  25
  26 # check for BASH v4+ (required for associative arrays)
  27 [ $(( ${BASH_VERSINFO[0]} )) -lt 4 ] && \
  28     echo >&2 "${me}: BASH version 4 or later is required (this is ${BASH_VERSION})." && \
  29     exit 1
  30
  31 # defaults to allow running this script by hand
  32 NETDATA_CONFIG_DIR="${NETDATA_CONFIG_DIR-/etc/netdata}"
  33 NETDATA_CACHE_DIR="${NETDATA_CACHE_DIR-/var/cache/netdata}"
  34 [ -z "${NETDATA_REGISTRY_URL}" ] && NETDATA_REGISTRY_URL="https://registry.my-netdata.io"
  35 [ -z "${NETDATA_HOSTNAME}" ] && NETDATA_HOSTNAME="$(hostname)"
  36 [ -z "${NETDATA_REGISTRY_HOSTNAME}" ] && NETDATA_REGISTRY_HOSTNAME="${NETDATA_HOSTNAME}"
  37
  38 # -----------------------------------------------------------------------------
  39 # parse command line parameters
  40
  41 roles="${1}"       # the roles that should be notified for this event
  42 host="${2}"        # the host generated this event
  43 unique_id="${3}"   # the unique id of this event
  44 alarm_id="${4}"    # the unique id of the alarm that generated this event
  45 event_id="${5}"    # the incremental id of the event, for this alarm id
  46 when="${6}"        # the timestamp this event occurred
  47 name="${7}"        # the name of the alarm, as given in netdata health.d entries
  48 chart="${8}"       # the name of the chart (type.id)
  49 family="${9}"      # the family of the chart
  50 status="${10}"     # the current status : REMOVED, UNITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
  51 old_status="${11}" # the previous status: REMOVED, UNITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
  52 value="${12}"      # the current value of the alarm
  53 old_value="${13}"  # the previous value of the alarm
  54 src="${14}"        # the line number and file the alarm has been configured
  55 duration="${15}"   # the duration in seconds of the previous alarm state
  56 non_clear_duration="${16}" # the total duration in seconds this is/was non-clear
  57 units="${17}"      # the units of the value
  58 info="${18}"       # a short description of the alarm
  59
  60 # -----------------------------------------------------------------------------
  61 # screen statuses we don't need to send a notification
  62
  63 # don't do anything if this is not WARNING, CRITICAL or CLEAR
  64 if [ "${status}" != "WARNING" -a "${status}" != "CRITICAL" -a "${status}" != "CLEAR" ]
  65 then
  66     echo >&2 "${me}: not sending notification for ${status} on '${chart}.${name}'"
  67     exit 1
  68 fi
  69
  70 # don't do anything if this is CLEAR, but it was not WARNING or CRITICAL
  71 if [ "${old_status}" != "WARNING" -a "${old_status}" != "CRITICAL" -a "${status}" = "CLEAR" ]
  72 then
  73     echo >&2 "${me}: not sending notification for ${status} on '${chart}.${name}' (last status was ${old_status})"
  74     exit 1
  75 fi
  76
  77 # -----------------------------------------------------------------------------
  78 # load configuration
  79
  80 # By default fetch images from the global public registry.
  81 # This is required by default, since all notification methods need to download
  82 # images via the Internet, and private registries might not be reachable.
  83 # This can be overwritten at the configuration file.
  84 images_base_url="https://registry.my-netdata.io"
  85
  86 # needed commands
  87 # if empty they will be searched in the system path
  88 curl=
  89 sendmail=
  90
  91 # enable / disable features
  92 SEND_SLACK="YES"
  93 SEND_PUSHOVER="YES"
  94 SEND_TELEGRAM="YES"
  95 SEND_EMAIL="YES"
  96 SEND_PUSHBULLET="YES"
  97
  98 # slack configs
  99 SLACK_WEBHOOK_URL=
 100 DEFAULT_RECIPIENT_SLACK=
 101 declare -A role_recipients_slack=()
 102
 103 # pushover configs
 104 PUSHOVER_APP_TOKEN=
 105 DEFAULT_RECIPIENT_PUSHOVER=
 106 declare -A role_recipients_pushover=()
 107
 108 # pushbullet configs
 109 PUSHBULLET_ACCESS_TOKEN=
 110 DEFAULT_RECIPIENT_PUSHBULLET=
 111 declare -A role_recipients_pushbullet=()
 112
 113 # telegram configs
 114 TELEGRAM_BOT_TOKEN=
 115 DEFAULT_RECIPIENT_TELEGRAM=
 116 declare -A role_recipients_telegram=()
 117
 118 # email configs
 119 DEFAULT_RECIPIENT_EMAIL="root"
 120 declare -A role_recipients_email=()
 121
 122 # load the user configuration
 123 # this will overwrite the variables above
 124 if [ -f "${NETDATA_CONFIG_DIR}/health_alarm_notify.conf" ]
 125     then
 126     source "${NETDATA_CONFIG_DIR}/health_alarm_notify.conf"
 127 fi
 128
 129 # -----------------------------------------------------------------------------
 130 # filter a recipient based on alarm event severity
 131
 132 filter_recipient_by_criticality() {
 133     local method="${1}" x="${2}" r s
 134     shift
 135
 136     r="${x/|*/}" # the recipient
 137     s="${x/*|/}" # the severity required for notifying this recipient
 138
 139     # no severity filtering for this person
 140     [ "${r}" = "${s}" ] && return 0
 141
 142     # the severity is invalid
 143     s="${s^^}"
 144     [ "${s}" != "CRITICAL" ] && return 0
 145
 146     # the new or the old status matches the severity
 147     if [ "${s}" = "${status}" -o "${s}" = "${old_status}" ]
 148         then
 149         [ ! -d "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}" ] && \
 150             mkdir -p "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}"
 151
 152         # we need to keep track of the notifications we sent
 153         # so that the same user will receive the recovery
 154         # even if old_status does not match the required severity
 155         touch "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}"
 156         return 0
 157     fi
 158
 159     # it is a cleared alarm we have sent notification for
 160     if [ "${status}" != "WARNING" -a "${status}" != "CRITICAL" -a -f "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}" ]
 161         then
 162         rm "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}"
 163         return 0
 164     fi
 165
 166     return 1
 167 }
 168
 169 # -----------------------------------------------------------------------------
 170 # find the recipients' addresses per method
 171
 172 declare -A arr_slack=()
 173 declare -A arr_pushover=()
 174 declare -A arr_pushbullet=()
 175 declare -A arr_telegram=()
 176 declare -A arr_email=()
 177
 178 # netdata may call us with multiple roles, and roles may have multiple but
 179 # overlapping recipients - so, here we find the unique recipients.
 180 for x in ${roles//,/ }
 181 do
 182     # the roles 'silent' and 'disabled' mean:
 183     # don't send a notification for this role
 184     [ "${x}" = "silent" -o "${x}" = "disabled" ] && continue
 185
 186     # email
 187     a="${role_recipients_email[${x}]}"
 188     [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_EMAIL}"
 189     for r in ${a//,/ }
 190     do
 191         [ "${r}" != "disabled" ] && filter_recipient_by_criticality email "${r}" && arr_email[${r/|*/}]="1"
 192     done
 193
 194     # pushover
 195     a="${role_recipients_pushover[${x}]}"
 196     [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_PUSHOVER}"
 197     for r in ${a//,/ }
 198     do
 199         [ "${r}" != "disabled" ] && filter_recipient_by_criticality pushover "${r}" && arr_pushover[${r/|*/}]="1"
 200     done
 201
 202     # pushbullet
 203     a="${role_recipients_pushbullet[${x}]}"
 204     [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_PUSHBULLET}"
 205     for r in ${a//,/ }
 206     do
 207         [ "${r}" != "disabled" ] && filter_recipient_by_criticality pushbullet "${r}" && arr_pushbullet[${r/|*/}]="1"
 208     done
 209
 210     # telegram
 211     a="${role_recipients_telegram[${x}]}"
 212     [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_TELEGRAM}"
 213     for r in ${a//,/ }
 214     do
 215         [ "${r}" != "disabled" ] && filter_recipient_by_criticality telegram "${r}" && arr_telegram[${r/|*/}]="1"
 216     done
 217
 218     # slack
 219     a="${role_recipients_slack[${x}]}"
 220     [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_SLACK}"
 221     for r in ${a//,/ }
 222     do
 223         [ "${r}" != "disabled" ] && filter_recipient_by_criticality slack "${r}" && arr_slack[${r/|*/}]="1"
 224     done
 225 done
 226
 227 # build the list of slack recipients (channels)
 228 to_slack="${!arr_slack[*]}"
 229 [ -z "${to_slack}" ] && SEND_SLACK="NO"
 230
 231 # build the list of pushover recipients (user tokens)
 232 to_pushover="${!arr_pushover[*]}"
 233 [ -z "${to_pushover}" ] && SEND_PUSHOVER="NO"
 234
 235 # build the list of pushbulet recipients (user tokens)
 236 to_pushbullet="${!arr_pushbullet[*]}"
 237 [ -z "${to_pushbullet}" ] && SEND_PUSHBULLET="NO"
 238
 239 # check array of telegram recipients (chat ids)
 240 to_telegram="${!arr_telegram[*]}"
 241 [ -z "${to_telegram}" ] && SEND_TELEGRAM="NO"
 242
 243 # build the list of email recipients (email addresses)
 244 to_email=
 245 for x in "${!arr_email[@]}"
 246 do
 247     [ ! -z "${to_email}" ] && to_email="${to_email}, "
 248     to_email="${to_email}${x}"
 249 done
 250 [ -z "${to_email}" ] && SEND_EMAIL="NO"
 251
 252
 253 # -----------------------------------------------------------------------------
 254 # verify the delivery methods supported
 255
 256 # check slack
 257 [ -z "${SLACK_WEBHOOK_URL}" ] && SEND_SLACK="NO"
 258
 259 # check pushover
 260 [ -z "${PUSHOVER_APP_TOKEN}" ] && SEND_PUSHOVER="NO"
 261
 262 # check pushbullet
 263 [ -z "${DEFAULT_RECIPIENT_PUSHBULLET}" ] && SEND_PUSHBULLET="NO"
 264
 265 # check telegram
 266 [ -z "${TELEGRAM_BOT_TOKEN}" ] && SEND_TELEGRAM="NO"
 267
 268 if [ \( "${SEND_PUSHOVER}" = "YES" -o "${SEND_SLACK}" = "YES" -o "${SEND_TELEGRAM}" = "YES" -o "${SEND_PUSHBULLET}" = "YES" \) -a -z "${curl}" ]
 269     then
 270     curl="$(which curl 2>/dev/null || command -v curl 2>/dev/null)"
 271     if [ -z "${curl}" ]
 272         then
 273         SEND_PUSHOVER="NO"
 274         SEND_PUSHBULLET="NO"
 275         SEND_TELEGRAM="NO"
 276         SEND_SLACK="NO"
 277     fi
 278 fi
 279
 280 if [ "${SEND_EMAIL}" = "YES" -a -z "${sendmail}" ]
 281     then
 282     sendmail="$(which sendmail 2>/dev/null || command -v sendmail 2>/dev/null)"
 283     [ -z "${sendmail}" ] && SEND_EMAIL="NO"
 284 fi
 285
 286 # check that we have at least a method enabled
 287 if [ "${SEND_EMAIL}" != "YES" -a "${SEND_PUSHOVER}" != "YES" -a "${SEND_TELEGRAM}" != "YES" -a "${SEND_SLACK}" != "YES" -a "${SEND_PUSHBULLET}" != "YES" ]
 288     then
 289     echo >&2 "All notification methods are disabled. Not sending a notification."
 290     exit 1
 291 fi
 292
 293 # -----------------------------------------------------------------------------
 294 # get the system hostname
 295
 296 [ -z "${host}" ] && host="${NETDATA_HOSTNAME}"
 297 [ -z "${host}" ] && host="${NETDATA_REGISTRY_HOSTNAME}"
 298 [ -z "${host}" ] && host="$(hostname 2>/dev/null)"
 299
 300 # -----------------------------------------------------------------------------
 301 # get the date the alarm happened
 302
 303 date="$(date --date=@${when} 2>/dev/null)"
 304 [ -z "${date}" ] && date="$(date 2>/dev/null)"
 305
 306 # -----------------------------------------------------------------------------
 307 # URL encode a string
 308
 309 urlencode() {
 310     local string="${1}" strlen encoded pos c o
 311
 312     strlen=${#string}
 313     for (( pos=0 ; pos<strlen ; pos++ ))
 314     do
 315         c=${string:$pos:1}
 316         case "$c" in
 317             [-_.~a-zA-Z0-9])
 318                 o="${c}"
 319                 ;;
 320
 321             *)
 322                 printf -v o '%%%02x' "'$c"
 323                 ;;
 324         esac
 325         encoded+="${o}"
 326     done
 327
 328     REPLY="${encoded}"
 329     echo "${REPLY}"
 330 }
 331
 332 # -----------------------------------------------------------------------------
 333 # convert a duration in seconds, to a human readable duration
 334 # using DAYS, MINUTES, SECONDS
 335
 336 duration4human() {
 337     local s="${1}" d=0 h=0 m=0 ds="day" hs="hour" ms="minute" ss="second" ret
 338     d=$(( s / 86400 ))
 339     s=$(( s - (d * 86400) ))
 340     h=$(( s / 3600 ))
 341     s=$(( s - (h * 3600) ))
 342     m=$(( s / 60 ))
 343     s=$(( s - (m * 60) ))
 344
 345     if [ ${d} -gt 0 ]
 346     then
 347         [ ${m} -ge 30 ] && h=$(( h + 1 ))
 348         [ ${d} -gt 1 ] && ds="days"
 349         [ ${h} -gt 1 ] && hs="hours"
 350         if [ ${h} -gt 0 ]
 351         then
 352             ret="${d} ${ds} and ${h} ${hs}"
 353         else
 354             ret="${d} ${ds}"
 355         fi
 356     elif [ ${h} -gt 0 ]
 357     then
 358         [ ${s} -ge 30 ] && m=$(( m + 1 ))
 359         [ ${h} -gt 1 ] && hs="hours"
 360         [ ${m} -gt 1 ] && ms="minutes"
 361         if [ ${m} -gt 0 ]
 362         then
 363             ret="${h} ${hs} and ${m} ${ms}"
 364         else
 365             ret="${h} ${hs}"
 366         fi
 367     elif [ ${m} -gt 0 ]
 368     then
 369         [ ${m} -gt 1 ] && ms="minutes"
 370         [ ${s} -gt 1 ] && ss="seconds"
 371         if [ ${s} -gt 0 ]
 372         then
 373             ret="${m} ${ms} and ${s} ${ss}"
 374         else
 375             ret="${m} ${ms}"
 376         fi
 377     else
 378         [ ${s} -gt 1 ] && ss="seconds"
 379         ret="${s} ${ss}"
 380     fi
 381
 382     REPLY="${ret}"
 383     echo "${REPLY}"
 384 }
 385
 386 # -----------------------------------------------------------------------------
 387 # email sender
 388
 389 send_email() {
 390     local ret=
 391     if [ "${SEND_EMAIL}" = "YES" ]
 392         then
 393
 394         "${sendmail}" -t
 395         ret=$?
 396
 397         if [ $ret -eq 0 ]
 398         then
 399             echo >&2 "${me}: Sent email notification for: ${host} ${chart}.${name} is ${status} to '${to_email}'"
 400             return 0
 401         else
 402             echo >&2 "${me}: Failed to send email notification for: ${host} ${chart}.${name} is ${status} to '${to_email}' with error code ${ret}."
 403             return 1
 404         fi
 405     fi
 406
 407     return 1
 408 }
 409
 410 # -----------------------------------------------------------------------------
 411 # pushover sender
 412
 413 send_pushover() {
 414     local apptoken="${1}" usertokens="${2}" when="${3}" url="${4}" status="${5}" title="${6}" message="${7}" httpcode sent=0 user priority
 415
 416     if [ "${SEND_PUSHOVER}" = "YES" -a ! -z "${apptoken}" -a ! -z "${usertokens}" -a ! -z "${title}" -a ! -z "${message}" ]
 417         then
 418
 419         # https://pushover.net/api
 420         priority=-2
 421         case "${status}" in
 422             CLEAR) priority=-1;;   # low priority: no sound or vibration
 423             WARNING) priotity=0;;  # normal priority: respect quiet hours
 424             CRITICAL) priority=1;; # high priority: bypass quiet hours
 425             *) priority=-2;;       # lowest priority: no notification at all
 426         esac
 427
 428         for user in ${usertokens}
 429         do
 430             httpcode=$(${curl} --write-out %{http_code} --silent --output /dev/null \
 431                 --form-string "token=${apptoken}" \
 432                 --form-string "user=${user}" \
 433                 --form-string "html=1" \
 434                 --form-string "title=${title}" \
 435                 --form-string "message=${message}" \
 436                 --form-string "timestamp=${when}" \
 437                 --form-string "url=${url}" \
 438                 --form-string "url_title=Open netdata dashboard to view the alarm" \
 439                 --form-string "priority=${priority}" \
 440                 https://api.pushover.net/1/messages.json)
 441
 442             if [ "${httpcode}" == "200" ]
 443             then
 444                 echo >&2 "${me}: Sent pushover notification for: ${host} ${chart}.${name} is ${status} to '${user}'"
 445                 sent=$((sent + 1))
 446             else
 447                 echo >&2 "${me}: Failed to send pushover notification for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP error code ${httpcode}."
 448             fi
 449         done
 450
 451         [ ${sent} -gt 0 ] && return 0
 452     fi
 453
 454     return 1
 455 }
 456
 457 # -----------------------------------------------------------------------------
 458 # pushbullet sender
 459
 460 send_pushbullet() {
 461 <<<<<<< HEAD
 462     local userapikey="${1}" recipients="${2}" message="${3}" title="${4}" httpcode sent=0 user
 463 =======
 464     local userapikey="${1}" recipients="${2}" message="${3}" title="${4}" httpcode sent=0 user priority
 465 >>>>>>> 863efae676a63da80a868908ad03eab89e7380ab
 466
 467     if [ "${SEND_PUSHBULLET}" = "YES" -a ! -z "${userapikey}" -a ! -z "${recipients}" -a ! -z "${message}" -a ! -z "${title}" ]
 468         then
 469         #https://docs.pushbullet.com/#create-push
 470         for user in ${recipients}
 471         do
 472             httpcode=$(${curl} --write-out %{http_code} --silent --output /dev/null \
 473                  -u "$userapikey": \
 474                  -d type="note" \
 475                  --data-urlencode email="${user}" \
 476                  --data-urlencode body="${message}" \
 477                  --data-urlencode title="${title}" \
 478                  "https://api.pushbullet.com/v2/pushes"
 479                 )
 480
 481             if [ "${httpcode}" == "200" ]
 482             then
 483                 echo >&2 "${me}: Sent pushbullet notification for: ${host} ${chart}.${name} is ${status} to '${user}'"
 484                 sent=$((sent + 1))
 485             else
 486                 echo >&2 "${me}: Failed to send pushbullet notification for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP error code ${httpcode}."
 487             fi
 488         done
 489
 490         [ ${sent} -gt 0 ] && return 0
 491     fi
 492
 493     return 1
 494 }
 495
 496
 497
 498 # -----------------------------------------------------------------------------
 499 # telegram sender
 500
 501 send_telegram() {
 502     local bottoken="${1}" chatids="${2}" message="${3}" httpcode sent=0 chatid disableNotification=""
 503
 504     if [ "${status}" = "CLEAR" ]; then disableNotification="--data-urlencode disable_notification=true"; fi
 505
 506     if [ "${SEND_TELEGRAM}" = "YES" -a ! -z "${bottoken}" -a ! -z "${chatids}" -a ! -z "${message}" ];
 507     then
 508         for chatid in ${chatids}
 509         do
 510             # https://core.telegram.org/bots/api#sendmessage
 511             httpcode=$(${curl} --write-out %{http_code} --silent --output /dev/null ${disableNotification} \
 512                 --data-urlencode "parse_mode=HTML" \
 513                 --data-urlencode "disable_web_page_preview=true" \
 514                 --data-urlencode "text=$message" \
 515                 "https://api.telegram.org/bot${bottoken}/sendMessage?chat_id=$chatid")
 516
 517             if [ "${httpcode}" == "200" ]
 518             then
 519                 echo >&2 "${me}: Sent telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}'"
 520                 sent=$((sent + 1))
 521             elif [ "${httpcode}" == "401" ]
 522             then
 523                 echo >&2 "${me}: Failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}': Wrong bot token."
 524             else
 525                 echo >&2 "${me}: Failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}' with HTTP error code ${httpcode}."
 526             fi
 527         done
 528
 529         [ ${sent} -gt 0 ] && return 0
 530     fi
 531
 532     return 1
 533 }
 534
 535 # -----------------------------------------------------------------------------
 536 # slack sender
 537
 538 send_slack() {
 539     local webhook="${1}" channels="${2}" httpcode sent=0 channel color payload
 540
 541     [ "${SEND_SLACK}" != "YES" ] && return 1
 542
 543     case "${status}" in
 544         WARNING) color="warning" ;;
 545         CRITICAL) color="danger" ;;
 546         CLEAR) color="good" ;;
 547         *) color="#777777" ;;
 548     esac
 549
 550     for channel in ${channels}
 551     do
 552         payload="$(cat <<EOF
 553         {
 554             "channel": "#${channel}",
 555             "username": "netdata on ${host}",
 556             "icon_url": "${images_base_url}/images/seo-performance-128.png",
 557             "text": "${host} ${status_message}, \`${chart}\` (_${family}_), *${alarm}*",
 558             "attachments": [
 559                 {
 560                     "fallback": "${alarm} - ${chart} (${family}) - ${info}",
 561                     "color": "${color}",
 562                     "title": "${alarm}",
 563                     "title_link": "${goto_url}",
 564                     "text": "${info}",
 565                     "fields": [
 566                         {
 567                             "title": "${chart}",
 568                             "short": true
 569                         },
 570                         {
 571                             "title": "${family}",
 572                             "short": true
 573                         }
 574                     ],
 575                     "thumb_url": "${image}",
 576                     "footer": "<${goto_url}|${host}>",
 577                     "ts": ${when}
 578                 }
 579             ]
 580         }
 581 EOF
 582         )"
 583
 584         httpcode=$(${curl} --write-out %{http_code} --silent --output /dev/null -X POST --data-urlencode "payload=${payload}" "${webhook}")
 585         if [ "${httpcode}" == "200" ]
 586         then
 587             echo >&2 "${me}: Sent slack notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
 588             sent=$((sent + 1))
 589         else
 590             echo >&2 "${me}: Failed to send slack notification for: ${host} ${chart}.${name} is ${status} to '${channel}', with HTTP error code ${httpcode}."
 591         fi
 592     done
 593
 594     [ ${sent} -gt 0 ] && return 0
 595
 596     return 1
 597 }
 598
 599
 600 # -----------------------------------------------------------------------------
 601 # prepare the content of the notification
 602
 603 # the url to send the user on click
 604 urlencode "${NETDATA_REGISTRY_HOSTNAME}" >/dev/null; url_host="${REPLY}"
 605 urlencode "${chart}" >/dev/null; url_chart="${REPLY}"
 606 urlencode "${family}" >/dev/null; url_family="${REPLY}"
 607 urlencode "${name}" >/dev/null; url_name="${REPLY}"
 608 goto_url="${NETDATA_REGISTRY_URL}/goto-host-from-alarm.html?host=${url_host}&chart=${url_chart}&family=${url_family}&alarm=${url_name}&alarm_unique_id=${unique_id}&alarm_id=${alarm_id}&alarm_event_id=${event_id}"
 609
 610 # the severity of the alarm
 611 severity="${status}"
 612
 613 # the time the alarm was raised
 614 duration4human ${duration} >/dev/null; duration_txt="${REPLY}"
 615 duration4human ${non_clear_duration} >/dev/null; non_clear_duration_txt="${REPLY}"
 616 raised_for="(was ${old_status,,} for ${duration_txt})"
 617
 618 # the key status message
 619 status_message="status unknown"
 620
 621 # the color of the alarm
 622 color="grey"
 623
 624 # the alarm value
 625 alarm="${name//_/ } = ${value} ${units}"
 626
 627 # the image of the alarm
 628 image="${images_base_url}/images/seo-performance-128.png"
 629
 630 # prepare the title based on status
 631 case "${status}" in
 632         CRITICAL)
 633         image="${images_base_url}/images/alert-128-red.png"
 634         status_message="is critical"
 635         color="#ca414b"
 636         ;;
 637
 638     WARNING)
 639         image="${images_base_url}/images/alert-128-orange.png"
 640         status_message="needs attention"
 641         color="#caca4b"
 642                 ;;
 643
 644         CLEAR)
 645         image="${images_base_url}/images/check-mark-2-128-green.png"
 646         status_message="recovered"
 647                 color="#77ca6d"
 648
 649                 # don't show the value when the status is CLEAR
 650                 # for certain alarms, this value might not have any meaning
 651                 alarm="${name//_/ } ${raised_for}"
 652                 ;;
 653 esac
 654
 655 if [ "${status}" = "CLEAR" ]
 656 then
 657     severity="Recovered from ${old_status}"
 658     if [ $non_clear_duration -gt $duration ]
 659     then
 660         raised_for="(alarm was raised for ${non_clear_duration_txt})"
 661     fi
 662
 663 elif [ "${old_status}" = "WARNING" -a "${status}" = "CRITICAL" ]
 664 then
 665     severity="Escalated to ${status}"
 666     if [ $non_clear_duration -gt $duration ]
 667     then
 668         raised_for="(alarm is raised for ${non_clear_duration_txt})"
 669     fi
 670
 671 elif [ "${old_status}" = "CRITICAL" -a "${status}" = "WARNING" ]
 672 then
 673     severity="Demoted to ${status}"
 674     if [ $non_clear_duration -gt $duration ]
 675     then
 676         raised_for="(alarm is raised for ${non_clear_duration_txt})"
 677     fi
 678
 679 else
 680     raised_for=
 681 fi
 682
 683 # prepare HTML versions of elements
 684 info_html=
 685 [ ! -z "${info}" ] && info_html=" <small><br/>${info}</small>"
 686
 687 raised_for_html=
 688 [ ! -z "${raised_for}" ] && raised_for_html="<br/><small>${raised_for}</small>"
 689
 690 # -----------------------------------------------------------------------------
 691 # send the slack notification
 692
 693 # slack aggregates posts from the same username
 694 # so we use "${host} ${status}" as the bot username, to make them diff
 695
 696 send_slack "${SLACK_WEBHOOK_URL}" "${to_slack}"
 697 SENT_SLACK=$?
 698
 699 # -----------------------------------------------------------------------------
 700 # send the pushover notification
 701
 702 send_pushover "${PUSHOVER_APP_TOKEN}" "${to_pushover}" "${when}" "${goto_url}" "${status}" "${host} ${status_message} - ${name//_/ } - ${chart}" "
 703 <font color=\"${color}\"><b>${alarm}</b></font>${info_html}<br/>&nbsp;
 704 <small><b>${chart}</b><br/>Chart<br/>&nbsp;</small>
 705 <small><b>${family}</b><br/>Family<br/>&nbsp;</small>
 706 <small><b>${severity}</b><br/>Severity<br/>&nbsp;</small>
 707 <small><b>${date}${raised_for_html}</b><br/>Time<br/>&nbsp;</small>
 708 <a href=\"${goto_url}\">View Netdata</a><br/>&nbsp;
 709 <small><small>The source of this alarm is line ${src}</small></small>
 710 "
 711
 712 SENT_PUSHOVER=$?
 713
 714 # -----------------------------------------------------------------------------
 715 # send the pushbullet notification
 716
 717 pushbullet_message="
 718 ${alarm}
 719 Severity: ${severity}
 720 Chart: ${chart}
 721 Family: ${family}
 722 To View Netdata go to: ${goto_url}
 723 The source of this alarm is line ${src}"
 724 pushbullet_title="${status} at ${host} ${status_message} - ${name//_/ } - ${chart}}"
 725 send_pushbullet "${PUSHBULLET_ACCESS_TOKEN}" "${to_pushbullet}" "$pushbullet_message" "$pushbullet_title"
 726
 727 SENT_PUSHBULLET=$?
 728
 729 # -----------------------------------------------------------------------------
 730 # send the telegram.org message
 731
 732 # https://core.telegram.org/bots/api#formatting-options
 733 telegram_message="<b>${severity}"
 734 [ "${status_message}" != "recovered" ] && telegram_message="${telegram_message}, ${status_message}"
 735 telegram_message="${telegram_message}
 736 ${chart} (${family})</b>
 737 <a href=\"${goto_url}\">${alarm}</a>
 738 <i>${info}</i>"
 739
 740 send_telegram "${TELEGRAM_BOT_TOKEN}" "${to_telegram}" "${telegram_message}"
 741
 742 SENT_TELEGRAM=$?
 743
 744 # -----------------------------------------------------------------------------
 745 # send the email
 746
 747 send_email <<EOF
 748 To: ${to_email}
 749 Subject: ${host} ${status_message} - ${name//_/ } - ${chart}
 750 Content-Type: text/html
 751
 752 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
 753 <html xmlns="http://www.w3.org/1999/xhtml" style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0; padding: 0;">
 754 <body style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 14px; width: 100% !important; min-height: 100%; line-height: 1.6; background: #f6f6f6; margin:0; padding: 0;">
 755 <table>
 756     <tbody>
 757     <tr>
 758         <td style="vertical-align: top;" valign="top"></td>
 759         <td width="700" style="vertical-align: top; display: block !important; max-width: 700px !important; clear: both !important; margin: 0 auto; padding: 0;" valign="top">
 760             <div style="max-width: 700px; display: block; margin: 0 auto; padding: 20px;">
 761                 <table width="100%" cellpadding="0" cellspacing="0" style="background: #fff; border: 1px solid #e9e9e9;">
 762                     <tbody>
 763                     <tr>
 764                         <td bgcolor="#eee" style="padding: 5px 20px 5px 20px; background-color: #eee;">
 765                             <div style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 20px; color: #777; font-weight: bold;">netdata notification</div>
 766                         </td>
 767                     </tr>
 768                     <tr>
 769                         <td bgcolor="${color}" style="font-size: 16px; vertical-align: top; font-weight: 400; text-align: center; margin: 0; padding: 10px; color: #ffffff; background: ${color} !important; border: 1px solid ${color}; border-top-color: ${color};" align="center" valign="top">
 770                             <h1 style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-weight: 400; margin: 0;">${host} ${status_message}</h1>
 771                         </td>
 772                     </tr>
 773                     <tr>
 774                         <td style="vertical-align: top;" valign="top">
 775                             <div style="margin: 0; padding: 20px; max-width: 700px;">
 776                                 <table width="100%" cellpadding="0" cellspacing="0" style="max-width:700px">
 777                                     <tbody>
 778                                     <tr>
 779                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding:0 0 20px;" align="left" valign="top">
 780                                             <span>${chart}</span>
 781                                             <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Chart</span>
 782                                         </td>
 783                                     </tr>
 784                                     <tr style="margin: 0; padding: 0;">
 785                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
 786                                             <span><b>${alarm}</b>${info_html}</span>
 787                                             <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Alarm</span>
 788                                         </td>
 789                                     </tr>
 790                                     <tr>
 791                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
 792                                             <span>${family}</span>
 793                                             <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Family</span>
 794                                         </td>
 795                                     </tr>
 796                                     <tr style="margin: 0; padding: 0;">
 797                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
 798                                             <span>${severity}</span>
 799                                             <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Severity</span>
 800                                         </td>
 801                                     </tr>
 802                                     <tr style="margin: 0; padding: 0;">
 803                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top"><span>${date}</span>
 804                                             <span>${raised_for_html}</span> <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Time</span>
 805                                         </td>
 806                                     </tr>
 807                                     <tr style="margin: 0; padding: 0;">
 808                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;">
 809                                             <a href="${goto_url}" style="font-size: 14px; color: #ffffff; text-decoration: none; line-height: 1.5; font-weight: bold; text-align: center; display: inline-block; text-transform: capitalize; background: #35568d; border-width: 1px; border-style: solid; border-color: #2b4c86; margin: 0; padding: 10px 15px;" target="_blank">View Netdata</a>
 810                                         </td>
 811                                     </tr>
 812                                     <tr style="text-align: center; margin: 0; padding: 0;">
 813                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 11px; vertical-align: top; margin: 0; padding: 10px 0 0 0; color: #666666;" align="center" valign="bottom">The source of this alarm is line <code>${src}</code>
 814                                         </td>
 815                                     </tr>
 816                                     <tr style="text-align: center; margin: 0; padding: 0;">
 817                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 12px; vertical-align: top; margin:0; padding: 20px 0 0 0; color: #666666; border-top: 1px solid #f0f0f0;" align="center" valign="bottom">Sent by
 818                                             <a href="https://mynetdata.io/" target="_blank">netdata</a>, the real-time performance monitoring.
 819                                         </td>
 820                                     </tr>
 821                                     </tbody>
 822                                 </table>
 823                             </div>
 824                         </td>
 825                     </tr>
 826                     </tbody>
 827                 </table>
 828             </div>
 829         </td>
 830     </tr>
 831     </tbody>
 832 </table>
 833 </body>
 834 </html>
 835 EOF
 836
 837 SENT_EMAIL=$?
 838
 839 # -----------------------------------------------------------------------------
 840 # let netdata know
 841
 842 # we did send something
 843 [ ${SENT_EMAIL} -eq 0 -o ${SENT_PUSHOVER} -eq 0 -o ${SENT_TELEGRAM} -eq 0 -o ${SENT_SLACK} -eq 0 -o ${SENT_PUSHBULLET} -eq 0 ] && exit 0
 844
 845 # we did not send anything
 846 exit 1