plugins.d/alarm-notify.sh

   1 #!/usr/bin/env bash
   2
   3 # netdata
   4 # real-time performance and health monitoring, done right!
   5 # (C) 2016 Costa Tsaousis <costa@tsaousis.gr>
   6 # GPL v3+
   7 #
   8 # Script to send alarm notifications for netdata
   9 #
  10 # Features:
  11 #  - multiple notification methods
  12 #  - multiple roles per alarm
  13 #  - multiple recipients per role
  14 #  - severity filtering per recipient
  15 #
  16 # Supported notification methods:
  17 #  - emails
  18 #  - slack.com notifications
  19 #  - pushover.net notifications
  20 #  - pushbullet.com push notifications by Tiago Peralta @tperalta82 PR #1070
  21 #  - telegram.org notifications by @hashworks PR #1002
  22 #
  23
  24 # -----------------------------------------------------------------------------
  25 # testing notifications
  26
  27 if [ \( "${1}" = "test" -o "${2}" = "test" \) -a "${#}" -le 2 ]
  28 then
  29     if [ "${2}" = "test" ]
  30     then
  31         recipient="${1}"
  32     else
  33         recipient="${2}"
  34     fi
  35
  36     [ -z "${recipient}" ] && recipient="sysadmin"
  37
  38     id=1
  39     last="CLEAR"
  40     for x in "CRITICAL" "WARNING" "CLEAR"
  41     do
  42         echo >&2
  43         echo >&2 "# SENDING TEST ${x} ALARM TO ROLE: ${recipient}"
  44
  45         "${0}" "${recipient}" "$(hostname)" 1 1 "${id}" "$(date +%s)" "test_alarm" "test.chart" "test.family" "${x}" "${last}" 100 90 "${0}" 1 $((0 + id)) "units" "this is a test alarm to verify notifications work"
  46         if [ $? -ne 0 ]
  47         then
  48             echo >&2 "# FAILED"
  49         else
  50             echo >&2 "# OK"
  51         fi
  52
  53         last="${x}"
  54         id=$((id + 1))
  55     done
  56
  57     exit 1
  58 fi
  59
  60 export PATH="${PATH}:/sbin:/usr/sbin:/usr/local/sbin"
  61 export LC_ALL=C
  62
  63 # -----------------------------------------------------------------------------
  64
  65 PROGRAM_NAME="$(basename "${0}")"
  66
  67 logdate() {
  68     date "+%Y-%m-%d %H:%M:%S"
  69 }
  70
  71 log() {
  72     local status="${1}"
  73     shift
  74
  75     echo >&2 "$(logdate): ${PROGRAM_NAME}: ${status}: ${*}"
  76
  77 }
  78
  79 warning() {
  80     log WARNING "${@}"
  81 }
  82
  83 error() {
  84     log ERROR "${@}"
  85 }
  86
  87 info() {
  88     log INFO "${@}"
  89 }
  90
  91 fatal() {
  92     log FATAL "${@}"
  93     exit 1
  94 }
  95
  96 debug=0
  97 debug() {
  98     [ ${debug} -eq 1 ] && log DEBUG "${@}"
  99 }
 100
 101 # -----------------------------------------------------------------------------
 102
 103 # check for BASH v4+ (required for associative arrays)
 104 [ $(( ${BASH_VERSINFO[0]} )) -lt 4 ] && \
 105     fatal "BASH version 4 or later is required (this is ${BASH_VERSION})."
 106
 107 # -----------------------------------------------------------------------------
 108 # defaults to allow running this script by hand
 109
 110 NETDATA_CONFIG_DIR="${NETDATA_CONFIG_DIR-/etc/netdata}"
 111 NETDATA_CACHE_DIR="${NETDATA_CACHE_DIR-/var/cache/netdata}"
 112 [ -z "${NETDATA_REGISTRY_URL}" ] && NETDATA_REGISTRY_URL="https://registry.my-netdata.io"
 113 [ -z "${NETDATA_HOSTNAME}" ] && NETDATA_HOSTNAME="$(hostname)"
 114 [ -z "${NETDATA_REGISTRY_HOSTNAME}" ] && NETDATA_REGISTRY_HOSTNAME="${NETDATA_HOSTNAME}"
 115
 116 # -----------------------------------------------------------------------------
 117 # parse command line parameters
 118
 119 roles="${1}"       # the roles that should be notified for this event
 120 host="${2}"        # the host generated this event
 121 unique_id="${3}"   # the unique id of this event
 122 alarm_id="${4}"    # the unique id of the alarm that generated this event
 123 event_id="${5}"    # the incremental id of the event, for this alarm id
 124 when="${6}"        # the timestamp this event occurred
 125 name="${7}"        # the name of the alarm, as given in netdata health.d entries
 126 chart="${8}"       # the name of the chart (type.id)
 127 family="${9}"      # the family of the chart
 128 status="${10}"     # the current status : REMOVED, UNITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
 129 old_status="${11}" # the previous status: REMOVED, UNITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
 130 value="${12}"      # the current value of the alarm
 131 old_value="${13}"  # the previous value of the alarm
 132 src="${14}"        # the line number and file the alarm has been configured
 133 duration="${15}"   # the duration in seconds of the previous alarm state
 134 non_clear_duration="${16}" # the total duration in seconds this is/was non-clear
 135 units="${17}"      # the units of the value
 136 info="${18}"       # a short description of the alarm
 137
 138 # -----------------------------------------------------------------------------
 139 # screen statuses we don't need to send a notification
 140
 141 # don't do anything if this is not WARNING, CRITICAL or CLEAR
 142 if [ "${status}" != "WARNING" -a "${status}" != "CRITICAL" -a "${status}" != "CLEAR" ]
 143 then
 144     info "not sending notification for ${status} on '${chart}.${name}'"
 145     exit 1
 146 fi
 147
 148 # don't do anything if this is CLEAR, but it was not WARNING or CRITICAL
 149 if [ "${old_status}" != "WARNING" -a "${old_status}" != "CRITICAL" -a "${status}" = "CLEAR" ]
 150 then
 151     info "not sending notification for ${status} on '${chart}.${name}' (last status was ${old_status})"
 152     exit 1
 153 fi
 154
 155 # -----------------------------------------------------------------------------
 156 # load configuration
 157
 158 # By default fetch images from the global public registry.
 159 # This is required by default, since all notification methods need to download
 160 # images via the Internet, and private registries might not be reachable.
 161 # This can be overwritten at the configuration file.
 162 images_base_url="https://registry.my-netdata.io"
 163
 164 # needed commands
 165 # if empty they will be searched in the system path
 166 curl=
 167 sendmail=
 168
 169 # enable / disable features
 170 SEND_SLACK="YES"
 171 SEND_PUSHOVER="YES"
 172 SEND_TWILIO="YES"
 173 SEND_TELEGRAM="YES"
 174 SEND_EMAIL="YES"
 175 SEND_PUSHBULLET="YES"
 176
 177 # slack configs
 178 SLACK_WEBHOOK_URL=
 179 DEFAULT_RECIPIENT_SLACK=
 180 declare -A role_recipients_slack=()
 181
 182 # pushover configs
 183 PUSHOVER_APP_TOKEN=
 184 DEFAULT_RECIPIENT_PUSHOVER=
 185 declare -A role_recipients_pushover=()
 186
 187 # pushbullet configs
 188 PUSHBULLET_ACCESS_TOKEN=
 189 DEFAULT_RECIPIENT_PUSHBULLET=
 190 declare -A role_recipients_pushbullet=()
 191
 192 # twilio configs
 193 TWILIO_ACCOUNT_SID=
 194 TWILIO_ACCOUNT_TOKEN=
 195 TWILIO_NUMBER=
 196 DEFAULT_RECIPIENT_TWILIO=
 197 declare -A role_recipients_twilio=()
 198
 199 # telegram configs
 200 TELEGRAM_BOT_TOKEN=
 201 DEFAULT_RECIPIENT_TELEGRAM=
 202 declare -A role_recipients_telegram=()
 203
 204 # email configs
 205 DEFAULT_RECIPIENT_EMAIL="root"
 206 declare -A role_recipients_email=()
 207
 208 # load the user configuration
 209 # this will overwrite the variables above
 210 if [ -f "${NETDATA_CONFIG_DIR}/health_alarm_notify.conf" ]
 211     then
 212     source "${NETDATA_CONFIG_DIR}/health_alarm_notify.conf"
 213 fi
 214
 215 # -----------------------------------------------------------------------------
 216 # filter a recipient based on alarm event severity
 217
 218 filter_recipient_by_criticality() {
 219     local method="${1}" x="${2}" r s
 220     shift
 221
 222     r="${x/|*/}" # the recipient
 223     s="${x/*|/}" # the severity required for notifying this recipient
 224
 225     # no severity filtering for this person
 226     [ "${r}" = "${s}" ] && return 0
 227
 228     # the severity is invalid
 229     s="${s^^}"
 230     [ "${s}" != "CRITICAL" ] && return 0
 231
 232     # the new or the old status matches the severity
 233     if [ "${s}" = "${status}" -o "${s}" = "${old_status}" ]
 234         then
 235         [ ! -d "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}" ] && \
 236             mkdir -p "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}"
 237
 238         # we need to keep track of the notifications we sent
 239         # so that the same user will receive the recovery
 240         # even if old_status does not match the required severity
 241         touch "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}"
 242         return 0
 243     fi
 244
 245     # it is a cleared alarm we have sent notification for
 246     if [ "${status}" != "WARNING" -a "${status}" != "CRITICAL" -a -f "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}" ]
 247         then
 248         rm "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}"
 249         return 0
 250     fi
 251
 252     return 1
 253 }
 254
 255 # -----------------------------------------------------------------------------
 256 # find the recipients' addresses per method
 257
 258 declare -A arr_slack=()
 259 declare -A arr_pushover=()
 260 declare -A arr_pushbullet=()
 261 declare -A arr_twilio=()
 262 declare -A arr_telegram=()
 263 declare -A arr_email=()
 264
 265 # netdata may call us with multiple roles, and roles may have multiple but
 266 # overlapping recipients - so, here we find the unique recipients.
 267 for x in ${roles//,/ }
 268 do
 269     # the roles 'silent' and 'disabled' mean:
 270     # don't send a notification for this role
 271     [ "${x}" = "silent" -o "${x}" = "disabled" ] && continue
 272
 273     # email
 274     a="${role_recipients_email[${x}]}"
 275     [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_EMAIL}"
 276     for r in ${a//,/ }
 277     do
 278         [ "${r}" != "disabled" ] && filter_recipient_by_criticality email "${r}" && arr_email[${r/|*/}]="1"
 279     done
 280
 281     # pushover
 282     a="${role_recipients_pushover[${x}]}"
 283     [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_PUSHOVER}"
 284     for r in ${a//,/ }
 285     do
 286         [ "${r}" != "disabled" ] && filter_recipient_by_criticality pushover "${r}" && arr_pushover[${r/|*/}]="1"
 287     done
 288
 289     # pushbullet
 290     a="${role_recipients_pushbullet[${x}]}"
 291     [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_PUSHBULLET}"
 292     for r in ${a//,/ }
 293     do
 294         [ "${r}" != "disabled" ] && filter_recipient_by_criticality pushbullet "${r}" && arr_pushbullet[${r/|*/}]="1"
 295     done
 296
 297     # twilio
 298     a="${role_recipients_twilio[${x}]}"
 299     [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_TWILIO}"
 300     for r in ${a//,/ }
 301     do
 302         [ "${r}" != "disabled" ] && filter_recipient_by_criticality twilio "${r}" && arr_twilio[${r/|*/}]="1"
 303     done
 304
 305     # telegram
 306     a="${role_recipients_telegram[${x}]}"
 307     [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_TELEGRAM}"
 308     for r in ${a//,/ }
 309     do
 310         [ "${r}" != "disabled" ] && filter_recipient_by_criticality telegram "${r}" && arr_telegram[${r/|*/}]="1"
 311     done
 312
 313     # slack
 314     a="${role_recipients_slack[${x}]}"
 315     [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_SLACK}"
 316     for r in ${a//,/ }
 317     do
 318         [ "${r}" != "disabled" ] && filter_recipient_by_criticality slack "${r}" && arr_slack[${r/|*/}]="1"
 319     done
 320 done
 321
 322 # build the list of slack recipients (channels)
 323 to_slack="${!arr_slack[*]}"
 324 [ -z "${to_slack}" ] && SEND_SLACK="NO"
 325
 326 # build the list of pushover recipients (user tokens)
 327 to_pushover="${!arr_pushover[*]}"
 328 [ -z "${to_pushover}" ] && SEND_PUSHOVER="NO"
 329
 330 # build the list of pushbulet recipients (user tokens)
 331 to_pushbullet="${!arr_pushbullet[*]}"
 332 [ -z "${to_pushbullet}" ] && SEND_PUSHBULLET="NO"
 333
 334 # build the list of twilio recipients (phone numbers)
 335 to_twilio="${!arr_twilio[*]}"
 336 [ -z "${to_twilio}" ] && SEND_TWILIO="NO"
 337
 338 # check array of telegram recipients (chat ids)
 339 to_telegram="${!arr_telegram[*]}"
 340 [ -z "${to_telegram}" ] && SEND_TELEGRAM="NO"
 341
 342 # build the list of email recipients (email addresses)
 343 to_email=
 344 for x in "${!arr_email[@]}"
 345 do
 346     [ ! -z "${to_email}" ] && to_email="${to_email}, "
 347     to_email="${to_email}${x}"
 348 done
 349 [ -z "${to_email}" ] && SEND_EMAIL="NO"
 350
 351
 352 # -----------------------------------------------------------------------------
 353 # verify the delivery methods supported
 354
 355 # check slack
 356 [ -z "${SLACK_WEBHOOK_URL}" ] && SEND_SLACK="NO"
 357
 358 # check pushover
 359 [ -z "${PUSHOVER_APP_TOKEN}" ] && SEND_PUSHOVER="NO"
 360
 361 # check pushbullet
 362 [ -z "${DEFAULT_RECIPIENT_PUSHBULLET}" ] && SEND_PUSHBULLET="NO"
 363
 364 # check twilio
 365 [ -z "${DEFAULT_RECIPIENT_TWILIO}" ] && SEND_TWILIO="NO"
 366
 367 # check telegram
 368 [ -z "${TELEGRAM_BOT_TOKEN}" ] && SEND_TELEGRAM="NO"
 369
 370 if [ \( "${SEND_PUSHOVER}" = "YES" -o "${SEND_SLACK}" = "YES" -o "${SEND_TWILIO}" = "YES" -o "${SEND_TELEGRAM}" = "YES" -o "${SEND_PUSHBULLET}" = "YES" \) -a -z "${curl}" ]
 371     then
 372     curl="$(which curl 2>/dev/null || command -v curl 2>/dev/null)"
 373     if [ -z "${curl}" ]
 374         then
 375         SEND_PUSHOVER="NO"
 376         SEND_PUSHBULLET="NO"
 377         SEND_TELEGRAM="NO"
 378         SEND_SLACK="NO"
 379         SEND_TWILIO="NO"
 380     fi
 381 fi
 382
 383 if [ "${SEND_EMAIL}" = "YES" -a -z "${sendmail}" ]
 384     then
 385     sendmail="$(which sendmail 2>/dev/null || command -v sendmail 2>/dev/null)"
 386     [ -z "${sendmail}" ] && SEND_EMAIL="NO"
 387 fi
 388
 389 # check that we have at least a method enabled
 390 if [ "${SEND_EMAIL}" != "YES" -a "${SEND_PUSHOVER}" != "YES" -a "${SEND_TELEGRAM}" != "YES" -a "${SEND_SLACK}" != "YES" -a "${SEND_TWILIO}" != "YES" -a  "${SEND_PUSHBULLET}" != "YES" ]
 391     then
 392     fatal "All notification methods are disabled. Not sending notification to '${role}' for '${name}' = '${value}' of chart '${chart}' for status '${status}'."
 393 fi
 394
 395 # -----------------------------------------------------------------------------
 396 # get the system hostname
 397
 398 [ -z "${host}" ] && host="${NETDATA_HOSTNAME}"
 399 [ -z "${host}" ] && host="${NETDATA_REGISTRY_HOSTNAME}"
 400 [ -z "${host}" ] && host="$(hostname 2>/dev/null)"
 401
 402 # -----------------------------------------------------------------------------
 403 # get the date the alarm happened
 404
 405 date="$(date --date=@${when} 2>/dev/null)"
 406 [ -z "${date}" ] && date="$(date 2>/dev/null)"
 407
 408 # -----------------------------------------------------------------------------
 409 # URL encode a string
 410
 411 urlencode() {
 412     local string="${1}" strlen encoded pos c o
 413
 414     strlen=${#string}
 415     for (( pos=0 ; pos<strlen ; pos++ ))
 416     do
 417         c=${string:${pos}:1}
 418         case "${c}" in
 419             [-_.~a-zA-Z0-9])
 420                 o="${c}"
 421                 ;;
 422
 423             *)
 424                 printf -v o '%%%02x' "'${c}"
 425                 ;;
 426         esac
 427         encoded+="${o}"
 428     done
 429
 430     REPLY="${encoded}"
 431     echo "${REPLY}"
 432 }
 433
 434 # -----------------------------------------------------------------------------
 435 # convert a duration in seconds, to a human readable duration
 436 # using DAYS, MINUTES, SECONDS
 437
 438 duration4human() {
 439     local s="${1}" d=0 h=0 m=0 ds="day" hs="hour" ms="minute" ss="second" ret
 440     d=$(( s / 86400 ))
 441     s=$(( s - (d * 86400) ))
 442     h=$(( s / 3600 ))
 443     s=$(( s - (h * 3600) ))
 444     m=$(( s / 60 ))
 445     s=$(( s - (m * 60) ))
 446
 447     if [ ${d} -gt 0 ]
 448     then
 449         [ ${m} -ge 30 ] && h=$(( h + 1 ))
 450         [ ${d} -gt 1 ] && ds="days"
 451         [ ${h} -gt 1 ] && hs="hours"
 452         if [ ${h} -gt 0 ]
 453         then
 454             ret="${d} ${ds} and ${h} ${hs}"
 455         else
 456             ret="${d} ${ds}"
 457         fi
 458     elif [ ${h} -gt 0 ]
 459     then
 460         [ ${s} -ge 30 ] && m=$(( m + 1 ))
 461         [ ${h} -gt 1 ] && hs="hours"
 462         [ ${m} -gt 1 ] && ms="minutes"
 463         if [ ${m} -gt 0 ]
 464         then
 465             ret="${h} ${hs} and ${m} ${ms}"
 466         else
 467             ret="${h} ${hs}"
 468         fi
 469     elif [ ${m} -gt 0 ]
 470     then
 471         [ ${m} -gt 1 ] && ms="minutes"
 472         [ ${s} -gt 1 ] && ss="seconds"
 473         if [ ${s} -gt 0 ]
 474         then
 475             ret="${m} ${ms} and ${s} ${ss}"
 476         else
 477             ret="${m} ${ms}"
 478         fi
 479     else
 480         [ ${s} -gt 1 ] && ss="seconds"
 481         ret="${s} ${ss}"
 482     fi
 483
 484     REPLY="${ret}"
 485     echo "${REPLY}"
 486 }
 487
 488 # -----------------------------------------------------------------------------
 489 # email sender
 490
 491 send_email() {
 492     local ret=
 493     if [ "${SEND_EMAIL}" = "YES" ]
 494         then
 495
 496         "${sendmail}" -t
 497         ret=$?
 498
 499         if [ ${ret} -eq 0 ]
 500         then
 501             info "sent email notification for: ${host} ${chart}.${name} is ${status} to '${to_email}'"
 502             return 0
 503         else
 504             error "failed to send email notification for: ${host} ${chart}.${name} is ${status} to '${to_email}' with error code ${ret}."
 505             return 1
 506         fi
 507     fi
 508
 509     return 1
 510 }
 511
 512 # -----------------------------------------------------------------------------
 513 # pushover sender
 514
 515 send_pushover() {
 516     local apptoken="${1}" usertokens="${2}" when="${3}" url="${4}" status="${5}" title="${6}" message="${7}" httpcode sent=0 user priority
 517
 518     if [ "${SEND_PUSHOVER}" = "YES" -a ! -z "${apptoken}" -a ! -z "${usertokens}" -a ! -z "${title}" -a ! -z "${message}" ]
 519         then
 520
 521         # https://pushover.net/api
 522         priority=-2
 523         case "${status}" in
 524             CLEAR) priority=-1;;   # low priority: no sound or vibration
 525             WARNING) priotity=0;;  # normal priority: respect quiet hours
 526             CRITICAL) priority=1;; # high priority: bypass quiet hours
 527             *) priority=-2;;       # lowest priority: no notification at all
 528         esac
 529
 530         for user in ${usertokens}
 531         do
 532             httpcode=$(${curl} --write-out %{http_code} --silent --output /dev/null \
 533                 --form-string "token=${apptoken}" \
 534                 --form-string "user=${user}" \
 535                 --form-string "html=1" \
 536                 --form-string "title=${title}" \
 537                 --form-string "message=${message}" \
 538                 --form-string "timestamp=${when}" \
 539                 --form-string "url=${url}" \
 540                 --form-string "url_title=Open netdata dashboard to view the alarm" \
 541                 --form-string "priority=${priority}" \
 542                 https://api.pushover.net/1/messages.json)
 543
 544             if [ "${httpcode}" == "200" ]
 545             then
 546                 info "sent pushover notification for: ${host} ${chart}.${name} is ${status} to '${user}'"
 547                 sent=$((sent + 1))
 548             else
 549                 error "failed to send pushover notification for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP error code ${httpcode}."
 550             fi
 551         done
 552
 553         [ ${sent} -gt 0 ] && return 0
 554     fi
 555
 556     return 1
 557 }
 558
 559 # -----------------------------------------------------------------------------
 560 # pushbullet sender
 561
 562 send_pushbullet() {
 563     local userapikey="${1}" recipients="${2}"  title="${3}" message="${4}" httpcode sent=0 user
 564     if [ "${SEND_PUSHBULLET}" = "YES" -a ! -z "${userapikey}" -a ! -z "${recipients}" -a ! -z "${message}" -a ! -z "${title}" ]
 565         then
 566         #https://docs.pushbullet.com/#create-push
 567         for user in ${recipients}
 568         do
 569             httpcode=$(${curl} --write-out %{http_code} --silent --output /dev/null \
 570               --header 'Access-Token: '${userapikey}'' \
 571               --header 'Content-Type: application/json' \
 572               --data-binary  @<(cat <<EOF
 573                               {"title": "${title}",
 574                               "type": "note",
 575                               "email": "${user}",
 576                               "body": "$( echo -n ${message})"}
 577 EOF
 578                ) "https://api.pushbullet.com/v2/pushes" -X POST)
 579
 580             if [ "${httpcode}" == "200" ]
 581             then
 582                 info "sent pushbullet notification for: ${host} ${chart}.${name} is ${status} to '${user}'"
 583                 sent=$((sent + 1))
 584             else
 585                 error "failed to send pushbullet notification for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP error code ${httpcode}."
 586             fi
 587         done
 588
 589         [ ${sent} -gt 0 ] && return 0
 590     fi
 591
 592     return 1
 593 }
 594
 595 # -----------------------------------------------------------------------------
 596 # twilio sender
 597
 598 send_twilio() {
 599     local accountsid="${1}" accounttoken="${2}" twilionumber="${3}" recipients="${4}"  title="${5}" message="${6}" httpcode sent=0 user
 600     if [ "${SEND_TWILIO}" = "YES" -a ! -z "${accountsid}" -a ! -z "${accounttoken}" -a ! -z "${twilionumber}" -a ! -z "${recipients}" -a ! -z "${message}" -a ! -z "${title}" ]
 601         then
 602         #https://www.twilio.com/packages/labs/code/bash/twilio-sms
 603         for user in ${recipients}
 604         do
 605             httpcode=$(${curl} -X POST --write-out %{http_code} --silent --output /dev/null \
 606                 --data-urlencode "From=${twilionumber}" \
 607                 --data-urlencode "To=${user}" \
 608                 --data-urlencode "Body=${title}${message}" \
 609                 -u "${accountsid}:${accounttoken}" \
 610                 "https://api.twilio.com/2010-04-01/Accounts/${accountsid}/Messages.json")
 611
 612             if [ "${httpcode}" == "201" ]
 613             then
 614                 info "sent Twilio SMS for: ${host} ${chart}.${name} is ${status} to '${user}'"
 615                 sent=$((sent + 1))
 616             else
 617                 error "failed to send Twilio SMS for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP error code ${httpcode}."
 618             fi
 619         done
 620
 621         [ ${sent} -gt 0 ] && return 0
 622     fi
 623
 624     return 1
 625 }
 626
 627 # -----------------------------------------------------------------------------
 628 # telegram sender
 629
 630 send_telegram() {
 631     local bottoken="${1}" chatids="${2}" message="${3}" httpcode sent=0 chatid disableNotification=""
 632
 633     if [ "${status}" = "CLEAR" ]; then disableNotification="--data-urlencode disable_notification=true"; fi
 634
 635     if [ "${SEND_TELEGRAM}" = "YES" -a ! -z "${bottoken}" -a ! -z "${chatids}" -a ! -z "${message}" ];
 636     then
 637         for chatid in ${chatids}
 638         do
 639             # https://core.telegram.org/bots/api#sendmessage
 640             httpcode=$(${curl} --write-out %{http_code} --silent --output /dev/null ${disableNotification} \
 641                 --data-urlencode "parse_mode=HTML" \
 642                 --data-urlencode "disable_web_page_preview=true" \
 643                 --data-urlencode "text=${message}" \
 644                 "https://api.telegram.org/bot${bottoken}/sendMessage?chat_id=${chatid}")
 645
 646             if [ "${httpcode}" == "200" ]
 647             then
 648                 info "sent telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}'"
 649                 sent=$((sent + 1))
 650             elif [ "${httpcode}" == "401" ]
 651             then
 652                 error "failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}': Wrong bot token."
 653             else
 654                 error "failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}' with HTTP error code ${httpcode}."
 655             fi
 656         done
 657
 658         [ ${sent} -gt 0 ] && return 0
 659     fi
 660
 661     return 1
 662 }
 663
 664 # -----------------------------------------------------------------------------
 665 # slack sender
 666
 667 send_slack() {
 668     local webhook="${1}" channels="${2}" httpcode sent=0 channel color payload
 669
 670     [ "${SEND_SLACK}" != "YES" ] && return 1
 671
 672     case "${status}" in
 673         WARNING)  color="warning" ;;
 674         CRITICAL) color="danger" ;;
 675         CLEAR)    color="good" ;;
 676         *)        color="#777777" ;;
 677     esac
 678
 679     for channel in ${channels}
 680     do
 681         payload="$(cat <<EOF
 682         {
 683             "channel": "#${channel}",
 684             "username": "netdata on ${host}",
 685             "icon_url": "${images_base_url}/images/seo-performance-128.png",
 686             "text": "${host} ${status_message}, \`${chart}\` (_${family}_), *${alarm}*",
 687             "attachments": [
 688                 {
 689                     "fallback": "${alarm} - ${chart} (${family}) - ${info}",
 690                     "color": "${color}",
 691                     "title": "${alarm}",
 692                     "title_link": "${goto_url}",
 693                     "text": "${info}",
 694                     "fields": [
 695                         {
 696                             "title": "${chart}",
 697                             "short": true
 698                         },
 699                         {
 700                             "title": "${family}",
 701                             "short": true
 702                         }
 703                     ],
 704                     "thumb_url": "${image}",
 705                     "footer": "<${goto_url}|${host}>",
 706                     "ts": ${when}
 707                 }
 708             ]
 709         }
 710 EOF
 711         )"
 712
 713         httpcode=$(${curl} --write-out %{http_code} --silent --output /dev/null -X POST --data-urlencode "payload=${payload}" "${webhook}")
 714         if [ "${httpcode}" == "200" ]
 715         then
 716             info "sent slack notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
 717             sent=$((sent + 1))
 718         else
 719             error "failed to send slack notification for: ${host} ${chart}.${name} is ${status} to '${channel}', with HTTP error code ${httpcode}."
 720         fi
 721     done
 722
 723     [ ${sent} -gt 0 ] && return 0
 724
 725     return 1
 726 }
 727
 728
 729 # -----------------------------------------------------------------------------
 730 # prepare the content of the notification
 731
 732 # the url to send the user on click
 733 urlencode "${NETDATA_REGISTRY_HOSTNAME}" >/dev/null; url_host="${REPLY}"
 734 urlencode "${chart}" >/dev/null; url_chart="${REPLY}"
 735 urlencode "${family}" >/dev/null; url_family="${REPLY}"
 736 urlencode "${name}" >/dev/null; url_name="${REPLY}"
 737 goto_url="${NETDATA_REGISTRY_URL}/goto-host-from-alarm.html?host=${url_host}&chart=${url_chart}&family=${url_family}&alarm=${url_name}&alarm_unique_id=${unique_id}&alarm_id=${alarm_id}&alarm_event_id=${event_id}"
 738
 739 # the severity of the alarm
 740 severity="${status}"
 741
 742 # the time the alarm was raised
 743 duration4human ${duration} >/dev/null; duration_txt="${REPLY}"
 744 duration4human ${non_clear_duration} >/dev/null; non_clear_duration_txt="${REPLY}"
 745 raised_for="(was ${old_status,,} for ${duration_txt})"
 746
 747 # the key status message
 748 status_message="status unknown"
 749
 750 # the color of the alarm
 751 color="grey"
 752
 753 # the alarm value
 754 alarm="${name//_/ } = ${value} ${units}"
 755
 756 # the image of the alarm
 757 image="${images_base_url}/images/seo-performance-128.png"
 758
 759 # prepare the title based on status
 760 case "${status}" in
 761         CRITICAL)
 762         image="${images_base_url}/images/alert-128-red.png"
 763         status_message="is critical"
 764         color="#ca414b"
 765         ;;
 766
 767     WARNING)
 768         image="${images_base_url}/images/alert-128-orange.png"
 769         status_message="needs attention"
 770         color="#caca4b"
 771                 ;;
 772
 773         CLEAR)
 774         image="${images_base_url}/images/check-mark-2-128-green.png"
 775         status_message="recovered"
 776                 color="#77ca6d"
 777                 ;;
 778 esac
 779
 780 if [ "${status}" = "CLEAR" ]
 781 then
 782     severity="Recovered from ${old_status}"
 783     if [ ${non_clear_duration} -gt ${duration} ]
 784     then
 785         raised_for="(alarm was raised for ${non_clear_duration_txt})"
 786     fi
 787
 788     # don't show the value when the status is CLEAR
 789     # for certain alarms, this value might not have any meaning
 790     alarm="${name//_/ } ${raised_for}"
 791
 792 elif [ "${old_status}" = "WARNING" -a "${status}" = "CRITICAL" ]
 793 then
 794     severity="Escalated to ${status}"
 795     if [ ${non_clear_duration} -gt ${duration} ]
 796     then
 797         raised_for="(alarm is raised for ${non_clear_duration_txt})"
 798     fi
 799
 800 elif [ "${old_status}" = "CRITICAL" -a "${status}" = "WARNING" ]
 801 then
 802     severity="Demoted to ${status}"
 803     if [ ${non_clear_duration} -gt ${duration} ]
 804     then
 805         raised_for="(alarm is raised for ${non_clear_duration_txt})"
 806     fi
 807
 808 else
 809     raised_for=
 810 fi
 811
 812 # prepare HTML versions of elements
 813 info_html=
 814 [ ! -z "${info}" ] && info_html=" <small><br/>${info}</small>"
 815
 816 raised_for_html=
 817 [ ! -z "${raised_for}" ] && raised_for_html="<br/><small>${raised_for}</small>"
 818
 819 # -----------------------------------------------------------------------------
 820 # send the slack notification
 821
 822 # slack aggregates posts from the same username
 823 # so we use "${host} ${status}" as the bot username, to make them diff
 824
 825 send_slack "${SLACK_WEBHOOK_URL}" "${to_slack}"
 826 SENT_SLACK=$?
 827
 828 # -----------------------------------------------------------------------------
 829 # send the pushover notification
 830
 831 send_pushover "${PUSHOVER_APP_TOKEN}" "${to_pushover}" "${when}" "${goto_url}" "${status}" "${host} ${status_message} - ${name//_/ } - ${chart}" "
 832 <font color=\"${color}\"><b>${alarm}</b></font>${info_html}<br/>&nbsp;
 833 <small><b>${chart}</b><br/>Chart<br/>&nbsp;</small>
 834 <small><b>${family}</b><br/>Family<br/>&nbsp;</small>
 835 <small><b>${severity}</b><br/>Severity<br/>&nbsp;</small>
 836 <small><b>${date}${raised_for_html}</b><br/>Time<br/>&nbsp;</small>
 837 <a href=\"${goto_url}\">View Netdata</a><br/>&nbsp;
 838 <small><small>The source of this alarm is line ${src}</small></small>
 839 "
 840
 841 SENT_PUSHOVER=$?
 842
 843 # -----------------------------------------------------------------------------
 844 # send the pushbullet notification
 845
 846 send_pushbullet "${PUSHBULLET_ACCESS_TOKEN}" "${to_pushbullet}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm}\n
 847 Severity: ${severity}\n
 848 Chart: ${chart}\n
 849 Family: ${family}\n
 850 To View Netdata go to: ${goto_url}\n
 851 The source of this alarm is line ${src}"
 852
 853 SENT_PUSHBULLET=$?
 854
 855 # send the twilio SMS
 856
 857 send_twilio "${TWILIO_ACCOUNT_SID}" "${TWILIO_ACCOUNT_TOKEN}" "${TWILIO_NUMBER}" "${to_twilio}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm}
 858 Severity: ${severity}
 859 Chart: ${chart}
 860 Family: ${family}
 861 The source of this alarm is line ${src}"
 862
 863 SENT_TWILIO=$?
 864
 865 # -----------------------------------------------------------------------------
 866 # send the telegram.org message
 867
 868 # https://core.telegram.org/bots/api#formatting-options
 869 send_telegram "${TELEGRAM_BOT_TOKEN}" "${to_telegram}" "${host} ${status_message} - <b>${name//_/ }</b>
 870 ${chart} (${family})
 871 <a href=\"${goto_url}\">${alarm}</a>
 872 <i>${info}</i>"
 873
 874 SENT_TELEGRAM=$?
 875
 876 # -----------------------------------------------------------------------------
 877 # send the email
 878
 879 send_email <<EOF
 880 To: ${to_email}
 881 Subject: ${host} ${status_message} - ${name//_/ } - ${chart}
 882 Content-Type: text/html
 883
 884 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
 885 <html xmlns="http://www.w3.org/1999/xhtml" style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0; padding: 0;">
 886 <body style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 14px; width: 100% !important; min-height: 100%; line-height: 1.6; background: #f6f6f6; margin:0; padding: 0;">
 887 <table>
 888     <tbody>
 889     <tr>
 890         <td style="vertical-align: top;" valign="top"></td>
 891         <td width="700" style="vertical-align: top; display: block !important; max-width: 700px !important; clear: both !important; margin: 0 auto; padding: 0;" valign="top">
 892             <div style="max-width: 700px; display: block; margin: 0 auto; padding: 20px;">
 893                 <table width="100%" cellpadding="0" cellspacing="0" style="background: #fff; border: 1px solid #e9e9e9;">
 894                     <tbody>
 895                     <tr>
 896                         <td bgcolor="#eee" style="padding: 5px 20px 5px 20px; background-color: #eee;">
 897                             <div style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 20px; color: #777; font-weight: bold;">netdata notification</div>
 898                         </td>
 899                     </tr>
 900                     <tr>
 901                         <td bgcolor="${color}" style="font-size: 16px; vertical-align: top; font-weight: 400; text-align: center; margin: 0; padding: 10px; color: #ffffff; background: ${color} !important; border: 1px solid ${color}; border-top-color: ${color};" align="center" valign="top">
 902                             <h1 style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-weight: 400; margin: 0;">${host} ${status_message}</h1>
 903                         </td>
 904                     </tr>
 905                     <tr>
 906                         <td style="vertical-align: top;" valign="top">
 907                             <div style="margin: 0; padding: 20px; max-width: 700px;">
 908                                 <table width="100%" cellpadding="0" cellspacing="0" style="max-width:700px">
 909                                     <tbody>
 910                                     <tr>
 911                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding:0 0 20px;" align="left" valign="top">
 912                                             <span>${chart}</span>
 913                                             <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Chart</span>
 914                                         </td>
 915                                     </tr>
 916                                     <tr style="margin: 0; padding: 0;">
 917                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
 918                                             <span><b>${alarm}</b>${info_html}</span>
 919                                             <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Alarm</span>
 920                                         </td>
 921                                     </tr>
 922                                     <tr>
 923                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
 924                                             <span>${family}</span>
 925                                             <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Family</span>
 926                                         </td>
 927                                     </tr>
 928                                     <tr style="margin: 0; padding: 0;">
 929                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
 930                                             <span>${severity}</span>
 931                                             <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Severity</span>
 932                                         </td>
 933                                     </tr>
 934                                     <tr style="margin: 0; padding: 0;">
 935                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top"><span>${date}</span>
 936                                             <span>${raised_for_html}</span> <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Time</span>
 937                                         </td>
 938                                     </tr>
 939                                     <tr style="margin: 0; padding: 0;">
 940                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;">
 941                                             <a href="${goto_url}" style="font-size: 14px; color: #ffffff; text-decoration: none; line-height: 1.5; font-weight: bold; text-align: center; display: inline-block; text-transform: capitalize; background: #35568d; border-width: 1px; border-style: solid; border-color: #2b4c86; margin: 0; padding: 10px 15px;" target="_blank">View Netdata</a>
 942                                         </td>
 943                                     </tr>
 944                                     <tr style="text-align: center; margin: 0; padding: 0;">
 945                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 11px; vertical-align: top; margin: 0; padding: 10px 0 0 0; color: #666666;" align="center" valign="bottom">The source of this alarm is line <code>${src}</code>
 946                                         </td>
 947                                     </tr>
 948                                     <tr style="text-align: center; margin: 0; padding: 0;">
 949                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 12px; vertical-align: top; margin:0; padding: 20px 0 0 0; color: #666666; border-top: 1px solid #f0f0f0;" align="center" valign="bottom">Sent by
 950                                             <a href="https://mynetdata.io/" target="_blank">netdata</a>, the real-time performance monitoring.
 951                                         </td>
 952                                     </tr>
 953                                     </tbody>
 954                                 </table>
 955                             </div>
 956                         </td>
 957                     </tr>
 958                     </tbody>
 959                 </table>
 960             </div>
 961         </td>
 962     </tr>
 963     </tbody>
 964 </table>
 965 </body>
 966 </html>
 967 EOF
 968
 969 SENT_EMAIL=$?
 970
 971 # -----------------------------------------------------------------------------
 972 # let netdata know
 973
 974 # we did send something
 975 [ ${SENT_EMAIL} -eq 0 -o ${SENT_PUSHOVER} -eq 0 -o ${SENT_TELEGRAM} -eq 0 -o ${SENT_SLACK} -eq 0 -o ${SENT_TWILIO} -eq 0 -o  ${SENT_PUSHBULLET} -eq 0 ] && exit 0
 976
 977 # we did not send anything
 978 exit 1