plugins.d/alarm-notify.sh

   1 #!/usr/bin/env bash
   2
   3 # netdata
   4 # real-time performance and health monitoring, done right!
   5 # (C) 2016 Costa Tsaousis <costa@tsaousis.gr>
   6 # GPL v3+
   7 #
   8 # Script to send alarm notifications for netdata
   9 #
  10 # Features:
  11 #  - multiple notification methods
  12 #  - multiple roles per alarm
  13 #  - multiple recipients per role
  14 #  - severity filtering per recipient
  15 #
  16 # Supported notification methods:
  17 #  - emails by @ktsaou
  18 #  - slack.com notifications by @ktsaou
  19 #  - pushover.net notifications by @ktsaou
  20 #  - pushbullet.com push notifications by Tiago Peralta @tperalta82 PR #1070
  21 #  - telegram.org notifications by @hashworks PR #1002
  22 #  - twilio.com notifications by Levi Blaney @shadycuz PR #1211
  23 #  - kafka notifications by @ktsaou #1342
  24 #  - pagerduty.com notifications by Jim Cooley @jimcooley PR #1373
  25 #  - messagebird.com notifications by @tech_no_logical #1453
  26 #  - hipchart notifications by @ktsaou
  27
  28 # -----------------------------------------------------------------------------
  29 # testing notifications
  30
  31 if [ \( "${1}" = "test" -o "${2}" = "test" \) -a "${#}" -le 2 ]
  32 then
  33     if [ "${2}" = "test" ]
  34     then
  35         recipient="${1}"
  36     else
  37         recipient="${2}"
  38     fi
  39
  40     [ -z "${recipient}" ] && recipient="sysadmin"
  41
  42     id=1
  43     last="CLEAR"
  44     for x in "CRITICAL" "WARNING" "CLEAR"
  45     do
  46         echo >&2
  47         echo >&2 "# SENDING TEST ${x} ALARM TO ROLE: ${recipient}"
  48
  49         "${0}" "${recipient}" "$(hostname)" 1 1 "${id}" "$(date +%s)" "test_alarm" "test.chart" "test.family" "${x}" "${last}" 100 90 "${0}" 1 $((0 + id)) "units" "this is a test alarm to verify notifications work"
  50         if [ $? -ne 0 ]
  51         then
  52             echo >&2 "# FAILED"
  53         else
  54             echo >&2 "# OK"
  55         fi
  56
  57         last="${x}"
  58         id=$((id + 1))
  59     done
  60
  61     exit 1
  62 fi
  63
  64 export PATH="${PATH}:/sbin:/usr/sbin:/usr/local/sbin"
  65 export LC_ALL=C
  66
  67 # -----------------------------------------------------------------------------
  68
  69 PROGRAM_NAME="$(basename "${0}")"
  70
  71 logdate() {
  72     date "+%Y-%m-%d %H:%M:%S"
  73 }
  74
  75 log() {
  76     local status="${1}"
  77     shift
  78
  79     echo >&2 "$(logdate): ${PROGRAM_NAME}: ${status}: ${*}"
  80
  81 }
  82
  83 warning() {
  84     log WARNING "${@}"
  85 }
  86
  87 error() {
  88     log ERROR "${@}"
  89 }
  90
  91 info() {
  92     log INFO "${@}"
  93 }
  94
  95 fatal() {
  96     log FATAL "${@}"
  97     exit 1
  98 }
  99
 100 debug=0
 101 debug() {
 102     [ ${debug} -eq 1 ] && log DEBUG "${@}"
 103 }
 104
 105 # -----------------------------------------------------------------------------
 106
 107 # check for BASH v4+ (required for associative arrays)
 108 [ $(( ${BASH_VERSINFO[0]} )) -lt 4 ] && \
 109     fatal "BASH version 4 or later is required (this is ${BASH_VERSION})."
 110
 111 # -----------------------------------------------------------------------------
 112 # defaults to allow running this script by hand
 113
 114 NETDATA_CONFIG_DIR="${NETDATA_CONFIG_DIR-/etc/netdata}"
 115 NETDATA_CACHE_DIR="${NETDATA_CACHE_DIR-/var/cache/netdata}"
 116 [ -z "${NETDATA_REGISTRY_URL}" ] && NETDATA_REGISTRY_URL="https://registry.my-netdata.io"
 117 [ -z "${NETDATA_HOSTNAME}" ] && NETDATA_HOSTNAME="$(hostname)"
 118 [ -z "${NETDATA_REGISTRY_HOSTNAME}" ] && NETDATA_REGISTRY_HOSTNAME="${NETDATA_HOSTNAME}"
 119
 120 # -----------------------------------------------------------------------------
 121 # parse command line parameters
 122
 123 roles="${1}"       # the roles that should be notified for this event
 124 host="${2}"        # the host generated this event
 125 unique_id="${3}"   # the unique id of this event
 126 alarm_id="${4}"    # the unique id of the alarm that generated this event
 127 event_id="${5}"    # the incremental id of the event, for this alarm id
 128 when="${6}"        # the timestamp this event occurred
 129 name="${7}"        # the name of the alarm, as given in netdata health.d entries
 130 chart="${8}"       # the name of the chart (type.id)
 131 family="${9}"      # the family of the chart
 132 status="${10}"     # the current status : REMOVED, UNITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
 133 old_status="${11}" # the previous status: REMOVED, UNITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
 134 value="${12}"      # the current value of the alarm
 135 old_value="${13}"  # the previous value of the alarm
 136 src="${14}"        # the line number and file the alarm has been configured
 137 duration="${15}"   # the duration in seconds of the previous alarm state
 138 non_clear_duration="${16}" # the total duration in seconds this is/was non-clear
 139 units="${17}"      # the units of the value
 140 info="${18}"       # a short description of the alarm
 141
 142 # -----------------------------------------------------------------------------
 143 # screen statuses we don't need to send a notification
 144
 145 # don't do anything if this is not WARNING, CRITICAL or CLEAR
 146 if [ "${status}" != "WARNING" -a "${status}" != "CRITICAL" -a "${status}" != "CLEAR" ]
 147 then
 148     info "not sending notification for ${status} on '${chart}.${name}'"
 149     exit 1
 150 fi
 151
 152 # don't do anything if this is CLEAR, but it was not WARNING or CRITICAL
 153 if [ "${old_status}" != "WARNING" -a "${old_status}" != "CRITICAL" -a "${status}" = "CLEAR" ]
 154 then
 155     info "not sending notification for ${status} on '${chart}.${name}' (last status was ${old_status})"
 156     exit 1
 157 fi
 158
 159 # -----------------------------------------------------------------------------
 160 # load configuration
 161
 162 # By default fetch images from the global public registry.
 163 # This is required by default, since all notification methods need to download
 164 # images via the Internet, and private registries might not be reachable.
 165 # This can be overwritten at the configuration file.
 166 images_base_url="https://registry.my-netdata.io"
 167
 168 # needed commands
 169 # if empty they will be searched in the system path
 170 curl=
 171 sendmail=
 172
 173 # enable / disable features
 174 SEND_SLACK="YES"
 175 SEND_PUSHOVER="YES"
 176 SEND_TWILIO="YES"
 177 SEND_HIPCHAT="YES"
 178 SEND_MESSAGEBIRD="YES"
 179 SEND_TELEGRAM="YES"
 180 SEND_EMAIL="YES"
 181 SEND_PUSHBULLET="YES"
 182 SEND_KAFKA="YES"
 183 SEND_PD="YES"
 184
 185 # slack configs
 186 SLACK_WEBHOOK_URL=
 187 DEFAULT_RECIPIENT_SLACK=
 188 declare -A role_recipients_slack=()
 189
 190 # pushover configs
 191 PUSHOVER_APP_TOKEN=
 192 DEFAULT_RECIPIENT_PUSHOVER=
 193 declare -A role_recipients_pushover=()
 194
 195 # pushbullet configs
 196 PUSHBULLET_ACCESS_TOKEN=
 197 DEFAULT_RECIPIENT_PUSHBULLET=
 198 declare -A role_recipients_pushbullet=()
 199
 200 # twilio configs
 201 TWILIO_ACCOUNT_SID=
 202 TWILIO_ACCOUNT_TOKEN=
 203 TWILIO_NUMBER=
 204 DEFAULT_RECIPIENT_TWILIO=
 205 declare -A role_recipients_twilio=()
 206
 207 # hipchat configs
 208 HIPCHAT_AUTH_TOKEN=
 209 DEFAULT_RECIPIENT_HIPCHAT=
 210 declare -A role_recipients_hipchat=()
 211
 212 # messagebird configs
 213 MESSAGEBIRD_ACCESS_KEY=
 214 MESSAGEBIRD_NUMBER=
 215 DEFAULT_RECIPIENT_MESSAGEBIRD=
 216 declare -A role_recipients_messagebird=()
 217
 218 # telegram configs
 219 TELEGRAM_BOT_TOKEN=
 220 DEFAULT_RECIPIENT_TELEGRAM=
 221 declare -A role_recipients_telegram=()
 222
 223 # kafka configs
 224 KAFKA_URL=
 225 KAFKA_SENDER_IP=
 226
 227 # pagerduty.com configs
 228 PD_SERVICE_KEY=
 229 declare -A role_recipients_pd=()
 230
 231 # email configs
 232 DEFAULT_RECIPIENT_EMAIL="root"
 233 declare -A role_recipients_email=()
 234
 235 # load the user configuration
 236 # this will overwrite the variables above
 237 if [ -f "${NETDATA_CONFIG_DIR}/health_alarm_notify.conf" ]
 238     then
 239     source "${NETDATA_CONFIG_DIR}/health_alarm_notify.conf"
 240 fi
 241
 242 # -----------------------------------------------------------------------------
 243 # filter a recipient based on alarm event severity
 244
 245 filter_recipient_by_criticality() {
 246     local method="${1}" x="${2}" r s
 247     shift
 248
 249     r="${x/|*/}" # the recipient
 250     s="${x/*|/}" # the severity required for notifying this recipient
 251
 252     # no severity filtering for this person
 253     [ "${r}" = "${s}" ] && return 0
 254
 255     # the severity is invalid
 256     s="${s^^}"
 257     [ "${s}" != "CRITICAL" ] && return 0
 258
 259     # the new or the old status matches the severity
 260     if [ "${s}" = "${status}" -o "${s}" = "${old_status}" ]
 261         then
 262         [ ! -d "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}" ] && \
 263             mkdir -p "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}"
 264
 265         # we need to keep track of the notifications we sent
 266         # so that the same user will receive the recovery
 267         # even if old_status does not match the required severity
 268         touch "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}"
 269         return 0
 270     fi
 271
 272     # it is a cleared alarm we have sent notification for
 273     if [ "${status}" != "WARNING" -a "${status}" != "CRITICAL" -a -f "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}" ]
 274         then
 275         rm "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}"
 276         return 0
 277     fi
 278
 279     return 1
 280 }
 281
 282 # -----------------------------------------------------------------------------
 283 # find the recipients' addresses per method
 284
 285 declare -A arr_slack=()
 286 declare -A arr_pushover=()
 287 declare -A arr_pushbullet=()
 288 declare -A arr_twilio=()
 289 declare -A arr_hipchat=()
 290 declare -A arr_telegram=()
 291 declare -A arr_pd=()
 292 declare -A arr_email=()
 293
 294 # netdata may call us with multiple roles, and roles may have multiple but
 295 # overlapping recipients - so, here we find the unique recipients.
 296 for x in ${roles//,/ }
 297 do
 298     # the roles 'silent' and 'disabled' mean:
 299     # don't send a notification for this role
 300     [ "${x}" = "silent" -o "${x}" = "disabled" ] && continue
 301
 302     # email
 303     a="${role_recipients_email[${x}]}"
 304     [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_EMAIL}"
 305     for r in ${a//,/ }
 306     do
 307         [ "${r}" != "disabled" ] && filter_recipient_by_criticality email "${r}" && arr_email[${r/|*/}]="1"
 308     done
 309
 310     # pushover
 311     a="${role_recipients_pushover[${x}]}"
 312     [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_PUSHOVER}"
 313     for r in ${a//,/ }
 314     do
 315         [ "${r}" != "disabled" ] && filter_recipient_by_criticality pushover "${r}" && arr_pushover[${r/|*/}]="1"
 316     done
 317
 318     # pushbullet
 319     a="${role_recipients_pushbullet[${x}]}"
 320     [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_PUSHBULLET}"
 321     for r in ${a//,/ }
 322     do
 323         [ "${r}" != "disabled" ] && filter_recipient_by_criticality pushbullet "${r}" && arr_pushbullet[${r/|*/}]="1"
 324     done
 325
 326     # twilio
 327     a="${role_recipients_twilio[${x}]}"
 328     [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_TWILIO}"
 329     for r in ${a//,/ }
 330     do
 331         [ "${r}" != "disabled" ] && filter_recipient_by_criticality twilio "${r}" && arr_twilio[${r/|*/}]="1"
 332     done
 333
 334     # hipchat
 335     a="${role_recipients_hipchat[${x}]}"
 336     [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_HIPCHAT}"
 337     for r in ${a//,/ }
 338     do
 339         [ "${r}" != "disabled" ] && filter_recipient_by_criticality hipchat "${r}" && arr_hipchat[${r/|*/}]="1"
 340     done
 341
 342     # messagebird
 343     a="${role_recipients_messagebird[${x}]}"
 344     [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_MESSAGEBIRD}"
 345     for r in ${a//,/ }
 346     do
 347         [ "${r}" != "disabled" ] && filter_recipient_by_criticality messagebird "${r}" && arr_messagebird[${r/|*/}]="1"
 348     done
 349
 350     # telegram
 351     a="${role_recipients_telegram[${x}]}"
 352     [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_TELEGRAM}"
 353     for r in ${a//,/ }
 354     do
 355         [ "${r}" != "disabled" ] && filter_recipient_by_criticality telegram "${r}" && arr_telegram[${r/|*/}]="1"
 356     done
 357
 358     # slack
 359     a="${role_recipients_slack[${x}]}"
 360     [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_SLACK}"
 361     for r in ${a//,/ }
 362     do
 363         [ "${r}" != "disabled" ] && filter_recipient_by_criticality slack "${r}" && arr_slack[${r/|*/}]="1"
 364     done
 365
 366     # pagerduty.com
 367     a="${role_recipients_pd[${x}]}"
 368     [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_PD}"
 369     for r in ${a//,/ }
 370     do
 371         [ "${r}" != "disabled" ] && filter_recipient_by_criticality pd "${r}" && arr_pd[${r/|*/}]="1"
 372     done
 373 done
 374
 375 # build the list of slack recipients (channels)
 376 to_slack="${!arr_slack[*]}"
 377 [ -z "${to_slack}" ] && SEND_SLACK="NO"
 378
 379 # build the list of pushover recipients (user tokens)
 380 to_pushover="${!arr_pushover[*]}"
 381 [ -z "${to_pushover}" ] && SEND_PUSHOVER="NO"
 382
 383 # build the list of pushbulet recipients (user tokens)
 384 to_pushbullet="${!arr_pushbullet[*]}"
 385 [ -z "${to_pushbullet}" ] && SEND_PUSHBULLET="NO"
 386
 387 # build the list of twilio recipients (phone numbers)
 388 to_twilio="${!arr_twilio[*]}"
 389 [ -z "${to_twilio}" ] && SEND_TWILIO="NO"
 390
 391 # build the list of hipchat recipients (rooms)
 392 to_hipchat="${!arr_hipchat[*]}"
 393 [ -z "${to_hipchat}" ] && SEND_HIPCHAT="NO"
 394
 395 # build the list of messagebird recipients (phone numbers)
 396 to_messagebird="${!arr_messagebird[*]}"
 397 [ -z "${to_messagebird}" ] && SEND_MESSAGEBIRD="NO"
 398
 399 # check array of telegram recipients (chat ids)
 400 to_telegram="${!arr_telegram[*]}"
 401 [ -z "${to_telegram}" ] && SEND_TELEGRAM="NO"
 402
 403 # build the list of pagerduty recipients (service keys)
 404 to_pd="${!arr_pd[*]}"
 405 [ -z "${to_pd}" ] && SEND_PD="NO"
 406
 407 # build the list of email recipients (email addresses)
 408 to_email=
 409 for x in "${!arr_email[@]}"
 410 do
 411     [ ! -z "${to_email}" ] && to_email="${to_email}, "
 412     to_email="${to_email}${x}"
 413 done
 414 [ -z "${to_email}" ] && SEND_EMAIL="NO"
 415
 416
 417 # -----------------------------------------------------------------------------
 418 # verify the delivery methods supported
 419
 420 # check slack
 421 [ -z "${SLACK_WEBHOOK_URL}" ] && SEND_SLACK="NO"
 422
 423 # check pushover
 424 [ -z "${PUSHOVER_APP_TOKEN}" ] && SEND_PUSHOVER="NO"
 425
 426 # check pushbullet
 427 [ -z "${PUSHBULLET_ACCESS_TOKEN}" ] && SEND_PUSHBULLET="NO"
 428
 429 # check twilio
 430 [ -z "${TWILIO_ACCOUNT_TOKEN}" -o -z "${TWILIO_ACCOUNT_SID}" -o -z "${TWILIO_NUMBER}" ] && SEND_TWILIO="NO"
 431
 432 # check hipchat
 433 [ -z "${HIPCHAT_AUTH_TOKEN}" ] && SEND_HIPCHAT="NO"
 434
 435 # check messagebird
 436 [ -z "${MESSAGEBIRD_ACCESS_KEY}" -o -z "${MESSAGEBIRD_NUMBER}" ] && SEND_MESSAGEBIRD="NO"
 437
 438 # check telegram
 439 [ -z "${TELEGRAM_BOT_TOKEN}" ] && SEND_TELEGRAM="NO"
 440
 441 # check kafka
 442 [ -z "${KAFKA_URL}" -o -z "${KAFKA_SENDER_IP}" ] && SEND_KAFKA="NO"
 443
 444 # check pagerduty.com
 445 # if we need pd-send, check for the pd-send command
 446 # https://www.pagerduty.com/docs/guides/agent-install-guide/
 447 if [ "${SEND_PD}" = "YES" ]
 448     then
 449     pd_send="$(which pd-send 2>/dev/null || command -v pd-send 2>/dev/null)"
 450     if [ -z "${pd_send}" ]
 451         then
 452         # no pd-send available
 453         # disable pagerduty.com
 454         SEND_PD="NO"
 455     fi
 456 fi
 457
 458 # if we need curl, check for the curl command
 459 if [ \( \
 460            "${SEND_PUSHOVER}"    = "YES" \
 461         -o "${SEND_SLACK}"       = "YES" \
 462         -o "${SEND_HIPCHAT}"     = "YES" \
 463         -o "${SEND_TWILIO}"      = "YES" \
 464         -o "${SEND_MESSAGEBIRD}" = "YES" \
 465         -o "${SEND_TELEGRAM}"    = "YES" \
 466         -o "${SEND_PUSHBULLET}"  = "YES" \
 467         -o "${SEND_KAFKA}"       = "YES" \
 468     \) -a -z "${curl}" ]
 469     then
 470     curl="$(which curl 2>/dev/null || command -v curl 2>/dev/null)"
 471     if [ -z "${curl}" ]
 472         then
 473         # no curl available
 474         # disable all curl based methods
 475         SEND_PUSHOVER="NO"
 476         SEND_PUSHBULLET="NO"
 477         SEND_TELEGRAM="NO"
 478         SEND_SLACK="NO"
 479         SEND_TWILIO="NO"
 480         SEND_HIPCHAT="NO"
 481         SEND_MESSAGEBIRD="NO"
 482         SEND_KAFKA="NO"
 483     fi
 484 fi
 485
 486 # if we need sendmail, check for the sendmail command
 487 if [ "${SEND_EMAIL}" = "YES" -a -z "${sendmail}" ]
 488     then
 489     sendmail="$(which sendmail 2>/dev/null || command -v sendmail 2>/dev/null)"
 490     [ -z "${sendmail}" ] && SEND_EMAIL="NO"
 491 fi
 492
 493 # check that we have at least a method enabled
 494 if [   "${SEND_EMAIL}"          != "YES" \
 495     -a "${SEND_PUSHOVER}"       != "YES" \
 496     -a "${SEND_TELEGRAM}"       != "YES" \
 497     -a "${SEND_SLACK}"          != "YES" \
 498     -a "${SEND_TWILIO}"         != "YES" \
 499     -a "${SEND_HIPCHAT}"        != "YES" \
 500     -a "${SEND_MESSAGEBIRD}"    != "YES" \
 501     -a "${SEND_PUSHBULLET}"     != "YES" \
 502     -a "${SEND_KAFKA}"          != "YES" \
 503     -a "${SEND_PD}"             != "YES" \
 504     ]
 505     then
 506     fatal "All notification methods are disabled. Not sending notification to '${roles}' for '${name}' = '${value}' of chart '${chart}' for status '${status}'."
 507 fi
 508
 509 # -----------------------------------------------------------------------------
 510 # find a suitable hostname to use, if netdata did not supply a hostname
 511
 512 [ -z "${host}" ] && host="${NETDATA_HOSTNAME}"
 513 [ -z "${host}" ] && host="${NETDATA_REGISTRY_HOSTNAME}"
 514 [ -z "${host}" ] && host="$(hostname 2>/dev/null)"
 515
 516 # -----------------------------------------------------------------------------
 517 # get the date the alarm happened
 518
 519 date="$(date --date=@${when} 2>/dev/null)"
 520 [ -z "${date}" ] && date="$(date 2>/dev/null)"
 521
 522 # -----------------------------------------------------------------------------
 523 # function to URL encode a string
 524
 525 urlencode() {
 526     local string="${1}" strlen encoded pos c o
 527
 528     strlen=${#string}
 529     for (( pos=0 ; pos<strlen ; pos++ ))
 530     do
 531         c=${string:${pos}:1}
 532         case "${c}" in
 533             [-_.~a-zA-Z0-9])
 534                 o="${c}"
 535                 ;;
 536
 537             *)
 538                 printf -v o '%%%02x' "'${c}"
 539                 ;;
 540         esac
 541         encoded+="${o}"
 542     done
 543
 544     REPLY="${encoded}"
 545     echo "${REPLY}"
 546 }
 547
 548 # -----------------------------------------------------------------------------
 549 # function to convert a duration in seconds, to a human readable duration
 550 # using DAYS, MINUTES, SECONDS
 551
 552 duration4human() {
 553     local s="${1}" d=0 h=0 m=0 ds="day" hs="hour" ms="minute" ss="second" ret
 554     d=$(( s / 86400 ))
 555     s=$(( s - (d * 86400) ))
 556     h=$(( s / 3600 ))
 557     s=$(( s - (h * 3600) ))
 558     m=$(( s / 60 ))
 559     s=$(( s - (m * 60) ))
 560
 561     if [ ${d} -gt 0 ]
 562     then
 563         [ ${m} -ge 30 ] && h=$(( h + 1 ))
 564         [ ${d} -gt 1 ] && ds="days"
 565         [ ${h} -gt 1 ] && hs="hours"
 566         if [ ${h} -gt 0 ]
 567         then
 568             ret="${d} ${ds} and ${h} ${hs}"
 569         else
 570             ret="${d} ${ds}"
 571         fi
 572     elif [ ${h} -gt 0 ]
 573     then
 574         [ ${s} -ge 30 ] && m=$(( m + 1 ))
 575         [ ${h} -gt 1 ] && hs="hours"
 576         [ ${m} -gt 1 ] && ms="minutes"
 577         if [ ${m} -gt 0 ]
 578         then
 579             ret="${h} ${hs} and ${m} ${ms}"
 580         else
 581             ret="${h} ${hs}"
 582         fi
 583     elif [ ${m} -gt 0 ]
 584     then
 585         [ ${m} -gt 1 ] && ms="minutes"
 586         [ ${s} -gt 1 ] && ss="seconds"
 587         if [ ${s} -gt 0 ]
 588         then
 589             ret="${m} ${ms} and ${s} ${ss}"
 590         else
 591             ret="${m} ${ms}"
 592         fi
 593     else
 594         [ ${s} -gt 1 ] && ss="seconds"
 595         ret="${s} ${ss}"
 596     fi
 597
 598     REPLY="${ret}"
 599     echo "${REPLY}"
 600 }
 601
 602 # -----------------------------------------------------------------------------
 603 # email sender
 604
 605 send_email() {
 606     local ret=
 607     if [ "${SEND_EMAIL}" = "YES" ]
 608         then
 609
 610         "${sendmail}" -t
 611         ret=$?
 612
 613         if [ ${ret} -eq 0 ]
 614         then
 615             info "sent email notification for: ${host} ${chart}.${name} is ${status} to '${to_email}'"
 616             return 0
 617         else
 618             error "failed to send email notification for: ${host} ${chart}.${name} is ${status} to '${to_email}' with error code ${ret}."
 619             return 1
 620         fi
 621     fi
 622
 623     return 1
 624 }
 625
 626 # -----------------------------------------------------------------------------
 627 # pushover sender
 628
 629 send_pushover() {
 630     local apptoken="${1}" usertokens="${2}" when="${3}" url="${4}" status="${5}" title="${6}" message="${7}" httpcode sent=0 user priority
 631
 632     if [ "${SEND_PUSHOVER}" = "YES" -a ! -z "${apptoken}" -a ! -z "${usertokens}" -a ! -z "${title}" -a ! -z "${message}" ]
 633         then
 634
 635         # https://pushover.net/api
 636         priority=-2
 637         case "${status}" in
 638             CLEAR) priority=-1;;   # low priority: no sound or vibration
 639             WARNING) priotity=0;;  # normal priority: respect quiet hours
 640             CRITICAL) priority=1;; # high priority: bypass quiet hours
 641             *) priority=-2;;       # lowest priority: no notification at all
 642         esac
 643
 644         for user in ${usertokens}
 645         do
 646             httpcode=$(${curl} --write-out %{http_code} --silent --output /dev/null \
 647                 --form-string "token=${apptoken}" \
 648                 --form-string "user=${user}" \
 649                 --form-string "html=1" \
 650                 --form-string "title=${title}" \
 651                 --form-string "message=${message}" \
 652                 --form-string "timestamp=${when}" \
 653                 --form-string "url=${url}" \
 654                 --form-string "url_title=Open netdata dashboard to view the alarm" \
 655                 --form-string "priority=${priority}" \
 656                 https://api.pushover.net/1/messages.json)
 657
 658             if [ "${httpcode}" == "200" ]
 659             then
 660                 info "sent pushover notification for: ${host} ${chart}.${name} is ${status} to '${user}'"
 661                 sent=$((sent + 1))
 662             else
 663                 error "failed to send pushover notification for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP error code ${httpcode}."
 664             fi
 665         done
 666
 667         [ ${sent} -gt 0 ] && return 0
 668     fi
 669
 670     return 1
 671 }
 672
 673 # -----------------------------------------------------------------------------
 674 # pushbullet sender
 675
 676 send_pushbullet() {
 677     local userapikey="${1}" recipients="${2}"  title="${3}" message="${4}" httpcode sent=0 user
 678     if [ "${SEND_PUSHBULLET}" = "YES" -a ! -z "${userapikey}" -a ! -z "${recipients}" -a ! -z "${message}" -a ! -z "${title}" ]
 679         then
 680         #https://docs.pushbullet.com/#create-push
 681         for user in ${recipients}
 682         do
 683             httpcode=$(${curl} --write-out %{http_code} --silent --output /dev/null \
 684               --header 'Access-Token: '${userapikey}'' \
 685               --header 'Content-Type: application/json' \
 686               --data-binary  @<(cat <<EOF
 687                               {"title": "${title}",
 688                               "type": "note",
 689                               "email": "${user}",
 690                               "body": "$( echo -n ${message})"}
 691 EOF
 692                ) "https://api.pushbullet.com/v2/pushes" -X POST)
 693
 694             if [ "${httpcode}" == "200" ]
 695             then
 696                 info "sent pushbullet notification for: ${host} ${chart}.${name} is ${status} to '${user}'"
 697                 sent=$((sent + 1))
 698             else
 699                 error "failed to send pushbullet notification for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP error code ${httpcode}."
 700             fi
 701         done
 702
 703         [ ${sent} -gt 0 ] && return 0
 704     fi
 705
 706     return 1
 707 }
 708
 709 # -----------------------------------------------------------------------------
 710 # kafka sender
 711
 712 send_kafka() {
 713     local httpcode sent=0
 714     if [ "${SEND_KAFKA}" = "YES" ]
 715         then
 716             httpcode=$(${curl} -X POST --write-out %{http_code} --silent --output /dev/null \
 717                 --data "{host_ip:\"${KAFKA_SENDER_IP}\",when:${when},name:\"${name}\",chart:\"${chart}\",family:\"${family}\",status:\"${status}\",old_status:\"${old_status}\",value:${value},old_value:${old_value},duration:${duration},non_clear_duration:${non_clear_duration},units:\"${units}\",info:\"${info}\"}" \
 718                 "${KAFKA_URL}")
 719
 720             if [ "${httpcode}" == "204" ]
 721             then
 722                 info "sent kafka data for: ${host} ${chart}.${name} is ${status} and ip '${KAFKA_SENDER_IP}'"
 723                 sent=$((sent + 1))
 724             else
 725                 error "failed to send kafka data for: ${host} ${chart}.${name} is ${status} and ip '${KAFKA_SENDER_IP}' with HTTP error code ${httpcode}."
 726             fi
 727
 728         [ ${sent} -gt 0 ] && return 0
 729     fi
 730
 731     return 1
 732 }
 733
 734 # -----------------------------------------------------------------------------
 735 # pagerduty.com sender
 736
 737 send_pd() {
 738     local recipients="${1}" sent=0
 739     unset t
 740     case ${status} in
 741         CLEAR)    t='resolve';;
 742         WARNING)  t='trigger';;
 743         CRITICAL) t='trigger';;
 744     esac
 745
 746     if [ ${SEND_PD} = "YES" -a ! -z "${t}" ]
 747         then
 748         for PD_SERVICE_KEY in ${recipients}
 749         do
 750             d="${status} ${name}=${value} ${units} - ${host}, ${family}"
 751             ${pd_send} -k ${PD_SERVICE_KEY} \
 752                        -t ${t} \
 753                        -d "${d}" \
 754                        -i ${alarm_id} \
 755                        -f 'info'="${info}" \
 756                        -f 'value_w_units'="${value} ${units}" \
 757                        -f 'when'="${when}" \
 758                        -f 'duration'="${duration}" \
 759                        -f 'roles'="${roles}" \
 760                        -f 'host'="${host}" \
 761                        -f 'unique_id'="${unique_id}" \
 762                        -f 'alarm_id'="${alarm_id}" \
 763                        -f 'event_id'="${event_id}" \
 764                        -f 'name'="${name}" \
 765                        -f 'chart'="${chart}" \
 766                        -f 'family'="${family}" \
 767                        -f 'status'="${status}" \
 768                        -f 'old_status'="${old_status}" \
 769                        -f 'value'="${value}" \
 770                        -f 'old_value'="${old_value}" \
 771                        -f 'src'="${src}" \
 772                        -f 'non_clear_duration'="${non_clear_duration}" \
 773                        -f 'units'="${units}"
 774             retval=$?
 775             if [ ${retval} -eq 0 ]
 776                 then
 777                     info "sent pagerduty.com notification using service key ${PD_SERVICE_KEY::-26}....: ${d}"
 778                     sent=$((sent + 1))
 779                 else
 780                     error "failed to send pagerduty.com notification using service key ${PD_SERVICE_KEY::-26}.... (error code ${retval}): ${d}"
 781             fi
 782         done
 783
 784         [ ${sent} -gt 0 ] && return 0
 785     fi
 786
 787     return 1
 788 }
 789
 790 # -----------------------------------------------------------------------------
 791 # twilio sender
 792
 793 send_twilio() {
 794     local accountsid="${1}" accounttoken="${2}" twilionumber="${3}" recipients="${4}"  title="${5}" message="${6}" httpcode sent=0 user
 795     if [ "${SEND_TWILIO}" = "YES" -a ! -z "${accountsid}" -a ! -z "${accounttoken}" -a ! -z "${twilionumber}" -a ! -z "${recipients}" -a ! -z "${message}" -a ! -z "${title}" ]
 796         then
 797         #https://www.twilio.com/packages/labs/code/bash/twilio-sms
 798         for user in ${recipients}
 799         do
 800             httpcode=$(${curl} -X POST --write-out %{http_code} --silent --output /dev/null \
 801                 --data-urlencode "From=${twilionumber}" \
 802                 --data-urlencode "To=${user}" \
 803                 --data-urlencode "Body=${title} ${message}" \
 804                 -u "${accountsid}:${accounttoken}" \
 805                 "https://api.twilio.com/2010-04-01/Accounts/${accountsid}/Messages.json")
 806
 807             if [ "${httpcode}" == "201" ]
 808             then
 809                 info "sent Twilio SMS for: ${host} ${chart}.${name} is ${status} to '${user}'"
 810                 sent=$((sent + 1))
 811             else
 812                 error "failed to send Twilio SMS for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP error code ${httpcode}."
 813             fi
 814         done
 815
 816         [ ${sent} -gt 0 ] && return 0
 817     fi
 818
 819     return 1
 820 }
 821
 822
 823 # -----------------------------------------------------------------------------
 824 # hipchat sender
 825
 826 send_hipchat() {
 827     local authtoken="${1}" recipients="${2}" message="${3}" httpcode sent=0 room color sender msg_format notify
 828
 829     if [ "${SEND_HIPCHAT}" = "YES" -a ! -z "${authtoken}" -a ! -z "${recipients}" -a ! -z "${message}" ]
 830         then
 831
 832         # A label to be shown in addition to the sender's name
 833         # Valid length range: 0 - 64.
 834         sender="netdata"
 835
 836         # Valid values: html, text.
 837         # Defaults to 'html'.
 838         msg_format="text"
 839
 840         # Background color for message. Valid values: yellow, green, red, purple, gray, random. Defaults to 'yellow'.
 841         case "${status}" in
 842             WARNING)  color="yellow" ;;
 843             CRITICAL) color="red" ;;
 844             CLEAR)    color="green" ;;
 845             *)        color="gray" ;;
 846         esac
 847
 848         # Whether this message should trigger a user notification (change the tab color, play a sound, notify mobile phones, etc).
 849         # Each recipient's notification preferences are taken into account.
 850         # Defaults to false.
 851         notify="true"
 852
 853         for room in ${recipients}
 854         do
 855             httpcode=$(${curl} -X POST --write-out %{http_code} --silent --output /dev/null \
 856                     -H "Content-type: application/json" \
 857                     -H "Authorization: Bearer ${authtoken}" \
 858                     -d "{\"color\": \"${color}\", \"from\": \"${netdata}\", \"message_format\": \"${msg_format}\", \"message\": \"${message}\", \"notify\": \"${notify}\"}" \
 859                     "https://api.hipchat.com/v2/room/${room}/notification")
 860
 861             if [ "${httpcode}" == "200" ]
 862             then
 863                 info "sent HipChat notification for: ${host} ${chart}.${name} is ${status} to '${room}'"
 864                 sent=$((sent + 1))
 865             else
 866                 error "failed to send HipChat notification for: ${host} ${chart}.${name} is ${status} to '${room}' with HTTP error code ${httpcode}."
 867             fi
 868         done
 869
 870         [ ${sent} -gt 0 ] && return 0
 871     fi
 872
 873     return 1
 874 }
 875
 876
 877 # -----------------------------------------------------------------------------
 878 # messagebird sender
 879
 880 send_messagebird() {
 881     local accesskey="${1}" messagebirdnumber="${2}" recipients="${3}"  title="${4}" message="${5}" httpcode sent=0 user
 882     if [ "${SEND_MESSAGEBIRD}" = "YES" -a ! -z "${accesskey}" -a ! -z "${messagebirdnumber}" -a ! -z "${recipients}" -a ! -z "${message}" -a ! -z "${title}" ]
 883         then
 884         #https://developers.messagebird.com/docs/messaging
 885         for user in ${recipients}
 886         do
 887             httpcode=$(${curl} -X POST --write-out %{http_code} --silent --output /dev/null \
 888                 --data-urlencode "originator=${messagebirdnumber}" \
 889                 --data-urlencode "recipients=${user}" \
 890                 --data-urlencode "body=${title} ${message}" \
 891                         --data-urlencode "datacoding=auto" \
 892                 -H "Authorization: AccessKey ${accesskey}" \
 893                 "https://rest.messagebird.com/messages")
 894
 895             if [ "${httpcode}" == "201" ]
 896             then
 897                 info "sent Messagebird SMS for: ${host} ${chart}.${name} is ${status} to '${user}'"
 898                 sent=$((sent + 1))
 899             else
 900                 error "failed to send Messagebird SMS for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP error code ${httpcode}."
 901             fi
 902         done
 903
 904         [ ${sent} -gt 0 ] && return 0
 905     fi
 906
 907     return 1
 908 }
 909
 910 # -----------------------------------------------------------------------------
 911 # telegram sender
 912
 913 send_telegram() {
 914     local bottoken="${1}" chatids="${2}" message="${3}" httpcode sent=0 chatid disableNotification=""
 915
 916     if [ "${status}" = "CLEAR" ]; then disableNotification="--data-urlencode disable_notification=true"; fi
 917
 918     if [ "${SEND_TELEGRAM}" = "YES" -a ! -z "${bottoken}" -a ! -z "${chatids}" -a ! -z "${message}" ];
 919     then
 920         for chatid in ${chatids}
 921         do
 922             # https://core.telegram.org/bots/api#sendmessage
 923             httpcode=$(${curl} --write-out %{http_code} --silent --output /dev/null ${disableNotification} \
 924                 --data-urlencode "parse_mode=HTML" \
 925                 --data-urlencode "disable_web_page_preview=true" \
 926                 --data-urlencode "text=${message}" \
 927                 "https://api.telegram.org/bot${bottoken}/sendMessage?chat_id=${chatid}")
 928
 929             if [ "${httpcode}" == "200" ]
 930             then
 931                 info "sent telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}'"
 932                 sent=$((sent + 1))
 933             elif [ "${httpcode}" == "401" ]
 934             then
 935                 error "failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}': Wrong bot token."
 936             else
 937                 error "failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}' with HTTP error code ${httpcode}."
 938             fi
 939         done
 940
 941         [ ${sent} -gt 0 ] && return 0
 942     fi
 943
 944     return 1
 945 }
 946
 947 # -----------------------------------------------------------------------------
 948 # slack sender
 949
 950 send_slack() {
 951     local webhook="${1}" channels="${2}" httpcode sent=0 channel color payload
 952
 953     [ "${SEND_SLACK}" != "YES" ] && return 1
 954
 955     case "${status}" in
 956         WARNING)  color="warning" ;;
 957         CRITICAL) color="danger" ;;
 958         CLEAR)    color="good" ;;
 959         *)        color="#777777" ;;
 960     esac
 961
 962     for channel in ${channels}
 963     do
 964         payload="$(cat <<EOF
 965         {
 966             "channel": "#${channel}",
 967             "username": "netdata on ${host}",
 968             "icon_url": "${images_base_url}/images/seo-performance-128.png",
 969             "text": "${host} ${status_message}, \`${chart}\` (_${family}_), *${alarm}*",
 970             "attachments": [
 971                 {
 972                     "fallback": "${alarm} - ${chart} (${family}) - ${info}",
 973                     "color": "${color}",
 974                     "title": "${alarm}",
 975                     "title_link": "${goto_url}",
 976                     "text": "${info}",
 977                     "fields": [
 978                         {
 979                             "title": "${chart}",
 980                             "short": true
 981                         },
 982                         {
 983                             "title": "${family}",
 984                             "short": true
 985                         }
 986                     ],
 987                     "thumb_url": "${image}",
 988                     "footer": "<${goto_url}|${host}>",
 989                     "ts": ${when}
 990                 }
 991             ]
 992         }
 993 EOF
 994         )"
 995
 996         httpcode=$(${curl} --write-out %{http_code} --silent --output /dev/null -X POST --data-urlencode "payload=${payload}" "${webhook}")
 997         if [ "${httpcode}" == "200" ]
 998         then
 999             info "sent slack notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
1000             sent=$((sent + 1))
1001         else
1002             error "failed to send slack notification for: ${host} ${chart}.${name} is ${status} to '${channel}', with HTTP error code ${httpcode}."
1003         fi
1004     done
1005
1006     [ ${sent} -gt 0 ] && return 0
1007
1008     return 1
1009 }
1010
1011
1012 # -----------------------------------------------------------------------------
1013 # prepare the content of the notification
1014
1015 # the url to send the user on click
1016 urlencode "${NETDATA_REGISTRY_HOSTNAME}" >/dev/null; url_host="${REPLY}"
1017 urlencode "${chart}" >/dev/null; url_chart="${REPLY}"
1018 urlencode "${family}" >/dev/null; url_family="${REPLY}"
1019 urlencode "${name}" >/dev/null; url_name="${REPLY}"
1020 goto_url="${NETDATA_REGISTRY_URL}/goto-host-from-alarm.html?host=${url_host}&chart=${url_chart}&family=${url_family}&alarm=${url_name}&alarm_unique_id=${unique_id}&alarm_id=${alarm_id}&alarm_event_id=${event_id}"
1021
1022 # the severity of the alarm
1023 severity="${status}"
1024
1025 # the time the alarm was raised
1026 duration4human ${duration} >/dev/null; duration_txt="${REPLY}"
1027 duration4human ${non_clear_duration} >/dev/null; non_clear_duration_txt="${REPLY}"
1028 raised_for="(was ${old_status,,} for ${duration_txt})"
1029
1030 # the key status message
1031 status_message="status unknown"
1032
1033 # the color of the alarm
1034 color="grey"
1035
1036 # the alarm value
1037 alarm="${name//_/ } = ${value} ${units}"
1038
1039 # the image of the alarm
1040 image="${images_base_url}/images/seo-performance-128.png"
1041
1042 # prepare the title based on status
1043 case "${status}" in
1044         CRITICAL)
1045         image="${images_base_url}/images/alert-128-red.png"
1046         status_message="is critical"
1047         color="#ca414b"
1048         ;;
1049
1050     WARNING)
1051         image="${images_base_url}/images/alert-128-orange.png"
1052         status_message="needs attention"
1053         color="#caca4b"
1054                 ;;
1055
1056         CLEAR)
1057         image="${images_base_url}/images/check-mark-2-128-green.png"
1058         status_message="recovered"
1059                 color="#77ca6d"
1060                 ;;
1061 esac
1062
1063 if [ "${status}" = "CLEAR" ]
1064 then
1065     severity="Recovered from ${old_status}"
1066     if [ ${non_clear_duration} -gt ${duration} ]
1067     then
1068         raised_for="(alarm was raised for ${non_clear_duration_txt})"
1069     fi
1070
1071     # don't show the value when the status is CLEAR
1072     # for certain alarms, this value might not have any meaning
1073     alarm="${name//_/ } ${raised_for}"
1074
1075 elif [ "${old_status}" = "WARNING" -a "${status}" = "CRITICAL" ]
1076 then
1077     severity="Escalated to ${status}"
1078     if [ ${non_clear_duration} -gt ${duration} ]
1079     then
1080         raised_for="(alarm is raised for ${non_clear_duration_txt})"
1081     fi
1082
1083 elif [ "${old_status}" = "CRITICAL" -a "${status}" = "WARNING" ]
1084 then
1085     severity="Demoted to ${status}"
1086     if [ ${non_clear_duration} -gt ${duration} ]
1087     then
1088         raised_for="(alarm is raised for ${non_clear_duration_txt})"
1089     fi
1090
1091 else
1092     raised_for=
1093 fi
1094
1095 # prepare HTML versions of elements
1096 info_html=
1097 [ ! -z "${info}" ] && info_html=" <small><br/>${info}</small>"
1098
1099 raised_for_html=
1100 [ ! -z "${raised_for}" ] && raised_for_html="<br/><small>${raised_for}</small>"
1101
1102 # -----------------------------------------------------------------------------
1103 # send the slack notification
1104
1105 # slack aggregates posts from the same username
1106 # so we use "${host} ${status}" as the bot username, to make them diff
1107
1108 send_slack "${SLACK_WEBHOOK_URL}" "${to_slack}"
1109 SENT_SLACK=$?
1110
1111 # -----------------------------------------------------------------------------
1112 # send the pushover notification
1113
1114 send_pushover "${PUSHOVER_APP_TOKEN}" "${to_pushover}" "${when}" "${goto_url}" "${status}" "${host} ${status_message} - ${name//_/ } - ${chart}" "
1115 <font color=\"${color}\"><b>${alarm}</b></font>${info_html}<br/>&nbsp;
1116 <small><b>${chart}</b><br/>Chart<br/>&nbsp;</small>
1117 <small><b>${family}</b><br/>Family<br/>&nbsp;</small>
1118 <small><b>${severity}</b><br/>Severity<br/>&nbsp;</small>
1119 <small><b>${date}${raised_for_html}</b><br/>Time<br/>&nbsp;</small>
1120 <a href=\"${goto_url}\">View Netdata</a><br/>&nbsp;
1121 <small><small>The source of this alarm is line ${src}</small></small>
1122 "
1123
1124 SENT_PUSHOVER=$?
1125
1126 # -----------------------------------------------------------------------------
1127 # send the pushbullet notification
1128
1129 send_pushbullet "${PUSHBULLET_ACCESS_TOKEN}" "${to_pushbullet}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm}\n
1130 Severity: ${severity}\n
1131 Chart: ${chart}\n
1132 Family: ${family}\n
1133 To View Netdata go to: ${goto_url}\n
1134 The source of this alarm is line ${src}"
1135
1136 SENT_PUSHBULLET=$?
1137
1138 # -----------------------------------------------------------------------------
1139 # send the twilio SMS
1140
1141 send_twilio "${TWILIO_ACCOUNT_SID}" "${TWILIO_ACCOUNT_TOKEN}" "${TWILIO_NUMBER}" "${to_twilio}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm}
1142 Severity: ${severity}
1143 Chart: ${chart}
1144 Family: ${family}
1145 ${info}"
1146
1147 SENT_TWILIO=$?
1148
1149 # -----------------------------------------------------------------------------
1150 # send the messagebird SMS
1151
1152 send_messagebird "${MESSAGEBIRD_ACCESS_KEY}" "${MESSAGEBIRD_NUMBER}" "${to_messagebird}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm}
1153 Severity: ${severity}
1154 Chart: ${chart}
1155 Family: ${family}
1156 ${info}"
1157
1158 SENT_MESSAGEBIRD=$?
1159
1160
1161 # -----------------------------------------------------------------------------
1162 # send the telegram.org message
1163
1164 # https://core.telegram.org/bots/api#formatting-options
1165 send_telegram "${TELEGRAM_BOT_TOKEN}" "${to_telegram}" "${host} ${status_message} - <b>${name//_/ }</b>
1166 ${chart} (${family})
1167 <a href=\"${goto_url}\">${alarm}</a>
1168 <i>${info}</i>"
1169
1170 SENT_TELEGRAM=$?
1171
1172
1173 # -----------------------------------------------------------------------------
1174 # send the kafka message
1175
1176 send_kafka
1177 SENT_KAFKA=$?
1178
1179
1180 # -----------------------------------------------------------------------------
1181 # send the pagerduty.com message
1182
1183 send_pd "${to_pd}"
1184 SENT_PD=$?
1185
1186
1187 # -----------------------------------------------------------------------------
1188 # send hipchat message
1189
1190 send_hipchat "${HIPCHAT_AUTH_TOKEN}" "${to_hipchat}" "
1191 <b>${alarm}</b> ${info_html}<br/>&nbsp;
1192 <small><b>${chart}</b><br/>Chart<br/>&nbsp;</small>
1193 <small><b>${family}</b><br/>Family<br/>&nbsp;</small>
1194 <small><b>${severity}</b><br/>Severity<br/>&nbsp;</small>
1195 <small><b>${date}${raised_for_html}</b><br/>Time<br/>&nbsp;</small>
1196 <a href=\"${goto_url}\">View Netdata</a><br/>&nbsp;
1197 <small><small>The source of this alarm is line ${src}</small></small>
1198 "
1199
1200 SENT_HIPCHAT=$?
1201
1202 # -----------------------------------------------------------------------------
1203 # send the email
1204
1205 send_email <<EOF
1206 To: ${to_email}
1207 Subject: ${host} ${status_message} - ${name//_/ } - ${chart}
1208 Content-Type: text/html
1209
1210 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
1211 <html xmlns="http://www.w3.org/1999/xhtml" style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0; padding: 0;">
1212 <body style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 14px; width: 100% !important; min-height: 100%; line-height: 1.6; background: #f6f6f6; margin:0; padding: 0;">
1213 <table>
1214     <tbody>
1215     <tr>
1216         <td style="vertical-align: top;" valign="top"></td>
1217         <td width="700" style="vertical-align: top; display: block !important; max-width: 700px !important; clear: both !important; margin: 0 auto; padding: 0;" valign="top">
1218             <div style="max-width: 700px; display: block; margin: 0 auto; padding: 20px;">
1219                 <table width="100%" cellpadding="0" cellspacing="0" style="background: #fff; border: 1px solid #e9e9e9;">
1220                     <tbody>
1221                     <tr>
1222                         <td bgcolor="#eee" style="padding: 5px 20px 5px 20px; background-color: #eee;">
1223                             <div style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 20px; color: #777; font-weight: bold;">netdata notification</div>
1224                         </td>
1225                     </tr>
1226                     <tr>
1227                         <td bgcolor="${color}" style="font-size: 16px; vertical-align: top; font-weight: 400; text-align: center; margin: 0; padding: 10px; color: #ffffff; background: ${color} !important; border: 1px solid ${color}; border-top-color: ${color};" align="center" valign="top">
1228                             <h1 style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-weight: 400; margin: 0;">${host} ${status_message}</h1>
1229                         </td>
1230                     </tr>
1231                     <tr>
1232                         <td style="vertical-align: top;" valign="top">
1233                             <div style="margin: 0; padding: 20px; max-width: 700px;">
1234                                 <table width="100%" cellpadding="0" cellspacing="0" style="max-width:700px">
1235                                     <tbody>
1236                                     <tr>
1237                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding:0 0 20px;" align="left" valign="top">
1238                                             <span>${chart}</span>
1239                                             <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Chart</span>
1240                                         </td>
1241                                     </tr>
1242                                     <tr style="margin: 0; padding: 0;">
1243                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
1244                                             <span><b>${alarm}</b>${info_html}</span>
1245                                             <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Alarm</span>
1246                                         </td>
1247                                     </tr>
1248                                     <tr>
1249                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
1250                                             <span>${family}</span>
1251                                             <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Family</span>
1252                                         </td>
1253                                     </tr>
1254                                     <tr style="margin: 0; padding: 0;">
1255                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
1256                                             <span>${severity}</span>
1257                                             <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Severity</span>
1258                                         </td>
1259                                     </tr>
1260                                     <tr style="margin: 0; padding: 0;">
1261                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top"><span>${date}</span>
1262                                             <span>${raised_for_html}</span> <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Time</span>
1263                                         </td>
1264                                     </tr>
1265                                     <tr style="margin: 0; padding: 0;">
1266                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;">
1267                                             <a href="${goto_url}" style="font-size: 14px; color: #ffffff; text-decoration: none; line-height: 1.5; font-weight: bold; text-align: center; display: inline-block; text-transform: capitalize; background: #35568d; border-width: 1px; border-style: solid; border-color: #2b4c86; margin: 0; padding: 10px 15px;" target="_blank">View Netdata</a>
1268                                         </td>
1269                                     </tr>
1270                                     <tr style="text-align: center; margin: 0; padding: 0;">
1271                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 11px; vertical-align: top; margin: 0; padding: 10px 0 0 0; color: #666666;" align="center" valign="bottom">The source of this alarm is line <code>${src}</code>
1272                                         </td>
1273                                     </tr>
1274                                     <tr style="text-align: center; margin: 0; padding: 0;">
1275                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 12px; vertical-align: top; margin:0; padding: 20px 0 0 0; color: #666666; border-top: 1px solid #f0f0f0;" align="center" valign="bottom">Sent by
1276                                             <a href="https://mynetdata.io/" target="_blank">netdata</a>, the real-time performance monitoring.
1277                                         </td>
1278                                     </tr>
1279                                     </tbody>
1280                                 </table>
1281                             </div>
1282                         </td>
1283                     </tr>
1284                     </tbody>
1285                 </table>
1286             </div>
1287         </td>
1288     </tr>
1289     </tbody>
1290 </table>
1291 </body>
1292 </html>
1293 EOF
1294
1295 SENT_EMAIL=$?
1296
1297 # -----------------------------------------------------------------------------
1298 # let netdata know
1299
1300 if [   ${SENT_EMAIL}        -eq 0 \
1301     -o ${SENT_PUSHOVER}     -eq 0 \
1302     -o ${SENT_TELEGRAM}     -eq 0 \
1303     -o ${SENT_SLACK}        -eq 0 \
1304     -o ${SENT_TWILIO}       -eq 0 \
1305     -o ${SENT_HIPCHAT}      -eq 0 \
1306     -o ${SENT_MESSAGEBIRD}  -eq 0 \
1307     -o ${SENT_PUSHBULLET}   -eq 0 \
1308     -o ${SENT_KAFKA}        -eq 0 \
1309     -o ${SENT_PD}           -eq 0 \
1310     ]
1311     then
1312     # we did send something
1313     exit 0
1314 fi
1315
1316 # we did not send anything
1317 exit 1