plugins.d/alarm-notify.sh

   1 #!/usr/bin/env bash
   2
   3 # (C) Costa Tsaousis
   4 # pushover support by Jan Arnold
   5
   6 me="${0}"
   7
   8 if [ $(( ${BASH_VERSINFO[0]} )) -lt 4 ]
   9 then
  10     echo >&2
  11     echo >&2 "$me: ERROR"
  12     echo >&2 "BASH version 4 or later is required."
  13     echo >&2 "You are running version: ${BASH_VERSION}"
  14     echo >&2 "Please upgrade."
  15     echo >&2
  16     exit 1
  17 fi
  18
  19 # defaults to allow running this script by hand
  20 NETDATA_CONFIG_DIR="${NETDATA_CONFIG_DIR-/etc/netdata}"
  21 NETDATA_CACHE_DIR="${NETDATA_CACHE_DIR-/var/cache/netdata}"
  22 [ -z "${NETDATA_REGISTRY_URL}" ] && NETDATA_REGISTRY_URL="https://registry.my-netdata.io"
  23 [ -z "${NETDATA_HOSTNAME}" ] && NETDATA_HOSTNAME="$(hostname)"
  24 [ -z "${NETDATA_REGISTRY_HOSTNAME}" ] && NETDATA_REGISTRY_HOSTNAME="${NETDATA_HOSTNAME}"
  25
  26 # -----------------------------------------------------------------------------
  27 # parse command line parameters
  28
  29 roles="${1}"       # the roles that should be notified for this event
  30 host="${2}"        # the host generated this event
  31 unique_id="${3}"   # the unique id of this event
  32 alarm_id="${4}"    # the unique id of the alarm that generated this event
  33 event_id="${5}"    # the incremental id of the event, for this alarm id
  34 when="${6}"        # the timestamp this event occurred
  35 name="${7}"        # the name of the alarm, as given in netdata health.d entries
  36 chart="${8}"       # the name of the chart (type.id)
  37 family="${9}"      # the family of the chart
  38 status="${10}"     # the current status : REMOVED, UNITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
  39 old_status="${11}" # the previous status: REMOVED, UNITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
  40 value="${12}"      # the current value of the alarm
  41 old_value="${13}"  # the previous value of the alarm
  42 src="${14}"        # the line number and file the alarm has been configured
  43 duration="${15}"   # the duration in seconds of the previous alarm state
  44 non_clear_duration="${16}" # the total duration in seconds this is/was non-clear
  45 units="${17}"      # the units of the value
  46 info="${18}"       # a short description of the alarm
  47
  48 # -----------------------------------------------------------------------------
  49 # screen statuses we don't need to send a notification
  50
  51 # don't do anything if this is not WARNING, CRITICAL or CLEAR
  52 if [ "${status}" != "WARNING" -a "${status}" != "CRITICAL" -a "${status}" != "CLEAR" ]
  53 then
  54     echo >&2 "${me}: not sending notification for ${status} on '${chart}.${name}'"
  55     exit 1
  56 fi
  57
  58 # don't do anything if this is CLEAR, but it was not WARNING or CRITICAL
  59 if [ "${old_status}" != "WARNING" -a "${old_status}" != "CRITICAL" -a "${status}" = "CLEAR" ]
  60 then
  61     echo >&2 "${me}: not sending notification for ${status} on '${chart}.${name}' (last status was ${old_status})"
  62     exit 1
  63 fi
  64
  65 # -----------------------------------------------------------------------------
  66 # load configuration
  67
  68 # By default fetch images from the global public registry.
  69 # This is required by default, since all notification methods need to download
  70 # images via the Internet, and private registries might not be reachable.
  71 # This can be overwritten at the configuration file.
  72 images_base_url="https://registry.my-netdata.io"
  73
  74 # needed commands
  75 # if empty they will be searched in the system path
  76 curl=
  77 sendmail=
  78
  79 # enable / disable features
  80 SEND_SLACK="YES"
  81 SEND_PUSHOVER="YES"
  82 SEND_TELEGRAM="YES"
  83 SEND_EMAIL="YES"
  84
  85 # slack configs
  86 SLACK_WEBHOOK_URL=
  87 DEFAULT_RECIPIENT_SLACK=
  88 declare -A role_recipients_slack=()
  89
  90 # pushover configs
  91 PUSHOVER_APP_TOKEN=
  92 DEFAULT_RECIPIENT_PUSHOVER=
  93 declare -A role_recipients_pushover=()
  94
  95 # telegram configs
  96 TELEGRAM_BOT_TOKEN=
  97 DEFAULT_RECIPIENT_TELEGRAM=
  98 declare -A role_recipients_telegram=()
  99
 100 # email configs
 101 DEFAULT_RECIPIENT_EMAIL="root"
 102 declare -A role_recipients_email=()
 103
 104 # load the user configuration
 105 # this will overwrite the variables above
 106 if [ -f "${NETDATA_CONFIG_DIR}/health_alarm_notify.conf" ]
 107     then
 108     source "${NETDATA_CONFIG_DIR}/health_alarm_notify.conf"
 109 fi
 110
 111 # -----------------------------------------------------------------------------
 112 # filter a recipient based on alarm event severity
 113
 114 filter_recipient_by_criticality() {
 115     local method="${1}" x="${2}" r s
 116     shift
 117
 118     r="${x/|*/}" # the recipient
 119     s="${x/*|/}" # the severity required for notifying this recipient
 120
 121     # no severity filtering for this person
 122     [ "${r}" = "${s}" ] && return 0
 123
 124     # the severity is invalid
 125     s="${s^^}"
 126     [ "${s}" != "CRITICAL" ] && return 0
 127
 128     # the new or the old status matches the severity
 129     if [ "${s}" = "${status}" -o "${s}" = "${old_status}" ]
 130         then
 131         [ ! -d "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}" ] && \
 132             mkdir -p "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}"
 133
 134         # we need to keep track of the notifications we sent
 135         # so that the same user will receive the recovery
 136         # even if old_status does not match the required severity
 137         touch "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}"
 138         return 0
 139     fi
 140
 141     # it is a cleared alarm we have sent notification for
 142     if [ "${status}" != "WARNING" -a "${status}" != "CRITICAL" -a -f "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}" ]
 143         then
 144         rm "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}"
 145         return 0
 146     fi
 147
 148     return 1
 149 }
 150
 151 # -----------------------------------------------------------------------------
 152 # find the recipients' addresses per method
 153
 154 declare -A arr_slack=()
 155 declare -A arr_pushover=()
 156 declare -A arr_telegram=()
 157 declare -A arr_email=()
 158
 159 # netdata may call us with multiple roles, and roles may have multiple but
 160 # overlapping recipients - so, here we find the unique recipients.
 161 for x in ${roles//,/ }
 162 do
 163     # the roles 'silent' and 'disabled' mean:
 164     # don't send a notification for this role
 165     [ "${x}" = "silent" -o "${x}" = "disabled" ] && continue
 166
 167     # email
 168     a="${role_recipients_email[${x}]}"
 169     [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_EMAIL}"
 170     for r in ${a//,/ }
 171     do
 172         [ "${r}" != "disabled" ] && filter_recipient_by_criticality email "${r}" && arr_email[${r/|*/}]="1"
 173     done
 174
 175     # pushover
 176     a="${role_recipients_pushover[${x}]}"
 177     [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_PUSHOVER}"
 178     for r in ${a//,/ }
 179     do
 180         [ "${r}" != "disabled" ] && filter_recipient_by_criticality pushover "${r}" && arr_pushover[${r/|*/}]="1"
 181     done
 182
 183     # telegram
 184     a="${role_recipients_telegram[${x}]}"
 185     [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_TELEGRAM}"
 186     for r in ${a//,/ }
 187     do
 188         [ "${r}" != "disabled" ] && filter_recipient_by_criticality telegram "${r}" && arr_telegram[${r/|*/}]="1"
 189     done
 190
 191     # slack
 192     a="${role_recipients_slack[${x}]}"
 193     [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_SLACK}"
 194     for r in ${a//,/ }
 195     do
 196         [ "${r}" != "disabled" ] && filter_recipient_by_criticality slack "${r}" && arr_slack[${r/|*/}]="1"
 197     done
 198 done
 199
 200 # build the list of slack recipients (channels)
 201 to_slack="${!arr_slack[*]}"
 202 [ -z "${to_slack}" ] && SEND_SLACK="NO"
 203
 204 # build the list of pushover recipients (user tokens)
 205 to_pushover="${!arr_pushover[*]}"
 206 [ -z "${to_pushover}" ] && SEND_PUSHOVER="NO"
 207
 208 # check array of telegram recipients (chat ids)
 209 to_telegram="${!arr_telegram[*]}"
 210 [ -z "${to_telegram}" ] && SEND_TELEGRAM="NO"
 211
 212 # build the list of email recipients (email addresses)
 213 to_email=
 214 for x in "${!arr_email[@]}"
 215 do
 216     [ ! -z "${to_email}" ] && to_email="${to_email}, "
 217     to_email="${to_email}${x}"
 218 done
 219 [ -z "${to_email}" ] && SEND_EMAIL="NO"
 220
 221
 222 # -----------------------------------------------------------------------------
 223 # verify the delivery methods supported
 224
 225 # check slack
 226 [ -z "${SLACK_WEBHOOK_URL}" ] && SEND_SLACK="NO"
 227
 228 # check pushover
 229 [ -z "${PUSHOVER_APP_TOKEN}" ] && SEND_PUSHOVER="NO"
 230
 231 # check telegram
 232 [ -z "${TELEGRAM_BOT_TOKEN}" ] && SEND_TELEGRAM="NO"
 233
 234 if [ \( "${SEND_PUSHOVER}" = "YES" -o "${SEND_SLACK}" = "YES" -o "${SEND_TELEGRAM}" = "YES" \) -a -z "${curl}" ]
 235     then
 236     curl="$(which curl 2>/dev/null || command -v curl 2>/dev/null)"
 237     if [ -z "${curl}" ]
 238         then
 239         SEND_PUSHOVER="NO"
 240         SEND_TELEGRAM="NO"
 241         SEND_SLACK="NO"
 242     fi
 243 fi
 244
 245 if [ "${SEND_EMAIL}" = "YES" -a -z "${sendmail}" ]
 246     then
 247     sendmail="$(which sendmail 2>/dev/null || command -v sendmail 2>/dev/null)"
 248     [ -z "${sendmail}" ] && SEND_EMAIL="NO"
 249 fi
 250
 251 # check that we have at least a method enabled
 252 if [ "${SEND_EMAIL}" != "YES" -a "${SEND_PUSHOVER}" != "YES" -a "${SEND_TELEGRAM}" != "YES" -a "${SEND_SLACK}" != "YES" ]
 253     then
 254     echo >&2 "All notification methods are disabled. Not sending a notification."
 255     exit 1
 256 fi
 257
 258 # -----------------------------------------------------------------------------
 259 # get the system hostname
 260
 261 [ -z "${host}" ] && host="${NETDATA_HOSTNAME}"
 262 [ -z "${host}" ] && host="${NETDATA_REGISTRY_HOSTNAME}"
 263 [ -z "${host}" ] && host="$(hostname 2>/dev/null)"
 264
 265 # -----------------------------------------------------------------------------
 266 # get the date the alarm happened
 267
 268 date="$(date --date=@${when} 2>/dev/null)"
 269 [ -z "${date}" ] && date="$(date 2>/dev/null)"
 270
 271 # -----------------------------------------------------------------------------
 272 # URL encode a string
 273
 274 urlencode() {
 275     local string="${1}" strlen encoded pos c o
 276
 277     strlen=${#string}
 278     for (( pos=0 ; pos<strlen ; pos++ ))
 279     do
 280         c=${string:$pos:1}
 281         case "$c" in
 282             [-_.~a-zA-Z0-9])
 283                 o="${c}"
 284                 ;;
 285
 286             *)
 287                 printf -v o '%%%02x' "'$c"
 288                 ;;
 289         esac
 290         encoded+="${o}"
 291     done
 292
 293     REPLY="${encoded}"
 294     echo "${REPLY}"
 295 }
 296
 297 # -----------------------------------------------------------------------------
 298 # convert a duration in seconds, to a human readable duration
 299 # using DAYS, MINUTES, SECONDS
 300
 301 duration4human() {
 302     local s="${1}" d=0 h=0 m=0 ds="day" hs="hour" ms="minute" ss="second" ret
 303     d=$(( s / 86400 ))
 304     s=$(( s - (d * 86400) ))
 305     h=$(( s / 3600 ))
 306     s=$(( s - (h * 3600) ))
 307     m=$(( s / 60 ))
 308     s=$(( s - (m * 60) ))
 309
 310     if [ ${d} -gt 0 ]
 311     then
 312         [ ${m} -ge 30 ] && h=$(( h + 1 ))
 313         [ ${d} -gt 1 ] && ds="days"
 314         [ ${h} -gt 1 ] && hs="hours"
 315         if [ ${h} -gt 0 ]
 316         then
 317             ret="${d} ${ds} and ${h} ${hs}"
 318         else
 319             ret="${d} ${ds}"
 320         fi
 321     elif [ ${h} -gt 0 ]
 322     then
 323         [ ${s} -ge 30 ] && m=$(( m + 1 ))
 324         [ ${h} -gt 1 ] && hs="hours"
 325         [ ${m} -gt 1 ] && ms="minutes"
 326         if [ ${m} -gt 0 ]
 327         then
 328             ret="${h} ${hs} and ${m} ${ms}"
 329         else
 330             ret="${h} ${hs}"
 331         fi
 332     elif [ ${m} -gt 0 ]
 333     then
 334         [ ${m} -gt 1 ] && ms="minutes"
 335         [ ${s} -gt 1 ] && ss="seconds"
 336         if [ ${s} -gt 0 ]
 337         then
 338             ret="${m} ${ms} and ${s} ${ss}"
 339         else
 340             ret="${m} ${ms}"
 341         fi
 342     else
 343         [ ${s} -gt 1 ] && ss="seconds"
 344         ret="${s} ${ss}"
 345     fi
 346
 347     REPLY="${ret}"
 348     echo "${REPLY}"
 349 }
 350
 351 # -----------------------------------------------------------------------------
 352 # email sender
 353
 354 send_email() {
 355     local ret=
 356     if [ "${SEND_EMAIL}" = "YES" ]
 357         then
 358
 359         "${sendmail}" -t
 360         ret=$?
 361
 362         if [ $ret -eq 0 ]
 363         then
 364             echo >&2 "${me}: Sent email notification for: ${host} ${chart}.${name} is ${status} to '${to_email}'"
 365             return 0
 366         else
 367             echo >&2 "${me}: Failed to send email notification for: ${host} ${chart}.${name} is ${status} to '${to_email}' with error code ${ret}."
 368             return 1
 369         fi
 370     fi
 371
 372     return 1
 373 }
 374
 375 # -----------------------------------------------------------------------------
 376 # pushover sender
 377
 378 send_pushover() {
 379     local apptoken="${1}" usertokens="${2}" when="${3}" url="${4}" status="${5}" title="${6}" message="${7}" httpcode sent=0 user priority
 380
 381     if [ "${SEND_PUSHOVER}" = "YES" -a ! -z "${apptoken}" -a ! -z "${usertokens}" -a ! -z "${title}" -a ! -z "${message}" ]
 382         then
 383
 384         # https://pushover.net/api
 385         priority=-2
 386         case "${status}" in
 387             CLEAR) priority=-1;;   # low priority: no sound or vibration
 388             WARNING) priotity=0;;  # normal priority: respect quiet hours
 389             CRITICAL) priority=1;; # high priority: bypass quiet hours
 390             *) priority=-2;;       # lowest priority: no notification at all
 391         esac
 392
 393         for user in ${usertokens}
 394         do
 395             httpcode=$(${curl} --write-out %{http_code} --silent --output /dev/null \
 396                 --form-string "token=${apptoken}" \
 397                 --form-string "user=${user}" \
 398                 --form-string "html=1" \
 399                 --form-string "title=${title}" \
 400                 --form-string "message=${message}" \
 401                 --form-string "timestamp=${when}" \
 402                 --form-string "url=${url}" \
 403                 --form-string "url_title=Open netdata dashboard to view the alarm" \
 404                 --form-string "priority=${priority}" \
 405                 https://api.pushover.net/1/messages.json)
 406
 407             if [ "${httpcode}" == "200" ]
 408             then
 409                 echo >&2 "${me}: Sent pushover notification for: ${host} ${chart}.${name} is ${status} to '${user}'"
 410                 sent=$((sent + 1))
 411             else
 412                 echo >&2 "${me}: Failed to send pushover notification for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP error code ${httpcode}."
 413             fi
 414         done
 415
 416         [ ${sent} -gt 0 ] && return 0
 417     fi
 418
 419     return 1
 420 }
 421
 422
 423 # -----------------------------------------------------------------------------
 424 # telegram sender
 425
 426 send_telegram() {
 427     local bottoken="${1}" chatids="${2}" message="${3}" httpcode sent=0 chatid disableNotification=""
 428
 429     if [ "${status}" = "CLEAR" ]; then disableNotification="--data-urlencode disable_notification=true"; fi
 430
 431     if [ "${SEND_TELEGRAM}" = "YES" -a ! -z "${bottoken}" -a ! -z "${chatids}" -a ! -z "${message}" ];
 432     then
 433         for chatid in ${chatids}
 434         do
 435             # https://core.telegram.org/bots/api#sendmessage
 436             httpcode=$(${curl} --write-out %{http_code} --silent --output /dev/null ${disableNotification} \
 437                 --data-urlencode "parse_mode=HTML" \
 438                 --data-urlencode "disable_web_page_preview=true" \
 439                 --data-urlencode "text=$message" \
 440                 "https://api.telegram.org/bot${bottoken}/sendMessage?chat_id=$chatid")
 441
 442             if [ "${httpcode}" == "200" ]
 443             then
 444                 echo >&2 "${me}: Sent telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}'"
 445                 sent=$((sent + 1))
 446             elif [ "${httpcode}" == "401" ]
 447             then
 448                 echo >&2 "${me}: Failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}': Wrong bot token."
 449             else
 450                 echo >&2 "${me}: Failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}' with HTTP error code ${httpcode}."
 451             fi
 452         done
 453
 454         [ ${sent} -gt 0 ] && return 0
 455     fi
 456
 457     return 1
 458 }
 459
 460 # -----------------------------------------------------------------------------
 461 # slack sender
 462
 463 send_slack() {
 464     local webhook="${1}" channels="${2}" httpcode sent=0 channel color payload
 465
 466     [ "${SEND_SLACK}" != "YES" ] && return 1
 467
 468     case "${status}" in
 469         WARNING) color="warning" ;;
 470         CRITICAL) color="danger" ;;
 471         CLEAR) color="good" ;;
 472         *) color="#777777" ;;
 473     esac
 474
 475     for channel in ${channels}
 476     do
 477         payload="$(cat <<EOF
 478         {
 479             "channel": "#${channel}",
 480             "username": "netdata on ${host}",
 481             "icon_url": "${images_base_url}/images/seo-performance-128.png",
 482             "text": "${host} ${status_message}, \`${chart}\` (_${family}_), *${alarm}*",
 483             "attachments": [
 484                 {
 485                     "fallback": "${alarm} - ${chart} (${family}) - ${info}",
 486                     "color": "${color}",
 487                     "title": "${alarm}",
 488                     "title_link": "${goto_url}",
 489                     "text": "${info}",
 490                     "fields": [
 491                         {
 492                             "title": "${chart}",
 493                             "short": true
 494                         },
 495                         {
 496                             "title": "${family}",
 497                             "short": true
 498                         }
 499                     ],
 500                     "thumb_url": "${image}",
 501                     "footer": "<${goto_url}|${host}>",
 502                     "ts": ${when}
 503                 }
 504             ]
 505         }
 506 EOF
 507         )"
 508
 509         httpcode=$(${curl} --write-out %{http_code} --silent --output /dev/null -X POST --data-urlencode "payload=${payload}" "${webhook}")
 510         if [ "${httpcode}" == "200" ]
 511         then
 512             echo >&2 "${me}: Sent slack notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
 513             sent=$((sent + 1))
 514         else
 515             echo >&2 "${me}: Failed to send slack notification for: ${host} ${chart}.${name} is ${status} to '${channel}', with HTTP error code ${httpcode}."
 516         fi
 517     done
 518
 519     [ ${sent} -gt 0 ] && return 0
 520
 521     return 1
 522 }
 523
 524
 525 # -----------------------------------------------------------------------------
 526 # prepare the content of the notification
 527
 528 # the url to send the user on click
 529 urlencode "${NETDATA_REGISTRY_HOSTNAME}" >/dev/null; url_host="${REPLY}"
 530 urlencode "${chart}" >/dev/null; url_chart="${REPLY}"
 531 urlencode "${family}" >/dev/null; url_family="${REPLY}"
 532 urlencode "${name}" >/dev/null; url_name="${REPLY}"
 533 goto_url="${NETDATA_REGISTRY_URL}/goto-host-from-alarm.html?host=${url_host}&chart=${url_chart}&family=${url_family}&alarm=${url_name}&alarm_unique_id=${unique_id}&alarm_id=${alarm_id}&alarm_event_id=${event_id}"
 534
 535 # the severity of the alarm
 536 severity="${status}"
 537
 538 # the time the alarm was raised
 539 duration4human ${duration} >/dev/null; duration_txt="${REPLY}"
 540 duration4human ${non_clear_duration} >/dev/null; non_clear_duration_txt="${REPLY}"
 541 raised_for="(was ${old_status,,} for ${duration_txt})"
 542
 543 # the key status message
 544 status_message="status unknown"
 545
 546 # the color of the alarm
 547 color="grey"
 548
 549 # the alarm value
 550 alarm="${name//_/ } = ${value} ${units}"
 551
 552 # the image of the alarm
 553 image="${images_base_url}/images/seo-performance-128.png"
 554
 555 # prepare the title based on status
 556 case "${status}" in
 557         CRITICAL)
 558         image="${images_base_url}/images/alert-128-red.png"
 559         status_message="is critical"
 560         color="#ca414b"
 561         ;;
 562
 563     WARNING)
 564         image="${images_base_url}/images/alert-128-orange.png"
 565         status_message="needs attention"
 566         color="#caca4b"
 567                 ;;
 568
 569         CLEAR)
 570         image="${images_base_url}/images/check-mark-2-128-green.png"
 571         status_message="recovered"
 572                 color="#77ca6d"
 573
 574                 # don't show the value when the status is CLEAR
 575                 # for certain alarms, this value might not have any meaning
 576                 alarm="${name//_/ } ${raised_for}"
 577                 ;;
 578 esac
 579
 580 if [ "${status}" = "CLEAR" ]
 581 then
 582     severity="Recovered from ${old_status}"
 583     if [ $non_clear_duration -gt $duration ]
 584     then
 585         raised_for="(alarm was raised for ${non_clear_duration_txt})"
 586     fi
 587
 588 elif [ "${old_status}" = "WARNING" -a "${status}" = "CRITICAL" ]
 589 then
 590     severity="Escalated to ${status}"
 591     if [ $non_clear_duration -gt $duration ]
 592     then
 593         raised_for="(alarm is raised for ${non_clear_duration_txt})"
 594     fi
 595
 596 elif [ "${old_status}" = "CRITICAL" -a "${status}" = "WARNING" ]
 597 then
 598     severity="Demoted to ${status}"
 599     if [ $non_clear_duration -gt $duration ]
 600     then
 601         raised_for="(alarm is raised for ${non_clear_duration_txt})"
 602     fi
 603
 604 else
 605     raised_for=
 606 fi
 607
 608 # prepare HTML versions of elements
 609 info_html=
 610 [ ! -z "${info}" ] && info_html=" <small><br/>${info}</small>"
 611
 612 raised_for_html=
 613 [ ! -z "${raised_for}" ] && raised_for_html="<br/><small>${raised_for}</small>"
 614
 615 # -----------------------------------------------------------------------------
 616 # send the slack notification
 617
 618 # slack aggregates posts from the same username
 619 # so we use "${host} ${status}" as the bot username, to make them diff
 620
 621 send_slack "${SLACK_WEBHOOK_URL}" "${to_slack}"
 622 SENT_SLACK=$?
 623
 624 # -----------------------------------------------------------------------------
 625 # send the pushover notification
 626
 627 send_pushover "${PUSHOVER_APP_TOKEN}" "${to_pushover}" "${when}" "${goto_url}" "${status}" "${host} ${status_message} - ${name//_/ } - ${chart}" "
 628 <font color=\"${color}\"><b>${alarm}</b></font>${info_html}<br/>&nbsp;
 629 <small><b>${chart}</b><br/>Chart<br/>&nbsp;</small>
 630 <small><b>${family}</b><br/>Family<br/>&nbsp;</small>
 631 <small><b>${severity}</b><br/>Severity<br/>&nbsp;</small>
 632 <small><b>${date}${raised_for_html}</b><br/>Time<br/>&nbsp;</small>
 633 <a href=\"${goto_url}\">View Netdata</a><br/>&nbsp;
 634 <small><small>The source of this alarm is line ${src}</small></small>
 635 "
 636
 637 SENT_PUSHOVER=$?
 638
 639 # -----------------------------------------------------------------------------
 640 # send the telegram.org message
 641
 642 # https://core.telegram.org/bots/api#formatting-options
 643 telegram_message="<b>${severity}"
 644 [ "${status_message}" != "recovered" ] && telegram_message="${telegram_message}, ${status_message}"
 645 telegram_message="${telegram_message}
 646 ${chart} (${family})</b>
 647 <a href=\"${goto_url}\">${alarm}</a>
 648 <i>${info}</i>"
 649
 650 send_telegram "${TELEGRAM_BOT_TOKEN}" "${to_telegram}" "${telegram_message}"
 651
 652 SENT_TELEGRAM=$?
 653
 654 # -----------------------------------------------------------------------------
 655 # send the email
 656
 657 send_email <<EOF
 658 To: ${to_email}
 659 Subject: ${host} ${status_message} - ${name//_/ } - ${chart}
 660 Content-Type: text/html
 661
 662 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
 663 <html xmlns="http://www.w3.org/1999/xhtml" style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0; padding: 0;">
 664 <body style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 14px; width: 100% !important; min-height: 100%; line-height: 1.6; background: #f6f6f6; margin:0; padding: 0;">
 665 <table>
 666     <tbody>
 667     <tr>
 668         <td style="vertical-align: top;" valign="top"></td>
 669         <td width="700" style="vertical-align: top; display: block !important; max-width: 700px !important; clear: both !important; margin: 0 auto; padding: 0;" valign="top">
 670             <div style="max-width: 700px; display: block; margin: 0 auto; padding: 20px;">
 671                 <table width="100%" cellpadding="0" cellspacing="0" style="background: #fff; border: 1px solid #e9e9e9;">
 672                     <tbody>
 673                     <tr>
 674                         <td bgcolor="#eee" style="padding: 5px 20px 5px 20px; background-color: #eee;">
 675                             <div style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 20px; color: #777; font-weight: bold;">netdata notification</div>
 676                         </td>
 677                     </tr>
 678                     <tr>
 679                         <td bgcolor="${color}" style="font-size: 16px; vertical-align: top; font-weight: 400; text-align: center; margin: 0; padding: 10px; color: #ffffff; background: ${color} !important; border: 1px solid ${color}; border-top-color: ${color};" align="center" valign="top">
 680                             <h1 style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-weight: 400; margin: 0;">${host} ${status_message}</h1>
 681                         </td>
 682                     </tr>
 683                     <tr>
 684                         <td style="vertical-align: top;" valign="top">
 685                             <div style="margin: 0; padding: 20px; max-width: 700px;">
 686                                 <table width="100%" cellpadding="0" cellspacing="0" style="max-width:700px">
 687                                     <tbody>
 688                                     <tr>
 689                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding:0 0 20px;" align="left" valign="top">
 690                                             <span>${chart}</span>
 691                                             <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Chart</span>
 692                                         </td>
 693                                     </tr>
 694                                     <tr style="margin: 0; padding: 0;">
 695                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
 696                                             <span><b>${alarm}</b>${info_html}</span>
 697                                             <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Alarm</span>
 698                                         </td>
 699                                     </tr>
 700                                     <tr>
 701                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
 702                                             <span>${family}</span>
 703                                             <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Family</span>
 704                                         </td>
 705                                     </tr>
 706                                     <tr style="margin: 0; padding: 0;">
 707                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
 708                                             <span>${severity}</span>
 709                                             <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Severity</span>
 710                                         </td>
 711                                     </tr>
 712                                     <tr style="margin: 0; padding: 0;">
 713                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top"><span>${date}</span>
 714                                             <span>${raised_for_html}</span> <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Time</span>
 715                                         </td>
 716                                     </tr>
 717                                     <tr style="margin: 0; padding: 0;">
 718                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;">
 719                                             <a href="${goto_url}" style="font-size: 14px; color: #ffffff; text-decoration: none; line-height: 1.5; font-weight: bold; text-align: center; display: inline-block; text-transform: capitalize; background: #35568d; border-width: 1px; border-style: solid; border-color: #2b4c86; margin: 0; padding: 10px 15px;" target="_blank">View Netdata</a>
 720                                         </td>
 721                                     </tr>
 722                                     <tr style="text-align: center; margin: 0; padding: 0;">
 723                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 11px; vertical-align: top; margin: 0; padding: 10px 0 0 0; color: #666666;" align="center" valign="bottom">The source of this alarm is line <code>${src}</code>
 724                                         </td>
 725                                     </tr>
 726                                     <tr style="text-align: center; margin: 0; padding: 0;">
 727                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 12px; vertical-align: top; margin:0; padding: 20px 0 0 0; color: #666666; border-top: 1px solid #f0f0f0;" align="center" valign="bottom">Sent by
 728                                             <a href="https://mynetdata.io/" target="_blank">netdata</a>, the real-time performance monitoring.
 729                                         </td>
 730                                     </tr>
 731                                     </tbody>
 732                                 </table>
 733                             </div>
 734                         </td>
 735                     </tr>
 736                     </tbody>
 737                 </table>
 738             </div>
 739         </td>
 740     </tr>
 741     </tbody>
 742 </table>
 743 </body>
 744 </html>
 745 EOF
 746
 747 SENT_EMAIL=$?
 748
 749 # -----------------------------------------------------------------------------
 750 # let netdata know
 751
 752 # we did send something
 753 [ ${SENT_EMAIL} -eq 0 -o ${SENT_PUSHOVER} -eq 0 -o ${SENT_TELEGRAM} -eq 0 -o ${SENT_SLACK} -eq 0 ] && exit 0
 754
 755 # we did not send anything
 756 exit 1