plugins.d/alarm-notify.sh

   1 #!/usr/bin/env bash
   2
   3 # (C) Costa Tsaousis
   4 # pushover support by Jan Arnold
   5
   6 me="${0}"
   7
   8 if [ $(( ${BASH_VERSINFO[0]} )) -lt 4 ]
   9 then
  10     echo >&2
  11     echo >&2 "$me: ERROR"
  12     echo >&2 "BASH version 4 or later is required."
  13     echo >&2 "You are running version: ${BASH_VERSION}"
  14     echo >&2 "Please upgrade."
  15     echo >&2
  16     exit 1
  17 fi
  18
  19 # defaults to allow running this script by hand
  20 NETDATA_CONFIG_DIR="${NETDATA_CONFIG_DIR-/etc/netdata}"
  21 NETDATA_CACHE_DIR="${NETDATA_CACHE_DIR-/var/cache/netdata}"
  22 [ -z "${NETDATA_REGISTRY_URL}" ] && NETDATA_REGISTRY_URL="https://registry.my-netdata.io"
  23 [ -z "${NETDATA_HOSTNAME}" ] && NETDATA_HOSTNAME="$(hostname)"
  24 [ -z "${NETDATA_REGISTRY_HOSTNAME}" ] && NETDATA_REGISTRY_HOSTNAME="${NETDATA_HOSTNAME}"
  25
  26 # -----------------------------------------------------------------------------
  27 # parse command line parameters
  28
  29 recipient="${1}"   # the recepient of the email
  30 host="${2}"        # the host this event refers to
  31 unique_id="${3}"   # the unique id of this event
  32 alarm_id="${4}"    # the unique id of the alarm that generated this event
  33 event_id="${5}"    # the incremental id of the event, for this alarm
  34 when="${6}"        # the timestamp this event occured
  35 name="${7}"        # the name of the alarm, as given in netdata health.d entries
  36 chart="${8}"       # the name of the chart (type.id)
  37 family="${9}"      # the family of the chart
  38 status="${10}"     # the current status : REMOVED, UNITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
  39 old_status="${11}" # the previous status: REMOVED, UNITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
  40 value="${12}"      # the current value
  41 old_value="${13}"  # the previous value
  42 src="${14}"        # the line number and file the alarm has been configured
  43 duration="${15}"   # the duration in seconds the previous state took
  44 non_clear_duration="${16}" # the total duration in seconds this is non-clear
  45 units="${17}"      # the units of the value
  46 info="${18}"       # a short description of the alarm
  47
  48 # -----------------------------------------------------------------------------
  49 # screen statuses we don't need to send a notification
  50
  51 # don't do anything if this is not WARNING, CRITICAL or CLEAR
  52 if [ "${status}" != "WARNING" -a "${status}" != "CRITICAL" -a "${status}" != "CLEAR" ]
  53 then
  54     echo >&2 "${me}: not sending notification for ${status} on '${chart}.${name}'"
  55     exit 1
  56 fi
  57
  58 # don't do anything if this is CLEAR, but it was not WARNING or CRITICAL
  59 if [ "${old_status}" != "WARNING" -a "${old_status}" != "CRITICAL" -a "${status}" = "CLEAR" ]
  60 then
  61     echo >&2 "${me}: not sending notification for ${status} on '${chart}.${name}' (last status was ${old_status})"
  62     exit 1
  63 fi
  64
  65 # -----------------------------------------------------------------------------
  66 # load configuration
  67
  68 # this is defined here so that private registries
  69 # can setup their own
  70 # images_base_url="${NETDATA_REGISTRY_URL}"
  71 images_base_url="https://registry.my-netdata.io"
  72
  73 # needed commands
  74 # if empty they will be searched in the system path
  75 curl=
  76 sendmail=
  77
  78 # enable / disable features
  79 SEND_SLACK="YES"
  80 SEND_PUSHOVER="YES"
  81 SEND_EMAIL="YES"
  82
  83 # slack configs
  84 SLACK_WEBHOOK_URL=
  85 DEFAULT_RECIPIENT_SLACK=
  86 declare -A role_recipients_slack=()
  87
  88 # pushover configs
  89 PUSHOVER_APP_TOKEN=
  90 DEFAULT_RECIPIENT_PUSHOVER=
  91 declare -A role_recipients_pushover=()
  92
  93 # email configs
  94 DEFAULT_RECIPIENT_EMAIL="root"
  95 declare -A role_recipients_email=()
  96
  97 # load the user configuration
  98 # this will overwrite the variables above
  99 if [ -f "${NETDATA_CONFIG_DIR}/health_alarm_notify.conf" ]
 100     then
 101     source "${NETDATA_CONFIG_DIR}/health_alarm_notify.conf"
 102 fi
 103
 104 # -----------------------------------------------------------------------------
 105 # filter recipients based on the criticality of each
 106
 107 filter_recipient_by_criticality() {
 108     local method="${1}" x="${2}" r s
 109     shift
 110
 111     r="${x/|*/}"
 112     s="${x/*|/}"
 113
 114     # no severity filtering for this person
 115     [ "${r}" = "${s}" ] && return 0
 116
 117     # the severity is invalid
 118     s="${s^^}"
 119     [ "${s}" != "CRITICAL" ] && return 0
 120
 121     # the new or the old status matches the severity
 122     if [ "${s}" = "${status}" -o "${s}" = "${old_status}" ]
 123         then
 124         [ ! -d "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}" ] && \
 125             mkdir -p "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}"
 126
 127         # we need to keep track of the notifications we sent
 128         # so that the same user will receive the recovery
 129         # even if old_status does not match the required severity
 130         touch "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}"
 131         return 0
 132     fi
 133
 134     # it is a cleared alarm we have sent notification for
 135     if [ "${status}" != "WARNING" -a "${status}" != "CRITICAL" -a -f "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}" ]
 136         then
 137         rm "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}"
 138         return 0
 139     fi
 140
 141     return 1
 142 }
 143
 144 # -----------------------------------------------------------------------------
 145 # find the recipient's addresses per method
 146
 147 declare -A arr_slack=()
 148 declare -A arr_pushover=()
 149 declare -A arr_email=()
 150
 151 # netdata may call us with multiple recipients
 152 # so, here we find the unique ones
 153 for x in ${recipient//,/ }
 154 do
 155     # email
 156     a="${role_recipients_email[${recipient}]}"
 157     [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_EMAIL}"
 158     for r in ${a//,/ }
 159     do
 160         [ "${r}" != "disabled" ] && filter_recipient_by_criticality email "${r}" && arr_email[${r/|*/}]="1"
 161     done
 162
 163     # pushover
 164     a="${role_recipients_pushover[${recipient}]}"
 165     [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_PUSHOVER}"
 166     for r in ${a//,/ }
 167     do
 168         [ "${r}" != "disabled" ] && filter_recipient_by_criticality pushover "${r}" && arr_pushover[${r/|*/}]="1"
 169     done
 170
 171     # slack
 172     a="${role_recipients_slack[${recipient}]}"
 173     [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_SLACK}"
 174     for r in ${a//,/ }
 175     do
 176         [ "${r}" != "disabled" ] && filter_recipient_by_criticality slack "${r}" && arr_slack[${r/|*/}]="1"
 177     done
 178 done
 179
 180 # build the list of slack recipients (channels)
 181 to_slack="${!arr_slack[*]}"
 182 [ -z "${to_slack}" ] && SEND_SLACK="NO"
 183
 184 # build the list of pushover recipients (user tokens)
 185 to_pushover="${!arr_pushover[*]}"
 186 [ -z "${to_pushover}" ] && SEND_PUSHOVER="NO"
 187
 188 # build the list of email recipients (email addresses)
 189 to_email=
 190 for x in "${!arr_email[@]}"
 191 do
 192     [ ! -z "${to_email}" ] && to_email="${to_email}, "
 193     to_email="${to_email}${x}"
 194 done
 195 [ -z "${to_email}" ] && SEND_EMAIL="NO"
 196
 197
 198 # -----------------------------------------------------------------------------
 199 # verify the delivery methods supported
 200
 201 # check slack
 202 [ -z "${SLACK_WEBHOOK_URL}" ] && SEND_SLACK="NO"
 203
 204 # check pushover
 205 [ -z "${PUSHOVER_APP_TOKEN}" ] && SEND_PUSHOVER="NO"
 206
 207 if [ \( "${SEND_PUSHOVER}" = "YES" -o "${SEND_SLACK}" = "YES" \) -a -z "${curl}" ]
 208     then
 209     curl="$(which curl 2>/dev/null || command -v curl 2>/dev/null)"
 210     if [ -z "${curl}" ]
 211         then
 212         SEND_PUSHOVER="NO"
 213         SEND_SLACK="NO"
 214     fi
 215 fi
 216
 217 if [ "${SEND_EMAIL}" = "YES" -a -z "${sendmail}" ]
 218     then
 219     sendmail="$(which sendmail 2>/dev/null || command -v sendmail 2>/dev/null)"
 220     [ -z "${sendmail}" ] && SEND_EMAIL="NO"
 221 fi
 222
 223 # check that we have at least a method enabled
 224 if [ "${SEND_EMAIL}" != "YES" -a "${SEND_PUSHOVER}" != "YES" -a "${SEND_SLACK}" != "YES" ]
 225     then
 226     echo >&2 "All notification methods are disabled. Not sending a notification."
 227     exit 1
 228 fi
 229
 230 # -----------------------------------------------------------------------------
 231 # get the system hostname
 232
 233 [ -z "${host}" ] && host="${NETDATA_HOSTNAME}"
 234 [ -z "${host}" ] && host="${NETDATA_REGISTRY_HOSTNAME}"
 235 [ -z "${host}" ] && host="$(hostname 2>/dev/null)"
 236
 237 # -----------------------------------------------------------------------------
 238 # get the date the alarm happened
 239
 240 date="$(date --date=@${when} 2>/dev/null)"
 241 [ -z "${date}" ] && date="$(date 2>/dev/null)"
 242
 243 # -----------------------------------------------------------------------------
 244 # URL encode a string
 245
 246 urlencode() {
 247     local string="${1}" strlen encoded pos c o
 248
 249     strlen=${#string}
 250     for (( pos=0 ; pos<strlen ; pos++ ))
 251     do
 252         c=${string:$pos:1}
 253         case "$c" in
 254             [-_.~a-zA-Z0-9])
 255                 o="${c}"
 256                 ;;
 257
 258             *)
 259                 printf -v o '%%%02x' "'$c"
 260                 ;;
 261         esac
 262         encoded+="${o}"
 263     done
 264
 265     REPLY="${encoded}"
 266     echo "${REPLY}"
 267 }
 268
 269 # -----------------------------------------------------------------------------
 270 # convert a duration in seconds, to a human readable duration
 271 # using DAYS, MINUTES, SECONDS
 272
 273 duration4human() {
 274     local s="${1}" d=0 h=0 m=0 ds="day" hs="hour" ms="minute" ss="second" ret
 275     d=$(( s / 86400 ))
 276     s=$(( s - (d * 86400) ))
 277     h=$(( s / 3600 ))
 278     s=$(( s - (h * 3600) ))
 279     m=$(( s / 60 ))
 280     s=$(( s - (m * 60) ))
 281
 282     if [ ${d} -gt 0 ]
 283     then
 284         [ ${m} -ge 30 ] && h=$(( h + 1 ))
 285         [ ${d} -gt 1 ] && ds="days"
 286         [ ${h} -gt 1 ] && hs="hours"
 287         if [ ${h} -gt 0 ]
 288         then
 289             ret="${d} ${ds} and ${h} ${hs}"
 290         else
 291             ret="${d} ${ds}"
 292         fi
 293     elif [ ${h} -gt 0 ]
 294     then
 295         [ ${s} -ge 30 ] && m=$(( m + 1 ))
 296         [ ${h} -gt 1 ] && hs="hours"
 297         [ ${m} -gt 1 ] && ms="minutes"
 298         if [ ${m} -gt 0 ]
 299         then
 300             ret="${h} ${hs} and ${m} ${ms}"
 301         else
 302             ret="${h} ${hs}"
 303         fi
 304     elif [ ${m} -gt 0 ]
 305     then
 306         [ ${m} -gt 1 ] && ms="minutes"
 307         [ ${s} -gt 1 ] && ss="seconds"
 308         if [ ${s} -gt 0 ]
 309         then
 310             ret="${m} ${ms} and ${s} ${ss}"
 311         else
 312             ret="${m} ${ms}"
 313         fi
 314     else
 315         [ ${s} -gt 1 ] && ss="seconds"
 316         ret="${s} ${ss}"
 317     fi
 318
 319     REPLY="${ret}"
 320     echo "${REPLY}"
 321 }
 322
 323 # -----------------------------------------------------------------------------
 324 # email sender
 325
 326 send_email() {
 327     local ret=
 328     if [ "${SEND_EMAIL}" = "YES" ]
 329         then
 330
 331         "${sendmail}" -t
 332         ret=$?
 333
 334         if [ $ret -eq 0 ]
 335         then
 336             echo >&2 "${me}: Sent email notification for: ${host} ${chart}.${name} is ${status} to '${to_email}'"
 337             return 0
 338         else
 339             echo >&2 "${me}: Failed to send email notification for: ${host} ${chart}.${name} is ${status} to '${to_email}' with error code ${ret}."
 340             return 1
 341         fi
 342     fi
 343
 344     return 1
 345 }
 346
 347 # -----------------------------------------------------------------------------
 348 # pushover sender
 349
 350 send_pushover() {
 351     local apptoken="${1}" usertokens="${2}" when="${3}" url="${4}" status="${5}" title="${6}" message="${7}" httpcode sent=0 user priority
 352
 353     if [ "${SEND_PUSHOVER}" = "YES" -a ! -z "${apptoken}" -a ! -z "${usertokens}" -a ! -z "${title}" -a ! -z "${message}" ]
 354         then
 355
 356         # https://pushover.net/api
 357         priority=-2
 358         case "${status}" in
 359             CLEAR) priority=-1;;   # low priority: no sound or vibration
 360             WARNING) priotity=0;;  # normal priority: respect quiet hours
 361             CRITICAL) priority=1;; # high priority: bypass quiet hours
 362             *) priority=-2;;       # lowest priority: no notification at all
 363         esac
 364
 365         for user in ${usertokens}
 366         do
 367             httpcode=$(${curl} --write-out %{http_code} --silent --output /dev/null \
 368                 --form-string "token=${apptoken}" \
 369                 --form-string "user=${user}" \
 370                 --form-string "html=1" \
 371                 --form-string "title=${title}" \
 372                 --form-string "message=${message}" \
 373                 --form-string "timestamp=${when}" \
 374                 --form-string "url=${url}" \
 375                 --form-string "url_title=Open netdata dashboard to view the alarm" \
 376                 --form-string "priority=${priority}" \
 377                 https://api.pushover.net/1/messages.json)
 378
 379             if [ "${httpcode}" == "200" ]
 380             then
 381                 echo >&2 "${me}: Sent pushover notification for: ${host} ${chart}.${name} is ${status} to '${user}'"
 382                 sent=$((sent + 1))
 383             else
 384                 echo >&2 "${me}: Failed to send pushover notification for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP error code ${httpcode}."
 385             fi
 386         done
 387
 388         [ ${sent} -gt 0 ] && return 0
 389     fi
 390
 391     return 1
 392 }
 393
 394 # -----------------------------------------------------------------------------
 395 # slack sender
 396
 397 send_slack() {
 398     local webhook="${1}" channels="${2}" httpcode sent=0 channel color payload
 399
 400     [ "${SEND_SLACK}" != "YES" ] && return 1
 401
 402     case "${status}" in
 403         WARNING) color="warning" ;;
 404         CRITICAL) color="danger" ;;
 405         CLEAR) color="good" ;;
 406         *) color="#777777" ;;
 407     esac
 408
 409     for channel in ${channels}
 410     do
 411         payload="$(cat <<EOF
 412         {
 413             "channel": "#${channel}",
 414             "username": "netdata on ${host}",
 415             "icon_url": "${images_base_url}/images/seo-performance-128.png",
 416             "text": "${host} ${status_message}, \`${chart}\` (_${family}_), *${alarm}*",
 417             "attachments": [
 418                 {
 419                     "fallback": "${alarm} - ${chart} (${family}) - ${info}",
 420                     "color": "${color}",
 421                     "title": "${alarm}",
 422                     "title_link": "${goto_url}",
 423                     "text": "${info}",
 424                     "fields": [
 425                         {
 426                             "title": "${chart}",
 427                             "short": true
 428                         },
 429                         {
 430                             "title": "${family}",
 431                             "short": true
 432                         }
 433                     ],
 434                     "thumb_url": "${image}",
 435                     "footer": "<${goto_url}|${host}>",
 436                     "ts": ${when}
 437                 }
 438             ]
 439         }
 440 EOF
 441         )"
 442
 443         httpcode=$(${curl} --write-out %{http_code} --silent --output /dev/null -X POST --data-urlencode "payload=${payload}" "${webhook}")
 444         if [ "${httpcode}" == "200" ]
 445         then
 446             echo >&2 "${me}: Sent slack notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
 447             sent=$((sent + 1))
 448         else
 449             echo >&2 "${me}: Failed to send slack notification for: ${host} ${chart}.${name} is ${status} to '${channel}', with HTTP error code ${httpcode}."
 450         fi
 451     done
 452
 453     [ ${sent} -gt 0 ] && return 0
 454
 455     return 1
 456 }
 457
 458
 459 # -----------------------------------------------------------------------------
 460 # prepare the content of the notification
 461
 462 # the url to send the user on click
 463 urlencode "${NETDATA_REGISTRY_HOSTNAME}" >/dev/null; url_host="${REPLY}"
 464 urlencode "${chart}" >/dev/null; url_chart="${REPLY}"
 465 urlencode "${family}" >/dev/null; url_family="${REPLY}"
 466 urlencode "${name}" >/dev/null; url_name="${REPLY}"
 467 goto_url="${NETDATA_REGISTRY_URL}/goto-host-from-alarm.html?host=${url_host}&chart=${url_chart}&family=${url_family}&alarm=${url_name}&alarm_unique_id=${unique_id}&alarm_id=${alarm_id}&alarm_event_id=${event_id}"
 468
 469 # the severity of the alarm
 470 severity="${status}"
 471
 472 # the time the alarm was raised
 473 duration4human ${duration} >/dev/null; duration_txt="${REPLY}"
 474 duration4human ${non_clear_duration} >/dev/null; non_clear_duration_txt="${REPLY}"
 475 raised_for="(was ${old_status,,} for ${duration_txt})"
 476
 477 # the key status message
 478 status_message="status unknown"
 479
 480 # the color of the alarm
 481 color="grey"
 482
 483 # the alarm value
 484 alarm="${name//_/ } = ${value} ${units}"
 485
 486 # the image of the alarm
 487 image="${images_base_url}/images/seo-performance-128.png"
 488
 489 # prepare the title based on status
 490 case "${status}" in
 491         CRITICAL)
 492         image="${images_base_url}/images/alert-128-red.png"
 493         status_message="is critical"
 494         color="#ca414b"
 495         ;;
 496
 497     WARNING)
 498         image="${images_base_url}/images/alert-128-orange.png"
 499         status_message="needs attention"
 500         color="#caca4b"
 501                 ;;
 502
 503         CLEAR)
 504         image="${images_base_url}/images/check-mark-2-128-green.png"
 505         status_message="recovered"
 506                 color="#77ca6d"
 507
 508                 # don't show the value when the status is CLEAR
 509                 # for certain alarms, this value might not have any meaning
 510                 alarm="${name//_/ } ${raised_for}"
 511                 ;;
 512 esac
 513
 514 if [ "${status}" = "CLEAR" ]
 515 then
 516     severity="Recovered from ${old_status}"
 517     if [ $non_clear_duration -gt $duration ]
 518     then
 519         raised_for="(alarm was raised for ${non_clear_duration_txt})"
 520     fi
 521
 522 elif [ "${old_status}" = "WARNING" -a "${status}" = "CRITICAL" ]
 523 then
 524     severity="Escalated to ${status}"
 525     if [ $non_clear_duration -gt $duration ]
 526     then
 527         raised_for="(alarm is raised for ${non_clear_duration_txt})"
 528     fi
 529
 530 elif [ "${old_status}" = "CRITICAL" -a "${status}" = "WARNING" ]
 531 then
 532     severity="Demoted to ${status}"
 533     if [ $non_clear_duration -gt $duration ]
 534     then
 535         raised_for="(alarm is raised for ${non_clear_duration_txt})"
 536     fi
 537
 538 else
 539     raised_for=
 540 fi
 541
 542 # prepare HTML versions of elements
 543 info_html=
 544 [ ! -z "${info}" ] && info_html=" <small><br/>${info}</small>"
 545
 546 raised_for_html=
 547 [ ! -z "${raised_for}" ] && raised_for_html="<br/><small>${raised_for}</small>"
 548
 549 # -----------------------------------------------------------------------------
 550 # send the slack notification
 551
 552 # slack aggregates posts from the same username
 553 # so we use "${host} ${status}" as the bot username, to make them diff
 554
 555 send_slack "${SLACK_WEBHOOK_URL}" "${to_slack}"
 556 SENT_SLACK=$?
 557
 558 # -----------------------------------------------------------------------------
 559 # send the pushover notification
 560
 561 send_pushover "${PUSHOVER_APP_TOKEN}" "${to_pushover}" "${when}" "${goto_url}" "${status}" "${host} ${status_message} - ${name//_/ } - ${chart}" "
 562 <font color=\"${color}\"><b>${alarm}</b></font>${info_html}<br/>&nbsp;
 563 <small><b>${chart}</b><br/>Chart<br/>&nbsp;</small>
 564 <small><b>${family}</b><br/>Family<br/>&nbsp;</small>
 565 <small><b>${severity}</b><br/>Severity<br/>&nbsp;</small>
 566 <small><b>${date}${raised_for_html}</b><br/>Time<br/>&nbsp;</small>
 567 <a href=\"${goto_url}\">View Netdata</a><br/>&nbsp;
 568 <small><small>The source of this alarm is line ${src}</small></small>
 569 "
 570
 571 SENT_PUSHOVER=$?
 572
 573 # -----------------------------------------------------------------------------
 574 # send the email
 575
 576 send_email <<EOF
 577 To: ${to_email}
 578 Subject: ${host} ${status_message} - ${name//_/ } - ${chart}
 579 Content-Type: text/html
 580
 581 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
 582 <html xmlns="http://www.w3.org/1999/xhtml" style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0; padding: 0;">
 583 <body style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 14px; width: 100% !important; min-height: 100%; line-height: 1.6; background: #f6f6f6; margin:0; padding: 0;">
 584 <table>
 585     <tbody>
 586     <tr>
 587         <td style="vertical-align: top;" valign="top"></td>
 588         <td width="700" style="vertical-align: top; display: block !important; max-width: 700px !important; clear: both !important; margin: 0 auto; padding: 0;" valign="top">
 589             <div style="max-width: 700px; display: block; margin: 0 auto; padding: 20px;">
 590                 <table width="100%" cellpadding="0" cellspacing="0" style="background: #fff; border: 1px solid #e9e9e9;">
 591                     <tbody>
 592                     <tr>
 593                         <td bgcolor="#eee" style="padding: 5px 20px 5px 20px; background-color: #eee;">
 594                             <div style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 20px; color: #777; font-weight: bold;">netdata notification</div>
 595                         </td>
 596                     </tr>
 597                     <tr>
 598                         <td bgcolor="${color}" style="font-size: 16px; vertical-align: top; font-weight: 400; text-align: center; margin: 0; padding: 10px; color: #ffffff; background: ${color} !important; border: 1px solid ${color}; border-top-color: ${color};" align="center" valign="top">
 599                             <h1 style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-weight: 400; margin: 0;">${host} ${status_message}</h1>
 600                         </td>
 601                     </tr>
 602                     <tr>
 603                         <td style="vertical-align: top;" valign="top">
 604                             <div style="margin: 0; padding: 20px; max-width: 700px;">
 605                                 <table width="100%" cellpadding="0" cellspacing="0" style="max-width:700px">
 606                                     <tbody>
 607                                     <tr>
 608                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding:0 0 20px;" align="left" valign="top">
 609                                             <span>${chart}</span>
 610                                             <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Chart</span>
 611                                         </td>
 612                                     </tr>
 613                                     <tr style="margin: 0; padding: 0;">
 614                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
 615                                             <span><b>${alarm}</b>${info_html}</span>
 616                                             <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Alarm</span>
 617                                         </td>
 618                                     </tr>
 619                                     <tr>
 620                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
 621                                             <span>${family}</span>
 622                                             <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Family</span>
 623                                         </td>
 624                                     </tr>
 625                                     <tr style="margin: 0; padding: 0;">
 626                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
 627                                             <span>${severity}</span>
 628                                             <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Severity</span>
 629                                         </td>
 630                                     </tr>
 631                                     <tr style="margin: 0; padding: 0;">
 632                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top"><span>${date}</span>
 633                                             <span>${raised_for_html}</span> <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Time</span>
 634                                         </td>
 635                                     </tr>
 636                                     <tr style="margin: 0; padding: 0;">
 637                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;">
 638                                             <a href="${goto_url}" style="font-size: 14px; color: #ffffff; text-decoration: none; line-height: 1.5; font-weight: bold; text-align: center; display: inline-block; text-transform: capitalize; background: #35568d; border-width: 1px; border-style: solid; border-color: #2b4c86; margin: 0; padding: 10px 15px;" target="_blank">View Netdata</a>
 639                                         </td>
 640                                     </tr>
 641                                     <tr style="text-align: center; margin: 0; padding: 0;">
 642                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 11px; vertical-align: top; margin: 0; padding: 10px 0 0 0; color: #666666;" align="center" valign="bottom">The source of this alarm is line <code>${src}</code>
 643                                         </td>
 644                                     </tr>
 645                                     <tr style="text-align: center; margin: 0; padding: 0;">
 646                                         <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 12px; vertical-align: top; margin:0; padding: 20px 0 0 0; color: #666666; border-top: 1px solid #f0f0f0;" align="center" valign="bottom">Sent by
 647                                             <a href="https://mynetdata.io/" target="_blank">netdata</a>, the real-time performance monitoring.
 648                                         </td>
 649                                     </tr>
 650                                     </tbody>
 651                                 </table>
 652                             </div>
 653                         </td>
 654                     </tr>
 655                     </tbody>
 656                 </table>
 657             </div>
 658         </td>
 659     </tr>
 660     </tbody>
 661 </table>
 662 </body>
 663 </html>
 664 EOF
 665
 666 SENT_EMAIL=$?
 667
 668 # -----------------------------------------------------------------------------
 669 # let netdata know
 670
 671 # we did send somehting
 672 [ ${SENT_EMAIL} -eq 0 -o ${SENT_PUSHOVER} -eq 0 -o ${SENT_SLACK} -eq 0 ] && exit 0
 673
 674 # we did not send anything
 675 exit 1