plugins.d/alarm-email.sh

   1 #!/usr/bin/env bash
   2
   3 me="${0}"
   4
   5 sendmail="$(which sendmail 2>/dev/null || command -v sendmail 2>/dev/null)"
   6 if [ -z "${sendmail}" ]
   7 then
   8     echo >&2 "I cannot send emails - there is no sendmail command available."
   9 fi
  10
  11 sendmail_from_pipe() {
  12     "${sendmail}" -t
  13
  14     if [ $? -eq 0 ]
  15     then
  16         echo >&2 "${me}: Sent notification email for ${status} on '${chart}.${name}'"
  17         return 0
  18     else
  19         echo >&2 "${me}: FAILED to send notification email for ${status} on '${chart}.${name}'"
  20         return 1
  21     fi
  22 }
  23
  24 name="${1}"       # the name of the alarm, as given in netdata health.d entries
  25 chart="${2}"      # the name of the chart (type.id)
  26 family="${3}"     # the family of the chart
  27 status="${4}"     # the current status : UNITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
  28 old_status="${5}" # the previous status: UNITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
  29 value="${6}"      # the current value
  30 old_value="${7}"  # the previous value
  31 src="${8}"        # the line number and file the alarm has been configured
  32 duration="${9}"   # the duration in seconds the previous state took
  33 non_clear_duration="${10}" # the total duration in seconds this is non-clear
  34 units="${11}"     # the units of the value
  35 info="${12}"      # a short description of the alarm
  36
  37 [ ! -z "${info}" ] && info=" <br/>${info}"
  38
  39 # get the system hostname
  40 hostname="${NETDATA_HOSTNAME}"
  41 [ -z "${hostname}" ] && hostname="${NETDATA_REGISTRY_HOSTNAME}"
  42 [ -z "${hostname}" ] && hostname="$(hostname)"
  43
  44 goto_url="${NETDATA_REGISTRY_URL}/goto-host-from-alarm.html?machine_guid=${NETDATA_REGISTRY_UNIQUE_ID}&chart=${chart}&family=${family}"
  45
  46 # get the current date
  47 date="$(date)"
  48
  49 duration4human() {
  50     local s="${1}" d=0 h=0 m=0 ds="day" hs="hour" ms="minute" ss="second"
  51     d=$(( s / 86400 ))
  52     s=$(( s - (d * 86400) ))
  53     h=$(( s / 3600 ))
  54     s=$(( s - (h * 3600) ))
  55     m=$(( s / 60 ))
  56     s=$(( s - (m * 60) ))
  57
  58     if [ ${d} -gt 0 ]
  59     then
  60         [ ${m} -ge 30 ] && h=$(( h + 1 ))
  61         [ ${d} -gt 1 ] && ds="days"
  62         [ ${h} -gt 1 ] && hs="hours"
  63         if [ ${h} -gt 0 ]
  64         then
  65             echo "${d} ${ds} and ${h} ${hs}"
  66         else
  67             echo "${d} ${ds}"
  68         fi
  69     elif [ ${h} -gt 0 ]
  70     then
  71         [ ${s} -ge 30 ] && m=$(( m + 1 ))
  72         [ ${h} -gt 1 ] && hs="hours"
  73         [ ${m} -gt 1 ] && ms="minutes"
  74         if [ ${m} -gt 0 ]
  75         then
  76             echo "${h} ${hs} and ${m} ${ms}"
  77         else
  78             echo "${h} ${hs}"
  79         fi
  80     elif [ ${m} -gt 0 ]
  81     then
  82         [ ${m} -gt 1 ] && ms="minutes"
  83         [ ${s} -gt 1 ] && ss="seconds"
  84         if [ ${s} -gt 0 ]
  85         then
  86             echo "${m} ${ms} and ${s} ${ss}"
  87         else
  88             echo "${m} ${ms}"
  89         fi
  90     else
  91         [ ${s} -gt 1 ] && ss="seconds"
  92         echo "${s} ${ss}"
  93     fi
  94 }
  95
  96 severity="${status}"
  97 raised_for="<br/>(was ${old_status,,} for $(duration4human ${duration}))"
  98 status_message="status unknown"
  99 color="grey"
 100 alarm="${name} = ${value} ${units}"
 101
 102 # prepare the title based on status
 103 case "${status}" in
 104         CRITICAL)
 105         status_message="is critical"
 106         color="#ca414b"
 107         ;;
 108
 109     WARNING)
 110         status_message="needs attention"
 111         color="#caca4b"
 112                 ;;
 113
 114         CLEAR)
 115         status_message="recovered"
 116                 color="#77ca6d"
 117
 118                 # don't show the value when the status is CLEAR
 119                 # for certain alarms, this value might not have any meaning
 120                 alarm="${name}"
 121                 ;;
 122 esac
 123
 124 if [ "${status}" != "WARNING" -a "${status}" != "CRITICAL" -a "${status}" != "CLEAR" ]
 125 then
 126     # don't do anything if this is not WARNING, CRITICAL or CLEAR
 127     echo >&2 "${me}: not sending notification email for ${status} on '${chart}.${name}'"
 128     exit 0
 129 elif [ "${old_status}" != "WARNING" -a "${old_status}" != "CRITICAL" -a "${status}" = "CLEAR" ]
 130 then
 131     # don't do anything if this is CLEAR, but it was not WARNING or CRITICAL
 132     echo >&2 "${me}: not sending notification email for ${status} on '${chart}.${name}' (last status was ${old_status})"
 133     exit 0
 134 elif [ "${status}" = "CLEAR" ]
 135 then
 136     severity="Recovered from ${old_status}"
 137     if [ $non_clear_duration -gt $duration ]
 138     then
 139         raised_for="<br/>(had issues for $(duration4human ${non_clear_duration}))"
 140     fi
 141
 142 elif [ "${old_status}" = "WARNING" -a "${status}" = "CRITICAL" ]
 143 then
 144     severity="Escalated to ${status}"
 145     if [ $non_clear_duration -gt $duration ]
 146     then
 147         raised_for="<br/>(has issues for $(duration4human ${non_clear_duration}))"
 148     fi
 149
 150 elif [ "${old_status}" = "CRITICAL" -a "${status}" = "WARNING" ]
 151 then
 152     severity="Demoted to ${status}"
 153     if [ $non_clear_duration -gt $duration ]
 154     then
 155         raised_for="<br/>(has issues for $(duration4human ${non_clear_duration}))"
 156     fi
 157
 158 else
 159     raised_for=
 160 fi
 161
 162 # send the email
 163 cat <<EOF | sendmail_from_pipe
 164 To: root
 165 Subject: ${hostname} ${status_message} - ${chart}.${name}
 166 Content-Type: text/html
 167
 168 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
 169 <html xmlns="http://www.w3.org/1999/xhtml" style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0; padding: 0;">
 170 <body style="font-family:'Helvetica Neue','Helvetica',Helvetica,Arial,sans-serif;font-size:14px;width:100%!important;min-height:100%;line-height:1.6;background:#f6f6f6;margin:0;padding:0">
 171 <table>
 172     <tbody>
 173     <tr>
 174         <td style="vertical-align:top;" valign="top"></td>
 175         <td width="700" style="vertical-align:top;display:block!important;max-width:700px!important;clear:both!important;margin:0 auto;padding:0" valign="top">
 176             <div style="max-width:700px;display:block;margin:0 auto;padding:20px">
 177                 <table width="100%" cellpadding="0" cellspacing="0"
 178                        style="background:#fff;border:1px solid #e9e9e9">
 179                     <tbody>
 180                     <tr>
 181                         <td bgcolor="#eee"
 182                             style="padding: 5px 20px 5px 20px;background-color:#eee;">
 183                             <div style="font-size:20px;color:#777;font-weight: bold;">netdata notification</div>
 184                         </td>
 185                     </tr>
 186                     <tr>
 187                         <td bgcolor="${color}"
 188                             style="font-size:16px;vertical-align:top;font-weight:400;text-align:center;margin:0;padding:10px;color:#ffffff;background:${color}!important;border:1px solid ${color};border-top-color:${color}" align="center" valign="top">
 189                             <h1 style="font-weight:400;margin:0">${hostname} ${status_message}</h1>
 190                         </td>
 191                     </tr>
 192                     <tr>
 193                         <td style="vertical-align:top" valign="top">
 194                             <div style="margin:0;padding:20px;max-width:700px">
 195                                 <table width="100%" cellpadding="0" cellspacing="0" style="max-width:700px">
 196                                     <tbody>
 197                                     <tr>
 198                                         <td style="font-size:18px;vertical-align:top;margin:0;padding:0 0 20px"
 199                                             align="left" valign="top">
 200                                             <span>${chart}</span>
 201                                             <span style="display:block;color:#666666;font-size:12px;font-weight:300;line-height:1;text-transform:uppercase">Chart</span>
 202                                         </td>
 203                                     </tr>
 204                                     <tr style="margin:0;padding:0">
 205                                         <td style="font-size:18px;vertical-align:top;margin:0;padding:0 0 20px"
 206                                             align="left" valign="top">
 207                                             <span>${alarm}${info}</span>
 208                                             <span style="display:block;color:#666666;font-size:12px;font-weight:300;line-height:1;text-transform:uppercase">Alarm</span>
 209                                         </td>
 210                                     </tr>
 211                                     <tr>
 212                                         <td style="font-size:18px;vertical-align:top;margin:0;padding:0 0 20px"
 213                                             align="left" valign="top">
 214                                             <span>${family}</span>
 215                                             <span style="display:block;color:#666666;font-size:12px;font-weight:300;line-height:1;text-transform:uppercase">Family</span>
 216                                         </td>
 217                                     </tr>
 218                                     <tr style="margin:0;padding:0">
 219                                         <td style="font-size:18px;vertical-align:top;margin:0;padding:0 0 20px"
 220                                             align="left" valign="top">
 221                                             <span>${severity}</span>
 222                                             <span style="display:block;color:#666666;font-size:12px;font-weight:300;line-height:1;text-transform:uppercase">Severity</span>
 223                                         </td>
 224                                     </tr>
 225                                     <tr style="margin:0;padding:0">
 226                                         <td style="font-size:18px;vertical-align:top;margin:0;padding:0 0 20px"
 227                                             align="left" valign="top"><span>${date}</span>
 228                                             <span>${raised_for}</span> <span
 229                                                     style="display:block;color:#666666;font-size:12px;font-weight:300;line-height:1;text-transform:uppercase">Time</span>
 230                                         </td>
 231                                     </tr>
 232                                     <!--
 233                                     <tr style="margin:0;padding:0">
 234                                         <td style="font-size:18px;vertical-align:top;margin:0;padding:0 0 20px">
 235                                             <a href="${goto_url}" style="font-size:14px;color:#ffffff;text-decoration:none;line-height:1.5;font-weight:bold;text-align:center;display:inline-block;text-transform:capitalize;background:#35568d;border-width:1px;border-style:solid;border-color:#2b4c86;margin:0;padding:10px 15px" target="_blank">View Netdata</a>
 236                                         </td>
 237                                     </tr>
 238                                     -->
 239                                     <tr style="text-align:center;margin:0;padding:0">
 240                                         <td style="font-size:11px;vertical-align:top;margin:0;padding:10px 0 0 0;color:#666666"
 241                                             align="center" valign="bottom">The source of this alarm is line <code>${src}</code>
 242                                         </td>
 243                                     </tr>
 244                                     <tr style="text-align:center;margin:0;padding:0">
 245                                         <td style="font-size:12px;vertical-align:top;margin:0;padding:20px 0 0 0;color:#666666;border-top:1px solid #f0f0f0"
 246                                             align="center" valign="bottom">Sent by
 247                                             <a href="https://mynetdata.io/" target="_blank">netdata</a>, the real-time performance monitoring.
 248                                         </td>
 249                                     </tr>
 250                                     </tbody>
 251                                 </table>
 252                             </div>
 253                         </td>
 254                     </tr>
 255                     </tbody>
 256                 </table>
 257             </div>
 258         </td>
 259     </tr>
 260     </tbody>
 261 </table>
 262 </body>
 263 </html>
 264 EOF