plugins.d/alarm-email.sh

   1 #!/usr/bin/env bash
   2
   3 me="${0}"
   4
   5 sendmail="$(which sendmail 2>/dev/null || command -v sendmail 2>/dev/null)"
   6 if [ -z "${sendmail}" ]
   7 then
   8     echo >&2 "I cannot send emails - there is no sendmail command available."
   9 fi
  10
  11 sendmail_from_pipe() {
  12     "${sendmail}" -t
  13
  14     if [ $? -eq 0 ]
  15     then
  16         echo >&2 "${me}: Sent notification email for ${status} on '${chart}.${name}'"
  17         return 0
  18     else
  19         echo >&2 "${me}: FAILED to send notification email for ${status} on '${chart}.${name}'"
  20         return 1
  21     fi
  22 }
  23
  24 name="${1}"       # the name of the alarm, as given in netdata health.d entries
  25 chart="${2}"      # the name of the chart (type.id)
  26 status="${3}"     # the current status : UNITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
  27 old_status="${4}" # the previous status: UNITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
  28 value="${5}"      # the current value
  29 old_value="${6}"  # the previous value
  30 src="${7}"        # the line number and file the alarm has been configured
  31 duration="${8}"   # the duration in seconds the previous state took
  32 non_clear_duration="${9}" # the total duration in seconds this is non-clear
  33
  34 # get the system hostname
  35 hostname="$(hostname)"
  36
  37 # get the current date
  38 date="$(date)"
  39
  40 duration4human() {
  41     local s="${1}" d=0 h=0 m=0 ds="day" hs="hour" ms="minute" ss="second"
  42     d=$(( s / 86400 ))
  43     s=$(( s - (d * 86400) ))
  44     h=$(( s / 3600 ))
  45     s=$(( s - (h * 3600) ))
  46     m=$(( s / 60 ))
  47     s=$(( s - (m * 60) ))
  48
  49     if [ ${d} -gt 0 ]
  50     then
  51         [ ${m} -ge 30 ] && h=$(( h + 1 ))
  52         [ ${d} -gt 1 ] && ds="days"
  53         [ ${h} -gt 1 ] && hs="hours"
  54         if [ ${h} -gt 0 ]
  55         then
  56             echo "${d} ${ds} and ${h} ${hs}"
  57         else
  58             echo "${d} ${ds}"
  59         fi
  60     elif [ ${h} -gt 0 ]
  61     then
  62         [ ${s} -ge 30 ] && m=$(( m + 1 ))
  63         [ ${h} -gt 1 ] && hs="hours"
  64         [ ${m} -gt 1 ] && ms="minutes"
  65         if [ ${m} -gt 0 ]
  66         then
  67             echo "${h} ${hs} and ${m} ${ms}"
  68         else
  69             echo "${h} ${hs}"
  70         fi
  71     elif [ ${m} -gt 0 ]
  72     then
  73         [ ${m} -gt 1 ] && ms="minutes"
  74         [ ${s} -gt 1 ] && ss="seconds"
  75         if [ ${s} -gt 0 ]
  76         then
  77             echo "${m} ${ms} and ${s} ${ss}"
  78         else
  79             echo "${m} ${ms}"
  80         fi
  81     else
  82         [ ${s} -gt 1 ] && ss="seconds"
  83         echo "${s} ${ss}"
  84     fi
  85 }
  86
  87 severity="${status}"
  88 raised_for="<br/>(was ${old_status,,} for $(duration4human ${duration}))"
  89 status_message="status unknown"
  90 color="grey"
  91 alarm="${name} = ${value}"
  92
  93 # prepare the title based on status
  94 case "${status}" in
  95         CRITICAL)
  96         status_message="is critical"
  97         color="#ca414b"
  98         ;;
  99
 100     WARNING)
 101         status_message="needs attention"
 102         color="#caca4b"
 103                 ;;
 104
 105         CLEAR)
 106         status_message="recovered"
 107                 color="#77ca6d"
 108
 109                 # don't show the value when the status is CLEAR
 110                 # for certain alarms, this value might not have any meaning
 111                 alarm="${name}"
 112                 ;;
 113 esac
 114
 115 if [ "${status}" != "WARNING" -a "${status}" != "CRITICAL" -a "${status}" != "CLEAR" ]
 116 then
 117     # don't do anything if this is not WARNING, CRITICAL or CLEAR
 118     echo >&2 "${me}: not sending notification email for ${status} on '${chart}.${name}'"
 119     exit 0
 120 elif [ "${old_status}" != "WARNING" -a "${old_status}" != "CRITICAL" -a "${status}" = "CLEAR" ]
 121 then
 122     # don't do anything if this is CLEAR, but it was not WARNING or CRITICAL
 123     echo >&2 "${me}: not sending notification email for ${status} on '${chart}.${name}' (last status was ${old_status})"
 124     exit 0
 125 elif [ "${status}" = "CLEAR" ]
 126 then
 127     severity="Recovered from ${old_status}"
 128     if [ $non_clear_duration > $duration ]
 129     then
 130         raised_for="<br/>(had issues for $(duration4human ${non_clear_duration}))"
 131     fi
 132
 133 elif [ "${old_status}" = "WARNING" -a "${status}" = "CRITICAL" ]
 134 then
 135     severity="Escalated to ${status}"
 136     if [ $non_clear_duration > $duration ]
 137     then
 138         raised_for="<br/>(has issues for $(duration4human ${non_clear_duration}))"
 139     fi
 140
 141 elif [ "${old_status}" = "CRITICAL" -a "${status}" = "WARNING" ]
 142 then
 143     severity="Demoted to ${status}"
 144     if [ $non_clear_duration > $duration ]
 145     then
 146         raised_for="<br/>(has issues for $(duration4human ${non_clear_duration}))"
 147     fi
 148
 149 else
 150     raised_for=
 151 fi
 152
 153 # send the email
 154 cat <<EOF | sendmail_from_pipe
 155 To: root
 156 Subject: ${hostname} ${status_message} - ${chart}.${name}
 157 Content-Type: text/html
 158
 159 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
 160 <html xmlns="http://www.w3.org/1999/xhtml" style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0; padding: 0;">
 161 <body style="font-family:'Helvetica Neue','Helvetica',Helvetica,Arial,sans-serif;font-size:14px;width:100%!important;min-height:100%;line-height:1.6;background:#f6f6f6;margin:0;padding:0">
 162 <table>
 163     <tbody>
 164     <tr>
 165         <td style="vertical-align:top;" valign="top"></td>
 166         <td width="700" style="vertical-align:top;display:block!important;max-width:700px!important;clear:both!important;margin:0 auto;padding:0" valign="top">
 167             <div style="max-width:700px;display:block;margin:0 auto;padding:20px">
 168                 <table width="100%" cellpadding="0" cellspacing="0"
 169                        style="background:#fff;border:1px solid #e9e9e9">
 170                     <tbody>
 171                     <tr>
 172                         <td bgcolor="#eee"
 173                             style="padding: 5px 20px 5px 20px;background-color:#eee;">
 174                             <div style="font-size:20px;color:#777;font-weight: bold;">netdata notification</div>
 175                         </td>
 176                     </tr>
 177                     <tr>
 178                         <td bgcolor="${color}"
 179                             style="font-size:16px;vertical-align:top;font-weight:400;text-align:center;margin:0;padding:10px;color:#ffffff;background:${color}!important;border:1px solid ${color};border-top-color:${color}" align="center" valign="top">
 180                             <h1 style="font-weight:400;margin:0">${hostname} ${status_message}</h1>
 181                         </td>
 182                     </tr>
 183                     <tr>
 184                         <td style="vertical-align:top" valign="top">
 185                             <div style="margin:0;padding:20px;max-width:700px">
 186                                 <table width="100%" cellpadding="0" cellspacing="0" style="max-width:700px">
 187                                     <tbody>
 188                                     <tr>
 189                                         <td style="font-size:18px;vertical-align:top;margin:0;padding:0 0 20px"
 190                                             align="left" valign="top">
 191                                             <span>${chart}</span>
 192                                             <span style="display:block;color:#666666;font-size:12px;font-weight:300;line-height:1;text-transform:uppercase">Chart</span>
 193                                         </td>
 194                                     </tr>
 195                                     <tr style="margin:0;padding:0">
 196                                         <td style="font-size:18px;vertical-align:top;margin:0;padding:0 0 20px"
 197                                             align="left" valign="top">
 198                                             <span>${alarm}</span>
 199                                             <span style="display:block;color:#666666;font-size:12px;font-weight:300;line-height:1;text-transform:uppercase">Alarm</span>
 200                                         </td>
 201                                     </tr>
 202                                     <tr style="margin:0;padding:0">
 203                                         <td style="font-size:18px;vertical-align:top;margin:0;padding:0 0 20px"
 204                                             align="left" valign="top">
 205                                             <span>${severity}</span>
 206                                             <span style="display:block;color:#666666;font-size:12px;font-weight:300;line-height:1;text-transform:uppercase">Severity</span>
 207                                         </td>
 208                                     </tr>
 209                                     <tr style="margin:0;padding:0">
 210                                         <td style="font-size:18px;vertical-align:top;margin:0;padding:0 0 20px"
 211                                             align="left" valign="top"><span>${date}</span>
 212                                             <span>${raised_for}</span> <span
 213                                                     style="display:block;color:#666666;font-size:12px;font-weight:300;line-height:1;text-transform:uppercase">Time</span>
 214                                         </td>
 215                                     </tr>
 216                                     <tr style="text-align:center;margin:0;padding:0">
 217                                         <td style="font-size:11px;vertical-align:top;margin:0;padding:10px 0 0 0;color:#666666"
 218                                             align="center" valign="bottom">The source of this alarm is line <code>${src}</code>
 219                                         </td>
 220                                     </tr>
 221                                     <tr style="text-align:center;margin:0;padding:0">
 222                                         <td style="font-size:12px;vertical-align:top;margin:0;padding:20px 0 0 0;color:#666666;border-top:1px solid #f0f0f0"
 223                                             align="center" valign="bottom">Sent by
 224                                             <a href="https://mynetdata.io/" target="_blank">netdata</a>, the real-time performance monitoring.
 225                                         </td>
 226                                     </tr>
 227                                     </tbody>
 228                                 </table>
 229                             </div>
 230                         </td>
 231                     </tr>
 232                     </tbody>
 233                 </table>
 234             </div>
 235         </td>
 236     </tr>
 237     </tbody>
 238 </table>
 239 </body>
 240 </html>
 241 EOF