plugins.d/alarm-email.sh

   1 #!/usr/bin/env bash
   2
   3 me="${0}"
   4
   5 sendmail="$(which sendmail 2>/dev/null || command -v sendmail 2>/dev/null)"
   6 if [ -z "${sendmail}" ]
   7 then
   8     echo >&2 "I cannot send emails - there is no sendmail command available."
   9 fi
  10
  11 sendmail_from_pipe() {
  12     "${sendmail}" -t
  13
  14     if [ $? -eq 0 ]
  15     then
  16         echo >&2 "${me}: Sent notification email for ${status} on '${chart}.${name}'"
  17         return 0
  18     else
  19         echo >&2 "${me}: FAILED to send notification email for ${status} on '${chart}.${name}'"
  20         return 1
  21     fi
  22 }
  23
  24 name="${1}"       # the name of the alarm, as given in netdata health.d entries
  25 chart="${2}"      # the name of the chart (type.id)
  26 family="${3}"     # the family of the chart
  27 status="${4}"     # the current status : UNITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
  28 old_status="${5}" # the previous status: UNITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
  29 value="${6}"      # the current value
  30 old_value="${7}"  # the previous value
  31 src="${8}"        # the line number and file the alarm has been configured
  32 duration="${9}"   # the duration in seconds the previous state took
  33 non_clear_duration="${10}" # the total duration in seconds this is non-clear
  34
  35 # get the system hostname
  36 hostname="${NETDATA_HOSTNAME}"
  37 [ -z "${hostname}" ] && hostname="${NETDATA_REGISTRY_HOSTNAME}"
  38 [ -z "${hostname}" ] && hostname="$(hostname)"
  39
  40 goto_url="${NETDATA_REGISTRY_URL}/alarmcallback.html?machine_guid=${NETDATA_REGISTRY_UNIQUE_ID}&chart=${chart}&family=${family}"
  41
  42 # get the current date
  43 date="$(date)"
  44
  45 duration4human() {
  46     local s="${1}" d=0 h=0 m=0 ds="day" hs="hour" ms="minute" ss="second"
  47     d=$(( s / 86400 ))
  48     s=$(( s - (d * 86400) ))
  49     h=$(( s / 3600 ))
  50     s=$(( s - (h * 3600) ))
  51     m=$(( s / 60 ))
  52     s=$(( s - (m * 60) ))
  53
  54     if [ ${d} -gt 0 ]
  55     then
  56         [ ${m} -ge 30 ] && h=$(( h + 1 ))
  57         [ ${d} -gt 1 ] && ds="days"
  58         [ ${h} -gt 1 ] && hs="hours"
  59         if [ ${h} -gt 0 ]
  60         then
  61             echo "${d} ${ds} and ${h} ${hs}"
  62         else
  63             echo "${d} ${ds}"
  64         fi
  65     elif [ ${h} -gt 0 ]
  66     then
  67         [ ${s} -ge 30 ] && m=$(( m + 1 ))
  68         [ ${h} -gt 1 ] && hs="hours"
  69         [ ${m} -gt 1 ] && ms="minutes"
  70         if [ ${m} -gt 0 ]
  71         then
  72             echo "${h} ${hs} and ${m} ${ms}"
  73         else
  74             echo "${h} ${hs}"
  75         fi
  76     elif [ ${m} -gt 0 ]
  77     then
  78         [ ${m} -gt 1 ] && ms="minutes"
  79         [ ${s} -gt 1 ] && ss="seconds"
  80         if [ ${s} -gt 0 ]
  81         then
  82             echo "${m} ${ms} and ${s} ${ss}"
  83         else
  84             echo "${m} ${ms}"
  85         fi
  86     else
  87         [ ${s} -gt 1 ] && ss="seconds"
  88         echo "${s} ${ss}"
  89     fi
  90 }
  91
  92 severity="${status}"
  93 raised_for="<br/>(was ${old_status,,} for $(duration4human ${duration}))"
  94 status_message="status unknown"
  95 color="grey"
  96 alarm="${name} = ${value}"
  97
  98 # prepare the title based on status
  99 case "${status}" in
 100         CRITICAL)
 101         status_message="is critical"
 102         color="#ca414b"
 103         ;;
 104
 105     WARNING)
 106         status_message="needs attention"
 107         color="#caca4b"
 108                 ;;
 109
 110         CLEAR)
 111         status_message="recovered"
 112                 color="#77ca6d"
 113
 114                 # don't show the value when the status is CLEAR
 115                 # for certain alarms, this value might not have any meaning
 116                 alarm="${name}"
 117                 ;;
 118 esac
 119
 120 if [ "${status}" != "WARNING" -a "${status}" != "CRITICAL" -a "${status}" != "CLEAR" ]
 121 then
 122     # don't do anything if this is not WARNING, CRITICAL or CLEAR
 123     echo >&2 "${me}: not sending notification email for ${status} on '${chart}.${name}'"
 124     exit 0
 125 elif [ "${old_status}" != "WARNING" -a "${old_status}" != "CRITICAL" -a "${status}" = "CLEAR" ]
 126 then
 127     # don't do anything if this is CLEAR, but it was not WARNING or CRITICAL
 128     echo >&2 "${me}: not sending notification email for ${status} on '${chart}.${name}' (last status was ${old_status})"
 129     exit 0
 130 elif [ "${status}" = "CLEAR" ]
 131 then
 132     severity="Recovered from ${old_status}"
 133     if [ $non_clear_duration -gt $duration ]
 134     then
 135         raised_for="<br/>(had issues for $(duration4human ${non_clear_duration}))"
 136     fi
 137
 138 elif [ "${old_status}" = "WARNING" -a "${status}" = "CRITICAL" ]
 139 then
 140     severity="Escalated to ${status}"
 141     if [ $non_clear_duration -gt $duration ]
 142     then
 143         raised_for="<br/>(has issues for $(duration4human ${non_clear_duration}))"
 144     fi
 145
 146 elif [ "${old_status}" = "CRITICAL" -a "${status}" = "WARNING" ]
 147 then
 148     severity="Demoted to ${status}"
 149     if [ $non_clear_duration -gt $duration ]
 150     then
 151         raised_for="<br/>(has issues for $(duration4human ${non_clear_duration}))"
 152     fi
 153
 154 else
 155     raised_for=
 156 fi
 157
 158 # send the email
 159 cat <<EOF | sendmail_from_pipe
 160 To: root
 161 Subject: ${hostname} ${status_message} - ${chart}.${name}
 162 Content-Type: text/html
 163
 164 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
 165 <html xmlns="http://www.w3.org/1999/xhtml" style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0; padding: 0;">
 166 <body style="font-family:'Helvetica Neue','Helvetica',Helvetica,Arial,sans-serif;font-size:14px;width:100%!important;min-height:100%;line-height:1.6;background:#f6f6f6;margin:0;padding:0">
 167 <table>
 168     <tbody>
 169     <tr>
 170         <td style="vertical-align:top;" valign="top"></td>
 171         <td width="700" style="vertical-align:top;display:block!important;max-width:700px!important;clear:both!important;margin:0 auto;padding:0" valign="top">
 172             <div style="max-width:700px;display:block;margin:0 auto;padding:20px">
 173                 <table width="100%" cellpadding="0" cellspacing="0"
 174                        style="background:#fff;border:1px solid #e9e9e9">
 175                     <tbody>
 176                     <tr>
 177                         <td bgcolor="#eee"
 178                             style="padding: 5px 20px 5px 20px;background-color:#eee;">
 179                             <div style="font-size:20px;color:#777;font-weight: bold;">netdata notification</div>
 180                         </td>
 181                     </tr>
 182                     <tr>
 183                         <td bgcolor="${color}"
 184                             style="font-size:16px;vertical-align:top;font-weight:400;text-align:center;margin:0;padding:10px;color:#ffffff;background:${color}!important;border:1px solid ${color};border-top-color:${color}" align="center" valign="top">
 185                             <h1 style="font-weight:400;margin:0">${hostname} ${status_message}</h1>
 186                         </td>
 187                     </tr>
 188                     <tr>
 189                         <td style="vertical-align:top" valign="top">
 190                             <div style="margin:0;padding:20px;max-width:700px">
 191                                 <table width="100%" cellpadding="0" cellspacing="0" style="max-width:700px">
 192                                     <tbody>
 193                                     <tr>
 194                                         <td style="font-size:18px;vertical-align:top;margin:0;padding:0 0 20px"
 195                                             align="left" valign="top">
 196                                             <span>${chart}</span>
 197                                             <span style="display:block;color:#666666;font-size:12px;font-weight:300;line-height:1;text-transform:uppercase">Chart</span>
 198                                         </td>
 199                                     </tr>
 200                                     <tr style="margin:0;padding:0">
 201                                         <td style="font-size:18px;vertical-align:top;margin:0;padding:0 0 20px"
 202                                             align="left" valign="top">
 203                                             <span>${alarm}</span>
 204                                             <span style="display:block;color:#666666;font-size:12px;font-weight:300;line-height:1;text-transform:uppercase">Alarm</span>
 205                                         </td>
 206                                     </tr>
 207                                     <tr style="margin:0;padding:0">
 208                                         <td style="font-size:18px;vertical-align:top;margin:0;padding:0 0 20px"
 209                                             align="left" valign="top">
 210                                             <span>${severity}</span>
 211                                             <span style="display:block;color:#666666;font-size:12px;font-weight:300;line-height:1;text-transform:uppercase">Severity</span>
 212                                         </td>
 213                                     </tr>
 214                                     <tr style="margin:0;padding:0">
 215                                         <td style="font-size:18px;vertical-align:top;margin:0;padding:0 0 20px"
 216                                             align="left" valign="top"><span>${date}</span>
 217                                             <span>${raised_for}</span> <span
 218                                                     style="display:block;color:#666666;font-size:12px;font-weight:300;line-height:1;text-transform:uppercase">Time</span>
 219                                         </td>
 220                                     </tr>
 221                                     <!--
 222                                     <tr style="margin:0;padding:0">
 223                                         <td style="font-size:18px;vertical-align:top;margin:0;padding:0 0 20px">
 224                                             <a href="${goto_url}" style="font-size:14px;color:#ffffff;text-decoration:none;line-height:1.5;font-weight:bold;text-align:center;display:inline-block;text-transform:capitalize;background:#35568d;border-width:1px;border-style:solid;border-color:#2b4c86;margin:0;padding:10px 15px" target="_blank">View Netdata</a>
 225                                         </td>
 226                                     </tr>
 227                                     -->
 228                                     <tr style="text-align:center;margin:0;padding:0">
 229                                         <td style="font-size:11px;vertical-align:top;margin:0;padding:10px 0 0 0;color:#666666"
 230                                             align="center" valign="bottom">The source of this alarm is line <code>${src}</code>
 231                                         </td>
 232                                     </tr>
 233                                     <tr style="text-align:center;margin:0;padding:0">
 234                                         <td style="font-size:12px;vertical-align:top;margin:0;padding:20px 0 0 0;color:#666666;border-top:1px solid #f0f0f0"
 235                                             align="center" valign="bottom">Sent by
 236                                             <a href="https://mynetdata.io/" target="_blank">netdata</a>, the real-time performance monitoring.
 237                                         </td>
 238                                     </tr>
 239                                     </tbody>
 240                                 </table>
 241                             </div>
 242                         </td>
 243                     </tr>
 244                     </tbody>
 245                 </table>
 246             </div>
 247         </td>
 248     </tr>
 249     </tbody>
 250 </table>
 251 </body>
 252 </html>
 253 EOF