added hipchat notifications; fixes #1526

diff --git a/conf.d/health_alarm_notify.conf b/conf.d/health_alarm_notify.conf
index edc55184fb3420c77925ed8574de4091a76a3360..b838e8024aac83257e72460a5f09beba0bfc9e38 100644 (file)
--- a/conf.d/health_alarm_notify.conf
+++ b/conf.d/health_alarm_notify.conf
@@ -49,8 +49,7 @@ sendmail=""
 
 # The full path of the curl command.
 # If empty, the system $PATH will be searched for it.
-# If not found, pushover, telegram and slack notifications will be
-# silently disabled.
+# If not found, most notifications will be silently disabled.
 curl=""
 
 
@@ -63,6 +62,7 @@ curl=""
 #  - pushover user tokens
 #  - telegram chat ids
 #  - slack channels
+#  - hipchat rooms
 #  - sms phone numbers
 #  - pagerduty.com (pd) services
 #
@@ -80,7 +80,7 @@ curl=""
 #  pd         : "<pd_service_key_1> <pd_service_key_2>|critical"
 #
 # If a recipient is set to empty string, the default recipient of the given
-# notification method (email, pushover, telegram, slack, pd) will be used.
+# notification method (email, pushover, telegram, slack, etc) will be used.
 # To disable a notification, use the recipient called: disabled
 # This works for all notification methods (including the default recipients).
 
@@ -141,6 +141,7 @@ SEND_PUSHBULLET="YES"
 PUSHBULLET_ACCESS_TOKEN=""
 DEFAULT_RECIPIENT_PUSHBULLET=""
 
+
 #------------------------------------------------------------------------------
 # Twilio (twilio.com) SMS options
 
@@ -162,6 +163,7 @@ TWILIO_ACCOUNT_TOKEN=""
 TWILIO_NUMBER=""
 DEFAULT_RECIPIENT_TWILIO=""
 
+
 #------------------------------------------------------------------------------
 # Messagebird (messagebird.com) SMS options
 
@@ -182,6 +184,7 @@ MESSAGEBIRD_ACCESS_KEY=""
 MESSAGEBIRD_NUMBER=""
 DEFAULT_RECIPIENT_MESSAGEBIRD=""
 
+
 #------------------------------------------------------------------------------
 # telegram (telegram.org) global notification options
 
@@ -224,6 +227,25 @@ SLACK_WEBHOOK_URL=""
 DEFAULT_RECIPIENT_SLACK=""
 
 
+#------------------------------------------------------------------------------
+# hipchat global notification options
+
+# multiple recipients can be given like this:
+#                  "ROOM1 ROOM2 ..."
+
+# enable/disable sending hipchat notifications
+SEND_HIPCHAT="YES"
+
+# api.hipchat.com authorization token
+# Without this, netdata cannot send hipchat notifications.
+HIPCHAT_AUTH_TOKEN=""
+
+# if a role's recipients are not configured, a notification will be send to
+# this hipchat room (empty = do not send a notification for unconfigured
+# roles):
+DEFAULT_RECIPIENT_HIPCHAT=""
+
+
 #------------------------------------------------------------------------------
 # kafka notification options
 
@@ -273,6 +295,8 @@ role_recipients_telegram[sysadmin]="${DEFAULT_RECIPIENT_TELEGRAM}"
 
 role_recipients_slack[sysadmin]="${DEFAULT_RECIPIENT_SLACK}"
 
+role_recipients_hipchat[sysadmin]="${DEFAULT_RECIPIENT_HIPCHAT}"
+
 role_recipients_twilio[sysadmin]="${DEFAULT_RECIPIENT_TWILIO}"
 
 role_recipients_messagebird[sysadmin]="${DEFAULT_RECIPIENT_MESSAGEBIRD}"
@@ -292,6 +316,8 @@ role_recipients_telegram[domainadmin]="${DEFAULT_RECIPIENT_TELEGRAM}"
 
 role_recipients_slack[domainadmin]="${DEFAULT_RECIPIENT_SLACK}"
 
+role_recipients_hipchat[domainadmin]="${DEFAULT_RECIPIENT_HIPCHAT}"
+
 role_recipients_twilio[domainadmin]="${DEFAULT_RECIPIENT_TWILIO}"
 
 role_recipients_messagebird[domainadmin]="${DEFAULT_RECIPIENT_MESSAGEBIRD}"
@@ -300,7 +326,7 @@ role_recipients_pd[domainadmin]="${DEFAULT_RECIPIENT_PD}"
 
 # -----------------------------------------------------------------------------
 # database servers alarms
-# mysql, redis, memcached, etc
+# mysql, redis, memcached, postgres, etc
 
 role_recipients_email[dba]="${DEFAULT_RECIPIENT_EMAIL}"
 
@@ -312,6 +338,8 @@ role_recipients_telegram[dba]="${DEFAULT_RECIPIENT_TELEGRAM}"
 
 role_recipients_slack[dba]="${DEFAULT_RECIPIENT_SLACK}"
 
+role_recipients_hipchat[dba]="${DEFAULT_RECIPIENT_HIPCHAT}"
+
 role_recipients_twilio[dba]="${DEFAULT_RECIPIENT_TWILIO}"
 
 role_recipients_messagebird[dba]="${DEFAULT_RECIPIENT_MESSAGEBIRD}"
@@ -320,7 +348,7 @@ role_recipients_pd[dba]="${DEFAULT_RECIPIENT_PD}"
 
 # -----------------------------------------------------------------------------
 # web servers alarms
-# apache, nginx, etc
+# apache, nginx, lighttpd, etc
 
 role_recipients_email[webmaster]="${DEFAULT_RECIPIENT_EMAIL}"
 
@@ -332,6 +360,8 @@ role_recipients_telegram[webmaster]="${DEFAULT_RECIPIENT_TELEGRAM}"
 
 role_recipients_slack[webmaster]="${DEFAULT_RECIPIENT_SLACK}"
 
+role_recipients_hipchat[webmaster]="${DEFAULT_RECIPIENT_HIPCHAT}"
+
 role_recipients_twilio[webmaster]="${DEFAULT_RECIPIENT_TWILIO}"
 
 role_recipients_messagebird[webmaster]="${DEFAULT_RECIPIENT_MESSAGEBIRD}"
@@ -340,7 +370,7 @@ role_recipients_pd[webmaster]="${DEFAULT_RECIPIENT_PD}"
 
 # -----------------------------------------------------------------------------
 # proxy servers alarms
-# apache, nginx, etc
+# squid, etc
 
 role_recipients_email[proxyadmin]="${DEFAULT_RECIPIENT_EMAIL}"
 
@@ -352,6 +382,8 @@ role_recipients_telegram[proxyadmin]="${DEFAULT_RECIPIENT_TELEGRAM}"
 
 role_recipients_slack[proxyadmin]="${DEFAULT_RECIPIENT_SLACK}"
 
+role_recipients_hipchat[proxyadmin]="${DEFAULT_RECIPIENT_HIPCHAT}"
+
 role_recipients_twilio[proxyadmin]="${DEFAULT_RECIPIENT_TWILIO}"
 
 role_recipients_messagebird[proxyadmin]="${DEFAULT_RECIPIENT_MESSAGEBIRD}"

diff --git a/plugins.d/alarm-notify.sh b/plugins.d/alarm-notify.sh
index 3dbc0c7c324003b185b07a41e1617155975da743..b9704162568a569d62e794a63fbca3eb008280dd 100755 (executable)
--- a/plugins.d/alarm-notify.sh
+++ b/plugins.d/alarm-notify.sh
@@ -14,15 +14,16 @@
 #  - severity filtering per recipient
 #
 # Supported notification methods:
-#  - emails
-#  - slack.com notifications
-#  - pushover.net notifications
+#  - emails by @ktsaou
+#  - slack.com notifications by @ktsaou
+#  - pushover.net notifications by @ktsaou
 #  - pushbullet.com push notifications by Tiago Peralta @tperalta82 PR #1070
 #  - telegram.org notifications by @hashworks PR #1002
 #  - twilio.com notifications by Levi Blaney @shadycuz PR #1211
-#  - kafka notifications
+#  - kafka notifications by @ktsaou #1342
 #  - pagerduty.com notifications by Jim Cooley @jimcooley PR #1373
-#  - messagebird.com notifications by @tech_no_logical
+#  - messagebird.com notifications by @tech_no_logical #1453
+#  - hipchart notifications by @ktsaou
 
 # -----------------------------------------------------------------------------
 # testing notifications
@@ -173,6 +174,7 @@ sendmail=
 SEND_SLACK="YES"
 SEND_PUSHOVER="YES"
 SEND_TWILIO="YES"
+SEND_HIPCHAT="YES"
 SEND_MESSAGEBIRD="YES"
 SEND_TELEGRAM="YES"
 SEND_EMAIL="YES"
@@ -202,6 +204,11 @@ TWILIO_NUMBER=
 DEFAULT_RECIPIENT_TWILIO=
 declare -A role_recipients_twilio=()
 
+# hipchat configs
+HIPCHAT_AUTH_TOKEN=
+DEFAULT_RECIPIENT_HIPCHAT=
+declare -A role_recipients_hipchat=()
+
 # messagebird configs
 MESSAGEBIRD_ACCESS_KEY=
 MESSAGEBIRD_NUMBER=
@@ -279,6 +286,7 @@ declare -A arr_slack=()
 declare -A arr_pushover=()
 declare -A arr_pushbullet=()
 declare -A arr_twilio=()
+declare -A arr_hipchat=()
 declare -A arr_telegram=()
 declare -A arr_pd=()
 declare -A arr_email=()
@@ -323,6 +331,14 @@ do
         [ "${r}" != "disabled" ] && filter_recipient_by_criticality twilio "${r}" && arr_twilio[${r/|*/}]="1"
     done
 
+    # hipchat
+    a="${role_recipients_hipchat[${x}]}"
+    [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_HIPCHAT}"
+    for r in ${a//,/ }
+    do
+        [ "${r}" != "disabled" ] && filter_recipient_by_criticality hipchat "${r}" && arr_hipchat[${r/|*/}]="1"
+    done
+
     # messagebird
     a="${role_recipients_messagebird[${x}]}"
     [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_MESSAGEBIRD}"
@@ -372,6 +388,10 @@ to_pushbullet="${!arr_pushbullet[*]}"
 to_twilio="${!arr_twilio[*]}"
 [ -z "${to_twilio}" ] && SEND_TWILIO="NO"
 
+# build the list of hipchat recipients (rooms)
+to_hipchat="${!arr_hipchat[*]}"
+[ -z "${to_hipchat}" ] && SEND_HIPCHAT="NO"
+
 # build the list of messagebird recipients (phone numbers)
 to_messagebird="${!arr_messagebird[*]}"
 [ -z "${to_messagebird}" ] && SEND_MESSAGEBIRD="NO"
@@ -409,6 +429,9 @@ done
 # check twilio
 [ -z "${TWILIO_ACCOUNT_TOKEN}" -o -z "${TWILIO_ACCOUNT_SID}" -o -z "${TWILIO_NUMBER}" ] && SEND_TWILIO="NO"
 
+# check hipchat
+[ -z "${HIPCHAT_AUTH_TOKEN}" ] && SEND_HIPCHAT="NO"
+
 # check messagebird
 [ -z "${MESSAGEBIRD_ACCESS_KEY}" -o -z "${MESSAGEBIRD_NUMBER}" ] && SEND_MESSAGEBIRD="NO"
 
@@ -433,7 +456,16 @@ if [ "${SEND_PD}" = "YES" ]
 fi
 
 # if we need curl, check for the curl command
-if [ \( "${SEND_PUSHOVER}" = "YES" -o "${SEND_SLACK}" = "YES" -o "${SEND_TWILIO}" = "YES" -o "${SEND_MESSAGEBIRD}" = "YES" -o "${SEND_TELEGRAM}" = "YES" -o "${SEND_PUSHBULLET}" = "YES" -o "${SEND_KAFKA}" = "YES" \) -a -z "${curl}" ]
+if [ \( \
+           "${SEND_PUSHOVER}"    = "YES" \
+        -o "${SEND_SLACK}"       = "YES" \
+        -o "${SEND_HIPCHAT}"     = "YES" \
+        -o "${SEND_TWILIO}"      = "YES" \
+        -o "${SEND_MESSAGEBIRD}" = "YES" \
+        -o "${SEND_TELEGRAM}"    = "YES" \
+        -o "${SEND_PUSHBULLET}"  = "YES" \
+        -o "${SEND_KAFKA}"       = "YES" \
+    \) -a -z "${curl}" ]
     then
     curl="$(which curl 2>/dev/null || command -v curl 2>/dev/null)"
     if [ -z "${curl}" ]
@@ -445,6 +477,7 @@ if [ \( "${SEND_PUSHOVER}" = "YES" -o "${SEND_SLACK}" = "YES" -o "${SEND_TWILIO}
         SEND_TELEGRAM="NO"
         SEND_SLACK="NO"
         SEND_TWILIO="NO"
+        SEND_HIPCHAT="NO"
         SEND_MESSAGEBIRD="NO"
         SEND_KAFKA="NO"
     fi
@@ -463,6 +496,7 @@ if [   "${SEND_EMAIL}"          != "YES" \
     -a "${SEND_TELEGRAM}"       != "YES" \
     -a "${SEND_SLACK}"          != "YES" \
     -a "${SEND_TWILIO}"         != "YES" \
+    -a "${SEND_HIPCHAT}"        != "YES" \
     -a "${SEND_MESSAGEBIRD}"    != "YES" \
     -a "${SEND_PUSHBULLET}"     != "YES" \
     -a "${SEND_KAFKA}"          != "YES" \
@@ -785,6 +819,61 @@ send_twilio() {
     return 1
 }
 
+
+# -----------------------------------------------------------------------------
+# hipchat sender
+
+send_hipchat() {
+    local authtoken="${1}" recipients="${2}" message="${3}" httpcode sent=0 room color sender msg_format notify
+
+    if [ "${SEND_HIPCHAT}" = "YES" -a ! -z "${authtoken}" -a ! -z "${recipients}" -a ! -z "${message}" ]
+        then
+
+        # A label to be shown in addition to the sender's name
+        # Valid length range: 0 - 64. 
+        sender="netdata"
+
+        # Valid values: html, text.
+        # Defaults to 'html'.
+        msg_format="text"
+
+        # Background color for message. Valid values: yellow, green, red, purple, gray, random. Defaults to 'yellow'.
+        case "${status}" in
+            WARNING)  color="yellow" ;;
+            CRITICAL) color="red" ;;
+            CLEAR)    color="green" ;;
+            *)        color="gray" ;;
+        esac
+
+        # Whether this message should trigger a user notification (change the tab color, play a sound, notify mobile phones, etc).
+        # Each recipient's notification preferences are taken into account.
+        # Defaults to false.
+        notify="true"
+
+        for room in ${recipients}
+        do
+            httpcode=$(${curl} -X POST --write-out %{http_code} --silent --output /dev/null \
+                    -H "Content-type: application/json" \
+                    -H "Authorization: Bearer ${authtoken}" \
+                    -d "{\"color\": \"${color}\", \"from\": \"${netdata}\", \"message_format\": \"${msg_format}\", \"message\": \"${message}\", \"notify\": \"${notify}\"}" \
+                    "https://api.hipchat.com/v2/room/${room}/notification")
+
+            if [ "${httpcode}" == "200" ]
+            then
+                info "sent HipChat notification for: ${host} ${chart}.${name} is ${status} to '${room}'"
+                sent=$((sent + 1))
+            else
+                error "failed to send HipChat notification for: ${host} ${chart}.${name} is ${status} to '${room}' with HTTP error code ${httpcode}."
+            fi
+        done
+
+        [ ${sent} -gt 0 ] && return 0
+    fi
+
+    return 1
+}
+
+
 # -----------------------------------------------------------------------------
 # messagebird sender
 
@@ -1095,6 +1184,21 @@ send_pd "${to_pd}"
 SENT_PD=$?
 
 
+# -----------------------------------------------------------------------------
+# send hipchat message
+
+send_hipchat "${HIPCHAT_AUTH_TOKEN}" "${to_hipchat}" "
+<b>${alarm}</b> ${info_html}<br/>&nbsp;
+<small><b>${chart}</b><br/>Chart<br/>&nbsp;</small>
+<small><b>${family}</b><br/>Family<br/>&nbsp;</small>
+<small><b>${severity}</b><br/>Severity<br/>&nbsp;</small>
+<small><b>${date}${raised_for_html}</b><br/>Time<br/>&nbsp;</small>
+<a href=\"${goto_url}\">View Netdata</a><br/>&nbsp;
+<small><small>The source of this alarm is line ${src}</small></small>
+"
+
+SENT_HIPCHAT=$?
+
 # -----------------------------------------------------------------------------
 # send the email
 
@@ -1198,6 +1302,7 @@ if [   ${SENT_EMAIL}        -eq 0 \
     -o ${SENT_TELEGRAM}     -eq 0 \
     -o ${SENT_SLACK}        -eq 0 \
     -o ${SENT_TWILIO}       -eq 0 \
+    -o ${SENT_HIPCHAT}      -eq 0 \
     -o ${SENT_MESSAGEBIRD}  -eq 0 \
     -o ${SENT_PUSHBULLET}   -eq 0 \
     -o ${SENT_KAFKA}        -eq 0 \