web_log alarms

diff --git a/conf.d/Makefile.am b/conf.d/Makefile.am
index e6f8777ee78aed7dfa147b57d647e21fe1fb06f9..4f4c88388c2dead0d24070162ad9f5285f9cbde1 100644 (file)
--- a/conf.d/Makefile.am
+++ b/conf.d/Makefile.am
@@ -78,6 +78,7 @@ dist_healthconfig_DATA = \
     health.d/retroshare.conf \
     health.d/squid.conf \
     health.d/varnish.conf \
+    health.d/web_log.conf \
     $(NULL)
 
 if LINUX

diff --git a/conf.d/health.d/web_log.conf b/conf.d/health.d/web_log.conf
new file mode 100644 (file)
index 0000000..82e7d98
--- /dev/null
+++ b/conf.d/health.d/web_log.conf
@@ -0,0 +1,118 @@
+
+# make sure we can collect web log data
+
+template: last_collected_secs
+      on: web_log.response_codes
+    calc: $now - $last_collected_t
+   units: seconds ago
+   every: 10s
+    warn: $this > (($status >= $WARNING)  ? ($update_every) : ( 5 * $update_every))
+    crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
+   delay: down 5m multiplier 1.5 max 1h
+    info: number of seconds since the last successful data collection
+      to: webmaster
+
+# -----------------------------------------------------------------------------
+# high level response code alarms
+
+template: 1m_2xx
+      on: web_log.response_codes
+  lookup: sum -1m unaligned of 2xx
+    calc: ($this == 0)?(1):($this)
+   units: requests
+   every: 10s
+    info: the sum of successful HTTP requests over the last minute
+
+template: 1m_redirects
+      on: web_log.response_codes
+  lookup: sum -1m unaligned of 3xx
+    calc: $this * 100 / ( $1m_2xx + $this )
+   units: %
+   every: 10s
+    warn: $this > (($status >= $WARNING)  ? ( 1 ) : ( 2 ))
+    crit: $this > (($status == $CRITICAL) ? ( 2 ) : ( 5 ))
+   delay: down 15m multiplier 1.5 max 1h
+    info: the ratio of HTTP redirects (3xx) vs the successful requests, \
+          over the last minute
+      to: webmaster
+
+template: 1m_bad_requests
+      on: web_log.response_codes
+  lookup: sum -1m unaligned of 4xx
+    calc: $this * 100 / ( $1m_2xx + $this )
+   units: %
+   every: 10s
+    warn: $this > (($status >= $WARNING)  ? ( 1 ) : (  5 ))
+    crit: $this > (($status == $CRITICAL) ? ( 5 ) : ( 10 ))
+   delay: down 15m multiplier 1.5 max 1h
+    info: the ratio of HTTP bad requests (4xx) vs the successful requests, \
+          over the last minute
+      to: webmaster
+
+template: 1m_internal_errors
+      on: web_log.response_codes
+  lookup: sum -1m unaligned of 5xx
+    calc: $this * 100 / ( $1m_2xx + $this )
+   units: %
+   every: 10s
+    warn: $this > (($status >= $WARNING)  ? ( 1 ) : ( 2 ))
+    crit: $this > (($status == $CRITICAL) ? ( 2 ) : ( 5 ))
+   delay: down 15m multiplier 1.5 max 1h
+    info: the ratio of HTTP internal server errors (5xx) vs the successful \
+          requests, over the last minute
+      to: webmaster
+
+# -----------------------------------------------------------------------------
+# web slow
+
+template: 10m_response_time
+      on: web_log.response_time
+  lookup: average -10m unaligned of avg
+   units: ms
+   every: 30s
+    info: the average time to respond to HTTP requests, over the last 10 minutes
+
+
+template: web_slow
+      on: web_log.response_time
+  lookup: sum -1m unaligned of avg
+   units: ms
+   every: 10s
+   green: 500
+     red: 1000
+    warn: $this > $green && $this > ($10m_response_time * 2)
+    crit: $this > $red   && $this > ($10m_response_time * 4)
+   delay: down 15m multiplier 1.5 max 1h
+    info: the average time to respond to HTTP requests, over the last 1 minute
+      to: webmaster
+
+# -----------------------------------------------------------------------------
+# web too many or too few requests
+
+template: 5m_2xx_last
+      on: web_log.response_codes
+  lookup: average -5m at -5m unaligned of 2xx
+   units: requests
+   every: 30s
+    info: average successful HTTP requests over the last 5 minutes
+
+template: 5m_2xx_now
+      on: web_log.response_codes
+  lookup: average -5m unaligned of 2xx
+   units: requests
+   every: 30s
+    info: average successful HTTP requests over the last 5 minutes
+
+template: 5m_requests_ratio
+      on: web_log.response_codes
+    calc: ($5m_2xx_last > 0)?($5m_2xx_now * 100 / $5m_2xx_last):(100)
+   units: %
+   every: 30s
+    warn: ($5m_2xx_last > 30)?($this > 200 OR $this < 50):(0)
+    crit: ($5m_2xx_last > 30)?($this > 400 OR $this < 25):(0)
+   delay: down 15m multiplier 1.5 max 1h
+options: no-clear-notification
+    info: the percentage of web requests over the last 5 minutes, \
+          compared with the previous 15 minutes
+      to: webmaster
+