alarms reworked to lower CPU pressure on large systems

diff --git a/conf.d/health.d/cpu.conf b/conf.d/health.d/cpu.conf
index 2b04c794d0919a203fe5ce6af204d785e4b256c2..60f494d70c078c1f885bae1b731c3378df22fd34 100644 (file)
--- a/conf.d/health.d/cpu.conf
+++ b/conf.d/health.d/cpu.conf
@@ -28,6 +28,6 @@ template: 20min_steal_cpu
    every: 5m
     warn: $this > (($status >= $WARNING)  ? (5)  : (10))
     crit: $this > (($status == $CRITICAL) ? (20) : (30))
-   delay: down 15m multiplier 1.5 max 1h
+   delay: down 1h multiplier 1.5 max 2h
     info: average CPU steal time for the last 20 minutes
       to: sysadmin

diff --git a/conf.d/health.d/entropy.conf b/conf.d/health.d/entropy.conf
index d0eca8a6c83bfc5191c026486698ae418cf149d5..5dd8af502e14c3e3b793f197767881b80af2853f 100644 (file)
--- a/conf.d/health.d/entropy.conf
+++ b/conf.d/health.d/entropy.conf
@@ -3,12 +3,12 @@
 # the alarm is checked every 1 minute
 # and examines the last hour of data
 
-   alarm: 1hour_lowest_entropy
+   alarm: lowest_entropy
       on: system.entropy
-  lookup: min -1h unaligned
+  lookup: min -10m unaligned
    units: entries
    every: 5m
     warn: $this < (($status >= $WARNING) ? (200) : (100))
-   delay: down 1h multiplier 1.5 max 1h
-    info: minimum entries in the random numbers pool in the last 30 minutes
+   delay: down 1h multiplier 1.5 max 2h
+    info: minimum entries in the random numbers pool in the last 10 minutes
       to: silent

diff --git a/conf.d/health.d/net.conf b/conf.d/health.d/net.conf
index 11f7c43e7bf9480c5b2ffbd09b6dfd7cc21fedb5..924acccc3d1b3d7f7e5fff3812854574b1379fba 100644 (file)
--- a/conf.d/health.d/net.conf
+++ b/conf.d/health.d/net.conf
@@ -19,54 +19,54 @@ families: *
 
 # check if an interface is dropping packets
 # the alarm is checked every 1 minute
-# and examines the last hour of data
+# and examines the last 10 minutes of data
 
-template: 1hour_packet_drops_inbound
+template: inbound_packets_dropped
       on: net.drops
 families: *
-  lookup: sum -1h unaligned absolute of inbound
+  lookup: sum -10m unaligned absolute of inbound
    units: packets
    every: 1m
     warn: $this > 0
-   delay: down 30m multiplier 1.5 max 1h
-    info: interface inbound dropped packets in the last hour
+   delay: down 1h multiplier 1.5 max 2h
+    info: interface inbound dropped packets in the last 10 minutes
       to: sysadmin
 
-template: 1hour_packet_drops_outbound
+template: outbound_packets_dropped
       on: net.drops
 families: *
-  lookup: sum -1h unaligned absolute of outbound
+  lookup: sum -10m unaligned absolute of outbound
    units: packets
    every: 1m
     warn: $this > 0
-   delay: down 30m multiplier 1.5 max 1h
-    info: interface outbound dropped packets in the last hour
+   delay: down 1h multiplier 1.5 max 2h
+    info: interface outbound dropped packets in the last 10 minutes
       to: sysadmin
 
-template: 1hour_packet_drops_ratio_inbound
+template: inbound_packets_dropped_ratio
       on: net.packets
 families: *
-  lookup: sum -1h unaligned absolute of received
-    calc: (($1hour_packet_drops_inbound != nan AND $this > 0) ? ($1hour_packet_drops_inbound * 100 / $this) : (0))
+  lookup: sum -10m unaligned absolute of received
+    calc: (($inbound_packets_dropped != nan AND $this > 0) ? ($inbound_packets_dropped * 100 / $this) : (0))
    units: %
    every: 1m
     warn: $this > 0.5
     crit: $this > 3
-   delay: down 30m multiplier 1.5 max 1h
-    info: the ratio of inbound dropped packets vs the total number of received packets of the network interface, during the last hour
+   delay: down 1h multiplier 1.5 max 2h
+    info: the ratio of inbound dropped packets vs the total number of received packets of the network interface, during the last 10 minutes
       to: sysadmin
 
-template: 1hour_packet_drops_ratio_outbound
+template: outbound_packets_dropped_ratio
       on: net.packets
 families: *
-  lookup: sum -1h unaligned absolute of sent
-    calc: (($1hour_packet_drops_outbound != nan AND $this > 0) ? ($1hour_packet_drops_outbound * 100 / $this) : (0))
+  lookup: sum -10m unaligned absolute of sent
+    calc: (($outbound_packets_dropped != nan AND $this > 0) ? ($outbound_packets_dropped * 100 / $this) : (0))
    units: %
    every: 1m
     warn: $this > 0.5
     crit: $this > 3
-   delay: down 30m multiplier 1.5 max 1h
-    info: the ratio of outbound dropped packets vs the total number of sent packets of the network interface, during the last hour
+   delay: down 1h multiplier 1.5 max 2h
+    info: the ratio of outbound dropped packets vs the total number of sent packets of the network interface, during the last 10 minutes
       to: sysadmin
 
 
@@ -76,17 +76,17 @@ families: *
 # check if an interface is having FIFO
 # buffer errors
 # the alarm is checked every 1 minute
-# and examines the last hour of data
+# and examines the last 10 minutes of data
 
-template: 1hour_fifo_errors
+template: 10min_fifo_errors
       on: net.fifo
 families: *
-  lookup: sum -1h unaligned absolute
+  lookup: sum -10m unaligned absolute
    units: errors
    every: 1m
     warn: $this > 0
-   delay: down 30m multiplier 1.5 max 1h
-    info: interface fifo errors in the last hour
+   delay: down 1h multiplier 1.5 max 2h
+    info: interface fifo errors in the last 10 minutes
       to: sysadmin
 
 

diff --git a/conf.d/health.d/softnet.conf b/conf.d/health.d/softnet.conf
index 420a45448a2e766711baecf3d2b347d77713be12..5faf9a9ee5f0b1f504132a5fd42b79cb91fcb3e3 100644 (file)
--- a/conf.d/health.d/softnet.conf
+++ b/conf.d/health.d/softnet.conf
@@ -1,21 +1,21 @@
 # check for common /proc/net/softnet_stat errors
 
-   alarm: 1hour_netdev_backlog_exceeded
+   alarm: 10min_netdev_backlog_exceeded
       on: system.softnet_stat
-  lookup: sum -1h unaligned absolute of dropped
+  lookup: sum -10m unaligned absolute of dropped
    units: packets
    every: 1m
     warn: $this > 0
-   delay: down 30m multiplier 1.5 max 1h
-    info: number of packets dropped because sysctl net.core.netdev_max_backlog was exceeded (this can be a cause for dropped packets)
+   delay: down 1h multiplier 1.5 max 2h
+    info: number of packets dropped in the last 10min, because sysctl net.core.netdev_max_backlog was exceeded (this can be a cause for dropped packets)
       to: sysadmin
 
-   alarm: 1hour_netdev_budget_ran_outs
+   alarm: 10min_netdev_budget_ran_outs
       on: system.softnet_stat
-  lookup: sum -1h unaligned absolute of squeezed
+  lookup: sum -10m unaligned absolute of squeezed
    units: events
    every: 1m
     warn: $this > (($status >= $WARNING)  ? (0) : (10))
-   delay: down 30m multiplier 1.5 max 1h
-    info: number of times ksoftirq ran out of sysctl net.core.netdev_budget or time slice, with work remaining (this can be a cause for dropped packets)
+   delay: down 1h multiplier 1.5 max 2h
+    info: number of times, during the last 10min, ksoftirq ran out of sysctl net.core.netdev_budget or time slice, with work remaining (this can be a cause for dropped packets)
       to: silent