+# -----------------------------------------------------------------------------
+# make sure we collect values for each disk
+
+# for mount points
+template: disk_space_last_collected_secs
+ on: disk.space
+families: *
+ calc: $now - $last_collected_t
+ units: seconds ago
+ every: 10s
+ warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
+ crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
+ delay: down 5m multiplier 1.5 max 1h
+ info: number of seconds since the last successful data collection of the mount point
+ to: sysadmin
+
+# for block devices
+template: disk_last_collected_secs
+ on: disk.io
+families: *
+ calc: $now - $last_collected_t
+ units: seconds ago
+ every: 10s
+ warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
+ crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
+ delay: down 5m multiplier 1.5 max 1h
+ info: number of seconds since the last successful data collection of the block device
+ to: sysadmin
+
+
# -----------------------------------------------------------------------------
# low disk space
# raise an alarm if the disk is low on
# available disk space
-template: low_disk_space
+template: disk_space_usage
on: disk.space
+families: *
+ calc: $used * 100 / ($avail + $used)
+ units: %
every: 1m
- warn: $avail * 100 / ($avail + $used) > 80
- crit: $avail * 100 / ($avail + $used) > 90
+ warn: $this > (($status >= $WARNING ) ? (80) : (90))
+ crit: $this > (($status == $CRITICAL) ? (90) : (98))
+ delay: up 1m down 15m multiplier 1.5 max 1h
+ info: current disk space usage
+ to: sysadmin
+
+template: disk_inode_usage
+ on: disk.inodes
+families: *
+ calc: $used * 100 / ($avail + $used)
+ units: %
+ every: 1m
+ warn: $this > (($status >= $WARNING) ? (80) : (90))
+ crit: $this > (($status == $CRITICAL) ? (90) : (98))
+ delay: up 1m down 15m multiplier 1.5 max 1h
+ info: current disk inode usage
+ to: sysadmin
+
# -----------------------------------------------------------------------------
# disk fill rate
# calculate the rate the disk fills
# use as base, the available space change
-# during the last minute
+# during the last hour
+
+# this is just a calculation - it has no alarm
+# we will use it in the next template to find
+# the hours remaining
-template: disk_fill_rate_1m
+template: disk_fill_rate
on: disk.space
- lookup: max -1s at -1m unaligned of avail
- calc: ($this - $avail) / (1 * 60)
+families: *
+ lookup: min -10m at -50m unaligned of avail
+ calc: ($this - $avail) / (($now - $after) / 3600)
every: 1m
- warn: $this * 2 * 86400 > $avail
- crit: $this * 1 * 86400 > $avail
+ units: GB/hour
+ info: average rate the disk fills up (positive), or frees up (negative) space, for the last hour
-# calculate the rate the disk fills
-# use as base, the available space change
-# during the last hour
-template: disk_fill_rate_59m
+
+# calculate the hours remaining
+# if the disk continues to fill
+# in this rate
+
+template: out_of_disk_space_time
on: disk.space
- lookup: max -1s at -59m unaligned of avail
- every: 1m
- calc: ($this - $avail) / (59 * 60)
- warn: $this * 2 * 86400 > $avail
- crit: $this * 1 * 86400 > $avail
+families: *
+ calc: ($disk_fill_rate > 0) ? ($avail / $disk_fill_rate) : (inf)
+ units: hours
+ every: 10s
+ warn: $this > 0 and $this < (($status >= $WARNING) ? (48) : (8))
+ crit: $this > 0 and $this < (($status == $CRITICAL) ? (24) : (2))
+ delay: down 15m multiplier 1.2 max 1h
+ info: estimated time the disk will run out of space, if the system continues to add data with the rate of the last hour
+ to: sysadmin
+
# -----------------------------------------------------------------------------
# disk congestion
# raise an alarm if the disk is congested
# by calculating the average disk utilization
-# for the last minute
+# for the last 10 minutes
-template: disk_congested
+template: 10min_disk_utilization
on: disk.util
- lookup: average -1m every 1m unaligned
- green: 70
- red: 95
- warn: $this > $green
- crit: $this > $red
+families: *
+ lookup: average -10m unaligned
+ units: %
+ every: 1m
+ green: 90
+ red: 98
+ warn: $this > $green * (($status >= $WARNING) ? (0.7) : (1))
+ crit: $this > $red * (($status == $CRITICAL) ? (0.7) : (1))
+ delay: down 15m multiplier 1.2 max 1h
+ info: the percentage of time the disk was busy, during the last 10 minutes
+ to: sysadmin
+
# raise an alarm if the disk backlog
# is above 1000ms (1s) per second
-# for 1 minute
+# for 10 minutes
# (i.e. the disk cannot catch up)
-template: disk_not_catching_up
+template: 10min_disk_backlog
on: disk.backlog
- lookup: average -1m every 1m unaligned
- green: 500
- red: 1000
- warn: $this > $green
- crit: $this > $red
-
+families: *
+ lookup: average -10m unaligned
+ units: ms
+ every: 1m
+ green: 2000
+ red: 5000
+ warn: $this > $green * (($status >= $WARNING) ? (0.7) : (1))
+ crit: $this > $red * (($status == $CRITICAL) ? (0.7) : (1))
+ delay: down 15m multiplier 1.2 max 1h
+ info: average of the kernel estimated disk backlog, for the last 10 minutes
+ to: sysadmin