python.d/apache_cache.conf \
python.d/bind_rndc.conf \
python.d/cpufreq.conf \
+ python.d/dns_query_time.conf \
python.d/dovecot.conf \
python.d/elasticsearch.conf \
python.d/example.conf \
# bind_rndc: yes
# cpufreq: yes
# cpuidle: yes
+# dns_query_time: yes
# dovecot: yes
# elasticsearch: yes
# memcached: yes
# mysql: yes
# nginx: yes
+# nsd: yes
# nginx_log has been replaced by web_log
nginx_log: no
--- /dev/null
+# netdata python.d.plugin configuration for dns_query_time
+#
+# This file is in YaML format. Generally the format is:
+#
+# name: value
+#
+# There are 2 sections:
+# - global variables
+# - one or more JOBS
+#
+# JOBS allow you to collect values from multiple sources.
+# Each source will have its own set of charts.
+#
+# JOB parameters have to be indented (using spaces only, example below).
+
+# ----------------------------------------------------------------------
+# Global Variables
+# These variables set the defaults for all JOBs, however each JOB
+# may define its own, overriding the defaults.
+
+# update_every sets the default data collection frequency.
+# If unset, the python.d.plugin default is used.
+# update_every: 1
+
+# priority controls the order of charts at the netdata dashboard.
+# Lower numbers move the charts towards the top of the page.
+# If unset, the default for python.d.plugin is used.
+# priority: 60000
+
+# retries sets the number of retries to be made in case of failures.
+# If unset, the default for python.d.plugin is used.
+# Attempts to restore the service are made once every update_every
+# and only if the module has collected values in the past.
+# retries: 5
+
+# ----------------------------------------------------------------------
+# JOBS (data collection sources)
+#
+# The default JOBS share the same *name*. JOBS with the same name
+# are mutually exclusive. Only one of them will be allowed running at
+# any time. This allows autodetection to try several alternatives and
+# pick the one that works.
+#
+# Any number of jobs is supported.
+#
+# All python.d.plugin JOBS (for all its modules) support a set of
+# predefined parameters. These are:
+#
+# job_name:
+# name: myname # the JOB's name as it will appear at the
+# # dashboard (by default is the job_name)
+# # JOBs sharing a name are mutually exclusive
+# update_every: 1 # the JOB's data collection frequency
+# priority: 60000 # the JOB's order on the dashboard
+# retries: 5 # the JOB's number of restoration attempts
+#
+# Additionally to the above, dns_query_time also supports the following:
+#
+# dns_servers: 'dns servers' # list of dns servers to query
+# domains: 'domains' # list of domains
+# aggregate: yes/no # Default: yes. Aggregate all servers in one chart or not
+# response_timeout: 4 # Defalt: 4. Dns query response timeout (query = -100 if response time > response_time)
+#
+#
+# ----------------------------------------------------------------------
+# AUTO-DETECTION JOBS
+# only one of them will run (they have the same name)
+#
+#
+#aggregate: yes
+#dns_servers: '8.8.8.8 8.8.4.4'
+#domains: 'python.org distrowatch.com linuxmint.com linux.com rsyslog.com liblognorm.com archlinux.org cisco.com debian.org kernel.org gns3.com opera.com github.com youtube.com amazon.co.uk kde.org netdata.firehol.org ubuntu.com redhat.com opensuse.org wireshark.org vmware.com microsoft.com elastic.co'
bind_rndc.chart.py \
cpufreq.chart.py \
cpuidle.chart.py \
+ dns_query_time.chart.py \
dovecot.chart.py \
elasticsearch.chart.py \
example.chart.py \
It produces one stacked chart per CPU, showing the percentage of time spent in
each state.
+---
+# dns_query_time
+
+This module provides dns query time statistics.
+
+**Requirement:**
+* `python-dnspython` package
+
+It produces one aggregate chart or one chart per dns server, showing the query time.
+
---
# dovecot
--- /dev/null
+try:
+ from time import monotonic as time
+except ImportError:
+ from time import time
+try:
+ import dns.message, dns.query, dns.name
+ DNS_PYTHON = True
+except ImportError:
+ DNS_PYTHON = False
+try:
+ from queue import Queue
+except ImportError:
+ from Queue import Queue
+from random import choice
+from threading import Thread
+from socket import gethostbyname, gaierror
+from base import SimpleService
+
+
+# default module values (can be overridden per job in `config`)
+update_every = 5
+priority = 60000
+retries = 60
+
+
+class Service(SimpleService):
+ def __init__(self, configuration=None, name=None):
+ SimpleService.__init__(self, configuration=configuration, name=name)
+ self.order = list()
+ self.definitions = dict()
+ self.timeout = self.configuration.get('response_timeout', 4)
+ self.aggregate = self.configuration.get('aggregate', True)
+ self.domains = self.configuration.get('domains')
+ self.server_list = self.configuration.get('dns_servers')
+
+ def check(self):
+ if not DNS_PYTHON:
+ self.error('\'python-dnspython\' package is needed to use dns_query_time.chart.py')
+ return False
+
+ self.timeout = self.timeout if isinstance(self.timeout, int) else 4
+ self.update_every = self.timeout + 1 if self.update_every <= self.timeout else self.update_every
+
+ if not all([self.domains, self.server_list,
+ isinstance(self.server_list, str), isinstance(self.domains, str)]):
+ self.error('server_list and domain_list can\'t be empty')
+ return False
+ else:
+ self.domains, self.server_list = self.domains.split(), self.server_list.split()
+
+ for ns in self.server_list:
+ if not check_ns(ns):
+ self.info('Bad NS: %s' % ns)
+ self.server_list.remove(ns)
+ if not self.server_list:
+ return False
+
+ data = self._get_data(timeout=1)
+
+ down_servers = [s[2:] for s in data if data[s] == -100]
+ if down_servers:
+ self.info('Removed due to non response %s' % down_servers)
+ self.server_list = [s for s in self.server_list if s not in down_servers]
+ if self.server_list:
+ self._data_from_check = data
+ self.order, self.definitions = create_charts(aggregate=self.aggregate, server_list=self.server_list)
+ self.info(str({'domains': len(self.domains), 'servers': self.server_list}))
+ return True
+ else:
+ return False
+
+ def _get_data(self, timeout=None):
+ return dns_request(self.server_list, timeout or self.timeout, self.domains)
+
+
+def dns_request(server_list, timeout, domains):
+ threads = list()
+ que = Queue()
+ result = dict()
+
+ def dns_req(ns, t, q):
+ domain = dns.name.from_text(choice(domains))
+ request = dns.message.make_query(domain, dns.rdatatype.A)
+
+ try:
+ dns_start = time()
+ dns.query.udp(request, ns, timeout=t)
+ dns_end = time()
+ query_time = round((dns_end - dns_start) * 1000)
+ q.put({''.join(['ns', ns]): query_time})
+ except dns.exception.Timeout:
+ q.put({''.join(['ns', ns]): -100})
+
+ for server in server_list:
+ th = Thread(target=dns_req, args=(server, timeout, que))
+ th.start()
+ threads.append(th)
+
+ for th in threads:
+ th.join()
+ result.update(que.get())
+
+ return result
+
+
+def check_ns(ns):
+ try:
+ return gethostbyname(ns)
+ except gaierror:
+ return False
+
+
+def create_charts(aggregate, server_list):
+ if aggregate:
+ order = ['dns_group']
+ definitions = {'dns_group': {'options': [None, "DNS Response Time", "ms", 'name servers',
+ 'resp.time', 'line'], 'lines': []}}
+ for ns in server_list:
+ definitions['dns_group']['lines'].append([''.join(['ns', ns]), ns, 'absolute'])
+
+ return order, definitions
+ else:
+ order = [''.join(['dns_', ns]) for ns in server_list]
+ definitions = dict()
+ for ns in server_list:
+ definitions[''.join(['dns_', ns])] = {'options': [None, "DNS Response Time", "ms", ns,
+ 'resp.time', 'area'],
+ 'lines': [[''.join(['ns', ns]), ns, 'absolute']]}
+ return order, definitions
# Author: l2isbad
from base import LogService
-from re import compile
-
-try:
- from itertools import filterfalse
-except ImportError:
- from itertools import ifilterfalse as filterfalse
+from re import compile as r_compile
from os import access as is_accessible, R_OK
from os.path import isdir
from glob import glob
+import bisect
priority = 60000
retries = 60
-REGEX = compile(r'\[([A-Za-z-_]+)][^\[\]]*?(?<!# )enabled = true')
-ORDER = ['jails_group']
+REGEX_JAILS = r_compile(r'\[([A-Za-z-_]+)][^\[\]]*?(?<!# )enabled = true')
+REGEX_DATA = r_compile(r'\[(?P<jail>[a-z]+)\] (?P<ban>[A-Z])[a-z]+ (?P<ipaddr>\d{1,3}(?:\.\d{1,3}){3})')
+ORDER = ['jails_bans', 'jails_in_jail']
class Service(LogService):
self.log_path = self.configuration.get('log_path', '/var/log/fail2ban.log')
self.conf_path = self.configuration.get('conf_path', '/etc/fail2ban/jail.local')
self.conf_dir = self.configuration.get('conf_dir', '')
+ self.bans = dict()
try:
self.exclude = self.configuration['exclude'].split()
except (KeyError, AttributeError):
- self.exclude = []
+ self.exclude = list()
def _get_data(self):
"""
Parse new log lines
:return: dict
"""
- try:
- raw = self._get_raw_data()
- if raw is None:
- return None
- elif not raw:
- return self.data
- except (ValueError, AttributeError):
+ raw = self._get_raw_data()
+ if raw is None:
return None
+ elif not raw:
+ return self.data
# Fail2ban logs looks like
# 2016-12-25 12:36:04,711 fail2ban.actions[2455]: WARNING [ssh] Ban 178.156.32.231
- data = dict(
- zip(
- self.jails_list,
- [len(list(filterfalse(lambda line: (jail + '] Ban') not in line, raw))) for jail in self.jails_list]
- ))
-
- for jail in data:
- self.data[jail] += data[jail]
+ for row in raw:
+ match = REGEX_DATA.search(row)
+ if match:
+ match_dict = match.groupdict()
+ jail, ban, ipaddr = match_dict['jail'], match_dict['ban'], match_dict['ipaddr']
+ if jail in self.jails_list:
+ if ban == 'B':
+ self.data[jail] += 1
+ if address_not_in_jail(self.bans[jail], ipaddr, self.data[jail + '_in_jail']):
+ self.data[jail + '_in_jail'] += 1
+ else:
+ if ipaddr in self.bans[jail]:
+ self.bans[jail].remove(ipaddr)
+ self.data[jail + '_in_jail'] -= 1
return self.data
# If for some reason parse failed we still can START with default jails_list.
self.jails_list = list(set(jails_list) - set(self.exclude)) or ['ssh']
+
self.data = dict([(jail, 0) for jail in self.jails_list])
+ self.data.update(dict([(jail + '_in_jail', 0) for jail in self.jails_list]))
+ self.bans = dict([(jail, list()) for jail in self.jails_list])
+
self.create_dimensions()
self.info('Plugin successfully started. Jails: %s' % self.jails_list)
return True
def create_dimensions(self):
self.definitions = {
- 'jails_group': {'options': [None, "Jails ban statistics", "bans/s", 'jails', 'jail.ban', 'line'],
- 'lines': []}}
+ 'jails_bans': {'options': [None, "Jails Ban Statistics", "bans/s", 'bans', 'jail.bans', 'line'],
+ 'lines': []},
+ 'jails_in_jail': {'options': [None, "Currently In Jail", "ip addresses", 'in jail', 'jail.in_jail', 'line'],
+ 'lines': []},
+ }
for jail in self.jails_list:
- self.definitions['jails_group']['lines'].append([jail, jail, 'incremental'])
+ self.definitions['jails_bans']['lines'].append([jail, jail, 'incremental'])
+ self.definitions['jails_in_jail']['lines'].append([jail + '_in_jail', jail, 'absolute'])
def parse_conf_dir(conf_dir):
raw_data = f.read()
data = ' '.join(raw_data.split())
- jails_list.extend(REGEX.findall(data))
+ jails_list.extend(REGEX_JAILS.findall(data))
jails_list = list(set(jails_list))
return jails_list, 'can\'t locate any jails in %s. Default jail is [\'ssh\']' % conf_dir
raw_data = jails_conf.read()
data = raw_data.split()
- jails_list = REGEX.findall(' '.join(data))
+ jails_list = REGEX_JAILS.findall(' '.join(data))
return jails_list, 'can\'t locate any jails in %s. Default jail is [\'ssh\']' % conf_path
+
+
+def address_not_in_jail(pool, address, pool_size):
+ index = bisect.bisect_left(pool, address)
+ if index < pool_size:
+ if pool[index] == address:
+ return False
+ else:
+ bisect.insort_left(pool, address)
+ return True
+ else:
+ bisect.insort_left(pool, address)
+ return True
projects / netdata.git / commitdiff