From: Costa Tsaousis Date: Sat, 25 Mar 2017 22:17:55 +0000 (+0200) Subject: Merge pull request #2020 from ktsaou/master X-Git-Tag: ab-debian_0.20170327.01-0ab1~1^2~2 X-Git-Url: https://arthur.barton.de/gitweb/?a=commitdiff_plain;h=1994f41b8a19e7c843b3d2b5a59dcd0c6aaa7e5b;hp=5ba248ecd55038de0744919f47c2b76c65f166ee;p=netdata.git Merge pull request #2020 from ktsaou/master ZFS plugin for Linux --- diff --git a/conf.d/Makefile.am b/conf.d/Makefile.am index db263e85..f5342992 100644 --- a/conf.d/Makefile.am +++ b/conf.d/Makefile.am @@ -28,6 +28,7 @@ dist_pythonconfig_DATA = \ python.d/apache_cache.conf \ python.d/bind_rndc.conf \ python.d/cpufreq.conf \ + python.d/dns_query_time.conf \ python.d/dovecot.conf \ python.d/elasticsearch.conf \ python.d/example.conf \ diff --git a/conf.d/python.d.conf b/conf.d/python.d.conf index 9ed346cd..22a18efa 100644 --- a/conf.d/python.d.conf +++ b/conf.d/python.d.conf @@ -31,6 +31,7 @@ log_interval: 3600 # bind_rndc: yes # cpufreq: yes # cpuidle: yes +# dns_query_time: yes # dovecot: yes # elasticsearch: yes @@ -52,6 +53,7 @@ gunicorn_log: no # memcached: yes # mysql: yes # nginx: yes +# nsd: yes # nginx_log has been replaced by web_log nginx_log: no diff --git a/conf.d/python.d/dns_query_time.conf b/conf.d/python.d/dns_query_time.conf new file mode 100644 index 00000000..f4d4dbf9 --- /dev/null +++ b/conf.d/python.d/dns_query_time.conf @@ -0,0 +1,72 @@ +# netdata python.d.plugin configuration for dns_query_time +# +# This file is in YaML format. Generally the format is: +# +# name: value +# +# There are 2 sections: +# - global variables +# - one or more JOBS +# +# JOBS allow you to collect values from multiple sources. +# Each source will have its own set of charts. +# +# JOB parameters have to be indented (using spaces only, example below). + +# ---------------------------------------------------------------------- +# Global Variables +# These variables set the defaults for all JOBs, however each JOB +# may define its own, overriding the defaults. + +# update_every sets the default data collection frequency. +# If unset, the python.d.plugin default is used. +# update_every: 1 + +# priority controls the order of charts at the netdata dashboard. +# Lower numbers move the charts towards the top of the page. +# If unset, the default for python.d.plugin is used. +# priority: 60000 + +# retries sets the number of retries to be made in case of failures. +# If unset, the default for python.d.plugin is used. +# Attempts to restore the service are made once every update_every +# and only if the module has collected values in the past. +# retries: 5 + +# ---------------------------------------------------------------------- +# JOBS (data collection sources) +# +# The default JOBS share the same *name*. JOBS with the same name +# are mutually exclusive. Only one of them will be allowed running at +# any time. This allows autodetection to try several alternatives and +# pick the one that works. +# +# Any number of jobs is supported. +# +# All python.d.plugin JOBS (for all its modules) support a set of +# predefined parameters. These are: +# +# job_name: +# name: myname # the JOB's name as it will appear at the +# # dashboard (by default is the job_name) +# # JOBs sharing a name are mutually exclusive +# update_every: 1 # the JOB's data collection frequency +# priority: 60000 # the JOB's order on the dashboard +# retries: 5 # the JOB's number of restoration attempts +# +# Additionally to the above, dns_query_time also supports the following: +# +# dns_servers: 'dns servers' # list of dns servers to query +# domains: 'domains' # list of domains +# aggregate: yes/no # Default: yes. Aggregate all servers in one chart or not +# response_timeout: 4 # Defalt: 4. Dns query response timeout (query = -100 if response time > response_time) +# +# +# ---------------------------------------------------------------------- +# AUTO-DETECTION JOBS +# only one of them will run (they have the same name) +# +# +#aggregate: yes +#dns_servers: '8.8.8.8 8.8.4.4' +#domains: 'python.org distrowatch.com linuxmint.com linux.com rsyslog.com liblognorm.com archlinux.org cisco.com debian.org kernel.org gns3.com opera.com github.com youtube.com amazon.co.uk kde.org netdata.firehol.org ubuntu.com redhat.com opensuse.org wireshark.org vmware.com microsoft.com elastic.co' diff --git a/python.d/Makefile.am b/python.d/Makefile.am index bfe28ff2..89ac2b78 100644 --- a/python.d/Makefile.am +++ b/python.d/Makefile.am @@ -18,6 +18,7 @@ dist_python_DATA = \ bind_rndc.chart.py \ cpufreq.chart.py \ cpuidle.chart.py \ + dns_query_time.chart.py \ dovecot.chart.py \ elasticsearch.chart.py \ example.chart.py \ diff --git a/python.d/README.md b/python.d/README.md index 7df6e3e8..e0e5893e 100644 --- a/python.d/README.md +++ b/python.d/README.md @@ -227,6 +227,16 @@ Your kernel needs to have `CONFIG_CPU_IDLE` enabled. It produces one stacked chart per CPU, showing the percentage of time spent in each state. +--- +# dns_query_time + +This module provides dns query time statistics. + +**Requirement:** +* `python-dnspython` package + +It produces one aggregate chart or one chart per dns server, showing the query time. + --- # dovecot diff --git a/python.d/dns_query_time.chart.py b/python.d/dns_query_time.chart.py new file mode 100644 index 00000000..527e5b8e --- /dev/null +++ b/python.d/dns_query_time.chart.py @@ -0,0 +1,129 @@ +try: + from time import monotonic as time +except ImportError: + from time import time +try: + import dns.message, dns.query, dns.name + DNS_PYTHON = True +except ImportError: + DNS_PYTHON = False +try: + from queue import Queue +except ImportError: + from Queue import Queue +from random import choice +from threading import Thread +from socket import gethostbyname, gaierror +from base import SimpleService + + +# default module values (can be overridden per job in `config`) +update_every = 5 +priority = 60000 +retries = 60 + + +class Service(SimpleService): + def __init__(self, configuration=None, name=None): + SimpleService.__init__(self, configuration=configuration, name=name) + self.order = list() + self.definitions = dict() + self.timeout = self.configuration.get('response_timeout', 4) + self.aggregate = self.configuration.get('aggregate', True) + self.domains = self.configuration.get('domains') + self.server_list = self.configuration.get('dns_servers') + + def check(self): + if not DNS_PYTHON: + self.error('\'python-dnspython\' package is needed to use dns_query_time.chart.py') + return False + + self.timeout = self.timeout if isinstance(self.timeout, int) else 4 + self.update_every = self.timeout + 1 if self.update_every <= self.timeout else self.update_every + + if not all([self.domains, self.server_list, + isinstance(self.server_list, str), isinstance(self.domains, str)]): + self.error('server_list and domain_list can\'t be empty') + return False + else: + self.domains, self.server_list = self.domains.split(), self.server_list.split() + + for ns in self.server_list: + if not check_ns(ns): + self.info('Bad NS: %s' % ns) + self.server_list.remove(ns) + if not self.server_list: + return False + + data = self._get_data(timeout=1) + + down_servers = [s[2:] for s in data if data[s] == -100] + if down_servers: + self.info('Removed due to non response %s' % down_servers) + self.server_list = [s for s in self.server_list if s not in down_servers] + if self.server_list: + self._data_from_check = data + self.order, self.definitions = create_charts(aggregate=self.aggregate, server_list=self.server_list) + self.info(str({'domains': len(self.domains), 'servers': self.server_list})) + return True + else: + return False + + def _get_data(self, timeout=None): + return dns_request(self.server_list, timeout or self.timeout, self.domains) + + +def dns_request(server_list, timeout, domains): + threads = list() + que = Queue() + result = dict() + + def dns_req(ns, t, q): + domain = dns.name.from_text(choice(domains)) + request = dns.message.make_query(domain, dns.rdatatype.A) + + try: + dns_start = time() + dns.query.udp(request, ns, timeout=t) + dns_end = time() + query_time = round((dns_end - dns_start) * 1000) + q.put({''.join(['ns', ns]): query_time}) + except dns.exception.Timeout: + q.put({''.join(['ns', ns]): -100}) + + for server in server_list: + th = Thread(target=dns_req, args=(server, timeout, que)) + th.start() + threads.append(th) + + for th in threads: + th.join() + result.update(que.get()) + + return result + + +def check_ns(ns): + try: + return gethostbyname(ns) + except gaierror: + return False + + +def create_charts(aggregate, server_list): + if aggregate: + order = ['dns_group'] + definitions = {'dns_group': {'options': [None, "DNS Response Time", "ms", 'name servers', + 'resp.time', 'line'], 'lines': []}} + for ns in server_list: + definitions['dns_group']['lines'].append([''.join(['ns', ns]), ns, 'absolute']) + + return order, definitions + else: + order = [''.join(['dns_', ns]) for ns in server_list] + definitions = dict() + for ns in server_list: + definitions[''.join(['dns_', ns])] = {'options': [None, "DNS Response Time", "ms", ns, + 'resp.time', 'area'], + 'lines': [[''.join(['ns', ns]), ns, 'absolute']]} + return order, definitions diff --git a/python.d/fail2ban.chart.py b/python.d/fail2ban.chart.py index c7d24e8c..ce95ff94 100644 --- a/python.d/fail2ban.chart.py +++ b/python.d/fail2ban.chart.py @@ -3,20 +3,17 @@ # Author: l2isbad from base import LogService -from re import compile - -try: - from itertools import filterfalse -except ImportError: - from itertools import ifilterfalse as filterfalse +from re import compile as r_compile from os import access as is_accessible, R_OK from os.path import isdir from glob import glob +import bisect priority = 60000 retries = 60 -REGEX = compile(r'\[([A-Za-z-_]+)][^\[\]]*?(?[a-z]+)\] (?P[A-Z])[a-z]+ (?P\d{1,3}(?:\.\d{1,3}){3})') +ORDER = ['jails_bans', 'jails_in_jail'] class Service(LogService): @@ -26,35 +23,39 @@ class Service(LogService): self.log_path = self.configuration.get('log_path', '/var/log/fail2ban.log') self.conf_path = self.configuration.get('conf_path', '/etc/fail2ban/jail.local') self.conf_dir = self.configuration.get('conf_dir', '') + self.bans = dict() try: self.exclude = self.configuration['exclude'].split() except (KeyError, AttributeError): - self.exclude = [] + self.exclude = list() def _get_data(self): """ Parse new log lines :return: dict """ - try: - raw = self._get_raw_data() - if raw is None: - return None - elif not raw: - return self.data - except (ValueError, AttributeError): + raw = self._get_raw_data() + if raw is None: return None + elif not raw: + return self.data # Fail2ban logs looks like # 2016-12-25 12:36:04,711 fail2ban.actions[2455]: WARNING [ssh] Ban 178.156.32.231 - data = dict( - zip( - self.jails_list, - [len(list(filterfalse(lambda line: (jail + '] Ban') not in line, raw))) for jail in self.jails_list] - )) - - for jail in data: - self.data[jail] += data[jail] + for row in raw: + match = REGEX_DATA.search(row) + if match: + match_dict = match.groupdict() + jail, ban, ipaddr = match_dict['jail'], match_dict['ban'], match_dict['ipaddr'] + if jail in self.jails_list: + if ban == 'B': + self.data[jail] += 1 + if address_not_in_jail(self.bans[jail], ipaddr, self.data[jail + '_in_jail']): + self.data[jail + '_in_jail'] += 1 + else: + if ipaddr in self.bans[jail]: + self.bans[jail].remove(ipaddr) + self.data[jail + '_in_jail'] -= 1 return self.data @@ -81,17 +82,25 @@ class Service(LogService): # If for some reason parse failed we still can START with default jails_list. self.jails_list = list(set(jails_list) - set(self.exclude)) or ['ssh'] + self.data = dict([(jail, 0) for jail in self.jails_list]) + self.data.update(dict([(jail + '_in_jail', 0) for jail in self.jails_list])) + self.bans = dict([(jail, list()) for jail in self.jails_list]) + self.create_dimensions() self.info('Plugin successfully started. Jails: %s' % self.jails_list) return True def create_dimensions(self): self.definitions = { - 'jails_group': {'options': [None, "Jails ban statistics", "bans/s", 'jails', 'jail.ban', 'line'], - 'lines': []}} + 'jails_bans': {'options': [None, "Jails Ban Statistics", "bans/s", 'bans', 'jail.bans', 'line'], + 'lines': []}, + 'jails_in_jail': {'options': [None, "Currently In Jail", "ip addresses", 'in jail', 'jail.in_jail', 'line'], + 'lines': []}, + } for jail in self.jails_list: - self.definitions['jails_group']['lines'].append([jail, jail, 'incremental']) + self.definitions['jails_bans']['lines'].append([jail, jail, 'incremental']) + self.definitions['jails_in_jail']['lines'].append([jail + '_in_jail', jail, 'absolute']) def parse_conf_dir(conf_dir): @@ -114,7 +123,7 @@ def parse_conf_dir(conf_dir): raw_data = f.read() data = ' '.join(raw_data.split()) - jails_list.extend(REGEX.findall(data)) + jails_list.extend(REGEX_JAILS.findall(data)) jails_list = list(set(jails_list)) return jails_list, 'can\'t locate any jails in %s. Default jail is [\'ssh\']' % conf_dir @@ -128,5 +137,18 @@ def parse_conf_path(conf_path): raw_data = jails_conf.read() data = raw_data.split() - jails_list = REGEX.findall(' '.join(data)) + jails_list = REGEX_JAILS.findall(' '.join(data)) return jails_list, 'can\'t locate any jails in %s. Default jail is [\'ssh\']' % conf_path + + +def address_not_in_jail(pool, address, pool_size): + index = bisect.bisect_left(pool, address) + if index < pool_size: + if pool[index] == address: + return False + else: + bisect.insort_left(pool, address) + return True + else: + bisect.insort_left(pool, address) + return True