X-Git-Url: https://arthur.barton.de/gitweb/?a=blobdiff_plain;f=python.d%2Fpython_modules%2Fbase.py;h=0482b9b8059de1c344142118e4f24e861184c091;hb=4bdf8543f617f8df20f5c2be289accc03bbb631a;hp=69af6285c6d54ba12d9d950d6858381ac6fc3184;hpb=1fb6da8a7c81509d36cd399bb0cb2d453ad233b6;p=netdata.git diff --git a/python.d/python_modules/base.py b/python.d/python_modules/base.py index 69af6285..0482b9b8 100644 --- a/python.d/python_modules/base.py +++ b/python.d/python_modules/base.py @@ -1,24 +1,55 @@ # -*- coding: utf-8 -*- -# Description: prototypes for netdata python.d modules +# Description: netdata python modules framework # Author: Pawel Krupa (paulfantom) +# Remember: +# ALL CODE NEEDS TO BE COMPATIBLE WITH Python > 2.7 and Python > 3.1 +# Follow PEP8 as much as it is possible +# "check" and "create" CANNOT be blocking. +# "update" CAN be blocking +# "update" function needs to be fast, so follow: +# https://wiki.python.org/moin/PythonSpeed/PerformanceTips +# basically: +# - use local variables wherever it is possible +# - avoid dots in expressions that are executed many times +# - use "join()" instead of "+" +# - use "import" only at the beginning +# +# using ".encode()" in one thread can block other threads as well (only in python2) + import time -import sys import os import socket import select +import threading +import msg +import ssl +from subprocess import Popen, PIPE +from sys import exc_info + try: import urllib.request as urllib2 except ImportError: import urllib2 -from subprocess import Popen, PIPE +try: + import MySQLdb + PYMYSQL = True +except ImportError: + try: + import pymysql as MySQLdb + PYMYSQL = True + except ImportError: + PYMYSQL = False -import threading -import msg +try: + PATH = os.getenv('PATH').split(':') +except AttributeError: + PATH = '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'.split(':') -class BaseService(threading.Thread): +# class BaseService(threading.Thread): +class SimpleService(threading.Thread): """ Prototype of Service class. Implemented basic functionality to run jobs by `python.d.plugin` @@ -42,6 +73,10 @@ class BaseService(threading.Thread): self._dimensions = [] self._charts = [] self.__chart_set = False + self.__first_run = True + self.order = [] + self.definitions = {} + self._data_from_check = dict() if configuration is None: self.error("BaseService: no configuration parameters supplied. Cannot create Service.") raise RuntimeError @@ -50,6 +85,8 @@ class BaseService(threading.Thread): self.timetable = {} self.create_timetable() + # --- BASIC SERVICE CONFIGURATION --- + def _extract_base_config(self, config): """ Get basic parameters to run service @@ -59,13 +96,14 @@ class BaseService(threading.Thread): 'retries':0} :param config: dict """ + pop = config.pop try: - self.override_name = config.pop('name') + self.override_name = pop('name') except KeyError: pass - self.update_every = int(config.pop('update_every')) - self.priority = int(config.pop('priority')) - self.retries = int(config.pop('retries')) + self.update_every = int(pop('update_every')) + self.priority = int(pop('priority')) + self.retries = int(pop('retries')) self.retries_left = self.retries self.configuration = config @@ -86,46 +124,33 @@ class BaseService(threading.Thread): 'next': now - (now % freq) + freq, 'freq': freq} + # --- THREAD CONFIGURATION --- + def _run_once(self): """ Executes self.update(interval) and draws run time chart. Return value presents exit status of update() :return: boolean """ - t_start = time.time() - # check if it is time to execute job update() function - if self.timetable['next'] > t_start: - #msg.debug(self.chart_name + " will be run in " + - # str(int((self.timetable['next'] - t_start) * 1000)) + " ms") - msg.debug(self.chart_name, "will be run in", str(int((self.timetable['next'] - t_start) * 1000)), "ms") - return True + t_start = float(time.time()) + chart_name = self.chart_name since_last = int((t_start - self.timetable['last']) * 1000000) - #msg.debug(self.chart_name + - # " ready to run, after " + str(int((t_start - self.timetable['last']) * 1000)) + - # " ms (update_every: " + str(self.timetable['freq'] * 1000) + - # " ms, latency: " + str(int((t_start - self.timetable['next']) * 1000)) + " ms)") - msg.debug(self.chart_name, - "ready to run, after", str(int((t_start - self.timetable['last']) * 1000)), - "ms (update_every:", str(self.timetable['freq'] * 1000), - "ms, latency:", str(int((t_start - self.timetable['next']) * 1000)), "ms") + if self.__first_run: + since_last = 0 + if not self.update(since_last): self.error("update function failed.") return False - t_end = time.time() - self.timetable['next'] = t_end - (t_end % self.timetable['freq']) + self.timetable['freq'] + # draw performance graph - run_time = str(int((t_end - t_start) * 1000)) - #run_time_chart = "BEGIN netdata.plugin_pythond_" + self.chart_name + " " + str(since_last) + '\n' - #run_time_chart += "SET run_time = " + run_time + '\n' - #run_time_chart += "END\n" - #sys.stdout.write(run_time_chart) - sys.stdout.write("BEGIN netdata.plugin_pythond_%s %s\nSET run_time = %s\nEND\n" % \ - (self.chart_name, str(since_last), run_time)) - - #msg.debug(self.chart_name + " updated in " + str(run_time) + " ms") - msg.debug(self.chart_name, "updated in", str(run_time), "ms") + run_time = int((time.time() - t_start) * 1000) + print("BEGIN netdata.plugin_pythond_%s %s\nSET run_time = %s\nEND\n" % + (self.chart_name, str(since_last), str(run_time))) + + self.debug(chart_name, "updated in", str(run_time), "ms") self.timetable['last'] = t_start + self.__first_run = False return True def run(self): @@ -134,26 +159,58 @@ class BaseService(threading.Thread): Exits when job failed or timed out. :return: None """ - self.timetable['last'] = time.time() - while True: + step = float(self.timetable['freq']) + penalty = 0 + self.timetable['last'] = float(time.time() - step) + self.debug("starting data collection - update frequency:", str(step), " retries allowed:", str(self.retries)) + while True: # run forever, unless something is wrong + now = float(time.time()) + next = self.timetable['next'] = now - (now % step) + step + penalty + + # it is important to do this in a loop + # sleep() is interruptable + while now < next: + self.debug("sleeping for", str(next - now), "secs to reach frequency of", str(step), "secs, now:", str(now), " next:", str(next), " penalty:", str(penalty)) + time.sleep(next - now) + now = float(time.time()) + + # do the job try: status = self._run_once() except Exception as e: - msg.error("Something wrong: ", str(e)) - return + status = False + if status: - time.sleep(self.timetable['next'] - time.time()) + # it is good self.retries_left = self.retries + penalty = 0 else: + # it failed self.retries_left -= 1 if self.retries_left <= 0: - msg.error("no more retries. Exiting") - return + if penalty == 0: + penalty = float(self.retries * step) / 2 + else: + penalty *= 1.5 + + if penalty > 600: + penalty = 600 + + self.retries_left = self.retries + self.alert("failed to collect data for " + str(self.retries) + " times - increasing penalty to " + str(penalty) + " sec and trying again") + else: - time.sleep(self.timetable['freq']) + self.error("failed to collect data - " + str(self.retries_left) + " retries left - penalty: " + str(penalty) + " sec") + + # --- CHART --- @staticmethod def _format(*args): + """ + Escape and convert passed arguments. + :param args: anything + :return: list + """ params = [] append = params.append for p in args: @@ -170,28 +227,14 @@ class BaseService(threading.Thread): def _line(self, instruction, *params): """ Converts *params to string and joins them with one space between every one. + Result is appended to self._data_stream :param params: str/int/float """ - #self._data_stream += instruction tmp = list(map((lambda x: "''" if x is None or len(x) == 0 else x), params)) - self._data_stream += "%s %s\n" % (instruction, str(" ".join(tmp))) - # self.error(str(" ".join(tmp))) - # for p in params: - # if p is None: - # p = "" - # else: - # p = str(p) - # if len(p) == 0: - # p = "''" - # if ' ' in p: - # p = "'" + p + "'" - # self._data_stream += " " + p - #self._data_stream += "\n" - def chart(self, type_id, name="", title="", units="", family="", - category="", charttype="line", priority="", update_every=""): + category="", chart_type="line", priority="", update_every=""): """ Defines a new chart. :param type_id: str @@ -200,14 +243,13 @@ class BaseService(threading.Thread): :param units: str :param family: str :param category: str - :param charttype: str + :param chart_type: str :param priority: int/str :param update_every: int/str """ self._charts.append(type_id) - #self._line("CHART", type_id, name, title, units, family, category, charttype, priority, update_every) - p = self._format(type_id, name, title, units, family, category, charttype, priority, update_every) + p = self._format(type_id, name, title, units, family, category, chart_type, priority, update_every) self._line("CHART", *p) def dimension(self, id, name=None, algorithm="absolute", multiplier=1, divisor=1, hidden=False): @@ -239,10 +281,8 @@ class BaseService(threading.Thread): self._dimensions.append(str(id)) if hidden: p = self._format(id, name, algorithm, multiplier, divisor, "hidden") - #self._line("DIMENSION", id, name, algorithm, str(multiplier), str(divisor), "hidden") else: p = self._format(id, name, algorithm, multiplier, divisor) - #self._line("DIMENSION", id, name, algorithm, str(multiplier), str(divisor)) self._line("DIMENSION", *p) @@ -278,7 +318,7 @@ class BaseService(threading.Thread): try: value = str(int(value)) except TypeError: - self.error("cannot set non-numeric value:", value) + self.error("cannot set non-numeric value:", str(value)) return False self._line("SET", id, "=", str(value)) self.__chart_set = True @@ -294,17 +334,28 @@ class BaseService(threading.Thread): def commit(self): """ - Upload new data to netdata + Upload new data to netdata. """ - print(self._data_stream) + try: + print(self._data_stream) + except Exception as e: + msg.fatal('cannot send data to netdata:', str(e)) self._data_stream = "" + # --- ERROR HANDLING --- + def error(self, *params): """ Show error message on stderr """ msg.error(self.chart_name, *params) + def alert(self, *params): + """ + Show error message on stderr + """ + msg.alert(self.chart_name, *params) + def debug(self, *params): """ Show debug message on stderr @@ -317,37 +368,7 @@ class BaseService(threading.Thread): """ msg.info(self.chart_name, *params) - def check(self): - """ - check() prototype - :return: boolean - """ - msg.error("Service " + str(self.__module__) + "doesn't implement check() function") - return False - - def create(self): - """ - create() prototype - :return: boolean - """ - msg.error("Service " + str(self.__module__) + "doesn't implement create() function?") - return False - - def update(self, interval): - """ - update() prototype - :param interval: int - :return: boolean - """ - msg.error("Service " + str(self.__module__) + "doesn't implement update() function") - return False - - -class SimpleService(BaseService): - def __init__(self, configuration=None, name=None): - self.order = [] - self.definitions = {} - BaseService.__init__(self, configuration=configuration, name=name) + # --- MAIN METHODS --- def _get_data(self): """ @@ -358,8 +379,21 @@ class SimpleService(BaseService): def check(self): """ - :return: + check() prototype + :return: boolean """ + self.debug("Module", str(self.__module__), "doesn't implement check() function. Using default.") + data = self._get_data() + + if data is None: + self.debug("failed to receive data during check().") + return False + + if len(data) == 0: + self.debug("empty data during check().") + return False + + self.debug("successfully received data during check(): '" + str(data) + "'") return True def create(self): @@ -367,8 +401,9 @@ class SimpleService(BaseService): Create charts :return: boolean """ - data = self._get_data() + data = self._data_from_check or self._get_data() if data is None: + self.debug("failed to receive data during create().") return False idx = 0 @@ -392,7 +427,7 @@ class SimpleService(BaseService): """ data = self._get_data() if data is None: - self.debug("_get_data() returned no data") + self.debug("failed to receive data during update().") return False updated = False @@ -412,20 +447,72 @@ class SimpleService(BaseService): return updated + @staticmethod + def find_binary(binary): + try: + if isinstance(binary, str): + binary = os.path.basename(binary) + return next(('/'.join([p, binary]) for p in PATH + if os.path.isfile('/'.join([p, binary])) + and os.access('/'.join([p, binary]), os.X_OK))) + else: + return None + except StopIteration: + return None + class UrlService(SimpleService): + # TODO add support for https connections def __init__(self, configuration=None, name=None): self.url = "" self.user = None self.password = None + self.proxies = {} SimpleService.__init__(self, configuration=configuration, name=name) - def __add_auth(self): - passman = urllib2.HTTPPasswordMgrWithDefaultRealm() - passman.add_password(None, self.url, self.user, self.password) - authhandler = urllib2.HTTPBasicAuthHandler(passman) - opener = urllib2.build_opener(authhandler) - urllib2.install_opener(opener) + def __add_openers(self): + # TODO add error handling + if self.ss_cert: + try: + ctx = ssl.create_default_context() + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + self.opener = urllib2.build_opener(urllib2.HTTPSHandler(context=ctx)) + except Exception as error: + self.error(str(error)) + self.opener = urllib2.build_opener() + else: + self.opener = urllib2.build_opener() + + # Proxy handling + # TODO currently self.proxies isn't parsed from configuration file + # if len(self.proxies) > 0: + # for proxy in self.proxies: + # url = proxy['url'] + # # TODO test this: + # if "user" in proxy and "pass" in proxy: + # if url.lower().startswith('https://'): + # url = 'https://' + proxy['user'] + ':' + proxy['pass'] + '@' + url[8:] + # else: + # url = 'http://' + proxy['user'] + ':' + proxy['pass'] + '@' + url[7:] + # # FIXME move proxy auth to sth like this: + # # passman = urllib2.HTTPPasswordMgrWithDefaultRealm() + # # passman.add_password(None, url, proxy['user'], proxy['password']) + # # opener.add_handler(urllib2.HTTPBasicAuthHandler(passman)) + # + # if url.lower().startswith('https://'): + # opener.add_handler(urllib2.ProxyHandler({'https': url})) + # else: + # opener.add_handler(urllib2.ProxyHandler({'https': url})) + + # HTTP Basic Auth + if self.user is not None and self.password is not None: + passman = urllib2.HTTPPasswordMgrWithDefaultRealm() + passman.add_password(None, self.url, self.user, self.password) + self.opener.add_handler(urllib2.HTTPBasicAuthHandler(passman)) + self.debug("Enabling HTTP basic auth") + + #urllib2.install_opener(opener) def _get_raw_data(self): """ @@ -434,13 +521,14 @@ class UrlService(SimpleService): """ raw = None try: - f = urllib2.urlopen(self.url, timeout=self.update_every) + f = self.opener.open(self.url, timeout=self.update_every * 2) + # f = urllib2.urlopen(self.url, timeout=self.update_every * 2) except Exception as e: self.error(str(e)) return None try: - raw = f.read().decode('utf-8') + raw = f.read().decode('utf-8', 'ignore') except Exception as e: self.error(str(e)) finally: @@ -469,14 +557,14 @@ class UrlService(SimpleService): self.password = str(self.configuration['pass']) except (KeyError, TypeError): pass + self.ss_cert = self.configuration.get('ss_cert') + self.__add_openers() - if self.user is not None and self.password is not None: - self.__add_auth() - - if self._get_data() is not None: - return True - else: + test = self._get_data() + if test is None or len(test) == 0: return False + else: + return True class SocketService(SimpleService): @@ -488,8 +576,83 @@ class SocketService(SimpleService): self.unix_socket = None self.request = "" self.__socket_config = None + self.__empty_request = "".encode() SimpleService.__init__(self, configuration=configuration, name=name) + def _socketerror(self, message=None): + if self.unix_socket is not None: + self.error("unix socket '" + self.unix_socket + "':", message) + else: + if self.__socket_config is not None: + af, socktype, proto, canonname, sa = self.__socket_config + self.error("socket to '" + str(sa[0]) + "' port " + str(sa[1]) + ":", message) + else: + self.error("unknown socket:", message) + + def _connect2socket(self, res=None): + """ + Connect to a socket, passing the result of getaddrinfo() + :return: boolean + """ + if res is None: + res = self.__socket_config + if res is None: + self.error("Cannot create socket to 'None':") + return False + + af, socktype, proto, canonname, sa = res + try: + self.debug("creating socket to '" + str(sa[0]) + "', port " + str(sa[1])) + self._sock = socket.socket(af, socktype, proto) + except socket.error as e: + self.error("Failed to create socket to '" + str(sa[0]) + "', port " + str(sa[1]) + ":", str(e)) + self._sock = None + self.__socket_config = None + return False + + try: + self.debug("connecting socket to '" + str(sa[0]) + "', port " + str(sa[1])) + self._sock.connect(sa) + except socket.error as e: + self.error("Failed to connect to '" + str(sa[0]) + "', port " + str(sa[1]) + ":", str(e)) + self._disconnect() + self.__socket_config = None + return False + + self.debug("connected to '" + str(sa[0]) + "', port " + str(sa[1])) + self.__socket_config = res + return True + + def _connect2unixsocket(self): + """ + Connect to a unix socket, given its filename + :return: boolean + """ + if self.unix_socket is None: + self.error("cannot connect to unix socket 'None'") + return False + + try: + self.debug("attempting DGRAM unix socket '" + str(self.unix_socket) + "'") + self._sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) + self._sock.connect(self.unix_socket) + self.debug("connected DGRAM unix socket '" + str(self.unix_socket) + "'") + return True + except socket.error as e: + self.debug("Failed to connect DGRAM unix socket '" + str(self.unix_socket) + "':", str(e)) + + try: + self.debug("attempting STREAM unix socket '" + str(self.unix_socket) + "'") + self._sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self._sock.connect(self.unix_socket) + self.debug("connected STREAM unix socket '" + str(self.unix_socket) + "'") + return True + except socket.error as e: + self.debug("Failed to connect STREAM unix socket '" + str(self.unix_socket) + "':", str(e)) + self.error("Failed to connect to unix socket '" + str(self.unix_socket) + "':", str(e)) + self._sock = None + return False + def _connect(self): """ Recreate socket and connect to it since sockets cannot be reused after closing @@ -497,54 +660,38 @@ class SocketService(SimpleService): :return: """ try: - if self.unix_socket is None: - if self.__socket_config is None: - # establish ipv6 or ipv4 connection. - for res in socket.getaddrinfo(self.host, self.port, socket.AF_UNSPEC, socket.SOCK_STREAM): - try: - af, socktype, proto, canonname, sa = res - self._sock = socket.socket(af, socktype, proto) - except socket.error as msg: - self._sock = None - continue - try: - self._sock.connect(sa) - except socket.error as msg: - self._disconnect() - continue - self.__socket_config = res - break - else: - # connect to socket with previously established configuration - try: - af, socktype, proto, canonname, sa = self.__socket_config - self._sock = socket.socket(af, socktype, proto) - self._sock.connect(sa) - except socket.error as msg: - self._disconnect() + if self.unix_socket is not None: + self._connect2unixsocket() + else: - # connect to unix socket - self._sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) - self._sock.connect(self.unix_socket) + if self.__socket_config is not None: + self._connect2socket() + else: + for res in socket.getaddrinfo(self.host, self.port, socket.AF_UNSPEC, socket.SOCK_STREAM): + if self._connect2socket(res): break + except Exception as e: - self.error(str(e), - "Cannot create socket with following configuration: host:", str(self.host), - "port:", str(self.port), - "socket:", str(self.unix_socket)) self._sock = None - self._sock.setblocking(0) + self.__socket_config = None + + if self._sock is not None: + self._sock.setblocking(0) + self._sock.settimeout(5) + self.debug("set socket timeout to: " + str(self._sock.gettimeout())) def _disconnect(self): """ Close socket connection :return: """ - try: - self._sock.shutdown(2) # 0 - read, 1 - write, 2 - all - self._sock.close() - except Exception: - pass - self._sock = None + if self._sock is not None: + try: + self.debug("closing socket") + self._sock.shutdown(2) # 0 - read, 1 - write, 2 - all + self._sock.close() + except Exception: + pass + self._sock = None def _send(self): """ @@ -552,15 +699,13 @@ class SocketService(SimpleService): :return: boolean """ # Send request if it is needed - if self.request != "".encode(): + if self.request != self.__empty_request: try: + self.debug("sending request:", str(self.request)) self._sock.send(self.request) except Exception as e: + self._socketerror("error sending request:" + str(e)) self._disconnect() - self.error(str(e), - "used configuration: host:", str(self.host), - "port:", str(self.port), - "socket:", str(self.unix_socket)) return False return True @@ -571,24 +716,28 @@ class SocketService(SimpleService): """ data = "" while True: + self.debug("receiving response") try: - ready_to_read, _, in_error = select.select([self._sock], [], [], 15) + buf = self._sock.recv(4096) except Exception as e: - self.debug("SELECT", str(e)) + self._socketerror("failed to receive response:" + str(e)) self._disconnect() break - if len(ready_to_read) > 0: - buf = self._sock.recv(4096) - if len(buf) == 0 or buf is None: # handle server disconnect - break - data += buf.decode() - if self._check_raw_data(data): - break - else: - self.error("Socket timed out.") + + if buf is None or len(buf) == 0: # handle server disconnect + if data == "": + self._socketerror("unexpectedly disconnected") + else: + self.debug("server closed the connection") self._disconnect() break + self.debug("received data:", str(buf)) + data += buf.decode('utf-8', 'ignore') + if self._check_raw_data(data): + break + + self.debug("final response:", str(data)) return data def _get_raw_data(self): @@ -598,6 +747,8 @@ class SocketService(SimpleService): """ if self._sock is None: self._connect() + if self._sock is None: + return None # Send request if it is needed if not self._send(): @@ -627,10 +778,12 @@ class SocketService(SimpleService): self.name = "" else: self.name = str(self.name) + try: self.unix_socket = str(self.configuration['socket']) except (KeyError, TypeError): self.debug("No unix socket specified. Trying TCP/IP socket.") + self.unix_socket = None try: self.host = str(self.configuration['host']) except (KeyError, TypeError): @@ -639,12 +792,18 @@ class SocketService(SimpleService): self.port = int(self.configuration['port']) except (KeyError, TypeError): self.debug("No port specified. Using: '" + str(self.port) + "'") + try: self.request = str(self.configuration['request']) except (KeyError, TypeError): self.debug("No request specified. Using: '" + str(self.request) + "'") + self.request = self.request.encode() + def check(self): + self._parse_config() + return SimpleService.check(self) + class LogService(SimpleService): def __init__(self, configuration=None, name=None): @@ -662,10 +821,10 @@ class LogService(SimpleService): lines = [] try: if os.path.getsize(self.log_path) < self._last_position: - self._last_position = 0 + self._last_position = 0 # read from beginning if file has shrunk elif os.path.getsize(self.log_path) == self._last_position: self.debug("Log file hasn't changed. No new data.") - return None + return [] # return empty list if nothing has changed with open(self.log_path, "r") as fp: fp.seek(self._last_position) for i, line in enumerate(fp): @@ -692,7 +851,7 @@ class LogService(SimpleService): try: self.log_path = str(self.configuration['path']) except (KeyError, TypeError): - self.error("No path to log specified. Using: '" + self.log_path + "'") + self.info("No path to log specified. Using: '" + self.log_path + "'") if os.access(self.log_path, os.R_OK): return True @@ -701,72 +860,208 @@ class LogService(SimpleService): return False def create(self): + # set cursor at last byte of log file + self._last_position = os.path.getsize(self.log_path) status = SimpleService.create(self) - self._last_position = 0 + # self._last_position = 0 return status class ExecutableService(SimpleService): - #command_whitelist = ['exim', 'postqueue'] - bad_substrings = ('&', '|', ';', '>', '<') def __init__(self, configuration=None, name=None): - self.command = "" SimpleService.__init__(self, configuration=configuration, name=name) + self.command = None def _get_raw_data(self): """ Get raw data from executed command - :return: str + :return: """ try: p = Popen(self.command, stdout=PIPE, stderr=PIPE) - except Exception as e: - self.error("Executing command", self.command, "resulted in error:", str(e)) + except Exception as error: + self.error("Executing command", self.command, "resulted in error:", str(error)) return None - data = [] + data = list() for line in p.stdout.readlines(): - data.append(str(line.decode())) + data.append(line.decode()) - if len(data) == 0: - self.error("No data collected.") - return None - - return data + return data or None def check(self): """ Parse basic configuration, check if command is whitelisted and is returning values - :return: boolean + :return: """ - if self.name is not None or self.name != str(None): - self.name = "" + # Preference: 1. "command" from configuration file 2. "command" from plugin (if specified) + if 'command' in self.configuration: + self.command = self.configuration['command'] + + # "command" must be: 1.not None 2. type + if not (self.command and isinstance(self.command, str)): + self.error('Command is not defined or command type is not ') + return False + + # Split "command" into: 1. command 2. options + command, opts = self.command.split()[0], self.command.split()[1:] + + # Check for "bad" symbols in options. No pipes, redirects etc. TODO: what is missing? + bad_opts = set(''.join(opts)) & set(['&', '|', ';', '>', '<']) + if bad_opts: + self.error("Bad command argument(s): %s" % bad_opts) + return False + + # Find absolute path ('echo' => '/bin/echo') + if '/' not in command: + command = self.find_binary(command) + if not command: + self.error('Can\'t locate "%s" binary in PATH(%s)' % (self.command, PATH)) + return False + # Check if binary exist and executable else: - self.name = str(self.name) - try: - self.command = str(self.configuration['command']) - except (KeyError, TypeError): - self.error("No command specified. Using: '" + self.command + "'") - self.command = self.command.split(' ') - #if self.command[0] not in self.command_whitelist: - # self.error("Command is not whitelisted.") - # return False - - for arg in self.command[1:]: - if any(st in arg for st in self.bad_substrings): - self.error("Bad command argument:" + " ".join(self.command[1:])) + if not (os.path.isfile(command) and os.access(command, os.X_OK)): + self.error('"%s" is not a file or not executable' % command) return False - # test command and search for it in /usr/sbin or /sbin when failed - base = self.command[0].split('/')[-1] - if self._get_raw_data() is None: - for prefix in ['/sbin/', '/usr/sbin/']: - self.command[0] = prefix + base - if os.path.isfile(self.command[0]): - break - #if self._get_raw_data() is not None: - # break - - if self._get_data() is None or len(self._get_data()) == 0: - self.error("Command", self.command, "returned no data") + + self.command = [command] + opts if opts else [command] + + try: + data = self._get_data() + except Exception as error: + self.error('_get_data() failed. Command: %s. Error: %s' % (self.command, error)) return False - return True + + if isinstance(data, dict) and data: + # We need this for create() method. No reason to execute get_data() again if result is not empty dict() + self._data_from_check = data + return True + else: + self.error("Command", str(self.command), "returned no data") + return False + + +class MySQLService(SimpleService): + + def __init__(self, configuration=None, name=None): + SimpleService.__init__(self, configuration=configuration, name=name) + self.__connection = None + self.conn_properties = dict() + self.queries = dict() + + def __connect(self): + try: + connection = MySQLdb.connect(connect_timeout=self.update_every, **self.conn_properties) + except (MySQLdb.MySQLError, TypeError) as error: + return None, str(error) + else: + return connection, None + + def check(self): + def get_connection_properties(conf): + properties = dict() + if 'user' in conf and conf['user']: + properties['user'] = conf['user'] + if 'pass' in conf and conf['pass']: + properties['passwd'] = conf['pass'] + if 'socket' in conf and conf['socket']: + properties['unix_socket'] = conf['socket'] + elif 'host' in conf and conf['host']: + properties['host'] = conf['host'] + properties['port'] = int(conf['port']) if conf.get('port') else 3306 + elif 'my.cnf' in conf and conf['my.cnf']: + properties['read_default_file'] = conf['my.cnf'] + + return properties or None + + def is_valid_queries_dict(raw_queries, log_error): + """ + :param raw_queries: dict: + :param log_error: function: + :return: dict or None + + raw_queries is valid when: type and not empty after is_valid_query(for all queries) + """ + def is_valid_query(query): + return all([isinstance(query, str), + query.startswith(('SELECT', 'select', 'SHOW', 'show'))]) + + if isinstance(raw_queries, dict) and raw_queries: + valid_queries = dict([(n, q) for n, q in raw_queries.items() if is_valid_query(q)]) + bad_queries = set(raw_queries) - set(valid_queries) + + if bad_queries: + log_error('Removed query(s): %s' % bad_queries) + return valid_queries + else: + log_error('Unsupported "queries" format. Must be not empty ') + return None + + if not PYMYSQL: + self.error('MySQLdb or PyMySQL module is needed to use mysql.chart.py plugin') + return False + + # Check if "self.queries" exist, not empty and all queries are in valid format + self.queries = is_valid_queries_dict(self.queries, self.error) + if not self.queries: + return None + + # Get connection properties + self.conn_properties = get_connection_properties(self.configuration) + if not self.conn_properties: + self.error('Connection properties are missing') + return False + + # Create connection to the database + self.__connection, error = self.__connect() + if error: + self.error('Can\'t establish connection to MySQL: %s' % error) + return False + + try: + data = self._get_data() + except Exception as error: + self.error('_get_data() failed. Error: %s' % error) + return False + + if isinstance(data, dict) and data: + # We need this for create() method + self._data_from_check = data + return True + else: + self.error("_get_data() returned no data or type is not ") + return False + + def _get_raw_data(self, description=None): + """ + Get raw data from MySQL server + :return: dict: fetchall() or (fetchall(), description) + """ + + if not self.__connection: + self.__connection, error = self.__connect() + if error: + return None + + raw_data = dict() + try: + with self.__connection as cursor: + for name, query in self.queries.items(): + try: + cursor.execute(query) + except (MySQLdb.ProgrammingError, MySQLdb.OperationalError) as error: + if exc_info()[0] == MySQLdb.OperationalError and 'denied' not in str(error): + raise RuntimeError + self.error('Removed query: %s[%s]. Error: %s' + % (name, query, error)) + self.queries.pop(name) + continue + else: + raw_data[name] = (cursor.fetchall(), cursor.description) if description else cursor.fetchall() + self.__connection.commit() + except (MySQLdb.MySQLError, RuntimeError, TypeError, AttributeError): + self.__connection.close() + self.__connection = None + return None + else: + return raw_data or None