# -*- coding: utf-8 -*-
-# Description: prototypes for netdata python.d modules
+# Description: netdata python modules framework
# Author: Pawel Krupa (paulfantom)
+# Remember:
+# ALL CODE NEEDS TO BE COMPATIBLE WITH Python > 2.7 and Python > 3.1
+# Follow PEP8 as much as it is possible
+# "check" and "create" CANNOT be blocking.
+# "update" CAN be blocking
+# "update" function needs to be fast, so follow:
+# https://wiki.python.org/moin/PythonSpeed/PerformanceTips
+# basically:
+# - use local variables wherever it is possible
+# - avoid dots in expressions that are executed many times
+# - use "join()" instead of "+"
+# - use "import" only at the beginning
+#
+# using ".encode()" in one thread can block other threads as well (only in python2)
+
import time
-import sys
import os
import socket
import select
+import threading
+import msg
+import ssl
+from subprocess import Popen, PIPE
+from sys import exc_info
+
try:
import urllib.request as urllib2
except ImportError:
import urllib2
-from subprocess import Popen, PIPE
+try:
+ import MySQLdb
+ PYMYSQL = True
+except ImportError:
+ try:
+ import pymysql as MySQLdb
+ PYMYSQL = True
+ except ImportError:
+ PYMYSQL = False
-import threading
-import msg
+try:
+ PATH = os.getenv('PATH').split(':')
+except AttributeError:
+ PATH = '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'.split(':')
-class BaseService(threading.Thread):
+# class BaseService(threading.Thread):
+class SimpleService(threading.Thread):
"""
Prototype of Service class.
Implemented basic functionality to run jobs by `python.d.plugin`
self._dimensions = []
self._charts = []
self.__chart_set = False
+ self.__first_run = True
+ self.order = []
+ self.definitions = {}
+ self._data_from_check = dict()
if configuration is None:
self.error("BaseService: no configuration parameters supplied. Cannot create Service.")
raise RuntimeError
self.timetable = {}
self.create_timetable()
+ # --- BASIC SERVICE CONFIGURATION ---
+
def _extract_base_config(self, config):
"""
Get basic parameters to run service
'retries':0}
:param config: dict
"""
+ pop = config.pop
try:
- self.override_name = config.pop('name')
+ self.override_name = pop('name')
except KeyError:
pass
- self.update_every = int(config.pop('update_every'))
- self.priority = int(config.pop('priority'))
- self.retries = int(config.pop('retries'))
+ self.update_every = int(pop('update_every'))
+ self.priority = int(pop('priority'))
+ self.retries = int(pop('retries'))
self.retries_left = self.retries
self.configuration = config
'next': now - (now % freq) + freq,
'freq': freq}
+ # --- THREAD CONFIGURATION ---
+
def _run_once(self):
"""
Executes self.update(interval) and draws run time chart.
Return value presents exit status of update()
:return: boolean
"""
- t_start = time.time()
- # check if it is time to execute job update() function
- if self.timetable['next'] > t_start:
- #msg.debug(self.chart_name + " will be run in " +
- # str(int((self.timetable['next'] - t_start) * 1000)) + " ms")
- msg.debug(self.chart_name, "will be run in", str(int((self.timetable['next'] - t_start) * 1000)), "ms")
- return True
+ t_start = float(time.time())
+ chart_name = self.chart_name
since_last = int((t_start - self.timetable['last']) * 1000000)
- #msg.debug(self.chart_name +
- # " ready to run, after " + str(int((t_start - self.timetable['last']) * 1000)) +
- # " ms (update_every: " + str(self.timetable['freq'] * 1000) +
- # " ms, latency: " + str(int((t_start - self.timetable['next']) * 1000)) + " ms)")
- msg.debug(self.chart_name,
- "ready to run, after", str(int((t_start - self.timetable['last']) * 1000)),
- "ms (update_every:", str(self.timetable['freq'] * 1000),
- "ms, latency:", str(int((t_start - self.timetable['next']) * 1000)), "ms")
+ if self.__first_run:
+ since_last = 0
+
if not self.update(since_last):
self.error("update function failed.")
return False
- t_end = time.time()
- self.timetable['next'] = t_end - (t_end % self.timetable['freq']) + self.timetable['freq']
+
# draw performance graph
- run_time = str(int((t_end - t_start) * 1000))
- #run_time_chart = "BEGIN netdata.plugin_pythond_" + self.chart_name + " " + str(since_last) + '\n'
- #run_time_chart += "SET run_time = " + run_time + '\n'
- #run_time_chart += "END\n"
- #sys.stdout.write(run_time_chart)
- sys.stdout.write("BEGIN netdata.plugin_pythond_%s %s\nSET run_time = %s\nEND\n" % \
- (self.chart_name, str(since_last), run_time))
-
- #msg.debug(self.chart_name + " updated in " + str(run_time) + " ms")
- msg.debug(self.chart_name, "updated in", str(run_time), "ms")
+ run_time = int((time.time() - t_start) * 1000)
+ print("BEGIN netdata.plugin_pythond_%s %s\nSET run_time = %s\nEND\n" %
+ (self.chart_name, str(since_last), str(run_time)))
+
+ self.debug(chart_name, "updated in", str(run_time), "ms")
self.timetable['last'] = t_start
+ self.__first_run = False
return True
def run(self):
Exits when job failed or timed out.
:return: None
"""
- self.timetable['last'] = time.time()
- while True:
+ step = float(self.timetable['freq'])
+ penalty = 0
+ self.timetable['last'] = float(time.time() - step)
+ self.debug("starting data collection - update frequency:", str(step), " retries allowed:", str(self.retries))
+ while True: # run forever, unless something is wrong
+ now = float(time.time())
+ next = self.timetable['next'] = now - (now % step) + step + penalty
+
+ # it is important to do this in a loop
+ # sleep() is interruptable
+ while now < next:
+ self.debug("sleeping for", str(next - now), "secs to reach frequency of", str(step), "secs, now:", str(now), " next:", str(next), " penalty:", str(penalty))
+ time.sleep(next - now)
+ now = float(time.time())
+
+ # do the job
try:
status = self._run_once()
except Exception as e:
- msg.error("Something wrong: ", str(e))
- return
+ status = False
+
if status:
- time.sleep(self.timetable['next'] - time.time())
+ # it is good
self.retries_left = self.retries
+ penalty = 0
else:
+ # it failed
self.retries_left -= 1
if self.retries_left <= 0:
- msg.error("no more retries. Exiting")
- return
+ if penalty == 0:
+ penalty = float(self.retries * step) / 2
+ else:
+ penalty *= 1.5
+
+ if penalty > 600:
+ penalty = 600
+
+ self.retries_left = self.retries
+ self.alert("failed to collect data for " + str(self.retries) + " times - increasing penalty to " + str(penalty) + " sec and trying again")
+
else:
- time.sleep(self.timetable['freq'])
+ self.error("failed to collect data - " + str(self.retries_left) + " retries left - penalty: " + str(penalty) + " sec")
+
+ # --- CHART ---
@staticmethod
def _format(*args):
+ """
+ Escape and convert passed arguments.
+ :param args: anything
+ :return: list
+ """
params = []
append = params.append
for p in args:
def _line(self, instruction, *params):
"""
Converts *params to string and joins them with one space between every one.
+ Result is appended to self._data_stream
:param params: str/int/float
"""
- #self._data_stream += instruction
tmp = list(map((lambda x: "''" if x is None or len(x) == 0 else x), params))
-
self._data_stream += "%s %s\n" % (instruction, str(" ".join(tmp)))
- # self.error(str(" ".join(tmp)))
- # for p in params:
- # if p is None:
- # p = ""
- # else:
- # p = str(p)
- # if len(p) == 0:
- # p = "''"
- # if ' ' in p:
- # p = "'" + p + "'"
- # self._data_stream += " " + p
- #self._data_stream += "\n"
-
def chart(self, type_id, name="", title="", units="", family="",
- category="", charttype="line", priority="", update_every=""):
+ category="", chart_type="line", priority="", update_every=""):
"""
Defines a new chart.
:param type_id: str
:param units: str
:param family: str
:param category: str
- :param charttype: str
+ :param chart_type: str
:param priority: int/str
:param update_every: int/str
"""
self._charts.append(type_id)
- #self._line("CHART", type_id, name, title, units, family, category, charttype, priority, update_every)
- p = self._format(type_id, name, title, units, family, category, charttype, priority, update_every)
+ p = self._format(type_id, name, title, units, family, category, chart_type, priority, update_every)
self._line("CHART", *p)
def dimension(self, id, name=None, algorithm="absolute", multiplier=1, divisor=1, hidden=False):
self._dimensions.append(str(id))
if hidden:
p = self._format(id, name, algorithm, multiplier, divisor, "hidden")
- #self._line("DIMENSION", id, name, algorithm, str(multiplier), str(divisor), "hidden")
else:
p = self._format(id, name, algorithm, multiplier, divisor)
- #self._line("DIMENSION", id, name, algorithm, str(multiplier), str(divisor))
self._line("DIMENSION", *p)
try:
value = str(int(value))
except TypeError:
- self.error("cannot set non-numeric value:", value)
+ self.error("cannot set non-numeric value:", str(value))
return False
self._line("SET", id, "=", str(value))
self.__chart_set = True
def commit(self):
"""
- Upload new data to netdata
+ Upload new data to netdata.
"""
- print(self._data_stream)
+ try:
+ print(self._data_stream)
+ except Exception as e:
+ msg.fatal('cannot send data to netdata:', str(e))
self._data_stream = ""
+ # --- ERROR HANDLING ---
+
def error(self, *params):
"""
Show error message on stderr
"""
msg.error(self.chart_name, *params)
+ def alert(self, *params):
+ """
+ Show error message on stderr
+ """
+ msg.alert(self.chart_name, *params)
+
def debug(self, *params):
"""
Show debug message on stderr
"""
msg.info(self.chart_name, *params)
- def check(self):
- """
- check() prototype
- :return: boolean
- """
- msg.error("Service " + str(self.__module__) + "doesn't implement check() function")
- return False
-
- def create(self):
- """
- create() prototype
- :return: boolean
- """
- msg.error("Service " + str(self.__module__) + "doesn't implement create() function?")
- return False
-
- def update(self, interval):
- """
- update() prototype
- :param interval: int
- :return: boolean
- """
- msg.error("Service " + str(self.__module__) + "doesn't implement update() function")
- return False
-
-
-class SimpleService(BaseService):
- def __init__(self, configuration=None, name=None):
- self.order = []
- self.definitions = {}
- BaseService.__init__(self, configuration=configuration, name=name)
+ # --- MAIN METHODS ---
def _get_data(self):
"""
def check(self):
"""
- :return:
+ check() prototype
+ :return: boolean
"""
+ self.debug("Module", str(self.__module__), "doesn't implement check() function. Using default.")
+ data = self._get_data()
+
+ if data is None:
+ self.debug("failed to receive data during check().")
+ return False
+
+ if len(data) == 0:
+ self.debug("empty data during check().")
+ return False
+
+ self.debug("successfully received data during check(): '" + str(data) + "'")
return True
def create(self):
Create charts
:return: boolean
"""
- data = self._get_data()
+ data = self._data_from_check or self._get_data()
if data is None:
+ self.debug("failed to receive data during create().")
return False
idx = 0
"""
data = self._get_data()
if data is None:
- self.debug("_get_data() returned no data")
+ self.debug("failed to receive data during update().")
return False
updated = False
return updated
+ @staticmethod
+ def find_binary(binary):
+ try:
+ if isinstance(binary, str):
+ binary = os.path.basename(binary)
+ return next(('/'.join([p, binary]) for p in PATH
+ if os.path.isfile('/'.join([p, binary]))
+ and os.access('/'.join([p, binary]), os.X_OK)))
+ else:
+ return None
+ except StopIteration:
+ return None
+
class UrlService(SimpleService):
+ # TODO add support for https connections
def __init__(self, configuration=None, name=None):
self.url = ""
self.user = None
self.password = None
+ self.proxies = {}
SimpleService.__init__(self, configuration=configuration, name=name)
- def __add_auth(self):
- passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
- passman.add_password(None, self.url, self.user, self.password)
- authhandler = urllib2.HTTPBasicAuthHandler(passman)
- opener = urllib2.build_opener(authhandler)
- urllib2.install_opener(opener)
+ def __add_openers(self):
+ # TODO add error handling
+ if self.ss_cert:
+ try:
+ ctx = ssl.create_default_context()
+ ctx.check_hostname = False
+ ctx.verify_mode = ssl.CERT_NONE
+ self.opener = urllib2.build_opener(urllib2.HTTPSHandler(context=ctx))
+ except Exception as error:
+ self.error(str(error))
+ self.opener = urllib2.build_opener()
+ else:
+ self.opener = urllib2.build_opener()
+
+ # Proxy handling
+ # TODO currently self.proxies isn't parsed from configuration file
+ # if len(self.proxies) > 0:
+ # for proxy in self.proxies:
+ # url = proxy['url']
+ # # TODO test this:
+ # if "user" in proxy and "pass" in proxy:
+ # if url.lower().startswith('https://'):
+ # url = 'https://' + proxy['user'] + ':' + proxy['pass'] + '@' + url[8:]
+ # else:
+ # url = 'http://' + proxy['user'] + ':' + proxy['pass'] + '@' + url[7:]
+ # # FIXME move proxy auth to sth like this:
+ # # passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
+ # # passman.add_password(None, url, proxy['user'], proxy['password'])
+ # # opener.add_handler(urllib2.HTTPBasicAuthHandler(passman))
+ #
+ # if url.lower().startswith('https://'):
+ # opener.add_handler(urllib2.ProxyHandler({'https': url}))
+ # else:
+ # opener.add_handler(urllib2.ProxyHandler({'https': url}))
+
+ # HTTP Basic Auth
+ if self.user is not None and self.password is not None:
+ passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
+ passman.add_password(None, self.url, self.user, self.password)
+ self.opener.add_handler(urllib2.HTTPBasicAuthHandler(passman))
+ self.debug("Enabling HTTP basic auth")
+
+ #urllib2.install_opener(opener)
def _get_raw_data(self):
"""
"""
raw = None
try:
- f = urllib2.urlopen(self.url, timeout=self.update_every)
+ f = self.opener.open(self.url, timeout=self.update_every * 2)
+ # f = urllib2.urlopen(self.url, timeout=self.update_every * 2)
except Exception as e:
self.error(str(e))
return None
try:
- raw = f.read().decode('utf-8')
+ raw = f.read().decode('utf-8', 'ignore')
except Exception as e:
self.error(str(e))
finally:
self.password = str(self.configuration['pass'])
except (KeyError, TypeError):
pass
+ self.ss_cert = self.configuration.get('ss_cert')
+ self.__add_openers()
- if self.user is not None and self.password is not None:
- self.__add_auth()
-
- if self._get_data() is not None:
- return True
- else:
+ test = self._get_data()
+ if test is None or len(test) == 0:
return False
+ else:
+ return True
class SocketService(SimpleService):
self.unix_socket = None
self.request = ""
self.__socket_config = None
+ self.__empty_request = "".encode()
SimpleService.__init__(self, configuration=configuration, name=name)
+ def _socketerror(self, message=None):
+ if self.unix_socket is not None:
+ self.error("unix socket '" + self.unix_socket + "':", message)
+ else:
+ if self.__socket_config is not None:
+ af, socktype, proto, canonname, sa = self.__socket_config
+ self.error("socket to '" + str(sa[0]) + "' port " + str(sa[1]) + ":", message)
+ else:
+ self.error("unknown socket:", message)
+
+ def _connect2socket(self, res=None):
+ """
+ Connect to a socket, passing the result of getaddrinfo()
+ :return: boolean
+ """
+ if res is None:
+ res = self.__socket_config
+ if res is None:
+ self.error("Cannot create socket to 'None':")
+ return False
+
+ af, socktype, proto, canonname, sa = res
+ try:
+ self.debug("creating socket to '" + str(sa[0]) + "', port " + str(sa[1]))
+ self._sock = socket.socket(af, socktype, proto)
+ except socket.error as e:
+ self.error("Failed to create socket to '" + str(sa[0]) + "', port " + str(sa[1]) + ":", str(e))
+ self._sock = None
+ self.__socket_config = None
+ return False
+
+ try:
+ self.debug("connecting socket to '" + str(sa[0]) + "', port " + str(sa[1]))
+ self._sock.connect(sa)
+ except socket.error as e:
+ self.error("Failed to connect to '" + str(sa[0]) + "', port " + str(sa[1]) + ":", str(e))
+ self._disconnect()
+ self.__socket_config = None
+ return False
+
+ self.debug("connected to '" + str(sa[0]) + "', port " + str(sa[1]))
+ self.__socket_config = res
+ return True
+
+ def _connect2unixsocket(self):
+ """
+ Connect to a unix socket, given its filename
+ :return: boolean
+ """
+ if self.unix_socket is None:
+ self.error("cannot connect to unix socket 'None'")
+ return False
+
+ try:
+ self.debug("attempting DGRAM unix socket '" + str(self.unix_socket) + "'")
+ self._sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
+ self._sock.connect(self.unix_socket)
+ self.debug("connected DGRAM unix socket '" + str(self.unix_socket) + "'")
+ return True
+ except socket.error as e:
+ self.debug("Failed to connect DGRAM unix socket '" + str(self.unix_socket) + "':", str(e))
+
+ try:
+ self.debug("attempting STREAM unix socket '" + str(self.unix_socket) + "'")
+ self._sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+ self._sock.connect(self.unix_socket)
+ self.debug("connected STREAM unix socket '" + str(self.unix_socket) + "'")
+ return True
+ except socket.error as e:
+ self.debug("Failed to connect STREAM unix socket '" + str(self.unix_socket) + "':", str(e))
+ self.error("Failed to connect to unix socket '" + str(self.unix_socket) + "':", str(e))
+ self._sock = None
+ return False
+
def _connect(self):
"""
Recreate socket and connect to it since sockets cannot be reused after closing
:return:
"""
try:
- if self.unix_socket is None:
- if self.__socket_config is None:
- # establish ipv6 or ipv4 connection.
- for res in socket.getaddrinfo(self.host, self.port, socket.AF_UNSPEC, socket.SOCK_STREAM):
- try:
- af, socktype, proto, canonname, sa = res
- self._sock = socket.socket(af, socktype, proto)
- except socket.error as msg:
- self._sock = None
- continue
- try:
- self._sock.connect(sa)
- except socket.error as msg:
- self._disconnect()
- continue
- self.__socket_config = res
- break
- else:
- # connect to socket with previously established configuration
- try:
- af, socktype, proto, canonname, sa = self.__socket_config
- self._sock = socket.socket(af, socktype, proto)
- self._sock.connect(sa)
- except socket.error as msg:
- self._disconnect()
+ if self.unix_socket is not None:
+ self._connect2unixsocket()
+
else:
- # connect to unix socket
- self._sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
- self._sock.connect(self.unix_socket)
+ if self.__socket_config is not None:
+ self._connect2socket()
+ else:
+ for res in socket.getaddrinfo(self.host, self.port, socket.AF_UNSPEC, socket.SOCK_STREAM):
+ if self._connect2socket(res): break
+
except Exception as e:
- self.error(str(e),
- "Cannot create socket with following configuration: host:", str(self.host),
- "port:", str(self.port),
- "socket:", str(self.unix_socket))
self._sock = None
- self._sock.setblocking(0)
+ self.__socket_config = None
+
+ if self._sock is not None:
+ self._sock.setblocking(0)
+ self._sock.settimeout(5)
+ self.debug("set socket timeout to: " + str(self._sock.gettimeout()))
def _disconnect(self):
"""
Close socket connection
:return:
"""
- try:
- self._sock.shutdown(2) # 0 - read, 1 - write, 2 - all
- self._sock.close()
- except Exception:
- pass
- self._sock = None
+ if self._sock is not None:
+ try:
+ self.debug("closing socket")
+ self._sock.shutdown(2) # 0 - read, 1 - write, 2 - all
+ self._sock.close()
+ except Exception:
+ pass
+ self._sock = None
def _send(self):
"""
:return: boolean
"""
# Send request if it is needed
- if self.request != "".encode():
+ if self.request != self.__empty_request:
try:
+ self.debug("sending request:", str(self.request))
self._sock.send(self.request)
except Exception as e:
+ self._socketerror("error sending request:" + str(e))
self._disconnect()
- self.error(str(e),
- "used configuration: host:", str(self.host),
- "port:", str(self.port),
- "socket:", str(self.unix_socket))
return False
return True
"""
data = ""
while True:
+ self.debug("receiving response")
try:
- ready_to_read, _, in_error = select.select([self._sock], [], [], 15)
+ buf = self._sock.recv(4096)
except Exception as e:
- self.debug("SELECT", str(e))
+ self._socketerror("failed to receive response:" + str(e))
self._disconnect()
break
- if len(ready_to_read) > 0:
- buf = self._sock.recv(4096)
- if len(buf) == 0 or buf is None: # handle server disconnect
- break
- data += buf.decode()
- if self._check_raw_data(data):
- break
- else:
- self.error("Socket timed out.")
+
+ if buf is None or len(buf) == 0: # handle server disconnect
+ if data == "":
+ self._socketerror("unexpectedly disconnected")
+ else:
+ self.debug("server closed the connection")
self._disconnect()
break
+ self.debug("received data:", str(buf))
+ data += buf.decode('utf-8', 'ignore')
+ if self._check_raw_data(data):
+ break
+
+ self.debug("final response:", str(data))
return data
def _get_raw_data(self):
"""
if self._sock is None:
self._connect()
+ if self._sock is None:
+ return None
# Send request if it is needed
if not self._send():
self.name = ""
else:
self.name = str(self.name)
+
try:
self.unix_socket = str(self.configuration['socket'])
except (KeyError, TypeError):
self.debug("No unix socket specified. Trying TCP/IP socket.")
+ self.unix_socket = None
try:
self.host = str(self.configuration['host'])
except (KeyError, TypeError):
self.port = int(self.configuration['port'])
except (KeyError, TypeError):
self.debug("No port specified. Using: '" + str(self.port) + "'")
+
try:
self.request = str(self.configuration['request'])
except (KeyError, TypeError):
self.debug("No request specified. Using: '" + str(self.request) + "'")
+
self.request = self.request.encode()
+ def check(self):
+ self._parse_config()
+ return SimpleService.check(self)
+
class LogService(SimpleService):
def __init__(self, configuration=None, name=None):
lines = []
try:
if os.path.getsize(self.log_path) < self._last_position:
- self._last_position = 0
+ self._last_position = 0 # read from beginning if file has shrunk
elif os.path.getsize(self.log_path) == self._last_position:
self.debug("Log file hasn't changed. No new data.")
- return None
+ return [] # return empty list if nothing has changed
with open(self.log_path, "r") as fp:
fp.seek(self._last_position)
for i, line in enumerate(fp):
try:
self.log_path = str(self.configuration['path'])
except (KeyError, TypeError):
- self.error("No path to log specified. Using: '" + self.log_path + "'")
+ self.info("No path to log specified. Using: '" + self.log_path + "'")
if os.access(self.log_path, os.R_OK):
return True
return False
def create(self):
+ # set cursor at last byte of log file
+ self._last_position = os.path.getsize(self.log_path)
status = SimpleService.create(self)
- self._last_position = 0
+ # self._last_position = 0
return status
class ExecutableService(SimpleService):
- #command_whitelist = ['exim', 'postqueue']
- bad_substrings = ('&', '|', ';', '>', '<')
def __init__(self, configuration=None, name=None):
- self.command = ""
SimpleService.__init__(self, configuration=configuration, name=name)
+ self.command = None
def _get_raw_data(self):
"""
Get raw data from executed command
- :return: str
+ :return: <list>
"""
try:
p = Popen(self.command, stdout=PIPE, stderr=PIPE)
- except Exception as e:
- self.error("Executing command", self.command, "resulted in error:", str(e))
+ except Exception as error:
+ self.error("Executing command", self.command, "resulted in error:", str(error))
return None
- data = []
+ data = list()
for line in p.stdout.readlines():
- data.append(str(line.decode()))
+ data.append(line.decode())
- if len(data) == 0:
- self.error("No data collected.")
- return None
-
- return data
+ return data or None
def check(self):
"""
Parse basic configuration, check if command is whitelisted and is returning values
- :return: boolean
+ :return: <boolean>
"""
- if self.name is not None or self.name != str(None):
- self.name = ""
+ # Preference: 1. "command" from configuration file 2. "command" from plugin (if specified)
+ if 'command' in self.configuration:
+ self.command = self.configuration['command']
+
+ # "command" must be: 1.not None 2. type <str>
+ if not (self.command and isinstance(self.command, str)):
+ self.error('Command is not defined or command type is not <str>')
+ return False
+
+ # Split "command" into: 1. command <str> 2. options <list>
+ command, opts = self.command.split()[0], self.command.split()[1:]
+
+ # Check for "bad" symbols in options. No pipes, redirects etc. TODO: what is missing?
+ bad_opts = set(''.join(opts)) & set(['&', '|', ';', '>', '<'])
+ if bad_opts:
+ self.error("Bad command argument(s): %s" % bad_opts)
+ return False
+
+ # Find absolute path ('echo' => '/bin/echo')
+ if '/' not in command:
+ command = self.find_binary(command)
+ if not command:
+ self.error('Can\'t locate "%s" binary in PATH(%s)' % (self.command, PATH))
+ return False
+ # Check if binary exist and executable
else:
- self.name = str(self.name)
- try:
- self.command = str(self.configuration['command'])
- except (KeyError, TypeError):
- self.error("No command specified. Using: '" + self.command + "'")
- self.command = self.command.split(' ')
- #if self.command[0] not in self.command_whitelist:
- # self.error("Command is not whitelisted.")
- # return False
-
- for arg in self.command[1:]:
- if any(st in arg for st in self.bad_substrings):
- self.error("Bad command argument:" + " ".join(self.command[1:]))
+ if not (os.path.isfile(command) and os.access(command, os.X_OK)):
+ self.error('"%s" is not a file or not executable' % command)
return False
- # test command and search for it in /usr/sbin or /sbin when failed
- base = self.command[0].split('/')[-1]
- if self._get_raw_data() is None:
- for prefix in ['/sbin/', '/usr/sbin/']:
- self.command[0] = prefix + base
- if os.path.isfile(self.command[0]):
- break
- #if self._get_raw_data() is not None:
- # break
-
- if self._get_data() is None or len(self._get_data()) == 0:
- self.error("Command", self.command, "returned no data")
+
+ self.command = [command] + opts if opts else [command]
+
+ try:
+ data = self._get_data()
+ except Exception as error:
+ self.error('_get_data() failed. Command: %s. Error: %s' % (self.command, error))
return False
- return True
+
+ if isinstance(data, dict) and data:
+ # We need this for create() method. No reason to execute get_data() again if result is not empty dict()
+ self._data_from_check = data
+ return True
+ else:
+ self.error("Command", str(self.command), "returned no data")
+ return False
+
+
+class MySQLService(SimpleService):
+
+ def __init__(self, configuration=None, name=None):
+ SimpleService.__init__(self, configuration=configuration, name=name)
+ self.__connection = None
+ self.conn_properties = dict()
+ self.queries = dict()
+
+ def __connect(self):
+ try:
+ connection = MySQLdb.connect(connect_timeout=self.update_every, **self.conn_properties)
+ except (MySQLdb.MySQLError, TypeError) as error:
+ return None, str(error)
+ else:
+ return connection, None
+
+ def check(self):
+ def get_connection_properties(conf):
+ properties = dict()
+ if 'user' in conf and conf['user']:
+ properties['user'] = conf['user']
+ if 'pass' in conf and conf['pass']:
+ properties['passwd'] = conf['pass']
+ if 'socket' in conf and conf['socket']:
+ properties['unix_socket'] = conf['socket']
+ elif 'host' in conf and conf['host']:
+ properties['host'] = conf['host']
+ properties['port'] = int(conf['port']) if conf.get('port') else 3306
+ elif 'my.cnf' in conf and conf['my.cnf']:
+ properties['read_default_file'] = conf['my.cnf']
+
+ return properties or None
+
+ def is_valid_queries_dict(raw_queries, log_error):
+ """
+ :param raw_queries: dict:
+ :param log_error: function:
+ :return: dict or None
+
+ raw_queries is valid when: type <dict> and not empty after is_valid_query(for all queries)
+ """
+ def is_valid_query(query):
+ return all([isinstance(query, str),
+ query.startswith(('SELECT', 'select', 'SHOW', 'show'))])
+
+ if isinstance(raw_queries, dict) and raw_queries:
+ valid_queries = dict([(n, q) for n, q in raw_queries.items() if is_valid_query(q)])
+ bad_queries = set(raw_queries) - set(valid_queries)
+
+ if bad_queries:
+ log_error('Removed query(s): %s' % bad_queries)
+ return valid_queries
+ else:
+ log_error('Unsupported "queries" format. Must be not empty <dict>')
+ return None
+
+ if not PYMYSQL:
+ self.error('MySQLdb or PyMySQL module is needed to use mysql.chart.py plugin')
+ return False
+
+ # Check if "self.queries" exist, not empty and all queries are in valid format
+ self.queries = is_valid_queries_dict(self.queries, self.error)
+ if not self.queries:
+ return None
+
+ # Get connection properties
+ self.conn_properties = get_connection_properties(self.configuration)
+ if not self.conn_properties:
+ self.error('Connection properties are missing')
+ return False
+
+ # Create connection to the database
+ self.__connection, error = self.__connect()
+ if error:
+ self.error('Can\'t establish connection to MySQL: %s' % error)
+ return False
+
+ try:
+ data = self._get_data()
+ except Exception as error:
+ self.error('_get_data() failed. Error: %s' % error)
+ return False
+
+ if isinstance(data, dict) and data:
+ # We need this for create() method
+ self._data_from_check = data
+ return True
+ else:
+ self.error("_get_data() returned no data or type is not <dict>")
+ return False
+
+ def _get_raw_data(self, description=None):
+ """
+ Get raw data from MySQL server
+ :return: dict: fetchall() or (fetchall(), description)
+ """
+
+ if not self.__connection:
+ self.__connection, error = self.__connect()
+ if error:
+ return None
+
+ raw_data = dict()
+ try:
+ with self.__connection as cursor:
+ for name, query in self.queries.items():
+ try:
+ cursor.execute(query)
+ except (MySQLdb.ProgrammingError, MySQLdb.OperationalError) as error:
+ if exc_info()[0] == MySQLdb.OperationalError and 'denied' not in str(error):
+ raise RuntimeError
+ self.error('Removed query: %s[%s]. Error: %s'
+ % (name, query, error))
+ self.queries.pop(name)
+ continue
+ else:
+ raw_data[name] = (cursor.fetchall(), cursor.description) if description else cursor.fetchall()
+ self.__connection.commit()
+ except (MySQLdb.MySQLError, RuntimeError, TypeError, AttributeError):
+ self.__connection.close()
+ self.__connection = None
+ return None
+ else:
+ return raw_data or None
projects / netdata.git / blobdiff