Merge pull request #591 from paulfantom/master

Fixing python

diff --git a/conf.d/python.d.conf b/conf.d/python.d.conf
index b467221116534cd753dbfae008c03284c0345ee0..e8f6e8923dba67b4c3bd48d06f076f45dbc23120 100644 (file)
--- a/conf.d/python.d.conf
+++ b/conf.d/python.d.conf
@@ -1,5 +1,8 @@
 # This is the configuration for python.d.plugin
 
+# Disable all python modules
+enabled: no
+
 # By default python.d.plugin enables all modules stored in python.d
 # Modules can be disabled with setting "module_name = no"
 example: no

diff --git a/plugins.d/python.d.plugin b/plugins.d/python.d.plugin
index db5afccb62eb41358f8a912318ee61141938f1be..a16e1b8ccfcdeece46becdc4498ea8984ba9c044 100755 (executable)
--- a/plugins.d/python.d.plugin
+++ b/plugins.d/python.d.plugin
@@ -8,79 +8,80 @@ import os
 import sys
 import time
 
-
+# -----------------------------------------------------------------------------
+# globals & environment setup
+# https://github.com/firehol/netdata/wiki/External-Plugins#environment-variables
 MODULE_EXTENSION = ".chart.py"
-BASE_CONFIG = {'update_every': 1,
+BASE_CONFIG = {'update_every': os.getenv('NETDATA_UPDATE_EVERY', 1),
                'priority': 90000,
                'retries': 10}
 
-
-# -----------------------------------------------------------------------------
-# logging functions
+MODULES_DIR = os.path.abspath(os.getenv('NETDATA_PLUGINS_DIR',
+                              os.path.dirname(__file__)) + "/../python.d") + "/"
+CONFIG_DIR = os.getenv('NETDATA_CONFIG_DIR', "/etc/netdata/")
+# directories should end with '/'
+if CONFIG_DIR[-1] != "/":
+    CONFIG_DIR += "/"
+sys.path.append(MODULES_DIR + "python_modules")
 
 PROGRAM = os.path.basename(__file__).replace(".plugin", "")
 DEBUG_FLAG = False
+OVERRIDE_UPDATE_EVERY = False
 
-def debug(*args):
+
+# -----------------------------------------------------------------------------
+# logging functions
+def log_msg(msg_type, *args):
     """
     Print message on stderr.
+    :param msg_type: str
     """
-    if not DEBUG_FLAG:
-        return
-    sys.stderr.write(PROGRAM + " DEBUG :")
+    sys.stderr.write(PROGRAM)
+    sys.stderr.write(" ")
+    sys.stderr.write(msg_type)
+    sys.stderr.write(": ")
     for i in args:
-        sys.stderr.write(" " + str(i))
+        sys.stderr.write(" ")
+        sys.stderr.write(str(i))
     sys.stderr.write("\n")
     sys.stderr.flush()
 
+
+def debug(*args):
+    """
+    Print debug message on stderr.
+    """
+    if not DEBUG_FLAG:
+        return
+
+    log_msg("DEBUG", *args)
+
+
 def error(*args):
     """
     Print message on stderr.
     """
-    sys.stderr.write(PROGRAM + " ERROR :")
-    for i in args:
-        sys.stderr.write(" " + str(i))
-    sys.stderr.write("\n")
-    sys.stderr.flush()
+    log_msg("ERROR", *args)
+
 
 def info(*args):
     """
     Print message on stderr.
     """
-    sys.stderr.write(PROGRAM + " INFO :")
-    for i in args:
-        sys.stderr.write(" " + str(i))
-    sys.stderr.write("\n")
-    sys.stderr.flush()
+    log_msg("INFO", *args)
+
 
 def fatal(*args):
     """
     Print message on stderr and exit.
     """
-    sys.stderr.write(PROGRAM + " FATAL :")
-    for i in args:
-        sys.stderr.write(" " + str(i))
-    sys.stderr.write("\n")
-    sys.stderr.flush()
+    log_msg("FATAL", *args)
     sys.stdout.write('DISABLE\n')
     sys.exit(1)
 
 
-
 # -----------------------------------------------------------------------------
-# globals & python modules management
-
-# setup environment
-# https://github.com/firehol/netdata/wiki/External-Plugins#environment-variables
-MODULES_DIR = os.path.abspath(os.getenv('NETDATA_PLUGINS_DIR',
-                        os.path.dirname(__file__)) + "/../python.d") + "/"
-CONFIG_DIR = os.getenv('NETDATA_CONFIG_DIR', "/etc/netdata/")
-UPDATE_EVERY = os.getenv('NETDATA_UPDATE_EVERY', 1)
-# directories should end with '/'
-if CONFIG_DIR[-1] != "/":
-    CONFIG_DIR += "/"
-sys.path.append(MODULES_DIR + "python_modules")
-
+# third party and version specific python modules management
 try:
     assert sys.version_info >= (3, 1)
     import importlib.machinery
@@ -106,18 +107,17 @@ try:
 except ImportError:
     fatal('Cannot find yaml library')
 
+
 class PythonCharts(object):
     """
     Main class used to control every python module.
     """
     def __init__(self,
-                 update_every=None,
                  modules=None,
                  modules_path='../python.d/',
                  modules_configs='../conf.d/',
                  modules_disabled=None):
         """
-        :param update_every: int
         :param modules: list
         :param modules_path: str
         :param modules_configs: str
@@ -141,9 +141,11 @@ class PythonCharts(object):
 
         # good economy and prosperity:
         self.jobs = self._create_jobs(configured_modules)  # type: list
-        if DEBUG_FLAG and update_every is not None:
+
+        # enable timetable override like `python.d.plugin mysql debug 1`
+        if DEBUG_FLAG and OVERRIDE_UPDATE_EVERY:
             for job in self.jobs:
-                job.create_timetable(update_every)
+                job.create_timetable(BASE_CONFIG['update_every'])
 
     @staticmethod
     def _import_module(path, name=None):
@@ -165,7 +167,7 @@ class PythonCharts(object):
             else:
                 return imp.load_source(name, path)
         except Exception as e:
-            debug(str(e))
+            debug("Problem loading", name, str(e))
             return None
 
     def _load_modules(self, path, modules, disabled):
@@ -227,6 +229,7 @@ class PythonCharts(object):
                 error(mod.__name__ + ": configuration file '" + configfile + "' not found. Using defaults.")
                 # set config if not found
                 if not hasattr(mod, 'config'):
+                    debug(mod.__name__ + ": setting configuration for only one job")
                     mod.config = {None: {}}
                     for var in BASE_CONFIG:
                         try:
@@ -252,6 +255,7 @@ class PythonCharts(object):
         # get default values
         defaults = {}
         for key in BASE_CONFIG:
+            debug(module.__name__ + ": reading configuration")
             try:
                 # get defaults from module config
                 defaults[key] = int(config.pop(key))
@@ -360,11 +364,12 @@ class PythonCharts(object):
                 if not job.check():
                     self._stop(job, "failed check")
                 else:
+                    debug(job.name, ": check succeeded")
                     i += 1
             except AttributeError:
                 self._stop(job, "no check")
             except (UnboundLocalError, Exception) as e:
-                self._stop(job, "misbehaving. Reason: " + str(e))
+                self._stop(job, "misbehaving. Reason:" + str(e))
 
     def create(self):
         """
@@ -390,6 +395,7 @@ class PythonCharts(object):
                         str(job.timetable['freq']) +
                         '\n')
                     sys.stdout.write("DIMENSION run_time 'run time' absolute 1 1\n\n")
+                    debug("created charts for", job.chart_name)
                     # sys.stdout.flush()
                     i += 1
             except AttributeError:
@@ -412,6 +418,7 @@ class PythonCharts(object):
         t_start = time.time()
         # check if it is time to execute job update() function
         if job.timetable['next'] > t_start:
+            debug("it is not a time to invoke update on", job.chart_name)
             return True
         try:
             if self.first_run:
@@ -434,8 +441,10 @@ class PythonCharts(object):
         t_end = time.time()
         job.timetable['next'] = t_end - (t_end % job.timetable['freq']) + job.timetable['freq']
         # draw performance graph
+        run_time = str(int((t_end - t_start) * 1000))
+        debug(job.chart_name, "updated in", run_time)
         sys.stdout.write("BEGIN netdata.plugin_pythond_" + job.chart_name + " " + str(since_last) + '\n')
-        sys.stdout.write("SET run_time = " + str(int((t_end - t_start) * 1000)) + '\n')
+        sys.stdout.write("SET run_time = " + run_time + '\n')
         sys.stdout.write("END\n")
         # sys.stdout.flush()
         job.timetable['last'] = t_start
@@ -491,8 +500,8 @@ def parse_cmdline(directory, *commands):
     :return: dict
     """
     global DEBUG_FLAG
-    global UPDATE_EVERY
-    update_every = UPDATE_EVERY
+    global OVERRIDE_UPDATE_EVERY
+    global BASE_CONFIG
 
     mods = []
     for cmd in commands[1:]:
@@ -506,14 +515,15 @@ def parse_cmdline(directory, *commands):
             mods.append(cmd.replace(".chart.py", ""))
         else:
             try:
-                update_every = int(cmd)
+                BASE_CONFIG['update_every'] = int(cmd)
+                OVERRIDE_UPDATE_EVERY = True
+                debug(PROGRAM, "overriding update interval to", str(int(cmd)))
             except ValueError:
-                update_every = UPDATE_EVERY
+                pass
 
     debug("started from", commands[0], "with options:", *commands[1:])
 
-    UPDATE_EVERY = update_every
-    return {'modules': mods}
+    return mods
 
 
 # if __name__ == '__main__':
@@ -521,24 +531,24 @@ def run():
     """
     Main program.
     """
-    global PROGRAM, DEBUG_FLAG
-    PROGRAM = sys.argv[0].split('/')[-1].split('.plugin')[0]
+    global DEBUG_FLAG, BASE_CONFIG
 
     # read configuration file
     disabled = []
     configfile = CONFIG_DIR + "python.d.conf"
+    debug(PROGRAM, "reading configuration file:", configfile)
 
-    update_every = UPDATE_EVERY
     conf = read_config(configfile)
     if conf is not None:
         try:
-            # FIXME: is this right? exit the whole plugin?
-            if str(conf['enable']) is False:
+            # exit the whole plugin when 'enabled: no' is set in 'python.d.conf'
+            if conf['enabled'] is False:
                 fatal('disabled in configuration file.\n')
         except (KeyError, TypeError):
             pass
         try:
-            update_every = conf['update_every']
+            for param in BASE_CONFIG:
+                BASE_CONFIG[param] = conf[param]
         except (KeyError, TypeError):
             pass  # use default update_every from NETDATA_UPDATE_EVERY
         try:
@@ -546,18 +556,20 @@ def run():
         except (KeyError, TypeError):
             pass
         for k, v in conf.items():
-            if k in ("update_every", "debug", "enable"):
+            if k in ("update_every", "debug", "enabled"):
                 continue
             if v is False:
                 disabled.append(k)
 
     # parse passed command line arguments
-    out = parse_cmdline(MODULES_DIR, *sys.argv)
-    modules = out['modules']
-    info("MODULES_DIR='" + MODULES_DIR + "', CONFIG_DIR='" + CONFIG_DIR + "', UPDATE_EVERY=" + str(UPDATE_EVERY) + ", ONLY_MODULES=" + str(out['modules']))
+    modules = parse_cmdline(MODULES_DIR, *sys.argv)
+    info("MODULES_DIR='" + MODULES_DIR +
+         "', CONFIG_DIR='" + CONFIG_DIR +
+         "', UPDATE_EVERY=" + str(BASE_CONFIG['update_every']) +
+         ", ONLY_MODULES=" + str(modules))
 
     # run plugins
-    charts = PythonCharts(update_every, modules, MODULES_DIR, CONFIG_DIR + "python.d/", disabled)
+    charts = PythonCharts(modules, MODULES_DIR, CONFIG_DIR + "python.d/", disabled)
     charts.check()
     charts.create()
     charts.update()

diff --git a/python.d/mysql.chart.py b/python.d/mysql.chart.py
index 22aa8f5eacc8e13a833809fe03fbcb4bc0452590..e43a72c4dd4f5d96e70dbada59b453ea179ab698 100644 (file)
--- a/python.d/mysql.chart.py
+++ b/python.d/mysql.chart.py
@@ -4,6 +4,7 @@
 
 import os
 import sys
+from base import BaseService
 
 NAME = os.path.basename(__file__).replace(".chart.py", "")
 
@@ -23,8 +24,6 @@ except ImportError:
         sys.stderr.write(NAME + ": You need to install MySQLdb or PyMySQL module to use mysql.chart.py plugin\n")
         raise ImportError
 
-from base import BaseService
-
 # default module values (can be overridden per job in `config`)
 update_every = 3
 priority = 90000
@@ -384,7 +383,7 @@ class Service(BaseService):
                                               unix_socket=self.configuration['socket'],
                                               host=self.configuration['host'],
                                               port=self.configuration['port'],
-                                              connect_timeout=self.configuration['update_every'])
+                                              connect_timeout=self.update_every)
         except Exception as e:
             self.error(NAME + " has problem connecting to server:", e)
             raise RuntimeError

diff --git a/python.d/python_modules/base.py b/python.d/python_modules/base.py
index 573700061e0a863acdb8ac1cd306927dabcacef8..a60afb26f2550e5d71d72931935ef8b18ec1589c 100644 (file)
--- a/python.d/python_modules/base.py
+++ b/python.d/python_modules/base.py
@@ -36,10 +36,11 @@ class BaseService(object):
                       'retries':0}
         :param config: dict
         """
-        self.update_every = int(config['update_every'])
-        self.priority = int(config['priority'])
-        self.retries = int(config['retries'])
+        self.update_every = int(config.pop('update_every'))
+        self.priority = int(config.pop('priority'))
+        self.retries = int(config.pop('retries'))
         self.retries_left = self.retries
+        self.configuration = config
 
     def create_timetable(self, freq=None):
         """