Merge pull request #1367 from vlvkobal/master

[netdata.git] / plugins.d / python.d.plugin

diff --git a/plugins.d/python.d.plugin b/plugins.d/python.d.plugin
index efccd3c10dc9435b698e39c08284de8a540c9ce3..05d23b9ae09f98ab8b165d929b3214fe6f2d9060 100755 (executable)
--- a/plugins.d/python.d.plugin
+++ b/plugins.d/python.d.plugin
@@ -28,6 +28,7 @@ sys.path.append(MODULES_DIR + "python_modules")
 
 PROGRAM = os.path.basename(__file__).replace(".plugin", "")
 DEBUG_FLAG = False
+TRACE_FLAG = False
 OVERRIDE_UPDATE_EVERY = False
 
 # -----------------------------------------------------------------------------
@@ -37,11 +38,10 @@ import msg
 try:
     assert sys.version_info >= (3, 1)
     import importlib.machinery
-
+    PY_VERSION = 3
     # change this hack below if we want PY_VERSION to be used in modules
     # import builtins
     # builtins.PY_VERSION = 3
-    PY_VERSION = 3
     msg.info('Using python v3')
 except (AssertionError, ImportError):
     try:
@@ -54,8 +54,15 @@ except (AssertionError, ImportError):
         msg.info('Using python v2')
     except ImportError:
         msg.fatal('Cannot start. No importlib.machinery on python3 or lack of imp on python2')
+# try:
+#     import yaml
+# except ImportError:
+#     msg.fatal('Cannot find yaml library')
 try:
-    import yaml
+    if PY_VERSION == 3:
+        import pyyaml3 as yaml
+    else:
+        import pyyaml2 as yaml
 except ImportError:
     msg.fatal('Cannot find yaml library')
 
@@ -286,8 +293,8 @@ class PythonCharts(object):
         if job.name is not None and len(job.name) != 0:
             prefix += "/" + job.name
         try:
+            msg.error("DISABLED:", prefix)
             self.jobs.remove(job)
-            msg.info("Disabled", prefix)
         except Exception as e:
             msg.debug("This shouldn't happen. NO " + prefix + " IN LIST:" + str(self.jobs) + " ERROR: " + str(e))
 
@@ -326,28 +333,29 @@ class PythonCharts(object):
             job = self.jobs[i]
             try:
                 if not job.check():
-                    msg.error(job.chart_name, "check function failed.")
+                    msg.error(job.chart_name, "check() failed - disabling job")
                     self._stop(job)
                 else:
-                    msg.debug(job.chart_name, "check succeeded")
+                    msg.info("CHECKED OK:", job.chart_name)
                     i += 1
                     try:
                         if job.override_name is not None:
                             new_name = job.__module__ + '_' + job.override_name
                             if new_name in overridden:
-                                msg.error(job.chart_name + " already exists. Stopping '" + job.name + "'")
+                                msg.info("DROPPED:", job.name, ", job '" + job.override_name + "' is already served by another job.")
                                 self._stop(job)
                                 i -= 1
                             else:
                                 job.name = job.override_name
-                                msg.debug(job.chart_name + " changing chart name to: '" + new_name + "'")
+                                msg.info("RENAMED:", new_name, ", from " + job.chart_name)
                                 job.chart_name = new_name
                                 overridden.append(job.chart_name)
                     except Exception:
                         pass
-            except AttributeError:
+            except AttributeError as e:
                 self._stop(job)
-                msg.error(job.chart_name, "cannot find check() function.")
+                msg.error(job.chart_name, "cannot find check() function or it thrown unhandled exception.")
+                msg.debug(str(e))
             except (UnboundLocalError, Exception) as e:
                 msg.error(job.chart_name, str(e))
                 self._stop(job)
@@ -383,7 +391,7 @@ class PythonCharts(object):
                     # sys.stdout.flush()
                     i += 1
             except AttributeError:
-                msg.error(job.chart_name, "cannot find create() function.")
+                msg.error(job.chart_name, "cannot find create() function or it thrown unhandled exception.")
                 self._stop(job)
             except (UnboundLocalError, Exception) as e:
                 msg.error(job.chart_name, str(e))
@@ -428,7 +436,7 @@ def parse_cmdline(directory, *commands):
     :param commands: list of str
     :return: dict
     """
-    global DEBUG_FLAG
+    global DEBUG_FLAG, TRACE_FLAG
     global OVERRIDE_UPDATE_EVERY
     global BASE_CONFIG
 
@@ -440,6 +448,8 @@ def parse_cmdline(directory, *commands):
         elif cmd == "debug" or cmd == "all":
             DEBUG_FLAG = True
             # redirect stderr to stdout?
+        elif cmd == "trace" or cmd == "all":
+            TRACE_FLAG = True
         elif os.path.isfile(directory + cmd + ".chart.py") or os.path.isfile(directory + cmd):
             #DEBUG_FLAG = True
             mods.append(cmd.replace(".chart.py", ""))
@@ -463,13 +473,15 @@ def run():
     """
     Main program.
     """
-    global DEBUG_FLAG, BASE_CONFIG
+    global DEBUG_FLAG, TRACE_FLAG, BASE_CONFIG
 
     # read configuration file
     disabled = []
     configfile = CONFIG_DIR + "python.d.conf"
     msg.PROGRAM = PROGRAM
     msg.info("reading configuration file:", configfile)
+    log_throttle = 200
+    log_interval = 3600
 
     conf = read_config(configfile)
     if conf is not None:
@@ -479,15 +491,33 @@ def run():
                 msg.fatal('disabled in configuration file.\n')
         except (KeyError, TypeError):
             pass
+
         try:
             for param in BASE_CONFIG:
                 BASE_CONFIG[param] = conf[param]
         except (KeyError, TypeError):
             pass  # use default update_every from NETDATA_UPDATE_EVERY
+
         try:
             DEBUG_FLAG = conf['debug']
         except (KeyError, TypeError):
             pass
+
+        try:
+            TRACE_FLAG = conf['trace']
+        except (KeyError, TypeError):
+            pass
+
+        try:
+            log_throttle = conf['logs_per_interval']
+        except (KeyError, TypeError):
+            pass
+
+        try:
+            log_interval = conf['log_interval']
+        except (KeyError, TypeError):
+            pass
+
         for k, v in conf.items():
             if k in ("update_every", "debug", "enabled"):
                 continue
@@ -497,6 +527,11 @@ def run():
     # parse passed command line arguments
     modules = parse_cmdline(MODULES_DIR, *sys.argv)
     msg.DEBUG_FLAG = DEBUG_FLAG
+    msg.TRACE_FLAG = TRACE_FLAG
+    msg.LOG_THROTTLE = log_throttle
+    msg.LOG_INTERVAL = log_interval
+    msg.LOG_COUNTER = 0
+    msg.LOG_NEXT_CHECK = 0
     msg.info("MODULES_DIR='" + MODULES_DIR +
              "', CONFIG_DIR='" + CONFIG_DIR +
              "', UPDATE_EVERY=" + str(BASE_CONFIG['update_every']) +