X-Git-Url: https://arthur.barton.de/gitweb/?a=blobdiff_plain;f=plugins.d%2Fpython.d.plugin;h=b4e6473a6223f09764aa60449e07c73763c73378;hb=c59a9ed846a1669d801892feee4af79046b2618b;hp=a6b410167e91fea93dc629f304bdc075fd2fa897;hpb=b40b0803968d63966aa0cc2168680766a0eeb5b1;p=netdata.git diff --git a/plugins.d/python.d.plugin b/plugins.d/python.d.plugin index a6b41016..b4e6473a 100755 --- a/plugins.d/python.d.plugin +++ b/plugins.d/python.d.plugin @@ -1,4 +1,5 @@ -#!/usr/bin/env python +#!/usr/bin/env bash +'''':; exec "$(command -v python || command -v python3 || command -v python2 || echo "ERROR python IS NOT AVAILABLE IN THIS SYSTEM")" "$0" "$@" # ''' # -*- coding: utf-8 -*- # Description: netdata python modules supervisor @@ -8,6 +9,7 @@ import os import sys import time import threading +from re import sub # ----------------------------------------------------------------------------- # globals & environment setup @@ -27,6 +29,7 @@ sys.path.append(MODULES_DIR + "python_modules") PROGRAM = os.path.basename(__file__).replace(".plugin", "") DEBUG_FLAG = False +TRACE_FLAG = False OVERRIDE_UPDATE_EVERY = False # ----------------------------------------------------------------------------- @@ -36,11 +39,10 @@ import msg try: assert sys.version_info >= (3, 1) import importlib.machinery - + PY_VERSION = 3 # change this hack below if we want PY_VERSION to be used in modules # import builtins # builtins.PY_VERSION = 3 - PY_VERSION = 3 msg.info('Using python v3') except (AssertionError, ImportError): try: @@ -53,8 +55,15 @@ except (AssertionError, ImportError): msg.info('Using python v2') except ImportError: msg.fatal('Cannot start. No importlib.machinery on python3 or lack of imp on python2') +# try: +# import yaml +# except ImportError: +# msg.fatal('Cannot find yaml library') try: - import yaml + if PY_VERSION == 3: + import pyyaml3 as yaml + else: + import pyyaml2 as yaml except ImportError: msg.fatal('Cannot find yaml library') @@ -206,6 +215,8 @@ class PythonCharts(object): :param config: dict :return: dict """ + if config is None: + config = {} # get default values defaults = {} msg.debug(module.__name__ + ": reading configuration") @@ -280,11 +291,16 @@ class PythonCharts(object): :param reason: str """ prefix = job.__module__ - if job.name is not None: + if job.name is not None and len(job.name) != 0: prefix += "/" + job.name - prefix += ": " + try: + msg.error("DISABLED:", prefix) + self.jobs.remove(job) + except Exception as e: + msg.debug("This shouldn't happen. NO " + prefix + " IN LIST:" + str(self.jobs) + " ERROR: " + str(e)) - self.jobs.remove(job) + # TODO remove section below and remove `reason`. + prefix += ": " if reason is None: return elif reason[:3] == "no ": @@ -312,18 +328,40 @@ class PythonCharts(object): If job.check() fails job is stopped """ i = 0 + overridden = [] + msg.debug("all job objects", str(self.jobs)) while i < len(self.jobs): job = self.jobs[i] try: if not job.check(): - self._stop(job, "failed check") + msg.error(job.chart_name, "check() failed - disabling job") + self._stop(job) else: - msg.debug(job.chart_name, ": check succeeded") + msg.info("CHECKED OK:", job.chart_name) i += 1 - except AttributeError: - self._stop(job, "no check") + try: + if job.override_name is not None: + new_name = job.__module__ + '_' + sub(r'\s+', '_', job.override_name) + if new_name in overridden: + msg.info("DROPPED:", job.name, ", job '" + job.override_name + "' is already served by another job.") + self._stop(job) + i -= 1 + else: + job.name = job.override_name + msg.info("RENAMED:", new_name, ", from " + job.chart_name) + job.chart_name = new_name + overridden.append(job.chart_name) + except Exception: + pass + except AttributeError as e: + self._stop(job) + msg.error(job.chart_name, "cannot find check() function or it thrown unhandled exception.") + msg.debug(str(e)) except (UnboundLocalError, Exception) as e: - self._stop(job, "misbehaving. Reason:" + str(e)) + msg.error(job.chart_name, str(e)) + self._stop(job) + msg.debug("overridden job names:", str(overridden)) + msg.debug("all remaining job objects:", str(self.jobs)) def create(self): """ @@ -337,7 +375,8 @@ class PythonCharts(object): job = self.jobs[i] try: if not job.create(): - self._stop(job, "failed create") + msg.error(job.chart_name, "create function failed.") + self._stop(job) else: chart = job.chart_name sys.stdout.write( @@ -353,11 +392,17 @@ class PythonCharts(object): # sys.stdout.flush() i += 1 except AttributeError: - self._stop(job, "no create") + msg.error(job.chart_name, "cannot find create() function or it thrown unhandled exception.") + self._stop(job) except (UnboundLocalError, Exception) as e: - self._stop(job, "misbehaving. Reason: " + str(e)) + msg.error(job.chart_name, str(e)) + self._stop(job) def update(self): + """ + Creates and supervises every job thread. + This will stay forever and ever and ever forever and ever it'll be the one... + """ for job in self.jobs: job.start() @@ -392,7 +437,7 @@ def parse_cmdline(directory, *commands): :param commands: list of str :return: dict """ - global DEBUG_FLAG + global DEBUG_FLAG, TRACE_FLAG global OVERRIDE_UPDATE_EVERY global BASE_CONFIG @@ -404,8 +449,10 @@ def parse_cmdline(directory, *commands): elif cmd == "debug" or cmd == "all": DEBUG_FLAG = True # redirect stderr to stdout? + elif cmd == "trace" or cmd == "all": + TRACE_FLAG = True elif os.path.isfile(directory + cmd + ".chart.py") or os.path.isfile(directory + cmd): - DEBUG_FLAG = True + #DEBUG_FLAG = True mods.append(cmd.replace(".chart.py", "")) else: try: @@ -427,13 +474,15 @@ def run(): """ Main program. """ - global DEBUG_FLAG, BASE_CONFIG + global DEBUG_FLAG, TRACE_FLAG, BASE_CONFIG # read configuration file disabled = [] configfile = CONFIG_DIR + "python.d.conf" msg.PROGRAM = PROGRAM msg.info("reading configuration file:", configfile) + log_throttle = 200 + log_interval = 3600 conf = read_config(configfile) if conf is not None: @@ -443,15 +492,33 @@ def run(): msg.fatal('disabled in configuration file.\n') except (KeyError, TypeError): pass + try: for param in BASE_CONFIG: BASE_CONFIG[param] = conf[param] except (KeyError, TypeError): pass # use default update_every from NETDATA_UPDATE_EVERY + try: DEBUG_FLAG = conf['debug'] except (KeyError, TypeError): pass + + try: + TRACE_FLAG = conf['trace'] + except (KeyError, TypeError): + pass + + try: + log_throttle = conf['logs_per_interval'] + except (KeyError, TypeError): + pass + + try: + log_interval = conf['log_interval'] + except (KeyError, TypeError): + pass + for k, v in conf.items(): if k in ("update_every", "debug", "enabled"): continue @@ -461,6 +528,11 @@ def run(): # parse passed command line arguments modules = parse_cmdline(MODULES_DIR, *sys.argv) msg.DEBUG_FLAG = DEBUG_FLAG + msg.TRACE_FLAG = TRACE_FLAG + msg.LOG_THROTTLE = log_throttle + msg.LOG_INTERVAL = log_interval + msg.LOG_COUNTER = 0 + msg.LOG_NEXT_CHECK = 0 msg.info("MODULES_DIR='" + MODULES_DIR + "', CONFIG_DIR='" + CONFIG_DIR + "', UPDATE_EVERY=" + str(BASE_CONFIG['update_every']) +