X-Git-Url: https://arthur.barton.de/gitweb/?a=blobdiff_plain;f=plugins.d%2Fpython.d.plugin;h=efa62cbc5a34affbd86651147a50e08871ff7e0f;hb=HEAD;hp=741ba1f7378a28c8010633b00f9de4964fb0f592;hpb=cd2deef34b42b8f1326493c2c40f71d2ebeaec24;p=netdata.git diff --git a/plugins.d/python.d.plugin b/plugins.d/python.d.plugin index 741ba1f7..efa62cbc 100755 --- a/plugins.d/python.d.plugin +++ b/plugins.d/python.d.plugin @@ -1,4 +1,5 @@ -#!/usr/bin/env python +#!/usr/bin/env bash +'''':; exec "$(command -v python || command -v python3 || command -v python2 || echo "ERROR python IS NOT AVAILABLE IN THIS SYSTEM")" "$0" "$@" # ''' # -*- coding: utf-8 -*- # Description: netdata python modules supervisor @@ -8,6 +9,7 @@ import os import sys import time import threading +from re import sub # ----------------------------------------------------------------------------- # globals & environment setup @@ -27,6 +29,7 @@ sys.path.append(MODULES_DIR + "python_modules") PROGRAM = os.path.basename(__file__).replace(".plugin", "") DEBUG_FLAG = False +TRACE_FLAG = False OVERRIDE_UPDATE_EVERY = False # ----------------------------------------------------------------------------- @@ -36,11 +39,10 @@ import msg try: assert sys.version_info >= (3, 1) import importlib.machinery - + PY_VERSION = 3 # change this hack below if we want PY_VERSION to be used in modules # import builtins # builtins.PY_VERSION = 3 - PY_VERSION = 3 msg.info('Using python v3') except (AssertionError, ImportError): try: @@ -53,11 +55,46 @@ except (AssertionError, ImportError): msg.info('Using python v2') except ImportError: msg.fatal('Cannot start. No importlib.machinery on python3 or lack of imp on python2') +# try: +# import yaml +# except ImportError: +# msg.fatal('Cannot find yaml library') try: - import yaml + if PY_VERSION == 3: + import pyyaml3 as yaml + else: + import pyyaml2 as yaml except ImportError: msg.fatal('Cannot find yaml library') +try: + from collections import OrderedDict + ORDERED = True + DICT = OrderedDict + msg.info('YAML output is ordered') +except ImportError: + try: + from ordereddict import OrderedDict + ORDERED = True + DICT = OrderedDict + msg.info('YAML output is ordered') + except ImportError: + ORDERED = False + DICT = dict + msg.info('YAML output is unordered') +if ORDERED: + def ordered_load(stream, Loader=yaml.Loader, object_pairs_hook=OrderedDict): + class OrderedLoader(Loader): + pass + + def construct_mapping(loader, node): + loader.flatten_mapping(node) + return object_pairs_hook(loader.construct_pairs(node)) + OrderedLoader.add_constructor( + yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, + construct_mapping) + return yaml.load(stream, OrderedLoader) + class PythonCharts(object): """ @@ -68,12 +105,16 @@ class PythonCharts(object): modules=None, modules_path='../python.d/', modules_configs='../conf.d/', - modules_disabled=None): + modules_disabled=None, + modules_enabled=None, + default_run=None): """ :param modules: list :param modules_path: str :param modules_configs: str :param modules_disabled: list + :param modules_enabled: list + :param default_run: bool """ if modules is None: @@ -86,13 +127,13 @@ class PythonCharts(object): self.configs = modules_configs # load modules - loaded_modules = self._load_modules(modules_path, modules, modules_disabled) + loaded_modules = self._load_modules(modules_path, modules, modules_disabled, modules_enabled, default_run) # load configuration files configured_modules = self._load_configs(loaded_modules) # good economy and prosperity: - self.jobs = self._create_jobs(configured_modules) # type: list + self.jobs = self._create_jobs(configured_modules) # type # enable timetable override like `python.d.plugin mysql debug 1` if DEBUG_FLAG and OVERRIDE_UPDATE_EVERY: @@ -122,7 +163,7 @@ class PythonCharts(object): msg.error("Problem loading", name, str(e)) return None - def _load_modules(self, path, modules, disabled): + def _load_modules(self, path, modules, disabled, enabled, default_run): """ Load modules from 'modules' list or dynamically every file from 'path' (only .chart.py files) :param path: str @@ -148,7 +189,10 @@ class PythonCharts(object): msg.fatal('no modules found.') else: # scan directory specified in path and load all modules from there - names = os.listdir(path) + if default_run is False: + names = [module for module in os.listdir(path) if module[:-9] in enabled] + else: + names = os.listdir(path) for mod in names: if mod.replace(MODULE_EXTENSION, "") in disabled: msg.error(mod + ": disabled module ", mod.replace(MODULE_EXTENSION, "")) @@ -206,6 +250,8 @@ class PythonCharts(object): :param config: dict :return: dict """ + if config is None: + config = {} # get default values defaults = {} msg.debug(module.__name__ + ": reading configuration") @@ -224,7 +270,7 @@ class PythonCharts(object): # check if there are dict in config dict many_jobs = False for name in config: - if type(config[name]) is dict: + if isinstance(config[name], DICT): many_jobs = True break @@ -280,11 +326,16 @@ class PythonCharts(object): :param reason: str """ prefix = job.__module__ - if job.name is not None: + if job.name is not None and len(job.name) != 0: prefix += "/" + job.name - prefix += ": " + try: + msg.error("DISABLED:", prefix) + self.jobs.remove(job) + except Exception as e: + msg.debug("This shouldn't happen. NO " + prefix + " IN LIST:" + str(self.jobs) + " ERROR: " + str(e)) - self.jobs.remove(job) + # TODO remove section below and remove `reason`. + prefix += ": " if reason is None: return elif reason[:3] == "no ": @@ -313,29 +364,40 @@ class PythonCharts(object): """ i = 0 overridden = [] + msg.debug("all job objects", str(self.jobs)) while i < len(self.jobs): job = self.jobs[i] - if job.name in overridden: - self._stop(job) - msg.error(job.name + " already exists") try: if not job.check(): - self._stop(job, "failed check") + msg.error(job.chart_name, "check() failed - disabling job") + self._stop(job) else: - msg.debug(job.chart_name, ": check succeeded") + msg.info("CHECKED OK:", job.chart_name) i += 1 try: if job.override_name is not None: - job.name = job.override_name - msg.debug(job.chart_name + " changing chart name to: " + job.__module__ + job.name) - job.chart_name = job.__module__ + job.name - overridden.append(job.name) + new_name = job.__module__ + '_' + sub(r'\s+', '_', job.override_name) + if new_name in overridden: + msg.info("DROPPED:", job.name, ", job '" + job.override_name + + "' is already served by another job.") + self._stop(job) + i -= 1 + else: + job.name = job.override_name + msg.info("RENAMED:", new_name, ", from " + job.chart_name) + job.chart_name = new_name + overridden.append(job.chart_name) except Exception: pass - except AttributeError: - self._stop(job, "no check") + except AttributeError as e: + self._stop(job) + msg.error(job.chart_name, "cannot find check() function or it thrown unhandled exception.") + msg.debug(str(e)) except (UnboundLocalError, Exception) as e: - self._stop(job, "misbehaving. Reason:" + str(e)) + msg.error(job.chart_name, str(e)) + self._stop(job) + msg.debug("overridden job names:", str(overridden)) + msg.debug("all remaining job objects:", str(self.jobs)) def create(self): """ @@ -349,7 +411,8 @@ class PythonCharts(object): job = self.jobs[i] try: if not job.create(): - self._stop(job, "failed create") + msg.error(job.chart_name, "create function failed.") + self._stop(job) else: chart = job.chart_name sys.stdout.write( @@ -365,9 +428,11 @@ class PythonCharts(object): # sys.stdout.flush() i += 1 except AttributeError: - self._stop(job, "no create") + msg.error(job.chart_name, "cannot find create() function or it thrown unhandled exception.") + self._stop(job) except (UnboundLocalError, Exception) as e: - self._stop(job, "misbehaving. Reason: " + str(e)) + msg.error(job.chart_name, str(e)) + self._stop(job) def update(self): """ @@ -391,7 +456,10 @@ def read_config(path): """ try: with open(path, 'r') as stream: - config = yaml.load(stream) + if ORDERED: + config = ordered_load(stream, yaml.SafeLoader) + else: + config = yaml.load(stream) except (OSError, IOError): msg.error(str(path), "is not a valid configuration file") return None @@ -408,7 +476,7 @@ def parse_cmdline(directory, *commands): :param commands: list of str :return: dict """ - global DEBUG_FLAG + global DEBUG_FLAG, TRACE_FLAG global OVERRIDE_UPDATE_EVERY global BASE_CONFIG @@ -420,8 +488,10 @@ def parse_cmdline(directory, *commands): elif cmd == "debug" or cmd == "all": DEBUG_FLAG = True # redirect stderr to stdout? + elif cmd == "trace" or cmd == "all": + TRACE_FLAG = True elif os.path.isfile(directory + cmd + ".chart.py") or os.path.isfile(directory + cmd): - DEBUG_FLAG = True + # DEBUG_FLAG = True mods.append(cmd.replace(".chart.py", "")) else: try: @@ -443,13 +513,17 @@ def run(): """ Main program. """ - global DEBUG_FLAG, BASE_CONFIG + global DEBUG_FLAG, TRACE_FLAG, BASE_CONFIG # read configuration file - disabled = [] + disabled = ['nginx_log', 'gunicorn_log'] + enabled = list() + default_run = True configfile = CONFIG_DIR + "python.d.conf" msg.PROGRAM = PROGRAM msg.info("reading configuration file:", configfile) + log_throttle = 200 + log_interval = 3600 conf = read_config(configfile) if conf is not None: @@ -459,31 +533,59 @@ def run(): msg.fatal('disabled in configuration file.\n') except (KeyError, TypeError): pass + try: for param in BASE_CONFIG: BASE_CONFIG[param] = conf[param] except (KeyError, TypeError): pass # use default update_every from NETDATA_UPDATE_EVERY + try: DEBUG_FLAG = conf['debug'] except (KeyError, TypeError): pass + + try: + TRACE_FLAG = conf['trace'] + except (KeyError, TypeError): + pass + + try: + log_throttle = conf['logs_per_interval'] + except (KeyError, TypeError): + pass + + try: + log_interval = conf['log_interval'] + except (KeyError, TypeError): + pass + + default_run = True if ('default_run' not in conf or conf.get('default_run')) else False + for k, v in conf.items(): - if k in ("update_every", "debug", "enabled"): + if k in ("update_every", "debug", "enabled", "default_run"): continue - if v is False: - disabled.append(k) - + if default_run: + if v is False: + disabled.append(k) + else: + if v is True: + enabled.append(k) # parse passed command line arguments modules = parse_cmdline(MODULES_DIR, *sys.argv) msg.DEBUG_FLAG = DEBUG_FLAG + msg.TRACE_FLAG = TRACE_FLAG + msg.LOG_THROTTLE = log_throttle + msg.LOG_INTERVAL = log_interval + msg.LOG_COUNTER = 0 + msg.LOG_NEXT_CHECK = 0 msg.info("MODULES_DIR='" + MODULES_DIR + "', CONFIG_DIR='" + CONFIG_DIR + "', UPDATE_EVERY=" + str(BASE_CONFIG['update_every']) + ", ONLY_MODULES=" + str(modules)) # run plugins - charts = PythonCharts(modules, MODULES_DIR, CONFIG_DIR + "python.d/", disabled) + charts = PythonCharts(modules, MODULES_DIR, CONFIG_DIR + "python.d/", disabled, enabled, default_run) charts.check() charts.create() charts.update()