2 '''':; exec "$(command -v python || command -v python3 || command -v python2 || echo "ERROR python IS NOT AVAILABLE IN THIS SYSTEM")" "$0" "$@" # '''
3 # -*- coding: utf-8 -*-
5 # Description: netdata python modules supervisor
6 # Author: Pawel Krupa (paulfantom)
13 # -----------------------------------------------------------------------------
14 # globals & environment setup
15 # https://github.com/firehol/netdata/wiki/External-Plugins#environment-variables
16 MODULE_EXTENSION = ".chart.py"
17 BASE_CONFIG = {'update_every': os.getenv('NETDATA_UPDATE_EVERY', 1),
21 MODULES_DIR = os.path.abspath(os.getenv('NETDATA_PLUGINS_DIR',
22 os.path.dirname(__file__)) + "/../python.d") + "/"
23 CONFIG_DIR = os.getenv('NETDATA_CONFIG_DIR', "/etc/netdata/")
24 # directories should end with '/'
25 if CONFIG_DIR[-1] != "/":
27 sys.path.append(MODULES_DIR + "python_modules")
29 PROGRAM = os.path.basename(__file__).replace(".plugin", "")
32 OVERRIDE_UPDATE_EVERY = False
34 # -----------------------------------------------------------------------------
35 # custom, third party and version specific python modules management
39 assert sys.version_info >= (3, 1)
40 import importlib.machinery
42 # change this hack below if we want PY_VERSION to be used in modules
44 # builtins.PY_VERSION = 3
45 msg.info('Using python v3')
46 except (AssertionError, ImportError):
50 # change this hack below if we want PY_VERSION to be used in modules
52 # __builtin__.PY_VERSION = 2
54 msg.info('Using python v2')
56 msg.fatal('Cannot start. No importlib.machinery on python3 or lack of imp on python2')
60 # msg.fatal('Cannot find yaml library')
63 import pyyaml3 as yaml
65 import pyyaml2 as yaml
67 msg.fatal('Cannot find yaml library')
70 class PythonCharts(object):
72 Main class used to control every python module.
77 modules_path='../python.d/',
78 modules_configs='../conf.d/',
79 modules_disabled=None):
82 :param modules_path: str
83 :param modules_configs: str
84 :param modules_disabled: list
89 if modules_disabled is None:
93 # set configuration directory
94 self.configs = modules_configs
97 loaded_modules = self._load_modules(modules_path, modules, modules_disabled)
99 # load configuration files
100 configured_modules = self._load_configs(loaded_modules)
102 # good economy and prosperity:
103 self.jobs = self._create_jobs(configured_modules) # type: list
105 # enable timetable override like `python.d.plugin mysql debug 1`
106 if DEBUG_FLAG and OVERRIDE_UPDATE_EVERY:
107 for job in self.jobs:
108 job.create_timetable(BASE_CONFIG['update_every'])
111 def _import_module(path, name=None):
113 Try to import module using only its path.
120 name = path.split('/')[-1]
121 if name[-len(MODULE_EXTENSION):] != MODULE_EXTENSION:
123 name = name[:-len(MODULE_EXTENSION)]
126 return importlib.machinery.SourceFileLoader(name, path).load_module()
128 return imp.load_source(name, path)
129 except Exception as e:
130 msg.error("Problem loading", name, str(e))
133 def _load_modules(self, path, modules, disabled):
135 Load modules from 'modules' list or dynamically every file from 'path' (only .chart.py files)
138 :param disabled: list
142 # check if plugin directory exists
143 if not os.path.isdir(path):
144 msg.fatal("cannot find charts directory ", path)
152 mod = self._import_module(path + m + MODULE_EXTENSION)
155 else: # exit if plugin is not found
156 msg.fatal('no modules found.')
158 # scan directory specified in path and load all modules from there
159 names = os.listdir(path)
161 if mod.replace(MODULE_EXTENSION, "") in disabled:
162 msg.error(mod + ": disabled module ", mod.replace(MODULE_EXTENSION, ""))
164 m = self._import_module(path + mod)
166 msg.debug(mod + ": loading module '" + path + mod + "'")
170 def _load_configs(self, modules):
172 Append configuration in list named `config` to every module.
173 For multi-job modules `config` list is created in _parse_config,
174 otherwise it is created here based on BASE_CONFIG prototype with None as identifier.
179 configfile = self.configs + mod.__name__ + ".conf"
180 if os.path.isfile(configfile):
181 msg.debug(mod.__name__ + ": loading module configuration: '" + configfile + "'")
183 if not hasattr(mod, 'config'):
187 self._parse_config(mod, read_config(configfile)))
188 except Exception as e:
189 msg.error(mod.__name__ + ": cannot parse configuration file '" + configfile + "':", str(e))
191 msg.error(mod.__name__ + ": configuration file '" + configfile + "' not found. Using defaults.")
192 # set config if not found
193 if not hasattr(mod, 'config'):
194 msg.debug(mod.__name__ + ": setting configuration for only one job")
195 mod.config = {None: {}}
196 for var in BASE_CONFIG:
198 mod.config[None][var] = getattr(mod, var)
199 except AttributeError:
200 mod.config[None][var] = BASE_CONFIG[var]
204 def _parse_config(module, config):
206 Parse configuration file or extract configuration from module file.
207 Example of returned dictionary:
213 :param module: object
221 msg.debug(module.__name__ + ": reading configuration")
222 for key in BASE_CONFIG:
224 # get defaults from module config
225 defaults[key] = int(config.pop(key))
226 except (KeyError, ValueError):
228 # get defaults from module source code
229 defaults[key] = getattr(module, key)
230 except (KeyError, ValueError, AttributeError):
231 # if above failed, get defaults from global dict
232 defaults[key] = BASE_CONFIG[key]
234 # check if there are dict in config dict
237 if type(config[name]) is dict:
241 # assign variables needed by supervisor to every job configuration
245 if key not in config[name]:
246 config[name][key] = defaults[key]
247 # if only one job is needed, values doesn't have to be in dict (in YAML)
249 config = {None: config.copy()}
250 config[None].update(defaults)
252 # return dictionary of jobs where every job has BASE_CONFIG variables
256 def _create_jobs(modules):
258 Create jobs based on module.config dictionary and module.Service class definition.
263 for module in modules:
264 for name in module.config:
266 conf = module.config[name]
268 job = module.Service(configuration=conf, name=name)
269 except Exception as e:
270 msg.error(module.__name__ +
271 ("/" + str(name) if name is not None else "") +
272 ": cannot start job: '" +
276 # set chart_name (needed to plot run time graphs)
277 job.chart_name = module.__name__
279 job.chart_name += "_" + name
281 msg.debug(module.__name__ + ("/" + str(name) if name is not None else "") + ": job added")
283 return [j for j in jobs if j is not None]
285 def _stop(self, job, reason=None):
287 Stop specified job and remove it from self.jobs list
288 Also notifies user about job failure if DEBUG_FLAG is set
292 prefix = job.__module__
293 if job.name is not None and len(job.name) != 0:
294 prefix += "/" + job.name
296 msg.error("DISABLED:", prefix)
297 self.jobs.remove(job)
298 except Exception as e:
299 msg.debug("This shouldn't happen. NO " + prefix + " IN LIST:" + str(self.jobs) + " ERROR: " + str(e))
301 # TODO remove section below and remove `reason`.
305 elif reason[:3] == "no ":
307 "does not seem to have " +
309 "() function. Disabling it.")
310 elif reason[:7] == "failed ":
313 "() function reports failure.")
314 elif reason[:13] == "configuration":
316 "configuration file '" +
319 ".conf' not found. Using defaults.")
320 elif reason[:11] == "misbehaving":
321 msg.error(prefix + "is " + reason)
325 Tries to execute check() on every job.
326 This cannot fail thus it is catching every exception
327 If job.check() fails job is stopped
331 msg.debug("all job objects", str(self.jobs))
332 while i < len(self.jobs):
336 msg.error(job.chart_name, "check() failed - disabling job")
339 msg.info("CHECKED OK:", job.chart_name)
342 if job.override_name is not None:
343 new_name = job.__module__ + '_' + job.override_name
344 if new_name in overridden:
345 msg.info("DROPPED:", job.name, ", job '" + job.override_name + "' is already served by another job.")
349 job.name = job.override_name
350 msg.info("RENAMED:", new_name, ", from " + job.chart_name)
351 job.chart_name = new_name
352 overridden.append(job.chart_name)
355 except AttributeError as e:
357 msg.error(job.chart_name, "cannot find check() function or it thrown unhandled exception.")
359 except (UnboundLocalError, Exception) as e:
360 msg.error(job.chart_name, str(e))
362 msg.debug("overridden job names:", str(overridden))
363 msg.debug("all remaining job objects:", str(self.jobs))
367 Tries to execute create() on every job.
368 This cannot fail thus it is catching every exception.
369 If job.create() fails job is stopped.
370 This is also creating job run time chart.
373 while i < len(self.jobs):
377 msg.error(job.chart_name, "create function failed.")
380 chart = job.chart_name
382 "CHART netdata.plugin_pythond_" +
384 " '' 'Execution time for " +
386 " plugin' 'milliseconds / run' python.d netdata.plugin_python area 145000 " +
387 str(job.timetable['freq']) +
389 sys.stdout.write("DIMENSION run_time 'run time' absolute 1 1\n\n")
390 msg.debug("created charts for", job.chart_name)
393 except AttributeError:
394 msg.error(job.chart_name, "cannot find create() function or it thrown unhandled exception.")
396 except (UnboundLocalError, Exception) as e:
397 msg.error(job.chart_name, str(e))
402 Creates and supervises every job thread.
403 This will stay forever and ever and ever forever and ever it'll be the one...
405 for job in self.jobs:
409 if threading.active_count() <= 1:
410 msg.fatal("no more jobs")
414 def read_config(path):
416 Read YAML configuration from specified file
421 with open(path, 'r') as stream:
422 config = yaml.load(stream)
423 except (OSError, IOError):
424 msg.error(str(path), "is not a valid configuration file")
426 except yaml.YAMLError as e:
427 msg.error(str(path), "is malformed:", e)
432 def parse_cmdline(directory, *commands):
434 Parse parameters from command line.
435 :param directory: str
436 :param commands: list of str
439 global DEBUG_FLAG, TRACE_FLAG
440 global OVERRIDE_UPDATE_EVERY
443 changed_update = False
445 for cmd in commands[1:]:
448 elif cmd == "debug" or cmd == "all":
450 # redirect stderr to stdout?
451 elif cmd == "trace" or cmd == "all":
453 elif os.path.isfile(directory + cmd + ".chart.py") or os.path.isfile(directory + cmd):
455 mods.append(cmd.replace(".chart.py", ""))
458 BASE_CONFIG['update_every'] = int(cmd)
459 changed_update = True
462 if changed_update and DEBUG_FLAG:
463 OVERRIDE_UPDATE_EVERY = True
464 msg.debug(PROGRAM, "overriding update interval to", str(BASE_CONFIG['update_every']))
466 msg.debug("started from", commands[0], "with options:", *commands[1:])
471 # if __name__ == '__main__':
476 global DEBUG_FLAG, TRACE_FLAG, BASE_CONFIG
478 # read configuration file
480 configfile = CONFIG_DIR + "python.d.conf"
481 msg.PROGRAM = PROGRAM
482 msg.info("reading configuration file:", configfile)
486 conf = read_config(configfile)
489 # exit the whole plugin when 'enabled: no' is set in 'python.d.conf'
490 if conf['enabled'] is False:
491 msg.fatal('disabled in configuration file.\n')
492 except (KeyError, TypeError):
496 for param in BASE_CONFIG:
497 BASE_CONFIG[param] = conf[param]
498 except (KeyError, TypeError):
499 pass # use default update_every from NETDATA_UPDATE_EVERY
502 DEBUG_FLAG = conf['debug']
503 except (KeyError, TypeError):
507 TRACE_FLAG = conf['trace']
508 except (KeyError, TypeError):
512 log_throttle = conf['logs_per_interval']
513 except (KeyError, TypeError):
517 log_interval = conf['log_interval']
518 except (KeyError, TypeError):
521 for k, v in conf.items():
522 if k in ("update_every", "debug", "enabled"):
527 # parse passed command line arguments
528 modules = parse_cmdline(MODULES_DIR, *sys.argv)
529 msg.DEBUG_FLAG = DEBUG_FLAG
530 msg.TRACE_FLAG = TRACE_FLAG
531 msg.LOG_THROTTLE = log_throttle
532 msg.LOG_INTERVAL = log_interval
534 msg.LOG_NEXT_CHECK = 0
535 msg.info("MODULES_DIR='" + MODULES_DIR +
536 "', CONFIG_DIR='" + CONFIG_DIR +
537 "', UPDATE_EVERY=" + str(BASE_CONFIG['update_every']) +
538 ", ONLY_MODULES=" + str(modules))
541 charts = PythonCharts(modules, MODULES_DIR, CONFIG_DIR + "python.d/", disabled)
545 msg.fatal("finished")
548 if __name__ == '__main__':