2 '''':; exec "$(command -v python || command -v python3 || command -v python2 || echo "ERROR python IS NOT AVAILABLE IN THIS SYSTEM")" "$0" "$@" # '''
3 # -*- coding: utf-8 -*-
5 # Description: netdata python modules supervisor
6 # Author: Pawel Krupa (paulfantom)
13 # -----------------------------------------------------------------------------
14 # globals & environment setup
15 # https://github.com/firehol/netdata/wiki/External-Plugins#environment-variables
16 MODULE_EXTENSION = ".chart.py"
17 BASE_CONFIG = {'update_every': os.getenv('NETDATA_UPDATE_EVERY', 1),
21 MODULES_DIR = os.path.abspath(os.getenv('NETDATA_PLUGINS_DIR',
22 os.path.dirname(__file__)) + "/../python.d") + "/"
23 CONFIG_DIR = os.getenv('NETDATA_CONFIG_DIR', "/etc/netdata/")
24 # directories should end with '/'
25 if CONFIG_DIR[-1] != "/":
27 sys.path.append(MODULES_DIR + "python_modules")
29 PROGRAM = os.path.basename(__file__).replace(".plugin", "")
31 OVERRIDE_UPDATE_EVERY = False
33 # -----------------------------------------------------------------------------
34 # custom, third party and version specific python modules management
38 assert sys.version_info >= (3, 1)
39 import importlib.machinery
41 # change this hack below if we want PY_VERSION to be used in modules
43 # builtins.PY_VERSION = 3
44 msg.info('Using python v3')
45 except (AssertionError, ImportError):
49 # change this hack below if we want PY_VERSION to be used in modules
51 # __builtin__.PY_VERSION = 2
53 msg.info('Using python v2')
55 msg.fatal('Cannot start. No importlib.machinery on python3 or lack of imp on python2')
59 # msg.fatal('Cannot find yaml library')
62 import pyyaml3 as yaml
64 import pyyaml2 as yaml
66 msg.fatal('Cannot find yaml library')
69 class PythonCharts(object):
71 Main class used to control every python module.
76 modules_path='../python.d/',
77 modules_configs='../conf.d/',
78 modules_disabled=None):
81 :param modules_path: str
82 :param modules_configs: str
83 :param modules_disabled: list
88 if modules_disabled is None:
92 # set configuration directory
93 self.configs = modules_configs
96 loaded_modules = self._load_modules(modules_path, modules, modules_disabled)
98 # load configuration files
99 configured_modules = self._load_configs(loaded_modules)
101 # good economy and prosperity:
102 self.jobs = self._create_jobs(configured_modules) # type: list
104 # enable timetable override like `python.d.plugin mysql debug 1`
105 if DEBUG_FLAG and OVERRIDE_UPDATE_EVERY:
106 for job in self.jobs:
107 job.create_timetable(BASE_CONFIG['update_every'])
110 def _import_module(path, name=None):
112 Try to import module using only its path.
119 name = path.split('/')[-1]
120 if name[-len(MODULE_EXTENSION):] != MODULE_EXTENSION:
122 name = name[:-len(MODULE_EXTENSION)]
125 return importlib.machinery.SourceFileLoader(name, path).load_module()
127 return imp.load_source(name, path)
128 except Exception as e:
129 msg.error("Problem loading", name, str(e))
132 def _load_modules(self, path, modules, disabled):
134 Load modules from 'modules' list or dynamically every file from 'path' (only .chart.py files)
137 :param disabled: list
141 # check if plugin directory exists
142 if not os.path.isdir(path):
143 msg.fatal("cannot find charts directory ", path)
151 mod = self._import_module(path + m + MODULE_EXTENSION)
154 else: # exit if plugin is not found
155 msg.fatal('no modules found.')
157 # scan directory specified in path and load all modules from there
158 names = os.listdir(path)
160 if mod.replace(MODULE_EXTENSION, "") in disabled:
161 msg.error(mod + ": disabled module ", mod.replace(MODULE_EXTENSION, ""))
163 m = self._import_module(path + mod)
165 msg.debug(mod + ": loading module '" + path + mod + "'")
169 def _load_configs(self, modules):
171 Append configuration in list named `config` to every module.
172 For multi-job modules `config` list is created in _parse_config,
173 otherwise it is created here based on BASE_CONFIG prototype with None as identifier.
178 configfile = self.configs + mod.__name__ + ".conf"
179 if os.path.isfile(configfile):
180 msg.debug(mod.__name__ + ": loading module configuration: '" + configfile + "'")
182 if not hasattr(mod, 'config'):
186 self._parse_config(mod, read_config(configfile)))
187 except Exception as e:
188 msg.error(mod.__name__ + ": cannot parse configuration file '" + configfile + "':", str(e))
190 msg.error(mod.__name__ + ": configuration file '" + configfile + "' not found. Using defaults.")
191 # set config if not found
192 if not hasattr(mod, 'config'):
193 msg.debug(mod.__name__ + ": setting configuration for only one job")
194 mod.config = {None: {}}
195 for var in BASE_CONFIG:
197 mod.config[None][var] = getattr(mod, var)
198 except AttributeError:
199 mod.config[None][var] = BASE_CONFIG[var]
203 def _parse_config(module, config):
205 Parse configuration file or extract configuration from module file.
206 Example of returned dictionary:
212 :param module: object
220 msg.debug(module.__name__ + ": reading configuration")
221 for key in BASE_CONFIG:
223 # get defaults from module config
224 defaults[key] = int(config.pop(key))
225 except (KeyError, ValueError):
227 # get defaults from module source code
228 defaults[key] = getattr(module, key)
229 except (KeyError, ValueError, AttributeError):
230 # if above failed, get defaults from global dict
231 defaults[key] = BASE_CONFIG[key]
233 # check if there are dict in config dict
236 if type(config[name]) is dict:
240 # assign variables needed by supervisor to every job configuration
244 if key not in config[name]:
245 config[name][key] = defaults[key]
246 # if only one job is needed, values doesn't have to be in dict (in YAML)
248 config = {None: config.copy()}
249 config[None].update(defaults)
251 # return dictionary of jobs where every job has BASE_CONFIG variables
255 def _create_jobs(modules):
257 Create jobs based on module.config dictionary and module.Service class definition.
262 for module in modules:
263 for name in module.config:
265 conf = module.config[name]
267 job = module.Service(configuration=conf, name=name)
268 except Exception as e:
269 msg.error(module.__name__ +
270 ("/" + str(name) if name is not None else "") +
271 ": cannot start job: '" +
275 # set chart_name (needed to plot run time graphs)
276 job.chart_name = module.__name__
278 job.chart_name += "_" + name
280 msg.debug(module.__name__ + ("/" + str(name) if name is not None else "") + ": job added")
282 return [j for j in jobs if j is not None]
284 def _stop(self, job, reason=None):
286 Stop specified job and remove it from self.jobs list
287 Also notifies user about job failure if DEBUG_FLAG is set
291 prefix = job.__module__
292 if job.name is not None and len(job.name) != 0:
293 prefix += "/" + job.name
295 msg.error("DISABLED:", prefix)
296 self.jobs.remove(job)
297 except Exception as e:
298 msg.debug("This shouldn't happen. NO " + prefix + " IN LIST:" + str(self.jobs) + " ERROR: " + str(e))
300 # TODO remove section below and remove `reason`.
304 elif reason[:3] == "no ":
306 "does not seem to have " +
308 "() function. Disabling it.")
309 elif reason[:7] == "failed ":
312 "() function reports failure.")
313 elif reason[:13] == "configuration":
315 "configuration file '" +
318 ".conf' not found. Using defaults.")
319 elif reason[:11] == "misbehaving":
320 msg.error(prefix + "is " + reason)
324 Tries to execute check() on every job.
325 This cannot fail thus it is catching every exception
326 If job.check() fails job is stopped
330 msg.debug("all job objects", str(self.jobs))
331 while i < len(self.jobs):
335 msg.error(job.chart_name, "check function failed.")
338 msg.info("CHECKED OK:", job.chart_name)
341 if job.override_name is not None:
342 new_name = job.__module__ + '_' + job.override_name
343 if new_name in overridden:
344 msg.info("DROPPED:", job.name, ", job '" + job.override_name + "' is already served by another job.")
348 job.name = job.override_name
349 msg.info("RENAMED:", new_name, ", from " + job.chart_name)
350 job.chart_name = new_name
351 overridden.append(job.chart_name)
354 except AttributeError as e:
356 msg.error(job.chart_name, "cannot find check() function or it thrown unhandled exception.")
358 except (UnboundLocalError, Exception) as e:
359 msg.error(job.chart_name, str(e))
361 msg.debug("overridden job names:", str(overridden))
362 msg.debug("all remaining job objects:", str(self.jobs))
366 Tries to execute create() on every job.
367 This cannot fail thus it is catching every exception.
368 If job.create() fails job is stopped.
369 This is also creating job run time chart.
372 while i < len(self.jobs):
376 msg.error(job.chart_name, "create function failed.")
379 chart = job.chart_name
381 "CHART netdata.plugin_pythond_" +
383 " '' 'Execution time for " +
385 " plugin' 'milliseconds / run' python.d netdata.plugin_python area 145000 " +
386 str(job.timetable['freq']) +
388 sys.stdout.write("DIMENSION run_time 'run time' absolute 1 1\n\n")
389 msg.debug("created charts for", job.chart_name)
392 except AttributeError:
393 msg.error(job.chart_name, "cannot find create() function or it thrown unhandled exception.")
395 except (UnboundLocalError, Exception) as e:
396 msg.error(job.chart_name, str(e))
401 Creates and supervises every job thread.
402 This will stay forever and ever and ever forever and ever it'll be the one...
404 for job in self.jobs:
408 if threading.active_count() <= 1:
409 msg.fatal("no more jobs")
413 def read_config(path):
415 Read YAML configuration from specified file
420 with open(path, 'r') as stream:
421 config = yaml.load(stream)
422 except (OSError, IOError):
423 msg.error(str(path), "is not a valid configuration file")
425 except yaml.YAMLError as e:
426 msg.error(str(path), "is malformed:", e)
431 def parse_cmdline(directory, *commands):
433 Parse parameters from command line.
434 :param directory: str
435 :param commands: list of str
439 global OVERRIDE_UPDATE_EVERY
442 changed_update = False
444 for cmd in commands[1:]:
447 elif cmd == "debug" or cmd == "all":
449 # redirect stderr to stdout?
450 elif os.path.isfile(directory + cmd + ".chart.py") or os.path.isfile(directory + cmd):
452 mods.append(cmd.replace(".chart.py", ""))
455 BASE_CONFIG['update_every'] = int(cmd)
456 changed_update = True
459 if changed_update and DEBUG_FLAG:
460 OVERRIDE_UPDATE_EVERY = True
461 msg.debug(PROGRAM, "overriding update interval to", str(BASE_CONFIG['update_every']))
463 msg.debug("started from", commands[0], "with options:", *commands[1:])
468 # if __name__ == '__main__':
473 global DEBUG_FLAG, BASE_CONFIG
475 # read configuration file
477 configfile = CONFIG_DIR + "python.d.conf"
478 msg.PROGRAM = PROGRAM
479 msg.info("reading configuration file:", configfile)
483 conf = read_config(configfile)
486 # exit the whole plugin when 'enabled: no' is set in 'python.d.conf'
487 if conf['enabled'] is False:
488 msg.fatal('disabled in configuration file.\n')
489 except (KeyError, TypeError):
492 for param in BASE_CONFIG:
493 BASE_CONFIG[param] = conf[param]
494 except (KeyError, TypeError):
495 pass # use default update_every from NETDATA_UPDATE_EVERY
497 DEBUG_FLAG = conf['debug']
498 except (KeyError, TypeError):
501 log_throttle = conf['logs_per_interval']
502 except (KeyError, TypeError):
505 log_interval = conf['log_interval']
506 except (KeyError, TypeError):
508 for k, v in conf.items():
509 if k in ("update_every", "debug", "enabled"):
514 # parse passed command line arguments
515 modules = parse_cmdline(MODULES_DIR, *sys.argv)
516 msg.DEBUG_FLAG = DEBUG_FLAG
517 msg.LOG_THROTTLE = log_throttle
518 msg.LOG_INTERVAL = log_interval
520 msg.LOG_NEXT_CHECK = 0
521 msg.info("MODULES_DIR='" + MODULES_DIR +
522 "', CONFIG_DIR='" + CONFIG_DIR +
523 "', UPDATE_EVERY=" + str(BASE_CONFIG['update_every']) +
524 ", ONLY_MODULES=" + str(modules))
527 charts = PythonCharts(modules, MODULES_DIR, CONFIG_DIR + "python.d/", disabled)
531 msg.fatal("finished")
534 if __name__ == '__main__':