2 '''':; exec "$(command -v python2 || command -v python3 || echo "ERROR python IS NOT AVAILABLE IN THIS SYSTEM")" "$0" "$@" # '''
3 # -*- coding: utf-8 -*-
5 # Description: netdata python modules supervisor
6 # Author: Pawel Krupa (paulfantom)
13 # -----------------------------------------------------------------------------
14 # globals & environment setup
15 # https://github.com/firehol/netdata/wiki/External-Plugins#environment-variables
16 MODULE_EXTENSION = ".chart.py"
17 BASE_CONFIG = {'update_every': os.getenv('NETDATA_UPDATE_EVERY', 1),
21 MODULES_DIR = os.path.abspath(os.getenv('NETDATA_PLUGINS_DIR',
22 os.path.dirname(__file__)) + "/../python.d") + "/"
23 CONFIG_DIR = os.getenv('NETDATA_CONFIG_DIR', "/etc/netdata/")
24 # directories should end with '/'
25 if CONFIG_DIR[-1] != "/":
27 sys.path.append(MODULES_DIR + "python_modules")
29 PROGRAM = os.path.basename(__file__).replace(".plugin", "")
31 OVERRIDE_UPDATE_EVERY = False
33 # -----------------------------------------------------------------------------
34 # custom, third party and version specific python modules management
38 assert sys.version_info >= (3, 1)
39 import importlib.machinery
41 # change this hack below if we want PY_VERSION to be used in modules
43 # builtins.PY_VERSION = 3
45 msg.info('Using python v3')
46 except (AssertionError, ImportError):
50 # change this hack below if we want PY_VERSION to be used in modules
52 # __builtin__.PY_VERSION = 2
54 msg.info('Using python v2')
56 msg.fatal('Cannot start. No importlib.machinery on python3 or lack of imp on python2')
60 msg.fatal('Cannot find yaml library')
63 class PythonCharts(object):
65 Main class used to control every python module.
70 modules_path='../python.d/',
71 modules_configs='../conf.d/',
72 modules_disabled=None):
75 :param modules_path: str
76 :param modules_configs: str
77 :param modules_disabled: list
82 if modules_disabled is None:
86 # set configuration directory
87 self.configs = modules_configs
90 loaded_modules = self._load_modules(modules_path, modules, modules_disabled)
92 # load configuration files
93 configured_modules = self._load_configs(loaded_modules)
95 # good economy and prosperity:
96 self.jobs = self._create_jobs(configured_modules) # type: list
98 # enable timetable override like `python.d.plugin mysql debug 1`
99 if DEBUG_FLAG and OVERRIDE_UPDATE_EVERY:
100 for job in self.jobs:
101 job.create_timetable(BASE_CONFIG['update_every'])
104 def _import_module(path, name=None):
106 Try to import module using only its path.
113 name = path.split('/')[-1]
114 if name[-len(MODULE_EXTENSION):] != MODULE_EXTENSION:
116 name = name[:-len(MODULE_EXTENSION)]
119 return importlib.machinery.SourceFileLoader(name, path).load_module()
121 return imp.load_source(name, path)
122 except Exception as e:
123 msg.error("Problem loading", name, str(e))
126 def _load_modules(self, path, modules, disabled):
128 Load modules from 'modules' list or dynamically every file from 'path' (only .chart.py files)
131 :param disabled: list
135 # check if plugin directory exists
136 if not os.path.isdir(path):
137 msg.fatal("cannot find charts directory ", path)
145 mod = self._import_module(path + m + MODULE_EXTENSION)
148 else: # exit if plugin is not found
149 msg.fatal('no modules found.')
151 # scan directory specified in path and load all modules from there
152 names = os.listdir(path)
154 if mod.replace(MODULE_EXTENSION, "") in disabled:
155 msg.error(mod + ": disabled module ", mod.replace(MODULE_EXTENSION, ""))
157 m = self._import_module(path + mod)
159 msg.debug(mod + ": loading module '" + path + mod + "'")
163 def _load_configs(self, modules):
165 Append configuration in list named `config` to every module.
166 For multi-job modules `config` list is created in _parse_config,
167 otherwise it is created here based on BASE_CONFIG prototype with None as identifier.
172 configfile = self.configs + mod.__name__ + ".conf"
173 if os.path.isfile(configfile):
174 msg.debug(mod.__name__ + ": loading module configuration: '" + configfile + "'")
176 if not hasattr(mod, 'config'):
180 self._parse_config(mod, read_config(configfile)))
181 except Exception as e:
182 msg.error(mod.__name__ + ": cannot parse configuration file '" + configfile + "':", str(e))
184 msg.error(mod.__name__ + ": configuration file '" + configfile + "' not found. Using defaults.")
185 # set config if not found
186 if not hasattr(mod, 'config'):
187 msg.debug(mod.__name__ + ": setting configuration for only one job")
188 mod.config = {None: {}}
189 for var in BASE_CONFIG:
191 mod.config[None][var] = getattr(mod, var)
192 except AttributeError:
193 mod.config[None][var] = BASE_CONFIG[var]
197 def _parse_config(module, config):
199 Parse configuration file or extract configuration from module file.
200 Example of returned dictionary:
206 :param module: object
214 msg.debug(module.__name__ + ": reading configuration")
215 for key in BASE_CONFIG:
217 # get defaults from module config
218 defaults[key] = int(config.pop(key))
219 except (KeyError, ValueError):
221 # get defaults from module source code
222 defaults[key] = getattr(module, key)
223 except (KeyError, ValueError, AttributeError):
224 # if above failed, get defaults from global dict
225 defaults[key] = BASE_CONFIG[key]
227 # check if there are dict in config dict
230 if type(config[name]) is dict:
234 # assign variables needed by supervisor to every job configuration
238 if key not in config[name]:
239 config[name][key] = defaults[key]
240 # if only one job is needed, values doesn't have to be in dict (in YAML)
242 config = {None: config.copy()}
243 config[None].update(defaults)
245 # return dictionary of jobs where every job has BASE_CONFIG variables
249 def _create_jobs(modules):
251 Create jobs based on module.config dictionary and module.Service class definition.
256 for module in modules:
257 for name in module.config:
259 conf = module.config[name]
261 job = module.Service(configuration=conf, name=name)
262 except Exception as e:
263 msg.error(module.__name__ +
264 ("/" + str(name) if name is not None else "") +
265 ": cannot start job: '" +
269 # set chart_name (needed to plot run time graphs)
270 job.chart_name = module.__name__
272 job.chart_name += "_" + name
274 msg.debug(module.__name__ + ("/" + str(name) if name is not None else "") + ": job added")
276 return [j for j in jobs if j is not None]
278 def _stop(self, job, reason=None):
280 Stop specified job and remove it from self.jobs list
281 Also notifies user about job failure if DEBUG_FLAG is set
285 prefix = job.__module__
286 if job.name is not None and len(job.name) != 0:
287 prefix += "/" + job.name
289 self.jobs.remove(job)
290 msg.info("Disabled", prefix)
291 except Exception as e:
292 msg.debug("This shouldn't happen. NO " + prefix + " IN LIST:" + str(self.jobs))
297 elif reason[:3] == "no ":
299 "does not seem to have " +
301 "() function. Disabling it.")
302 elif reason[:7] == "failed ":
305 "() function reports failure.")
306 elif reason[:13] == "configuration":
308 "configuration file '" +
311 ".conf' not found. Using defaults.")
312 elif reason[:11] == "misbehaving":
313 msg.error(prefix + "is " + reason)
317 Tries to execute check() on every job.
318 This cannot fail thus it is catching every exception
319 If job.check() fails job is stopped
323 while i < len(self.jobs):
327 self._stop(job, "failed check")
329 msg.debug(job.chart_name, ": check succeeded")
332 if job.override_name is not None:
333 new_name = job.__module__ + '_' + job.override_name
334 if new_name in overridden:
335 msg.error(job.chart_name + " already exists. Stopping '" + job.name + "'")
338 job.name = job.override_name
339 msg.debug(job.chart_name + " changing chart name to: '" + new_name + "'")
340 job.chart_name = new_name
341 overridden.append(job.chart_name)
344 except AttributeError:
345 self._stop(job, "no check")
346 except (UnboundLocalError, Exception) as e:
347 self._stop(job, "misbehaving. Reason:" + str(e))
351 Tries to execute create() on every job.
352 This cannot fail thus it is catching every exception.
353 If job.create() fails job is stopped.
354 This is also creating job run time chart.
357 while i < len(self.jobs):
361 self._stop(job, "failed create")
363 chart = job.chart_name
365 "CHART netdata.plugin_pythond_" +
367 " '' 'Execution time for " +
369 " plugin' 'milliseconds / run' python.d netdata.plugin_python area 145000 " +
370 str(job.timetable['freq']) +
372 sys.stdout.write("DIMENSION run_time 'run time' absolute 1 1\n\n")
373 msg.debug("created charts for", job.chart_name)
376 except AttributeError:
377 self._stop(job, "no create")
378 except (UnboundLocalError, Exception) as e:
379 self._stop(job, "misbehaving. Reason: " + str(e))
383 Creates and supervises every job thread.
384 This will stay forever and ever and ever forever and ever it'll be the one...
386 for job in self.jobs:
390 if threading.active_count() <= 1:
391 msg.fatal("no more jobs")
395 def read_config(path):
397 Read YAML configuration from specified file
402 with open(path, 'r') as stream:
403 config = yaml.load(stream)
404 except (OSError, IOError):
405 msg.error(str(path), "is not a valid configuration file")
407 except yaml.YAMLError as e:
408 msg.error(str(path), "is malformed:", e)
413 def parse_cmdline(directory, *commands):
415 Parse parameters from command line.
416 :param directory: str
417 :param commands: list of str
421 global OVERRIDE_UPDATE_EVERY
424 changed_update = False
426 for cmd in commands[1:]:
429 elif cmd == "debug" or cmd == "all":
431 # redirect stderr to stdout?
432 elif os.path.isfile(directory + cmd + ".chart.py") or os.path.isfile(directory + cmd):
434 mods.append(cmd.replace(".chart.py", ""))
437 BASE_CONFIG['update_every'] = int(cmd)
438 changed_update = True
441 if changed_update and DEBUG_FLAG:
442 OVERRIDE_UPDATE_EVERY = True
443 msg.debug(PROGRAM, "overriding update interval to", str(BASE_CONFIG['update_every']))
445 msg.debug("started from", commands[0], "with options:", *commands[1:])
450 # if __name__ == '__main__':
455 global DEBUG_FLAG, BASE_CONFIG
457 # read configuration file
459 configfile = CONFIG_DIR + "python.d.conf"
460 msg.PROGRAM = PROGRAM
461 msg.info("reading configuration file:", configfile)
463 conf = read_config(configfile)
466 # exit the whole plugin when 'enabled: no' is set in 'python.d.conf'
467 if conf['enabled'] is False:
468 msg.fatal('disabled in configuration file.\n')
469 except (KeyError, TypeError):
472 for param in BASE_CONFIG:
473 BASE_CONFIG[param] = conf[param]
474 except (KeyError, TypeError):
475 pass # use default update_every from NETDATA_UPDATE_EVERY
477 DEBUG_FLAG = conf['debug']
478 except (KeyError, TypeError):
480 for k, v in conf.items():
481 if k in ("update_every", "debug", "enabled"):
486 # parse passed command line arguments
487 modules = parse_cmdline(MODULES_DIR, *sys.argv)
488 msg.DEBUG_FLAG = DEBUG_FLAG
489 msg.info("MODULES_DIR='" + MODULES_DIR +
490 "', CONFIG_DIR='" + CONFIG_DIR +
491 "', UPDATE_EVERY=" + str(BASE_CONFIG['update_every']) +
492 ", ONLY_MODULES=" + str(modules))
495 charts = PythonCharts(modules, MODULES_DIR, CONFIG_DIR + "python.d/", disabled)
499 msg.fatal("finished")
502 if __name__ == '__main__':