2 # -*- coding: utf-8 -*-
4 # Description: netdata python modules supervisor
5 # Author: Pawel Krupa (paulfantom)
12 # -----------------------------------------------------------------------------
13 # globals & environment setup
14 # https://github.com/firehol/netdata/wiki/External-Plugins#environment-variables
15 MODULE_EXTENSION = ".chart.py"
16 BASE_CONFIG = {'update_every': os.getenv('NETDATA_UPDATE_EVERY', 1),
20 MODULES_DIR = os.path.abspath(os.getenv('NETDATA_PLUGINS_DIR',
21 os.path.dirname(__file__)) + "/../python.d") + "/"
22 CONFIG_DIR = os.getenv('NETDATA_CONFIG_DIR', "/etc/netdata/")
23 # directories should end with '/'
24 if CONFIG_DIR[-1] != "/":
26 sys.path.append(MODULES_DIR + "python_modules")
28 PROGRAM = os.path.basename(__file__).replace(".plugin", "")
30 OVERRIDE_UPDATE_EVERY = False
32 # -----------------------------------------------------------------------------
33 # custom, third party and version specific python modules management
37 assert sys.version_info >= (3, 1)
38 import importlib.machinery
40 # change this hack below if we want PY_VERSION to be used in modules
42 # builtins.PY_VERSION = 3
44 msg.info('Using python v3')
45 except (AssertionError, ImportError):
49 # change this hack below if we want PY_VERSION to be used in modules
51 # __builtin__.PY_VERSION = 2
53 msg.info('Using python v2')
55 msg.fatal('Cannot start. No importlib.machinery on python3 or lack of imp on python2')
59 msg.fatal('Cannot find yaml library')
62 class PythonCharts(object):
64 Main class used to control every python module.
69 modules_path='../python.d/',
70 modules_configs='../conf.d/',
71 modules_disabled=None):
74 :param modules_path: str
75 :param modules_configs: str
76 :param modules_disabled: list
81 if modules_disabled is None:
85 # set configuration directory
86 self.configs = modules_configs
89 loaded_modules = self._load_modules(modules_path, modules, modules_disabled)
91 # load configuration files
92 configured_modules = self._load_configs(loaded_modules)
94 # good economy and prosperity:
95 self.jobs = self._create_jobs(configured_modules) # type: list
97 # enable timetable override like `python.d.plugin mysql debug 1`
98 if DEBUG_FLAG and OVERRIDE_UPDATE_EVERY:
100 job.create_timetable(BASE_CONFIG['update_every'])
103 def _import_module(path, name=None):
105 Try to import module using only its path.
112 name = path.split('/')[-1]
113 if name[-len(MODULE_EXTENSION):] != MODULE_EXTENSION:
115 name = name[:-len(MODULE_EXTENSION)]
118 return importlib.machinery.SourceFileLoader(name, path).load_module()
120 return imp.load_source(name, path)
121 except Exception as e:
122 msg.error("Problem loading", name, str(e))
125 def _load_modules(self, path, modules, disabled):
127 Load modules from 'modules' list or dynamically every file from 'path' (only .chart.py files)
130 :param disabled: list
134 # check if plugin directory exists
135 if not os.path.isdir(path):
136 msg.fatal("cannot find charts directory ", path)
144 mod = self._import_module(path + m + MODULE_EXTENSION)
147 else: # exit if plugin is not found
148 msg.fatal('no modules found.')
150 # scan directory specified in path and load all modules from there
151 names = os.listdir(path)
153 if mod.replace(MODULE_EXTENSION, "") in disabled:
154 msg.error(mod + ": disabled module ", mod.replace(MODULE_EXTENSION, ""))
156 m = self._import_module(path + mod)
158 msg.debug(mod + ": loading module '" + path + mod + "'")
162 def _load_configs(self, modules):
164 Append configuration in list named `config` to every module.
165 For multi-job modules `config` list is created in _parse_config,
166 otherwise it is created here based on BASE_CONFIG prototype with None as identifier.
171 configfile = self.configs + mod.__name__ + ".conf"
172 if os.path.isfile(configfile):
173 msg.debug(mod.__name__ + ": loading module configuration: '" + configfile + "'")
175 if not hasattr(mod, 'config'):
179 self._parse_config(mod, read_config(configfile)))
180 except Exception as e:
181 msg.error(mod.__name__ + ": cannot parse configuration file '" + configfile + "':", str(e))
183 msg.error(mod.__name__ + ": configuration file '" + configfile + "' not found. Using defaults.")
184 # set config if not found
185 if not hasattr(mod, 'config'):
186 msg.debug(mod.__name__ + ": setting configuration for only one job")
187 mod.config = {None: {}}
188 for var in BASE_CONFIG:
190 mod.config[None][var] = getattr(mod, var)
191 except AttributeError:
192 mod.config[None][var] = BASE_CONFIG[var]
196 def _parse_config(module, config):
198 Parse configuration file or extract configuration from module file.
199 Example of returned dictionary:
205 :param module: object
211 msg.debug(module.__name__ + ": reading configuration")
212 for key in BASE_CONFIG:
214 # get defaults from module config
215 defaults[key] = int(config.pop(key))
216 except (KeyError, ValueError):
218 # get defaults from module source code
219 defaults[key] = getattr(module, key)
220 except (KeyError, ValueError, AttributeError):
221 # if above failed, get defaults from global dict
222 defaults[key] = BASE_CONFIG[key]
224 # check if there are dict in config dict
227 if type(config[name]) is dict:
231 # assign variables needed by supervisor to every job configuration
235 if key not in config[name]:
236 config[name][key] = defaults[key]
237 # if only one job is needed, values doesn't have to be in dict (in YAML)
239 config = {None: config.copy()}
240 config[None].update(defaults)
242 # return dictionary of jobs where every job has BASE_CONFIG variables
246 def _create_jobs(modules):
248 Create jobs based on module.config dictionary and module.Service class definition.
253 for module in modules:
254 for name in module.config:
256 conf = module.config[name]
258 job = module.Service(configuration=conf, name=name)
259 except Exception as e:
260 msg.error(module.__name__ +
261 ("/" + str(name) if name is not None else "") +
262 ": cannot start job: '" +
266 # set chart_name (needed to plot run time graphs)
267 job.chart_name = module.__name__
269 job.chart_name += "_" + name
271 msg.debug(module.__name__ + ("/" + str(name) if name is not None else "") + ": job added")
273 return [j for j in jobs if j is not None]
275 def _stop(self, job, reason=None):
277 Stop specified job and remove it from self.jobs list
278 Also notifies user about job failure if DEBUG_FLAG is set
282 prefix = job.__module__
283 if job.name is not None:
284 prefix += "/" + job.name
287 self.jobs.remove(job)
290 elif reason[:3] == "no ":
292 "does not seem to have " +
294 "() function. Disabling it.")
295 elif reason[:7] == "failed ":
298 "() function reports failure.")
299 elif reason[:13] == "configuration":
301 "configuration file '" +
304 ".conf' not found. Using defaults.")
305 elif reason[:11] == "misbehaving":
306 msg.error(prefix + "is " + reason)
310 Tries to execute check() on every job.
311 This cannot fail thus it is catching every exception
312 If job.check() fails job is stopped
315 while i < len(self.jobs):
319 self._stop(job, "failed check")
321 msg.debug(job.chart_name, ": check succeeded")
323 except AttributeError:
324 self._stop(job, "no check")
325 except (UnboundLocalError, Exception) as e:
326 self._stop(job, "misbehaving. Reason:" + str(e))
330 Tries to execute create() on every job.
331 This cannot fail thus it is catching every exception.
332 If job.create() fails job is stopped.
333 This is also creating job run time chart.
336 while i < len(self.jobs):
340 self._stop(job, "failed create")
342 chart = job.chart_name
344 "CHART netdata.plugin_pythond_" +
346 " '' 'Execution time for " +
348 " plugin' 'milliseconds / run' python.d netdata.plugin_python area 145000 " +
349 str(job.timetable['freq']) +
351 sys.stdout.write("DIMENSION run_time 'run time' absolute 1 1\n\n")
352 msg.debug("created charts for", job.chart_name)
355 except AttributeError:
356 self._stop(job, "no create")
357 except (UnboundLocalError, Exception) as e:
358 self._stop(job, "misbehaving. Reason: " + str(e))
362 Creates and supervises every job thread.
363 This will stay forever and ever and ever forever and ever it'll be the one...
365 for job in self.jobs:
369 if threading.active_count() <= 1:
370 msg.fatal("no more jobs")
374 def read_config(path):
376 Read YAML configuration from specified file
381 with open(path, 'r') as stream:
382 config = yaml.load(stream)
383 except (OSError, IOError):
384 msg.error(str(path), "is not a valid configuration file")
386 except yaml.YAMLError as e:
387 msg.error(str(path), "is malformed:", e)
392 def parse_cmdline(directory, *commands):
394 Parse parameters from command line.
395 :param directory: str
396 :param commands: list of str
400 global OVERRIDE_UPDATE_EVERY
403 changed_update = False
405 for cmd in commands[1:]:
408 elif cmd == "debug" or cmd == "all":
410 # redirect stderr to stdout?
411 elif os.path.isfile(directory + cmd + ".chart.py") or os.path.isfile(directory + cmd):
413 mods.append(cmd.replace(".chart.py", ""))
416 BASE_CONFIG['update_every'] = int(cmd)
417 changed_update = True
420 if changed_update and DEBUG_FLAG:
421 OVERRIDE_UPDATE_EVERY = True
422 msg.debug(PROGRAM, "overriding update interval to", str(BASE_CONFIG['update_every']))
424 msg.debug("started from", commands[0], "with options:", *commands[1:])
429 # if __name__ == '__main__':
434 global DEBUG_FLAG, BASE_CONFIG
436 # read configuration file
438 configfile = CONFIG_DIR + "python.d.conf"
439 msg.PROGRAM = PROGRAM
440 msg.info("reading configuration file:", configfile)
442 conf = read_config(configfile)
445 # exit the whole plugin when 'enabled: no' is set in 'python.d.conf'
446 if conf['enabled'] is False:
447 msg.fatal('disabled in configuration file.\n')
448 except (KeyError, TypeError):
451 for param in BASE_CONFIG:
452 BASE_CONFIG[param] = conf[param]
453 except (KeyError, TypeError):
454 pass # use default update_every from NETDATA_UPDATE_EVERY
456 DEBUG_FLAG = conf['debug']
457 except (KeyError, TypeError):
459 for k, v in conf.items():
460 if k in ("update_every", "debug", "enabled"):
465 # parse passed command line arguments
466 modules = parse_cmdline(MODULES_DIR, *sys.argv)
467 msg.DEBUG_FLAG = DEBUG_FLAG
468 msg.info("MODULES_DIR='" + MODULES_DIR +
469 "', CONFIG_DIR='" + CONFIG_DIR +
470 "', UPDATE_EVERY=" + str(BASE_CONFIG['update_every']) +
471 ", ONLY_MODULES=" + str(modules))
474 charts = PythonCharts(modules, MODULES_DIR, CONFIG_DIR + "python.d/", disabled)
478 msg.fatal("finished")
481 if __name__ == '__main__':