2 '''':; exec "$(command -v python2 || command -v python3 || echo "ERROR python IS NOT AVAILABLE IN THIS SYSTEM")" "$0" "$@" # '''
3 # -*- coding: utf-8 -*-
5 # Description: netdata python modules supervisor
6 # Author: Pawel Krupa (paulfantom)
13 # -----------------------------------------------------------------------------
14 # globals & environment setup
15 # https://github.com/firehol/netdata/wiki/External-Plugins#environment-variables
16 MODULE_EXTENSION = ".chart.py"
17 BASE_CONFIG = {'update_every': os.getenv('NETDATA_UPDATE_EVERY', 1),
21 MODULES_DIR = os.path.abspath(os.getenv('NETDATA_PLUGINS_DIR',
22 os.path.dirname(__file__)) + "/../python.d") + "/"
23 CONFIG_DIR = os.getenv('NETDATA_CONFIG_DIR', "/etc/netdata/")
24 # directories should end with '/'
25 if CONFIG_DIR[-1] != "/":
27 sys.path.append(MODULES_DIR + "python_modules")
29 PROGRAM = os.path.basename(__file__).replace(".plugin", "")
31 OVERRIDE_UPDATE_EVERY = False
33 # -----------------------------------------------------------------------------
34 # custom, third party and version specific python modules management
38 assert sys.version_info >= (3, 1)
39 import importlib.machinery
41 # change this hack below if we want PY_VERSION to be used in modules
43 # builtins.PY_VERSION = 3
45 msg.info('Using python v3')
46 except (AssertionError, ImportError):
50 # change this hack below if we want PY_VERSION to be used in modules
52 # __builtin__.PY_VERSION = 2
54 msg.info('Using python v2')
56 msg.fatal('Cannot start. No importlib.machinery on python3 or lack of imp on python2')
60 msg.fatal('Cannot find yaml library')
63 class PythonCharts(object):
65 Main class used to control every python module.
70 modules_path='../python.d/',
71 modules_configs='../conf.d/',
72 modules_disabled=None):
75 :param modules_path: str
76 :param modules_configs: str
77 :param modules_disabled: list
82 if modules_disabled is None:
86 # set configuration directory
87 self.configs = modules_configs
90 loaded_modules = self._load_modules(modules_path, modules, modules_disabled)
92 # load configuration files
93 configured_modules = self._load_configs(loaded_modules)
95 # good economy and prosperity:
96 self.jobs = self._create_jobs(configured_modules) # type: list
98 # enable timetable override like `python.d.plugin mysql debug 1`
99 if DEBUG_FLAG and OVERRIDE_UPDATE_EVERY:
100 for job in self.jobs:
101 job.create_timetable(BASE_CONFIG['update_every'])
104 def _import_module(path, name=None):
106 Try to import module using only its path.
113 name = path.split('/')[-1]
114 if name[-len(MODULE_EXTENSION):] != MODULE_EXTENSION:
116 name = name[:-len(MODULE_EXTENSION)]
119 return importlib.machinery.SourceFileLoader(name, path).load_module()
121 return imp.load_source(name, path)
122 except Exception as e:
123 msg.error("Problem loading", name, str(e))
126 def _load_modules(self, path, modules, disabled):
128 Load modules from 'modules' list or dynamically every file from 'path' (only .chart.py files)
131 :param disabled: list
135 # check if plugin directory exists
136 if not os.path.isdir(path):
137 msg.fatal("cannot find charts directory ", path)
145 mod = self._import_module(path + m + MODULE_EXTENSION)
148 else: # exit if plugin is not found
149 msg.fatal('no modules found.')
151 # scan directory specified in path and load all modules from there
152 names = os.listdir(path)
154 if mod.replace(MODULE_EXTENSION, "") in disabled:
155 msg.error(mod + ": disabled module ", mod.replace(MODULE_EXTENSION, ""))
157 m = self._import_module(path + mod)
159 msg.debug(mod + ": loading module '" + path + mod + "'")
163 def _load_configs(self, modules):
165 Append configuration in list named `config` to every module.
166 For multi-job modules `config` list is created in _parse_config,
167 otherwise it is created here based on BASE_CONFIG prototype with None as identifier.
172 configfile = self.configs + mod.__name__ + ".conf"
173 if os.path.isfile(configfile):
174 msg.debug(mod.__name__ + ": loading module configuration: '" + configfile + "'")
176 if not hasattr(mod, 'config'):
180 self._parse_config(mod, read_config(configfile)))
181 except Exception as e:
182 msg.error(mod.__name__ + ": cannot parse configuration file '" + configfile + "':", str(e))
184 msg.error(mod.__name__ + ": configuration file '" + configfile + "' not found. Using defaults.")
185 # set config if not found
186 if not hasattr(mod, 'config'):
187 msg.debug(mod.__name__ + ": setting configuration for only one job")
188 mod.config = {None: {}}
189 for var in BASE_CONFIG:
191 mod.config[None][var] = getattr(mod, var)
192 except AttributeError:
193 mod.config[None][var] = BASE_CONFIG[var]
197 def _parse_config(module, config):
199 Parse configuration file or extract configuration from module file.
200 Example of returned dictionary:
206 :param module: object
214 msg.debug(module.__name__ + ": reading configuration")
215 for key in BASE_CONFIG:
217 # get defaults from module config
218 defaults[key] = int(config.pop(key))
219 except (KeyError, ValueError):
221 # get defaults from module source code
222 defaults[key] = getattr(module, key)
223 except (KeyError, ValueError, AttributeError):
224 # if above failed, get defaults from global dict
225 defaults[key] = BASE_CONFIG[key]
227 # check if there are dict in config dict
230 if type(config[name]) is dict:
234 # assign variables needed by supervisor to every job configuration
238 if key not in config[name]:
239 config[name][key] = defaults[key]
240 # if only one job is needed, values doesn't have to be in dict (in YAML)
242 config = {None: config.copy()}
243 config[None].update(defaults)
245 # return dictionary of jobs where every job has BASE_CONFIG variables
249 def _create_jobs(modules):
251 Create jobs based on module.config dictionary and module.Service class definition.
256 for module in modules:
257 for name in module.config:
259 conf = module.config[name]
261 job = module.Service(configuration=conf, name=name)
262 except Exception as e:
263 msg.error(module.__name__ +
264 ("/" + str(name) if name is not None else "") +
265 ": cannot start job: '" +
269 # set chart_name (needed to plot run time graphs)
270 job.chart_name = module.__name__
272 job.chart_name += "_" + name
274 msg.debug(module.__name__ + ("/" + str(name) if name is not None else "") + ": job added")
276 return [j for j in jobs if j is not None]
278 def _stop(self, job, reason=None):
280 Stop specified job and remove it from self.jobs list
281 Also notifies user about job failure if DEBUG_FLAG is set
285 prefix = job.__module__
286 if job.name is not None:
287 prefix += "/" + job.name
290 self.jobs.remove(job)
293 elif reason[:3] == "no ":
295 "does not seem to have " +
297 "() function. Disabling it.")
298 elif reason[:7] == "failed ":
301 "() function reports failure.")
302 elif reason[:13] == "configuration":
304 "configuration file '" +
307 ".conf' not found. Using defaults.")
308 elif reason[:11] == "misbehaving":
309 msg.error(prefix + "is " + reason)
313 Tries to execute check() on every job.
314 This cannot fail thus it is catching every exception
315 If job.check() fails job is stopped
319 while i < len(self.jobs):
321 if job.name in overridden:
323 msg.error(job.name + " already exists")
326 self._stop(job, "failed check")
328 msg.debug(job.chart_name, ": check succeeded")
331 if job.override_name is not None:
332 job.name = job.override_name
333 msg.debug(job.chart_name + " changing chart name to: " + job.__module__ + job.name)
334 job.chart_name = job.__module__ + '_' + job.name
335 overridden.append(job.name)
338 except AttributeError:
339 self._stop(job, "no check")
340 except (UnboundLocalError, Exception) as e:
341 self._stop(job, "misbehaving. Reason:" + str(e))
345 Tries to execute create() on every job.
346 This cannot fail thus it is catching every exception.
347 If job.create() fails job is stopped.
348 This is also creating job run time chart.
351 while i < len(self.jobs):
355 self._stop(job, "failed create")
357 chart = job.chart_name
359 "CHART netdata.plugin_pythond_" +
361 " '' 'Execution time for " +
363 " plugin' 'milliseconds / run' python.d netdata.plugin_python area 145000 " +
364 str(job.timetable['freq']) +
366 sys.stdout.write("DIMENSION run_time 'run time' absolute 1 1\n\n")
367 msg.debug("created charts for", job.chart_name)
370 except AttributeError:
371 self._stop(job, "no create")
372 except (UnboundLocalError, Exception) as e:
373 self._stop(job, "misbehaving. Reason: " + str(e))
377 Creates and supervises every job thread.
378 This will stay forever and ever and ever forever and ever it'll be the one...
380 for job in self.jobs:
384 if threading.active_count() <= 1:
385 msg.fatal("no more jobs")
389 def read_config(path):
391 Read YAML configuration from specified file
396 with open(path, 'r') as stream:
397 config = yaml.load(stream)
398 except (OSError, IOError):
399 msg.error(str(path), "is not a valid configuration file")
401 except yaml.YAMLError as e:
402 msg.error(str(path), "is malformed:", e)
407 def parse_cmdline(directory, *commands):
409 Parse parameters from command line.
410 :param directory: str
411 :param commands: list of str
415 global OVERRIDE_UPDATE_EVERY
418 changed_update = False
420 for cmd in commands[1:]:
423 elif cmd == "debug" or cmd == "all":
425 # redirect stderr to stdout?
426 elif os.path.isfile(directory + cmd + ".chart.py") or os.path.isfile(directory + cmd):
428 mods.append(cmd.replace(".chart.py", ""))
431 BASE_CONFIG['update_every'] = int(cmd)
432 changed_update = True
435 if changed_update and DEBUG_FLAG:
436 OVERRIDE_UPDATE_EVERY = True
437 msg.debug(PROGRAM, "overriding update interval to", str(BASE_CONFIG['update_every']))
439 msg.debug("started from", commands[0], "with options:", *commands[1:])
444 # if __name__ == '__main__':
449 global DEBUG_FLAG, BASE_CONFIG
451 # read configuration file
453 configfile = CONFIG_DIR + "python.d.conf"
454 msg.PROGRAM = PROGRAM
455 msg.info("reading configuration file:", configfile)
457 conf = read_config(configfile)
460 # exit the whole plugin when 'enabled: no' is set in 'python.d.conf'
461 if conf['enabled'] is False:
462 msg.fatal('disabled in configuration file.\n')
463 except (KeyError, TypeError):
466 for param in BASE_CONFIG:
467 BASE_CONFIG[param] = conf[param]
468 except (KeyError, TypeError):
469 pass # use default update_every from NETDATA_UPDATE_EVERY
471 DEBUG_FLAG = conf['debug']
472 except (KeyError, TypeError):
474 for k, v in conf.items():
475 if k in ("update_every", "debug", "enabled"):
480 # parse passed command line arguments
481 modules = parse_cmdline(MODULES_DIR, *sys.argv)
482 msg.DEBUG_FLAG = DEBUG_FLAG
483 msg.info("MODULES_DIR='" + MODULES_DIR +
484 "', CONFIG_DIR='" + CONFIG_DIR +
485 "', UPDATE_EVERY=" + str(BASE_CONFIG['update_every']) +
486 ", ONLY_MODULES=" + str(modules))
489 charts = PythonCharts(modules, MODULES_DIR, CONFIG_DIR + "python.d/", disabled)
493 msg.fatal("finished")
496 if __name__ == '__main__':