2 '''':; exec "$(command -v python2 || command -v python3 || echo "ERROR python IS NOT AVAILABLE IN THIS SYSTEM")" "$0" "$@" # '''
3 # -*- coding: utf-8 -*-
5 # Description: netdata python modules supervisor
6 # Author: Pawel Krupa (paulfantom)
13 # -----------------------------------------------------------------------------
14 # globals & environment setup
15 # https://github.com/firehol/netdata/wiki/External-Plugins#environment-variables
16 MODULE_EXTENSION = ".chart.py"
17 BASE_CONFIG = {'update_every': os.getenv('NETDATA_UPDATE_EVERY', 1),
21 MODULES_DIR = os.path.abspath(os.getenv('NETDATA_PLUGINS_DIR',
22 os.path.dirname(__file__)) + "/../python.d") + "/"
23 CONFIG_DIR = os.getenv('NETDATA_CONFIG_DIR', "/etc/netdata/")
24 # directories should end with '/'
25 if CONFIG_DIR[-1] != "/":
27 sys.path.append(MODULES_DIR + "python_modules")
29 PROGRAM = os.path.basename(__file__).replace(".plugin", "")
31 OVERRIDE_UPDATE_EVERY = False
33 # -----------------------------------------------------------------------------
34 # custom, third party and version specific python modules management
38 assert sys.version_info >= (3, 1)
39 import importlib.machinery
41 # change this hack below if we want PY_VERSION to be used in modules
43 # builtins.PY_VERSION = 3
45 msg.info('Using python v3')
46 except (AssertionError, ImportError):
50 # change this hack below if we want PY_VERSION to be used in modules
52 # __builtin__.PY_VERSION = 2
54 msg.info('Using python v2')
56 msg.fatal('Cannot start. No importlib.machinery on python3 or lack of imp on python2')
60 msg.fatal('Cannot find yaml library')
63 class PythonCharts(object):
65 Main class used to control every python module.
70 modules_path='../python.d/',
71 modules_configs='../conf.d/',
72 modules_disabled=None):
75 :param modules_path: str
76 :param modules_configs: str
77 :param modules_disabled: list
82 if modules_disabled is None:
86 # set configuration directory
87 self.configs = modules_configs
90 loaded_modules = self._load_modules(modules_path, modules, modules_disabled)
92 # load configuration files
93 configured_modules = self._load_configs(loaded_modules)
95 # good economy and prosperity:
96 self.jobs = self._create_jobs(configured_modules) # type: list
98 # enable timetable override like `python.d.plugin mysql debug 1`
99 if DEBUG_FLAG and OVERRIDE_UPDATE_EVERY:
100 for job in self.jobs:
101 job.create_timetable(BASE_CONFIG['update_every'])
104 def _import_module(path, name=None):
106 Try to import module using only its path.
113 name = path.split('/')[-1]
114 if name[-len(MODULE_EXTENSION):] != MODULE_EXTENSION:
116 name = name[:-len(MODULE_EXTENSION)]
119 return importlib.machinery.SourceFileLoader(name, path).load_module()
121 return imp.load_source(name, path)
122 except Exception as e:
123 msg.error("Problem loading", name, str(e))
126 def _load_modules(self, path, modules, disabled):
128 Load modules from 'modules' list or dynamically every file from 'path' (only .chart.py files)
131 :param disabled: list
135 # check if plugin directory exists
136 if not os.path.isdir(path):
137 msg.fatal("cannot find charts directory ", path)
145 mod = self._import_module(path + m + MODULE_EXTENSION)
148 else: # exit if plugin is not found
149 msg.fatal('no modules found.')
151 # scan directory specified in path and load all modules from there
152 names = os.listdir(path)
154 if mod.replace(MODULE_EXTENSION, "") in disabled:
155 msg.error(mod + ": disabled module ", mod.replace(MODULE_EXTENSION, ""))
157 m = self._import_module(path + mod)
159 msg.debug(mod + ": loading module '" + path + mod + "'")
163 def _load_configs(self, modules):
165 Append configuration in list named `config` to every module.
166 For multi-job modules `config` list is created in _parse_config,
167 otherwise it is created here based on BASE_CONFIG prototype with None as identifier.
172 configfile = self.configs + mod.__name__ + ".conf"
173 if os.path.isfile(configfile):
174 msg.debug(mod.__name__ + ": loading module configuration: '" + configfile + "'")
176 if not hasattr(mod, 'config'):
180 self._parse_config(mod, read_config(configfile)))
181 except Exception as e:
182 msg.error(mod.__name__ + ": cannot parse configuration file '" + configfile + "':", str(e))
184 msg.error(mod.__name__ + ": configuration file '" + configfile + "' not found. Using defaults.")
185 # set config if not found
186 if not hasattr(mod, 'config'):
187 msg.debug(mod.__name__ + ": setting configuration for only one job")
188 mod.config = {None: {}}
189 for var in BASE_CONFIG:
191 mod.config[None][var] = getattr(mod, var)
192 except AttributeError:
193 mod.config[None][var] = BASE_CONFIG[var]
197 def _parse_config(module, config):
199 Parse configuration file or extract configuration from module file.
200 Example of returned dictionary:
206 :param module: object
214 msg.debug(module.__name__ + ": reading configuration")
215 for key in BASE_CONFIG:
217 # get defaults from module config
218 defaults[key] = int(config.pop(key))
219 except (KeyError, ValueError):
221 # get defaults from module source code
222 defaults[key] = getattr(module, key)
223 except (KeyError, ValueError, AttributeError):
224 # if above failed, get defaults from global dict
225 defaults[key] = BASE_CONFIG[key]
227 # check if there are dict in config dict
230 if type(config[name]) is dict:
234 # assign variables needed by supervisor to every job configuration
238 if key not in config[name]:
239 config[name][key] = defaults[key]
240 # if only one job is needed, values doesn't have to be in dict (in YAML)
242 config = {None: config.copy()}
243 config[None].update(defaults)
245 # return dictionary of jobs where every job has BASE_CONFIG variables
249 def _create_jobs(modules):
251 Create jobs based on module.config dictionary and module.Service class definition.
256 for module in modules:
257 for name in module.config:
259 conf = module.config[name]
261 job = module.Service(configuration=conf, name=name)
262 except Exception as e:
263 msg.error(module.__name__ +
264 ("/" + str(name) if name is not None else "") +
265 ": cannot start job: '" +
269 # set chart_name (needed to plot run time graphs)
270 job.chart_name = module.__name__
272 job.chart_name += "_" + name
274 msg.debug(module.__name__ + ("/" + str(name) if name is not None else "") + ": job added")
276 return [j for j in jobs if j is not None]
278 def _stop(self, job, reason=None):
280 Stop specified job and remove it from self.jobs list
281 Also notifies user about job failure if DEBUG_FLAG is set
285 prefix = job.__module__
286 if job.name is not None and len(job.name) != 0:
287 prefix += "/" + job.name
290 self.jobs.remove(job)
291 except Exception as e:
292 msg.debug("This shouldn't happen. NO " + prefix + " IN LIST:" + str(self.jobs))
295 elif reason[:3] == "no ":
297 "does not seem to have " +
299 "() function. Disabling it.")
300 elif reason[:7] == "failed ":
303 "() function reports failure.")
304 elif reason[:13] == "configuration":
306 "configuration file '" +
309 ".conf' not found. Using defaults.")
310 elif reason[:11] == "misbehaving":
311 msg.error(prefix + "is " + reason)
315 Tries to execute check() on every job.
316 This cannot fail thus it is catching every exception
317 If job.check() fails job is stopped
321 while i < len(self.jobs):
322 msg.error(*overridden)
326 self._stop(job, "failed check")
328 msg.debug(job.chart_name, ": check succeeded")
331 if job.override_name is not None:
333 job.name = job.override_name
334 msg.debug(job.chart_name + " changing chart name to: " + job.__module__ + '_' + job.name)
335 job.chart_name = job.__module__ + '_' + job.name
336 if job.chart_name in overridden:
338 msg.error(job.chart_name + " already exists. Created with '" + tmp + "'")
340 overridden.append(job.chart_name)
343 except AttributeError:
344 self._stop(job, "no check")
345 except (UnboundLocalError, Exception) as e:
346 self._stop(job, "misbehaving. Reason:" + str(e))
350 Tries to execute create() on every job.
351 This cannot fail thus it is catching every exception.
352 If job.create() fails job is stopped.
353 This is also creating job run time chart.
356 while i < len(self.jobs):
360 self._stop(job, "failed create")
362 chart = job.chart_name
364 "CHART netdata.plugin_pythond_" +
366 " '' 'Execution time for " +
368 " plugin' 'milliseconds / run' python.d netdata.plugin_python area 145000 " +
369 str(job.timetable['freq']) +
371 sys.stdout.write("DIMENSION run_time 'run time' absolute 1 1\n\n")
372 msg.debug("created charts for", job.chart_name)
375 except AttributeError:
376 self._stop(job, "no create")
377 except (UnboundLocalError, Exception) as e:
378 self._stop(job, "misbehaving. Reason: " + str(e))
382 Creates and supervises every job thread.
383 This will stay forever and ever and ever forever and ever it'll be the one...
385 for job in self.jobs:
389 if threading.active_count() <= 1:
390 msg.fatal("no more jobs")
394 def read_config(path):
396 Read YAML configuration from specified file
401 with open(path, 'r') as stream:
402 config = yaml.load(stream)
403 except (OSError, IOError):
404 msg.error(str(path), "is not a valid configuration file")
406 except yaml.YAMLError as e:
407 msg.error(str(path), "is malformed:", e)
412 def parse_cmdline(directory, *commands):
414 Parse parameters from command line.
415 :param directory: str
416 :param commands: list of str
420 global OVERRIDE_UPDATE_EVERY
423 changed_update = False
425 for cmd in commands[1:]:
428 elif cmd == "debug" or cmd == "all":
430 # redirect stderr to stdout?
431 elif os.path.isfile(directory + cmd + ".chart.py") or os.path.isfile(directory + cmd):
433 mods.append(cmd.replace(".chart.py", ""))
436 BASE_CONFIG['update_every'] = int(cmd)
437 changed_update = True
440 if changed_update and DEBUG_FLAG:
441 OVERRIDE_UPDATE_EVERY = True
442 msg.debug(PROGRAM, "overriding update interval to", str(BASE_CONFIG['update_every']))
444 msg.debug("started from", commands[0], "with options:", *commands[1:])
449 # if __name__ == '__main__':
454 global DEBUG_FLAG, BASE_CONFIG
456 # read configuration file
458 configfile = CONFIG_DIR + "python.d.conf"
459 msg.PROGRAM = PROGRAM
460 msg.info("reading configuration file:", configfile)
462 conf = read_config(configfile)
465 # exit the whole plugin when 'enabled: no' is set in 'python.d.conf'
466 if conf['enabled'] is False:
467 msg.fatal('disabled in configuration file.\n')
468 except (KeyError, TypeError):
471 for param in BASE_CONFIG:
472 BASE_CONFIG[param] = conf[param]
473 except (KeyError, TypeError):
474 pass # use default update_every from NETDATA_UPDATE_EVERY
476 DEBUG_FLAG = conf['debug']
477 except (KeyError, TypeError):
479 for k, v in conf.items():
480 if k in ("update_every", "debug", "enabled"):
485 # parse passed command line arguments
486 modules = parse_cmdline(MODULES_DIR, *sys.argv)
487 msg.DEBUG_FLAG = DEBUG_FLAG
488 modules = ['sensors']
489 msg.info("MODULES_DIR='" + MODULES_DIR +
490 "', CONFIG_DIR='" + CONFIG_DIR +
491 "', UPDATE_EVERY=" + str(BASE_CONFIG['update_every']) +
492 ", ONLY_MODULES=" + str(modules))
495 charts = PythonCharts(modules, MODULES_DIR, CONFIG_DIR + "python.d/", disabled)
499 msg.fatal("finished")
502 if __name__ == '__main__':