2 '''':; exec "$(command -v python || command -v python3 || command -v python2 || echo "ERROR python IS NOT AVAILABLE IN THIS SYSTEM")" "$0" "$@" # '''
3 # -*- coding: utf-8 -*-
5 # Description: netdata python modules supervisor
6 # Author: Pawel Krupa (paulfantom)
13 # -----------------------------------------------------------------------------
14 # globals & environment setup
15 # https://github.com/firehol/netdata/wiki/External-Plugins#environment-variables
16 MODULE_EXTENSION = ".chart.py"
17 BASE_CONFIG = {'update_every': os.getenv('NETDATA_UPDATE_EVERY', 1),
21 MODULES_DIR = os.path.abspath(os.getenv('NETDATA_PLUGINS_DIR',
22 os.path.dirname(__file__)) + "/../python.d") + "/"
23 CONFIG_DIR = os.getenv('NETDATA_CONFIG_DIR', "/etc/netdata/")
24 # directories should end with '/'
25 if CONFIG_DIR[-1] != "/":
27 sys.path.append(MODULES_DIR + "python_modules")
29 PROGRAM = os.path.basename(__file__).replace(".plugin", "")
31 OVERRIDE_UPDATE_EVERY = False
33 # -----------------------------------------------------------------------------
34 # custom, third party and version specific python modules management
38 assert sys.version_info >= (3, 1)
39 import importlib.machinery
41 # change this hack below if we want PY_VERSION to be used in modules
43 # builtins.PY_VERSION = 3
44 msg.info('Using python v3')
45 except (AssertionError, ImportError):
49 # change this hack below if we want PY_VERSION to be used in modules
51 # __builtin__.PY_VERSION = 2
53 msg.info('Using python v2')
55 msg.fatal('Cannot start. No importlib.machinery on python3 or lack of imp on python2')
59 # msg.fatal('Cannot find yaml library')
62 import pyyaml3 as yaml
64 import pyyaml2 as yaml
66 msg.fatal('Cannot find yaml library')
69 class PythonCharts(object):
71 Main class used to control every python module.
76 modules_path='../python.d/',
77 modules_configs='../conf.d/',
78 modules_disabled=None):
81 :param modules_path: str
82 :param modules_configs: str
83 :param modules_disabled: list
88 if modules_disabled is None:
92 # set configuration directory
93 self.configs = modules_configs
96 loaded_modules = self._load_modules(modules_path, modules, modules_disabled)
98 # load configuration files
99 configured_modules = self._load_configs(loaded_modules)
101 # good economy and prosperity:
102 self.jobs = self._create_jobs(configured_modules) # type: list
104 # enable timetable override like `python.d.plugin mysql debug 1`
105 if DEBUG_FLAG and OVERRIDE_UPDATE_EVERY:
106 for job in self.jobs:
107 job.create_timetable(BASE_CONFIG['update_every'])
110 def _import_module(path, name=None):
112 Try to import module using only its path.
119 name = path.split('/')[-1]
120 if name[-len(MODULE_EXTENSION):] != MODULE_EXTENSION:
122 name = name[:-len(MODULE_EXTENSION)]
125 return importlib.machinery.SourceFileLoader(name, path).load_module()
127 return imp.load_source(name, path)
128 except Exception as e:
129 msg.error("Problem loading", name, str(e))
132 def _load_modules(self, path, modules, disabled):
134 Load modules from 'modules' list or dynamically every file from 'path' (only .chart.py files)
137 :param disabled: list
141 # check if plugin directory exists
142 if not os.path.isdir(path):
143 msg.fatal("cannot find charts directory ", path)
151 mod = self._import_module(path + m + MODULE_EXTENSION)
154 else: # exit if plugin is not found
155 msg.fatal('no modules found.')
157 # scan directory specified in path and load all modules from there
158 names = os.listdir(path)
160 if mod.replace(MODULE_EXTENSION, "") in disabled:
161 msg.error(mod + ": disabled module ", mod.replace(MODULE_EXTENSION, ""))
163 m = self._import_module(path + mod)
165 msg.debug(mod + ": loading module '" + path + mod + "'")
169 def _load_configs(self, modules):
171 Append configuration in list named `config` to every module.
172 For multi-job modules `config` list is created in _parse_config,
173 otherwise it is created here based on BASE_CONFIG prototype with None as identifier.
178 configfile = self.configs + mod.__name__ + ".conf"
179 if os.path.isfile(configfile):
180 msg.debug(mod.__name__ + ": loading module configuration: '" + configfile + "'")
182 if not hasattr(mod, 'config'):
186 self._parse_config(mod, read_config(configfile)))
187 except Exception as e:
188 msg.error(mod.__name__ + ": cannot parse configuration file '" + configfile + "':", str(e))
190 msg.error(mod.__name__ + ": configuration file '" + configfile + "' not found. Using defaults.")
191 # set config if not found
192 if not hasattr(mod, 'config'):
193 msg.debug(mod.__name__ + ": setting configuration for only one job")
194 mod.config = {None: {}}
195 for var in BASE_CONFIG:
197 mod.config[None][var] = getattr(mod, var)
198 except AttributeError:
199 mod.config[None][var] = BASE_CONFIG[var]
203 def _parse_config(module, config):
205 Parse configuration file or extract configuration from module file.
206 Example of returned dictionary:
212 :param module: object
220 msg.debug(module.__name__ + ": reading configuration")
221 for key in BASE_CONFIG:
223 # get defaults from module config
224 defaults[key] = int(config.pop(key))
225 except (KeyError, ValueError):
227 # get defaults from module source code
228 defaults[key] = getattr(module, key)
229 except (KeyError, ValueError, AttributeError):
230 # if above failed, get defaults from global dict
231 defaults[key] = BASE_CONFIG[key]
233 # check if there are dict in config dict
236 if type(config[name]) is dict:
240 # assign variables needed by supervisor to every job configuration
244 if key not in config[name]:
245 config[name][key] = defaults[key]
246 # if only one job is needed, values doesn't have to be in dict (in YAML)
248 config = {None: config.copy()}
249 config[None].update(defaults)
251 # return dictionary of jobs where every job has BASE_CONFIG variables
255 def _create_jobs(modules):
257 Create jobs based on module.config dictionary and module.Service class definition.
262 for module in modules:
263 for name in module.config:
265 conf = module.config[name]
267 job = module.Service(configuration=conf, name=name)
268 except Exception as e:
269 msg.error(module.__name__ +
270 ("/" + str(name) if name is not None else "") +
271 ": cannot start job: '" +
275 # set chart_name (needed to plot run time graphs)
276 job.chart_name = module.__name__
278 job.chart_name += "_" + name
280 msg.debug(module.__name__ + ("/" + str(name) if name is not None else "") + ": job added")
282 return [j for j in jobs if j is not None]
284 def _stop(self, job, reason=None):
286 Stop specified job and remove it from self.jobs list
287 Also notifies user about job failure if DEBUG_FLAG is set
291 prefix = job.__module__
292 if job.name is not None and len(job.name) != 0:
293 prefix += "/" + job.name
295 self.jobs.remove(job)
296 msg.info("Disabled", prefix)
297 except Exception as e:
298 msg.debug("This shouldn't happen. NO " + prefix + " IN LIST:" + str(self.jobs) + " ERROR: " + str(e))
300 # TODO remove section below and remove `reason`.
304 elif reason[:3] == "no ":
306 "does not seem to have " +
308 "() function. Disabling it.")
309 elif reason[:7] == "failed ":
312 "() function reports failure.")
313 elif reason[:13] == "configuration":
315 "configuration file '" +
318 ".conf' not found. Using defaults.")
319 elif reason[:11] == "misbehaving":
320 msg.error(prefix + "is " + reason)
324 Tries to execute check() on every job.
325 This cannot fail thus it is catching every exception
326 If job.check() fails job is stopped
330 msg.debug("all job objects", str(self.jobs))
331 while i < len(self.jobs):
335 msg.error(job.chart_name, "check function failed.")
338 msg.debug(job.chart_name, "check succeeded")
341 if job.override_name is not None:
342 new_name = job.__module__ + '_' + job.override_name
343 if new_name in overridden:
344 msg.error(job.chart_name + " already exists. Stopping '" + job.name + "'")
348 job.name = job.override_name
349 msg.debug(job.chart_name + " changing chart name to: '" + new_name + "'")
350 job.chart_name = new_name
351 overridden.append(job.chart_name)
354 except AttributeError:
356 msg.error(job.chart_name, "cannot find check() function.")
357 except (UnboundLocalError, Exception) as e:
358 msg.error(job.chart_name, str(e))
360 msg.debug("overridden job names:", str(overridden))
361 msg.debug("all remaining job objects:", str(self.jobs))
365 Tries to execute create() on every job.
366 This cannot fail thus it is catching every exception.
367 If job.create() fails job is stopped.
368 This is also creating job run time chart.
371 while i < len(self.jobs):
375 msg.error(job.chart_name, "create function failed.")
378 chart = job.chart_name
380 "CHART netdata.plugin_pythond_" +
382 " '' 'Execution time for " +
384 " plugin' 'milliseconds / run' python.d netdata.plugin_python area 145000 " +
385 str(job.timetable['freq']) +
387 sys.stdout.write("DIMENSION run_time 'run time' absolute 1 1\n\n")
388 msg.debug("created charts for", job.chart_name)
391 except AttributeError:
392 msg.error(job.chart_name, "cannot find create() function.")
394 except (UnboundLocalError, Exception) as e:
395 msg.error(job.chart_name, str(e))
400 Creates and supervises every job thread.
401 This will stay forever and ever and ever forever and ever it'll be the one...
403 for job in self.jobs:
407 if threading.active_count() <= 1:
408 msg.fatal("no more jobs")
412 def read_config(path):
414 Read YAML configuration from specified file
419 with open(path, 'r') as stream:
420 config = yaml.load(stream)
421 except (OSError, IOError):
422 msg.error(str(path), "is not a valid configuration file")
424 except yaml.YAMLError as e:
425 msg.error(str(path), "is malformed:", e)
430 def parse_cmdline(directory, *commands):
432 Parse parameters from command line.
433 :param directory: str
434 :param commands: list of str
438 global OVERRIDE_UPDATE_EVERY
441 changed_update = False
443 for cmd in commands[1:]:
446 elif cmd == "debug" or cmd == "all":
448 # redirect stderr to stdout?
449 elif os.path.isfile(directory + cmd + ".chart.py") or os.path.isfile(directory + cmd):
451 mods.append(cmd.replace(".chart.py", ""))
454 BASE_CONFIG['update_every'] = int(cmd)
455 changed_update = True
458 if changed_update and DEBUG_FLAG:
459 OVERRIDE_UPDATE_EVERY = True
460 msg.debug(PROGRAM, "overriding update interval to", str(BASE_CONFIG['update_every']))
462 msg.debug("started from", commands[0], "with options:", *commands[1:])
467 # if __name__ == '__main__':
472 global DEBUG_FLAG, BASE_CONFIG
474 # read configuration file
476 configfile = CONFIG_DIR + "python.d.conf"
477 msg.PROGRAM = PROGRAM
478 msg.info("reading configuration file:", configfile)
480 conf = read_config(configfile)
483 # exit the whole plugin when 'enabled: no' is set in 'python.d.conf'
484 if conf['enabled'] is False:
485 msg.fatal('disabled in configuration file.\n')
486 except (KeyError, TypeError):
489 for param in BASE_CONFIG:
490 BASE_CONFIG[param] = conf[param]
491 except (KeyError, TypeError):
492 pass # use default update_every from NETDATA_UPDATE_EVERY
494 DEBUG_FLAG = conf['debug']
495 except (KeyError, TypeError):
497 for k, v in conf.items():
498 if k in ("update_every", "debug", "enabled"):
503 # parse passed command line arguments
504 modules = parse_cmdline(MODULES_DIR, *sys.argv)
505 msg.DEBUG_FLAG = DEBUG_FLAG
506 msg.info("MODULES_DIR='" + MODULES_DIR +
507 "', CONFIG_DIR='" + CONFIG_DIR +
508 "', UPDATE_EVERY=" + str(BASE_CONFIG['update_every']) +
509 ", ONLY_MODULES=" + str(modules))
512 charts = PythonCharts(modules, MODULES_DIR, CONFIG_DIR + "python.d/", disabled)
516 msg.fatal("finished")
519 if __name__ == '__main__':