2 '''':; exec "$(command -v python2 || command -v python3 || echo "ERROR python IS NOT AVAILABLE IN THIS SYSTEM")" "$0" "$@" # '''
3 # -*- coding: utf-8 -*-
5 # Description: netdata python modules supervisor
6 # Author: Pawel Krupa (paulfantom)
13 # -----------------------------------------------------------------------------
14 # globals & environment setup
15 # https://github.com/firehol/netdata/wiki/External-Plugins#environment-variables
16 MODULE_EXTENSION = ".chart.py"
17 BASE_CONFIG = {'update_every': os.getenv('NETDATA_UPDATE_EVERY', 1),
21 MODULES_DIR = os.path.abspath(os.getenv('NETDATA_PLUGINS_DIR',
22 os.path.dirname(__file__)) + "/../python.d") + "/"
23 CONFIG_DIR = os.getenv('NETDATA_CONFIG_DIR', "/etc/netdata/")
24 # directories should end with '/'
25 if CONFIG_DIR[-1] != "/":
27 sys.path.append(MODULES_DIR + "python_modules")
29 PROGRAM = os.path.basename(__file__).replace(".plugin", "")
31 OVERRIDE_UPDATE_EVERY = False
33 # -----------------------------------------------------------------------------
34 # custom, third party and version specific python modules management
38 assert sys.version_info >= (3, 1)
39 import importlib.machinery
41 # change this hack below if we want PY_VERSION to be used in modules
43 # builtins.PY_VERSION = 3
45 msg.info('Using python v3')
46 except (AssertionError, ImportError):
50 # change this hack below if we want PY_VERSION to be used in modules
52 # __builtin__.PY_VERSION = 2
54 msg.info('Using python v2')
56 msg.fatal('Cannot start. No importlib.machinery on python3 or lack of imp on python2')
60 msg.fatal('Cannot find yaml library')
63 class PythonCharts(object):
65 Main class used to control every python module.
70 modules_path='../python.d/',
71 modules_configs='../conf.d/',
72 modules_disabled=None):
75 :param modules_path: str
76 :param modules_configs: str
77 :param modules_disabled: list
82 if modules_disabled is None:
86 # set configuration directory
87 self.configs = modules_configs
90 loaded_modules = self._load_modules(modules_path, modules, modules_disabled)
92 # load configuration files
93 configured_modules = self._load_configs(loaded_modules)
95 # good economy and prosperity:
96 self.jobs = self._create_jobs(configured_modules) # type: list
98 # enable timetable override like `python.d.plugin mysql debug 1`
99 if DEBUG_FLAG and OVERRIDE_UPDATE_EVERY:
100 for job in self.jobs:
101 job.create_timetable(BASE_CONFIG['update_every'])
104 def _import_module(path, name=None):
106 Try to import module using only its path.
113 name = path.split('/')[-1]
114 if name[-len(MODULE_EXTENSION):] != MODULE_EXTENSION:
116 name = name[:-len(MODULE_EXTENSION)]
119 return importlib.machinery.SourceFileLoader(name, path).load_module()
121 return imp.load_source(name, path)
122 except Exception as e:
123 msg.error("Problem loading", name, str(e))
126 def _load_modules(self, path, modules, disabled):
128 Load modules from 'modules' list or dynamically every file from 'path' (only .chart.py files)
131 :param disabled: list
135 # check if plugin directory exists
136 if not os.path.isdir(path):
137 msg.fatal("cannot find charts directory ", path)
145 mod = self._import_module(path + m + MODULE_EXTENSION)
148 else: # exit if plugin is not found
149 msg.fatal('no modules found.')
151 # scan directory specified in path and load all modules from there
152 names = os.listdir(path)
154 if mod.replace(MODULE_EXTENSION, "") in disabled:
155 msg.error(mod + ": disabled module ", mod.replace(MODULE_EXTENSION, ""))
157 m = self._import_module(path + mod)
159 msg.debug(mod + ": loading module '" + path + mod + "'")
163 def _load_configs(self, modules):
165 Append configuration in list named `config` to every module.
166 For multi-job modules `config` list is created in _parse_config,
167 otherwise it is created here based on BASE_CONFIG prototype with None as identifier.
172 configfile = self.configs + mod.__name__ + ".conf"
173 if os.path.isfile(configfile):
174 msg.debug(mod.__name__ + ": loading module configuration: '" + configfile + "'")
176 if not hasattr(mod, 'config'):
180 self._parse_config(mod, read_config(configfile)))
181 except Exception as e:
182 msg.error(mod.__name__ + ": cannot parse configuration file '" + configfile + "':", str(e))
184 msg.error(mod.__name__ + ": configuration file '" + configfile + "' not found. Using defaults.")
185 # set config if not found
186 if not hasattr(mod, 'config'):
187 msg.debug(mod.__name__ + ": setting configuration for only one job")
188 mod.config = {None: {}}
189 for var in BASE_CONFIG:
191 mod.config[None][var] = getattr(mod, var)
192 except AttributeError:
193 mod.config[None][var] = BASE_CONFIG[var]
197 def _parse_config(module, config):
199 Parse configuration file or extract configuration from module file.
200 Example of returned dictionary:
206 :param module: object
214 msg.debug(module.__name__ + ": reading configuration")
215 for key in BASE_CONFIG:
217 # get defaults from module config
218 defaults[key] = int(config.pop(key))
219 except (KeyError, ValueError):
221 # get defaults from module source code
222 defaults[key] = getattr(module, key)
223 except (KeyError, ValueError, AttributeError):
224 # if above failed, get defaults from global dict
225 defaults[key] = BASE_CONFIG[key]
227 # check if there are dict in config dict
230 if type(config[name]) is dict:
234 # assign variables needed by supervisor to every job configuration
238 if key not in config[name]:
239 config[name][key] = defaults[key]
240 # if only one job is needed, values doesn't have to be in dict (in YAML)
242 config = {None: config.copy()}
243 config[None].update(defaults)
245 # return dictionary of jobs where every job has BASE_CONFIG variables
249 def _create_jobs(modules):
251 Create jobs based on module.config dictionary and module.Service class definition.
256 for module in modules:
257 for name in module.config:
259 conf = module.config[name]
261 job = module.Service(configuration=conf, name=name)
262 except Exception as e:
263 msg.error(module.__name__ +
264 ("/" + str(name) if name is not None else "") +
265 ": cannot start job: '" +
269 # set chart_name (needed to plot run time graphs)
270 job.chart_name = module.__name__
272 job.chart_name += "_" + name
274 msg.debug(module.__name__ + ("/" + str(name) if name is not None else "") + ": job added")
276 return [j for j in jobs if j is not None]
278 def _stop(self, job, reason=None):
280 Stop specified job and remove it from self.jobs list
281 Also notifies user about job failure if DEBUG_FLAG is set
285 prefix = job.__module__
286 if job.name is not None and len(job.name) != 0:
287 prefix += "/" + job.name
289 self.jobs.remove(job)
290 msg.info("Disabled", prefix)
291 except Exception as e:
292 msg.debug("This shouldn't happen. NO " + prefix + " IN LIST:" + str(self.jobs) + " ERROR: " + str(e))
297 elif reason[:3] == "no ":
299 "does not seem to have " +
301 "() function. Disabling it.")
302 elif reason[:7] == "failed ":
305 "() function reports failure.")
306 elif reason[:13] == "configuration":
308 "configuration file '" +
311 ".conf' not found. Using defaults.")
312 elif reason[:11] == "misbehaving":
313 msg.error(prefix + "is " + reason)
317 Tries to execute check() on every job.
318 This cannot fail thus it is catching every exception
319 If job.check() fails job is stopped
323 while i < len(self.jobs):
327 self._stop(job, "failed check")
329 msg.debug(job.chart_name, ": check succeeded")
332 if job.override_name is not None:
333 new_name = job.__module__ + '_' + job.override_name
334 if new_name in overridden:
335 msg.error(job.chart_name + " already exists. Stopping '" + job.name + "'")
339 job.name = job.override_name
340 msg.debug(job.chart_name + " changing chart name to: '" + new_name + "'")
341 job.chart_name = new_name
342 overridden.append(job.chart_name)
345 except AttributeError:
346 self._stop(job, "no check")
347 except (UnboundLocalError, Exception) as e:
348 self._stop(job, "misbehaving. Reason:" + str(e))
352 Tries to execute create() on every job.
353 This cannot fail thus it is catching every exception.
354 If job.create() fails job is stopped.
355 This is also creating job run time chart.
358 while i < len(self.jobs):
362 self._stop(job, "failed create")
364 chart = job.chart_name
366 "CHART netdata.plugin_pythond_" +
368 " '' 'Execution time for " +
370 " plugin' 'milliseconds / run' python.d netdata.plugin_python area 145000 " +
371 str(job.timetable['freq']) +
373 sys.stdout.write("DIMENSION run_time 'run time' absolute 1 1\n\n")
374 msg.debug("created charts for", job.chart_name)
377 except AttributeError:
378 self._stop(job, "no create")
379 except (UnboundLocalError, Exception) as e:
380 self._stop(job, "misbehaving. Reason: " + str(e))
384 Creates and supervises every job thread.
385 This will stay forever and ever and ever forever and ever it'll be the one...
387 for job in self.jobs:
391 if threading.active_count() <= 1:
392 msg.fatal("no more jobs")
396 def read_config(path):
398 Read YAML configuration from specified file
403 with open(path, 'r') as stream:
404 config = yaml.load(stream)
405 except (OSError, IOError):
406 msg.error(str(path), "is not a valid configuration file")
408 except yaml.YAMLError as e:
409 msg.error(str(path), "is malformed:", e)
414 def parse_cmdline(directory, *commands):
416 Parse parameters from command line.
417 :param directory: str
418 :param commands: list of str
422 global OVERRIDE_UPDATE_EVERY
425 changed_update = False
427 for cmd in commands[1:]:
430 elif cmd == "debug" or cmd == "all":
432 # redirect stderr to stdout?
433 elif os.path.isfile(directory + cmd + ".chart.py") or os.path.isfile(directory + cmd):
435 mods.append(cmd.replace(".chart.py", ""))
438 BASE_CONFIG['update_every'] = int(cmd)
439 changed_update = True
442 if changed_update and DEBUG_FLAG:
443 OVERRIDE_UPDATE_EVERY = True
444 msg.debug(PROGRAM, "overriding update interval to", str(BASE_CONFIG['update_every']))
446 msg.debug("started from", commands[0], "with options:", *commands[1:])
451 # if __name__ == '__main__':
456 global DEBUG_FLAG, BASE_CONFIG
458 # read configuration file
460 configfile = CONFIG_DIR + "python.d.conf"
461 msg.PROGRAM = PROGRAM
462 msg.info("reading configuration file:", configfile)
464 conf = read_config(configfile)
467 # exit the whole plugin when 'enabled: no' is set in 'python.d.conf'
468 if conf['enabled'] is False:
469 msg.fatal('disabled in configuration file.\n')
470 except (KeyError, TypeError):
473 for param in BASE_CONFIG:
474 BASE_CONFIG[param] = conf[param]
475 except (KeyError, TypeError):
476 pass # use default update_every from NETDATA_UPDATE_EVERY
478 DEBUG_FLAG = conf['debug']
479 except (KeyError, TypeError):
481 for k, v in conf.items():
482 if k in ("update_every", "debug", "enabled"):
487 # parse passed command line arguments
488 modules = parse_cmdline(MODULES_DIR, *sys.argv)
489 msg.DEBUG_FLAG = DEBUG_FLAG
490 msg.info("MODULES_DIR='" + MODULES_DIR +
491 "', CONFIG_DIR='" + CONFIG_DIR +
492 "', UPDATE_EVERY=" + str(BASE_CONFIG['update_every']) +
493 ", ONLY_MODULES=" + str(modules))
496 charts = PythonCharts(modules, MODULES_DIR, CONFIG_DIR + "python.d/", disabled)
500 msg.fatal("finished")
503 if __name__ == '__main__':