2 '''':; exec "$(command -v python || command -v python3 || command -v python2 || echo "ERROR python IS NOT AVAILABLE IN THIS SYSTEM")" "$0" "$@" # '''
3 # -*- coding: utf-8 -*-
5 # Description: netdata python modules supervisor
6 # Author: Pawel Krupa (paulfantom)
14 # -----------------------------------------------------------------------------
15 # globals & environment setup
16 # https://github.com/firehol/netdata/wiki/External-Plugins#environment-variables
17 MODULE_EXTENSION = ".chart.py"
18 BASE_CONFIG = {'update_every': os.getenv('NETDATA_UPDATE_EVERY', 1),
22 MODULES_DIR = os.path.abspath(os.getenv('NETDATA_PLUGINS_DIR',
23 os.path.dirname(__file__)) + "/../python.d") + "/"
24 CONFIG_DIR = os.getenv('NETDATA_CONFIG_DIR', "/etc/netdata/")
25 # directories should end with '/'
26 if CONFIG_DIR[-1] != "/":
28 sys.path.append(MODULES_DIR + "python_modules")
30 PROGRAM = os.path.basename(__file__).replace(".plugin", "")
33 OVERRIDE_UPDATE_EVERY = False
35 # -----------------------------------------------------------------------------
36 # custom, third party and version specific python modules management
40 assert sys.version_info >= (3, 1)
41 import importlib.machinery
43 # change this hack below if we want PY_VERSION to be used in modules
45 # builtins.PY_VERSION = 3
46 msg.info('Using python v3')
47 except (AssertionError, ImportError):
51 # change this hack below if we want PY_VERSION to be used in modules
53 # __builtin__.PY_VERSION = 2
55 msg.info('Using python v2')
57 msg.fatal('Cannot start. No importlib.machinery on python3 or lack of imp on python2')
61 # msg.fatal('Cannot find yaml library')
64 import pyyaml3 as yaml
66 import pyyaml2 as yaml
68 msg.fatal('Cannot find yaml library')
71 class PythonCharts(object):
73 Main class used to control every python module.
78 modules_path='../python.d/',
79 modules_configs='../conf.d/',
80 modules_disabled=None):
83 :param modules_path: str
84 :param modules_configs: str
85 :param modules_disabled: list
90 if modules_disabled is None:
94 # set configuration directory
95 self.configs = modules_configs
98 loaded_modules = self._load_modules(modules_path, modules, modules_disabled)
100 # load configuration files
101 configured_modules = self._load_configs(loaded_modules)
103 # good economy and prosperity:
104 self.jobs = self._create_jobs(configured_modules) # type: list
106 # enable timetable override like `python.d.plugin mysql debug 1`
107 if DEBUG_FLAG and OVERRIDE_UPDATE_EVERY:
108 for job in self.jobs:
109 job.create_timetable(BASE_CONFIG['update_every'])
112 def _import_module(path, name=None):
114 Try to import module using only its path.
121 name = path.split('/')[-1]
122 if name[-len(MODULE_EXTENSION):] != MODULE_EXTENSION:
124 name = name[:-len(MODULE_EXTENSION)]
127 return importlib.machinery.SourceFileLoader(name, path).load_module()
129 return imp.load_source(name, path)
130 except Exception as e:
131 msg.error("Problem loading", name, str(e))
134 def _load_modules(self, path, modules, disabled):
136 Load modules from 'modules' list or dynamically every file from 'path' (only .chart.py files)
139 :param disabled: list
143 # check if plugin directory exists
144 if not os.path.isdir(path):
145 msg.fatal("cannot find charts directory ", path)
153 mod = self._import_module(path + m + MODULE_EXTENSION)
156 else: # exit if plugin is not found
157 msg.fatal('no modules found.')
159 # scan directory specified in path and load all modules from there
160 names = os.listdir(path)
162 if mod.replace(MODULE_EXTENSION, "") in disabled:
163 msg.error(mod + ": disabled module ", mod.replace(MODULE_EXTENSION, ""))
165 m = self._import_module(path + mod)
167 msg.debug(mod + ": loading module '" + path + mod + "'")
171 def _load_configs(self, modules):
173 Append configuration in list named `config` to every module.
174 For multi-job modules `config` list is created in _parse_config,
175 otherwise it is created here based on BASE_CONFIG prototype with None as identifier.
180 configfile = self.configs + mod.__name__ + ".conf"
181 if os.path.isfile(configfile):
182 msg.debug(mod.__name__ + ": loading module configuration: '" + configfile + "'")
184 if not hasattr(mod, 'config'):
188 self._parse_config(mod, read_config(configfile)))
189 except Exception as e:
190 msg.error(mod.__name__ + ": cannot parse configuration file '" + configfile + "':", str(e))
192 msg.error(mod.__name__ + ": configuration file '" + configfile + "' not found. Using defaults.")
193 # set config if not found
194 if not hasattr(mod, 'config'):
195 msg.debug(mod.__name__ + ": setting configuration for only one job")
196 mod.config = {None: {}}
197 for var in BASE_CONFIG:
199 mod.config[None][var] = getattr(mod, var)
200 except AttributeError:
201 mod.config[None][var] = BASE_CONFIG[var]
205 def _parse_config(module, config):
207 Parse configuration file or extract configuration from module file.
208 Example of returned dictionary:
214 :param module: object
222 msg.debug(module.__name__ + ": reading configuration")
223 for key in BASE_CONFIG:
225 # get defaults from module config
226 defaults[key] = int(config.pop(key))
227 except (KeyError, ValueError):
229 # get defaults from module source code
230 defaults[key] = getattr(module, key)
231 except (KeyError, ValueError, AttributeError):
232 # if above failed, get defaults from global dict
233 defaults[key] = BASE_CONFIG[key]
235 # check if there are dict in config dict
238 if type(config[name]) is dict:
242 # assign variables needed by supervisor to every job configuration
246 if key not in config[name]:
247 config[name][key] = defaults[key]
248 # if only one job is needed, values doesn't have to be in dict (in YAML)
250 config = {None: config.copy()}
251 config[None].update(defaults)
253 # return dictionary of jobs where every job has BASE_CONFIG variables
257 def _create_jobs(modules):
259 Create jobs based on module.config dictionary and module.Service class definition.
264 for module in modules:
265 for name in module.config:
267 conf = module.config[name]
269 job = module.Service(configuration=conf, name=name)
270 except Exception as e:
271 msg.error(module.__name__ +
272 ("/" + str(name) if name is not None else "") +
273 ": cannot start job: '" +
277 # set chart_name (needed to plot run time graphs)
278 job.chart_name = module.__name__
280 job.chart_name += "_" + name
282 msg.debug(module.__name__ + ("/" + str(name) if name is not None else "") + ": job added")
284 return [j for j in jobs if j is not None]
286 def _stop(self, job, reason=None):
288 Stop specified job and remove it from self.jobs list
289 Also notifies user about job failure if DEBUG_FLAG is set
293 prefix = job.__module__
294 if job.name is not None and len(job.name) != 0:
295 prefix += "/" + job.name
297 msg.error("DISABLED:", prefix)
298 self.jobs.remove(job)
299 except Exception as e:
300 msg.debug("This shouldn't happen. NO " + prefix + " IN LIST:" + str(self.jobs) + " ERROR: " + str(e))
302 # TODO remove section below and remove `reason`.
306 elif reason[:3] == "no ":
308 "does not seem to have " +
310 "() function. Disabling it.")
311 elif reason[:7] == "failed ":
314 "() function reports failure.")
315 elif reason[:13] == "configuration":
317 "configuration file '" +
320 ".conf' not found. Using defaults.")
321 elif reason[:11] == "misbehaving":
322 msg.error(prefix + "is " + reason)
326 Tries to execute check() on every job.
327 This cannot fail thus it is catching every exception
328 If job.check() fails job is stopped
332 msg.debug("all job objects", str(self.jobs))
333 while i < len(self.jobs):
337 msg.error(job.chart_name, "check() failed - disabling job")
340 msg.info("CHECKED OK:", job.chart_name)
343 if job.override_name is not None:
344 new_name = job.__module__ + '_' + sub(r'\s+', '_', job.override_name)
345 if new_name in overridden:
346 msg.info("DROPPED:", job.name, ", job '" + job.override_name + "' is already served by another job.")
350 job.name = job.override_name
351 msg.info("RENAMED:", new_name, ", from " + job.chart_name)
352 job.chart_name = new_name
353 overridden.append(job.chart_name)
356 except AttributeError as e:
358 msg.error(job.chart_name, "cannot find check() function or it thrown unhandled exception.")
360 except (UnboundLocalError, Exception) as e:
361 msg.error(job.chart_name, str(e))
363 msg.debug("overridden job names:", str(overridden))
364 msg.debug("all remaining job objects:", str(self.jobs))
368 Tries to execute create() on every job.
369 This cannot fail thus it is catching every exception.
370 If job.create() fails job is stopped.
371 This is also creating job run time chart.
374 while i < len(self.jobs):
378 msg.error(job.chart_name, "create function failed.")
381 chart = job.chart_name
383 "CHART netdata.plugin_pythond_" +
385 " '' 'Execution time for " +
387 " plugin' 'milliseconds / run' python.d netdata.plugin_python area 145000 " +
388 str(job.timetable['freq']) +
390 sys.stdout.write("DIMENSION run_time 'run time' absolute 1 1\n\n")
391 msg.debug("created charts for", job.chart_name)
394 except AttributeError:
395 msg.error(job.chart_name, "cannot find create() function or it thrown unhandled exception.")
397 except (UnboundLocalError, Exception) as e:
398 msg.error(job.chart_name, str(e))
403 Creates and supervises every job thread.
404 This will stay forever and ever and ever forever and ever it'll be the one...
406 for job in self.jobs:
410 if threading.active_count() <= 1:
411 msg.fatal("no more jobs")
415 def read_config(path):
417 Read YAML configuration from specified file
422 with open(path, 'r') as stream:
423 config = yaml.load(stream)
424 except (OSError, IOError):
425 msg.error(str(path), "is not a valid configuration file")
427 except yaml.YAMLError as e:
428 msg.error(str(path), "is malformed:", e)
433 def parse_cmdline(directory, *commands):
435 Parse parameters from command line.
436 :param directory: str
437 :param commands: list of str
440 global DEBUG_FLAG, TRACE_FLAG
441 global OVERRIDE_UPDATE_EVERY
444 changed_update = False
446 for cmd in commands[1:]:
449 elif cmd == "debug" or cmd == "all":
451 # redirect stderr to stdout?
452 elif cmd == "trace" or cmd == "all":
454 elif os.path.isfile(directory + cmd + ".chart.py") or os.path.isfile(directory + cmd):
456 mods.append(cmd.replace(".chart.py", ""))
459 BASE_CONFIG['update_every'] = int(cmd)
460 changed_update = True
463 if changed_update and DEBUG_FLAG:
464 OVERRIDE_UPDATE_EVERY = True
465 msg.debug(PROGRAM, "overriding update interval to", str(BASE_CONFIG['update_every']))
467 msg.debug("started from", commands[0], "with options:", *commands[1:])
472 # if __name__ == '__main__':
477 global DEBUG_FLAG, TRACE_FLAG, BASE_CONFIG
479 # read configuration file
481 configfile = CONFIG_DIR + "python.d.conf"
482 msg.PROGRAM = PROGRAM
483 msg.info("reading configuration file:", configfile)
487 conf = read_config(configfile)
490 # exit the whole plugin when 'enabled: no' is set in 'python.d.conf'
491 if conf['enabled'] is False:
492 msg.fatal('disabled in configuration file.\n')
493 except (KeyError, TypeError):
497 for param in BASE_CONFIG:
498 BASE_CONFIG[param] = conf[param]
499 except (KeyError, TypeError):
500 pass # use default update_every from NETDATA_UPDATE_EVERY
503 DEBUG_FLAG = conf['debug']
504 except (KeyError, TypeError):
508 TRACE_FLAG = conf['trace']
509 except (KeyError, TypeError):
513 log_throttle = conf['logs_per_interval']
514 except (KeyError, TypeError):
518 log_interval = conf['log_interval']
519 except (KeyError, TypeError):
522 for k, v in conf.items():
523 if k in ("update_every", "debug", "enabled"):
528 # parse passed command line arguments
529 modules = parse_cmdline(MODULES_DIR, *sys.argv)
530 msg.DEBUG_FLAG = DEBUG_FLAG
531 msg.TRACE_FLAG = TRACE_FLAG
532 msg.LOG_THROTTLE = log_throttle
533 msg.LOG_INTERVAL = log_interval
535 msg.LOG_NEXT_CHECK = 0
536 msg.info("MODULES_DIR='" + MODULES_DIR +
537 "', CONFIG_DIR='" + CONFIG_DIR +
538 "', UPDATE_EVERY=" + str(BASE_CONFIG['update_every']) +
539 ", ONLY_MODULES=" + str(modules))
542 charts = PythonCharts(modules, MODULES_DIR, CONFIG_DIR + "python.d/", disabled)
546 msg.fatal("finished")
549 if __name__ == '__main__':