2 # -*- coding: utf-8 -*-
4 # Description: netdata python modules supervisor
5 # Author: Pawel Krupa (paulfantom)
11 # -----------------------------------------------------------------------------
12 # globals & environment setup
13 # https://github.com/firehol/netdata/wiki/External-Plugins#environment-variables
14 MODULE_EXTENSION = ".chart.py"
15 BASE_CONFIG = {'update_every': os.getenv('NETDATA_UPDATE_EVERY', 1),
19 MODULES_DIR = os.path.abspath(os.getenv('NETDATA_PLUGINS_DIR',
20 os.path.dirname(__file__)) + "/../python.d") + "/"
21 CONFIG_DIR = os.getenv('NETDATA_CONFIG_DIR', "/etc/netdata/")
22 # directories should end with '/'
23 if CONFIG_DIR[-1] != "/":
25 sys.path.append(MODULES_DIR + "python_modules")
27 PROGRAM = os.path.basename(__file__).replace(".plugin", "")
29 OVERRIDE_UPDATE_EVERY = False
32 # -----------------------------------------------------------------------------
36 Print message on stderr.
40 sys.stderr.write(PROGRAM + " DEBUG :")
42 sys.stderr.write(" " + str(i))
43 sys.stderr.write("\n")
49 Print message on stderr.
51 sys.stderr.write(PROGRAM + " ERROR :")
53 sys.stderr.write(" " + str(i))
54 sys.stderr.write("\n")
60 Print message on stderr.
62 sys.stderr.write(PROGRAM + " INFO :")
64 sys.stderr.write(" " + str(i))
65 sys.stderr.write("\n")
71 Print message on stderr and exit.
73 sys.stderr.write(PROGRAM + " FATAL :")
75 sys.stderr.write(" " + str(i))
76 sys.stderr.write("\n")
78 sys.stdout.write('DISABLE\n')
82 # -----------------------------------------------------------------------------
83 # third party and version specific python modules management
85 assert sys.version_info >= (3, 1)
86 import importlib.machinery
88 # change this hack below if we want PY_VERSION to be used in modules
90 # builtins.PY_VERSION = 3
92 info('Using python v3')
93 except (AssertionError, ImportError):
97 # change this hack below if we want PY_VERSION to be used in modules
99 # __builtin__.PY_VERSION = 2
101 info('Using python v2')
103 fatal('Cannot start. No importlib.machinery on python3 or lack of imp on python2')
107 fatal('Cannot find yaml library')
110 class PythonCharts(object):
112 Main class used to control every python module.
116 modules_path='../python.d/',
117 modules_configs='../conf.d/',
118 modules_disabled=None):
121 :param modules_path: str
122 :param modules_configs: str
123 :param modules_disabled: list
128 if modules_disabled is None:
129 modules_disabled = []
131 self.first_run = True
132 # set configuration directory
133 self.configs = modules_configs
136 loaded_modules = self._load_modules(modules_path, modules, modules_disabled)
138 # load configuration files
139 configured_modules = self._load_configs(loaded_modules)
141 # good economy and prosperity:
142 self.jobs = self._create_jobs(configured_modules) # type: list
144 # enable timetable override like `python.d.plugin mysql debug 1`
145 if DEBUG_FLAG and OVERRIDE_UPDATE_EVERY:
146 for job in self.jobs:
147 job.create_timetable(BASE_CONFIG['update_every'])
150 def _import_module(path, name=None):
152 Try to import module using only its path.
159 name = path.split('/')[-1]
160 if name[-len(MODULE_EXTENSION):] != MODULE_EXTENSION:
162 name = name[:-len(MODULE_EXTENSION)]
165 return importlib.machinery.SourceFileLoader(name, path).load_module()
167 return imp.load_source(name, path)
168 except Exception as e:
172 def _load_modules(self, path, modules, disabled):
174 Load modules from 'modules' list or dynamically every file from 'path' (only .chart.py files)
177 :param disabled: list
181 # check if plugin directory exists
182 if not os.path.isdir(path):
183 fatal("cannot find charts directory ", path)
191 mod = self._import_module(path + m + MODULE_EXTENSION)
194 else: # exit if plugin is not found
195 fatal('no modules found.')
197 # scan directory specified in path and load all modules from there
198 names = os.listdir(path)
200 if mod.strip(MODULE_EXTENSION) in disabled:
201 error(mod + ": disabled module ", mod.strip(MODULE_EXTENSION))
203 m = self._import_module(path + mod)
205 debug(mod + ": loading module '" + path + mod + "'")
209 def _load_configs(self, modules):
211 Append configuration in list named `config` to every module.
212 For multi-job modules `config` list is created in _parse_config,
213 otherwise it is created here based on BASE_CONFIG prototype with None as identifier.
218 configfile = self.configs + mod.__name__ + ".conf"
219 if os.path.isfile(configfile):
220 debug(mod.__name__ + ": loading module configuration: '" + configfile + "'")
224 self._parse_config(mod, read_config(configfile)))
225 except Exception as e:
226 error(mod.__name__ + ": cannot parse configuration file '" + configfile + "':", str(e))
228 error(mod.__name__ + ": configuration file '" + configfile + "' not found. Using defaults.")
229 # set config if not found
230 if not hasattr(mod, 'config'):
231 mod.config = {None: {}}
232 for var in BASE_CONFIG:
234 mod.config[None][var] = getattr(mod, var)
235 except AttributeError:
236 mod.config[None][var] = BASE_CONFIG[var]
240 def _parse_config(module, config):
242 Parse configuration file or extract configuration from module file.
243 Example of returned dictionary:
249 :param module: object
255 for key in BASE_CONFIG:
257 # get defaults from module config
258 defaults[key] = int(config.pop(key))
259 except (KeyError, ValueError):
261 # get defaults from module source code
262 defaults[key] = getattr(module, key)
263 except (KeyError, ValueError):
264 # if above failed, get defaults from global dict
265 defaults[key] = BASE_CONFIG[key]
267 # check if there are dict in config dict
270 if type(config[name]) is dict:
274 # assign variables needed by supervisor to every job configuration
278 if key not in config[name]:
279 config[name][key] = defaults[key]
280 # if only one job is needed, values doesn't have to be in dict (in YAML)
282 config = {None: config.copy()}
283 config[None].update(defaults)
285 # return dictionary of jobs where every job has BASE_CONFIG variables
289 def _create_jobs(modules):
291 Create jobs based on module.config dictionary and module.Service class definition.
296 for module in modules:
297 for name in module.config:
299 conf = module.config[name]
301 job = module.Service(configuration=conf, name=name)
302 except Exception as e:
303 error(module.__name__ +
304 ("/" + str(name) if name is not None else "") +
305 ": cannot start job: '" +
309 # set chart_name (needed to plot run time graphs)
310 job.chart_name = module.__name__
312 job.chart_name += "_" + name
314 debug(module.__name__ + ("/" + str(name) if name is not None else "") + ": job added")
316 return [j for j in jobs if j is not None]
318 def _stop(self, job, reason=None):
320 Stop specified job and remove it from self.jobs list
321 Also notifies user about job failure if DEBUG_FLAG is set
325 prefix = job.__module__
326 if job.name is not None:
327 prefix += "/" + job.name
330 self.jobs.remove(job)
333 elif reason[:3] == "no ":
335 "does not seem to have " +
337 "() function. Disabling it.")
338 elif reason[:7] == "failed ":
341 "() function reports failure.")
342 elif reason[:13] == "configuration":
344 "configuration file '" +
347 ".conf' not found. Using defaults.")
348 elif reason[:11] == "misbehaving":
349 error(prefix + "is " + reason)
353 Tries to execute check() on every job.
354 This cannot fail thus it is catching every exception
355 If job.check() fails job is stopped
358 while i < len(self.jobs):
362 self._stop(job, "failed check")
365 except AttributeError:
366 self._stop(job, "no check")
367 except (UnboundLocalError, Exception) as e:
368 self._stop(job, "misbehaving. Reason: " + str(e))
372 Tries to execute create() on every job.
373 This cannot fail thus it is catching every exception.
374 If job.create() fails job is stopped.
375 This is also creating job run time chart.
378 while i < len(self.jobs):
382 self._stop(job, "failed create")
384 chart = job.chart_name
386 "CHART netdata.plugin_pythond_" +
388 " '' 'Execution time for " +
390 " plugin' 'milliseconds / run' python.d netdata.plugin_python area 145000 " +
391 str(job.timetable['freq']) +
393 sys.stdout.write("DIMENSION run_time 'run time' absolute 1 1\n\n")
396 except AttributeError:
397 self._stop(job, "no create")
398 except (UnboundLocalError, Exception) as e:
399 self._stop(job, "misbehaving. Reason: " + str(e))
401 def _update_job(self, job):
403 Tries to execute update() on specified job.
404 This cannot fail thus it is catching every exception.
405 If job.update() returns False, number of retries_left is decremented.
406 If there are no more retries, job is stopped.
407 Job is also stopped if it throws an exception.
408 This is also updating job run time chart.
409 Return False if job is stopped
413 t_start = time.time()
414 # check if it is time to execute job update() function
415 if job.timetable['next'] > t_start:
421 since_last = int((t_start - job.timetable['last']) * 1000000)
422 if not job.update(since_last):
423 if job.retries_left <= 0:
424 self._stop(job, "update failed")
426 job.retries_left -= 1
427 job.timetable['next'] += job.timetable['freq']
429 except AttributeError:
430 self._stop(job, "no update")
432 except (UnboundLocalError, Exception) as e:
433 self._stop(job, "misbehaving. Reason: " + str(e))
436 job.timetable['next'] = t_end - (t_end % job.timetable['freq']) + job.timetable['freq']
437 # draw performance graph
438 sys.stdout.write("BEGIN netdata.plugin_pythond_" + job.chart_name + " " + str(since_last) + '\n')
439 sys.stdout.write("SET run_time = " + str(int((t_end - t_start) * 1000)) + '\n')
440 sys.stdout.write("END\n")
442 job.timetable['last'] = t_start
443 job.retries_left = job.retries
444 self.first_run = False
449 Tries to execute update() on every job by using _update_job()
450 This will stay forever and ever and ever forever and ever it'll be the one...
452 self.first_run = True
456 while i < len(self.jobs):
458 if self._update_job(job):
460 next_runs.append(job.timetable['next'])
464 if len(next_runs) == 0:
465 fatal('no python.d modules loaded.')
466 time.sleep(min(next_runs) - time.time())
469 def read_config(path):
471 Read YAML configuration from specified file
476 with open(path, 'r') as stream:
477 config = yaml.load(stream)
478 except (OSError, IOError):
479 error(str(path), "is not a valid configuration file")
481 except yaml.YAMLError as e:
482 error(str(path), "is malformed:", e)
487 def parse_cmdline(directory, *commands):
489 Parse parameters from command line.
490 :param directory: str
491 :param commands: list of str
495 global OVERRIDE_UPDATE_EVERY
499 for cmd in commands[1:]:
502 elif cmd == "debug" or cmd == "all":
504 # redirect stderr to stdout?
505 elif os.path.isfile(directory + cmd + ".chart.py") or os.path.isfile(directory + cmd):
507 mods.append(cmd.replace(".chart.py", ""))
510 BASE_CONFIG['update_every'] = int(cmd)
511 OVERRIDE_UPDATE_EVERY = True
515 debug("started from", commands[0], "with options:", *commands[1:])
520 # if __name__ == '__main__':
525 global DEBUG_FLAG, BASE_CONFIG
527 # read configuration file
529 configfile = CONFIG_DIR + "python.d.conf"
531 conf = read_config(configfile)
534 # exit the whole plugin when 'enabled: no' is set in 'python.d.conf'
535 if str(conf['enabled']) is False:
536 fatal('disabled in configuration file.\n')
537 except (KeyError, TypeError):
540 for param in BASE_CONFIG:
541 BASE_CONFIG[param] = conf[param]
542 except (KeyError, TypeError):
543 pass # use default update_every from NETDATA_UPDATE_EVERY
545 DEBUG_FLAG = conf['debug']
546 except (KeyError, TypeError):
548 for k, v in conf.items():
549 if k in ("update_every", "debug", "enable"):
554 # parse passed command line arguments
555 modules = parse_cmdline(MODULES_DIR, *sys.argv)
556 info("MODULES_DIR='" + MODULES_DIR +
557 "', CONFIG_DIR='" + CONFIG_DIR +
558 "', UPDATE_EVERY=" + str(BASE_CONFIG['update_every']) +
559 ", ONLY_MODULES=" + str(modules))
563 charts = PythonCharts(modules, MODULES_DIR, CONFIG_DIR + "python.d/", disabled)
570 if __name__ == '__main__':