2 # -*- coding: utf-8 -*-
4 # Description: netdata python modules supervisor
5 # Author: Pawel Krupa (paulfantom)
12 MODULE_EXTENSION = ".chart.py"
13 BASE_CONFIG = {'update_every': 1,
18 # -----------------------------------------------------------------------------
21 PROGRAM = os.path.basename(__file__).replace(".plugin", "")
26 Print message on stderr.
30 sys.stderr.write(PROGRAM + " DEBUG :")
32 sys.stderr.write(" " + str(i))
33 sys.stderr.write("\n")
38 Print message on stderr.
40 sys.stderr.write(PROGRAM + " ERROR :")
42 sys.stderr.write(" " + str(i))
43 sys.stderr.write("\n")
48 Print message on stderr.
50 sys.stderr.write(PROGRAM + " INFO :")
52 sys.stderr.write(" " + str(i))
53 sys.stderr.write("\n")
58 Print message on stderr and exit.
60 sys.stderr.write(PROGRAM + " FATAL :")
62 sys.stderr.write(" " + str(i))
63 sys.stderr.write("\n")
65 sys.stdout.write('DISABLE\n')
70 # -----------------------------------------------------------------------------
71 # globals & python modules management
74 # https://github.com/firehol/netdata/wiki/External-Plugins#environment-variables
75 MODULES_DIR = os.path.abspath(os.getenv('NETDATA_PLUGINS_DIR',
76 os.path.dirname(__file__)) + "/../python.d") + "/"
77 CONFIG_DIR = os.getenv('NETDATA_CONFIG_DIR', "/etc/netdata/")
78 UPDATE_EVERY = os.getenv('NETDATA_UPDATE_EVERY', 1)
79 # directories should end with '/'
80 if CONFIG_DIR[-1] != "/":
82 sys.path.append(MODULES_DIR + "python_modules")
85 assert sys.version_info >= (3, 1)
86 import importlib.machinery
88 # change this hack below if we want PY_VERSION to be used in modules
90 # builtins.PY_VERSION = 3
92 info('Using python v3')
93 except (AssertionError, ImportError):
97 # change this hack below if we want PY_VERSION to be used in modules
99 # __builtin__.PY_VERSION = 2
101 info('Using python v2')
103 fatal('Cannot start. No importlib.machinery on python3 or lack of imp on python2')
107 fatal('Cannot find yaml library')
109 class PythonCharts(object):
111 Main class used to control every python module.
116 modules_path='../python.d/',
117 modules_configs='../conf.d/',
118 modules_disabled=None):
120 :param update_every: int
122 :param modules_path: str
123 :param modules_configs: str
124 :param modules_disabled: list
129 if modules_disabled is None:
130 modules_disabled = []
132 self.first_run = True
133 # set configuration directory
134 self.configs = modules_configs
137 loaded_modules = self._load_modules(modules_path, modules, modules_disabled)
139 # load configuration files
140 configured_modules = self._load_configs(loaded_modules)
142 # good economy and prosperity:
143 self.jobs = self._create_jobs(configured_modules) # type: list
144 if DEBUG_FLAG and update_every is not None:
145 for job in self.jobs:
146 job.create_timetable(update_every)
149 def _import_module(path, name=None):
151 Try to import module using only its path.
158 name = path.split('/')[-1]
159 if name[-len(MODULE_EXTENSION):] != MODULE_EXTENSION:
161 name = name[:-len(MODULE_EXTENSION)]
164 return importlib.machinery.SourceFileLoader(name, path).load_module()
166 return imp.load_source(name, path)
167 except Exception as e:
171 def _load_modules(self, path, modules, disabled):
173 Load modules from 'modules' list or dynamically every file from 'path' (only .chart.py files)
176 :param disabled: list
180 # check if plugin directory exists
181 if not os.path.isdir(path):
182 fatal("cannot find charts directory ", path)
190 mod = self._import_module(path + m + MODULE_EXTENSION)
193 else: # exit if plugin is not found
194 fatal('no modules found.')
196 # scan directory specified in path and load all modules from there
197 names = os.listdir(path)
199 if mod.strip(MODULE_EXTENSION) in disabled:
200 error(mod + ": disabled module ", mod.strip(MODULE_EXTENSION))
202 m = self._import_module(path + mod)
204 debug(mod + ": loading module '" + path + mod + "'")
208 def _load_configs(self, modules):
210 Append configuration in list named `config` to every module.
211 For multi-job modules `config` list is created in _parse_config,
212 otherwise it is created here based on BASE_CONFIG prototype with None as identifier.
217 configfile = self.configs + mod.__name__ + ".conf"
218 if os.path.isfile(configfile):
219 debug(mod.__name__ + ": loading module configuration: '" + configfile + "'")
223 self._parse_config(mod, read_config(configfile)))
224 except Exception as e:
225 error(mod.__name__ + ": cannot parse configuration file '" + configfile + "':", str(e))
227 error(mod.__name__ + ": configuration file '" + configfile + "' not found. Using defaults.")
228 # set config if not found
229 if not hasattr(mod, 'config'):
230 mod.config = {None: {}}
231 for var in BASE_CONFIG:
233 mod.config[None][var] = getattr(mod, var)
234 except AttributeError:
235 mod.config[None][var] = BASE_CONFIG[var]
239 def _parse_config(module, config):
241 Parse configuration file or extract configuration from module file.
242 Example of returned dictionary:
248 :param module: object
254 for key in BASE_CONFIG:
256 # get defaults from module config
257 defaults[key] = int(config.pop(key))
258 except (KeyError, ValueError):
260 # get defaults from module source code
261 defaults[key] = getattr(module, key)
262 except (KeyError, ValueError):
263 # if above failed, get defaults from global dict
264 defaults[key] = BASE_CONFIG[key]
266 # check if there are dict in config dict
269 if type(config[name]) is dict:
273 # assign variables needed by supervisor to every job configuration
277 if key not in config[name]:
278 config[name][key] = defaults[key]
279 # if only one job is needed, values doesn't have to be in dict (in YAML)
281 config = {None: config.copy()}
282 config[None].update(defaults)
284 # return dictionary of jobs where every job has BASE_CONFIG variables
288 def _create_jobs(modules):
290 Create jobs based on module.config dictionary and module.Service class definition.
295 for module in modules:
296 for name in module.config:
298 conf = module.config[name]
300 job = module.Service(configuration=conf, name=name)
301 except Exception as e:
302 error(module.__name__ +
303 ("/" + str(name) if name is not None else "") +
304 ": cannot start job: '" +
308 # set chart_name (needed to plot run time graphs)
309 job.chart_name = module.__name__
311 job.chart_name += "_" + name
313 debug(module.__name__ + ("/" + str(name) if name is not None else "") + ": job added")
315 return [j for j in jobs if j is not None]
317 def _stop(self, job, reason=None):
319 Stop specified job and remove it from self.jobs list
320 Also notifies user about job failure if DEBUG_FLAG is set
324 prefix = job.__module__
325 if job.name is not None:
326 prefix += "/" + job.name
329 self.jobs.remove(job)
332 elif reason[:3] == "no ":
334 "does not seem to have " +
336 "() function. Disabling it.")
337 elif reason[:7] == "failed ":
340 "() function reports failure.")
341 elif reason[:13] == "configuration":
343 "configuration file '" +
346 ".conf' not found. Using defaults.")
347 elif reason[:11] == "misbehaving":
348 error(prefix + "is " + reason)
352 Tries to execute check() on every job.
353 This cannot fail thus it is catching every exception
354 If job.check() fails job is stopped
357 while i < len(self.jobs):
361 self._stop(job, "failed check")
364 except AttributeError:
365 self._stop(job, "no check")
366 except (UnboundLocalError, Exception) as e:
367 self._stop(job, "misbehaving. Reason: " + str(e))
371 Tries to execute create() on every job.
372 This cannot fail thus it is catching every exception.
373 If job.create() fails job is stopped.
374 This is also creating job run time chart.
377 while i < len(self.jobs):
381 self._stop(job, "failed create")
383 chart = job.chart_name
385 "CHART netdata.plugin_pythond_" +
387 " '' 'Execution time for " +
389 " plugin' 'milliseconds / run' python.d netdata.plugin_python area 145000 " +
390 str(job.timetable['freq']) +
392 sys.stdout.write("DIMENSION run_time 'run time' absolute 1 1\n\n")
395 except AttributeError:
396 self._stop(job, "no create")
397 except (UnboundLocalError, Exception) as e:
398 self._stop(job, "misbehaving. Reason: " + str(e))
400 def _update_job(self, job):
402 Tries to execute update() on specified job.
403 This cannot fail thus it is catching every exception.
404 If job.update() returns False, number of retries_left is decremented.
405 If there are no more retries, job is stopped.
406 Job is also stopped if it throws an exception.
407 This is also updating job run time chart.
408 Return False if job is stopped
412 t_start = time.time()
413 # check if it is time to execute job update() function
414 if job.timetable['next'] > t_start:
420 since_last = int((t_start - job.timetable['last']) * 1000000)
421 if not job.update(since_last):
422 if job.retries_left <= 0:
423 self._stop(job, "update failed")
425 job.retries_left -= 1
426 job.timetable['next'] += job.timetable['freq']
428 except AttributeError:
429 self._stop(job, "no update")
431 except (UnboundLocalError, Exception) as e:
432 self._stop(job, "misbehaving. Reason: " + str(e))
435 job.timetable['next'] = t_end - (t_end % job.timetable['freq']) + job.timetable['freq']
436 # draw performance graph
437 sys.stdout.write("BEGIN netdata.plugin_pythond_" + job.chart_name + " " + str(since_last) + '\n')
438 sys.stdout.write("SET run_time = " + str(int((t_end - t_start) * 1000)) + '\n')
439 sys.stdout.write("END\n")
441 job.timetable['last'] = t_start
442 job.retries_left = job.retries
443 self.first_run = False
448 Tries to execute update() on every job by using _update_job()
449 This will stay forever and ever and ever forever and ever it'll be the one...
451 self.first_run = True
455 while i < len(self.jobs):
457 if self._update_job(job):
459 next_runs.append(job.timetable['next'])
463 if len(next_runs) == 0:
464 fatal('no python.d modules loaded.')
465 time.sleep(min(next_runs) - time.time())
468 def read_config(path):
470 Read YAML configuration from specified file
475 with open(path, 'r') as stream:
476 config = yaml.load(stream)
477 except (OSError, IOError):
478 error(str(path), "is not a valid configuration file")
480 except yaml.YAMLError as e:
481 error(str(path), "is malformed:", e)
486 def parse_cmdline(directory, *commands):
488 Parse parameters from command line.
489 :param directory: str
490 :param commands: list of str
495 update_every = UPDATE_EVERY
498 for cmd in commands[1:]:
501 elif cmd == "debug" or cmd == "all":
503 # redirect stderr to stdout?
504 elif os.path.isfile(directory + cmd + ".chart.py") or os.path.isfile(directory + cmd):
506 mods.append(cmd.replace(".chart.py", ""))
509 update_every = int(cmd)
511 update_every = UPDATE_EVERY
513 debug("started from", commands[0], "with options:", *commands[1:])
515 UPDATE_EVERY = update_every
516 return {'modules': mods}
519 # if __name__ == '__main__':
524 global PROGRAM, DEBUG_FLAG
525 PROGRAM = sys.argv[0].split('/')[-1].split('.plugin')[0]
527 # read configuration file
529 configfile = CONFIG_DIR + "python.d.conf"
531 update_every = UPDATE_EVERY
532 conf = read_config(configfile)
535 # FIXME: is this right? exit the whole plugin?
536 if str(conf['enable']) is False:
537 fatal('disabled in configuration file.\n')
538 except (KeyError, TypeError):
541 update_every = conf['update_every']
542 except (KeyError, TypeError):
543 pass # use default update_every from NETDATA_UPDATE_EVERY
545 DEBUG_FLAG = conf['debug']
546 except (KeyError, TypeError):
548 for k, v in conf.items():
549 if k in ("update_every", "debug", "enable"):
554 # parse passed command line arguments
555 out = parse_cmdline(MODULES_DIR, *sys.argv)
556 modules = out['modules']
557 info("MODULES_DIR='" + MODULES_DIR + "', CONFIG_DIR='" + CONFIG_DIR + "', UPDATE_EVERY=" + str(UPDATE_EVERY) + ", ONLY_MODULES=" + str(out['modules']))
560 charts = PythonCharts(update_every, modules, MODULES_DIR, CONFIG_DIR + "python.d/", disabled)
567 if __name__ == '__main__':