2 # -*- coding: utf-8 -*-
4 # Description: netdata python modules supervisor
5 # Author: Pawel Krupa (paulfantom)
11 # -----------------------------------------------------------------------------
12 # globals & environment setup
13 # https://github.com/firehol/netdata/wiki/External-Plugins#environment-variables
14 MODULE_EXTENSION = ".chart.py"
15 BASE_CONFIG = {'update_every': os.getenv('NETDATA_UPDATE_EVERY', 1),
19 MODULES_DIR = os.path.abspath(os.getenv('NETDATA_PLUGINS_DIR',
20 os.path.dirname(__file__)) + "/../python.d") + "/"
21 CONFIG_DIR = os.getenv('NETDATA_CONFIG_DIR', "/etc/netdata/")
22 # directories should end with '/'
23 if CONFIG_DIR[-1] != "/":
25 sys.path.append(MODULES_DIR + "python_modules")
27 PROGRAM = os.path.basename(__file__).replace(".plugin", "")
29 OVERRIDE_UPDATE_EVERY = False
32 # -----------------------------------------------------------------------------
34 def log_msg(msg_type, *args):
36 Print message on stderr.
39 sys.stderr.write(PROGRAM)
41 sys.stderr.write(msg_type)
42 sys.stderr.write(": ")
45 sys.stderr.write(str(i))
46 sys.stderr.write("\n")
52 Print debug message on stderr.
57 log_msg("DEBUG", *args)
62 Print message on stderr.
64 log_msg("ERROR", *args)
69 Print message on stderr.
71 log_msg("INFO", *args)
76 Print message on stderr and exit.
78 log_msg("FATAL", *args)
79 sys.stdout.write('DISABLE\n')
83 # -----------------------------------------------------------------------------
84 # third party and version specific python modules management
86 assert sys.version_info >= (3, 1)
87 import importlib.machinery
89 # change this hack below if we want PY_VERSION to be used in modules
91 # builtins.PY_VERSION = 3
93 info('Using python v3')
94 except (AssertionError, ImportError):
98 # change this hack below if we want PY_VERSION to be used in modules
100 # __builtin__.PY_VERSION = 2
102 info('Using python v2')
104 fatal('Cannot start. No importlib.machinery on python3 or lack of imp on python2')
108 fatal('Cannot find yaml library')
111 class PythonCharts(object):
113 Main class used to control every python module.
117 modules_path='../python.d/',
118 modules_configs='../conf.d/',
119 modules_disabled=None):
122 :param modules_path: str
123 :param modules_configs: str
124 :param modules_disabled: list
129 if modules_disabled is None:
130 modules_disabled = []
132 self.first_run = True
133 # set configuration directory
134 self.configs = modules_configs
137 loaded_modules = self._load_modules(modules_path, modules, modules_disabled)
139 # load configuration files
140 configured_modules = self._load_configs(loaded_modules)
142 # good economy and prosperity:
143 self.jobs = self._create_jobs(configured_modules) # type: list
145 # enable timetable override like `python.d.plugin mysql debug 1`
146 if DEBUG_FLAG and OVERRIDE_UPDATE_EVERY:
147 for job in self.jobs:
148 job.create_timetable(BASE_CONFIG['update_every'])
151 def _import_module(path, name=None):
153 Try to import module using only its path.
160 name = path.split('/')[-1]
161 if name[-len(MODULE_EXTENSION):] != MODULE_EXTENSION:
163 name = name[:-len(MODULE_EXTENSION)]
166 return importlib.machinery.SourceFileLoader(name, path).load_module()
168 return imp.load_source(name, path)
169 except Exception as e:
170 error("Problem loading", name, str(e))
173 def _load_modules(self, path, modules, disabled):
175 Load modules from 'modules' list or dynamically every file from 'path' (only .chart.py files)
178 :param disabled: list
182 # check if plugin directory exists
183 if not os.path.isdir(path):
184 fatal("cannot find charts directory ", path)
192 mod = self._import_module(path + m + MODULE_EXTENSION)
195 else: # exit if plugin is not found
196 fatal('no modules found.')
198 # scan directory specified in path and load all modules from there
199 names = os.listdir(path)
201 if mod.replace(MODULE_EXTENSION, "") in disabled:
202 error(mod + ": disabled module ", mod.replace(MODULE_EXTENSION, ""))
204 m = self._import_module(path + mod)
206 debug(mod + ": loading module '" + path + mod + "'")
210 def _load_configs(self, modules):
212 Append configuration in list named `config` to every module.
213 For multi-job modules `config` list is created in _parse_config,
214 otherwise it is created here based on BASE_CONFIG prototype with None as identifier.
219 configfile = self.configs + mod.__name__ + ".conf"
220 if os.path.isfile(configfile):
221 debug(mod.__name__ + ": loading module configuration: '" + configfile + "'")
223 if not hasattr(mod, 'config'):
227 self._parse_config(mod, read_config(configfile)))
228 except Exception as e:
229 error(mod.__name__ + ": cannot parse configuration file '" + configfile + "':", str(e))
231 error(mod.__name__ + ": configuration file '" + configfile + "' not found. Using defaults.")
232 # set config if not found
233 if not hasattr(mod, 'config'):
234 debug(mod.__name__ + ": setting configuration for only one job")
235 mod.config = {None: {}}
236 for var in BASE_CONFIG:
238 mod.config[None][var] = getattr(mod, var)
239 except AttributeError:
240 mod.config[None][var] = BASE_CONFIG[var]
244 def _parse_config(module, config):
246 Parse configuration file or extract configuration from module file.
247 Example of returned dictionary:
253 :param module: object
259 debug(module.__name__ + ": reading configuration")
260 for key in BASE_CONFIG:
262 # get defaults from module config
263 defaults[key] = int(config.pop(key))
264 except (KeyError, ValueError):
266 # get defaults from module source code
267 defaults[key] = getattr(module, key)
268 except (KeyError, ValueError, AttributeError):
269 # if above failed, get defaults from global dict
270 defaults[key] = BASE_CONFIG[key]
272 # check if there are dict in config dict
275 if type(config[name]) is dict:
279 # assign variables needed by supervisor to every job configuration
283 if key not in config[name]:
284 config[name][key] = defaults[key]
285 # if only one job is needed, values doesn't have to be in dict (in YAML)
287 config = {None: config.copy()}
288 config[None].update(defaults)
290 # return dictionary of jobs where every job has BASE_CONFIG variables
294 def _create_jobs(modules):
296 Create jobs based on module.config dictionary and module.Service class definition.
301 for module in modules:
302 for name in module.config:
304 conf = module.config[name]
306 job = module.Service(configuration=conf, name=name)
307 except Exception as e:
308 error(module.__name__ +
309 ("/" + str(name) if name is not None else "") +
310 ": cannot start job: '" +
314 # set chart_name (needed to plot run time graphs)
315 job.chart_name = module.__name__
317 job.chart_name += "_" + name
319 debug(module.__name__ + ("/" + str(name) if name is not None else "") + ": job added")
321 return [j for j in jobs if j is not None]
323 def _stop(self, job, reason=None):
325 Stop specified job and remove it from self.jobs list
326 Also notifies user about job failure if DEBUG_FLAG is set
330 prefix = job.__module__
331 if job.name is not None:
332 prefix += "/" + job.name
335 self.jobs.remove(job)
338 elif reason[:3] == "no ":
340 "does not seem to have " +
342 "() function. Disabling it.")
343 elif reason[:7] == "failed ":
346 "() function reports failure.")
347 elif reason[:13] == "configuration":
349 "configuration file '" +
352 ".conf' not found. Using defaults.")
353 elif reason[:11] == "misbehaving":
354 error(prefix + "is " + reason)
358 Tries to execute check() on every job.
359 This cannot fail thus it is catching every exception
360 If job.check() fails job is stopped
363 while i < len(self.jobs):
367 self._stop(job, "failed check")
369 debug(job.chart_name, ": check succeeded")
371 except AttributeError:
372 self._stop(job, "no check")
373 except (UnboundLocalError, Exception) as e:
374 self._stop(job, "misbehaving. Reason:" + str(e))
378 Tries to execute create() on every job.
379 This cannot fail thus it is catching every exception.
380 If job.create() fails job is stopped.
381 This is also creating job run time chart.
384 while i < len(self.jobs):
388 self._stop(job, "failed create")
390 chart = job.chart_name
392 "CHART netdata.plugin_pythond_" +
394 " '' 'Execution time for " +
396 " plugin' 'milliseconds / run' python.d netdata.plugin_python area 145000 " +
397 str(job.timetable['freq']) +
399 sys.stdout.write("DIMENSION run_time 'run time' absolute 1 1\n\n")
400 debug("created charts for", job.chart_name)
403 except AttributeError:
404 self._stop(job, "no create")
405 except (UnboundLocalError, Exception) as e:
406 self._stop(job, "misbehaving. Reason: " + str(e))
408 def _update_job(self, job):
410 Tries to execute update() on specified job.
411 This cannot fail thus it is catching every exception.
412 If job.update() returns False, number of retries_left is decremented.
413 If there are no more retries, job is stopped.
414 Job is also stopped if it throws an exception.
415 This is also updating job run time chart.
416 Return False if job is stopped
420 t_start = time.time()
421 # check if it is time to execute job update() function
422 if job.timetable['next'] > t_start:
423 debug(job.chart_name + " will be run in " + str(int((job.timetable['next'] - t_start) * 1000)) + " ms")
429 since_last = int((t_start - job.timetable['last']) * 1000000)
430 debug(job.chart_name +
431 " ready to run, after " + str(int((t_start - job.timetable['last']) * 1000)) +
432 " ms (update_every: " + str(job.timetable['freq'] * 1000) +
433 " ms, latency: " + str(int((t_start - job.timetable['next']) * 1000)) + " ms)")
434 if not job.update(since_last):
435 if job.retries_left <= 0:
436 self._stop(job, "update failed")
438 job.retries_left -= 1
439 job.timetable['next'] += job.timetable['freq']
441 except AttributeError:
442 self._stop(job, "no update")
444 except (UnboundLocalError, Exception) as e:
445 self._stop(job, "misbehaving. Reason: " + str(e))
448 job.timetable['next'] = t_end - (t_end % job.timetable['freq']) + job.timetable['freq']
449 # draw performance graph
450 run_time = str(int((t_end - t_start) * 1000))
451 debug(job.chart_name, "updated in", run_time, "ms")
452 sys.stdout.write("BEGIN netdata.plugin_pythond_" + job.chart_name + " " + str(since_last) + '\n')
453 sys.stdout.write("SET run_time = " + run_time + '\n')
454 sys.stdout.write("END\n")
456 job.timetable['last'] = t_start
457 job.retries_left = job.retries
458 self.first_run = False
463 Tries to execute update() on every job by using _update_job()
464 This will stay forever and ever and ever forever and ever it'll be the one...
466 self.first_run = True
470 while i < len(self.jobs):
472 if self._update_job(job):
474 next_runs.append(job.timetable['next'])
478 if len(next_runs) == 0:
479 fatal('no python.d modules loaded.')
480 time.sleep(min(next_runs) - time.time())
483 def read_config(path):
485 Read YAML configuration from specified file
490 with open(path, 'r') as stream:
491 config = yaml.load(stream)
492 except (OSError, IOError):
493 error(str(path), "is not a valid configuration file")
495 except yaml.YAMLError as e:
496 error(str(path), "is malformed:", e)
501 def parse_cmdline(directory, *commands):
503 Parse parameters from command line.
504 :param directory: str
505 :param commands: list of str
509 global OVERRIDE_UPDATE_EVERY
512 changed_update = False
514 for cmd in commands[1:]:
517 elif cmd == "debug" or cmd == "all":
519 # redirect stderr to stdout?
520 elif os.path.isfile(directory + cmd + ".chart.py") or os.path.isfile(directory + cmd):
522 mods.append(cmd.replace(".chart.py", ""))
525 BASE_CONFIG['update_every'] = int(cmd)
526 changed_update = True
529 if changed_update and DEBUG_FLAG:
530 OVERRIDE_UPDATE_EVERY = True
531 debug(PROGRAM, "overriding update interval to", str(BASE_CONFIG['update_every']))
533 debug("started from", commands[0], "with options:", *commands[1:])
538 # if __name__ == '__main__':
543 global DEBUG_FLAG, BASE_CONFIG
545 # read configuration file
547 configfile = CONFIG_DIR + "python.d.conf"
548 debug(PROGRAM, "reading configuration file:", configfile)
550 conf = read_config(configfile)
553 # exit the whole plugin when 'enabled: no' is set in 'python.d.conf'
554 if conf['enabled'] is False:
555 fatal('disabled in configuration file.\n')
556 except (KeyError, TypeError):
559 for param in BASE_CONFIG:
560 BASE_CONFIG[param] = conf[param]
561 except (KeyError, TypeError):
562 pass # use default update_every from NETDATA_UPDATE_EVERY
564 DEBUG_FLAG = conf['debug']
565 except (KeyError, TypeError):
567 for k, v in conf.items():
568 if k in ("update_every", "debug", "enabled"):
573 # parse passed command line arguments
574 modules = parse_cmdline(MODULES_DIR, *sys.argv)
575 info("MODULES_DIR='" + MODULES_DIR +
576 "', CONFIG_DIR='" + CONFIG_DIR +
577 "', UPDATE_EVERY=" + str(BASE_CONFIG['update_every']) +
578 ", ONLY_MODULES=" + str(modules))
581 charts = PythonCharts(modules, MODULES_DIR, CONFIG_DIR + "python.d/", disabled)
588 if __name__ == '__main__':