8 assert sys.version_info >= (3, 1)
9 import importlib.machinery
11 # change this hack below if we want PY_VERSION to be used in modules
13 # builtins.PY_VERSION = 3
15 sys.stderr.write('python.d.plugin: Using python 3\n')
16 except (AssertionError, ImportError):
20 # change this hack below if we want PY_VERSION to be used in modules
22 # __builtin__.PY_VERSION = 2
24 sys.stderr.write('python.d.plugin: Using python 2\n')
25 except (AssertionError, ImportError):
26 sys.stderr.write('python.d.plugin: Cannot start. No importlib.machinery on python3 or lack of imp on python2\n')
27 sys.stdout.write('DISABLE\n')
32 sys.stderr.write('python.d.plugin: Cannot find yaml library\n')
33 sys.stdout.write('DISABLE\n')
37 PROGRAM = "python.d.plugin"
38 MODULE_EXTENSION = ".chart.py"
39 BASE_CONFIG = {'update_every': 10,
44 class PythonCharts(object):
48 modules_path='../python.d/',
49 modules_configs='../conf.d/',
50 modules_disabled=None):
54 if modules_disabled is None:
58 # set configuration directory
59 self.configs = modules_configs
62 loaded_modules = self._load_modules(modules_path, modules, modules_disabled)
64 # load configuration files
65 configured_modules = self._load_configs(loaded_modules)
67 # good economy and prosperity:
68 self.jobs = self._create_jobs(configured_modules)
69 if DEBUG_FLAG and interval is not None:
71 job.create_timetable(interval)
74 def _import_module(path, name=None):
75 # try to import module using only its path
77 name = path.split('/')[-1]
78 if name[-len(MODULE_EXTENSION):] != MODULE_EXTENSION:
80 name = name[:-len(MODULE_EXTENSION)]
83 return importlib.machinery.SourceFileLoader(name, path).load_module()
85 return imp.load_source(name, path)
86 # return importlib.import_module(path, name)
87 except Exception as e:
91 def _load_modules(self, path, modules, disabled):
92 # check if plugin directory exists
93 if not os.path.isdir(path):
94 debug("cannot find charts directory ", path)
95 sys.stdout.write("DISABLE\n")
104 mod = self._import_module(path + m + MODULE_EXTENSION)
107 else: # exit if plugin is not found
108 sys.stdout.write("DISABLE")
112 # scan directory specified in path and load all modules from there
113 names = os.listdir(path)
115 if mod.strip(MODULE_EXTENSION) in disabled:
116 debug("disabling:", mod.strip(MODULE_EXTENSION))
118 m = self._import_module(path + mod)
120 debug("loading chart: '" + path + mod + "'")
124 def _load_configs(self, modules):
125 # function loads configuration files to modules
127 configfile = self.configs + mod.__name__ + ".conf"
128 if os.path.isfile(configfile):
129 debug("loading chart options: '" + configfile + "'")
133 self._parse_config(mod, read_config(configfile)))
134 except Exception as e:
135 debug("something went wrong while loading configuration", e)
138 ": configuration file '" +
140 "' not found. Using defaults.")
141 # set config if not found
142 if not hasattr(mod, 'config'):
143 mod.config = {None: {}}
144 for var in BASE_CONFIG:
146 mod.config[None][var] = getattr(mod, var)
147 except AttributeError:
148 mod.config[None][var] = BASE_CONFIG[var]
152 def _parse_config(module, config):
155 for key in BASE_CONFIG:
157 # get defaults from module config
158 defaults[key] = int(config.pop(key))
159 except (KeyError, ValueError):
161 # get defaults from module source code
162 defaults[key] = getattr(module, key)
163 except (KeyError, ValueError):
164 # if above failed, get defaults from global dict
165 defaults[key] = BASE_CONFIG[key]
167 # check if there are dict in config dict
170 if type(config[name]) is dict:
174 # assign variables needed by supervisor to every job configuration
178 if key not in config[name]:
179 config[name][key] = defaults[key]
180 # if only one job is needed, values doesn't have to be in dict (in YAML)
182 config = {None: config.copy()}
183 config[None].update(defaults)
185 # return dictionary of jobs where every job has BASE_CONFIG variables
189 def _create_jobs(modules):
190 # module store a definition of Service class
191 # module store configuration in module.config
192 # configs are list of dicts or a dict
193 # one dict is one service
194 # iterate over list of modules and inside one loop iterate over configs
196 for module in modules:
197 for name in module.config:
199 conf = module.config[name]
201 job = module.Service(configuration=conf, name=name)
202 except Exception as e:
203 debug(module.__name__ +
204 ": Couldn't start job named " +
210 # set execution_name (needed to plot run time graphs)
211 job.execution_name = module.__name__
213 job.execution_name += "_" + name
215 return [j for j in jobs if j is not None]
217 def _stop(self, job, reason=None):
219 self.jobs.remove(job)
222 elif reason[:3] == "no ":
225 "' does not seem to have " +
227 "() function. Disabling it.")
228 elif reason[:7] == "failed ":
230 job.execution_name + "' " +
232 "() function reports failure.")
233 elif reason[:13] == "configuration":
234 debug(job.execution_name,
235 "configuration file '" +
238 ".conf' not found. Using defaults.")
239 elif reason[:11] == "misbehaving":
240 debug(job.execution_name, "is " + reason)
243 # try to execute check() on every job
244 for job in self.jobs:
247 self._stop(job, "failed check")
248 except AttributeError:
249 self._stop(job, "no check")
250 except (UnboundLocalError, Exception) as e:
251 self._stop(job, "misbehaving. Reason: " + str(e))
254 # try to execute create() on every job
255 for job in self.jobs:
258 self._stop(job, "failed create")
260 chart = job.execution_name
262 "CHART netdata.plugin_pythond_" +
264 " '' 'Execution time for " +
266 " plugin' 'milliseconds / run' python.d netdata.plugin_python area 145000 " +
267 str(job.timetable['freq']) +
269 sys.stdout.write("DIMENSION run_time 'run time' absolute 1 1\n\n")
271 except AttributeError:
272 self._stop(job, "no create")
273 except (UnboundLocalError, Exception) as e:
274 self._stop(job, "misbehaving. Reason: " + str(e))
276 def _update_job(self, job):
277 # try to execute update() on every job and draw run time graph
278 t_start = time.time()
279 # check if it is time to execute job update() function
280 if job.timetable['next'] > t_start:
286 since_last = int((t_start - job.timetable['last']) * 1000000)
287 if not job.update(since_last):
288 self._stop(job, "update failed")
290 except AttributeError:
291 self._stop(job, "no update")
293 except (UnboundLocalError, Exception) as e:
294 self._stop(job, "misbehaving. Reason: " + str(e))
297 job.timetable['next'] = t_end - (t_end % job.timetable['freq']) + job.timetable['freq']
298 # draw performance graph
299 sys.stdout.write("BEGIN netdata.plugin_pythond_" + job.execution_name + " " + str(since_last) + '\n')
300 sys.stdout.write("SET run_time = " + str(int((t_end - t_start) * 1000)) + '\n')
301 sys.stdout.write("END\n")
303 job.timetable['last'] = t_start
304 self.first_run = False
307 # run updates (this will stay forever and ever and ever forever and ever it'll be the one...)
308 self.first_run = True
311 for job in self.jobs:
312 self._update_job(job)
314 next_runs.append(job.timetable['next'])
317 if len(next_runs) == 0:
318 debug("No plugins loaded")
319 sys.stdout.write("DISABLE\n")
321 time.sleep(min(next_runs) - time.time())
324 def read_config(path):
326 with open(path, 'r') as stream:
327 config = yaml.load(stream)
328 except (OSError, IOError):
329 debug(str(path), "is not a valid configuration file")
331 except yaml.YAMLError as e:
332 debug(str(path), "is malformed:", e)
340 sys.stderr.write(PROGRAM + ":")
342 sys.stderr.write(" " + str(i))
343 sys.stderr.write("\n")
347 def parse_cmdline(directory, *commands):
352 for cmd in commands[1:]:
355 elif cmd == "debug" or cmd == "all":
357 # redirect stderr to stdout?
358 elif os.path.isfile(directory + cmd + ".chart.py") or os.path.isfile(directory + cmd):
360 mods.append(cmd.replace(".chart.py", ""))
368 debug("started from", commands[0], "with options:", *commands[1:])
369 if len(mods) == 0 and DEBUG_FLAG is False:
372 return {'interval': interval,
376 # if __name__ == '__main__':
378 global PROGRAM, DEBUG_FLAG
379 PROGRAM = sys.argv[0].split('/')[-1].split('.plugin')[0]
380 # parse env variables
381 # https://github.com/firehol/netdata/wiki/External-Plugins#environment-variables
382 main_dir = os.getenv('NETDATA_PLUGINS_DIR',
383 os.path.abspath(__file__).strip("python.d.plugin.py"))
384 config_dir = os.getenv('NETDATA_CONFIG_DIR', "/etc/netdata/")
385 interval = os.getenv('NETDATA_UPDATE_EVERY', None)
387 # read configuration file
389 if config_dir[-1] != '/':
391 configfile = config_dir + "python.d.conf"
393 conf = read_config(configfile)
396 if str(conf['enable']) is False:
397 debug("disabled in configuration file")
398 sys.stdout.write("DISABLE\n")
400 except (KeyError, TypeError):
403 modules_conf = conf['plugins_config_dir']
404 except (KeyError, TypeError):
405 modules_conf = config_dir + "python.d/" # default configuration directory
407 modules_dir = conf['plugins_dir']
408 except (KeyError, TypeError):
409 modules_dir = main_dir.replace("plugins.d", "python.d")
411 interval = conf['interval']
412 except (KeyError, TypeError):
413 pass # use default interval from NETDATA_UPDATE_EVERY
415 DEBUG_FLAG = conf['debug']
416 except (KeyError, TypeError):
418 for k, v in conf.items():
419 if k in ("plugins_config_dir", "plugins_dir", "interval", "debug"):
424 modules_conf = config_dir + "python.d/"
425 modules_dir = main_dir.replace("plugins.d", "python.d")
427 # directories should end with '/'
428 if modules_dir[-1] != '/':
430 if modules_conf[-1] != '/':
433 # parse passed command line arguments
434 out = parse_cmdline(modules_dir, *sys.argv)
435 modules = out['modules']
436 if out['interval'] is not None:
437 interval = out['interval']
439 # configure environment to run modules
440 sys.path.append(modules_dir + "python_modules") # append path to directory with modules dependencies
443 charts = PythonCharts(interval, modules, modules_dir, modules_conf, disabled)
447 sys.stdout.write("DISABLE")
450 if __name__ == '__main__':