1 # -*- coding: utf-8 -*-
2 # Description: prototypes for netdata python.d modules
3 # Author: Pawel Krupa (paulfantom)
10 import urllib.request as urllib2
14 from subprocess import Popen, PIPE
20 class BaseService(threading.Thread):
22 Prototype of Service class.
23 Implemented basic functionality to run jobs by `python.d.plugin`
25 def __init__(self, configuration=None, name=None):
27 This needs to be initialized in child classes
28 :param configuration: dict
31 threading.Thread.__init__(self)
32 self._data_stream = ""
36 self.priority = 140000
39 self.override_name = None
43 if configuration is None:
44 self.error("BaseService: no configuration parameters supplied. Cannot create Service.")
47 self._extract_base_config(configuration)
49 self.create_timetable()
51 def _extract_base_config(self, config):
53 Get basic parameters to run service
55 config = {'update_every':1,
61 self.override_name = config.pop('name')
64 self.update_every = int(config.pop('update_every'))
65 self.priority = int(config.pop('priority'))
66 self.retries = int(config.pop('retries'))
67 self.retries_left = self.retries
68 self.configuration = config
70 def create_timetable(self, freq=None):
72 Create service timetable.
75 timetable = {'last': 1466370091.3767564,
81 freq = self.update_every
83 self.timetable = {'last': now,
84 'next': now - (now % freq) + freq,
89 Executes self.update(interval) and draws run time chart.
90 Return value presents exit status of update()
94 # check if it is time to execute job update() function
95 if self.timetable['next'] > t_start:
96 #msg.debug(self.chart_name + " will be run in " +
97 # str(int((self.timetable['next'] - t_start) * 1000)) + " ms")
98 msg.debug(self.chart_name, "will be run in", str(int((self.timetable['next'] - t_start) * 1000)), "ms")
101 since_last = int((t_start - self.timetable['last']) * 1000000)
102 #msg.debug(self.chart_name +
103 # " ready to run, after " + str(int((t_start - self.timetable['last']) * 1000)) +
104 # " ms (update_every: " + str(self.timetable['freq'] * 1000) +
105 # " ms, latency: " + str(int((t_start - self.timetable['next']) * 1000)) + " ms)")
106 msg.debug(self.chart_name,
107 "ready to run, after", str(int((t_start - self.timetable['last']) * 1000)),
108 "ms (update_every:", str(self.timetable['freq'] * 1000),
109 "ms, latency:", str(int((t_start - self.timetable['next']) * 1000)), "ms")
110 if not self.update(since_last):
111 self.error("update function failed.")
114 self.timetable['next'] = t_end - (t_end % self.timetable['freq']) + self.timetable['freq']
115 # draw performance graph
116 run_time = str(int((t_end - t_start) * 1000))
117 #run_time_chart = "BEGIN netdata.plugin_pythond_" + self.chart_name + " " + str(since_last) + '\n'
118 #run_time_chart += "SET run_time = " + run_time + '\n'
119 #run_time_chart += "END\n"
120 #sys.stdout.write(run_time_chart)
121 sys.stdout.write("BEGIN netdata.plugin_pythond_%s %s\nSET run_time = %s\nEND\n" % \
122 (self.chart_name, str(since_last), run_time))
124 #msg.debug(self.chart_name + " updated in " + str(run_time) + " ms")
125 msg.debug(self.chart_name, "updated in", str(run_time), "ms")
126 self.timetable['last'] = t_start
131 Runs job in thread. Handles retries.
132 Exits when job failed or timed out.
135 self.timetable['last'] = time.time()
138 status = self._run_once()
139 except Exception as e:
140 msg.error("Something wrong: ", str(e))
143 time.sleep(self.timetable['next'] - time.time())
144 self.retries_left = self.retries
146 self.retries_left -= 1
147 if self.retries_left <= 0:
148 msg.error("no more retries. Exiting")
151 time.sleep(self.timetable['freq'])
156 append = params.append
161 if type(p) is not str:
168 def _line(self, instruction, *params):
170 Converts *params to string and joins them with one space between every one.
171 :param params: str/int/float
173 #self._data_stream += instruction
174 tmp = list(map((lambda x: "''" if x is None or len(x) == 0 else x), params))
176 self._data_stream += "%s %s\n" % (instruction, str(" ".join(tmp)))
178 # self.error(str(" ".join(tmp)))
188 # self._data_stream += " " + p
189 #self._data_stream += "\n"
191 def chart(self, type_id, name="", title="", units="", family="",
192 category="", charttype="line", priority="", update_every=""):
201 :param charttype: str
202 :param priority: int/str
203 :param update_every: int/str
205 self._charts.append(type_id)
206 #self._line("CHART", type_id, name, title, units, family, category, charttype, priority, update_every)
208 p = self._format(type_id, name, title, units, family, category, charttype, priority, update_every)
209 self._line("CHART", *p)
211 def dimension(self, id, name=None, algorithm="absolute", multiplier=1, divisor=1, hidden=False):
213 Defines a new dimension for the chart
216 :param algorithm: str
217 :param multiplier: int/str
218 :param divisor: int/str
219 :param hidden: boolean
225 self.error("malformed dimension: multiplier is not a number:", multiplier)
230 self.error("malformed dimension: divisor is not a number:", divisor)
234 if algorithm not in ("absolute", "incremental", "percentage-of-absolute-row", "percentage-of-incremental-row"):
235 algorithm = "absolute"
237 self._dimensions.append(str(id))
239 p = self._format(id, name, algorithm, multiplier, divisor, "hidden")
240 #self._line("DIMENSION", id, name, algorithm, str(multiplier), str(divisor), "hidden")
242 p = self._format(id, name, algorithm, multiplier, divisor)
243 #self._line("DIMENSION", id, name, algorithm, str(multiplier), str(divisor))
245 self._line("DIMENSION", *p)
247 def begin(self, type_id, microseconds=0):
251 :param microseconds: int
254 if type_id not in self._charts:
255 self.error("wrong chart type_id:", type_id)
260 self.error("malformed begin statement: microseconds are not a number:", microseconds)
263 self._line("BEGIN", type_id, str(microseconds))
266 def set(self, id, value):
268 Set value to dimension
270 :param value: int/float
273 if id not in self._dimensions:
274 self.error("wrong dimension id:", id, "Available dimensions are:", *self._dimensions)
277 value = str(int(value))
279 self.error("cannot set non-numeric value:", value)
281 self._line("SET", id, "=", str(value))
289 Upload new data to netdata
291 print(self._data_stream)
292 self._data_stream = ""
294 def error(self, *params):
296 Show error message on stderr
298 msg.error(self.chart_name, *params)
300 def debug(self, *params):
302 Show debug message on stderr
304 msg.debug(self.chart_name, *params)
306 def info(self, *params):
308 Show information message on stderr
310 msg.info(self.chart_name, *params)
317 msg.error("Service " + str(self.__module__) + "doesn't implement check() function")
325 msg.error("Service " + str(self.__module__) + "doesn't implement create() function?")
328 def update(self, interval):
334 msg.error("Service " + str(self.__module__) + "doesn't implement update() function")
338 class SimpleService(BaseService):
339 def __init__(self, configuration=None, name=None):
341 self.definitions = {}
342 BaseService.__init__(self, configuration=configuration, name=name)
362 data = self._get_data()
367 for name in self.order:
368 options = self.definitions[name]['options'] + [self.priority + idx, self.update_every]
369 self.chart(self.chart_name + "." + name, *options)
370 # check if server has this datapoint
371 for line in self.definitions[name]['lines']:
373 self.dimension(*line)
379 def update(self, interval):
385 data = self._get_data()
387 self.debug("_get_data() returned no data")
391 for chart in self.order:
392 if self.begin(self.chart_name + "." + chart, interval):
394 for dim in self.definitions[chart]['lines']:
396 self.set(dim[0], data[dim[0]])
403 self.error("no charts to update")
408 class UrlService(SimpleService):
409 def __init__(self, configuration=None, name=None):
413 SimpleService.__init__(self, configuration=configuration, name=name)
415 def __add_auth(self):
416 passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
417 passman.add_password(None, self.url, self.user, self.password)
418 authhandler = urllib2.HTTPBasicAuthHandler(passman)
419 opener = urllib2.build_opener(authhandler)
420 urllib2.install_opener(opener)
422 def _get_raw_data(self):
424 Get raw data from http request
429 f = urllib2.urlopen(self.url, timeout=self.update_every)
430 except Exception as e:
435 raw = f.read().decode('utf-8')
436 except Exception as e:
444 Format configuration data and try to connect to server
447 if self.name is None or self.name == str(None):
449 self.chart_name += "_" + self.name
451 self.name = str(self.name)
453 self.url = str(self.configuration['url'])
454 except (KeyError, TypeError):
457 self.user = str(self.configuration['user'])
458 except (KeyError, TypeError):
461 self.password = str(self.configuration['pass'])
462 except (KeyError, TypeError):
465 if self.user is not None and self.password is not None:
468 if self._get_data() is not None:
474 class SocketService(SimpleService):
475 def __init__(self, configuration=None, name=None):
477 self._keep_alive = True
478 self.host = "localhost"
480 self.unix_socket = None
482 SimpleService.__init__(self, configuration=configuration, name=name)
484 def _get_raw_data(self):
486 Get raw data with low-level "socket" module.
489 # Recreate socket since they cannot be reused
490 if self._sock is None:
492 if self.unix_socket is None:
493 self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
494 self._sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
495 #sock.setsockopt(socket.SOL_SOCKET, socket.TCP_NODELAY, 1)
496 #sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
497 #sock.settimeout(self.update_every)
498 self._sock.settimeout(0.5)
499 self._sock.connect((self.host, self.port))
500 self._sock.settimeout(0.5) # Just to be sure
502 self._sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
503 #self._sock.settimeout(self.update_every)
504 self._sock.settimeout(0.05)
505 self._sock.connect(self.unix_socket)
506 self._sock.settimeout(0.05) # Just to be sure
507 except Exception as e:
509 "Cannot create socket with following configuration: host:", str(self.host),
510 "port:", str(self.port),
511 "socket:", str(self.unix_socket))
515 # Send request if it is needed
516 if self.request != "".encode():
518 self._sock.send(self.request)
519 except Exception as e:
521 self._sock.shutdown(0)
527 "used configuration: host:", str(self.host),
528 "port:", str(self.port),
529 "socket:", str(self.unix_socket))
532 # Receive first two bytes from socket. This will test if there is anything to receive
535 data = self._sock.recv(size).decode()
536 except Exception as e:
538 "used configuration: host:", str(self.host),
539 "port:", str(self.port),
540 "socket:", str(self.unix_socket))
541 self._sock.shutdown(0)
548 # implement something like TCP Window Scaling
551 buf = self._sock.recv(size)
553 if len(buf) == 0: break # precaution
554 if len(buf) < size and not self._more_data_available():
559 if not self._keep_alive:
560 self._sock.shutdown(0)
566 def _more_data_available(self):
569 def _parse_config(self):
571 Parse configuration data
574 if self.name is None or self.name == str(None):
577 self.name = str(self.name)
579 self.unix_socket = str(self.configuration['socket'])
580 except (KeyError, TypeError):
581 self.debug("No unix socket specified. Trying TCP/IP socket.")
583 self.host = str(self.configuration['host'])
584 except (KeyError, TypeError):
585 self.debug("No host specified. Using: '" + self.host + "'")
587 self.port = int(self.configuration['port'])
588 except (KeyError, TypeError):
589 self.debug("No port specified. Using: '" + str(self.port) + "'")
591 self.request = str(self.configuration['request'])
592 except (KeyError, TypeError):
593 self.debug("No request specified. Using: '" + str(self.request) + "'")
594 self.request = self.request.encode()
597 class LogService(SimpleService):
598 def __init__(self, configuration=None, name=None):
600 self._last_position = 0
601 # self._log_reader = None
602 SimpleService.__init__(self, configuration=configuration, name=name)
603 self.retries = 100000 # basically always retry
605 def _get_raw_data(self):
607 Get log lines since last poll
612 if os.path.getsize(self.log_path) < self._last_position:
613 self._last_position = 0
614 elif os.path.getsize(self.log_path) == self._last_position:
615 self.debug("Log file hasn't changed. No new data.")
617 with open(self.log_path, "r") as fp:
618 fp.seek(self._last_position)
619 for i, line in enumerate(fp):
621 self._last_position = fp.tell()
622 except Exception as e:
628 self.error("No data collected.")
633 Parse basic configuration and check if log file exists
636 if self.name is not None or self.name != str(None):
639 self.name = str(self.name)
641 self.log_path = str(self.configuration['path'])
642 except (KeyError, TypeError):
643 self.error("No path to log specified. Using: '" + self.log_path + "'")
645 if os.access(self.log_path, os.R_OK):
648 self.error("Cannot access file: '" + self.log_path + "'")
652 status = SimpleService.create(self)
653 self._last_position = 0
657 class ExecutableService(SimpleService):
658 #command_whitelist = ['exim', 'postqueue']
659 bad_substrings = ('&', '|', ';', '>', '<')
661 def __init__(self, configuration=None, name=None):
663 SimpleService.__init__(self, configuration=configuration, name=name)
665 def _get_raw_data(self):
667 Get raw data from executed command
671 p = Popen(self.command, stdout=PIPE, stderr=PIPE)
672 except Exception as e:
673 self.error("Executing command", self.command, "resulted in error:", str(e))
676 for line in p.stdout.readlines():
677 data.append(str(line.decode()))
680 self.error("No data collected.")
687 Parse basic configuration, check if command is whitelisted and is returning values
690 if self.name is not None or self.name != str(None):
693 self.name = str(self.name)
695 self.command = str(self.configuration['command'])
696 except (KeyError, TypeError):
697 self.error("No command specified. Using: '" + self.command + "'")
698 self.command = self.command.split(' ')
699 #if self.command[0] not in self.command_whitelist:
700 # self.error("Command is not whitelisted.")
703 for arg in self.command[1:]:
704 if any(st in arg for st in self.bad_substrings):
705 self.error("Bad command argument:" + " ".join(self.command[1:]))
707 # test command and search for it in /usr/sbin or /sbin when failed
708 base = self.command[0].split('/')[-1]
709 if self._get_raw_data() is None:
710 for prefix in ['/sbin/', '/usr/sbin/']:
711 self.command[0] = prefix + base
712 if os.path.isfile(self.command[0]):
714 #if self._get_raw_data() is not None:
717 if self._get_data() is None or len(self._get_data()) == 0:
718 self.error("Command", self.command, "returned no data")