better debugging and logging for SocketService modules; redis enhancements

diff --git a/python.d/memcached.chart.py b/python.d/memcached.chart.py
index 09ceed2210fc322a917aaf895c3ca87f127ba820..f0b782dea091985e4a8d3de63f4282b134b418f8 100644 (file)
--- a/python.d/memcached.chart.py
+++ b/python.d/memcached.chart.py
@@ -33,7 +33,7 @@ CHARTS = {
         'options': [None, 'Network', 'kilobytes/s', 'Network', 'memcached.net', 'line'],
         'lines': [
             ['bytes_read', 'read', 'incremental', 1, 1024],
-            ['bytes_written', 'written', 'incremental', 1, 1024]
+            ['bytes_written', 'write', 'incremental', 1, 1024]
         ]},
     'connections': {
         'options': [None, 'Connections', 'connections/s', 'Cluster', 'memcached.connections', 'line'],
@@ -125,46 +125,52 @@ class Service(SocketService):
         Get data from socket
         :return: dict
         """
+        response = self._get_raw_data()
+        if response is None:
+            # error has already been logged
+            return None
+
+        if response.startswith('ERROR'):
+            self.error("received ERROR")
+            return None
+
         try:
-            raw = self._get_raw_data().split("\n")
+            parsed = response.split("\n")
         except AttributeError:
-            self.error("no data received")
-            return None
-        if raw[0].startswith('ERROR'):
-            self.error("memcached returned ERROR")
+            self.error("response is invalid/empty")
             return None
+
+        # split the response
         data = {}
-        for line in raw:
+        for line in parsed:
             if line.startswith('STAT'):
                 try:
                     t = line[5:].split(' ')
-                    data[t[0]] = int(t[1])
+                    data[t[0]] = t[1]
                 except (IndexError, ValueError):
+                    self.debug("invalid line received: " + str(line))
                     pass
-        try:
-            data['hit_rate'] = int((data['keyspace_hits'] / float(data['keyspace_hits'] + data['keyspace_misses'])) * 100)
-        except:
-            data['hit_rate'] = 0
 
+        if len(data) == 0:
+            self.error("received data doesn't have any records")
+            return None
+
+        # custom calculations
         try:
             data['avail'] = int(data['limit_maxbytes']) - int(data['bytes'])
-            data['used'] = data['bytes']
+            data['used'] = int(data['bytes'])
         except:
             pass
 
-        if len(data) == 0:
-            self.error("received data doesn't have needed records")
-            return None
-        else:
-            return data
+        return data
 
     def _check_raw_data(self, data):
         if data.endswith('END\r\n'):
-            self.debug("received response")
+            self.debug("received full response from memcached")
             return True
-        else:
-            self.debug("waiting response")
-            return False
+
+        self.debug("waiting more data from memcached")
+        return False
 
     def check(self):
         """

diff --git a/python.d/python_modules/base.py b/python.d/python_modules/base.py
index 10ad7c93e6c43b9732574a3a707c70a6a99323cc..c92b9c9749476b32f7ab16d5f6c607a2dba53e1a 100644 (file)
--- a/python.d/python_modules/base.py
+++ b/python.d/python_modules/base.py
@@ -506,6 +506,7 @@ class SocketService(SimpleService):
         self.unix_socket = None
         self.request = ""
         self.__socket_config = None
+        self.__empty_request = "".encode()
         SimpleService.__init__(self, configuration=configuration, name=name)
 
     def _socketerror(self, msg=None):
@@ -614,7 +615,7 @@ class SocketService(SimpleService):
         """
         if self._sock is not None:
             try:
-                self.error("closing socket")
+                self.debug("closing socket")
                 self._sock.shutdown(2)  # 0 - read, 1 - write, 2 - all
                 self._sock.close()
             except Exception:
@@ -627,9 +628,9 @@ class SocketService(SimpleService):
         :return: boolean
         """
         # Send request if it is needed
-        if self.request != "".encode():
+        if self.request != self.__empty_request:
             try:
-                self.debug("sending request")
+                self.debug("sending request:", str(self.request))
                 self._sock.send(self.request)
             except Exception as e:
                 self._socketerror("error sending request:" + str(e))
@@ -657,10 +658,12 @@ class SocketService(SimpleService):
                 self._disconnect()
                 break
 
+            self.debug("received data:", str(buf))
             data += buf.decode(errors='ignore')
             if self._check_raw_data(data):
                 break
 
+        self.debug("final response:", str(data))
         return data
 
     def _get_raw_data(self):

diff --git a/python.d/redis.chart.py b/python.d/redis.chart.py
index 218401e1276d081a052dff0322cb607984723579..8258c28744b2a2edadabb9e30c8ddb951a6df2cf 100644 (file)
--- a/python.d/redis.chart.py
+++ b/python.d/redis.chart.py
@@ -19,40 +19,52 @@ retries = 60
 #             'unix_socket': None
 #          }}
 
-ORDER = ['operations', 'hit_rate', 'memory', 'keys', 'clients', 'slaves']
+ORDER = ['operations', 'hit_rate', 'memory', 'keys', 'connections', 'clients', 'slaves', 'persistence']
 
 CHARTS = {
     'operations': {
-        'options': [None, 'Operations', 'operations/s', 'Statistics', 'redis.operations', 'line'],
+        'options': [None, 'Operations', 'operations/s', 'operations', 'redis.operations', 'line'],
         'lines': [
+            ['total_commands_processed', 'commands', 'incremental'],
             ['instantaneous_ops_per_sec', 'operations', 'absolute']
         ]},
     'hit_rate': {
-        'options': [None, 'Hit rate', 'percent', 'Statistics', 'redis.hit_rate', 'line'],
+        'options': [None, 'Hit rate', 'percent', 'hits', 'redis.hit_rate', 'line'],
         'lines': [
             ['hit_rate', 'rate', 'absolute']
         ]},
     'memory': {
-        'options': [None, 'Memory utilization', 'kilobytes', 'Memory', 'redis.memory', 'line'],
+        'options': [None, 'Memory utilization', 'kilobytes', 'memory', 'redis.memory', 'line'],
         'lines': [
             ['used_memory', 'total', 'absolute', 1, 1024],
             ['used_memory_lua', 'lua', 'absolute', 1, 1024]
         ]},
     'keys': {
-        'options': [None, 'Database keys', 'keys', 'Keys', 'redis.keys', 'line'],
+        'options': [None, 'Database keys', 'keys', 'keys', 'redis.keys', 'line'],
         'lines': [
             # lines are created dynamically in `check()` method
         ]},
+    'connections': {
+        'options': [None, 'Connections', 'connections/s', 'connections', 'redis.connections', 'line'],
+        'lines': [
+            ['total_connections_received', 'received', 'incremental'],
+            ['rejected_connections', 'rejected', 'incremental']
+        ]},
     'clients': {
-        'options': [None, 'Clients', 'clients', 'Clients', 'redis.clients', 'line'],
+        'options': [None, 'Clients', 'clients', 'connections', 'redis.clients', 'line'],
         'lines': [
             ['connected_clients', 'connected', 'absolute'],
             ['blocked_clients', 'blocked', 'absolute']
         ]},
     'slaves': {
-        'options': [None, 'Slaves', 'slaves', 'Replication', 'redis.slaves', 'line'],
+        'options': [None, 'Slaves', 'slaves', 'replication', 'redis.slaves', 'line'],
         'lines': [
             ['connected_slaves', 'connected', 'absolute']
+        ]},
+    'persistence': {
+        'options': [None, 'Changes since last save', 'changes', 'persistence', 'redis.rdb_changes', 'line'],
+        'lines': [
+            ['rdb_changes_since_last_save', 'changes', 'absolute']
         ]}
 }
 
@@ -74,33 +86,45 @@ class Service(SocketService):
         Get data from socket
         :return: dict
         """
+        response = self._get_raw_data()
+        if response is None:
+            # error has already been logged
+            return None
+
         try:
-            raw = self._get_raw_data().split("\n")
+            parsed = response.split("\n")
         except AttributeError:
-            self.error("no data received")
+            self.error("response is invalid/empty")
             return None
+
         data = {}
-        for line in raw:
-            if line.startswith(('instantaneous', 'keyspace', 'used_memory', 'connected', 'blocked')):
-                try:
-                    t = line.split(':')
-                    data[t[0]] = int(t[1])
-                except (IndexError, ValueError):
-                    pass
-            elif line.startswith('db'):
+        for line in parsed:
+            if len(line) < 5 or line[0] == '$' or line[0] == '#':
+                continue
+
+            if line.startswith('db'):
                 tmp = line.split(',')[0].replace('keys=', '')
                 record = tmp.split(':')
-                data[record[0]] = int(record[1])
+                data[record[0]] = record[1]
+                continue
+
+            try:
+                t = line.split(':')
+                data[t[0]] = t[1]
+            except (IndexError, ValueError):
+                self.debug("invalid line received: " + str(line))
+                pass
+
+        if len(data) == 0:
+            self.error("received data doesn't have any records")
+            return None
+
         try:
-            data['hit_rate'] = int((data['keyspace_hits'] / float(data['keyspace_hits'] + data['keyspace_misses'])) * 100)
+            data['hit_rate'] = (int(data['keyspace_hits']) * 100) / (int(data['keyspace_hits']) + int(data['keyspace_misses']))
         except:
             data['hit_rate'] = 0
 
-        if len(data) == 0:
-            self.error("received data doesn't have needed records")
-            return None
-        else:
-            return data
+        return data
 
     def _check_raw_data(self, data):
         """
@@ -113,11 +137,12 @@ class Service(SocketService):
         supposed = data.split('\n')[0][1:]
         offset = len(supposed) + 4  # 1 dollar sing, 1 new line character + 1 ending sequence '\r\n'
         supposed = int(supposed)
+
         if length - offset >= supposed:
+            self.debug("received full response from redis")
             return True
-        else:
-            return False
 
+        self.debug("waiting more data from redis")
         return False
 
     def check(self):

diff --git a/python.d/squid.chart.py b/python.d/squid.chart.py
index 8300d9bad515ec04cf422789e876ca6a47139fe5..e9e8f1d08bbb1f263136a3e59ff0f59d5960d134 100644 (file)
--- a/python.d/squid.chart.py
+++ b/python.d/squid.chart.py
@@ -58,20 +58,25 @@ class Service(SocketService):
         Get data via http request
         :return: dict
         """
+        response = self._get_raw_data()
+
         data = {}
         try:
             raw = ""
-            for tmp in self._get_raw_data().split('\r\n'):
+            for tmp in response.split('\r\n'):
                 if tmp.startswith("sample_time"):
                     raw = tmp
                     break
+
             if raw.startswith('<'):
                 self.error("invalid data received")
                 return None
+
             for row in raw.split('\n'):
                 if row.startswith(("client", "server.all")):
                     tmp = row.split("=")
                     data[tmp[0].replace('.', '_').strip(' ')] = int(tmp[1])
+
         except (ValueError, AttributeError, TypeError):
             self.error("invalid data received")
             return None
@@ -83,15 +88,19 @@ class Service(SocketService):
             return data
 
     def _check_raw_data(self, data):
-        if "Connection: keep-alive" in data[:1024]:
+        header = data[:1024].lower()
+
+        if "connection: keep-alive" in header:
             self._keep_alive = True
         else:
             self._keep_alive = False
 
-        if data[-7:] == "\r\n0\r\n\r\n" and "Transfer-Encoding: chunked" in data[:1024]:  # HTTP/1.1 response
+        if data[-7:] == "\r\n0\r\n\r\n" and "transfer-encoding: chunked" in header:  # HTTP/1.1 response
+            self.debug("received full response from squid")
             return True
-        else:
-            return False
+
+        self.debug("waiting more data from squid")
+        return False
 
     def check(self):
         """