Merge pull request #696 from paulfantom/master

better python logging

diff --git a/plugins.d/python.d.plugin b/plugins.d/python.d.plugin
index 10d881b33bf0fe5db732c2ae27b4fad44d80d807..746802cbe24a6dc7f2777128ad712bf2349bf07b 100755 (executable)
--- a/plugins.d/python.d.plugin
+++ b/plugins.d/python.d.plugin
@@ -291,6 +291,7 @@ class PythonCharts(object):
         except Exception as e:
             msg.debug("This shouldn't happen. NO " + prefix + " IN LIST:" + str(self.jobs) + " ERROR: " + str(e))
 
+        # TODO remove section below and remove `reason`.
         prefix += ": "
         if reason is None:
             return
@@ -320,13 +321,15 @@ class PythonCharts(object):
         """
         i = 0
         overridden = []
+        msg.debug("all job objects", str(self.jobs))
         while i < len(self.jobs):
             job = self.jobs[i]
             try:
                 if not job.check():
-                    self._stop(job, "failed check")
+                    msg.error(job.chart_name, "check function failed.")
+                    self._stop(job)
                 else:
-                    msg.debug(job.chart_name, ": check succeeded")
+                    msg.debug(job.chart_name, "check succeeded")
                     i += 1
                     try:
                         if job.override_name is not None:
@@ -343,9 +346,13 @@ class PythonCharts(object):
                     except Exception:
                         pass
             except AttributeError:
-                self._stop(job, "no check")
+                self._stop(job)
+                msg.error(job.chart_name, "cannot find check() function.")
             except (UnboundLocalError, Exception) as e:
-                self._stop(job, "misbehaving. Reason:" + str(e))
+                msg.error(job.chart_name, str(e))
+                self._stop(job)
+        msg.debug("overridden job names:", str(overridden))
+        msg.debug("all remaining job objects:", str(self.jobs))
 
     def create(self):
         """
@@ -359,7 +366,8 @@ class PythonCharts(object):
             job = self.jobs[i]
             try:
                 if not job.create():
-                    self._stop(job, "failed create")
+                    msg.error(job.chart_name, "create function failed.")
+                    self._stop(job)
                 else:
                     chart = job.chart_name
                     sys.stdout.write(
@@ -375,9 +383,11 @@ class PythonCharts(object):
                     # sys.stdout.flush()
                     i += 1
             except AttributeError:
-                self._stop(job, "no create")
+                msg.error(job.chart_name, "cannot find create() function.")
+                self._stop(job)
             except (UnboundLocalError, Exception) as e:
-                self._stop(job, "misbehaving. Reason: " + str(e))
+                msg.error(job.chart_name, str(e))
+                self._stop(job)
 
     def update(self):
         """

diff --git a/python.d/cpufreq.chart.py b/python.d/cpufreq.chart.py
index 5246e3424ee85eec45ef52bbc9e17fb6048629f4..600786bfeeefd85454fb80b5e90db03041d604ae 100644 (file)
--- a/python.d/cpufreq.chart.py
+++ b/python.d/cpufreq.chart.py
@@ -53,6 +53,7 @@ class Service(SimpleService):
                 self.paths.append(dirpath + "/" + self.filename)
 
         if len(self.paths) == 0:
+            self.error("cannot find", self.filename)
             return False
 
         self.paths.sort()

diff --git a/python.d/hddtemp.chart.py b/python.d/hddtemp.chart.py
index 92b1b7a614113beb1b0a12725ba7b84d3c3e75c9..487e0ad330696aa1a4eaaa0436ecab3cf1e86076 100644 (file)
--- a/python.d/hddtemp.chart.py
+++ b/python.d/hddtemp.chart.py
@@ -46,6 +46,7 @@ class Service(SocketService):
         try:
             raw = self._get_raw_data().split("|")[:-1]
         except AttributeError:
+            self.error("no data received")
             return None
         data = {}
         for i in range(len(raw) // 5):
@@ -54,7 +55,12 @@ class Service(SocketService):
             except ValueError:
                 val = 0
             data[raw[i*5+1].replace("/dev/", "")] = val
-        return data
+
+        if len(data) == 0:
+            self.error("received data doesn't have needed records")
+            return None
+        else:
+            return data
 
     def check(self):
         """
@@ -64,7 +70,6 @@ class Service(SocketService):
         self._parse_config()
         data = self._get_data()
         if data is None:
-            self.error("No data received")
             return False
 
         for name in data:

diff --git a/python.d/python_modules/base.py b/python.d/python_modules/base.py
index b8fa3e9e3acf17f7f6e3a171e61c1788f7725dd6..c8309dbf69070b07fe0f28918dcd821214764f21 100644 (file)
--- a/python.d/python_modules/base.py
+++ b/python.d/python_modules/base.py
@@ -6,7 +6,6 @@ import time
 import sys
 import os
 import socket
-import resource
 try:
     import urllib.request as urllib2
 except ImportError:
@@ -96,7 +95,7 @@ class BaseService(threading.Thread):
         if self.timetable['next'] > t_start:
             #msg.debug(self.chart_name + " will be run in " +
             #          str(int((self.timetable['next'] - t_start) * 1000)) + " ms")
-            msg.debug(self.chart_name,"will be run in", str(int((self.timetable['next'] - t_start) * 1000)), "ms")
+            msg.debug(self.chart_name, "will be run in", str(int((self.timetable['next'] - t_start) * 1000)), "ms")
             return True
 
         since_last = int((t_start - self.timetable['last']) * 1000000)
@@ -109,6 +108,7 @@ class BaseService(threading.Thread):
                   "ms (update_every:", str(self.timetable['freq'] * 1000),
                   "ms, latency:", str(int((t_start - self.timetable['next']) * 1000)), "ms")
         if not self.update(since_last):
+            self.error("update function failed.")
             return False
         t_end = time.time()
         self.timetable['next'] = t_end - (t_end % self.timetable['freq']) + self.timetable['freq']
@@ -137,7 +137,7 @@ class BaseService(threading.Thread):
             try:
                 status = self._run_once()
             except Exception as e:
-                msg.error("Something wrong: " + str(e))
+                msg.error("Something wrong: ", str(e))
                 return
             if status:
                 time.sleep(self.timetable['next'] - time.time())
@@ -150,7 +150,8 @@ class BaseService(threading.Thread):
                 else:
                     time.sleep(self.timetable['freq'])
 
-    def _format(self, *args):
+    @staticmethod
+    def _format(*args):
         params = []
         append = params.append
         for p in args:
@@ -383,6 +384,7 @@ class SimpleService(BaseService):
         """
         data = self._get_data()
         if data is None:
+            self.debug("_get_data() returned no data")
             return False
 
         updated = False
@@ -397,6 +399,8 @@ class SimpleService(BaseService):
                 self.end()
 
         self.commit()
+        if not updated:
+            self.error("no charts to update")
 
         return updated
 
@@ -424,13 +428,13 @@ class UrlService(SimpleService):
         try:
             f = urllib2.urlopen(self.url, timeout=self.update_every)
         except Exception as e:
-            msg.error(self.__module__, str(e))
+            self.error(str(e))
             return None
 
         try:
             raw = f.read().decode('utf-8')
         except Exception as e:
-            msg.error(self.__module__, str(e))
+            self.error(str(e))
         finally:
             f.close()
         return raw
@@ -500,27 +504,28 @@ class SocketService(SimpleService):
                     sock.settimeout(0.05)  # Just to be sure
 
             except Exception as e:
-                self.error(str(e))
+                self.error(str(e), "used configuration: host:", str(self.host), "port:", str(self.port), "socket:", str(self.unix_socket))
                 self.sock = None
                 return None
 
         if self.request != "".encode():
             try:
                 sock.send(self.request)
-            except Exception:
+            except Exception as e:
                 try:
                     sock.shutdown(1)
                     sock.close()
                 except:
                     pass
                 self.sock = None
+                self.error(str(e), "used configuration: host:", str(self.host), "port:", str(self.port), "socket:", str(self.unix_socket))
                 return None
 
         size = 2
         try:
             data = sock.recv(size).decode()
         except Exception as e:
-            self.error(str(e))
+            self.error(str(e), "used configuration: host:", str(self.host), "port:", str(self.port), "socket:", str(self.unix_socket))
             sock.close()
             return None
 
@@ -547,19 +552,19 @@ class SocketService(SimpleService):
         try:
             self.unix_socket = str(self.configuration['socket'])
         except (KeyError, TypeError):
-            self.error("No unix socket specified. Trying TCP/IP socket.")
+            self.debug("No unix socket specified. Trying TCP/IP socket.")
             try:
                 self.host = str(self.configuration['host'])
             except (KeyError, TypeError):
-                self.error("No host specified. Using: '" + self.host + "'")
+                self.debug("No host specified. Using: '" + self.host + "'")
             try:
                 self.port = int(self.configuration['port'])
             except (KeyError, TypeError):
-                self.error("No port specified. Using: '" + str(self.port) + "'")
+                self.debug("No port specified. Using: '" + str(self.port) + "'")
         try:
             self.request = str(self.configuration['request'])
         except (KeyError, TypeError):
-            self.error("No request specified. Using: '" + str(self.request) + "'")
+            self.debug("No request specified. Using: '" + str(self.request) + "'")
         self.request = self.request.encode()
 
 
@@ -581,6 +586,7 @@ class LogService(SimpleService):
             if os.path.getsize(self.log_path) < self._last_position:
                 self._last_position = 0
             elif os.path.getsize(self.log_path) == self._last_position:
+                self.debug("Log file hasn't changed. No new data.")
                 return None
             with open(self.log_path, "r") as fp:
                 fp.seek(self._last_position)
@@ -588,11 +594,13 @@ class LogService(SimpleService):
                     lines.append(line)
                 self._last_position = fp.tell()
         except Exception as e:
-            self.error(self.__module__, str(e))
+            self.error(str(e))
 
         if len(lines) != 0:
             return lines
-        return None
+        else:
+            self.error("No data collected.")
+            return None
 
     def check(self):
         """
@@ -636,13 +644,14 @@ class ExecutableService(SimpleService):
         try:
             p = Popen(self.command, stdout=PIPE, stderr=PIPE)
         except Exception as e:
-            self.error(self.__module__, str(e))
+            self.error(str(e))
             return None
         data = []
         for line in p.stdout.readlines():
             data.append(str(line.decode()))
 
         if len(data) == 0:
+            self.error("No data collected.")
             return None
 
         return data

diff --git a/python.d/redis.chart.py b/python.d/redis.chart.py
index f3f4152ece48071c856d58ca18d4cd09a2af98ad..8268ff14dd4b2bdae8a86386b4f3051f1bbddfbb 100644 (file)
--- a/python.d/redis.chart.py
+++ b/python.d/redis.chart.py
@@ -75,6 +75,7 @@ class Service(SocketService):
         try:
             raw = self._get_raw_data().split("\n")
         except AttributeError:
+            self.error("no data received")
             return None
         data = {}
         for line in raw:
@@ -93,7 +94,11 @@ class Service(SocketService):
         except:
             data['hit_rate'] = 0
 
-        return data
+        if len(data) == 0:
+            self.error("received data doesn't have needed records")
+            return None
+        else:
+            return data
 
     def check(self):
         """
@@ -106,7 +111,6 @@ class Service(SocketService):
         self.chart_name += "_" + self.name
         data = self._get_data()
         if data is None:
-            self.error("No data received")
             return False
 
         for name in data:

diff --git a/python.d/squid.chart.py b/python.d/squid.chart.py
index fd1904d520f518a705c2af0ce5c36071a8b09715..716ee9ce57bfc0aef2752e56989ca46c738a5ed9 100644 (file)
--- a/python.d/squid.chart.py
+++ b/python.d/squid.chart.py
@@ -60,15 +60,18 @@ class Service(SocketService):
         try:
             raw = self._get_raw_data().split('\r\n')[-1]
             if raw.startswith('<'):
+                self.error("invalid data received")
                 return None
             for row in raw.split('\n'):
                 if row.startswith(("client", "server.all")):
                     tmp = row.split("=")
                     data[tmp[0].replace('.', '_').strip(' ')] = int(tmp[1])
         except (ValueError, AttributeError, TypeError):
+            self.error("invalid data received")
             return None
 
         if len(data) == 0:
+            self.error("no data received")
             return None
         else:
             return data
@@ -89,5 +92,4 @@ class Service(SocketService):
         if self._get_data() is not None:
             return True
         else:
-            self.error("No data returned")
             return False