]> arthur.barton.de Git - netdata.git/commitdiff
Merge pull request #643 from paulfantom/master
authorCosta Tsaousis <costa@tsaousis.gr>
Wed, 6 Jul 2016 19:02:17 +0000 (22:02 +0300)
committerGitHub <noreply@github.com>
Wed, 6 Jul 2016 19:02:17 +0000 (22:02 +0300)
python.d squid + hddtemp + tomcat + minor fixes

21 files changed:
conf.d/Makefile.am
conf.d/python.d.conf
conf.d/python.d/apache.conf [new file with mode: 0644]
conf.d/python.d/apache_cache.conf [new file with mode: 0644]
conf.d/python.d/hddtemp.conf [new file with mode: 0644]
conf.d/python.d/mysql.conf
conf.d/python.d/nginx.conf [new file with mode: 0644]
conf.d/python.d/phpfpm.conf
conf.d/python.d/squid.conf [new file with mode: 0644]
conf.d/python.d/tomcat.conf [new file with mode: 0644]
python.d/Makefile.am
python.d/README.md
python.d/apache.chart.py
python.d/apache_cache.chart.py
python.d/hddtemp.chart.py [new file with mode: 0644]
python.d/mysql.chart.py
python.d/nginx.chart.py
python.d/phpfpm.chart.py
python.d/python_modules/base.py
python.d/squid.chart.py [new file with mode: 0644]
python.d/tomcat.chart.py [new file with mode: 0644]

index 5dbc29e19c72fe1bf66013902a4fe41899f42a88..faf4ecdfbc654b2b5871be6c05c4142e3d5906ee 100644 (file)
@@ -19,9 +19,15 @@ dist_nodeconfig_DATA = \
 
 pythonconfigdir=$(configdir)/python.d
 dist_pythonconfig_DATA = \
+       python.d/apache.conf \
+       python.d/apache_cache.conf \
        python.d/example.conf \
+       python.d/hddtemp.conf \
        python.d/mysql.conf \
+       python.d/nginx.conf \
        python.d/phpfpm.conf \
+       python.d/squid.conf \
+       python.d/tomcat.conf \
        $(NULL)
 
 
index d416cf9c9eef071688cd5d62205952fea6f6a348..195165634f9e90fe2206956d486ed37e5dcfde38 100644 (file)
@@ -5,8 +5,12 @@ enabled: no
 
 # By default python.d.plugin enables all modules stored in python.d
 # Modules can be disabled with setting "module_name = no"
+apache: yes
+apache_cache: yes
 example: yes
+hddtemp: yes
 mysql: yes
-phpfpm: yes
-apache: yes
 nginx: yes
+phpfpm: yes
+squid: yes
+
diff --git a/conf.d/python.d/apache.conf b/conf.d/python.d/apache.conf
new file mode 100644 (file)
index 0000000..0cc2183
--- /dev/null
@@ -0,0 +1,6 @@
+# Example configuration of apache.chart.py
+# YAML format
+
+local:
+  url: "http://localhost/server-status?auto"
+  retries: 10
diff --git a/conf.d/python.d/apache_cache.conf b/conf.d/python.d/apache_cache.conf
new file mode 100644 (file)
index 0000000..a6276de
--- /dev/null
@@ -0,0 +1,4 @@
+# Example configuration of apache_cache.chart.py
+# YAML format
+
+path: "/var/log/apache2/cache.log"
diff --git a/conf.d/python.d/hddtemp.conf b/conf.d/python.d/hddtemp.conf
new file mode 100644 (file)
index 0000000..19e7bfd
--- /dev/null
@@ -0,0 +1,8 @@
+# Example configuration of hddtemp.chart.py
+# YAML format
+
+update_every: 1
+retries: 10
+host: 'localhost'
+port: 7634
+
index d175af1e6256119eafd9cb3f221a7702b515cee8..6df941d59935a9391e6e5640be41366a1af63713 100644 (file)
@@ -1,18 +1,47 @@
 # Example configuration of mysql.chart.py
 # YAML format
 
-update_every: 5
+update_every: 1
+retries: 10
 
-local: 
+mycnf:
+  name     : 'local'
   'my.cnf' : '/etc/mysql/my.cnf'
 
-local_s:
-  user     : 'root'
-  password : 'blablablabla'
+socket1:
+  name     : 'local'
+  #user     : 'root'
+  #password : ''
   socket   : '/var/run/mysqld/mysqld.sock'
 
-remote:
-  user     : 'admin'
-  password : 'bla'
-  host     : 'example.org'
+socket2:
+  name     : 'local'
+  #user     : 'root'
+  #password : ''
+  socket   : '/var/lib/mysql/mysql.sock'
+
+tcp:
+  name     : 'local'
+  #user     : 'root'
+  #password : ''
+  host     : 'localhost'
   port     : '3306'
+
+socket1_root:
+  name     : 'local'
+  user     : 'root'
+  #password : ''
+  socket   : '/var/run/mysqld/mysqld.sock'
+
+socket2_root:
+  name     : 'local'
+  user     : 'root'
+  #password : ''
+  socket   : '/var/lib/mysql/mysql.sock'
+
+tcp_root:
+  name     : 'local'
+  user     : 'root'
+  #password : ''
+  host     : 'localhost'
+  port     : '3306'
\ No newline at end of file
diff --git a/conf.d/python.d/nginx.conf b/conf.d/python.d/nginx.conf
new file mode 100644 (file)
index 0000000..cbb862c
--- /dev/null
@@ -0,0 +1,7 @@
+# Example configuration of nginx.chart.py
+# YAML format
+
+retries: 10
+
+local:
+  url: "http://localhost/stub_status"
index df3e5df9d32f3915e2c119f046759666902e1219..3c68e8955287c1352c93b92951d16886b4d33742 100644 (file)
@@ -1,4 +1,7 @@
-#update_every: 3
+# Example configuration of phpfpm.chart.py
+# YAML format
+
+retries: 10
 
 local:
   url: "http://localhost/status"
diff --git a/conf.d/python.d/squid.conf b/conf.d/python.d/squid.conf
new file mode 100644 (file)
index 0000000..22cec78
--- /dev/null
@@ -0,0 +1,30 @@
+# Example configuration of squid.chart.py
+# YAML format
+
+update_every: 1
+retries: 10
+
+tcp1:
+  name     : 'local'
+  host     : 'localhost'
+  port     : 3128
+  request  : 'cache_object://localhost:3128/counters'
+
+tcp2:
+  name     : 'local'
+  host     : 'localhost'
+  port     : 8080
+  request  : 'cache_object://localhost:3128/counters'
+
+tcp3:
+  name     : 'local'
+  host     : 'localhost'
+  port     : 3128
+  request  : '/squid-internal-mgr/counters'
+
+tcp4:
+  name     : 'local'
+  host     : 'localhost'
+  port     : 8080
+  request  : '/squid-internal-mgr/counters'
+
diff --git a/conf.d/python.d/tomcat.conf b/conf.d/python.d/tomcat.conf
new file mode 100644 (file)
index 0000000..efcb671
--- /dev/null
@@ -0,0 +1,9 @@
+# Example configuration of tomcat.chart.py
+# YAML format
+
+retries: 10
+
+local:
+  url: "http://localhost:8080/manager/status?XML=true"
+  user: ""
+  password: ""
index bf000989f2cee8fa59cb6883fe3cee6a40803a08..7f14bffa0ea840246992f483f8d0af171f9c7cc3 100644 (file)
@@ -14,6 +14,9 @@ dist_python_SCRIPTS = \
        apache.chart.py \
        nginx.chart.py \
        apache_cache.chart.py \
+       hddtemp.chart.py \
+       squid.chart.py \
+       tomcat.chart.py \
        python-modules-installer.sh \
        $(NULL)
 
index 2cf6c52d9e55decacdee478833652a119f0c7ffe..a6092053bafa36afee37b31a99447a41f430704d 100644 (file)
@@ -1,8 +1,8 @@
 # Disclaimer
 
-**Python plugin support is experimental and implementation may change in the future**
+**Python support is experimental and implementation may change in the future**
 
-Every plugin should be compatible with python2 and python3.
+Every module should be compatible with python2 and python3.
 All third party libraries should be installed system-wide or in `python_modules` directory.
 Module configurations are written in YAML and **pyYAML is required**.
 
@@ -29,7 +29,6 @@ priority     : 20000
 
 local:  # job name
   update_every : 5 # job update frequency
-  retries      : 2 # job retries
   other_var1   : some_val # module specific variable
 
 other_job: 
@@ -42,14 +41,121 @@ other_job:
 
 ---
 
-The following python.d plugins are supported:
+The following python.d modules are supported:
+
+# apache
+
+This module will monitor one or more apache servers depending on configuration. 
+
+**Requirements:**
+ * apache with enabled `mod_status`
+
+It produces following charts:
+
+1. **Requests** in requests/s
+ * requests
+
+2. **Connections**
+ * connections
+
+3. **Async Connections**
+ * keepalive
+ * closing
+ * writing
+4. **Bandwidth** in kilobytes/s
+ * sent
+5. **Workers**
+ * idle
+ * busy
+6. **Lifetime Avg. Requests/s** in requests/s
+ * requests_sec
+7. **Lifetime Avg. Bandwidth/s** in kilobytes/s
+ * size_sec
+8. **Lifetime Avg. Response Size** in bytes/request
+ * size_req
+
+### configuration
+
+Needs only `url` to server's `server-status?auto`
+
+Here is an example for 2 servers:
+
+```yaml
+update_every : 10
+priority     : 90100
+
+local:
+  url      : 'http://localhost/server-status?auto'
+  retries  : 20
+
+remote:
+  url          : 'http://www.apache.org/server-status?auto'
+  update_every : 5
+  retries      : 4
+```
+
+Without configuration, module attempts to connect to `http://localhost/server-status?auto`
+
+---
+
+# apache_cache
+
+Module monitors apache mod_cache log and produces only one chart:
+
+**cached responses** in percent cached
+ * hit
+ * miss
+ * other
+### configuration
+
+Sample:
+
+```yaml
+update_every : 10
+priority     : 120000
+retries      : 5
+log_path     : '/var/log/apache2/cache.log'
+```
+
+If no configuration is given, module will attempt to read log file at `/var/log/apache2/cache.log`
+
+---
+
+# hddtemp
+Module monitors disk temperatures from one or more hddtemp daemons
+
+**Requirement:**
+Running `hddtemp` in daemonized mode with access on tcp port
+
+It produces one chart **Temperature** with dynamic number of dimensions (one per disk)
+
+### configuration
+
+Sample:
+
+```yaml
+update_every: 3
+host: "127.0.0.1"
+port: 7634
+```
+
+If no configuration is given, module will attempt to connect to hddtemp daemon on `127.0.0.1:7634` address
+
+---
 
 # mysql
 
-The plugin will monitor one or more mysql servers
+Module monitors one or more mysql servers
 
 **Requirements:**
- * python module [MySQLdb](https://github.com/PyMySQL/mysqlclient-python) (faster) or [PyMySQL](https://github.com/PyMySQL/PyMySQL) (slower)
+ * python library [MySQLdb](https://github.com/PyMySQL/mysqlclient-python) (faster) or [PyMySQL](https://github.com/PyMySQL/PyMySQL) (slower)
 
 It will produce following charts (if data is available):
 
@@ -99,13 +205,12 @@ It will produce following charts (if data is available):
 
 You can provide, per server, the following:
 
-1. a name, anything you like, but keep it short
-2. username which have access to database (deafults to 'root')
-3. password (defaults to none)
-4. mysql my.cnf configuration file
-5. mysql socket (optional)
-6. mysql host (ip or hostname)
-7. mysql port (defaults to 3306)
+1. username which have access to database (deafults to 'root')
+2. password (defaults to none)
+3. mysql my.cnf configuration file
+4. mysql socket (optional)
+5. mysql host (ip or hostname)
+6. mysql port (defaults to 3306)
 
 Here is an example for 3 servers:
 
@@ -132,7 +237,129 @@ remote:
   retries  : 20
 ```
 
-If no configuration is given, the plugin will attempt to connect to mysql server via unix socket at `/var/run/mysqld/mysqld.sock` without password and with username `root`
+If no configuration is given, module will attempt to connect to mysql server via unix socket at `/var/run/mysqld/mysqld.sock` without password and with username `root`
+
+---
+
+# nginx
+
+This module will monitor one or more nginx servers depending on configuration. 
+
+**Requirements:**
+ * nginx with configured `stub_status`
+
+It produces following charts:
+
+1. **Active Connections**
+ * active
+
+2. **Requests** in requests/s
+ * requests
+
+3. **Active Connections by Status**
+ * reading
+ * writing
+ * waiting
+4. **Connections Rate** in connections/s
+ * accepts
+ * handled
+### configuration
+
+Needs only `url` to server's `stub_status`
+
+Here is an example for local server:
+
+```yaml
+update_every : 10
+priority     : 90100
+
+local:
+  url     : 'http://localhost/stub_status'
+  retries : 10
+```
+
+Without configuration, module attempts to connect to `http://localhost/stub_status`
 
 ---
 
+# phpfpm
+
+This module will monitor one or more php-fpm instances depending on configuration. 
+
+**Requirements:**
+ * php-fpm with enabled `status` page
+ * access to `status` page via web server
+It produces following charts:
+
+1. **Active Connections**
+ * active
+ * maxActive
+ * idle
+
+2. **Requests** in requests/s
+ * requests
+3. **Performance**
+ * reached
+ * slow
+### configuration
+
+Needs only `url` to server's `status`
+Here is an example for local instance:
+
+```yaml
+update_every : 3
+priority     : 90100
+
+local:
+  url     : 'http://localhost/status'
+  retries : 10
+```
+
+Without configuration, module attempts to connect to `http://localhost/status`
+
+---
+
+# squid
+
+This module will monitor one or more squid instances depending on configuration.
+
+It produces following charts:
+
+1. **Client Bandwidth** in kilobits/s
+ * in
+ * out
+ * hits
+
+2. **Client Requests** in requests/s
+ * requests
+ * hits
+ * errors
+
+3. **Server Bandwidth** in kilobits/s
+ * in
+ * out
+4. **Server Requests** in requests/s
+ * requests
+ * errors
+### configuration
+
+```yaml
+priority     : 50000
+
+local:
+  request : 'cache_object://localhost:3128/counters'
+  host    : 'localhost'
+  port    : 3128
+```
+
+Without any configuration module will try to autodetect where squid presents its `counters` data
+---
\ No newline at end of file
index 53058388e5983128221f0ee3fa64e1efe9a35b96..f0f4fe2e13c53db0d2d0fdae85e07a7eae720e2a 100644 (file)
@@ -86,13 +86,13 @@ class Service(UrlService):
                            "ConnsAsyncClosing": 'closing',
                            "ConnsAsyncWriting": 'writing'}
 
-    def _format_data(self):
+    def _get_data(self):
         """
         Format data received from http request
         :return: dict
         """
         try:
-            raw = self._get_data().split("\n")
+            raw = self._get_raw_data().split("\n")
         except AttributeError:
             return None
         data = {}
index 69da50c1f514124449130f973c03e1bc2230f4a3..026d612da539495caf838ba45382fccc02a2bd0d 100644 (file)
@@ -6,7 +6,7 @@ from base import LogService
 
 priority = 60000
 retries = 5
-update_every = 3
+update_every = 3
 
 ORDER = ['cache']
 CHARTS = {
@@ -28,13 +28,13 @@ class Service(LogService):
         self.order = ORDER
         self.definitions = CHARTS
 
-    def _format_data(self):
+    def _get_data(self):
         """
         Parse new log lines
         :return: dict
         """
         try:
-            raw = self._get_data()
+            raw = self._get_raw_data()
             if raw is None:
                 return None
         except (ValueError, AttributeError):
diff --git a/python.d/hddtemp.chart.py b/python.d/hddtemp.chart.py
new file mode 100644 (file)
index 0000000..2a549e7
--- /dev/null
@@ -0,0 +1,75 @@
+# -*- coding: utf-8 -*-
+# Description: hddtemp netdata python.d plugin
+# Author: Pawel Krupa (paulfantom)
+
+from base import NetSocketService
+
+# default module values (can be overridden per job in `config`)
+#update_every = 2
+priority = 60000
+retries = 5
+
+# default job configuration (overridden by python.d.plugin)
+# config = {'local': {
+#             'update_every': update_every,
+#             'retries': retries,
+#             'priority': priority,
+#             'host': 'localhost',
+#             'port': 7634
+#          }}
+
+ORDER = ['temperatures']
+
+CHARTS = {
+    'temperatures': {
+        'options': ['disks_temp', 'temperature', 'Celsius', 'Disks temperature', 'hddtemp.temp', 'line'],
+        'lines': [
+            # lines are created dynamically in `check()` method
+        ]}
+}
+
+
+class Service(NetSocketService):
+    def __init__(self, configuration=None, name=None):
+        NetSocketService.__init__(self, configuration=configuration, name=name)
+        self.request = ""
+        self.host = "127.0.0.1"
+        self.port = 7634
+        self.order = ORDER
+        self.definitions = CHARTS
+
+    def _get_data(self):
+        """
+        Get data from TCP/IP socket
+        :return: dict
+        """
+        try:
+            raw = self._get_raw_data().split("|")[:-1]
+        except AttributeError:
+            return None
+        data = {}
+        for i in range(len(raw) // 5):
+            try:
+                val = int(raw[i*5+3])
+            except ValueError:
+                val = 0
+            data[raw[i*5+1].replace("/dev/", "")] = val
+        return data
+
+    def check(self):
+        """
+        Parse configuration, check if hddtemp is available, and dynamically create chart lines data
+        :return: boolean
+        """
+        self._parse_config()
+        data = self._get_data()
+        if data is None:
+            self.error("No data received")
+            return False
+
+        for name in data:
+            self.definitions[ORDER[0]]['lines'].append([name])
+
+        return True
+
+
index 982d2513507a64f7fe0114d999e538b9f3bb92f3..951daf2d2321db046cb1d0d9c56cc648c5f435f7 100644 (file)
@@ -355,7 +355,7 @@ class Service(SimpleService):
             self.error("problem connecting to server:", e)
             raise RuntimeError
 
-    def _format_data(self):
+    def _get_data(self):
         """
         Get raw data from MySQL server
         :return: dict
index 8fcac7d181410d9b17028c2eff3e7b68a009d3f0..c50d0b72c675990e16c054e9980ac4f0111f71bd 100644 (file)
@@ -55,13 +55,13 @@ class Service(UrlService):
         self.order = ORDER
         self.definitions = CHARTS
 
-    def _format_data(self):
+    def _get_data(self):
         """
         Format data received from http request
         :return: dict
         """
         try:
-            raw = self._get_data().split(" ")
+            raw = self._get_raw_data().split(" ")
             return {'active': int(raw[2]),
                     'requests': int(raw[7]),
                     'reading': int(raw[11]),
index 938c0edf5a16365319934736008fc28c2d1642c2..024e6f5c7383afda668a15629708fa302b77efaa 100755 (executable)
@@ -56,13 +56,13 @@ class Service(UrlService):
                            "max children reached": 'reached',
                            "slow requests": 'slow'}
 
-    def _format_data(self):
+    def _get_data(self):
         """
         Format data received from http request
         :return: dict
         """
         try:
-            raw = self._get_data().split('\n')
+            raw = self._get_raw_data().split('\n')
         except AttributeError:
             return None
         data = {}
index e868ffc3a9e4ba2fc0beec7c8104e185a72626e9..c025f16dae4cb298633e83d7ac5d2bec95256ae5 100644 (file)
@@ -5,12 +5,14 @@
 import time
 import sys
 import os
+import socket
 try:
-    from urllib.request import urlopen
+    import urllib.request as urllib2
 except ImportError:
-    from urllib2 import urlopen
+    import urllib2
+
+from subprocess import Popen, PIPE
 
-# from subprocess import STDOUT, PIPE, Popen
 import threading
 import msg
 
@@ -56,7 +58,7 @@ class BaseService(threading.Thread):
         :param config: dict
         """
         try:
-            self.override_name = config.pop('override_name')
+            self.override_name = config.pop('name')
         except KeyError:
             pass
         self.update_every = int(config.pop('update_every'))
@@ -132,7 +134,7 @@ class BaseService(threading.Thread):
                 time.sleep(self.timetable['next'] - time.time())
                 self.retries_left = self.retries
             else:
-                self.retries -= 1
+                self.retries_left -= 1
                 if self.retries_left <= 0:
                     msg.error("no more retries. Exiting")
                     return
@@ -305,14 +307,7 @@ class SimpleService(BaseService):
 
     def _get_data(self):
         """
-        Get raw data from http request
-        :return: str
-        """
-        return ""
-
-    def _format_data(self):
-        """
-        Format data received from http request
+        Get some data
         :return: dict
         """
         return {}
@@ -328,14 +323,18 @@ class SimpleService(BaseService):
         Create charts
         :return: boolean
         """
-        data = self._format_data()
+        data = self._get_data()
         if data is None:
             return False
 
         idx = 0
         for name in self.order:
             options = self.definitions[name]['options'] + [self.priority + idx, self.update_every]
-            self.chart(self.__module__ + "_" + self.name + "." + name, *options)
+            if self.name == "":
+                type_id = self.__module__
+            else:
+                type_id = self.__module__ + "_" + self.name
+            self.chart(type_id + "." + name, *options)
             # check if server has this datapoint
             for line in self.definitions[name]['lines']:
                 if line[0] in data:
@@ -351,13 +350,17 @@ class SimpleService(BaseService):
         :param interval: int
         :return: boolean
         """
-        data = self._format_data()
+        data = self._get_data()
         if data is None:
             return False
 
         updated = False
         for chart in self.order:
-            if self.begin(self.__module__ + "_" + str(self.name) + "." + chart, interval):
+            if str(self.name) == "":
+                type_id = self.__module__
+            else:
+                type_id = self.__module__ + "_" + self.name
+            if self.begin(type_id + "." + chart, interval):
                 updated = True
                 for dim in self.definitions[chart]['lines']:
                     try:
@@ -373,29 +376,36 @@ class SimpleService(BaseService):
 
 class UrlService(SimpleService):
     def __init__(self, configuration=None, name=None):
-        # definitions are created dynamically in create() method based on 'charts' dictionary. format:
-        # definitions = {
-        #     'chart_name_in_netdata' : [ charts['chart_name_in_netdata']['lines']['name'] ]
-        # }
         self.url = ""
+        self.user = None
+        self.password = None
         SimpleService.__init__(self, configuration=configuration, name=name)
 
-    def _get_data(self):
+    def __add_auth(self):
+        passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
+        passman.add_password(None, self.url, self.user, self.password)
+        authhandler = urllib2.HTTPBasicAuthHandler(passman)
+        opener = urllib2.build_opener(authhandler)
+        urllib2.install_opener(opener)
+
+    def _get_raw_data(self):
         """
         Get raw data from http request
         :return: str
         """
         raw = None
         try:
-            f = urlopen(self.url, timeout=self.update_every)
+            f = urllib2.urlopen(self.url, timeout=self.update_every)
+        except Exception as e:
+            msg.error(self.__module__, str(e))
+            return None
+
+        try:
             raw = f.read().decode('utf-8')
         except Exception as e:
             msg.error(self.__module__, str(e))
         finally:
-            try:
-                f.close()
-            except:
-                pass
+            f.close()
         return raw
 
     def check(self):
@@ -411,26 +421,111 @@ class UrlService(SimpleService):
             self.url = str(self.configuration['url'])
         except (KeyError, TypeError):
             pass
+        try:
+            self.user = str(self.configuration['user'])
+        except (KeyError, TypeError):
+            pass
+        try:
+            self.password = str(self.configuration['password'])
+        except (KeyError, TypeError):
+            pass
 
-        if self._format_data() is not None:
+        if self.user is not None and self.password is not None:
+            self.__add_auth()
+
+        if self._get_data() is not None:
             return True
         else:
             return False
 
 
+class NetSocketService(SimpleService):
+    def __init__(self, configuration=None, name=None):
+        self.host = "localhost"
+        self.port = None
+        self.sock = None
+        self.request = ""
+        SimpleService.__init__(self, configuration=configuration, name=name)
+
+    def _get_raw_data(self):
+        """
+        Get raw data with low-level "socket" module.
+        :return: str
+        """
+        if self.sock is None:
+            try:
+                sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+                sock.settimeout(self.update_every)
+                sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+                sock.connect((self.host, self.port))
+            except Exception as e:
+                print(e)
+                self.sock = None
+                return None
+
+        if self.request != "".encode():
+            try:
+                sock.send(self.request)
+            except Exception:
+                try:
+                    sock.shutdown(1)
+                    sock.close()
+                except:
+                    pass
+                self.sock = None
+                return None
+
+        data = sock.recv(1024)
+        try:
+            while True:
+                buf = sock.recv(1024)
+                if not buf:
+                    break
+                else:
+                    data += buf
+        except:
+            sock.close()
+            return None
+
+        return data.decode()
+
+    def _parse_config(self):
+        """
+        Parse configuration data
+        :return: boolean
+        """
+        if self.name is not None or self.name != str(None):
+            self.name = ""
+        else:
+            self.name = str(self.name)
+        try:
+            self.host = str(self.configuration['host'])
+        except (KeyError, TypeError):
+            self.error("No host specified. Using: '" + self.host + "'")
+        try:
+            self.port = int(self.configuration['port'])
+        except (KeyError, TypeError):
+            self.error("No port specified. Using: '" + str(self.port) + "'")
+        try:
+            self.request = str(self.configuration['request'])
+        except (KeyError, TypeError):
+            self.error("No request specified. Using: '" + str(self.request) + "'")
+        self.request = self.request.encode()
+
+
 class LogService(SimpleService):
     def __init__(self, configuration=None, name=None):
-        # definitions are created dynamically in create() method based on 'charts' dictionary. format:
-        # definitions = {
-        #     'chart_name_in_netdata' : [ charts['chart_name_in_netdata']['lines']['name'] ]
-        # }
         self.log_path = ""
         self._last_position = 0
         # self._log_reader = None
         SimpleService.__init__(self, configuration=configuration, name=name)
         self.retries = 100000  # basically always retry
 
-    def _get_data(self):
+    def _get_raw_data(self):
+        """
+        Get log lines since last poll
+        :return: list
+        """
         lines = []
         try:
             if os.path.getsize(self.log_path) < self._last_position:
@@ -443,13 +538,17 @@ class LogService(SimpleService):
                     lines.append(line)
                 self._last_position = fp.tell()
         except Exception as e:
-            msg.error(self.__module__, str(e))
+            self.error(self.__module__, str(e))
 
         if len(lines) != 0:
             return lines
         return None
 
     def check(self):
+        """
+        Parse basic configuration and check if log file exists
+        :return: boolean
+        """
         if self.name is not None or self.name != str(None):
             self.name = ""
         else:
@@ -470,3 +569,50 @@ class LogService(SimpleService):
         self._last_position = 0
         return status
 
+
+class ExecutableService(SimpleService):
+    command_whitelist = ['exim']
+
+    def __init__(self, configuration=None, name=None):
+        self.command = ""
+        SimpleService.__init__(self, configuration=configuration, name=name)
+
+    def _get_raw_data(self):
+        """
+        Get raw data from executed command
+        :return: str
+        """
+        try:
+            p = Popen(self.command, stdout=PIPE, stderr=PIPE)
+        except Exception as e:
+            self.error(self.__module__, str(e))
+            return None
+        data = []
+        for line in p.stdout.readlines():
+            data.append(line)
+
+        return data
+
+    def check(self):
+        """
+        Parse basic configuration, check if command is whitelisted and is returning values
+        :return: boolean
+        """
+        if self.name is not None or self.name != str(None):
+            self.name = ""
+        else:
+            self.name = str(self.name)
+        # try:
+        #     self.command = str(self.configuration['path'])
+        # except (KeyError, TypeError):
+        #     self.error("No command specified. Using: '" + self.command + "'")
+        self.command = self.command.split(' ')
+        for i in self.command:
+            if i.startswith('-') or i in self.command_whitelist:
+                pass
+            else:
+                self.error("Wrong command. Probably not on whitelist.")
+                return False
+        if self._get_data() is None or len(self._get_data()) == 0:
+            return False
+        return True
diff --git a/python.d/squid.chart.py b/python.d/squid.chart.py
new file mode 100644 (file)
index 0000000..a179dd9
--- /dev/null
@@ -0,0 +1,112 @@
+# -*- coding: utf-8 -*-
+# Description: squid netdata python.d plugin
+# Author: Pawel Krupa (paulfantom)
+
+from base import NetSocketService
+
+# default module values (can be overridden per job in `config`)
+# update_every = 2
+priority = 60000
+retries = 5
+
+# charts order (can be overridden if you want less charts, or different order)
+ORDER = ['clients_net', 'clients_requests', 'servers_net', 'servers_requests']
+
+CHARTS = {
+    'clients_net': {
+        'options': [None, "Squid Client Bandwidth", "kilobits/s", "clients", "squid.clients.net" "area"],
+        'lines': [
+            ["client_http_kbytes_in", "in", "incremental", 8, 1],
+            ["client_http_kbytes_out", "out", "incremental", -8, 1],
+            ["client_http_hit_kbytes_out", "hits", "incremental", -8, 1]
+        ]},
+    'clients_requests': {
+        'options': [None, "Squid Client Requests", "requests/s", "clients", "squid.clients.requests", 'line'],
+        'lines': [
+            ["client_http_requests", "requests"],
+            ["client_http_hits", "hits"],
+            ["client_http_errors", "errors", "incremental", -1, 1]
+        ]},
+    'servers_net': {
+        'options': [None, "Squid Server Bandwidth", "kilobits/s", "servers", "squid.servers.net" "area"],
+        'lines': [
+            ["server_all_kbytes_in", "in", "incremental", 8, 1],
+            ["server_all_kbytes_out", "out", "incremental", -8, 1]
+        ]},
+    'servers_requests': {
+        'options': [None, "Squid Server Requests", "requests/s", "servers", "squid.servers.requests", 'line'],
+        'lines': [
+            ["server_all_requests", "requests"],
+            ["server_all_errors", "errors", "incremental", -1, 1]
+        ]}
+}
+
+
+class Service(NetSocketService):
+    def __init__(self, configuration=None, name=None):
+        NetSocketService.__init__(self, configuration=configuration, name=name)
+        self.request = ""
+        self.host = "localhost"
+        self.port = 3128
+        self.order = ORDER
+        self.definitions = CHARTS
+
+    def _get_data(self):
+        """
+        Get data via http request
+        :return: dict
+        """
+        try:
+            raw = self._get_raw_data().split('\n')
+            if "200 OK" not in raw[0]:
+                return None
+            data = {}
+            for row in raw:
+                if row.startswith(("client", "server.all")):
+                    tmp = row.split("=")
+                    data[tmp[0].replace('.', '_').strip(' ')] = int(tmp[1])
+
+            return data
+        except (ValueError, AttributeError):
+            return None
+
+    def check(self):
+        """
+        Parse essential configuration, autodetect squid configuration (if needed), and check if data is available
+        :return: boolean
+        """
+        self._parse_config()
+        # format request
+        req = self.request.decode()
+        if not req.startswith("GET"):
+            req = "GET " + req
+        if not req.endswith(" HTTP/1.0\r\n\r\n"):
+            req += " HTTP/1.0\r\n\r\n"
+        self.request = req.encode()
+        #
+        # # autodetect squid
+        # if type(self.port) is tuple:
+        #     ports = self.port
+        #     for port in ports:
+        #         self.port = port
+        #         urls = ["cache_object://" + self.host + ":" + str(port) + "/counters",
+        #                 "/squid-internal-mgr/counters"]
+        #         for url in urls:
+        #             tmp = "GET " + url + " HTTP/1.0\r\n\r\n"
+        #             self.request = tmp.encode()
+        #             if self._get_data() is not None:
+        #                 return True
+        # else:
+        if True:
+            if self._get_data() is not None:
+                return True
+            else:
+                return False
+            
+                
+                
+                
+            
+                
+            
+
diff --git a/python.d/tomcat.chart.py b/python.d/tomcat.chart.py
new file mode 100644 (file)
index 0000000..a4b2e71
--- /dev/null
@@ -0,0 +1,78 @@
+# -*- coding: utf-8 -*-
+# Description: tomcat netdata python.d plugin
+# Author: Pawel Krupa (paulfantom)
+
+from base import UrlService
+import xml.etree.ElementTree as ET  # phone home...
+
+# default module values (can be overridden per job in `config`)
+# update_every = 2
+priority = 60000
+retries = 5
+
+# charts order (can be overridden if you want less charts, or different order)
+ORDER = ['accesses', 'volume', 'threads', 'jvm']
+
+CHARTS = {
+    'accesses': {
+        'options': [None, "tomcat requests", "requests/s", "statistics", "tomcat.accesses", "area"],
+        'lines': [
+            ["accesses"]
+        ]},
+    'volume': {
+        'options': [None, "tomcat volume", "KB/s", "volume", "tomcat.volume", "area"],
+        'lines': [
+            ["volume", None, 'incremental']
+        ]},
+    'threads': {
+        'options': [None, "tomcat threads", "current threads", "statistics", "tomcat.threads", "line"],
+        'lines': [
+            ["current", None, "absolute"],
+            ["busy", None, "absolute"]
+        ]},
+    'jvm': {
+        'options': [None, "JVM Free Memory", "MB", "statistics", "tomcat.jvm", "area"],
+        'lines': [
+            ["jvm", None, "absolute"]
+        ]}
+}
+
+
+class Service(UrlService):
+    def __init__(self, configuration=None, name=None):
+        UrlService.__init__(self, configuration=configuration, name=name)
+        if len(self.url) == 0:
+            self.url = "http://localhost:8080/manager/status?XML=true"
+        self.order = ORDER
+        self.definitions = CHARTS
+        # get port from url
+        self.port = 0
+        for i in self.url.split('/'):
+            try:
+                int(i[-1])
+                self.port = i.split(':')[-1]
+                break
+            except:
+                pass
+        if self.port == 0:
+            self.port = 80
+
+    def _get_data(self):
+        """
+        Format data received from http request
+        :return: dict
+        """
+        try:
+            raw = self._get_raw_data()
+            data = ET.fromstring(raw)
+            memory = data.find('./jvm/memory')
+            threads = data.find("./connector[@name='\"http-bio-" + str(self.port) + "\"']/threadInfo")
+            requests = data.find("./connector[@name='\"http-bio-" + str(self.port) + "\"']/requestInfo")
+
+            return {'accesses': requests.attrib['requestCount'],
+                    'volume': requests.attrib['bytesSent'],
+                    'current': threads.attrib['currentThreadCount'],
+                    'busy': threads.attrib['currentThreadsBusy'],
+                    'jvm': memory.attrib['free']}
+        except (ValueError, AttributeError):
+            return None