From: Ilya <ilyamaschenko@gmail.com>
Date: Thu, 9 Feb 2017 02:21:06 +0000 (+0900)
Subject: web_log pugin: context renamed, ipv6 address support added, unmatched dim for respons... 
X-Git-Tag: ab-debian_0.20170213.01-0ab1~1^2~17^2~12
X-Git-Url: https://arthur.barton.de/gitweb/?p=netdata.git;a=commitdiff_plain;h=8848f136c475ea59f654a96cde9ca08261d4e361

web_log pugin: context renamed, ipv6 address support added, unmatched dim for response codes chart added
---

diff --git a/python.d/web_log.chart.py b/python.d/web_log.chart.py
index 6b66763f..c8e11910 100644
--- a/python.d/web_log.chart.py
+++ b/python.d/web_log.chart.py
@@ -12,40 +12,43 @@ from collections import defaultdict, namedtuple
 priority = 60000
 retries = 60
 
-ORDER = ['response_codes', 'request_time', 'requests_per_url', 'http_method', 'bandwidth', 'clients_cur', 'clients_all']
+ORDER = ['response_codes', 'response_time', 'requests_per_url', 'http_method', 'bandwidth', 'clients', 'clients_all']
 CHARTS = {
     'response_codes': {
-        'options': [None, 'Response Codes', 'requests/s', 'responses', 'web_log.response', 'stacked'],
+        'options': [None, 'Response Codes', 'requests/s', 'responses', 'web_log.response_codes', 'stacked'],
         'lines': [
             ['2xx', '2xx', 'absolute'],
             ['5xx', '5xx', 'absolute'],
             ['3xx', '3xx', 'absolute'],
             ['4xx', '4xx', 'absolute'],
             ['1xx', '1xx', 'absolute'],
-            ['0xx', 'other', 'absolute']
+            ['0xx', 'other', 'absolute'],
+            ['unmatched', 'unmatched', 'absolute']
         ]},
     'bandwidth': {
-        'options': [None, 'Bandwidth', 'KB/s', 'bandwidth', 'web_log.bw', 'area'],
+        'options': [None, 'Bandwidth', 'KB/s', 'bandwidth', 'web_log.bandwidth', 'area'],
         'lines': [
             ['resp_length', 'received', 'absolute', 1, 1024],
             ['bytes_sent', 'sent', 'absolute', -1, 1024]
         ]},
-    'request_time': {
-        'options': [None, 'Processing Time', 'milliseconds', 'timings', 'web_log.request', 'area'],
+    'response_time': {
+        'options': [None, 'Processing Time', 'milliseconds', 'timings', 'web_log.response_time', 'area'],
         'lines': [
             ['resp_time_min', 'min', 'absolute', 1, 1],
             ['resp_time_max', 'max', 'absolute', 1, 1],
             ['resp_time_avg', 'avg', 'absolute', 1, 1]
         ]},
-    'clients_cur': {
+    'clients': {
         'options': [None, 'Current Poll Unique Client IPs', 'unique ips', 'unique clients', 'web_log.clients', 'line'],
         'lines': [
-            ['unique_cur', 'clients', 'absolute', 1, 1]
+            ['unique_cur_ipv4', 'ipv4', 'absolute', 1, 1],
+            ['unique_cur_ipv6', 'ipv6', 'absolute', 1, 1]
         ]},
     'clients_all': {
-        'options': [None, 'All Time Unique Client IPs', 'unique ips', 'unique clients', 'web_log.clients', 'line'],
+        'options': [None, 'All Time Unique Client IPs', 'unique ips', 'unique clients', 'web_log.clients_all', 'line'],
         'lines': [
-            ['unique_tot', 'clients', 'absolute', 1, 1]
+            ['unique_tot_ipv4', 'ipv4', 'absolute', 1, 1],
+            ['unique_tot_ipv6', 'ipv6', 'absolute', 1, 1]
         ]},
     'http_method': {
         'options': [None, 'Requests Per HTTP Method', 'requests/s', 'requests', 'web_log.http_method', 'stacked'],
@@ -66,16 +69,22 @@ class Service(LogService):
         self.url_pattern = self.configuration.get('categories')  # dict
         # REGEX: 1.IPv4 address 2.HTTP method 3. URL 4. Response code
         # 5. Bytes sent 6. Response length 7. Response process time
-        #self.regex = re.compile(r'(\d{1,3}(?:\.\d{1,3}){3}) .*?"([A-Z]+) (.*?)" ([1-9]\d{2}) (\d+) (\d+)? ?([\d.]+)? ?"(.*?)" "(.*?)"')
-        self.regex = re.compile(r'(\d{1,3}(?:\.\d{1,3}){3}) .*?"([A-Z]+) (.*?)" ([1-9]\d{2}) (\d+) (\d+)? ?([\d.]+)?')
+        self.regex = re.compile(r'([\da-f.:]+)'
+                                r' -.*?"([A-Z]+)'
+                                r' (.*?)"'
+                                r' ([1-9]\d{2})'
+                                r' (\d+)'
+                                r' (\d+)?'
+                                r' ?([\d.]+)?')
         # sorted list of unique IPs
         self.unique_all_time = list()
         # dict for values that should not be zeroed every poll
-        self.storage = {'unique_tot': 0}
+        self.storage = {'unique_tot_ipv4': 0, 'unique_tot_ipv6': 0}
         # if there is no new logs this dict + self.storage returned to netdata
         self.data = {'bytes_sent': 0, 'resp_length': 0, 'resp_time_min': 0,
-                     'resp_time_max': 0, 'resp_time_avg': 0, 'unique_cur': 0,
-                     '2xx': 0, '5xx': 0, '3xx': 0, '4xx': 0, '1xx': 0, '0xx': 0}
+                     'resp_time_max': 0, 'resp_time_avg': 0, 'unique_cur_ipv4': 0,
+                     'unique_cur_ipv6': 0, '2xx': 0, '5xx': 0, '3xx': 0, '4xx': 0,
+                     '1xx': 0, '0xx': 0, 'unmatched': 0}
 
     def check(self):
         if not self.log_path:
@@ -208,10 +217,20 @@ class Service(LogService):
                     request_counter['count'] += 1
                     request_counter['sum'] += resp_time
                 # unique clients ips
-                if address_not_in_pool(self.unique_all_time, match_dict['address'], self.storage['unique_tot']):
-                    self.storage['unique_tot'] += 1
-                if address_not_in_pool(unique_current, match_dict['address'], to_netdata['unique_cur']):
-                    to_netdata['unique_cur'] += 1
+                if address_not_in_pool(self.unique_all_time, match_dict['address'],
+                                       self.storage['unique_tot_ipv4'] + self.storage['unique_tot_ipv6']):
+                    if '.' in match_dict['address']:
+                        self.storage['unique_tot_ipv4'] += 1
+                    else:
+                        self.storage['unique_tot_ipv6'] += 1
+                if address_not_in_pool(unique_current, match_dict['address'],
+                                       to_netdata['unique_cur_ipv4'] + to_netdata['unique_cur_ipv6']):
+                    if '.' in match_dict['address']:
+                        to_netdata['unique_cur_ipv4'] += 1
+                    else:
+                        to_netdata['unique_cur_ipv6'] += 1
+            else:
+                to_netdata['unmatched'] += 1
         # timings
         if request_time:
             to_netdata['resp_time_min'] = request_time[0]