# pick the one that works.
#
# Any number of jobs is supported.
+
+# ----------------------------------------------------------------------
+# PLUGIN CONFIGURATION
#
# All python.d.plugin JOBS (for all its modules) support a set of
# predefined parameters. These are:
# cacti: 'cacti.*' # name(dimension): REGEX to match
# observium: 'observium.*' # name(dimension): REGEX to match
# stub_status: 'stub_status' # name(dimension): REGEX to match
+
+# ----------------------------------------------------------------------
+# WEB SERVER CONFIGURATION
+#
+# Make sure the log directory and file can be read by user 'netdata'.
#
# Preferable Log Format. You need to change to this to collect all metrics.
-# Nginx:
-# log_format netdata '$remote_addr - $remote_user [$time_local] '
-# '"$request" $status $body_bytes_sent '
-# '$request_length $request_time '
-# '"$http_referer" "$http_user_agent"';
-# access_log /var/log/nginx/access.log netdata;
-# Apache:
-# LogFormat "%h %l %u %t \"%r\" %>s %O %I %D \"%{Referer}i\" \"%{User-Agent}i\"" vhost_combined
-# LogFormat "%h %l %u %t \"%r\" %>s %O %I %D \"%{Referer}i\" \"%{User-Agent}i\"" combined
+#
+# nginx:
+# log_format netdata '$remote_addr - $remote_user [$time_local] '
+# '"$request" $status $body_bytes_sent '
+# '$request_length $request_time '
+# '"$http_referer" "$http_user_agent"';
+# access_log /var/log/nginx/access.log netdata;
+#
+# apache:
+# LogFormat "%h %l %u %t \"%r\" %>s %O %I %D \"%{Referer}i\" \"%{User-Agent}i\"" vhost_combined
+# LogFormat "%h %l %u %t \"%r\" %>s %O %I %D \"%{Referer}i\" \"%{User-Agent}i\"" combined
+
# ----------------------------------------------------------------------
# AUTO-DETECTION JOBS
-# only one of them will run (if they have the same name)
+# only one of them per web server will run (when they have the same name)
-# debian
+
+# -------------------------------------------
+# nginx log on various distros
+
+# debian, arch
nginx_log:
name: 'nginx'
path: '/var/log/nginx/access.log'
name: 'nginx'
path: '/var/log/nginx/localhost.access_log'
+
+# -------------------------------------------
+# apache log on various distros
+
# debian
apache_log:
name: 'apache'
name: 'apache'
path: '/var/log/httpd/access_log'
+# debian
apache_vhosts_log:
name: 'apache_vhosts'
path: '/var/log/apache2/other_vhosts_access.log'
+
+# -------------------------------------------
+# gunicorn log on various distros
+
gunicorn_log:
name: 'gunicorn'
path: '/var/log/gunicorn/access.log'
priority = 60000
retries = 60
-ORDER = ['response_codes', 'response_time', 'requests_per_url', 'http_method', 'requests_per_ipproto', 'bandwidth', 'clients', 'clients_all']
+ORDER = ['response_codes', 'bandwidth', 'response_time', 'requests_per_url', 'http_method', 'requests_per_ipproto', 'clients', 'clients_all']
CHARTS = {
'response_codes': {
'options': [None, 'Response Codes', 'requests/s', 'responses', 'web_log.response_codes', 'stacked'],
['resp_time_avg', 'avg', 'absolute', 1, 1]
]},
'clients': {
- 'options': [None, 'Current Poll Unique Client IPs', 'unique ips', 'unique clients', 'web_log.clients', 'stacked'],
+ 'options': [None, 'Current Poll Unique Client IPs', 'unique ips', 'clients', 'web_log.clients', 'stacked'],
'lines': [
['unique_cur_ipv4', 'ipv4', 'absolute', 1, 1],
['unique_cur_ipv6', 'ipv6', 'absolute', 1, 1]
]},
'clients_all': {
- 'options': [None, 'All Time Unique Client IPs', 'unique ips', 'unique clients', 'web_log.clients_all', 'stacked'],
+ 'options': [None, 'All Time Unique Client IPs', 'unique ips', 'clients', 'web_log.clients_all', 'stacked'],
'lines': [
['unique_tot_ipv4', 'ipv4', 'absolute', 1, 1],
['unique_tot_ipv6', 'ipv6', 'absolute', 1, 1]
]},
'http_method': {
- 'options': [None, 'Requests Per HTTP Method', 'requests/s', 'requests', 'web_log.http_method', 'stacked'],
+ 'options': [None, 'Requests Per HTTP Method', 'requests/s', 'http methods', 'web_log.http_method', 'stacked'],
'lines': [
]},
'requests_per_ipproto': {
- 'options': [None, 'Requests Per IP Protocol', 'requests/s', 'requests', 'web_log.requests_per_ipproto', 'stacked'],
+ 'options': [None, 'Requests Per IP Protocol', 'requests/s', 'ip protocols', 'web_log.requests_per_ipproto', 'stacked'],
'lines': [
['req_ipv4', 'ipv4', 'absolute', 1, 1],
['req_ipv6', 'ipv6', 'absolute', 1, 1]
if self.detailed_response_codes:
self.order.append('detailed_response_codes')
self.definitions['detailed_response_codes'] = {'options': [None, 'Detailed Response Codes', 'requests/s',
- 'responses', 'web_log.detailed_resp', 'stacked'],
+ 'responses', 'web_log.detailed_response_codes', 'stacked'],
'lines': []}
# Add 'requests_per_url' chart if specified in the configuration
if self.url_pattern:
self.url_pattern = [NAMED_URL_PATTERN(description=k, pattern=re.compile(v)) for k, v in self.url_pattern.items()]
self.definitions['requests_per_url'] = {'options': [None, 'Requests Per Url', 'requests/s',
- 'requests', 'web_log.url_pattern', 'stacked'],
+ 'urls', 'web_log.requests_per_url', 'stacked'],
'lines': [['other_url', 'other', 'absolute']]}
for elem in self.url_pattern:
self.definitions['requests_per_url']['lines'].append([elem.description, elem.description, 'absolute'])
var netdataDashboard = window.netdataDashboard || {};
-// menu
+// ----------------------------------------------------------------------------
+// menus
+
// information about the main menus
netdataDashboard.menu = {
info: undefined
},
+ 'web_log': {
+ title: undefined,
+ icon: '<i class="fa fa-file-text-o" aria-hidden="true"></i>',
+ info: 'Information extracted from the web server log file. netdata <code>python.d/web_log</code> plugin incrementally parses web server log files to provide a real-time break down of key web server performance metrics. A special log file format is also supported (for <code>nginx</code> and <code>apache</code>) that allows netdata to extract timing information for the web server responses and bandwidth for both requests and responses. The <code>web_log</code> plugin can also be configured to provide a break down of requests per URL pattern (check <a href="https://github.com/firehol/netdata/blob/master/conf.d/python.d/web_log.conf" target="_blank"><code>/etc/netdata/python.d/web_log.conf</code></a>). netdata provides also several alarms based on the information on these charts, such as <b>too many bad requests</b>, <b>too many redirects</b>, <b>too many internal errros</b>, <b>unreasonably slow responses</b>, <b>too many requests</b> and <b>too few requests</b>.'
+ },
+
'named': {
title: 'named',
icon: '<i class="fa fa-tag" aria-hidden="true"></i>',
}
};
-// submenu
+
+
+// ----------------------------------------------------------------------------
+// submenus
+
+// information to be shown, just below each submenu
+
// information about the submenus
netdataDashboard.submenu = {
+ 'web_log.bandwidth': {
+ info: 'Bandwidth of requests (<code>received</code>) and responses (<code>sent</code>). <code>received</code> requires a special file format (without it, the web server log does not have this information). This chart may present unusual spikes, since the whole bandwidth will be accounted at the time the log line is saved by the web server, even if the time needed to serve it spans across a longer duration. We suggest to use QoS (e.g. <a href="http://firehol.org/#fireqos" target="_blank">FireQOS</a>) for accurate accounting of the web server bandwidth.'
+ },
+
+ 'web_log.urls': {
+ info: 'Number of requests for each URL <code>category</code> (URL pattern) defined in <a href="https://github.com/firehol/netdata/blob/master/conf.d/python.d/web_log.conf" target="_blank"><code>/etc/netdata/python.d/web_log.conf</code></a>. This chart counts all requests matching the URL patterns defined, independently of the web server response codes (i.e. both successful and unsuccessful).'
+ },
+
+ 'web_log.clients': {
+ info: 'Charts showing the number of unique client IPs, accessing the web server.'
+ },
+
+ 'web_log.timings': {
+ info: 'Web server response timings - the time the web server needed to prepare and respond to requests. This requires a special log format and its meaning is web server specific. For most web servers this accounts the time from the reception of a complete request, to the dispatch of the last byte of the response. So, it includes the network delays of responses, but it does not include the network delays of requests.'
+ },
+
'mem.ksm': {
title: 'Memory Deduper',
info: 'Kernel Same-page Merging (KSM) performance monitoring, read from several files in <code>/sys/kernel/mm/ksm/</code>. KSM is a memory-saving de-duplication feature in the Linux kernel (since version 2.6.32). The KSM daemon ksmd periodically scans those areas of user memory which have been registered with it, looking for pages of identical content which can be replaced by a single write-protected page (which is automatically copied if a process later wants to update its content). KSM was originally developed for use with KVM (where it was known as Kernel Shared Memory), to fit more virtual machines into physical memory, by sharing the data common between them. But it can be useful to any application which generates many instances of the same data.'
}
};
+
+
+// ----------------------------------------------------------------------------
// chart
+
// information works on the context of a chart
// Its purpose is to set:
//
'fping.packets': {
height: 0.5
+ },
+
+ 'web_log.response_codes': {
+ info: 'Break down of web server responses by response code type. <code>1xx</code> are informational responses, <code>2xx</code> are successful responses, <code>3xx</code> are redirects, <code>4xx</code> are bad requests, <code>5xx</code> are internal server errors, <code>other</code> are non-standard responses, <code>unmatched</code> counts the lines in the log file that are not matched by the plugin (please <a href="https://github.com/firehol/netdata/issues/new?title=web_log%20reports%20unmatched%20lines&body=web_log%20plugin%20reports%20unmatched%20lines.%0A%0AThis%20is%20my%20log:%0A%0A%60%60%60txt%0A%0Aplease%20paste%20your%20web%20server%20log%20here%0A%0A%60%60%60" target="_blank">open a github issue</a> to help us fix it, if you have any unmatched lines).',
+
+ mainheads: [
+ function(os, id) {
+ void(os);
+ return '<div data-netdata="' + id + '"'
+ + ' data-dimensions="2xx"'
+ + ' data-chart-library="gauge"'
+ + ' data-title="Successful"'
+ + ' data-units="requests"'
+ + ' data-gauge-adjust="width"'
+ + ' data-width="12%"'
+ + ' data-before="0"'
+ + ' data-after="-CHART_DURATION"'
+ + ' data-points="CHART_DURATION"'
+ + ' data-common-max="' + id + '"'
+ + ' data-colors="' + NETDATA.colors[0] + '"'
+ + ' role="application"></div>';
+ },
+
+ function(os, id) {
+ void(os);
+ return '<div data-netdata="' + id + '"'
+ + ' data-dimensions="3xx"'
+ + ' data-chart-library="gauge"'
+ + ' data-title="Redirects"'
+ + ' data-units="requests"'
+ + ' data-gauge-adjust="width"'
+ + ' data-width="12%"'
+ + ' data-before="0"'
+ + ' data-after="-CHART_DURATION"'
+ + ' data-points="CHART_DURATION"'
+ + ' data-common-max="' + id + '"'
+ + ' data-colors="' + NETDATA.colors[2] + '"'
+ + ' role="application"></div>';
+ },
+
+ function(os, id) {
+ void(os);
+ return '<div data-netdata="' + id + '"'
+ + ' data-dimensions="4xx"'
+ + ' data-chart-library="gauge"'
+ + ' data-title="Bad Requests"'
+ + ' data-units="requests"'
+ + ' data-gauge-adjust="width"'
+ + ' data-width="12%"'
+ + ' data-before="0"'
+ + ' data-after="-CHART_DURATION"'
+ + ' data-points="CHART_DURATION"'
+ + ' data-common-max="' + id + '"'
+ + ' data-colors="' + NETDATA.colors[3] + '"'
+ + ' role="application"></div>';
+ },
+
+ function(os, id) {
+ void(os);
+ return '<div data-netdata="' + id + '"'
+ + ' data-dimensions="5xx"'
+ + ' data-chart-library="gauge"'
+ + ' data-title="Server Errors"'
+ + ' data-units="requests"'
+ + ' data-gauge-adjust="width"'
+ + ' data-width="12%"'
+ + ' data-before="0"'
+ + ' data-after="-CHART_DURATION"'
+ + ' data-points="CHART_DURATION"'
+ + ' data-common-max="' + id + '"'
+ + ' data-colors="' + NETDATA.colors[1] + '"'
+ + ' role="application"></div>';
+ }
+ ]
+ },
+
+ 'web_log.detailed_response_codes': {
+ info: 'Number of responses for each response code.'
+ },
+
+ 'web_log.requests_per_ipproto': {
+ info: 'Web server requests received per IP protocol version.'
+ },
+
+ 'web_log.clients': {
+ info: 'Unique client IPs accessing the web server, within each data collection iteration. So, if data collection is <b>per second</b>, this chart shows <b>unique client IPs per second</b>.'
+ },
+
+ 'web_log.clients_all': {
+ info: 'Unique client IPs accessing the web server since the last restart of netdata. This plugin keeps in memory all the unique IPs that have accessed the web server. On very busy web servers (several millions of unique IPs) you may want to disable this chart (check <a href="https://github.com/firehol/netdata/blob/master/conf.d/python.d/web_log.conf" target="_blank"><code>/etc/netdata/python.d/web_log.conf</code></a>).'
}
};
submenuTitle: function(menu, submenu) {
var key = menu + '.' + submenu;
+ // console.log(key);
var title = this.anyAttribute(this.submenu, 'title', key, submenu).toString().replace(/_/g, ' ');
if(title.length > 28) {
var a = title.substring(0, 13);
// enrich the data structure returned by netdata
// to reflect our menu system and content
+ // FIXME: this is a shame - we should fix charts naming (issue #807)
function enrichChartData(chart) {
- var tmp = chart.type.split('_')[0];
+ var parts = chart.type.split('_');
+ var tmp = parts[0];
switch(tmp) {
case 'ap':
chart.menu = tmp;
break;
+ case 'apache':
+ chart.menu = chart.type;
+ if(parts.length > 2 && parts[1] === 'cache')
+ chart.menu_pattern = tmp + '_' + parts[1];
+ break;
+
+ case 'bind':
+ chart.menu = chart.type;
+ if(parts.length > 2 && parts[1] === 'rndc')
+ chart.menu_pattern = tmp + '_' + parts[1];
+ break;
+
case 'cgroup':
chart.menu = chart.type;
if(chart.id.match(/.*[\._\/-:]qemu[\._\/-:]*/) || chart.id.match(/.*[\._\/-:]kvm[\._\/-:]*/))
chart.menu_pattern = 'cgroup';
break;
- case 'apache':
- case 'exim':
+ case 'isc':
+ chart.menu = chart.type;
+ if(parts.length > 2 && parts[1] === 'dhcpd')
+ chart.menu_pattern = tmp + '_' + parts[1];
+ break;
+
+ case 'ovpn':
+ chart.menu = chart.type;
+ if(parts.length > 3 && parts[1] === 'status' && parts[2] === 'log')
+ chart.menu_pattern = tmp + '_' + parts[1];
+ break;
+
+ case 'smartd':
+ case 'web':
+ chart.menu = chart.type;
+ if(parts.length > 2 && parts[1] === 'log')
+ chart.menu_pattern = tmp + '_' + parts[1];
+ break;
+
case 'dovecot':
+ case 'exim':
case 'hddtemp':
case 'ipfs':
case 'memcached':
function renderChartsAndMenu(data) {
var menus = options.menus;
var charts = data.charts;
+ var m, menu_key;
for(var c in charts) {
if(!charts.hasOwnProperty(c)) continue;
- enrichChartData(charts[c]);
+ var chart = charts[c];
+ enrichChartData(chart);
+ m = chart.menu;
// create the menu
- if(typeof menus[charts[c].menu] === 'undefined') {
- menus[charts[c].menu] = {
- priority: charts[c].priority,
+ if(typeof menus[m] === 'undefined') {
+ menus[m] = {
+ menu_pattern: chart.menu_pattern,
+ priority: chart.priority,
submenus: {},
- title: netdataDashboard.menuTitle(charts[c]),
- icon: netdataDashboard.menuIcon(charts[c]),
- info: netdataDashboard.menuInfo(charts[c]),
- height: netdataDashboard.menuHeight(charts[c]) * options.chartsHeight
+ title: netdataDashboard.menuTitle(chart),
+ icon: netdataDashboard.menuIcon(chart),
+ info: netdataDashboard.menuInfo(chart),
+ height: netdataDashboard.menuHeight(chart) * options.chartsHeight
};
}
+ else {
+ if(typeof(menus[m].menu_pattern) === 'undefined')
+ menus[m].menu_pattern = chart.menu_pattern;
+
+ if(chart.priority < menus[m].priority)
+ menus[m].priority = chart.priority;
+ }
- if(charts[c].priority < menus[charts[c].menu].priority)
- menus[charts[c].menu].priority = charts[c].priority;
+ menu_key = (typeof(menus[m].menu_pattern) !== 'undefined')?menus[m].menu_pattern:m;
// create the submenu
- if(typeof menus[charts[c].menu].submenus[charts[c].submenu] === 'undefined') {
- menus[charts[c].menu].submenus[charts[c].submenu] = {
- priority: charts[c].priority,
+ if(typeof menus[m].submenus[chart.submenu] === 'undefined') {
+ menus[m].submenus[chart.submenu] = {
+ priority: chart.priority,
charts: [],
title: null,
- info: netdataDashboard.submenuInfo(charts[c].menu, charts[c].submenu),
- height: netdataDashboard.submenuHeight(charts[c].menu, charts[c].submenu, menus[charts[c].menu].height)
+ info: netdataDashboard.submenuInfo(menu_key, chart.submenu),
+ height: netdataDashboard.submenuHeight(menu_key, chart.submenu, menus[m].height)
};
}
-
- if(charts[c].priority < menus[charts[c].menu].submenus[charts[c].submenu].priority)
- menus[charts[c].menu].submenus[charts[c].submenu].priority = charts[c].priority;
+ else {
+ if (chart.priority < menus[m].submenus[chart.submenu].priority)
+ menus[m].submenus[chart.submenu].priority = chart.priority;
+ }
// index the chart in the menu/submenu
- menus[charts[c].menu].submenus[charts[c].submenu].charts.push(charts[c]);
+ menus[m].submenus[chart.submenu].charts.push(chart);
}
// propagate the descriptive subname given to QoS
// to all the other submenus with the same name
- for(var m in menus) {
+ for(m in menus) {
if(!menus.hasOwnProperty(m)) continue;
for(var s in menus[m].submenus) {
menus[m].submenus[s].title = s + ' (' + options.submenu_names[s] + ')';
}
else {
- menus[m].submenus[s].title = netdataDashboard.submenuTitle(m, s);
+ menu_key = (typeof(menus[m].menu_pattern) !== 'undefined')?menus[m].menu_pattern:m;
+ menus[m].submenus[s].title = netdataDashboard.submenuTitle(menu_key, s);
}
}
}
});
NETDATA.requiredJs.push({
- url: NETDATA.serverDefault + 'dashboard_info.js?v20170208-8',
+ url: NETDATA.serverDefault + 'dashboard_info.js?v20170211-18',
async: false,
isAlreadyLoaded: function() { return false; }
});
projects / netdata.git / commitdiff