From: Ilya Date: Mon, 20 Feb 2017 05:38:15 +0000 (+0900) Subject: mongodb_plugin: initial version added X-Git-Tag: ab-debian_0.20170227.01-0ab1~1^2^2~9 X-Git-Url: https://arthur.barton.de/gitweb/?a=commitdiff_plain;h=159d817bc2abf7a4dc0bceb6c3153a2485c59c89;hp=b00e65483e916f956bb26a74a264146522e1ed8f;p=netdata.git mongodb_plugin: initial version added --- diff --git a/python.d/mongodb.chart.py b/python.d/mongodb.chart.py new file mode 100644 index 00000000..f2b49822 --- /dev/null +++ b/python.d/mongodb.chart.py @@ -0,0 +1,370 @@ +# -*- coding: utf-8 -*- +# Description: mongodb netdata python.d module +# Author: l2isbad + +from base import SimpleService +try: + from pymongo import MongoClient + from pymongo.errors import PyMongoError + PYMONGO = True +except ImportError: + PYMONGO = False + +# default module values (can be overridden per job in `config`) +# update_every = 2 +priority = 60000 +retries = 60 + +# charts order (can be overridden if you want less charts, or different order) +ORDER = ['read_operations', 'write_operations', 'active_clients', 'journaling_transactions', + 'journaling_volume', 'background_flush_average', 'background_flush_last', 'background_flush_rate', + 'wiredtiger_read', 'wiredtiger_write', 'cursors', 'connections', 'memory', 'page_faults', + 'queued_requests', 'record_moves', 'wiredtiger_cache', 'wiredtiger_pages_evicted', 'asserts', + 'dbstats_objects', 'tcmalloc_generic', 'tcmalloc_metrics', 'command_total_rate', 'command_failed_rate'] + +CHARTS = { + 'read_operations': { + 'options': [None, "Received read requests", "requests/s", 'throughput metrics', + 'mongodb.read_operations', 'line'], + 'lines': [ + ['readWriteOper_query', 'query', 'incremental'], + ['readWriteOper_getmore', 'getmore', 'incremental'] + ]}, + 'write_operations': { + 'options': [None, "Received write requests", "requests/s", 'throughput metrics', + 'mongodb.write_operations', 'line'], + 'lines': [ + ['readWriteOper_insert', 'insert', 'incremental'], + ['readWriteOper_update', 'update', 'incremental'], + ['readWriteOper_delete', 'delete', 'incremental'] + ]}, + 'active_clients': { + 'options': [None, "Clients with read or write operations in progress or queued", "clients", + 'throughput metrics', 'mongodb.active_clients', 'line'], + 'lines': [ + ['activeClients_readers', 'readers', 'absolute'], + ['activeClients_writers', 'writers', 'absolute'] + ]}, + 'journaling_transactions': { + 'options': [None, "Transactions that have been written to the journal", "commits", + 'database performance', 'mongodb.journaling_transactions', 'line'], + 'lines': [ + ['journalTrans_commits', 'commits', 'absolute'] + ]}, + 'journaling_volume': { + 'options': [None, "Volume of data written to the journal", "MB", 'database performance', + 'mongodb.journaling_volume', 'line'], + 'lines': [ + ['journalTrans_journaled', 'volume', 'absolute', 1, 100] + ]}, + 'background_flush_average': { + 'options': [None, "Average time taken by flushes to execute", "ms", 'database performance', + 'mongodb.background_flush_average', 'line'], + 'lines': [ + ['background_flush_average', 'time', 'absolute', 1, 100] + ]}, + 'background_flush_last': { + 'options': [None, "Time taken by the last flush operation to execute", "ms", 'database performance', + 'mongodb.background_flush_last', 'line'], + 'lines': [ + ['background_flush_last', 'time', 'absolute', 1, 100] + ]}, + 'background_flush_rate': { + 'options': [None, "Flushes rate", "flushes", 'database performance', 'mongodb.background_flush_rate', 'line'], + 'lines': [ + ['background_flush_rate', 'flushes', 'incremental', 1, 1] + ]}, + 'wiredtiger_read': { + 'options': [None, "Read tickets in use and remaining", "tickets", 'database performance', + 'mongodb.wiredtiger_read', 'stacked'], + 'lines': [ + ['wiredTigerRead_available', 'available', 'absolute', 1, 1], + ['wiredTigerRead_out', 'inuse', 'absolute', 1, 1] + ]}, + 'wiredtiger_write': { + 'options': [None, "Write tickets in use and remaining", "tickets", 'database performance', + 'mongodb.wiredtiger_write', 'stacked'], + 'lines': [ + ['wiredTigerWrite_available', 'available', 'absolute', 1, 1], + ['wiredTigerWrite_out', 'inuse', 'absolute', 1, 1] + ]}, + 'cursors': { + 'options': [None, "Currently openned cursors, cursors with timeout disabled and timed out cursors", + "cursors", 'database performance', 'mongodb.cursors', 'stacked'], + 'lines': [ + ['cursor_total', 'openned', 'absolute', 1, 1], + ['cursor_noTimeout', 'notimeout', 'absolute', 1, 1], + ['cursor_timedOut', 'timedout', 'incremental', 1, 1] + ]}, + 'connections': { + 'options': [None, "Currently connected clients and unused connections", "connections", + 'resource utilization', 'mongodb.connections', 'stacked'], + 'lines': [ + ['connections_available', 'unused', 'absolute', 1, 1], + ['connections_current', 'connected', 'absolute', 1, 1] + ]}, + 'memory': { + 'options': [None, "Memory metrics", "MB", 'resource utilization', 'mongodb.memory', 'stacked'], + 'lines': [ + ['memory_virtual', 'virtual', 'absolute', 1, 1], + ['memory_resident', 'resident', 'absolute', 1, 1], + ['memory_mapped', 'mapped', 'absolute', 1, 1] + ]}, + 'page_faults': { + 'options': [None, "Number of times MongoDB had to fetch data from disk", "request/s", + 'resource utilization', 'mongodb.page_faults', 'line'], + 'lines': [ + ['page_faults', 'page_faults', 'incremental', 1, 1] + ]}, + 'queued_requests': { + 'options': [None, "Currently queued read and wrire requests", "requests", 'resource saturation', + 'mongodb.queued_requests', 'line'], + 'lines': [ + ['currentQueue_readers', 'readers', 'absolute', 1, 1], + ['currentQueue_writers', 'writers', 'absolute', 1, 1] + ]}, + 'record_moves': { + 'options': [None, "Number of times documents had to be moved on-disk", "number", + 'resource saturation', 'mongodb.record_moves', 'line'], + 'lines': [ + ['record_moves', 'moves', 'incremental', 1, 1] + ]}, + 'asserts': { + 'options': [None, "Number of message, warning, regular, corresponding to errors generated" + " by users assertions raised", "number", 'errors (asserts)', 'mongodb.asserts', 'line'], + 'lines': [ + ['errors_msg', 'msg', 'incremental', 1, 1], + ['errors_warning', 'warning', 'incremental', 1, 1], + ['errors_regular', 'regular', 'incremental', 1, 1], + ['errors_user', 'user', 'incremental', 1, 1] + ]}, + 'wiredtiger_cache': { + 'options': [None, "Amount of space taken by cached data/dirty data in the cache and maximum cache size", + "KB", 'resource utilization', 'mongodb.wiredtiger_cache', 'stacked'], + 'lines': [ + ['wiredTiger_bytes_in_cache', 'cached', 'absolute', 1, 1024], + ['wiredTiger_dirty_in_cache', 'dirty', 'absolute', 1, 1024], + ['wiredTiger_maximum_in_conf', 'maximum', 'absolute', 1, 1024] + ]}, + 'wiredtiger_pages_evicted': { + 'options': [None, "Pages evicted from the cache", + "pages", 'resource utilization', 'mongodb.wiredtiger_pages_evicted', 'stacked'], + 'lines': [ + ['wiredTiger_unmodified_pages_evicted', 'unmodified', 'absolute', 1, 1], + ['wiredTiger_modified_pages_evicted', 'modified', 'absolute', 1, 1] + ]}, + 'dbstats_objects': { + 'options': [None, "Number of documents in the database among all the collections", "documents", + 'storage size metrics', 'mongodb.dbstats_objects', 'stacked'], + 'lines': [ + ]}, + 'tcmalloc_generic': { + 'options': [None, "Tcmalloc generic metrics", "MB", 'tcmalloc', 'mongodb.tcmalloc_generic', 'stacked'], + 'lines': [ + ['current_allocated_bytes', 'allocated', 'absolute', 1, 1048576], + ['heap_size', 'heap_size', 'absolute', 1, 1048576] + ]}, + 'tcmalloc_metrics': { + 'options': [None, "Tcmalloc metrics", "KB", 'tcmalloc', 'mongodb.tcmalloc_metrics', 'stacked'], + 'lines': [ + ['central_cache_free_bytes', 'central_cache_free', 'absolute', 1, 1024], + ['current_total_thread_cache_bytes', 'current_total_thread_cache', 'absolute', 1, 1024], + ['pageheap_free_bytes', 'pageheap_free', 'absolute', 1, 1024], + ['pageheap_unmapped_bytes', 'pageheap_unmapped', 'absolute', 1, 1024], + ['thread_cache_free_bytes', 'thread_cache_free', 'absolute', 1, 1024], + ['transfer_cache_free_bytes', 'transfer_cache_free', 'absolute', 1, 1024] + ]}, + 'command_total_rate': { + 'options': [None, "Commands total rate", "commands/s", 'commands', 'mongodb.command_total_rate', 'stacked'], + 'lines': [ + ['count_total', 'count', 'incremental', 1, 1], + ['createIndexes_total', 'createIndexes', 'incremental', 1, 1], + ['delete_total', 'delete', 'incremental', 1, 1], + ['eval_total', 'eval', 'incremental', 1, 1], + ['findAndModify_total', 'findAndModify', 'incremental', 1, 1], + ['insert_total', 'insert', 'incremental', 1, 1], + ['update_total', 'update', 'incremental', 1, 1] + ]}, + 'command_failed_rate': { + 'options': [None, "Commands failed rate", "commands/s", 'commands', 'mongodb.command_failed_rate', 'stacked'], + 'lines': [ + ['count_failed', 'count', 'incremental', 1, 1], + ['createIndexes_failed', 'createIndexes', 'incremental', 1, 1], + ['delete_dailed', 'delete', 'incremental', 1, 1], + ['eval_failed', 'eval', 'incremental', 1, 1], + ['findAndModify_failed', 'findAndModify', 'incremental', 1, 1], + ['insert_failed', 'insert', 'incremental', 1, 1], + ['update_failed', 'update', 'incremental', 1, 1] + ]} +} + + +class Service(SimpleService): + def __init__(self, configuration=None, name=None): + SimpleService.__init__(self, configuration=configuration, name=name) + self.user = self.configuration.get('user') + self.password = self.configuration.get('pass') + self.host = self.configuration.get('host', '127.0.0.1') + self.port = self.configuration.get('port', 27017) + self.timeout = self.configuration.get('timeout', 100) + + def check(self): + if not PYMONGO: + self.error('Pymongo module is needed to use mongodb.chart.py') + return False + + self.connection, server_status, error = self._create_connection() + if error: + self.error(error) + return False + + self._create_charts(server_status) + + return True + + def _create_charts(self, server_status): + + self.order = ORDER[:] + self.definitions = CHARTS + self.ss = dict() + + for elem in ['dur', 'backgroundFlushing', 'wiredTiger', 'tcmalloc', 'cursor', 'commands']: + self.ss[elem] = in_server_status(elem, server_status) + + if not self.ss['dur']: + self.order.remove('journaling_transactions') + self.order.remove('journaling_volume') + + if not self.ss['backgroundFlushing']: + self.order.remove('background_flush_average') + self.order.remove('background_flush_last') + + if not self.ss['cursor']: + self.order.remove('cursors') + + if not self.ss['wiredTiger']: + self.order.remove('wiredtiger_write') + self.order.remove('wiredtiger_read') + self.order.remove('wiredtiger_cache') + + if not self.ss['tcmalloc']: + self.order.remove('tcmalloc_generic') + self.order.remove('tcmalloc_metrics') + + if not self.ss['commands']: + self.order.remove('command_total_rate') + self.order.remove('command_failed_rate') + + self.databases = self.connection.database_names() + + for dbase in self.databases: + self.order.append('_'.join([dbase, 'dbstats'])) + self.definitions['_'.join([dbase, 'dbstats'])] = { + 'options': [None, "%s: size of all documents, indexes, extents" % dbase, "KB", + 'storage size metrics', 'mongodb.dbstats', 'line'], + 'lines': [ + ['_'.join([dbase, 'dataSize']), 'documents', 'absolute', 1, 1024], + ['_'.join([dbase, 'indexSize']), 'indexes', 'absolute', 1, 1024], + ['_'.join([dbase, 'storageSize']), 'extents', 'absolute', 1, 1024] + ]} + self.definitions['dbstats_objects']['lines'].append(['_'.join([dbase, 'objects']), dbase, 'absolute']) + + + def _get_raw_data(self): + raw_data = dict() + + try: + raw_data['serverStatus'] = self.connection.admin.command('serverStatus') + for dbase in self.databases: + raw_data[dbase] = self.connection[dbase].command('dbStats') + except PyMongoError: + return None + return raw_data + + def _get_data(self): + """ + :return: dict + """ + raw_data = self._get_raw_data() + + if not raw_data: + return None + + to_netdata = dict() + server_status = raw_data['serverStatus'] + + to_netdata.update(update_dict_key(server_status['opcounters'], 'readWriteOper')) + to_netdata.update(update_dict_key(server_status['globalLock']['activeClients'], 'activeClients')) + to_netdata.update(update_dict_key(server_status['connections'], 'connections')) + to_netdata.update(update_dict_key(server_status['mem'], 'memory')) + to_netdata.update(update_dict_key(server_status['globalLock']['currentQueue'], 'currentQueue')) + to_netdata.update(update_dict_key(server_status['asserts'], 'errors')) + to_netdata['page_faults'] = server_status['extra_info']['page_faults'] + to_netdata['record_moves'] = server_status['metrics']['record']['moves'] + + if self.ss['dur']: + to_netdata['journalTrans_commits'] = server_status['dur']['commits'] + to_netdata['journalTrans_journaled'] = int(server_status['dur']['journaledMB'] * 100) + + if self.ss['backgroundFlushing']: + to_netdata['background_flush_average'] = int(server_status['backgroundFlushing']['average_ms'] * 100) + to_netdata['background_flush_last'] = int(server_status['backgroundFlushing']['last_ms'] * 100) + to_netdata['background_flush_rate'] = server_status['backgroundFlushing']['flushes'] + + if self.ss['cursor']: + to_netdata['cursor_timedOut'] = server_status['metrics']['cursor']['timedOut'] + to_netdata.update(update_dict_key(server_status['metrics']['cursor']['open'], 'cursor')) + + if self.ss['wiredTiger']: + wired_tiger = server_status['wiredTiger'] + to_netdata.update(update_dict_key(server_status['wiredTiger']['concurrentTransactions']['read'], + 'wiredTigerRead')) + to_netdata.update(update_dict_key(server_status['wiredTiger']['concurrentTransactions']['write'], + 'wiredTigerWrite')) + to_netdata['wiredTiger_bytes_in_cache'] = wired_tiger['cache']['bytes currently in the cache'] + to_netdata['wiredTiger_maximum_in_conf'] = wired_tiger['cache']['maximum bytes configured'] + to_netdata['wiredTiger_dirty_in_cache'] = wired_tiger['cache']['tracked dirty bytes in the cache'] + to_netdata['wiredTiger_unmodified_pages_evicted'] = wired_tiger['cache']['unmodified pages evicted'] + to_netdata['wiredTiger_modified_pages_evicted'] = wired_tiger['cache']['modified pages evicted'] + + if self.ss['tcmalloc']: + to_netdata.update(server_status['tcmalloc']['generic']) + to_netdata.update(dict([(k, v) for k, v in server_status['tcmalloc']['tcmalloc'].items() + if int_or_float(v)])) + + if self.ss['commands']: + for elem in ['count', 'createIndexes', 'delete', 'eval', 'findAndModify', 'insert', 'update']: + to_netdata.update(update_dict_key(server_status['metrics']['commands'][elem], elem)) + + for dbase in self.databases: + dbase_dbstats = raw_data[dbase] + dbase_dbstats = dict([(k, v) for k, v in dbase_dbstats.items() if int_or_float(v)]) + to_netdata.update(update_dict_key(dbase_dbstats, dbase)) + + return to_netdata + + def _create_connection(self): + conn_vars = {'host': self.host, 'port': self.port} + if 'server_selection_timeout' in dir(MongoClient): + conn_vars.update({'serverselectiontimeoutms': self.timeout}) + try: + connection = MongoClient(**conn_vars) + if self.user and self.password: + connection.admin.authenticate(name=self.user, password=self.password) + server_status = connection.admin.command('serverStatus') + except PyMongoError as error: + return None, None, str(error) + else: + return connection, server_status, None + + +def update_dict_key(collection, string): + return dict([('_'.join([string, k]), int(round(v))) for k, v in collection.items()]) + + +def int_or_float(value): + return isinstance(value, int) or isinstance(value, float) + + +def in_server_status(elem, server_status): + return elem in server_status or elem in server_status['metrics']