1 # -*- coding: utf-8 -*-
2 # Description: MySQL netdata python.d module
3 # Author: Pawel Krupa (paulfantom)
5 from base import SimpleService
8 # import 3rd party library to handle MySQL communication
12 # https://github.com/PyMySQL/mysqlclient-python
13 msg.info("using MySQLdb")
16 import pymysql as MySQLdb
18 # https://github.com/PyMySQL/PyMySQL
19 msg.info("using pymysql")
21 msg.error("MySQLdb or PyMySQL module is needed to use mysql.chart.py plugin")
24 # default module values (can be overridden per job in `config`)
29 # default configuration (overridden by python.d.plugin)
34 # 'socket': '/var/run/mysqld/mysqld.sock',
35 # 'update_every': update_every,
37 # 'priority': priority
41 # query executed on MySQL server
42 QUERY = "SHOW GLOBAL STATUS;"
43 QUERY_SLAVE = "SHOW SLAVE STATUS;"
49 'join_issues', 'sort_issues',
51 'connections', 'connection_errors',
52 'binlog_cache', 'binlog_stmt_cache',
53 'threads', 'thread_cache_misses',
54 'innodb_io', 'innodb_io_ops', 'innodb_io_pending_ops', 'innodb_log', 'innodb_os_log', 'innodb_os_log_io',
55 'innodb_cur_row_lock', 'innodb_rows', 'innodb_buffer_pool_pages', 'innodb_buffer_pool_bytes',
56 'innodb_buffer_pool_read_ahead', 'innodb_buffer_pool_reqs', 'innodb_buffer_pool_ops',
57 'qcache_ops', 'qcache', 'qcache_freemem', 'qcache_memblocks',
58 'key_blocks', 'key_requests', 'key_disk_ops',
59 'files', 'files_rate', 'slave_behind', 'slave_status']
63 'options': [None, 'mysql Bandwidth', 'kilobits/s', 'bandwidth', 'mysql.net', 'area'],
65 ["Bytes_received", "in", "incremental", 8, 1024],
66 ["Bytes_sent", "out", "incremental", -8, 1024]
69 'options': [None, 'mysql Queries', 'queries/s', 'queries', 'mysql.queries', 'line'],
71 ["Queries", "queries", "incremental"],
72 ["Questions", "questions", "incremental"],
73 ["Slow_queries", "slow_queries", "incremental"]
76 'options': [None, 'mysql Handlers', 'handlers/s', 'handlers', 'mysql.handlers', 'line'],
78 ["Handler_commit", "commit", "incremental"],
79 ["Handler_delete", "delete", "incremental"],
80 ["Handler_prepare", "prepare", "incremental"],
81 ["Handler_read_first", "read_first", "incremental"],
82 ["Handler_read_key", "read_key", "incremental"],
83 ["Handler_read_next", "read_next", "incremental"],
84 ["Handler_read_prev", "read_prev", "incremental"],
85 ["Handler_read_rnd", "read_rnd", "incremental"],
86 ["Handler_read_rnd_next", "read_rnd_next", "incremental"],
87 ["Handler_rollback", "rollback", "incremental"],
88 ["Handler_savepoint", "savepoint", "incremental"],
89 ["Handler_savepoint_rollback", "savepoint_rollback", "incremental"],
90 ["Handler_update", "update", "incremental"],
91 ["Handler_write", "write", "incremental"]
94 'options': [None, 'mysql Tables Locks', 'locks/s', 'locks', 'mysql.table_locks', 'line'],
96 ["Table_locks_immediate", "immediate", "incremental"],
97 ["Table_locks_waited", "waited", "incremental", -1, 1]
100 'options': [None, 'mysql Select Join Issues', 'joins/s', 'issues', 'mysql.join_issues', 'line'],
102 ["Select_full_join", "full_join", "incremental"],
103 ["Select_full_range_join", "full_range_join", "incremental"],
104 ["Select_range", "range", "incremental"],
105 ["Select_range_check", "range_check", "incremental"],
106 ["Select_scan", "scan", "incremental"]
109 'options': [None, 'mysql Sort Issues', 'issues/s', 'issues', 'mysql.sort_issues', 'line'],
111 ["Sort_merge_passes", "merge_passes", "incremental"],
112 ["Sort_range", "range", "incremental"],
113 ["Sort_scan", "scan", "incremental"]
116 'options': [None, 'mysql Tmp Operations', 'counter', 'temporaries', 'mysql.tmp', 'line'],
118 ["Created_tmp_disk_tables", "disk_tables", "incremental"],
119 ["Created_tmp_files", "files", "incremental"],
120 ["Created_tmp_tables", "tables", "incremental"]
123 'options': [None, 'mysql Connections', 'connections/s', 'connections', 'mysql.connections', 'line'],
125 ["Connections", "all", "incremental"],
126 ["Aborted_connects", "aborted", "incremental"]
129 'options': [None, 'mysql Binlog Cache', 'transactions/s', 'binlog', 'mysql.binlog_cache', 'line'],
131 ["Binlog_cache_disk_use", "disk", "incremental"],
132 ["Binlog_cache_use", "all", "incremental"]
135 'options': [None, 'mysql Threads', 'threads', 'threads', 'mysql.threads', 'line'],
137 ["Threads_connected", "connected", "absolute"],
138 ["Threads_created", "created", "incremental"],
139 ["Threads_cached", "cached", "absolute", -1, 1],
140 ["Threads_running", "running", "absolute"],
142 'thread_cache_misses': {
143 'options': [None, 'mysql Threads Cache Misses', 'misses', 'threads', 'mysql.thread_cache_misses', 'area'],
145 ["Thread_cache_misses", "misses", "absolute", 1, 100]
148 'options': [None, 'mysql InnoDB I/O Bandwidth', 'kilobytes/s', 'innodb', 'mysql.innodb_io', 'area'],
150 ["Innodb_data_read", "read", "incremental", 1, 1024],
151 ["Innodb_data_written", "write", "incremental", -1, 1024]
154 'options': [None, 'mysql InnoDB I/O Operations', 'operations/s', 'innodb', 'mysql.innodb_io_ops', 'line'],
156 ["Innodb_data_reads", "reads", "incremental"],
157 ["Innodb_data_writes", "writes", "incremental", -1, 1],
158 ["Innodb_data_fsyncs", "fsyncs", "incremental"]
160 'innodb_io_pending_ops': {
161 'options': [None, 'mysql InnoDB Pending I/O Operations', 'operations', 'innodb', 'mysql.innodb_io_pending_ops', 'line'],
163 ["Innodb_data_pending_reads", "reads", "absolute"],
164 ["Innodb_data_pending_writes", "writes", "absolute", -1, 1],
165 ["Innodb_data_pending_fsyncs", "fsyncs", "absolute"]
168 'options': [None, 'mysql InnoDB Log Operations', 'operations/s', 'innodb', 'mysql.innodb_log', 'line'],
170 ["Innodb_log_waits", "waits", "incremental"],
171 ["Innodb_log_write_requests", "write_requests", "incremental", -1, 1],
172 ["Innodb_log_writes", "writes", "incremental", -1, 1],
175 'options': [None, 'mysql InnoDB OS Log Operations', 'operations', 'innodb', 'mysql.innodb_os_log', 'line'],
177 ["Innodb_os_log_fsyncs", "fsyncs", "incremental"],
178 ["Innodb_os_log_pending_fsyncs", "pending_fsyncs", "absolute"],
179 ["Innodb_os_log_pending_writes", "pending_writes", "absolute", -1, 1],
181 'innodb_os_log_io': {
182 'options': [None, 'mysql InnoDB OS Log Bandwidth', 'kilobytes/s', 'innodb', 'mysql.innodb_os_log_io', 'area'],
184 ["Innodb_os_log_written", "write", "incremental", -1, 1024],
186 'innodb_cur_row_lock': {
187 'options': [None, 'mysql InnoDB Current Row Locks', 'operations', 'innodb', 'mysql.innodb_cur_row_lock', 'area'],
189 ["Innodb_row_lock_current_waits", "current_waits", "absolute"]
192 'options': [None, 'mysql InnoDB Row Operations', 'operations/s', 'innodb', 'mysql.innodb_rows', 'area'],
194 ["Innodb_rows_inserted", "read", "incremental"],
195 ["Innodb_rows_read", "deleted", "incremental", -1, 1],
196 ["Innodb_rows_updated", "inserted", "incremental", 1, 1],
197 ["Innodb_rows_deleted", "updated", "incremental", -1, 1],
199 'innodb_buffer_pool_pages': {
200 'options': [None, 'mysql InnoDB Buffer Pool Pages', 'pages', 'innodb', 'mysql.innodb_buffer_pool_pages', 'line'],
202 ["Innodb_buffer_pool_pages_data", "data", "absolute"],
203 ["Innodb_buffer_pool_pages_dirty", "dirty", "absolute", -1, 1],
204 ["Innodb_buffer_pool_pages_free", "free", "absolute"],
205 ["Innodb_buffer_pool_pages_flushed", "flushed", "incremental", -1, 1],
206 ["Innodb_buffer_pool_pages_misc", "misc", "absolute", -1, 1],
207 ["Innodb_buffer_pool_pages_total", "total", "absolute"]
209 'innodb_buffer_pool_bytes': {
210 'options': [None, 'mysql InnoDB Buffer Pool Bytes', 'MB', 'innodb', 'mysql.innodb_buffer_pool_bytes', 'area'],
212 ["Innodb_buffer_pool_bytes_data", "data", "absolute", 1, 1024 * 1024],
213 ["Innodb_buffer_pool_bytes_dirty", "dirty", "absolute", -1, 1024 * 1024]
215 'innodb_buffer_pool_read_ahead': {
216 'options': [None, 'mysql InnoDB Buffer Pool Read Ahead', 'operations/s', 'innodb', 'mysql.innodb_buffer_pool_read_ahead', 'area'],
218 ["Innodb_buffer_pool_read_ahead", "all", "incremental"],
219 ["Innodb_buffer_pool_read_ahead_evicted", "evicted", "incremental", -1, 1],
220 ["Innodb_buffer_pool_read_ahead_rnd", "random", "incremental"]
222 'innodb_buffer_pool_reqs': {
223 'options': [None, 'mysql InnoDB Buffer Pool Requests', 'requests/s', 'innodb', 'mysql.innodb_buffer_pool_reqs', 'area'],
225 ["Innodb_buffer_pool_read_requests", "reads", "incremental"],
226 ["Innodb_buffer_pool_write_requests", "writes", "incremental", -1, 1]
228 'innodb_buffer_pool_ops': {
229 'options': [None, 'mysql InnoDB Buffer Pool Operations', 'operations/s', 'innodb', 'mysql.innodb_buffer_pool_ops', 'area'],
231 ["Innodb_buffer_pool_reads", "disk reads", "incremental"],
232 ["Innodb_buffer_pool_wait_free", "wait free", "incremental", -1, 1]
235 'options': [None, 'mysql QCache Operations', 'queries/s', 'qcache', 'mysql.qcache_ops', 'line'],
237 ["Qcache_hits", "hits", "incremental"],
238 ["Qcache_lowmem_prunes", "lowmem prunes", "incremental", -1, 1],
239 ["Qcache_inserts", "inserts", "incremental"],
240 ["Qcache_not_cached", "not cached", "incremental", -1, 1]
243 'options': [None, 'mysql QCache Queries in Cache', 'queries', 'qcache', 'mysql.qcache', 'line'],
245 ["Qcache_queries_in_cache", "queries", "absolute"]
248 'options': [None, 'mysql QCache Free Memory', 'MB', 'qcache', 'mysql.qcache_freemem', 'area'],
250 ["Qcache_free_memory", "free", "absolute", 1, 1024 * 1024]
252 'qcache_memblocks': {
253 'options': [None, 'mysql QCache Memory Blocks', 'blocks', 'qcache', 'mysql.qcache_memblocks', 'line'],
255 ["Qcache_free_blocks", "free", "absolute"],
256 ["Qcache_total_blocks", "total", "absolute"]
259 'options': [None, 'mysql MyISAM Key Cache Blocks', 'blocks', 'myisam', 'mysql.key_blocks', 'line'],
261 ["Key_blocks_unused", "unused", "absolute"],
262 ["Key_blocks_used", "used", "absolute", -1, 1],
263 ["Key_blocks_not_flushed", "not flushed", "absolute"]
266 'options': [None, 'mysql MyISAM Key Cache Requests', 'requests/s', 'myisam', 'mysql.key_requests', 'area'],
268 ["Key_read_requests", "reads", "incremental"],
269 ["Key_write_requests", "writes", "incremental", -1, 1]
272 'options': [None, 'mysql MyISAM Key Cache Disk Operations', 'operations/s', 'myisam', 'mysql.key_disk_ops', 'area'],
274 ["Key_reads", "reads", "incremental"],
275 ["Key_writes", "writes", "incremental", -1, 1]
278 'options': [None, 'mysql Open Files', 'files', 'files', 'mysql.files', 'line'],
280 ["Open_files", "files", "absolute"]
283 'options': [None, 'mysql Opened Files Rate', 'files/s', 'files', 'mysql.files_rate', 'line'],
285 ["Opened_files", "files", "incremental"]
287 'binlog_stmt_cache': {
288 'options': [None, 'mysql Binlog Statement Cache', 'statements/s', 'binlog', 'mysql.binlog_stmt_cache', 'line'],
290 ["Binlog_stmt_cache_disk_use", "disk", "incremental"],
291 ["Binlog_stmt_cache_use", "all", "incremental"]
293 'connection_errors': {
294 'options': [None, 'mysql Connection Errors', 'connections/s', 'connections', 'mysql.connection_errors', 'line'],
296 ["Connection_errors_accept", "accept", "incremental"],
297 ["Connection_errors_internal", "internal", "incremental"],
298 ["Connection_errors_max_connections", "max", "incremental"],
299 ["Connection_errors_peer_address", "peer_addr", "incremental"],
300 ["Connection_errors_select", "select", "incremental"],
301 ["Connection_errors_tcpwrap", "tcpwrap", "incremental"]
304 'options': [None, 'Slave Behind Seconds', 'seconds', 'slave', 'mysql.slave_behind', 'line'],
306 ["slave_behind", "seconds", "absolute"]
309 'options': [None, 'Slave Status', 'status', 'slave', 'mysql.slave_status', 'line'],
311 ["slave_sql", "sql_running", "absolute"],
312 ["slave_io", "io_running", "absolute"]
317 class Service(SimpleService):
318 def __init__(self, configuration=None, name=None):
319 SimpleService.__init__(self, configuration=configuration, name=name)
320 self._parse_config(configuration)
322 self.definitions = CHARTS
323 self.connection = None
326 def _parse_config(self, configuration):
328 Parse configuration to collect data from MySQL server
329 :param configuration: dict
333 if self.name is None:
335 if 'user' in configuration:
336 parameters['user'] = self.configuration['user']
337 if 'pass' in configuration:
338 parameters['passwd'] = self.configuration['pass']
339 if 'my.cnf' in configuration:
340 parameters['read_default_file'] = self.configuration['my.cnf']
341 elif 'socket' in configuration:
342 parameters['unix_socket'] = self.configuration['socket']
343 elif 'host' in configuration:
344 parameters['host'] = self.configuration['host']
345 if 'port' in configuration:
346 parameters['port'] = int(self.configuration['port'])
347 self.connection_parameters = parameters
351 Try to connect to MySQL server
354 self.connection = MySQLdb.connect(connect_timeout=self.update_every, **self.connection_parameters)
355 except MySQLdb.OperationalError as e:
356 self.error("Cannot establish connection to MySQL.")
359 except Exception as e:
360 self.error("problem connecting to server:", e)
363 def _get_data_slave(self):
365 Get slave raw data from MySQL server
368 if self.connection is None:
375 slave_raw_data = None
377 cursor = self.connection.cursor()
378 if cursor.execute(QUERY_SLAVE):
379 slave_raw_data = dict(list(zip([elem[0] for elem in cursor.description], cursor.fetchone())))
381 except MySQLdb.OperationalError as e:
382 self.debug("Reconnecting for query", QUERY_SLAVE, ":", str(e))
385 cursor = self.connection.cursor()
386 if cursor.execute(QUERY_SLAVE):
387 slave_raw_data = dict(list(zip([elem[0] for elem in cursor.description], cursor.fetchone())))
388 except Exception as e:
389 self.error("retried, but cannot execute query", QUERY_SLAVE, ":", str(e))
390 self.connection.close()
391 self.connection = None
394 except Exception as e:
395 self.error("cannot execute query", QUERY_SLAVE, ":", str(e))
396 self.connection.close()
397 self.connection = None
400 if slave_raw_data is not None:
402 'slave_behind': None,
408 slave_data['slave_behind'] = int(slave_raw_data.setdefault('Seconds_Behind_Master', -1))
410 slave_data['slave_behind'] = None
413 slave_data['slave_sql'] = 1 if slave_raw_data.get('Slave_SQL_Running') == 'Yes' else -1
415 slave_data['slave_sql'] = None
418 slave_data['slave_io'] = 1 if slave_raw_data.get('Slave_IO_Running') == 'Yes' else -1
420 slave_data['slave_io'] = None
426 Get raw data from MySQL server
429 if self.connection is None:
435 cursor = self.connection.cursor()
436 cursor.execute(QUERY)
437 raw_data = cursor.fetchall()
439 except MySQLdb.OperationalError as e:
440 self.debug("Reconnecting for query", QUERY, ":", str(e))
443 cursor = self.connection.cursor()
444 cursor.execute(QUERY)
445 raw_data = cursor.fetchall()
446 except Exception as e:
447 self.error("retried, but cannot execute query", QUERY, ":", str(e))
448 self.connection.close()
449 self.connection = None
452 except Exception as e:
453 self.error("cannot execute query", QUERY, ":", str(e))
454 self.connection.close()
455 self.connection = None
458 data = dict(raw_data)
460 # check for slave data
461 # the first time is -1 (so we do it)
462 # then it is set to 1 or 0 and we keep it like that
463 if self.do_slave != 0:
464 slave_data = self._get_data_slave()
465 if slave_data is not None:
466 data.update(slave_data)
467 if self.do_slave == -1:
470 if self.do_slave == -1:
471 self.error("replication metrics will be disabled - please allow netdata to collect them.")
476 data["Thread_cache_misses"] = round(float(data["Threads_created"]) / float(data["Connections"]) * 10000)
478 data["Thread_cache_misses"] = None
484 Check if service is able to connect to server
488 self.connection = self._connect()
491 self.connection = None