src/proc_diskstats.c

   1 #ifdef HAVE_CONFIG_H
   2 #include <config.h>
   3 #endif
   4 #include <stdio.h>
   5 #include <stdlib.h>
   6 #include <string.h>
   7
   8 #include "common.h"
   9 #include "log.h"
  10 #include "appconfig.h"
  11 #include "procfile.h"
  12 #include "rrd.h"
  13 #include "plugin_proc.h"
  14
  15 #define RRD_TYPE_DISK                           "disk"
  16 #define RRD_TYPE_DISK_LEN                       strlen(RRD_TYPE_DISK)
  17
  18 #define MAX_PROC_DISKSTATS_LINE 4096
  19
  20 int do_proc_diskstats(int update_every, unsigned long long dt) {
  21         static procfile *ff = NULL;
  22         static char path_to_get_hw_sector_size[FILENAME_MAX + 1] = "";
  23         static int enable_new_disks = -1;
  24         static int do_io = -1, do_ops = -1, do_mops = -1, do_iotime = -1, do_qops = -1, do_util = -1, do_backlog = -1;
  25
  26         if(enable_new_disks == -1)      enable_new_disks = config_get_boolean_ondemand("plugin:proc:/proc/diskstats", "enable new disks detected at runtime", CONFIG_ONDEMAND_ONDEMAND);
  27
  28         if(do_io == -1)         do_io           = config_get_boolean_ondemand("plugin:proc:/proc/diskstats", "bandwidth for all disks", CONFIG_ONDEMAND_ONDEMAND);
  29         if(do_ops == -1)        do_ops          = config_get_boolean_ondemand("plugin:proc:/proc/diskstats", "operations for all disks", CONFIG_ONDEMAND_ONDEMAND);
  30         if(do_mops == -1)       do_mops         = config_get_boolean_ondemand("plugin:proc:/proc/diskstats", "merged operations for all disks", CONFIG_ONDEMAND_ONDEMAND);
  31         if(do_iotime == -1)     do_iotime       = config_get_boolean_ondemand("plugin:proc:/proc/diskstats", "i/o time for all disks", CONFIG_ONDEMAND_ONDEMAND);
  32         if(do_qops == -1)       do_qops         = config_get_boolean_ondemand("plugin:proc:/proc/diskstats", "queued operations for all disks", CONFIG_ONDEMAND_ONDEMAND);
  33         if(do_util == -1)       do_util         = config_get_boolean_ondemand("plugin:proc:/proc/diskstats", "utilization percentage for all disks", CONFIG_ONDEMAND_ONDEMAND);
  34         if(do_backlog == -1)do_backlog  = config_get_boolean_ondemand("plugin:proc:/proc/diskstats", "backlog for all disks", CONFIG_ONDEMAND_ONDEMAND);
  35
  36         if(!ff) {
  37                 char filename[FILENAME_MAX + 1];
  38                 snprintf(filename, FILENAME_MAX, "%s%s", global_host_prefix, "/proc/diskstats");
  39                 ff = procfile_open(config_get("plugin:proc:/proc/diskstats", "filename to monitor", filename), " \t", PROCFILE_FLAG_DEFAULT);
  40         }
  41         if(!ff) return 1;
  42
  43         if(!path_to_get_hw_sector_size[0]) {
  44                 char filename[FILENAME_MAX + 1];
  45                 snprintf(filename, FILENAME_MAX, "%s%s", global_host_prefix, "/sys/block/%s/queue/hw_sector_size");
  46                 snprintf(path_to_get_hw_sector_size, FILENAME_MAX, "%s%s", global_host_prefix, config_get("plugin:proc:/proc/diskstats", "path to get h/w sector size", filename));
  47         }
  48
  49         ff = procfile_readall(ff);
  50         if(!ff) return 0; // we return 0, so that we will retry to open it next time
  51
  52         uint32_t lines = procfile_lines(ff), l;
  53         uint32_t words;
  54
  55         for(l = 0; l < lines ;l++) {
  56                 char *disk;
  57                 unsigned long long      major = 0, minor = 0,
  58                                                         reads = 0,  mreads = 0,  readsectors = 0,  readms = 0,
  59                                                         writes = 0, mwrites = 0, writesectors = 0, writems = 0,
  60                                                         queued_ios = 0, busy_ms = 0, backlog_ms = 0;
  61
  62                 unsigned long long      last_reads = 0,  last_readsectors = 0,  last_readms = 0,
  63                                                         last_writes = 0, last_writesectors = 0, last_writems = 0,
  64                                                         last_busy_ms = 0;
  65
  66                 words = procfile_linewords(ff, l);
  67                 if(words < 14) continue;
  68
  69                 major                   = strtoull(procfile_lineword(ff, l, 0), NULL, 10);
  70                 minor                   = strtoull(procfile_lineword(ff, l, 1), NULL, 10);
  71                 disk                    = procfile_lineword(ff, l, 2);
  72
  73                 // # of reads completed # of writes completed
  74                 // This is the total number of reads or writes completed successfully.
  75                 reads                   = strtoull(procfile_lineword(ff, l, 3), NULL, 10);      // rd_ios
  76                 writes                  = strtoull(procfile_lineword(ff, l, 7), NULL, 10);      // wr_ios
  77
  78                 // # of reads merged # of writes merged
  79                 // Reads and writes which are adjacent to each other may be merged for
  80             // efficiency.  Thus two 4K reads may become one 8K read before it is
  81             // ultimately handed to the disk, and so it will be counted (and queued)
  82                 mreads                  = strtoull(procfile_lineword(ff, l, 4), NULL, 10);      // rd_merges_or_rd_sec
  83                 mwrites                 = strtoull(procfile_lineword(ff, l, 8), NULL, 10);      // wr_merges
  84
  85                 // # of sectors read # of sectors written
  86                 // This is the total number of sectors read or written successfully.
  87                 readsectors     = strtoull(procfile_lineword(ff, l, 5), NULL, 10);      // rd_sec_or_wr_ios
  88                 writesectors    = strtoull(procfile_lineword(ff, l, 9), NULL, 10);      // wr_sec
  89
  90                 // # of milliseconds spent reading # of milliseconds spent writing
  91                 // This is the total number of milliseconds spent by all reads or writes (as
  92                 // measured from __make_request() to end_that_request_last()).
  93                 readms                  = strtoull(procfile_lineword(ff, l, 6), NULL, 10);      // rd_ticks_or_wr_sec
  94                 writems                 = strtoull(procfile_lineword(ff, l, 10), NULL, 10);     // wr_ticks
  95
  96                 // # of I/Os currently in progress
  97                 // The only field that should go to zero. Incremented as requests are
  98                 // given to appropriate struct request_queue and decremented as they finish.
  99                 queued_ios              = strtoull(procfile_lineword(ff, l, 11), NULL, 10);     // ios_pgr
 100
 101                 // # of milliseconds spent doing I/Os
 102                 // This field increases so long as field queued_ios is nonzero.
 103                 busy_ms                 = strtoull(procfile_lineword(ff, l, 12), NULL, 10);     // tot_ticks
 104
 105                 // weighted # of milliseconds spent doing I/Os
 106                 // This field is incremented at each I/O start, I/O completion, I/O
 107                 // merge, or read of these stats by the number of I/Os in progress
 108                 // (field queued_ios) times the number of milliseconds spent doing I/O since the
 109                 // last update of this field.  This can provide an easy measure of both
 110                 // I/O completion time and the backlog that may be accumulating.
 111                 backlog_ms              = strtoull(procfile_lineword(ff, l, 13), NULL, 10);     // rq_ticks
 112
 113
 114                 int def_enabled = 0;
 115
 116                 // remove slashes from disk names
 117                 char *s;
 118                 for(s = disk; *s ;s++) if(*s == '/') *s = '_';
 119
 120                 switch(major) {
 121                         case 9: // MDs
 122                         case 43: // network block
 123                         case 144: // nfs
 124                         case 145: // nfs
 125                         case 146: // nfs
 126                         case 199: // veritas
 127                         case 201: // veritas
 128                         case 251: // dm
 129                                 def_enabled = enable_new_disks;
 130                                 break;
 131
 132                         case 48: // RAID
 133                         case 49: // RAID
 134                         case 50: // RAID
 135                         case 51: // RAID
 136                         case 52: // RAID
 137                         case 53: // RAID
 138                         case 54: // RAID
 139                         case 55: // RAID
 140                         case 112: // RAID
 141                         case 136: // RAID
 142                         case 137: // RAID
 143                         case 138: // RAID
 144                         case 139: // RAID
 145                         case 140: // RAID
 146                         case 141: // RAID
 147                         case 142: // RAID
 148                         case 143: // RAID
 149                         case 179: // MMC
 150                         case 180: // USB
 151                                 if(minor % 8) def_enabled = 0; // partitions
 152                                 else def_enabled = enable_new_disks;
 153                                 break;
 154
 155                         case 8: // scsi disks
 156                         case 65: // scsi disks
 157                         case 66: // scsi disks
 158                         case 67: // scsi disks
 159                         case 68: // scsi disks
 160                         case 69: // scsi disks
 161                         case 70: // scsi disks
 162                         case 71: // scsi disks
 163                         case 72: // scsi disks
 164                         case 73: // scsi disks
 165                         case 74: // scsi disks
 166                         case 75: // scsi disks
 167                         case 76: // scsi disks
 168                         case 77: // scsi disks
 169                         case 78: // scsi disks
 170                         case 79: // scsi disks
 171                         case 80: // i2o
 172                         case 81: // i2o
 173                         case 82: // i2o
 174                         case 83: // i2o
 175                         case 84: // i2o
 176                         case 85: // i2o
 177                         case 86: // i2o
 178                         case 87: // i2o
 179                         case 101: // hyperdisk
 180                         case 102: // compressed
 181                         case 104: // scsi
 182                         case 105: // scsi
 183                         case 106: // scsi
 184                         case 107: // scsi
 185                         case 108: // scsi
 186                         case 109: // scsi
 187                         case 110: // scsi
 188                         case 111: // scsi
 189                         case 114: // bios raid
 190                         case 116: // ram board
 191                         case 128: // scsi
 192                         case 129: // scsi
 193                         case 130: // scsi
 194                         case 131: // scsi
 195                         case 132: // scsi
 196                         case 133: // scsi
 197                         case 134: // scsi
 198                         case 135: // scsi
 199                         case 153: // raid
 200                         case 202: // xen
 201                         case 256: // flash
 202                         case 257: // flash
 203                                 if(minor % 16) def_enabled = 0; // partitions
 204                                 else def_enabled = enable_new_disks;
 205                                 break;
 206
 207                         case 160: // raid
 208                         case 161: // raid
 209                                 if(minor % 32) def_enabled = 0; // partitions
 210                                 else def_enabled = enable_new_disks;
 211                                 break;
 212
 213                         case 3: // ide
 214                         case 13: // 8bit ide
 215                         case 22: // ide
 216                         case 33: // ide
 217                         case 34: // ide
 218                         case 56: // ide
 219                         case 57: // ide
 220                         case 88: // ide
 221                         case 89: // ide
 222                         case 90: // ide
 223                         case 91: // ide
 224                                 if(minor % 64) def_enabled = 0; // partitions
 225                                 else def_enabled = enable_new_disks;
 226                                 break;
 227
 228                         case 252: // zram
 229                                 def_enabled = 0;
 230                                 break;
 231
 232                         default:
 233                                 def_enabled = 0;
 234                                 break;
 235                 }
 236
 237                 int ddo_io = do_io, ddo_ops = do_ops, ddo_mops = do_mops, ddo_iotime = do_iotime, ddo_qops = do_qops, ddo_util = do_util, ddo_backlog = do_backlog;
 238
 239                 // check which charts are enabled for this disk
 240                 {
 241                         char var_name[4096 + 1];
 242                         snprintf(var_name, 4096, "plugin:proc:/proc/diskstats:%s", disk);
 243                         def_enabled = config_get_boolean_ondemand(var_name, "enabled", def_enabled);
 244                         if(def_enabled == CONFIG_ONDEMAND_NO) continue;
 245                         if(def_enabled == CONFIG_ONDEMAND_ONDEMAND && !reads && !writes) continue;
 246
 247
 248                         ddo_io          = config_get_boolean_ondemand(var_name, "bandwidth", ddo_io);
 249                         ddo_ops         = config_get_boolean_ondemand(var_name, "operations", ddo_ops);
 250                         ddo_mops        = config_get_boolean_ondemand(var_name, "merged operations", ddo_mops);
 251                         ddo_iotime      = config_get_boolean_ondemand(var_name, "i/o time", ddo_iotime);
 252                         ddo_qops        = config_get_boolean_ondemand(var_name, "queued operations", ddo_qops);
 253                         ddo_util        = config_get_boolean_ondemand(var_name, "utilization percentage", ddo_util);
 254                         ddo_backlog = config_get_boolean_ondemand(var_name, "backlog", ddo_backlog);
 255
 256                         // by default, do not add charts that do not have values
 257                         if(ddo_io == CONFIG_ONDEMAND_ONDEMAND && !reads && !writes) ddo_io = 0;
 258                         if(ddo_mops == CONFIG_ONDEMAND_ONDEMAND && mreads == 0 && mwrites == 0) ddo_mops = 0;
 259                         if(ddo_iotime == CONFIG_ONDEMAND_ONDEMAND && readms == 0 && writems == 0) ddo_iotime = 0;
 260                         if(ddo_util == CONFIG_ONDEMAND_ONDEMAND && busy_ms == 0) ddo_util = 0;
 261                         if(ddo_backlog == CONFIG_ONDEMAND_ONDEMAND && backlog_ms == 0) ddo_backlog = 0;
 262                         if(ddo_qops == CONFIG_ONDEMAND_ONDEMAND && backlog_ms == 0) ddo_qops = 0;
 263
 264                         // for absolute values, we need to switch the setting to 'yes'
 265                         // to allow it refresh from now on
 266                         if(ddo_qops == CONFIG_ONDEMAND_ONDEMAND) config_set(var_name, "queued operations", "yes");
 267                 }
 268
 269                 RRDSET *st;
 270
 271                 // --------------------------------------------------------------------
 272
 273                 int sector_size = 512;
 274                 if(ddo_io) {
 275                         st = rrdset_find_bytype(RRD_TYPE_DISK, disk);
 276                         if(!st) {
 277                                 char tf[FILENAME_MAX + 1], *t;
 278                                 char ssfilename[FILENAME_MAX + 1];
 279
 280                                 strncpy(tf, disk, FILENAME_MAX);
 281                                 tf[FILENAME_MAX] = '\0';
 282
 283                                 // replace all / with !
 284                                 while((t = strchr(tf, '/'))) *t = '!';
 285
 286                                 snprintf(ssfilename, FILENAME_MAX, path_to_get_hw_sector_size, tf);
 287                                 FILE *fpss = fopen(ssfilename, "r");
 288                                 if(fpss) {
 289                                         char ssbuffer[1025];
 290                                         char *tmp = fgets(ssbuffer, 1024, fpss);
 291
 292                                         if(tmp) {
 293                                                 sector_size = atoi(tmp);
 294                                                 if(sector_size <= 0) {
 295                                                         error("Invalid sector size %d for device %s in %s. Assuming 512.", sector_size, disk, ssfilename);
 296                                                         sector_size = 512;
 297                                                 }
 298                                         }
 299                                         else error("Cannot read data for sector size for device %s from %s. Assuming 512.", disk, ssfilename);
 300
 301                                         fclose(fpss);
 302                                 }
 303                                 else error("Cannot read sector size for device %s from %s. Assuming 512.", disk, ssfilename);
 304
 305                                 st = rrdset_create(RRD_TYPE_DISK, disk, NULL, disk, "Disk I/O Bandwidth", "kilobytes/s", 2000, update_every, RRDSET_TYPE_AREA);
 306
 307                                 rrddim_add(st, "reads", NULL, sector_size, 1024, RRDDIM_INCREMENTAL);
 308                                 rrddim_add(st, "writes", NULL, sector_size * -1, 1024, RRDDIM_INCREMENTAL);
 309                         }
 310                         else rrdset_next_usec(st, dt);
 311
 312                         last_readsectors  = rrddim_set(st, "reads", readsectors);
 313                         last_writesectors = rrddim_set(st, "writes", writesectors);
 314                         rrdset_done(st);
 315                 }
 316
 317                 // --------------------------------------------------------------------
 318
 319                 if(ddo_ops) {
 320                         st = rrdset_find_bytype("disk_ops", disk);
 321                         if(!st) {
 322                                 st = rrdset_create("disk_ops", disk, NULL, disk, "Disk Completed I/O Operations", "operations/s", 2001, update_every, RRDSET_TYPE_LINE);
 323                                 st->isdetail = 1;
 324
 325                                 rrddim_add(st, "reads", NULL, 1, 1, RRDDIM_INCREMENTAL);
 326                                 rrddim_add(st, "writes", NULL, -1, 1, RRDDIM_INCREMENTAL);
 327                         }
 328                         else rrdset_next_usec(st, dt);
 329
 330                         last_reads  = rrddim_set(st, "reads", reads);
 331                         last_writes = rrddim_set(st, "writes", writes);
 332                         rrdset_done(st);
 333                 }
 334
 335                 // --------------------------------------------------------------------
 336
 337                 if(ddo_qops) {
 338                         st = rrdset_find_bytype("disk_qops", disk);
 339                         if(!st) {
 340                                 st = rrdset_create("disk_qops", disk, NULL, disk, "Disk Queued I/O Operations", "operations", 2002, update_every, RRDSET_TYPE_LINE);
 341                                 st->isdetail = 1;
 342
 343                                 rrddim_add(st, "operations", NULL, 1, 1, RRDDIM_ABSOLUTE);
 344                         }
 345                         else rrdset_next_usec(st, dt);
 346
 347                         rrddim_set(st, "operations", queued_ios);
 348                         rrdset_done(st);
 349                 }
 350
 351                 // --------------------------------------------------------------------
 352
 353                 if(ddo_backlog) {
 354                         st = rrdset_find_bytype("disk_backlog", disk);
 355                         if(!st) {
 356                                 st = rrdset_create("disk_backlog", disk, NULL, disk, "Disk Backlog", "backlog (ms)", 2003, update_every, RRDSET_TYPE_AREA);
 357                                 st->isdetail = 1;
 358
 359                                 rrddim_add(st, "backlog", NULL, 1, 10, RRDDIM_INCREMENTAL);
 360                         }
 361                         else rrdset_next_usec(st, dt);
 362
 363                         rrddim_set(st, "backlog", backlog_ms);
 364                         rrdset_done(st);
 365                 }
 366
 367                 // --------------------------------------------------------------------
 368
 369                 if(ddo_util) {
 370                         st = rrdset_find_bytype("disk_util", disk);
 371                         if(!st) {
 372                                 st = rrdset_create("disk_util", disk, NULL, disk, "Disk Utilization Time", "% of time working", 2004, update_every, RRDSET_TYPE_AREA);
 373                                 st->isdetail = 1;
 374
 375                                 rrddim_add(st, "utilization", NULL, 1, 10, RRDDIM_INCREMENTAL);
 376                         }
 377                         else rrdset_next_usec(st, dt);
 378
 379                         last_busy_ms = rrddim_set(st, "utilization", busy_ms);
 380                         rrdset_done(st);
 381                 }
 382
 383                 // --------------------------------------------------------------------
 384
 385                 if(ddo_mops) {
 386                         st = rrdset_find_bytype("disk_mops", disk);
 387                         if(!st) {
 388                                 st = rrdset_create("disk_mops", disk, NULL, disk, "Disk Merged Operations", "merged operations/s", 2021, update_every, RRDSET_TYPE_LINE);
 389                                 st->isdetail = 1;
 390
 391                                 rrddim_add(st, "reads", NULL, 1, 1, RRDDIM_INCREMENTAL);
 392                                 rrddim_add(st, "writes", NULL, -1, 1, RRDDIM_INCREMENTAL);
 393                         }
 394                         else rrdset_next_usec(st, dt);
 395
 396                         rrddim_set(st, "reads", mreads);
 397                         rrddim_set(st, "writes", mwrites);
 398                         rrdset_done(st);
 399                 }
 400
 401                 // --------------------------------------------------------------------
 402
 403                 if(ddo_iotime) {
 404                         st = rrdset_find_bytype("disk_iotime", disk);
 405                         if(!st) {
 406                                 st = rrdset_create("disk_iotime", disk, NULL, disk, "Disk Total I/O Time", "milliseconds/s", 2022, update_every, RRDSET_TYPE_LINE);
 407                                 st->isdetail = 1;
 408
 409                                 rrddim_add(st, "reads", NULL, 1, 1, RRDDIM_INCREMENTAL);
 410                                 rrddim_add(st, "writes", NULL, -1, 1, RRDDIM_INCREMENTAL);
 411                         }
 412                         else rrdset_next_usec(st, dt);
 413
 414                         last_readms  = rrddim_set(st, "reads", readms);
 415                         last_writems = rrddim_set(st, "writes", writems);
 416                         rrdset_done(st);
 417                 }
 418
 419                 // --------------------------------------------------------------------
 420                 // calculate differential charts
 421                 // only if this is not the first time we run
 422
 423                 if(dt) {
 424                         if(ddo_iotime && ddo_ops) {
 425                                 st = rrdset_find_bytype("disk_await", disk);
 426                                 if(!st) {
 427                                         st = rrdset_create("disk_await", disk, NULL, disk, "Average Completed I/O Operation Time", "ms per operation", 2005, update_every, RRDSET_TYPE_LINE);
 428                                         st->isdetail = 1;
 429
 430                                         rrddim_add(st, "reads", NULL, 1, 1, RRDDIM_ABSOLUTE);
 431                                         rrddim_add(st, "writes", NULL, -1, 1, RRDDIM_ABSOLUTE);
 432                                 }
 433                                 else rrdset_next_usec(st, dt);
 434
 435                                 rrddim_set(st, "reads", (reads - last_reads) ? (readms - last_readms) / (reads - last_reads) : 0);
 436                                 rrddim_set(st, "writes", (writes - last_writes) ? (writems - last_writems) / (writes - last_writes) : 0);
 437                                 rrdset_done(st);
 438                         }
 439
 440                         if(ddo_io && ddo_ops) {
 441                                 st = rrdset_find_bytype("disk_avgsz", disk);
 442                                 if(!st) {
 443                                         st = rrdset_create("disk_avgsz", disk, NULL, disk, "Average Completed I/O Operation Bandwidth", "kilobytes per operation", 2006, update_every, RRDSET_TYPE_AREA);
 444                                         st->isdetail = 1;
 445
 446                                         rrddim_add(st, "reads", NULL, sector_size, 1024, RRDDIM_ABSOLUTE);
 447                                         rrddim_add(st, "writes", NULL, -sector_size, 1024, RRDDIM_ABSOLUTE);
 448                                 }
 449                                 else rrdset_next_usec(st, dt);
 450
 451                                 rrddim_set(st, "reads", (reads - last_reads) ? (readsectors - last_readsectors) / (reads - last_reads) : 0);
 452                                 rrddim_set(st, "writes", (writes - last_writes) ? (writesectors - last_writesectors) / (writes - last_writes) : 0);
 453                                 rrdset_done(st);
 454                         }
 455
 456                         if(ddo_util && ddo_ops) {
 457                                 st = rrdset_find_bytype("disk_svctm", disk);
 458                                 if(!st) {
 459                                         st = rrdset_create("disk_svctm", disk, NULL, disk, "Average Service Time", "ms per operation", 2007, update_every, RRDSET_TYPE_LINE);
 460                                         st->isdetail = 1;
 461
 462                                         rrddim_add(st, "svctm", NULL, 1, 1, RRDDIM_ABSOLUTE);
 463                                 }
 464                                 else rrdset_next_usec(st, dt);
 465
 466                                 rrddim_set(st, "svctm", ((reads - last_reads) + (writes - last_writes)) ? (busy_ms - last_busy_ms) / ((reads - last_reads) + (writes - last_writes)) : 0);
 467                                 rrdset_done(st);
 468                         }
 469                 }
 470         }
 471
 472         return 0;
 473 }