src/proc_diskstats.c

   1 #ifdef HAVE_CONFIG_H
   2 #include <config.h>
   3 #endif
   4 #include <stdio.h>
   5 #include <stdlib.h>
   6 #include <string.h>
   7
   8 #include "common.h"
   9 #include "log.h"
  10 #include "appconfig.h"
  11 #include "procfile.h"
  12 #include "rrd.h"
  13 #include "plugin_proc.h"
  14
  15 #define RRD_TYPE_DISK "disk"
  16
  17 int do_proc_diskstats(int update_every, unsigned long long dt) {
  18         static procfile *ff = NULL;
  19         static char path_to_get_hw_sector_size[FILENAME_MAX + 1] = "";
  20         static int enable_new_disks = -1;
  21         static int do_io = -1, do_ops = -1, do_mops = -1, do_iotime = -1, do_qops = -1, do_util = -1, do_backlog = -1;
  22
  23         if(enable_new_disks == -1)      enable_new_disks = config_get_boolean_ondemand("plugin:proc:/proc/diskstats", "enable new disks detected at runtime", CONFIG_ONDEMAND_ONDEMAND);
  24
  25         if(do_io == -1)         do_io           = config_get_boolean_ondemand("plugin:proc:/proc/diskstats", "bandwidth for all disks", CONFIG_ONDEMAND_ONDEMAND);
  26         if(do_ops == -1)        do_ops          = config_get_boolean_ondemand("plugin:proc:/proc/diskstats", "operations for all disks", CONFIG_ONDEMAND_ONDEMAND);
  27         if(do_mops == -1)       do_mops         = config_get_boolean_ondemand("plugin:proc:/proc/diskstats", "merged operations for all disks", CONFIG_ONDEMAND_ONDEMAND);
  28         if(do_iotime == -1)     do_iotime       = config_get_boolean_ondemand("plugin:proc:/proc/diskstats", "i/o time for all disks", CONFIG_ONDEMAND_ONDEMAND);
  29         if(do_qops == -1)       do_qops         = config_get_boolean_ondemand("plugin:proc:/proc/diskstats", "queued operations for all disks", CONFIG_ONDEMAND_ONDEMAND);
  30         if(do_util == -1)       do_util         = config_get_boolean_ondemand("plugin:proc:/proc/diskstats", "utilization percentage for all disks", CONFIG_ONDEMAND_ONDEMAND);
  31         if(do_backlog == -1)do_backlog  = config_get_boolean_ondemand("plugin:proc:/proc/diskstats", "backlog for all disks", CONFIG_ONDEMAND_ONDEMAND);
  32
  33         if(!ff) {
  34                 char filename[FILENAME_MAX + 1];
  35                 snprintf(filename, FILENAME_MAX, "%s%s", global_host_prefix, "/proc/diskstats");
  36                 ff = procfile_open(config_get("plugin:proc:/proc/diskstats", "filename to monitor", filename), " \t", PROCFILE_FLAG_DEFAULT);
  37         }
  38         if(!ff) return 1;
  39
  40         if(!path_to_get_hw_sector_size[0]) {
  41                 char filename[FILENAME_MAX + 1];
  42                 snprintf(filename, FILENAME_MAX, "%s%s", global_host_prefix, "/sys/block/%s/queue/hw_sector_size");
  43                 snprintf(path_to_get_hw_sector_size, FILENAME_MAX, "%s%s", global_host_prefix, config_get("plugin:proc:/proc/diskstats", "path to get h/w sector size", filename));
  44         }
  45
  46         ff = procfile_readall(ff);
  47         if(!ff) return 0; // we return 0, so that we will retry to open it next time
  48
  49         uint32_t lines = procfile_lines(ff), l;
  50         uint32_t words;
  51
  52         for(l = 0; l < lines ;l++) {
  53                 char *disk;
  54                 unsigned long long      major = 0, minor = 0,
  55                                                         reads = 0,  mreads = 0,  readsectors = 0,  readms = 0,
  56                                                         writes = 0, mwrites = 0, writesectors = 0, writems = 0,
  57                                                         queued_ios = 0, busy_ms = 0, backlog_ms = 0;
  58
  59                 unsigned long long      last_reads = 0,  last_readsectors = 0,  last_readms = 0,
  60                                                         last_writes = 0, last_writesectors = 0, last_writems = 0,
  61                                                         last_busy_ms = 0;
  62
  63                 words = procfile_linewords(ff, l);
  64                 if(words < 14) continue;
  65
  66                 major                   = strtoull(procfile_lineword(ff, l, 0), NULL, 10);
  67                 minor                   = strtoull(procfile_lineword(ff, l, 1), NULL, 10);
  68                 disk                    = procfile_lineword(ff, l, 2);
  69
  70                 // # of reads completed # of writes completed
  71                 // This is the total number of reads or writes completed successfully.
  72                 reads                   = strtoull(procfile_lineword(ff, l, 3), NULL, 10);      // rd_ios
  73                 writes                  = strtoull(procfile_lineword(ff, l, 7), NULL, 10);      // wr_ios
  74
  75                 // # of reads merged # of writes merged
  76                 // Reads and writes which are adjacent to each other may be merged for
  77             // efficiency.  Thus two 4K reads may become one 8K read before it is
  78             // ultimately handed to the disk, and so it will be counted (and queued)
  79                 mreads                  = strtoull(procfile_lineword(ff, l, 4), NULL, 10);      // rd_merges_or_rd_sec
  80                 mwrites                 = strtoull(procfile_lineword(ff, l, 8), NULL, 10);      // wr_merges
  81
  82                 // # of sectors read # of sectors written
  83                 // This is the total number of sectors read or written successfully.
  84                 readsectors     = strtoull(procfile_lineword(ff, l, 5), NULL, 10);      // rd_sec_or_wr_ios
  85                 writesectors    = strtoull(procfile_lineword(ff, l, 9), NULL, 10);      // wr_sec
  86
  87                 // # of milliseconds spent reading # of milliseconds spent writing
  88                 // This is the total number of milliseconds spent by all reads or writes (as
  89                 // measured from __make_request() to end_that_request_last()).
  90                 readms                  = strtoull(procfile_lineword(ff, l, 6), NULL, 10);      // rd_ticks_or_wr_sec
  91                 writems                 = strtoull(procfile_lineword(ff, l, 10), NULL, 10);     // wr_ticks
  92
  93                 // # of I/Os currently in progress
  94                 // The only field that should go to zero. Incremented as requests are
  95                 // given to appropriate struct request_queue and decremented as they finish.
  96                 queued_ios              = strtoull(procfile_lineword(ff, l, 11), NULL, 10);     // ios_pgr
  97
  98                 // # of milliseconds spent doing I/Os
  99                 // This field increases so long as field queued_ios is nonzero.
 100                 busy_ms                 = strtoull(procfile_lineword(ff, l, 12), NULL, 10);     // tot_ticks
 101
 102                 // weighted # of milliseconds spent doing I/Os
 103                 // This field is incremented at each I/O start, I/O completion, I/O
 104                 // merge, or read of these stats by the number of I/Os in progress
 105                 // (field queued_ios) times the number of milliseconds spent doing I/O since the
 106                 // last update of this field.  This can provide an easy measure of both
 107                 // I/O completion time and the backlog that may be accumulating.
 108                 backlog_ms              = strtoull(procfile_lineword(ff, l, 13), NULL, 10);     // rq_ticks
 109
 110
 111                 int def_enabled = 0;
 112
 113                 // remove slashes from disk names
 114                 char *s;
 115                 for(s = disk; *s ;s++) if(*s == '/') *s = '_';
 116
 117                 switch(major) {
 118                         case 9: // MDs
 119                         case 43: // network block
 120                         case 144: // nfs
 121                         case 145: // nfs
 122                         case 146: // nfs
 123                         case 199: // veritas
 124                         case 201: // veritas
 125                         case 251: // dm
 126                         case 253: // virtio
 127                                 def_enabled = enable_new_disks;
 128                                 break;
 129
 130                         case 48: // RAID
 131                         case 49: // RAID
 132                         case 50: // RAID
 133                         case 51: // RAID
 134                         case 52: // RAID
 135                         case 53: // RAID
 136                         case 54: // RAID
 137                         case 55: // RAID
 138                         case 112: // RAID
 139                         case 136: // RAID
 140                         case 137: // RAID
 141                         case 138: // RAID
 142                         case 139: // RAID
 143                         case 140: // RAID
 144                         case 141: // RAID
 145                         case 142: // RAID
 146                         case 143: // RAID
 147                         case 179: // MMC
 148                         case 180: // USB
 149                                 if(minor % 8) def_enabled = 0; // partitions
 150                                 else def_enabled = enable_new_disks;
 151                                 break;
 152
 153                         case 8: // scsi disks
 154                         case 65: // scsi disks
 155                         case 66: // scsi disks
 156                         case 67: // scsi disks
 157                         case 68: // scsi disks
 158                         case 69: // scsi disks
 159                         case 70: // scsi disks
 160                         case 71: // scsi disks
 161                         case 72: // scsi disks
 162                         case 73: // scsi disks
 163                         case 74: // scsi disks
 164                         case 75: // scsi disks
 165                         case 76: // scsi disks
 166                         case 77: // scsi disks
 167                         case 78: // scsi disks
 168                         case 79: // scsi disks
 169                         case 80: // i2o
 170                         case 81: // i2o
 171                         case 82: // i2o
 172                         case 83: // i2o
 173                         case 84: // i2o
 174                         case 85: // i2o
 175                         case 86: // i2o
 176                         case 87: // i2o
 177                         case 101: // hyperdisk
 178                         case 102: // compressed
 179                         case 104: // scsi
 180                         case 105: // scsi
 181                         case 106: // scsi
 182                         case 107: // scsi
 183                         case 108: // scsi
 184                         case 109: // scsi
 185                         case 110: // scsi
 186                         case 111: // scsi
 187                         case 114: // bios raid
 188                         case 116: // ram board
 189                         case 128: // scsi
 190                         case 129: // scsi
 191                         case 130: // scsi
 192                         case 131: // scsi
 193                         case 132: // scsi
 194                         case 133: // scsi
 195                         case 134: // scsi
 196                         case 135: // scsi
 197                         case 153: // raid
 198                         case 202: // xen
 199                         case 256: // flash
 200                         case 257: // flash
 201                         case 259: // nvme0n1 issue #119
 202                                 if(minor % 16) def_enabled = 0; // partitions
 203                                 else def_enabled = enable_new_disks;
 204                                 break;
 205
 206                         case 160: // raid
 207                         case 161: // raid
 208                                 if(minor % 32) def_enabled = 0; // partitions
 209                                 else def_enabled = enable_new_disks;
 210                                 break;
 211
 212                         case 3: // ide
 213                         case 13: // 8bit ide
 214                         case 22: // ide
 215                         case 33: // ide
 216                         case 34: // ide
 217                         case 56: // ide
 218                         case 57: // ide
 219                         case 88: // ide
 220                         case 89: // ide
 221                         case 90: // ide
 222                         case 91: // ide
 223                                 if(minor % 64) def_enabled = 0; // partitions
 224                                 else def_enabled = enable_new_disks;
 225                                 break;
 226
 227                         case 252: // zram
 228                                 def_enabled = 0;
 229                                 break;
 230
 231                         default:
 232                                 def_enabled = 0;
 233                                 break;
 234                 }
 235
 236                 int ddo_io = do_io, ddo_ops = do_ops, ddo_mops = do_mops, ddo_iotime = do_iotime, ddo_qops = do_qops, ddo_util = do_util, ddo_backlog = do_backlog;
 237
 238                 // check which charts are enabled for this disk
 239                 {
 240                         char var_name[4096 + 1];
 241                         snprintf(var_name, 4096, "plugin:proc:/proc/diskstats:%s", disk);
 242                         def_enabled = config_get_boolean_ondemand(var_name, "enabled", def_enabled);
 243                         if(def_enabled == CONFIG_ONDEMAND_NO) continue;
 244                         if(def_enabled == CONFIG_ONDEMAND_ONDEMAND && !reads && !writes) continue;
 245
 246
 247                         ddo_io          = config_get_boolean_ondemand(var_name, "bandwidth", ddo_io);
 248                         ddo_ops         = config_get_boolean_ondemand(var_name, "operations", ddo_ops);
 249                         ddo_mops        = config_get_boolean_ondemand(var_name, "merged operations", ddo_mops);
 250                         ddo_iotime      = config_get_boolean_ondemand(var_name, "i/o time", ddo_iotime);
 251                         ddo_qops        = config_get_boolean_ondemand(var_name, "queued operations", ddo_qops);
 252                         ddo_util        = config_get_boolean_ondemand(var_name, "utilization percentage", ddo_util);
 253                         ddo_backlog = config_get_boolean_ondemand(var_name, "backlog", ddo_backlog);
 254
 255                         // by default, do not add charts that do not have values
 256                         if(ddo_io == CONFIG_ONDEMAND_ONDEMAND && !reads && !writes) ddo_io = 0;
 257                         if(ddo_mops == CONFIG_ONDEMAND_ONDEMAND && mreads == 0 && mwrites == 0) ddo_mops = 0;
 258                         if(ddo_iotime == CONFIG_ONDEMAND_ONDEMAND && readms == 0 && writems == 0) ddo_iotime = 0;
 259                         if(ddo_util == CONFIG_ONDEMAND_ONDEMAND && busy_ms == 0) ddo_util = 0;
 260                         if(ddo_backlog == CONFIG_ONDEMAND_ONDEMAND && backlog_ms == 0) ddo_backlog = 0;
 261                         if(ddo_qops == CONFIG_ONDEMAND_ONDEMAND && backlog_ms == 0) ddo_qops = 0;
 262
 263                         // for absolute values, we need to switch the setting to 'yes'
 264                         // to allow it refresh from now on
 265                         if(ddo_qops == CONFIG_ONDEMAND_ONDEMAND) config_set(var_name, "queued operations", "yes");
 266                 }
 267
 268                 RRDSET *st;
 269
 270                 // --------------------------------------------------------------------
 271
 272                 int sector_size = 512;
 273                 if(ddo_io) {
 274                         st = rrdset_find_bytype(RRD_TYPE_DISK, disk);
 275                         if(!st) {
 276                                 char tf[FILENAME_MAX + 1], *t;
 277                                 char ssfilename[FILENAME_MAX + 1];
 278
 279                                 strncpy(tf, disk, FILENAME_MAX);
 280                                 tf[FILENAME_MAX] = '\0';
 281
 282                                 // replace all / with !
 283                                 while((t = strchr(tf, '/'))) *t = '!';
 284
 285                                 snprintf(ssfilename, FILENAME_MAX, path_to_get_hw_sector_size, tf);
 286                                 FILE *fpss = fopen(ssfilename, "r");
 287                                 if(fpss) {
 288                                         char ssbuffer[1025];
 289                                         char *tmp = fgets(ssbuffer, 1024, fpss);
 290
 291                                         if(tmp) {
 292                                                 sector_size = atoi(tmp);
 293                                                 if(sector_size <= 0) {
 294                                                         error("Invalid sector size %d for device %s in %s. Assuming 512.", sector_size, disk, ssfilename);
 295                                                         sector_size = 512;
 296                                                 }
 297                                         }
 298                                         else error("Cannot read data for sector size for device %s from %s. Assuming 512.", disk, ssfilename);
 299
 300                                         fclose(fpss);
 301                                 }
 302                                 else error("Cannot read sector size for device %s from %s. Assuming 512.", disk, ssfilename);
 303
 304                                 st = rrdset_create(RRD_TYPE_DISK, disk, NULL, disk, "disk.io", "Disk I/O Bandwidth", "kilobytes/s", 2000, update_every, RRDSET_TYPE_AREA);
 305
 306                                 rrddim_add(st, "reads", NULL, sector_size, 1024, RRDDIM_INCREMENTAL);
 307                                 rrddim_add(st, "writes", NULL, sector_size * -1, 1024, RRDDIM_INCREMENTAL);
 308                         }
 309                         else rrdset_next_usec(st, dt);
 310
 311                         last_readsectors  = rrddim_set(st, "reads", readsectors);
 312                         last_writesectors = rrddim_set(st, "writes", writesectors);
 313                         rrdset_done(st);
 314                 }
 315
 316                 // --------------------------------------------------------------------
 317
 318                 if(ddo_ops) {
 319                         st = rrdset_find_bytype("disk_ops", disk);
 320                         if(!st) {
 321                                 st = rrdset_create("disk_ops", disk, NULL, disk, "disk.ops", "Disk Completed I/O Operations", "operations/s", 2001, update_every, RRDSET_TYPE_LINE);
 322                                 st->isdetail = 1;
 323
 324                                 rrddim_add(st, "reads", NULL, 1, 1, RRDDIM_INCREMENTAL);
 325                                 rrddim_add(st, "writes", NULL, -1, 1, RRDDIM_INCREMENTAL);
 326                         }
 327                         else rrdset_next_usec(st, dt);
 328
 329                         last_reads  = rrddim_set(st, "reads", reads);
 330                         last_writes = rrddim_set(st, "writes", writes);
 331                         rrdset_done(st);
 332                 }
 333
 334                 // --------------------------------------------------------------------
 335
 336                 if(ddo_qops) {
 337                         st = rrdset_find_bytype("disk_qops", disk);
 338                         if(!st) {
 339                                 st = rrdset_create("disk_qops", disk, NULL, disk, "disk.qops", "Disk Current I/O Operations", "operations", 2002, update_every, RRDSET_TYPE_LINE);
 340                                 st->isdetail = 1;
 341
 342                                 rrddim_add(st, "operations", NULL, 1, 1, RRDDIM_ABSOLUTE);
 343                         }
 344                         else rrdset_next_usec(st, dt);
 345
 346                         rrddim_set(st, "operations", queued_ios);
 347                         rrdset_done(st);
 348                 }
 349
 350                 // --------------------------------------------------------------------
 351
 352                 if(ddo_backlog) {
 353                         st = rrdset_find_bytype("disk_backlog", disk);
 354                         if(!st) {
 355                                 st = rrdset_create("disk_backlog", disk, NULL, disk, "disk.backlog", "Disk Backlog", "backlog (ms)", 2003, update_every, RRDSET_TYPE_AREA);
 356                                 st->isdetail = 1;
 357
 358                                 rrddim_add(st, "backlog", NULL, 1, 10, RRDDIM_INCREMENTAL);
 359                         }
 360                         else rrdset_next_usec(st, dt);
 361
 362                         rrddim_set(st, "backlog", backlog_ms);
 363                         rrdset_done(st);
 364                 }
 365
 366                 // --------------------------------------------------------------------
 367
 368                 if(ddo_util) {
 369                         st = rrdset_find_bytype("disk_util", disk);
 370                         if(!st) {
 371                                 st = rrdset_create("disk_util", disk, NULL, disk, "disk.util", "Disk Utilization Time", "% of time working", 2004, update_every, RRDSET_TYPE_AREA);
 372                                 st->isdetail = 1;
 373
 374                                 rrddim_add(st, "utilization", NULL, 1, 10, RRDDIM_INCREMENTAL);
 375                         }
 376                         else rrdset_next_usec(st, dt);
 377
 378                         last_busy_ms = rrddim_set(st, "utilization", busy_ms);
 379                         rrdset_done(st);
 380                 }
 381
 382                 // --------------------------------------------------------------------
 383
 384                 if(ddo_mops) {
 385                         st = rrdset_find_bytype("disk_mops", disk);
 386                         if(!st) {
 387                                 st = rrdset_create("disk_mops", disk, NULL, disk, "disk.mops", "Disk Merged Operations", "merged operations/s", 2021, update_every, RRDSET_TYPE_LINE);
 388                                 st->isdetail = 1;
 389
 390                                 rrddim_add(st, "reads", NULL, 1, 1, RRDDIM_INCREMENTAL);
 391                                 rrddim_add(st, "writes", NULL, -1, 1, RRDDIM_INCREMENTAL);
 392                         }
 393                         else rrdset_next_usec(st, dt);
 394
 395                         rrddim_set(st, "reads", mreads);
 396                         rrddim_set(st, "writes", mwrites);
 397                         rrdset_done(st);
 398                 }
 399
 400                 // --------------------------------------------------------------------
 401
 402                 if(ddo_iotime) {
 403                         st = rrdset_find_bytype("disk_iotime", disk);
 404                         if(!st) {
 405                                 st = rrdset_create("disk_iotime", disk, NULL, disk, "disk.iotime", "Disk Total I/O Time", "milliseconds/s", 2022, update_every, RRDSET_TYPE_LINE);
 406                                 st->isdetail = 1;
 407
 408                                 rrddim_add(st, "reads", NULL, 1, 1, RRDDIM_INCREMENTAL);
 409                                 rrddim_add(st, "writes", NULL, -1, 1, RRDDIM_INCREMENTAL);
 410                         }
 411                         else rrdset_next_usec(st, dt);
 412
 413                         last_readms  = rrddim_set(st, "reads", readms);
 414                         last_writems = rrddim_set(st, "writes", writems);
 415                         rrdset_done(st);
 416                 }
 417
 418                 // --------------------------------------------------------------------
 419                 // calculate differential charts
 420                 // only if this is not the first time we run
 421
 422                 if(dt) {
 423                         if(ddo_iotime && ddo_ops) {
 424                                 st = rrdset_find_bytype("disk_await", disk);
 425                                 if(!st) {
 426                                         st = rrdset_create("disk_await", disk, NULL, disk, "disk.await", "Average Completed I/O Operation Time", "ms per operation", 2005, update_every, RRDSET_TYPE_LINE);
 427                                         st->isdetail = 1;
 428
 429                                         rrddim_add(st, "reads", NULL, 1, 1, RRDDIM_ABSOLUTE);
 430                                         rrddim_add(st, "writes", NULL, -1, 1, RRDDIM_ABSOLUTE);
 431                                 }
 432                                 else rrdset_next_usec(st, dt);
 433
 434                                 rrddim_set(st, "reads", (reads - last_reads) ? (readms - last_readms) / (reads - last_reads) : 0);
 435                                 rrddim_set(st, "writes", (writes - last_writes) ? (writems - last_writems) / (writes - last_writes) : 0);
 436                                 rrdset_done(st);
 437                         }
 438
 439                         if(ddo_io && ddo_ops) {
 440                                 st = rrdset_find_bytype("disk_avgsz", disk);
 441                                 if(!st) {
 442                                         st = rrdset_create("disk_avgsz", disk, NULL, disk, "disk.avgsz", "Average Completed I/O Operation Bandwidth", "kilobytes per operation", 2006, update_every, RRDSET_TYPE_AREA);
 443                                         st->isdetail = 1;
 444
 445                                         rrddim_add(st, "reads", NULL, sector_size, 1024, RRDDIM_ABSOLUTE);
 446                                         rrddim_add(st, "writes", NULL, -sector_size, 1024, RRDDIM_ABSOLUTE);
 447                                 }
 448                                 else rrdset_next_usec(st, dt);
 449
 450                                 rrddim_set(st, "reads", (reads - last_reads) ? (readsectors - last_readsectors) / (reads - last_reads) : 0);
 451                                 rrddim_set(st, "writes", (writes - last_writes) ? (writesectors - last_writesectors) / (writes - last_writes) : 0);
 452                                 rrdset_done(st);
 453                         }
 454
 455                         if(ddo_util && ddo_ops) {
 456                                 st = rrdset_find_bytype("disk_svctm", disk);
 457                                 if(!st) {
 458                                         st = rrdset_create("disk_svctm", disk, NULL, disk, "disk.svctm", "Average Service Time", "ms per operation", 2007, update_every, RRDSET_TYPE_LINE);
 459                                         st->isdetail = 1;
 460
 461                                         rrddim_add(st, "svctm", NULL, 1, 1, RRDDIM_ABSOLUTE);
 462                                 }
 463                                 else rrdset_next_usec(st, dt);
 464
 465                                 rrddim_set(st, "svctm", ((reads - last_reads) + (writes - last_writes)) ? (busy_ms - last_busy_ms) / ((reads - last_reads) + (writes - last_writes)) : 0);
 466                                 rrdset_done(st);
 467                         }
 468                 }
 469         }
 470
 471         return 0;
 472 }