src/sys_fs_cgroup.c

   1 #include "common.h"
   2
   3 // ----------------------------------------------------------------------------
   4 // cgroup globals
   5
   6 #define CHART_PRIORITY_SYSTEMD_SERVICES 19000
   7 #define CHART_PRIORITY_CONTAINERS       40000
   8
   9 static long system_page_size = 4096; // system will be queried via sysconf() in configuration()
  10
  11 static int cgroup_enable_cpuacct_stat = CONFIG_BOOLEAN_AUTO;
  12 static int cgroup_enable_cpuacct_usage = CONFIG_BOOLEAN_AUTO;
  13 static int cgroup_enable_memory = CONFIG_BOOLEAN_AUTO;
  14 static int cgroup_enable_detailed_memory = CONFIG_BOOLEAN_AUTO;
  15 static int cgroup_enable_memory_failcnt = CONFIG_BOOLEAN_AUTO;
  16 static int cgroup_enable_swap = CONFIG_BOOLEAN_AUTO;
  17 static int cgroup_enable_blkio_io = CONFIG_BOOLEAN_AUTO;
  18 static int cgroup_enable_blkio_ops = CONFIG_BOOLEAN_AUTO;
  19 static int cgroup_enable_blkio_throttle_io = CONFIG_BOOLEAN_AUTO;
  20 static int cgroup_enable_blkio_throttle_ops = CONFIG_BOOLEAN_AUTO;
  21 static int cgroup_enable_blkio_merged_ops = CONFIG_BOOLEAN_AUTO;
  22 static int cgroup_enable_blkio_queued_ops = CONFIG_BOOLEAN_AUTO;
  23
  24 static int cgroup_enable_systemd_services = CONFIG_BOOLEAN_YES;
  25 static int cgroup_enable_systemd_services_detailed_memory = CONFIG_BOOLEAN_NO;
  26 static int cgroup_used_memory_without_cache = CONFIG_BOOLEAN_YES;
  27
  28 static int cgroup_search_in_devices = 1;
  29
  30 static int cgroup_enable_new_cgroups_detected_at_runtime = 1;
  31 static int cgroup_check_for_new_every = 10;
  32 static int cgroup_update_every = 1;
  33
  34 static int cgroup_recheck_zero_blkio_every_iterations = 10;
  35 static int cgroup_recheck_zero_mem_failcnt_every_iterations = 10;
  36 static int cgroup_recheck_zero_mem_detailed_every_iterations = 10;
  37
  38 static char *cgroup_cpuacct_base = NULL;
  39 static char *cgroup_blkio_base = NULL;
  40 static char *cgroup_memory_base = NULL;
  41 static char *cgroup_devices_base = NULL;
  42
  43 static int cgroup_root_count = 0;
  44 static int cgroup_root_max = 500;
  45 static int cgroup_max_depth = 0;
  46
  47 static SIMPLE_PATTERN *enabled_cgroup_patterns = NULL;
  48 static SIMPLE_PATTERN *enabled_cgroup_paths = NULL;
  49 static SIMPLE_PATTERN *enabled_cgroup_renames = NULL;
  50 static SIMPLE_PATTERN *systemd_services_cgroups = NULL;
  51
  52 static char *cgroups_rename_script = NULL;
  53
  54 static int cgroups_check = 0;
  55
  56 static uint32_t Read_hash = 0;
  57 static uint32_t Write_hash = 0;
  58 static uint32_t user_hash = 0;
  59 static uint32_t system_hash = 0;
  60
  61 void read_cgroup_plugin_configuration() {
  62     system_page_size = sysconf(_SC_PAGESIZE);
  63
  64     Read_hash = simple_hash("Read");
  65     Write_hash = simple_hash("Write");
  66     user_hash = simple_hash("user");
  67     system_hash = simple_hash("system");
  68
  69     cgroup_update_every = (int)config_get_number("plugin:cgroups", "update every", localhost->rrd_update_every);
  70     if(cgroup_update_every < localhost->rrd_update_every)
  71         cgroup_update_every = localhost->rrd_update_every;
  72
  73     cgroup_check_for_new_every = (int)config_get_number("plugin:cgroups", "check for new cgroups every", cgroup_check_for_new_every * cgroup_update_every);
  74     if(cgroup_check_for_new_every < cgroup_update_every)
  75         cgroup_check_for_new_every = cgroup_update_every;
  76
  77     cgroup_enable_cpuacct_stat = config_get_boolean_ondemand("plugin:cgroups", "enable cpuacct stat (total CPU)", cgroup_enable_cpuacct_stat);
  78     cgroup_enable_cpuacct_usage = config_get_boolean_ondemand("plugin:cgroups", "enable cpuacct usage (per core CPU)", cgroup_enable_cpuacct_usage);
  79
  80     cgroup_enable_memory = config_get_boolean_ondemand("plugin:cgroups", "enable memory (used mem including cache)", cgroup_enable_memory);
  81     cgroup_enable_detailed_memory = config_get_boolean_ondemand("plugin:cgroups", "enable detailed memory", cgroup_enable_detailed_memory);
  82     cgroup_enable_memory_failcnt = config_get_boolean_ondemand("plugin:cgroups", "enable memory limits fail count", cgroup_enable_memory_failcnt);
  83     cgroup_enable_swap = config_get_boolean_ondemand("plugin:cgroups", "enable swap memory", cgroup_enable_swap);
  84
  85     cgroup_enable_blkio_io = config_get_boolean_ondemand("plugin:cgroups", "enable blkio bandwidth", cgroup_enable_blkio_io);
  86     cgroup_enable_blkio_ops = config_get_boolean_ondemand("plugin:cgroups", "enable blkio operations", cgroup_enable_blkio_ops);
  87     cgroup_enable_blkio_throttle_io = config_get_boolean_ondemand("plugin:cgroups", "enable blkio throttle bandwidth", cgroup_enable_blkio_throttle_io);
  88     cgroup_enable_blkio_throttle_ops = config_get_boolean_ondemand("plugin:cgroups", "enable blkio throttle operations", cgroup_enable_blkio_throttle_ops);
  89     cgroup_enable_blkio_queued_ops = config_get_boolean_ondemand("plugin:cgroups", "enable blkio queued operations", cgroup_enable_blkio_queued_ops);
  90     cgroup_enable_blkio_merged_ops = config_get_boolean_ondemand("plugin:cgroups", "enable blkio merged operations", cgroup_enable_blkio_merged_ops);
  91
  92     cgroup_recheck_zero_blkio_every_iterations = (int)config_get_number("plugin:cgroups", "recheck zero blkio every iterations", cgroup_recheck_zero_blkio_every_iterations);
  93     cgroup_recheck_zero_mem_failcnt_every_iterations = (int)config_get_number("plugin:cgroups", "recheck zero memory failcnt every iterations", cgroup_recheck_zero_mem_failcnt_every_iterations);
  94     cgroup_recheck_zero_mem_detailed_every_iterations = (int)config_get_number("plugin:cgroups", "recheck zero detailed memory every iterations", cgroup_recheck_zero_mem_detailed_every_iterations);
  95
  96     cgroup_enable_systemd_services = config_get_boolean("plugin:cgroups", "enable systemd services", cgroup_enable_systemd_services);
  97     cgroup_enable_systemd_services_detailed_memory = config_get_boolean("plugin:cgroups", "enable systemd services detailed memory", cgroup_enable_systemd_services_detailed_memory);
  98     cgroup_used_memory_without_cache = config_get_boolean("plugin:cgroups", "report used memory without cache", cgroup_used_memory_without_cache);
  99
 100     char filename[FILENAME_MAX + 1], *s;
 101     struct mountinfo *mi, *root = mountinfo_read(0);
 102
 103     mi = mountinfo_find_by_filesystem_super_option(root, "cgroup", "cpuacct");
 104     if(!mi) mi = mountinfo_find_by_filesystem_mount_source(root, "cgroup", "cpuacct");
 105     if(!mi) {
 106         error("Cannot find cgroup cpuacct mountinfo. Assuming default: /sys/fs/cgroup/cpuacct");
 107         s = "/sys/fs/cgroup/cpuacct";
 108     }
 109     else s = mi->mount_point;
 110     snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, s);
 111     cgroup_cpuacct_base = config_get("plugin:cgroups", "path to /sys/fs/cgroup/cpuacct", filename);
 112
 113     mi = mountinfo_find_by_filesystem_super_option(root, "cgroup", "blkio");
 114     if(!mi) mi = mountinfo_find_by_filesystem_mount_source(root, "cgroup", "blkio");
 115     if(!mi) {
 116         error("Cannot find cgroup blkio mountinfo. Assuming default: /sys/fs/cgroup/blkio");
 117         s = "/sys/fs/cgroup/blkio";
 118     }
 119     else s = mi->mount_point;
 120     snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, s);
 121     cgroup_blkio_base = config_get("plugin:cgroups", "path to /sys/fs/cgroup/blkio", filename);
 122
 123     mi = mountinfo_find_by_filesystem_super_option(root, "cgroup", "memory");
 124     if(!mi) mi = mountinfo_find_by_filesystem_mount_source(root, "cgroup", "memory");
 125     if(!mi) {
 126         error("Cannot find cgroup memory mountinfo. Assuming default: /sys/fs/cgroup/memory");
 127         s = "/sys/fs/cgroup/memory";
 128     }
 129     else s = mi->mount_point;
 130     snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, s);
 131     cgroup_memory_base = config_get("plugin:cgroups", "path to /sys/fs/cgroup/memory", filename);
 132
 133     mi = mountinfo_find_by_filesystem_super_option(root, "cgroup", "devices");
 134     if(!mi) mi = mountinfo_find_by_filesystem_mount_source(root, "cgroup", "devices");
 135     if(!mi) {
 136         error("Cannot find cgroup devices mountinfo. Assuming default: /sys/fs/cgroup/devices");
 137         s = "/sys/fs/cgroup/devices";
 138     }
 139     else s = mi->mount_point;
 140     snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, s);
 141     cgroup_devices_base = config_get("plugin:cgroups", "path to /sys/fs/cgroup/devices", filename);
 142
 143     cgroup_root_max = (int)config_get_number("plugin:cgroups", "max cgroups to allow", cgroup_root_max);
 144     cgroup_max_depth = (int)config_get_number("plugin:cgroups", "max cgroups depth to monitor", cgroup_max_depth);
 145
 146     cgroup_enable_new_cgroups_detected_at_runtime = config_get_boolean("plugin:cgroups", "enable new cgroups detected at run time", cgroup_enable_new_cgroups_detected_at_runtime);
 147
 148     enabled_cgroup_patterns = simple_pattern_create(
 149             config_get("plugin:cgroups", "enable by default cgroups matching",
 150                     " /system.slice/docker-*.scope "
 151                     " /qemu.slice/*.scope "                // #1949
 152                     " !*.mount "
 153                     " !*.partition "
 154                     " !*.scope "
 155                     " !*.service "
 156                     " !*.slice "
 157                     " !*.swap "
 158                     " !*.user "
 159                     " !/ "
 160                     " !/docker "
 161                     " !/libvirt "
 162                     " !/lxc "
 163                     " !/lxc/*/ns "                         //  #1397
 164                     " !/machine "
 165                     " !/qemu "
 166                     " !/system "
 167                     " !/systemd "
 168                     " !/user "
 169                     " * "                                  // enable anything else
 170             ), SIMPLE_PATTERN_EXACT);
 171
 172     enabled_cgroup_paths = simple_pattern_create(
 173             config_get("plugin:cgroups", "search for cgroups in subpaths matching",
 174                     " !*-qemu "                           //  #345
 175                     " !/init.scope "
 176                     " !/system "
 177                     " !/systemd "
 178                     " !/user "
 179                     " !/user.slice "
 180                     " * "
 181             ), SIMPLE_PATTERN_EXACT);
 182
 183     snprintfz(filename, FILENAME_MAX, "%s/cgroup-name.sh", netdata_configured_plugins_dir);
 184     cgroups_rename_script = config_get("plugin:cgroups", "script to get cgroup names", filename);
 185
 186     enabled_cgroup_renames = simple_pattern_create(
 187             config_get("plugin:cgroups", "run script to rename cgroups matching",
 188                     " /qemu.slice/*.scope "                // #1949
 189                     " *docker* "
 190                     " *lxc* "
 191                     " !/ "
 192                     " !*.mount "
 193                     " !*.partition "
 194                     " !*.scope "
 195                     " !*.service "
 196                     " !*.slice "
 197                     " !*.swap "
 198                     " !*.user "
 199                     " * "
 200             ), SIMPLE_PATTERN_EXACT);
 201
 202     if(cgroup_enable_systemd_services) {
 203         systemd_services_cgroups = simple_pattern_create(
 204                 config_get("plugin:cgroups", "cgroups to match as systemd services",
 205                         " !/system.slice/*/*.service "
 206                         " /system.slice/*.service "
 207                 ), SIMPLE_PATTERN_EXACT);
 208     }
 209
 210     mountinfo_free(root);
 211 }
 212
 213 // ----------------------------------------------------------------------------
 214 // cgroup objects
 215
 216 struct blkio {
 217     int updated;
 218     int enabled; // CONFIG_BOOLEAN_YES or CONFIG_BOOLEAN_AUTO
 219     int delay_counter;
 220
 221     char *filename;
 222
 223     unsigned long long Read;
 224     unsigned long long Write;
 225 /*
 226     unsigned long long Sync;
 227     unsigned long long Async;
 228     unsigned long long Total;
 229 */
 230 };
 231
 232 // https://www.kernel.org/doc/Documentation/cgroup-v1/memory.txt
 233 struct memory {
 234     ARL_BASE *arl_base;
 235     ARL_ENTRY *arl_dirty;
 236     ARL_ENTRY *arl_swap;
 237
 238     int updated_detailed;
 239     int updated_usage_in_bytes;
 240     int updated_msw_usage_in_bytes;
 241     int updated_failcnt;
 242
 243     int enabled_detailed;           // CONFIG_BOOLEAN_YES or CONFIG_BOOLEAN_AUTO
 244     int enabled_usage_in_bytes;     // CONFIG_BOOLEAN_YES or CONFIG_BOOLEAN_AUTO
 245     int enabled_msw_usage_in_bytes; // CONFIG_BOOLEAN_YES or CONFIG_BOOLEAN_AUTO
 246     int enabled_failcnt;            // CONFIG_BOOLEAN_YES or CONFIG_BOOLEAN_AUTO
 247
 248     int delay_counter_detailed;
 249     int delay_counter_failcnt;
 250
 251     char *filename_detailed;
 252     char *filename_usage_in_bytes;
 253     char *filename_msw_usage_in_bytes;
 254     char *filename_failcnt;
 255
 256     int detailed_has_dirty;
 257     int detailed_has_swap;
 258
 259     // detailed metrics
 260     unsigned long long cache;
 261     unsigned long long rss;
 262     unsigned long long rss_huge;
 263     unsigned long long mapped_file;
 264     unsigned long long writeback;
 265     unsigned long long dirty;
 266     unsigned long long swap;
 267     unsigned long long pgpgin;
 268     unsigned long long pgpgout;
 269     unsigned long long pgfault;
 270     unsigned long long pgmajfault;
 271 /*
 272     unsigned long long inactive_anon;
 273     unsigned long long active_anon;
 274     unsigned long long inactive_file;
 275     unsigned long long active_file;
 276     unsigned long long unevictable;
 277     unsigned long long hierarchical_memory_limit;
 278     unsigned long long total_cache;
 279     unsigned long long total_rss;
 280     unsigned long long total_rss_huge;
 281     unsigned long long total_mapped_file;
 282     unsigned long long total_writeback;
 283     unsigned long long total_dirty;
 284     unsigned long long total_swap;
 285     unsigned long long total_pgpgin;
 286     unsigned long long total_pgpgout;
 287     unsigned long long total_pgfault;
 288     unsigned long long total_pgmajfault;
 289     unsigned long long total_inactive_anon;
 290     unsigned long long total_active_anon;
 291     unsigned long long total_inactive_file;
 292     unsigned long long total_active_file;
 293     unsigned long long total_unevictable;
 294 */
 295
 296     // single file metrics
 297     unsigned long long usage_in_bytes;
 298     unsigned long long msw_usage_in_bytes;
 299     unsigned long long failcnt;
 300 };
 301
 302 // https://www.kernel.org/doc/Documentation/cgroup-v1/cpuacct.txt
 303 struct cpuacct_stat {
 304     int updated;
 305     int enabled; // CONFIG_BOOLEAN_YES or CONFIG_BOOLEAN_AUTO
 306
 307     char *filename;
 308
 309     unsigned long long user;
 310     unsigned long long system;
 311 };
 312
 313 // https://www.kernel.org/doc/Documentation/cgroup-v1/cpuacct.txt
 314 struct cpuacct_usage {
 315     int updated;
 316     int enabled; // CONFIG_BOOLEAN_YES or CONFIG_BOOLEAN_AUTO
 317
 318     char *filename;
 319
 320     unsigned int cpus;
 321     unsigned long long *cpu_percpu;
 322 };
 323
 324 #define CGROUP_OPTIONS_DISABLED_DUPLICATE   0x00000001
 325 #define CGROUP_OPTIONS_SYSTEM_SLICE_SERVICE 0x00000002
 326
 327 struct cgroup {
 328     uint32_t options;
 329
 330     char available;      // found in the filesystem
 331     char enabled;        // enabled in the config
 332
 333     char *id;
 334     uint32_t hash;
 335
 336     char *chart_id;
 337     uint32_t hash_chart;
 338
 339     char *chart_title;
 340
 341     struct cpuacct_stat cpuacct_stat;
 342     struct cpuacct_usage cpuacct_usage;
 343
 344     struct memory memory;
 345
 346     struct blkio io_service_bytes;              // bytes
 347     struct blkio io_serviced;                   // operations
 348
 349     struct blkio throttle_io_service_bytes;     // bytes
 350     struct blkio throttle_io_serviced;          // operations
 351
 352     struct blkio io_merged;                     // operations
 353     struct blkio io_queued;                     // operations
 354
 355     // per cgroup charts
 356     RRDSET *st_cpu;
 357     RRDSET *st_cpu_per_core;
 358     RRDSET *st_mem;
 359     RRDSET *st_writeback;
 360     RRDSET *st_mem_activity;
 361     RRDSET *st_pgfaults;
 362     RRDSET *st_mem_usage;
 363     RRDSET *st_mem_failcnt;
 364     RRDSET *st_io;
 365     RRDSET *st_serviced_ops;
 366     RRDSET *st_throttle_io;
 367     RRDSET *st_throttle_serviced_ops;
 368     RRDSET *st_queued_ops;
 369     RRDSET *st_merged_ops;
 370
 371     // services
 372     RRDDIM *rd_cpu;
 373     RRDDIM *rd_mem_usage;
 374     RRDDIM *rd_mem_failcnt;
 375     RRDDIM *rd_swap_usage;
 376
 377     RRDDIM *rd_mem_detailed_cache;
 378     RRDDIM *rd_mem_detailed_rss;
 379     RRDDIM *rd_mem_detailed_mapped;
 380     RRDDIM *rd_mem_detailed_writeback;
 381     RRDDIM *rd_mem_detailed_pgpgin;
 382     RRDDIM *rd_mem_detailed_pgpgout;
 383     RRDDIM *rd_mem_detailed_pgfault;
 384     RRDDIM *rd_mem_detailed_pgmajfault;
 385
 386     RRDDIM *rd_io_service_bytes_read;
 387     RRDDIM *rd_io_serviced_read;
 388     RRDDIM *rd_throttle_io_read;
 389     RRDDIM *rd_throttle_io_serviced_read;
 390     RRDDIM *rd_io_queued_read;
 391     RRDDIM *rd_io_merged_read;
 392
 393     RRDDIM *rd_io_service_bytes_write;
 394     RRDDIM *rd_io_serviced_write;
 395     RRDDIM *rd_throttle_io_write;
 396     RRDDIM *rd_throttle_io_serviced_write;
 397     RRDDIM *rd_io_queued_write;
 398     RRDDIM *rd_io_merged_write;
 399
 400     struct cgroup *next;
 401
 402 } *cgroup_root = NULL;
 403
 404 // ----------------------------------------------------------------------------
 405 // read values from /sys
 406
 407 static inline void cgroup_read_cpuacct_stat(struct cpuacct_stat *cp) {
 408     static procfile *ff = NULL;
 409
 410     if(likely(cp->filename)) {
 411         ff = procfile_reopen(ff, cp->filename, NULL, PROCFILE_FLAG_DEFAULT);
 412         if(unlikely(!ff)) {
 413             cp->updated = 0;
 414             cgroups_check = 1;
 415             return;
 416         }
 417
 418         ff = procfile_readall(ff);
 419         if(unlikely(!ff)) {
 420             cp->updated = 0;
 421             cgroups_check = 1;
 422             return;
 423         }
 424
 425         unsigned long i, lines = procfile_lines(ff);
 426
 427         if(unlikely(lines < 1)) {
 428             error("File '%s' should have 1+ lines.", cp->filename);
 429             cp->updated = 0;
 430             return;
 431         }
 432
 433         for(i = 0; i < lines ; i++) {
 434             char *s = procfile_lineword(ff, i, 0);
 435             uint32_t hash = simple_hash(s);
 436
 437             if(unlikely(hash == user_hash && !strcmp(s, "user")))
 438                 cp->user = str2ull(procfile_lineword(ff, i, 1));
 439
 440             else if(unlikely(hash == system_hash && !strcmp(s, "system")))
 441                 cp->system = str2ull(procfile_lineword(ff, i, 1));
 442         }
 443
 444         cp->updated = 1;
 445
 446         if(unlikely(cp->enabled == CONFIG_BOOLEAN_AUTO && (cp->user || cp->system)))
 447             cp->enabled = CONFIG_BOOLEAN_YES;
 448     }
 449 }
 450
 451 static inline void cgroup_read_cpuacct_usage(struct cpuacct_usage *ca) {
 452     static procfile *ff = NULL;
 453
 454     if(likely(ca->filename)) {
 455         ff = procfile_reopen(ff, ca->filename, NULL, PROCFILE_FLAG_DEFAULT);
 456         if(unlikely(!ff)) {
 457             ca->updated = 0;
 458             cgroups_check = 1;
 459             return;
 460         }
 461
 462         ff = procfile_readall(ff);
 463         if(unlikely(!ff)) {
 464             ca->updated = 0;
 465             cgroups_check = 1;
 466             return;
 467         }
 468
 469         if(unlikely(procfile_lines(ff) < 1)) {
 470             error("File '%s' should have 1+ lines but has %zu.", ca->filename, procfile_lines(ff));
 471             ca->updated = 0;
 472             return;
 473         }
 474
 475         unsigned long i = procfile_linewords(ff, 0);
 476         if(unlikely(i == 0)) {
 477             ca->updated = 0;
 478             return;
 479         }
 480
 481         // we may have 1 more CPU reported
 482         while(i > 0) {
 483             char *s = procfile_lineword(ff, 0, i - 1);
 484             if(!*s) i--;
 485             else break;
 486         }
 487
 488         if(unlikely(i != ca->cpus)) {
 489             freez(ca->cpu_percpu);
 490             ca->cpu_percpu = mallocz(sizeof(unsigned long long) * i);
 491             ca->cpus = (unsigned int)i;
 492         }
 493
 494         unsigned long long total = 0;
 495         for(i = 0; i < ca->cpus ;i++) {
 496             unsigned long long n = str2ull(procfile_lineword(ff, 0, i));
 497             ca->cpu_percpu[i] = n;
 498             total += n;
 499         }
 500
 501         ca->updated = 1;
 502
 503         if(unlikely(ca->enabled == CONFIG_BOOLEAN_AUTO && total))
 504             ca->enabled = CONFIG_BOOLEAN_YES;
 505     }
 506 }
 507
 508 static inline void cgroup_read_blkio(struct blkio *io) {
 509     static procfile *ff = NULL;
 510
 511     if(unlikely(io->enabled == CONFIG_BOOLEAN_AUTO && io->delay_counter > 0)) {
 512         io->delay_counter--;
 513         return;
 514     }
 515
 516     if(likely(io->filename)) {
 517         ff = procfile_reopen(ff, io->filename, NULL, PROCFILE_FLAG_DEFAULT);
 518         if(unlikely(!ff)) {
 519             io->updated = 0;
 520             cgroups_check = 1;
 521             return;
 522         }
 523
 524         ff = procfile_readall(ff);
 525         if(unlikely(!ff)) {
 526             io->updated = 0;
 527             cgroups_check = 1;
 528             return;
 529         }
 530
 531         unsigned long i, lines = procfile_lines(ff);
 532
 533         if(unlikely(lines < 1)) {
 534             error("File '%s' should have 1+ lines.", io->filename);
 535             io->updated = 0;
 536             return;
 537         }
 538
 539         io->Read = 0;
 540         io->Write = 0;
 541 /*
 542         io->Sync = 0;
 543         io->Async = 0;
 544         io->Total = 0;
 545 */
 546
 547         for(i = 0; i < lines ; i++) {
 548             char *s = procfile_lineword(ff, i, 1);
 549             uint32_t hash = simple_hash(s);
 550
 551             if(unlikely(hash == Read_hash && !strcmp(s, "Read")))
 552                 io->Read += str2ull(procfile_lineword(ff, i, 2));
 553
 554             else if(unlikely(hash == Write_hash && !strcmp(s, "Write")))
 555                 io->Write += str2ull(procfile_lineword(ff, i, 2));
 556
 557 /*
 558             else if(unlikely(hash == Sync_hash && !strcmp(s, "Sync")))
 559                 io->Sync += str2ull(procfile_lineword(ff, i, 2));
 560
 561             else if(unlikely(hash == Async_hash && !strcmp(s, "Async")))
 562                 io->Async += str2ull(procfile_lineword(ff, i, 2));
 563
 564             else if(unlikely(hash == Total_hash && !strcmp(s, "Total")))
 565                 io->Total += str2ull(procfile_lineword(ff, i, 2));
 566 */
 567         }
 568
 569         io->updated = 1;
 570
 571         if(unlikely(io->enabled == CONFIG_BOOLEAN_AUTO)) {
 572             if(unlikely(io->Read || io->Write))
 573                 io->enabled = CONFIG_BOOLEAN_YES;
 574             else
 575                 io->delay_counter = cgroup_recheck_zero_blkio_every_iterations;
 576         }
 577     }
 578 }
 579
 580 static inline void cgroup_read_memory(struct memory *mem) {
 581     static procfile *ff = NULL;
 582
 583     // read detailed ram usage
 584     if(likely(mem->filename_detailed)) {
 585         if(unlikely(mem->enabled_detailed == CONFIG_BOOLEAN_AUTO && mem->delay_counter_detailed > 0)) {
 586             mem->delay_counter_detailed--;
 587             goto memory_next;
 588         }
 589
 590         ff = procfile_reopen(ff, mem->filename_detailed, NULL, PROCFILE_FLAG_DEFAULT);
 591         if(unlikely(!ff)) {
 592             mem->updated_detailed = 0;
 593             cgroups_check = 1;
 594             goto memory_next;
 595         }
 596
 597         ff = procfile_readall(ff);
 598         if(unlikely(!ff)) {
 599             mem->updated_detailed = 0;
 600             cgroups_check = 1;
 601             goto memory_next;
 602         }
 603
 604         unsigned long i, lines = procfile_lines(ff);
 605
 606         if(unlikely(lines < 1)) {
 607             error("File '%s' should have 1+ lines.", mem->filename_detailed);
 608             mem->updated_detailed = 0;
 609             goto memory_next;
 610         }
 611
 612         if(unlikely(!mem->arl_base)) {
 613             mem->arl_base = arl_create("cgroup/memory", NULL, 60);
 614
 615             arl_expect(mem->arl_base, "cache", &mem->cache);
 616             arl_expect(mem->arl_base, "rss", &mem->rss);
 617             arl_expect(mem->arl_base, "rss_huge", &mem->rss_huge);
 618             arl_expect(mem->arl_base, "mapped_file", &mem->mapped_file);
 619             arl_expect(mem->arl_base, "writeback", &mem->writeback);
 620             mem->arl_dirty = arl_expect(mem->arl_base, "dirty", &mem->dirty);
 621             mem->arl_swap  = arl_expect(mem->arl_base, "swap", &mem->swap);
 622             arl_expect(mem->arl_base, "pgpgin", &mem->pgpgin);
 623             arl_expect(mem->arl_base, "pgpgout", &mem->pgpgout);
 624             arl_expect(mem->arl_base, "pgfault", &mem->pgfault);
 625             arl_expect(mem->arl_base, "pgmajfault", &mem->pgmajfault);
 626         }
 627
 628         arl_begin(mem->arl_base);
 629
 630         for(i = 0; i < lines ; i++) {
 631             if(arl_check(mem->arl_base,
 632                     procfile_lineword(ff, i, 0),
 633                     procfile_lineword(ff, i, 1))) break;
 634         }
 635
 636         if(unlikely(mem->arl_dirty->flags & ARL_ENTRY_FLAG_FOUND))
 637             mem->detailed_has_dirty = 1;
 638
 639         if(unlikely(mem->arl_swap->flags & ARL_ENTRY_FLAG_FOUND))
 640             mem->detailed_has_swap = 1;
 641
 642         // fprintf(stderr, "READ: '%s', cache: %llu, rss: %llu, rss_huge: %llu, mapped_file: %llu, writeback: %llu, dirty: %llu, swap: %llu, pgpgin: %llu, pgpgout: %llu, pgfault: %llu, pgmajfault: %llu, inactive_anon: %llu, active_anon: %llu, inactive_file: %llu, active_file: %llu, unevictable: %llu, hierarchical_memory_limit: %llu, total_cache: %llu, total_rss: %llu, total_rss_huge: %llu, total_mapped_file: %llu, total_writeback: %llu, total_dirty: %llu, total_swap: %llu, total_pgpgin: %llu, total_pgpgout: %llu, total_pgfault: %llu, total_pgmajfault: %llu, total_inactive_anon: %llu, total_active_anon: %llu, total_inactive_file: %llu, total_active_file: %llu, total_unevictable: %llu\n", mem->filename, mem->cache, mem->rss, mem->rss_huge, mem->mapped_file, mem->writeback, mem->dirty, mem->swap, mem->pgpgin, mem->pgpgout, mem->pgfault, mem->pgmajfault, mem->inactive_anon, mem->active_anon, mem->inactive_file, mem->active_file, mem->unevictable, mem->hierarchical_memory_limit, mem->total_cache, mem->total_rss, mem->total_rss_huge, mem->total_mapped_file, mem->total_writeback, mem->total_dirty, mem->total_swap, mem->total_pgpgin, mem->total_pgpgout, mem->total_pgfault, mem->total_pgmajfault, mem->total_inactive_anon, mem->total_active_anon, mem->total_inactive_file, mem->total_active_file, mem->total_unevictable);
 643
 644         mem->updated_detailed = 1;
 645
 646         if(unlikely(mem->enabled_detailed == CONFIG_BOOLEAN_AUTO)) {
 647             if(mem->cache || mem->dirty || mem->rss || mem->rss_huge || mem->mapped_file || mem->writeback || mem->swap || mem->pgpgin || mem->pgpgout || mem->pgfault || mem->pgmajfault)
 648                 mem->enabled_detailed = CONFIG_BOOLEAN_YES;
 649             else
 650                 mem->delay_counter_detailed = cgroup_recheck_zero_mem_detailed_every_iterations;
 651         }
 652     }
 653
 654 memory_next:
 655
 656     // read usage_in_bytes
 657     if(likely(mem->filename_usage_in_bytes)) {
 658         mem->updated_usage_in_bytes = !read_single_number_file(mem->filename_usage_in_bytes, &mem->usage_in_bytes);
 659         if(unlikely(mem->updated_usage_in_bytes && mem->enabled_usage_in_bytes == CONFIG_BOOLEAN_AUTO && mem->usage_in_bytes))
 660             mem->enabled_usage_in_bytes = CONFIG_BOOLEAN_YES;
 661     }
 662
 663     // read msw_usage_in_bytes
 664     if(likely(mem->filename_msw_usage_in_bytes)) {
 665         mem->updated_msw_usage_in_bytes = !read_single_number_file(mem->filename_msw_usage_in_bytes, &mem->msw_usage_in_bytes);
 666         if(unlikely(mem->updated_msw_usage_in_bytes && mem->enabled_msw_usage_in_bytes == CONFIG_BOOLEAN_AUTO && mem->msw_usage_in_bytes))
 667             mem->enabled_msw_usage_in_bytes = CONFIG_BOOLEAN_YES;
 668     }
 669
 670     // read failcnt
 671     if(likely(mem->filename_failcnt)) {
 672         if(unlikely(mem->enabled_failcnt == CONFIG_BOOLEAN_AUTO && mem->delay_counter_failcnt > 0)) {
 673             mem->updated_failcnt = 0;
 674             mem->delay_counter_failcnt--;
 675         }
 676         else {
 677             mem->updated_failcnt = !read_single_number_file(mem->filename_failcnt, &mem->failcnt);
 678             if(unlikely(mem->updated_failcnt && mem->enabled_failcnt == CONFIG_BOOLEAN_AUTO)) {
 679                 if(unlikely(!mem->failcnt))
 680                     mem->delay_counter_failcnt = cgroup_recheck_zero_mem_failcnt_every_iterations;
 681                 else
 682                     mem->enabled_failcnt = CONFIG_BOOLEAN_YES;
 683             }
 684         }
 685     }
 686 }
 687
 688 static inline void cgroup_read(struct cgroup *cg) {
 689     debug(D_CGROUP, "reading metrics for cgroups '%s'", cg->id);
 690
 691     cgroup_read_cpuacct_stat(&cg->cpuacct_stat);
 692     cgroup_read_cpuacct_usage(&cg->cpuacct_usage);
 693     cgroup_read_memory(&cg->memory);
 694     cgroup_read_blkio(&cg->io_service_bytes);
 695     cgroup_read_blkio(&cg->io_serviced);
 696     cgroup_read_blkio(&cg->throttle_io_service_bytes);
 697     cgroup_read_blkio(&cg->throttle_io_serviced);
 698     cgroup_read_blkio(&cg->io_merged);
 699     cgroup_read_blkio(&cg->io_queued);
 700 }
 701
 702 static inline void read_all_cgroups(struct cgroup *root) {
 703     debug(D_CGROUP, "reading metrics for all cgroups");
 704
 705     struct cgroup *cg;
 706
 707     for(cg = root; cg ; cg = cg->next)
 708         if(cg->enabled && cg->available)
 709             cgroup_read(cg);
 710 }
 711
 712 // ----------------------------------------------------------------------------
 713 // add/remove/find cgroup objects
 714
 715 #define CGROUP_CHARTID_LINE_MAX 1024
 716
 717 static inline char *cgroup_title_strdupz(const char *s) {
 718     if(!s || !*s) s = "/";
 719
 720     if(*s == '/' && s[1] != '\0') s++;
 721
 722     char *r = strdupz(s);
 723     netdata_fix_chart_name(r);
 724
 725     return r;
 726 }
 727
 728 static inline char *cgroup_chart_id_strdupz(const char *s) {
 729     if(!s || !*s) s = "/";
 730
 731     if(*s == '/' && s[1] != '\0') s++;
 732
 733     char *r = strdupz(s);
 734     netdata_fix_chart_id(r);
 735
 736     return r;
 737 }
 738
 739 static inline void cgroup_get_chart_name(struct cgroup *cg) {
 740     debug(D_CGROUP, "looking for the name of cgroup '%s' with chart id '%s' and title '%s'", cg->id, cg->chart_id, cg->chart_title);
 741
 742     pid_t cgroup_pid;
 743     char buffer[CGROUP_CHARTID_LINE_MAX + 1];
 744
 745     snprintfz(buffer, CGROUP_CHARTID_LINE_MAX, "exec %s '%s'", cgroups_rename_script, cg->chart_id);
 746
 747     debug(D_CGROUP, "executing command '%s' for cgroup '%s'", buffer, cg->id);
 748     FILE *fp = mypopen(buffer, &cgroup_pid);
 749     if(fp) {
 750         // debug(D_CGROUP, "reading from command '%s' for cgroup '%s'", buffer, cg->id);
 751         char *s = fgets(buffer, CGROUP_CHARTID_LINE_MAX, fp);
 752         // debug(D_CGROUP, "closing command for cgroup '%s'", cg->id);
 753         mypclose(fp, cgroup_pid);
 754         // debug(D_CGROUP, "closed command for cgroup '%s'", cg->id);
 755
 756         if(s && *s && *s != '\n') {
 757             debug(D_CGROUP, "cgroup '%s' should be renamed to '%s'", cg->id, s);
 758
 759             trim(s);
 760
 761             freez(cg->chart_title);
 762             cg->chart_title = cgroup_title_strdupz(s);
 763
 764             freez(cg->chart_id);
 765             cg->chart_id = cgroup_chart_id_strdupz(s);
 766             cg->hash_chart = simple_hash(cg->chart_id);
 767         }
 768     }
 769     else
 770         error("CGROUP: Cannot popen(\"%s\", \"r\").", buffer);
 771 }
 772
 773 static inline struct cgroup *cgroup_add(const char *id) {
 774     if(!id || !*id) id = "/";
 775     debug(D_CGROUP, "adding to list, cgroup with id '%s'", id);
 776
 777     if(cgroup_root_count >= cgroup_root_max) {
 778         info("Maximum number of cgroups reached (%d). Not adding cgroup '%s'", cgroup_root_count, id);
 779         return NULL;
 780     }
 781
 782     int def = simple_pattern_matches(enabled_cgroup_patterns, id)?cgroup_enable_new_cgroups_detected_at_runtime:0;
 783     struct cgroup *cg = callocz(1, sizeof(struct cgroup));
 784
 785     cg->id = strdupz(id);
 786     cg->hash = simple_hash(cg->id);
 787
 788     cg->chart_title = cgroup_title_strdupz(id);
 789
 790     cg->chart_id = cgroup_chart_id_strdupz(id);
 791     cg->hash_chart = simple_hash(cg->chart_id);
 792
 793     if(!cgroup_root)
 794         cgroup_root = cg;
 795     else {
 796         // append it
 797         struct cgroup *e;
 798         for(e = cgroup_root; e->next ;e = e->next) ;
 799         e->next = cg;
 800     }
 801
 802     cgroup_root_count++;
 803
 804     // fix the chart_id and title by calling the external script
 805     if(simple_pattern_matches(enabled_cgroup_renames, cg->id)) {
 806
 807         cgroup_get_chart_name(cg);
 808
 809         debug(D_CGROUP, "cgroup '%s' renamed to '%s' (title: '%s')", cg->id, cg->chart_id, cg->chart_title);
 810     }
 811     else
 812         debug(D_CGROUP, "cgroup '%s' will not be renamed - it matches the list of disabled cgroup renames (will be shown as '%s')", cg->id, cg->chart_id);
 813
 814     int user_configurable = 1;
 815
 816     // check if this cgroup should be a systemd service
 817     if(cgroup_enable_systemd_services) {
 818         if(simple_pattern_matches(systemd_services_cgroups, cg->id) ||
 819                 simple_pattern_matches(systemd_services_cgroups, cg->chart_id)) {
 820             debug(D_CGROUP, "cgroup '%s' with chart id '%s' (title: '%s') matches systemd services cgroups", cg->id, cg->chart_id, cg->chart_title);
 821
 822             char buffer[CGROUP_CHARTID_LINE_MAX + 1];
 823             cg->options |= CGROUP_OPTIONS_SYSTEM_SLICE_SERVICE;
 824
 825             strncpy(buffer, cg->id, CGROUP_CHARTID_LINE_MAX);
 826             char *s = buffer;
 827
 828             //freez(cg->chart_id);
 829             //cg->chart_id = cgroup_chart_id_strdupz(s);
 830             //cg->hash_chart = simple_hash(cg->chart_id);
 831
 832             // skip to the last slash
 833             size_t len = strlen(s);
 834             while(len--) if(unlikely(s[len] == '/')) break;
 835             if(len) s = &s[len + 1];
 836
 837             // remove extension
 838             len = strlen(s);
 839             while(len--) if(unlikely(s[len] == '.')) break;
 840             if(len) s[len] = '\0';
 841
 842             freez(cg->chart_title);
 843             cg->chart_title = cgroup_title_strdupz(s);
 844
 845             cg->enabled = 1;
 846             user_configurable = 0;
 847
 848             debug(D_CGROUP, "cgroup '%s' renamed to '%s' (title: '%s')", cg->id, cg->chart_id, cg->chart_title);
 849         }
 850         else
 851             debug(D_CGROUP, "cgroup '%s' with chart id '%s' (title: '%s') does not match systemd services groups", cg->id, cg->chart_id, cg->chart_title);
 852     }
 853
 854     if(user_configurable) {
 855         // allow the user to enable/disable this individualy
 856         char option[FILENAME_MAX + 1];
 857         snprintfz(option, FILENAME_MAX, "enable cgroup %s", cg->chart_title);
 858         cg->enabled = (char) config_get_boolean("plugin:cgroups", option, def);
 859     }
 860
 861     // detect duplicate cgroups
 862     if(cg->enabled) {
 863         struct cgroup *t;
 864         for (t = cgroup_root; t; t = t->next) {
 865             if (t != cg && t->enabled && t->hash_chart == cg->hash_chart && !strcmp(t->chart_id, cg->chart_id)) {
 866                 if (!strncmp(t->chart_id, "/system.slice/", 14) && !strncmp(cg->chart_id, "/init.scope/system.slice/", 25)) {
 867                     error("Control group with chart id '%s' already exists with id '%s' and is enabled. Swapping them by enabling cgroup with id '%s' and disabling cgroup with id '%s'.",
 868                           cg->chart_id, t->id, cg->id, t->id);
 869                     debug(D_CGROUP, "Control group with chart id '%s' already exists with id '%s' and is enabled. Swapping them by enabling cgroup with id '%s' and disabling cgroup with id '%s'.",
 870                           cg->chart_id, t->id, cg->id, t->id);
 871                     t->enabled = 0;
 872                     t->options |= CGROUP_OPTIONS_DISABLED_DUPLICATE;
 873                 }
 874                 else {
 875                     error("Control group with chart id '%s' already exists with id '%s' and is enabled and available. Disabling cgroup with id '%s'.",
 876                           cg->chart_id, t->id, cg->id);
 877                     debug(D_CGROUP, "Control group with chart id '%s' already exists with id '%s' and is enabled and available. Disabling cgroup with id '%s'.",
 878                           cg->chart_id, t->id, cg->id);
 879                     cg->enabled = 0;
 880                     cg->options |= CGROUP_OPTIONS_DISABLED_DUPLICATE;
 881                 }
 882
 883                 break;
 884             }
 885         }
 886     }
 887
 888     debug(D_CGROUP, "ADDED CGROUP: '%s' with chart id '%s' and title '%s' as %s (default was %s)", cg->id, cg->chart_id, cg->chart_title, (cg->enabled)?"enabled":"disabled", (def)?"enabled":"disabled");
 889
 890     return cg;
 891 }
 892
 893 static inline void cgroup_free(struct cgroup *cg) {
 894     debug(D_CGROUP, "Removing cgroup '%s' with chart id '%s' (was %s and %s)", cg->id, cg->chart_id, (cg->enabled)?"enabled":"disabled", (cg->available)?"available":"not available");
 895
 896     if(cg->st_cpu)                   rrdset_flag_set(cg->st_cpu,                   RRDSET_FLAG_OBSOLETE);
 897     if(cg->st_cpu_per_core)          rrdset_flag_set(cg->st_cpu_per_core,          RRDSET_FLAG_OBSOLETE);
 898     if(cg->st_mem)                   rrdset_flag_set(cg->st_mem,                   RRDSET_FLAG_OBSOLETE);
 899     if(cg->st_writeback)             rrdset_flag_set(cg->st_writeback,             RRDSET_FLAG_OBSOLETE);
 900     if(cg->st_mem_activity)          rrdset_flag_set(cg->st_mem_activity,          RRDSET_FLAG_OBSOLETE);
 901     if(cg->st_pgfaults)              rrdset_flag_set(cg->st_pgfaults,              RRDSET_FLAG_OBSOLETE);
 902     if(cg->st_mem_usage)             rrdset_flag_set(cg->st_mem_usage,             RRDSET_FLAG_OBSOLETE);
 903     if(cg->st_mem_failcnt)           rrdset_flag_set(cg->st_mem_failcnt,           RRDSET_FLAG_OBSOLETE);
 904     if(cg->st_io)                    rrdset_flag_set(cg->st_io,                    RRDSET_FLAG_OBSOLETE);
 905     if(cg->st_serviced_ops)          rrdset_flag_set(cg->st_serviced_ops,          RRDSET_FLAG_OBSOLETE);
 906     if(cg->st_throttle_io)           rrdset_flag_set(cg->st_throttle_io,           RRDSET_FLAG_OBSOLETE);
 907     if(cg->st_throttle_serviced_ops) rrdset_flag_set(cg->st_throttle_serviced_ops, RRDSET_FLAG_OBSOLETE);
 908     if(cg->st_queued_ops)            rrdset_flag_set(cg->st_queued_ops,            RRDSET_FLAG_OBSOLETE);
 909     if(cg->st_merged_ops)            rrdset_flag_set(cg->st_merged_ops,            RRDSET_FLAG_OBSOLETE);
 910
 911     freez(cg->cpuacct_usage.cpu_percpu);
 912
 913     freez(cg->cpuacct_stat.filename);
 914     freez(cg->cpuacct_usage.filename);
 915
 916     arl_free(cg->memory.arl_base);
 917     freez(cg->memory.filename_detailed);
 918     freez(cg->memory.filename_failcnt);
 919     freez(cg->memory.filename_usage_in_bytes);
 920     freez(cg->memory.filename_msw_usage_in_bytes);
 921
 922     freez(cg->io_service_bytes.filename);
 923     freez(cg->io_serviced.filename);
 924
 925     freez(cg->throttle_io_service_bytes.filename);
 926     freez(cg->throttle_io_serviced.filename);
 927
 928     freez(cg->io_merged.filename);
 929     freez(cg->io_queued.filename);
 930
 931     freez(cg->id);
 932     freez(cg->chart_id);
 933     freez(cg->chart_title);
 934
 935     freez(cg);
 936
 937     cgroup_root_count--;
 938 }
 939
 940 // find if a given cgroup exists
 941 static inline struct cgroup *cgroup_find(const char *id) {
 942     debug(D_CGROUP, "searching for cgroup '%s'", id);
 943
 944     uint32_t hash = simple_hash(id);
 945
 946     struct cgroup *cg;
 947     for(cg = cgroup_root; cg ; cg = cg->next) {
 948         if(hash == cg->hash && strcmp(id, cg->id) == 0)
 949             break;
 950     }
 951
 952     debug(D_CGROUP, "cgroup '%s' %s in memory", id, (cg)?"found":"not found");
 953     return cg;
 954 }
 955
 956 // ----------------------------------------------------------------------------
 957 // detect running cgroups
 958
 959 // callback for find_file_in_subdirs()
 960 static inline void found_subdir_in_dir(const char *dir) {
 961     debug(D_CGROUP, "examining cgroup dir '%s'", dir);
 962
 963     struct cgroup *cg = cgroup_find(dir);
 964     if(!cg) {
 965         if(*dir && cgroup_max_depth > 0) {
 966             int depth = 0;
 967             const char *s;
 968
 969             for(s = dir; *s ;s++)
 970                 if(unlikely(*s == '/'))
 971                     depth++;
 972
 973             if(depth > cgroup_max_depth) {
 974                 info("cgroup '%s' is too deep (%d, while max is %d)", dir, depth, cgroup_max_depth);
 975                 return;
 976             }
 977         }
 978         // debug(D_CGROUP, "will add dir '%s' as cgroup", dir);
 979         cg = cgroup_add(dir);
 980     }
 981
 982     if(cg) cg->available = 1;
 983 }
 984
 985 static inline int find_dir_in_subdirs(const char *base, const char *this, void (*callback)(const char *)) {
 986     if(!this) this = base;
 987     debug(D_CGROUP, "searching for directories in '%s' (base '%s')", this?this:"", base);
 988
 989     size_t dirlen = strlen(this), baselen = strlen(base);
 990
 991     int ret = -1;
 992     int enabled = -1;
 993
 994     const char *relative_path = &this[baselen];
 995     if(!*relative_path) relative_path = "/";
 996
 997     DIR *dir = opendir(this);
 998     if(!dir) {
 999         error("Cannot read cgroups directory '%s'", base);
1000         return ret;
1001     }
1002     ret = 1;
1003
1004     callback(relative_path);
1005
1006     struct dirent *de = NULL;
1007     while((de = readdir(dir))) {
1008         if(de->d_type == DT_DIR
1009             && (
1010                 (de->d_name[0] == '.' && de->d_name[1] == '\0')
1011                 || (de->d_name[0] == '.' && de->d_name[1] == '.' && de->d_name[2] == '\0')
1012                 ))
1013             continue;
1014
1015         if(de->d_type == DT_DIR) {
1016             if(enabled == -1) {
1017                 const char *r = relative_path;
1018                 if(*r == '\0') r = "/";
1019
1020                 // do not decent in directories we are not interested
1021                 int def = simple_pattern_matches(enabled_cgroup_paths, r);
1022
1023                 // we check for this option here
1024                 // so that the config will not have settings
1025                 // for leaf directories
1026                 char option[FILENAME_MAX + 1];
1027                 snprintfz(option, FILENAME_MAX, "search for cgroups under %s", r);
1028                 option[FILENAME_MAX] = '\0';
1029                 enabled = config_get_boolean("plugin:cgroups", option, def);
1030             }
1031
1032             if(enabled) {
1033                 char *s = mallocz(dirlen + strlen(de->d_name) + 2);
1034                 strcpy(s, this);
1035                 strcat(s, "/");
1036                 strcat(s, de->d_name);
1037                 int ret2 = find_dir_in_subdirs(base, s, callback);
1038                 if(ret2 > 0) ret += ret2;
1039                 freez(s);
1040             }
1041         }
1042     }
1043
1044     closedir(dir);
1045     return ret;
1046 }
1047
1048 static inline void mark_all_cgroups_as_not_available() {
1049     debug(D_CGROUP, "marking all cgroups as not available");
1050
1051     struct cgroup *cg;
1052
1053     // mark all as not available
1054     for(cg = cgroup_root; cg ; cg = cg->next) {
1055         cg->available = 0;
1056     }
1057 }
1058
1059 static inline void cleanup_all_cgroups() {
1060     struct cgroup *cg = cgroup_root, *last = NULL;
1061
1062     for(; cg ;) {
1063         if(!cg->available) {
1064             // enable the first duplicate cgroup
1065             {
1066                 struct cgroup *t;
1067                 for(t = cgroup_root; t ; t = t->next) {
1068                     if(t != cg && t->available && !t->enabled && t->options & CGROUP_OPTIONS_DISABLED_DUPLICATE && t->hash_chart == cg->hash_chart && !strcmp(t->chart_id, cg->chart_id)) {
1069                         debug(D_CGROUP, "Enabling duplicate of cgroup '%s' with id '%s', because the original with id '%s' stopped.", t->chart_id, t->id, cg->id);
1070                         t->enabled = 1;
1071                         t->options &= ~CGROUP_OPTIONS_DISABLED_DUPLICATE;
1072                         break;
1073                     }
1074                 }
1075             }
1076
1077             if(!last)
1078                 cgroup_root = cg->next;
1079             else
1080                 last->next = cg->next;
1081
1082             cgroup_free(cg);
1083
1084             if(!last)
1085                 cg = cgroup_root;
1086             else
1087                 cg = last->next;
1088         }
1089         else {
1090             last = cg;
1091             cg = cg->next;
1092         }
1093     }
1094 }
1095
1096 static inline void find_all_cgroups() {
1097     debug(D_CGROUP, "searching for cgroups");
1098
1099     mark_all_cgroups_as_not_available();
1100
1101     if(cgroup_enable_cpuacct_stat || cgroup_enable_cpuacct_usage) {
1102         if(find_dir_in_subdirs(cgroup_cpuacct_base, NULL, found_subdir_in_dir) == -1) {
1103             cgroup_enable_cpuacct_stat =
1104             cgroup_enable_cpuacct_usage = CONFIG_BOOLEAN_NO;
1105             error("disabled CGROUP cpu statistics.");
1106         }
1107     }
1108
1109     if(cgroup_enable_blkio_io || cgroup_enable_blkio_ops || cgroup_enable_blkio_throttle_io || cgroup_enable_blkio_throttle_ops || cgroup_enable_blkio_merged_ops || cgroup_enable_blkio_queued_ops) {
1110         if(find_dir_in_subdirs(cgroup_blkio_base, NULL, found_subdir_in_dir) == -1) {
1111             cgroup_enable_blkio_io =
1112             cgroup_enable_blkio_ops =
1113             cgroup_enable_blkio_throttle_io =
1114             cgroup_enable_blkio_throttle_ops =
1115             cgroup_enable_blkio_merged_ops =
1116             cgroup_enable_blkio_queued_ops = CONFIG_BOOLEAN_NO;
1117             error("disabled CGROUP blkio statistics.");
1118         }
1119     }
1120
1121     if(cgroup_enable_memory || cgroup_enable_detailed_memory || cgroup_enable_swap || cgroup_enable_memory_failcnt) {
1122         if(find_dir_in_subdirs(cgroup_memory_base, NULL, found_subdir_in_dir) == -1) {
1123             cgroup_enable_memory =
1124             cgroup_enable_detailed_memory =
1125             cgroup_enable_swap =
1126             cgroup_enable_memory_failcnt = CONFIG_BOOLEAN_NO;
1127             error("disabled CGROUP memory statistics.");
1128         }
1129     }
1130
1131     if(cgroup_search_in_devices) {
1132         if(find_dir_in_subdirs(cgroup_devices_base, NULL, found_subdir_in_dir) == -1) {
1133             cgroup_search_in_devices = 0;
1134             error("disabled CGROUP devices statistics.");
1135         }
1136     }
1137
1138     // remove any non-existing cgroups
1139     cleanup_all_cgroups();
1140
1141     struct cgroup *cg;
1142     struct stat buf;
1143     for(cg = cgroup_root; cg ; cg = cg->next) {
1144         // fprintf(stderr, " >>> CGROUP '%s' (%u - %s) with name '%s'\n", cg->id, cg->hash, cg->available?"available":"stopped", cg->name);
1145
1146         if(unlikely(!cg->available))
1147             continue;
1148
1149         debug(D_CGROUP, "checking paths for cgroup '%s'", cg->id);
1150
1151         // check for newly added cgroups
1152         // and update the filenames they read
1153         char filename[FILENAME_MAX + 1];
1154         if(unlikely(cgroup_enable_cpuacct_stat && !cg->cpuacct_stat.filename)) {
1155             snprintfz(filename, FILENAME_MAX, "%s%s/cpuacct.stat", cgroup_cpuacct_base, cg->id);
1156             if(likely(stat(filename, &buf) != -1)) {
1157                 cg->cpuacct_stat.filename = strdupz(filename);
1158                 cg->cpuacct_stat.enabled = cgroup_enable_cpuacct_stat;
1159                 debug(D_CGROUP, "cpuacct.stat filename for cgroup '%s': '%s'", cg->id, cg->cpuacct_stat.filename);
1160             }
1161             else
1162                 debug(D_CGROUP, "cpuacct.stat file for cgroup '%s': '%s' does not exist.", cg->id, filename);
1163         }
1164
1165         if(unlikely(cgroup_enable_cpuacct_usage && !cg->cpuacct_usage.filename && !(cg->options & CGROUP_OPTIONS_SYSTEM_SLICE_SERVICE))) {
1166             snprintfz(filename, FILENAME_MAX, "%s%s/cpuacct.usage_percpu", cgroup_cpuacct_base, cg->id);
1167             if(likely(stat(filename, &buf) != -1)) {
1168                 cg->cpuacct_usage.filename = strdupz(filename);
1169                 cg->cpuacct_usage.enabled = cgroup_enable_cpuacct_usage;
1170                 debug(D_CGROUP, "cpuacct.usage_percpu filename for cgroup '%s': '%s'", cg->id, cg->cpuacct_usage.filename);
1171             }
1172             else
1173                 debug(D_CGROUP, "cpuacct.usage_percpu file for cgroup '%s': '%s' does not exist.", cg->id, filename);
1174         }
1175
1176         if(unlikely((cgroup_enable_detailed_memory || cgroup_used_memory_without_cache) && !cg->memory.filename_detailed && (cgroup_used_memory_without_cache || cgroup_enable_systemd_services_detailed_memory || !(cg->options & CGROUP_OPTIONS_SYSTEM_SLICE_SERVICE)))) {
1177             snprintfz(filename, FILENAME_MAX, "%s%s/memory.stat", cgroup_memory_base, cg->id);
1178             if(likely(stat(filename, &buf) != -1)) {
1179                 cg->memory.filename_detailed = strdupz(filename);
1180                 cg->memory.enabled_detailed = (cgroup_enable_detailed_memory == CONFIG_BOOLEAN_YES)?CONFIG_BOOLEAN_YES:CONFIG_BOOLEAN_AUTO;
1181                 debug(D_CGROUP, "memory.stat filename for cgroup '%s': '%s'", cg->id, cg->memory.filename_detailed);
1182             }
1183             else
1184                 debug(D_CGROUP, "memory.stat file for cgroup '%s': '%s' does not exist.", cg->id, filename);
1185         }
1186
1187         if(unlikely(cgroup_enable_memory && !cg->memory.filename_usage_in_bytes)) {
1188             snprintfz(filename, FILENAME_MAX, "%s%s/memory.usage_in_bytes", cgroup_memory_base, cg->id);
1189             if(likely(stat(filename, &buf) != -1)) {
1190                 cg->memory.filename_usage_in_bytes = strdupz(filename);
1191                 cg->memory.enabled_usage_in_bytes = cgroup_enable_memory;
1192                 debug(D_CGROUP, "memory.usage_in_bytes filename for cgroup '%s': '%s'", cg->id, cg->memory.filename_usage_in_bytes);
1193             }
1194             else
1195                 debug(D_CGROUP, "memory.usage_in_bytes file for cgroup '%s': '%s' does not exist.", cg->id, filename);
1196         }
1197
1198         if(unlikely(cgroup_enable_swap && !cg->memory.filename_msw_usage_in_bytes)) {
1199             snprintfz(filename, FILENAME_MAX, "%s%s/memory.msw_usage_in_bytes", cgroup_memory_base, cg->id);
1200             if(likely(stat(filename, &buf) != -1)) {
1201                 cg->memory.filename_msw_usage_in_bytes = strdupz(filename);
1202                 cg->memory.enabled_msw_usage_in_bytes = cgroup_enable_swap;
1203                 debug(D_CGROUP, "memory.msw_usage_in_bytes filename for cgroup '%s': '%s'", cg->id, cg->memory.filename_msw_usage_in_bytes);
1204             }
1205             else
1206                 debug(D_CGROUP, "memory.msw_usage_in_bytes file for cgroup '%s': '%s' does not exist.", cg->id, filename);
1207         }
1208
1209         if(unlikely(cgroup_enable_memory_failcnt && !cg->memory.filename_failcnt)) {
1210             snprintfz(filename, FILENAME_MAX, "%s%s/memory.failcnt", cgroup_memory_base, cg->id);
1211             if(likely(stat(filename, &buf) != -1)) {
1212                 cg->memory.filename_failcnt = strdupz(filename);
1213                 cg->memory.enabled_failcnt = cgroup_enable_memory_failcnt;
1214                 debug(D_CGROUP, "memory.failcnt filename for cgroup '%s': '%s'", cg->id, cg->memory.filename_failcnt);
1215             }
1216             else
1217                 debug(D_CGROUP, "memory.failcnt file for cgroup '%s': '%s' does not exist.", cg->id, filename);
1218         }
1219
1220         if(unlikely(cgroup_enable_blkio_io && !cg->io_service_bytes.filename)) {
1221             snprintfz(filename, FILENAME_MAX, "%s%s/blkio.io_service_bytes", cgroup_blkio_base, cg->id);
1222             if(likely(stat(filename, &buf) != -1)) {
1223                 cg->io_service_bytes.filename = strdupz(filename);
1224                 cg->io_service_bytes.enabled = cgroup_enable_blkio_io;
1225                 debug(D_CGROUP, "io_service_bytes filename for cgroup '%s': '%s'", cg->id, cg->io_service_bytes.filename);
1226             }
1227             else
1228                 debug(D_CGROUP, "io_service_bytes file for cgroup '%s': '%s' does not exist.", cg->id, filename);
1229         }
1230
1231         if(unlikely(cgroup_enable_blkio_ops && !cg->io_serviced.filename)) {
1232             snprintfz(filename, FILENAME_MAX, "%s%s/blkio.io_serviced", cgroup_blkio_base, cg->id);
1233             if(likely(stat(filename, &buf) != -1)) {
1234                 cg->io_serviced.filename = strdupz(filename);
1235                 cg->io_serviced.enabled = cgroup_enable_blkio_ops;
1236                 debug(D_CGROUP, "io_serviced filename for cgroup '%s': '%s'", cg->id, cg->io_serviced.filename);
1237             }
1238             else
1239                 debug(D_CGROUP, "io_serviced file for cgroup '%s': '%s' does not exist.", cg->id, filename);
1240         }
1241
1242         if(unlikely(cgroup_enable_blkio_throttle_io && !cg->throttle_io_service_bytes.filename)) {
1243             snprintfz(filename, FILENAME_MAX, "%s%s/blkio.throttle.io_service_bytes", cgroup_blkio_base, cg->id);
1244             if(likely(stat(filename, &buf) != -1)) {
1245                 cg->throttle_io_service_bytes.filename = strdupz(filename);
1246                 cg->throttle_io_service_bytes.enabled = cgroup_enable_blkio_throttle_io;
1247                 debug(D_CGROUP, "throttle_io_service_bytes filename for cgroup '%s': '%s'", cg->id, cg->throttle_io_service_bytes.filename);
1248             }
1249             else
1250                 debug(D_CGROUP, "throttle_io_service_bytes file for cgroup '%s': '%s' does not exist.", cg->id, filename);
1251         }
1252
1253         if(unlikely(cgroup_enable_blkio_throttle_ops && !cg->throttle_io_serviced.filename)) {
1254             snprintfz(filename, FILENAME_MAX, "%s%s/blkio.throttle.io_serviced", cgroup_blkio_base, cg->id);
1255             if(likely(stat(filename, &buf) != -1)) {
1256                 cg->throttle_io_serviced.filename = strdupz(filename);
1257                 cg->throttle_io_serviced.enabled = cgroup_enable_blkio_throttle_ops;
1258                 debug(D_CGROUP, "throttle_io_serviced filename for cgroup '%s': '%s'", cg->id, cg->throttle_io_serviced.filename);
1259             }
1260             else
1261                 debug(D_CGROUP, "throttle_io_serviced file for cgroup '%s': '%s' does not exist.", cg->id, filename);
1262         }
1263
1264         if(unlikely(cgroup_enable_blkio_merged_ops && !cg->io_merged.filename)) {
1265             snprintfz(filename, FILENAME_MAX, "%s%s/blkio.io_merged", cgroup_blkio_base, cg->id);
1266             if(likely(stat(filename, &buf) != -1)) {
1267                 cg->io_merged.filename = strdupz(filename);
1268                 cg->io_merged.enabled = cgroup_enable_blkio_merged_ops;
1269                 debug(D_CGROUP, "io_merged filename for cgroup '%s': '%s'", cg->id, cg->io_merged.filename);
1270             }
1271             else
1272                 debug(D_CGROUP, "io_merged file for cgroup '%s': '%s' does not exist.", cg->id, filename);
1273         }
1274
1275         if(unlikely(cgroup_enable_blkio_queued_ops && !cg->io_queued.filename)) {
1276             snprintfz(filename, FILENAME_MAX, "%s%s/blkio.io_queued", cgroup_blkio_base, cg->id);
1277             if(likely(stat(filename, &buf) != -1)) {
1278                 cg->io_queued.filename = strdupz(filename);
1279                 cg->io_queued.enabled = cgroup_enable_blkio_queued_ops;
1280                 debug(D_CGROUP, "io_queued filename for cgroup '%s': '%s'", cg->id, cg->io_queued.filename);
1281             }
1282             else
1283                 debug(D_CGROUP, "io_queued file for cgroup '%s': '%s' does not exist.", cg->id, filename);
1284         }
1285     }
1286
1287     debug(D_CGROUP, "done searching for cgroups");
1288     return;
1289 }
1290
1291 // ----------------------------------------------------------------------------
1292 // generate charts
1293
1294 #define CHART_TITLE_MAX 300
1295
1296 void update_systemd_services_charts(
1297           int update_every
1298         , int do_cpu
1299         , int do_mem_usage
1300         , int do_mem_detailed
1301         , int do_mem_failcnt
1302         , int do_swap_usage
1303         , int do_io
1304         , int do_io_ops
1305         , int do_throttle_io
1306         , int do_throttle_ops
1307         , int do_queued_ops
1308         , int do_merged_ops
1309 ) {
1310     static RRDSET
1311         *st_cpu = NULL,
1312         *st_mem_usage = NULL,
1313         *st_mem_failcnt = NULL,
1314         *st_swap_usage = NULL,
1315
1316         *st_mem_detailed_cache = NULL,
1317         *st_mem_detailed_rss = NULL,
1318         *st_mem_detailed_mapped = NULL,
1319         *st_mem_detailed_writeback = NULL,
1320         *st_mem_detailed_pgfault = NULL,
1321         *st_mem_detailed_pgmajfault = NULL,
1322         *st_mem_detailed_pgpgin = NULL,
1323         *st_mem_detailed_pgpgout = NULL,
1324
1325         *st_io_read = NULL,
1326         *st_io_serviced_read = NULL,
1327         *st_throttle_io_read = NULL,
1328         *st_throttle_ops_read = NULL,
1329         *st_queued_ops_read = NULL,
1330         *st_merged_ops_read = NULL,
1331
1332         *st_io_write = NULL,
1333         *st_io_serviced_write = NULL,
1334         *st_throttle_io_write = NULL,
1335         *st_throttle_ops_write = NULL,
1336         *st_queued_ops_write = NULL,
1337         *st_merged_ops_write = NULL;
1338
1339     // create the charts
1340
1341     if(likely(do_cpu)) {
1342         if(unlikely(!st_cpu)) {
1343             char title[CHART_TITLE_MAX + 1];
1344             snprintfz(title, CHART_TITLE_MAX, "Systemd Services CPU utilization (%d%% = %d core%s)", (processors * 100), processors, (processors > 1) ? "s" : "");
1345
1346             st_cpu = rrdset_create_localhost(
1347                     "services"
1348                     , "cpu"
1349                     , NULL
1350                     , "cpu"
1351                     , "services.cpu"
1352                     , title
1353                     , "%"
1354                     , CHART_PRIORITY_SYSTEMD_SERVICES
1355                     , update_every
1356                     , RRDSET_TYPE_STACKED
1357             );
1358
1359         }
1360         else
1361             rrdset_next(st_cpu);
1362     }
1363
1364     if(likely(do_mem_usage)) {
1365         if(unlikely(!st_mem_usage)) {
1366
1367             st_mem_usage = rrdset_create_localhost(
1368                     "services"
1369                     , "mem_usage"
1370                     , NULL
1371                     , "mem"
1372                     , "services.mem_usage"
1373                     , (cgroup_used_memory_without_cache) ? "Systemd Services Used Memory without Cache"
1374                                                          : "Systemd Services Used Memory"
1375                     , "MB"
1376                     , CHART_PRIORITY_SYSTEMD_SERVICES + 10
1377                     , update_every
1378                     , RRDSET_TYPE_STACKED
1379             );
1380
1381         }
1382         else
1383             rrdset_next(st_mem_usage);
1384     }
1385
1386     if(likely(do_mem_detailed)) {
1387         if(unlikely(!st_mem_detailed_rss)) {
1388
1389             st_mem_detailed_rss = rrdset_create_localhost(
1390                     "services"
1391                     , "mem_rss"
1392                     , NULL
1393                     , "mem"
1394                     , "services.mem_rss"
1395                     , "Systemd Services RSS Memory"
1396                     , "MB"
1397                     , CHART_PRIORITY_SYSTEMD_SERVICES + 20
1398                     , update_every
1399                     , RRDSET_TYPE_STACKED
1400             );
1401
1402         }
1403         else
1404             rrdset_next(st_mem_detailed_rss);
1405
1406         if(unlikely(!st_mem_detailed_mapped)) {
1407
1408             st_mem_detailed_mapped = rrdset_create_localhost(
1409                     "services"
1410                     , "mem_mapped"
1411                     , NULL
1412                     , "mem"
1413                     , "services.mem_mapped"
1414                     , "Systemd Services Mapped Memory"
1415                     , "MB"
1416                     , CHART_PRIORITY_SYSTEMD_SERVICES + 30
1417                     , update_every
1418                     , RRDSET_TYPE_STACKED
1419             );
1420
1421         }
1422         else
1423             rrdset_next(st_mem_detailed_mapped);
1424
1425         if(unlikely(!st_mem_detailed_cache)) {
1426
1427             st_mem_detailed_cache = rrdset_create_localhost(
1428                     "services"
1429                     , "mem_cache"
1430                     , NULL
1431                     , "mem"
1432                     , "services.mem_cache"
1433                     , "Systemd Services Cache Memory"
1434                     , "MB"
1435                     , CHART_PRIORITY_SYSTEMD_SERVICES + 40
1436                     , update_every
1437                     , RRDSET_TYPE_STACKED
1438             );
1439
1440         }
1441         else
1442             rrdset_next(st_mem_detailed_cache);
1443
1444         if(unlikely(!st_mem_detailed_writeback)) {
1445
1446             st_mem_detailed_writeback = rrdset_create_localhost(
1447                     "services"
1448                     , "mem_writeback"
1449                     , NULL
1450                     , "mem"
1451                     , "services.mem_writeback"
1452                     , "Systemd Services Writeback Memory"
1453                     , "MB"
1454                     , CHART_PRIORITY_SYSTEMD_SERVICES + 50
1455                     , update_every
1456                     , RRDSET_TYPE_STACKED
1457             );
1458
1459         }
1460         else
1461             rrdset_next(st_mem_detailed_writeback);
1462
1463         if(unlikely(!st_mem_detailed_pgfault)) {
1464
1465             st_mem_detailed_pgfault = rrdset_create_localhost(
1466                     "services"
1467                     , "mem_pgfault"
1468                     , NULL
1469                     , "mem"
1470                     , "services.mem_pgfault"
1471                     , "Systemd Services Memory Minor Page Faults"
1472                     , "MB/s"
1473                     , CHART_PRIORITY_SYSTEMD_SERVICES + 60
1474                     , update_every
1475                     , RRDSET_TYPE_STACKED
1476             );
1477         }
1478         else
1479             rrdset_next(st_mem_detailed_pgfault);
1480
1481         if(unlikely(!st_mem_detailed_pgmajfault)) {
1482
1483             st_mem_detailed_pgmajfault = rrdset_create_localhost(
1484                     "services"
1485                     , "mem_pgmajfault"
1486                     , NULL
1487                     , "mem"
1488                     , "services.mem_pgmajfault"
1489                     , "Systemd Services Memory Major Page Faults"
1490                     , "MB/s"
1491                     , CHART_PRIORITY_SYSTEMD_SERVICES + 70
1492                     , update_every
1493                     , RRDSET_TYPE_STACKED
1494             );
1495
1496         }
1497         else
1498             rrdset_next(st_mem_detailed_pgmajfault);
1499
1500         if(unlikely(!st_mem_detailed_pgpgin)) {
1501
1502             st_mem_detailed_pgpgin = rrdset_create_localhost(
1503                     "services"
1504                     , "mem_pgpgin"
1505                     , NULL
1506                     , "mem"
1507                     , "services.mem_pgpgin"
1508                     , "Systemd Services Memory Charging Activity"
1509                     , "MB/s"
1510                     , CHART_PRIORITY_SYSTEMD_SERVICES + 80
1511                     , update_every
1512                     , RRDSET_TYPE_STACKED
1513             );
1514
1515         }
1516         else
1517             rrdset_next(st_mem_detailed_pgpgin);
1518
1519         if(unlikely(!st_mem_detailed_pgpgout)) {
1520
1521             st_mem_detailed_pgpgout = rrdset_create_localhost(
1522                     "services"
1523                     , "mem_pgpgout"
1524                     , NULL
1525                     , "mem"
1526                     , "services.mem_pgpgout"
1527                     , "Systemd Services Memory Uncharging Activity"
1528                     , "MB/s"
1529                     , CHART_PRIORITY_SYSTEMD_SERVICES + 90
1530                     , update_every
1531                     , RRDSET_TYPE_STACKED
1532             );
1533
1534         }
1535         else
1536             rrdset_next(st_mem_detailed_pgpgout);
1537     }
1538
1539     if(likely(do_mem_failcnt)) {
1540         if(unlikely(!st_mem_failcnt)) {
1541
1542             st_mem_failcnt = rrdset_create_localhost(
1543                     "services"
1544                     , "mem_failcnt"
1545                     , NULL
1546                     , "mem"
1547                     , "services.mem_failcnt"
1548                     , "Systemd Services Memory Limit Failures"
1549                     , "MB"
1550                     , CHART_PRIORITY_SYSTEMD_SERVICES + 110
1551                     , update_every
1552                     , RRDSET_TYPE_STACKED
1553             );
1554
1555         }
1556         else
1557             rrdset_next(st_mem_failcnt);
1558     }
1559
1560     if(likely(do_swap_usage)) {
1561         if(unlikely(!st_swap_usage)) {
1562
1563             st_swap_usage = rrdset_create_localhost(
1564                     "services"
1565                     , "swap_usage"
1566                     , NULL
1567                     , "swap"
1568                     , "services.swap_usage"
1569                     , "Systemd Services Swap Memory Used"
1570                     , "MB"
1571                     , CHART_PRIORITY_SYSTEMD_SERVICES + 100
1572                     , update_every
1573                     , RRDSET_TYPE_STACKED
1574             );
1575
1576         }
1577         else
1578             rrdset_next(st_swap_usage);
1579     }
1580
1581     if(likely(do_io)) {
1582         if(unlikely(!st_io_read)) {
1583
1584             st_io_read = rrdset_create_localhost(
1585                     "services"
1586                     , "io_read"
1587                     , NULL
1588                     , "disk"
1589                     , "services.io_read"
1590                     , "Systemd Services Disk Read Bandwidth"
1591                     , "KB/s"
1592                     , CHART_PRIORITY_SYSTEMD_SERVICES + 120
1593                     , update_every
1594                     , RRDSET_TYPE_STACKED
1595             );
1596
1597         }
1598         else
1599             rrdset_next(st_io_read);
1600
1601         if(unlikely(!st_io_write)) {
1602
1603             st_io_write = rrdset_create_localhost(
1604                     "services"
1605                     , "io_write"
1606                     , NULL
1607                     , "disk"
1608                     , "services.io_write"
1609                     , "Systemd Services Disk Write Bandwidth"
1610                     , "KB/s"
1611                     , CHART_PRIORITY_SYSTEMD_SERVICES + 130
1612                     , update_every
1613                     , RRDSET_TYPE_STACKED
1614             );
1615
1616         }
1617         else
1618             rrdset_next(st_io_write);
1619     }
1620
1621     if(likely(do_io_ops)) {
1622         if(unlikely(!st_io_serviced_read)) {
1623
1624             st_io_serviced_read = rrdset_create_localhost(
1625                     "services"
1626                     , "io_ops_read"
1627                     , NULL
1628                     , "disk"
1629                     , "services.io_ops_read"
1630                     , "Systemd Services Disk Read Operations"
1631                     , "operations/s"
1632                     , CHART_PRIORITY_SYSTEMD_SERVICES + 140
1633                     , update_every
1634                     , RRDSET_TYPE_STACKED
1635             );
1636
1637         }
1638         else
1639             rrdset_next(st_io_serviced_read);
1640
1641         if(unlikely(!st_io_serviced_write)) {
1642
1643             st_io_serviced_write = rrdset_create_localhost(
1644                     "services"
1645                     , "io_ops_write"
1646                     , NULL
1647                     , "disk"
1648                     , "services.io_ops_write"
1649                     , "Systemd Services Disk Write Operations"
1650                     , "operations/s"
1651                     , CHART_PRIORITY_SYSTEMD_SERVICES + 150
1652                     , update_every
1653                     , RRDSET_TYPE_STACKED
1654             );
1655
1656         }
1657         else
1658             rrdset_next(st_io_serviced_write);
1659     }
1660
1661     if(likely(do_throttle_io)) {
1662         if(unlikely(!st_throttle_io_read)) {
1663
1664             st_throttle_io_read = rrdset_create_localhost(
1665                     "services"
1666                     , "throttle_io_read"
1667                     , NULL
1668                     , "disk"
1669                     , "services.throttle_io_read"
1670                     , "Systemd Services Throttle Disk Read Bandwidth"
1671                     , "KB/s"
1672                     , CHART_PRIORITY_SYSTEMD_SERVICES + 160
1673                     , update_every
1674                     , RRDSET_TYPE_STACKED
1675             );
1676
1677         }
1678         else
1679             rrdset_next(st_throttle_io_read);
1680
1681         if(unlikely(!st_throttle_io_write)) {
1682
1683             st_throttle_io_write = rrdset_create_localhost(
1684                     "services"
1685                     , "throttle_io_write"
1686                     , NULL
1687                     , "disk"
1688                     , "services.throttle_io_write"
1689                     , "Systemd Services Throttle Disk Write Bandwidth"
1690                     , "KB/s"
1691                     , CHART_PRIORITY_SYSTEMD_SERVICES + 170
1692                     , update_every
1693                     , RRDSET_TYPE_STACKED
1694             );
1695
1696         }
1697         else
1698             rrdset_next(st_throttle_io_write);
1699     }
1700
1701     if(likely(do_throttle_ops)) {
1702         if(unlikely(!st_throttle_ops_read)) {
1703
1704             st_throttle_ops_read = rrdset_create_localhost(
1705                     "services"
1706                     , "throttle_io_ops_read"
1707                     , NULL
1708                     , "disk"
1709                     , "services.throttle_io_ops_read"
1710                     , "Systemd Services Throttle Disk Read Operations"
1711                     , "operations/s"
1712                     , CHART_PRIORITY_SYSTEMD_SERVICES + 180
1713                     , update_every
1714                     , RRDSET_TYPE_STACKED
1715             );
1716
1717         }
1718         else
1719             rrdset_next(st_throttle_ops_read);
1720
1721         if(unlikely(!st_throttle_ops_write)) {
1722
1723             st_throttle_ops_write = rrdset_create_localhost(
1724                     "services"
1725                     , "throttle_io_ops_write"
1726                     , NULL
1727                     , "disk"
1728                     , "services.throttle_io_ops_write"
1729                     , "Systemd Services Throttle Disk Write Operations"
1730                     , "operations/s"
1731                     , CHART_PRIORITY_SYSTEMD_SERVICES + 190
1732                     , update_every
1733                     , RRDSET_TYPE_STACKED
1734             );
1735
1736         }
1737         else
1738             rrdset_next(st_throttle_ops_write);
1739     }
1740
1741     if(likely(do_queued_ops)) {
1742         if(unlikely(!st_queued_ops_read)) {
1743
1744             st_queued_ops_read = rrdset_create_localhost(
1745                     "services"
1746                     , "queued_io_ops_read"
1747                     , NULL
1748                     , "disk"
1749                     , "services.queued_io_ops_read"
1750                     , "Systemd Services Queued Disk Read Operations"
1751                     , "operations/s"
1752                     , CHART_PRIORITY_SYSTEMD_SERVICES + 200
1753                     , update_every
1754                     , RRDSET_TYPE_STACKED
1755             );
1756
1757         }
1758         else
1759             rrdset_next(st_queued_ops_read);
1760
1761         if(unlikely(!st_queued_ops_write)) {
1762
1763             st_queued_ops_write = rrdset_create_localhost(
1764                     "services"
1765                     , "queued_io_ops_write"
1766                     , NULL
1767                     , "disk"
1768                     , "services.queued_io_ops_write"
1769                     , "Systemd Services Queued Disk Write Operations"
1770                     , "operations/s"
1771                     , CHART_PRIORITY_SYSTEMD_SERVICES + 210
1772                     , update_every
1773                     , RRDSET_TYPE_STACKED
1774             );
1775
1776         }
1777         else
1778             rrdset_next(st_queued_ops_write);
1779     }
1780
1781     if(likely(do_merged_ops)) {
1782         if(unlikely(!st_merged_ops_read)) {
1783
1784             st_merged_ops_read = rrdset_create_localhost(
1785                     "services"
1786                     , "merged_io_ops_read"
1787                     , NULL
1788                     , "disk"
1789                     , "services.merged_io_ops_read"
1790                     , "Systemd Services Merged Disk Read Operations"
1791                     , "operations/s"
1792                     , CHART_PRIORITY_SYSTEMD_SERVICES + 220
1793                     , update_every
1794                     , RRDSET_TYPE_STACKED
1795             );
1796
1797         }
1798         else
1799             rrdset_next(st_merged_ops_read);
1800
1801         if(unlikely(!st_merged_ops_write)) {
1802
1803             st_merged_ops_write = rrdset_create_localhost(
1804                     "services"
1805                     , "merged_io_ops_write"
1806                     , NULL
1807                     , "disk"
1808                     , "services.merged_io_ops_write"
1809                     , "Systemd Services Merged Disk Write Operations"
1810                     , "operations/s"
1811                     , CHART_PRIORITY_SYSTEMD_SERVICES + 230
1812                     , update_every
1813                     , RRDSET_TYPE_STACKED
1814             );
1815
1816         }
1817         else
1818             rrdset_next(st_merged_ops_write);
1819     }
1820
1821     // update the values
1822     struct cgroup *cg;
1823     for(cg = cgroup_root; cg ; cg = cg->next) {
1824         if(unlikely(!cg->available || !cg->enabled || !(cg->options & CGROUP_OPTIONS_SYSTEM_SLICE_SERVICE)))
1825             continue;
1826
1827         if(likely(do_cpu && cg->cpuacct_stat.updated)) {
1828             if(unlikely(!cg->rd_cpu))
1829                 cg->rd_cpu = rrddim_add(st_cpu, cg->chart_id, cg->chart_title, 100, hz, RRD_ALGORITHM_INCREMENTAL);
1830
1831             rrddim_set_by_pointer(st_cpu, cg->rd_cpu, cg->cpuacct_stat.user + cg->cpuacct_stat.system);
1832         }
1833
1834         if(likely(do_mem_usage && cg->memory.updated_usage_in_bytes)) {
1835             if(unlikely(!cg->rd_mem_usage))
1836                 cg->rd_mem_usage = rrddim_add(st_mem_usage, cg->chart_id, cg->chart_title, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
1837
1838             rrddim_set_by_pointer(st_mem_usage, cg->rd_mem_usage, cg->memory.usage_in_bytes - ((cgroup_used_memory_without_cache)?cg->memory.cache:0));
1839         }
1840
1841         if(likely(do_mem_detailed && cg->memory.updated_detailed)) {
1842             if(unlikely(!cg->rd_mem_detailed_rss))
1843                 cg->rd_mem_detailed_rss = rrddim_add(st_mem_detailed_rss, cg->chart_id, cg->chart_title, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
1844
1845             rrddim_set_by_pointer(st_mem_detailed_rss, cg->rd_mem_detailed_rss, cg->memory.rss + cg->memory.rss_huge);
1846
1847             if(unlikely(!cg->rd_mem_detailed_mapped))
1848                 cg->rd_mem_detailed_mapped = rrddim_add(st_mem_detailed_mapped, cg->chart_id, cg->chart_title, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
1849
1850             rrddim_set_by_pointer(st_mem_detailed_mapped, cg->rd_mem_detailed_mapped, cg->memory.mapped_file);
1851
1852             if(unlikely(!cg->rd_mem_detailed_cache))
1853                 cg->rd_mem_detailed_cache = rrddim_add(st_mem_detailed_cache, cg->chart_id, cg->chart_title, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
1854
1855             rrddim_set_by_pointer(st_mem_detailed_cache, cg->rd_mem_detailed_cache, cg->memory.cache);
1856
1857             if(unlikely(!cg->rd_mem_detailed_writeback))
1858                 cg->rd_mem_detailed_writeback = rrddim_add(st_mem_detailed_writeback, cg->chart_id, cg->chart_title, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
1859
1860             rrddim_set_by_pointer(st_mem_detailed_writeback, cg->rd_mem_detailed_writeback, cg->memory.writeback);
1861
1862             if(unlikely(!cg->rd_mem_detailed_pgfault))
1863                 cg->rd_mem_detailed_pgfault = rrddim_add(st_mem_detailed_pgfault, cg->chart_id, cg->chart_title, system_page_size, 1024 * 1024, RRD_ALGORITHM_INCREMENTAL);
1864
1865             rrddim_set_by_pointer(st_mem_detailed_pgfault, cg->rd_mem_detailed_pgfault, cg->memory.pgfault);
1866
1867             if(unlikely(!cg->rd_mem_detailed_pgmajfault))
1868                 cg->rd_mem_detailed_pgmajfault = rrddim_add(st_mem_detailed_pgmajfault, cg->chart_id, cg->chart_title, system_page_size, 1024 * 1024, RRD_ALGORITHM_INCREMENTAL);
1869
1870             rrddim_set_by_pointer(st_mem_detailed_pgmajfault, cg->rd_mem_detailed_pgmajfault, cg->memory.pgmajfault);
1871
1872             if(unlikely(!cg->rd_mem_detailed_pgpgin))
1873                 cg->rd_mem_detailed_pgpgin = rrddim_add(st_mem_detailed_pgpgin, cg->chart_id, cg->chart_title, system_page_size, 1024 * 1024, RRD_ALGORITHM_INCREMENTAL);
1874
1875             rrddim_set_by_pointer(st_mem_detailed_pgpgin, cg->rd_mem_detailed_pgpgin, cg->memory.pgpgin);
1876
1877             if(unlikely(!cg->rd_mem_detailed_pgpgout))
1878                 cg->rd_mem_detailed_pgpgout = rrddim_add(st_mem_detailed_pgpgout, cg->chart_id, cg->chart_title, system_page_size, 1024 * 1024, RRD_ALGORITHM_INCREMENTAL);
1879
1880             rrddim_set_by_pointer(st_mem_detailed_pgpgout, cg->rd_mem_detailed_pgpgout, cg->memory.pgpgout);
1881         }
1882
1883         if(likely(do_mem_failcnt && cg->memory.updated_failcnt)) {
1884             if(unlikely(!cg->rd_mem_failcnt))
1885                 cg->rd_mem_failcnt = rrddim_add(st_mem_failcnt, cg->chart_id, cg->chart_title, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1886
1887             rrddim_set_by_pointer(st_mem_failcnt, cg->rd_mem_failcnt, cg->memory.failcnt);
1888         }
1889
1890         if(likely(do_swap_usage && cg->memory.updated_msw_usage_in_bytes)) {
1891             if(unlikely(!cg->rd_swap_usage))
1892                 cg->rd_swap_usage = rrddim_add(st_swap_usage, cg->chart_id, cg->chart_title, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
1893
1894             rrddim_set_by_pointer(st_swap_usage, cg->rd_swap_usage, cg->memory.msw_usage_in_bytes);
1895         }
1896
1897         if(likely(do_io && cg->io_service_bytes.updated)) {
1898             if(unlikely(!cg->rd_io_service_bytes_read))
1899                 cg->rd_io_service_bytes_read = rrddim_add(st_io_read, cg->chart_id, cg->chart_title, 1, 1024, RRD_ALGORITHM_INCREMENTAL);
1900
1901             rrddim_set_by_pointer(st_io_read, cg->rd_io_service_bytes_read, cg->io_service_bytes.Read);
1902
1903             if(unlikely(!cg->rd_io_service_bytes_write))
1904                 cg->rd_io_service_bytes_write = rrddim_add(st_io_write, cg->chart_id, cg->chart_title, 1, 1024, RRD_ALGORITHM_INCREMENTAL);
1905
1906             rrddim_set_by_pointer(st_io_write, cg->rd_io_service_bytes_write, cg->io_service_bytes.Write);
1907         }
1908
1909         if(likely(do_io_ops && cg->io_serviced.updated)) {
1910             if(unlikely(!cg->rd_io_serviced_read))
1911                 cg->rd_io_serviced_read = rrddim_add(st_io_serviced_read, cg->chart_id, cg->chart_title, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1912
1913             rrddim_set_by_pointer(st_io_serviced_read, cg->rd_io_serviced_read, cg->io_serviced.Read);
1914
1915             if(unlikely(!cg->rd_io_serviced_write))
1916                 cg->rd_io_serviced_write = rrddim_add(st_io_serviced_write, cg->chart_id, cg->chart_title, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1917
1918             rrddim_set_by_pointer(st_io_serviced_write, cg->rd_io_serviced_write, cg->io_serviced.Write);
1919         }
1920
1921         if(likely(do_throttle_io && cg->throttle_io_service_bytes.updated)) {
1922             if(unlikely(!cg->rd_throttle_io_read))
1923                 cg->rd_throttle_io_read = rrddim_add(st_throttle_io_read, cg->chart_id, cg->chart_title, 1, 1024, RRD_ALGORITHM_INCREMENTAL);
1924
1925             rrddim_set_by_pointer(st_throttle_io_read, cg->rd_throttle_io_read, cg->throttle_io_service_bytes.Read);
1926
1927             if(unlikely(!cg->rd_throttle_io_write))
1928                 cg->rd_throttle_io_write = rrddim_add(st_throttle_io_write, cg->chart_id, cg->chart_title, 1, 1024, RRD_ALGORITHM_INCREMENTAL);
1929
1930             rrddim_set_by_pointer(st_throttle_io_write, cg->rd_throttle_io_write, cg->throttle_io_service_bytes.Write);
1931         }
1932
1933         if(likely(do_throttle_ops && cg->throttle_io_serviced.updated)) {
1934             if(unlikely(!cg->rd_throttle_io_serviced_read))
1935                 cg->rd_throttle_io_serviced_read = rrddim_add(st_throttle_ops_read, cg->chart_id, cg->chart_title, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1936
1937             rrddim_set_by_pointer(st_throttle_ops_read, cg->rd_throttle_io_serviced_read, cg->throttle_io_serviced.Read);
1938
1939             if(unlikely(!cg->rd_throttle_io_serviced_write))
1940                 cg->rd_throttle_io_serviced_write = rrddim_add(st_throttle_ops_write, cg->chart_id, cg->chart_title, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1941
1942             rrddim_set_by_pointer(st_throttle_ops_write, cg->rd_throttle_io_serviced_write, cg->throttle_io_serviced.Write);
1943         }
1944
1945         if(likely(do_queued_ops && cg->io_queued.updated)) {
1946             if(unlikely(!cg->rd_io_queued_read))
1947                 cg->rd_io_queued_read = rrddim_add(st_queued_ops_read, cg->chart_id, cg->chart_title, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1948
1949             rrddim_set_by_pointer(st_queued_ops_read, cg->rd_io_queued_read, cg->io_queued.Read);
1950
1951             if(unlikely(!cg->rd_io_queued_write))
1952                 cg->rd_io_queued_write = rrddim_add(st_queued_ops_write, cg->chart_id, cg->chart_title, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1953
1954             rrddim_set_by_pointer(st_queued_ops_write, cg->rd_io_queued_write, cg->io_queued.Write);
1955         }
1956
1957         if(likely(do_merged_ops && cg->io_merged.updated)) {
1958             if(unlikely(!cg->rd_io_merged_read))
1959                 cg->rd_io_merged_read = rrddim_add(st_merged_ops_read, cg->chart_id, cg->chart_title, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1960
1961             rrddim_set_by_pointer(st_merged_ops_read, cg->rd_io_merged_read, cg->io_merged.Read);
1962
1963             if(unlikely(!cg->rd_io_merged_write))
1964                 cg->rd_io_merged_write = rrddim_add(st_merged_ops_write, cg->chart_id, cg->chart_title, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1965
1966             rrddim_set_by_pointer(st_merged_ops_write, cg->rd_io_merged_write, cg->io_merged.Write);
1967         }
1968     }
1969
1970     // complete the iteration
1971     if(likely(do_cpu))
1972         rrdset_done(st_cpu);
1973
1974     if(likely(do_mem_usage))
1975         rrdset_done(st_mem_usage);
1976
1977     if(unlikely(do_mem_detailed)) {
1978         rrdset_done(st_mem_detailed_cache);
1979         rrdset_done(st_mem_detailed_rss);
1980         rrdset_done(st_mem_detailed_mapped);
1981         rrdset_done(st_mem_detailed_writeback);
1982         rrdset_done(st_mem_detailed_pgfault);
1983         rrdset_done(st_mem_detailed_pgmajfault);
1984         rrdset_done(st_mem_detailed_pgpgin);
1985         rrdset_done(st_mem_detailed_pgpgout);
1986     }
1987
1988     if(likely(do_mem_failcnt))
1989         rrdset_done(st_mem_failcnt);
1990
1991     if(likely(do_swap_usage))
1992         rrdset_done(st_swap_usage);
1993
1994     if(likely(do_io)) {
1995         rrdset_done(st_io_read);
1996         rrdset_done(st_io_write);
1997     }
1998
1999     if(likely(do_io_ops)) {
2000         rrdset_done(st_io_serviced_read);
2001         rrdset_done(st_io_serviced_write);
2002     }
2003
2004     if(likely(do_throttle_io)) {
2005         rrdset_done(st_throttle_io_read);
2006         rrdset_done(st_throttle_io_write);
2007     }
2008
2009     if(likely(do_throttle_ops)) {
2010         rrdset_done(st_throttle_ops_read);
2011         rrdset_done(st_throttle_ops_write);
2012     }
2013
2014     if(likely(do_queued_ops)) {
2015         rrdset_done(st_queued_ops_read);
2016         rrdset_done(st_queued_ops_write);
2017     }
2018
2019     if(likely(do_merged_ops)) {
2020         rrdset_done(st_merged_ops_read);
2021         rrdset_done(st_merged_ops_write);
2022     }
2023 }
2024
2025 static inline char *cgroup_chart_type(char *buffer, const char *id, size_t len) {
2026     if(buffer[0]) return buffer;
2027
2028     if(id[0] == '\0' || (id[0] == '/' && id[1] == '\0'))
2029         strncpy(buffer, "cgroup_root", len);
2030     else
2031         snprintfz(buffer, len, "cgroup_%s", id);
2032
2033     netdata_fix_chart_id(buffer);
2034     return buffer;
2035 }
2036
2037 void update_cgroup_charts(int update_every) {
2038     debug(D_CGROUP, "updating cgroups charts");
2039
2040     char type[RRD_ID_LENGTH_MAX + 1];
2041     char title[CHART_TITLE_MAX + 1];
2042
2043     int services_do_cpu = 0,
2044             services_do_mem_usage = 0,
2045             services_do_mem_detailed = 0,
2046             services_do_mem_failcnt = 0,
2047             services_do_swap_usage = 0,
2048             services_do_io = 0,
2049             services_do_io_ops = 0,
2050             services_do_throttle_io = 0,
2051             services_do_throttle_ops = 0,
2052             services_do_queued_ops = 0,
2053             services_do_merged_ops = 0;
2054
2055     struct cgroup *cg;
2056     for(cg = cgroup_root; cg ; cg = cg->next) {
2057         if(unlikely(!cg->available || !cg->enabled))
2058             continue;
2059
2060         if(likely(cgroup_enable_systemd_services && cg->options & CGROUP_OPTIONS_SYSTEM_SLICE_SERVICE)) {
2061             if(cg->cpuacct_stat.updated && cg->cpuacct_stat.enabled == CONFIG_BOOLEAN_YES) services_do_cpu++;
2062
2063             if(cgroup_enable_systemd_services_detailed_memory && cg->memory.updated_detailed && cg->memory.enabled_detailed) services_do_mem_detailed++;
2064             if(cg->memory.updated_usage_in_bytes && cg->memory.enabled_usage_in_bytes == CONFIG_BOOLEAN_YES) services_do_mem_usage++;
2065             if(cg->memory.updated_failcnt && cg->memory.enabled_failcnt == CONFIG_BOOLEAN_YES) services_do_mem_failcnt++;
2066             if(cg->memory.updated_msw_usage_in_bytes && cg->memory.enabled_msw_usage_in_bytes == CONFIG_BOOLEAN_YES) services_do_swap_usage++;
2067
2068             if(cg->io_service_bytes.updated && cg->io_service_bytes.enabled == CONFIG_BOOLEAN_YES) services_do_io++;
2069             if(cg->io_serviced.updated && cg->io_serviced.enabled == CONFIG_BOOLEAN_YES) services_do_io_ops++;
2070             if(cg->throttle_io_service_bytes.updated && cg->throttle_io_service_bytes.enabled == CONFIG_BOOLEAN_YES) services_do_throttle_io++;
2071             if(cg->throttle_io_serviced.updated && cg->throttle_io_serviced.enabled == CONFIG_BOOLEAN_YES) services_do_throttle_ops++;
2072             if(cg->io_queued.updated && cg->io_queued.enabled == CONFIG_BOOLEAN_YES) services_do_queued_ops++;
2073             if(cg->io_merged.updated && cg->io_merged.enabled == CONFIG_BOOLEAN_YES) services_do_merged_ops++;
2074             continue;
2075         }
2076
2077         type[0] = '\0';
2078
2079         if(likely(cg->cpuacct_stat.updated && cg->cpuacct_stat.enabled == CONFIG_BOOLEAN_YES)) {
2080             if(unlikely(!cg->st_cpu)) {
2081                 snprintfz(title, CHART_TITLE_MAX, "CPU Usage (%d%% = %d core%s) for cgroup %s", (processors * 100), processors, (processors > 1) ? "s" : "", cg->chart_title);
2082
2083                 cg->st_cpu = rrdset_create_localhost(
2084                         cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX)
2085                         , "cpu"
2086                         , NULL
2087                         , "cpu"
2088                         , "cgroup.cpu"
2089                         , title
2090                         , "%"
2091                         , CHART_PRIORITY_CONTAINERS
2092                         , update_every
2093                         , RRDSET_TYPE_STACKED
2094                 );
2095
2096                 rrddim_add(cg->st_cpu, "user", NULL, 100, hz, RRD_ALGORITHM_INCREMENTAL);
2097                 rrddim_add(cg->st_cpu, "system", NULL, 100, hz, RRD_ALGORITHM_INCREMENTAL);
2098             }
2099             else
2100                 rrdset_next(cg->st_cpu);
2101
2102             rrddim_set(cg->st_cpu, "user", cg->cpuacct_stat.user);
2103             rrddim_set(cg->st_cpu, "system", cg->cpuacct_stat.system);
2104             rrdset_done(cg->st_cpu);
2105         }
2106
2107         if(likely(cg->cpuacct_usage.updated && cg->cpuacct_usage.enabled == CONFIG_BOOLEAN_YES)) {
2108             char id[RRD_ID_LENGTH_MAX + 1];
2109             unsigned int i;
2110
2111             if(unlikely(!cg->st_cpu_per_core)) {
2112                 snprintfz(title, CHART_TITLE_MAX, "CPU Usage (%d%% = %d core%s) Per Core for cgroup %s", (processors * 100), processors, (processors > 1) ? "s" : "", cg->chart_title);
2113
2114                 cg->st_cpu_per_core = rrdset_create_localhost(
2115                         cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX)
2116                         , "cpu_per_core"
2117                         , NULL
2118                         , "cpu"
2119                         , "cgroup.cpu_per_core"
2120                         , title
2121                         , "%"
2122                         , CHART_PRIORITY_CONTAINERS + 100
2123                         , update_every
2124                         , RRDSET_TYPE_STACKED
2125                 );
2126
2127                 for(i = 0; i < cg->cpuacct_usage.cpus; i++) {
2128                     snprintfz(id, RRD_ID_LENGTH_MAX, "cpu%u", i);
2129                     rrddim_add(cg->st_cpu_per_core, id, NULL, 100, 1000000000, RRD_ALGORITHM_INCREMENTAL);
2130                 }
2131             }
2132             else
2133                 rrdset_next(cg->st_cpu_per_core);
2134
2135             for(i = 0; i < cg->cpuacct_usage.cpus ;i++) {
2136                 snprintfz(id, RRD_ID_LENGTH_MAX, "cpu%u", i);
2137                 rrddim_set(cg->st_cpu_per_core, id, cg->cpuacct_usage.cpu_percpu[i]);
2138             }
2139             rrdset_done(cg->st_cpu_per_core);
2140         }
2141
2142         if(likely(cg->memory.updated_detailed && cg->memory.enabled_detailed == CONFIG_BOOLEAN_YES)) {
2143             if(unlikely(!cg->st_mem)) {
2144                 snprintfz(title, CHART_TITLE_MAX, "Memory Usage for cgroup %s", cg->chart_title);
2145
2146                 cg->st_mem = rrdset_create_localhost(
2147                         cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX)
2148                         , "mem"
2149                         , NULL
2150                         , "mem"
2151                         , "cgroup.mem"
2152                         , title
2153                         , "MB"
2154                         , CHART_PRIORITY_CONTAINERS + 210
2155                         , update_every
2156                         , RRDSET_TYPE_STACKED
2157                 );
2158
2159                 rrddim_add(cg->st_mem, "cache", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
2160                 rrddim_add(cg->st_mem, "rss", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
2161
2162                 if(cg->memory.detailed_has_swap)
2163                     rrddim_add(cg->st_mem, "swap", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
2164
2165                 rrddim_add(cg->st_mem, "rss_huge", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
2166                 rrddim_add(cg->st_mem, "mapped_file", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
2167             }
2168             else
2169                 rrdset_next(cg->st_mem);
2170
2171             rrddim_set(cg->st_mem, "cache", cg->memory.cache);
2172             rrddim_set(cg->st_mem, "rss", cg->memory.rss);
2173
2174             if(cg->memory.detailed_has_swap)
2175                 rrddim_set(cg->st_mem, "swap", cg->memory.swap);
2176
2177             rrddim_set(cg->st_mem, "rss_huge", cg->memory.rss_huge);
2178             rrddim_set(cg->st_mem, "mapped_file", cg->memory.mapped_file);
2179             rrdset_done(cg->st_mem);
2180
2181             if(unlikely(!cg->st_writeback)) {
2182                 snprintfz(title, CHART_TITLE_MAX, "Writeback Memory for cgroup %s", cg->chart_title);
2183
2184                 cg->st_writeback = rrdset_create_localhost(
2185                         cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX)
2186                         , "writeback"
2187                         , NULL
2188                         , "mem"
2189                         , "cgroup.writeback"
2190                         , title
2191                         , "MB"
2192                         , CHART_PRIORITY_CONTAINERS + 300
2193                         , update_every
2194                         , RRDSET_TYPE_AREA
2195                 );
2196
2197                 if(cg->memory.detailed_has_dirty)
2198                     rrddim_add(cg->st_writeback, "dirty", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
2199
2200                 rrddim_add(cg->st_writeback, "writeback", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
2201             }
2202             else
2203                 rrdset_next(cg->st_writeback);
2204
2205             if(cg->memory.detailed_has_dirty)
2206                 rrddim_set(cg->st_writeback, "dirty", cg->memory.dirty);
2207
2208             rrddim_set(cg->st_writeback, "writeback", cg->memory.writeback);
2209             rrdset_done(cg->st_writeback);
2210
2211             if(unlikely(!cg->st_mem_activity)) {
2212                 snprintfz(title, CHART_TITLE_MAX, "Memory Activity for cgroup %s", cg->chart_title);
2213
2214                 cg->st_mem_activity = rrdset_create_localhost(
2215                         cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX)
2216                         , "mem_activity"
2217                         , NULL
2218                         , "mem"
2219                         , "cgroup.mem_activity"
2220                         , title
2221                         , "MB/s"
2222                         , CHART_PRIORITY_CONTAINERS + 400
2223                         , update_every
2224                         , RRDSET_TYPE_LINE
2225                 );
2226
2227                 rrddim_add(cg->st_mem_activity, "pgpgin", "in", system_page_size, 1024 * 1024, RRD_ALGORITHM_INCREMENTAL);
2228                 rrddim_add(cg->st_mem_activity, "pgpgout", "out", -system_page_size, 1024 * 1024, RRD_ALGORITHM_INCREMENTAL);
2229             }
2230             else
2231                 rrdset_next(cg->st_mem_activity);
2232
2233             rrddim_set(cg->st_mem_activity, "pgpgin", cg->memory.pgpgin);
2234             rrddim_set(cg->st_mem_activity, "pgpgout", cg->memory.pgpgout);
2235             rrdset_done(cg->st_mem_activity);
2236
2237             if(unlikely(!cg->st_pgfaults)) {
2238                 snprintfz(title, CHART_TITLE_MAX, "Memory Page Faults for cgroup %s", cg->chart_title);
2239
2240                 cg->st_pgfaults = rrdset_create_localhost(
2241                         cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX)
2242                         , "pgfaults"
2243                         , NULL
2244                         , "mem"
2245                         , "cgroup.pgfaults"
2246                         , title
2247                         , "MB/s"
2248                         , CHART_PRIORITY_CONTAINERS + 500
2249                         , update_every
2250                         , RRDSET_TYPE_LINE
2251                 );
2252
2253                 rrddim_add(cg->st_pgfaults, "pgfault", NULL, system_page_size, 1024 * 1024, RRD_ALGORITHM_INCREMENTAL);
2254                 rrddim_add(cg->st_pgfaults, "pgmajfault", "swap", -system_page_size, 1024 * 1024, RRD_ALGORITHM_INCREMENTAL);
2255             }
2256             else
2257                 rrdset_next(cg->st_pgfaults);
2258
2259             rrddim_set(cg->st_pgfaults, "pgfault", cg->memory.pgfault);
2260             rrddim_set(cg->st_pgfaults, "pgmajfault", cg->memory.pgmajfault);
2261             rrdset_done(cg->st_pgfaults);
2262         }
2263
2264         if(likely(cg->memory.updated_usage_in_bytes && cg->memory.enabled_usage_in_bytes == CONFIG_BOOLEAN_YES)) {
2265             if(unlikely(!cg->st_mem_usage)) {
2266                 snprintfz(title, CHART_TITLE_MAX, "Used Memory %sfor cgroup %s", (cgroup_used_memory_without_cache && cg->memory.updated_detailed)?"without Cache ":"", cg->chart_title);
2267
2268                 cg->st_mem_usage = rrdset_create_localhost(
2269                         cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX)
2270                         , "mem_usage"
2271                         , NULL
2272                         , "mem"
2273                         , "cgroup.mem_usage"
2274                         , title
2275                         , "MB"
2276                         , CHART_PRIORITY_CONTAINERS + 200
2277                         , update_every
2278                         , RRDSET_TYPE_STACKED
2279                 );
2280
2281                 rrddim_add(cg->st_mem_usage, "ram", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
2282                 rrddim_add(cg->st_mem_usage, "swap", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
2283             }
2284             else
2285                 rrdset_next(cg->st_mem_usage);
2286
2287             rrddim_set(cg->st_mem_usage, "ram", cg->memory.usage_in_bytes - ((cgroup_used_memory_without_cache)?cg->memory.cache:0));
2288             rrddim_set(cg->st_mem_usage, "swap", (cg->memory.msw_usage_in_bytes > cg->memory.usage_in_bytes)?cg->memory.msw_usage_in_bytes - cg->memory.usage_in_bytes:0);
2289             rrdset_done(cg->st_mem_usage);
2290         }
2291
2292         if(likely(cg->memory.updated_failcnt && cg->memory.enabled_failcnt == CONFIG_BOOLEAN_YES)) {
2293             if(unlikely(!cg->st_mem_failcnt)) {
2294                 snprintfz(title, CHART_TITLE_MAX, "Memory Limit Failures for cgroup %s", cg->chart_title);
2295
2296                 cg->st_mem_failcnt = rrdset_create_localhost(
2297                         cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX)
2298                         , "mem_failcnt"
2299                         , NULL
2300                         , "mem"
2301                         , "cgroup.mem_failcnt"
2302                         , title
2303                         , "count"
2304                         , CHART_PRIORITY_CONTAINERS + 250
2305                         , update_every
2306                         , RRDSET_TYPE_LINE
2307                 );
2308
2309                 rrddim_add(cg->st_mem_failcnt, "failures", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
2310             }
2311             else
2312                 rrdset_next(cg->st_mem_failcnt);
2313
2314             rrddim_set(cg->st_mem_failcnt, "failures", cg->memory.failcnt);
2315             rrdset_done(cg->st_mem_failcnt);
2316         }
2317
2318         if(likely(cg->io_service_bytes.updated && cg->io_service_bytes.enabled == CONFIG_BOOLEAN_YES)) {
2319             if(unlikely(!cg->st_io)) {
2320                 snprintfz(title, CHART_TITLE_MAX, "I/O Bandwidth (all disks) for cgroup %s", cg->chart_title);
2321
2322                 cg->st_io = rrdset_create_localhost(
2323                         cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX)
2324                         , "io"
2325                         , NULL
2326                         , "disk"
2327                         , "cgroup.io"
2328                         , title
2329                         , "KB/s"
2330                         , CHART_PRIORITY_CONTAINERS + 1200
2331                         , update_every
2332                         , RRDSET_TYPE_AREA
2333                 );
2334
2335                 rrddim_add(cg->st_io, "read", NULL, 1, 1024, RRD_ALGORITHM_INCREMENTAL);
2336                 rrddim_add(cg->st_io, "write", NULL, -1, 1024, RRD_ALGORITHM_INCREMENTAL);
2337             }
2338             else
2339                 rrdset_next(cg->st_io);
2340
2341             rrddim_set(cg->st_io, "read", cg->io_service_bytes.Read);
2342             rrddim_set(cg->st_io, "write", cg->io_service_bytes.Write);
2343             rrdset_done(cg->st_io);
2344         }
2345
2346         if(likely(cg->io_serviced.updated && cg->io_serviced.enabled == CONFIG_BOOLEAN_YES)) {
2347             if(unlikely(!cg->st_serviced_ops)) {
2348                 snprintfz(title, CHART_TITLE_MAX, "Serviced I/O Operations (all disks) for cgroup %s", cg->chart_title);
2349
2350                 cg->st_serviced_ops = rrdset_create_localhost(
2351                         cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX)
2352                         , "serviced_ops"
2353                         , NULL
2354                         , "disk"
2355                         , "cgroup.serviced_ops"
2356                         , title
2357                         , "operations/s"
2358                         , CHART_PRIORITY_CONTAINERS + 1200
2359                         , update_every
2360                         , RRDSET_TYPE_LINE
2361                 );
2362
2363                 rrddim_add(cg->st_serviced_ops, "read", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
2364                 rrddim_add(cg->st_serviced_ops, "write", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
2365             }
2366             else
2367                 rrdset_next(cg->st_serviced_ops);
2368
2369             rrddim_set(cg->st_serviced_ops, "read", cg->io_serviced.Read);
2370             rrddim_set(cg->st_serviced_ops, "write", cg->io_serviced.Write);
2371             rrdset_done(cg->st_serviced_ops);
2372         }
2373
2374         if(likely(cg->throttle_io_service_bytes.updated && cg->throttle_io_service_bytes.enabled == CONFIG_BOOLEAN_YES)) {
2375             if(unlikely(!cg->st_throttle_io)) {
2376                 snprintfz(title, CHART_TITLE_MAX, "Throttle I/O Bandwidth (all disks) for cgroup %s", cg->chart_title);
2377
2378                 cg->st_throttle_io = rrdset_create_localhost(
2379                         cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX)
2380                         , "throttle_io"
2381                         , NULL
2382                         , "disk"
2383                         , "cgroup.throttle_io"
2384                         , title
2385                         , "KB/s"
2386                         , CHART_PRIORITY_CONTAINERS + 1200
2387                         , update_every
2388                         , RRDSET_TYPE_AREA
2389                 );
2390
2391                 rrddim_add(cg->st_throttle_io, "read", NULL, 1, 1024, RRD_ALGORITHM_INCREMENTAL);
2392                 rrddim_add(cg->st_throttle_io, "write", NULL, -1, 1024, RRD_ALGORITHM_INCREMENTAL);
2393             }
2394             else
2395                 rrdset_next(cg->st_throttle_io);
2396
2397             rrddim_set(cg->st_throttle_io, "read", cg->throttle_io_service_bytes.Read);
2398             rrddim_set(cg->st_throttle_io, "write", cg->throttle_io_service_bytes.Write);
2399             rrdset_done(cg->st_throttle_io);
2400         }
2401
2402         if(likely(cg->throttle_io_serviced.updated && cg->throttle_io_serviced.enabled == CONFIG_BOOLEAN_YES)) {
2403             if(unlikely(!cg->st_throttle_serviced_ops)) {
2404                 snprintfz(title, CHART_TITLE_MAX, "Throttle Serviced I/O Operations (all disks) for cgroup %s", cg->chart_title);
2405
2406                 cg->st_throttle_serviced_ops = rrdset_create_localhost(
2407                         cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX)
2408                         , "throttle_serviced_ops"
2409                         , NULL
2410                         , "disk"
2411                         , "cgroup.throttle_serviced_ops"
2412                         , title
2413                         , "operations/s"
2414                         , CHART_PRIORITY_CONTAINERS + 1200
2415                         , update_every
2416                         , RRDSET_TYPE_LINE
2417                 );
2418
2419                 rrddim_add(cg->st_throttle_serviced_ops, "read", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
2420                 rrddim_add(cg->st_throttle_serviced_ops, "write", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
2421             }
2422             else
2423                 rrdset_next(cg->st_throttle_serviced_ops);
2424
2425             rrddim_set(cg->st_throttle_serviced_ops, "read", cg->throttle_io_serviced.Read);
2426             rrddim_set(cg->st_throttle_serviced_ops, "write", cg->throttle_io_serviced.Write);
2427             rrdset_done(cg->st_throttle_serviced_ops);
2428         }
2429
2430         if(likely(cg->io_queued.updated && cg->io_queued.enabled == CONFIG_BOOLEAN_YES)) {
2431             if(unlikely(!cg->st_queued_ops)) {
2432                 snprintfz(title, CHART_TITLE_MAX, "Queued I/O Operations (all disks) for cgroup %s", cg->chart_title);
2433
2434                 cg->st_queued_ops = rrdset_create_localhost(
2435                         cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX)
2436                         , "queued_ops"
2437                         , NULL
2438                         , "disk"
2439                         , "cgroup.queued_ops"
2440                         , title
2441                         , "operations"
2442                         , CHART_PRIORITY_CONTAINERS + 2000
2443                         , update_every
2444                         , RRDSET_TYPE_LINE
2445                 );
2446
2447                 rrddim_add(cg->st_queued_ops, "read", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
2448                 rrddim_add(cg->st_queued_ops, "write", NULL, -1, 1, RRD_ALGORITHM_ABSOLUTE);
2449             }
2450             else
2451                 rrdset_next(cg->st_queued_ops);
2452
2453             rrddim_set(cg->st_queued_ops, "read", cg->io_queued.Read);
2454             rrddim_set(cg->st_queued_ops, "write", cg->io_queued.Write);
2455             rrdset_done(cg->st_queued_ops);
2456         }
2457
2458         if(likely(cg->io_merged.updated && cg->io_merged.enabled == CONFIG_BOOLEAN_YES)) {
2459             if(unlikely(!cg->st_merged_ops)) {
2460                 snprintfz(title, CHART_TITLE_MAX, "Merged I/O Operations (all disks) for cgroup %s", cg->chart_title);
2461
2462                 cg->st_merged_ops = rrdset_create_localhost(
2463                         cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX)
2464                         , "merged_ops"
2465                         , NULL
2466                         , "disk"
2467                         , "cgroup.merged_ops"
2468                         , title
2469                         , "operations/s"
2470                         , CHART_PRIORITY_CONTAINERS + 2100
2471                         , update_every
2472                         , RRDSET_TYPE_LINE
2473                 );
2474
2475                 rrddim_add(cg->st_merged_ops, "read", NULL, 1, 1024, RRD_ALGORITHM_INCREMENTAL);
2476                 rrddim_add(cg->st_merged_ops, "write", NULL, -1, 1024, RRD_ALGORITHM_INCREMENTAL);
2477             }
2478             else
2479                 rrdset_next(cg->st_merged_ops);
2480
2481             rrddim_set(cg->st_merged_ops, "read", cg->io_merged.Read);
2482             rrddim_set(cg->st_merged_ops, "write", cg->io_merged.Write);
2483             rrdset_done(cg->st_merged_ops);
2484         }
2485     }
2486
2487     if(likely(cgroup_enable_systemd_services))
2488         update_systemd_services_charts(update_every, services_do_cpu, services_do_mem_usage, services_do_mem_detailed
2489                                        , services_do_mem_failcnt, services_do_swap_usage, services_do_io
2490                                        , services_do_io_ops, services_do_throttle_io, services_do_throttle_ops
2491                                        , services_do_queued_ops, services_do_merged_ops
2492         );
2493
2494     debug(D_CGROUP, "done updating cgroups charts");
2495 }
2496
2497 // ----------------------------------------------------------------------------
2498 // cgroups main
2499
2500 void *cgroups_main(void *ptr) {
2501     struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr;
2502
2503     info("CGROUP Plugin thread created with task id %d", gettid());
2504
2505     if(pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL) != 0)
2506         error("Cannot set pthread cancel type to DEFERRED.");
2507
2508     if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0)
2509         error("Cannot set pthread cancel state to ENABLE.");
2510
2511     struct rusage thread;
2512
2513     // when ZERO, attempt to do it
2514     int vdo_cpu_netdata = config_get_boolean("plugin:cgroups", "cgroups plugin resource charts", 1);
2515
2516     read_cgroup_plugin_configuration();
2517
2518     RRDSET *stcpu_thread = NULL;
2519
2520     heartbeat_t hb;
2521     heartbeat_init(&hb);
2522     usec_t step = cgroup_update_every * USEC_PER_SEC;
2523     usec_t find_every = cgroup_check_for_new_every * USEC_PER_SEC, find_dt = 0;
2524     for(;;) {
2525         usec_t hb_dt = heartbeat_next(&hb, step);
2526         if(unlikely(netdata_exit)) break;
2527
2528         // BEGIN -- the job to be done
2529
2530         find_dt += hb_dt;
2531         if(unlikely(find_dt >= find_every || cgroups_check)) {
2532             find_all_cgroups();
2533             find_dt = 0;
2534             cgroups_check = 0;
2535         }
2536
2537         read_all_cgroups(cgroup_root);
2538         update_cgroup_charts(cgroup_update_every);
2539
2540         // END -- the job is done
2541
2542         // --------------------------------------------------------------------
2543
2544         if(vdo_cpu_netdata) {
2545             getrusage(RUSAGE_THREAD, &thread);
2546
2547             if(unlikely(!stcpu_thread)) {
2548
2549                 stcpu_thread = rrdset_create_localhost(
2550                         "netdata"
2551                         , "plugin_cgroups_cpu"
2552                         , NULL
2553                         , "cgroups"
2554                         , NULL
2555                         , "NetData CGroups Plugin CPU usage"
2556                         , "milliseconds/s"
2557                         , 132000
2558                         , cgroup_update_every
2559                         , RRDSET_TYPE_STACKED
2560                 );
2561
2562                 rrddim_add(stcpu_thread, "user",  NULL,  1, 1000, RRD_ALGORITHM_INCREMENTAL);
2563                 rrddim_add(stcpu_thread, "system", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
2564             }
2565             else
2566                 rrdset_next(stcpu_thread);
2567
2568             rrddim_set(stcpu_thread, "user"  , thread.ru_utime.tv_sec * 1000000ULL + thread.ru_utime.tv_usec);
2569             rrddim_set(stcpu_thread, "system", thread.ru_stime.tv_sec * 1000000ULL + thread.ru_stime.tv_usec);
2570             rrdset_done(stcpu_thread);
2571         }
2572     }
2573
2574     info("CGROUP thread exiting");
2575
2576     static_thread->enabled = 0;
2577     pthread_exit(NULL);
2578     return NULL;
2579 }