reduce netdata virtual memory size by setting MALLOC_ARENA_MAX; portable way of detec...

diff --git a/configure.ac b/configure.ac
index b86131d008b39a573ba8bfa3252e7454fbe4ccc0..1684178efb5a17b7be5198d519cabec5f96557f2 100644 (file)
--- a/configure.ac
+++ b/configure.ac
@@ -88,6 +88,8 @@ AC_TYPE_UINT16_T
 AC_TYPE_UINT32_T
 AC_C_INLINE
 AC_FUNC_STRERROR_R
+AC_C_MALLOPT
+AC_C_MALLINFO
 AC_C__GENERIC
 AC_C___ATOMIC
 AC_CHECK_SIZEOF([void *])

diff --git a/m4/ax_c_mallinfo.m4 b/m4/ax_c_mallinfo.m4
new file mode 100755 (executable)
index 0000000..a8f8cc2
--- /dev/null
+++ b/m4/ax_c_mallinfo.m4
@@ -0,0 +1,21 @@
+# AC_C_MALLINFO
+# -------------
+# Define HAVE_C_MALLINFO if mallinfo() works.
+AN_IDENTIFIER([mallinfo], [AC_C_MALLINFO])
+AC_DEFUN([AC_C_MALLINFO],
+[AC_CACHE_CHECK([for mallinfo], ac_cv_c_mallinfo,
+[AC_LINK_IFELSE(
+   [AC_LANG_SOURCE(
+      [[#include <malloc.h>
+        int main(int argc, char **argv) {
+          struct mallinfo mi = mallinfo();
+          printf("%d%d%d\n", mi.uordblks, mi.hblkhd, mi.arena);
+        }
+      ]])],
+   [ac_cv_c_mallinfo=yes],
+   [ac_cv_c_mallinfo=no])])
+if test $ac_cv_c_mallinfo = yes; then
+  AC_DEFINE([HAVE_C_MALLINFO], 1,
+           [Define to 1 if glibc mallinfo exists.])
+fi
+])# AC_C_MALLINFO

diff --git a/m4/ax_c_mallopt.m4 b/m4/ax_c_mallopt.m4
new file mode 100755 (executable)
index 0000000..31c4fdc
--- /dev/null
+++ b/m4/ax_c_mallopt.m4
@@ -0,0 +1,20 @@
+# AC_C_MALLOPT
+# -------------
+# Define HAVE_C_MALLOPT if mallopt() works.
+AN_IDENTIFIER([mallopt], [AC_C_MALLOPT])
+AC_DEFUN([AC_C_MALLOPT],
+[AC_CACHE_CHECK([for mallopt], ac_cv_c_mallopt,
+[AC_LINK_IFELSE(
+   [AC_LANG_SOURCE(
+      [[#include <malloc.h>
+        int main(int argc, char **argv) {
+          mallopt(M_ARENA_MAX, 1);
+        }
+      ]])],
+   [ac_cv_c_mallopt=yes],
+   [ac_cv_c_mallopt=no])])
+if test $ac_cv_c_mallopt = yes; then
+  AC_DEFINE([HAVE_C_MALLOPT], 1,
+           [Define to 1 if glibc mallopt exists.])
+fi
+])# AC_C_MALLOPT

diff --git a/src/apps_plugin.c b/src/apps_plugin.c
index 31899eac7babde120c8629ec26e580b66918291d..f22a575ba596f9bcb9805d08e24d7b9fe71150b1 100644 (file)
--- a/src/apps_plugin.c
+++ b/src/apps_plugin.c
@@ -66,23 +66,23 @@ struct target {
     unsigned long long cstime;
     unsigned long long cgtime;
     unsigned long long num_threads;
-    unsigned long long rss;
+    // unsigned long long rss;
 
     unsigned long long statm_size;
     unsigned long long statm_resident;
     unsigned long long statm_share;
-    unsigned long long statm_text;
-    unsigned long long statm_lib;
-    unsigned long long statm_data;
-    unsigned long long statm_dirty;
+    // unsigned long long statm_text;
+    // unsigned long long statm_lib;
+    // unsigned long long statm_data;
+    // unsigned long long statm_dirty;
 
     unsigned long long io_logical_bytes_read;
     unsigned long long io_logical_bytes_written;
-    unsigned long long io_read_calls;
-    unsigned long long io_write_calls;
+    // unsigned long long io_read_calls;
+    // unsigned long long io_write_calls;
     unsigned long long io_storage_bytes_read;
     unsigned long long io_storage_bytes_written;
-    unsigned long long io_cancelled_write_bytes;
+    // unsigned long long io_cancelled_write_bytes;
 
     int *fds;
     unsigned long long openfiles;
@@ -385,7 +385,7 @@ struct pid_stat {
     // int64_t itrealvalue;
     // unsigned long long starttime;
     // unsigned long long vsize;
-    unsigned long long rss;
+    // unsigned long long rss;
     // unsigned long long rsslim;
     // unsigned long long starcode;
     // unsigned long long endcode;
@@ -411,26 +411,26 @@ struct pid_stat {
     unsigned long long statm_size;
     unsigned long long statm_resident;
     unsigned long long statm_share;
-    unsigned long long statm_text;
-    unsigned long long statm_lib;
-    unsigned long long statm_data;
-    unsigned long long statm_dirty;
+    // unsigned long long statm_text;
+    // unsigned long long statm_lib;
+    // unsigned long long statm_data;
+    // unsigned long long statm_dirty;
 
     unsigned long long io_logical_bytes_read_raw;
     unsigned long long io_logical_bytes_written_raw;
-    unsigned long long io_read_calls_raw;
-    unsigned long long io_write_calls_raw;
+    // unsigned long long io_read_calls_raw;
+    // unsigned long long io_write_calls_raw;
     unsigned long long io_storage_bytes_read_raw;
     unsigned long long io_storage_bytes_written_raw;
-    unsigned long long io_cancelled_write_bytes_raw;
+    // unsigned long long io_cancelled_write_bytes_raw;
 
     unsigned long long io_logical_bytes_read;
     unsigned long long io_logical_bytes_written;
-    unsigned long long io_read_calls;
-    unsigned long long io_write_calls;
+    // unsigned long long io_read_calls;
+    // unsigned long long io_write_calls;
     unsigned long long io_storage_bytes_read;
     unsigned long long io_storage_bytes_written;
-    unsigned long long io_cancelled_write_bytes;
+    // unsigned long long io_cancelled_write_bytes;
 
     int *fds;                       // array of fds it uses
     int fds_size;                   // the size of the fds array
@@ -645,7 +645,7 @@ int read_proc_pid_stat(struct pid_stat *p) {
     // p->itrealvalue   = strtoull(procfile_lineword(ff, 0, 20), NULL, 10);
     // p->starttime     = strtoull(procfile_lineword(ff, 0, 21), NULL, 10);
     // p->vsize         = strtoull(procfile_lineword(ff, 0, 22), NULL, 10);
-    p->rss              = strtoull(procfile_lineword(ff, 0, 23), NULL, 10);
+    // p->rss           = strtoull(procfile_lineword(ff, 0, 23), NULL, 10);
     // p->rsslim        = strtoull(procfile_lineword(ff, 0, 24), NULL, 10);
     // p->starcode      = strtoull(procfile_lineword(ff, 0, 25), NULL, 10);
     // p->endcode       = strtoull(procfile_lineword(ff, 0, 26), NULL, 10);
@@ -711,7 +711,7 @@ cleanup:
     p->cstime           = 0;
     p->cgtime           = 0;
     p->num_threads      = 0;
-    p->rss              = 0;
+    // p->rss              = 0;
     return 0;
 }
 
@@ -735,10 +735,10 @@ int read_proc_pid_statm(struct pid_stat *p) {
     p->statm_size           = strtoull(procfile_lineword(ff, 0, 0), NULL, 10);
     p->statm_resident       = strtoull(procfile_lineword(ff, 0, 1), NULL, 10);
     p->statm_share          = strtoull(procfile_lineword(ff, 0, 2), NULL, 10);
-    p->statm_text           = strtoull(procfile_lineword(ff, 0, 3), NULL, 10);
-    p->statm_lib            = strtoull(procfile_lineword(ff, 0, 4), NULL, 10);
-    p->statm_data           = strtoull(procfile_lineword(ff, 0, 5), NULL, 10);
-    p->statm_dirty          = strtoull(procfile_lineword(ff, 0, 6), NULL, 10);
+    // p->statm_text           = strtoull(procfile_lineword(ff, 0, 3), NULL, 10);
+    // p->statm_lib            = strtoull(procfile_lineword(ff, 0, 4), NULL, 10);
+    // p->statm_data           = strtoull(procfile_lineword(ff, 0, 5), NULL, 10);
+    // p->statm_dirty          = strtoull(procfile_lineword(ff, 0, 6), NULL, 10);
 
     return 1;
 
@@ -746,10 +746,10 @@ cleanup:
     p->statm_size           = 0;
     p->statm_resident       = 0;
     p->statm_share          = 0;
-    p->statm_text           = 0;
-    p->statm_lib            = 0;
-    p->statm_data           = 0;
-    p->statm_dirty          = 0;
+    // p->statm_text           = 0;
+    // p->statm_lib            = 0;
+    // p->statm_data           = 0;
+    // p->statm_dirty          = 0;
     return 0;
 }
 
@@ -784,13 +784,13 @@ int read_proc_pid_io(struct pid_stat *p) {
     p->io_logical_bytes_written_raw = strtoull(procfile_lineword(ff, 1, 1), NULL, 10);
     p->io_logical_bytes_written = (p->io_logical_bytes_written_raw - last) * (1000000ULL * RATES_DETAIL) / (p->io_collected_usec - p->last_io_collected_usec);
 
-    last = p->io_read_calls_raw;
-    p->io_read_calls_raw = strtoull(procfile_lineword(ff, 2, 1), NULL, 10);
-    p->io_read_calls = (p->io_read_calls_raw - last) * (1000000ULL * RATES_DETAIL) / (p->io_collected_usec - p->last_io_collected_usec);
+    // last = p->io_read_calls_raw;
+    // p->io_read_calls_raw = strtoull(procfile_lineword(ff, 2, 1), NULL, 10);
+    // p->io_read_calls = (p->io_read_calls_raw - last) * (1000000ULL * RATES_DETAIL) / (p->io_collected_usec - p->last_io_collected_usec);
 
-    last = p->io_write_calls_raw;
-    p->io_write_calls_raw = strtoull(procfile_lineword(ff, 3, 1), NULL, 10);
-    p->io_write_calls = (p->io_write_calls_raw - last) * (1000000ULL * RATES_DETAIL) / (p->io_collected_usec - p->last_io_collected_usec);
+    // last = p->io_write_calls_raw;
+    // p->io_write_calls_raw = strtoull(procfile_lineword(ff, 3, 1), NULL, 10);
+    // p->io_write_calls = (p->io_write_calls_raw - last) * (1000000ULL * RATES_DETAIL) / (p->io_collected_usec - p->last_io_collected_usec);
 
     last = p->io_storage_bytes_read_raw;
     p->io_storage_bytes_read_raw = strtoull(procfile_lineword(ff, 4, 1), NULL, 10);
@@ -800,18 +800,18 @@ int read_proc_pid_io(struct pid_stat *p) {
     p->io_storage_bytes_written_raw = strtoull(procfile_lineword(ff, 5, 1), NULL, 10);
     p->io_storage_bytes_written = (p->io_storage_bytes_written_raw - last) * (1000000ULL * RATES_DETAIL) / (p->io_collected_usec - p->last_io_collected_usec);
 
-    last = p->io_cancelled_write_bytes_raw;
-    p->io_cancelled_write_bytes_raw = strtoull(procfile_lineword(ff, 6, 1), NULL, 10);
-    p->io_cancelled_write_bytes = (p->io_cancelled_write_bytes_raw - last) * (1000000ULL * RATES_DETAIL) / (p->io_collected_usec - p->last_io_collected_usec);
+    // last = p->io_cancelled_write_bytes_raw;
+    // p->io_cancelled_write_bytes_raw = strtoull(procfile_lineword(ff, 6, 1), NULL, 10);
+    // p->io_cancelled_write_bytes = (p->io_cancelled_write_bytes_raw - last) * (1000000ULL * RATES_DETAIL) / (p->io_collected_usec - p->last_io_collected_usec);
 
     if(unlikely(global_iterations_counter == 1)) {
         p->io_logical_bytes_read        = 0;
         p->io_logical_bytes_written     = 0;
-        p->io_read_calls                = 0;
-        p->io_write_calls               = 0;
+        // p->io_read_calls             = 0;
+        // p->io_write_calls            = 0;
         p->io_storage_bytes_read        = 0;
         p->io_storage_bytes_written     = 0;
-        p->io_cancelled_write_bytes     = 0;
+        // p->io_cancelled_write_bytes  = 0;
     }
 
     return 1;
@@ -819,11 +819,11 @@ int read_proc_pid_io(struct pid_stat *p) {
 cleanup:
     p->io_logical_bytes_read        = 0;
     p->io_logical_bytes_written     = 0;
-    p->io_read_calls                = 0;
-    p->io_write_calls               = 0;
+    // p->io_read_calls             = 0;
+    // p->io_write_calls            = 0;
     p->io_storage_bytes_read        = 0;
     p->io_storage_bytes_written     = 0;
-    p->io_cancelled_write_bytes     = 0;
+    // p->io_cancelled_write_bytes  = 0;
     return 0;
 }
 
@@ -1902,24 +1902,24 @@ long zero_all_targets(struct target *root) {
         w->cstime = 0;
         w->cgtime = 0;
         w->num_threads = 0;
-        w->rss = 0;
+        // w->rss = 0;
         w->processes = 0;
 
         w->statm_size = 0;
         w->statm_resident = 0;
         w->statm_share = 0;
-        w->statm_text = 0;
-        w->statm_lib = 0;
-        w->statm_data = 0;
-        w->statm_dirty = 0;
+        // w->statm_text = 0;
+        // w->statm_lib = 0;
+        // w->statm_data = 0;
+        // w->statm_dirty = 0;
 
         w->io_logical_bytes_read = 0;
         w->io_logical_bytes_written = 0;
-        w->io_read_calls = 0;
-        w->io_write_calls = 0;
+        // w->io_read_calls = 0;
+        // w->io_write_calls = 0;
         w->io_storage_bytes_read = 0;
         w->io_storage_bytes_written = 0;
-        w->io_cancelled_write_bytes = 0;
+        // w->io_cancelled_write_bytes = 0;
     }
 
     return count;
@@ -1944,23 +1944,23 @@ void aggregate_pid_on_target(struct target *w, struct pid_stat *p, struct target
         w->minflt += p->minflt;
         w->majflt += p->majflt;
 
-        w->rss += p->rss;
+        // w->rss += p->rss;
 
         w->statm_size += p->statm_size;
         w->statm_resident += p->statm_resident;
         w->statm_share += p->statm_share;
-        w->statm_text += p->statm_text;
-        w->statm_lib += p->statm_lib;
-        w->statm_data += p->statm_data;
-        w->statm_dirty += p->statm_dirty;
+        // w->statm_text += p->statm_text;
+        // w->statm_lib += p->statm_lib;
+        // w->statm_data += p->statm_data;
+        // w->statm_dirty += p->statm_dirty;
 
         w->io_logical_bytes_read    += p->io_logical_bytes_read;
         w->io_logical_bytes_written += p->io_logical_bytes_written;
-        w->io_read_calls            += p->io_read_calls;
-        w->io_write_calls           += p->io_write_calls;
+        // w->io_read_calls            += p->io_read_calls;
+        // w->io_write_calls           += p->io_write_calls;
         w->io_storage_bytes_read    += p->io_storage_bytes_read;
         w->io_storage_bytes_written += p->io_storage_bytes_written;
-        w->io_cancelled_write_bytes += p->io_cancelled_write_bytes;
+        // w->io_cancelled_write_bytes += p->io_cancelled_write_bytes;
 
         w->processes++;
         w->num_threads += p->num_threads;
@@ -2433,6 +2433,13 @@ void send_collected_data_to_netdata(struct target *root, const char *type, unsig
     }
     send_END();
 
+    send_BEGIN(type, "vmem", usec);
+    for (w = root; w ; w = w->next) {
+        if(unlikely(w->exposed))
+            send_SET(w->name, w->statm_size);
+    }
+    send_END();
+
     send_BEGIN(type, "minor_faults", usec);
     for (w = root; w ; w = w->next) {
         if(unlikely(w->exposed))
@@ -2529,7 +2536,13 @@ void send_charts_updates_to_netdata(struct target *root, const char *type, const
             buffer_sprintf(output, "DIMENSION %s '' absolute 1 %llu %s\n", w->name, hz * RATES_DETAIL / 100, w->hidden ? "hidden" : "");
     }
 
-    buffer_sprintf(output, "CHART %s.mem '' '%s Dedicated Memory (w/o shared)' 'MB' mem %s.mem stacked 20003 %d\n", type, title, type, update_every);
+    buffer_sprintf(output, "CHART %s.mem '' '%s Real Memory (w/o shared)' 'MB' mem %s.mem stacked 20003 %d\n", type, title, type, update_every);
+    for (w = root; w ; w = w->next) {
+        if(unlikely(w->exposed))
+            buffer_sprintf(output, "DIMENSION %s '' absolute %ld %ld\n", w->name, sysconf(_SC_PAGESIZE), 1024L*1024L);
+    }
+
+    buffer_sprintf(output, "CHART %s.vmem '' '%s Virtual Memory Size' 'MB' mem %s.vmem stacked 20004 %d\n", type, title, type, update_every);
     for (w = root; w ; w = w->next) {
         if(unlikely(w->exposed))
             buffer_sprintf(output, "DIMENSION %s '' absolute %ld %ld\n", w->name, sysconf(_SC_PAGESIZE), 1024L*1024L);

diff --git a/src/main.c b/src/main.c
index 79b09237875134ca78c0f24b051265fd3860e6f8..0ea80c2c66440b190b9b0d8df2e661007e331fbf 100644 (file)
--- a/src/main.c
+++ b/src/main.c
@@ -286,8 +286,6 @@ int main(int argc, char **argv)
     // set the name for logging
     program_name = "netdata";
 
-    // parse command line.
-
     // parse depercated options
     // TODO: Remove this block with the next major release.
     {
@@ -409,6 +407,16 @@ int main(int argc, char **argv)
         load_config(NULL, 0);
 
     {
+        char *pmax = config_get("global", "glibc malloc arena max for plugins", "1");
+        if(pmax && *pmax)
+            setenv("MALLOC_ARENA_MAX", pmax, 1);
+
+#if defined(HAVE_C_MALLOPT)
+        int i = config_get_number("global", "glibc malloc arena max for netdata", 1);
+        if(i > 0)
+            mallopt(M_ARENA_MAX, 1);
+#endif
+
         char *config_dir = config_get("global", "config directory", CONFIG_DIR);
 
         // prepare configuration environment variables for the plugins

diff --git a/src/web_server.c b/src/web_server.c
index 71781cdb631cac54602254a5adb8bac70a2f39ff..8c1b53ecec8676ef15f999b6dbaef9356ff1d494 100644 (file)
--- a/src/web_server.c
+++ b/src/web_server.c
@@ -10,21 +10,35 @@ int web_server_mode = WEB_SERVER_MODE_MULTI_THREADED;
 #ifdef NETDATA_INTERNAL_CHECKS
 static void log_allocations(void)
 {
-    static int mem = 0;
+#ifdef HAVE_C_MALLINFO
+    static int heap = 0, used = 0, mmap = 0;
 
     struct mallinfo mi;
 
     mi = mallinfo();
-    if(mi.uordblks > mem) {
+    if(mi.uordblks > used) {
         int clients = 0;
         struct web_client *w;
         for(w = web_clients; w ; w = w->next) clients++;
 
-        info("Allocated memory increased from %d to %d (increased by %d bytes). There are %d web clients connected.", mem, mi.uordblks, mi.uordblks - mem, clients);
-        mem = mi.uordblks;
+        info("Allocated memory: used %d KB (+%d B), mmap %d KB (+%d B), heap %d KB (+%d B). %d web clients connected.",
+            mi.uordblks / 1024,
+            mi.uordblks - used,
+            mi.hblkhd / 1024,
+            mi.hblkhd - mmap,
+            mi.arena / 1024,
+            mi.arena - heap,
+            clients);
+
+        used = mi.uordblks;
+        heap = mi.arena;
+        mmap = mi.hblkhd;
     }
+#else /* ! HAVE_C_MALLINFO */
+    ;
+#endif /* ! HAVE_C_MALLINFO */
 }
-#endif
+#endif /* NETDATA_INTERNAL_CHECKS */
 
 #ifndef HAVE_ACCEPT4
 int accept4(int sock, struct sockaddr *addr, socklen_t *addrlen, int flags) {