From ee7b844c041ec1453684dedefd3aed9e7de00c4d Mon Sep 17 00:00:00 2001 From: Costa Tsaousis Date: Fri, 30 Sep 2016 14:49:19 +0300 Subject: [PATCH] fix unecessary calls to realloc(); added memory allocations logger - enabled with -DNETDATA_LOG_ALLOCATIONS=1 --- CMakeLists.txt | 2 +- src/common.c | 116 ++++++++++++++++++++++++++++++++++++++++++ src/common.h | 16 +++++- src/proc_interrupts.c | 3 +- src/proc_softirqs.c | 3 +- src/rrd.c | 27 ++++++++-- 6 files changed, 158 insertions(+), 9 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a713732c..e973f6eb 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,7 +7,7 @@ project(netdata C) find_package (Threads) -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -Wformat-signedness -Werror=format-security") +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -Wformat-signedness -Werror=format-security -DNETDATA_INTERNAL_CHECKS=1") set(NETDATA_SOURCE_FILES src/appconfig.c diff --git a/src/common.c b/src/common.c index 3205b46b..64d50f63 100644 --- a/src/common.c +++ b/src/common.c @@ -14,31 +14,141 @@ volatile sig_atomic_t netdata_exit = 0; // its lifetime), these can be used to override the default system allocation // routines. +#ifdef NETDATA_LOG_ALLOCATIONS +static struct memory_statistics { + size_t malloc_calls_made; + size_t calloc_calls_made; + size_t realloc_calls_made; + size_t strdup_calls_made; + size_t free_calls_made; + size_t memory_calls_made; + size_t allocated_memory; + size_t mmapped_memory; +} memory_statistics = { + 0, 0, 0, 0, 0, 0, 0, 0 +}; + +static inline void print_allocations(const char *file, const char *function, const unsigned long line) { + static struct memory_statistics old = { 0, 0, 0, 0, 0, 0, 0, 0 }; + + //if(unlikely(!(memory_statistics.memory_calls_made % 5))) { + fprintf(stderr, "(%04lu@%-10.10s:%-15.15s): Allocated %zu KB (+%zu B), mmapped %zu KB (+%zu B): malloc %zu (+%zu), calloc %zu (+%zu), realloc %zu (+%zu), strdup %zu (+%zu), free %zu (+%zu)\n", + line, file, function, + (memory_statistics.allocated_memory + 512) / 1024, memory_statistics.allocated_memory - old.allocated_memory, + (memory_statistics.mmapped_memory + 512) / 1024, memory_statistics.mmapped_memory - old.mmapped_memory, + memory_statistics.malloc_calls_made, memory_statistics.malloc_calls_made - old.malloc_calls_made, + memory_statistics.calloc_calls_made, memory_statistics.calloc_calls_made - old.calloc_calls_made, + memory_statistics.realloc_calls_made, memory_statistics.realloc_calls_made - old.realloc_calls_made, + memory_statistics.strdup_calls_made, memory_statistics.strdup_calls_made - old.strdup_calls_made, + memory_statistics.free_calls_made, memory_statistics.free_calls_made - old.free_calls_made + ); + + memcpy(&old, &memory_statistics, sizeof(struct memory_statistics)); + //} +} + +static inline void malloc_accounting(const char *file, const char *function, const unsigned long line, size_t size) { + memory_statistics.memory_calls_made++; + memory_statistics.malloc_calls_made++; + memory_statistics.allocated_memory += size; + print_allocations(file, function, line); +} + +static inline void mmap_accounting(size_t size) { + memory_statistics.memory_calls_made++; + memory_statistics.mmapped_memory += size; +} + +static inline void calloc_accounting(const char *file, const char *function, const unsigned long line, size_t size) { + memory_statistics.memory_calls_made++; + memory_statistics.calloc_calls_made++; + memory_statistics.allocated_memory += size; + print_allocations(file, function, line); +} + +static inline void realloc_accounting(const char *file, const char *function, const unsigned long line, void *ptr, size_t size) { + (void)ptr; + + memory_statistics.memory_calls_made++; + memory_statistics.realloc_calls_made++; + memory_statistics.allocated_memory += size; + print_allocations(file, function, line); +} + +static inline void strdup_accounting(const char *file, const char *function, const unsigned long line, const char *s) { + memory_statistics.memory_calls_made++; + memory_statistics.strdup_calls_made++; + memory_statistics.allocated_memory += strlen(s) + 1; + print_allocations(file, function, line); +} + +static inline void free_accounting(const char *file, const char *function, const unsigned long line, void *ptr) { + (void)file; + (void)function; + (void)line; + + if(likely(ptr)) { + memory_statistics.memory_calls_made++; + memory_statistics.free_calls_made++; + } +} +#endif + +#ifdef NETDATA_LOG_ALLOCATIONS +char *strdupz_int(const char *file, const char *function, const unsigned long line, const char *s) { + strdup_accounting(file, function, line, s); +#else char *strdupz(const char *s) { +#endif + char *t = strdup(s); if (unlikely(!t)) fatal("Cannot strdup() string '%s'", s); return t; } +#ifdef NETDATA_LOG_ALLOCATIONS +void *mallocz_int(const char *file, const char *function, const unsigned long line, size_t size) { + malloc_accounting(file, function, line, size); +#else void *mallocz(size_t size) { +#endif + void *p = malloc(size); if (unlikely(!p)) fatal("Cannot allocate %zu bytes of memory.", size); return p; } +#ifdef NETDATA_LOG_ALLOCATIONS +void *callocz_int(const char *file, const char *function, const unsigned long line, size_t nmemb, size_t size) { + calloc_accounting(file, function, line, nmemb * size); +#else void *callocz(size_t nmemb, size_t size) { +#endif + void *p = calloc(nmemb, size); if (unlikely(!p)) fatal("Cannot allocate %zu bytes of memory.", nmemb * size); return p; } +#ifdef NETDATA_LOG_ALLOCATIONS +void *reallocz_int(const char *file, const char *function, const unsigned long line, void *ptr, size_t size) { + realloc_accounting(file, function, line, ptr, size); +#else void *reallocz(void *ptr, size_t size) { +#endif + void *p = realloc(ptr, size); if (unlikely(!p)) fatal("Cannot re-allocate memory to %zu bytes.", size); return p; } +#ifdef NETDATA_LOG_ALLOCATIONS +void freez_int(const char *file, const char *function, const unsigned long line, void *ptr) { + free_accounting(file, function, line, ptr); +#else void freez(void *ptr) { +#endif + free(ptr); } @@ -775,6 +885,9 @@ void *mymmap(const char *filename, size_t size, int flags, int ksm) { mem = NULL; } else { +#ifdef NETDATA_LOG_ALLOCATIONS + mmap_accounting(size); +#endif int advise = MADV_SEQUENTIAL | MADV_DONTFORK; if (flags & MAP_SHARED) advise |= MADV_WILLNEED; @@ -804,6 +917,9 @@ void *mymmap(const char *filename, size_t size, int flags, int ksm) { mem = NULL; } else { +#ifdef NETDATA_LOG_ALLOCATIONS + mmap_accounting(size); +#endif if (lseek(fd, 0, SEEK_SET) == 0) { if (read(fd, mem, size) != (ssize_t) size) error("Cannot read from file '%s'", filename); diff --git a/src/common.h b/src/common.h index 8af318cc..1a928917 100644 --- a/src/common.h +++ b/src/common.h @@ -131,11 +131,25 @@ extern int vsnprintfz(char *dst, size_t n, const char *fmt, va_list args); extern int snprintfz(char *dst, size_t n, const char *fmt, ...) __attribute__ (( format (printf, 3, 4))); // memory allocation functions that handle failures +#ifdef NETDATA_LOG_ALLOCATIONS +#define strdupz(s) strdupz_int(__FILE__, __FUNCTION__, __LINE__, s) +#define callocz(nmemb, size) callocz_int(__FILE__, __FUNCTION__, __LINE__, nmemb, size) +#define mallocz(size) mallocz_int(__FILE__, __FUNCTION__, __LINE__, size) +#define reallocz(ptr, size) reallocz_int(__FILE__, __FUNCTION__, __LINE__, ptr, size) +#define freez(ptr) freez_int(__FILE__, __FUNCTION__, __LINE__, ptr) + +extern char *strdupz_int(const char *file, const char *function, const unsigned long line, const char *s); +extern void *callocz_int(const char *file, const char *function, const unsigned long line, size_t nmemb, size_t size); +extern void *mallocz_int(const char *file, const char *function, const unsigned long line, size_t size); +extern void *reallocz_int(const char *file, const char *function, const unsigned long line, void *ptr, size_t size); +extern void freez_int(const char *file, const char *function, const unsigned long line, void *ptr); +#else extern char *strdupz(const char *s); extern void *callocz(size_t nmemb, size_t size); extern void *mallocz(size_t size); -extern void freez(void *ptr); extern void *reallocz(void *ptr, size_t size); +extern void freez(void *ptr); +#endif extern void *mymmap(const char *filename, size_t size, int flags, int ksm); extern int savememory(const char *filename, void *mem, size_t size); diff --git a/src/proc_interrupts.c b/src/proc_interrupts.c index be3792f2..f277a5a9 100644 --- a/src/proc_interrupts.c +++ b/src/proc_interrupts.c @@ -21,8 +21,7 @@ static inline struct interrupt *get_interrupts_array(int lines, int cpus) { static struct interrupt *irrs = NULL; static int allocated = 0; - if(lines < allocated) return irrs; - else { + if(lines > allocated) { irrs = (struct interrupt *)reallocz(irrs, lines * recordsize(cpus)); allocated = lines; } diff --git a/src/proc_softirqs.c b/src/proc_softirqs.c index 701153e6..ebbbf2ae 100644 --- a/src/proc_softirqs.c +++ b/src/proc_softirqs.c @@ -21,8 +21,7 @@ static inline struct interrupt *get_interrupts_array(int lines, int cpus) { static struct interrupt *irrs = NULL; static int allocated = 0; - if(lines < allocated) return irrs; - else { + if(lines > allocated) { irrs = (struct interrupt *)reallocz(irrs, lines * recordsize(cpus)); allocated = lines; } diff --git a/src/rrd.c b/src/rrd.c index 14fbbaf6..f9b5d4e0 100644 --- a/src/rrd.c +++ b/src/rrd.c @@ -425,6 +425,27 @@ void rrdset_reset(RRDSET *st) memset(rd->values, 0, rd->entries * sizeof(storage_number)); } } +static long align_entries_to_pagesize(long entries) { + if(entries < 5) entries = 5; + if(entries > RRD_HISTORY_ENTRIES_MAX) entries = RRD_HISTORY_ENTRIES_MAX; + +#ifdef NETDATA_LOG_ALLOCATIONS + long page = (size_t)sysconf(_SC_PAGESIZE); + + long size = sizeof(RRDDIM) + entries * sizeof(storage_number); + if(size % page) { + size -= (size % page); + size += page; + + long n = (size - sizeof(RRDDIM)) / sizeof(storage_number); + return n; + } + + return entries; +#else + return entries; +#endif +} RRDSET *rrdset_create(const char *type, const char *id, const char *name, const char *family, const char *context, const char *title, const char *units, long priority, int update_every, int chart_type) { @@ -450,9 +471,9 @@ RRDSET *rrdset_create(const char *type, const char *id, const char *name, const return st; } - long entries = config_get_number(fullid, "history", rrd_default_history_entries); - if(entries < 5) entries = config_set_number(fullid, "history", 5); - if(entries > RRD_HISTORY_ENTRIES_MAX) entries = config_set_number(fullid, "history", RRD_HISTORY_ENTRIES_MAX); + long rentries = config_get_number(fullid, "history", rrd_default_history_entries); + long entries = align_entries_to_pagesize(rentries); + if(entries != rentries) entries = config_set_number(fullid, "history", entries); int enabled = config_get_boolean(fullid, "enabled", 1); if(!enabled) entries = 5; -- 2.39.2