5 # define MADV_DONTFORK INHERIT_NONE
6 #endif /* __FreeBSD__ */
8 char *global_host_prefix = "";
11 volatile sig_atomic_t netdata_exit = 0;
13 // ----------------------------------------------------------------------------
14 // memory allocation functions that handle failures
16 // although netdata does not use memory allocations too often (netdata tries to
17 // maintain its memory footprint stable during runtime, i.e. all buffers are
18 // allocated during initialization and are adapted to current use throughout
19 // its lifetime), these can be used to override the default system allocation
22 #ifdef NETDATA_LOG_ALLOCATIONS
23 static struct memory_statistics {
24 volatile size_t malloc_calls_made;
25 volatile size_t calloc_calls_made;
26 volatile size_t realloc_calls_made;
27 volatile size_t strdup_calls_made;
28 volatile size_t free_calls_made;
29 volatile size_t memory_calls_made;
30 volatile size_t allocated_memory;
31 volatile size_t mmapped_memory;
34 static inline void print_allocations(const char *file, const char *function, const unsigned long line) {
35 static struct memory_statistics old = { 0, 0, 0, 0, 0, 0, 0, 0 };
37 //if(unlikely(!(memory_statistics.memory_calls_made % 5))) {
38 fprintf(stderr, "(%04lu@%-10.10s:%-15.15s): Allocated %zu KB (+%zu B), mmapped %zu KB (+%zu B): malloc %zu (+%zu), calloc %zu (+%zu), realloc %zu (+%zu), strdup %zu (+%zu), free %zu (+%zu)\n",
40 (memory_statistics.allocated_memory + 512) / 1024, memory_statistics.allocated_memory - old.allocated_memory,
41 (memory_statistics.mmapped_memory + 512) / 1024, memory_statistics.mmapped_memory - old.mmapped_memory,
42 memory_statistics.malloc_calls_made, memory_statistics.malloc_calls_made - old.malloc_calls_made,
43 memory_statistics.calloc_calls_made, memory_statistics.calloc_calls_made - old.calloc_calls_made,
44 memory_statistics.realloc_calls_made, memory_statistics.realloc_calls_made - old.realloc_calls_made,
45 memory_statistics.strdup_calls_made, memory_statistics.strdup_calls_made - old.strdup_calls_made,
46 memory_statistics.free_calls_made, memory_statistics.free_calls_made - old.free_calls_made
49 memcpy(&old, &memory_statistics, sizeof(struct memory_statistics));
53 static inline void malloc_accounting(const char *file, const char *function, const unsigned long line, size_t size) {
54 #if defined(HAVE_C___ATOMIC) && !defined(NETDATA_NO_ATOMIC_INSTRUCTIONS)
55 __atomic_fetch_add(&memory_statistics.memory_calls_made, 1, __ATOMIC_SEQ_CST);
56 __atomic_fetch_add(&memory_statistics.malloc_calls_made, 1, __ATOMIC_SEQ_CST);
57 __atomic_fetch_add(&memory_statistics.allocated_memory, size, __ATOMIC_SEQ_CST);
59 // this is for debugging - we don't care locking it
60 memory_statistics.memory_calls_made++;
61 memory_statistics.malloc_calls_made++;
62 memory_statistics.allocated_memory += size;
64 print_allocations(file, function, line);
67 static inline void mmap_accounting(size_t size) {
68 #if defined(HAVE_C___ATOMIC) && !defined(NETDATA_NO_ATOMIC_INSTRUCTIONS)
69 __atomic_fetch_add(&memory_statistics.malloc_calls_made, 1, __ATOMIC_SEQ_CST);
70 __atomic_fetch_add(&memory_statistics.mmapped_memory, size, __ATOMIC_SEQ_CST);
72 // this is for debugging - we don't care locking it
73 memory_statistics.memory_calls_made++;
74 memory_statistics.mmapped_memory += size;
78 static inline void calloc_accounting(const char *file, const char *function, const unsigned long line, size_t size) {
79 #if defined(HAVE_C___ATOMIC) && !defined(NETDATA_NO_ATOMIC_INSTRUCTIONS)
80 __atomic_fetch_add(&memory_statistics.memory_calls_made, 1, __ATOMIC_SEQ_CST);
81 __atomic_fetch_add(&memory_statistics.calloc_calls_made, 1, __ATOMIC_SEQ_CST);
82 __atomic_fetch_add(&memory_statistics.allocated_memory, size, __ATOMIC_SEQ_CST);
84 // this is for debugging - we don't care locking it
85 memory_statistics.memory_calls_made++;
86 memory_statistics.calloc_calls_made++;
87 memory_statistics.allocated_memory += size;
89 print_allocations(file, function, line);
92 static inline void realloc_accounting(const char *file, const char *function, const unsigned long line, void *ptr, size_t size) {
95 #if defined(HAVE_C___ATOMIC) && !defined(NETDATA_NO_ATOMIC_INSTRUCTIONS)
96 __atomic_fetch_add(&memory_statistics.memory_calls_made, 1, __ATOMIC_SEQ_CST);
97 __atomic_fetch_add(&memory_statistics.realloc_calls_made, 1, __ATOMIC_SEQ_CST);
98 __atomic_fetch_add(&memory_statistics.allocated_memory, size, __ATOMIC_SEQ_CST);
100 // this is for debugging - we don't care locking it
101 memory_statistics.memory_calls_made++;
102 memory_statistics.realloc_calls_made++;
103 memory_statistics.allocated_memory += size;
105 print_allocations(file, function, line);
108 static inline void strdup_accounting(const char *file, const char *function, const unsigned long line, const char *s) {
109 size_t size = strlen(s) + 1;
111 #if defined(HAVE_C___ATOMIC) && !defined(NETDATA_NO_ATOMIC_INSTRUCTIONS)
112 __atomic_fetch_add(&memory_statistics.memory_calls_made, 1, __ATOMIC_SEQ_CST);
113 __atomic_fetch_add(&memory_statistics.strdup_calls_made, 1, __ATOMIC_SEQ_CST);
114 __atomic_fetch_add(&memory_statistics.allocated_memory, size, __ATOMIC_SEQ_CST);
116 // this is for debugging - we don't care locking it
117 memory_statistics.memory_calls_made++;
118 memory_statistics.strdup_calls_made++;
119 memory_statistics.allocated_memory += size;
121 print_allocations(file, function, line);
124 static inline void free_accounting(const char *file, const char *function, const unsigned long line, void *ptr) {
130 #if defined(HAVE_C___ATOMIC) && !defined(NETDATA_NO_ATOMIC_INSTRUCTIONS)
131 __atomic_fetch_add(&memory_statistics.memory_calls_made, 1, __ATOMIC_SEQ_CST);
132 __atomic_fetch_add(&memory_statistics.free_calls_made, 1, __ATOMIC_SEQ_CST);
134 // this is for debugging - we don't care locking it
135 memory_statistics.memory_calls_made++;
136 memory_statistics.free_calls_made++;
142 #ifdef NETDATA_LOG_ALLOCATIONS
143 char *strdupz_int(const char *file, const char *function, const unsigned long line, const char *s) {
144 strdup_accounting(file, function, line, s);
146 char *strdupz(const char *s) {
150 if (unlikely(!t)) fatal("Cannot strdup() string '%s'", s);
154 #ifdef NETDATA_LOG_ALLOCATIONS
155 void *mallocz_int(const char *file, const char *function, const unsigned long line, size_t size) {
156 malloc_accounting(file, function, line, size);
158 void *mallocz(size_t size) {
161 void *p = malloc(size);
162 if (unlikely(!p)) fatal("Cannot allocate %zu bytes of memory.", size);
166 #ifdef NETDATA_LOG_ALLOCATIONS
167 void *callocz_int(const char *file, const char *function, const unsigned long line, size_t nmemb, size_t size) {
168 calloc_accounting(file, function, line, nmemb * size);
170 void *callocz(size_t nmemb, size_t size) {
173 void *p = calloc(nmemb, size);
174 if (unlikely(!p)) fatal("Cannot allocate %zu bytes of memory.", nmemb * size);
178 #ifdef NETDATA_LOG_ALLOCATIONS
179 void *reallocz_int(const char *file, const char *function, const unsigned long line, void *ptr, size_t size) {
180 realloc_accounting(file, function, line, ptr, size);
182 void *reallocz(void *ptr, size_t size) {
185 void *p = realloc(ptr, size);
186 if (unlikely(!p)) fatal("Cannot re-allocate memory to %zu bytes.", size);
190 #ifdef NETDATA_LOG_ALLOCATIONS
191 void freez_int(const char *file, const char *function, const unsigned long line, void *ptr) {
192 free_accounting(file, function, line, ptr);
194 void freez(void *ptr) {
200 int sleep_usec(usec_t usec) {
202 #ifndef NETDATA_WITH_USLEEP
203 // we expect microseconds (1.000.000 per second)
204 // but timespec is nanoseconds (1.000.000.000 per second)
205 struct timespec rem, req = {
206 .tv_sec = (time_t) (usec / 1000000),
207 .tv_nsec = (suseconds_t) ((usec % 1000000) * 1000)
210 while (nanosleep(&req, &rem) == -1) {
211 if (likely(errno == EINTR)) {
212 debug(D_SYSTEM, "nanosleep() interrupted (while sleeping for %llu microseconds).", usec);
213 req.tv_sec = rem.tv_sec;
214 req.tv_nsec = rem.tv_nsec;
216 error("Cannot nanosleep() for %llu microseconds.", usec);
223 int ret = usleep(usec);
224 if(unlikely(ret == -1 && errno == EINVAL)) {
225 // on certain systems, usec has to be up to 999999
227 int counter = usec / 999999;
231 usleep(usec % 999999);
234 error("Cannot usleep() for %llu microseconds.", usec);
240 error("usleep() failed for %llu microseconds.", usec);
246 unsigned char netdata_map_chart_names[256] = {
339 [92] = '/', // backslash
505 // make sure the supplied string
506 // is good for a netdata chart/dimension ID/NAME
507 void netdata_fix_chart_name(char *s) {
508 while ((*s = netdata_map_chart_names[(unsigned char) *s])) s++;
511 unsigned char netdata_map_chart_ids[256] = {
604 [92] = '/', // backslash
770 // make sure the supplied string
771 // is good for a netdata chart/dimension ID/NAME
772 void netdata_fix_chart_id(char *s) {
773 while ((*s = netdata_map_chart_ids[(unsigned char) *s])) s++;
777 // http://stackoverflow.com/questions/7666509/hash-function-for-string
778 uint32_t simple_hash(const char *name)
780 const char *s = name;
781 uint32_t hash = 5381;
784 while((i = *s++)) hash = ((hash << 5) + hash) + i;
786 // fprintf(stderr, "HASH: %lu %s\n", hash, name);
793 // http://isthe.com/chongo/tech/comp/fnv/#FNV-1a
794 uint32_t simple_hash(const char *name) {
795 unsigned char *s = (unsigned char *) name;
796 uint32_t hval = 0x811c9dc5;
800 // multiply by the 32 bit FNV magic prime mod 2^32
801 // NOTE: No need to optimize with left shifts.
802 // GCC will use imul instruction anyway.
803 // Tested with 'gcc -O3 -S'
804 //hval += (hval<<1) + (hval<<4) + (hval<<7) + (hval<<8) + (hval<<24);
807 // xor the bottom with the current octet
808 hval ^= (uint32_t) *s++;
811 // fprintf(stderr, "HASH: %u = %s\n", hval, name);
815 uint32_t simple_uhash(const char *name) {
816 unsigned char *s = (unsigned char *) name;
817 uint32_t hval = 0x811c9dc5, c;
821 if (unlikely(c >= 'A' && c <= 'Z')) c += 'a' - 'A';
829 // http://eternallyconfuzzled.com/tuts/algorithms/jsw_tut_hashing.aspx
830 // one at a time hash
831 uint32_t simple_hash(const char *name) {
832 unsigned char *s = (unsigned char *)name;
845 // fprintf(stderr, "HASH: %u = %s\n", h, name);
851 void strreverse(char *begin, char *end) {
852 while (end > begin) {
860 char *mystrsep(char **ptr, char *s) {
862 while (p && !p[0] && *ptr) p = strsep(ptr, s);
866 char *trim(char *s) {
867 // skip leading spaces
868 // and 'comments' as well!?
869 while (*s && isspace(*s)) s++;
870 if (!*s || *s == '#') return NULL;
872 // skip tailing spaces
873 // this way is way faster. Writes only one NUL char.
874 ssize_t l = strlen(s);
877 while (p > s && isspace(*p)) p--;
881 if (!*s) return NULL;
886 void *mymmap(const char *filename, size_t size, int flags, int ksm) {
887 static int log_madvise_1 = 1;
888 #ifdef MADV_MERGEABLE
889 static int log_madvise_2 = 1, log_madvise_3 = 1;
894 int fd = open(filename, O_RDWR | O_CREAT | O_NOATIME, 0664);
896 if (lseek(fd, size, SEEK_SET) == (off_t) size) {
897 if (write(fd, "", 1) == 1) {
898 if (ftruncate(fd, size))
899 error("Cannot truncate file '%s' to size %zu. Will use the larger file.", filename, size);
901 #ifdef MADV_MERGEABLE
902 if (flags & MAP_SHARED || !enable_ksm || !ksm) {
904 mem = mmap(NULL, size, PROT_READ | PROT_WRITE, flags, fd, 0);
905 if (mem == MAP_FAILED) {
906 error("Cannot allocate SHARED memory for file '%s'.", filename);
910 #ifdef NETDATA_LOG_ALLOCATIONS
911 mmap_accounting(size);
913 int advise = MADV_SEQUENTIAL | MADV_DONTFORK;
914 if (flags & MAP_SHARED) advise |= MADV_WILLNEED;
916 if (madvise(mem, size, advise) != 0 && log_madvise_1) {
917 error("Cannot advise the kernel about the memory usage of file '%s'.", filename);
921 #ifdef MADV_MERGEABLE
925 // test - load the file into memory
926 mem = calloc(1, size);
928 if(lseek(fd, 0, SEEK_SET) == 0) {
929 if(read(fd, mem, size) != (ssize_t)size)
930 error("Cannot read from file '%s'", filename);
933 error("Cannot seek to beginning of file '%s'.", filename);
936 mem = mmap(NULL, size, PROT_READ | PROT_WRITE, flags | MAP_ANONYMOUS, -1, 0);
937 if (mem == MAP_FAILED) {
938 error("Cannot allocate PRIVATE ANONYMOUS memory for KSM for file '%s'.", filename);
942 #ifdef NETDATA_LOG_ALLOCATIONS
943 mmap_accounting(size);
945 if (lseek(fd, 0, SEEK_SET) == 0) {
946 if (read(fd, mem, size) != (ssize_t) size)
947 error("Cannot read from file '%s'", filename);
949 error("Cannot seek to beginning of file '%s'.", filename);
951 // don't use MADV_SEQUENTIAL|MADV_DONTFORK, they disable MADV_MERGEABLE
952 if (madvise(mem, size, MADV_SEQUENTIAL | MADV_DONTFORK) != 0 && log_madvise_2) {
953 error("Cannot advise the kernel about the memory usage (MADV_SEQUENTIAL|MADV_DONTFORK) of file '%s'.",
958 if (madvise(mem, size, MADV_MERGEABLE) != 0 && log_madvise_3) {
959 error("Cannot advise the kernel about the memory usage (MADV_MERGEABLE) of file '%s'.",
968 error("Cannot write to file '%s' at position %zu.", filename, size);
971 error("Cannot seek file '%s' to size %zu.", filename, size);
976 error("Cannot create/open file '%s'.", filename);
981 int savememory(const char *filename, void *mem, size_t size) {
982 char tmpfilename[FILENAME_MAX + 1];
984 snprintfz(tmpfilename, FILENAME_MAX, "%s.%ld.tmp", filename, (long) getpid());
986 int fd = open(tmpfilename, O_RDWR | O_CREAT | O_NOATIME, 0664);
988 error("Cannot create/open file '%s'.", filename);
992 if (write(fd, mem, size) != (ssize_t) size) {
993 error("Cannot write to file '%s' %ld bytes.", filename, (long) size);
1000 if (rename(tmpfilename, filename)) {
1001 error("Cannot rename '%s' to '%s'", tmpfilename, filename);
1008 int fd_is_valid(int fd) {
1009 return fcntl(fd, F_GETFD) != -1 || errno != EBADF;
1012 pid_t gettid(void) {
1014 return (pid_t)pthread_getthreadid_np();
1016 return (pid_t)syscall(SYS_gettid);
1017 #endif /* __FreeBSD__ */
1020 char *fgets_trim_len(char *buf, size_t buf_size, FILE *fp, size_t *len) {
1021 char *s = fgets(buf, (int)buf_size, fp);
1022 if (!s) return NULL;
1026 // find the string end
1027 while (*++t != '\0');
1029 // trim trailing spaces/newlines/tabs
1030 while (--t > s && *t == '\n')
1040 char *strncpyz(char *dst, const char *src, size_t n) {
1051 int vsnprintfz(char *dst, size_t n, const char *fmt, va_list args) {
1052 int size = vsnprintf(dst, n, fmt, args);
1054 if (unlikely((size_t) size > n)) {
1063 int snprintfz(char *dst, size_t n, const char *fmt, ...) {
1066 va_start(args, fmt);
1067 int ret = vsnprintfz(dst, n, fmt, args);
1073 // ----------------------------------------------------------------------------
1075 // to retrieve settings of the system
1078 long get_system_cpus(void) {
1081 char filename[FILENAME_MAX + 1];
1082 snprintfz(filename, FILENAME_MAX, "%s/proc/stat", global_host_prefix);
1084 procfile *ff = procfile_open(filename, NULL, PROCFILE_FLAG_DEFAULT);
1086 error("Cannot open file '%s'. Assuming system has %d processors.", filename, processors);
1090 ff = procfile_readall(ff);
1092 error("Cannot open file '%s'. Assuming system has %d processors.", filename, processors);
1098 for(i = 0; i < procfile_lines(ff); i++) {
1099 if(!procfile_linewords(ff, i)) continue;
1101 if(strncmp(procfile_lineword(ff, i, 0), "cpu", 3) == 0) processors++;
1104 if(processors < 1) processors = 1;
1108 debug(D_SYSTEM, "System has %d processors.", processors);
1112 pid_t pid_max = 32768;
1113 pid_t get_system_pid_max(void) {
1114 char filename[FILENAME_MAX + 1];
1115 snprintfz(filename, FILENAME_MAX, "%s/proc/sys/kernel/pid_max", global_host_prefix);
1116 procfile *ff = procfile_open(filename, NULL, PROCFILE_FLAG_DEFAULT);
1118 error("Cannot open file '%s'. Assuming system supports %d pids.", filename, pid_max);
1122 ff = procfile_readall(ff);
1124 error("Cannot read file '%s'. Assuming system supports %d pids.", filename, pid_max);
1128 pid_max = (pid_t)atoi(procfile_lineword(ff, 0, 0));
1132 error("Cannot parse file '%s'. Assuming system supports %d pids.", filename, pid_max);
1137 debug(D_SYSTEM, "System supports %d pids.", pid_max);
1142 void get_system_HZ(void) {
1145 if ((ticks = sysconf(_SC_CLK_TCK)) == -1) {
1149 hz = (unsigned int) ticks;
1152 int read_single_number_file(const char *filename, unsigned long long *result) {
1153 char buffer[1024 + 1];
1155 int fd = open(filename, O_RDONLY, 0666);
1156 if(unlikely(fd == -1)) return 1;
1158 ssize_t r = read(fd, buffer, 1024);
1159 if(unlikely(r == -1)) {
1165 *result = strtoull(buffer, NULL, 0);