6 #if defined(__FreeBSD__) || defined(__APPLE__)
8 # define MADV_DONTFORK INHERIT_NONE
9 #endif /* __FreeBSD__ || __APPLE__*/
11 char *netdata_configured_hostname = NULL;
12 char *netdata_configured_config_dir = NULL;
13 char *netdata_configured_log_dir = NULL;
14 char *netdata_configured_plugins_dir = NULL;
15 char *netdata_configured_web_dir = NULL;
16 char *netdata_configured_cache_dir = NULL;
17 char *netdata_configured_varlib_dir = NULL;
18 char *netdata_configured_home_dir = NULL;
19 char *netdata_configured_host_prefix = NULL;
23 volatile sig_atomic_t netdata_exit = 0;
24 const char *os_type = NETDATA_OS_TYPE;
25 const char *program_version = VERSION;
27 // ----------------------------------------------------------------------------
28 // memory allocation functions that handle failures
30 // although netdata does not use memory allocations too often (netdata tries to
31 // maintain its memory footprint stable during runtime, i.e. all buffers are
32 // allocated during initialization and are adapted to current use throughout
33 // its lifetime), these can be used to override the default system allocation
36 #ifdef NETDATA_LOG_ALLOCATIONS
37 static struct memory_statistics {
38 volatile size_t malloc_calls_made;
39 volatile size_t calloc_calls_made;
40 volatile size_t realloc_calls_made;
41 volatile size_t strdup_calls_made;
42 volatile size_t free_calls_made;
43 volatile size_t memory_calls_made;
44 volatile size_t allocated_memory;
45 volatile size_t mmapped_memory;
48 static inline void print_allocations(const char *file, const char *function, const unsigned long line) {
49 static struct memory_statistics old = { 0, 0, 0, 0, 0, 0, 0, 0 };
51 //if(unlikely(!(memory_statistics.memory_calls_made % 5))) {
52 fprintf(stderr, "(%04lu@%-10.10s:%-15.15s): Allocated %zu KB (+%zu B), mmapped %zu KB (+%zu B): malloc %zu (+%zu), calloc %zu (+%zu), realloc %zu (+%zu), strdup %zu (+%zu), free %zu (+%zu)\n",
54 (memory_statistics.allocated_memory + 512) / 1024, memory_statistics.allocated_memory - old.allocated_memory,
55 (memory_statistics.mmapped_memory + 512) / 1024, memory_statistics.mmapped_memory - old.mmapped_memory,
56 memory_statistics.malloc_calls_made, memory_statistics.malloc_calls_made - old.malloc_calls_made,
57 memory_statistics.calloc_calls_made, memory_statistics.calloc_calls_made - old.calloc_calls_made,
58 memory_statistics.realloc_calls_made, memory_statistics.realloc_calls_made - old.realloc_calls_made,
59 memory_statistics.strdup_calls_made, memory_statistics.strdup_calls_made - old.strdup_calls_made,
60 memory_statistics.free_calls_made, memory_statistics.free_calls_made - old.free_calls_made
63 memcpy(&old, &memory_statistics, sizeof(struct memory_statistics));
67 static inline void malloc_accounting(const char *file, const char *function, const unsigned long line, size_t size) {
68 #if defined(HAVE_C___ATOMIC) && !defined(NETDATA_NO_ATOMIC_INSTRUCTIONS)
69 __atomic_fetch_add(&memory_statistics.memory_calls_made, 1, __ATOMIC_SEQ_CST);
70 __atomic_fetch_add(&memory_statistics.malloc_calls_made, 1, __ATOMIC_SEQ_CST);
71 __atomic_fetch_add(&memory_statistics.allocated_memory, size, __ATOMIC_SEQ_CST);
73 // this is for debugging - we don't care locking it
74 memory_statistics.memory_calls_made++;
75 memory_statistics.malloc_calls_made++;
76 memory_statistics.allocated_memory += size;
78 print_allocations(file, function, line);
81 static inline void mmap_accounting(size_t size) {
82 #if defined(HAVE_C___ATOMIC) && !defined(NETDATA_NO_ATOMIC_INSTRUCTIONS)
83 __atomic_fetch_add(&memory_statistics.malloc_calls_made, 1, __ATOMIC_SEQ_CST);
84 __atomic_fetch_add(&memory_statistics.mmapped_memory, size, __ATOMIC_SEQ_CST);
86 // this is for debugging - we don't care locking it
87 memory_statistics.memory_calls_made++;
88 memory_statistics.mmapped_memory += size;
92 static inline void calloc_accounting(const char *file, const char *function, const unsigned long line, size_t size) {
93 #if defined(HAVE_C___ATOMIC) && !defined(NETDATA_NO_ATOMIC_INSTRUCTIONS)
94 __atomic_fetch_add(&memory_statistics.memory_calls_made, 1, __ATOMIC_SEQ_CST);
95 __atomic_fetch_add(&memory_statistics.calloc_calls_made, 1, __ATOMIC_SEQ_CST);
96 __atomic_fetch_add(&memory_statistics.allocated_memory, size, __ATOMIC_SEQ_CST);
98 // this is for debugging - we don't care locking it
99 memory_statistics.memory_calls_made++;
100 memory_statistics.calloc_calls_made++;
101 memory_statistics.allocated_memory += size;
103 print_allocations(file, function, line);
106 static inline void realloc_accounting(const char *file, const char *function, const unsigned long line, void *ptr, size_t size) {
109 #if defined(HAVE_C___ATOMIC) && !defined(NETDATA_NO_ATOMIC_INSTRUCTIONS)
110 __atomic_fetch_add(&memory_statistics.memory_calls_made, 1, __ATOMIC_SEQ_CST);
111 __atomic_fetch_add(&memory_statistics.realloc_calls_made, 1, __ATOMIC_SEQ_CST);
112 __atomic_fetch_add(&memory_statistics.allocated_memory, size, __ATOMIC_SEQ_CST);
114 // this is for debugging - we don't care locking it
115 memory_statistics.memory_calls_made++;
116 memory_statistics.realloc_calls_made++;
117 memory_statistics.allocated_memory += size;
119 print_allocations(file, function, line);
122 static inline void strdup_accounting(const char *file, const char *function, const unsigned long line, const char *s) {
123 size_t size = strlen(s) + 1;
125 #if defined(HAVE_C___ATOMIC) && !defined(NETDATA_NO_ATOMIC_INSTRUCTIONS)
126 __atomic_fetch_add(&memory_statistics.memory_calls_made, 1, __ATOMIC_SEQ_CST);
127 __atomic_fetch_add(&memory_statistics.strdup_calls_made, 1, __ATOMIC_SEQ_CST);
128 __atomic_fetch_add(&memory_statistics.allocated_memory, size, __ATOMIC_SEQ_CST);
130 // this is for debugging - we don't care locking it
131 memory_statistics.memory_calls_made++;
132 memory_statistics.strdup_calls_made++;
133 memory_statistics.allocated_memory += size;
135 print_allocations(file, function, line);
138 static inline void free_accounting(const char *file, const char *function, const unsigned long line, void *ptr) {
144 #if defined(HAVE_C___ATOMIC) && !defined(NETDATA_NO_ATOMIC_INSTRUCTIONS)
145 __atomic_fetch_add(&memory_statistics.memory_calls_made, 1, __ATOMIC_SEQ_CST);
146 __atomic_fetch_add(&memory_statistics.free_calls_made, 1, __ATOMIC_SEQ_CST);
148 // this is for debugging - we don't care locking it
149 memory_statistics.memory_calls_made++;
150 memory_statistics.free_calls_made++;
156 #ifdef NETDATA_LOG_ALLOCATIONS
157 char *strdupz_int(const char *file, const char *function, const unsigned long line, const char *s) {
158 strdup_accounting(file, function, line, s);
160 char *strdupz(const char *s) {
164 if (unlikely(!t)) fatal("Cannot strdup() string '%s'", s);
168 #ifdef NETDATA_LOG_ALLOCATIONS
169 void *mallocz_int(const char *file, const char *function, const unsigned long line, size_t size) {
170 malloc_accounting(file, function, line, size);
172 void *mallocz(size_t size) {
175 void *p = malloc(size);
176 if (unlikely(!p)) fatal("Cannot allocate %zu bytes of memory.", size);
180 #ifdef NETDATA_LOG_ALLOCATIONS
181 void *callocz_int(const char *file, const char *function, const unsigned long line, size_t nmemb, size_t size) {
182 calloc_accounting(file, function, line, nmemb * size);
184 void *callocz(size_t nmemb, size_t size) {
187 void *p = calloc(nmemb, size);
188 if (unlikely(!p)) fatal("Cannot allocate %zu bytes of memory.", nmemb * size);
192 #ifdef NETDATA_LOG_ALLOCATIONS
193 void *reallocz_int(const char *file, const char *function, const unsigned long line, void *ptr, size_t size) {
194 realloc_accounting(file, function, line, ptr, size);
196 void *reallocz(void *ptr, size_t size) {
199 void *p = realloc(ptr, size);
200 if (unlikely(!p)) fatal("Cannot re-allocate memory to %zu bytes.", size);
204 #ifdef NETDATA_LOG_ALLOCATIONS
205 void freez_int(const char *file, const char *function, const unsigned long line, void *ptr) {
206 free_accounting(file, function, line, ptr);
208 void freez(void *ptr) {
214 void json_escape_string(char *dst, const char *src, size_t size) {
216 char *d = dst, *e = &dst[size - 1];
218 for(t = src; *t && d < e ;t++) {
219 if(unlikely(*t == '\\' || *t == '"')) {
220 if(unlikely(d + 1 >= e)) break;
229 int sleep_usec(usec_t usec) {
231 #ifndef NETDATA_WITH_USLEEP
232 // we expect microseconds (1.000.000 per second)
233 // but timespec is nanoseconds (1.000.000.000 per second)
234 struct timespec rem, req = {
235 .tv_sec = (time_t) (usec / 1000000),
236 .tv_nsec = (suseconds_t) ((usec % 1000000) * 1000)
239 while (nanosleep(&req, &rem) == -1) {
240 if (likely(errno == EINTR)) {
241 debug(D_SYSTEM, "nanosleep() interrupted (while sleeping for %llu microseconds).", usec);
242 req.tv_sec = rem.tv_sec;
243 req.tv_nsec = rem.tv_nsec;
245 error("Cannot nanosleep() for %llu microseconds.", usec);
252 int ret = usleep(usec);
253 if(unlikely(ret == -1 && errno == EINVAL)) {
254 // on certain systems, usec has to be up to 999999
256 int counter = usec / 999999;
260 usleep(usec % 999999);
263 error("Cannot usleep() for %llu microseconds.", usec);
269 error("usleep() failed for %llu microseconds.", usec);
275 unsigned char netdata_map_chart_names[256] = {
368 [92] = '/', // backslash
534 // make sure the supplied string
535 // is good for a netdata chart/dimension ID/NAME
536 void netdata_fix_chart_name(char *s) {
537 while ((*s = netdata_map_chart_names[(unsigned char) *s])) s++;
540 unsigned char netdata_map_chart_ids[256] = {
633 [92] = '/', // backslash
799 // make sure the supplied string
800 // is good for a netdata chart/dimension ID/NAME
801 void netdata_fix_chart_id(char *s) {
802 while ((*s = netdata_map_chart_ids[(unsigned char) *s])) s++;
806 // http://stackoverflow.com/questions/7666509/hash-function-for-string
807 uint32_t simple_hash(const char *name)
809 const char *s = name;
810 uint32_t hash = 5381;
813 while((i = *s++)) hash = ((hash << 5) + hash) + i;
815 // fprintf(stderr, "HASH: %lu %s\n", hash, name);
822 // http://isthe.com/chongo/tech/comp/fnv/#FNV-1a
823 uint32_t simple_hash(const char *name) {
824 unsigned char *s = (unsigned char *) name;
825 uint32_t hval = 0x811c9dc5;
829 // multiply by the 32 bit FNV magic prime mod 2^32
830 // NOTE: No need to optimize with left shifts.
831 // GCC will use imul instruction anyway.
832 // Tested with 'gcc -O3 -S'
833 //hval += (hval<<1) + (hval<<4) + (hval<<7) + (hval<<8) + (hval<<24);
836 // xor the bottom with the current octet
837 hval ^= (uint32_t) *s++;
840 // fprintf(stderr, "HASH: %u = %s\n", hval, name);
844 uint32_t simple_uhash(const char *name) {
845 unsigned char *s = (unsigned char *) name;
846 uint32_t hval = 0x811c9dc5, c;
850 if (unlikely(c >= 'A' && c <= 'Z')) c += 'a' - 'A';
859 // http://eternallyconfuzzled.com/tuts/algorithms/jsw_tut_hashing.aspx
860 // one at a time hash
861 uint32_t simple_hash(const char *name) {
862 unsigned char *s = (unsigned char *)name;
875 // fprintf(stderr, "HASH: %u = %s\n", h, name);
881 void strreverse(char *begin, char *end) {
882 while (end > begin) {
890 char *mystrsep(char **ptr, char *s) {
892 while (p && !p[0] && *ptr) p = strsep(ptr, s);
896 char *trim(char *s) {
897 // skip leading spaces
898 // and 'comments' as well!?
899 while (*s && isspace(*s)) s++;
900 if (!*s || *s == '#') return NULL;
902 // skip tailing spaces
903 // this way is way faster. Writes only one NUL char.
904 ssize_t l = strlen(s);
907 while (p > s && isspace(*p)) p--;
911 if (!*s) return NULL;
916 void *mymmap(const char *filename, size_t size, int flags, int ksm) {
917 #ifndef MADV_MERGEABLE
920 static int log_madvise_1 = 1;
921 #ifdef MADV_MERGEABLE
922 static int log_madvise_2 = 1, log_madvise_3 = 1;
927 int fd = open(filename, O_RDWR | O_CREAT | O_NOATIME, 0664);
929 if (lseek(fd, size, SEEK_SET) == (off_t) size) {
930 if (write(fd, "", 1) == 1) {
931 if (ftruncate(fd, size))
932 error("Cannot truncate file '%s' to size %zu. Will use the larger file.", filename, size);
934 #ifdef MADV_MERGEABLE
935 if (flags & MAP_SHARED || !enable_ksm || !ksm) {
937 mem = mmap(NULL, size, PROT_READ | PROT_WRITE, flags, fd, 0);
938 if (mem == MAP_FAILED) {
939 error("Cannot allocate SHARED memory for file '%s'.", filename);
943 #ifdef NETDATA_LOG_ALLOCATIONS
944 mmap_accounting(size);
946 int advise = MADV_SEQUENTIAL | MADV_DONTFORK;
947 if (flags & MAP_SHARED) advise |= MADV_WILLNEED;
949 if (madvise(mem, size, advise) != 0 && log_madvise_1) {
950 error("Cannot advise the kernel about the memory usage of file '%s'.", filename);
954 #ifdef MADV_MERGEABLE
958 // test - load the file into memory
959 mem = calloc(1, size);
961 if(lseek(fd, 0, SEEK_SET) == 0) {
962 if(read(fd, mem, size) != (ssize_t)size)
963 error("Cannot read from file '%s'", filename);
966 error("Cannot seek to beginning of file '%s'.", filename);
969 mem = mmap(NULL, size, PROT_READ | PROT_WRITE, flags | MAP_ANONYMOUS, -1, 0);
970 if (mem == MAP_FAILED) {
971 error("Cannot allocate PRIVATE ANONYMOUS memory for KSM for file '%s'.", filename);
975 #ifdef NETDATA_LOG_ALLOCATIONS
976 mmap_accounting(size);
978 if (lseek(fd, 0, SEEK_SET) == 0) {
979 if (read(fd, mem, size) != (ssize_t) size)
980 error("Cannot read from file '%s'", filename);
982 error("Cannot seek to beginning of file '%s'.", filename);
984 // don't use MADV_SEQUENTIAL|MADV_DONTFORK, they disable MADV_MERGEABLE
985 if (madvise(mem, size, MADV_SEQUENTIAL | MADV_DONTFORK) != 0 && log_madvise_2) {
986 error("Cannot advise the kernel about the memory usage (MADV_SEQUENTIAL|MADV_DONTFORK) of file '%s'.",
991 if (madvise(mem, size, MADV_MERGEABLE) != 0 && log_madvise_3) {
992 error("Cannot advise the kernel about the memory usage (MADV_MERGEABLE) of file '%s'.",
1001 error("Cannot write to file '%s' at position %zu.", filename, size);
1004 error("Cannot seek file '%s' to size %zu.", filename, size);
1009 error("Cannot create/open file '%s'.", filename);
1014 int savememory(const char *filename, void *mem, size_t size) {
1015 char tmpfilename[FILENAME_MAX + 1];
1017 snprintfz(tmpfilename, FILENAME_MAX, "%s.%ld.tmp", filename, (long) getpid());
1019 int fd = open(tmpfilename, O_RDWR | O_CREAT | O_NOATIME, 0664);
1021 error("Cannot create/open file '%s'.", filename);
1025 if (write(fd, mem, size) != (ssize_t) size) {
1026 error("Cannot write to file '%s' %ld bytes.", filename, (long) size);
1033 if (rename(tmpfilename, filename)) {
1034 error("Cannot rename '%s' to '%s'", tmpfilename, filename);
1041 int fd_is_valid(int fd) {
1042 return fcntl(fd, F_GETFD) != -1 || errno != EBADF;
1045 pid_t gettid(void) {
1047 return (pid_t)pthread_getthreadid_np();
1048 #elif defined(__APPLE__)
1049 uint64_t curthreadid;
1050 pthread_threadid_np(NULL, &curthreadid);
1051 return (pid_t)curthreadid;
1053 return (pid_t)syscall(SYS_gettid);
1054 #endif /* __FreeBSD__, __APPLE__*/
1057 char *fgets_trim_len(char *buf, size_t buf_size, FILE *fp, size_t *len) {
1058 char *s = fgets(buf, (int)buf_size, fp);
1059 if (!s) return NULL;
1063 // find the string end
1064 while (*++t != '\0');
1066 // trim trailing spaces/newlines/tabs
1067 while (--t > s && *t == '\n')
1077 int vsnprintfz(char *dst, size_t n, const char *fmt, va_list args) {
1078 int size = vsnprintf(dst, n, fmt, args);
1080 if (unlikely((size_t) size > n)) {
1089 int snprintfz(char *dst, size_t n, const char *fmt, ...) {
1092 va_start(args, fmt);
1093 int ret = vsnprintfz(dst, n, fmt, args);
1099 // ----------------------------------------------------------------------------
1101 // to retrieve settings of the system
1104 long get_system_cpus(void) {
1108 int32_t tmp_processors;
1110 if (unlikely(GETSYSCTL_BY_NAME("hw.logicalcpu", tmp_processors))) {
1111 error("Assuming system has %d processors.", processors);
1113 processors = tmp_processors;
1118 int32_t tmp_processors;
1120 if (unlikely(GETSYSCTL_BY_NAME("hw.ncpu", tmp_processors))) {
1121 error("Assuming system has %d processors.", processors);
1123 processors = tmp_processors;
1129 char filename[FILENAME_MAX + 1];
1130 snprintfz(filename, FILENAME_MAX, "%s/proc/stat", netdata_configured_host_prefix);
1132 procfile *ff = procfile_open(filename, NULL, PROCFILE_FLAG_DEFAULT);
1134 error("Cannot open file '%s'. Assuming system has %d processors.", filename, processors);
1138 ff = procfile_readall(ff);
1140 error("Cannot open file '%s'. Assuming system has %d processors.", filename, processors);
1146 for(i = 0; i < procfile_lines(ff); i++) {
1147 if(!procfile_linewords(ff, i)) continue;
1149 if(strncmp(procfile_lineword(ff, i, 0), "cpu", 3) == 0) processors++;
1152 if(processors < 1) processors = 1;
1156 debug(D_SYSTEM, "System has %d processors.", processors);
1159 #endif /* __APPLE__, __FreeBSD__ */
1162 pid_t pid_max = 32768;
1163 pid_t get_system_pid_max(void) {
1165 // As we currently do not know a solution to query pid_max from the os
1166 // we use the number defined in bsd/sys/proc_internal.h in XNU sources
1170 int32_t tmp_pid_max;
1172 if (unlikely(GETSYSCTL_BY_NAME("kern.pid_max", tmp_pid_max))) {
1174 error("Assuming system's maximum pid is %d.", pid_max);
1176 pid_max = tmp_pid_max;
1182 static char read = 0;
1183 if(unlikely(read)) return pid_max;
1186 char filename[FILENAME_MAX + 1];
1187 snprintfz(filename, FILENAME_MAX, "%s/proc/sys/kernel/pid_max", netdata_configured_host_prefix);
1189 unsigned long long max = 0;
1190 if(read_single_number_file(filename, &max) != 0) {
1191 error("Cannot open file '%s'. Assuming system supports %d pids.", filename, pid_max);
1196 error("Cannot parse file '%s'. Assuming system supports %d pids.", filename, pid_max);
1200 pid_max = (pid_t) max;
1203 #endif /* __APPLE__, __FreeBSD__ */
1207 void get_system_HZ(void) {
1210 if ((ticks = sysconf(_SC_CLK_TCK)) == -1) {
1211 error("Cannot get system clock ticks");
1214 hz = (unsigned int) ticks;
1218 // poor man cycle counting
1219 static unsigned long tsc;
1220 void begin_tsc(void) {
1222 asm volatile ("cpuid\nrdtsc" : "=a" (a), "=d" (d) : "0" (0) : "ebx", "ecx");
1223 tsc = ((unsigned long)d << 32) | (unsigned long)a;
1225 unsigned long end_tsc(void) {
1227 asm volatile ("rdtscp" : "=a" (a), "=d" (d) : : "ecx");
1228 return (((unsigned long)d << 32) | (unsigned long)a) - tsc;