From a1abb2f3398730f13a0704133ee1e52b4b6561a9 Mon Sep 17 00:00:00 2001 From: Costa Tsaousis Date: Sun, 8 May 2016 22:43:44 +0300 Subject: [PATCH] registry: working prototype with load/save --- CMakeLists.txt | 2 +- configure.ac | 1 + netdata-installer.sh | 5 +- profile/benchmark-dictionary.c | 4 +- src/Makefile.am | 1 + src/common.c | 20 + src/common.h | 3 + src/dictionary.c | 34 +- src/dictionary.h | 3 +- src/main.c | 1 + src/registry.c | 730 +++++++++++++++++++++++---------- 11 files changed, 581 insertions(+), 223 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1c42a913..967df77e 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -91,7 +91,7 @@ set(APPS_PLUGIN_SOURCE_FILES include_directories(AFTER .) -add_definitions(-DHAVE_CONFIG_H -DNETDATA_INTERNAL_CHECKS=1 -DCACHE_DIR="/tmp" -DCONFIG_DIR="/tmp" -DLOG_DIR="/tmp" -DPLUGINS_DIR="/tmp" -DWEB_DIR="/tmp") +add_definitions(-DHAVE_CONFIG_H -DNETDATA_INTERNAL_CHECKS=1 -DCACHE_DIR="/tmp" -DCONFIG_DIR="/tmp" -DLOG_DIR="/tmp" -DPLUGINS_DIR="/tmp" -DWEB_DIR="/tmp" -DVARLIB_DIR="/tmp") add_executable(netdata ${NETDATA_SOURCE_FILES}) add_executable(apps.plugin ${APPS_PLUGIN_SOURCE_FILES}) diff --git a/configure.ac b/configure.ac index 7979bfbf..fd69e71d 100644 --- a/configure.ac +++ b/configure.ac @@ -143,6 +143,7 @@ fi AC_DEFINE_UNQUOTED([NETDATA_USER], ["${with_user}"], [use this user to drop privileged]) +AC_SUBST([varlibdir], ["\$(localstatedir)/lib/netdata"]) AC_SUBST([cachedir], ["\$(localstatedir)/cache/netdata"]) AC_SUBST([chartsdir], ["\$(libexecdir)/netdata/charts.d"]) AC_SUBST([nodedir], ["\$(libexecdir)/netdata/node.d"]) diff --git a/netdata-installer.sh b/netdata-installer.sh index cc27a61c..2125fd4f 100755 --- a/netdata-installer.sh +++ b/netdata-installer.sh @@ -120,6 +120,7 @@ It will be installed at these locations: - web files at ${NETDATA_PREFIX}/usr/share/netdata - plugins at ${NETDATA_PREFIX}/usr/libexec/netdata - cache files at ${NETDATA_PREFIX}/var/cache/netdata + - db files at ${NETDATA_PREFIX}/var/lib/netdata - log files at ${NETDATA_PREFIX}/var/log/netdata - pid file at ${NETDATA_PREFIX}/var/run @@ -407,6 +408,7 @@ defport=19999 NETDATA_PORT="$( config_option "port" ${defport} )" # directories +NETDATA_LIB_DIR="$( config_option "lib directory" "${NETDATA_PREFIX}/var/lib/netdata" )" NETDATA_CACHE_DIR="$( config_option "cache directory" "${NETDATA_PREFIX}/var/cache/netdata" )" NETDATA_WEB_DIR="$( config_option "web files directory" "${NETDATA_PREFIX}/usr/share/netdata/web" )" NETDATA_LOG_DIR="$( config_option "log directory" "${NETDATA_PREFIX}/var/log/netdata" )" @@ -425,7 +427,7 @@ if [ ! -d "${NETDATA_RUN_DIR}" ] fi echo >&2 "Fixing directories (user: ${NETDATA_USER})..." -for x in "${NETDATA_WEB_DIR}" "${NETDATA_CONF_DIR}" "${NETDATA_CACHE_DIR}" "${NETDATA_LOG_DIR}" +for x in "${NETDATA_WEB_DIR}" "${NETDATA_CONF_DIR}" "${NETDATA_CACHE_DIR}" "${NETDATA_LOG_DIR}" "${NETDATA_LIB_DIR}" do if [ ! -d "${x}" ] then @@ -716,6 +718,7 @@ else deletedir "${NETDATA_PREFIX}/etc/netdata" deletedir "${NETDATA_PREFIX}/usr/share/netdata" deletedir "${NETDATA_PREFIX}/usr/libexec/netdata" + deletedir "${NETDATA_PREFIX}/var/lib/netdata" deletedir "${NETDATA_PREFIX}/var/cache/netdata" deletedir "${NETDATA_PREFIX}/var/log/netdata" fi diff --git a/profile/benchmark-dictionary.c b/profile/benchmark-dictionary.c index 163f4425..8c414db8 100644 --- a/profile/benchmark-dictionary.c +++ b/profile/benchmark-dictionary.c @@ -22,14 +22,14 @@ struct myvalue { int main(int argc, char **argv) { if(argc || argv) {;} - DICTIONARY *dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED); + DICTIONARY *dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED|DICTIONARY_FLAG_NAME_CORRUPTION_CHECK); if(!dict) fatal("Cannot create dictionary."); struct rusage start, end; unsigned long long dt; char buf[100 + 1]; struct myvalue value, *v; - int i, max = 1000000, max2; + int i, max = 100000, max2; // ------------------------------------------------------------------------ diff --git a/src/Makefile.am b/src/Makefile.am index 9226e4c6..ef7d20fd 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -4,6 +4,7 @@ MAINTAINERCLEANFILES= $(srcdir)/Makefile.in AM_CPPFLAGS = \ + -DVARLIB_DIR="\"$(varlibdir)\"" \ -DCACHE_DIR="\"$(cachedir)\"" \ -DCONFIG_DIR="\"$(configdir)\"" \ -DLOG_DIR="\"$(logdir)\"" \ diff --git a/src/common.c b/src/common.c index ed04fdb9..ca4b8753 100644 --- a/src/common.c +++ b/src/common.c @@ -762,3 +762,23 @@ pid_t gettid(void) return syscall(SYS_gettid); } +char *fgets_trim_len(char *buf, size_t buf_size, FILE *fp, size_t *len) { + char *s = fgets(buf, buf_size, fp); + if(!s) return NULL; + + char *t = s; + if(*t != '\0') { + // find the string end + while (*++t != '\0'); + + // trim trailing spaces/newlines/tabs + while (--t > s && *t == '\n') + *t = '\0'; + } + + if(len) + *len = t - s + 1; + + return s; +} + diff --git a/src/common.h b/src/common.h index a028da7c..cc276560 100644 --- a/src/common.h +++ b/src/common.h @@ -1,5 +1,6 @@ #include #include +#include #ifndef NETDATA_COMMON_H #define NETDATA_COMMON_H 1 @@ -39,6 +40,8 @@ extern pid_t gettid(void); extern unsigned long long timems(void); +extern char *fgets_trim_len(char *buf, size_t buf_size, FILE *fp, size_t *len); + /* fix for alpine linux */ #ifndef RUSAGE_THREAD #ifdef RUSAGE_CHILDREN diff --git a/src/dictionary.c b/src/dictionary.c index b95a8f9c..b7c904ce 100644 --- a/src/dictionary.c +++ b/src/dictionary.c @@ -33,6 +33,7 @@ static inline NAME_VALUE *dictionary_name_value_index_find_nolock(DICTIONARY *di dict->searches++; avl_search(&(dict->values_index), (avl *)&tmp, name_value_iterator, (avl **)&result); + return result; } @@ -226,20 +227,37 @@ int dictionary_del(DICTIONARY *dict, const char *name) { } -static void dictionary_walker(avl *a, int (*callback)(void *entry, void *data), void *data) { - if(unlikely(!a)) return; +static int dictionary_walker(avl *a, int (*callback)(void *entry, void *data), void *data) { + int total = 0, ret = 0; - if(a->right) - dictionary_walker(a->right, callback, data); + if(a->right) { + ret = dictionary_walker(a->right, callback, data); + if(ret < 0) return ret; + total += ret; + } - callback(((NAME_VALUE *)a)->value, data); + ret = callback(((NAME_VALUE *)a)->value, data); + if(ret < 0) return ret; + total += ret; - if(a->left) + if(a->left) { dictionary_walker(a->left, callback, data); + if (ret < 0) return ret; + total += ret; + } + + return total; } -void dictionary_get_all(DICTIONARY *dict, int (*callback)(void *entry, void *data), void *data) { +int dictionary_get_all(DICTIONARY *dict, int (*callback)(void *entry, void *data), void *data) { + int ret = 0; + dictionary_read_lock(dict); - dictionary_walker(dict->values_index.root, callback, data); + + if(likely(dict->values_index.root)) + ret = dictionary_walker(dict->values_index.root, callback, data); + dictionary_unlock(dict); + + return ret; } diff --git a/src/dictionary.h b/src/dictionary.h index a9ad2997..c659d580 100644 --- a/src/dictionary.h +++ b/src/dictionary.h @@ -11,6 +11,7 @@ typedef struct name_value { uint32_t hash; // a simple hash to speed up searching // we first compare hashes, and only if the hashes are equal we do string comparisons + char *name; void *value; } NAME_VALUE; @@ -38,6 +39,6 @@ extern void *dictionary_set(DICTIONARY *dict, const char *name, void *value, siz extern void *dictionary_get(DICTIONARY *dict, const char *name); extern int dictionary_del(DICTIONARY *dict, const char *name); -extern void dictionary_get_all(DICTIONARY *dict, int (*callback)(void *entry, void *data), void *data); +extern int dictionary_get_all(DICTIONARY *dict, int (*callback)(void *entry, void *data), void *data); #endif /* NETDATA_DICTIONARY_H */ diff --git a/src/main.c b/src/main.c index 760ab641..0740d617 100644 --- a/src/main.c +++ b/src/main.c @@ -275,6 +275,7 @@ int main(int argc, char **argv) setenv("NETDATA_PLUGINS_DIR", config_get("global", "plugins directory" , PLUGINS_DIR), 1); setenv("NETDATA_WEB_DIR" , config_get("global", "web files directory", WEB_DIR) , 1); setenv("NETDATA_CACHE_DIR" , config_get("global", "cache directory" , CACHE_DIR) , 1); + setenv("NETDATA_LIB_DIR" , config_get("global", "lib directory" , VARLIB_DIR) , 1); setenv("NETDATA_LOG_DIR" , config_get("global", "log directory" , LOG_DIR) , 1); setenv("NETDATA_HOST_PREFIX", config_get("global", "host access prefix" , "") , 1); setenv("HOME" , config_get("global", "home directory" , CACHE_DIR) , 1); diff --git a/src/registry.c b/src/registry.c index 2c8a653e..0944585c 100644 --- a/src/registry.c +++ b/src/registry.c @@ -2,22 +2,28 @@ #include #endif -// gcc -O1 -ggdb -Wall -Wextra -I ../src/ -I ../ -o registry ../src/registry.c ../src/dictionary.o ../src/log.o ../src/avl.o ../src/common.o -pthread -luuid -DHAVE_CONFIG_H +// gcc -O1 -ggdb -Wall -Wextra -I ../src/ -I ../ -o registry ../src/registry.c ../src/dictionary.o ../src/log.o ../src/avl.o ../src/common.o ../src/appconfig.o ../src/web_buffer.o ../src/storage_number.o -pthread -luuid -lm -DHAVE_CONFIG_H -DVARLIB_DIR="\"/tmp\"" #include #include #include #include +#include +#include +#include +#include +#include #include "log.h" #include "common.h" #include "dictionary.h" - -#define REGISTRY_MEMORY_ATTRIBUTE aligned +#include "appconfig.h" #define REGISTRY_URL_FLAGS_DEFAULT 0x00 #define REGISTRY_URL_FLAGS_EXPIRED 0x01 +#define DICTIONARY_FLAGS DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE | DICTIONARY_FLAG_NAME_LINK_DONT_CLONE + // ---------------------------------------------------------------------------- // COMMON structures @@ -29,9 +35,15 @@ struct registry { unsigned long long persons_urls_count; unsigned long long machines_urls_count; + char *pathname; + char *db_filename; + char *log_filename; + FILE *registry_log_fp; + DICTIONARY *persons; // dictionary of PERSON *, with key the PERSON.guid DICTIONARY *machines; // dictionary of MACHINE *, with key the MACHINE.guid DICTIONARY *urls; // dictionary of URL *, with key the URL.url + } registry; @@ -41,9 +53,9 @@ struct registry { struct url { uint32_t links; - uint16_t url_length; - char url[]; -} __attribute__ (( REGISTRY_MEMORY_ATTRIBUTE )); + uint16_t len; + char url[1]; +}; typedef struct url URL; @@ -59,7 +71,7 @@ struct machine_url { uint32_t first_t; // the first time we saw this uint32_t last_t; // the last time we saw this uint32_t usages; // how many times this has been accessed -} __attribute__ (( REGISTRY_MEMORY_ATTRIBUTE )); +}; typedef struct machine_url MACHINE_URL; // A machine @@ -71,7 +83,7 @@ struct machine { uint32_t first_t; // the first time we saw this uint32_t last_t; // the last time we saw this uint32_t usages; // how many times this has been accessed -} __attribute__ (( REGISTRY_MEMORY_ATTRIBUTE )); +}; typedef struct machine MACHINE; @@ -87,7 +99,7 @@ struct person_url { uint32_t first_t; // the first time we saw this uint32_t last_t; // the last time we saw this uint32_t usages; // how many times this has been accessed -} __attribute__ (( REGISTRY_MEMORY_ATTRIBUTE )); +}; typedef struct person_url PERSON_URL; // A person @@ -99,30 +111,37 @@ struct person { uint32_t first_t; // the first time we saw this uint32_t last_t; // the last time we saw this uint32_t usages; // how many times this has been accessed -} __attribute__ (( REGISTRY_MEMORY_ATTRIBUTE )); +}; typedef struct person PERSON; +extern PERSON *registry_request(const char *person_guid, const char *machine_guid, const char *url, time_t when); // ---------------------------------------------------------------------------- // URL -static inline URL *registry_url_get(const char *url) { - debug(D_DICTIONARY, "Registry: registry_url_get('%s')", url); - URL *u = dictionary_get(registry.urls, url); - if(!u) { - size_t len = strlen(url); +static inline URL *registry_url_allocate(const char *url) { + size_t len = strlen(url); - debug(D_DICTIONARY, "Registry: registry_url_get('%s'): allocating %zu bytes", url, sizeof(URL) + len + 1); - u = malloc(sizeof(URL) + len + 1); - if(!u) fatal("Cannot allocate %zu bytes for URL '%s'", sizeof(URL) + len + 1, url); + debug(D_REGISTRY, "Registry: registry_url_allocate('%s'): allocating %zu bytes", url, sizeof(URL) + len); + URL *u = malloc(sizeof(URL) + len); + if(!u) fatal("Cannot allocate %zu bytes for URL '%s'", sizeof(URL) + len); - strcpy(u->url, url); - u->links = 0; - u->url_length = len; + strcpy(u->url, url); + u->len = len; + u->links = 0; - debug(D_DICTIONARY, "Registry: registry_url_get('%s'): indexing it", url); - dictionary_set(registry.urls, url, u, sizeof(URL) + len + 1); + debug(D_REGISTRY, "Registry: registry_url_allocate('%s'): indexing it", url); + dictionary_set(registry.urls, u->url, u, sizeof(URL)); + return u; +} + +static inline URL *registry_url_get(const char *url) { + debug(D_REGISTRY, "Registry: registry_url_get('%s')", url); + + URL *u = dictionary_get(registry.urls, url); + if(!u) { + u = registry_url_allocate(url); registry.urls_count++; } @@ -131,81 +150,51 @@ static inline URL *registry_url_get(const char *url) { static inline void registry_url_link(URL *u) { u->links++; + debug(D_REGISTRY, "Registry: registry_url_unlink('%s'): This URL has now %u links", u->url, u->links); } static inline void registry_url_unlink(URL *u) { u->links--; if(!u->links) { + debug(D_REGISTRY, "Registry: registry_url_unlink('%s'): No more links for this URL", u->url); dictionary_del(registry.urls, u->url); free(u); } + else + debug(D_REGISTRY, "Registry: registry_url_unlink('%s'): This URL has %u links left", u->url, u->links); } // ---------------------------------------------------------------------------- // MACHINE -int dump_machine_url(void *entry, void *nothing) { - (void)nothing; - - MACHINE_URL *mu = entry; - - fprintf(stderr, "\n\tURL '%s'\n\t\tfirst seen %u, last seen %u, usages %u, flags 0x%02x\n", - mu->url->url, - mu->first_t, - mu->last_t, - mu->usages, - mu->flags - ); - - return 0; -} - -int dump_machine(void *entry, void *nothing) { - (void)nothing; - - MACHINE *m = entry; - - fprintf(stderr, "MACHINE '%s'\n\tfirst seen %u, last seen %u, usages %u\n", - m->guid, - (uint32_t)m->first_t, - (uint32_t)m->last_t, - m->usages - ); - - dictionary_get_all(m->urls, dump_machine_url, NULL); - return 0; +static inline MACHINE *registry_machine_find(const char *machine_guid) { + debug(D_REGISTRY, "Registry: registry_machine_find('%s')", machine_guid); + return dictionary_get(registry.machines, machine_guid); } -MACHINE *registry_machine_load(const char *machine_guid) { - debug(D_REGISTRY, "Registry: registry_machine_load('%s')", machine_guid); - (void)machine_guid; +static inline MACHINE_URL *registry_machine_url_allocate(MACHINE *m, URL *u, time_t when) { + debug(D_REGISTRY, "registry_machine_link_to_url('%s', '%s'): allocating %zu bytes", m->guid, u->url, sizeof(MACHINE_URL)); - debug(D_REGISTRY, "Registry: registry_machine_load('%s'): not found", machine_guid); - return NULL; -} + MACHINE_URL *mu = malloc(sizeof(MACHINE_URL)); + if(!mu) fatal("registry_machine_link_to_url('%s', '%s'): cannot allocate %zu bytes.", m->guid, u->url, sizeof(MACHINE_URL)); -int registry_machine_save(MACHINE *m) { - debug(D_REGISTRY, "Registry: registry_machine_save('%s')", m->guid); + // mu->persons = dictionary_create(DICTIONARY_FLAGS); + // dictionary_set(mu->persons, p->guid, p, sizeof(PERSON)); -#ifdef REGISTRY_STDOUT_DUMP - fprintf(stderr, "\nSAVING "); - dump_machine(m, NULL); -#endif /* REGISTRY_STDOUT_DUMP */ + mu->first_t = mu->last_t = when; + mu->usages = 1; + mu->url = u; + mu->flags = REGISTRY_URL_FLAGS_DEFAULT; - return -1; -} - -MACHINE *registry_machine_find(const char *machine_guid) { - debug(D_REGISTRY, "Registry: registry_machine_find('%s')", machine_guid); + debug(D_REGISTRY, "registry_machine_link_to_url('%s', '%s'): indexing URL in machine", m->guid, u->url); + dictionary_set(m->urls, u->url, mu, sizeof(MACHINE_URL)); + registry_url_link(u); - MACHINE *m = dictionary_get(registry.machines, machine_guid); - if(!m) m = registry_machine_load(machine_guid); - return m; + return mu; } - -MACHINE *registry_machine_allocate(const char *machine_guid) { +static inline MACHINE *registry_machine_allocate(const char *machine_guid, time_t when) { debug(D_REGISTRY, "Registry: registry_machine_allocate('%s'): creating new machine, sizeof(MACHINE)=%zu", machine_guid, sizeof(MACHINE)); MACHINE *m = calloc(1, sizeof(MACHINE)); @@ -214,9 +203,9 @@ MACHINE *registry_machine_allocate(const char *machine_guid) { strncpy(m->guid, machine_guid, 36); debug(D_REGISTRY, "Registry: registry_machine_allocate('%s'): creating dictionary of urls", machine_guid); - m->urls = dictionary_create(DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_SINGLE_THREADED); + m->urls = dictionary_create(DICTIONARY_FLAGS); - m->first_t = m->last_t = time(NULL); + m->first_t = m->last_t = when; m->usages = 0; dictionary_set(registry.machines, m->guid, m, sizeof(MACHINE)); @@ -224,7 +213,10 @@ MACHINE *registry_machine_allocate(const char *machine_guid) { return m; } -MACHINE *registry_machine_get(const char *machine_guid) { +// 1. validate machine GUID +// 2. if it is valid, find it or create it and return it +// 3. if it is not valid, return NULL +static inline MACHINE *registry_machine_get(const char *machine_guid, time_t when) { MACHINE *m = NULL; if(likely(machine_guid && *machine_guid)) { @@ -242,7 +234,7 @@ MACHINE *registry_machine_get(const char *machine_guid) { machine_guid = buf; m = registry_machine_find(machine_guid); if(!m) { - m = registry_machine_allocate(machine_guid); + m = registry_machine_allocate(machine_guid, when); registry.machines_count++; } } @@ -255,65 +247,30 @@ MACHINE *registry_machine_get(const char *machine_guid) { // ---------------------------------------------------------------------------- // PERSON -int dump_person_url(void *entry, void *nothing) { - (void)nothing; - - PERSON_URL *pu = entry; - - fprintf(stderr, "\n\tURL '%s'\n\t\tfirst seen %u, last seen %u, usages %u, flags 0x%02x\n", - pu->url->url, - pu->first_t, - pu->last_t, - pu->usages, - pu->flags - ); - - return 0; -} - -int dump_person(void *entry, void *nothing) { - (void)nothing; - - PERSON *p = entry; - - fprintf(stderr, "PERSON '%s'\n\tfirst seen %u, last seen %u, usages %u\n", - p->guid, - p->first_t, - p->last_t, - p->usages - ); - - dictionary_get_all(p->urls, dump_person_url, NULL); - return 0; +static inline PERSON *registry_person_find(const char *person_guid) { + debug(D_REGISTRY, "Registry: registry_person_find('%s')", person_guid); + return dictionary_get(registry.persons, person_guid); } -int registry_person_save(PERSON *p) { - debug(D_REGISTRY, "Registry: registry_person_save('%s')", p->guid); +static inline PERSON_URL *registry_person_url_allocate(PERSON *p, MACHINE *m, URL *u, time_t when) { + debug(D_REGISTRY, "registry_person_link_to_url('%s', '%s', '%s'): allocating %zu bytes", p->guid, m->guid, u->url, sizeof(PERSON_URL)); + PERSON_URL *pu = malloc(sizeof(PERSON_URL)); + if(!pu) fatal("registry_person_link_to_url('%s', '%s', '%s'): cannot allocate %zu bytes.", p->guid, m->guid, u->url, sizeof(PERSON_URL)); -#ifdef REGISTRY_STDOUT_DUMP - fprintf(stderr, "\nSAVING "); - dump_person(p, NULL); -#endif /* REGISTRY_STDOUT_DUMP */ + pu->machine = m; + pu->first_t = pu->last_t = when; + pu->usages = 1; + pu->url = u; + pu->flags = REGISTRY_URL_FLAGS_DEFAULT; - return -1; -} + debug(D_REGISTRY, "registry_person_link_to_url('%s', '%s', '%s'): indexing URL in person", p->guid, m->guid, u->url); + dictionary_set(p->urls, u->url, pu, sizeof(PERSON_URL)); + registry_url_link(u); -PERSON *registry_person_load(const char *person_guid) { - debug(D_REGISTRY, "Registry: registry_person_load('%s')", person_guid); - (void)person_guid; - - debug(D_REGISTRY, "Registry: registry_person_load('%s'): not found", person_guid); - return NULL; -} - -PERSON *registry_person_find(const char *person_guid) { - debug(D_REGISTRY, "Registry: registry_person_find('%s')", person_guid); - PERSON *p = dictionary_get(registry.persons, person_guid); - if(!p) p = registry_person_load(person_guid); - return p; + return pu; } -PERSON *registry_person_allocate(const char *person_guid) { +static inline PERSON *registry_person_allocate(const char *person_guid, time_t when) { PERSON *p = NULL; debug(D_REGISTRY, "Registry: registry_person_allocate('%s'): allocating new person, sizeof(PERSON)=%zu", (person_guid)?person_guid:"", sizeof(PERSON)); @@ -344,16 +301,21 @@ PERSON *registry_person_allocate(const char *person_guid) { } debug(D_REGISTRY, "Registry: registry_person_allocate('%s'): creating dictionary of urls", p->guid); - p->urls = dictionary_create(DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_SINGLE_THREADED); + p->urls = dictionary_create(DICTIONARY_FLAGS); - p->first_t = p->last_t = time(NULL); + p->first_t = p->last_t = when; p->usages = 0; dictionary_set(registry.persons, p->guid, p, sizeof(PERSON)); return p; } -PERSON *registry_person_get(const char *person_guid) { + +// 1. validate person GUID +// 2. if it is valid, find it +// 3. if it is not valid, create a new one +// 4. return it +static inline PERSON *registry_person_get(const char *person_guid, time_t when) { PERSON *p = NULL; if(person_guid && *person_guid) { @@ -375,33 +337,23 @@ PERSON *registry_person_get(const char *person_guid) { } if(!p) { - p = registry_person_allocate(NULL); + p = registry_person_allocate(NULL, when); registry.persons_count++; } return p; } - // ---------------------------------------------------------------------------- // LINKING OF OBJECTS -PERSON_URL *registry_person_link_to_url(PERSON *p, MACHINE *m, URL *u) { +static inline PERSON_URL *registry_person_link_to_url(PERSON *p, MACHINE *m, URL *u, time_t when) { debug(D_REGISTRY, "registry_person_link_to_url('%s', '%s', '%s'): searching for URL in person", p->guid, m->guid, u->url); PERSON_URL *pu = dictionary_get(p->urls, u->url); if(!pu) { - debug(D_REGISTRY, "registry_person_link_to_url('%s', '%s', '%s'): not found, allocating %zu bytes", p->guid, m->guid, u->url, sizeof(PERSON_URL)); - pu = malloc(sizeof(PERSON_URL)); - pu->machine = m; - pu->first_t = pu->last_t = time(NULL); - pu->usages = 1; - pu->url = u; - pu->flags = REGISTRY_URL_FLAGS_DEFAULT; - - debug(D_REGISTRY, "registry_person_link_to_url('%s', '%s', '%s'): indexing URL in person", p->guid, m->guid, u->url); - dictionary_set(p->urls, u->url, pu, sizeof(PERSON_URL)); - registry_url_link(u); + debug(D_REGISTRY, "registry_person_link_to_url('%s', '%s', '%s'): not found", p->guid, m->guid, u->url); + pu = registry_person_url_allocate(p, m, u, when); registry.persons_urls_count++; } else { @@ -421,6 +373,7 @@ PERSON_URL *registry_person_link_to_url(PERSON *p, MACHINE *m, URL *u) { } p->usages++; + p->last_t = when; if(pu->flags & REGISTRY_URL_FLAGS_EXPIRED) info("registry_person_link_to_url('%s', '%s', '%s'): accessing an expired URL.", p->guid, m->guid, u->url); @@ -428,22 +381,13 @@ PERSON_URL *registry_person_link_to_url(PERSON *p, MACHINE *m, URL *u) { return pu; } -MACHINE_URL *registry_machine_link_to_url(PERSON *p, MACHINE *m, URL *u) { +static inline MACHINE_URL *registry_machine_link_to_url(PERSON *p, MACHINE *m, URL *u, time_t when) { debug(D_REGISTRY, "registry_machine_link_to_url('%s', '%s', '%s'): searching for URL in machine", p->guid, m->guid, u->url); MACHINE_URL *mu = dictionary_get(m->urls, u->url); if(!mu) { - debug(D_REGISTRY, "registry_machine_link_to_url('%s', '%s', '%s'): not found, allocating %zu bytes", p->guid, m->guid, u->url, sizeof(MACHINE_URL)); - mu = malloc(sizeof(MACHINE_URL)); - //mu->persons = dictionary_create(DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_SINGLE_THREADED); - mu->first_t = mu->last_t = time(NULL); - mu->usages = 1; - mu->url = u; - mu->flags = REGISTRY_URL_FLAGS_DEFAULT; - - debug(D_REGISTRY, "registry_machine_link_to_url('%s', '%s', '%s'): indexing URL in machine", p->guid, m->guid, u->url); - dictionary_set(m->urls, u->url, mu, sizeof(MACHINE_URL)); - registry_url_link(u); + debug(D_REGISTRY, "registry_machine_link_to_url('%s', '%s', '%s'): not found", p->guid, m->guid, u->url); + mu = registry_machine_url_allocate(m, u, when); registry.machines_urls_count++; } else { @@ -455,6 +399,7 @@ MACHINE_URL *registry_machine_link_to_url(PERSON *p, MACHINE *m, URL *u) { //dictionary_set(mu->persons, p->guid, p, sizeof(PERSON)); m->usages++; + m->last_t = when; if(mu->flags & REGISTRY_URL_FLAGS_EXPIRED) info("registry_person_link_to_url('%s', '%s', '%s'): accessing an expired URL.", p->guid, m->guid, u->url); @@ -462,40 +407,388 @@ MACHINE_URL *registry_machine_link_to_url(PERSON *p, MACHINE *m, URL *u) { return mu; } - // ---------------------------------------------------------------------------- -// REGISTRY REQUESTS +// REGISTRY LOG LOAD/SAVE + +static inline void registry_log(const char action, PERSON *p, MACHINE *m, URL *u) { + if(likely(registry.registry_log_fp)) + fprintf(registry.registry_log_fp, "%c\t%08x\t%s\t%s\t%s\n", + action, + p->last_t, + p->guid, + m->guid, + u->url + ); +} -int registry_save(void) { - return -1; +void registry_log_open(void) { + registry.registry_log_fp = fopen(registry.log_filename, "a"); + + if(registry.registry_log_fp) { + if (setvbuf(registry.registry_log_fp, NULL, _IOLBF, 0) != 0) + error("Cannot set line buffering on registry log file."); + } +} + +void registry_log_close(void) { + if(registry.registry_log_fp) + fclose(registry.registry_log_fp); + + registry.registry_log_fp = NULL; +} + +void registry_log_recreate(void) { + if(registry.registry_log_fp != NULL) { + fclose(registry.registry_log_fp); + + registry.registry_log_fp = fopen(registry.log_filename, "w"); + if(registry.registry_log_fp) fclose(registry.registry_log_fp); + registry.registry_log_fp = NULL; + + registry_log_open(); + } } -PERSON *registry_request(const char *person_guid, const char *machine_guid, const char *url) { +int registry_log_load(void) { + char *s, buf[4096 + 1]; + size_t line = 0; + + registry_log_close(); + + debug(D_REGISTRY, "Registry: loading active db from: %s", registry.log_filename); + FILE *fp = fopen(registry.log_filename, "r"); + if(!fp) { + error("Registry: cannot open registry file: %s", registry.log_filename); + return 0; + } + + size_t len = 0; + while((s = fgets_trim_len(buf, 4096, fp, &len))) { + line++; + + switch(s[0]) { + case 'A': + // verify it is valid + if(unlikely(len < 85 || s[1] != '\t' || s[10] != '\t' || s[47] != '\t' || s[84] != '\t')) + error("Registry log line %u is wrong (len = %zu).", line, len); + + s[1] = s[10] = s[47] = s[84] = '\0'; + registry_request(&s[11], &s[48], &s[85], strtoul(&s[2], NULL, 16)); + break; + + default: + error("Registry: ignoring line %zu of filename '%s': %s.", line, registry.log_filename, s); + break; + } + } + + registry_log_open(); + + return 0; +} + + +// ---------------------------------------------------------------------------- +// REGISTRY REQUESTS + +PERSON *registry_request(const char *person_guid, const char *machine_guid, const char *url, time_t when) { debug(D_REGISTRY, "registry_request('%s', '%s', '%s'): NEW REQUEST", (person_guid)?person_guid:"", machine_guid, url); - MACHINE *m = registry_machine_get(machine_guid); + MACHINE *m = registry_machine_get(machine_guid, when); if(!m) return NULL; URL *u = registry_url_get(url); + PERSON *p = registry_person_get(person_guid, when); - PERSON *p = registry_person_get(person_guid); - registry_person_link_to_url(p, m, u); - registry_machine_link_to_url(p, m, u); + registry_person_link_to_url(p, m, u, when); + registry_machine_link_to_url(p, m, u, when); - debug(D_REGISTRY, "Registry: registry_request('%s', '%s', '%s'): saving", person_guid, machine_guid, url); - registry_person_save(p); - registry_machine_save(m); + registry_log('A', p, m, u); registry.usages_count++; - registry_save(); - return p; } + +// ---------------------------------------------------------------------------- +// REGISTRY LOAD/SAVE + +int registry_machine_save_url(void *entry, void *file) { + MACHINE_URL *mu = entry; + FILE *fp = file; + + debug(D_REGISTRY, "Registry: registry_machine_save_url('%s')", mu->url->url); + + int ret = fprintf(fp, "V\t%08x\t%08x\t%08x\t%02x\t%s\n", + mu->first_t, + mu->last_t, + mu->usages, + mu->flags, + mu->url->url + ); + + return ret; +} + +int registry_machine_save(void *entry, void *file) { + MACHINE *m = entry; + FILE *fp = file; + + debug(D_REGISTRY, "Registry: registry_machine_save('%s')", m->guid); + + int ret = fprintf(fp, "M\t%08x\t%08x\t%08x\t%s\n", + m->first_t, + m->last_t, + m->usages, + m->guid + ); + + if(ret >= 0) { + int ret2 = dictionary_get_all(m->urls, registry_machine_save_url, fp); + if(ret2 < 0) return ret2; + ret += ret2; + } + + return ret; +} + +static inline int registry_person_save_url(void *entry, void *file) { + PERSON_URL *pu = entry; + FILE *fp = file; + + debug(D_REGISTRY, "Registry: registry_person_save_url('%s')", pu->url->url); + + int ret = fprintf(fp, "U\t%08x\t%08x\t%08x\t%02x\t%s\t%s\n", + pu->first_t, + pu->last_t, + pu->usages, + pu->flags, + pu->machine->guid, + pu->url->url + ); + + return ret; +} + +static inline int registry_person_save(void *entry, void *file) { + PERSON *p = entry; + FILE *fp = file; + + debug(D_REGISTRY, "Registry: registry_person_save('%s')", p->guid); + + int ret = fprintf(fp, "P\t%08x\t%08x\t%08x\t%s\n", + p->first_t, + p->last_t, + p->usages, + p->guid + ); + + if(ret >= 0) { + int ret2 = dictionary_get_all(p->urls, registry_person_save_url, fp); + if (ret2 < 0) return ret2; + ret += ret2; + } + + return ret; +} + +static int registry_save(void) { + char tmp_filename[FILENAME_MAX + 1]; + char old_filename[FILENAME_MAX + 1]; + + snprintf(old_filename, FILENAME_MAX, "%s.old", registry.db_filename); + snprintf(tmp_filename, FILENAME_MAX, "%s.tmp", registry.db_filename); + + debug(D_REGISTRY, "Registry: Creating file '%s'", tmp_filename); + FILE *fp = fopen(tmp_filename, "w"); + if(!fp) { + error("Registry: Cannot create file: %s", tmp_filename); + return -1; + } + + debug(D_REGISTRY, "Saving all machines"); + int bytes1 = dictionary_get_all(registry.machines, registry_machine_save, fp); + if(bytes1 < 0) { + error("Registry: Cannot save registry machines - return value %d", bytes1); + fclose(fp); + return bytes1; + } + debug(D_REGISTRY, "Registry: saving machines took %d bytes", bytes1); + + debug(D_REGISTRY, "Saving all persons"); + int bytes2 = dictionary_get_all(registry.persons, registry_person_save, fp); + if(bytes2 < 0) { + error("Registry: Cannot save registry persons - return value %d", bytes2); + fclose(fp); + return bytes2; + } + debug(D_REGISTRY, "Registry: saving persons took %d bytes", bytes2); + + fclose(fp); + + errno = 0; + + // remove the .old db + debug(D_REGISTRY, "Registry: Removing old db '%s'", old_filename); + if(unlink(old_filename) == -1 && errno != ENOENT) + error("Registry: cannot remove old registry file '%s'", old_filename); + + // rename the db to .old + debug(D_REGISTRY, "Registry: Link current db '%s' to .old: '%s'", registry.db_filename, old_filename); + if(link(registry.db_filename, old_filename) == -1 && errno != ENOENT) + error("Registry: cannot move file '%s' to '%s'. Saving registry DB failed!", tmp_filename, registry.db_filename); + + else { + // remove the database (it is saved in .old) + debug(D_REGISTRY, "Registry: removing db '%s'", registry.db_filename); + if (unlink(registry.db_filename) == -1 && errno != ENOENT) + error("Registry: cannot remove old registry file '%s'", registry.db_filename); + + // move the .tmp to make it active + debug(D_REGISTRY, "Registry: linking tmp db '%s' to active db '%s'", tmp_filename, registry.db_filename); + if (link(tmp_filename, registry.db_filename) == -1) { + error("Registry: cannot move file '%s' to '%s'. Saving registry DB failed!", tmp_filename, + registry.db_filename); + + // move the .old back + debug(D_REGISTRY, "Registry: linking old db '%s' to active db '%s'", old_filename, registry.db_filename); + if(link(old_filename, registry.db_filename) == -1) + error("Registry: cannot move file '%s' to '%s'. Recovering the old registry DB failed!", old_filename, registry.db_filename); + } + else { + debug(D_REGISTRY, "Registry: removing tmp db '%s'", tmp_filename); + if(unlink(tmp_filename) == -1) + error("Registry: cannot remove tmp registry file '%s'", tmp_filename); + + // it has been moved successfully + // discard the current registry log + registry_log_recreate(); + } + } + + return -1; +} + +static inline size_t registry_load(void) { + char *s, buf[4096 + 1]; + PERSON *p = NULL; + MACHINE *m = NULL; + URL *u = NULL; + size_t line = 0; + + debug(D_REGISTRY, "Registry: loading active db from: %s", registry.db_filename); + FILE *fp = fopen(registry.db_filename, "r"); + if(!fp) { + error("Registry: cannot open registry file: %s", registry.db_filename); + return 0; + } + + size_t len = 0; + while((s = fgets_trim_len(buf, 4096, fp, &len))) { + line++; + + debug(D_REGISTRY, "Registry: read line %zu to length %zu: %s", line, len, s); + + switch(*s) { + case 'P': // person + m = NULL; + // verify it is valid + if(unlikely(len != 65 || s[1] != '\t' || s[10] != '\t' || s[19] != '\t' || s[28] != '\t' || s[65] != '\0')) + error("Registry person line %u is wrong (len = %zu).", line, len); + + s[1] = s[10] = s[19] = s[28] = '\0'; + p = registry_person_allocate(&s[29], strtoul(&s[2], NULL, 16)); + p->last_t = strtoul(&s[11], NULL, 16); + p->usages = strtoul(&s[20], NULL, 16); + debug(D_REGISTRY, "Registry loaded person '%s', first: %u, last: %u, usages: %u", p->guid, p->first_t, p->last_t, p->usages); + break; + + case 'M': // machine + p = NULL; + // verify it is valid + if(unlikely(len != 65 || s[1] != '\t' || s[10] != '\t' || s[19] != '\t' || s[28] != '\t' || s[65] != '\0')) + error("Registry person line %u is wrong (len = %zu).", line, len); + + s[1] = s[10] = s[19] = s[28] = '\0'; + m = registry_machine_allocate(&s[29], strtoul(&s[2], NULL, 16)); + m->last_t = strtoul(&s[11], NULL, 16); + m->usages = strtoul(&s[20], NULL, 16); + debug(D_REGISTRY, "Registry loaded machine '%s', first: %u, last: %u, usages: %u", m->guid, m->first_t, m->last_t, m->usages); + break; + + case 'U': // person URL + if(unlikely(!p)) { + error("Registry: ignoring line %zu, no person loaded: %s", line, s); + break; + } + + // verify it is valid + if(len < 69 || s[1] != '\t' || s[10] != '\t' || s[19] != '\t' || s[28] != '\t' || s[31] != '\t' || s[68] != '\t') + error("Registry person URL line %u is wrong (len = %zu).", line, len); + + s[1] = s[10] = s[19] = s[28] = s[31] = s[68] = '\0'; + u = registry_url_allocate(&s[69]); + + time_t first_t = strtoul(&s[2], NULL, 16); + + m = registry_machine_find(&s[32]); + if(!m) m = registry_machine_allocate(&s[32], first_t); + + PERSON_URL *pu = registry_person_url_allocate(p, m, u, first_t); + pu->last_t = strtoul(&s[11], NULL, 16); + pu->usages = strtoul(&s[20], NULL, 16); + pu->flags = strtoul(&s[29], NULL, 16); + debug(D_REGISTRY, "Registry loaded person URL '%s', machine '%s', first: %u, last: %u, usages: %u, flags: %02x", u->url, m->guid, pu->first_t, pu->last_t, pu->usages, pu->flags); + break; + + case 'V': // machine URL + if(unlikely(!m)) { + error("Registry: ignoring line %zu, no machine loaded: %s", line, s); + break; + } + + // verify it is valid + if(len < 32 || s[1] != '\t' || s[10] != '\t' || s[19] != '\t' || s[28] != '\t' || s[31] != '\t') + error("Registry person URL line %u is wrong (len = %zu).", line, len); + + s[1] = s[10] = s[19] = s[28] = s[31] = '\0'; + u = registry_url_allocate(&s[32]); + + MACHINE_URL *mu = registry_machine_url_allocate(m, u, strtoul(&s[2], NULL, 16)); + mu->last_t = strtoul(&s[11], NULL, 16); + mu->usages = strtoul(&s[20], NULL, 16); + mu->flags = strtoul(&s[29], NULL, 16); + debug(D_REGISTRY, "Registry loaded machine URL '%s', machine '%s', first: %u, last: %u, usages: %u, flags: %02x", u->url, m->guid, mu->first_t, mu->last_t, mu->usages, mu->flags); + break; + + default: + error("Registry: ignoring line %zu of filename '%s': %s.", line, registry.db_filename, s); + break; + } + } + fclose(fp); + + registry_log_load(); + + return line; +} + // ---------------------------------------------------------------------------- // REGISTRY void registry_init(void) { + char filename[FILENAME_MAX + 1]; + + registry.pathname = config_get("registry", "registry db directory", VARLIB_DIR); + if(mkdir(registry.pathname, 0644) == -1 && errno != EEXIST) + error("Cannot create directory '%s'", registry.pathname); + + snprintf(filename, FILENAME_MAX, "%s/%s", registry.pathname, "registry.db"); + registry.db_filename = config_get("registry", "registry db file", filename); + + snprintf(filename, FILENAME_MAX, "%s/%s", registry.pathname, "registry-log.db"); + registry.log_filename = config_get("registry", "registry log file", filename); + registry.persons_count = 0; registry.machines_count = 0; registry.usages_count = 0; @@ -504,34 +797,33 @@ void registry_init(void) { registry.machines_urls_count = 0; debug(D_REGISTRY, "Registry: creating global registry dictionary for persons."); - registry.persons = dictionary_create(DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_SINGLE_THREADED); + registry.persons = dictionary_create(DICTIONARY_FLAGS); debug(D_REGISTRY, "Registry: creating global registry dictionary for machines."); - registry.machines = dictionary_create(DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_SINGLE_THREADED); + registry.machines = dictionary_create(DICTIONARY_FLAGS); debug(D_REGISTRY, "Registry: creating global registry dictionary for urls."); - registry.urls = dictionary_create(DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_SINGLE_THREADED); + registry.urls = dictionary_create(DICTIONARY_FLAGS); + + registry_log_open(); + registry_load(); } void registry_free(void) { while(registry.persons->values_index.root) { PERSON *p = ((NAME_VALUE *)registry.persons->values_index.root)->value; -#ifdef REGISTRY_STDOUT_DUMP - fprintf(stderr, "\nPERSON: '%s', first: %u, last: %u, usages: %u\n", p->guid, p->first_t, p->last_t, p->usages); -#endif /* REGISTRY_STDOUT_DUMP */ + // fprintf(stderr, "\nPERSON: '%s', first: %u, last: %u, usages: %u\n", p->guid, p->first_t, p->last_t, p->usages); while(p->urls->values_index.root) { PERSON_URL *pu = ((NAME_VALUE *)p->urls->values_index.root)->value; -#ifdef REGISTRY_STDOUT_DUMP - fprintf(stderr, "\tURL: '%s', first: %u, last: %u, usages: %u, flags: 0x%02x\n", pu->url->url, pu->first_t, pu->last_t, pu->usages, pu->flags); -#endif /* REGISTRY_STDOUT_DUMP */ + // fprintf(stderr, "\tURL: '%s', first: %u, last: %u, usages: %u, flags: 0x%02x\n", pu->url->url, pu->first_t, pu->last_t, pu->usages, pu->flags); debug(D_REGISTRY, "Registry: deleting url '%s' from person '%s'", pu->url->url, p->guid); dictionary_del(p->urls, pu->url->url); - debug(D_REGISTRY, "Registry: unlinking url '%s'", pu->url->url); + debug(D_REGISTRY, "Registry: unlinking url '%s' from person", pu->url->url); registry_url_unlink(pu->url); debug(D_REGISTRY, "Registry: freeing person url"); @@ -551,16 +843,12 @@ void registry_free(void) { while(registry.machines->values_index.root) { MACHINE *m = ((NAME_VALUE *)registry.machines->values_index.root)->value; -#ifdef REGISTRY_STDOUT_DUMP - fprintf(stderr, "\nMACHINE: '%s', first: %u, last: %u, usages: %u\n", m->guid, m->first_t, m->last_t, m->usages); -#endif /* REGISTRY_STDOUT_DUMP */ + // fprintf(stderr, "\nMACHINE: '%s', first: %u, last: %u, usages: %u\n", m->guid, m->first_t, m->last_t, m->usages); while(m->urls->values_index.root) { MACHINE_URL *mu = ((NAME_VALUE *)m->urls->values_index.root)->value; -#ifdef REGISTRY_STDOUT_DUMP - fprintf(stderr, "\tURL: '%s', first: %u, last: %u, usages: %u, flags: 0x%02x\n", mu->url->url, mu->first_t, mu->last_t, mu->usages, mu->flags); -#endif /* REGISTRY_STDOUT_DUMP */ + // fprintf(stderr, "\tURL: '%s', first: %u, last: %u, usages: %u, flags: 0x%02x\n", mu->url->url, mu->first_t, mu->last_t, mu->usages, mu->flags); //debug(D_REGISTRY, "Registry: destroying persons dictionary from url '%s'", mu->url->url); //dictionary_destroy(mu->persons); @@ -568,7 +856,7 @@ void registry_free(void) { debug(D_REGISTRY, "Registry: deleting url '%s' from person '%s'", mu->url->url, m->guid); dictionary_del(m->urls, mu->url->url); - debug(D_REGISTRY, "Registry: unlinking url '%s'", mu->url); + debug(D_REGISTRY, "Registry: unlinking url '%s' from machine", mu->url->url); registry_url_unlink(mu->url); debug(D_REGISTRY, "Registry: freeing machine url"); @@ -595,6 +883,10 @@ void registry_free(void) { dictionary_destroy(registry.urls); } + +// ---------------------------------------------------------------------------- +// TESTS + int test1(int argc, char **argv) { void print_stats(uint32_t requests, unsigned long long start, unsigned long long end) { @@ -609,8 +901,8 @@ int test1(int argc, char **argv) { (void) argc; (void) argv; - uint32_t u, users = 500000; - uint32_t m, machines = 50000; + uint32_t u, users = 1000000; + uint32_t m, machines = 200000; uint32_t machines2 = machines * 2; char **users_guids = malloc(users * sizeof(char *)); @@ -637,10 +929,11 @@ int test1(int argc, char **argv) { start = timems(); fprintf(stderr, "\nGenerating %u users accessing %u machines\n", users, machines); m = 0; + time_t now = time(NULL); for(u = 0; u < users ; u++) { if(++m == machines) m = 0; - PERSON *p = registry_request(NULL, machines_guids[m], machines_urls[m]); + PERSON *p = registry_request(NULL, machines_guids[m], machines_urls[m], now); users_guids[u] = p->guid; } print_stats(u, start, timems()); @@ -648,10 +941,11 @@ int test1(int argc, char **argv) { start = timems(); fprintf(stderr, "\nAll %u users accessing again the same %u servers\n", users, machines); m = 0; + now = time(NULL); for(u = 0; u < users ; u++) { if(++m == machines) m = 0; - PERSON *p = registry_request(users_guids[u], machines_guids[m], machines_urls[m]); + PERSON *p = registry_request(users_guids[u], machines_guids[m], machines_urls[m], now); if(p->guid != users_guids[u]) fprintf(stderr, "ERROR: expected to get user guid '%s' but git '%s'", users_guids[u], p->guid); @@ -661,10 +955,11 @@ int test1(int argc, char **argv) { start = timems(); fprintf(stderr, "\nAll %u users accessing a new server, out of the %u servers\n", users, machines); m = 1; + now = time(NULL); for(u = 0; u < users ; u++) { if(++m == machines) m = 0; - PERSON *p = registry_request(users_guids[u], machines_guids[m], machines_urls[m]); + PERSON *p = registry_request(users_guids[u], machines_guids[m], machines_urls[m], now); if(p->guid != users_guids[u]) fprintf(stderr, "ERROR: expected to get user guid '%s' but git '%s'", users_guids[u], p->guid); @@ -673,11 +968,12 @@ int test1(int argc, char **argv) { start = timems(); fprintf(stderr, "\n%u random users accessing a random server, out of the %u servers\n", users, machines); + now = time(NULL); for(u = 0; u < users ; u++) { uint32_t tu = random() * users / RAND_MAX; uint32_t tm = random() * machines / RAND_MAX; - PERSON *p = registry_request(users_guids[tu], machines_guids[tm], machines_urls[tm]); + PERSON *p = registry_request(users_guids[tu], machines_guids[tm], machines_urls[tm], now); if(p->guid != users_guids[tu]) fprintf(stderr, "ERROR: expected to get user guid '%s' but git '%s'", users_guids[tu], p->guid); @@ -686,11 +982,12 @@ int test1(int argc, char **argv) { start = timems(); fprintf(stderr, "\n%u random users accessing a random server, out of %u servers\n", users, machines2); + now = time(NULL); for(u = 0; u < users ; u++) { uint32_t tu = random() * users / RAND_MAX; uint32_t tm = random() * machines2 / RAND_MAX; - PERSON *p = registry_request(users_guids[tu], machines_guids[tm], machines_urls[tm]); + PERSON *p = registry_request(users_guids[tu], machines_guids[tm], machines_urls[tm], now); if(p->guid != users_guids[tu]) fprintf(stderr, "ERROR: expected to get user guid '%s' but git '%s'", users_guids[tu], p->guid); @@ -702,6 +999,7 @@ int test1(int argc, char **argv) { fprintf(stderr, "\n%u random user accesses to a random server, out of %u servers,\n > using 1/10000 with a random url, 1/1000 with a mismatched url\n", users * 2, machines2); + now = time(NULL); for (u = 0; u < users * 2; u++) { uint32_t tu = random() * users / RAND_MAX; uint32_t tm = random() * machines2 / RAND_MAX; @@ -715,7 +1013,7 @@ int test1(int argc, char **argv) { else if (random() % 1000 == 123) url = machines_urls[random() * machines2 / RAND_MAX]; - PERSON *p = registry_request(users_guids[tu], machines_guids[tm], url); + PERSON *p = registry_request(users_guids[tu], machines_guids[tm], url, now); if (p->guid != users_guids[tu]) fprintf(stderr, "ERROR: expected to get user guid '%s' but git '%s'", users_guids[tu], p->guid); @@ -723,6 +1021,11 @@ int test1(int argc, char **argv) { print_stats(u, start, timems()); } + fprintf(stderr, "\n\nSAVE\n"); + start = timems(); + registry_save(); + print_stats(registry.persons_count, start, timems()); + fprintf(stderr, "\n\nCLEANUP\n"); start = timems(); registry_free(); @@ -734,9 +1037,9 @@ int test1(int argc, char **argv) { // TESTING int main(int argc, char **argv) { - // debug_flags = 0xFFFFFFFF; - test1(argc, argv); - exit(0); + debug_flags = 0xFFFFFFFF; + //test1(argc, argv); + //exit(0); (void)argc; (void)argv; @@ -748,41 +1051,48 @@ int main(int argc, char **argv) { registry_init(); + int i = 2; + + fprintf(stderr, "\n\nADDING ENTRY\n"); + // p1 = registry_request("2c95abd0-1542-11e6-8c66-00508db7e9c9", "7c173980-145c-11e6-b86f-00508db7e9c1", "http://localhost:19999/", time(NULL)); - int i = 100000; + if(0) while(i--) { #ifdef REGISTRY_STDOUT_DUMP fprintf(stderr, "\n\nADDING ENTRY\n"); #endif /* REGISTRY_STDOUT_DUMP */ - p1 = registry_request(NULL, "7c173980-145c-11e6-b86f-00508db7e9c1", "http://localhost:19999/"); + p1 = registry_request(NULL, "7c173980-145c-11e6-b86f-00508db7e9c1", "http://localhost:19999/", time(NULL)); #ifdef REGISTRY_STDOUT_DUMP fprintf(stderr, "\n\nADDING ANOTHER URL\n"); #endif /* REGISTRY_STDOUT_DUMP */ - p1 = registry_request(p1->guid, "7c173980-145c-11e6-b86f-00508db7e9c1", "http://127.0.0.1:19999/"); + p1 = registry_request(p1->guid, "7c173980-145c-11e6-b86f-00508db7e9c1", "http://127.0.0.1:19999/", time(NULL)); #ifdef REGISTRY_STDOUT_DUMP fprintf(stderr, "\n\nADDING ANOTHER URL\n"); #endif /* REGISTRY_STDOUT_DUMP */ - p1 = registry_request(p1->guid, "7c173980-145c-11e6-b86f-00508db7e9c1", "http://my.server:19999/"); + p1 = registry_request(p1->guid, "7c173980-145c-11e6-b86f-00508db7e9c1", "http://my.server:19999/", time(NULL)); #ifdef REGISTRY_STDOUT_DUMP fprintf(stderr, "\n\nADDING ANOTHER MACHINE\n"); #endif /* REGISTRY_STDOUT_DUMP */ - p1 = registry_request(p1->guid, "7c173980-145c-11e6-b86f-00508db7e9c1", "http://my.server:19999/"); + p1 = registry_request(p1->guid, "7c173980-145c-11e6-b86f-00508db7e9c1", "http://my.server:19999/", time(NULL)); #ifdef REGISTRY_STDOUT_DUMP fprintf(stderr, "\n\nADDING ANOTHER PERSON\n"); #endif /* REGISTRY_STDOUT_DUMP */ - p2 = registry_request(NULL, "7c173980-145c-11e6-b86f-00508db7e9c3", "http://localhost:19999/"); + p2 = registry_request(NULL, "7c173980-145c-11e6-b86f-00508db7e9c3", "http://localhost:19999/", time(NULL)); #ifdef REGISTRY_STDOUT_DUMP fprintf(stderr, "\n\nADDING ANOTHER MACHINE\n"); #endif /* REGISTRY_STDOUT_DUMP */ - p2 = registry_request(p2->guid, "7c173980-145c-11e6-b86f-00508db7e9c3", "http://localhost:19999/"); + p2 = registry_request(p2->guid, "7c173980-145c-11e6-b86f-00508db7e9c3", "http://localhost:19999/", time(NULL)); } + fprintf(stderr, "\n\nSAVE\n"); + registry_save(); + fprintf(stderr, "\n\nCLEANUP\n"); - //registry_free(); + registry_free(); return 0; } -- 2.39.2