on disk rebuild db

diff --git a/NEWS b/NEWS
index 13dac554a6fb6c1bbfc5a5bba311f5a89fbf26b5..cdb59217852d1fab7773d12904c6ab16ccc7a028 100644 (file)
--- a/NEWS
+++ b/NEWS
@@ -9,6 +9,7 @@ Changes in 2.1.6
 * FIX: Solaris: configure script failed to enable DDP module
 * FIX: AppleDouble buffer overrun by extremely long filename
 * UPD: afpd: return version info with machine type in DSIGetStatus
+* UPD: dbd: use on-disk temporary rebuild db instead of in-memory db
 
 Changes in 2.1.5
 ================

diff --git a/etc/cnid_dbd/cmd_dbd.c b/etc/cnid_dbd/cmd_dbd.c
index dc4c3923d90eeb1a0e47c3c24bb4b4df32d4333f..357b1d226854a8cdbb7460abc0c7445f19e88578 100644 (file)
--- a/etc/cnid_dbd/cmd_dbd.c
+++ b/etc/cnid_dbd/cmd_dbd.c
@@ -1,6 +1,4 @@
 /* 
-   $Id: cmd_dbd.c,v 1.26 2010-04-20 16:46:20 hat001 Exp $
-
    Copyright (c) 2009 Frank Lahm <franklahm@gmail.com>
    
    This program is free software; you can redistribute it and/or modify
@@ -229,7 +227,7 @@ static void free_lock(int lockfd)
 
 static void usage (void)
 {
-    printf("Usage: dbd [-e|-v|-x] -d [-i] | -s [-c|-n]| -r [-c|-f] | -u <path to netatalk volume>\n"
+    printf("Usage: dbd [-e|-t|-v|-x] -d [-i] | -s [-c|-n]| -r [-c|-f] | -u <path to netatalk volume>\n"
            "dbd can dump, scan, reindex and rebuild Netatalk dbd CNID databases.\n"
            "dbd must be run with appropiate permissions i.e. as root.\n\n"
            "Main commands are:\n"
@@ -266,6 +264,7 @@ static void usage (void)
            "General options:\n"
            "   -e only work on inactive volumes and lock them (exclusive)\n"
            "   -x rebuild indexes (just for completeness, mostly useless!)\n"
+           "   -t show statistics while running\n"
            "   -v verbose\n\n"
            "WARNING:\n"
            "For -r -f restore of the CNID database from the adouble files, the CNID must of course\n"
@@ -289,7 +288,7 @@ int main(int argc, char **argv)
     /* Inhereting perms in ad_mkdir etc requires this */
     ad_setfuid(0);
 
-    while ((c = getopt(argc, argv, ":cdefinrsuvx")) != -1) {
+    while ((c = getopt(argc, argv, ":cdefinrstuvx")) != -1) {
         switch(c) {
         case 'c':
             flags |= DBD_FLAGS_CLEANUP;
@@ -310,6 +309,9 @@ int main(int argc, char **argv)
         case 'r':
             rebuild = 1;
             break;
+        case 't':
+            flags |= DBD_FLAGS_STATS;
+            break;
         case 'u':
             prep_upgrade = 1;
             break;
@@ -464,6 +466,7 @@ int main(int argc, char **argv)
     }
 
     /* Cleanup */
+    dbd_log(LOGDEBUG, "Closing db");
     if (! nocniddb && dbif_close(dbd) < 0) {
         dbd_log( LOGSTD, "Error closing database");
         goto exit_failure;

diff --git a/etc/cnid_dbd/cmd_dbd.h b/etc/cnid_dbd/cmd_dbd.h
index a1d51b67dcdd77dddf648997f7cdefef2faebd19..84b1191abc187384d91c89ab4c79743bae171965 100644 (file)
--- a/etc/cnid_dbd/cmd_dbd.h
+++ b/etc/cnid_dbd/cmd_dbd.h
@@ -14,6 +14,7 @@ typedef unsigned int dbd_flags_t;
 #define DBD_FLAGS_FORCE    (1 << 1)
 #define DBD_FLAGS_EXCL     (1 << 2)
 #define DBD_FLAGS_CLEANUP  (1 << 3) /* Dont create AD stuff, but cleanup orphaned */
+#define DBD_FLAGS_STATS    (1 << 4)
 
 #define ADv2_DIRNAME ".AppleDouble"
 

diff --git a/etc/cnid_dbd/cmd_dbd_scanvol.c b/etc/cnid_dbd/cmd_dbd_scanvol.c
index 45d67ae1bd5b2c5dcfc09ce618fa3f95fb768372..fbb0b7dbf15a60ff3eb8d1c3a8adfcebdd777979 100644 (file)
--- a/etc/cnid_dbd/cmd_dbd_scanvol.c
+++ b/etc/cnid_dbd/cmd_dbd_scanvol.c
@@ -923,6 +923,22 @@ static int dbd_readdir(int volroot, cnid_t did)
             continue;
         }
 
+        /**************************************************************************
+           Statistics
+         **************************************************************************/
+        static unsigned long long statcount = 0;
+        static time_t t = 0;
+
+        if (t == 0)
+            t = time(NULL);
+
+        statcount++;
+        if ((statcount % 10000) == 0) {
+            if (dbd_flags & DBD_FLAGS_STATS)            
+                dbd_log(LOGSTD, "Scanned: %10llu, time: %10llu s",
+                        statcount, (unsigned long long)(time(NULL) - t));
+        }
+
         /**************************************************************************
            Tests
         **************************************************************************/
@@ -946,7 +962,8 @@ static int dbd_readdir(int volroot, cnid_t did)
             /* Now add this object to our rebuild dbd */
             if (cnid) {
                 rqst.cnid = rply.cnid;
-                dbd_rebuild_add(dbd_rebuild, &rqst, &rply);
+                ret = dbd_rebuild_add(dbd_rebuild, &rqst, &rply);
+                dbif_txn_close(dbd_rebuild, ret);
                 if (rply.result != CNID_DBD_RES_OK) {
                     dbd_log( LOGDEBUG, "Fatal error adding CNID: %u for '%s/%s' to in-memory rebuild-db",
                              cnid, cwdbuf, ep->d_name);
@@ -1032,7 +1049,7 @@ static int scanvol(struct volinfo *vi, dbd_flags_t flags)
 */
 static void delete_orphaned_cnids(DBD *dbd, DBD *dbd_rebuild, dbd_flags_t flags)
 {
-    int ret, deleted = 0;
+    int ret = 0, deleted = 0;
     cnid_t dbd_cnid = 0, rebuild_cnid = 0;
     struct cnid_dbd_rqst rqst;
     struct cnid_dbd_rply rply;
@@ -1053,7 +1070,7 @@ static void delete_orphaned_cnids(DBD *dbd, DBD *dbd_rebuild, dbd_flags_t flags)
         if (alarmed)
             longjmp(jmp, 1); /* this jumps back to cmd_dbd_scanvol() */
 
-        if (deleted > 50) {
+        if (deleted > 1000) {
             deleted = 0;
             if (dbif_txn_checkpoint(dbd, 0, 0, 0) < 0) {
                 dbd_log(LOGSTD, "Error checkpointing!");
@@ -1116,6 +1133,16 @@ cleanup:
     return;
 }
 
+static const char *get_tmpdb_path(void)
+{
+    pid_t pid = getpid();
+    static char path[MAXPATHLEN];
+    snprintf(path, MAXPATHLEN, "/tmp/tmpdb-dbd.%u", pid);
+    if (mkdir(path, 0755) != 0)
+        return NULL;
+    return path;
+}
+
 /*
   Main func called from cmd_dbd.c
 */
@@ -1123,6 +1150,7 @@ int cmd_dbd_scanvol(DBD *dbd_ref, struct volinfo *volinfo, dbd_flags_t flags)
 {
     int ret = 0;
     struct db_param db_param = { 0 };
+    const char *tmpdb_path = NULL;
 
     /* Set cachesize for in-memory rebuild db */
     db_param.cachesize = 128 * 1024 * 1024; /* 128 MB */
@@ -1140,24 +1168,44 @@ int cmd_dbd_scanvol(DBD *dbd_ref, struct volinfo *volinfo, dbd_flags_t flags)
         /* Get volume stamp */
         dbd_getstamp(dbd, &rqst, &rply);
         if (rply.result != CNID_DBD_RES_OK)
-            goto exit_cleanup;
+            goto exit;
         memcpy(stamp, rply.name, CNID_DEV_LEN);
 
         /* open/create rebuild dbd, copy rootinfo key */
-        if (NULL == (dbd_rebuild = dbif_init(NULL, NULL)))
-            return -1;
-        if (0 != (dbif_open(dbd_rebuild, &db_param, 0)))
-            return -1;
-        if (0 != (dbif_copy_rootinfokey(dbd, dbd_rebuild)))
-            goto exit_cleanup;
+        tmpdb_path = get_tmpdb_path();
+        if (NULL == (dbd_rebuild = dbif_init(tmpdb_path, "cnid2.db"))) {
+            ret = -1;
+            goto exit;
+        }
+
+        if (dbif_env_open(dbd_rebuild,
+                          &db_param,
+                          DB_CREATE | DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN) < 0) {
+            dbd_log(LOGSTD, "error opening tmp database!");
+            goto exit;
+        }
+
+        if (0 != (dbif_open(dbd_rebuild, NULL, 0))) {
+            ret = -1;
+            goto exit;
+        }
+
+        if (0 != (dbif_copy_rootinfokey(dbd, dbd_rebuild))) {
+            ret = -1;
+            goto exit;
+        }
     }
 
-    if (setjmp(jmp) != 0)
-        goto exit_cleanup;      /* Got signal, jump from dbd_readdir */
+    if (setjmp(jmp) != 0) {
+        ret = 0;                /* Got signal, jump from dbd_readdir */
+        goto exit;              
+    }
 
     /* scanvol */
-    if ( (scanvol(volinfo, flags)) != 0)
-        return -1;
+    if ( (scanvol(volinfo, flags)) != 0) {
+        ret = -1;
+        goto exit;
+    }
 
     if (! nocniddb) {
         /* We can only do this in exclusive mode, otherwise we might delete CNIDs added from
@@ -1166,8 +1214,20 @@ int cmd_dbd_scanvol(DBD *dbd_ref, struct volinfo *volinfo, dbd_flags_t flags)
             delete_orphaned_cnids(dbd, dbd_rebuild, flags);
     }
 
-exit_cleanup:
-    if (! nocniddb)
+exit:
+    if (dbd_rebuild) {
+        dbd_log(LOGDEBUG, "Closing tmp db");
         dbif_close(dbd_rebuild);
+#if 0
+        if (tmpdb_path) {
+            char cmd[8 + MAXPATHLEN];
+            snprintf(cmd, 8 + MAXPATHLEN, "rm -f %s/*", tmpdb_path);
+            dbd_log( LOGDEBUG, "Removing temp database '%s'", tmpdb_path);
+            system(cmd);
+            snprintf(cmd, 8 + MAXPATHLEN, "rmdir %s", tmpdb_path);
+            system(cmd);
+        }        
+#endif
+    }
     return ret;
 }