Merge from branch 2-1

diff --git a/etc/cnid_dbd/cmd_dbd.c b/etc/cnid_dbd/cmd_dbd.c
index 5c29e19ca8fc6be1af70a8b1f8be2b4bff493e57..532b7b8a6bd3b67d3966e374185ea83f02e91b04 100644 (file)
--- a/etc/cnid_dbd/cmd_dbd.c
+++ b/etc/cnid_dbd/cmd_dbd.c
@@ -93,12 +93,13 @@ static struct db_param db_param = {
     64 * 1024,                  /* bdb cachesize (64 MB) */
     DEFAULT_MAXLOCKS,           /* maxlocks */
     DEFAULT_MAXLOCKOBJS,        /* maxlockobjs */
-    -1,                         /* not used ... */
-    -1,
-    "",
-    -1,
-    -1,
-    -1
+    0,                          /* flush_interval */
+    0,                          /* flush_frequency */
+    1000,                       /* txn_frequency */
+    0,                          /* usock_file */
+    -1,                         /* fd_table_size */
+    -1,                         /* idle_timeout */
+    -1                          /* max_vols */
 };
 static char dbpath[MAXPATHLEN+1];   /* Path to the dbd database */
 
@@ -481,9 +482,11 @@ int main(int argc, char **argv)
 
     /* Cleanup */
     dbd_log(LOGDEBUG, "Closing db");
-    if (! nocniddb && dbif_close(dbd) < 0) {
-        dbd_log( LOGSTD, "Error closing database");
-        goto exit_failure;
+    if (! nocniddb) {
+        if (dbif_close(dbd) < 0) {
+            dbd_log( LOGSTD, "Error closing database");
+            goto exit_failure;
+        }
     }
 
 exit_success:

diff --git a/etc/cnid_dbd/cmd_dbd_scanvol.c b/etc/cnid_dbd/cmd_dbd_scanvol.c
index f263329df364ddcb9185ce65601cd68de4560009..0f105dd580e6e5db53f63bc1bd03e070e1f5c7ac 100644 (file)
--- a/etc/cnid_dbd/cmd_dbd_scanvol.c
+++ b/etc/cnid_dbd/cmd_dbd_scanvol.c
@@ -673,6 +673,17 @@ static cnid_t check_cnid(const char *name, cnid_t did, struct stat *st, int adfi
     cnid_t db_cnid, ad_cnid;
     struct adouble ad;
 
+    /* Force checkout every X items */
+    static int cnidcount = 0;
+    cnidcount++;
+    if (cnidcount > 10000) {
+        cnidcount = 0;
+        if (dbif_txn_checkpoint(dbd, 0, 0, 0) < 0) {
+            dbd_log(LOGSTD, "Error checkpointing!");
+            return 0;
+        }
+    }
+
     /* Get CNID from ad-file if volume is using AFPVOL_CACHE */
     ad_cnid = 0;
     if ( (myvolinfo->v_flags & AFPVOL_CACHE) && ADFILE_OK) {
@@ -959,6 +970,7 @@ static int dbd_readdir(int volroot, cnid_t did)
 
             /* Now add this object to our rebuild dbd */
             if (cnid) {
+                static uint count = 0;
                 rqst.cnid = rply.cnid;
                 ret = dbd_rebuild_add(dbd_rebuild, &rqst, &rply);
                 dbif_txn_close(dbd_rebuild, ret);
@@ -967,6 +979,14 @@ static int dbd_readdir(int volroot, cnid_t did)
                              cnid, cwdbuf, ep->d_name);
                     longjmp(jmp, 1); /* this jumps back to cmd_dbd_scanvol() */
                 }
+                count++;
+                if (count == 10000) {
+                    if (dbif_txn_checkpoint(dbd_rebuild, 0, 0, 0) < 0) {
+                        dbd_log(LOGSTD, "Error checkpointing!");
+                        return -1;
+                    }
+                    count = 0;
+                }
             }
         }
 
@@ -1052,6 +1072,8 @@ static void delete_orphaned_cnids(DBD *dbd, DBD *dbd_rebuild, dbd_flags_t flags)
     struct cnid_dbd_rqst rqst;
     struct cnid_dbd_rply rply;
 
+    dbd->db_param.txn_frequency = 0;
+
     /* jump over rootinfo key */
     if ( dbif_idwalk(dbd, &dbd_cnid, 0) != 1)
         return;
@@ -1124,12 +1146,13 @@ static void delete_orphaned_cnids(DBD *dbd, DBD *dbd_rebuild, dbd_flags_t flags)
         }
 
         if (dbd_cnid > rebuild_cnid) {
+            dbif_idwalk(dbd, NULL, 1); /* Close cursor */
+            dbif_idwalk(dbd_rebuild, NULL, 1); /* Close cursor */
+            dbif_txn_close(dbd, 2);
+            dbif_txn_close(dbd_rebuild, 2);
             dbd_log(LOGSTD, "Ghost CNID: %u. This is fatal! Dumping rebuild db:\n", rebuild_cnid);
             dbif_dump(dbd_rebuild, 0);
             dbd_log(LOGSTD, "Send this dump and a `dbd -d ...` dump to the Netatalk Dev team!");
-            dbif_txn_close(dbd, ret);
-            dbif_idwalk(dbd, NULL, 1); /* Close cursor */
-            dbif_idwalk(dbd_rebuild, NULL, 1); /* Close cursor */
             goto cleanup;
         }
     } /* while ((dbif_idwalk(dbd, &dbd_cnid, 0)) == 1) */
@@ -1163,6 +1186,8 @@ int cmd_dbd_scanvol(DBD *dbd_ref, struct volinfo *vi, dbd_flags_t flags)
     db_param.cachesize = 128 * 1024; /* 128 MB */
     db_param.maxlocks = DEFAULT_MAXLOCKS;
     db_param.maxlockobjs = DEFAULT_MAXLOCKOBJS;
+    db_param.txn_frequency = 1000;          /* close txn every 1000 objects */
+    db_param.logfile_autoremove = 1;
 
     /* Make it accessible for all funcs */
     dbd = dbd_ref;
@@ -1207,7 +1232,7 @@ int cmd_dbd_scanvol(DBD *dbd_ref, struct volinfo *vi, dbd_flags_t flags)
 
     if (setjmp(jmp) != 0) {
         ret = 0;                /* Got signal, jump from dbd_readdir */
-        goto exit;              
+        goto exit_cleanup;              
     }
 
     /* scanvol */
@@ -1216,10 +1241,13 @@ int cmd_dbd_scanvol(DBD *dbd_ref, struct volinfo *vi, dbd_flags_t flags)
         goto exit;
     }
 
+exit_cleanup:
     if (! nocniddb) {
-        /* We can only do this in exclusive mode, otherwise we might delete CNIDs added from
-           other clients in between our pass 1 and 2 */
-        if (flags & DBD_FLAGS_EXCL)
+        dbif_txn_close(dbd, 2);
+        dbif_txn_close(dbd_rebuild, 2);
+        if ((flags & DBD_FLAGS_EXCL) && !(flags & DBD_FLAGS_FORCE))
+            /* We can only do this in exclusive mode, otherwise we might delete CNIDs added from
+               other clients in between our pass 1 and 2 */
             delete_orphaned_cnids(dbd, dbd_rebuild, flags);
     }
 
@@ -1227,7 +1255,7 @@ exit:
     if (dbd_rebuild) {
         dbd_log(LOGDEBUG, "Closing tmp db");
         dbif_close(dbd_rebuild);
-#if 0
+
         if (tmpdb_path) {
             char cmd[8 + MAXPATHLEN];
             snprintf(cmd, 8 + MAXPATHLEN, "rm -f %s/*", tmpdb_path);
@@ -1236,7 +1264,6 @@ exit:
             snprintf(cmd, 8 + MAXPATHLEN, "rmdir %s", tmpdb_path);
             system(cmd);
         }        
-#endif
     }
     return ret;
 }

diff --git a/etc/cnid_dbd/db_param.h b/etc/cnid_dbd/db_param.h
index f3a4827a031a5f99e5fa055c2232d228539024f2..302b16db93632b6f7617eeafbd635fb97926ea95 100644 (file)
--- a/etc/cnid_dbd/db_param.h
+++ b/etc/cnid_dbd/db_param.h
@@ -28,6 +28,7 @@ struct db_param {
     int maxlockobjs;
     int flush_interval;
     int flush_frequency;
+    int txn_frequency;
     char usock_file[MAXPATHLEN + 1];    
     int fd_table_size;
     int idle_timeout;

diff --git a/etc/cnid_dbd/dbif.c b/etc/cnid_dbd/dbif.c
index 22237cc49ba065c6642e0e3aa53b04a5b68a4253..125be0093e55cf53ef2312a9ad7eef56bb263dec 100644 (file)
--- a/etc/cnid_dbd/dbif.c
+++ b/etc/cnid_dbd/dbif.c
@@ -386,6 +386,8 @@ int dbif_env_open(DBD *dbd, struct db_param *dbp, uint32_t dbenv_oflags)
         return -1;
     }
 
+    dbd->db_param = *dbp;
+
     if ((dbif_openlog(dbd)) != 0)
         return -1;
 
@@ -1005,9 +1007,15 @@ int dbif_txn_abort(DBD *dbd)
 }
 
 /* 
-   ret = 1 -> commit txn
-   ret = 0 -> abort txn -> exit!
+   ret = 2 -> commit txn regardless of db_param.txn_frequency
+   ret = 1 -> commit txn if db_param.txn_frequency
+   ret = 0 -> abort txn db_param.txn_frequency -> exit!
    anything else -> exit!
+
+   db_param of the db environment might specify txn_frequency > 1 in which case
+   we only close a txn every txn_frequency time. the `dbd` command uses this for the
+   temp rebuild db, cnid_dbd keeps it at 0. For increasing cnid_dbd throughput this
+   should be tuned and testes as well.
 */
 void dbif_txn_close(DBD *dbd, int ret)
 {
@@ -1016,7 +1024,13 @@ void dbif_txn_close(DBD *dbd, int ret)
             LOG( log_error, logtype_cnid, "Fatal error aborting transaction. Exiting!");
             exit(EXIT_FAILURE);
         }
-    } else if (ret == 1) {
+    } else if (ret == 1 || ret == 2) {
+        static uint count;
+        if (ret != 2 && dbd->db_param.txn_frequency > 1) {
+            count++;
+            if ((count % dbd->db_param.txn_frequency) != 0)
+                return;
+        }
         ret = dbif_txn_commit(dbd);
         if (  ret < 0) {
             LOG( log_error, logtype_cnid, "Fatal error committing transaction. Exiting!");

diff --git a/etc/cnid_dbd/dbif.h b/etc/cnid_dbd/dbif.h
index 9c886fb1f9d60e960c099cb0b5c4920ab2af575c..bcdf46d14716278df1f07f59f1000b24e10d3b16 100644 (file)
--- a/etc/cnid_dbd/dbif.h
+++ b/etc/cnid_dbd/dbif.h
@@ -76,6 +76,7 @@ typedef struct {
 
 typedef struct {
     DB_ENV   *db_env;
+    struct db_param db_param;
     DB_TXN   *db_txn;
     DBC      *db_cur;              /* for dbif_walk */
     char     *db_envhome;