X-Git-Url: https://arthur.barton.de/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=etc%2Fcnid_dbd%2Fcmd_dbd_scanvol.c;h=588da0db777591d0e25a4000d03b1d1b1ffb1ef2;hb=1a6013c7ecd313e03eee1192d54bee624d4a9293;hp=240d4150d3312ef2dec686526e541f9b1a3ed0f3;hpb=aea0dd669334a8e576f6fda99a4aef2852a7bcb2;p=netatalk.git diff --git a/etc/cnid_dbd/cmd_dbd_scanvol.c b/etc/cnid_dbd/cmd_dbd_scanvol.c index 240d4150..588da0db 100644 --- a/etc/cnid_dbd/cmd_dbd_scanvol.c +++ b/etc/cnid_dbd/cmd_dbd_scanvol.c @@ -666,6 +666,8 @@ static int read_addir(void) /* Check CNID for a file/dir, both from db and from ad-file. For detailed specs see intro. + + @return Correct CNID of object or CNID_INVALID (ie 0) on error */ static cnid_t check_cnid(const char *name, cnid_t did, struct stat *st, int adfile_ok, int adflags) { @@ -673,6 +675,17 @@ static cnid_t check_cnid(const char *name, cnid_t did, struct stat *st, int adfi cnid_t db_cnid, ad_cnid; struct adouble ad; + /* Force checkout every X items */ + static int cnidcount = 0; + cnidcount++; + if (cnidcount > 10000) { + cnidcount = 0; + if (dbif_txn_checkpoint(dbd, 0, 0, 0) < 0) { + dbd_log(LOGSTD, "Error checkpointing!"); + return CNID_INVALID; + } + } + /* Get CNID from ad-file if volume is using AFPVOL_CACHE */ ad_cnid = 0; if ( (myvolinfo->v_flags & AFPVOL_CACHE) && ADFILE_OK) { @@ -680,10 +693,10 @@ static cnid_t check_cnid(const char *name, cnid_t did, struct stat *st, int adfi if (ad_open_metadata( name, adflags, O_RDWR, &ad) != 0) { if (dbd_flags & DBD_FLAGS_CLEANUP) - return 0; + return CNID_INVALID; dbd_log( LOGSTD, "Error opening AppleDouble file for '%s/%s': %s", cwdbuf, name, strerror(errno)); - return 0; + return CNID_INVALID; } if (dbd_flags & DBD_FLAGS_FORCE) { @@ -719,7 +732,8 @@ static cnid_t check_cnid(const char *name, cnid_t did, struct stat *st, int adfi /* Query the database */ ret = dbd_lookup(dbd, &rqst, &rply, (dbd_flags & DBD_FLAGS_SCAN) ? 1 : 0); - dbif_txn_close(dbd, ret); + if (dbif_txn_close(dbd, ret) != 0) + return CNID_INVALID; if (rply.result == CNID_DBD_RES_OK) { db_cnid = rply.cnid; } else if (rply.result == CNID_DBD_RES_NOTFOUND) { @@ -741,14 +755,17 @@ static cnid_t check_cnid(const char *name, cnid_t did, struct stat *st, int adfi if ( ! (dbd_flags & DBD_FLAGS_SCAN)) { rqst.cnid = db_cnid; ret = dbd_delete(dbd, &rqst, &rply, DBIF_CNID); - dbif_txn_close(dbd, ret); + if (dbif_txn_close(dbd, ret) != 0) + return CNID_INVALID; rqst.cnid = ad_cnid; ret = dbd_delete(dbd, &rqst, &rply, DBIF_CNID); - dbif_txn_close(dbd, ret); + if (dbif_txn_close(dbd, ret) != 0) + return CNID_INVALID; ret = dbd_rebuild_add(dbd, &rqst, &rply); - dbif_txn_close(dbd, ret); + if (dbif_txn_close(dbd, ret) != 0) + return CNID_INVALID; } return ad_cnid; } else if (ad_cnid && (db_cnid == 0)) { @@ -763,7 +780,8 @@ static cnid_t check_cnid(const char *name, cnid_t did, struct stat *st, int adfi if (ret == CNID_DBD_RES_OK) { /* Occupied! Choose another, update ad-file */ ret = dbd_add(dbd, &rqst, &rply, 1); - dbif_txn_close(dbd, ret); + if (dbif_txn_close(dbd, ret) != 0) + return CNID_INVALID; db_cnid = rply.cnid; dbd_log(LOGSTD, "New CNID for '%s/%s': %u", cwdbuf, name, ntohl(db_cnid)); @@ -776,7 +794,7 @@ static cnid_t check_cnid(const char *name, cnid_t did, struct stat *st, int adfi if (ad_open_metadata( name, adflags, O_RDWR, &ad) != 0) { dbd_log(LOGSTD, "Error opening AppleDouble file for '%s/%s': %s", cwdbuf, name, strerror(errno)); - return 0; + return CNID_INVALID; } ad_setid( &ad, st->st_dev, st->st_ino, db_cnid, did, stamp); ad_flush(&ad); @@ -789,7 +807,8 @@ static cnid_t check_cnid(const char *name, cnid_t did, struct stat *st, int adfi cwdbuf, name, ntohl(ad_cnid)); rqst.cnid = ad_cnid; ret = dbd_rebuild_add(dbd, &rqst, &rply); - dbif_txn_close(dbd, ret); + if (dbif_txn_close(dbd, ret) != 0) + return CNID_INVALID; } return ad_cnid; } else if ((db_cnid == 0) && (ad_cnid == 0)) { @@ -798,7 +817,8 @@ static cnid_t check_cnid(const char *name, cnid_t did, struct stat *st, int adfi if ( ! (dbd_flags & DBD_FLAGS_SCAN)) { /* add to db */ ret = dbd_add(dbd, &rqst, &rply, 1); - dbif_txn_close(dbd, ret); + if (dbif_txn_close(dbd, ret) != 0) + return CNID_INVALID; db_cnid = rply.cnid; dbd_log(LOGSTD, "New CNID for '%s/%s': %u", cwdbuf, name, ntohl(db_cnid)); } @@ -814,7 +834,7 @@ static cnid_t check_cnid(const char *name, cnid_t did, struct stat *st, int adfi if (ad_open_metadata( name, adflags, O_RDWR, &ad) != 0) { dbd_log(LOGSTD, "Error opening AppleDouble file for '%s/%s': %s", cwdbuf, name, strerror(errno)); - return 0; + return CNID_INVALID; } ad_setid( &ad, st->st_dev, st->st_ino, db_cnid, did, stamp); ad_flush(&ad); @@ -824,7 +844,7 @@ static cnid_t check_cnid(const char *name, cnid_t did, struct stat *st, int adfi return db_cnid; } - return 0; + return CNID_INVALID; } /* @@ -842,6 +862,11 @@ static int dbd_readdir(int volroot, cnid_t did) struct dirent *ep; static struct stat st; /* Save some stack space */ + /* keep trying to get the lock */ + if (!db_locked) + if ((db_locked = get_lock(1, NULL)) == -1) + return -1; + /* Check again for .AppleDouble folder, check_adfile also checks/creates it */ if ((addir_ok = check_addir(volroot)) != 0) if ( ! (dbd_flags & DBD_FLAGS_SCAN)) @@ -921,6 +946,22 @@ static int dbd_readdir(int volroot, cnid_t did) continue; } + /************************************************************************** + Statistics + **************************************************************************/ + static unsigned long long statcount = 0; + static time_t t = 0; + + if (t == 0) + t = time(NULL); + + statcount++; + if ((statcount % 10000) == 0) { + if (dbd_flags & DBD_FLAGS_STATS) + dbd_log(LOGSTD, "Scanned: %10llu, time: %10llu s", + statcount, (unsigned long long)(time(NULL) - t)); + } + /************************************************************************** Tests **************************************************************************/ @@ -942,13 +983,24 @@ static int dbd_readdir(int volroot, cnid_t did) cnid = check_cnid(ep->d_name, did, &st, adfile_ok, adflags); /* Now add this object to our rebuild dbd */ - if (cnid) { + if (cnid && dbd_rebuild) { + static uint count = 0; rqst.cnid = rply.cnid; - dbd_rebuild_add(dbd_rebuild, &rqst, &rply); + ret = dbd_rebuild_add(dbd_rebuild, &rqst, &rply); + if (dbif_txn_close(dbd_rebuild, ret) != 0) + return -1; if (rply.result != CNID_DBD_RES_OK) { - dbd_log( LOGDEBUG, "Fatal error adding CNID: %u for '%s/%s' to in-memory rebuild-db", + dbd_log( LOGSTD, "Fatal error adding CNID: %u for '%s/%s' to in-memory rebuild-db", cnid, cwdbuf, ep->d_name); - longjmp(jmp, 1); /* this jumps back to cmd_dbd_scanvol() */ + return -1; + } + count++; + if (count == 10000) { + if (dbif_txn_checkpoint(dbd_rebuild, 0, 0, 0) < 0) { + dbd_log(LOGSTD, "Error checkpointing!"); + return -1; + } + count = 0; } } } @@ -980,7 +1032,7 @@ static int dbd_readdir(int volroot, cnid_t did) close(cwd); *(strrchr(cwdbuf, '/')) = 0; if (ret < 0) - continue; + return -1; } } @@ -1030,7 +1082,7 @@ static int scanvol(struct volinfo *vi, dbd_flags_t flags) */ static void delete_orphaned_cnids(DBD *dbd, DBD *dbd_rebuild, dbd_flags_t flags) { - int ret, deleted = 0; + int ret = 0, deleted = 0; cnid_t dbd_cnid = 0, rebuild_cnid = 0; struct cnid_dbd_rqst rqst; struct cnid_dbd_rply rply; @@ -1051,7 +1103,7 @@ static void delete_orphaned_cnids(DBD *dbd, DBD *dbd_rebuild, dbd_flags_t flags) if (alarmed) longjmp(jmp, 1); /* this jumps back to cmd_dbd_scanvol() */ - if (deleted > 50) { + if (deleted > 1000) { deleted = 0; if (dbif_txn_checkpoint(dbd, 0, 0, 0) < 0) { dbd_log(LOGSTD, "Error checkpointing!"); @@ -1077,7 +1129,8 @@ static void delete_orphaned_cnids(DBD *dbd, DBD *dbd_rebuild, dbd_flags_t flags) goto cleanup; } - dbif_txn_close(dbd, ret); + if (dbif_txn_close(dbd, ret) != 0) + return; deleted++; } /* Check if we got a termination signal */ @@ -1100,19 +1153,21 @@ static void delete_orphaned_cnids(DBD *dbd, DBD *dbd_rebuild, dbd_flags_t flags) (void)dbif_txn_abort(dbd); goto cleanup; } - dbif_txn_close(dbd, ret); + if (dbif_txn_close(dbd, ret) != 0) + return; deleted++; } continue; } if (dbd_cnid > rebuild_cnid) { + dbif_idwalk(dbd, NULL, 1); /* Close cursor */ + dbif_idwalk(dbd_rebuild, NULL, 1); /* Close cursor */ + (void)dbif_txn_close(dbd, 2); + (void)dbif_txn_close(dbd_rebuild, 2); dbd_log(LOGSTD, "Ghost CNID: %u. This is fatal! Dumping rebuild db:\n", rebuild_cnid); dbif_dump(dbd_rebuild, 0); dbd_log(LOGSTD, "Send this dump and a `dbd -d ...` dump to the Netatalk Dev team!"); - dbif_txn_close(dbd, ret); - dbif_idwalk(dbd, NULL, 1); /* Close cursor */ - dbif_idwalk(dbd_rebuild, NULL, 1); /* Close cursor */ goto cleanup; } } /* while ((dbif_idwalk(dbd, &dbd_cnid, 0)) == 1) */ @@ -1123,6 +1178,16 @@ cleanup: return; } +static const char *get_tmpdb_path(void) +{ + pid_t pid = getpid(); + static char path[MAXPATHLEN]; + snprintf(path, MAXPATHLEN, "/tmp/tmpdb-dbd.%u", pid); + if (mkdir(path, 0755) != 0) + return NULL; + return path; +} + /* Main func called from cmd_dbd.c */ @@ -1130,9 +1195,13 @@ int cmd_dbd_scanvol(DBD *dbd_ref, struct volinfo *vi, dbd_flags_t flags) { int ret = 0; struct db_param db_param = { 0 }; + const char *tmpdb_path = NULL; /* Set cachesize for in-memory rebuild db */ - db_param.cachesize = 128 * 1024 * 1024; /* 128 MB */ + db_param.cachesize = 64 * 1024; /* 64 MB */ + db_param.maxlocks = DEFAULT_MAXLOCKS; + db_param.maxlockobjs = DEFAULT_MAXLOCKOBJS; + db_param.logfile_autoremove = 1; /* Make it accessible for all funcs */ dbd = dbd_ref; @@ -1143,38 +1212,75 @@ int cmd_dbd_scanvol(DBD *dbd_ref, struct volinfo *vi, dbd_flags_t flags) return -1; } - if (! nocniddb) { + /* temporary rebuild db, used with -re rebuild to delete unused CNIDs, not used with -f */ + if (! nocniddb && !(flags & DBD_FLAGS_FORCE)) { /* Get volume stamp */ dbd_getstamp(dbd, &rqst, &rply); if (rply.result != CNID_DBD_RES_OK) - goto exit_cleanup; + goto exit; memcpy(stamp, rply.name, CNID_DEV_LEN); /* open/create rebuild dbd, copy rootinfo key */ - if (NULL == (dbd_rebuild = dbif_init(NULL, NULL))) - return -1; - if (0 != (dbif_open(dbd_rebuild, &db_param, 0))) - return -1; - if (0 != (dbif_copy_rootinfokey(dbd, dbd_rebuild))) - goto exit_cleanup; + tmpdb_path = get_tmpdb_path(); + if (NULL == (dbd_rebuild = dbif_init(tmpdb_path, "cnid2.db"))) { + ret = -1; + goto exit; + } + + if (dbif_env_open(dbd_rebuild, + &db_param, + DB_CREATE | DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN) < 0) { + dbd_log(LOGSTD, "error opening tmp database!"); + goto exit; + } + + if (0 != (dbif_open(dbd_rebuild, NULL, 0))) { + ret = -1; + goto exit; + } + + if (0 != (dbif_copy_rootinfokey(dbd, dbd_rebuild))) { + ret = -1; + goto exit; + } } - if (setjmp(jmp) != 0) - goto exit_cleanup; /* Got signal, jump from dbd_readdir */ + if (setjmp(jmp) != 0) { + ret = 0; /* Got signal, jump from dbd_readdir */ + goto exit; + } /* scanvol */ - if ( (scanvol(vi, flags)) != 0) - return -1; + if ( (scanvol(vi, flags)) != 0) { + ret = -1; + goto exit; + } +exit: if (! nocniddb) { - /* We can only do this in exclusive mode, otherwise we might delete CNIDs added from - other clients in between our pass 1 and 2 */ - if (flags & DBD_FLAGS_EXCL) + if (dbif_txn_close(dbd, 2) != 0) + ret = -1; + if (dbd_rebuild) + if (dbif_txn_close(dbd_rebuild, 2) != 0) + ret = -1; + if ((ret == 0) && dbd_rebuild && (flags & DBD_FLAGS_EXCL) && !(flags & DBD_FLAGS_FORCE)) + /* We can only do this in exclusive mode, otherwise we might delete CNIDs added from + other clients in between our pass 1 and 2 */ delete_orphaned_cnids(dbd, dbd_rebuild, flags); } -exit_cleanup: - if (! nocniddb) + if (dbd_rebuild) { + dbd_log(LOGDEBUG, "Closing tmp db"); dbif_close(dbd_rebuild); + + if (tmpdb_path) { + char cmd[8 + MAXPATHLEN]; + snprintf(cmd, 8 + MAXPATHLEN, "rm -f %s/*", tmpdb_path); + dbd_log( LOGDEBUG, "Removing temp database '%s'", tmpdb_path); + system(cmd); + snprintf(cmd, 8 + MAXPATHLEN, "rmdir %s", tmpdb_path); + system(cmd); + } + } return ret; }