You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucy.apache.org by nw...@apache.org on 2017/04/16 10:28:39 UTC
[01/16] lucy git commit: Rework snapshot and deletion locks
Repository: lucy
Updated Branches:
refs/heads/master 7a64a288c -> d7feb9970
Rework snapshot and deletion locks
Request exclusive or shared snapshot locks instead of the global
deletion lock. This removes the need to retry lock requests and
reduces contention.
Project: http://git-wip-us.apache.org/repos/asf/lucy/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucy/commit/5270b988
Tree: http://git-wip-us.apache.org/repos/asf/lucy/tree/5270b988
Diff: http://git-wip-us.apache.org/repos/asf/lucy/diff/5270b988
Branch: refs/heads/master
Commit: 5270b988c477a2e1c110e2069c66bb2d3f27a60f
Parents: 79161a4
Author: Nick Wellnhofer <we...@aevum.de>
Authored: Fri Feb 17 17:19:43 2017 +0100
Committer: Nick Wellnhofer <we...@aevum.de>
Committed: Mon Feb 20 16:26:21 2017 +0100
----------------------------------------------------------------------
core/Lucy/Index/FilePurger.c | 133 ++++++++++++++++------------------
core/Lucy/Index/IndexManager.c | 43 +----------
core/Lucy/Index/IndexManager.cfh | 32 +-------
core/Lucy/Index/IndexReader.c | 18 ++---
core/Lucy/Index/IndexReader.cfh | 3 +-
core/Lucy/Index/PolyReader.c | 123 ++++++++++++-------------------
go/lucy/index_test.go | 13 +---
perl/t/109-read_locking.t | 53 ++++++--------
perl/t/111-index_manager.t | 7 +-
9 files changed, 145 insertions(+), 280 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucy/blob/5270b988/core/Lucy/Index/FilePurger.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/FilePurger.c b/core/Lucy/Index/FilePurger.c
index 6811c33..433dec3 100644
--- a/core/Lucy/Index/FilePurger.c
+++ b/core/Lucy/Index/FilePurger.c
@@ -33,7 +33,7 @@
// obsolete snapshots to snapshots array.
static void
S_discover_unused(FilePurger *self, Snapshot *current, Hash *spared,
- Hash *purged, Vector *snapshots);
+ Hash *purged, Vector *snapshots, Vector *locks);
// Add filepath entries referenced by a snapshot to a Hash. Note that
// it's assumed that snapshots only list entries local to the index
@@ -75,69 +75,65 @@ FilePurger_Destroy_IMP(FilePurger *self) {
void
FilePurger_Purge_Snapshots_IMP(FilePurger *self, Snapshot *current) {
FilePurgerIVARS *const ivars = FilePurger_IVARS(self);
- Lock *deletion_lock = IxManager_Make_Deletion_Lock(ivars->manager);
-
- // Obtain deletion lock, purge files, release deletion lock.
- if (Lock_Obtain_Exclusive(deletion_lock)) {
- Folder *folder = ivars->folder;
- Hash *failures = Hash_new(16);
- Hash *spared = Hash_new(32);
- Hash *purged = Hash_new(32);
- Vector *snapshots = Vec_new(16);
-
- // Don't allow the locks directory to be zapped.
- Hash_Store_Utf8(spared, "locks", 5, (Obj*)CFISH_TRUE);
-
- S_discover_unused(self, current, spared, purged, snapshots);
-
- // Attempt to delete entries -- if failure, no big deal, just try
- // again later.
- HashIterator *iter = HashIter_new(purged);
- while (HashIter_Next(iter)) {
- String *entry = HashIter_Get_Key(iter);
- if (Hash_Fetch(spared, entry)) { continue; }
- if (!S_delete_entry(folder, entry)) {
- if (Folder_Exists(folder, entry)) {
- Hash_Store(failures, entry, (Obj*)CFISH_TRUE);
- }
+
+ Folder *folder = ivars->folder;
+ Hash *failures = Hash_new(16);
+ Hash *spared = Hash_new(32);
+ Hash *purged = Hash_new(32);
+ Vector *snapshots = Vec_new(16);
+ Vector *locks = Vec_new(16);
+
+ // Don't allow the locks directory to be zapped.
+ Hash_Store_Utf8(spared, "locks", 5, (Obj*)CFISH_TRUE);
+
+ S_discover_unused(self, current, spared, purged, snapshots, locks);
+
+ // Attempt to delete entries -- if failure, no big deal, just try
+ // again later.
+ HashIterator *iter = HashIter_new(purged);
+ while (HashIter_Next(iter)) {
+ String *entry = HashIter_Get_Key(iter);
+ if (Hash_Fetch(spared, entry)) { continue; }
+ if (!S_delete_entry(folder, entry)) {
+ if (Folder_Exists(folder, entry)) {
+ Hash_Store(failures, entry, (Obj*)CFISH_TRUE);
}
}
+ }
- for (size_t i = 0, max = Vec_Get_Size(snapshots); i < max; i++) {
- Snapshot *snapshot = (Snapshot*)Vec_Fetch(snapshots, i);
- bool snapshot_has_failures = false;
- if (Hash_Get_Size(failures)) {
- // Only delete snapshot files if all of their entries were
- // successfully deleted.
- Vector *entries = Snapshot_List(snapshot);
- for (size_t j = Vec_Get_Size(entries); j--;) {
- String *entry = (String*)Vec_Fetch(entries, j);
- if (Hash_Fetch(failures, entry)) {
- snapshot_has_failures = true;
- break;
- }
+ for (size_t i = 0, max = Vec_Get_Size(snapshots); i < max; i++) {
+ Snapshot *snapshot = (Snapshot*)Vec_Fetch(snapshots, i);
+ bool snapshot_has_failures = false;
+ if (Hash_Get_Size(failures)) {
+ // Only delete snapshot files if all of their entries were
+ // successfully deleted.
+ Vector *entries = Snapshot_List(snapshot);
+ for (size_t j = Vec_Get_Size(entries); j--;) {
+ String *entry = (String*)Vec_Fetch(entries, j);
+ if (Hash_Fetch(failures, entry)) {
+ snapshot_has_failures = true;
+ break;
}
- DECREF(entries);
- }
- if (!snapshot_has_failures) {
- String *snapfile = Snapshot_Get_Path(snapshot);
- Folder_Local_Delete(folder, snapfile);
}
+ DECREF(entries);
+ }
+ if (!snapshot_has_failures) {
+ String *snapfile = Snapshot_Get_Path(snapshot);
+ Folder_Local_Delete(folder, snapfile);
}
-
- DECREF(iter);
- DECREF(failures);
- DECREF(purged);
- DECREF(spared);
- DECREF(snapshots);
- Lock_Release(deletion_lock);
}
- else {
- WARN("Can't obtain deletion lock, skipping deletion of "
- "obsolete files");
+
+ // Release snapshot locks.
+ for (size_t i = 0, max = Vec_Get_Size(locks); i < max; i++) {
+ Lock_Release((Lock*)Vec_Fetch(locks, i));
}
- DECREF(deletion_lock);
+ DECREF(iter);
+ DECREF(failures);
+ DECREF(purged);
+ DECREF(spared);
+ DECREF(snapshots);
+ DECREF(locks);
}
void
@@ -185,7 +181,7 @@ FilePurger_Purge_Aborted_Merge_IMP(FilePurger *self) {
static void
S_discover_unused(FilePurger *self, Snapshot *current, Hash *spared,
- Hash *purged, Vector *snapshots) {
+ Hash *purged, Vector *snapshots, Vector *locks) {
FilePurgerIVARS *const ivars = FilePurger_IVARS(self);
Folder *folder = ivars->folder;
DirHandle *dh = Folder_Open_Dir(folder, NULL);
@@ -206,26 +202,23 @@ S_discover_unused(FilePurger *self, Snapshot *current, Hash *spared,
) {
Snapshot *snapshot
= Snapshot_Read_File(Snapshot_new(), folder, entry);
- Lock *lock
- = IxManager_Make_Snapshot_Read_Lock(ivars->manager, entry);
+ Lock *lock = IxManager_Make_Snapshot_Lock(ivars->manager, entry);
- // DON'T obtain the lock -- only see whether another
- // entity holds a lock on the snapshot file.
- if (lock && Lock_Is_Locked(lock)) {
+ if (Lock_Request_Exclusive(lock)) {
+ // No one's using this snapshot, so all of its entries are
+ // candidates for deletion.
+ Vec_Push(snapshots, (Obj*)snapshot);
+ Vec_Push(locks, (Obj*)lock);
+ S_find_all_referenced(snapshot, purged);
+ }
+ else {
// The snapshot file is locked, which means someone's using
// that version of the index -- protect all of its entries.
Hash_Store(spared, entry, (Obj*)CFISH_TRUE);
S_find_all_referenced(snapshot, spared);
+ DECREF(snapshot);
+ DECREF(lock);
}
- else {
- // No one's using this snapshot, so all of its entries are
- // candidates for deletion.
- Vec_Push(snapshots, INCREF(snapshot));
- S_find_all_referenced(snapshot, purged);
- }
-
- DECREF(snapshot);
- DECREF(lock);
}
DECREF(entry);
}
http://git-wip-us.apache.org/repos/asf/lucy/blob/5270b988/core/Lucy/Index/IndexManager.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/IndexManager.c b/core/Lucy/Index/IndexManager.c
index 82b9012..bd3aa30 100644
--- a/core/Lucy/Index/IndexManager.c
+++ b/core/Lucy/Index/IndexManager.c
@@ -99,8 +99,6 @@ IxManager_init(IndexManager *self, String *host) {
ivars->write_lock_interval = 100;
ivars->merge_lock_timeout = 0;
ivars->merge_lock_interval = 1000;
- ivars->deletion_lock_timeout = 1000;
- ivars->deletion_lock_interval = 100;
return self;
}
@@ -263,16 +261,6 @@ IxManager_Make_Write_Lock_IMP(IndexManager *self) {
}
Lock*
-IxManager_Make_Deletion_Lock_IMP(IndexManager *self) {
- IndexManagerIVARS *const ivars = IxManager_IVARS(self);
- String *lock_name = SSTR_WRAP_C("deletion");
- return (Lock*)LFLock_new(ivars->folder, lock_name, ivars->host,
- (int32_t)ivars->deletion_lock_timeout,
- (int32_t)ivars->deletion_lock_interval,
- true);
-}
-
-Lock*
IxManager_Make_Merge_Lock_IMP(IndexManager *self) {
IndexManagerIVARS *const ivars = IxManager_IVARS(self);
String *merge_lock_name = SSTR_WRAP_C("merge");
@@ -323,8 +311,7 @@ IxManager_Remove_Merge_Data_IMP(IndexManager *self) {
}
Lock*
-IxManager_Make_Snapshot_Read_Lock_IMP(IndexManager *self,
- String *filename) {
+IxManager_Make_Snapshot_Lock_IMP(IndexManager *self, String *filename) {
IndexManagerIVARS *const ivars = IxManager_IVARS(self);
if (!Str_Starts_With_Utf8(filename, "snapshot_", 9)
@@ -382,16 +369,6 @@ IxManager_Get_Merge_Lock_Interval_IMP(IndexManager *self) {
return IxManager_IVARS(self)->merge_lock_interval;
}
-uint32_t
-IxManager_Get_Deletion_Lock_Timeout_IMP(IndexManager *self) {
- return IxManager_IVARS(self)->deletion_lock_timeout;
-}
-
-uint32_t
-IxManager_Get_Deletion_Lock_Interval_IMP(IndexManager *self) {
- return IxManager_IVARS(self)->deletion_lock_interval;
-}
-
void
IxManager_Set_Write_Lock_Timeout_IMP(IndexManager *self, uint32_t timeout) {
if (timeout > INT32_MAX) {
@@ -424,22 +401,4 @@ IxManager_Set_Merge_Lock_Interval_IMP(IndexManager *self, uint32_t interval) {
IxManager_IVARS(self)->merge_lock_interval = interval;
}
-void
-IxManager_Set_Deletion_Lock_Timeout_IMP(IndexManager *self,
- uint32_t timeout) {
- if (timeout > INT32_MAX) {
- THROW(ERR, "Timeout can't be greater than INT32_MAX: %u32", timeout);
- }
- IxManager_IVARS(self)->deletion_lock_timeout = timeout;
-}
-
-void
-IxManager_Set_Deletion_Lock_Interval_IMP(IndexManager *self,
- uint32_t interval) {
- if (interval > INT32_MAX) {
- THROW(ERR, "Interval can't be greater than INT32_MAX: %u32", interval);
- }
- IxManager_IVARS(self)->deletion_lock_interval = interval;
-}
-
http://git-wip-us.apache.org/repos/asf/lucy/blob/5270b988/core/Lucy/Index/IndexManager.cfh
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/IndexManager.cfh b/core/Lucy/Index/IndexManager.cfh
index a94517d..e73ef14 100644
--- a/core/Lucy/Index/IndexManager.cfh
+++ b/core/Lucy/Index/IndexManager.cfh
@@ -35,8 +35,6 @@ public class Lucy::Index::IndexManager nickname IxManager
uint32_t write_lock_interval;
uint32_t merge_lock_timeout;
uint32_t merge_lock_interval;
- uint32_t deletion_lock_timeout;
- uint32_t deletion_lock_interval;
/** Create a new IndexManager.
*
@@ -104,12 +102,6 @@ public class Lucy::Index::IndexManager nickname IxManager
incremented Lock*
Make_Write_Lock(IndexManager *self);
- /** Create the Lock which grants permission to delete obsolete snapshot
- * files or any file listed within an existing snapshot file.
- */
- incremented Lock*
- Make_Deletion_Lock(IndexManager *self);
-
incremented Lock*
Make_Merge_Lock(IndexManager *self);
@@ -129,11 +121,11 @@ public class Lucy::Index::IndexManager nickname IxManager
bool
Remove_Merge_Data(IndexManager *self);
- /** Create a shared lock on a snapshot file, which serves as a proxy for
+ /** Create a lock on a snapshot file, which serves as a proxy for
* all the files it lists and indicates that they must not be deleted.
*/
incremented Lock*
- Make_Snapshot_Read_Lock(IndexManager *self, String *filename);
+ Make_Snapshot_Lock(IndexManager *self, String *filename);
/** Return the highest number for a segment directory which contains a
* segmeta file in the snapshot.
@@ -187,26 +179,6 @@ public class Lucy::Index::IndexManager nickname IxManager
*/
uint32_t
Get_Merge_Lock_Interval(IndexManager *self);
-
- /** Setter for deletion lock timeout. Default: 1000 milliseconds.
- */
- void
- Set_Deletion_Lock_Timeout(IndexManager *self, uint32_t timeout);
-
- /** Getter for deletion lock timeout.
- */
- uint32_t
- Get_Deletion_Lock_Timeout(IndexManager *self);
-
- /** Setter for deletion lock retry interval. Default: 100 milliseconds.
- */
- void
- Set_Deletion_Lock_Interval(IndexManager *self, uint32_t timeout);
-
- /** Getter for deletion lock retry interval.
- */
- uint32_t
- Get_Deletion_Lock_Interval(IndexManager *self);
}
http://git-wip-us.apache.org/repos/asf/lucy/blob/5270b988/core/Lucy/Index/IndexReader.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/IndexReader.c b/core/Lucy/Index/IndexReader.c
index 98ef4dc..dabb3be 100644
--- a/core/Lucy/Index/IndexReader.c
+++ b/core/Lucy/Index/IndexReader.c
@@ -55,8 +55,7 @@ IxReader_init(IndexReader *self, Schema *schema, Folder *folder,
DECREF(snapshot);
IndexReaderIVARS *const ivars = IxReader_IVARS(self);
ivars->components = Hash_new(0);
- ivars->read_lock = NULL;
- ivars->deletion_lock = NULL;
+ ivars->snapshot_lock = NULL;
if (manager) {
ivars->manager = (IndexManager*)INCREF(manager);
IxManager_Set_Folder(ivars->manager, ivars->folder);
@@ -81,10 +80,10 @@ IxReader_Close_IMP(IndexReader *self) {
DECREF(iter);
Hash_Clear(ivars->components);
}
- if (ivars->read_lock) {
- Lock_Release(ivars->read_lock);
- DECREF(ivars->read_lock);
- ivars->read_lock = NULL;
+ if (ivars->snapshot_lock) {
+ Lock_Release(ivars->snapshot_lock);
+ DECREF(ivars->snapshot_lock);
+ ivars->snapshot_lock = NULL;
}
}
@@ -92,12 +91,11 @@ void
IxReader_Destroy_IMP(IndexReader *self) {
IndexReaderIVARS *const ivars = IxReader_IVARS(self);
DECREF(ivars->components);
- if (ivars->read_lock) {
- Lock_Release(ivars->read_lock);
- DECREF(ivars->read_lock);
+ if (ivars->snapshot_lock) {
+ Lock_Release(ivars->snapshot_lock);
+ DECREF(ivars->snapshot_lock);
}
DECREF(ivars->manager);
- DECREF(ivars->deletion_lock);
SUPER_DESTROY(self, INDEXREADER);
}
http://git-wip-us.apache.org/repos/asf/lucy/blob/5270b988/core/Lucy/Index/IndexReader.cfh
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/IndexReader.cfh b/core/Lucy/Index/IndexReader.cfh
index 96be62a..501318c 100644
--- a/core/Lucy/Index/IndexReader.cfh
+++ b/core/Lucy/Index/IndexReader.cfh
@@ -39,8 +39,7 @@ public class Lucy::Index::IndexReader nickname IxReader
Hash *components;
IndexManager *manager;
- Lock *read_lock;
- Lock *deletion_lock;
+ Lock *snapshot_lock;
/** Abstract initializer.
*/
http://git-wip-us.apache.org/repos/asf/lucy/blob/5270b988/core/Lucy/Index/PolyReader.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/PolyReader.c b/core/Lucy/Index/PolyReader.c
index ea5e35f..6b9e943 100644
--- a/core/Lucy/Index/PolyReader.c
+++ b/core/Lucy/Index/PolyReader.c
@@ -34,15 +34,11 @@
#include "Lucy/Util/Freezer.h"
#include "Lucy/Util/IndexFileNames.h"
-// Obtain/release read locks and commit locks.
+// Request/release snapshot locks.
static bool
-S_obtain_read_lock(PolyReader *self, String *snapshot_filename);
-static bool
-S_obtain_deletion_lock(PolyReader *self);
+S_request_snapshot_lock(PolyReader *self, String *snapshot_filename);
static void
-S_release_read_lock(PolyReader *self);
-static void
-S_release_deletion_lock(PolyReader *self);
+S_release_snapshot_lock(PolyReader *self);
// Try to open all SegReaders.
struct try_open_elements_context {
@@ -334,19 +330,13 @@ PolyReader*
PolyReader_do_open(PolyReader *self, Obj *index, Snapshot *snapshot,
IndexManager *manager) {
PolyReaderIVARS *const ivars = PolyReader_IVARS(self);
- Folder *folder = S_derive_folder(index);
- uint64_t last_gen = 0;
+ Folder *folder = S_derive_folder(index);
+ Err *last_error = NULL;
+ uint64_t last_gen = 0;
PolyReader_init(self, NULL, folder, snapshot, manager, NULL);
DECREF(folder);
- if (manager) {
- if (!S_obtain_deletion_lock(self)) {
- DECREF(self);
- THROW(LOCKERR, "Couldn't get deletion lock");
- }
- }
-
while (1) {
String *target_snap_file;
@@ -372,12 +362,26 @@ PolyReader_do_open(PolyReader *self, Obj *index, Snapshot *snapshot,
// Derive "generation" of this snapshot file from its name.
uint64_t gen = IxFileNames_extract_gen(target_snap_file);
+ if (gen <= last_gen) {
+ // If a snapshot was supplied, we couldn't read it. Otherwise,
+ // no new snapshot was found which should never happen. Throw
+ // error from previous attempt to read the snapshot.
+ DECREF(self);
+ RETHROW(last_error);
+ }
+
+ last_gen = gen;
+
// Get a read lock on the most recent snapshot file if indicated.
+ // There's no need to retry. In the unlikely case that we fail to
+ // request a lock on the latest snapshot, there must be a
+ // FilePurger deleting the snapshot, which means that a newer
+ // snapshot just became available.
if (manager) {
- if (!S_obtain_read_lock(self, target_snap_file)) {
- DECREF(self);
- THROW(LOCKERR, "Couldn't get read lock for %o",
- target_snap_file);
+ if (!S_request_snapshot_lock(self, target_snap_file)) {
+ // Index updated, so try again.
+ last_error = (Err*)INCREF(Err_get_error());
+ continue;
}
}
@@ -399,20 +403,13 @@ PolyReader_do_open(PolyReader *self, Obj *index, Snapshot *snapshot,
context.snapshot = ivars->snapshot;
context.folder = folder;
context.path = target_snap_file;
- Err *error = Err_trap(S_try_read_snapshot, &context);
+ last_error = Err_trap(S_try_read_snapshot, &context);
- if (error) {
- S_release_read_lock(self);
+ if (last_error) {
+ S_release_snapshot_lock(self);
DECREF(target_snap_file);
- if (last_gen < gen) { // Index updated, so try again.
- DECREF(error);
- last_gen = gen;
- continue;
- }
- else { // Real error.
- if (manager) { S_release_deletion_lock(self); }
- RETHROW(error);
- }
+ // Index updated, so try again.
+ continue;
}
}
@@ -425,18 +422,12 @@ PolyReader_do_open(PolyReader *self, Obj *index, Snapshot *snapshot,
struct try_open_elements_context context;
context.self = self;
context.seg_readers = NULL;
- Err *error = Err_trap(S_try_open_elements, &context);
- if (error) {
- S_release_read_lock(self);
+ last_error = Err_trap(S_try_open_elements, &context);
+ if (last_error) {
+ S_release_snapshot_lock(self);
DECREF(target_snap_file);
- if (last_gen < gen) { // Index updated, so try again.
- DECREF(error);
- last_gen = gen;
- }
- else { // Real error.
- if (manager) { S_release_deletion_lock(self); }
- RETHROW(error);
- }
+ // Index updated, so try again.
+ continue;
}
else { // Succeeded.
S_init_sub_readers(self, (Vector*)context.seg_readers);
@@ -446,8 +437,6 @@ PolyReader_do_open(PolyReader *self, Obj *index, Snapshot *snapshot,
}
}
- if (manager) { S_release_deletion_lock(self); }
-
return self;
}
@@ -466,49 +455,27 @@ S_derive_folder(Obj *index) {
return folder;
}
-static bool
-S_obtain_deletion_lock(PolyReader *self) {
- PolyReaderIVARS *const ivars = PolyReader_IVARS(self);
- ivars->deletion_lock = IxManager_Make_Deletion_Lock(ivars->manager);
- if (!Lock_Obtain_Exclusive(ivars->deletion_lock)) {
- DECREF(ivars->deletion_lock);
- ivars->deletion_lock = NULL;
- return false;
- }
- return true;
-}
-
static bool
-S_obtain_read_lock(PolyReader *self, String *snapshot_file_name) {
+S_request_snapshot_lock(PolyReader *self, String *snapshot_file_name) {
PolyReaderIVARS *const ivars = PolyReader_IVARS(self);
- ivars->read_lock = IxManager_Make_Snapshot_Read_Lock(ivars->manager,
- snapshot_file_name);
+ ivars->snapshot_lock = IxManager_Make_Snapshot_Lock(ivars->manager,
+ snapshot_file_name);
- if (!Lock_Obtain_Shared(ivars->read_lock)) {
- DECREF(ivars->read_lock);
- ivars->read_lock = NULL;
+ if (!Lock_Request_Shared(ivars->snapshot_lock)) {
+ DECREF(ivars->snapshot_lock);
+ ivars->snapshot_lock = NULL;
return false;
}
return true;
}
static void
-S_release_read_lock(PolyReader *self) {
- PolyReaderIVARS *const ivars = PolyReader_IVARS(self);
- if (ivars->read_lock) {
- Lock_Release(ivars->read_lock);
- DECREF(ivars->read_lock);
- ivars->read_lock = NULL;
- }
-}
-
-static void
-S_release_deletion_lock(PolyReader *self) {
+S_release_snapshot_lock(PolyReader *self) {
PolyReaderIVARS *const ivars = PolyReader_IVARS(self);
- if (ivars->deletion_lock) {
- Lock_Release(ivars->deletion_lock);
- DECREF(ivars->deletion_lock);
- ivars->deletion_lock = NULL;
+ if (ivars->snapshot_lock) {
+ Lock_Release(ivars->snapshot_lock);
+ DECREF(ivars->snapshot_lock);
+ ivars->snapshot_lock = NULL;
}
}
http://git-wip-us.apache.org/repos/asf/lucy/blob/5270b988/go/lucy/index_test.go
----------------------------------------------------------------------
diff --git a/go/lucy/index_test.go b/go/lucy/index_test.go
index bf1ba1e..a47c364 100644
--- a/go/lucy/index_test.go
+++ b/go/lucy/index_test.go
@@ -173,14 +173,6 @@ func TestIndexManagerAccessors(t *testing.T) {
if got := manager.getMergeLockInterval(); got != 43 {
t.Errorf("set/getMergeLockInterval: %d", got)
}
- manager.setDeletionLockTimeout(71)
- if got := manager.getDeletionLockTimeout(); got != 71 {
- t.Errorf("set/getDeletionLockTimeout: %d", got)
- }
- manager.setDeletionLockInterval(41)
- if got := manager.getDeletionLockInterval(); got != 41 {
- t.Errorf("set/getDeletionLockInterval: %d", got)
- }
}
func TestIndexManagerLocks(t *testing.T) {
@@ -192,11 +184,8 @@ func TestIndexManagerLocks(t *testing.T) {
if _, ok := manager.makeMergeLock().(Lock); !ok {
t.Errorf("makeMergeLock")
}
- if _, ok := manager.makeDeletionLock().(Lock); !ok {
- t.Errorf("makeDeletionLock")
- }
snapFile := "snapshot_4a.json"
- if _, ok := manager.makeSnapshotReadLock(snapFile).(Lock); !ok {
+ if _, ok := manager.makeSnapshotLock(snapFile).(Lock); !ok {
t.Errorf("makeDeletionLock")
}
}
http://git-wip-us.apache.org/repos/asf/lucy/blob/5270b988/perl/t/109-read_locking.t
----------------------------------------------------------------------
diff --git a/perl/t/109-read_locking.t b/perl/t/109-read_locking.t
index 6a39813..60dbc1c 100644
--- a/perl/t/109-read_locking.t
+++ b/perl/t/109-read_locking.t
@@ -17,19 +17,10 @@ use strict;
use warnings;
use lib 'buildlib';
-use Test::More tests => 15;
-
-package FastIndexManager;
-use base qw( Lucy::Index::IndexManager );
-
-sub new {
- my $self = shift->SUPER::new(@_);
- $self->set_deletion_lock_timeout(100);
- return $self;
-}
+use Test::More tests => 14;
package NonMergingIndexManager;
-use base qw( FastIndexManager );
+use base qw( Lucy::Index::IndexManager );
sub recycle { [] }
package main;
@@ -43,49 +34,51 @@ my $schema = Lucy::Test::TestSchema->new;
my $indexer = Lucy::Index::Indexer->new(
index => $folder,
schema => $schema,
- manager => FastIndexManager->new,
+ manager => Lucy::Index::IndexManager->new,
create => 1,
);
$indexer->delete_by_term( field => 'content', term => $_ ) for qw( a b c );
$indexer->add_doc( { content => 'x' } );
-# Artificially create deletion lock.
-my $outstream = $folder->open_out('locks/deletion.lock')
+# Artificially create snapshot lock.
+my $outstream = $folder->open_out('locks/snapshot_1.lock')
or die Clownfish->error;
$outstream->print("{}");
$outstream->close;
-{
- my $captured;
- local $SIG{__WARN__} = sub { $captured = shift; };
- $indexer->commit;
- like( $captured, qr/obsolete/,
- "Indexer warns if it can't get a deletion lock" );
-}
-ok( $folder->exists('locks/deletion.lock'),
- "Indexer doesn't delete deletion lock when it can't get it" );
+$indexer->commit;
+
+ok( $folder->exists('locks/snapshot_1.lock'),
+ "Indexer doesn't delete snapshot lock when it can't get it" );
my $num_ds_files = grep {m/documents\.dat$/} @{ $folder->list_r };
cmp_ok( $num_ds_files, '>', 1,
"Indexer doesn't process deletions when it can't get deletion lock" );
-my $num_snap_files = grep {m/snapshot/} @{ $folder->list_r };
+my $num_snap_files = grep {m/snapshot.*\.json/} @{ $folder->list_r };
is( $num_snap_files, 2, "didn't zap the old snap file" );
my $reader;
SKIP: {
skip( "IndexReader opening failure leaks", 1 )
if $ENV{LUCY_VALGRIND};
+ my $snapshot = Lucy::Index::Snapshot->new;
+ $snapshot->read_file(
+ folder => $folder,
+ path => 'snapshot_1.json',
+ );
eval {
$reader = Lucy::Index::IndexReader->open(
- index => $folder,
- manager => FastIndexManager->new( host => 'me' ),
+ index => $folder,
+ snapshot => $snapshot,
+ manager => Lucy::Index::IndexManager->new( host => 'me' ),
);
};
ok( blessed($@) && $@->isa("Lucy::Store::LockErr"),
- "IndexReader dies if it can't get deletion lock"
+ "IndexReader dies if it can't get snapshot lock"
);
}
-$folder->delete('locks/deletion.lock') or die "Can't delete 'deletion.lock'";
+$folder->delete('locks/snapshot_1.lock')
+ or die "Can't delete 'snapshot_1.lock'";
Test_race_condition_1: {
my $latest_snapshot_file = latest_snapshot($folder);
@@ -103,7 +96,7 @@ Test_race_condition_1: {
$reader = Lucy::Index::IndexReader->open(
index => $folder,
- manager => FastIndexManager->new( host => 'me' ),
+ manager => Lucy::Index::IndexManager->new( host => 'me' ),
);
is( $reader->doc_count, 1,
"reader overcomes race condition of index update after read lock" );
@@ -149,7 +142,7 @@ $folder = create_index(qw( a b c x ));
# Establish read lock.
$reader = Lucy::Index::IndexReader->open(
index => $folder,
- manager => FastIndexManager->new( host => 'me' ),
+ manager => Lucy::Index::IndexManager->new( host => 'me' ),
);
$indexer = Lucy::Index::Indexer->new(
http://git-wip-us.apache.org/repos/asf/lucy/blob/5270b988/perl/t/111-index_manager.t
----------------------------------------------------------------------
diff --git a/perl/t/111-index_manager.t b/perl/t/111-index_manager.t
index 3cb616b..d42ae98 100644
--- a/perl/t/111-index_manager.t
+++ b/perl/t/111-index_manager.t
@@ -32,7 +32,7 @@ sub recycle {
package main;
-use Test::More tests => 13;
+use Test::More tests => 11;
use Lucy::Test;
my $folder = Lucy::Store::RAMFolder->new;
@@ -101,11 +101,6 @@ $manager->set_merge_lock_timeout(3);
is( $manager->get_merge_lock_timeout, 3, "set/get merge lock timeout" );
$manager->set_merge_lock_interval(4);
is( $manager->get_merge_lock_interval, 4, "set/get merge lock interval" );
-$manager->set_deletion_lock_timeout(5);
-is( $manager->get_deletion_lock_timeout, 5, "set/get deletion lock timeout" );
-$manager->set_deletion_lock_interval(6);
-is( $manager->get_deletion_lock_interval,
- 6, "set/get deletion lock interval" );
my $indexer = Lucy::Index::Indexer->new(
index => $folder,
[15/16] lucy git commit: Merge branch 'master' into improve-locking
Posted by nw...@apache.org.
Merge branch 'master' into improve-locking
Project: http://git-wip-us.apache.org/repos/asf/lucy/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucy/commit/a3cf33f1
Tree: http://git-wip-us.apache.org/repos/asf/lucy/tree/a3cf33f1
Diff: http://git-wip-us.apache.org/repos/asf/lucy/diff/a3cf33f1
Branch: refs/heads/master
Commit: a3cf33f1efc796a284d6a5761142f89a1319ac52
Parents: 6e8538a 7a64a28
Author: Nick Wellnhofer <we...@aevum.de>
Authored: Fri Mar 17 17:10:51 2017 +0100
Committer: Nick Wellnhofer <we...@aevum.de>
Committed: Fri Mar 17 17:10:51 2017 +0100
----------------------------------------------------------------------
core/Lucy/Analysis/Analyzer.cfh | 2 +
core/Lucy/Index/BackgroundMerger.c | 43 ++--
core/Lucy/Index/IndexManager.c | 27 ---
core/Lucy/Index/IndexManager.cfh | 7 -
core/Lucy/Index/Indexer.c | 25 +--
core/Lucy/Index/Indexer.cfh | 1 +
core/Lucy/Store/FSFileHandle.c | 7 +-
core/Lucy/Store/FSFolder.c | 241 +++++++++++++++-------
core/Lucy/Util/IndexFileNames.c | 8 +
core/Lucy/Util/IndexFileNames.cfh | 6 +
go/build.go | 1 -
go/lucy/index.go | 12 --
go/lucy/index_test.go | 5 -
perl/buildlib/Lucy/Build/Binding/Analysis.pm | 59 +++++-
perl/t/051-fsfile.t | 1 +
perl/t/240-custom-analyzer.t | 69 +++++++
16 files changed, 350 insertions(+), 164 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucy/blob/a3cf33f1/core/Lucy/Index/BackgroundMerger.c
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucy/blob/a3cf33f1/core/Lucy/Index/IndexManager.c
----------------------------------------------------------------------
diff --cc core/Lucy/Index/IndexManager.c
index 8e56509,24eed1d..b99254d
--- a/core/Lucy/Index/IndexManager.c
+++ b/core/Lucy/Index/IndexManager.c
@@@ -25,10 -25,10 +25,9 @@@
#include "Lucy/Index/Snapshot.h"
#include "Lucy/Store/DirHandle.h"
#include "Lucy/Store/Folder.h"
-#include "Lucy/Store/Lock.h"
-#include "Lucy/Store/LockFactory.h"
+#include "Lucy/Store/LockFileLock.h"
#include "Lucy/Util/IndexFileNames.h"
#include "Lucy/Util/Json.h"
- #include "Lucy/Util/StringHelper.h"
#include <stdlib.h>
http://git-wip-us.apache.org/repos/asf/lucy/blob/a3cf33f1/core/Lucy/Index/IndexManager.cfh
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucy/blob/a3cf33f1/core/Lucy/Index/Indexer.c
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucy/blob/a3cf33f1/go/build.go
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucy/blob/a3cf33f1/go/lucy/index_test.go
----------------------------------------------------------------------
diff --cc go/lucy/index_test.go
index a47c364,0049ee4..341bf64
--- a/go/lucy/index_test.go
+++ b/go/lucy/index_test.go
@@@ -212,11 -222,7 +211,7 @@@ func TestIndexManagerMergeData(t *testi
}
func TestIndexManagerMisc(t *testing.T) {
- manager := NewIndexManager("", nil)
+ manager := NewIndexManager("")
- manager.SetFolder(NewRAMFolder(""))
- if got, err := manager.MakeSnapshotFilename(); !strings.Contains(got, "snapshot") || err != nil {
- t.Errorf("MakeSnapshotFilename: %s, %v", got, err)
- }
snapshot := NewSnapshot()
snapshot.AddEntry("seg_4")
snapshot.AddEntry("seg_5")
[02/16] lucy git commit: Don't allow double obtain/release
Posted by nw...@apache.org.
Don't allow double obtain/release
Project: http://git-wip-us.apache.org/repos/asf/lucy/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucy/commit/79161a4b
Tree: http://git-wip-us.apache.org/repos/asf/lucy/tree/79161a4b
Diff: http://git-wip-us.apache.org/repos/asf/lucy/diff/79161a4b
Branch: refs/heads/master
Commit: 79161a4b233ce3570ecf4c48745b469ccb50aec7
Parents: 3e22986
Author: Nick Wellnhofer <we...@aevum.de>
Authored: Fri Feb 17 13:53:59 2017 +0100
Committer: Nick Wellnhofer <we...@aevum.de>
Committed: Mon Feb 20 16:26:21 2017 +0100
----------------------------------------------------------------------
core/Lucy/Store/Lock.c | 51 ++++++++++++++++++++++++++++---------------
core/Lucy/Store/Lock.cfh | 1 +
2 files changed, 34 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucy/blob/79161a4b/core/Lucy/Store/Lock.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Store/Lock.c b/core/Lucy/Store/Lock.c
index 6d39362..d187405 100644
--- a/core/Lucy/Store/Lock.c
+++ b/core/Lucy/Store/Lock.c
@@ -123,6 +123,10 @@ Lock_Obtain_Exclusive_IMP(Lock *self) {
/***************************************************************************/
+#define LFLOCK_STATE_UNLOCKED 0
+#define LFLOCK_STATE_LOCKED_SHARED 1
+#define LFLOCK_STATE_LOCKED_EXCLUSIVE 2
+
static bool
S_request(LockFileLockIVARS *ivars, String *lock_path);
@@ -182,6 +186,10 @@ LFLock_Request_Shared_IMP(LockFileLock *self) {
THROW(ERR, "Can't request shared lock if exclusive_only is set");
}
+ if (ivars->state != LFLOCK_STATE_UNLOCKED) {
+ THROW(ERR, "Lock already acquired");
+ }
+
// TODO: The is_locked test and subsequent file creation is prone to a
// race condition. We could protect the whole process with an internal
// exclusive lock.
@@ -192,16 +200,7 @@ LFLock_Request_Shared_IMP(LockFileLock *self) {
return false;
}
- String *path = ivars->shared_lock_path;
-
- // Null shared_lock_path indicates whether this particular instance is
- // locked.
- if (path && Folder_Exists(ivars->folder, path)) {
- // Don't allow double obtain.
- String *msg = Str_newf("Lock already obtained via '%o'", path);
- Err_set_error((Err*)LockErr_new(msg));
- return false;
- }
+ String *path = NULL;
uint32_t i = 0;
do {
@@ -211,11 +210,11 @@ LFLock_Request_Shared_IMP(LockFileLock *self) {
if (S_request(ivars, path)) {
ivars->shared_lock_path = path;
+ ivars->state = LFLOCK_STATE_LOCKED_SHARED;
return true;
}
else {
DECREF(path);
- ivars->shared_lock_path = NULL;
return false;
}
}
@@ -224,6 +223,10 @@ bool
LFLock_Request_Exclusive_IMP(LockFileLock *self) {
LockFileLockIVARS *const ivars = LFLock_IVARS(self);
+ if (ivars->state != LFLOCK_STATE_UNLOCKED) {
+ THROW(ERR, "Lock already acquired");
+ }
+
// TODO: The is_locked test and subsequent file creation is prone to a
// race condition. We could protect the whole process with an internal
// exclusive lock.
@@ -237,7 +240,13 @@ LFLock_Request_Exclusive_IMP(LockFileLock *self) {
return false;
}
- return S_request(ivars, ivars->lock_path);
+ if (S_request(ivars, ivars->lock_path)) {
+ ivars->state = LFLOCK_STATE_LOCKED_EXCLUSIVE;
+ return true;
+ }
+ else {
+ return false;
+ }
}
static bool
@@ -321,7 +330,16 @@ void
LFLock_Release_IMP(LockFileLock *self) {
LockFileLockIVARS *const ivars = LFLock_IVARS(self);
- if (ivars->shared_lock_path) {
+ if (ivars->state == LFLOCK_STATE_UNLOCKED) {
+ THROW(ERR, "Lock not acquired");
+ }
+
+ if (ivars->state == LFLOCK_STATE_LOCKED_EXCLUSIVE) {
+ if (Folder_Exists(ivars->folder, ivars->lock_path)) {
+ S_maybe_delete_file(ivars, ivars->lock_path, true, false);
+ }
+ }
+ else { // Shared lock.
if (Folder_Exists(ivars->folder, ivars->shared_lock_path)) {
S_maybe_delete_file(ivars, ivars->shared_lock_path, true, false);
}
@@ -330,11 +348,8 @@ LFLock_Release_IMP(LockFileLock *self) {
DECREF(ivars->shared_lock_path);
ivars->shared_lock_path = NULL;
}
- else {
- if (Folder_Exists(ivars->folder, ivars->lock_path)) {
- S_maybe_delete_file(ivars, ivars->lock_path, true, false);
- }
- }
+
+ ivars->state = LFLOCK_STATE_UNLOCKED;
}
bool
http://git-wip-us.apache.org/repos/asf/lucy/blob/79161a4b/core/Lucy/Store/Lock.cfh
----------------------------------------------------------------------
diff --git a/core/Lucy/Store/Lock.cfh b/core/Lucy/Store/Lock.cfh
index bef6dae..fb813e1 100644
--- a/core/Lucy/Store/Lock.cfh
+++ b/core/Lucy/Store/Lock.cfh
@@ -127,6 +127,7 @@ class Lucy::Store::LockFileLock nickname LFLock
String *shared_lock_path;
String *link_path;
+ int state;
bool exclusive_only;
inert incremented LockFileLock*
[06/16] lucy git commit: Separate purging of snapshots and aborted
merges
Posted by nw...@apache.org.
Separate purging of snapshots and aborted merges
Purge aborted merge when starting an Indexer/BGMerger session and
purge snapshots at the end of Commit.
Rework FilePurger to use a single hash for purged and spared files
when purging snapshots. Optimize Folder handling assuming that
Snapshots only contain entries local to the index folder.
Make sure to list the contents of the real directory, not the
virtual compound file directory. CFReaderDirHandle lists both real
and virtual files which might be a bug in itself.
Port BGMerger tests to C.
TODO: Move Perl BGMerge test to t/binding.
Project: http://git-wip-us.apache.org/repos/asf/lucy/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucy/commit/2051451d
Tree: http://git-wip-us.apache.org/repos/asf/lucy/tree/2051451d
Diff: http://git-wip-us.apache.org/repos/asf/lucy/diff/2051451d
Branch: refs/heads/master
Commit: 2051451de456f51e35345f12206ca4cb0040dd7c
Parents: 5f15a92
Author: Nick Wellnhofer <we...@aevum.de>
Authored: Wed Feb 15 05:10:07 2017 +0100
Committer: Nick Wellnhofer <we...@aevum.de>
Committed: Mon Feb 20 16:26:21 2017 +0100
----------------------------------------------------------------------
core/Lucy/Index/BackgroundMerger.c | 6 +-
core/Lucy/Index/FilePurger.c | 210 ++++++++++-----------
core/Lucy/Index/FilePurger.cfh | 15 +-
core/Lucy/Index/Indexer.c | 17 +-
go/lucy/index_test.go | 5 +-
perl/t/core/233-background_merger.t | 23 +++
test/Lucy/Test.c | 2 +
test/Lucy/Test/Index/NoMergeManager.c | 42 +++++
test/Lucy/Test/Index/NoMergeManager.cfh | 30 +++
test/Lucy/Test/Index/TestBackgroundMerger.c | 208 ++++++++++++++++++++
test/Lucy/Test/Index/TestBackgroundMerger.cfh | 29 +++
test/Lucy/Test/Index/TestSortWriter.c | 28 +--
test/Lucy/Test/Index/TestSortWriter.cfh | 15 --
test/Lucy/Test/TestUtils.c | 17 +-
14 files changed, 462 insertions(+), 185 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucy/blob/2051451d/core/Lucy/Index/BackgroundMerger.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/BackgroundMerger.c b/core/Lucy/Index/BackgroundMerger.c
index c729bda..8f36050 100644
--- a/core/Lucy/Index/BackgroundMerger.c
+++ b/core/Lucy/Index/BackgroundMerger.c
@@ -109,8 +109,8 @@ BGMerger_init(BackgroundMerger *self, Obj *index, IndexManager *manager) {
}
// Create FilePurger. Zap detritus from previous sessions.
- ivars->file_purger = FilePurger_new(folder, ivars->snapshot, ivars->manager);
- FilePurger_Purge(ivars->file_purger);
+ ivars->file_purger = FilePurger_new(folder, ivars->manager);
+ FilePurger_Purge_Aborted_Merge(ivars->file_purger);
// Open a PolyReader, passing in the IndexManager so we get a read lock on
// the Snapshot's files -- so that Indexers don't zap our files while
@@ -518,7 +518,7 @@ BGMerger_Commit_IMP(BackgroundMerger *self) {
if (ivars->needs_commit) {
// Purge obsolete files.
- FilePurger_Purge(ivars->file_purger);
+ FilePurger_Purge_Snapshots(ivars->file_purger, ivars->snapshot);
}
// Release the write lock.
http://git-wip-us.apache.org/repos/asf/lucy/blob/2051451d/core/Lucy/Index/FilePurger.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/FilePurger.c b/core/Lucy/Index/FilePurger.c
index 8c71678..55ade17 100644
--- a/core/Lucy/Index/FilePurger.c
+++ b/core/Lucy/Index/FilePurger.c
@@ -15,55 +15,52 @@
*/
#define C_LUCY_FILEPURGER
-#include <ctype.h>
#include "Lucy/Util/ToolSet.h"
#include "Lucy/Index/FilePurger.h"
#include "Clownfish/Boolean.h"
+#include "Clownfish/HashIterator.h"
#include "Lucy/Index/IndexManager.h"
#include "Lucy/Index/Segment.h"
#include "Lucy/Index/Snapshot.h"
-#include "Lucy/Plan/Schema.h"
+#include "Lucy/Store/CompoundFileReader.h"
#include "Lucy/Store/DirHandle.h"
#include "Lucy/Store/Folder.h"
#include "Lucy/Store/Lock.h"
#include "Lucy/Util/Json.h"
-// Place unused files into purgables array and obsolete Snapshots into
-// snapshots array.
+// Add unused files to purged hash, used files to spared hash, and
+// obsolete snapshots to snapshots array.
static void
-S_discover_unused(FilePurger *self, Vector **purgables, Vector **snapshots);
+S_discover_unused(FilePurger *self, Snapshot *current, Hash *spared,
+ Hash *purged, Vector *snapshots);
-// Clean up after a failed background merge session, adding all dead files to
-// the list of candidates to be zapped.
+// Add filepath entries referenced by a snapshot to a Hash. Note that
+// it's assumed that snapshots only list entries local to the index
+// folder.
static void
-S_zap_dead_merge(FilePurger *self, Hash *candidates);
+S_find_all_referenced(Snapshot *snapshot, Hash *set);
-// Return an array of recursively expanded filepath entries.
-static Vector*
-S_find_all_referenced(Folder *folder, Vector *entries);
+// Delete local entry from a folder. Handles CompoundFileReaders efficiently
+// but doesn't support subdirectories in 'entry'.
+static bool
+S_delete_entry(Folder *folder, String *entry);
FilePurger*
-FilePurger_new(Folder *folder, Snapshot *snapshot, IndexManager *manager) {
+FilePurger_new(Folder *folder, IndexManager *manager) {
FilePurger *self = (FilePurger*)Class_Make_Obj(FILEPURGER);
- return FilePurger_init(self, folder, snapshot, manager);
+ return FilePurger_init(self, folder, manager);
}
FilePurger*
-FilePurger_init(FilePurger *self, Folder *folder, Snapshot *snapshot,
- IndexManager *manager) {
+FilePurger_init(FilePurger *self, Folder *folder, IndexManager *manager) {
FilePurgerIVARS *const ivars = FilePurger_IVARS(self);
ivars->folder = (Folder*)INCREF(folder);
- ivars->snapshot = (Snapshot*)INCREF(snapshot);
ivars->manager = manager
? (IndexManager*)INCREF(manager)
: IxManager_new(NULL, NULL);
IxManager_Set_Folder(ivars->manager, folder);
- // Don't allow the locks directory to be zapped.
- ivars->disallowed = Hash_new(0);
- Hash_Store_Utf8(ivars->disallowed, "locks", 5, (Obj*)CFISH_TRUE);
-
return self;
}
@@ -71,35 +68,36 @@ void
FilePurger_Destroy_IMP(FilePurger *self) {
FilePurgerIVARS *const ivars = FilePurger_IVARS(self);
DECREF(ivars->folder);
- DECREF(ivars->snapshot);
DECREF(ivars->manager);
- DECREF(ivars->disallowed);
SUPER_DESTROY(self, FILEPURGER);
}
void
-FilePurger_Purge_IMP(FilePurger *self) {
+FilePurger_Purge_Snapshots_IMP(FilePurger *self, Snapshot *current) {
FilePurgerIVARS *const ivars = FilePurger_IVARS(self);
Lock *deletion_lock = IxManager_Make_Deletion_Lock(ivars->manager);
// Obtain deletion lock, purge files, release deletion lock.
Lock_Clear_Stale(deletion_lock);
if (Lock_Obtain(deletion_lock)) {
- Folder *folder = ivars->folder;
- Hash *failures = Hash_new(0);
- Vector *purgables;
- Vector *snapshots;
+ Folder *folder = ivars->folder;
+ Hash *failures = Hash_new(16);
+ Hash *spared = Hash_new(32);
+ Hash *purged = Hash_new(32);
+ Vector *snapshots = Vec_new(16);
+
+ // Don't allow the locks directory to be zapped.
+ Hash_Store_Utf8(spared, "locks", 5, (Obj*)CFISH_TRUE);
- S_discover_unused(self, &purgables, &snapshots);
+ S_discover_unused(self, current, spared, purged, snapshots);
// Attempt to delete entries -- if failure, no big deal, just try
- // again later. Proceed in reverse lexical order so that directories
- // get deleted after they've been emptied.
- Vec_Sort(purgables);
- for (size_t i = Vec_Get_Size(purgables); i--;) {
- String *entry = (String*)Vec_Fetch(purgables, i);
- if (Hash_Fetch(ivars->disallowed, entry)) { continue; }
- if (!Folder_Delete(folder, entry)) {
+ // again later.
+ HashIterator *iter = HashIter_new(purged);
+ while (HashIter_Next(iter)) {
+ String *entry = HashIter_Get_Key(iter);
+ if (Hash_Fetch(spared, entry)) { continue; }
+ if (!S_delete_entry(folder, entry)) {
if (Folder_Exists(folder, entry)) {
Hash_Store(failures, entry, (Obj*)CFISH_TRUE);
}
@@ -124,12 +122,14 @@ FilePurger_Purge_IMP(FilePurger *self) {
}
if (!snapshot_has_failures) {
String *snapfile = Snapshot_Get_Path(snapshot);
- Folder_Delete(folder, snapfile);
+ Folder_Local_Delete(folder, snapfile);
}
}
+ DECREF(iter);
DECREF(failures);
- DECREF(purgables);
+ DECREF(purged);
+ DECREF(spared);
DECREF(snapshots);
Lock_Release(deletion_lock);
}
@@ -141,8 +141,8 @@ FilePurger_Purge_IMP(FilePurger *self) {
DECREF(deletion_lock);
}
-static void
-S_zap_dead_merge(FilePurger *self, Hash *candidates) {
+void
+FilePurger_Purge_Aborted_Merge_IMP(FilePurger *self) {
FilePurgerIVARS *const ivars = FilePurger_IVARS(self);
IndexManager *manager = ivars->manager;
Lock *merge_lock = IxManager_Make_Merge_Lock(manager);
@@ -155,27 +155,26 @@ S_zap_dead_merge(FilePurger *self, Hash *candidates) {
: NULL;
if (cutoff) {
- String *cutoff_seg = Seg_num_to_name(Json_obj_to_i64(cutoff));
- if (Folder_Exists(ivars->folder, cutoff_seg)) {
- String *merge_json = SSTR_WRAP_C("merge.json");
- DirHandle *dh = Folder_Open_Dir(ivars->folder, cutoff_seg);
+ Folder *folder = ivars->folder;
- if (!dh) {
- THROW(ERR, "Can't open segment dir '%o'", cutoff_seg);
+ String *cutoff_seg = Seg_num_to_name(Json_obj_to_i64(cutoff));
+ if (Folder_Local_Exists(folder, cutoff_seg)) {
+ if (!S_delete_entry(folder, cutoff_seg)) {
+ if (Folder_Local_Exists(folder, cutoff_seg)) {
+ WARN("Couldn't delete '%o' from aborted merge",
+ cutoff_seg);
+ }
}
+ }
- Hash_Store(candidates, cutoff_seg, (Obj*)CFISH_TRUE);
- Hash_Store(candidates, merge_json, (Obj*)CFISH_TRUE);
- while (DH_Next(dh)) {
- // TODO: recursively delete subdirs within seg dir.
- String *entry = DH_Get_Entry(dh);
- String *filepath = Str_newf("%o/%o", cutoff_seg, entry);
- Hash_Store(candidates, filepath, (Obj*)CFISH_TRUE);
- DECREF(filepath);
- DECREF(entry);
+ String *merge_json = SSTR_WRAP_C("merge.json");
+ if (!Folder_Local_Delete(folder, merge_json)) {
+ if (Folder_Local_Exists(folder, merge_json)) {
+ WARN("Couldn't delete '%o' from aborted merge",
+ merge_json);
}
- DECREF(dh);
}
+
DECREF(cutoff_seg);
}
@@ -187,28 +186,20 @@ S_zap_dead_merge(FilePurger *self, Hash *candidates) {
}
static void
-S_discover_unused(FilePurger *self, Vector **purgables_ptr,
- Vector **snapshots_ptr) {
+S_discover_unused(FilePurger *self, Snapshot *current, Hash *spared,
+ Hash *purged, Vector *snapshots) {
FilePurgerIVARS *const ivars = FilePurger_IVARS(self);
Folder *folder = ivars->folder;
DirHandle *dh = Folder_Open_Dir(folder, NULL);
if (!dh) { RETHROW(INCREF(Err_get_error())); }
- Vector *spared = Vec_new(1);
- Vector *snapshots = Vec_new(1);
- String *snapfile = NULL;
-
- // Start off with the list of files in the current snapshot.
- if (ivars->snapshot) {
- Vector *entries = Snapshot_List(ivars->snapshot);
- Vector *referenced = S_find_all_referenced(folder, entries);
- Vec_Push_All(spared, referenced);
- DECREF(entries);
- DECREF(referenced);
- snapfile = Snapshot_Get_Path(ivars->snapshot);
- if (snapfile) { Vec_Push(spared, INCREF(snapfile)); }
+ String *snapfile = Snapshot_Get_Path(current);
+
+ snapfile = Snapshot_Get_Path(current);
+ if (snapfile) {
+ Hash_Store(spared, snapfile, (Obj*)CFISH_TRUE);
}
+ S_find_all_referenced(current, spared);
- Hash *candidates = Hash_new(64);
while (DH_Next(dh)) {
String *entry = DH_Get_Entry(dh);
if (Str_Starts_With_Utf8(entry, "snapshot_", 9)
@@ -219,8 +210,6 @@ S_discover_unused(FilePurger *self, Vector **purgables_ptr,
= Snapshot_Read_File(Snapshot_new(), folder, entry);
Lock *lock
= IxManager_Make_Snapshot_Read_Lock(ivars->manager, entry);
- Vector *snap_list = Snapshot_List(snapshot);
- Vector *referenced = S_find_all_referenced(folder, snap_list);
// DON'T obtain the lock -- only see whether another
// entity holds a lock on the snapshot file.
@@ -230,67 +219,60 @@ S_discover_unused(FilePurger *self, Vector **purgables_ptr,
if (lock && Lock_Is_Locked(lock)) {
// The snapshot file is locked, which means someone's using
// that version of the index -- protect all of its entries.
- size_t new_size = Vec_Get_Size(spared)
- + Vec_Get_Size(referenced)
- + 1;
- Vec_Grow(spared, new_size);
- Vec_Push(spared, (Obj*)Str_Clone(entry));
- Vec_Push_All(spared, referenced);
+ Hash_Store(spared, entry, (Obj*)CFISH_TRUE);
+ S_find_all_referenced(snapshot, spared);
}
else {
// No one's using this snapshot, so all of its entries are
// candidates for deletion.
- for (size_t i = 0, max = Vec_Get_Size(referenced); i < max; i++) {
- String *file = (String*)Vec_Fetch(referenced, i);
- Hash_Store(candidates, file, (Obj*)CFISH_TRUE);
- }
Vec_Push(snapshots, INCREF(snapshot));
+ S_find_all_referenced(snapshot, purged);
}
- DECREF(referenced);
- DECREF(snap_list);
DECREF(snapshot);
DECREF(lock);
}
DECREF(entry);
}
+
DECREF(dh);
+}
- // Clean up after a dead segment consolidation.
- S_zap_dead_merge(self, candidates);
+static void
+S_find_all_referenced(Snapshot *snapshot, Hash *set) {
+ Vector *snap_list = Snapshot_List(snapshot);
- // Eliminate any current files from the list of files to be purged.
- for (size_t i = 0, max = Vec_Get_Size(spared); i < max; i++) {
- String *filename = (String*)Vec_Fetch(spared, i);
- DECREF(Hash_Delete(candidates, filename));
+ for (size_t i = 0, max = Vec_Get_Size(snap_list); i < max; i++) {
+ String *entry = (String*)Vec_Fetch(snap_list, i);
+ Hash_Store(set, entry, (Obj*)CFISH_TRUE);
}
- // Pass back purgables and Snapshots.
- *purgables_ptr = Hash_Keys(candidates);
- *snapshots_ptr = snapshots;
-
- DECREF(candidates);
- DECREF(spared);
+ DECREF(snap_list);
}
-static Vector*
-S_find_all_referenced(Folder *folder, Vector *entries) {
- Hash *uniqued = Hash_new(Vec_Get_Size(entries));
- for (size_t i = 0, max = Vec_Get_Size(entries); i < max; i++) {
- String *entry = (String*)Vec_Fetch(entries, i);
- Hash_Store(uniqued, entry, (Obj*)CFISH_TRUE);
- if (Folder_Is_Directory(folder, entry)) {
- Vector *contents = Folder_List_R(folder, entry);
- for (size_t j = Vec_Get_Size(contents); j--;) {
- String *sub_entry = (String*)Vec_Fetch(contents, j);
- Hash_Store(uniqued, sub_entry, (Obj*)CFISH_TRUE);
- }
- DECREF(contents);
+static bool
+S_delete_entry(Folder *folder, String *folder_entry) {
+ if (Folder_Local_Is_Directory(folder, folder_entry)) {
+ // CFReader has the nasty habit of listing both real and virtual
+ // files. Get the real folder.
+ Folder *inner = Folder_Local_Find_Folder(folder, folder_entry);
+ if (inner == NULL) { return false; }
+ if (Folder_is_a(inner, COMPOUNDFILEREADER)) {
+ inner = CFReader_Get_Real_Folder((CompoundFileReader*)inner);
+ }
+
+ Vector *entries = Folder_List(inner, NULL);
+ if (entries == NULL) { return false; }
+
+ for (size_t i = 0, max = Vec_Get_Size(entries); i < max; i++) {
+ // TODO: recursively delete subdirs within seg dir.
+ String *entry = (String*)Vec_Fetch(entries, i);
+ Folder_Local_Delete(inner, entry);
}
+
+ DECREF(entries);
}
- Vector *referenced = Hash_Keys(uniqued);
- DECREF(uniqued);
- return referenced;
-}
+ return Folder_Local_Delete(folder, folder_entry);
+}
http://git-wip-us.apache.org/repos/asf/lucy/blob/2051451d/core/Lucy/Index/FilePurger.cfh
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/FilePurger.cfh b/core/Lucy/Index/FilePurger.cfh
index c03ca18..4f3923d 100644
--- a/core/Lucy/Index/FilePurger.cfh
+++ b/core/Lucy/Index/FilePurger.cfh
@@ -22,22 +22,23 @@ parcel Lucy;
class Lucy::Index::FilePurger inherits Clownfish::Obj {
Folder *folder;
- Snapshot *snapshot;
IndexManager *manager;
- Hash *disallowed;
inert incremented FilePurger*
- new(Folder *folder, Snapshot *snapshot = NULL,
- IndexManager *manager = NULL);
+ new(Folder *folder, IndexManager *manager = NULL);
inert FilePurger*
- init(FilePurger *self, Folder *folder, Snapshot *snapshot = NULL,
- IndexManager *manager = NULL);
+ init(FilePurger *self, Folder *folder, IndexManager *manager = NULL);
/** Purge obsolete files from the index.
*/
void
- Purge(FilePurger *self);
+ Purge_Snapshots(FilePurger *self, Snapshot *current);
+
+ /** Purge files left behind by an aborted background merge.
+ */
+ void
+ Purge_Aborted_Merge(FilePurger *self);
public void
Destroy(FilePurger *self);
http://git-wip-us.apache.org/repos/asf/lucy/blob/2051451d/core/Lucy/Index/Indexer.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/Indexer.c b/core/Lucy/Index/Indexer.c
index 16b915b..031237f 100644
--- a/core/Lucy/Index/Indexer.c
+++ b/core/Lucy/Index/Indexer.c
@@ -161,13 +161,10 @@ Indexer_init(Indexer *self, Schema *schema, Obj *index,
}
}
- // Zap detritus from previous sessions.
- // Note: we have to feed FilePurger with the most recent snapshot file
- // now, but with the Indexer's snapshot later.
- FilePurger *file_purger
- = FilePurger_new(folder, latest_snapshot, ivars->manager);
- FilePurger_Purge(file_purger);
- DECREF(file_purger);
+ // Create new FilePurger and zap detritus from an aborted background
+ // merge.
+ ivars->file_purger = FilePurger_new(folder, ivars->manager);
+ FilePurger_Purge_Aborted_Merge(ivars->file_purger);
// Create a new segment.
int64_t new_seg_num
@@ -204,9 +201,7 @@ Indexer_init(Indexer *self, Schema *schema, Obj *index,
DECREF(merge_lock);
- // Create new SegWriter and FilePurger.
- ivars->file_purger
- = FilePurger_new(folder, ivars->snapshot, ivars->manager);
+ // Create new SegWriter.
ivars->seg_writer = SegWriter_new(ivars->schema, ivars->snapshot,
ivars->segment, ivars->polyreader);
SegWriter_Prep_Seg_Dir(ivars->seg_writer);
@@ -571,7 +566,7 @@ Indexer_Commit_IMP(Indexer *self) {
if (!success) { RETHROW(INCREF(Err_get_error())); }
// Purge obsolete files.
- FilePurger_Purge(ivars->file_purger);
+ FilePurger_Purge_Snapshots(ivars->file_purger, ivars->snapshot);
}
// Release locks, invalidating the Indexer.
http://git-wip-us.apache.org/repos/asf/lucy/blob/2051451d/go/lucy/index_test.go
----------------------------------------------------------------------
diff --git a/go/lucy/index_test.go b/go/lucy/index_test.go
index b8823e8..30f7c4a 100644
--- a/go/lucy/index_test.go
+++ b/go/lucy/index_test.go
@@ -541,8 +541,9 @@ func TestFilePurgerMisc(t *testing.T) {
snapshot := NewSnapshot()
snapshot.WriteFile(folder, "")
- purger := NewFilePurger(folder, snapshot, nil)
- purger.purge()
+ purger := NewFilePurger(folder, nil)
+ purger.purgeSnapshots(snapshot)
+ purger.purgeAbortedMerge()
if folder.exists("foo") {
t.Errorf("Failed to purge file")
}
http://git-wip-us.apache.org/repos/asf/lucy/blob/2051451d/perl/t/core/233-background_merger.t
----------------------------------------------------------------------
diff --git a/perl/t/core/233-background_merger.t b/perl/t/core/233-background_merger.t
new file mode 100644
index 0000000..b5cd424
--- /dev/null
+++ b/perl/t/core/233-background_merger.t
@@ -0,0 +1,23 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+my $success = Lucy::Test::run_tests("Lucy::Test::Index::TestBackgroundMerger");
+
+exit($success ? 0 : 1);
+
http://git-wip-us.apache.org/repos/asf/lucy/blob/2051451d/test/Lucy/Test.c
----------------------------------------------------------------------
diff --git a/test/Lucy/Test.c b/test/Lucy/Test.c
index 97d1854..22223e1 100644
--- a/test/Lucy/Test.c
+++ b/test/Lucy/Test.c
@@ -32,6 +32,7 @@
#include "Lucy/Test/Analysis/TestStandardTokenizer.h"
#include "Lucy/Test/Highlight/TestHeatMap.h"
#include "Lucy/Test/Highlight/TestHighlighter.h"
+#include "Lucy/Test/Index/TestBackgroundMerger.h"
#include "Lucy/Test/Index/TestDocWriter.h"
#include "Lucy/Test/Index/TestHighlightWriter.h"
#include "Lucy/Test/Index/TestIndexManager.h"
@@ -138,6 +139,7 @@ Test_create_test_suite() {
TestSuite_Add_Batch(suite, (TestBatch*)TestBlobType_new());
TestSuite_Add_Batch(suite, (TestBatch*)TestNumericType_new());
TestSuite_Add_Batch(suite, (TestBatch*)TestFType_new());
+ TestSuite_Add_Batch(suite, (TestBatch*)TestBGMerger_new());
TestSuite_Add_Batch(suite, (TestBatch*)TestSeg_new());
TestSuite_Add_Batch(suite, (TestBatch*)TestHighlighter_new());
TestSuite_Add_Batch(suite, (TestBatch*)TestSimple_new());
http://git-wip-us.apache.org/repos/asf/lucy/blob/2051451d/test/Lucy/Test/Index/NoMergeManager.c
----------------------------------------------------------------------
diff --git a/test/Lucy/Test/Index/NoMergeManager.c b/test/Lucy/Test/Index/NoMergeManager.c
new file mode 100644
index 0000000..7c0804b
--- /dev/null
+++ b/test/Lucy/Test/Index/NoMergeManager.c
@@ -0,0 +1,42 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define TESTLUCY_USE_SHORT_NAMES
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Test/Index/NoMergeManager.h"
+#include "Lucy/Index/DeletionsWriter.h"
+#include "Lucy/Index/IndexManager.h"
+#include "Lucy/Index/PolyReader.h"
+
+NoMergeManager*
+NoMergeManager_new() {
+ NoMergeManager *self = (NoMergeManager*)Class_Make_Obj(NOMERGEMANAGER);
+ return (NoMergeManager*)IxManager_init((IndexManager*)self, NULL, NULL);
+}
+
+Vector*
+NoMergeManager_Recycle_IMP(NoMergeManager *self, PolyReader *reader,
+ DeletionsWriter *del_writer, int64_t cutoff,
+ bool optimize) {
+ UNUSED_VAR(self);
+ UNUSED_VAR(reader);
+ UNUSED_VAR(del_writer);
+ UNUSED_VAR(cutoff);
+ UNUSED_VAR(optimize);
+ return Vec_new(0);
+}
+
http://git-wip-us.apache.org/repos/asf/lucy/blob/2051451d/test/Lucy/Test/Index/NoMergeManager.cfh
----------------------------------------------------------------------
diff --git a/test/Lucy/Test/Index/NoMergeManager.cfh b/test/Lucy/Test/Index/NoMergeManager.cfh
new file mode 100644
index 0000000..0b667ab
--- /dev/null
+++ b/test/Lucy/Test/Index/NoMergeManager.cfh
@@ -0,0 +1,30 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel TestLucy;
+
+class Lucy::Test::Index::NoMergeManager inherits Lucy::Index::IndexManager {
+
+ inert incremented NoMergeManager*
+ new();
+
+ public incremented Vector*
+ Recycle(NoMergeManager *self, PolyReader *reader,
+ DeletionsWriter *del_writer, int64_t cutoff,
+ bool optimize = false);
+}
+
+
http://git-wip-us.apache.org/repos/asf/lucy/blob/2051451d/test/Lucy/Test/Index/TestBackgroundMerger.c
----------------------------------------------------------------------
diff --git a/test/Lucy/Test/Index/TestBackgroundMerger.c b/test/Lucy/Test/Index/TestBackgroundMerger.c
new file mode 100644
index 0000000..084c49d
--- /dev/null
+++ b/test/Lucy/Test/Index/TestBackgroundMerger.c
@@ -0,0 +1,208 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define TESTLUCY_USE_SHORT_NAMES
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Test/Index/TestBackgroundMerger.h"
+#include "Clownfish/TestHarness/TestBatchRunner.h"
+#include "Lucy/Document/Doc.h"
+#include "Lucy/Index/BackgroundMerger.h"
+#include "Lucy/Index/Indexer.h"
+#include "Lucy/Index/Segment.h"
+#include "Lucy/Search/Hits.h"
+#include "Lucy/Search/IndexSearcher.h"
+#include "Lucy/Store/FSFolder.h"
+#include "Lucy/Store/RAMFolder.h"
+#include "Lucy/Test/Index/NoMergeManager.h"
+#include "Lucy/Test/TestSchema.h"
+#include "Lucy/Test/TestUtils.h"
+#include "Lucy/Util/Json.h"
+
+TestBackgroundMerger*
+TestBGMerger_new() {
+ return (TestBackgroundMerger*)Class_Make_Obj(TESTBACKGROUNDMERGER);
+}
+
+static void
+S_add_doc(Schema *schema, Folder *folder, int code_point,
+ IndexManager *manager) {
+ Indexer *indexer = Indexer_new(schema, (Obj*)folder, manager, 0);
+ Doc *doc = Doc_new(NULL, 0);
+ String *field = SSTR_WRAP_C("content");
+ String *content = Str_new_from_char(code_point);
+
+ Doc_Store(doc, field, (Obj*)content);
+ Indexer_Add_Doc(indexer, doc, 1.0f);
+ Indexer_Commit(indexer);
+
+ DECREF(content);
+ DECREF(doc);
+ DECREF(indexer);
+}
+
+static int
+S_count_segs(Folder *folder) {
+ Vector *entries = Folder_List_R(folder, NULL);
+ int count = 0;
+
+ for (size_t i = 0, max = Vec_Get_Size(entries); i < max; i++) {
+ String *entry = (String*)Vec_Fetch(entries, i);
+ if (Str_Ends_With_Utf8(entry, "segmeta.json", 12)) { count++; }
+ }
+
+ DECREF(entries);
+ return count;
+}
+
+static void
+S_test_terms(TestBatchRunner *runner, Folder *folder, const char *msg) {
+ IndexSearcher *searcher = IxSearcher_new((Obj*)folder);
+
+ for (int i = 'a'; i <= 'e'; i++) {
+ const char *status;
+ uint32_t expected;
+
+ if (i == 'b') {
+ expected = 0;
+ status = "deleted";
+ }
+ else {
+ expected = 1;
+ status = "present";
+ }
+
+ String *query = Str_new_from_char(i);
+ Hits *hits = IxSearcher_Hits(searcher, (Obj*)query, 0, 10, NULL);
+ TEST_UINT_EQ(runner, Hits_Total_Hits(hits), expected,
+ "term %c still %s%s", i, status, msg);
+ DECREF(hits);
+ DECREF(query);
+ }
+
+ DECREF(searcher);
+}
+
+static void
+test_bg_merger(TestBatchRunner *runner) {
+ Schema *schema = (Schema*)TestSchema_new(false);
+#if 1
+ Folder *folder = (Folder*)RAMFolder_new(NULL);
+#else
+ Folder *folder = (Folder*)FSFolder_new(SSTR_WRAP_C("_test_index"));
+#endif
+ IndexManager *no_merge = (IndexManager*)NoMergeManager_new();
+
+ Folder_Initialize(folder);
+
+ for (int i = 'a'; i <= 'c'; i++) {
+ S_add_doc(schema, folder, i, no_merge);
+ }
+
+ BackgroundMerger *bg_merger = BGMerger_new((Obj*)folder, NULL);
+ S_add_doc(schema, folder, 'd', NULL);
+ TEST_INT_EQ(runner, S_count_segs(folder), 4,
+ "BackgroundMerger prevents Indexer from merging claimed"
+ " segments");
+
+ {
+ Indexer *indexer = Indexer_new(NULL, (Obj*)folder, NULL, 0);
+ Doc *doc = Doc_new(NULL, 0);
+ String *field = SSTR_WRAP_C("content");
+ String *content = Str_new_from_char('e');
+
+ Doc_Store(doc, field, (Obj*)content);
+ Indexer_Add_Doc(indexer, doc, 1.0f);
+ Indexer_Delete_By_Term(indexer, field, (Obj*)SSTR_WRAP_C("b"));
+ Indexer_Commit(indexer);
+ TEST_INT_EQ(runner, S_count_segs(folder), 4,
+ "Indexer may still merge unclaimed segments");
+
+ DECREF(content);
+ DECREF(doc);
+ DECREF(indexer);
+ }
+
+ BGMerger_Commit(bg_merger);
+ TEST_INT_EQ(runner, S_count_segs(folder), 3, "Background merge completes");
+ String *del_file = SSTR_WRAP_C("seg_7/deletions-seg_4.bv");
+ TEST_TRUE(runner, Folder_Exists(folder, del_file),
+ "deletions carried forward");
+
+ S_test_terms(runner, folder, "");
+
+ String *merge_json = SSTR_WRAP_C("merge.json");
+ String *cutoff_seg = NULL;
+
+ {
+ // Simulate failed background merge.
+ DECREF(bg_merger);
+ IndexManager *manager = IxManager_new(NULL, NULL);
+ bg_merger = BGMerger_new((Obj*)folder, manager);
+ BGMerger_Prepare_Commit(bg_merger);
+ DECREF(bg_merger);
+
+ TEST_TRUE(runner, Folder_Local_Exists(folder, merge_json),
+ "merge.json exists");
+ Hash *merge_data = IxManager_Read_Merge_Data(manager);
+ Obj *cutoff = Hash_Fetch_Utf8(merge_data, "cutoff", 6);
+
+ cutoff_seg = Seg_num_to_name(Json_obj_to_i64(cutoff));
+ TEST_TRUE(runner, Folder_Local_Exists(folder, cutoff_seg),
+ "cutoff segment exists");
+
+ Indexer *indexer = Indexer_new(NULL, (Obj*)folder, no_merge, 0);
+ Indexer_Commit(indexer);
+
+ DECREF(indexer);
+ DECREF(cutoff_seg);
+ DECREF(merge_data);
+ DECREF(manager);
+ }
+
+ TEST_FALSE(runner, Folder_Local_Exists(folder, merge_json),
+ "merge.json deleted after failed bg merge");
+ // Doesn't work because an indexing session always creates an empty
+ // segment directory even if no documents were added and nothing
+ // was merged.
+#if 0
+ TEST_FALSE(runner, Folder_Local_Exists(folder, cutoff_seg),
+ "cutoff segment deleted after failed bg merge");
+#endif
+
+ {
+ Indexer *indexer = Indexer_new(NULL, (Obj*)folder, NULL, 0);
+ Indexer_Optimize(indexer);
+ Indexer_Commit(indexer);
+ DECREF(indexer);
+ }
+
+ TEST_INT_EQ(runner, S_count_segs(folder), 1,
+ "Only a single segment remains after full optimize");
+ S_test_terms(runner, folder, " after full optimize");
+
+ DECREF(no_merge);
+ DECREF(folder);
+ DECREF(schema);
+}
+
+void
+TestBGMerger_Run_IMP(TestBackgroundMerger *self, TestBatchRunner *runner) {
+ TestBatchRunner_Plan(runner, (TestBatch*)self, 18);
+ test_bg_merger(runner);
+}
+
+
http://git-wip-us.apache.org/repos/asf/lucy/blob/2051451d/test/Lucy/Test/Index/TestBackgroundMerger.cfh
----------------------------------------------------------------------
diff --git a/test/Lucy/Test/Index/TestBackgroundMerger.cfh b/test/Lucy/Test/Index/TestBackgroundMerger.cfh
new file mode 100644
index 0000000..273e68d
--- /dev/null
+++ b/test/Lucy/Test/Index/TestBackgroundMerger.cfh
@@ -0,0 +1,29 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel TestLucy;
+
+class Lucy::Test::Index::TestBackgroundMerger nickname TestBGMerger
+ inherits Clownfish::TestHarness::TestBatch {
+
+ inert incremented TestBackgroundMerger*
+ new();
+
+ void
+ Run(TestBackgroundMerger *self, TestBatchRunner *runner);
+}
+
+
http://git-wip-us.apache.org/repos/asf/lucy/blob/2051451d/test/Lucy/Test/Index/TestSortWriter.c
----------------------------------------------------------------------
diff --git a/test/Lucy/Test/Index/TestSortWriter.c b/test/Lucy/Test/Index/TestSortWriter.c
index 539cc9e..0d876a0 100644
--- a/test/Lucy/Test/Index/TestSortWriter.c
+++ b/test/Lucy/Test/Index/TestSortWriter.c
@@ -38,6 +38,7 @@
#include "Lucy/Plan/Schema.h"
#include "Lucy/Plan/StringType.h"
#include "Lucy/Store/RAMFolder.h"
+#include "Lucy/Test/Index/NoMergeManager.h"
static String *name_str;
static String *speed_str;
@@ -226,7 +227,7 @@ test_sort_writer(TestBatchRunner *runner) {
{
// Add a second segment.
- NonMergingIndexManager *manager = NMIxManager_new();
+ NoMergeManager *manager = NoMergeManager_new();
Indexer *indexer
= Indexer_new(schema, (Obj*)folder, (IndexManager*)manager, 0);
// no "wheels" field -- test NULL/undef
@@ -295,29 +296,4 @@ TestSortWriter_Run_IMP(TestSortWriter *self, TestBatchRunner *runner) {
S_destroy_strings();
}
-NonMergingIndexManager*
-NMIxManager_new() {
- NonMergingIndexManager *self
- = (NonMergingIndexManager*)Class_Make_Obj(NONMERGINGINDEXMANAGER);
- return NMIxManager_init(self);
-}
-
-NonMergingIndexManager*
-NMIxManager_init(NonMergingIndexManager *self) {
- IxManager_init((IndexManager*)self, NULL, NULL);
- return self;
-}
-
-Vector*
-NMIxManager_Recycle_IMP(NonMergingIndexManager *self, PolyReader *reader,
- lucy_DeletionsWriter *del_writer, int64_t cutoff,
- bool optimize) {
- UNUSED_VAR(self);
- UNUSED_VAR(reader);
- UNUSED_VAR(del_writer);
- UNUSED_VAR(cutoff);
- UNUSED_VAR(optimize);
- return Vec_new(0);
-}
-
http://git-wip-us.apache.org/repos/asf/lucy/blob/2051451d/test/Lucy/Test/Index/TestSortWriter.cfh
----------------------------------------------------------------------
diff --git a/test/Lucy/Test/Index/TestSortWriter.cfh b/test/Lucy/Test/Index/TestSortWriter.cfh
index 5aa63ab..3855cc2 100644
--- a/test/Lucy/Test/Index/TestSortWriter.cfh
+++ b/test/Lucy/Test/Index/TestSortWriter.cfh
@@ -26,18 +26,3 @@ class Lucy::Test::Index::TestSortWriter
Run(TestSortWriter *self, TestBatchRunner *runner);
}
-class Lucy::Test::Index::NonMergingIndexManager nickname NMIxManager
- inherits Lucy::Index::IndexManager {
-
- public inert incremented NonMergingIndexManager*
- new();
-
- public inert NonMergingIndexManager*
- init(NonMergingIndexManager *self);
-
- public incremented Vector*
- Recycle(NonMergingIndexManager *self, PolyReader *reader,
- DeletionsWriter *del_writer, int64_t cutoff,
- bool optimize = false);
-}
-
http://git-wip-us.apache.org/repos/asf/lucy/blob/2051451d/test/Lucy/Test/TestUtils.c
----------------------------------------------------------------------
diff --git a/test/Lucy/Test/TestUtils.c b/test/Lucy/Test/TestUtils.c
index 0f579b0..614e84c 100644
--- a/test/Lucy/Test/TestUtils.c
+++ b/test/Lucy/Test/TestUtils.c
@@ -83,15 +83,18 @@ TestUtils_create_index(Vector *doc_set) {
Folder*
TestUtils_create_index_c(const char *first, ...) {
Vector *doc_set = Vec_new(1);
- Vec_Push(doc_set, (Obj*)Str_new_from_utf8(first, strlen(first)));
- va_list ap;
- va_start(ap, first);
- const char *next;
- while ((next = va_arg(ap, const char*))) {
- Vec_Push(doc_set, (Obj*)Str_new_from_utf8(next, strlen(next)));
+ if (first != NULL) {
+ Vec_Push(doc_set, (Obj*)Str_new_from_utf8(first, strlen(first)));
+
+ va_list ap;
+ va_start(ap, first);
+ const char *next;
+ while ((next = va_arg(ap, const char*))) {
+ Vec_Push(doc_set, (Obj*)Str_new_from_utf8(next, strlen(next)));
+ }
+ va_end(ap);
}
- va_end(ap);
Folder *folder = TestUtils_create_index(doc_set);
[05/16] lucy git commit: Remove LockFactory and make Lock a private
class
Posted by nw...@apache.org.
Remove LockFactory and make Lock a private class
If there are users who really use custom locks, this can be reverted.
Project: http://git-wip-us.apache.org/repos/asf/lucy/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucy/commit/3e229865
Tree: http://git-wip-us.apache.org/repos/asf/lucy/tree/3e229865
Diff: http://git-wip-us.apache.org/repos/asf/lucy/diff/3e229865
Branch: refs/heads/master
Commit: 3e2298650d8fc0213ce25078cadeb4f22d45bb5f
Parents: 1a8e02d
Author: Nick Wellnhofer <we...@aevum.de>
Authored: Thu Feb 16 17:47:33 2017 +0100
Committer: Nick Wellnhofer <we...@aevum.de>
Committed: Mon Feb 20 16:26:21 2017 +0100
----------------------------------------------------------------------
core/Lucy/Docs/Cookbook/FastUpdates.md | 2 +-
core/Lucy/Index/BackgroundMerger.c | 2 +-
core/Lucy/Index/FilePurger.c | 2 +-
core/Lucy/Index/IndexManager.c | 54 ++++--------
core/Lucy/Index/IndexManager.cfh | 10 +--
core/Lucy/Index/Indexer.c | 2 +-
core/Lucy/Store/Lock.cfh | 2 +-
core/Lucy/Store/LockFactory.c | 57 -------------
core/Lucy/Store/LockFactory.cfh | 65 ---------------
go/lucy/index_test.go | 14 ++--
go/lucy/store_test.go | 13 ---
perl/buildlib/Lucy/Build/Binding/Store.pm | 100 -----------------------
perl/lib/Lucy/Store/Lock.pm | 25 ------
perl/lib/Lucy/Store/LockFactory.pm | 25 ------
perl/t/111-index_manager.t | 9 +-
test/Lucy/Test/Index/NoMergeManager.c | 2 +-
test/Lucy/Test/Index/TestBackgroundMerger.c | 2 +-
test/Lucy/Test/Index/TestIndexManager.c | 2 +-
18 files changed, 39 insertions(+), 349 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucy/blob/3e229865/core/Lucy/Docs/Cookbook/FastUpdates.md
----------------------------------------------------------------------
diff --git a/core/Lucy/Docs/Cookbook/FastUpdates.md b/core/Lucy/Docs/Cookbook/FastUpdates.md
index 03f152c..2a65b47 100644
--- a/core/Lucy/Docs/Cookbook/FastUpdates.md
+++ b/core/Lucy/Docs/Cookbook/FastUpdates.md
@@ -197,7 +197,7 @@ void indexing_process(Obj *index, Doc *doc) {
void
background_merge_process(Obj *index) {
- IndexManager *manager = IxManager_new(NULL, NULL);
+ IndexManager *manager = IxManager_new(NULL);
IxManager_Set_Write_Lock_Timeout(manager, 60000);
BackgroundMerger bg_merger = BGMerger_new(index, manager);
http://git-wip-us.apache.org/repos/asf/lucy/blob/3e229865/core/Lucy/Index/BackgroundMerger.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/BackgroundMerger.c b/core/Lucy/Index/BackgroundMerger.c
index e9a1ae8..144139d 100644
--- a/core/Lucy/Index/BackgroundMerger.c
+++ b/core/Lucy/Index/BackgroundMerger.c
@@ -83,7 +83,7 @@ BGMerger_init(BackgroundMerger *self, Obj *index, IndexManager *manager) {
ivars->manager = (IndexManager*)INCREF(manager);
}
else {
- ivars->manager = IxManager_new(NULL, NULL);
+ ivars->manager = IxManager_new(NULL);
IxManager_Set_Write_Lock_Timeout(ivars->manager, 10000);
}
IxManager_Set_Folder(ivars->manager, folder);
http://git-wip-us.apache.org/repos/asf/lucy/blob/3e229865/core/Lucy/Index/FilePurger.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/FilePurger.c b/core/Lucy/Index/FilePurger.c
index 161f030..6811c33 100644
--- a/core/Lucy/Index/FilePurger.c
+++ b/core/Lucy/Index/FilePurger.c
@@ -58,7 +58,7 @@ FilePurger_init(FilePurger *self, Folder *folder, IndexManager *manager) {
ivars->folder = (Folder*)INCREF(folder);
ivars->manager = manager
? (IndexManager*)INCREF(manager)
- : IxManager_new(NULL, NULL);
+ : IxManager_new(NULL);
IxManager_Set_Folder(ivars->manager, folder);
return self;
http://git-wip-us.apache.org/repos/asf/lucy/blob/3e229865/core/Lucy/Index/IndexManager.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/IndexManager.c b/core/Lucy/Index/IndexManager.c
index 15cbbd9..82b9012 100644
--- a/core/Lucy/Index/IndexManager.c
+++ b/core/Lucy/Index/IndexManager.c
@@ -26,7 +26,6 @@
#include "Lucy/Store/DirHandle.h"
#include "Lucy/Store/Folder.h"
#include "Lucy/Store/Lock.h"
-#include "Lucy/Store/LockFactory.h"
#include "Lucy/Util/IndexFileNames.h"
#include "Lucy/Util/Json.h"
#include "Lucy/Util/StringHelper.h"
@@ -84,19 +83,17 @@ static const int32_t S_fibonacci[47] = {
};
IndexManager*
-IxManager_new(String *host, LockFactory *lock_factory) {
+IxManager_new(String *host) {
IndexManager *self = (IndexManager*)Class_Make_Obj(INDEXMANAGER);
- return IxManager_init(self, host, lock_factory);
+ return IxManager_init(self, host);
}
IndexManager*
-IxManager_init(IndexManager *self, String *host,
- LockFactory *lock_factory) {
+IxManager_init(IndexManager *self, String *host) {
IndexManagerIVARS *const ivars = IxManager_IVARS(self);
ivars->host = host
? Str_Clone(host)
: Str_new_from_trusted_utf8("", 0);
- ivars->lock_factory = (LockFactory*)INCREF(lock_factory);
ivars->folder = NULL;
ivars->write_lock_timeout = 1000;
ivars->write_lock_interval = 100;
@@ -113,7 +110,6 @@ IxManager_Destroy_IMP(IndexManager *self) {
IndexManagerIVARS *const ivars = IxManager_IVARS(self);
DECREF(ivars->host);
DECREF(ivars->folder);
- DECREF(ivars->lock_factory);
SUPER_DESTROY(self, INDEXMANAGER);
}
@@ -256,49 +252,34 @@ IxManager_Choose_Sparse_IMP(IndexManager *self, I32Array *doc_counts) {
return threshold;
}
-static LockFactory*
-S_obtain_lock_factory(IndexManager *self) {
- IndexManagerIVARS *const ivars = IxManager_IVARS(self);
- if (!ivars->lock_factory) {
- if (!ivars->folder) {
- THROW(ERR, "Can't create a LockFactory without a Folder");
- }
- ivars->lock_factory = LockFact_new(ivars->folder, ivars->host);
- }
- return ivars->lock_factory;
-}
-
Lock*
IxManager_Make_Write_Lock_IMP(IndexManager *self) {
IndexManagerIVARS *const ivars = IxManager_IVARS(self);
String *write_lock_name = SSTR_WRAP_C("write");
- LockFactory *lock_factory = S_obtain_lock_factory(self);
- return LockFact_Make_Lock(lock_factory, write_lock_name,
- (int32_t)ivars->write_lock_timeout,
- (int32_t)ivars->write_lock_interval,
- true);
+ return (Lock*)LFLock_new(ivars->folder, write_lock_name, ivars->host,
+ (int32_t)ivars->write_lock_timeout,
+ (int32_t)ivars->write_lock_interval,
+ true);
}
Lock*
IxManager_Make_Deletion_Lock_IMP(IndexManager *self) {
IndexManagerIVARS *const ivars = IxManager_IVARS(self);
String *lock_name = SSTR_WRAP_C("deletion");
- LockFactory *lock_factory = S_obtain_lock_factory(self);
- return LockFact_Make_Lock(lock_factory, lock_name,
- (int32_t)ivars->deletion_lock_timeout,
- (int32_t)ivars->deletion_lock_interval,
- true);
+ return (Lock*)LFLock_new(ivars->folder, lock_name, ivars->host,
+ (int32_t)ivars->deletion_lock_timeout,
+ (int32_t)ivars->deletion_lock_interval,
+ true);
}
Lock*
IxManager_Make_Merge_Lock_IMP(IndexManager *self) {
IndexManagerIVARS *const ivars = IxManager_IVARS(self);
String *merge_lock_name = SSTR_WRAP_C("merge");
- LockFactory *lock_factory = S_obtain_lock_factory(self);
- return LockFact_Make_Lock(lock_factory, merge_lock_name,
- (int32_t)ivars->merge_lock_timeout,
- (int32_t)ivars->merge_lock_interval,
- true);
+ return (Lock*)LFLock_new(ivars->folder, merge_lock_name, ivars->host,
+ (int32_t)ivars->merge_lock_timeout,
+ (int32_t)ivars->merge_lock_interval,
+ true);
}
void
@@ -344,7 +325,7 @@ IxManager_Remove_Merge_Data_IMP(IndexManager *self) {
Lock*
IxManager_Make_Snapshot_Read_Lock_IMP(IndexManager *self,
String *filename) {
- LockFactory *lock_factory = S_obtain_lock_factory(self);
+ IndexManagerIVARS *const ivars = IxManager_IVARS(self);
if (!Str_Starts_With_Utf8(filename, "snapshot_", 9)
|| !Str_Ends_With_Utf8(filename, ".json", 5)
@@ -356,7 +337,8 @@ IxManager_Make_Snapshot_Read_Lock_IMP(IndexManager *self,
size_t lock_name_len = Str_Length(filename) - (sizeof(".json") - 1);
String *lock_name = Str_SubString(filename, 0, lock_name_len);
- Lock *lock = LockFact_Make_Lock(lock_factory, lock_name, 1000, 100, false);
+ Lock *lock = (Lock*)LFLock_new(ivars->folder, lock_name, ivars->host,
+ 1000, 100, false);
DECREF(lock_name);
return lock;
http://git-wip-us.apache.org/repos/asf/lucy/blob/3e229865/core/Lucy/Index/IndexManager.cfh
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/IndexManager.cfh b/core/Lucy/Index/IndexManager.cfh
index be6389a..a94517d 100644
--- a/core/Lucy/Index/IndexManager.cfh
+++ b/core/Lucy/Index/IndexManager.cfh
@@ -31,7 +31,6 @@ public class Lucy::Index::IndexManager nickname IxManager
Folder *folder;
String *host;
- LockFactory *lock_factory;
uint32_t write_lock_timeout;
uint32_t write_lock_interval;
uint32_t merge_lock_timeout;
@@ -42,19 +41,16 @@ public class Lucy::Index::IndexManager nickname IxManager
/** Create a new IndexManager.
*
* @param host An identifier which should be unique per-machine.
- * @param lock_factory A LockFactory.
*/
public inert incremented IndexManager*
- new(String *host = NULL, LockFactory *lock_factory = NULL);
+ new(String *host = NULL);
/** Initialize an IndexManager.
*
* @param host An identifier which should be unique per-machine.
- * @param lock_factory A LockFactory.
*/
public inert IndexManager*
- init(IndexManager *self, String *host = NULL,
- LockFactory *lock_factory = NULL);
+ init(IndexManager *self, String *host = NULL);
public void
Destroy(IndexManager *self);
@@ -105,7 +101,7 @@ public class Lucy::Index::IndexManager nickname IxManager
/** Create the Lock which controls access to modifying the logical content
* of the index.
*/
- public incremented Lock*
+ incremented Lock*
Make_Write_Lock(IndexManager *self);
/** Create the Lock which grants permission to delete obsolete snapshot
http://git-wip-us.apache.org/repos/asf/lucy/blob/3e229865/core/Lucy/Index/Indexer.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/Indexer.c b/core/Lucy/Index/Indexer.c
index 2381b26..be5519a 100644
--- a/core/Lucy/Index/Indexer.c
+++ b/core/Lucy/Index/Indexer.c
@@ -92,7 +92,7 @@ Indexer_init(Indexer *self, Schema *schema, Obj *index,
ivars->folder = folder;
ivars->manager = manager
? (IndexManager*)INCREF(manager)
- : IxManager_new(NULL, NULL);
+ : IxManager_new(NULL);
IxManager_Set_Folder(ivars->manager, folder);
// Get a write lock for this folder.
http://git-wip-us.apache.org/repos/asf/lucy/blob/3e229865/core/Lucy/Store/Lock.cfh
----------------------------------------------------------------------
diff --git a/core/Lucy/Store/Lock.cfh b/core/Lucy/Store/Lock.cfh
index 598c777..bef6dae 100644
--- a/core/Lucy/Store/Lock.cfh
+++ b/core/Lucy/Store/Lock.cfh
@@ -27,7 +27,7 @@ parcel Lucy;
* help clear away stale locks.
*/
-public abstract class Lucy::Store::Lock inherits Clownfish::Obj {
+abstract class Lucy::Store::Lock inherits Clownfish::Obj {
Folder *folder;
String *name;
http://git-wip-us.apache.org/repos/asf/lucy/blob/3e229865/core/Lucy/Store/LockFactory.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Store/LockFactory.c b/core/Lucy/Store/LockFactory.c
deleted file mode 100644
index bd501f3..0000000
--- a/core/Lucy/Store/LockFactory.c
+++ /dev/null
@@ -1,57 +0,0 @@
-/* Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#define C_LUCY_LOCKFACTORY
-#include "Lucy/Util/ToolSet.h"
-
-#include <stdio.h>
-#include <ctype.h>
-
-#include "Lucy/Store/LockFactory.h"
-#include "Lucy/Store/Folder.h"
-#include "Lucy/Store/Lock.h"
-
-LockFactory*
-LockFact_new(Folder *folder, String *host) {
- LockFactory *self = (LockFactory*)Class_Make_Obj(LOCKFACTORY);
- return LockFact_init(self, folder, host);
-}
-
-LockFactory*
-LockFact_init(LockFactory *self, Folder *folder, String *host) {
- LockFactoryIVARS *const ivars = LockFact_IVARS(self);
- ivars->folder = (Folder*)INCREF(folder);
- ivars->host = Str_Clone(host);
- return self;
-}
-
-void
-LockFact_Destroy_IMP(LockFactory *self) {
- LockFactoryIVARS *const ivars = LockFact_IVARS(self);
- DECREF(ivars->folder);
- DECREF(ivars->host);
- SUPER_DESTROY(self, LOCKFACTORY);
-}
-
-Lock*
-LockFact_Make_Lock_IMP(LockFactory *self, String *name, int32_t timeout,
- int32_t interval, bool exclusive_only) {
- LockFactoryIVARS *const ivars = LockFact_IVARS(self);
- return (Lock*)LFLock_new(ivars->folder, name, ivars->host, timeout,
- interval, exclusive_only);
-}
-
-
http://git-wip-us.apache.org/repos/asf/lucy/blob/3e229865/core/Lucy/Store/LockFactory.cfh
----------------------------------------------------------------------
diff --git a/core/Lucy/Store/LockFactory.cfh b/core/Lucy/Store/LockFactory.cfh
deleted file mode 100644
index 47152b6..0000000
--- a/core/Lucy/Store/LockFactory.cfh
+++ /dev/null
@@ -1,65 +0,0 @@
-/* Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-parcel Lucy;
-
-/** Create Locks.
- *
- * LockFactory is used to spin off interprocess mutex locks used by various
- * index reading and writing components. The default implementation uses
- * lockfiles, but LockFactory subclasses which are implemented using
- * alternatives such as flock() are possible.
- */
-
-public class Lucy::Store::LockFactory nickname LockFact
- inherits Clownfish::Obj {
-
- Folder *folder;
- String *host;
-
- /** Create a new LockFactory.
- *
- * @param folder A [](cfish:Folder).
- * @param host An identifier which should be unique per-machine.
- */
- public inert incremented LockFactory*
- new(Folder *folder, String *host);
-
- /** Initialize a LockFactory.
- *
- * @param folder A [](cfish:Folder).
- * @param host An identifier which should be unique per-machine.
- */
- public inert LockFactory*
- init(LockFactory *self, Folder *folder, String *host);
-
- /** Return a Lock object.
- *
- * @param name A file-system-friendly id which identifies the
- * resource to be locked.
- * @param timeout Time in milliseconds to keep retrying before abandoning
- * the attempt to obtain a lock.
- * @param interval Time in milliseconds between retries.
- */
- public incremented Lock*
- Make_Lock(LockFactory *self, String *name, int32_t timeout = 0,
- int32_t interval = 100, bool exclusive_only);
-
- public void
- Destroy(LockFactory *self);
-}
-
-
http://git-wip-us.apache.org/repos/asf/lucy/blob/3e229865/go/lucy/index_test.go
----------------------------------------------------------------------
diff --git a/go/lucy/index_test.go b/go/lucy/index_test.go
index d47785e..bf1ba1e 100644
--- a/go/lucy/index_test.go
+++ b/go/lucy/index_test.go
@@ -148,7 +148,7 @@ func TestBackgroundMergerMisc(t *testing.T) {
func TestIndexManagerAccessors(t *testing.T) {
host := "dev.example.com"
- manager := NewIndexManager(host, nil)
+ manager := NewIndexManager(host)
if got := manager.GetHost(); got != host {
t.Errorf("GetHost: %v", got)
}
@@ -184,9 +184,9 @@ func TestIndexManagerAccessors(t *testing.T) {
}
func TestIndexManagerLocks(t *testing.T) {
- manager := NewIndexManager("", nil)
+ manager := NewIndexManager("")
manager.SetFolder(NewRAMFolder(""))
- if _, ok := manager.MakeWriteLock().(Lock); !ok {
+ if _, ok := manager.makeWriteLock().(Lock); !ok {
t.Errorf("MakeWriteLock")
}
if _, ok := manager.makeMergeLock().(Lock); !ok {
@@ -203,7 +203,7 @@ func TestIndexManagerLocks(t *testing.T) {
func TestIndexManagerMergeData(t *testing.T) {
var err error
- manager := NewIndexManager("", nil)
+ manager := NewIndexManager("")
manager.SetFolder(NewRAMFolder(""))
err = manager.WriteMergeData(42)
if err != nil {
@@ -223,7 +223,7 @@ func TestIndexManagerMergeData(t *testing.T) {
}
func TestIndexManagerMisc(t *testing.T) {
- manager := NewIndexManager("", nil)
+ manager := NewIndexManager("")
manager.SetFolder(NewRAMFolder(""))
if got, err := manager.MakeSnapshotFilename(); !strings.Contains(got, "snapshot") || err != nil {
t.Errorf("MakeSnapshotFilename: %s, %v", got, err)
@@ -238,7 +238,7 @@ func TestIndexManagerMisc(t *testing.T) {
func TestIndexManagerRecycle(t *testing.T) {
index := createTestIndex("foo", "bar", "baz")
- manager := NewIndexManager("", nil)
+ manager := NewIndexManager("")
manager.SetFolder(index)
indexer, _ := OpenIndexer(&OpenIndexerArgs{Index: index})
searcher, _ := OpenIndexSearcher(index)
@@ -788,7 +788,7 @@ func TestIndexReaderOpen(t *testing.T) {
if got, err := OpenIndexReader(folder, snapshot, nil); got == nil || err != nil {
t.Errorf("With Snapshot: %v", err)
}
- manager := NewIndexManager("", nil)
+ manager := NewIndexManager("")
manager.SetFolder(folder)
if got, err := OpenIndexReader(folder, nil, manager); got == nil || err != nil {
t.Errorf("With IndexManager: %v", err)
http://git-wip-us.apache.org/repos/asf/lucy/blob/3e229865/go/lucy/store_test.go
----------------------------------------------------------------------
diff --git a/go/lucy/store_test.go b/go/lucy/store_test.go
index ec51244..89f5057 100644
--- a/go/lucy/store_test.go
+++ b/go/lucy/store_test.go
@@ -712,19 +712,6 @@ func TestLockFileLockAll(t *testing.T) {
lock.Release()
}
-func TestLockFactoryAll(t *testing.T) {
- folder := NewRAMFolder("")
- factory := NewLockFactory(folder, "dev.example.com")
- lock := factory.MakeLock("write", 10, 42)
- if _, ok := lock.(Lock); !ok {
- t.Errorf("MakeLock")
- }
- shlock := factory.MakeSharedLock("read", 10, 42)
- if _, ok := shlock.(SharedLock); !ok {
- t.Errorf("MakeSharedLock")
- }
-}
-
func TestCompoundFiles(t *testing.T) {
var err error
folder := NewRAMFolder("seg_6b")
http://git-wip-us.apache.org/repos/asf/lucy/blob/3e229865/perl/buildlib/Lucy/Build/Binding/Store.pm
----------------------------------------------------------------------
diff --git a/perl/buildlib/Lucy/Build/Binding/Store.pm b/perl/buildlib/Lucy/Build/Binding/Store.pm
index 4a0dc54..da0ebe4 100644
--- a/perl/buildlib/Lucy/Build/Binding/Store.pm
+++ b/perl/buildlib/Lucy/Build/Binding/Store.pm
@@ -26,9 +26,7 @@ sub bind_all {
$class->bind_filehandle;
$class->bind_folder;
$class->bind_instream;
- $class->bind_lock;
$class->bind_lockerr;
- $class->bind_lockfactory;
$class->bind_outstream;
$class->bind_ramfilehandle;
$class->bind_ramfolder;
@@ -213,68 +211,6 @@ END_XS_CODE
Clownfish::CFC::Binding::Perl::Class->register($binding);
}
-sub bind_lock {
- my $pod_spec = Clownfish::CFC::Binding::Perl::Pod->new;
- my $synopsis = <<'END_SYNOPSIS';
- my $lock = $lock_factory->make_lock(
- name => 'write',
- timeout => 5000,
- );
- $lock->obtain or die "can't get lock for " . $lock->get_name;
- do_stuff();
- $lock->release;
-END_SYNOPSIS
- my $constructor = <<'END_CONSTRUCTOR';
-=head2 new
-
- my $lock = Lucy::Store::Lock->new(
- name => 'commit', # required
- folder => $folder, # required
- host => $hostname, # required
- timeout => 5000, # default: 0
- interval => 1000, # default: 100
- );
-
-Abstract constructor.
-
-=over
-
-=item *
-
-B<folder> - A Folder.
-
-=item *
-
-B<name> - String identifying the resource to be locked, which must
-consist solely of characters matching [-_.A-Za-z0-9].
-
-=item *
-
-B<host> - A unique per-machine identifier.
-
-=item *
-
-B<timeout> - Time in milliseconds to keep retrying before abandoning
-the attempt to obtain a lock.
-
-=item *
-
-B<interval> - Time in milliseconds between retries.
-
-=back
-END_CONSTRUCTOR
- $pod_spec->set_synopsis($synopsis);
- $pod_spec->add_constructor( alias => 'new', pod => $constructor, );
-
- my $binding = Clownfish::CFC::Binding::Perl::Class->new(
- parcel => "Lucy",
- class_name => "Lucy::Store::Lock",
- );
- $binding->set_pod_spec($pod_spec);
-
- Clownfish::CFC::Binding::Perl::Class->register($binding);
-}
-
sub bind_lockerr {
my $pod_spec = Clownfish::CFC::Binding::Perl::Pod->new;
my $synopsis = <<'END_SYNOPSIS';
@@ -303,42 +239,6 @@ END_SYNOPSIS
Clownfish::CFC::Binding::Perl::Class->register($binding);
}
-sub bind_lockfactory {
- my $pod_spec = Clownfish::CFC::Binding::Perl::Pod->new;
- my $synopsis = <<'END_SYNOPSIS';
- use Sys::Hostname qw( hostname );
- my $hostname = hostname() or die "Can't get unique hostname";
- my $folder = Lucy::Store::FSFolder->new(
- path => '/path/to/index',
- );
- my $lock_factory = Lucy::Store::LockFactory->new(
- folder => $folder,
- host => $hostname,
- );
- my $write_lock = $lock_factory->make_lock(
- name => 'write',
- timeout => 5000,
- interval => 100,
- );
-END_SYNOPSIS
- my $constructor = <<'END_CONSTRUCTOR';
- my $lock_factory = Lucy::Store::LockFactory->new(
- folder => $folder, # required
- host => $hostname, # required
- );
-END_CONSTRUCTOR
- $pod_spec->set_synopsis($synopsis);
- $pod_spec->add_constructor( alias => 'new', sample => $constructor, );
-
- my $binding = Clownfish::CFC::Binding::Perl::Class->new(
- parcel => "Lucy",
- class_name => "Lucy::Store::LockFactory",
- );
- $binding->set_pod_spec($pod_spec);
-
- Clownfish::CFC::Binding::Perl::Class->register($binding);
-}
-
sub bind_outstream {
my $xs_code = <<'END_XS_CODE';
MODULE = Lucy PACKAGE = Lucy::Store::OutStream
http://git-wip-us.apache.org/repos/asf/lucy/blob/3e229865/perl/lib/Lucy/Store/Lock.pm
----------------------------------------------------------------------
diff --git a/perl/lib/Lucy/Store/Lock.pm b/perl/lib/Lucy/Store/Lock.pm
deleted file mode 100644
index 44e14ef..0000000
--- a/perl/lib/Lucy/Store/Lock.pm
+++ /dev/null
@@ -1,25 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-package Lucy::Store::Lock;
-use Lucy;
-our $VERSION = '0.006000';
-$VERSION = eval $VERSION;
-
-1;
-
-__END__
-
-
http://git-wip-us.apache.org/repos/asf/lucy/blob/3e229865/perl/lib/Lucy/Store/LockFactory.pm
----------------------------------------------------------------------
diff --git a/perl/lib/Lucy/Store/LockFactory.pm b/perl/lib/Lucy/Store/LockFactory.pm
deleted file mode 100644
index b8fc429..0000000
--- a/perl/lib/Lucy/Store/LockFactory.pm
+++ /dev/null
@@ -1,25 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-package Lucy::Store::LockFactory;
-use Lucy;
-our $VERSION = '0.006000';
-$VERSION = eval $VERSION;
-
-1;
-
-__END__
-
-
http://git-wip-us.apache.org/repos/asf/lucy/blob/3e229865/perl/t/111-index_manager.t
----------------------------------------------------------------------
diff --git a/perl/t/111-index_manager.t b/perl/t/111-index_manager.t
index 16c98ea..3cb616b 100644
--- a/perl/t/111-index_manager.t
+++ b/perl/t/111-index_manager.t
@@ -37,12 +37,9 @@ use Lucy::Test;
my $folder = Lucy::Store::RAMFolder->new;
-my $lock_factory = Lucy::Store::LockFactory->new(
- folder => $folder,
- host => 'me',
-);
-
-my $lock = $lock_factory->make_lock(
+my $lock = Lucy::Store::LockFileLock->new(
+ folder => $folder,
+ host => 'me',
name => 'angie',
timeout => 1000,
exclusive_only => 0,
http://git-wip-us.apache.org/repos/asf/lucy/blob/3e229865/test/Lucy/Test/Index/NoMergeManager.c
----------------------------------------------------------------------
diff --git a/test/Lucy/Test/Index/NoMergeManager.c b/test/Lucy/Test/Index/NoMergeManager.c
index 7c0804b..2a2b71c 100644
--- a/test/Lucy/Test/Index/NoMergeManager.c
+++ b/test/Lucy/Test/Index/NoMergeManager.c
@@ -25,7 +25,7 @@
NoMergeManager*
NoMergeManager_new() {
NoMergeManager *self = (NoMergeManager*)Class_Make_Obj(NOMERGEMANAGER);
- return (NoMergeManager*)IxManager_init((IndexManager*)self, NULL, NULL);
+ return (NoMergeManager*)IxManager_init((IndexManager*)self, NULL);
}
Vector*
http://git-wip-us.apache.org/repos/asf/lucy/blob/3e229865/test/Lucy/Test/Index/TestBackgroundMerger.c
----------------------------------------------------------------------
diff --git a/test/Lucy/Test/Index/TestBackgroundMerger.c b/test/Lucy/Test/Index/TestBackgroundMerger.c
index 084c49d..4852d7f 100644
--- a/test/Lucy/Test/Index/TestBackgroundMerger.c
+++ b/test/Lucy/Test/Index/TestBackgroundMerger.c
@@ -150,7 +150,7 @@ test_bg_merger(TestBatchRunner *runner) {
{
// Simulate failed background merge.
DECREF(bg_merger);
- IndexManager *manager = IxManager_new(NULL, NULL);
+ IndexManager *manager = IxManager_new(NULL);
bg_merger = BGMerger_new((Obj*)folder, manager);
BGMerger_Prepare_Commit(bg_merger);
DECREF(bg_merger);
http://git-wip-us.apache.org/repos/asf/lucy/blob/3e229865/test/Lucy/Test/Index/TestIndexManager.c
----------------------------------------------------------------------
diff --git a/test/Lucy/Test/Index/TestIndexManager.c b/test/Lucy/Test/Index/TestIndexManager.c
index 7bc0661..4177606 100644
--- a/test/Lucy/Test/Index/TestIndexManager.c
+++ b/test/Lucy/Test/Index/TestIndexManager.c
@@ -29,7 +29,7 @@ TestIxManager_new() {
static void
test_Choose_Sparse(TestBatchRunner *runner) {
- IndexManager *manager = IxManager_new(NULL, NULL);
+ IndexManager *manager = IxManager_new(NULL);
for (uint32_t num_segs = 2; num_segs < 20; num_segs++) {
I32Array *doc_counts = I32Arr_new_blank(num_segs);
[04/16] lucy git commit: Merge SharedLock into LockFileLock
Posted by nw...@apache.org.
Merge SharedLock into LockFileLock
Supplying the lock type when requesting a lock better matches other
file lock APIs. Merging the implementations allows to convert
Maybe_Delete_File into a static function.
Make shared and exclusive locks check for locks of the other type
except for exclusive locks created with `exclusive_only`.
Clear_Stale is broken for shared locks but will be removed with the
following commit.
Project: http://git-wip-us.apache.org/repos/asf/lucy/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucy/commit/bb83d0d7
Tree: http://git-wip-us.apache.org/repos/asf/lucy/tree/bb83d0d7
Diff: http://git-wip-us.apache.org/repos/asf/lucy/diff/bb83d0d7
Branch: refs/heads/master
Commit: bb83d0d77bf99ed09b2bbad782c3c1d3b94f04a9
Parents: 2051451
Author: Nick Wellnhofer <we...@aevum.de>
Authored: Thu Feb 16 15:58:45 2017 +0100
Committer: Nick Wellnhofer <we...@aevum.de>
Committed: Mon Feb 20 16:26:21 2017 +0100
----------------------------------------------------------------------
core/Lucy/Docs/FileLocking.md | 5 +-
core/Lucy/Index/BackgroundMerger.c | 4 +-
core/Lucy/Index/FilePurger.c | 4 +-
core/Lucy/Index/IndexManager.c | 11 +-
core/Lucy/Index/Indexer.c | 6 +-
core/Lucy/Index/PolyReader.c | 4 +-
core/Lucy/Store/Lock.c | 326 +++++++++++++++++++------
core/Lucy/Store/Lock.cfh | 86 ++++---
core/Lucy/Store/LockFactory.c | 15 +-
core/Lucy/Store/LockFactory.cfh | 21 +-
core/Lucy/Store/SharedLock.c | 173 -------------
core/Lucy/Store/SharedLock.cfh | 77 ------
go/build.go | 6 +-
go/lucy/index_test.go | 2 +-
go/lucy/store.go | 28 ++-
go/lucy/store_test.go | 62 +++--
perl/buildlib/Lucy/Build/Binding/Store.pm | 2 +-
perl/t/105-folder.t | 34 +--
perl/t/106-locking.t | 9 +-
perl/t/110-shared_lock.t | 57 +++--
perl/t/111-index_manager.t | 15 +-
21 files changed, 449 insertions(+), 498 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucy/blob/bb83d0d7/core/Lucy/Docs/FileLocking.md
----------------------------------------------------------------------
diff --git a/core/Lucy/Docs/FileLocking.md b/core/Lucy/Docs/FileLocking.md
index 911df21..2a6f5b8 100644
--- a/core/Lucy/Docs/FileLocking.md
+++ b/core/Lucy/Docs/FileLocking.md
@@ -43,9 +43,8 @@ process will crash with a "Stale NFS filehandle" exception.
Under normal circumstances, it is neither necessary nor desirable for
IndexReaders to secure read locks against an index, but for NFS we have to
-make an exception. LockFactory's [](lucy.LockFactory.Make_Shared_Lock) method exists for this
-reason; supplying an IndexManager instance to IndexReader's constructor
-activates an internal locking mechanism using [](lucy.LockFactory.Make_Shared_Lock) which
+make an exception. Supplying an IndexManager instance to IndexReader's
+constructor activates an internal locking mechanism using shared locks which
prevents concurrent indexing processes from deleting files that are needed
by active readers.
http://git-wip-us.apache.org/repos/asf/lucy/blob/bb83d0d7/core/Lucy/Index/BackgroundMerger.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/BackgroundMerger.c b/core/Lucy/Index/BackgroundMerger.c
index 8f36050..661b9a6 100644
--- a/core/Lucy/Index/BackgroundMerger.c
+++ b/core/Lucy/Index/BackgroundMerger.c
@@ -530,7 +530,7 @@ S_obtain_write_lock(BackgroundMerger *self) {
BackgroundMergerIVARS *const ivars = BGMerger_IVARS(self);
Lock *write_lock = IxManager_Make_Write_Lock(ivars->manager);
Lock_Clear_Stale(write_lock);
- if (Lock_Obtain(write_lock)) {
+ if (Lock_Obtain_Exclusive(write_lock)) {
// Only assign if successful, otherwise DESTROY unlocks -- bad!
ivars->write_lock = write_lock;
}
@@ -544,7 +544,7 @@ S_obtain_merge_lock(BackgroundMerger *self) {
BackgroundMergerIVARS *const ivars = BGMerger_IVARS(self);
Lock *merge_lock = IxManager_Make_Merge_Lock(ivars->manager);
Lock_Clear_Stale(merge_lock);
- if (Lock_Obtain(merge_lock)) {
+ if (Lock_Obtain_Exclusive(merge_lock)) {
// Only assign if successful, same rationale as above.
ivars->merge_lock = merge_lock;
}
http://git-wip-us.apache.org/repos/asf/lucy/blob/bb83d0d7/core/Lucy/Index/FilePurger.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/FilePurger.c b/core/Lucy/Index/FilePurger.c
index 55ade17..3980c8f 100644
--- a/core/Lucy/Index/FilePurger.c
+++ b/core/Lucy/Index/FilePurger.c
@@ -79,7 +79,7 @@ FilePurger_Purge_Snapshots_IMP(FilePurger *self, Snapshot *current) {
// Obtain deletion lock, purge files, release deletion lock.
Lock_Clear_Stale(deletion_lock);
- if (Lock_Obtain(deletion_lock)) {
+ if (Lock_Obtain_Exclusive(deletion_lock)) {
Folder *folder = ivars->folder;
Hash *failures = Hash_new(16);
Hash *spared = Hash_new(32);
@@ -148,7 +148,7 @@ FilePurger_Purge_Aborted_Merge_IMP(FilePurger *self) {
Lock *merge_lock = IxManager_Make_Merge_Lock(manager);
Lock_Clear_Stale(merge_lock);
- if (!Lock_Is_Locked(merge_lock)) {
+ if (!Lock_Is_Locked_Exclusive(merge_lock)) {
Hash *merge_data = IxManager_Read_Merge_Data(manager);
Obj *cutoff = merge_data
? Hash_Fetch_Utf8(merge_data, "cutoff", 6)
http://git-wip-us.apache.org/repos/asf/lucy/blob/bb83d0d7/core/Lucy/Index/IndexManager.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/IndexManager.c b/core/Lucy/Index/IndexManager.c
index 15367fa..15cbbd9 100644
--- a/core/Lucy/Index/IndexManager.c
+++ b/core/Lucy/Index/IndexManager.c
@@ -275,7 +275,8 @@ IxManager_Make_Write_Lock_IMP(IndexManager *self) {
LockFactory *lock_factory = S_obtain_lock_factory(self);
return LockFact_Make_Lock(lock_factory, write_lock_name,
(int32_t)ivars->write_lock_timeout,
- (int32_t)ivars->write_lock_interval);
+ (int32_t)ivars->write_lock_interval,
+ true);
}
Lock*
@@ -285,7 +286,8 @@ IxManager_Make_Deletion_Lock_IMP(IndexManager *self) {
LockFactory *lock_factory = S_obtain_lock_factory(self);
return LockFact_Make_Lock(lock_factory, lock_name,
(int32_t)ivars->deletion_lock_timeout,
- (int32_t)ivars->deletion_lock_interval);
+ (int32_t)ivars->deletion_lock_interval,
+ true);
}
Lock*
@@ -295,7 +297,8 @@ IxManager_Make_Merge_Lock_IMP(IndexManager *self) {
LockFactory *lock_factory = S_obtain_lock_factory(self);
return LockFact_Make_Lock(lock_factory, merge_lock_name,
(int32_t)ivars->merge_lock_timeout,
- (int32_t)ivars->merge_lock_interval);
+ (int32_t)ivars->merge_lock_interval,
+ true);
}
void
@@ -353,7 +356,7 @@ IxManager_Make_Snapshot_Read_Lock_IMP(IndexManager *self,
size_t lock_name_len = Str_Length(filename) - (sizeof(".json") - 1);
String *lock_name = Str_SubString(filename, 0, lock_name_len);
- Lock *lock = LockFact_Make_Shared_Lock(lock_factory, lock_name, 1000, 100);
+ Lock *lock = LockFact_Make_Lock(lock_factory, lock_name, 1000, 100, false);
DECREF(lock_name);
return lock;
http://git-wip-us.apache.org/repos/asf/lucy/blob/bb83d0d7/core/Lucy/Index/Indexer.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/Indexer.c b/core/Lucy/Index/Indexer.c
index 031237f..97763e3 100644
--- a/core/Lucy/Index/Indexer.c
+++ b/core/Lucy/Index/Indexer.c
@@ -98,7 +98,7 @@ Indexer_init(Indexer *self, Schema *schema, Obj *index,
// Get a write lock for this folder.
Lock *write_lock = IxManager_Make_Write_Lock(ivars->manager);
Lock_Clear_Stale(write_lock);
- if (Lock_Obtain(write_lock)) {
+ if (Lock_Obtain_Exclusive(write_lock)) {
// Only assign if successful, otherwise DESTROY unlocks -- bad!
ivars->write_lock = write_lock;
}
@@ -170,7 +170,7 @@ Indexer_init(Indexer *self, Schema *schema, Obj *index,
int64_t new_seg_num
= IxManager_Highest_Seg_Num(ivars->manager, latest_snapshot) + 1;
Lock *merge_lock = IxManager_Make_Merge_Lock(ivars->manager);
- if (Lock_Is_Locked(merge_lock)) {
+ if (Lock_Is_Locked_Exclusive(merge_lock)) {
// If there's a background merge process going on, stay out of its
// way.
Hash *merge_data = IxManager_Read_Merge_Data(ivars->manager);
@@ -398,7 +398,7 @@ S_maybe_merge(Indexer *self, Vector *seg_readers) {
bool merge_happened = false;
size_t num_seg_readers = Vec_Get_Size(seg_readers);
Lock *merge_lock = IxManager_Make_Merge_Lock(ivars->manager);
- bool got_merge_lock = Lock_Obtain(merge_lock);
+ bool got_merge_lock = Lock_Obtain_Exclusive(merge_lock);
int64_t cutoff;
if (got_merge_lock) {
http://git-wip-us.apache.org/repos/asf/lucy/blob/bb83d0d7/core/Lucy/Index/PolyReader.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/PolyReader.c b/core/Lucy/Index/PolyReader.c
index e843bcd..d4166c6 100644
--- a/core/Lucy/Index/PolyReader.c
+++ b/core/Lucy/Index/PolyReader.c
@@ -471,7 +471,7 @@ S_obtain_deletion_lock(PolyReader *self) {
PolyReaderIVARS *const ivars = PolyReader_IVARS(self);
ivars->deletion_lock = IxManager_Make_Deletion_Lock(ivars->manager);
Lock_Clear_Stale(ivars->deletion_lock);
- if (!Lock_Obtain(ivars->deletion_lock)) {
+ if (!Lock_Obtain_Exclusive(ivars->deletion_lock)) {
DECREF(ivars->deletion_lock);
ivars->deletion_lock = NULL;
return false;
@@ -486,7 +486,7 @@ S_obtain_read_lock(PolyReader *self, String *snapshot_file_name) {
snapshot_file_name);
Lock_Clear_Stale(ivars->read_lock);
- if (!Lock_Obtain(ivars->read_lock)) {
+ if (!Lock_Obtain_Shared(ivars->read_lock)) {
DECREF(ivars->read_lock);
ivars->read_lock = NULL;
return false;
http://git-wip-us.apache.org/repos/asf/lucy/blob/bb83d0d7/core/Lucy/Store/Lock.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Store/Lock.c b/core/Lucy/Store/Lock.c
index 97592b0..0eb42da 100644
--- a/core/Lucy/Store/Lock.c
+++ b/core/Lucy/Store/Lock.c
@@ -22,6 +22,7 @@
#include <ctype.h>
#include "Lucy/Store/Lock.h"
+#include "Lucy/Store/DirHandle.h"
#include "Lucy/Store/Folder.h"
#include "Lucy/Store/OutStream.h"
#include "Lucy/Util/Json.h"
@@ -82,26 +83,38 @@ Lock_Get_Name_IMP(Lock *self) {
}
String*
-Lock_Get_Lock_Path_IMP(Lock *self) {
- return Lock_IVARS(self)->lock_path;
-}
-
-String*
Lock_Get_Host_IMP(Lock *self) {
return Lock_IVARS(self)->host;
}
bool
-Lock_Obtain_IMP(Lock *self) {
+Lock_Obtain_Shared_IMP(Lock *self) {
+ LockIVARS *const ivars = Lock_IVARS(self);
+ int32_t time_left = ivars->interval == 0 ? 0 : ivars->timeout;
+ bool locked = Lock_Request_Shared(self);
+
+ while (!locked) {
+ time_left -= ivars->interval;
+ if (time_left <= 0) { break; }
+ Sleep_millisleep((uint32_t)ivars->interval);
+ locked = Lock_Request_Shared(self);
+ }
+
+ if (!locked) { ERR_ADD_FRAME(Err_get_error()); }
+ return locked;
+}
+
+bool
+Lock_Obtain_Exclusive_IMP(Lock *self) {
LockIVARS *const ivars = Lock_IVARS(self);
int32_t time_left = ivars->interval == 0 ? 0 : ivars->timeout;
- bool locked = Lock_Request(self);
+ bool locked = Lock_Request_Exclusive(self);
while (!locked) {
time_left -= ivars->interval;
if (time_left <= 0) { break; }
Sleep_millisleep((uint32_t)ivars->interval);
- locked = Lock_Request(self);
+ locked = Lock_Request_Exclusive(self);
}
if (!locked) { ERR_ADD_FRAME(Err_get_error()); }
@@ -110,27 +123,42 @@ Lock_Obtain_IMP(Lock *self) {
/***************************************************************************/
+static bool
+S_request(LockFileLockIVARS *ivars, String *lock_path);
+
+static bool
+S_is_shared_lock_file(LockFileLockIVARS *ivars, String *entry);
+
+static bool
+S_maybe_delete_file(LockFileLockIVARS *ivars, String *path,
+ bool delete_mine, bool delete_other);
+
LockFileLock*
-LFLock_new(Folder *folder, String *name, String *host,
- int32_t timeout, int32_t interval) {
+LFLock_new(Folder *folder, String *name, String *host, int32_t timeout,
+ int32_t interval, bool exclusive_only) {
LockFileLock *self = (LockFileLock*)Class_Make_Obj(LOCKFILELOCK);
- return LFLock_init(self, folder, name, host, timeout, interval);
+ return LFLock_init(self, folder, name, host, timeout, interval,
+ exclusive_only);
}
LockFileLock*
-LFLock_init(LockFileLock *self, Folder *folder, String *name,
- String *host, int32_t timeout, int32_t interval) {
+LFLock_init(LockFileLock *self, Folder *folder, String *name, String *host,
+ int32_t timeout, int32_t interval, bool exclusive_only) {
int pid = PID_getpid();
Lock_init((Lock*)self, folder, name, host, timeout, interval);
LockFileLockIVARS *const ivars = LFLock_IVARS(self);
ivars->link_path = Str_newf("%o.%o.%i64", ivars->lock_path, host,
(int64_t)pid);
+ ivars->exclusive_only = exclusive_only;
return self;
}
-bool
-LFLock_Shared_IMP(LockFileLock *self) {
- UNUSED_VAR(self); return false;
+String*
+LFLock_Get_Lock_Path_IMP(LockFileLock *self) {
+ LockFileLockIVARS *const ivars = LFLock_IVARS(self);
+ return ivars->shared_lock_path
+ ? ivars->shared_lock_path
+ : ivars->lock_path;
}
struct lockfile_context {
@@ -147,16 +175,75 @@ S_write_lockfile_json(void *context) {
}
bool
-LFLock_Request_IMP(LockFileLock *self) {
+LFLock_Request_Shared_IMP(LockFileLock *self) {
LockFileLockIVARS *const ivars = LFLock_IVARS(self);
- bool success = false;
- if (Folder_Exists(ivars->folder, ivars->lock_path)) {
- Err_set_error((Err*)LockErr_new(Str_newf("Can't obtain lock: '%o' exists",
- ivars->lock_path)));
+ if (ivars->exclusive_only) {
+ THROW(ERR, "Can't request shared lock if exclusive_only is set");
+ }
+
+ // TODO: The is_locked test and subsequent file creation is prone to a
+ // race condition. We could protect the whole process with an internal
+ // exclusive lock.
+
+ if (LFLock_Is_Locked_Exclusive(self)) {
+ String *msg = Str_newf("'%o.lock' is locked", ivars->name);
+ Err_set_error((Err*)LockErr_new(msg));
+ return false;
+ }
+
+ String *path = ivars->shared_lock_path;
+
+ // Null shared_lock_path indicates whether this particular instance is
+ // locked.
+ if (path && Folder_Exists(ivars->folder, path)) {
+ // Don't allow double obtain.
+ String *msg = Str_newf("Lock already obtained via '%o'", path);
+ Err_set_error((Err*)LockErr_new(msg));
+ return false;
+ }
+
+ uint32_t i = 0;
+ do {
+ DECREF(path);
+ path = Str_newf("locks/%o-%u32.lock", ivars->name, ++i);
+ } while (Folder_Exists(ivars->folder, path));
+
+ if (S_request(ivars, path)) {
+ ivars->shared_lock_path = path;
+ return true;
+ }
+ else {
+ DECREF(path);
+ ivars->shared_lock_path = NULL;
+ return false;
+ }
+}
+
+bool
+LFLock_Request_Exclusive_IMP(LockFileLock *self) {
+ LockFileLockIVARS *const ivars = LFLock_IVARS(self);
+
+ // TODO: The is_locked test and subsequent file creation is prone to a
+ // race condition. We could protect the whole process with an internal
+ // exclusive lock.
+
+ if (ivars->exclusive_only
+ ? LFLock_Is_Locked_Exclusive(self)
+ : LFLock_Is_Locked(self)
+ ) {
+ String *msg = Str_newf("'%o.lock' is locked", ivars->name);
+ Err_set_error((Err*)LockErr_new(msg));
return false;
}
+ return S_request(ivars, ivars->lock_path);
+}
+
+static bool
+S_request(LockFileLockIVARS *ivars, String *lock_path) {
+ bool success = false;
+
// Create the "locks" subdirectory if necessary.
String *lock_dir_name = SSTR_WRAP_C("locks");
if (!Folder_Exists(ivars->folder, lock_dir_name)) {
@@ -205,17 +292,17 @@ LFLock_Request_IMP(LockFileLock *self) {
DECREF(json);
if (wrote_json) {
success = Folder_Hard_Link(ivars->folder, ivars->link_path,
- ivars->lock_path);
+ lock_path);
if (!success) {
Err *hard_link_err = (Err*)CERTIFY(Err_get_error(), ERR);
Err_set_error((Err*)LockErr_new(Str_newf("Failed to obtain lock at '%o': %o",
- ivars->lock_path,
+ lock_path,
Err_Get_Mess(hard_link_err))));
}
}
else {
Err_set_error((Err*)LockErr_new(Str_newf("Failed to obtain lock at '%o': %o",
- ivars->lock_path,
+ lock_path,
Err_Get_Mess(json_error))));
DECREF(json_error);
}
@@ -233,79 +320,167 @@ LFLock_Request_IMP(LockFileLock *self) {
void
LFLock_Release_IMP(LockFileLock *self) {
LockFileLockIVARS *const ivars = LFLock_IVARS(self);
- if (Folder_Exists(ivars->folder, ivars->lock_path)) {
- LFLock_Maybe_Delete_File(self, ivars->lock_path, true, false);
+
+ if (ivars->shared_lock_path) {
+ if (Folder_Exists(ivars->folder, ivars->shared_lock_path)) {
+ S_maybe_delete_file(ivars, ivars->shared_lock_path, true, false);
+ }
+
+ // Empty out lock_path.
+ DECREF(ivars->shared_lock_path);
+ ivars->shared_lock_path = NULL;
+ }
+ else {
+ if (Folder_Exists(ivars->folder, ivars->lock_path)) {
+ S_maybe_delete_file(ivars, ivars->lock_path, true, false);
+ }
}
}
bool
-LFLock_Is_Locked_IMP(LockFileLock *self) {
+LFLock_Is_Locked_Exclusive_IMP(LockFileLock *self) {
LockFileLockIVARS *const ivars = LFLock_IVARS(self);
+
return Folder_Exists(ivars->folder, ivars->lock_path);
}
-void
-LFLock_Clear_Stale_IMP(LockFileLock *self) {
+bool
+LFLock_Is_Locked_IMP(LockFileLock *self) {
LockFileLockIVARS *const ivars = LFLock_IVARS(self);
- LFLock_Maybe_Delete_File(self, ivars->lock_path, false, true);
+
+ // Check for exclusive lock.
+ if (Folder_Exists(ivars->folder, ivars->lock_path)) {
+ return true;
+ }
+
+ // Check for shared lock.
+
+ String *lock_dir_name = SSTR_WRAP_C("locks");
+ if (!Folder_Find_Folder(ivars->folder, lock_dir_name)) {
+ return false;
+ }
+
+ DirHandle *dh = Folder_Open_Dir(ivars->folder, lock_dir_name);
+ if (!dh) { RETHROW(INCREF(Err_get_error())); }
+
+ while (DH_Next(dh)) {
+ String *entry = DH_Get_Entry(dh);
+ if (S_is_shared_lock_file(ivars, entry)) {
+ DECREF(entry);
+ DECREF(dh);
+ return true;
+ }
+ DECREF(entry);
+ }
+
+ DECREF(dh);
+ return false;
}
-bool
-LFLock_Maybe_Delete_File_IMP(LockFileLock *self, String *path,
- bool delete_mine, bool delete_other) {
+void
+LFLock_Clear_Stale_IMP(LockFileLock *self) {
LockFileLockIVARS *const ivars = LFLock_IVARS(self);
- Folder *folder = ivars->folder;
- bool success = false;
- // Only delete locks that start with our lock name.
- if (!Str_Starts_With_Utf8(path, "locks", 5)) {
- return false;
+ if (ivars->shared_lock_path) {
+ String *lock_dir_name = SSTR_WRAP_C("locks");
+ if (!Folder_Find_Folder(ivars->folder, lock_dir_name)) {
+ return;
+ }
+
+ DirHandle *dh = Folder_Open_Dir(ivars->folder, lock_dir_name);
+ if (!dh) { RETHROW(INCREF(Err_get_error())); }
+
+ while (DH_Next(dh)) {
+ String *entry = DH_Get_Entry(dh);
+ if (S_is_shared_lock_file(ivars, entry)) {
+ String *candidate = Str_newf("%o/%o", lock_dir_name, entry);
+ S_maybe_delete_file(ivars, candidate, false, true);
+ DECREF(candidate);
+ }
+ DECREF(entry);
+ }
+
+ DECREF(dh);
}
- StringIterator *iter = Str_Top(path);
- StrIter_Advance(iter, 5 + 1);
- if (!StrIter_Starts_With(iter, ivars->name)) {
+ else {
+ S_maybe_delete_file(ivars, ivars->lock_path, false, true);
+ }
+}
+
+static bool
+S_is_shared_lock_file(LockFileLockIVARS *ivars, String *entry) {
+ // Translation: $match = $entry =~ /^\Q$name-\d+\.lock\z/
+ bool match = false;
+
+ // $name
+ if (Str_Starts_With(entry, ivars->name)) {
+ StringIterator *iter = Str_Top(entry);
+ StrIter_Advance(iter, Str_Length(ivars->name));
+
+ // Hyphen-minus
+ if (StrIter_Next(iter) == '-') {
+ int32_t code_point = StrIter_Next(iter);
+
+ // Digit
+ if (code_point >= '0' && code_point <= '9') {
+ // Optional digits
+ do {
+ code_point = StrIter_Next(iter);
+ } while (code_point >= '0' && code_point <= '9');
+
+ // ".lock"
+ match = code_point == '.'
+ && StrIter_Starts_With_Utf8(iter, "lock", 4)
+ && StrIter_Advance(iter, SIZE_MAX) == 4;
+ }
+ }
+
DECREF(iter);
- return false;
}
- DECREF(iter);
- // Attempt to delete dead lock file.
- if (Folder_Exists(folder, path)) {
- Hash *hash = (Hash*)Json_slurp_json(folder, path);
- if (hash != NULL && Obj_is_a((Obj*)hash, HASH)) {
- String *pid_buf = (String*)Hash_Fetch_Utf8(hash, "pid", 3);
- String *host = (String*)Hash_Fetch_Utf8(hash, "host", 4);
- String *name = (String*)Hash_Fetch_Utf8(hash, "name", 4);
-
- // Match hostname and lock name.
- if (host != NULL
- && Str_is_a(host, STRING)
- && Str_Equals(host, (Obj*)ivars->host)
- && name != NULL
- && Str_is_a(name, STRING)
- && Str_Equals(name, (Obj*)ivars->name)
- && pid_buf != NULL
- && Str_is_a(pid_buf, STRING)
+ return match;
+}
+
+static bool
+S_maybe_delete_file(LockFileLockIVARS *ivars, String *path,
+ bool delete_mine, bool delete_other) {
+ Folder *folder = ivars->folder;
+ bool success = false;
+
+ Hash *hash = (Hash*)Json_slurp_json(folder, path);
+ if (hash != NULL && Obj_is_a((Obj*)hash, HASH)) {
+ String *pid_buf = (String*)Hash_Fetch_Utf8(hash, "pid", 3);
+ String *host = (String*)Hash_Fetch_Utf8(hash, "host", 4);
+ String *name = (String*)Hash_Fetch_Utf8(hash, "name", 4);
+
+ // Match hostname and lock name.
+ if (host != NULL
+ && Str_is_a(host, STRING)
+ && Str_Equals(host, (Obj*)ivars->host)
+ && name != NULL
+ && Str_is_a(name, STRING)
+ && Str_Equals(name, (Obj*)ivars->name)
+ && pid_buf != NULL
+ && Str_is_a(pid_buf, STRING)
+ ) {
+ // Verify that pid is either mine or dead.
+ int pid = (int)Str_To_I64(pid_buf);
+ if ((delete_mine && pid == PID_getpid()) // This process.
+ || (delete_other && !PID_active(pid)) // Dead pid.
) {
- // Verify that pid is either mine or dead.
- int pid = (int)Str_To_I64(pid_buf);
- if ((delete_mine && pid == PID_getpid()) // This process.
- || (delete_other && !PID_active(pid)) // Dead pid.
- ) {
- if (Folder_Delete(folder, path)) {
- success = true;
- }
- else {
- String *mess
- = MAKE_MESS("Can't delete '%o'", path);
- DECREF(hash);
- Err_throw_mess(ERR, mess);
- }
+ if (Folder_Delete(folder, path)) {
+ success = true;
+ }
+ else {
+ String *mess
+ = MAKE_MESS("Can't delete '%o'", path);
+ DECREF(hash);
+ Err_throw_mess(ERR, mess);
}
}
}
- DECREF(hash);
}
+ DECREF(hash);
return success;
}
@@ -313,6 +488,7 @@ LFLock_Maybe_Delete_File_IMP(LockFileLock *self, String *path,
void
LFLock_Destroy_IMP(LockFileLock *self) {
LockFileLockIVARS *const ivars = LFLock_IVARS(self);
+ DECREF(ivars->shared_lock_path);
DECREF(ivars->link_path);
SUPER_DESTROY(self, LOCKFILELOCK);
}
http://git-wip-us.apache.org/repos/asf/lucy/blob/bb83d0d7/core/Lucy/Store/Lock.cfh
----------------------------------------------------------------------
diff --git a/core/Lucy/Store/Lock.cfh b/core/Lucy/Store/Lock.cfh
index 111a045..3f93dc0 100644
--- a/core/Lucy/Store/Lock.cfh
+++ b/core/Lucy/Store/Lock.cfh
@@ -43,41 +43,53 @@ public abstract class Lucy::Store::Lock inherits Clownfish::Obj {
* consist solely of characters matching [-_.A-Za-z0-9].
* @param host A unique per-machine identifier.
* @param timeout Time in milliseconds to keep retrying before abandoning
- * the attempt to [](cfish:.Obtain) a lock.
+ * the attempt to obtain a lock.
* @param interval Time in milliseconds between retries.
*/
public inert Lock*
init(Lock *self, Folder *folder, String *name,
String *host, int32_t timeout = 0, int32_t interval = 100);
- /** Returns true if the Lock is shared, false if the Lock is exclusive.
+ /** Call [](.Request_Shared) once per `interval` until [](.Request_Shared)
+ * returns success or the `timeout` has been reached.
+ *
+ * @return true on success, false on failure (sets the global error object
+ * returned by [](cfish:cfish.Err.get_error)).
*/
- public abstract bool
- Shared(Lock *self);
+ public bool
+ Obtain_Shared(Lock *self);
- /** Call [](cfish:.Request) once per `interval` until [](cfish:.Request) returns
- * success or the `timeout` has been reached.
+ /** Call [](.Request_Exclusive) once per `interval` until
+ * [](.Request_Exclusive) returns success or the `timeout` has been
+ * reached.
*
* @return true on success, false on failure (sets the global error object
* returned by [](cfish:cfish.Err.get_error)).
*/
public bool
- Obtain(Lock *self);
+ Obtain_Exclusive(Lock *self);
+
+ /** Make one attempt to acquire a shared lock.
+ *
+ * [](.Request_Shared) should not fail if another shared lock is held
+ * against the resource identified by `name` (though it might fail for
+ * other reasons).
+ *
+ * @return true on success, false on failure (sets the global error object
+ * returned by [](cfish:cfish.Err.get_error)).
+ */
+ public abstract bool
+ Request_Shared(Lock *self);
- /** Make one attempt to acquire the lock.
+ /** Make one attempt to acquire an exclusive lock.
*
- * The semantics of [](cfish:.Request) differ depending on whether [](cfish:.Shared) returns
- * true. If the Lock is [](cfish:.Shared), then [](cfish:.Request) should not fail if
- * another lock is held against the resource identified by
- * `name` (though it might fail for other reasons). If it is
- * not [](cfish:.Shared) -- i.e. it's an exclusive (write) lock -- then other locks
- * should cause [](cfish:.Request) to fail.
+ * Other locks should cause [](.Request_Exclusive) to fail.
*
* @return true on success, false on failure (sets the global error object
* returned by [](cfish:cfish.Err.get_error)).
*/
public abstract bool
- Request(Lock *self);
+ Request_Exclusive(Lock *self);
/** Release the lock.
*/
@@ -85,13 +97,21 @@ public abstract class Lucy::Store::Lock inherits Clownfish::Obj {
Release(Lock *self);
/** Indicate whether the resource identified by this lock's name is
- * currently locked.
+ * currently locked in shared or exclusive mode.
*
* @return true if the resource is locked, false otherwise.
*/
public abstract bool
Is_Locked(Lock *self);
+ /** Indicate whether the resource identified by this lock's name is
+ * currently locked in exclusive mode.
+ *
+ * @return true if the resource is locked, false otherwise.
+ */
+ public abstract bool
+ Is_Locked_Exclusive(Lock *self);
+
/** Release all locks that meet the following three conditions: the lock
* name matches, the host id matches, and the process id that the lock
* was created under no longer identifies an active process.
@@ -105,9 +125,6 @@ public abstract class Lucy::Store::Lock inherits Clownfish::Obj {
String*
Get_Host(Lock *self);
- String*
- Get_Lock_Path(Lock *self);
-
public void
Destroy(Lock *self);
}
@@ -115,21 +132,23 @@ public abstract class Lucy::Store::Lock inherits Clownfish::Obj {
class Lucy::Store::LockFileLock nickname LFLock
inherits Lucy::Store::Lock {
+ String *shared_lock_path;
String *link_path;
+ bool exclusive_only;
inert incremented LockFileLock*
- new(Folder *folder, String *name, String *host,
- int32_t timeout = 0, int32_t interval = 100);
+ new(Folder *folder, String *name, String *host, int32_t timeout = 0,
+ int32_t interval = 100, bool exclusive_only);
public inert LockFileLock*
- init(LockFileLock *self, Folder *folder, String *name,
- String *host, int32_t timeout = 0, int32_t interval = 100);
+ init(LockFileLock *self, Folder *folder, String *name, String *host,
+ int32_t timeout = 0, int32_t interval = 100, bool exclusive_only);
public bool
- Shared(LockFileLock *self);
+ Request_Shared(LockFileLock *self);
public bool
- Request(LockFileLock *self);
+ Request_Exclusive(LockFileLock *self);
public void
Release(LockFileLock *self);
@@ -137,21 +156,14 @@ class Lucy::Store::LockFileLock nickname LFLock
public bool
Is_Locked(LockFileLock *self);
+ public bool
+ Is_Locked_Exclusive(LockFileLock *self);
+
public void
Clear_Stale(LockFileLock *self);
- /** Delete a given lock file which meets these conditions...
- *
- * - lock name matches.
- * - host id matches.
- *
- * If delete_mine is false, don't delete a lock file which matches this
- * process's pid. If delete_other is false, don't delete lock files which
- * don't match this process's pid.
- */
- bool
- Maybe_Delete_File(LockFileLock *self, String *filepath,
- bool delete_mine, bool delete_other);
+ String*
+ Get_Lock_Path(LockFileLock *self);
public void
Destroy(LockFileLock *self);
http://git-wip-us.apache.org/repos/asf/lucy/blob/bb83d0d7/core/Lucy/Store/LockFactory.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Store/LockFactory.c b/core/Lucy/Store/LockFactory.c
index d2042cd..bd501f3 100644
--- a/core/Lucy/Store/LockFactory.c
+++ b/core/Lucy/Store/LockFactory.c
@@ -23,7 +23,6 @@
#include "Lucy/Store/LockFactory.h"
#include "Lucy/Store/Folder.h"
#include "Lucy/Store/Lock.h"
-#include "Lucy/Store/SharedLock.h"
LockFactory*
LockFact_new(Folder *folder, String *host) {
@@ -48,19 +47,11 @@ LockFact_Destroy_IMP(LockFactory *self) {
}
Lock*
-LockFact_Make_Lock_IMP(LockFactory *self, String *name,
- int32_t timeout, int32_t interval) {
+LockFact_Make_Lock_IMP(LockFactory *self, String *name, int32_t timeout,
+ int32_t interval, bool exclusive_only) {
LockFactoryIVARS *const ivars = LockFact_IVARS(self);
return (Lock*)LFLock_new(ivars->folder, name, ivars->host, timeout,
- interval);
-}
-
-Lock*
-LockFact_Make_Shared_Lock_IMP(LockFactory *self, String *name,
- int32_t timeout, int32_t interval) {
- LockFactoryIVARS *const ivars = LockFact_IVARS(self);
- return (Lock*)ShLock_new(ivars->folder, name, ivars->host, timeout,
- interval);
+ interval, exclusive_only);
}
http://git-wip-us.apache.org/repos/asf/lucy/blob/bb83d0d7/core/Lucy/Store/LockFactory.cfh
----------------------------------------------------------------------
diff --git a/core/Lucy/Store/LockFactory.cfh b/core/Lucy/Store/LockFactory.cfh
index 383d271..47152b6 100644
--- a/core/Lucy/Store/LockFactory.cfh
+++ b/core/Lucy/Store/LockFactory.cfh
@@ -46,32 +46,17 @@ public class Lucy::Store::LockFactory nickname LockFact
public inert LockFactory*
init(LockFactory *self, Folder *folder, String *host);
- /** Return a Lock object, which, once [](cfish:Lock.Obtain) returns successfully,
- * maintains an exclusive lock on a resource.
+ /** Return a Lock object.
*
* @param name A file-system-friendly id which identifies the
* resource to be locked.
* @param timeout Time in milliseconds to keep retrying before abandoning
- * the attempt to [](cfish:Lock.Obtain) a lock.
+ * the attempt to obtain a lock.
* @param interval Time in milliseconds between retries.
*/
public incremented Lock*
Make_Lock(LockFactory *self, String *name, int32_t timeout = 0,
- int32_t interval = 100);
-
- /** Return a Lock object for which [](cfish:Lock.Shared) returns true, and which
- * maintains a non-exclusive lock on a resource once [](cfish:Lock.Obtain) returns
- * success.
- *
- * @param name A file-system-friendly id which identifies the
- * resource to be locked.
- * @param timeout Time in milliseconds to keep retrying before abandoning
- * the attempt to [](cfish:Lock.Obtain) a lock.
- * @param interval Time in milliseconds between retries.
- */
- public incremented Lock*
- Make_Shared_Lock(LockFactory *self, String *name,
- int32_t timeout = 0, int32_t interval = 100);
+ int32_t interval = 100, bool exclusive_only);
public void
Destroy(LockFactory *self);
http://git-wip-us.apache.org/repos/asf/lucy/blob/bb83d0d7/core/Lucy/Store/SharedLock.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Store/SharedLock.c b/core/Lucy/Store/SharedLock.c
deleted file mode 100644
index 2563e5f..0000000
--- a/core/Lucy/Store/SharedLock.c
+++ /dev/null
@@ -1,173 +0,0 @@
-/* Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#define C_LUCY_SHAREDLOCK
-#include "Lucy/Util/ToolSet.h"
-
-#include <errno.h>
-#include <stdio.h>
-#include <ctype.h>
-
-#include "Lucy/Store/SharedLock.h"
-#include "Lucy/Store/DirHandle.h"
-#include "Lucy/Store/Folder.h"
-#include "Lucy/Store/OutStream.h"
-
-SharedLock*
-ShLock_new(Folder *folder, String *name, String *host,
- int32_t timeout, int32_t interval) {
- SharedLock *self = (SharedLock*)Class_Make_Obj(SHAREDLOCK);
- return ShLock_init(self, folder, name, host, timeout, interval);
-}
-
-SharedLock*
-ShLock_init(SharedLock *self, Folder *folder, String *name,
- String *host, int32_t timeout, int32_t interval) {
- LFLock_init((LockFileLock*)self, folder, name, host, timeout, interval);
- SharedLockIVARS *const ivars = ShLock_IVARS(self);
-
- // Override.
- DECREF(ivars->lock_path);
- ivars->lock_path = Str_newf("");
-
- return self;
-}
-
-bool
-ShLock_Shared_IMP(SharedLock *self) {
- UNUSED_VAR(self);
- return true;
-}
-
-bool
-ShLock_Request_IMP(SharedLock *self) {
- SharedLockIVARS *const ivars = ShLock_IVARS(self);
- uint32_t i = 0;
- ShLock_Request_t super_request
- = SUPER_METHOD_PTR(SHAREDLOCK, LUCY_ShLock_Request);
-
- // Empty lock_path indicates whether this particular instance is locked.
- if (ivars->lock_path
- && !Str_Equals_Utf8(ivars->lock_path, "", 0)
- && Folder_Exists(ivars->folder, ivars->lock_path)
- ) {
- // Don't allow double obtain.
- Err_set_error((Err*)LockErr_new(Str_newf("Lock already obtained via '%o'",
- ivars->lock_path)));
- return false;
- }
-
- do {
- DECREF(ivars->lock_path);
- ivars->lock_path = Str_newf("locks/%o-%u32.lock", ivars->name, ++i);
- } while (Folder_Exists(ivars->folder, ivars->lock_path));
-
- bool success = super_request(self);
- if (!success) { ERR_ADD_FRAME(Err_get_error()); }
- return success;
-}
-
-void
-ShLock_Release_IMP(SharedLock *self) {
- SharedLockIVARS *const ivars = ShLock_IVARS(self);
- if (ivars->lock_path && !Str_Equals_Utf8(ivars->lock_path, "", 0)) {
- ShLock_Release_t super_release
- = SUPER_METHOD_PTR(SHAREDLOCK, LUCY_ShLock_Release);
- super_release(self);
-
- // Empty out lock_path.
- DECREF(ivars->lock_path);
- ivars->lock_path = Str_newf("");
- }
-}
-
-
-void
-ShLock_Clear_Stale_IMP(SharedLock *self) {
- SharedLockIVARS *const ivars = ShLock_IVARS(self);
-
- String *lock_dir_name = SSTR_WRAP_C("locks");
- if (!Folder_Find_Folder(ivars->folder, lock_dir_name)) {
- return;
- }
-
- DirHandle *dh = Folder_Open_Dir(ivars->folder, lock_dir_name);
- if (!dh) { RETHROW(INCREF(Err_get_error())); }
-
- // Take a stab at any file that begins with our lock name.
- while (DH_Next(dh)) {
- String *entry = DH_Get_Entry(dh);
- if (Str_Starts_With(entry, ivars->name)
- && Str_Ends_With_Utf8(entry, ".lock", 5)
- ) {
- String *candidate = Str_newf("%o/%o", lock_dir_name, entry);
- ShLock_Maybe_Delete_File(self, candidate, false, true);
- DECREF(candidate);
- }
- DECREF(entry);
- }
-
- DECREF(dh);
-}
-
-bool
-ShLock_Is_Locked_IMP(SharedLock *self) {
- SharedLockIVARS *const ivars = ShLock_IVARS(self);
-
- String *lock_dir_name = SSTR_WRAP_C("locks");
- if (!Folder_Find_Folder(ivars->folder, lock_dir_name)) {
- return false;
- }
-
- DirHandle *dh = Folder_Open_Dir(ivars->folder, lock_dir_name);
- if (!dh) { RETHROW(INCREF(Err_get_error())); }
-
- while (DH_Next(dh)) {
- String *entry = DH_Get_Entry(dh);
- // Translation: $locked = 1 if $entry =~ /^\Q$name-\d+\.lock$/
- if (Str_Starts_With(entry, ivars->name)) {
- StringIterator *iter = Str_Top(entry);
- StrIter_Advance(iter, Str_Length(ivars->name));
- int32_t code_point = StrIter_Next(iter);
- if (code_point == '-') {
- code_point = StrIter_Next(iter);
- if (code_point != STR_OOB && isdigit(code_point)) {
- while (STR_OOB != (code_point = StrIter_Next(iter))) {
- if (!isdigit(code_point)) { break; }
- }
- if (code_point == '.'
- && StrIter_Starts_With_Utf8(iter, "lock", 4)
- ) {
- StrIter_Advance(iter, 4);
- if (!StrIter_Has_Next(iter)) {
- DECREF(iter);
- DECREF(entry);
- DECREF(dh);
- return true;
- }
- }
- }
- }
- DECREF(iter);
- }
- DECREF(entry);
- }
-
- DECREF(dh);
- return false;
-}
-
-
http://git-wip-us.apache.org/repos/asf/lucy/blob/bb83d0d7/core/Lucy/Store/SharedLock.cfh
----------------------------------------------------------------------
diff --git a/core/Lucy/Store/SharedLock.cfh b/core/Lucy/Store/SharedLock.cfh
deleted file mode 100644
index ba0fca2..0000000
--- a/core/Lucy/Store/SharedLock.cfh
+++ /dev/null
@@ -1,77 +0,0 @@
-/* Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-parcel Lucy;
-
-/** Shared (read) lock.
- *
- * SharedLock's interface is nearly identical to that of its parent class
- * [](cfish:Lock), taking the same constructor arguments and
- * implementing the same list of methods. It differs from Lock only in the
- * semantics of two methods.
- *
- * First, [](cfish:.Obtain) will not fail if another lock is held against the resource
- * identified by `name` (though it might fail for other reasons).
- *
- * Second, [](cfish:.Is_Locked) returns true so long as some lock, somewhere is holding
- * a lock on `name`. That lock could be this instance, or it could
- * be another -- so is entirely possible to call [](cfish:.Release) successfully on a
- * SharedLock object yet still have [](cfish:.Is_Locked) return true.
- *
- * As currently implemented, SharedLock differs from Lock in that each caller
- * gets its own lockfile. Lockfiles still have filenames which begin with the
- * lock name and end with ".lock", but each is also assigned a unique number
- * which gets pasted between: "foo-44.lock" instead of "foo.lock". A
- * SharedLock is considered fully released when no lock files with a given
- * lock name are left.
- */
-class Lucy::Store::SharedLock nickname ShLock
- inherits Lucy::Store::LockFileLock {
-
- inert incremented SharedLock*
- new(Folder *folder, String *name, String *host,
- int32_t timeout = 0, int32_t interval = 100);
-
- /**
- * @param folder The Lucy::Store::Folder where the lock file will
- * reside.
- * @param name String identifying the resource to be locked.
- * @param host An identifier which should be unique per-machine.
- * @param timeout Time in milliseconds to keep retrying before abandoning
- * the attempt to [](cfish:.Obtain) a lock.
- * @param interval Time in milliseconds between retries.
- */
- public inert SharedLock*
- init(SharedLock *self, Folder *folder, String *name,
- String *host, int32_t timeout = 0, int32_t interval = 100);
-
- public bool
- Shared(SharedLock *self);
-
- public bool
- Request(SharedLock *self);
-
- public void
- Release(SharedLock *self);
-
- public bool
- Is_Locked(SharedLock *self);
-
- public void
- Clear_Stale(SharedLock *self);
-}
-
-
http://git-wip-us.apache.org/repos/asf/lucy/blob/bb83d0d7/go/build.go
----------------------------------------------------------------------
diff --git a/go/build.go b/go/build.go
index b751ba5..16c31f2 100644
--- a/go/build.go
+++ b/go/build.go
@@ -453,8 +453,10 @@ func specClasses(parcel *cfc.Parcel) {
dhBinding.Register()
lockBinding := cfc.NewGoClass(parcel, "Lucy::Store::Lock")
- lockBinding.SpecMethod("Request", "Request() error")
- lockBinding.SpecMethod("Obtain", "Obtain() error")
+ lockBinding.SpecMethod("Request_Shared", "RequestShared() error")
+ lockBinding.SpecMethod("Request_Exclusive", "RequestExclusive() error")
+ lockBinding.SpecMethod("Obtain_Shared", "ObtainShared() error")
+ lockBinding.SpecMethod("Obtain_Exclusive", "ObtainExclusive() error")
lockBinding.SpecMethod("Release", "Release() error")
lockBinding.SpecMethod("Clear_Stale", "ClearStale() error")
lockBinding.Register()
http://git-wip-us.apache.org/repos/asf/lucy/blob/bb83d0d7/go/lucy/index_test.go
----------------------------------------------------------------------
diff --git a/go/lucy/index_test.go b/go/lucy/index_test.go
index 30f7c4a..d47785e 100644
--- a/go/lucy/index_test.go
+++ b/go/lucy/index_test.go
@@ -196,7 +196,7 @@ func TestIndexManagerLocks(t *testing.T) {
t.Errorf("makeDeletionLock")
}
snapFile := "snapshot_4a.json"
- if _, ok := manager.makeSnapshotReadLock(snapFile).(SharedLock); !ok {
+ if _, ok := manager.makeSnapshotReadLock(snapFile).(Lock); !ok {
t.Errorf("makeDeletionLock")
}
}
http://git-wip-us.apache.org/repos/asf/lucy/blob/bb83d0d7/go/lucy/store.go
----------------------------------------------------------------------
diff --git a/go/lucy/store.go b/go/lucy/store.go
index a543ece..9b9d8c3 100644
--- a/go/lucy/store.go
+++ b/go/lucy/store.go
@@ -718,9 +718,9 @@ func OpenFSDirHandle(path string) (dh FSDirHandle, err error) {
return dh, err
}
-func (lock *LockIMP) Request() error {
+func (lock *LockIMP) RequestShared() error {
self := (*C.lucy_Lock)(clownfish.Unwrap(lock, "lock"))
- success := C.LUCY_Lock_Request(self)
+ success := C.LUCY_Lock_Request_Shared(self)
if !success {
cfErr := C.cfish_Err_get_error();
return clownfish.WRAPAny(unsafe.Pointer(C.cfish_incref(unsafe.Pointer(cfErr)))).(error)
@@ -728,9 +728,29 @@ func (lock *LockIMP) Request() error {
return nil
}
-func (lock *LockIMP) Obtain() error {
+func (lock *LockIMP) RequestExclusive() error {
self := (*C.lucy_Lock)(clownfish.Unwrap(lock, "lock"))
- success := C.LUCY_Lock_Obtain(self)
+ success := C.LUCY_Lock_Request_Exclusive(self)
+ if !success {
+ cfErr := C.cfish_Err_get_error();
+ return clownfish.WRAPAny(unsafe.Pointer(C.cfish_incref(unsafe.Pointer(cfErr)))).(error)
+ }
+ return nil
+}
+
+func (lock *LockIMP) ObtainShared() error {
+ self := (*C.lucy_Lock)(clownfish.Unwrap(lock, "lock"))
+ success := C.LUCY_Lock_Obtain_Shared(self)
+ if !success {
+ cfErr := C.cfish_Err_get_error();
+ return clownfish.WRAPAny(unsafe.Pointer(C.cfish_incref(unsafe.Pointer(cfErr)))).(error)
+ }
+ return nil
+}
+
+func (lock *LockIMP) ObtainExclusive() error {
+ self := (*C.lucy_Lock)(clownfish.Unwrap(lock, "lock"))
+ success := C.LUCY_Lock_Obtain_Exclusive(self)
if !success {
cfErr := C.cfish_Err_get_error();
return clownfish.WRAPAny(unsafe.Pointer(C.cfish_incref(unsafe.Pointer(cfErr)))).(error)
http://git-wip-us.apache.org/repos/asf/lucy/blob/bb83d0d7/go/lucy/store_test.go
----------------------------------------------------------------------
diff --git a/go/lucy/store_test.go b/go/lucy/store_test.go
index d87d539..680b9b2 100644
--- a/go/lucy/store_test.go
+++ b/go/lucy/store_test.go
@@ -646,10 +646,11 @@ func TestFSDirHandleAll(t *testing.T) {
runDirHandleCommonTests(t, folder, makeDH)
}
-func runLockCommonTests(t *testing.T, makeLock func(string, string) Lock) {
+func TestLockFileLockAll(t *testing.T) {
var err error
- lock := makeLock("foo", "dev.example.com")
- other := makeLock("foo", "dev.example.com")
+ folder := NewRAMFolder("myindex")
+ lock := NewLockFileLock(folder, "foo", "dev.example.com", 0, 1, false)
+ other := NewLockFileLock(folder, "foo", "dev.example.com", 0, 1, false)
if got := lock.getName(); got != "foo" {
t.Errorf("getName: %v", got)
@@ -658,7 +659,7 @@ func runLockCommonTests(t *testing.T, makeLock func(string, string) Lock) {
t.Errorf("getHost: %v", got)
}
- err = lock.Request()
+ err = lock.RequestShared()
if err != nil {
t.Errorf("Request: %v", err)
}
@@ -669,42 +670,51 @@ func runLockCommonTests(t *testing.T, makeLock func(string, string) Lock) {
// Lock path only valid when locked for shared locks.
t.Errorf("getLockPath should work")
}
- err = other.Request()
- if other.Shared() && err != nil {
- t.Errorf("SharedLock Request should succeed: %v", err)
- } else if !other.Shared() && err == nil {
- t.Errorf("Request should fail for locked resource")
+ err = other.RequestShared()
+ if err != nil {
+ t.Errorf("Shared lock Request should succeed: %v", err)
}
err = lock.Release()
if err != nil {
- t.Errorf("Request: %v", err)
+ t.Errorf("Release: %v", err)
}
other.Release()
- err = lock.Obtain()
+ err = lock.ObtainShared()
if err != nil {
t.Errorf("Obtain: %v", err)
}
+ lock.Release()
- err = lock.ClearStale()
+ err = lock.RequestExclusive()
if err != nil {
- t.Errorf("Nothing for ClearStale to do, but should still suceed: %v", err)
+ t.Errorf("Request: %v", err)
}
-}
-
-func TestLockFileLockAll(t *testing.T) {
- folder := NewRAMFolder("myindex")
- makeLock := func(name, host string) Lock {
- return NewLockFileLock(folder, name, host, 0, 1)
+ if !lock.IsLocked() {
+ t.Errorf("Lock should be locked, but IsLocked returned false")
}
- runLockCommonTests(t, makeLock)
-}
+ if got := lock.getLockPath(); len(got) == 0 {
+ // Lock path only valid when locked for shared locks.
+ t.Errorf("getLockPath should work")
+ }
+ err = other.RequestExclusive()
+ if err == nil {
+ t.Errorf("Request should fail for locked resource")
+ }
+ err = lock.Release()
+ if err != nil {
+ t.Errorf("Release: %v", err)
+ }
+ other.Release()
+ err = lock.ObtainExclusive()
+ if err != nil {
+ t.Errorf("Obtain: %v", err)
+ }
+ lock.Release()
-func TestSharedLockAll(t *testing.T) {
- folder := NewRAMFolder("myindex")
- makeLock := func(name, host string) Lock {
- return NewSharedLock(folder, name, host, 0, 1)
+ err = lock.ClearStale()
+ if err != nil {
+ t.Errorf("Nothing for ClearStale to do, but should still suceed: %v", err)
}
- runLockCommonTests(t, makeLock)
}
func TestLockFactoryAll(t *testing.T) {
http://git-wip-us.apache.org/repos/asf/lucy/blob/bb83d0d7/perl/buildlib/Lucy/Build/Binding/Store.pm
----------------------------------------------------------------------
diff --git a/perl/buildlib/Lucy/Build/Binding/Store.pm b/perl/buildlib/Lucy/Build/Binding/Store.pm
index 4eedcba..4a0dc54 100644
--- a/perl/buildlib/Lucy/Build/Binding/Store.pm
+++ b/perl/buildlib/Lucy/Build/Binding/Store.pm
@@ -255,7 +255,7 @@ B<host> - A unique per-machine identifier.
=item *
B<timeout> - Time in milliseconds to keep retrying before abandoning
-the attempt to L<obtain()|/obtain> a lock.
+the attempt to obtain a lock.
=item *
http://git-wip-us.apache.org/repos/asf/lucy/blob/bb83d0d7/perl/t/105-folder.t
----------------------------------------------------------------------
diff --git a/perl/t/105-folder.t b/perl/t/105-folder.t
index 735c162..3d00195 100644
--- a/perl/t/105-folder.t
+++ b/perl/t/105-folder.t
@@ -47,27 +47,33 @@ for my $folder ( $fs_folder, $ram_folder ) {
is( $slurped, $king, "slurp_file works" );
my $lock = Lucy::Store::LockFileLock->new(
- host => '',
- folder => $folder,
- name => 'lock_robster',
- timeout => 0,
+ host => '',
+ folder => $folder,
+ name => 'lock_robster',
+ timeout => 0,
+ exclusive_only => 1,
);
my $competing_lock = Lucy::Store::LockFileLock->new(
- host => '',
- folder => $folder,
- name => 'lock_robster',
- timeout => 0,
+ host => '',
+ folder => $folder,
+ name => 'lock_robster',
+ timeout => 0,
+ exclusive_only => 1,
);
- $lock->obtain;
- ok( $lock->is_locked, "lock is locked" );
- ok( !$competing_lock->obtain, "shouldn't get lock on existing resource" );
- ok( $lock->is_locked, "lock still locked after competing attempt" );
+ $lock->obtain_exclusive();
+ ok( $lock->is_locked_exclusive(), "lock is locked" );
+ ok( !$competing_lock->obtain_exclusive(),
+ "shouldn't get lock on existing resource"
+ );
+ ok( $lock->is_locked_exclusive(),
+ "lock still locked after competing attempt"
+ );
$lock->release;
- ok( !$lock->is_locked, "release works" );
+ ok( !$lock->is_locked_exclusive(), "release works" );
- $lock->obtain;
+ $lock->obtain_exclusive();
$folder->rename( from => 'king_of_rock', to => 'king_of_lock' );
$lock->release;
http://git-wip-us.apache.org/repos/asf/lucy/blob/bb83d0d7/perl/t/106-locking.t
----------------------------------------------------------------------
diff --git a/perl/t/106-locking.t b/perl/t/106-locking.t
index b002b0a..3a1670c 100644
--- a/perl/t/106-locking.t
+++ b/perl/t/106-locking.t
@@ -44,13 +44,14 @@ Dead_locks_are_removed: {
sub make_lock {
my $lock = Lucy::Store::LockFileLock->new(
- timeout => 0,
- name => 'foo',
- host => '',
+ timeout => 0,
+ name => 'foo',
+ host => '',
+ exclusive_only => 1,
@_
);
$lock->clear_stale;
- $lock->obtain or die "no dice";
+ $lock->obtain_exclusive() or die "no dice";
return $lock;
}
http://git-wip-us.apache.org/repos/asf/lucy/blob/bb83d0d7/perl/t/110-shared_lock.t
----------------------------------------------------------------------
diff --git a/perl/t/110-shared_lock.t b/perl/t/110-shared_lock.t
index 0164cd5..5281c9f 100644
--- a/perl/t/110-shared_lock.t
+++ b/perl/t/110-shared_lock.t
@@ -21,40 +21,43 @@ use Lucy::Test;
my $folder = Lucy::Store::RAMFolder->new;
-my $lock = Lucy::Store::SharedLock->new(
- folder => $folder,
- name => 'ness',
- timeout => 0,
- host => 'nessie',
+my $lock = Lucy::Store::LockFileLock->new(
+ folder => $folder,
+ name => 'ness',
+ timeout => 0,
+ host => 'nessie',
+ exclusive_only => 0,
);
-ok( !$lock->is_locked, "not locked yet" );
+ok( !$lock->is_locked(), "not locked yet" );
-ok( $lock->obtain, "obtain" );
-ok( $lock->is_locked, "is_locked" );
+ok( $lock->obtain_shared(), "obtain" );
+ok( $lock->is_locked(), "is_locked" );
ok( $folder->exists('locks/ness-1.lock'), "lockfile exists" );
-my $another_lock = Lucy::Store::SharedLock->new(
- folder => $folder,
- name => 'ness',
- timeout => 0,
- host => 'nessie',
+my $another_lock = Lucy::Store::LockFileLock->new(
+ folder => $folder,
+ name => 'ness',
+ timeout => 0,
+ host => 'nessie',
+ exclusive_only => 0,
);
-ok( $another_lock->obtain, "got a second lock on the same resource" );
+ok( $another_lock->obtain_shared(), "got a second lock on the same resource" );
$lock->release;
-ok( $lock->is_locked,
+ok( $lock->is_locked(),
"first lock released but still is_locked because of other lock" );
-my $ya_lock = Lucy::Store::SharedLock->new(
- folder => $folder,
- name => 'ness',
- timeout => 0,
- host => 'nessie',
+my $ya_lock = Lucy::Store::LockFileLock->new(
+ folder => $folder,
+ name => 'ness',
+ timeout => 0,
+ host => 'nessie',
+ exclusive_only => 0,
);
-ok( $ya_lock->obtain, "got yet another lock" );
+ok( $ya_lock->obtain_shared(), "got yet another lock" );
-ok( $lock->obtain, "got first lock again" );
+ok( $lock->obtain_shared(), "got first lock again" );
is( $lock->get_lock_path, "locks/ness-3.lock",
"first lock uses a different lock_path now" );
@@ -71,12 +74,12 @@ $lock->release;
$another_lock->release;
$ya_lock->release;
-ok( $lock->is_locked, "failed to release a lock with a different pid" );
+ok( $lock->is_locked(), "failed to release a lock with a different pid" );
$lock->clear_stale;
-ok( !$lock->is_locked, "clear_stale" );
+ok( !$lock->is_locked(), "clear_stale" );
-ok( $lock->obtain, "got lock again" );
-ok( $lock->is_locked, "it's locked" );
+ok( $lock->obtain_shared(), "got lock again" );
+ok( $lock->is_locked(), "it's locked" );
# Rewrite lock file to spec a different host.
$content = $folder->slurp_file("locks/ness-1.lock");
@@ -87,4 +90,4 @@ $outstream->print($content);
$outstream->close;
$lock->release;
-ok( $lock->is_locked, "don't delete lock belonging to another host" );
+ok( $lock->is_locked(), "don't delete lock belonging to another host" );
http://git-wip-us.apache.org/repos/asf/lucy/blob/bb83d0d7/perl/t/111-index_manager.t
----------------------------------------------------------------------
diff --git a/perl/t/111-index_manager.t b/perl/t/111-index_manager.t
index 76b4a09..16c98ea 100644
--- a/perl/t/111-index_manager.t
+++ b/perl/t/111-index_manager.t
@@ -32,7 +32,7 @@ sub recycle {
package main;
-use Test::More tests => 16;
+use Test::More tests => 13;
use Lucy::Test;
my $folder = Lucy::Store::RAMFolder->new;
@@ -43,21 +43,14 @@ my $lock_factory = Lucy::Store::LockFactory->new(
);
my $lock = $lock_factory->make_lock(
- name => 'angie',
- timeout => 1000,
+ name => 'angie',
+ timeout => 1000,
+ exclusive_only => 0,
);
isa_ok( $lock, 'Lucy::Store::Lock', "make_lock" );
is( $lock->get_name, "angie", "correct lock name" );
is( $lock->get_host, "me", "correct host" );
-$lock = $lock_factory->make_shared_lock(
- name => 'fred',
- timeout => 0,
-);
-is( ref($lock), 'Lucy::Store::SharedLock', "make_shared_lock" );
-is( $lock->get_name, "fred", "correct lock name" );
-is( $lock->get_host, "me", "correct host" );
-
my $schema = Lucy::Test::TestSchema->new;
$folder = Lucy::Store::RAMFolder->new;
[11/16] lucy git commit: Port LockFileLock tests to C
Posted by nw...@apache.org.
Port LockFileLock tests to C
Project: http://git-wip-us.apache.org/repos/asf/lucy/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucy/commit/35388cd2
Tree: http://git-wip-us.apache.org/repos/asf/lucy/tree/35388cd2
Diff: http://git-wip-us.apache.org/repos/asf/lucy/diff/35388cd2
Branch: refs/heads/master
Commit: 35388cd22600cb3c202cf11c8c9afd661c8f4fe4
Parents: e06cdbf
Author: Nick Wellnhofer <we...@aevum.de>
Authored: Fri Feb 17 21:38:56 2017 +0100
Committer: Nick Wellnhofer <we...@aevum.de>
Committed: Mon Feb 20 16:48:17 2017 +0100
----------------------------------------------------------------------
core/Lucy/Store/Lock.c | 18 --
core/Lucy/Store/Lock.cfh | 9 -
go/lucy/store_test.go | 15 --
perl/t/105-folder.t | 36 +---
perl/t/106-locking.t | 89 ----------
perl/t/110-shared_lock.t | 93 -----------
perl/t/111-index_manager.t | 17 +-
perl/t/core/106-lf_lock.t | 23 +++
test/Lucy/Test.c | 2 +
test/Lucy/Test/Store/TestLock.c | 297 +++++++++++++++++++++++++++++++++
test/Lucy/Test/Store/TestLock.cfh | 29 ++++
11 files changed, 356 insertions(+), 272 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucy/blob/35388cd2/core/Lucy/Store/Lock.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Store/Lock.c b/core/Lucy/Store/Lock.c
index 90a6aa0..5cddcd9 100644
--- a/core/Lucy/Store/Lock.c
+++ b/core/Lucy/Store/Lock.c
@@ -77,16 +77,6 @@ Lock_Destroy_IMP(Lock *self) {
SUPER_DESTROY(self, LOCK);
}
-String*
-Lock_Get_Name_IMP(Lock *self) {
- return Lock_IVARS(self)->name;
-}
-
-String*
-Lock_Get_Host_IMP(Lock *self) {
- return Lock_IVARS(self)->host;
-}
-
bool
Lock_Obtain_Shared_IMP(Lock *self) {
LockIVARS *const ivars = Lock_IVARS(self);
@@ -163,14 +153,6 @@ LFLock_init(LockFileLock *self, Folder *folder, String *name, String *host,
return self;
}
-String*
-LFLock_Get_Lock_Path_IMP(LockFileLock *self) {
- LockFileLockIVARS *const ivars = LFLock_IVARS(self);
- return ivars->shared_lock_path
- ? ivars->shared_lock_path
- : ivars->lock_path;
-}
-
struct lockfile_context {
OutStream *outstream;
String *json;
http://git-wip-us.apache.org/repos/asf/lucy/blob/35388cd2/core/Lucy/Store/Lock.cfh
----------------------------------------------------------------------
diff --git a/core/Lucy/Store/Lock.cfh b/core/Lucy/Store/Lock.cfh
index 6688e95..30940b4 100644
--- a/core/Lucy/Store/Lock.cfh
+++ b/core/Lucy/Store/Lock.cfh
@@ -96,12 +96,6 @@ abstract class Lucy::Store::Lock inherits Clownfish::Obj {
public abstract void
Release(Lock *self);
- String*
- Get_Name(Lock *self);
-
- String*
- Get_Host(Lock *self);
-
public void
Destroy(Lock *self);
}
@@ -131,9 +125,6 @@ class Lucy::Store::LockFileLock nickname LFLock
public void
Release(LockFileLock *self);
- String*
- Get_Lock_Path(LockFileLock *self);
-
public void
Destroy(LockFileLock *self);
}
http://git-wip-us.apache.org/repos/asf/lucy/blob/35388cd2/go/lucy/store_test.go
----------------------------------------------------------------------
diff --git a/go/lucy/store_test.go b/go/lucy/store_test.go
index ae70069..2b24ff4 100644
--- a/go/lucy/store_test.go
+++ b/go/lucy/store_test.go
@@ -652,21 +652,10 @@ func TestLockFileLockAll(t *testing.T) {
lock := NewLockFileLock(folder, "foo", "dev.example.com", 0, 1, false)
other := NewLockFileLock(folder, "foo", "dev.example.com", 0, 1, false)
- if got := lock.getName(); got != "foo" {
- t.Errorf("getName: %v", got)
- }
- if got := lock.getHost(); got != "dev.example.com" {
- t.Errorf("getHost: %v", got)
- }
-
err = lock.RequestShared()
if err != nil {
t.Errorf("Request: %v", err)
}
- if got := lock.getLockPath(); len(got) == 0 {
- // Lock path only valid when locked for shared locks.
- t.Errorf("getLockPath should work")
- }
err = other.RequestShared()
if err != nil {
t.Errorf("Shared lock Request should succeed: %v", err)
@@ -686,10 +675,6 @@ func TestLockFileLockAll(t *testing.T) {
if err != nil {
t.Errorf("Request: %v", err)
}
- if got := lock.getLockPath(); len(got) == 0 {
- // Lock path only valid when locked for shared locks.
- t.Errorf("getLockPath should work")
- }
err = other.RequestExclusive()
if err == nil {
t.Errorf("Request should fail for locked resource")
http://git-wip-us.apache.org/repos/asf/lucy/blob/35388cd2/perl/t/105-folder.t
----------------------------------------------------------------------
diff --git a/perl/t/105-folder.t b/perl/t/105-folder.t
index ba1255e..002642f 100644
--- a/perl/t/105-folder.t
+++ b/perl/t/105-folder.t
@@ -17,7 +17,7 @@ use strict;
use warnings;
use lib 'buildlib';
-use Test::More tests => 25;
+use Test::More tests => 17;
use File::Spec::Functions qw( catfile );
use Fcntl;
use Lucy::Test::TestUtils qw( init_test_index_loc );
@@ -46,43 +46,13 @@ for my $folder ( $fs_folder, $ram_folder ) {
my $slurped = $folder->slurp_file('king_of_rock');
is( $slurped, $king, "slurp_file works" );
- my $lock = Lucy::Store::LockFileLock->new(
- host => '',
- folder => $folder,
- name => 'lock_robster',
- timeout => 0,
- exclusive_only => 1,
- );
- my $competing_lock = Lucy::Store::LockFileLock->new(
- host => '',
- folder => $folder,
- name => 'lock_robster',
- timeout => 0,
- exclusive_only => 1,
- );
-
- $lock->obtain_exclusive();
- my $lock_path = $lock->get_lock_path;
- ok( $folder->exists($lock_path), "lock is locked" );
- ok( !$competing_lock->obtain_exclusive(),
- "shouldn't get lock on existing resource"
- );
- ok( $folder->exists($lock_path),
- "lock still locked after competing attempt"
- );
-
- $lock->release;
- ok( !$folder->exists($lock_path), "release works" );
-
- $lock->obtain_exclusive();
$folder->rename( from => 'king_of_rock', to => 'king_of_lock' );
- $lock->release;
ok( !$folder->exists('king_of_rock'),
- "file successfully removed while locked"
+ "file successfully removed"
);
is( $folder->exists('king_of_lock'),
- 1, "file successfully moved while locked" );
+ 1, "file successfully moved" );
is( $folder->open_out("king_of_lock"),
undef, "open_out returns undef when file exists" );
http://git-wip-us.apache.org/repos/asf/lucy/blob/35388cd2/perl/t/106-locking.t
----------------------------------------------------------------------
diff --git a/perl/t/106-locking.t b/perl/t/106-locking.t
deleted file mode 100644
index 15c6a3a..0000000
--- a/perl/t/106-locking.t
+++ /dev/null
@@ -1,89 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-use strict;
-use warnings;
-use lib 'buildlib';
-
-use Time::HiRes qw( sleep );
-use Test::More;
-use File::Spec::Functions qw( catfile );
-use Lucy::Test::TestUtils qw( init_test_index_loc );
-
-BEGIN {
- if ( $^O =~ /(mswin|cygwin)/i ) {
- plan( 'skip_all', "fork on Windows not supported by Lucy" );
- }
- else {
- plan( tests => 3 );
- }
-}
-
-my $path = init_test_index_loc();
-
-Dead_locks_are_removed: {
- my $lock_path = catfile( $path, 'locks', 'foo.lock' );
-
- # Remove any existing lockfile
- unlink $lock_path;
- die "Can't unlink '$lock_path'" if -e $lock_path;
-
- my $folder = Lucy::Store::FSFolder->new( path => $path );
-
- sub make_lock {
- my $lock = Lucy::Store::LockFileLock->new(
- timeout => 0,
- name => 'foo',
- host => '',
- exclusive_only => 1,
- @_
- );
- $lock->obtain_exclusive() or die "no dice";
- return $lock;
- }
-
- # Fork a process that will create a lock and then exit
- my $pid = fork();
- if ( $pid == 0 ) { # child
- make_lock( folder => $folder );
- exit;
- }
- else {
- waitpid( $pid, 0 );
- }
-
- sleep .1;
- ok( -e $lock_path, "child secured lock" );
-
- # The locking attempt will fail if the pid from the process that made the
- # lock is active, so do the best we can to see whether another process
- # started up with the child's pid (which would be weird).
- my $pid_active = kill( 0, $pid );
-
- eval { make_lock( folder => $folder, host => 'somebody_else' ) };
- like( $@, qr/no dice/, "different host fails to get lock" );
-
- eval { make_lock( folder => $folder ) };
- warn $@ if $@;
- my $saved_err = $@;
- $pid_active ||= kill( 0, $pid );
-SKIP: {
- skip( "Child's pid is active", 1 ) if $pid_active;
- ok( !$saved_err,
- 'second lock attempt clobbered dead lock file and did not die' );
- }
-
- undef $folder;
-}
http://git-wip-us.apache.org/repos/asf/lucy/blob/35388cd2/perl/t/110-shared_lock.t
----------------------------------------------------------------------
diff --git a/perl/t/110-shared_lock.t b/perl/t/110-shared_lock.t
deleted file mode 100644
index 86e7d6c..0000000
--- a/perl/t/110-shared_lock.t
+++ /dev/null
@@ -1,93 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-use strict;
-use warnings;
-
-use Test::More tests => 14;
-use Lucy::Test;
-
-my $folder = Lucy::Store::RAMFolder->new;
-my @args = (
- folder => $folder,
- name => 'ness',
- timeout => 0,
- host => 'nessie',
- exclusive_only => 0,
-);
-
-my $ex_lock = Lucy::Store::LockFileLock->new(@args);
-
-sub is_locked {
- if ($ex_lock->request_exclusive) {
- $ex_lock->release;
- return 0;
- }
- else {
- return 1;
- }
-}
-
-my $lock = Lucy::Store::LockFileLock->new(@args);
-
-ok( !is_locked(), "not locked yet" );
-
-ok( $lock->obtain_shared(), "obtain" );
-ok( is_locked(), "is_locked" );
-ok( $folder->exists('locks/ness-1.lock'), "lockfile exists" );
-
-my $another_lock = Lucy::Store::LockFileLock->new(@args);
-ok( $another_lock->obtain_shared(), "got a second lock on the same resource" );
-
-$lock->release;
-ok( is_locked(),
- "first lock released but still is_locked because of other lock" );
-
-my $ya_lock = Lucy::Store::LockFileLock->new(@args);
-ok( $ya_lock->obtain_shared(), "got yet another lock" );
-
-ok( $lock->obtain_shared(), "got first lock again" );
-is( $lock->get_lock_path, "locks/ness-3.lock",
- "first lock uses a different lock_path now" );
-
-# Rewrite lock file to spec a different pid.
-my $content = $folder->slurp_file("locks/ness-3.lock");
-$content =~ s/$$/123456789/;
-$folder->delete('locks/ness-3.lock') or die "Can't delete 'ness-3.lock'";
-my $outstream = $folder->open_out('locks/ness-3.lock')
- or die Clownfish->error;
-$outstream->print($content);
-$outstream->close;
-
-$lock->release;
-$another_lock->release;
-$ya_lock->release;
-
-ok( $lock->get_lock_path, "failed to release a lock with a different pid" );
-ok( !is_locked(), "is_locked clears stale locks" );
-
-ok( $lock->obtain_shared(), "got lock again" );
-ok( is_locked(), "it's locked" );
-
-# Rewrite lock file to spec a different host.
-$content = $folder->slurp_file("locks/ness-1.lock");
-$content =~ s/nessie/sting/;
-$folder->delete('locks/ness-1.lock') or die "Can't delete 'ness-1.lock'";
-$outstream = $folder->open_out('locks/ness-1.lock') or die Clownfish->error;
-$outstream->print($content);
-$outstream->close;
-
-$lock->release;
-ok( is_locked(), "don't delete lock belonging to another host" );
http://git-wip-us.apache.org/repos/asf/lucy/blob/35388cd2/perl/t/111-index_manager.t
----------------------------------------------------------------------
diff --git a/perl/t/111-index_manager.t b/perl/t/111-index_manager.t
index d42ae98..dd7d8d0 100644
--- a/perl/t/111-index_manager.t
+++ b/perl/t/111-index_manager.t
@@ -32,24 +32,11 @@ sub recycle {
package main;
-use Test::More tests => 11;
+use Test::More tests => 8;
use Lucy::Test;
-my $folder = Lucy::Store::RAMFolder->new;
-
-my $lock = Lucy::Store::LockFileLock->new(
- folder => $folder,
- host => 'me',
- name => 'angie',
- timeout => 1000,
- exclusive_only => 0,
-);
-isa_ok( $lock, 'Lucy::Store::Lock', "make_lock" );
-is( $lock->get_name, "angie", "correct lock name" );
-is( $lock->get_host, "me", "correct host" );
-
my $schema = Lucy::Test::TestSchema->new;
-$folder = Lucy::Store::RAMFolder->new;
+my $folder = Lucy::Store::RAMFolder->new;
for ( 1 .. 20 ) {
my $indexer = Lucy::Index::Indexer->new(
http://git-wip-us.apache.org/repos/asf/lucy/blob/35388cd2/perl/t/core/106-lf_lock.t
----------------------------------------------------------------------
diff --git a/perl/t/core/106-lf_lock.t b/perl/t/core/106-lf_lock.t
new file mode 100644
index 0000000..db99bc4
--- /dev/null
+++ b/perl/t/core/106-lf_lock.t
@@ -0,0 +1,23 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+my $success = Lucy::Test::run_tests("Lucy::Test::Store::TestLockFileLock");
+
+exit($success ? 0 : 1);
+
http://git-wip-us.apache.org/repos/asf/lucy/blob/35388cd2/test/Lucy/Test.c
----------------------------------------------------------------------
diff --git a/test/Lucy/Test.c b/test/Lucy/Test.c
index 22223e1..2a30a68 100644
--- a/test/Lucy/Test.c
+++ b/test/Lucy/Test.c
@@ -75,6 +75,7 @@
#include "Lucy/Test/Store/TestIOChunks.h"
#include "Lucy/Test/Store/TestIOPrimitives.h"
#include "Lucy/Test/Store/TestInStream.h"
+#include "Lucy/Test/Store/TestLock.h"
#include "Lucy/Test/Store/TestRAMDirHandle.h"
#include "Lucy/Test/Store/TestRAMFileHandle.h"
#include "Lucy/Test/Store/TestRAMFolder.h"
@@ -114,6 +115,7 @@ Test_create_test_suite() {
TestSuite_Add_Batch(suite, (TestBatch*)TestFSFolder_new());
TestSuite_Add_Batch(suite, (TestBatch*)TestRAMFolder_new());
TestSuite_Add_Batch(suite, (TestBatch*)TestFolder_new());
+ TestSuite_Add_Batch(suite, (TestBatch*)TestLFLock_new());
TestSuite_Add_Batch(suite, (TestBatch*)TestIxManager_new());
TestSuite_Add_Batch(suite, (TestBatch*)TestCFWriter_new());
TestSuite_Add_Batch(suite, (TestBatch*)TestCFReader_new());
http://git-wip-us.apache.org/repos/asf/lucy/blob/35388cd2/test/Lucy/Test/Store/TestLock.c
----------------------------------------------------------------------
diff --git a/test/Lucy/Test/Store/TestLock.c b/test/Lucy/Test/Store/TestLock.c
new file mode 100644
index 0000000..f606e7b
--- /dev/null
+++ b/test/Lucy/Test/Store/TestLock.c
@@ -0,0 +1,297 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define TESTLUCY_USE_SHORT_NAMES
+#include "Lucy/Util/ToolSet.h"
+
+#include "charmony.h"
+
+// rmdir
+#ifdef CHY_HAS_DIRECT_H
+ #include <direct.h>
+#endif
+#ifdef CHY_HAS_UNISTD_H
+ #include <unistd.h>
+#endif
+
+#include "Lucy/Test/Store/TestLock.h"
+#include "Clownfish/TestHarness/TestBatchRunner.h"
+#include "Lucy/Store/FSFolder.h"
+#include "Lucy/Store/Lock.h"
+#include "Lucy/Store/RAMFolder.h"
+#include "Lucy/Util/Json.h"
+
+TestLockFileLock*
+TestLFLock_new() {
+ return (TestLockFileLock*)Class_Make_Obj(TESTLOCKFILELOCK);
+}
+
+static Folder*
+S_create_fs_folder() {
+ Folder *folder = (Folder*)FSFolder_new(SSTR_WRAP_C("_locktest"));
+ Folder_Initialize(folder);
+ return folder;
+}
+
+static void
+S_destroy_fs_folder(Folder *folder) {
+ Folder_Delete_Tree(folder, SSTR_WRAP_C("locks"));
+ rmdir("_locktest");
+ DECREF(folder);
+}
+
+static void
+test_exclusive_only(TestBatchRunner *runner, Lock *lock1, Lock *lock2,
+ const char *tag) {
+ TEST_TRUE(runner, Lock_Request_Exclusive(lock1),
+ "Request_Exclusive (only) succeeds %s", tag);
+ Err_set_error(NULL);
+ TEST_FALSE(runner, Lock_Request_Exclusive(lock2),
+ "Request_Exclusive (only) after Request_Exclusive fails %s",
+ tag);
+ TEST_TRUE(runner, CERTIFY(Err_get_error(), LOCKERR),
+ "Request_Exclusive (only) after Request_Exclusive sets"
+ " LockErr %s", tag);
+ Lock_Release(lock1);
+
+ TEST_TRUE(runner, Lock_Request_Exclusive(lock2),
+ "Request_Exclusive (only) succeeds after Release %s", tag);
+ TEST_FALSE(runner, Lock_Request_Exclusive(lock1),
+ "Request_Exclusive (only) fails if lock2 is locked %s", tag);
+ Lock_Release(lock2);
+
+ DECREF(lock2);
+ DECREF(lock1);
+}
+
+static void
+test_lock(TestBatchRunner *runner, Lock *lock1, Lock *lock2, Lock *lock3,
+ const char *tag) {
+ TEST_TRUE(runner, Lock_Request_Shared(lock1), "Request_Shared succeeds %s",
+ tag);
+ TEST_TRUE(runner, Lock_Request_Shared(lock2),
+ "Request_Shared on another lock succeeds %s", tag);
+ Err_set_error(NULL);
+ TEST_FALSE(runner, Lock_Request_Exclusive(lock3),
+ "Request_Exclusive after Request_Shared fails %s", tag);
+ TEST_TRUE(runner, CERTIFY(Err_get_error(), LOCKERR),
+ "Request_Exclusive after Request_Shared sets LockErr %s", tag);
+ Lock_Release(lock1);
+ Lock_Release(lock2);
+
+ TEST_TRUE(runner, Lock_Request_Exclusive(lock1),
+ "Request_Exclusive succeeds %s", tag);
+ Err_set_error(NULL);
+ TEST_FALSE(runner, Lock_Request_Shared(lock2),
+ "Request_Shared after Request_Exclusive fails %s", tag);
+ TEST_TRUE(runner, CERTIFY(Err_get_error(), LOCKERR),
+ "Request_Shared after Request_Exclusive sets LockErr %s", tag);
+ Err_set_error(NULL);
+ TEST_FALSE(runner, Lock_Request_Exclusive(lock3),
+ "Request_Exclusive after Request_Exclusive fails %s", tag);
+ TEST_TRUE(runner, CERTIFY(Err_get_error(), LOCKERR),
+ "Request_Exclusive after Request_Exclusive sets LockErr %s",
+ tag);
+ Lock_Release(lock1);
+
+ TEST_TRUE(runner, Lock_Request_Exclusive(lock3),
+ "Request_Exclusive succeeds after Release %s", tag);
+ Lock_Release(lock3);
+
+ DECREF(lock3);
+ DECREF(lock2);
+ DECREF(lock1);
+}
+
+static void
+test_change_pid(TestBatchRunner *runner, Folder *folder, String *path,
+ const char *tag) {
+ Hash *hash = (Hash*)Json_slurp_json(folder, path);
+ TEST_TRUE(runner, CERTIFY(hash, HASH), "Lock file %s exists %s",
+ Str_Get_Ptr8(path), tag);
+ Hash_Store(hash, SSTR_WRAP_C("pid"), (Obj*)Str_newf("10000000"));
+ Folder_Delete(folder, path);
+ Json_spew_json((Obj*)hash, folder, path);
+ DECREF(hash);
+}
+
+static void
+test_stale(TestBatchRunner *runner, Folder *folder, const char *tag) {
+ String *name = SSTR_WRAP_C("test");
+ String *host1 = SSTR_WRAP_C("fuchur");
+ String *host2 = SSTR_WRAP_C("drogon");
+ LockFileLock *lock1 = LFLock_new(folder, name, host1, 0, 100, false);
+ LockFileLock *lock2 = LFLock_new(folder, name, host2, 0, 100, false);
+ LockFileLock *tmp;
+
+ tmp = LFLock_new(folder, name, host1, 0, 100, false);
+ LFLock_Request_Exclusive(tmp);
+ DECREF(tmp);
+ test_change_pid(runner, folder, SSTR_WRAP_C("locks/test.lock"), tag);
+ TEST_FALSE(runner, LFLock_Request_Exclusive(lock2),
+ "Lock_Exclusive fails with other host's exclusive lock %s",
+ tag);
+ TEST_TRUE(runner, LFLock_Request_Exclusive(lock1),
+ "Lock_Exclusive succeeds with stale exclusive lock %s", tag);
+ LFLock_Release(lock1);
+
+ tmp = LFLock_new(folder, name, host1, 0, 100, false);
+ LFLock_Request_Shared(tmp);
+ DECREF(tmp);
+ test_change_pid(runner, folder, SSTR_WRAP_C("locks/test-1.lock"), tag);
+ TEST_FALSE(runner, LFLock_Request_Exclusive(lock2),
+ "Lock_Exclusive fails with other host's shared lock %s", tag);
+ TEST_TRUE(runner, LFLock_Request_Exclusive(lock1),
+ "Lock_Exclusive succeeds with stale shared files %s", tag);
+ LFLock_Release(lock1);
+
+ DECREF(lock2);
+ DECREF(lock1);
+}
+
+static void
+test_Obtain(TestBatchRunner *runner, Lock *lock1, Lock *lock2,
+ const char *tag) {
+ TEST_TRUE(runner, Lock_Obtain_Shared(lock1), "Obtain_Shared succeeds %s",
+ tag);
+ TEST_FALSE(runner, Lock_Obtain_Exclusive(lock2),
+ "Obtain_Exclusive after Obtain_Shared fails %s", tag);
+ Lock_Release(lock1);
+
+ TEST_TRUE(runner, Lock_Obtain_Exclusive(lock1),
+ "Obtain_Exclusive succeeds %s", tag);
+ TEST_FALSE(runner, Lock_Obtain_Shared(lock2),
+ "Obtain_Shared after Obtain_Exclusive fails %s", tag);
+ Lock_Release(lock1);
+
+ DECREF(lock2);
+ DECREF(lock1);
+}
+
+static void
+S_try_request_shared(void *context) {
+ Lock_Request_Shared((Lock*)context);
+}
+
+static void
+S_try_request_exclusive(void *context) {
+ Lock_Request_Exclusive((Lock*)context);
+}
+
+static void
+S_try_release(void *context) {
+ Lock_Release((Lock*)context);
+}
+
+static void
+test_double_request_release(TestBatchRunner *runner, Lock *lock,
+ const char *tag) {
+ Err *err;
+
+ Lock_Request_Shared(lock);
+ err = Err_trap(S_try_request_shared, lock);
+ TEST_TRUE(runner, err, "Request_Shared/Request_Shared fails %s",
+ tag);
+ DECREF(err);
+ Lock_Release(lock);
+
+ Lock_Request_Exclusive(lock);
+ err = Err_trap(S_try_request_shared, lock);
+ TEST_TRUE(runner, err, "Request_Shared/Request_Exclusive fails %s",
+ tag);
+ DECREF(err);
+ Lock_Release(lock);
+
+ Lock_Request_Shared(lock);
+ err = Err_trap(S_try_request_exclusive, lock);
+ TEST_TRUE(runner, err, "Request_Exclusive/Request_Shared fails %s",
+ tag);
+ DECREF(err);
+ Lock_Release(lock);
+
+ Lock_Request_Exclusive(lock);
+ err = Err_trap(S_try_request_exclusive, lock);
+ TEST_TRUE(runner, err,
+ "Request_Exclusive/Request_Exclusive fails %s", tag);
+ DECREF(err);
+ Lock_Release(lock);
+
+ err = Err_trap(S_try_release, lock);
+ TEST_TRUE(runner, err, "Double Release fails");
+ DECREF(err);
+
+ DECREF(lock);
+}
+
+static void
+test_empty_lock_folder(TestBatchRunner *runner, Folder *folder,
+ const char *name, const char *tag) {
+ Vector *files = Folder_List_R(folder, SSTR_WRAP_C("locks"));
+ TEST_TRUE(runner, files, "Lock folder exists after %s tests %s", name,
+ tag);
+ TEST_UINT_EQ(runner, Vec_Get_Size(files), 0,
+ "Lock folder is empty after %s tests %s", name, tag);
+ DECREF(files);
+}
+
+static void
+test_lf_lock_with_folder(TestBatchRunner *runner, Folder *folder,
+ const char *tag) {
+ String *name = SSTR_WRAP_C("test");
+ String *host = SSTR_WRAP_C("fuchur");
+ LockFileLock *lock1, *lock2, *lock3;
+
+ lock1 = LFLock_new(folder, name, host, 0, 100, true);
+ lock2 = LFLock_new(folder, name, host, 0, 100, true);
+ test_exclusive_only(runner, (Lock*)lock1, (Lock*)lock2, tag);
+ test_empty_lock_folder(runner, folder, "exclusive only", tag);
+
+ lock1 = LFLock_new(folder, name, host, 0, 100, false);
+ lock2 = LFLock_new(folder, name, host, 0, 100, false);
+ lock3 = LFLock_new(folder, name, host, 0, 100, false);
+ test_lock(runner, (Lock*)lock1, (Lock*)lock2, (Lock*)lock3, tag);
+ test_empty_lock_folder(runner, folder, "lock", tag);
+
+ lock1 = LFLock_new(folder, name, host, 10, 1, false);
+ lock2 = LFLock_new(folder, name, host, 10, 1, false);
+ test_Obtain(runner, (Lock*)lock1, (Lock*)lock2, tag);
+ test_empty_lock_folder(runner, folder, "Obtain", tag);
+
+ lock1 = LFLock_new(folder, name, host, 0, 100, false);
+ test_double_request_release(runner, (Lock*)lock1, tag);
+ test_empty_lock_folder(runner, folder, "double", tag);
+
+ test_stale(runner, folder, tag);
+ test_empty_lock_folder(runner, folder, "stale", tag);
+}
+
+static void
+test_lf_lock(TestBatchRunner *runner) {
+ Folder *fs_folder = S_create_fs_folder();
+ test_lf_lock_with_folder(runner, fs_folder, "(FSFolder)");
+ S_destroy_fs_folder(fs_folder);
+
+ Folder *ram_folder = (Folder*)RAMFolder_new(NULL);
+ test_lf_lock_with_folder(runner, ram_folder, "(RAMFolder)");
+ DECREF(ram_folder);
+}
+
+void
+TestLFLock_Run_IMP(TestLockFileLock *self, TestBatchRunner *runner) {
+ TestBatchRunner_Plan(runner, (TestBatch*)self, 80);
+ test_lf_lock(runner);
+}
+
http://git-wip-us.apache.org/repos/asf/lucy/blob/35388cd2/test/Lucy/Test/Store/TestLock.cfh
----------------------------------------------------------------------
diff --git a/test/Lucy/Test/Store/TestLock.cfh b/test/Lucy/Test/Store/TestLock.cfh
new file mode 100644
index 0000000..b87cb7d
--- /dev/null
+++ b/test/Lucy/Test/Store/TestLock.cfh
@@ -0,0 +1,29 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel TestLucy;
+
+class Lucy::Test::Store::TestLockFileLock nickname TestLFLock
+ inherits Clownfish::TestHarness::TestBatch {
+
+ inert incremented TestLockFileLock*
+ new();
+
+ void
+ Run(TestLockFileLock *self, TestBatchRunner *runner);
+}
+
+
[13/16] lucy git commit: Move host ivar from Lock to LockFileLock
Posted by nw...@apache.org.
Move host ivar from Lock to LockFileLock
Project: http://git-wip-us.apache.org/repos/asf/lucy/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucy/commit/2bd2bc61
Tree: http://git-wip-us.apache.org/repos/asf/lucy/tree/2bd2bc61
Diff: http://git-wip-us.apache.org/repos/asf/lucy/diff/2bd2bc61
Branch: refs/heads/master
Commit: 2bd2bc611325ee643f325ecfd11d5227319949a8
Parents: d23b560
Author: Nick Wellnhofer <we...@aevum.de>
Authored: Sun Feb 19 16:21:00 2017 +0100
Committer: Nick Wellnhofer <we...@aevum.de>
Committed: Mon Feb 20 16:51:32 2017 +0100
----------------------------------------------------------------------
core/Lucy/Store/Lock.c | 10 +++++-----
core/Lucy/Store/Lock.cfh | 6 +++---
2 files changed, 8 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucy/blob/2bd2bc61/core/Lucy/Store/Lock.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Store/Lock.c b/core/Lucy/Store/Lock.c
index c4542a6..e716099 100644
--- a/core/Lucy/Store/Lock.c
+++ b/core/Lucy/Store/Lock.c
@@ -30,8 +30,8 @@
#include "Lucy/Util/Sleep.h"
Lock*
-Lock_init(Lock *self, Folder *folder, String *name,
- String *host, int32_t timeout, int32_t interval) {
+Lock_init(Lock *self, Folder *folder, String *name, int32_t timeout,
+ int32_t interval) {
LockIVARS *const ivars = Lock_IVARS(self);
// Validate.
@@ -58,7 +58,6 @@ Lock_init(Lock *self, Folder *folder, String *name,
ivars->folder = (Folder*)INCREF(folder);
ivars->timeout = timeout;
ivars->name = Str_Clone(name);
- ivars->host = Str_Clone(host);
ivars->interval = interval;
// Derive.
@@ -71,7 +70,6 @@ void
Lock_Destroy_IMP(Lock *self) {
LockIVARS *const ivars = Lock_IVARS(self);
DECREF(ivars->folder);
- DECREF(ivars->host);
DECREF(ivars->name);
DECREF(ivars->lock_path);
SUPER_DESTROY(self, LOCK);
@@ -145,8 +143,9 @@ LockFileLock*
LFLock_init(LockFileLock *self, Folder *folder, String *name, String *host,
int32_t timeout, int32_t interval, bool exclusive_only) {
int pid = PID_getpid();
- Lock_init((Lock*)self, folder, name, host, timeout, interval);
+ Lock_init((Lock*)self, folder, name, timeout, interval);
LockFileLockIVARS *const ivars = LFLock_IVARS(self);
+ ivars->host = (String*)INCREF(host);
ivars->link_path = Str_newf("%o.%o.%i64", ivars->lock_path, host,
(int64_t)pid);
ivars->exclusive_only = exclusive_only;
@@ -455,6 +454,7 @@ void
LFLock_Destroy_IMP(LockFileLock *self) {
LockFileLockIVARS *const ivars = LFLock_IVARS(self);
if (ivars->state != LFLOCK_STATE_UNLOCKED) { LFLock_Release(self); }
+ DECREF(ivars->host);
DECREF(ivars->link_path);
SUPER_DESTROY(self, LOCKFILELOCK);
}
http://git-wip-us.apache.org/repos/asf/lucy/blob/2bd2bc61/core/Lucy/Store/Lock.cfh
----------------------------------------------------------------------
diff --git a/core/Lucy/Store/Lock.cfh b/core/Lucy/Store/Lock.cfh
index 6a4b719..11f31c0 100644
--- a/core/Lucy/Store/Lock.cfh
+++ b/core/Lucy/Store/Lock.cfh
@@ -32,7 +32,6 @@ abstract class Lucy::Store::Lock inherits Clownfish::Obj {
Folder *folder;
String *name;
String *lock_path;
- String *host;
int32_t timeout;
int32_t interval;
@@ -47,8 +46,8 @@ abstract class Lucy::Store::Lock inherits Clownfish::Obj {
* @param interval Time in milliseconds between retries.
*/
public inert Lock*
- init(Lock *self, Folder *folder, String *name,
- String *host, int32_t timeout = 0, int32_t interval = 100);
+ init(Lock *self, Folder *folder, String *name, int32_t timeout = 0,
+ int32_t interval = 100);
/** Call [](.Request_Shared) once per `interval` until [](.Request_Shared)
* returns success or the `timeout` has been reached.
@@ -103,6 +102,7 @@ abstract class Lucy::Store::Lock inherits Clownfish::Obj {
class Lucy::Store::LockFileLock nickname LFLock
inherits Lucy::Store::Lock {
+ String *host;
String *shared_lock_path;
String *link_path;
int state;
[03/16] lucy git commit: Always check for stale locks in Is_Locked
Posted by nw...@apache.org.
Always check for stale locks in Is_Locked
Note that checking for stale locks isn't necessary when requesting
shared locks.
Project: http://git-wip-us.apache.org/repos/asf/lucy/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucy/commit/1a8e02df
Tree: http://git-wip-us.apache.org/repos/asf/lucy/tree/1a8e02df
Diff: http://git-wip-us.apache.org/repos/asf/lucy/diff/1a8e02df
Branch: refs/heads/master
Commit: 1a8e02df0a0257e4ba70de4a41504507b0ba8a47
Parents: bb83d0d
Author: Nick Wellnhofer <we...@aevum.de>
Authored: Thu Feb 16 16:47:03 2017 +0100
Committer: Nick Wellnhofer <we...@aevum.de>
Committed: Mon Feb 20 16:26:21 2017 +0100
----------------------------------------------------------------------
core/Lucy/Index/BackgroundMerger.c | 2 --
core/Lucy/Index/FilePurger.c | 5 ----
core/Lucy/Index/Indexer.c | 1 -
core/Lucy/Index/PolyReader.c | 2 --
core/Lucy/Store/Lock.c | 48 +++++++++------------------------
core/Lucy/Store/Lock.cfh | 10 -------
go/build.go | 1 -
go/lucy/store.go | 7 -----
go/lucy/store_test.go | 5 ----
perl/t/106-locking.t | 1 -
perl/t/110-shared_lock.t | 5 ++--
11 files changed, 14 insertions(+), 73 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucy/blob/1a8e02df/core/Lucy/Index/BackgroundMerger.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/BackgroundMerger.c b/core/Lucy/Index/BackgroundMerger.c
index 661b9a6..e9a1ae8 100644
--- a/core/Lucy/Index/BackgroundMerger.c
+++ b/core/Lucy/Index/BackgroundMerger.c
@@ -529,7 +529,6 @@ static void
S_obtain_write_lock(BackgroundMerger *self) {
BackgroundMergerIVARS *const ivars = BGMerger_IVARS(self);
Lock *write_lock = IxManager_Make_Write_Lock(ivars->manager);
- Lock_Clear_Stale(write_lock);
if (Lock_Obtain_Exclusive(write_lock)) {
// Only assign if successful, otherwise DESTROY unlocks -- bad!
ivars->write_lock = write_lock;
@@ -543,7 +542,6 @@ static void
S_obtain_merge_lock(BackgroundMerger *self) {
BackgroundMergerIVARS *const ivars = BGMerger_IVARS(self);
Lock *merge_lock = IxManager_Make_Merge_Lock(ivars->manager);
- Lock_Clear_Stale(merge_lock);
if (Lock_Obtain_Exclusive(merge_lock)) {
// Only assign if successful, same rationale as above.
ivars->merge_lock = merge_lock;
http://git-wip-us.apache.org/repos/asf/lucy/blob/1a8e02df/core/Lucy/Index/FilePurger.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/FilePurger.c b/core/Lucy/Index/FilePurger.c
index 3980c8f..161f030 100644
--- a/core/Lucy/Index/FilePurger.c
+++ b/core/Lucy/Index/FilePurger.c
@@ -78,7 +78,6 @@ FilePurger_Purge_Snapshots_IMP(FilePurger *self, Snapshot *current) {
Lock *deletion_lock = IxManager_Make_Deletion_Lock(ivars->manager);
// Obtain deletion lock, purge files, release deletion lock.
- Lock_Clear_Stale(deletion_lock);
if (Lock_Obtain_Exclusive(deletion_lock)) {
Folder *folder = ivars->folder;
Hash *failures = Hash_new(16);
@@ -147,7 +146,6 @@ FilePurger_Purge_Aborted_Merge_IMP(FilePurger *self) {
IndexManager *manager = ivars->manager;
Lock *merge_lock = IxManager_Make_Merge_Lock(manager);
- Lock_Clear_Stale(merge_lock);
if (!Lock_Is_Locked_Exclusive(merge_lock)) {
Hash *merge_data = IxManager_Read_Merge_Data(manager);
Obj *cutoff = merge_data
@@ -213,9 +211,6 @@ S_discover_unused(FilePurger *self, Snapshot *current, Hash *spared,
// DON'T obtain the lock -- only see whether another
// entity holds a lock on the snapshot file.
- if (lock) {
- Lock_Clear_Stale(lock);
- }
if (lock && Lock_Is_Locked(lock)) {
// The snapshot file is locked, which means someone's using
// that version of the index -- protect all of its entries.
http://git-wip-us.apache.org/repos/asf/lucy/blob/1a8e02df/core/Lucy/Index/Indexer.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/Indexer.c b/core/Lucy/Index/Indexer.c
index 97763e3..2381b26 100644
--- a/core/Lucy/Index/Indexer.c
+++ b/core/Lucy/Index/Indexer.c
@@ -97,7 +97,6 @@ Indexer_init(Indexer *self, Schema *schema, Obj *index,
// Get a write lock for this folder.
Lock *write_lock = IxManager_Make_Write_Lock(ivars->manager);
- Lock_Clear_Stale(write_lock);
if (Lock_Obtain_Exclusive(write_lock)) {
// Only assign if successful, otherwise DESTROY unlocks -- bad!
ivars->write_lock = write_lock;
http://git-wip-us.apache.org/repos/asf/lucy/blob/1a8e02df/core/Lucy/Index/PolyReader.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/PolyReader.c b/core/Lucy/Index/PolyReader.c
index d4166c6..ea5e35f 100644
--- a/core/Lucy/Index/PolyReader.c
+++ b/core/Lucy/Index/PolyReader.c
@@ -470,7 +470,6 @@ static bool
S_obtain_deletion_lock(PolyReader *self) {
PolyReaderIVARS *const ivars = PolyReader_IVARS(self);
ivars->deletion_lock = IxManager_Make_Deletion_Lock(ivars->manager);
- Lock_Clear_Stale(ivars->deletion_lock);
if (!Lock_Obtain_Exclusive(ivars->deletion_lock)) {
DECREF(ivars->deletion_lock);
ivars->deletion_lock = NULL;
@@ -485,7 +484,6 @@ S_obtain_read_lock(PolyReader *self, String *snapshot_file_name) {
ivars->read_lock = IxManager_Make_Snapshot_Read_Lock(ivars->manager,
snapshot_file_name);
- Lock_Clear_Stale(ivars->read_lock);
if (!Lock_Obtain_Shared(ivars->read_lock)) {
DECREF(ivars->read_lock);
ivars->read_lock = NULL;
http://git-wip-us.apache.org/repos/asf/lucy/blob/1a8e02df/core/Lucy/Store/Lock.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Store/Lock.c b/core/Lucy/Store/Lock.c
index 0eb42da..6d39362 100644
--- a/core/Lucy/Store/Lock.c
+++ b/core/Lucy/Store/Lock.c
@@ -341,7 +341,8 @@ bool
LFLock_Is_Locked_Exclusive_IMP(LockFileLock *self) {
LockFileLockIVARS *const ivars = LFLock_IVARS(self);
- return Folder_Exists(ivars->folder, ivars->lock_path);
+ return Folder_Exists(ivars->folder, ivars->lock_path)
+ && !S_maybe_delete_file(ivars, ivars->lock_path, false, true);
}
bool
@@ -349,7 +350,9 @@ LFLock_Is_Locked_IMP(LockFileLock *self) {
LockFileLockIVARS *const ivars = LFLock_IVARS(self);
// Check for exclusive lock.
- if (Folder_Exists(ivars->folder, ivars->lock_path)) {
+ if (Folder_Exists(ivars->folder, ivars->lock_path)
+ && !S_maybe_delete_file(ivars, ivars->lock_path, false, true)
+ ) {
return true;
}
@@ -360,51 +363,24 @@ LFLock_Is_Locked_IMP(LockFileLock *self) {
return false;
}
+ bool locked = false;
DirHandle *dh = Folder_Open_Dir(ivars->folder, lock_dir_name);
if (!dh) { RETHROW(INCREF(Err_get_error())); }
while (DH_Next(dh)) {
String *entry = DH_Get_Entry(dh);
if (S_is_shared_lock_file(ivars, entry)) {
- DECREF(entry);
- DECREF(dh);
- return true;
+ String *candidate = Str_newf("%o/%o", lock_dir_name, entry);
+ if (!S_maybe_delete_file(ivars, candidate, false, true)) {
+ locked = true;
+ }
+ DECREF(candidate);
}
DECREF(entry);
}
DECREF(dh);
- return false;
-}
-
-void
-LFLock_Clear_Stale_IMP(LockFileLock *self) {
- LockFileLockIVARS *const ivars = LFLock_IVARS(self);
-
- if (ivars->shared_lock_path) {
- String *lock_dir_name = SSTR_WRAP_C("locks");
- if (!Folder_Find_Folder(ivars->folder, lock_dir_name)) {
- return;
- }
-
- DirHandle *dh = Folder_Open_Dir(ivars->folder, lock_dir_name);
- if (!dh) { RETHROW(INCREF(Err_get_error())); }
-
- while (DH_Next(dh)) {
- String *entry = DH_Get_Entry(dh);
- if (S_is_shared_lock_file(ivars, entry)) {
- String *candidate = Str_newf("%o/%o", lock_dir_name, entry);
- S_maybe_delete_file(ivars, candidate, false, true);
- DECREF(candidate);
- }
- DECREF(entry);
- }
-
- DECREF(dh);
- }
- else {
- S_maybe_delete_file(ivars, ivars->lock_path, false, true);
- }
+ return locked;
}
static bool
http://git-wip-us.apache.org/repos/asf/lucy/blob/1a8e02df/core/Lucy/Store/Lock.cfh
----------------------------------------------------------------------
diff --git a/core/Lucy/Store/Lock.cfh b/core/Lucy/Store/Lock.cfh
index 3f93dc0..598c777 100644
--- a/core/Lucy/Store/Lock.cfh
+++ b/core/Lucy/Store/Lock.cfh
@@ -112,13 +112,6 @@ public abstract class Lucy::Store::Lock inherits Clownfish::Obj {
public abstract bool
Is_Locked_Exclusive(Lock *self);
- /** Release all locks that meet the following three conditions: the lock
- * name matches, the host id matches, and the process id that the lock
- * was created under no longer identifies an active process.
- */
- public abstract void
- Clear_Stale(Lock *self);
-
String*
Get_Name(Lock *self);
@@ -159,9 +152,6 @@ class Lucy::Store::LockFileLock nickname LFLock
public bool
Is_Locked_Exclusive(LockFileLock *self);
- public void
- Clear_Stale(LockFileLock *self);
-
String*
Get_Lock_Path(LockFileLock *self);
http://git-wip-us.apache.org/repos/asf/lucy/blob/1a8e02df/go/build.go
----------------------------------------------------------------------
diff --git a/go/build.go b/go/build.go
index 16c31f2..d64cbd2 100644
--- a/go/build.go
+++ b/go/build.go
@@ -458,7 +458,6 @@ func specClasses(parcel *cfc.Parcel) {
lockBinding.SpecMethod("Obtain_Shared", "ObtainShared() error")
lockBinding.SpecMethod("Obtain_Exclusive", "ObtainExclusive() error")
lockBinding.SpecMethod("Release", "Release() error")
- lockBinding.SpecMethod("Clear_Stale", "ClearStale() error")
lockBinding.Register()
cfWriterBinding := cfc.NewGoClass(parcel, "Lucy::Store::CompoundFileWriter")
http://git-wip-us.apache.org/repos/asf/lucy/blob/1a8e02df/go/lucy/store.go
----------------------------------------------------------------------
diff --git a/go/lucy/store.go b/go/lucy/store.go
index 9b9d8c3..7f00a94 100644
--- a/go/lucy/store.go
+++ b/go/lucy/store.go
@@ -766,13 +766,6 @@ func (lock *LockIMP) Release() error {
}
-func (lock *LockIMP) ClearStale() error {
- return clownfish.TrapErr(func() {
- self := (*C.lucy_Lock)(clownfish.Unwrap(lock, "lock"))
- C.LUCY_Lock_Clear_Stale(self)
- })
-}
-
func OpenCompoundFileReader(folder Folder) (reader CompoundFileReader, err error) {
err = clownfish.TrapErr(func() {
folderC := (*C.lucy_Folder)(clownfish.Unwrap(folder, "Folder"))
http://git-wip-us.apache.org/repos/asf/lucy/blob/1a8e02df/go/lucy/store_test.go
----------------------------------------------------------------------
diff --git a/go/lucy/store_test.go b/go/lucy/store_test.go
index 680b9b2..ec51244 100644
--- a/go/lucy/store_test.go
+++ b/go/lucy/store_test.go
@@ -710,11 +710,6 @@ func TestLockFileLockAll(t *testing.T) {
t.Errorf("Obtain: %v", err)
}
lock.Release()
-
- err = lock.ClearStale()
- if err != nil {
- t.Errorf("Nothing for ClearStale to do, but should still suceed: %v", err)
- }
}
func TestLockFactoryAll(t *testing.T) {
http://git-wip-us.apache.org/repos/asf/lucy/blob/1a8e02df/perl/t/106-locking.t
----------------------------------------------------------------------
diff --git a/perl/t/106-locking.t b/perl/t/106-locking.t
index 3a1670c..15c6a3a 100644
--- a/perl/t/106-locking.t
+++ b/perl/t/106-locking.t
@@ -50,7 +50,6 @@ Dead_locks_are_removed: {
exclusive_only => 1,
@_
);
- $lock->clear_stale;
$lock->obtain_exclusive() or die "no dice";
return $lock;
}
http://git-wip-us.apache.org/repos/asf/lucy/blob/1a8e02df/perl/t/110-shared_lock.t
----------------------------------------------------------------------
diff --git a/perl/t/110-shared_lock.t b/perl/t/110-shared_lock.t
index 5281c9f..77c6862 100644
--- a/perl/t/110-shared_lock.t
+++ b/perl/t/110-shared_lock.t
@@ -74,9 +74,8 @@ $lock->release;
$another_lock->release;
$ya_lock->release;
-ok( $lock->is_locked(), "failed to release a lock with a different pid" );
-$lock->clear_stale;
-ok( !$lock->is_locked(), "clear_stale" );
+ok( $lock->get_lock_path, "failed to release a lock with a different pid" );
+ok( !$lock->is_locked(), "is_locked clears stale locks" );
ok( $lock->obtain_shared(), "got lock again" );
ok( $lock->is_locked(), "it's locked" );
[10/16] lucy git commit: Change lock acquitision order in
BGMerger_init
Posted by nw...@apache.org.
Change lock acquitision order in BGMerger_init
Obtain the merge lock before the write lock so that an already running
background merger won't abort when failing to acquire the write lock
during commit.
Release the merge lock at the very end of BGMerger_Commit. It doesn't
really matter when we release the merge lock after we acquired the
write lock in BGMerger_Prepare_Commit. I think it makes the code
clearer to release it at the very end of BGMerger_Commit.
Project: http://git-wip-us.apache.org/repos/asf/lucy/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucy/commit/de14d4ca
Tree: http://git-wip-us.apache.org/repos/asf/lucy/tree/de14d4ca
Diff: http://git-wip-us.apache.org/repos/asf/lucy/diff/de14d4ca
Branch: refs/heads/master
Commit: de14d4ca31ab27fc1569966c1b182fa4829afd40
Parents: 4e6e5bb
Author: Nick Wellnhofer <we...@aevum.de>
Authored: Fri Feb 17 18:34:31 2017 +0100
Committer: Nick Wellnhofer <we...@aevum.de>
Committed: Mon Feb 20 16:26:22 2017 +0100
----------------------------------------------------------------------
core/Lucy/Index/BackgroundMerger.c | 18 ++++++++++--------
1 file changed, 10 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucy/blob/de14d4ca/core/Lucy/Index/BackgroundMerger.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/BackgroundMerger.c b/core/Lucy/Index/BackgroundMerger.c
index 144139d..be69d90 100644
--- a/core/Lucy/Index/BackgroundMerger.c
+++ b/core/Lucy/Index/BackgroundMerger.c
@@ -88,14 +88,16 @@ BGMerger_init(BackgroundMerger *self, Obj *index, IndexManager *manager) {
}
IxManager_Set_Folder(ivars->manager, folder);
- // Obtain write lock (which we'll only hold briefly), then merge lock.
- S_obtain_write_lock(self);
- if (!ivars->write_lock) {
+ // Obtain merge lock first so that a running background merger won't
+ // abort when failing to acquire the write lock during commit.
+ S_obtain_merge_lock(self);
+ if (!ivars->merge_lock) {
DECREF(self);
RETHROW(INCREF(Err_get_error()));
}
- S_obtain_merge_lock(self);
- if (!ivars->merge_lock) {
+ // Obtain write lock (which we'll only hold briefly).
+ S_obtain_write_lock(self);
+ if (!ivars->write_lock) {
DECREF(self);
RETHROW(INCREF(Err_get_error()));
}
@@ -512,8 +514,7 @@ BGMerger_Commit_IMP(BackgroundMerger *self) {
DECREF(temp_snapfile);
}
- // Release the merge lock and remove the merge data file.
- S_release_merge_lock(self);
+ // Remove the merge data file.
IxManager_Remove_Merge_Data(ivars->manager);
if (ivars->needs_commit) {
@@ -521,8 +522,9 @@ BGMerger_Commit_IMP(BackgroundMerger *self) {
FilePurger_Purge_Snapshots(ivars->file_purger, ivars->snapshot);
}
- // Release the write lock.
+ // Release write and merge locks.
S_release_write_lock(self);
+ S_release_merge_lock(self);
}
static void
[09/16] lucy git commit: Don't hide I/O errors behind LockErr
Posted by nw...@apache.org.
Don't hide I/O errors behind LockErr
Only return a LockErr if there's lock contention, not on unrelated
I/O errors.
Project: http://git-wip-us.apache.org/repos/asf/lucy/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucy/commit/e06cdbf6
Tree: http://git-wip-us.apache.org/repos/asf/lucy/tree/e06cdbf6
Diff: http://git-wip-us.apache.org/repos/asf/lucy/diff/e06cdbf6
Branch: refs/heads/master
Commit: e06cdbf63d3036c3a33e5265444d827065539e92
Parents: 38a33ba
Author: Nick Wellnhofer <we...@aevum.de>
Authored: Fri Feb 17 20:26:13 2017 +0100
Committer: Nick Wellnhofer <we...@aevum.de>
Committed: Mon Feb 20 16:26:22 2017 +0100
----------------------------------------------------------------------
core/Lucy/Store/Lock.c | 26 +++++++++++---------------
1 file changed, 11 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucy/blob/e06cdbf6/core/Lucy/Store/Lock.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Store/Lock.c b/core/Lucy/Store/Lock.c
index 0879640..90a6aa0 100644
--- a/core/Lucy/Store/Lock.c
+++ b/core/Lucy/Store/Lock.c
@@ -263,16 +263,14 @@ S_request(LockFileLockIVARS *ivars, String *lock_path) {
String *lock_dir_name = SSTR_WRAP_C("locks");
if (!Folder_Exists(ivars->folder, lock_dir_name)) {
if (!Folder_MkDir(ivars->folder, lock_dir_name)) {
- Err *mkdir_err = (Err*)CERTIFY(Err_get_error(), ERR);
- LockErr *err = LockErr_new(Str_newf("Can't create 'locks' directory: %o",
- Err_Get_Mess(mkdir_err)));
+ Err *err = (Err*)INCREF(Err_get_error());
// Maybe our attempt failed because another process succeeded.
if (Folder_Find_Folder(ivars->folder, lock_dir_name)) {
DECREF(err);
}
else {
// Nope, everything failed, so bail out.
- Err_set_error((Err*)err);
+ Err_set_error(err);
return false;
}
}
@@ -302,25 +300,23 @@ S_request(LockFileLockIVARS *ivars, String *lock_path) {
context.outstream = outstream;
context.json = json;
Err *json_error = Err_trap(S_write_lockfile_json, &context);
- bool wrote_json = !json_error;
DECREF(outstream);
DECREF(json);
- if (wrote_json) {
+ if (json_error) {
+ Err_set_error(json_error);
+ }
+ else {
success = Folder_Hard_Link(ivars->folder, ivars->link_path,
lock_path);
if (!success) {
+ // TODO: Only return a LockErr if errno == EEXIST, otherwise
+ // return a normal Err.
Err *hard_link_err = (Err*)CERTIFY(Err_get_error(), ERR);
- Err_set_error((Err*)LockErr_new(Str_newf("Failed to obtain lock at '%o': %o",
- lock_path,
- Err_Get_Mess(hard_link_err))));
+ String *msg = Str_newf("Failed to obtain lock at '%o': %o",
+ lock_path, Err_Get_Mess(hard_link_err));
+ Err_set_error((Err*)LockErr_new(msg));
}
}
- else {
- Err_set_error((Err*)LockErr_new(Str_newf("Failed to obtain lock at '%o': %o",
- lock_path,
- Err_Get_Mess(json_error))));
- DECREF(json_error);
- }
// Verify that our temporary file got zapped.
bool deletion_failed = !Folder_Delete(ivars->folder, ivars->link_path);
[16/16] lucy git commit: Merge branch 'improve-locking'
Posted by nw...@apache.org.
Merge branch 'improve-locking'
Fixes LUCY-323.
- Major rewrite of FilePurger, making it do less work.
- Eliminate global deletion lock, making PolyReader and FilePurger
never wait for a lock.
- Redesign Lock API in preparation for native locks.
- Remove LockFactory.
- Fix locking bug with non-default merge lock timeout.
- Fix locking bug with concurrent BackgroundMergers.
- Port some tests to C.
Project: http://git-wip-us.apache.org/repos/asf/lucy/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucy/commit/d7feb997
Tree: http://git-wip-us.apache.org/repos/asf/lucy/tree/d7feb997
Diff: http://git-wip-us.apache.org/repos/asf/lucy/diff/d7feb997
Branch: refs/heads/master
Commit: d7feb997003ae1fc84a348e8584ea97b69b02d95
Parents: 7a64a28 a3cf33f
Author: Nick Wellnhofer <we...@aevum.de>
Authored: Sun Apr 16 12:15:01 2017 +0200
Committer: Nick Wellnhofer <we...@aevum.de>
Committed: Sun Apr 16 12:15:01 2017 +0200
----------------------------------------------------------------------
core/Lucy/Docs/Cookbook/FastUpdates.md | 2 +-
core/Lucy/Docs/FileLocking.md | 5 +-
core/Lucy/Index/BackgroundMerger.c | 34 +-
core/Lucy/Index/FilePurger.c | 328 ++++++++-----------
core/Lucy/Index/FilePurger.cfh | 19 +-
core/Lucy/Index/IndexManager.c | 88 +----
core/Lucy/Index/IndexManager.cfh | 42 +--
core/Lucy/Index/IndexReader.c | 16 +-
core/Lucy/Index/IndexReader.cfh | 3 +-
core/Lucy/Index/Indexer.c | 61 ++--
core/Lucy/Index/PolyReader.c | 124 +++----
core/Lucy/Store/Lock.c | 258 ++-------------
core/Lucy/Store/Lock.cfh | 117 ++-----
core/Lucy/Store/LockFactory.c | 66 ----
core/Lucy/Store/LockFactory.cfh | 80 -----
core/Lucy/Store/LockFileLock.c | 363 +++++++++++++++++++++
core/Lucy/Store/LockFileLock.cfh | 49 +++
core/Lucy/Store/SharedLock.c | 173 ----------
core/Lucy/Store/SharedLock.cfh | 77 -----
go/build.go | 7 +-
go/lucy/index_test.go | 32 +-
go/lucy/store.go | 35 +-
go/lucy/store_test.go | 77 ++---
perl/buildlib/Lucy/Build/Binding/Store.pm | 100 ------
perl/lib/Lucy/Store/Lock.pm | 25 --
perl/lib/Lucy/Store/LockFactory.pm | 25 --
perl/t/105-folder.t | 29 +-
perl/t/106-locking.t | 89 -----
perl/t/109-read_locking.t | 53 ++-
perl/t/110-shared_lock.t | 90 -----
perl/t/111-index_manager.t | 32 +-
perl/t/core/106-lf_lock.t | 23 ++
perl/t/core/233-background_merger.t | 23 ++
test/Lucy/Test.c | 4 +
test/Lucy/Test/Index/NoMergeManager.c | 42 +++
test/Lucy/Test/Index/NoMergeManager.cfh | 30 ++
test/Lucy/Test/Index/TestBackgroundMerger.c | 208 ++++++++++++
test/Lucy/Test/Index/TestBackgroundMerger.cfh | 29 ++
test/Lucy/Test/Index/TestIndexManager.c | 2 +-
test/Lucy/Test/Index/TestSortWriter.c | 28 +-
test/Lucy/Test/Index/TestSortWriter.cfh | 15 -
test/Lucy/Test/Store/TestLock.c | 296 +++++++++++++++++
test/Lucy/Test/Store/TestLock.cfh | 29 ++
test/Lucy/Test/TestUtils.c | 17 +-
44 files changed, 1531 insertions(+), 1714 deletions(-)
----------------------------------------------------------------------
[08/16] lucy git commit: Improve merge lock handling
Posted by nw...@apache.org.
Improve merge lock handling
Don't check the merge lock with Is_Locked in
FilePurger_Purge_Aborted_Merge. This prevented background mergers from
cleaning up because they already held the merge lock. Require that
callers hold the merge lock instead.
Don't check the merge lock with Is_Locked in Indexer_init. Since we
hold the write lock, and the merge.json file from aborted merge has
been removed, the presence of a merge.json file is a reliable indicator
that it belongs to an active background merger. Move the merge lock
request form S_maybe_merge to Indexer_init.
Also, S_maybe_merge used to Obtain the merge lock instead of merely
Requesting it. Waiting for a potentially long-running background merger
slows things down unnecessarily and increases the chance for write lock
contention. This didn't affect the default configuration which has a
zero merge lock timeout.
Project: http://git-wip-us.apache.org/repos/asf/lucy/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucy/commit/4e6e5bbf
Tree: http://git-wip-us.apache.org/repos/asf/lucy/tree/4e6e5bbf
Diff: http://git-wip-us.apache.org/repos/asf/lucy/diff/4e6e5bbf
Branch: refs/heads/master
Commit: 4e6e5bbf2814e8e2a824b5ae1b91d068233cd248
Parents: 5270b98
Author: Nick Wellnhofer <we...@aevum.de>
Authored: Fri Feb 17 18:25:35 2017 +0100
Committer: Nick Wellnhofer <we...@aevum.de>
Committed: Mon Feb 20 16:26:22 2017 +0100
----------------------------------------------------------------------
core/Lucy/Index/FilePurger.c | 53 ++++++++++++++++---------------------
core/Lucy/Index/FilePurger.cfh | 6 +++--
core/Lucy/Index/Indexer.c | 39 ++++++++++++---------------
3 files changed, 44 insertions(+), 54 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucy/blob/4e6e5bbf/core/Lucy/Index/FilePurger.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/FilePurger.c b/core/Lucy/Index/FilePurger.c
index 433dec3..5795bb0 100644
--- a/core/Lucy/Index/FilePurger.c
+++ b/core/Lucy/Index/FilePurger.c
@@ -139,44 +139,37 @@ FilePurger_Purge_Snapshots_IMP(FilePurger *self, Snapshot *current) {
void
FilePurger_Purge_Aborted_Merge_IMP(FilePurger *self) {
FilePurgerIVARS *const ivars = FilePurger_IVARS(self);
- IndexManager *manager = ivars->manager;
- Lock *merge_lock = IxManager_Make_Merge_Lock(manager);
-
- if (!Lock_Is_Locked_Exclusive(merge_lock)) {
- Hash *merge_data = IxManager_Read_Merge_Data(manager);
- Obj *cutoff = merge_data
- ? Hash_Fetch_Utf8(merge_data, "cutoff", 6)
- : NULL;
-
- if (cutoff) {
- Folder *folder = ivars->folder;
-
- String *cutoff_seg = Seg_num_to_name(Json_obj_to_i64(cutoff));
- if (Folder_Local_Exists(folder, cutoff_seg)) {
- if (!S_delete_entry(folder, cutoff_seg)) {
- if (Folder_Local_Exists(folder, cutoff_seg)) {
- WARN("Couldn't delete '%o' from aborted merge",
- cutoff_seg);
- }
- }
- }
-
- String *merge_json = SSTR_WRAP_C("merge.json");
- if (!Folder_Local_Delete(folder, merge_json)) {
- if (Folder_Local_Exists(folder, merge_json)) {
+ IndexManager *manager = ivars->manager;
+ Hash *merge_data = IxManager_Read_Merge_Data(manager);
+ Obj *cutoff = merge_data
+ ? Hash_Fetch_Utf8(merge_data, "cutoff", 6)
+ : NULL;
+
+ if (cutoff) {
+ Folder *folder = ivars->folder;
+
+ String *cutoff_seg = Seg_num_to_name(Json_obj_to_i64(cutoff));
+ if (Folder_Local_Exists(folder, cutoff_seg)) {
+ if (!S_delete_entry(folder, cutoff_seg)) {
+ if (Folder_Local_Exists(folder, cutoff_seg)) {
WARN("Couldn't delete '%o' from aborted merge",
- merge_json);
+ cutoff_seg);
}
}
+ }
- DECREF(cutoff_seg);
+ String *merge_json = SSTR_WRAP_C("merge.json");
+ if (!Folder_Local_Delete(folder, merge_json)) {
+ if (Folder_Local_Exists(folder, merge_json)) {
+ WARN("Couldn't delete '%o' from aborted merge",
+ merge_json);
+ }
}
- DECREF(merge_data);
+ DECREF(cutoff_seg);
}
- DECREF(merge_lock);
- return;
+ DECREF(merge_data);
}
static void
http://git-wip-us.apache.org/repos/asf/lucy/blob/4e6e5bbf/core/Lucy/Index/FilePurger.cfh
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/FilePurger.cfh b/core/Lucy/Index/FilePurger.cfh
index 4f3923d..c781170 100644
--- a/core/Lucy/Index/FilePurger.cfh
+++ b/core/Lucy/Index/FilePurger.cfh
@@ -30,12 +30,14 @@ class Lucy::Index::FilePurger inherits Clownfish::Obj {
inert FilePurger*
init(FilePurger *self, Folder *folder, IndexManager *manager = NULL);
- /** Purge obsolete files from the index.
+ /** Purge obsolete files from the index. The caller must hold the
+ * write lock.
*/
void
Purge_Snapshots(FilePurger *self, Snapshot *current);
- /** Purge files left behind by an aborted background merge.
+ /** Purge files left behind by an aborted background merge. The caller
+ * must hold the merge lock and the write lock.
*/
void
Purge_Aborted_Merge(FilePurger *self);
http://git-wip-us.apache.org/repos/asf/lucy/blob/4e6e5bbf/core/Lucy/Index/Indexer.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/Indexer.c b/core/Lucy/Index/Indexer.c
index be5519a..9190b93 100644
--- a/core/Lucy/Index/Indexer.c
+++ b/core/Lucy/Index/Indexer.c
@@ -132,6 +132,7 @@ Indexer_init(Indexer *self, Schema *schema, Obj *index,
schema_file = NULL;
}
else {
+ S_release_write_lock(self);
THROW(ERR, "Failed to parse %o", schema_file);
}
}
@@ -163,25 +164,25 @@ Indexer_init(Indexer *self, Schema *schema, Obj *index,
// Create new FilePurger and zap detritus from an aborted background
// merge.
ivars->file_purger = FilePurger_new(folder, ivars->manager);
- FilePurger_Purge_Aborted_Merge(ivars->file_purger);
+ Lock *merge_lock = IxManager_Make_Merge_Lock(ivars->manager);
+ if (Lock_Request_Exclusive(merge_lock)) {
+ // No background merger is running.
+ ivars->merge_lock = merge_lock;
+ FilePurger_Purge_Aborted_Merge(ivars->file_purger);
+ }
+ else {
+ DECREF(merge_lock);
+ }
// Create a new segment.
int64_t new_seg_num
= IxManager_Highest_Seg_Num(ivars->manager, latest_snapshot) + 1;
- Lock *merge_lock = IxManager_Make_Merge_Lock(ivars->manager);
- if (Lock_Is_Locked_Exclusive(merge_lock)) {
- // If there's a background merge process going on, stay out of its
- // way.
- Hash *merge_data = IxManager_Read_Merge_Data(ivars->manager);
- Obj *cutoff_obj = merge_data
- ? Hash_Fetch_Utf8(merge_data, "cutoff", 6)
- : NULL;
- if (!cutoff_obj) {
- DECREF(merge_lock);
- DECREF(merge_data);
- THROW(ERR, "Background merge detected, but can't read merge data");
- }
- else {
+ // If there's a background merge process going on, stay out of its
+ // way.
+ Hash *merge_data = IxManager_Read_Merge_Data(ivars->manager);
+ if (merge_data) {
+ Obj *cutoff_obj = Hash_Fetch_Utf8(merge_data, "cutoff", 6);
+ if (cutoff_obj) {
int64_t cutoff = Json_obj_to_i64(cutoff_obj);
if (cutoff >= new_seg_num) {
new_seg_num = cutoff + 1;
@@ -198,8 +199,6 @@ Indexer_init(Indexer *self, Schema *schema, Obj *index,
}
DECREF(fields);
- DECREF(merge_lock);
-
// Create new SegWriter.
ivars->seg_writer = SegWriter_new(ivars->schema, ivars->snapshot,
ivars->segment, ivars->polyreader);
@@ -396,12 +395,9 @@ S_maybe_merge(Indexer *self, Vector *seg_readers) {
IndexerIVARS *const ivars = Indexer_IVARS(self);
bool merge_happened = false;
size_t num_seg_readers = Vec_Get_Size(seg_readers);
- Lock *merge_lock = IxManager_Make_Merge_Lock(ivars->manager);
- bool got_merge_lock = Lock_Obtain_Exclusive(merge_lock);
int64_t cutoff;
- if (got_merge_lock) {
- ivars->merge_lock = merge_lock;
+ if (ivars->merge_lock) {
cutoff = 0;
}
else {
@@ -420,7 +416,6 @@ S_maybe_merge(Indexer *self, Vector *seg_readers) {
else {
cutoff = INT64_MAX;
}
- DECREF(merge_lock);
}
// Get a list of segments to recycle. Validate and confirm that there are
[12/16] lucy git commit: Release locks on destruction
Posted by nw...@apache.org.
Release locks on destruction
Project: http://git-wip-us.apache.org/repos/asf/lucy/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucy/commit/d23b560d
Tree: http://git-wip-us.apache.org/repos/asf/lucy/tree/d23b560d
Diff: http://git-wip-us.apache.org/repos/asf/lucy/diff/d23b560d
Branch: refs/heads/master
Commit: d23b560dc34d1c1cbc731fdc745b4e72324d8717
Parents: 35388cd
Author: Nick Wellnhofer <we...@aevum.de>
Authored: Sun Feb 19 14:09:02 2017 +0100
Committer: Nick Wellnhofer <we...@aevum.de>
Committed: Mon Feb 20 16:51:30 2017 +0100
----------------------------------------------------------------------
core/Lucy/Index/BackgroundMerger.c | 2 --
core/Lucy/Index/FilePurger.c | 7 +----
core/Lucy/Index/IndexReader.c | 6 +---
core/Lucy/Index/Indexer.c | 2 --
core/Lucy/Index/PolyReader.c | 1 -
core/Lucy/Store/Lock.c | 2 +-
core/Lucy/Store/Lock.cfh | 2 +-
test/Lucy/Test/Store/TestLock.c | 51 ++++++++++++++++-----------------
8 files changed, 29 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucy/blob/d23b560d/core/Lucy/Index/BackgroundMerger.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/BackgroundMerger.c b/core/Lucy/Index/BackgroundMerger.c
index be69d90..d326140 100644
--- a/core/Lucy/Index/BackgroundMerger.c
+++ b/core/Lucy/Index/BackgroundMerger.c
@@ -559,7 +559,6 @@ static void
S_release_write_lock(BackgroundMerger *self) {
BackgroundMergerIVARS *const ivars = BGMerger_IVARS(self);
if (ivars->write_lock) {
- Lock_Release(ivars->write_lock);
DECREF(ivars->write_lock);
ivars->write_lock = NULL;
}
@@ -569,7 +568,6 @@ static void
S_release_merge_lock(BackgroundMerger *self) {
BackgroundMergerIVARS *const ivars = BGMerger_IVARS(self);
if (ivars->merge_lock) {
- Lock_Release(ivars->merge_lock);
DECREF(ivars->merge_lock);
ivars->merge_lock = NULL;
}
http://git-wip-us.apache.org/repos/asf/lucy/blob/d23b560d/core/Lucy/Index/FilePurger.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/FilePurger.c b/core/Lucy/Index/FilePurger.c
index 5795bb0..211f0ea 100644
--- a/core/Lucy/Index/FilePurger.c
+++ b/core/Lucy/Index/FilePurger.c
@@ -123,17 +123,12 @@ FilePurger_Purge_Snapshots_IMP(FilePurger *self, Snapshot *current) {
}
}
- // Release snapshot locks.
- for (size_t i = 0, max = Vec_Get_Size(locks); i < max; i++) {
- Lock_Release((Lock*)Vec_Fetch(locks, i));
- }
-
DECREF(iter);
DECREF(failures);
DECREF(purged);
DECREF(spared);
DECREF(snapshots);
- DECREF(locks);
+ DECREF(locks); // Will release locks.
}
void
http://git-wip-us.apache.org/repos/asf/lucy/blob/d23b560d/core/Lucy/Index/IndexReader.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/IndexReader.c b/core/Lucy/Index/IndexReader.c
index dabb3be..ce3a205 100644
--- a/core/Lucy/Index/IndexReader.c
+++ b/core/Lucy/Index/IndexReader.c
@@ -81,7 +81,6 @@ IxReader_Close_IMP(IndexReader *self) {
Hash_Clear(ivars->components);
}
if (ivars->snapshot_lock) {
- Lock_Release(ivars->snapshot_lock);
DECREF(ivars->snapshot_lock);
ivars->snapshot_lock = NULL;
}
@@ -91,10 +90,7 @@ void
IxReader_Destroy_IMP(IndexReader *self) {
IndexReaderIVARS *const ivars = IxReader_IVARS(self);
DECREF(ivars->components);
- if (ivars->snapshot_lock) {
- Lock_Release(ivars->snapshot_lock);
- DECREF(ivars->snapshot_lock);
- }
+ DECREF(ivars->snapshot_lock);
DECREF(ivars->manager);
SUPER_DESTROY(self, INDEXREADER);
}
http://git-wip-us.apache.org/repos/asf/lucy/blob/d23b560d/core/Lucy/Index/Indexer.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/Indexer.c b/core/Lucy/Index/Indexer.c
index 9190b93..fc6afec 100644
--- a/core/Lucy/Index/Indexer.c
+++ b/core/Lucy/Index/Indexer.c
@@ -587,7 +587,6 @@ static void
S_release_write_lock(Indexer *self) {
IndexerIVARS *const ivars = Indexer_IVARS(self);
if (ivars->write_lock) {
- Lock_Release(ivars->write_lock);
DECREF(ivars->write_lock);
ivars->write_lock = NULL;
}
@@ -597,7 +596,6 @@ static void
S_release_merge_lock(Indexer *self) {
IndexerIVARS *const ivars = Indexer_IVARS(self);
if (ivars->merge_lock) {
- Lock_Release(ivars->merge_lock);
DECREF(ivars->merge_lock);
ivars->merge_lock = NULL;
}
http://git-wip-us.apache.org/repos/asf/lucy/blob/d23b560d/core/Lucy/Index/PolyReader.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/PolyReader.c b/core/Lucy/Index/PolyReader.c
index 6b9e943..572cc2b 100644
--- a/core/Lucy/Index/PolyReader.c
+++ b/core/Lucy/Index/PolyReader.c
@@ -473,7 +473,6 @@ static void
S_release_snapshot_lock(PolyReader *self) {
PolyReaderIVARS *const ivars = PolyReader_IVARS(self);
if (ivars->snapshot_lock) {
- Lock_Release(ivars->snapshot_lock);
DECREF(ivars->snapshot_lock);
ivars->snapshot_lock = NULL;
}
http://git-wip-us.apache.org/repos/asf/lucy/blob/d23b560d/core/Lucy/Store/Lock.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Store/Lock.c b/core/Lucy/Store/Lock.c
index 5cddcd9..c4542a6 100644
--- a/core/Lucy/Store/Lock.c
+++ b/core/Lucy/Store/Lock.c
@@ -454,7 +454,7 @@ S_maybe_delete_file(LockFileLockIVARS *ivars, String *path,
void
LFLock_Destroy_IMP(LockFileLock *self) {
LockFileLockIVARS *const ivars = LFLock_IVARS(self);
- DECREF(ivars->shared_lock_path);
+ if (ivars->state != LFLOCK_STATE_UNLOCKED) { LFLock_Release(self); }
DECREF(ivars->link_path);
SUPER_DESTROY(self, LOCKFILELOCK);
}
http://git-wip-us.apache.org/repos/asf/lucy/blob/d23b560d/core/Lucy/Store/Lock.cfh
----------------------------------------------------------------------
diff --git a/core/Lucy/Store/Lock.cfh b/core/Lucy/Store/Lock.cfh
index 30940b4..6a4b719 100644
--- a/core/Lucy/Store/Lock.cfh
+++ b/core/Lucy/Store/Lock.cfh
@@ -91,7 +91,7 @@ abstract class Lucy::Store::Lock inherits Clownfish::Obj {
public abstract bool
Request_Exclusive(Lock *self);
- /** Release the lock.
+ /** Release the lock. Locks are always released by the destructor.
*/
public abstract void
Release(Lock *self);
http://git-wip-us.apache.org/repos/asf/lucy/blob/d23b560d/test/Lucy/Test/Store/TestLock.c
----------------------------------------------------------------------
diff --git a/test/Lucy/Test/Store/TestLock.c b/test/Lucy/Test/Store/TestLock.c
index f606e7b..b52f366 100644
--- a/test/Lucy/Test/Store/TestLock.c
+++ b/test/Lucy/Test/Store/TestLock.c
@@ -71,9 +71,10 @@ test_exclusive_only(TestBatchRunner *runner, Lock *lock1, Lock *lock2,
"Request_Exclusive (only) succeeds after Release %s", tag);
TEST_FALSE(runner, Lock_Request_Exclusive(lock1),
"Request_Exclusive (only) fails if lock2 is locked %s", tag);
- Lock_Release(lock2);
-
DECREF(lock2);
+
+ TEST_TRUE(runner, Lock_Request_Exclusive(lock1),
+ "Request_Exclusive (only) succeeds after Destroy %s", tag);
DECREF(lock1);
}
@@ -105,27 +106,12 @@ test_lock(TestBatchRunner *runner, Lock *lock1, Lock *lock2, Lock *lock3,
TEST_TRUE(runner, CERTIFY(Err_get_error(), LOCKERR),
"Request_Exclusive after Request_Exclusive sets LockErr %s",
tag);
- Lock_Release(lock1);
-
- TEST_TRUE(runner, Lock_Request_Exclusive(lock3),
- "Request_Exclusive succeeds after Release %s", tag);
- Lock_Release(lock3);
-
- DECREF(lock3);
- DECREF(lock2);
DECREF(lock1);
-}
-static void
-test_change_pid(TestBatchRunner *runner, Folder *folder, String *path,
- const char *tag) {
- Hash *hash = (Hash*)Json_slurp_json(folder, path);
- TEST_TRUE(runner, CERTIFY(hash, HASH), "Lock file %s exists %s",
- Str_Get_Ptr8(path), tag);
- Hash_Store(hash, SSTR_WRAP_C("pid"), (Obj*)Str_newf("10000000"));
- Folder_Delete(folder, path);
- Json_spew_json((Obj*)hash, folder, path);
- DECREF(hash);
+ TEST_TRUE(runner, Lock_Request_Shared(lock2),
+ "Request_Shared succeeds after Destroy %s", tag);
+ DECREF(lock2);
+ DECREF(lock3);
}
static void
@@ -136,11 +122,19 @@ test_stale(TestBatchRunner *runner, Folder *folder, const char *tag) {
LockFileLock *lock1 = LFLock_new(folder, name, host1, 0, 100, false);
LockFileLock *lock2 = LFLock_new(folder, name, host2, 0, 100, false);
LockFileLock *tmp;
+ Hash *hash;
tmp = LFLock_new(folder, name, host1, 0, 100, false);
LFLock_Request_Exclusive(tmp);
+ String *ex_path = SSTR_WRAP_C("locks/test.lock");
+ hash = (Hash*)Json_slurp_json(folder, ex_path);
+ TEST_TRUE(runner, CERTIFY(hash, HASH), "Lock file %s exists %s",
+ Str_Get_Ptr8(ex_path), tag);
DECREF(tmp);
- test_change_pid(runner, folder, SSTR_WRAP_C("locks/test.lock"), tag);
+ // Write lock file with different pid.
+ Hash_Store(hash, SSTR_WRAP_C("pid"), (Obj*)Str_newf("10000000"));
+ Json_spew_json((Obj*)hash, folder, ex_path);
+ DECREF(hash);
TEST_FALSE(runner, LFLock_Request_Exclusive(lock2),
"Lock_Exclusive fails with other host's exclusive lock %s",
tag);
@@ -150,8 +144,15 @@ test_stale(TestBatchRunner *runner, Folder *folder, const char *tag) {
tmp = LFLock_new(folder, name, host1, 0, 100, false);
LFLock_Request_Shared(tmp);
+ String *sh_path = SSTR_WRAP_C("locks/test-1.lock");
+ hash = (Hash*)Json_slurp_json(folder, sh_path);
+ TEST_TRUE(runner, CERTIFY(hash, HASH), "Lock file %s exists %s",
+ Str_Get_Ptr8(sh_path), tag);
DECREF(tmp);
- test_change_pid(runner, folder, SSTR_WRAP_C("locks/test-1.lock"), tag);
+ // Write lock file with different pid.
+ Hash_Store(hash, SSTR_WRAP_C("pid"), (Obj*)Str_newf("10000000"));
+ Json_spew_json((Obj*)hash, folder, SSTR_WRAP_C("locks/test-98765.lock"));
+ DECREF(hash);
TEST_FALSE(runner, LFLock_Request_Exclusive(lock2),
"Lock_Exclusive fails with other host's shared lock %s", tag);
TEST_TRUE(runner, LFLock_Request_Exclusive(lock1),
@@ -175,8 +176,6 @@ test_Obtain(TestBatchRunner *runner, Lock *lock1, Lock *lock2,
"Obtain_Exclusive succeeds %s", tag);
TEST_FALSE(runner, Lock_Obtain_Shared(lock2),
"Obtain_Shared after Obtain_Exclusive fails %s", tag);
- Lock_Release(lock1);
-
DECREF(lock2);
DECREF(lock1);
}
@@ -291,7 +290,7 @@ test_lf_lock(TestBatchRunner *runner) {
void
TestLFLock_Run_IMP(TestLockFileLock *self, TestBatchRunner *runner) {
- TestBatchRunner_Plan(runner, (TestBatch*)self, 80);
+ TestBatchRunner_Plan(runner, (TestBatch*)self, 82);
test_lf_lock(runner);
}
[14/16] lucy git commit: Move LockFileLock to separate file
Posted by nw...@apache.org.
Move LockFileLock to separate file
Project: http://git-wip-us.apache.org/repos/asf/lucy/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucy/commit/6e8538a8
Tree: http://git-wip-us.apache.org/repos/asf/lucy/tree/6e8538a8
Diff: http://git-wip-us.apache.org/repos/asf/lucy/diff/6e8538a8
Branch: refs/heads/master
Commit: 6e8538a8834a49da7e8cce8b94decc130467b0f3
Parents: 2bd2bc6
Author: Nick Wellnhofer <we...@aevum.de>
Authored: Sun Feb 19 17:01:50 2017 +0100
Committer: Nick Wellnhofer <we...@aevum.de>
Committed: Mon Feb 20 16:51:32 2017 +0100
----------------------------------------------------------------------
core/Lucy/Index/IndexManager.c | 2 +-
core/Lucy/Store/Lock.c | 376 ++--------------------------------
core/Lucy/Store/Lock.cfh | 33 +--
core/Lucy/Store/LockFileLock.c | 363 ++++++++++++++++++++++++++++++++
core/Lucy/Store/LockFileLock.cfh | 49 +++++
test/Lucy/Test/Store/TestLock.c | 2 +-
6 files changed, 437 insertions(+), 388 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucy/blob/6e8538a8/core/Lucy/Index/IndexManager.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/IndexManager.c b/core/Lucy/Index/IndexManager.c
index bd3aa30..8e56509 100644
--- a/core/Lucy/Index/IndexManager.c
+++ b/core/Lucy/Index/IndexManager.c
@@ -25,7 +25,7 @@
#include "Lucy/Index/Snapshot.h"
#include "Lucy/Store/DirHandle.h"
#include "Lucy/Store/Folder.h"
-#include "Lucy/Store/Lock.h"
+#include "Lucy/Store/LockFileLock.h"
#include "Lucy/Util/IndexFileNames.h"
#include "Lucy/Util/Json.h"
#include "Lucy/Util/StringHelper.h"
http://git-wip-us.apache.org/repos/asf/lucy/blob/6e8538a8/core/Lucy/Store/Lock.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Store/Lock.c b/core/Lucy/Store/Lock.c
index e716099..7a4a060 100644
--- a/core/Lucy/Store/Lock.c
+++ b/core/Lucy/Store/Lock.c
@@ -15,18 +15,12 @@
*/
#define C_LUCY_LOCK
-#define C_LUCY_LOCKFILELOCK
#include "Lucy/Util/ToolSet.h"
-#include <stdio.h>
#include <ctype.h>
#include "Lucy/Store/Lock.h"
-#include "Lucy/Store/DirHandle.h"
#include "Lucy/Store/Folder.h"
-#include "Lucy/Store/OutStream.h"
-#include "Lucy/Util/Json.h"
-#include "Lucy/Util/ProcessID.h"
#include "Lucy/Util/Sleep.h"
Lock*
@@ -76,6 +70,26 @@ Lock_Destroy_IMP(Lock *self) {
}
bool
+Lock_make_lock_dir(Folder *folder) {
+ String *lock_dir_name = SSTR_WRAP_C("locks");
+
+ if (!Folder_MkDir(folder, lock_dir_name)) {
+ Err *err = (Err*)INCREF(Err_get_error());
+ // Maybe our attempt failed because another process succeeded.
+ if (Folder_Find_Folder(folder, lock_dir_name)) {
+ DECREF(err);
+ }
+ else {
+ // Nope, everything failed, so bail out.
+ Err_set_error(err);
+ return false;
+ }
+ }
+
+ return true;
+}
+
+bool
Lock_Obtain_Shared_IMP(Lock *self) {
LockIVARS *const ivars = Lock_IVARS(self);
int32_t time_left = ivars->interval == 0 ? 0 : ivars->timeout;
@@ -111,356 +125,6 @@ Lock_Obtain_Exclusive_IMP(Lock *self) {
/***************************************************************************/
-#define LFLOCK_STATE_UNLOCKED 0
-#define LFLOCK_STATE_LOCKED_SHARED 1
-#define LFLOCK_STATE_LOCKED_EXCLUSIVE 2
-
-static bool
-S_request(LockFileLockIVARS *ivars, String *lock_path);
-
-static bool
-S_is_locked_exclusive(LockFileLockIVARS *ivars);
-
-static bool
-S_is_locked(LockFileLockIVARS *ivars);
-
-static bool
-S_is_shared_lock_file(LockFileLockIVARS *ivars, String *entry);
-
-static bool
-S_maybe_delete_file(LockFileLockIVARS *ivars, String *path,
- bool delete_mine, bool delete_other);
-
-LockFileLock*
-LFLock_new(Folder *folder, String *name, String *host, int32_t timeout,
- int32_t interval, bool exclusive_only) {
- LockFileLock *self = (LockFileLock*)Class_Make_Obj(LOCKFILELOCK);
- return LFLock_init(self, folder, name, host, timeout, interval,
- exclusive_only);
-}
-
-LockFileLock*
-LFLock_init(LockFileLock *self, Folder *folder, String *name, String *host,
- int32_t timeout, int32_t interval, bool exclusive_only) {
- int pid = PID_getpid();
- Lock_init((Lock*)self, folder, name, timeout, interval);
- LockFileLockIVARS *const ivars = LFLock_IVARS(self);
- ivars->host = (String*)INCREF(host);
- ivars->link_path = Str_newf("%o.%o.%i64", ivars->lock_path, host,
- (int64_t)pid);
- ivars->exclusive_only = exclusive_only;
- return self;
-}
-
-struct lockfile_context {
- OutStream *outstream;
- String *json;
-};
-
-static void
-S_write_lockfile_json(void *context) {
- struct lockfile_context *stuff = (struct lockfile_context*)context;
- size_t size = Str_Get_Size(stuff->json);
- OutStream_Write_Bytes(stuff->outstream, Str_Get_Ptr8(stuff->json), size);
- OutStream_Close(stuff->outstream);
-}
-
-bool
-LFLock_Request_Shared_IMP(LockFileLock *self) {
- LockFileLockIVARS *const ivars = LFLock_IVARS(self);
-
- if (ivars->exclusive_only) {
- THROW(ERR, "Can't request shared lock if exclusive_only is set");
- }
-
- if (ivars->state != LFLOCK_STATE_UNLOCKED) {
- THROW(ERR, "Lock already acquired");
- }
-
- // TODO: The is_locked test and subsequent file creation is prone to a
- // race condition. We could protect the whole process with an internal
- // exclusive lock.
-
- if (S_is_locked_exclusive(ivars)) {
- String *msg = Str_newf("'%o.lock' is locked", ivars->name);
- Err_set_error((Err*)LockErr_new(msg));
- return false;
- }
-
- String *path = NULL;
-
- uint32_t i = 0;
- do {
- DECREF(path);
- path = Str_newf("locks/%o-%u32.lock", ivars->name, ++i);
- } while (Folder_Exists(ivars->folder, path));
-
- if (S_request(ivars, path)) {
- ivars->shared_lock_path = path;
- ivars->state = LFLOCK_STATE_LOCKED_SHARED;
- return true;
- }
- else {
- DECREF(path);
- return false;
- }
-}
-
-bool
-LFLock_Request_Exclusive_IMP(LockFileLock *self) {
- LockFileLockIVARS *const ivars = LFLock_IVARS(self);
-
- if (ivars->state != LFLOCK_STATE_UNLOCKED) {
- THROW(ERR, "Lock already acquired");
- }
-
- // TODO: The is_locked test and subsequent file creation is prone to a
- // race condition. We could protect the whole process with an internal
- // exclusive lock.
-
- if (ivars->exclusive_only
- ? S_is_locked_exclusive(ivars)
- : S_is_locked(ivars)
- ) {
- String *msg = Str_newf("'%o.lock' is locked", ivars->name);
- Err_set_error((Err*)LockErr_new(msg));
- return false;
- }
-
- if (S_request(ivars, ivars->lock_path)) {
- ivars->state = LFLOCK_STATE_LOCKED_EXCLUSIVE;
- return true;
- }
- else {
- return false;
- }
-}
-
-static bool
-S_request(LockFileLockIVARS *ivars, String *lock_path) {
- bool success = false;
-
- // Create the "locks" subdirectory if necessary.
- String *lock_dir_name = SSTR_WRAP_C("locks");
- if (!Folder_Exists(ivars->folder, lock_dir_name)) {
- if (!Folder_MkDir(ivars->folder, lock_dir_name)) {
- Err *err = (Err*)INCREF(Err_get_error());
- // Maybe our attempt failed because another process succeeded.
- if (Folder_Find_Folder(ivars->folder, lock_dir_name)) {
- DECREF(err);
- }
- else {
- // Nope, everything failed, so bail out.
- Err_set_error(err);
- return false;
- }
- }
- }
-
- // Prepare to write pid, lock name, and host to the lock file as JSON.
- Hash *file_data = Hash_new(3);
- Hash_Store_Utf8(file_data, "pid", 3,
- (Obj*)Str_newf("%i32", (int32_t)PID_getpid()));
- Hash_Store_Utf8(file_data, "host", 4, INCREF(ivars->host));
- Hash_Store_Utf8(file_data, "name", 4, INCREF(ivars->name));
- String *json = Json_to_json((Obj*)file_data);
- DECREF(file_data);
-
- // Write to a temporary file, then use the creation of a hard link to
- // ensure atomic but non-destructive creation of the lockfile with its
- // complete contents.
-
- OutStream *outstream = Folder_Open_Out(ivars->folder, ivars->link_path);
- if (!outstream) {
- ERR_ADD_FRAME(Err_get_error());
- DECREF(json);
- return false;
- }
-
- struct lockfile_context context;
- context.outstream = outstream;
- context.json = json;
- Err *json_error = Err_trap(S_write_lockfile_json, &context);
- DECREF(outstream);
- DECREF(json);
- if (json_error) {
- Err_set_error(json_error);
- }
- else {
- success = Folder_Hard_Link(ivars->folder, ivars->link_path,
- lock_path);
- if (!success) {
- // TODO: Only return a LockErr if errno == EEXIST, otherwise
- // return a normal Err.
- Err *hard_link_err = (Err*)CERTIFY(Err_get_error(), ERR);
- String *msg = Str_newf("Failed to obtain lock at '%o': %o",
- lock_path, Err_Get_Mess(hard_link_err));
- Err_set_error((Err*)LockErr_new(msg));
- }
- }
-
- // Verify that our temporary file got zapped.
- bool deletion_failed = !Folder_Delete(ivars->folder, ivars->link_path);
- if (deletion_failed) {
- String *mess = MAKE_MESS("Failed to delete '%o'", ivars->link_path);
- Err_throw_mess(ERR, mess);
- }
-
- return success;
-}
-
-void
-LFLock_Release_IMP(LockFileLock *self) {
- LockFileLockIVARS *const ivars = LFLock_IVARS(self);
-
- if (ivars->state == LFLOCK_STATE_UNLOCKED) {
- THROW(ERR, "Lock not acquired");
- }
-
- if (ivars->state == LFLOCK_STATE_LOCKED_EXCLUSIVE) {
- if (Folder_Exists(ivars->folder, ivars->lock_path)) {
- S_maybe_delete_file(ivars, ivars->lock_path, true, false);
- }
- }
- else { // Shared lock.
- if (Folder_Exists(ivars->folder, ivars->shared_lock_path)) {
- S_maybe_delete_file(ivars, ivars->shared_lock_path, true, false);
- }
-
- // Empty out lock_path.
- DECREF(ivars->shared_lock_path);
- ivars->shared_lock_path = NULL;
- }
-
- ivars->state = LFLOCK_STATE_UNLOCKED;
-}
-
-static bool
-S_is_locked_exclusive(LockFileLockIVARS *ivars) {
- return Folder_Exists(ivars->folder, ivars->lock_path)
- && !S_maybe_delete_file(ivars, ivars->lock_path, false, true);
-}
-
-static bool
-S_is_locked(LockFileLockIVARS *ivars) {
- if (S_is_locked_exclusive(ivars)) { return true; }
-
- // Check for shared lock.
-
- String *lock_dir_name = SSTR_WRAP_C("locks");
- if (!Folder_Find_Folder(ivars->folder, lock_dir_name)) {
- return false;
- }
-
- bool locked = false;
- DirHandle *dh = Folder_Open_Dir(ivars->folder, lock_dir_name);
- if (!dh) { RETHROW(INCREF(Err_get_error())); }
-
- while (DH_Next(dh)) {
- String *entry = DH_Get_Entry(dh);
- if (S_is_shared_lock_file(ivars, entry)) {
- String *candidate = Str_newf("%o/%o", lock_dir_name, entry);
- if (!S_maybe_delete_file(ivars, candidate, false, true)) {
- locked = true;
- }
- DECREF(candidate);
- }
- DECREF(entry);
- }
-
- DECREF(dh);
- return locked;
-}
-
-static bool
-S_is_shared_lock_file(LockFileLockIVARS *ivars, String *entry) {
- // Translation: $match = $entry =~ /^\Q$name-\d+\.lock\z/
- bool match = false;
-
- // $name
- if (Str_Starts_With(entry, ivars->name)) {
- StringIterator *iter = Str_Top(entry);
- StrIter_Advance(iter, Str_Length(ivars->name));
-
- // Hyphen-minus
- if (StrIter_Next(iter) == '-') {
- int32_t code_point = StrIter_Next(iter);
-
- // Digit
- if (code_point >= '0' && code_point <= '9') {
- // Optional digits
- do {
- code_point = StrIter_Next(iter);
- } while (code_point >= '0' && code_point <= '9');
-
- // ".lock"
- match = code_point == '.'
- && StrIter_Starts_With_Utf8(iter, "lock", 4)
- && StrIter_Advance(iter, SIZE_MAX) == 4;
- }
- }
-
- DECREF(iter);
- }
-
- return match;
-}
-
-static bool
-S_maybe_delete_file(LockFileLockIVARS *ivars, String *path,
- bool delete_mine, bool delete_other) {
- Folder *folder = ivars->folder;
- bool success = false;
-
- Hash *hash = (Hash*)Json_slurp_json(folder, path);
- if (hash != NULL && Obj_is_a((Obj*)hash, HASH)) {
- String *pid_buf = (String*)Hash_Fetch_Utf8(hash, "pid", 3);
- String *host = (String*)Hash_Fetch_Utf8(hash, "host", 4);
- String *name = (String*)Hash_Fetch_Utf8(hash, "name", 4);
-
- // Match hostname and lock name.
- if (host != NULL
- && Str_is_a(host, STRING)
- && Str_Equals(host, (Obj*)ivars->host)
- && name != NULL
- && Str_is_a(name, STRING)
- && Str_Equals(name, (Obj*)ivars->name)
- && pid_buf != NULL
- && Str_is_a(pid_buf, STRING)
- ) {
- // Verify that pid is either mine or dead.
- int pid = (int)Str_To_I64(pid_buf);
- if ((delete_mine && pid == PID_getpid()) // This process.
- || (delete_other && !PID_active(pid)) // Dead pid.
- ) {
- if (Folder_Delete(folder, path)) {
- success = true;
- }
- else {
- String *mess
- = MAKE_MESS("Can't delete '%o'", path);
- DECREF(hash);
- Err_throw_mess(ERR, mess);
- }
- }
- }
- }
- DECREF(hash);
-
- return success;
-}
-
-void
-LFLock_Destroy_IMP(LockFileLock *self) {
- LockFileLockIVARS *const ivars = LFLock_IVARS(self);
- if (ivars->state != LFLOCK_STATE_UNLOCKED) { LFLock_Release(self); }
- DECREF(ivars->host);
- DECREF(ivars->link_path);
- SUPER_DESTROY(self, LOCKFILELOCK);
-}
-
-/***************************************************************************/
-
LockErr*
LockErr_new(String *message) {
LockErr *self = (LockErr*)Class_Make_Obj(LOCKERR);
http://git-wip-us.apache.org/repos/asf/lucy/blob/6e8538a8/core/Lucy/Store/Lock.cfh
----------------------------------------------------------------------
diff --git a/core/Lucy/Store/Lock.cfh b/core/Lucy/Store/Lock.cfh
index 11f31c0..09c2c54 100644
--- a/core/Lucy/Store/Lock.cfh
+++ b/core/Lucy/Store/Lock.cfh
@@ -49,6 +49,9 @@ abstract class Lucy::Store::Lock inherits Clownfish::Obj {
init(Lock *self, Folder *folder, String *name, int32_t timeout = 0,
int32_t interval = 100);
+ inert bool
+ make_lock_dir(Folder *folder);
+
/** Call [](.Request_Shared) once per `interval` until [](.Request_Shared)
* returns success or the `timeout` has been reached.
*
@@ -99,36 +102,6 @@ abstract class Lucy::Store::Lock inherits Clownfish::Obj {
Destroy(Lock *self);
}
-class Lucy::Store::LockFileLock nickname LFLock
- inherits Lucy::Store::Lock {
-
- String *host;
- String *shared_lock_path;
- String *link_path;
- int state;
- bool exclusive_only;
-
- inert incremented LockFileLock*
- new(Folder *folder, String *name, String *host, int32_t timeout = 0,
- int32_t interval = 100, bool exclusive_only);
-
- public inert LockFileLock*
- init(LockFileLock *self, Folder *folder, String *name, String *host,
- int32_t timeout = 0, int32_t interval = 100, bool exclusive_only);
-
- public bool
- Request_Shared(LockFileLock *self);
-
- public bool
- Request_Exclusive(LockFileLock *self);
-
- public void
- Release(LockFileLock *self);
-
- public void
- Destroy(LockFileLock *self);
-}
-
/** Lock exception.
*
* LockErr is a subclass of [Err](cfish:cfish.Err) which indicates
http://git-wip-us.apache.org/repos/asf/lucy/blob/6e8538a8/core/Lucy/Store/LockFileLock.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Store/LockFileLock.c b/core/Lucy/Store/LockFileLock.c
new file mode 100644
index 0000000..19d624e
--- /dev/null
+++ b/core/Lucy/Store/LockFileLock.c
@@ -0,0 +1,363 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_LOCKFILELOCK
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Store/LockFileLock.h"
+#include "Lucy/Store/DirHandle.h"
+#include "Lucy/Store/Folder.h"
+#include "Lucy/Store/OutStream.h"
+#include "Lucy/Util/Json.h"
+#include "Lucy/Util/ProcessID.h"
+
+#define LFLOCK_STATE_UNLOCKED 0
+#define LFLOCK_STATE_LOCKED_SHARED 1
+#define LFLOCK_STATE_LOCKED_EXCLUSIVE 2
+
+static bool
+S_request(LockFileLockIVARS *ivars, String *lock_path);
+
+static bool
+S_is_locked_exclusive(LockFileLockIVARS *ivars);
+
+static bool
+S_is_locked(LockFileLockIVARS *ivars);
+
+static bool
+S_is_shared_lock_file(LockFileLockIVARS *ivars, String *entry);
+
+static bool
+S_maybe_delete_file(LockFileLockIVARS *ivars, String *path,
+ bool delete_mine, bool delete_other);
+
+LockFileLock*
+LFLock_new(Folder *folder, String *name, String *host, int32_t timeout,
+ int32_t interval, bool exclusive_only) {
+ LockFileLock *self = (LockFileLock*)Class_Make_Obj(LOCKFILELOCK);
+ return LFLock_init(self, folder, name, host, timeout, interval,
+ exclusive_only);
+}
+
+LockFileLock*
+LFLock_init(LockFileLock *self, Folder *folder, String *name, String *host,
+ int32_t timeout, int32_t interval, bool exclusive_only) {
+ int pid = PID_getpid();
+ Lock_init((Lock*)self, folder, name, timeout, interval);
+ LockFileLockIVARS *const ivars = LFLock_IVARS(self);
+ ivars->host = (String*)INCREF(host);
+ ivars->link_path = Str_newf("%o.%o.%i64", ivars->lock_path, host,
+ (int64_t)pid);
+ ivars->exclusive_only = exclusive_only;
+ return self;
+}
+
+struct lockfile_context {
+ OutStream *outstream;
+ String *json;
+};
+
+static void
+S_write_lockfile_json(void *context) {
+ struct lockfile_context *stuff = (struct lockfile_context*)context;
+ size_t size = Str_Get_Size(stuff->json);
+ OutStream_Write_Bytes(stuff->outstream, Str_Get_Ptr8(stuff->json), size);
+ OutStream_Close(stuff->outstream);
+}
+
+bool
+LFLock_Request_Shared_IMP(LockFileLock *self) {
+ LockFileLockIVARS *const ivars = LFLock_IVARS(self);
+
+ if (ivars->exclusive_only) {
+ THROW(ERR, "Can't request shared lock if exclusive_only is set");
+ }
+
+ if (ivars->state != LFLOCK_STATE_UNLOCKED) {
+ THROW(ERR, "Lock already acquired");
+ }
+
+ // TODO: The is_locked test and subsequent file creation is prone to a
+ // race condition. We could protect the whole process with an internal
+ // exclusive lock.
+
+ if (S_is_locked_exclusive(ivars)) {
+ String *msg = Str_newf("'%o.lock' is locked", ivars->name);
+ Err_set_error((Err*)LockErr_new(msg));
+ return false;
+ }
+
+ String *path = NULL;
+
+ uint32_t i = 0;
+ do {
+ DECREF(path);
+ path = Str_newf("locks/%o-%u32.lock", ivars->name, ++i);
+ } while (Folder_Exists(ivars->folder, path));
+
+ if (S_request(ivars, path)) {
+ ivars->shared_lock_path = path;
+ ivars->state = LFLOCK_STATE_LOCKED_SHARED;
+ return true;
+ }
+ else {
+ DECREF(path);
+ return false;
+ }
+}
+
+bool
+LFLock_Request_Exclusive_IMP(LockFileLock *self) {
+ LockFileLockIVARS *const ivars = LFLock_IVARS(self);
+
+ if (ivars->state != LFLOCK_STATE_UNLOCKED) {
+ THROW(ERR, "Lock already acquired");
+ }
+
+ // TODO: The is_locked test and subsequent file creation is prone to a
+ // race condition. We could protect the whole process with an internal
+ // exclusive lock.
+
+ if (ivars->exclusive_only
+ ? S_is_locked_exclusive(ivars)
+ : S_is_locked(ivars)
+ ) {
+ String *msg = Str_newf("'%o.lock' is locked", ivars->name);
+ Err_set_error((Err*)LockErr_new(msg));
+ return false;
+ }
+
+ if (S_request(ivars, ivars->lock_path)) {
+ ivars->state = LFLOCK_STATE_LOCKED_EXCLUSIVE;
+ return true;
+ }
+ else {
+ return false;
+ }
+}
+
+static bool
+S_request(LockFileLockIVARS *ivars, String *lock_path) {
+ bool success = false;
+
+ // Create the "locks" subdirectory if necessary.
+ String *lock_dir_name = SSTR_WRAP_C("locks");
+ if (!Folder_Exists(ivars->folder, lock_dir_name)) {
+ if (!Lock_make_lock_dir(ivars->folder)) { return false; }
+ }
+
+ // Prepare to write pid, lock name, and host to the lock file as JSON.
+ Hash *file_data = Hash_new(3);
+ Hash_Store_Utf8(file_data, "pid", 3,
+ (Obj*)Str_newf("%i32", (int32_t)PID_getpid()));
+ Hash_Store_Utf8(file_data, "host", 4, INCREF(ivars->host));
+ Hash_Store_Utf8(file_data, "name", 4, INCREF(ivars->name));
+ String *json = Json_to_json((Obj*)file_data);
+ DECREF(file_data);
+
+ // Write to a temporary file, then use the creation of a hard link to
+ // ensure atomic but non-destructive creation of the lockfile with its
+ // complete contents.
+
+ OutStream *outstream = Folder_Open_Out(ivars->folder, ivars->link_path);
+ if (!outstream) {
+ ERR_ADD_FRAME(Err_get_error());
+ DECREF(json);
+ return false;
+ }
+
+ struct lockfile_context context;
+ context.outstream = outstream;
+ context.json = json;
+ Err *json_error = Err_trap(S_write_lockfile_json, &context);
+ DECREF(outstream);
+ DECREF(json);
+ if (json_error) {
+ Err_set_error(json_error);
+ }
+ else {
+ success = Folder_Hard_Link(ivars->folder, ivars->link_path,
+ lock_path);
+ if (!success) {
+ // TODO: Only return a LockErr if errno == EEXIST, otherwise
+ // return a normal Err.
+ Err *hard_link_err = (Err*)CERTIFY(Err_get_error(), ERR);
+ String *msg = Str_newf("Failed to obtain lock at '%o': %o",
+ lock_path, Err_Get_Mess(hard_link_err));
+ Err_set_error((Err*)LockErr_new(msg));
+ }
+ }
+
+ // Verify that our temporary file got zapped.
+ bool deletion_failed = !Folder_Delete(ivars->folder, ivars->link_path);
+ if (deletion_failed) {
+ String *mess = MAKE_MESS("Failed to delete '%o'", ivars->link_path);
+ Err_throw_mess(ERR, mess);
+ }
+
+ return success;
+}
+
+void
+LFLock_Release_IMP(LockFileLock *self) {
+ LockFileLockIVARS *const ivars = LFLock_IVARS(self);
+
+ if (ivars->state == LFLOCK_STATE_UNLOCKED) {
+ THROW(ERR, "Lock not acquired");
+ }
+
+ if (ivars->state == LFLOCK_STATE_LOCKED_EXCLUSIVE) {
+ if (Folder_Exists(ivars->folder, ivars->lock_path)) {
+ S_maybe_delete_file(ivars, ivars->lock_path, true, false);
+ }
+ }
+ else { // Shared lock.
+ if (Folder_Exists(ivars->folder, ivars->shared_lock_path)) {
+ S_maybe_delete_file(ivars, ivars->shared_lock_path, true, false);
+ }
+
+ // Empty out lock_path.
+ DECREF(ivars->shared_lock_path);
+ ivars->shared_lock_path = NULL;
+ }
+
+ ivars->state = LFLOCK_STATE_UNLOCKED;
+}
+
+static bool
+S_is_locked_exclusive(LockFileLockIVARS *ivars) {
+ return Folder_Exists(ivars->folder, ivars->lock_path)
+ && !S_maybe_delete_file(ivars, ivars->lock_path, false, true);
+}
+
+static bool
+S_is_locked(LockFileLockIVARS *ivars) {
+ if (S_is_locked_exclusive(ivars)) { return true; }
+
+ // Check for shared lock.
+
+ String *lock_dir_name = SSTR_WRAP_C("locks");
+ if (!Folder_Find_Folder(ivars->folder, lock_dir_name)) {
+ return false;
+ }
+
+ bool locked = false;
+ DirHandle *dh = Folder_Open_Dir(ivars->folder, lock_dir_name);
+ if (!dh) { RETHROW(INCREF(Err_get_error())); }
+
+ while (DH_Next(dh)) {
+ String *entry = DH_Get_Entry(dh);
+ if (S_is_shared_lock_file(ivars, entry)) {
+ String *candidate = Str_newf("%o/%o", lock_dir_name, entry);
+ if (!S_maybe_delete_file(ivars, candidate, false, true)) {
+ locked = true;
+ }
+ DECREF(candidate);
+ }
+ DECREF(entry);
+ }
+
+ DECREF(dh);
+ return locked;
+}
+
+static bool
+S_is_shared_lock_file(LockFileLockIVARS *ivars, String *entry) {
+ // Translation: $match = $entry =~ /^\Q$name-\d+\.lock\z/
+ bool match = false;
+
+ // $name
+ if (Str_Starts_With(entry, ivars->name)) {
+ StringIterator *iter = Str_Top(entry);
+ StrIter_Advance(iter, Str_Length(ivars->name));
+
+ // Hyphen-minus
+ if (StrIter_Next(iter) == '-') {
+ int32_t code_point = StrIter_Next(iter);
+
+ // Digit
+ if (code_point >= '0' && code_point <= '9') {
+ // Optional digits
+ do {
+ code_point = StrIter_Next(iter);
+ } while (code_point >= '0' && code_point <= '9');
+
+ // ".lock"
+ match = code_point == '.'
+ && StrIter_Starts_With_Utf8(iter, "lock", 4)
+ && StrIter_Advance(iter, SIZE_MAX) == 4;
+ }
+ }
+
+ DECREF(iter);
+ }
+
+ return match;
+}
+
+static bool
+S_maybe_delete_file(LockFileLockIVARS *ivars, String *path,
+ bool delete_mine, bool delete_other) {
+ Folder *folder = ivars->folder;
+ bool success = false;
+
+ Hash *hash = (Hash*)Json_slurp_json(folder, path);
+ if (hash != NULL && Obj_is_a((Obj*)hash, HASH)) {
+ String *pid_buf = (String*)Hash_Fetch_Utf8(hash, "pid", 3);
+ String *host = (String*)Hash_Fetch_Utf8(hash, "host", 4);
+ String *name = (String*)Hash_Fetch_Utf8(hash, "name", 4);
+
+ // Match hostname and lock name.
+ if (host != NULL
+ && Str_is_a(host, STRING)
+ && Str_Equals(host, (Obj*)ivars->host)
+ && name != NULL
+ && Str_is_a(name, STRING)
+ && Str_Equals(name, (Obj*)ivars->name)
+ && pid_buf != NULL
+ && Str_is_a(pid_buf, STRING)
+ ) {
+ // Verify that pid is either mine or dead.
+ int pid = (int)Str_To_I64(pid_buf);
+ if ((delete_mine && pid == PID_getpid()) // This process.
+ || (delete_other && !PID_active(pid)) // Dead pid.
+ ) {
+ if (Folder_Delete(folder, path)) {
+ success = true;
+ }
+ else {
+ String *mess
+ = MAKE_MESS("Can't delete '%o'", path);
+ DECREF(hash);
+ Err_throw_mess(ERR, mess);
+ }
+ }
+ }
+ }
+ DECREF(hash);
+
+ return success;
+}
+
+void
+LFLock_Destroy_IMP(LockFileLock *self) {
+ LockFileLockIVARS *const ivars = LFLock_IVARS(self);
+ if (ivars->state != LFLOCK_STATE_UNLOCKED) { LFLock_Release(self); }
+ DECREF(ivars->host);
+ DECREF(ivars->link_path);
+ SUPER_DESTROY(self, LOCKFILELOCK);
+}
+
http://git-wip-us.apache.org/repos/asf/lucy/blob/6e8538a8/core/Lucy/Store/LockFileLock.cfh
----------------------------------------------------------------------
diff --git a/core/Lucy/Store/LockFileLock.cfh b/core/Lucy/Store/LockFileLock.cfh
new file mode 100644
index 0000000..30fc23a
--- /dev/null
+++ b/core/Lucy/Store/LockFileLock.cfh
@@ -0,0 +1,49 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+class Lucy::Store::LockFileLock nickname LFLock
+ inherits Lucy::Store::Lock {
+
+ String *host;
+ String *shared_lock_path;
+ String *link_path;
+ int state;
+ bool exclusive_only;
+
+ inert incremented LockFileLock*
+ new(Folder *folder, String *name, String *host, int32_t timeout = 0,
+ int32_t interval = 100, bool exclusive_only);
+
+ public inert LockFileLock*
+ init(LockFileLock *self, Folder *folder, String *name, String *host,
+ int32_t timeout = 0, int32_t interval = 100, bool exclusive_only);
+
+ public bool
+ Request_Shared(LockFileLock *self);
+
+ public bool
+ Request_Exclusive(LockFileLock *self);
+
+ public void
+ Release(LockFileLock *self);
+
+ public void
+ Destroy(LockFileLock *self);
+}
+
+
http://git-wip-us.apache.org/repos/asf/lucy/blob/6e8538a8/test/Lucy/Test/Store/TestLock.c
----------------------------------------------------------------------
diff --git a/test/Lucy/Test/Store/TestLock.c b/test/Lucy/Test/Store/TestLock.c
index b52f366..df3eb20 100644
--- a/test/Lucy/Test/Store/TestLock.c
+++ b/test/Lucy/Test/Store/TestLock.c
@@ -30,7 +30,7 @@
#include "Lucy/Test/Store/TestLock.h"
#include "Clownfish/TestHarness/TestBatchRunner.h"
#include "Lucy/Store/FSFolder.h"
-#include "Lucy/Store/Lock.h"
+#include "Lucy/Store/LockFileLock.h"
#include "Lucy/Store/RAMFolder.h"
#include "Lucy/Util/Json.h"
[07/16] lucy git commit: Remove Lock_Is_Locked
Posted by nw...@apache.org.
Remove Lock_Is_Locked
Project: http://git-wip-us.apache.org/repos/asf/lucy/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucy/commit/38a33bad
Tree: http://git-wip-us.apache.org/repos/asf/lucy/tree/38a33bad
Diff: http://git-wip-us.apache.org/repos/asf/lucy/diff/38a33bad
Branch: refs/heads/master
Commit: 38a33badb310ba9fea5bf9a923277da4a82ec369
Parents: de14d4c
Author: Nick Wellnhofer <we...@aevum.de>
Authored: Fri Feb 17 19:13:59 2017 +0100
Committer: Nick Wellnhofer <we...@aevum.de>
Committed: Mon Feb 20 16:26:22 2017 +0100
----------------------------------------------------------------------
core/Lucy/Store/Lock.c | 31 ++++++++++++++---------------
core/Lucy/Store/Lock.cfh | 22 ---------------------
go/lucy/store_test.go | 6 ------
perl/t/105-folder.t | 7 ++++---
perl/t/110-shared_lock.t | 45 ++++++++++++++++++++++---------------------
5 files changed, 41 insertions(+), 70 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucy/blob/38a33bad/core/Lucy/Store/Lock.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Store/Lock.c b/core/Lucy/Store/Lock.c
index d187405..0879640 100644
--- a/core/Lucy/Store/Lock.c
+++ b/core/Lucy/Store/Lock.c
@@ -131,6 +131,12 @@ static bool
S_request(LockFileLockIVARS *ivars, String *lock_path);
static bool
+S_is_locked_exclusive(LockFileLockIVARS *ivars);
+
+static bool
+S_is_locked(LockFileLockIVARS *ivars);
+
+static bool
S_is_shared_lock_file(LockFileLockIVARS *ivars, String *entry);
static bool
@@ -194,7 +200,7 @@ LFLock_Request_Shared_IMP(LockFileLock *self) {
// race condition. We could protect the whole process with an internal
// exclusive lock.
- if (LFLock_Is_Locked_Exclusive(self)) {
+ if (S_is_locked_exclusive(ivars)) {
String *msg = Str_newf("'%o.lock' is locked", ivars->name);
Err_set_error((Err*)LockErr_new(msg));
return false;
@@ -232,8 +238,8 @@ LFLock_Request_Exclusive_IMP(LockFileLock *self) {
// exclusive lock.
if (ivars->exclusive_only
- ? LFLock_Is_Locked_Exclusive(self)
- : LFLock_Is_Locked(self)
+ ? S_is_locked_exclusive(ivars)
+ : S_is_locked(ivars)
) {
String *msg = Str_newf("'%o.lock' is locked", ivars->name);
Err_set_error((Err*)LockErr_new(msg));
@@ -352,24 +358,15 @@ LFLock_Release_IMP(LockFileLock *self) {
ivars->state = LFLOCK_STATE_UNLOCKED;
}
-bool
-LFLock_Is_Locked_Exclusive_IMP(LockFileLock *self) {
- LockFileLockIVARS *const ivars = LFLock_IVARS(self);
-
+static bool
+S_is_locked_exclusive(LockFileLockIVARS *ivars) {
return Folder_Exists(ivars->folder, ivars->lock_path)
&& !S_maybe_delete_file(ivars, ivars->lock_path, false, true);
}
-bool
-LFLock_Is_Locked_IMP(LockFileLock *self) {
- LockFileLockIVARS *const ivars = LFLock_IVARS(self);
-
- // Check for exclusive lock.
- if (Folder_Exists(ivars->folder, ivars->lock_path)
- && !S_maybe_delete_file(ivars, ivars->lock_path, false, true)
- ) {
- return true;
- }
+static bool
+S_is_locked(LockFileLockIVARS *ivars) {
+ if (S_is_locked_exclusive(ivars)) { return true; }
// Check for shared lock.
http://git-wip-us.apache.org/repos/asf/lucy/blob/38a33bad/core/Lucy/Store/Lock.cfh
----------------------------------------------------------------------
diff --git a/core/Lucy/Store/Lock.cfh b/core/Lucy/Store/Lock.cfh
index fb813e1..6688e95 100644
--- a/core/Lucy/Store/Lock.cfh
+++ b/core/Lucy/Store/Lock.cfh
@@ -96,22 +96,6 @@ abstract class Lucy::Store::Lock inherits Clownfish::Obj {
public abstract void
Release(Lock *self);
- /** Indicate whether the resource identified by this lock's name is
- * currently locked in shared or exclusive mode.
- *
- * @return true if the resource is locked, false otherwise.
- */
- public abstract bool
- Is_Locked(Lock *self);
-
- /** Indicate whether the resource identified by this lock's name is
- * currently locked in exclusive mode.
- *
- * @return true if the resource is locked, false otherwise.
- */
- public abstract bool
- Is_Locked_Exclusive(Lock *self);
-
String*
Get_Name(Lock *self);
@@ -147,12 +131,6 @@ class Lucy::Store::LockFileLock nickname LFLock
public void
Release(LockFileLock *self);
- public bool
- Is_Locked(LockFileLock *self);
-
- public bool
- Is_Locked_Exclusive(LockFileLock *self);
-
String*
Get_Lock_Path(LockFileLock *self);
http://git-wip-us.apache.org/repos/asf/lucy/blob/38a33bad/go/lucy/store_test.go
----------------------------------------------------------------------
diff --git a/go/lucy/store_test.go b/go/lucy/store_test.go
index 89f5057..ae70069 100644
--- a/go/lucy/store_test.go
+++ b/go/lucy/store_test.go
@@ -663,9 +663,6 @@ func TestLockFileLockAll(t *testing.T) {
if err != nil {
t.Errorf("Request: %v", err)
}
- if !lock.IsLocked() {
- t.Errorf("Lock should be locked, but IsLocked returned false")
- }
if got := lock.getLockPath(); len(got) == 0 {
// Lock path only valid when locked for shared locks.
t.Errorf("getLockPath should work")
@@ -689,9 +686,6 @@ func TestLockFileLockAll(t *testing.T) {
if err != nil {
t.Errorf("Request: %v", err)
}
- if !lock.IsLocked() {
- t.Errorf("Lock should be locked, but IsLocked returned false")
- }
if got := lock.getLockPath(); len(got) == 0 {
// Lock path only valid when locked for shared locks.
t.Errorf("getLockPath should work")
http://git-wip-us.apache.org/repos/asf/lucy/blob/38a33bad/perl/t/105-folder.t
----------------------------------------------------------------------
diff --git a/perl/t/105-folder.t b/perl/t/105-folder.t
index 3d00195..ba1255e 100644
--- a/perl/t/105-folder.t
+++ b/perl/t/105-folder.t
@@ -62,16 +62,17 @@ for my $folder ( $fs_folder, $ram_folder ) {
);
$lock->obtain_exclusive();
- ok( $lock->is_locked_exclusive(), "lock is locked" );
+ my $lock_path = $lock->get_lock_path;
+ ok( $folder->exists($lock_path), "lock is locked" );
ok( !$competing_lock->obtain_exclusive(),
"shouldn't get lock on existing resource"
);
- ok( $lock->is_locked_exclusive(),
+ ok( $folder->exists($lock_path),
"lock still locked after competing attempt"
);
$lock->release;
- ok( !$lock->is_locked_exclusive(), "release works" );
+ ok( !$folder->exists($lock_path), "release works" );
$lock->obtain_exclusive();
$folder->rename( from => 'king_of_rock', to => 'king_of_lock' );
http://git-wip-us.apache.org/repos/asf/lucy/blob/38a33bad/perl/t/110-shared_lock.t
----------------------------------------------------------------------
diff --git a/perl/t/110-shared_lock.t b/perl/t/110-shared_lock.t
index 77c6862..86e7d6c 100644
--- a/perl/t/110-shared_lock.t
+++ b/perl/t/110-shared_lock.t
@@ -20,8 +20,7 @@ use Test::More tests => 14;
use Lucy::Test;
my $folder = Lucy::Store::RAMFolder->new;
-
-my $lock = Lucy::Store::LockFileLock->new(
+my @args = (
folder => $folder,
name => 'ness',
timeout => 0,
@@ -29,32 +28,34 @@ my $lock = Lucy::Store::LockFileLock->new(
exclusive_only => 0,
);
-ok( !$lock->is_locked(), "not locked yet" );
+my $ex_lock = Lucy::Store::LockFileLock->new(@args);
+
+sub is_locked {
+ if ($ex_lock->request_exclusive) {
+ $ex_lock->release;
+ return 0;
+ }
+ else {
+ return 1;
+ }
+}
+
+my $lock = Lucy::Store::LockFileLock->new(@args);
+
+ok( !is_locked(), "not locked yet" );
ok( $lock->obtain_shared(), "obtain" );
-ok( $lock->is_locked(), "is_locked" );
+ok( is_locked(), "is_locked" );
ok( $folder->exists('locks/ness-1.lock'), "lockfile exists" );
-my $another_lock = Lucy::Store::LockFileLock->new(
- folder => $folder,
- name => 'ness',
- timeout => 0,
- host => 'nessie',
- exclusive_only => 0,
-);
+my $another_lock = Lucy::Store::LockFileLock->new(@args);
ok( $another_lock->obtain_shared(), "got a second lock on the same resource" );
$lock->release;
-ok( $lock->is_locked(),
+ok( is_locked(),
"first lock released but still is_locked because of other lock" );
-my $ya_lock = Lucy::Store::LockFileLock->new(
- folder => $folder,
- name => 'ness',
- timeout => 0,
- host => 'nessie',
- exclusive_only => 0,
-);
+my $ya_lock = Lucy::Store::LockFileLock->new(@args);
ok( $ya_lock->obtain_shared(), "got yet another lock" );
ok( $lock->obtain_shared(), "got first lock again" );
@@ -75,10 +76,10 @@ $another_lock->release;
$ya_lock->release;
ok( $lock->get_lock_path, "failed to release a lock with a different pid" );
-ok( !$lock->is_locked(), "is_locked clears stale locks" );
+ok( !is_locked(), "is_locked clears stale locks" );
ok( $lock->obtain_shared(), "got lock again" );
-ok( $lock->is_locked(), "it's locked" );
+ok( is_locked(), "it's locked" );
# Rewrite lock file to spec a different host.
$content = $folder->slurp_file("locks/ness-1.lock");
@@ -89,4 +90,4 @@ $outstream->print($content);
$outstream->close;
$lock->release;
-ok( $lock->is_locked(), "don't delete lock belonging to another host" );
+ok( is_locked(), "don't delete lock belonging to another host" );