You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@directory.apache.org by ka...@apache.org on 2015/05/13 10:47:00 UTC
svn commit: r1679161 - in /directory/mavibot/trunk/mavibot/src:
main/java/org/apache/directory/mavibot/btree/
test/java/org/apache/directory/mavibot/btree/
Author: kayyagari
Date: Wed May 13 08:47:00 2015
New Revision: 1679161
URL: http://svn.apache.org/r1679161
Log:
o changed the reclaimer threshold to 70
o updated space reclaimer to clean the BTree of BTrees
o added a multi threaded test
Modified:
directory/mavibot/trunk/mavibot/src/main/java/org/apache/directory/mavibot/btree/RecordManager.java
directory/mavibot/trunk/mavibot/src/main/java/org/apache/directory/mavibot/btree/SpaceReclaimer.java
directory/mavibot/trunk/mavibot/src/test/java/org/apache/directory/mavibot/btree/SpaceReclaimerTest.java
Modified: directory/mavibot/trunk/mavibot/src/main/java/org/apache/directory/mavibot/btree/RecordManager.java
URL: http://svn.apache.org/viewvc/directory/mavibot/trunk/mavibot/src/main/java/org/apache/directory/mavibot/btree/RecordManager.java?rev=1679161&r1=1679160&r2=1679161&view=diff
==============================================================================
--- directory/mavibot/trunk/mavibot/src/main/java/org/apache/directory/mavibot/btree/RecordManager.java (original)
+++ directory/mavibot/trunk/mavibot/src/main/java/org/apache/directory/mavibot/btree/RecordManager.java Wed May 13 08:47:00 2015
@@ -162,7 +162,7 @@ public class RecordManager extends Abstr
public static final boolean NORMAL_BTREE = false;
/** The B-tree of B-trees */
- private BTree<NameRevision, Long> btreeOfBtrees;
+ /* no qualifier */ BTree<NameRevision, Long> btreeOfBtrees;
/** The B-tree of B-trees management btree name */
/* no qualifier */static final String BTREE_OF_BTREES_NAME = "_btree_of_btrees_";
@@ -216,7 +216,9 @@ public class RecordManager extends Abstr
private int commitCount = 0;
/** the threshold at which the SpaceReclaimer will be run to free the copied pages */
- private int spaceReclaimerThreshold = 200;
+ // FIXME the below value is derived after seeing that anything higher than that
+ // is resulting in a "This thread does not hold the transactionLock" error
+ private int spaceReclaimerThreshold = 70;
public Map<Long, Integer> writeCounter = new HashMap<Long, Integer>();
Modified: directory/mavibot/trunk/mavibot/src/main/java/org/apache/directory/mavibot/btree/SpaceReclaimer.java
URL: http://svn.apache.org/viewvc/directory/mavibot/trunk/mavibot/src/main/java/org/apache/directory/mavibot/btree/SpaceReclaimer.java?rev=1679161&r1=1679160&r2=1679161&view=diff
==============================================================================
--- directory/mavibot/trunk/mavibot/src/main/java/org/apache/directory/mavibot/btree/SpaceReclaimer.java (original)
+++ directory/mavibot/trunk/mavibot/src/main/java/org/apache/directory/mavibot/btree/SpaceReclaimer.java Wed May 13 08:47:00 2015
@@ -72,14 +72,14 @@ public class SpaceReclaimer
running = true;
- rm.beginTransaction();
-
Set<String> managed = rm.getManagedTrees();
for ( String name : managed )
{
PersistedBTree tree = ( PersistedBTree ) rm.getManagedTree( name );
+ long latestRev = tree.getRevision();
+
Set<Long> inUseRevisions = new TreeSet<Long>();
// the tree might have been removed
@@ -94,6 +94,13 @@ public class SpaceReclaimer
List<RevisionOffset> copiedRevisions = getRevisions( name );
+ // the revision last removed from copiedPage BTree
+ long lastRemovedRev = -1;
+
+ // FIXME an additional txn needs to be started to safeguard the copiedPage BTree changes
+ // no clue yet on why this is needed
+ rm.beginTransaction();
+
for ( RevisionOffset ro : copiedRevisions )
{
long rv = ro.getRevision();
@@ -112,11 +119,30 @@ public class SpaceReclaimer
RevisionName key = new RevisionName( rv, name );
rm.copiedPageBtree.delete( key );
+ lastRemovedRev = rv;
+ }
+
+ rm.commit();
+
+ // no new txn is needed for the operations on BoB
+ if ( lastRemovedRev != -1 )
+ {
+ // we SHOULD NOT delete the latest revision from BoB
+ NameRevision nr = new NameRevision( name, latestRev );
+ TupleCursor<NameRevision, Long> cursor = rm.btreeOfBtrees.browseFrom( nr );
+
+ while ( cursor.hasPrev() )
+ {
+ Tuple<NameRevision, Long> t = cursor.prev();
+ //System.out.println( "deleting BoB rev " + t.getKey() + " latest rev " + latestRev );
+ rm.btreeOfBtrees.delete( t.getKey() );
+ }
+
+ cursor.close();
}
}
running = false;
- rm.commit();
}
catch ( Exception e )
{
Modified: directory/mavibot/trunk/mavibot/src/test/java/org/apache/directory/mavibot/btree/SpaceReclaimerTest.java
URL: http://svn.apache.org/viewvc/directory/mavibot/trunk/mavibot/src/test/java/org/apache/directory/mavibot/btree/SpaceReclaimerTest.java?rev=1679161&r1=1679160&r2=1679161&view=diff
==============================================================================
--- directory/mavibot/trunk/mavibot/src/test/java/org/apache/directory/mavibot/btree/SpaceReclaimerTest.java (original)
+++ directory/mavibot/trunk/mavibot/src/test/java/org/apache/directory/mavibot/btree/SpaceReclaimerTest.java Wed May 13 08:47:00 2015
@@ -20,7 +20,13 @@
package org.apache.directory.mavibot.btree;
+import static org.junit.Assert.assertEquals;
+
import java.io.File;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
import org.apache.directory.mavibot.btree.serializer.IntSerializer;
import org.apache.directory.mavibot.btree.serializer.StringSerializer;
@@ -29,7 +35,6 @@ import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
-import static org.junit.Assert.*;
/**
* Tests for free page reclaimer.
@@ -147,4 +152,85 @@ public class SpaceReclaimerTest
assertEquals( count, total );
}
+
+
+ /**
+ * Test reclaimer functionality while multiple threads writing to the same BTree
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testReclaimerWithMultiThreads() throws Exception
+ {
+ final int numEntriesPerThread = 11;
+ final int numThreads = 5;
+
+ final int total = numThreads * numEntriesPerThread;
+
+ final Map<Integer, Integer> keyMap = new ConcurrentHashMap<Integer, Integer>();
+
+ final Random rnd = new Random();
+
+ final CountDownLatch latch = new CountDownLatch( numThreads );
+
+ Runnable r = new Runnable()
+ {
+ @Override
+ public void run()
+ {
+ for ( int i=0; i < numEntriesPerThread; i++ )
+ {
+ try
+ {
+ int key = rnd.nextInt( total );
+ while( true )
+ {
+ if( !keyMap.containsKey( key ) )
+ {
+ keyMap.put( key, key );
+ break;
+ }
+
+ //System.out.println( "duplicate " + key );
+ key = rnd.nextInt( total );
+ }
+
+ uidTree.insert( key, String.valueOf( key ) );
+ }
+ catch( Exception e )
+ {
+ throw new RuntimeException(e);
+ }
+ }
+
+ latch.countDown();
+ }
+ };
+
+ for ( int i=0; i<numThreads; i++ )
+ {
+ Thread t = new Thread( r );
+ t.start();
+ }
+
+ latch.await();
+
+ System.out.println( "Total size before closing " + dbFile.length() );
+ closeAndReopenRM();
+ System.out.println( "Total size AFTER closing " + dbFile.length() );
+
+ int count = 0;
+ TupleCursor<Integer, String> cursor = uidTree.browse();
+ while ( cursor.hasNext() )
+ {
+ Tuple<Integer, String> t = cursor.next();
+ assertEquals( t.key, Integer.valueOf( count ) );
+ count++;
+ }
+
+ cursor.close();
+
+ assertEquals( count, total );
+ }
+
}