You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@jena.apache.org by an...@apache.org on 2011/08/04 20:37:06 UTC
svn commit: r1153963 - in /incubator/jena/Experimental/TxTDB/trunk: ./
src/main/java/com/hp/hpl/jena/tdb/base/buffer/
src/main/java/com/hp/hpl/jena/tdb/transaction/
src/test/java/com/hp/hpl/jena/tdb/transaction/
Author: andy
Date: Thu Aug 4 18:37:04 2011
New Revision: 1153963
URL: http://svn.apache.org/viewvc?rev=1153963&view=rev
Log:
Tidy
Modified:
incubator/jena/Experimental/TxTDB/trunk/log4j.properties
incubator/jena/Experimental/TxTDB/trunk/src/main/java/com/hp/hpl/jena/tdb/base/buffer/RecordBuffer.java
incubator/jena/Experimental/TxTDB/trunk/src/main/java/com/hp/hpl/jena/tdb/transaction/SysTxnState.java
incubator/jena/Experimental/TxTDB/trunk/src/main/java/com/hp/hpl/jena/tdb/transaction/TransactionManager.java
incubator/jena/Experimental/TxTDB/trunk/src/test/java/com/hp/hpl/jena/tdb/transaction/TestTransSystem.java
Modified: incubator/jena/Experimental/TxTDB/trunk/log4j.properties
URL: http://svn.apache.org/viewvc/incubator/jena/Experimental/TxTDB/trunk/log4j.properties?rev=1153963&r1=1153962&r2=1153963&view=diff
==============================================================================
--- incubator/jena/Experimental/TxTDB/trunk/log4j.properties (original)
+++ incubator/jena/Experimental/TxTDB/trunk/log4j.properties Thu Aug 4 18:37:04 2011
@@ -22,8 +22,8 @@ log4j.logger.TDB=INFO
log4j.logger.com.hp.hpl.jena.tdb=INFO
log4j.logger.com.hp.hpl.jena.tdb.transaction=INFO
-## log4j.logger.com.hp.hpl.jena.tdb.transaction.TransactionManager=ALL
-## log4j.logger.com.hp.hpl.jena.tdb.transaction.TestTransSystem=ALL
+##log4j.logger.com.hp.hpl.jena.tdb.transaction.TransactionManager=ALL
+log4j.logger.com.hp.hpl.jena.tdb.transaction.TestTransSystem=ALL
# Joseki server
log4j.logger.org.joseki=INFO
Modified: incubator/jena/Experimental/TxTDB/trunk/src/main/java/com/hp/hpl/jena/tdb/base/buffer/RecordBuffer.java
URL: http://svn.apache.org/viewvc/incubator/jena/Experimental/TxTDB/trunk/src/main/java/com/hp/hpl/jena/tdb/base/buffer/RecordBuffer.java?rev=1153963&r1=1153962&r2=1153963&view=diff
==============================================================================
--- incubator/jena/Experimental/TxTDB/trunk/src/main/java/com/hp/hpl/jena/tdb/base/buffer/RecordBuffer.java (original)
+++ incubator/jena/Experimental/TxTDB/trunk/src/main/java/com/hp/hpl/jena/tdb/base/buffer/RecordBuffer.java Thu Aug 4 18:37:04 2011
@@ -44,14 +44,14 @@ public class RecordBuffer extends Buffer
public Record getLow()
{
if ( numSlot == 0 )
- throw new IllegalArgumentException("Empty RecordBuffer") ;
+ throw new IllegalArgumentException("getLow: Empty RecordBuffer") ;
return _get(0) ;
}
public Record getHigh()
{
if ( numSlot == 0 )
- throw new IllegalArgumentException("Empty RecordBuffer") ;
+ throw new IllegalArgumentException("getHigh: Empty RecordBuffer") ;
return _get(numSlot-1) ;
}
Modified: incubator/jena/Experimental/TxTDB/trunk/src/main/java/com/hp/hpl/jena/tdb/transaction/SysTxnState.java
URL: http://svn.apache.org/viewvc/incubator/jena/Experimental/TxTDB/trunk/src/main/java/com/hp/hpl/jena/tdb/transaction/SysTxnState.java?rev=1153963&r1=1153962&r2=1153963&view=diff
==============================================================================
--- incubator/jena/Experimental/TxTDB/trunk/src/main/java/com/hp/hpl/jena/tdb/transaction/SysTxnState.java (original)
+++ incubator/jena/Experimental/TxTDB/trunk/src/main/java/com/hp/hpl/jena/tdb/transaction/SysTxnState.java Thu Aug 4 18:37:04 2011
@@ -20,20 +20,20 @@ package com.hp.hpl.jena.tdb.transaction;
public class SysTxnState
{
- final public int activeReaders ;
- final public int activeWriters ;
- final public int finishedReaders ;
- final public int committedWriters ;
- final public int abortedWriters ;
- final public int queuedCommits ;
+ final public long activeReaders ;
+ final public long activeWriters ;
+ final public long finishedReaders ;
+ final public long committedWriters ;
+ final public long abortedWriters ;
+ final public long queuedCommits ;
SysTxnState(TransactionManager tm)
{
- activeReaders = tm.activeReaders ;
- activeWriters = tm.activeWriters ;
- finishedReaders = tm.finishedReaders ;
- committedWriters = tm.committedWriters ;
- abortedWriters = tm.abortedWriters ;
+ activeReaders = tm.activeReaders.get() ;
+ activeWriters = tm.activeWriters.get() ;
+ finishedReaders = tm.finishedReaders.get() ;
+ committedWriters = tm.committedWriters.get() ;
+ abortedWriters = tm.abortedWriters.get() ;
queuedCommits = tm.commitedAwaitingFlush.size() ;
}
Modified: incubator/jena/Experimental/TxTDB/trunk/src/main/java/com/hp/hpl/jena/tdb/transaction/TransactionManager.java
URL: http://svn.apache.org/viewvc/incubator/jena/Experimental/TxTDB/trunk/src/main/java/com/hp/hpl/jena/tdb/transaction/TransactionManager.java?rev=1153963&r1=1153962&r2=1153963&view=diff
==============================================================================
--- incubator/jena/Experimental/TxTDB/trunk/src/main/java/com/hp/hpl/jena/tdb/transaction/TransactionManager.java (original)
+++ incubator/jena/Experimental/TxTDB/trunk/src/main/java/com/hp/hpl/jena/tdb/transaction/TransactionManager.java Thu Aug 4 18:37:04 2011
@@ -26,6 +26,7 @@ import org.openjena.atlas.logging.Log ;
import org.slf4j.Logger ;
import org.slf4j.LoggerFactory ;
+import com.hp.hpl.jena.shared.Lock ;
import com.hp.hpl.jena.tdb.DatasetGraphTxn ;
import com.hp.hpl.jena.tdb.ReadWrite ;
import com.hp.hpl.jena.tdb.store.DatasetGraphTDB ;
@@ -36,6 +37,8 @@ public class TransactionManager
// TODO Don't keep counters, keep lists.
// TODO Useful logging.
+ private static boolean checking = true ;
+
private static Logger log = LoggerFactory.getLogger(TransactionManager.class) ;
private Set<Transaction> activeTransactions = new HashSet<Transaction>() ;
@@ -63,13 +66,13 @@ public class TransactionManager
static AtomicLong transactionId = new AtomicLong(1) ;
- int activeReaders = 0 ;
- int activeWriters = 0 ; // 0 or 1
+ AtomicLong activeReaders = new AtomicLong(0) ;
+ AtomicLong activeWriters = new AtomicLong(0) ; // 0 or 1
// Misc stats
- int finishedReaders = 0 ;
- int committedWriters = 0 ;
- int abortedWriters = 0 ;
+ AtomicLong finishedReaders = new AtomicLong(0) ;
+ AtomicLong committedWriters = new AtomicLong(0) ;
+ AtomicLong abortedWriters = new AtomicLong(0) ;
// Ensure single writer.
private Semaphore writersWaiting = new Semaphore(1, true) ;
@@ -138,13 +141,18 @@ public class TransactionManager
@Override public void transactionStarts(Transaction txn) { activeTransactions.add(txn) ; }
@Override public void transactionFinishes(Transaction txn) { activeTransactions.remove(txn) ; }
@Override public void transactionCloses(Transaction txn) { }
- @Override public void readerStarts(Transaction txn) { activeReaders++ ; }
- @Override public void readerFinishes(Transaction txn) { activeReaders-- ; finishedReaders++ ; }
- @Override public void writerStarts(Transaction txn) { activeWriters++ ; }
- @Override public void writerCommits(Transaction txn) { activeWriters-- ; committedWriters++ ; }
- @Override public void writerAborts(Transaction txn) { activeWriters-- ; abortedWriters++ ; }
+ @Override public void readerStarts(Transaction txn) { inc(activeReaders) ; }
+ @Override public void readerFinishes(Transaction txn) { dec(activeReaders) ; inc(finishedReaders); }
+ @Override public void writerStarts(Transaction txn) { inc(activeWriters) ; }
+ @Override public void writerCommits(Transaction txn) { dec(activeWriters) ; inc(committedWriters) ; }
+ @Override public void writerAborts(Transaction txn) { dec(activeWriters) ; inc(abortedWriters) ; }
}
+ // Short name: x++
+ static long inc(AtomicLong x) { return x.getAndIncrement() ; }
+ // Short name: --x
+ static long dec(AtomicLong x) { return x.decrementAndGet() ; }
+
// Transaction policy:
// TSM + WriterEnters, WriterLeaves which may use the semaphore. (+ReaderEnters, ReaderLeaves ??)
@@ -153,15 +161,25 @@ public class TransactionManager
// Queue cleared at en dof any transaction finding itself the only transaction.
class TSM_WriteBackEndTxn extends TSM_Base
{
+ // Safe mode.
+ // Take a READ lock over the base dataset.
+ // Write-back takes a WRITE lock.
+ @Override public void readerStarts(Transaction txn) { txn.getBaseDataset().getLock().enterCriticalSection(Lock.READ) ; }
+ @Override public void writerStarts(Transaction txn) { txn.getBaseDataset().getLock().enterCriticalSection(Lock.READ) ; }
// Currently, the writer semaphore is managed explicitly in the main code.
@Override public void readerFinishes(Transaction txn)
- { processDelayedReplayQueue(txn) ; }
+ {
+ txn.getBaseDataset().getLock().leaveCriticalSection() ;
+ processDelayedReplayQueue(txn) ;
+ }
@Override public void writerCommits(Transaction txn)
{
- if ( activeReaders == 0 )
+ txn.getBaseDataset().getLock().leaveCriticalSection() ;
+
+ if ( activeReaders.get() == 0 )
{
// Can commit immediately.
// Ensure the queue is empty though.
@@ -179,7 +197,10 @@ public class TransactionManager
}
@Override public void writerAborts(Transaction txn)
- { processDelayedReplayQueue(txn) ;}
+ {
+ txn.getBaseDataset().getLock().leaveCriticalSection() ;
+ processDelayedReplayQueue(txn) ;
+ }
}
// Policy for writing back that always writes from the writer by using a
@@ -270,7 +291,7 @@ public class TransactionManager
// // create new transaction
// }
- if ( mode == WRITE && activeWriters > 0 ) // Guard
+ if ( mode == WRITE && activeWriters.get() > 0 ) // Guard
throw new TDBTransactionException("Existing active write transaction") ;
// Even flush queue here.
@@ -345,7 +366,7 @@ public class TransactionManager
// Sync'ed by notifyCommit.
// If we knew which version of the DB each was looking at, we could reduce more often here.
// [TxTDB:TODO]
- if ( activeReaders != 0 || activeWriters != 0 )
+ if ( activeReaders.get() != 0 || activeWriters.get() != 0 )
{
if ( queue.size() > 0 )
if ( log() ) log(format("Pending transactions: R=%d / W=%d", activeReaders, activeWriters), txn) ;
@@ -365,8 +386,8 @@ public class TransactionManager
continue ;
log("Flush delayed commit", txn2) ;
// This takes a Write lock on the DSG - this is where it blocks.
- // **** REPLAYS WHOLE JOURNAL
// **** Related NodeFileTrans: writes at "prepare"
+ checkReplaySafe() ;
enactTransaction(txn2) ;
commitedAwaitingFlush.remove(txn2) ;
@@ -375,10 +396,21 @@ public class TransactionManager
} catch (InterruptedException ex)
{ Log.fatal(this, "Interruped!", ex) ; }
}
+ checkReplaySafe() ;
+
// Whole journal to base database
JournalControl.replay(journal, baseDataset) ;
+
+ checkReplaySafe() ;
}
+ private void checkReplaySafe()
+ {
+ if ( ! checking ) return ;
+ if ( activeReaders.get() != 0 || activeWriters.get() != 0 )
+ log.error("There are now active transactions") ;
+ }
+
synchronized
public void notifyClose(Transaction txn)
{
Modified: incubator/jena/Experimental/TxTDB/trunk/src/test/java/com/hp/hpl/jena/tdb/transaction/TestTransSystem.java
URL: http://svn.apache.org/viewvc/incubator/jena/Experimental/TxTDB/trunk/src/test/java/com/hp/hpl/jena/tdb/transaction/TestTransSystem.java?rev=1153963&r1=1153962&r2=1153963&view=diff
==============================================================================
--- incubator/jena/Experimental/TxTDB/trunk/src/test/java/com/hp/hpl/jena/tdb/transaction/TestTransSystem.java (original)
+++ incubator/jena/Experimental/TxTDB/trunk/src/test/java/com/hp/hpl/jena/tdb/transaction/TestTransSystem.java Thu Aug 4 18:37:04 2011
@@ -32,7 +32,6 @@ import org.junit.BeforeClass ;
import org.openjena.atlas.lib.FileOps ;
import org.openjena.atlas.lib.Lib ;
import org.openjena.atlas.lib.RandomLib ;
-import org.openjena.atlas.logging.Log ;
import org.slf4j.Logger ;
import org.slf4j.LoggerFactory ;
@@ -48,44 +47,46 @@ import com.hp.hpl.jena.tdb.base.file.Loc
/** System testing of the transactions. */
public class TestTransSystem
{
- static { Log.setLog4j() ; }
+ static { org.openjena.atlas.logging.Log.setLog4j() ; }
private static Logger log = LoggerFactory.getLogger(TestTransSystem.class) ;
- static final int Iterations = 10000 ;
- static boolean progress = (! log.isDebugEnabled()) && Iterations > 20 ;
+ static final int Iterations = 10 ;
+ // Output style.
+ static boolean inlineProgress = false ; // (! log.isDebugEnabled()) && Iterations > 20 ;
+ static boolean logging = ! inlineProgress ; // (! log.isDebugEnabled()) && Iterations > 20 ;
/*
* 5/0/5 blocks. with 50/50 pause, 50R/ 20W
* Others?
*/
+// static final int numReaderTasks = 5 ; // Add some
+// static final int numWriterTasksA = 1 ;
+// static final int numWriterTasksC = 5 ;
+//
+// static final int readerSeqRepeats = 5 ;
+// static final int readerMaxPause = 50 ;
+//
+// static final int writerAbortSeqRepeats = 0 ;
+// static final int writerCommitSeqRepeats = 5 ;
+ // static final int writerMaxPause = 20 ;
+
static final int numReaderTasks = 5 ; // Add some
- static final int numWriterTasksA = 1 ;
+ static final int numWriterTasksA = 5 ;
static final int numWriterTasksC = 5 ;
-
- static final int readerSeqRepeats = 5 ;
- static final int readerMaxPause = 50 ;
-
- static final int writerAbortSeqRepeats = 0 ;
- static final int writerCommitSeqRepeats = 5 ;
- static final int writerMaxPause = 20 ;
-
-// static final int numReaderTasks = 5 ;
-// static final int numWriterTasksA = 0 ;
-// static final int numWriterTasksC = 2 ;
-//
-// static final int readerSeqRepeats = 2 ;
-// static final int readerMaxPause = 100 ;
-//
-// static final int writerAbortSeqRepeats = 0 ;
-// static final int writerCommitSeqRepeats = 2 ;
-// static final int writerMaxPause = 20 ;
-
+
+ static final int readerSeqRepeats = 8 ;
+ static final int readerMaxPause = 100 ;
+
+ static final int writerAbortSeqRepeats = 4 ;
+ static final int writerCommitSeqRepeats = 4 ;
+ static final int writerMaxPause = 50 ;
+
public static void main(String...args)
{
- if ( ! progress )
- System.out.println("START") ;
+ if ( logging )
+ log.info("START");
int N = (Iterations < 10) ? 1 : Iterations / 10 ;
N = Math.min(N, 100) ;
@@ -93,22 +94,29 @@ public class TestTransSystem
for ( i = 0 ; i < Iterations ; i++ )
{
- if (!progress)
- System.out.printf("Iteration: %d\n", i) ;
-
- if ( i%N == 0 )
- printf("%03d: ",i) ;
- printf(".") ;
- if ( i%N == (N-1) )
- println() ;
+ if (!logging)
+ log.info("Iteration: %d\n", i) ;
+ if ( inlineProgress )
+ {
+ if ( i%N == 0 )
+ printf("%03d: ",i) ;
+ printf(".") ;
+ if ( i%N == (N-1) )
+ println() ;
+ }
new TestTransSystem().manyReaderAndOneWriter() ;
}
- if ( i%N != 0 )
- System.out.println() ;
- println() ;
- printf("DONE (%03d)\n",i) ;
- if ( ! progress )
- System.out.println("FINISH") ;
+ if ( inlineProgress )
+ {
+ if ( i%N != 0 )
+ System.out.println() ;
+ println() ;
+ printf("DONE (%03d)\n",i) ;
+ }
+ if (inlineProgress)
+ printf("DONE (%03d)\n",i) ;
+ if (logging)
+ log.info("FINISH ({})", i) ;
}
static class Reader implements Callable<Object>
@@ -127,26 +135,34 @@ public class TestTransSystem
@Override
public Object call()
{
+ DatasetGraphTxn dsg = null ;
try
{
int id = gen.incrementAndGet() ;
for (int i = 0; i < repeats; i++)
{
- DatasetGraphTxn dsg = sConn.begin(ReadWrite.READ) ;
+ dsg = sConn.begin(ReadWrite.READ) ;
log.debug("reader start " + id + "/" + i) ;
int x1 = count("SELECT * { ?s ?p ?o }", dsg) ;
pause(maxpause) ;
int x2 = count("SELECT * { ?s ?p ?o }", dsg) ;
- if (x1 != x2) log.warn(format("%s Change seen: %d/%d : id=%d: i=%d",
+ if (x1 != x2) log.warn(format("READER: %s Change seen: %d/%d : id=%d: i=%d",
dsg.getTransaction().getLabel(), x1, x2, id, i)) ;
log.debug("reader finish " + id + "/" + i) ;
dsg.close() ;
+ dsg = null ;
}
return null ;
} catch (RuntimeException ex)
{
System.err.println(ex.getMessage()) ;
+ if ( dsg != null )
+ {
+ dsg.abort() ;
+ dsg.close() ;
+ dsg = null ;
+ }
return null ;
}
}
@@ -170,12 +186,13 @@ public class TestTransSystem
@Override
public Object call()
{
+ DatasetGraphTxn dsg = null ;
try {
int id = gen.incrementAndGet() ;
for ( int i = 0 ; i < repeats ; i++ )
{
log.debug("writer start "+id+"/"+i) ;
- DatasetGraphTxn dsg = sConn.begin(ReadWrite.WRITE) ;
+ dsg = sConn.begin(ReadWrite.WRITE) ;
int x1 = count("SELECT * { ?s ?p ?o }", dsg) ;
int z = change(dsg, id, i) ;
@@ -186,9 +203,12 @@ public class TestTransSystem
TransactionManager txnMgr = dsg.getTransaction().getTxnMgr() ;
SysTxnState state = txnMgr.state() ;
String label = dsg.getTransaction().getLabel() ;
- log.warn(format("%s Change seen: %d + %d != %d : id=%d: i=%d", label, x1, z, x2, id, i)) ;
+ log.warn(format("WRITER: %s Change seen: %d + %d != %d : id=%d: i=%d", label, x1, z, x2, id, i)) ;
log.warn(state.toString()) ;
- System.exit(0) ;
+ dsg.abort() ;
+ dsg.close() ;
+ dsg = null ;
+ return null ;
}
if (commit)
dsg.commit() ;
@@ -198,10 +218,21 @@ public class TestTransSystem
log.debug(state.toString()) ;
log.debug("writer finish "+id+"/"+i) ;
dsg.close() ;
+ dsg = null ;
+ }
+ return null ;
+ }
+ catch (RuntimeException ex)
+ {
+ System.err.println(ex.getMessage()) ;
+ if ( dsg != null )
+ {
+ dsg.abort() ;
+ dsg.close() ;
+ dsg = null ;
}
return null ;
}
- catch (RuntimeException ex) { System.err.println(ex.getMessage()) ; return null ; }
}
// return the delta.
@@ -339,7 +370,7 @@ public class TestTransSystem
private static void printf(String string, Object...args)
{
- if ( progress )
+ if ( inlineProgress )
System.out.printf(string, args) ;
}