You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@jena.apache.org by an...@apache.org on 2011/08/04 20:37:06 UTC

svn commit: r1153963 - in /incubator/jena/Experimental/TxTDB/trunk: ./ src/main/java/com/hp/hpl/jena/tdb/base/buffer/ src/main/java/com/hp/hpl/jena/tdb/transaction/ src/test/java/com/hp/hpl/jena/tdb/transaction/

Author: andy
Date: Thu Aug  4 18:37:04 2011
New Revision: 1153963

URL: http://svn.apache.org/viewvc?rev=1153963&view=rev
Log:
Tidy

Modified:
    incubator/jena/Experimental/TxTDB/trunk/log4j.properties
    incubator/jena/Experimental/TxTDB/trunk/src/main/java/com/hp/hpl/jena/tdb/base/buffer/RecordBuffer.java
    incubator/jena/Experimental/TxTDB/trunk/src/main/java/com/hp/hpl/jena/tdb/transaction/SysTxnState.java
    incubator/jena/Experimental/TxTDB/trunk/src/main/java/com/hp/hpl/jena/tdb/transaction/TransactionManager.java
    incubator/jena/Experimental/TxTDB/trunk/src/test/java/com/hp/hpl/jena/tdb/transaction/TestTransSystem.java

Modified: incubator/jena/Experimental/TxTDB/trunk/log4j.properties
URL: http://svn.apache.org/viewvc/incubator/jena/Experimental/TxTDB/trunk/log4j.properties?rev=1153963&r1=1153962&r2=1153963&view=diff
==============================================================================
--- incubator/jena/Experimental/TxTDB/trunk/log4j.properties (original)
+++ incubator/jena/Experimental/TxTDB/trunk/log4j.properties Thu Aug  4 18:37:04 2011
@@ -22,8 +22,8 @@ log4j.logger.TDB=INFO
 
 log4j.logger.com.hp.hpl.jena.tdb=INFO
 log4j.logger.com.hp.hpl.jena.tdb.transaction=INFO
-## log4j.logger.com.hp.hpl.jena.tdb.transaction.TransactionManager=ALL
-## log4j.logger.com.hp.hpl.jena.tdb.transaction.TestTransSystem=ALL
+##log4j.logger.com.hp.hpl.jena.tdb.transaction.TransactionManager=ALL
+log4j.logger.com.hp.hpl.jena.tdb.transaction.TestTransSystem=ALL
 
 # Joseki server
 log4j.logger.org.joseki=INFO

Modified: incubator/jena/Experimental/TxTDB/trunk/src/main/java/com/hp/hpl/jena/tdb/base/buffer/RecordBuffer.java
URL: http://svn.apache.org/viewvc/incubator/jena/Experimental/TxTDB/trunk/src/main/java/com/hp/hpl/jena/tdb/base/buffer/RecordBuffer.java?rev=1153963&r1=1153962&r2=1153963&view=diff
==============================================================================
--- incubator/jena/Experimental/TxTDB/trunk/src/main/java/com/hp/hpl/jena/tdb/base/buffer/RecordBuffer.java (original)
+++ incubator/jena/Experimental/TxTDB/trunk/src/main/java/com/hp/hpl/jena/tdb/base/buffer/RecordBuffer.java Thu Aug  4 18:37:04 2011
@@ -44,14 +44,14 @@ public class RecordBuffer extends Buffer
     public Record getLow()
     { 
         if ( numSlot == 0 )
-            throw new IllegalArgumentException("Empty RecordBuffer") ;
+            throw new IllegalArgumentException("getLow: Empty RecordBuffer") ;
         return _get(0) ;
     }
     
     public Record getHigh()
     { 
         if ( numSlot == 0 )
-            throw new IllegalArgumentException("Empty RecordBuffer") ;
+            throw new IllegalArgumentException("getHigh: Empty RecordBuffer") ;
         return _get(numSlot-1) ;
     }
     

Modified: incubator/jena/Experimental/TxTDB/trunk/src/main/java/com/hp/hpl/jena/tdb/transaction/SysTxnState.java
URL: http://svn.apache.org/viewvc/incubator/jena/Experimental/TxTDB/trunk/src/main/java/com/hp/hpl/jena/tdb/transaction/SysTxnState.java?rev=1153963&r1=1153962&r2=1153963&view=diff
==============================================================================
--- incubator/jena/Experimental/TxTDB/trunk/src/main/java/com/hp/hpl/jena/tdb/transaction/SysTxnState.java (original)
+++ incubator/jena/Experimental/TxTDB/trunk/src/main/java/com/hp/hpl/jena/tdb/transaction/SysTxnState.java Thu Aug  4 18:37:04 2011
@@ -20,20 +20,20 @@ package com.hp.hpl.jena.tdb.transaction;
 
 public class SysTxnState
 {
-    final public int activeReaders ; 
-    final public int activeWriters ;
-    final public int finishedReaders ;
-    final public int committedWriters ;
-    final public int abortedWriters ;
-    final public int queuedCommits ;
+    final public long activeReaders ; 
+    final public long activeWriters ;
+    final public long finishedReaders ;
+    final public long committedWriters ;
+    final public long abortedWriters ;
+    final public long queuedCommits ;
     
     SysTxnState(TransactionManager tm)
     {
-        activeReaders = tm.activeReaders ;
-        activeWriters = tm.activeWriters ;
-        finishedReaders = tm.finishedReaders ;
-        committedWriters = tm.committedWriters ;
-        abortedWriters = tm.abortedWriters ;
+        activeReaders = tm.activeReaders.get() ;
+        activeWriters = tm.activeWriters.get() ;
+        finishedReaders = tm.finishedReaders.get() ;
+        committedWriters = tm.committedWriters.get() ;
+        abortedWriters = tm.abortedWriters.get() ;
         queuedCommits = tm.commitedAwaitingFlush.size() ;
     }
     

Modified: incubator/jena/Experimental/TxTDB/trunk/src/main/java/com/hp/hpl/jena/tdb/transaction/TransactionManager.java
URL: http://svn.apache.org/viewvc/incubator/jena/Experimental/TxTDB/trunk/src/main/java/com/hp/hpl/jena/tdb/transaction/TransactionManager.java?rev=1153963&r1=1153962&r2=1153963&view=diff
==============================================================================
--- incubator/jena/Experimental/TxTDB/trunk/src/main/java/com/hp/hpl/jena/tdb/transaction/TransactionManager.java (original)
+++ incubator/jena/Experimental/TxTDB/trunk/src/main/java/com/hp/hpl/jena/tdb/transaction/TransactionManager.java Thu Aug  4 18:37:04 2011
@@ -26,6 +26,7 @@ import org.openjena.atlas.logging.Log ;
 import org.slf4j.Logger ;
 import org.slf4j.LoggerFactory ;
 
+import com.hp.hpl.jena.shared.Lock ;
 import com.hp.hpl.jena.tdb.DatasetGraphTxn ;
 import com.hp.hpl.jena.tdb.ReadWrite ;
 import com.hp.hpl.jena.tdb.store.DatasetGraphTDB ;
@@ -36,6 +37,8 @@ public class TransactionManager
     // TODO Don't keep counters, keep lists.
     // TODO Useful logging.
     
+    private static boolean checking = true ;
+    
     private static Logger log = LoggerFactory.getLogger(TransactionManager.class) ;
     private Set<Transaction> activeTransactions = new HashSet<Transaction>() ;
     
@@ -63,13 +66,13 @@ public class TransactionManager
     
     static AtomicLong transactionId = new AtomicLong(1) ;
     
-    int activeReaders = 0 ; 
-    int activeWriters = 0 ;  // 0 or 1
+    AtomicLong activeReaders = new AtomicLong(0) ; 
+    AtomicLong activeWriters = new AtomicLong(0) ; // 0 or 1
     
     // Misc stats
-    int finishedReaders = 0 ;
-    int committedWriters = 0 ;
-    int abortedWriters = 0 ;
+    AtomicLong finishedReaders = new AtomicLong(0) ;
+    AtomicLong committedWriters = new AtomicLong(0) ;
+    AtomicLong abortedWriters = new AtomicLong(0) ;
     
     // Ensure single writer.
     private Semaphore writersWaiting = new Semaphore(1, true) ;
@@ -138,13 +141,18 @@ public class TransactionManager
         @Override public void transactionStarts(Transaction txn)    { activeTransactions.add(txn) ; }
         @Override public void transactionFinishes(Transaction txn)  { activeTransactions.remove(txn) ; }
         @Override public void transactionCloses(Transaction txn)    { }
-        @Override public void readerStarts(Transaction txn)         { activeReaders++ ; }
-        @Override public void readerFinishes(Transaction txn)       { activeReaders-- ; finishedReaders++ ; }
-        @Override public void writerStarts(Transaction txn)         { activeWriters++ ; }
-        @Override public void writerCommits(Transaction txn)        { activeWriters-- ; committedWriters++ ; }
-        @Override public void writerAborts(Transaction txn)         { activeWriters-- ; abortedWriters++ ; }
+        @Override public void readerStarts(Transaction txn)         { inc(activeReaders) ; }
+        @Override public void readerFinishes(Transaction txn)       { dec(activeReaders) ; inc(finishedReaders); }
+        @Override public void writerStarts(Transaction txn)         { inc(activeWriters) ; }
+        @Override public void writerCommits(Transaction txn)        { dec(activeWriters) ; inc(committedWriters) ; }
+        @Override public void writerAborts(Transaction txn)         { dec(activeWriters) ; inc(abortedWriters) ; }
     }
     
+    // Short name: x++
+    static long inc(AtomicLong x)   { return x.getAndIncrement() ; }
+    // Short name: --x
+    static long dec(AtomicLong x)   { return x.decrementAndGet() ; }
+    
     // Transaction policy:
     // TSM + WriterEnters, WriterLeaves which may use the semaphore. (+ReaderEnters, ReaderLeaves ??)
     
@@ -153,15 +161,25 @@ public class TransactionManager
     // Queue cleared at en dof any transaction finding itself the only transaction.
     class TSM_WriteBackEndTxn extends TSM_Base
     {
+        // Safe mode.
+        // Take a READ lock over the base dataset.
+        // Write-back takes a WRITE lock.
+        @Override public void readerStarts(Transaction txn)    { txn.getBaseDataset().getLock().enterCriticalSection(Lock.READ) ; }
+        @Override public void writerStarts(Transaction txn)    { txn.getBaseDataset().getLock().enterCriticalSection(Lock.READ) ; }
         
         // Currently, the writer semaphore is managed explicitly in the main code.
         
         @Override public void readerFinishes(Transaction txn)       
-        { processDelayedReplayQueue(txn) ; }
+        { 
+            txn.getBaseDataset().getLock().leaveCriticalSection() ;
+            processDelayedReplayQueue(txn) ;
+        }
         
         @Override public void writerCommits(Transaction txn)
         {
-            if ( activeReaders == 0 )
+            txn.getBaseDataset().getLock().leaveCriticalSection() ;
+
+            if ( activeReaders.get() == 0 )
             {
                 // Can commit immediately.
                 // Ensure the queue is empty though.
@@ -179,7 +197,10 @@ public class TransactionManager
         }
         
         @Override public void writerAborts(Transaction txn)
-        { processDelayedReplayQueue(txn) ;}
+        { 
+            txn.getBaseDataset().getLock().leaveCriticalSection() ;
+            processDelayedReplayQueue(txn) ;
+        }
     }
     
     // Policy for writing back that always writes from the writer by using a
@@ -270,7 +291,7 @@ public class TransactionManager
 //            //   create new transaction 
 //        }
         
-        if ( mode == WRITE && activeWriters > 0 )    // Guard
+        if ( mode == WRITE && activeWriters.get() > 0 )    // Guard
             throw new TDBTransactionException("Existing active write transaction") ;
 
         // Even flush queue here.
@@ -345,7 +366,7 @@ public class TransactionManager
         // Sync'ed by notifyCommit.
         // If we knew which version of the DB each was looking at, we could reduce more often here.
         // [TxTDB:TODO]
-        if ( activeReaders != 0 || activeWriters != 0 )
+        if ( activeReaders.get() != 0 || activeWriters.get() != 0 )
         {
             if ( queue.size() > 0 )
                 if ( log() ) log(format("Pending transactions: R=%d / W=%d", activeReaders, activeWriters), txn) ;
@@ -365,8 +386,8 @@ public class TransactionManager
                     continue ;
                 log("Flush delayed commit", txn2) ;
                 // This takes a Write lock on the  DSG - this is where it blocks.
-                // **** REPLAYS WHOLE JOURNAL
                 // **** Related NodeFileTrans: writes at "prepare" 
+                checkReplaySafe() ;
                 enactTransaction(txn2) ;
                 commitedAwaitingFlush.remove(txn2) ;
                 
@@ -375,10 +396,21 @@ public class TransactionManager
             } catch (InterruptedException ex)
             { Log.fatal(this, "Interruped!", ex) ; }
         }
+        checkReplaySafe() ;
+
         // Whole journal to base database
         JournalControl.replay(journal, baseDataset) ;
+        
+        checkReplaySafe() ;
     }
 
+    private void checkReplaySafe()
+    {
+        if ( ! checking ) return ;
+        if ( activeReaders.get() != 0 || activeWriters.get() != 0 )
+            log.error("There are now active transactions") ;
+    }
+    
     synchronized
     public void notifyClose(Transaction txn)
     {

Modified: incubator/jena/Experimental/TxTDB/trunk/src/test/java/com/hp/hpl/jena/tdb/transaction/TestTransSystem.java
URL: http://svn.apache.org/viewvc/incubator/jena/Experimental/TxTDB/trunk/src/test/java/com/hp/hpl/jena/tdb/transaction/TestTransSystem.java?rev=1153963&r1=1153962&r2=1153963&view=diff
==============================================================================
--- incubator/jena/Experimental/TxTDB/trunk/src/test/java/com/hp/hpl/jena/tdb/transaction/TestTransSystem.java (original)
+++ incubator/jena/Experimental/TxTDB/trunk/src/test/java/com/hp/hpl/jena/tdb/transaction/TestTransSystem.java Thu Aug  4 18:37:04 2011
@@ -32,7 +32,6 @@ import org.junit.BeforeClass ;
 import org.openjena.atlas.lib.FileOps ;
 import org.openjena.atlas.lib.Lib ;
 import org.openjena.atlas.lib.RandomLib ;
-import org.openjena.atlas.logging.Log ;
 import org.slf4j.Logger ;
 import org.slf4j.LoggerFactory ;
 
@@ -48,44 +47,46 @@ import com.hp.hpl.jena.tdb.base.file.Loc
 /** System testing of the transactions. */
 public class TestTransSystem
 {
-    static { Log.setLog4j() ; }
+    static { org.openjena.atlas.logging.Log.setLog4j() ; }
     private static Logger log = LoggerFactory.getLogger(TestTransSystem.class) ;
 
-    static final int Iterations       = 10000 ;
-    static boolean progress           = (! log.isDebugEnabled()) && Iterations > 20 ;
+    static final int Iterations       = 10 ;
+    // Output style.
+    static boolean inlineProgress           = false ; // (! log.isDebugEnabled()) && Iterations > 20 ;
+    static boolean logging                  = ! inlineProgress ; // (! log.isDebugEnabled()) && Iterations > 20 ;
     
     /*
      * 5/0/5 blocks. with 50/50 pause, 50R/ 20W
      * Others?
      */
     
+//    static final int numReaderTasks         = 5 ;   // Add some
+//    static final int numWriterTasksA        = 1 ;
+//    static final int numWriterTasksC        = 5 ;
+//    
+//    static final int readerSeqRepeats       = 5 ;    
+//    static final int readerMaxPause         = 50 ;
+//    
+//    static final int writerAbortSeqRepeats  = 0 ;
+//    static final int writerCommitSeqRepeats = 5 ;
+    //    static final int writerMaxPause         = 20 ;
+
     static final int numReaderTasks         = 5 ;   // Add some
-    static final int numWriterTasksA        = 1 ;
+    static final int numWriterTasksA        = 5 ;
     static final int numWriterTasksC        = 5 ;
-    
-    static final int readerSeqRepeats       = 5 ;    
-    static final int readerMaxPause         = 50 ;
-    
-    static final int writerAbortSeqRepeats  = 0 ;
-    static final int writerCommitSeqRepeats = 5 ;
-    static final int writerMaxPause         = 20 ;
-    
-//  static final int numReaderTasks         = 5 ;
-//  static final int numWriterTasksA        = 0 ;
-//  static final int numWriterTasksC        = 2 ;
-//  
-//  static final int readerSeqRepeats       = 2 ;    
-//  static final int readerMaxPause         = 100 ;
-//  
-//  static final int writerAbortSeqRepeats  = 0 ;
-//  static final int writerCommitSeqRepeats = 2 ;
-//  static final int writerMaxPause         = 20 ;
-  
+
+    static final int readerSeqRepeats       = 8 ;
+    static final int readerMaxPause         = 100 ;
+
+    static final int writerAbortSeqRepeats  = 4 ;
+    static final int writerCommitSeqRepeats = 4 ;
+    static final int writerMaxPause         = 50 ;
+
     
     public static void main(String...args)
     {
-        if ( ! progress )
-            System.out.println("START") ;
+        if ( logging )
+            log.info("START");
         
         int N = (Iterations < 10) ? 1 : Iterations / 10 ;
         N = Math.min(N, 100) ;
@@ -93,22 +94,29 @@ public class TestTransSystem
         
         for ( i = 0 ; i < Iterations ; i++ )
         {
-            if (!progress)
-               System.out.printf("Iteration: %d\n", i) ;
-            
-            if ( i%N == 0 )
-                printf("%03d: ",i) ;
-            printf(".") ;
-            if ( i%N == (N-1) )
-                println() ;
+            if (!logging)
+                log.info("Iteration: %d\n", i) ;
+            if ( inlineProgress )
+            {
+                if ( i%N == 0 )
+                    printf("%03d: ",i) ;
+                printf(".") ;
+                if ( i%N == (N-1) )
+                    println() ;
+            }
             new TestTransSystem().manyReaderAndOneWriter() ;
         }
-        if ( i%N != 0 )
-            System.out.println() ;
-        println() ;
-        printf("DONE (%03d)\n",i) ;
-        if ( ! progress )
-            System.out.println("FINISH") ;
+        if ( inlineProgress )
+        {
+            if ( i%N != 0 )
+                System.out.println() ;
+            println() ;
+            printf("DONE (%03d)\n",i) ;
+        }
+        if (inlineProgress)
+            printf("DONE (%03d)\n",i) ;
+        if (logging)
+            log.info("FINISH ({})", i) ;
     }
     
     static class Reader implements Callable<Object>
@@ -127,26 +135,34 @@ public class TestTransSystem
         @Override
         public Object call()
         {
+            DatasetGraphTxn dsg = null ;
             try
             {
                 int id = gen.incrementAndGet() ;
                 for (int i = 0; i < repeats; i++)
                 {
-                    DatasetGraphTxn dsg = sConn.begin(ReadWrite.READ) ;
+                    dsg = sConn.begin(ReadWrite.READ) ;
                     log.debug("reader start " + id + "/" + i) ;
 
                     int x1 = count("SELECT * { ?s ?p ?o }", dsg) ;
                     pause(maxpause) ;
                     int x2 = count("SELECT * { ?s ?p ?o }", dsg) ;
-                    if (x1 != x2) log.warn(format("%s Change seen: %d/%d : id=%d: i=%d",
+                    if (x1 != x2) log.warn(format("READER: %s Change seen: %d/%d : id=%d: i=%d",
                                                   dsg.getTransaction().getLabel(), x1, x2, id, i)) ;
                     log.debug("reader finish " + id + "/" + i) ;
                     dsg.close() ;
+                    dsg = null ;
                 }
                 return null ;
             } catch (RuntimeException ex)
             {
                 System.err.println(ex.getMessage()) ;
+                if ( dsg != null )
+                {
+                    dsg.abort() ;
+                    dsg.close() ;
+                    dsg = null ;
+                }
                 return null ;
             }
         }
@@ -170,12 +186,13 @@ public class TestTransSystem
         @Override
         public Object call()
         {
+            DatasetGraphTxn dsg = null ;
             try { 
                 int id = gen.incrementAndGet() ;
                 for ( int i = 0 ; i < repeats ; i++ )
                 {
                     log.debug("writer start "+id+"/"+i) ;                
-                    DatasetGraphTxn dsg = sConn.begin(ReadWrite.WRITE) ;
+                    dsg = sConn.begin(ReadWrite.WRITE) ;
 
                     int x1 = count("SELECT * { ?s ?p ?o }", dsg) ;
                     int z = change(dsg, id, i) ;
@@ -186,9 +203,12 @@ public class TestTransSystem
                         TransactionManager txnMgr = dsg.getTransaction().getTxnMgr() ;
                         SysTxnState state = txnMgr.state() ;
                         String label = dsg.getTransaction().getLabel() ; 
-                        log.warn(format("%s Change seen: %d + %d != %d : id=%d: i=%d", label, x1, z, x2, id, i)) ;
+                        log.warn(format("WRITER: %s Change seen: %d + %d != %d : id=%d: i=%d", label, x1, z, x2, id, i)) ;
                         log.warn(state.toString()) ;
-                        System.exit(0) ;
+                        dsg.abort() ;
+                        dsg.close() ;
+                        dsg = null ;
+                        return null ;
                     }
                     if (commit) 
                         dsg.commit() ;
@@ -198,10 +218,21 @@ public class TestTransSystem
                     log.debug(state.toString()) ;
                     log.debug("writer finish "+id+"/"+i) ;                
                     dsg.close() ;
+                    dsg = null ;
+                }
+                return null ;
+            }
+            catch (RuntimeException ex)
+            { 
+                System.err.println(ex.getMessage()) ;
+                if ( dsg != null )
+                {
+                    dsg.abort() ;
+                    dsg.close() ;
+                    dsg = null ;
                 }
                 return null ;
             }
-            catch (RuntimeException ex) { System.err.println(ex.getMessage()) ; return null ; }
         }
     
         // return the delta.
@@ -339,7 +370,7 @@ public class TestTransSystem
 
     private static void printf(String string, Object...args)
     {
-        if ( progress )
+        if ( inlineProgress )
             System.out.printf(string, args) ;
     }