You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@jena.apache.org by an...@apache.org on 2012/01/25 18:33:20 UTC
svn commit: r1235846 - in /incubator/jena/Jena2/TDB/trunk: src-dev/dev/
src/main/java/com/hp/hpl/jena/tdb/
src/main/java/com/hp/hpl/jena/tdb/base/file/
src/main/java/com/hp/hpl/jena/tdb/transaction/
src/test/java/com/hp/hpl/jena/tdb/transaction/
Author: andy
Date: Wed Jan 25 17:33:20 2012
New Revision: 1235846
URL: http://svn.apache.org/viewvc?rev=1235846&view=rev
Log:
Avoid opening the journal twice during recovery.
Add .isEmpty to BufferChannel.
Added:
incubator/jena/Jena2/TDB/trunk/src-dev/dev/Jena202_TDBDatasetSetChurn.java
Modified:
incubator/jena/Jena2/TDB/trunk/src/main/java/com/hp/hpl/jena/tdb/StoreConnection.java
incubator/jena/Jena2/TDB/trunk/src/main/java/com/hp/hpl/jena/tdb/TDBFactory.java
incubator/jena/Jena2/TDB/trunk/src/main/java/com/hp/hpl/jena/tdb/TDBFactoryTxn.java
incubator/jena/Jena2/TDB/trunk/src/main/java/com/hp/hpl/jena/tdb/base/file/BufferChannel.java
incubator/jena/Jena2/TDB/trunk/src/main/java/com/hp/hpl/jena/tdb/base/file/BufferChannelFile.java
incubator/jena/Jena2/TDB/trunk/src/main/java/com/hp/hpl/jena/tdb/base/file/BufferChannelMem.java
incubator/jena/Jena2/TDB/trunk/src/main/java/com/hp/hpl/jena/tdb/base/file/FileBase.java
incubator/jena/Jena2/TDB/trunk/src/main/java/com/hp/hpl/jena/tdb/transaction/Journal.java
incubator/jena/Jena2/TDB/trunk/src/main/java/com/hp/hpl/jena/tdb/transaction/JournalControl.java
incubator/jena/Jena2/TDB/trunk/src/main/java/com/hp/hpl/jena/tdb/transaction/TransactionManager.java
incubator/jena/Jena2/TDB/trunk/src/test/java/com/hp/hpl/jena/tdb/transaction/TestTransactionTDB.java
Added: incubator/jena/Jena2/TDB/trunk/src-dev/dev/Jena202_TDBDatasetSetChurn.java
URL: http://svn.apache.org/viewvc/incubator/jena/Jena2/TDB/trunk/src-dev/dev/Jena202_TDBDatasetSetChurn.java?rev=1235846&view=auto
==============================================================================
--- incubator/jena/Jena2/TDB/trunk/src-dev/dev/Jena202_TDBDatasetSetChurn.java (added)
+++ incubator/jena/Jena2/TDB/trunk/src-dev/dev/Jena202_TDBDatasetSetChurn.java Wed Jan 25 17:33:20 2012
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package dev;
+
+import org.openjena.atlas.iterator.Iter ;
+import org.openjena.atlas.lib.FileOps ;
+import org.openjena.atlas.logging.Log ;
+
+import com.hp.hpl.jena.graph.Triple ;
+import com.hp.hpl.jena.query.Dataset ;
+import com.hp.hpl.jena.query.ReadWrite ;
+import com.hp.hpl.jena.sparql.sse.SSE ;
+import com.hp.hpl.jena.tdb.TDBFactory ;
+
+public class Jena202_TDBDatasetSetChurn
+{
+ static { Log.setLog4j() ; }
+
+ public static void main(String[] argv)
+ {
+ String DB = "DB" ;
+
+ FileOps.ensureDir(DB) ;
+
+ Triple t = SSE.parseTriple("(<s> <p> <o>)") ;
+
+ int i ;
+ int Chunk = 1000 ;
+ int tock = 100 ;
+ for ( i = 0 ; i < 10000 ; i ++ )
+ {
+ if ( i != 0 && i%tock == 0 )
+ System.out.println() ;
+ System.out.print(".") ;
+
+ Dataset ds = TDBFactory.createDataset(DB) ;
+ ds.begin(ReadWrite.WRITE) ;
+ for ( int j = i*Chunk; j < (i+1)*Chunk ; j++ )
+ ds.getDefaultModel().getGraph().add(SSE.parseTriple("(<s> <p> 'X"+j+"')")) ;
+ ds.commit();
+ ds.end() ;
+
+ ds.begin(ReadWrite.READ) ;
+ Iter.count(ds.getDefaultModel().getGraph().find(t)) ;
+ ds.end() ;
+ //ds.close() ;
+
+ //TDBFactory.release(ds) ;
+
+ }
+ System.out.println() ;
+ System.out.println("DONE") ;
+ System.exit(0) ;
+ }
+}
Modified: incubator/jena/Jena2/TDB/trunk/src/main/java/com/hp/hpl/jena/tdb/StoreConnection.java
URL: http://svn.apache.org/viewvc/incubator/jena/Jena2/TDB/trunk/src/main/java/com/hp/hpl/jena/tdb/StoreConnection.java?rev=1235846&r1=1235845&r2=1235846&view=diff
==============================================================================
--- incubator/jena/Jena2/TDB/trunk/src/main/java/com/hp/hpl/jena/tdb/StoreConnection.java (original)
+++ incubator/jena/Jena2/TDB/trunk/src/main/java/com/hp/hpl/jena/tdb/StoreConnection.java Wed Jan 25 17:33:20 2012
@@ -139,13 +139,24 @@ public class StoreConnection
*/
public static synchronized StoreConnection make(DatasetGraphTDB dsg)
{
+ if ( dsg instanceof DatasetGraphTxn )
+ {
+ //((DatasetGraphTxn)dsg).getTransaction().getBaseDataset() ;
+ throw new TDBTransactionException("Can't make a StoreConnection from a transaction instance - need the base storage DatasetGraphTDB") ;
+ }
Location location = dsg.getLocation() ;
+
StoreConnection sConn = cache.get(location) ;
if ( sConn == null )
sConn = _makeAndCache(dsg) ;
return sConn ;
}
+ /** Return the StoreConnection if one already exists for this location, else return null */
+ public static synchronized StoreConnection getExisting(Location location)
+ {
+ return cache.get(location) ;
+ }
private static StoreConnection _makeAndCache(DatasetGraphTDB dsg)
{
@@ -155,7 +166,7 @@ public class StoreConnection
if ( sConn == null )
{
sConn = new StoreConnection(dsg) ;
- JournalControl.recovery(sConn.baseDSG) ;
+ JournalControl.recoverFromJournal(dsg, sConn.transactionManager.getJournal()) ;
if ( ! location.isMemUnique() )
// Don't cache use-once in-memory datasets.
cache.put(location, sConn) ;
Modified: incubator/jena/Jena2/TDB/trunk/src/main/java/com/hp/hpl/jena/tdb/TDBFactory.java
URL: http://svn.apache.org/viewvc/incubator/jena/Jena2/TDB/trunk/src/main/java/com/hp/hpl/jena/tdb/TDBFactory.java?rev=1235846&r1=1235845&r2=1235846&view=diff
==============================================================================
--- incubator/jena/Jena2/TDB/trunk/src/main/java/com/hp/hpl/jena/tdb/TDBFactory.java (original)
+++ incubator/jena/Jena2/TDB/trunk/src/main/java/com/hp/hpl/jena/tdb/TDBFactory.java Wed Jan 25 17:33:20 2012
@@ -119,7 +119,7 @@ public class TDBFactory
return asTransactional(dsg) ;
}
- /** By default, TDBFcatory returns Datasets and DatasetGraphs that can be used in
+ /** By default, TDBFactory returns Datasets and DatasetGraphs that can be used in
* transactions. To force a return to TDB 0.8.x behaviour of returning
* Datasets and DatasetGraphs attached directly to the storage, set this
* to false. Warning: it's global.
Modified: incubator/jena/Jena2/TDB/trunk/src/main/java/com/hp/hpl/jena/tdb/TDBFactoryTxn.java
URL: http://svn.apache.org/viewvc/incubator/jena/Jena2/TDB/trunk/src/main/java/com/hp/hpl/jena/tdb/TDBFactoryTxn.java?rev=1235846&r1=1235845&r2=1235846&view=diff
==============================================================================
--- incubator/jena/Jena2/TDB/trunk/src/main/java/com/hp/hpl/jena/tdb/TDBFactoryTxn.java (original)
+++ incubator/jena/Jena2/TDB/trunk/src/main/java/com/hp/hpl/jena/tdb/TDBFactoryTxn.java Wed Jan 25 17:33:20 2012
@@ -28,7 +28,7 @@ import com.hp.hpl.jena.tdb.store.Dataset
import com.hp.hpl.jena.tdb.transaction.DatasetGraphTransaction ;
/** factory for creating objects datasets backed by TDB storage which support transactions */
-public class TDBFactoryTxn
+class TDBFactoryTxn
{
// This is TDBFactory with a different maker.??
Modified: incubator/jena/Jena2/TDB/trunk/src/main/java/com/hp/hpl/jena/tdb/base/file/BufferChannel.java
URL: http://svn.apache.org/viewvc/incubator/jena/Jena2/TDB/trunk/src/main/java/com/hp/hpl/jena/tdb/base/file/BufferChannel.java?rev=1235846&r1=1235845&r2=1235846&view=diff
==============================================================================
--- incubator/jena/Jena2/TDB/trunk/src/main/java/com/hp/hpl/jena/tdb/base/file/BufferChannel.java (original)
+++ incubator/jena/Jena2/TDB/trunk/src/main/java/com/hp/hpl/jena/tdb/base/file/BufferChannel.java Wed Jan 25 17:33:20 2012
@@ -84,6 +84,9 @@ public interface BufferChannel extends S
/** Length of storage, in bytes.*/
public long size() ;
+ /** Is it empty? */
+ public boolean isEmpty() ;
+
/** useful display string */
public String getLabel() ;
Modified: incubator/jena/Jena2/TDB/trunk/src/main/java/com/hp/hpl/jena/tdb/base/file/BufferChannelFile.java
URL: http://svn.apache.org/viewvc/incubator/jena/Jena2/TDB/trunk/src/main/java/com/hp/hpl/jena/tdb/base/file/BufferChannelFile.java?rev=1235846&r1=1235845&r2=1235846&view=diff
==============================================================================
--- incubator/jena/Jena2/TDB/trunk/src/main/java/com/hp/hpl/jena/tdb/base/file/BufferChannelFile.java (original)
+++ incubator/jena/Jena2/TDB/trunk/src/main/java/com/hp/hpl/jena/tdb/base/file/BufferChannelFile.java Wed Jan 25 17:33:20 2012
@@ -103,6 +103,13 @@ public class BufferChannelFile implement
}
@Override
+ public boolean isEmpty()
+ {
+ try { return file.channel().size() == 0 ; }
+ catch (IOException e) { IO.exception(e) ; return false ; }
+ }
+
+ @Override
public void sync()
{
file.sync() ;
Modified: incubator/jena/Jena2/TDB/trunk/src/main/java/com/hp/hpl/jena/tdb/base/file/BufferChannelMem.java
URL: http://svn.apache.org/viewvc/incubator/jena/Jena2/TDB/trunk/src/main/java/com/hp/hpl/jena/tdb/base/file/BufferChannelMem.java?rev=1235846&r1=1235845&r2=1235846&view=diff
==============================================================================
--- incubator/jena/Jena2/TDB/trunk/src/main/java/com/hp/hpl/jena/tdb/base/file/BufferChannelMem.java (original)
+++ incubator/jena/Jena2/TDB/trunk/src/main/java/com/hp/hpl/jena/tdb/base/file/BufferChannelMem.java Wed Jan 25 17:33:20 2012
@@ -183,6 +183,14 @@ public class BufferChannelMem implements
@Override
synchronized
+ public boolean isEmpty()
+ {
+ checkIfClosed() ;
+ return size() == 0 ;
+ }
+
+ @Override
+ synchronized
public void sync()
{
checkIfClosed() ;
Modified: incubator/jena/Jena2/TDB/trunk/src/main/java/com/hp/hpl/jena/tdb/base/file/FileBase.java
URL: http://svn.apache.org/viewvc/incubator/jena/Jena2/TDB/trunk/src/main/java/com/hp/hpl/jena/tdb/base/file/FileBase.java?rev=1235846&r1=1235845&r2=1235846&view=diff
==============================================================================
--- incubator/jena/Jena2/TDB/trunk/src/main/java/com/hp/hpl/jena/tdb/base/file/FileBase.java (original)
+++ incubator/jena/Jena2/TDB/trunk/src/main/java/com/hp/hpl/jena/tdb/base/file/FileBase.java Wed Jan 25 17:33:20 2012
@@ -48,10 +48,10 @@ public final class FileBase implements S
private /*public*/ FileBase(String filename, String mode)
{
- DebugThis = DEBUG && filename.contains("nodes.dat-jrnl") ;
+ DebugThis = false ;
id = (counter++) ;
- if ( DebugThis )
+ if ( DebugThis && log.isDebugEnabled() )
log.debug("open: ["+id+"]"+filename) ;
this.filename = filename ;
channel = ChannelManager.acquire(filename, mode) ;
Modified: incubator/jena/Jena2/TDB/trunk/src/main/java/com/hp/hpl/jena/tdb/transaction/Journal.java
URL: http://svn.apache.org/viewvc/incubator/jena/Jena2/TDB/trunk/src/main/java/com/hp/hpl/jena/tdb/transaction/Journal.java?rev=1235846&r1=1235845&r2=1235846&view=diff
==============================================================================
--- incubator/jena/Jena2/TDB/trunk/src/main/java/com/hp/hpl/jena/tdb/transaction/Journal.java (original)
+++ incubator/jena/Jena2/TDB/trunk/src/main/java/com/hp/hpl/jena/tdb/transaction/Journal.java Wed Jan 25 17:33:20 2012
@@ -300,6 +300,8 @@ class Journal implements Iterable<Journa
public long size() { return channel.size() ; }
+ public boolean isEmpty() { return channel.size() == 0 ; }
+
public void truncate(long size) { channel.truncate(size) ; }
public void append() { position(size()) ; }
Modified: incubator/jena/Jena2/TDB/trunk/src/main/java/com/hp/hpl/jena/tdb/transaction/JournalControl.java
URL: http://svn.apache.org/viewvc/incubator/jena/Jena2/TDB/trunk/src/main/java/com/hp/hpl/jena/tdb/transaction/JournalControl.java?rev=1235846&r1=1235845&r2=1235846&view=diff
==============================================================================
--- incubator/jena/Jena2/TDB/trunk/src/main/java/com/hp/hpl/jena/tdb/transaction/JournalControl.java (original)
+++ incubator/jena/Jena2/TDB/trunk/src/main/java/com/hp/hpl/jena/tdb/transaction/JournalControl.java Wed Jan 25 17:33:20 2012
@@ -63,6 +63,7 @@ public class JournalControl
}
}
+ /** Recover a base storage DatasetGraph */
public static void recovery(DatasetGraphTDB dsg)
{
if ( dsg instanceof DatasetGraphTxn )
@@ -76,13 +77,13 @@ public class JournalControl
// Do we need to recover?
Journal journal = findJournal(dsg) ;
- if ( journal == null )
+ if ( journal == null || journal.isEmpty() )
return ;
for ( FileRef fileRef : dsg.getConfig().nodeTables.keySet() )
recoverNodeDat(dsg, fileRef) ;
- recoverSystemJournal(journal, dsg) ;
-
+ recoverFromJournal(dsg, journal) ;
+ journal.close() ;
// Recovery complete. Tidy up. Node journal files have already been handled.
if ( journal.getFilename() != null )
{
@@ -97,6 +98,7 @@ public class JournalControl
String journalFilename = loc.absolute(Names.journalFile) ;
File f = new File(journalFilename) ;
//if ( FileOps.exists(journalFilename)
+
if ( f.exists() && f.isFile() && f.length() > 0 )
return Journal.create(loc) ;
else
@@ -105,12 +107,18 @@ public class JournalControl
// New recovery - scan to commit, enact, scan, ....
- /** Recovery from the system journal.
+ /** Recovery from a journal.
* Find if there is a commit record; if so, reply the journal to that point.
* Try to see if there is another commit record ...
*/
- private static void recoverSystemJournal(Journal jrnl, DatasetGraphTDB dsg)
+ public static void recoverFromJournal(DatasetGraphTDB dsg, Journal jrnl )
{
+ if ( dsg instanceof DatasetGraphTxn )
+ throw new TDBTransactionException("Recovery works on the base dataset, not a transactional one") ;
+
+ if ( jrnl.isEmpty() )
+ return ;
+
long posn = 0 ;
for ( ;; )
{
@@ -122,7 +130,6 @@ public class JournalControl
// We have replayed the journals - clean up.
jrnl.truncate(0) ;
- jrnl.close();
dsg.sync() ;
}
@@ -169,47 +176,47 @@ public class JournalControl
} finally { Iter.close(iter) ; }
}
- /** Recovery from the system journal.
- * Find is there is a commit record; if so, reply the journal.
- */
- private static void recoverSystemJournal_0(DatasetGraphTDB dsg)
- {
- Location loc = dsg.getLocation() ;
- String journalFilename = loc.absolute(Names.journalFile) ;
- File f = new File(journalFilename) ;
- //if ( FileOps.exists(journalFilename)
- if ( f.exists() && f.isFile() && f.length() > 0 )
- {
- Journal jrnl = Journal.create(loc) ;
- // Scan for commit.
- boolean committed = false ;
- for ( JournalEntry e : jrnl )
- {
- if ( e.getType() == JournalEntryType.Commit )
- committed = true ;
- else
- {
- if ( committed )
- {
- errlog.warn("Extra journal entries ("+loc+")") ;
- break ;
- }
- }
- }
- if ( committed )
- {
- syslog.info("Recovering committed transaction") ;
- // The NodeTable Journal has already been done!
- JournalControl.replay(jrnl, dsg) ;
- }
- jrnl.truncate(0) ;
- jrnl.close();
- dsg.sync() ;
- }
-
- if ( f.exists() )
- FileOps.delete(journalFilename) ;
- }
+// /** Recovery from the system journal.
+// * Find is there is a commit record; if so, reply the journal.
+// */
+// private static void recoverSystemJournal_0(DatasetGraphTDB dsg)
+// {
+// Location loc = dsg.getLocation() ;
+// String journalFilename = loc.absolute(Names.journalFile) ;
+// File f = new File(journalFilename) ;
+// //if ( FileOps.exists(journalFilename)
+// if ( f.exists() && f.isFile() && f.length() > 0 )
+// {
+// Journal jrnl = Journal.create(loc) ;
+// // Scan for commit.
+// boolean committed = false ;
+// for ( JournalEntry e : jrnl )
+// {
+// if ( e.getType() == JournalEntryType.Commit )
+// committed = true ;
+// else
+// {
+// if ( committed )
+// {
+// errlog.warn("Extra journal entries ("+loc+")") ;
+// break ;
+// }
+// }
+// }
+// if ( committed )
+// {
+// syslog.info("Recovering committed transaction") ;
+// // The NodeTable Journal has already been done!
+// JournalControl.replay(jrnl, dsg) ;
+// }
+// jrnl.truncate(0) ;
+// jrnl.close();
+// dsg.sync() ;
+// }
+//
+// if ( f.exists() )
+// FileOps.delete(journalFilename) ;
+// }
/** Recover a node data file (".dat").
* Node data files are append-only so recovering, then not using the data is safe.
Modified: incubator/jena/Jena2/TDB/trunk/src/main/java/com/hp/hpl/jena/tdb/transaction/TransactionManager.java
URL: http://svn.apache.org/viewvc/incubator/jena/Jena2/TDB/trunk/src/main/java/com/hp/hpl/jena/tdb/transaction/TransactionManager.java?rev=1235846&r1=1235845&r2=1235846&view=diff
==============================================================================
--- incubator/jena/Jena2/TDB/trunk/src/main/java/com/hp/hpl/jena/tdb/transaction/TransactionManager.java (original)
+++ incubator/jena/Jena2/TDB/trunk/src/main/java/com/hp/hpl/jena/tdb/transaction/TransactionManager.java Wed Jan 25 17:33:20 2012
@@ -573,9 +573,7 @@ public class TransactionManager
transactionStateTransition = new ArrayList<Pair<Transaction, TxnPoint>>() ;
}
- // ---- Recording
-
- Journal getJournal()
+ public Journal getJournal()
{
return journal ;
}
Modified: incubator/jena/Jena2/TDB/trunk/src/test/java/com/hp/hpl/jena/tdb/transaction/TestTransactionTDB.java
URL: http://svn.apache.org/viewvc/incubator/jena/Jena2/TDB/trunk/src/test/java/com/hp/hpl/jena/tdb/transaction/TestTransactionTDB.java?rev=1235846&r1=1235845&r2=1235846&view=diff
==============================================================================
--- incubator/jena/Jena2/TDB/trunk/src/test/java/com/hp/hpl/jena/tdb/transaction/TestTransactionTDB.java (original)
+++ incubator/jena/Jena2/TDB/trunk/src/test/java/com/hp/hpl/jena/tdb/transaction/TestTransactionTDB.java Wed Jan 25 17:33:20 2012
@@ -28,7 +28,7 @@ import com.hp.hpl.jena.query.Dataset ;
import com.hp.hpl.jena.query.ReadWrite ;
import com.hp.hpl.jena.sparql.sse.SSE ;
import com.hp.hpl.jena.tdb.ConfigTest ;
-import com.hp.hpl.jena.tdb.TDBFactoryTxn ;
+import com.hp.hpl.jena.tdb.TDBFactory ;
import com.hp.hpl.jena.tdb.migrate.AbstractTestTransaction ;
public class TestTransactionTDB extends AbstractTestTransaction
@@ -48,8 +48,7 @@ public class TestTransactionTDB extends
@Override
protected Dataset create()
{
- //return TDBFactory.createDataset(DIR) ;
- return TDBFactoryTxn.XcreateDataset(DIR) ;
+ return TDBFactory.createDataset(DIR) ;
}
private static Triple triple1 = SSE.parseTriple("(<s> <p> <o>)") ;