You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@jena.apache.org by an...@apache.org on 2013/01/26 18:28:08 UTC
svn commit: r1438920 - in /jena/trunk/jena-tdb: ReleaseNotes.txt
src/main/java/com/hp/hpl/jena/tdb/index/TupleTable.java
src/main/java/com/hp/hpl/jena/tdb/nodetable/NodeTupleTableConcrete.java
src/main/java/com/hp/hpl/jena/tdb/solver/StageMatchTuple.java
Author: andy
Date: Sat Jan 26 17:28:08 2013
New Revision: 1438920
URL: http://svn.apache.org/viewvc?rev=1438920&view=rev
Log:
Improve the performance of union default graph by efficiently handling potential duplicates.
Lowers the memory needed; avoids the "growing set" problem.
Modified:
jena/trunk/jena-tdb/ReleaseNotes.txt
jena/trunk/jena-tdb/src/main/java/com/hp/hpl/jena/tdb/index/TupleTable.java
jena/trunk/jena-tdb/src/main/java/com/hp/hpl/jena/tdb/nodetable/NodeTupleTableConcrete.java
jena/trunk/jena-tdb/src/main/java/com/hp/hpl/jena/tdb/solver/StageMatchTuple.java
Modified: jena/trunk/jena-tdb/ReleaseNotes.txt
URL: http://svn.apache.org/viewvc/jena/trunk/jena-tdb/ReleaseNotes.txt?rev=1438920&r1=1438919&r2=1438920&view=diff
==============================================================================
--- jena/trunk/jena-tdb/ReleaseNotes.txt (original)
+++ jena/trunk/jena-tdb/ReleaseNotes.txt Sat Jan 26 17:28:08 2013
@@ -5,6 +5,7 @@ ChangeLog for TDB
** TDB 0.10.0 depends on Jena 2.10.0 (ARQ 2.10.0)
++ Improve the performance of union default graph by efficiently handling potential duplicates.
+ Command line tdbquery: --mem (or --data) reads into an in-memory dadaset and executes the query.
+ JENA-289 : QueryExecution timeouts now cause TDB BGPs to timeout even if they produce no output.
+ JENA-363 : Fix to handling of integers with absolute values between 2^56 and 2^63.
@@ -15,7 +16,7 @@ ChangeLog for TDB
==== TDB 0.9.4
+ Enable batching of flushing commits to the main database -
- The improves transactional write performance.
+ This improves transactional write performance.
+ Extensive testing and fixing for recovery due to abrupt termination of the JVM.
+ JENA-318 : Fix rejection of bad language tags.
+ JENA_317 : Fix to handling decimal numbers. Decimals of around 15 digits could be corrupted.
Modified: jena/trunk/jena-tdb/src/main/java/com/hp/hpl/jena/tdb/index/TupleTable.java
URL: http://svn.apache.org/viewvc/jena/trunk/jena-tdb/src/main/java/com/hp/hpl/jena/tdb/index/TupleTable.java?rev=1438920&r1=1438919&r2=1438920&view=diff
==============================================================================
--- jena/trunk/jena-tdb/src/main/java/com/hp/hpl/jena/tdb/index/TupleTable.java (original)
+++ jena/trunk/jena-tdb/src/main/java/com/hp/hpl/jena/tdb/index/TupleTable.java Sat Jan 26 17:28:08 2013
@@ -25,11 +25,13 @@ import java.util.Iterator ;
import org.apache.jena.atlas.lib.Closeable ;
import org.apache.jena.atlas.lib.Sync ;
import org.apache.jena.atlas.lib.Tuple ;
+import org.apache.jena.atlas.logging.Log ;
import org.slf4j.Logger ;
import org.slf4j.LoggerFactory ;
import com.hp.hpl.jena.tdb.TDBException ;
import com.hp.hpl.jena.tdb.store.NodeId ;
+import com.hp.hpl.jena.tdb.sys.SystemTDB ;
/** A TupleTable is a set of TupleIndexes. The first TupleIndex is the "primary" index and must exist */
public class TupleTable implements Sync, Closeable
@@ -37,6 +39,7 @@ public class TupleTable implements Sync,
private static Logger log = LoggerFactory.getLogger(TupleTable.class) ;
private final TupleIndex[] indexes ;
+ private final TupleIndex scanAllIndex ; // Use this index if a complete scan is needed.
private final int tupleLen ;
private boolean syncNeeded = false ;
@@ -51,12 +54,35 @@ public class TupleTable implements Sync,
if ( index != null && index.getTupleLength() != tupleLen )
throw new TDBException("Incompatible index: "+index.getMapping()) ;
}
-
+ scanAllIndex = chooseScanAllIndex(tupleLen, indexes) ;
}
- int x = 0 ;
- int added = 0 ;
-
+ /** Choose an index to scan in case we are asked for everything
+ * This needs to be ???G for the distinctAdjacent filter in union query to work.
+ */
+ private static TupleIndex chooseScanAllIndex(int tupleLen, TupleIndex[] indexes)
+ {
+ if ( tupleLen != 4 )
+ return indexes[0] ;
+
+ for ( TupleIndex index : indexes )
+ {
+ // First look for SPOG
+ if ( index.getName().equals("SPOG") )
+ return index ;
+ }
+
+ for ( TupleIndex index : indexes )
+ {
+ // Then look for any ???G
+ if ( index.getName().endsWith("G") )
+ return index ;
+ }
+
+ Log.warn(SystemTDB.errlog, "Did not find a ???G index for full scans") ;
+ return indexes[0] ;
+ }
+
/** Insert a tuple - return true if it was really added, false if it was a duplicate */
public boolean add(Tuple<NodeId> t)
{
@@ -147,7 +173,7 @@ public class TupleTable implements Sync,
}
if ( numSlots == 0 )
- return indexes[0].all() ;
+ return scanAllIndex.all() ;
int indexNumSlots = 0 ;
TupleIndex index = null ;
@@ -168,7 +194,6 @@ public class TupleTable implements Sync,
if ( index == null )
// No index at all. Scan.
index = indexes[0] ;
-
return index.find(pattern) ;
}
Modified: jena/trunk/jena-tdb/src/main/java/com/hp/hpl/jena/tdb/nodetable/NodeTupleTableConcrete.java
URL: http://svn.apache.org/viewvc/jena/trunk/jena-tdb/src/main/java/com/hp/hpl/jena/tdb/nodetable/NodeTupleTableConcrete.java?rev=1438920&r1=1438919&r2=1438920&view=diff
==============================================================================
--- jena/trunk/jena-tdb/src/main/java/com/hp/hpl/jena/tdb/nodetable/NodeTupleTableConcrete.java (original)
+++ jena/trunk/jena-tdb/src/main/java/com/hp/hpl/jena/tdb/nodetable/NodeTupleTableConcrete.java Sat Jan 26 17:28:08 2013
@@ -42,8 +42,8 @@ public class NodeTupleTableConcrete impl
private final DatasetControl dsPolicy ;
/*
- * Concurrency checking: Everything goes through one of addRow, deleteRow or
- * find*
+ * Concurrency checking: Everything goes through one of
+ * addRow, deleteRow or find*
*/
public NodeTupleTableConcrete(int N, TupleIndex[] indexes, NodeTable nodeTable, DatasetControl dsControl)
Modified: jena/trunk/jena-tdb/src/main/java/com/hp/hpl/jena/tdb/solver/StageMatchTuple.java
URL: http://svn.apache.org/viewvc/jena/trunk/jena-tdb/src/main/java/com/hp/hpl/jena/tdb/solver/StageMatchTuple.java?rev=1438920&r1=1438919&r2=1438920&view=diff
==============================================================================
--- jena/trunk/jena-tdb/src/main/java/com/hp/hpl/jena/tdb/solver/StageMatchTuple.java (original)
+++ jena/trunk/jena-tdb/src/main/java/com/hp/hpl/jena/tdb/solver/StageMatchTuple.java Sat Jan 26 17:28:08 2013
@@ -88,9 +88,6 @@ public class StageMatchTuple extends Rep
prepare(nodeTupleTable.getNodeTable(), patternTuple, input, ids, var) ;
- // Go directly to the tuple table
- //Iterator<Tuple<NodeId>> iterMatches = nodeTupleTable.getTupleTable().find(Tuple.create(ids)) ;
-
Iterator<Tuple<NodeId>> iterMatches = nodeTupleTable.find(Tuple.create(ids)) ;
// ** Allow a triple or quad filter here.
@@ -106,10 +103,23 @@ public class StageMatchTuple extends Rep
if ( anyGraphs )
{
iterMatches = Iter.operate(iterMatches, quadsToTriples) ;
- // If any slots were set, then the inde would be ???G and we can use distinctAdjacent.
+ // If any slots were set, then the index would be ???G and we can use distinctAdjacent.
// If all slots are unset, the index is probably GSPO (SPOG would be better in this one case).
// This is a safe, if potentially costly, choice.
- iterMatches = Iter.distinct(iterMatches) ; // WRT only three varying slots.
+
+ //Guaranteed
+ //iterMatches = Iter.distinct(iterMatches) ;
+
+ // This depends on the way indexes are choose and
+ // the indexing pattern. It assumes that the index
+ // chosen ends in G so same triples are adjacent
+ // in a union query.
+ // If any slot is defined, then the index will be X??G.
+ // if no slot is defined, then the index will be X???G.
+ // See TupleTable.scanAllIndex that ensures the latter.
+ // The former assumes indexes are either G... or ...G.
+ // No G part way through.
+ iterMatches = Iter.distinctAdjacent(iterMatches) ;
}
// Map Tuple<NodeId> to BindingNodeId