You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@jena.apache.org by an...@apache.org on 2012/12/23 23:27:46 UTC
svn commit: r1425541 - in /jena/trunk/jena-tdb: ./
src/main/java/com/hp/hpl/jena/tdb/solver/
src/test/java/com/hp/hpl/jena/tdb/extra/
Author: andy
Date: Sun Dec 23 22:27:45 2012
New Revision: 1425541
URL: http://svn.apache.org/viewvc?rev=1425541&view=rev
Log:
JENA-289 : Add timeout support for TDB BGP which cause a lot of work but little output.
Added:
jena/trunk/jena-tdb/src/main/java/com/hp/hpl/jena/tdb/solver/Abortable.java
jena/trunk/jena-tdb/src/test/java/com/hp/hpl/jena/tdb/extra/T_TimeoutTDBPattern.java
- copied, changed from r1422302, jena/trunk/jena-tdb/src/test/java/com/hp/hpl/jena/tdb/extra/T_TimeoutUnionGraph.java
Removed:
jena/trunk/jena-tdb/src/test/java/com/hp/hpl/jena/tdb/extra/T_TimeoutUnionGraph.java
Modified:
jena/trunk/jena-tdb/ReleaseNotes.txt
jena/trunk/jena-tdb/src/main/java/com/hp/hpl/jena/tdb/solver/QueryIterTDB.java
jena/trunk/jena-tdb/src/main/java/com/hp/hpl/jena/tdb/solver/SolverLib.java
Modified: jena/trunk/jena-tdb/ReleaseNotes.txt
URL: http://svn.apache.org/viewvc/jena/trunk/jena-tdb/ReleaseNotes.txt?rev=1425541&r1=1425540&r2=1425541&view=diff
==============================================================================
--- jena/trunk/jena-tdb/ReleaseNotes.txt (original)
+++ jena/trunk/jena-tdb/ReleaseNotes.txt Sun Dec 23 22:27:45 2012
@@ -5,6 +5,7 @@ ChangeLog for TDB
** TDB 0.10.0 depends on Jena 2.10.0 (ARQ 2.10.0)
++ JENA-289 : QueryExecution timeouts now cause TDB BGPs to timeout even if they produce no output.
+ JENA-363 : Fix to handling of integers with absolute values between 2^56 and 2^63.
+ JENA-346 : Further fixes for default union graph + write transactions.
+ Consolidation of versions numbers.
Added: jena/trunk/jena-tdb/src/main/java/com/hp/hpl/jena/tdb/solver/Abortable.java
URL: http://svn.apache.org/viewvc/jena/trunk/jena-tdb/src/main/java/com/hp/hpl/jena/tdb/solver/Abortable.java?rev=1425541&view=auto
==============================================================================
--- jena/trunk/jena-tdb/src/main/java/com/hp/hpl/jena/tdb/solver/Abortable.java (added)
+++ jena/trunk/jena-tdb/src/main/java/com/hp/hpl/jena/tdb/solver/Abortable.java Sun Dec 23 22:27:45 2012
@@ -0,0 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.hp.hpl.jena.tdb.solver;
+
+interface Abortable { public void abort() ; }
Modified: jena/trunk/jena-tdb/src/main/java/com/hp/hpl/jena/tdb/solver/QueryIterTDB.java
URL: http://svn.apache.org/viewvc/jena/trunk/jena-tdb/src/main/java/com/hp/hpl/jena/tdb/solver/QueryIterTDB.java?rev=1425541&r1=1425540&r2=1425541&view=diff
==============================================================================
--- jena/trunk/jena-tdb/src/main/java/com/hp/hpl/jena/tdb/solver/QueryIterTDB.java (original)
+++ jena/trunk/jena-tdb/src/main/java/com/hp/hpl/jena/tdb/solver/QueryIterTDB.java Sun Dec 23 22:27:45 2012
@@ -18,23 +18,26 @@
package com.hp.hpl.jena.tdb.solver;
-import java.util.Iterator;
+import java.util.Iterator ;
+import java.util.List ;
-import com.hp.hpl.jena.sparql.engine.ExecutionContext;
-import com.hp.hpl.jena.sparql.engine.QueryIterator;
-import com.hp.hpl.jena.sparql.engine.binding.Binding;
-import com.hp.hpl.jena.sparql.engine.iterator.QueryIterPlainWrapper;
+import com.hp.hpl.jena.sparql.engine.ExecutionContext ;
+import com.hp.hpl.jena.sparql.engine.QueryIterator ;
+import com.hp.hpl.jena.sparql.engine.binding.Binding ;
+import com.hp.hpl.jena.sparql.engine.iterator.QueryIterPlainWrapper ;
public class QueryIterTDB extends QueryIterPlainWrapper
{
// Rename as QueryIterCloseOther?
final private QueryIterator originalInput ;
+ private List<Abortable> killList ;
// The original input needs closing as well.
- public QueryIterTDB(Iterator<Binding> iterBinding, QueryIterator originalInput, ExecutionContext execCxt)
+ public QueryIterTDB(Iterator<Binding> iterBinding, List<Abortable> killList , QueryIterator originalInput, ExecutionContext execCxt)
{
super(iterBinding, execCxt) ;
this.originalInput = originalInput ;
+ this.killList = killList ;
}
@Override
@@ -44,4 +47,12 @@ public class QueryIterTDB extends QueryI
originalInput.close();
super.closeIterator() ;
}
+
+ @Override
+ protected void requestCancel()
+ {
+ if ( killList != null )
+ for ( Abortable it : killList )
+ it.abort() ;
+ }
}
Modified: jena/trunk/jena-tdb/src/main/java/com/hp/hpl/jena/tdb/solver/SolverLib.java
URL: http://svn.apache.org/viewvc/jena/trunk/jena-tdb/src/main/java/com/hp/hpl/jena/tdb/solver/SolverLib.java?rev=1425541&r1=1425540&r2=1425541&view=diff
==============================================================================
--- jena/trunk/jena-tdb/src/main/java/com/hp/hpl/jena/tdb/solver/SolverLib.java (original)
+++ jena/trunk/jena-tdb/src/main/java/com/hp/hpl/jena/tdb/solver/SolverLib.java Sun Dec 23 22:27:45 2012
@@ -20,14 +20,11 @@ package com.hp.hpl.jena.tdb.solver;
import static com.hp.hpl.jena.tdb.lib.Lib2.printAbbrev ;
-import java.util.Collection ;
-import java.util.HashSet ;
-import java.util.Iterator ;
-import java.util.List ;
-import java.util.Set ;
+import java.util.* ;
import org.apache.jena.atlas.iterator.Filter ;
import org.apache.jena.atlas.iterator.Iter ;
+import org.apache.jena.atlas.iterator.IteratorWrapper ;
import org.apache.jena.atlas.iterator.Transform ;
import org.apache.jena.atlas.lib.Tuple ;
import org.slf4j.Logger ;
@@ -35,6 +32,7 @@ import org.slf4j.LoggerFactory ;
import com.hp.hpl.jena.graph.Node ;
import com.hp.hpl.jena.graph.Triple ;
+import com.hp.hpl.jena.query.QueryCancelledException ;
import com.hp.hpl.jena.sparql.core.BasicPattern ;
import com.hp.hpl.jena.sparql.core.Var ;
import com.hp.hpl.jena.sparql.engine.ExecutionContext ;
@@ -94,7 +92,6 @@ public class SolverLib
public static Iterator<Binding> convertToNodes(Iterator<BindingNodeId> iterBindingIds, NodeTable nodeTable)
{ return Iter.map(iterBindingIds, convToBinding(nodeTable)) ; }
-
// The worker. Callers choose the NodeTupleTable.
// graphNode may be Node.ANY, meaning we should make triples unique.
@@ -120,6 +117,7 @@ public class SolverLib
NodeTable nodeTable = nodeTupleTable.getNodeTable() ;
Iterator<BindingNodeId> chain = Iter.map(input, SolverLib.convFromBinding(nodeTable)) ;
+ List<Abortable> killList = new ArrayList<Abortable>() ;
for ( Triple triple : triples )
{
@@ -131,6 +129,7 @@ public class SolverLib
// 4-tuples.
tuple = Tuple.create(graphNode, triple.getSubject(), triple.getPredicate(), triple.getObject()) ;
chain = solve(nodeTupleTable, tuple, anyGraph, chain, filter, execCxt) ;
+ chain = makeAbortable(chain, killList) ;
}
// DEBUG POINT
@@ -142,13 +141,67 @@ public class SolverLib
System.out.println("No results") ;
}
- // XXX
+ // Timeout wrapper ****
+ // QueryIterTDB gets called async.
+ // Iter.abortable?
+ // Or each iterator has a place to test.
+ // or pass in a thing to test?
+
// Need to make sure the bindings here point to parent.
Iterator<Binding> iterBinding = converter.convert(nodeTable, chain) ;
// "input" will be closed by QueryIterTDB but is otherwise unused.
- return new QueryIterTDB(iterBinding, input, execCxt) ;
+ // "killList" wil be aborted on timeout.
+ return new QueryIterTDB(iterBinding, killList, input, execCxt) ;
+ }
+
+ /** Create an abortable iterator, storing it in the killList.
+ * Just return the input iterator if kilList is null.
+ */
+ static <T> Iterator<T> makeAbortable(Iterator<T> iter, List<Abortable> killList)
+ {
+ if ( killList == null )
+ return iter ;
+ IterAbortable<T> k = new IterAbortable<T>(iter) ;
+ killList.add(k) ;
+ return k ;
+ }
+
+ /** Iterator that adds an abort operation which can be called
+ * at any time, including from another thread, and causes the
+ * iterator to throw an exception when next touched (hasNext, next).
+ */
+ static class IterAbortable<T> extends IteratorWrapper<T> implements Abortable
+ {
+ volatile boolean abortFlag = false ;
+
+ public IterAbortable(Iterator<T> iterator)
+ {
+ super(iterator) ;
+ }
+
+ /** Can call asynchronously at anytime */
+ @Override
+ public void abort() {
+ abortFlag = true ;
+ }
+
+ @Override
+ public boolean hasNext()
+ {
+ if ( abortFlag )
+ throw new QueryCancelledException() ;
+ return iterator.hasNext() ;
+ }
+
+ @Override
+ public T next()
+ {
+ if ( abortFlag )
+ throw new QueryCancelledException() ;
+ return iterator.next() ;
+ }
}
private static Iterator<BindingNodeId> solve(NodeTupleTable nodeTupleTable,
@@ -231,11 +284,18 @@ public class SolverLib
QueryIterator input, Filter<Tuple<NodeId>> filter,
ExecutionContext execCxt)
{
+ List<Abortable> killList = new ArrayList<Abortable>() ;
Iterator<Tuple<NodeId>> iter1 = ds.getQuadTable().getNodeTupleTable().find(NodeId.NodeIdAny, NodeId.NodeIdAny, NodeId.NodeIdAny, NodeId.NodeIdAny) ;
if ( filter != null )
iter1 = Iter.filter(iter1, filter) ;
+
Iterator<NodeId> iter2 = Tuple.project(0, iter1) ;
+ // Project is cheap - don't brother wrapping iter1
+ iter2 = makeAbortable(iter2, killList) ;
+
Iterator<NodeId> iter3 = Iter.distinct(iter2) ;
+ iter3 = makeAbortable(iter3, killList) ;
+
Iterator<Node> iter4 = NodeLib.nodes(ds.getQuadTable().getNodeTupleTable().getNodeTable(), iter3) ;
final Var var = Var.alloc(graphNode) ;
@@ -248,8 +308,8 @@ public class SolverLib
} ;
Iterator<Binding> iterBinding = Iter.map(iter4, bindGraphName) ;
-
- return new QueryIterTDB(iterBinding, input, execCxt) ;
+ // Not abortable.
+ return new QueryIterTDB(iterBinding, killList, input, execCxt) ;
}
/** Turn a BasicPattern into an abbreviated string for debugging */
Copied: jena/trunk/jena-tdb/src/test/java/com/hp/hpl/jena/tdb/extra/T_TimeoutTDBPattern.java (from r1422302, jena/trunk/jena-tdb/src/test/java/com/hp/hpl/jena/tdb/extra/T_TimeoutUnionGraph.java)
URL: http://svn.apache.org/viewvc/jena/trunk/jena-tdb/src/test/java/com/hp/hpl/jena/tdb/extra/T_TimeoutTDBPattern.java?p2=jena/trunk/jena-tdb/src/test/java/com/hp/hpl/jena/tdb/extra/T_TimeoutTDBPattern.java&p1=jena/trunk/jena-tdb/src/test/java/com/hp/hpl/jena/tdb/extra/T_TimeoutUnionGraph.java&r1=1422302&r2=1425541&rev=1425541&view=diff
==============================================================================
--- jena/trunk/jena-tdb/src/test/java/com/hp/hpl/jena/tdb/extra/T_TimeoutUnionGraph.java (original)
+++ jena/trunk/jena-tdb/src/test/java/com/hp/hpl/jena/tdb/extra/T_TimeoutTDBPattern.java Sun Dec 23 22:27:45 2012
@@ -16,110 +16,81 @@
* limitations under the License.
*/
-/*******************************************************************************
- * Licensed Materials - Property of IBM
- * (c) Copyright IBM Corporation 2011, 2012. All Rights Reserved.
- * Note to U.S. Government Users Restricted Rights: Use,
- * duplication or disclosure restricted by GSA ADP Schedule
- * Contract with IBM Corp.
- *******************************************************************************/
-
-// Licensed to ASF in JENA-289
-
-package com.hp.hpl.jena.tdb.extra ;
+package com.hp.hpl.jena.tdb.extra;
import java.text.MessageFormat ;
import java.util.Date ;
import java.util.concurrent.TimeUnit ;
-import org.apache.jena.atlas.lib.FileOps ;
-
import com.hp.hpl.jena.query.* ;
import com.hp.hpl.jena.rdf.model.* ;
-import com.hp.hpl.jena.sparql.util.Timer ;
-import com.hp.hpl.jena.tdb.TDB ;
import com.hp.hpl.jena.tdb.TDBFactory ;
-import com.hp.hpl.jena.tdb.base.block.FileMode ;
-import com.hp.hpl.jena.tdb.sys.SystemTDB ;
-import com.hp.hpl.jena.tdb.transaction.TransactionManager ;
-
-public class T_TimeoutUnionGraph {
-
- private static String location = "C:\\temp\\TestTimeout"; //$NON-NLS-1$
- private static final String sparql = "SELECT * WHERE { ?a ?b ?c . ?c ?d ?e }"; //$NON-NLS-1$
- //private static final String sparql = "SELECT * WHERE { { ?a ?b ?c . FILTER (!isLiteral(?c)) } . ?c ?d ?e }"; //$NON-NLS-1$
- private static final int limit = 1000;
- private static final int timeout1_sec = 10;
- private static final int timeout2_sec = 10;
-
- private static boolean CREATE = false;
- private static final int RESOURCES = 100000;
- private static final int COMMIT_EVERY = 1000;
- private static final int TRIPLES_PER_RESOURCE = 100;
- private static final String RES_NS = "http://example.com/"; //$NON-NLS-1$
- private static final String PROP_NS = "http://example.org/ns/1.0/"; //$NON-NLS-1$
-
- public static void main(String[] args) {
-
- System.out.printf("Max mem = %,dM\n", Runtime.getRuntime().maxMemory()/(1000*1000)) ;
-
- if ( false )
- SystemTDB.setFileMode(FileMode.direct) ;
-
- location = "DBX" ;
- FileOps.ensureDir(location) ;
- TDB.getContext().set(TDB.symUnionDefaultGraph, true);
-
+
+// From Jena-289.
+
+public class T_TimeoutTDBPattern
+{
+ private static final int timeout1_sec = 3;
+ private static final int timeout2_sec = 5;
+
+ private static final int RESOURCES = 100000;
+ private static final int COMMIT_EVERY = 1000;
+ private static final int TRIPLES_PER_RESOURCE = 100;
+ private static final String RES_NS = "http://example.com/";
+ private static final String PROP_NS = "http://example.org/ns/1.0/";
+
+ public static void main(String[] args) {
+ String location = "DB_Jena289" ;
Dataset ds = TDBFactory.createDataset(location);
- if ( ds.asDatasetGraph().isEmpty() )
- create(ds) ;
- Query query = QueryFactory.create(sparql, Syntax.syntaxSPARQL_11);
- query.setLimit(limit);
- System.out.println(query) ;
-
- ds.begin(ReadWrite.READ);
- QueryExecution qexec = null;
- Timer timer = new Timer() ;
-
- TransactionManager.QueueBatchSize = 0 ;
-
- System.out.println() ;
- System.out.println("Start query") ;
+ if (ds.asDatasetGraph().isEmpty())
+ create(ds) ;
- //ARQ.getContext().set(ARQ.queryTimeout, "10000,10000") ;
- //ARQ.getContext().set(ARQ.symLogExec, "true") ;
+ // 10M triples.
+ // No match to { ?a ?b ?c . ?c ?d ?e }
+
+ final String sparql = "SELECT * WHERE { ?a ?b ?c . ?c ?d ?e }";
- try {
- System.out.println(MessageFormat.format(
- "{0,date} {0,time} Executing query [limit={1} timeout1={2}s timeout2={3}s]: {4}", //$NON-NLS-1$
- new Date(System.currentTimeMillis()), limit, timeout1_sec, timeout2_sec, sparql));
- timer.startTimer() ;
- qexec = QueryExecutionFactory.create(query, ds);
- qexec.setTimeout(timeout1_sec, TimeUnit.SECONDS, timeout2_sec, TimeUnit.SECONDS);
-
- ResultSet rs = qexec.execSelect();
- ResultSetFormatter.outputAsXML(System.out, rs);
- } catch (Throwable t) {
- t.printStackTrace(); // OOME
- } finally {
- long x = timer.readTimer() ;
- System.out.printf("Time = %,dms\n", x) ;
- if (qexec != null)
- qexec.close();
- ds.end();
- ds.close();
- System.out.println(MessageFormat.format("{0,date} {0,time} Finished", //$NON-NLS-1$
- new Date(System.currentTimeMillis())));
- }
- }
+ Query query = QueryFactory.create(sparql);
+
+ ds.begin(ReadWrite.READ);
+ QueryExecution qexec = null;
+ try {
+ System.out.println(MessageFormat.format(
+ "{0,date} {0,time} Executing query [timeout1={1}s timeout2={2}s]: {3}",
+ new Date(System.currentTimeMillis()), timeout1_sec, timeout2_sec, sparql));
+ qexec = QueryExecutionFactory.create(query, ds);
+ if ( true )
+ qexec.setTimeout(timeout1_sec, TimeUnit.SECONDS, timeout2_sec, TimeUnit.SECONDS);
+ long start = System.nanoTime() ;
+ long finish = start ;
+ ResultSet rs = qexec.execSelect();
+
+ try {
+ long x = ResultSetFormatter.consume(rs) ;
+ finish = System.nanoTime() ;
+ System.out.println("Results: "+x) ;
+ } catch (QueryCancelledException ex)
+ {
+ finish = System.nanoTime() ;
+ System.out.println("Cancelled") ;
+ }
+ System.out.printf("%.2fs\n",(finish-start)/(1000.0*1000.0*1000.0)) ;
+ } catch (Throwable t) {
+ t.printStackTrace(); // OOME
+ } finally {
+ if (qexec != null)
+ qexec.close();
+ ds.end();
+ ds.close();
+ System.out.println(MessageFormat.format("{0,date} {0,time} Finished",
+ new Date(System.currentTimeMillis())));
+ }
+ }
private static void create(Dataset ds)
{
- System.out.println("Start create") ;
- int iR = 0 ;
-
- for (iR = 0; iR < RESOURCES; iR++) {
+ for (int iR = 0; iR < RESOURCES; iR++) { // 100,000
if (iR % COMMIT_EVERY == 0) {
if (ds.isInTransaction()) {
ds.commit();
@@ -129,22 +100,18 @@ public class T_TimeoutUnionGraph {
}
Model model = ModelFactory.createDefaultModel();
- Resource res = ResourceFactory.createResource(RES_NS + "resource" + iR); //$NON-NLS-1$
- //Model model = ds.getNamedModel(res.getURI()) ;
- for (int iP = 0; iP < TRIPLES_PER_RESOURCE; iP++) {
- Property prop = ResourceFactory.createProperty(PROP_NS, "property" + iP); //$NON-NLS-1$
- model.add(res, prop, model.createTypedLiteral("Property value " + iP)); //$NON-NLS-1$
+ Resource res = model.createResource(RES_NS + "resource" + iR);
+ for (int iP = 0; iP < TRIPLES_PER_RESOURCE; iP++) { // 100
+ Property prop = ResourceFactory.createProperty(PROP_NS, "property" + iP);
+ model.add(res, prop, model.createTypedLiteral("Property value " + iP));
}
- ds.addNamedModel(res.getURI(), model);
- //System.out.println("Created " + res.getURI()); //$NON-NLS-1$
+ //ds.addNamedModel(res.getURI(), model);
+ ds.getDefaultModel().add(model);
+ System.out.println("Created " + res.getURI());
}
-
- if (ds.isInTransaction()) {
- ds.commit();
- ds.end();
- }
-
- System.out.println("Finish create") ;
+ ds.commit();
+ ds.end();
}
-
}
+
+