You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@jena.apache.org by an...@apache.org on 2022/12/11 11:26:35 UTC
[jena] branch main updated: GH-1656 Service enhancer: bug fix for bulk requests with order+limit; improvement that sub-requests work with union default graph
This is an automated email from the ASF dual-hosted git repository.
andy pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/jena.git
The following commit(s) were added to refs/heads/main by this push:
new 3a79ea2083 GH-1656 Service enhancer: bug fix for bulk requests with order+limit; improvement that sub-requests work with union default graph
new b0f24f4b9d Merge pull request #1657 from Aklakan/GH-1656
3a79ea2083 is described below
commit 3a79ea208373a31423d51fc7c18663595780dac5
Author: Claus Stadler <Ra...@googlemail.com>
AuthorDate: Thu Oct 13 14:51:52 2022 +0200
GH-1656 Service enhancer: bug fix for bulk requests with order+limit; improvement that sub-requests work with union default graph
---
jena-extras/jena-serviceenhancer/pom.xml | 9 ++-
.../service/enhancer/impl/BatchQueryRewriter.java | 66 +++-------------
.../sparql/service/enhancer/impl/Estimate.java | 4 +-
.../service/enhancer/impl/PartitionRequest.java | 6 +-
.../enhancer/impl/QueryIterServiceBulk.java | 2 +-
.../enhancer/impl/util/DynamicDatasetUtils.java | 90 ++++++++++++++++++++++
.../service/enhancer/impl/util/VarScopeUtils.java | 3 -
.../service/enhancer/init/ServiceEnhancerInit.java | 87 +++++++++++++++++++--
...va => TestServiceEnhancerDatasetAssembler.java} | 61 ++++++++++++++-
...AbstractTestServiceEnhancerResultSetLimits.java | 45 +----------
.../TestServiceEnhancerBatchQueryRewriter.java | 89 +++++++++++++++++++++
...atcher.java => TestServiceEnhancerBatcher.java} | 2 +-
12 files changed, 352 insertions(+), 112 deletions(-)
diff --git a/jena-extras/jena-serviceenhancer/pom.xml b/jena-extras/jena-serviceenhancer/pom.xml
index 3b9b47e178..b30c0c0831 100644
--- a/jena-extras/jena-serviceenhancer/pom.xml
+++ b/jena-extras/jena-serviceenhancer/pom.xml
@@ -37,7 +37,7 @@
<dependency>
<groupId>org.apache.jena</groupId>
<artifactId>jena-arq</artifactId>
- <version>4.7.0-SNAPSHOT</version>
+ <version>${project.version}</version>
</dependency>
<dependency>
@@ -67,6 +67,13 @@
<artifactId>log4j-slf4j-impl</artifactId>
<scope>test</scope>
</dependency>
+
+ <dependency>
+ <groupId>org.apache.jena</groupId>
+ <artifactId>jena-tdb2</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
diff --git a/jena-extras/jena-serviceenhancer/src/main/java/org/apache/jena/sparql/service/enhancer/impl/BatchQueryRewriter.java b/jena-extras/jena-serviceenhancer/src/main/java/org/apache/jena/sparql/service/enhancer/impl/BatchQueryRewriter.java
index 35b8bcbcf6..a4c8e331d2 100644
--- a/jena-extras/jena-serviceenhancer/src/main/java/org/apache/jena/sparql/service/enhancer/impl/BatchQueryRewriter.java
+++ b/jena-extras/jena-serviceenhancer/src/main/java/org/apache/jena/sparql/service/enhancer/impl/BatchQueryRewriter.java
@@ -33,7 +33,6 @@ import java.util.Set;
import org.apache.jena.atlas.logging.Log;
import org.apache.jena.query.Query;
import org.apache.jena.query.SortCondition;
-import org.apache.jena.sparql.algebra.Algebra;
import org.apache.jena.sparql.algebra.Op;
import org.apache.jena.sparql.algebra.OpAsQuery;
import org.apache.jena.sparql.algebra.op.OpExtend;
@@ -85,11 +84,6 @@ public class BatchQueryRewriter {
static int REMOTE_END_MARKER = 1000000000;
static NodeValue NV_REMOTE_END_MARKER = NodeValue.makeInteger(REMOTE_END_MARKER);
- /** True if either local or remote end marker */
-// public static boolean isLocalOrRemoteEndMarker(int id) {
-// return isRemoteEndMarker(id) || isLocalEndMarker(id);
-// }
-
public static boolean isRemoteEndMarker(int id) {
return id == REMOTE_END_MARKER;
}
@@ -98,21 +92,6 @@ public class BatchQueryRewriter {
return Objects.equals(id, REMOTE_END_MARKER);
}
-
-
- // Local end marker is not returned by the remote service
-// static int LOCAL_END_MARKER = 1000000001;
-// static NodeValue NV_LOCAL_END_MARKER = NodeValue.makeInteger(LOCAL_END_MARKER);
-//
-// public static boolean isLocalEndMarker(int id) {
-// return id == LOCAL_END_MARKER;
-// }
-//
-// public static boolean isLocalEndMarker(Integer id) {
-// return Objects.equals(id, LOCAL_END_MARKER);
-// }
-
-
public BatchQueryRewriter(OpServiceInfo serviceInfo, Var idxVar,
boolean sequentialUnion, boolean orderRetainingUnion,
boolean omitEndMarker) {
@@ -136,18 +115,17 @@ public class BatchQueryRewriter {
}
public BatchQueryRewriteResult rewrite(Batch<Integer, PartitionRequest<Binding>> batchRequest) {
-
Op newOp = null;
List<Entry<Integer, PartitionRequest<Binding>>> es = new ArrayList<>(batchRequest.getItems().entrySet());
Collections.reverse(es);
- Query normQuery = serviceInfo.getNormedQuery();
- Op normOp = serviceInfo.getNormedQueryOp();
+ Query normedQuery = serviceInfo.getNormedQuery();
+ Op normedOp = serviceInfo.getNormedQueryOp();
// Prepare the sort conditions
List<SortCondition> sortConditions = new ArrayList<>();
List<SortCondition> localSortConditions =
- Optional.ofNullable(normQuery.getOrderBy()).orElse(Collections.emptyList());
+ Optional.ofNullable(normedQuery.getOrderBy()).orElse(Collections.emptyList());
boolean noOrderNeeded =
orderRetainingUnion || sequentialUnion && localSortConditions.isEmpty();
@@ -162,9 +140,14 @@ public class BatchQueryRewriter {
sortConditions.addAll(localSortConditions);
- for (Entry<Integer, PartitionRequest<Binding>> e : es) { // batchRequest.getItems().entrySet()) {
+ if (!omitEndMarker) {
+ Op endMarker = OpExtend.create(OpTable.unit(), idxVar, NV_REMOTE_END_MARKER);
+ newOp = newOp == null ? endMarker : OpUnion.create(newOp, endMarker);
+ }
- PartitionRequest<Binding> req = e.getValue(); // batchRequest.get(i);
+ for (Entry<Integer, PartitionRequest<Binding>> e : es) {
+
+ PartitionRequest<Binding> req = e.getValue();
long idx = e.getKey();
Binding scopedBinding = req.getPartitionKey();
@@ -173,37 +156,15 @@ public class BatchQueryRewriter {
Map<Var, Var> varMapScopedToNormed = ServiceCacheKeyFactory
.createJoinVarMapScopedToNormed(serviceInfo, scopedBindingVars);
- // Binding plainBinding = BindingUtils.renameKeys(scopedBinding, serviceInfo.getMentionedSubOpVarsScopedToPlain());
Binding normedBinding = BindingUtils.renameKeys(scopedBinding, varMapScopedToNormed);
// Op op = serviceInfo.getNormedQueryOp();
- Op op = normOp;
+ Op op = normedOp;
// Note: QC.substitute does not remove variables being substituted from projections
// This may cause unbound variables to be projected
- // If the union is sequential and order retaining we can retain the order on the members
- // otherwise, we can remove any ordering on the member
- if ((sequentialUnion && orderRetainingUnion) || localSortConditions.isEmpty()) {
- // If the union is sequential and order retaining we can retain the order on the members
- // otherwise, we can remove any ordering on the member
- } else {
- // Member order may not be retained - remove it from the query
-
- // TODO This should be done once OpServiceInfo
- Query tmp = normQuery.cloneQuery();
- if (tmp.hasOrderBy()) {
- tmp.getOrderBy().clear();
- }
-
- op = Algebra.compile(tmp);
- // TODO Something is odd with ordering here
- // Add the sort conditions
- // op = new OpOrder(op, localSortConditions);
- }
-
op = QC.substitute(op, normedBinding);
- op = OpExtend.create(op, idxVar, NodeValue.makeInteger(idx));
long o = req.hasOffset() ? req.getOffset() : Query.NOLIMIT;
long l = req.hasLimit() ? req.getLimit() : Query.NOLIMIT;
@@ -211,14 +172,11 @@ public class BatchQueryRewriter {
if (o != Query.NOLIMIT || l != Query.NOLIMIT) {
op = new OpSlice(op, o, l);
}
+ op = OpExtend.create(op, idxVar, NodeValue.makeInteger(idx));
newOp = newOp == null ? op : OpUnion.create(op, newOp);
}
- if (!omitEndMarker) {
- Op endMarker = OpExtend.create(OpTable.unit(), idxVar, NV_REMOTE_END_MARKER);
- newOp = newOp == null ? endMarker : OpUnion.create(newOp, endMarker);
- }
if (orderNeeded) {
newOp = new OpOrder(newOp, sortConditions);
diff --git a/jena-extras/jena-serviceenhancer/src/main/java/org/apache/jena/sparql/service/enhancer/impl/Estimate.java b/jena-extras/jena-serviceenhancer/src/main/java/org/apache/jena/sparql/service/enhancer/impl/Estimate.java
index 53e0969bd8..d262e797b2 100644
--- a/jena-extras/jena-serviceenhancer/src/main/java/org/apache/jena/sparql/service/enhancer/impl/Estimate.java
+++ b/jena-extras/jena-serviceenhancer/src/main/java/org/apache/jena/sparql/service/enhancer/impl/Estimate.java
@@ -21,7 +21,9 @@ package org.apache.jena.sparql.service.enhancer.impl;
import java.io.Serializable;
import java.util.Objects;
-/** An estimated value with a flag that indicates whether it's exact */
+/**
+ * An estimated value with a flag that indicates whether it's exact.
+ */
public class Estimate<T>
implements Serializable
{
diff --git a/jena-extras/jena-serviceenhancer/src/main/java/org/apache/jena/sparql/service/enhancer/impl/PartitionRequest.java b/jena-extras/jena-serviceenhancer/src/main/java/org/apache/jena/sparql/service/enhancer/impl/PartitionRequest.java
index 2c08334955..5753ac4f0c 100644
--- a/jena-extras/jena-serviceenhancer/src/main/java/org/apache/jena/sparql/service/enhancer/impl/PartitionRequest.java
+++ b/jena-extras/jena-serviceenhancer/src/main/java/org/apache/jena/sparql/service/enhancer/impl/PartitionRequest.java
@@ -18,7 +18,11 @@
package org.apache.jena.sparql.service.enhancer.impl;
-/** Helper class to capture a range of data w.r.t. a partition key (typically a binding) and assign it an id */
+/**
+ * Helper class to capture a range of data (specified by limit + offset)
+ * w.r.t. a partition key (typically a {@link org.apache.jena.sparql.engine.binding.Binding}
+ * and give that information an id.
+ */
public class PartitionRequest<I>
{
protected long outputId;
diff --git a/jena-extras/jena-serviceenhancer/src/main/java/org/apache/jena/sparql/service/enhancer/impl/QueryIterServiceBulk.java b/jena-extras/jena-serviceenhancer/src/main/java/org/apache/jena/sparql/service/enhancer/impl/QueryIterServiceBulk.java
index 3df6bec0df..407e605f16 100644
--- a/jena-extras/jena-serviceenhancer/src/main/java/org/apache/jena/sparql/service/enhancer/impl/QueryIterServiceBulk.java
+++ b/jena-extras/jena-serviceenhancer/src/main/java/org/apache/jena/sparql/service/enhancer/impl/QueryIterServiceBulk.java
@@ -208,7 +208,7 @@ public class QueryIterServiceBulk
resultSizeCache.updateLimit(targetService, backendResultSetLimit);
- // We obtained to few data for the current id - repeat the request
+ // We obtained too little data for the current id - repeat the request
prepareNextBatchExec(false);
continue;
}
diff --git a/jena-extras/jena-serviceenhancer/src/main/java/org/apache/jena/sparql/service/enhancer/impl/util/DynamicDatasetUtils.java b/jena-extras/jena-serviceenhancer/src/main/java/org/apache/jena/sparql/service/enhancer/impl/util/DynamicDatasetUtils.java
new file mode 100644
index 0000000000..4df92f9fc3
--- /dev/null
+++ b/jena-extras/jena-serviceenhancer/src/main/java/org/apache/jena/sparql/service/enhancer/impl/util/DynamicDatasetUtils.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.jena.sparql.service.enhancer.impl.util;
+
+import org.apache.jena.atlas.lib.Pair;
+import org.apache.jena.query.Query;
+import org.apache.jena.sparql.core.DatasetGraph;
+import org.apache.jena.sparql.core.DynamicDatasets.DynamicDatasetGraph;
+
+public class DynamicDatasetUtils {
+
+ /**
+ * Returns true iff the argument's original default and named graphs are non-null.
+ * Typically, this shouldn't happen. This method mainly exists for robustness as to
+ * avoid null pointer exceptions.
+ */
+ public static boolean isUnwrappable(DynamicDatasetGraph ddg) {
+ boolean result = ddg.getOriginal() != null &&
+ ddg.getOriginalDefaultGraphs() != null && ddg.getOriginalNamedGraphs() != null;
+ return result;
+ }
+
+ /** Convenience method to cast the argument as a DynamicDatasetGraph if it is an unwrappable one. */
+ public static DynamicDatasetGraph asUnwrappableDynamicDatasetOrNull(DatasetGraph dg) {
+ DynamicDatasetGraph result = null;
+ if (dg instanceof DynamicDatasetGraph) {
+ DynamicDatasetGraph ddg = (DynamicDatasetGraph)dg;
+ if (isUnwrappable(ddg)) {
+ result = ddg;
+ }
+ }
+ return result;
+ }
+
+ /**
+ * Returns the argument unless it is a DynamicDataset for which
+ * {@link #isUnwrappable(DynamicDatasetGraph)} is true. In that case
+ * {@link DynamicDatasetGraph#getOriginal()} is returned.
+ */
+ public static DatasetGraph unwrapOriginal(DatasetGraph dg) {
+ DatasetGraph result = dg;
+ if (dg instanceof DynamicDatasetGraph) {
+ DynamicDatasetGraph ddg = (DynamicDatasetGraph)dg;
+ if (isUnwrappable(ddg)) {
+ result = ddg.getOriginal();
+ }
+ }
+ return result;
+ }
+
+ /**
+ * Returns the given arguments as a pair unless the dataset is an unwrappable DynamicDataset.
+ * In that case, query's default/named graphs are modified in-place to match those of the dataset.
+ * The returned pair holds the modified query and the original (unwrapped) dataset.
+ */
+ public static Pair<Query, DatasetGraph> unwrap(Query query, DatasetGraph dataset) {
+ Pair<Query, DatasetGraph> result = null;
+ if (dataset instanceof DynamicDatasetGraph) {
+ DynamicDatasetGraph ddg = (DynamicDatasetGraph)dataset;
+ if (isUnwrappable(ddg)) {
+ // Query copy = query.cloneQuery(); Should we clone to be safe?
+ query.getGraphURIs().clear();
+ query.getNamedGraphURIs().clear();
+ ddg.getOriginalDefaultGraphs().forEach(n -> query.getGraphURIs().add(n.getURI()));
+ ddg.getOriginalNamedGraphs().forEach(n -> query.getNamedGraphURIs().add(n.getURI()));
+ result = Pair.create(query, ddg.getOriginal());
+ }
+ }
+ if (result == null) {
+ result = Pair.create(query, dataset);
+ }
+ return result;
+ }
+}
diff --git a/jena-extras/jena-serviceenhancer/src/main/java/org/apache/jena/sparql/service/enhancer/impl/util/VarScopeUtils.java b/jena-extras/jena-serviceenhancer/src/main/java/org/apache/jena/sparql/service/enhancer/impl/util/VarScopeUtils.java
index f8292c6eed..e71be965f7 100644
--- a/jena-extras/jena-serviceenhancer/src/main/java/org/apache/jena/sparql/service/enhancer/impl/util/VarScopeUtils.java
+++ b/jena-extras/jena-serviceenhancer/src/main/java/org/apache/jena/sparql/service/enhancer/impl/util/VarScopeUtils.java
@@ -124,9 +124,7 @@ public class VarScopeUtils {
public static BiMap<Var, Var> normalizeVarScopes(Collection<Var> vars) {
Map<String, Integer> varToMinLevel = getMinimumScopeLevels(vars);
-
BiMap<Var, Var> result = HashBiMap.create();
-
for (Var from : vars) {
String fromName = from.getName();
int fromLevel = getScopeLevel(fromName);
@@ -137,7 +135,6 @@ public class VarScopeUtils {
Var to = allocScoped(plainName, normalizedLevel);
result.put(from, to);
}
-
return result;
}
diff --git a/jena-extras/jena-serviceenhancer/src/main/java/org/apache/jena/sparql/service/enhancer/init/ServiceEnhancerInit.java b/jena-extras/jena-serviceenhancer/src/main/java/org/apache/jena/sparql/service/enhancer/init/ServiceEnhancerInit.java
index 19f0953037..67d81f3dab 100644
--- a/jena-extras/jena-serviceenhancer/src/main/java/org/apache/jena/sparql/service/enhancer/init/ServiceEnhancerInit.java
+++ b/jena-extras/jena-serviceenhancer/src/main/java/org/apache/jena/sparql/service/enhancer/init/ServiceEnhancerInit.java
@@ -18,23 +18,41 @@
package org.apache.jena.sparql.service.enhancer.init;
+import java.util.Map;
+import java.util.Set;
+
import org.apache.jena.assembler.Assembler;
import org.apache.jena.assembler.assemblers.AssemblerGroup;
+import org.apache.jena.atlas.lib.Pair;
import org.apache.jena.graph.Node;
import org.apache.jena.graph.NodeFactory;
import org.apache.jena.query.ARQ;
+import org.apache.jena.query.Query;
import org.apache.jena.sparql.ARQConstants;
import org.apache.jena.sparql.algebra.Op;
+import org.apache.jena.sparql.algebra.OpAsQuery;
+import org.apache.jena.sparql.algebra.OpVars;
import org.apache.jena.sparql.algebra.Transformer;
import org.apache.jena.sparql.algebra.op.OpService;
import org.apache.jena.sparql.algebra.optimize.Optimize;
import org.apache.jena.sparql.algebra.optimize.Rewrite;
import org.apache.jena.sparql.algebra.optimize.RewriteFactory;
import org.apache.jena.sparql.core.DatasetGraph;
+import org.apache.jena.sparql.core.DynamicDatasets.DynamicDatasetGraph;
+import org.apache.jena.sparql.core.Var;
import org.apache.jena.sparql.core.assembler.AssemblerUtils;
import org.apache.jena.sparql.core.assembler.DatasetAssembler;
import org.apache.jena.sparql.engine.ExecutionContext;
+import org.apache.jena.sparql.engine.Plan;
+import org.apache.jena.sparql.engine.QueryEngineFactory;
+import org.apache.jena.sparql.engine.QueryEngineRegistry;
import org.apache.jena.sparql.engine.QueryIterator;
+import org.apache.jena.sparql.engine.Rename;
+import org.apache.jena.sparql.engine.binding.Binding;
+import org.apache.jena.sparql.engine.binding.BindingFactory;
+import org.apache.jena.sparql.engine.iterator.QueryIter;
+import org.apache.jena.sparql.engine.iterator.QueryIterCommonParent;
+import org.apache.jena.sparql.engine.iterator.QueryIteratorMapped;
import org.apache.jena.sparql.engine.main.QC;
import org.apache.jena.sparql.function.FunctionRegistry;
import org.apache.jena.sparql.pfunction.PropertyFunctionRegistry;
@@ -48,6 +66,8 @@ import org.apache.jena.sparql.service.enhancer.impl.ChainingServiceExecutorBulkS
import org.apache.jena.sparql.service.enhancer.impl.ServiceOpts;
import org.apache.jena.sparql.service.enhancer.impl.ServiceResponseCache;
import org.apache.jena.sparql.service.enhancer.impl.ServiceResultSizeCache;
+import org.apache.jena.sparql.service.enhancer.impl.util.DynamicDatasetUtils;
+import org.apache.jena.sparql.service.enhancer.impl.util.VarScopeUtils;
import org.apache.jena.sparql.service.enhancer.pfunction.cacheLs;
import org.apache.jena.sparql.service.single.ChainingServiceExecutor;
import org.apache.jena.sparql.util.Context;
@@ -100,20 +120,32 @@ public class ServiceEnhancerInit
QueryIterator r;
ServiceOpts so = ServiceOpts.getEffectiveService(opExec);
OpService target = so.getTargetService();
+ DatasetGraph dataset = execCxt.getDataset();
// It seems that we always need to run the optimizer here
// in order to have property functions recognized properly
if (ServiceEnhancerConstants.SELF.equals(target.getService())) {
String optimizerMode = so.getFirstValue(ServiceOpts.SO_OPTIMIZE, "on", "on");
Op op = opExec.getSubOp();
- // Run the optimizer unless disabled
- if (!"off".equals(optimizerMode)) {
- Context cxt = execCxt.getContext();
- RewriteFactory rf = decideOptimizer(cxt);
- Rewrite rw = rf.create(cxt);
- op = rw.rewrite(op);
+
+ boolean useQc = false;
+ if (useQc) {
+ // Run the optimizer unless disabled
+ if (!"off".equals(optimizerMode)) {
+ Context cxt = execCxt.getContext();
+ RewriteFactory rf = decideOptimizer(cxt);
+ Rewrite rw = rf.create(cxt);
+ op = rw.rewrite(op);
+ }
+ // Using QC with e.g. TDB2 breaks unionDefaultGraph mode.
+ // Issue seems to be mitigated going through QueryEngineRegistry.
+ r = QC.execute(op, binding, execCxt);
+ } else {
+ // A context copy is needed in order to isolate changes from further executions;
+ // without a copy query engines may e.g. overwrite the context value for the NOW() function.
+ Context cxtCopy = execCxt.getContext().copy();
+ r = execute(op, dataset, binding, cxtCopy);
}
- r = QC.execute(op, binding, execCxt);
} else {
r = chain.createExecution(opExec, opOrig, binding, execCxt);
}
@@ -122,6 +154,47 @@ public class ServiceEnhancerInit
registry.addSingleLink(selfExec);
}
+ /** Special processing that unwraps dynamic datasets */
+ private static QueryIterator execute(Op op, DatasetGraph dataset, Binding binding, Context cxt) {
+ QueryIterator innerIter = null;
+ QueryIterator outerIter = null;
+ ExecutionContext execCxt = null;
+
+ DynamicDatasetGraph ddg = DynamicDatasetUtils.asUnwrappableDynamicDatasetOrNull(dataset);
+ if (ddg != null) {
+ // We are about to create a query from the op which loses scope information
+ // Set up the map that allows for mapping the query's result set variables's
+ // to the appropriately scoped ones
+ Set<Var> visibleVars = OpVars.visibleVars(op);
+ Map<Var, Var> normedToScoped = VarScopeUtils.normalizeVarScopes(visibleVars).inverse();
+
+ Op opRestored = Rename.reverseVarRename(op, true);
+ Query baseQuery = OpAsQuery.asQuery(opRestored);
+ Pair<Query, DatasetGraph> pair = DynamicDatasetUtils.unwrap(baseQuery, ddg);
+ Query effQuery = pair.getLeft();
+ DatasetGraph effDataset = pair.getRight();
+
+ QueryEngineFactory qef = QueryEngineRegistry.findFactory(effQuery, effDataset, cxt);
+ // The scoping of the binding does not match with that of the query.
+ // Therefore pass on an empty binding and rename the result set variables
+ // back into their proper scope
+ Plan plan = qef.create(effQuery, effDataset, BindingFactory.empty(), cxt);
+ innerIter = plan.iterator();
+ outerIter = new QueryIteratorMapped(innerIter, normedToScoped);
+ }
+
+ if (innerIter == null) {
+ QueryEngineFactory qef = QueryEngineRegistry.findFactory(op, dataset, cxt);
+ Plan plan = qef.create(op, dataset, BindingFactory.empty(), cxt);
+ innerIter = plan.iterator();
+ outerIter = innerIter;
+ }
+
+ execCxt = innerIter instanceof QueryIter ? ((QueryIter)innerIter).getExecContext() : null;
+ QueryIterator result = new QueryIterCommonParent(outerIter, binding, execCxt);
+ return result;
+ }
+
static void registerWith(AssemblerGroup g)
{
AssemblerUtils.register(g, ServiceEnhancerVocab.DatasetServiceEnhancer, new DatasetAssemblerServiceEnhancer(), DatasetAssembler.getType());
diff --git a/jena-extras/jena-serviceenhancer/src/test/java/org/apache/jena/sparql/service/enhancer/assembler/TestDatasetAssemblerServiceEnhancer.java b/jena-extras/jena-serviceenhancer/src/test/java/org/apache/jena/sparql/service/enhancer/assembler/TestServiceEnhancerDatasetAssembler.java
similarity index 64%
rename from jena-extras/jena-serviceenhancer/src/test/java/org/apache/jena/sparql/service/enhancer/assembler/TestDatasetAssemblerServiceEnhancer.java
rename to jena-extras/jena-serviceenhancer/src/test/java/org/apache/jena/sparql/service/enhancer/assembler/TestServiceEnhancerDatasetAssembler.java
index e9aea003c1..f624ee2416 100644
--- a/jena-extras/jena-serviceenhancer/src/test/java/org/apache/jena/sparql/service/enhancer/assembler/TestDatasetAssemblerServiceEnhancer.java
+++ b/jena-extras/jena-serviceenhancer/src/test/java/org/apache/jena/sparql/service/enhancer/assembler/TestServiceEnhancerDatasetAssembler.java
@@ -18,25 +18,34 @@
package org.apache.jena.sparql.service.enhancer.assembler;
+import java.io.IOException;
import java.io.StringReader;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import org.apache.jena.ext.com.google.common.io.MoreFiles;
import org.apache.jena.query.Dataset;
import org.apache.jena.query.DatasetFactory;
import org.apache.jena.query.QueryExecException;
import org.apache.jena.query.QueryExecution;
import org.apache.jena.query.QueryExecutionFactory;
+import org.apache.jena.query.ResultSet;
import org.apache.jena.query.ResultSetFormatter;
import org.apache.jena.rdf.model.Model;
import org.apache.jena.rdf.model.ModelFactory;
+import org.apache.jena.rdf.model.Resource;
import org.apache.jena.riot.Lang;
import org.apache.jena.riot.RDFDataMgr;
import org.apache.jena.sparql.service.enhancer.init.ServiceEnhancerConstants;
import org.apache.jena.sparql.util.Context;
+import org.apache.jena.system.Txn;
+import org.apache.jena.tdb2.assembler.VocabTDB2;
import org.apache.jena.vocabulary.RDF;
import org.junit.Assert;
import org.junit.Test;
-public class TestDatasetAssemblerServiceEnhancer
+
+public class TestServiceEnhancerDatasetAssembler
{
private static final String SPEC_STR_01 = String.join("\n",
"PREFIX ja: <http://jena.hpl.hp.com/2005/11/Assembler#>",
@@ -115,4 +124,54 @@ public class TestDatasetAssemblerServiceEnhancer
Assert.assertEquals(1, ResultSetFormatter.consume(qe.execSelect()));
}
}
+
+ /**
+ * Test case to check that when {@code OpService('loop:', subOp)} is evaluated against an union default
+ * graph (UDG) enabled dataset then the subOp must be evaluated on the UDG too.
+ * This also applies to {@code cache:} and {@code bulk:}.
+ */
+ @Test
+ public void testAssemblerTdbUnionDefaultGraph() throws IOException {
+ Path tdb2TmpFolder = Files.createTempDirectory("jena-service-enhancer-tdb2").toAbsolutePath();
+ try {
+ String assemblerStr = String.join("\n",
+ "PREFIX ja: <http://jena.hpl.hp.com/2005/11/Assembler#>",
+ "PREFIX se: <http://jena.apache.org/service-enhancer#>",
+ "PREFIX tdb2: <http://jena.apache.org/2016/tdb#>",
+ "<urn:example:root> a se:DatasetServiceEnhancer ; ja:baseDataset <urn:example:base> .",
+ "<urn:example:root> se:cacheMaxEntryCount 5 ; se:cachePageSize 1000 ; se:cacheMaxPageCount 10 .",
+ "<urn:example:root> se:bulkMaxSize 20 ; se:bulkSize 10 ; se:bulkMaxOutOfBandSize 5 .",
+ "<urn:example:base> a tdb2:DatasetTDB2 .",
+ "<urn:example:base> tdb2:unionDefaultGraph true ."
+ );
+
+ String dataStr = String.join("\n",
+ "PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>",
+ "<urn:example:g1> { <urn:example:s1> a <urn:example:class1> ; rdfs:label \"s1\" }",
+ "<urn:example:g2> { <urn:example:s2> a <urn:example:class2> ; rdfs:label \"s2\" }");
+
+ String queryStr = "SELECT * { ?s a [] BIND(NOW() AS ?outer) SERVICE <loop:> { ?s ?p ?o BIND(NOW() AS ?inner) } }";
+
+ Model confModel = ModelFactory.createDefaultModel();
+ RDFDataMgr.read(confModel, new StringReader(assemblerStr), null, Lang.TURTLE);
+ Resource tdb2Conf = confModel.getResource("urn:example:base");
+ tdb2Conf.addProperty(VocabTDB2.pLocation, tdb2TmpFolder.toString());
+
+ Dataset tdb2Ds = DatasetFactory.assemble(confModel.getResource("urn:example:root"));
+
+ Txn.executeWrite(tdb2Ds, () -> RDFDataMgr.read(tdb2Ds, new StringReader(dataStr), null, Lang.TRIG));
+
+ int actualRowCount = Txn.calculateRead(tdb2Ds, () -> {
+ try (QueryExecution qe = QueryExecution.create(queryStr, tdb2Ds)) {
+ ResultSet rs = qe.execSelect();
+ // System.out.println(ResultSetFormatter.asText(rs));
+ return ResultSetFormatter.consume(rs);
+ }
+ });
+
+ Assert.assertEquals(4, actualRowCount);
+ } finally {
+ MoreFiles.deleteRecursively(tdb2TmpFolder);
+ }
+ }
}
diff --git a/jena-extras/jena-serviceenhancer/src/test/java/org/apache/jena/sparql/service/enhancer/impl/AbstractTestServiceEnhancerResultSetLimits.java b/jena-extras/jena-serviceenhancer/src/test/java/org/apache/jena/sparql/service/enhancer/impl/AbstractTestServiceEnhancerResultSetLimits.java
index a9511b2f4f..0a949a953b 100644
--- a/jena-extras/jena-serviceenhancer/src/test/java/org/apache/jena/sparql/service/enhancer/impl/AbstractTestServiceEnhancerResultSetLimits.java
+++ b/jena-extras/jena-serviceenhancer/src/test/java/org/apache/jena/sparql/service/enhancer/impl/AbstractTestServiceEnhancerResultSetLimits.java
@@ -163,51 +163,12 @@ public abstract class AbstractTestServiceEnhancerResultSetLimits {
}
public static int testCore(Dataset dataset, String queryStr, int hiddenLimit) {
-
-
Query query = QueryFactory.create(queryStr);
- // Register a service plugin that slices the result
+ // Set up a custom service executor registry that "secretly" slices
+ // (i.e. cuts off) result sets after a certain number of rows
+ // This simulates result set limits which one typically encounters with public SPARQL endpoints
ServiceExecutorRegistry reg = ServiceExecutorRegistry.get().copy();
-/*
- reg.addBulkLink(
- (op, iter, execCxt, chain) -> {
- // apply the slice to each input binding
- QueryIterator x = new QueryIterRepeatApply(iter, execCxt) {
- @Override
- protected QueryIterator nextStage(Binding binding) {
- QueryIterator qi = QueryIterPlainWrapper.create(Collections.singleton(binding).iterator(), execCxt);
- return new QueryIterSlice(chain.createExecution(op, qi, execCxt), 0, hiddenLimit, execCxt);
- }
- };
-
- QueryIterator r;
- boolean failOnPurpose = false;
- if (failOnPurpose) {
- QueryIterator y = QueryIterPlainWrapper.create(new AbstractIterator<Binding>() {
- @Override
- protected Binding computeNext() {
- throw new RuntimeException("Synthetic error to test for resource leaks on failure");
- }
- }, execCxt);
-
- QueryIterConcat z = new QueryIterConcat(execCxt);
- z.add(x);
- z.add(y);
- r = z;
- } else {
- r = x;
- }
-
- return r;
- });
-*/
-
-// reg.addBulkLink(
-// (op, iter, execCxt, chain) -> {
-// return new QueryIterSlice(chain.createExecution(op, iter, execCxt), 0, hiddenLimit, execCxt);
-// }
-// );
reg.addSingleLink((opExec, opOrig, binding, execCxt, chain) -> {
return new QueryIterSlice(chain.createExecution(opExec, opOrig, binding, execCxt), 0, hiddenLimit, execCxt);
diff --git a/jena-extras/jena-serviceenhancer/src/test/java/org/apache/jena/sparql/service/enhancer/impl/TestServiceEnhancerBatchQueryRewriter.java b/jena-extras/jena-serviceenhancer/src/test/java/org/apache/jena/sparql/service/enhancer/impl/TestServiceEnhancerBatchQueryRewriter.java
new file mode 100644
index 0000000000..11512e904d
--- /dev/null
+++ b/jena-extras/jena-serviceenhancer/src/test/java/org/apache/jena/sparql/service/enhancer/impl/TestServiceEnhancerBatchQueryRewriter.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.jena.sparql.service.enhancer.impl;
+
+import org.apache.jena.graph.NodeFactory;
+import org.apache.jena.query.Query;
+import org.apache.jena.query.QueryFactory;
+import org.apache.jena.query.Syntax;
+import org.apache.jena.sparql.algebra.Algebra;
+import org.apache.jena.sparql.algebra.Op;
+import org.apache.jena.sparql.algebra.OpAsQuery;
+import org.apache.jena.sparql.algebra.op.OpService;
+import org.apache.jena.sparql.core.Var;
+import org.apache.jena.sparql.engine.binding.Binding;
+import org.apache.jena.sparql.engine.binding.BindingFactory;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestServiceEnhancerBatchQueryRewriter {
+
+ /**
+ * Test that the generation of a bulk request from a query that uses limit and order by uses the correct rewrite.
+ * Specifically, ORDER BY must be retained on the union members even if they appear without slice because
+ * it might make a difference when the result set is cut off by the service's result set limit.
+ */
+ @Test
+ public void testOrderBy_01() {
+ OpService op = (OpService)Algebra.compile(QueryFactory.create(String.join("\n",
+ "SELECT * {",
+ " SERVICE <http://example.org/> { SELECT * { ?s ?p ?o } ORDER BY ?s LIMIT 1 }",
+ "}"), Syntax.syntaxARQ).getQueryPattern());
+
+ Batch<Integer, PartitionRequest<Binding>> batch = BatchImpl.forInteger();
+ Var o = Var.alloc("o");
+ batch.put(0, new PartitionRequest<>(0, BindingFactory.binding(o, NodeFactory.createLiteral("x1")), 1, 5));
+ batch.put(1, new PartitionRequest<>(1, BindingFactory.binding(o, NodeFactory.createLiteral("x2")), 2, 6));
+
+ BatchQueryRewriter rewriter = new BatchQueryRewriter(new OpServiceInfo(op), Var.alloc("idx"), false, false, false);
+ BatchQueryRewriteResult rewrite = rewriter.rewrite(batch);
+ Op resultOp = rewrite.getOp();
+ Query actualQuery = OpAsQuery.asQuery(resultOp);
+
+ Query expectedQuery = QueryFactory.create(String.join("\n",
+ "SELECT *",
+ "WHERE",
+ " { { { SELECT *",
+ " WHERE",
+ " { ?s ?p \"x1\" }",
+ " ORDER BY ?s",
+ " OFFSET 1",
+ " LIMIT 5",
+ " }",
+ " BIND(0 AS ?idx)",
+ " }",
+ " UNION",
+ " { { { SELECT *",
+ " WHERE",
+ " { ?s ?p \"x2\" }",
+ " ORDER BY ?s",
+ " OFFSET 2",
+ " LIMIT 6",
+ " }",
+ " BIND(1 AS ?idx)",
+ " }",
+ " UNION",
+ " { BIND(1000000000 AS ?idx) }",
+ " }",
+ " }",
+ "ORDER BY ASC(?idx) ?s"));
+
+ Assert.assertEquals(expectedQuery, actualQuery);
+ }
+}
diff --git a/jena-extras/jena-serviceenhancer/src/test/java/org/apache/jena/sparql/service/enhancer/impl/TestBatcher.java b/jena-extras/jena-serviceenhancer/src/test/java/org/apache/jena/sparql/service/enhancer/impl/TestServiceEnhancerBatcher.java
similarity index 99%
rename from jena-extras/jena-serviceenhancer/src/test/java/org/apache/jena/sparql/service/enhancer/impl/TestBatcher.java
rename to jena-extras/jena-serviceenhancer/src/test/java/org/apache/jena/sparql/service/enhancer/impl/TestServiceEnhancerBatcher.java
index e5e53dd60d..7950e72541 100644
--- a/jena-extras/jena-serviceenhancer/src/test/java/org/apache/jena/sparql/service/enhancer/impl/TestBatcher.java
+++ b/jena-extras/jena-serviceenhancer/src/test/java/org/apache/jena/sparql/service/enhancer/impl/TestServiceEnhancerBatcher.java
@@ -32,7 +32,7 @@ import org.apache.jena.ext.com.google.common.collect.Streams;
import org.junit.Assert;
import org.junit.Test;
-public class TestBatcher {
+public class TestServiceEnhancerBatcher {
private static final List<Entry<String, Integer>> testData01 = List.<Entry<String, Integer>>of(
Map.entry("a", 0),