You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@jena.apache.org by an...@apache.org on 2022/12/11 11:26:35 UTC

[jena] branch main updated: GH-1656 Service enhancer: bug fix for bulk requests with order+limit; improvement that sub-requests work with union default graph

This is an automated email from the ASF dual-hosted git repository.

andy pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/jena.git


The following commit(s) were added to refs/heads/main by this push:
     new 3a79ea2083 GH-1656 Service enhancer: bug fix for bulk requests with order+limit; improvement that sub-requests work with union default graph
     new b0f24f4b9d Merge pull request #1657 from Aklakan/GH-1656
3a79ea2083 is described below

commit 3a79ea208373a31423d51fc7c18663595780dac5
Author: Claus Stadler <Ra...@googlemail.com>
AuthorDate: Thu Oct 13 14:51:52 2022 +0200

    GH-1656 Service enhancer: bug fix for bulk requests with order+limit; improvement that sub-requests work with union default graph
---
 jena-extras/jena-serviceenhancer/pom.xml           |  9 ++-
 .../service/enhancer/impl/BatchQueryRewriter.java  | 66 +++-------------
 .../sparql/service/enhancer/impl/Estimate.java     |  4 +-
 .../service/enhancer/impl/PartitionRequest.java    |  6 +-
 .../enhancer/impl/QueryIterServiceBulk.java        |  2 +-
 .../enhancer/impl/util/DynamicDatasetUtils.java    | 90 ++++++++++++++++++++++
 .../service/enhancer/impl/util/VarScopeUtils.java  |  3 -
 .../service/enhancer/init/ServiceEnhancerInit.java | 87 +++++++++++++++++++--
 ...va => TestServiceEnhancerDatasetAssembler.java} | 61 ++++++++++++++-
 ...AbstractTestServiceEnhancerResultSetLimits.java | 45 +----------
 .../TestServiceEnhancerBatchQueryRewriter.java     | 89 +++++++++++++++++++++
 ...atcher.java => TestServiceEnhancerBatcher.java} |  2 +-
 12 files changed, 352 insertions(+), 112 deletions(-)

diff --git a/jena-extras/jena-serviceenhancer/pom.xml b/jena-extras/jena-serviceenhancer/pom.xml
index 3b9b47e178..b30c0c0831 100644
--- a/jena-extras/jena-serviceenhancer/pom.xml
+++ b/jena-extras/jena-serviceenhancer/pom.xml
@@ -37,7 +37,7 @@
     <dependency>
       <groupId>org.apache.jena</groupId>
       <artifactId>jena-arq</artifactId>
-      <version>4.7.0-SNAPSHOT</version>
+      <version>${project.version}</version>
     </dependency>
 
     <dependency>
@@ -67,6 +67,13 @@
       <artifactId>log4j-slf4j-impl</artifactId>
       <scope>test</scope>
     </dependency>
+
+    <dependency>
+      <groupId>org.apache.jena</groupId>
+      <artifactId>jena-tdb2</artifactId>
+      <version>${project.version}</version>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 
   <build>
diff --git a/jena-extras/jena-serviceenhancer/src/main/java/org/apache/jena/sparql/service/enhancer/impl/BatchQueryRewriter.java b/jena-extras/jena-serviceenhancer/src/main/java/org/apache/jena/sparql/service/enhancer/impl/BatchQueryRewriter.java
index 35b8bcbcf6..a4c8e331d2 100644
--- a/jena-extras/jena-serviceenhancer/src/main/java/org/apache/jena/sparql/service/enhancer/impl/BatchQueryRewriter.java
+++ b/jena-extras/jena-serviceenhancer/src/main/java/org/apache/jena/sparql/service/enhancer/impl/BatchQueryRewriter.java
@@ -33,7 +33,6 @@ import java.util.Set;
 import org.apache.jena.atlas.logging.Log;
 import org.apache.jena.query.Query;
 import org.apache.jena.query.SortCondition;
-import org.apache.jena.sparql.algebra.Algebra;
 import org.apache.jena.sparql.algebra.Op;
 import org.apache.jena.sparql.algebra.OpAsQuery;
 import org.apache.jena.sparql.algebra.op.OpExtend;
@@ -85,11 +84,6 @@ public class BatchQueryRewriter {
     static int REMOTE_END_MARKER = 1000000000;
     static NodeValue NV_REMOTE_END_MARKER = NodeValue.makeInteger(REMOTE_END_MARKER);
 
-    /** True if either local or remote end marker */
-//    public static boolean isLocalOrRemoteEndMarker(int id) {
-//        return isRemoteEndMarker(id) || isLocalEndMarker(id);
-//    }
-
     public static boolean isRemoteEndMarker(int id) {
         return id == REMOTE_END_MARKER;
     }
@@ -98,21 +92,6 @@ public class BatchQueryRewriter {
         return Objects.equals(id, REMOTE_END_MARKER);
     }
 
-
-
-    // Local end marker is not returned by the remote service
-//    static int LOCAL_END_MARKER = 1000000001;
-//    static NodeValue NV_LOCAL_END_MARKER = NodeValue.makeInteger(LOCAL_END_MARKER);
-//
-//    public static boolean isLocalEndMarker(int id) {
-//        return id == LOCAL_END_MARKER;
-//    }
-//
-//    public static boolean isLocalEndMarker(Integer id) {
-//        return Objects.equals(id, LOCAL_END_MARKER);
-//    }
-
-
     public BatchQueryRewriter(OpServiceInfo serviceInfo, Var idxVar,
             boolean sequentialUnion, boolean orderRetainingUnion,
             boolean omitEndMarker) {
@@ -136,18 +115,17 @@ public class BatchQueryRewriter {
     }
 
     public BatchQueryRewriteResult rewrite(Batch<Integer, PartitionRequest<Binding>> batchRequest) {
-
         Op newOp = null;
         List<Entry<Integer, PartitionRequest<Binding>>> es = new ArrayList<>(batchRequest.getItems().entrySet());
         Collections.reverse(es);
 
-        Query normQuery = serviceInfo.getNormedQuery();
-        Op normOp = serviceInfo.getNormedQueryOp();
+        Query normedQuery = serviceInfo.getNormedQuery();
+        Op normedOp = serviceInfo.getNormedQueryOp();
 
         // Prepare the sort conditions
         List<SortCondition> sortConditions = new ArrayList<>();
         List<SortCondition> localSortConditions =
-                Optional.ofNullable(normQuery.getOrderBy()).orElse(Collections.emptyList());
+                Optional.ofNullable(normedQuery.getOrderBy()).orElse(Collections.emptyList());
 
         boolean noOrderNeeded =
                 orderRetainingUnion || sequentialUnion && localSortConditions.isEmpty();
@@ -162,9 +140,14 @@ public class BatchQueryRewriter {
 
         sortConditions.addAll(localSortConditions);
 
-        for (Entry<Integer, PartitionRequest<Binding>> e : es) { // batchRequest.getItems().entrySet()) {
+        if (!omitEndMarker) {
+            Op endMarker = OpExtend.create(OpTable.unit(), idxVar, NV_REMOTE_END_MARKER);
+            newOp = newOp == null ? endMarker : OpUnion.create(newOp, endMarker);
+        }
 
-            PartitionRequest<Binding> req = e.getValue(); // batchRequest.get(i);
+        for (Entry<Integer, PartitionRequest<Binding>> e : es) {
+
+            PartitionRequest<Binding> req = e.getValue();
             long idx = e.getKey();
             Binding scopedBinding = req.getPartitionKey();
 
@@ -173,37 +156,15 @@ public class BatchQueryRewriter {
             Map<Var, Var> varMapScopedToNormed = ServiceCacheKeyFactory
                     .createJoinVarMapScopedToNormed(serviceInfo, scopedBindingVars);
 
-            // Binding plainBinding = BindingUtils.renameKeys(scopedBinding, serviceInfo.getMentionedSubOpVarsScopedToPlain());
             Binding normedBinding = BindingUtils.renameKeys(scopedBinding, varMapScopedToNormed);
 
             // Op op = serviceInfo.getNormedQueryOp();
-            Op op = normOp;
+            Op op = normedOp;
 
             // Note: QC.substitute does not remove variables being substituted from projections
             //   This may cause unbound variables to be projected
 
-            // If the union is sequential and order retaining we can retain the order on the members
-            // otherwise, we can remove any ordering on the member
-            if ((sequentialUnion && orderRetainingUnion) || localSortConditions.isEmpty()) {
-                // If the union is sequential and order retaining we can retain the order on the members
-                // otherwise, we can remove any ordering on the member
-            } else {
-                // Member order may not be retained - remove it from the query
-
-                // TODO This should be done once OpServiceInfo
-                Query tmp = normQuery.cloneQuery();
-                if (tmp.hasOrderBy()) {
-                    tmp.getOrderBy().clear();
-                }
-
-                op = Algebra.compile(tmp);
-                // TODO Something is odd with ordering here
-                // Add the sort conditions
-                // op = new OpOrder(op, localSortConditions);
-            }
-
             op = QC.substitute(op, normedBinding);
-            op = OpExtend.create(op, idxVar, NodeValue.makeInteger(idx));
 
             long o = req.hasOffset() ? req.getOffset() : Query.NOLIMIT;
             long l = req.hasLimit() ? req.getLimit() : Query.NOLIMIT;
@@ -211,14 +172,11 @@ public class BatchQueryRewriter {
             if (o != Query.NOLIMIT || l != Query.NOLIMIT) {
                 op = new OpSlice(op, o, l);
             }
+            op = OpExtend.create(op, idxVar, NodeValue.makeInteger(idx));
 
             newOp = newOp == null ? op : OpUnion.create(op, newOp);
         }
 
-        if (!omitEndMarker) {
-            Op endMarker = OpExtend.create(OpTable.unit(), idxVar, NV_REMOTE_END_MARKER);
-            newOp = newOp == null ? endMarker : OpUnion.create(newOp, endMarker);
-        }
 
         if (orderNeeded) {
             newOp = new OpOrder(newOp, sortConditions);
diff --git a/jena-extras/jena-serviceenhancer/src/main/java/org/apache/jena/sparql/service/enhancer/impl/Estimate.java b/jena-extras/jena-serviceenhancer/src/main/java/org/apache/jena/sparql/service/enhancer/impl/Estimate.java
index 53e0969bd8..d262e797b2 100644
--- a/jena-extras/jena-serviceenhancer/src/main/java/org/apache/jena/sparql/service/enhancer/impl/Estimate.java
+++ b/jena-extras/jena-serviceenhancer/src/main/java/org/apache/jena/sparql/service/enhancer/impl/Estimate.java
@@ -21,7 +21,9 @@ package org.apache.jena.sparql.service.enhancer.impl;
 import java.io.Serializable;
 import java.util.Objects;
 
-/** An estimated value with a flag that indicates whether it's exact */
+/**
+ * An estimated value with a flag that indicates whether it's exact.
+ */
 public class Estimate<T>
     implements Serializable
 {
diff --git a/jena-extras/jena-serviceenhancer/src/main/java/org/apache/jena/sparql/service/enhancer/impl/PartitionRequest.java b/jena-extras/jena-serviceenhancer/src/main/java/org/apache/jena/sparql/service/enhancer/impl/PartitionRequest.java
index 2c08334955..5753ac4f0c 100644
--- a/jena-extras/jena-serviceenhancer/src/main/java/org/apache/jena/sparql/service/enhancer/impl/PartitionRequest.java
+++ b/jena-extras/jena-serviceenhancer/src/main/java/org/apache/jena/sparql/service/enhancer/impl/PartitionRequest.java
@@ -18,7 +18,11 @@
 
 package org.apache.jena.sparql.service.enhancer.impl;
 
-/** Helper class to capture a range of data w.r.t. a partition key (typically a binding) and assign it an id */
+/**
+ * Helper class to capture a range of data (specified by limit + offset)
+ * w.r.t. a partition key (typically a {@link org.apache.jena.sparql.engine.binding.Binding}
+ * and give that information an id.
+ */
 public class PartitionRequest<I>
 {
     protected long outputId;
diff --git a/jena-extras/jena-serviceenhancer/src/main/java/org/apache/jena/sparql/service/enhancer/impl/QueryIterServiceBulk.java b/jena-extras/jena-serviceenhancer/src/main/java/org/apache/jena/sparql/service/enhancer/impl/QueryIterServiceBulk.java
index 3df6bec0df..407e605f16 100644
--- a/jena-extras/jena-serviceenhancer/src/main/java/org/apache/jena/sparql/service/enhancer/impl/QueryIterServiceBulk.java
+++ b/jena-extras/jena-serviceenhancer/src/main/java/org/apache/jena/sparql/service/enhancer/impl/QueryIterServiceBulk.java
@@ -208,7 +208,7 @@ public class QueryIterServiceBulk
 
                 resultSizeCache.updateLimit(targetService, backendResultSetLimit);
 
-                // We obtained to few data for the current id - repeat the request
+                // We obtained too little data for the current id - repeat the request
                 prepareNextBatchExec(false);
                 continue;
             }
diff --git a/jena-extras/jena-serviceenhancer/src/main/java/org/apache/jena/sparql/service/enhancer/impl/util/DynamicDatasetUtils.java b/jena-extras/jena-serviceenhancer/src/main/java/org/apache/jena/sparql/service/enhancer/impl/util/DynamicDatasetUtils.java
new file mode 100644
index 0000000000..4df92f9fc3
--- /dev/null
+++ b/jena-extras/jena-serviceenhancer/src/main/java/org/apache/jena/sparql/service/enhancer/impl/util/DynamicDatasetUtils.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.jena.sparql.service.enhancer.impl.util;
+
+import org.apache.jena.atlas.lib.Pair;
+import org.apache.jena.query.Query;
+import org.apache.jena.sparql.core.DatasetGraph;
+import org.apache.jena.sparql.core.DynamicDatasets.DynamicDatasetGraph;
+
+public class DynamicDatasetUtils {
+
+    /**
+     * Returns true iff the argument's original default and named graphs are non-null.
+     * Typically, this shouldn't happen. This method mainly exists for robustness as to
+     * avoid null pointer exceptions.
+     */
+    public static boolean isUnwrappable(DynamicDatasetGraph ddg) {
+        boolean result = ddg.getOriginal() != null &&
+                ddg.getOriginalDefaultGraphs() != null && ddg.getOriginalNamedGraphs() != null;
+        return result;
+    }
+
+    /** Convenience method to cast the argument as a DynamicDatasetGraph if it is an unwrappable one. */
+    public static DynamicDatasetGraph asUnwrappableDynamicDatasetOrNull(DatasetGraph dg) {
+        DynamicDatasetGraph result = null;
+        if (dg instanceof DynamicDatasetGraph) {
+            DynamicDatasetGraph ddg = (DynamicDatasetGraph)dg;
+            if (isUnwrappable(ddg)) {
+                result = ddg;
+            }
+        }
+        return result;
+    }
+
+    /**
+     * Returns the argument unless it is a DynamicDataset for which
+     * {@link #isUnwrappable(DynamicDatasetGraph)} is true. In that case
+     * {@link DynamicDatasetGraph#getOriginal()} is returned.
+     */
+    public static DatasetGraph unwrapOriginal(DatasetGraph dg) {
+        DatasetGraph result = dg;
+        if (dg instanceof DynamicDatasetGraph) {
+            DynamicDatasetGraph ddg = (DynamicDatasetGraph)dg;
+            if (isUnwrappable(ddg)) {
+                result = ddg.getOriginal();
+            }
+        }
+        return result;
+    }
+
+    /**
+     * Returns the given arguments as a pair unless the dataset is an unwrappable DynamicDataset.
+     * In that case, query's default/named graphs are modified in-place to match those of the dataset.
+     * The returned pair holds the modified query and the original (unwrapped) dataset.
+     */
+    public static Pair<Query, DatasetGraph> unwrap(Query query, DatasetGraph dataset) {
+        Pair<Query, DatasetGraph> result = null;
+        if (dataset instanceof DynamicDatasetGraph) {
+            DynamicDatasetGraph ddg = (DynamicDatasetGraph)dataset;
+            if (isUnwrappable(ddg)) {
+                // Query copy = query.cloneQuery(); Should we clone to be safe?
+                query.getGraphURIs().clear();
+                query.getNamedGraphURIs().clear();
+                ddg.getOriginalDefaultGraphs().forEach(n -> query.getGraphURIs().add(n.getURI()));
+                ddg.getOriginalNamedGraphs().forEach(n -> query.getNamedGraphURIs().add(n.getURI()));
+                result = Pair.create(query, ddg.getOriginal());
+            }
+        }
+        if (result == null) {
+            result = Pair.create(query, dataset);
+        }
+        return result;
+    }
+}
diff --git a/jena-extras/jena-serviceenhancer/src/main/java/org/apache/jena/sparql/service/enhancer/impl/util/VarScopeUtils.java b/jena-extras/jena-serviceenhancer/src/main/java/org/apache/jena/sparql/service/enhancer/impl/util/VarScopeUtils.java
index f8292c6eed..e71be965f7 100644
--- a/jena-extras/jena-serviceenhancer/src/main/java/org/apache/jena/sparql/service/enhancer/impl/util/VarScopeUtils.java
+++ b/jena-extras/jena-serviceenhancer/src/main/java/org/apache/jena/sparql/service/enhancer/impl/util/VarScopeUtils.java
@@ -124,9 +124,7 @@ public class VarScopeUtils {
 
     public static BiMap<Var, Var> normalizeVarScopes(Collection<Var> vars) {
         Map<String, Integer> varToMinLevel = getMinimumScopeLevels(vars);
-
         BiMap<Var, Var> result = HashBiMap.create();
-
         for (Var from : vars) {
             String fromName = from.getName();
             int fromLevel = getScopeLevel(fromName);
@@ -137,7 +135,6 @@ public class VarScopeUtils {
             Var to = allocScoped(plainName, normalizedLevel);
             result.put(from, to);
         }
-
         return result;
     }
 
diff --git a/jena-extras/jena-serviceenhancer/src/main/java/org/apache/jena/sparql/service/enhancer/init/ServiceEnhancerInit.java b/jena-extras/jena-serviceenhancer/src/main/java/org/apache/jena/sparql/service/enhancer/init/ServiceEnhancerInit.java
index 19f0953037..67d81f3dab 100644
--- a/jena-extras/jena-serviceenhancer/src/main/java/org/apache/jena/sparql/service/enhancer/init/ServiceEnhancerInit.java
+++ b/jena-extras/jena-serviceenhancer/src/main/java/org/apache/jena/sparql/service/enhancer/init/ServiceEnhancerInit.java
@@ -18,23 +18,41 @@
 
 package org.apache.jena.sparql.service.enhancer.init;
 
+import java.util.Map;
+import java.util.Set;
+
 import org.apache.jena.assembler.Assembler;
 import org.apache.jena.assembler.assemblers.AssemblerGroup;
+import org.apache.jena.atlas.lib.Pair;
 import org.apache.jena.graph.Node;
 import org.apache.jena.graph.NodeFactory;
 import org.apache.jena.query.ARQ;
+import org.apache.jena.query.Query;
 import org.apache.jena.sparql.ARQConstants;
 import org.apache.jena.sparql.algebra.Op;
+import org.apache.jena.sparql.algebra.OpAsQuery;
+import org.apache.jena.sparql.algebra.OpVars;
 import org.apache.jena.sparql.algebra.Transformer;
 import org.apache.jena.sparql.algebra.op.OpService;
 import org.apache.jena.sparql.algebra.optimize.Optimize;
 import org.apache.jena.sparql.algebra.optimize.Rewrite;
 import org.apache.jena.sparql.algebra.optimize.RewriteFactory;
 import org.apache.jena.sparql.core.DatasetGraph;
+import org.apache.jena.sparql.core.DynamicDatasets.DynamicDatasetGraph;
+import org.apache.jena.sparql.core.Var;
 import org.apache.jena.sparql.core.assembler.AssemblerUtils;
 import org.apache.jena.sparql.core.assembler.DatasetAssembler;
 import org.apache.jena.sparql.engine.ExecutionContext;
+import org.apache.jena.sparql.engine.Plan;
+import org.apache.jena.sparql.engine.QueryEngineFactory;
+import org.apache.jena.sparql.engine.QueryEngineRegistry;
 import org.apache.jena.sparql.engine.QueryIterator;
+import org.apache.jena.sparql.engine.Rename;
+import org.apache.jena.sparql.engine.binding.Binding;
+import org.apache.jena.sparql.engine.binding.BindingFactory;
+import org.apache.jena.sparql.engine.iterator.QueryIter;
+import org.apache.jena.sparql.engine.iterator.QueryIterCommonParent;
+import org.apache.jena.sparql.engine.iterator.QueryIteratorMapped;
 import org.apache.jena.sparql.engine.main.QC;
 import org.apache.jena.sparql.function.FunctionRegistry;
 import org.apache.jena.sparql.pfunction.PropertyFunctionRegistry;
@@ -48,6 +66,8 @@ import org.apache.jena.sparql.service.enhancer.impl.ChainingServiceExecutorBulkS
 import org.apache.jena.sparql.service.enhancer.impl.ServiceOpts;
 import org.apache.jena.sparql.service.enhancer.impl.ServiceResponseCache;
 import org.apache.jena.sparql.service.enhancer.impl.ServiceResultSizeCache;
+import org.apache.jena.sparql.service.enhancer.impl.util.DynamicDatasetUtils;
+import org.apache.jena.sparql.service.enhancer.impl.util.VarScopeUtils;
 import org.apache.jena.sparql.service.enhancer.pfunction.cacheLs;
 import org.apache.jena.sparql.service.single.ChainingServiceExecutor;
 import org.apache.jena.sparql.util.Context;
@@ -100,20 +120,32 @@ public class ServiceEnhancerInit
             QueryIterator r;
             ServiceOpts so = ServiceOpts.getEffectiveService(opExec);
             OpService target = so.getTargetService();
+            DatasetGraph dataset = execCxt.getDataset();
 
             // It seems that we always need to run the optimizer here
             // in order to have property functions recognized properly
             if (ServiceEnhancerConstants.SELF.equals(target.getService())) {
                 String optimizerMode = so.getFirstValue(ServiceOpts.SO_OPTIMIZE, "on", "on");
                 Op op = opExec.getSubOp();
-                // Run the optimizer unless disabled
-                if (!"off".equals(optimizerMode)) {
-                    Context cxt = execCxt.getContext();
-                    RewriteFactory rf = decideOptimizer(cxt);
-                    Rewrite rw = rf.create(cxt);
-                    op = rw.rewrite(op);
+
+                boolean useQc = false;
+                if (useQc) {
+                    // Run the optimizer unless disabled
+                    if (!"off".equals(optimizerMode)) {
+                        Context cxt = execCxt.getContext();
+                        RewriteFactory rf = decideOptimizer(cxt);
+                        Rewrite rw = rf.create(cxt);
+                        op = rw.rewrite(op);
+                    }
+                    // Using QC with e.g. TDB2 breaks unionDefaultGraph mode.
+                    //   Issue seems to be mitigated going through QueryEngineRegistry.
+                    r = QC.execute(op, binding, execCxt);
+                } else {
+                    // A context copy is needed in order to isolate changes from further executions;
+                    //   without a copy query engines may e.g. overwrite the context value for the NOW() function.
+                    Context cxtCopy = execCxt.getContext().copy();
+                    r = execute(op, dataset, binding, cxtCopy);
                 }
-                r = QC.execute(op, binding, execCxt);
             } else {
                 r = chain.createExecution(opExec, opOrig, binding, execCxt);
             }
@@ -122,6 +154,47 @@ public class ServiceEnhancerInit
         registry.addSingleLink(selfExec);
     }
 
+    /** Special processing that unwraps dynamic datasets */
+    private static QueryIterator execute(Op op, DatasetGraph dataset, Binding binding, Context cxt) {
+        QueryIterator innerIter = null;
+        QueryIterator outerIter = null;
+        ExecutionContext execCxt = null;
+
+        DynamicDatasetGraph ddg = DynamicDatasetUtils.asUnwrappableDynamicDatasetOrNull(dataset);
+        if (ddg != null) {
+            // We are about to create a query from the op which loses scope information
+            // Set up the map that allows for mapping the query's result set variables's
+            // to the appropriately scoped ones
+            Set<Var> visibleVars = OpVars.visibleVars(op);
+            Map<Var, Var> normedToScoped = VarScopeUtils.normalizeVarScopes(visibleVars).inverse();
+
+            Op opRestored = Rename.reverseVarRename(op, true);
+            Query baseQuery = OpAsQuery.asQuery(opRestored);
+            Pair<Query, DatasetGraph> pair = DynamicDatasetUtils.unwrap(baseQuery, ddg);
+            Query effQuery = pair.getLeft();
+            DatasetGraph effDataset = pair.getRight();
+
+            QueryEngineFactory qef = QueryEngineRegistry.findFactory(effQuery, effDataset, cxt);
+            // The scoping of the binding does not match with that of the query.
+            // Therefore pass on an empty binding and rename the result set variables
+            // back into their proper scope
+            Plan plan = qef.create(effQuery, effDataset, BindingFactory.empty(), cxt);
+            innerIter = plan.iterator();
+            outerIter = new QueryIteratorMapped(innerIter, normedToScoped);
+        }
+
+        if (innerIter == null) {
+            QueryEngineFactory qef = QueryEngineRegistry.findFactory(op, dataset, cxt);
+            Plan plan = qef.create(op, dataset, BindingFactory.empty(), cxt);
+            innerIter = plan.iterator();
+            outerIter = innerIter;
+        }
+
+        execCxt = innerIter instanceof QueryIter ? ((QueryIter)innerIter).getExecContext() : null;
+        QueryIterator result = new QueryIterCommonParent(outerIter, binding, execCxt);
+        return result;
+    }
+
     static void registerWith(AssemblerGroup g)
     {
         AssemblerUtils.register(g, ServiceEnhancerVocab.DatasetServiceEnhancer, new DatasetAssemblerServiceEnhancer(), DatasetAssembler.getType());
diff --git a/jena-extras/jena-serviceenhancer/src/test/java/org/apache/jena/sparql/service/enhancer/assembler/TestDatasetAssemblerServiceEnhancer.java b/jena-extras/jena-serviceenhancer/src/test/java/org/apache/jena/sparql/service/enhancer/assembler/TestServiceEnhancerDatasetAssembler.java
similarity index 64%
rename from jena-extras/jena-serviceenhancer/src/test/java/org/apache/jena/sparql/service/enhancer/assembler/TestDatasetAssemblerServiceEnhancer.java
rename to jena-extras/jena-serviceenhancer/src/test/java/org/apache/jena/sparql/service/enhancer/assembler/TestServiceEnhancerDatasetAssembler.java
index e9aea003c1..f624ee2416 100644
--- a/jena-extras/jena-serviceenhancer/src/test/java/org/apache/jena/sparql/service/enhancer/assembler/TestDatasetAssemblerServiceEnhancer.java
+++ b/jena-extras/jena-serviceenhancer/src/test/java/org/apache/jena/sparql/service/enhancer/assembler/TestServiceEnhancerDatasetAssembler.java
@@ -18,25 +18,34 @@
 
 package org.apache.jena.sparql.service.enhancer.assembler;
 
+import java.io.IOException;
 import java.io.StringReader;
+import java.nio.file.Files;
+import java.nio.file.Path;
 
+import org.apache.jena.ext.com.google.common.io.MoreFiles;
 import org.apache.jena.query.Dataset;
 import org.apache.jena.query.DatasetFactory;
 import org.apache.jena.query.QueryExecException;
 import org.apache.jena.query.QueryExecution;
 import org.apache.jena.query.QueryExecutionFactory;
+import org.apache.jena.query.ResultSet;
 import org.apache.jena.query.ResultSetFormatter;
 import org.apache.jena.rdf.model.Model;
 import org.apache.jena.rdf.model.ModelFactory;
+import org.apache.jena.rdf.model.Resource;
 import org.apache.jena.riot.Lang;
 import org.apache.jena.riot.RDFDataMgr;
 import org.apache.jena.sparql.service.enhancer.init.ServiceEnhancerConstants;
 import org.apache.jena.sparql.util.Context;
+import org.apache.jena.system.Txn;
+import org.apache.jena.tdb2.assembler.VocabTDB2;
 import org.apache.jena.vocabulary.RDF;
 import org.junit.Assert;
 import org.junit.Test;
 
-public class TestDatasetAssemblerServiceEnhancer
+
+public class TestServiceEnhancerDatasetAssembler
 {
     private static final String SPEC_STR_01 = String.join("\n",
             "PREFIX ja: <http://jena.hpl.hp.com/2005/11/Assembler#>",
@@ -115,4 +124,54 @@ public class TestDatasetAssemblerServiceEnhancer
             Assert.assertEquals(1, ResultSetFormatter.consume(qe.execSelect()));
         }
     }
+
+    /**
+     * Test case to check that when {@code OpService('loop:', subOp)} is evaluated against an union default
+     * graph (UDG) enabled dataset then the subOp must be evaluated on the UDG too.
+     * This also applies to {@code cache:} and {@code bulk:}.
+     */
+    @Test
+    public void testAssemblerTdbUnionDefaultGraph() throws IOException {
+        Path tdb2TmpFolder = Files.createTempDirectory("jena-service-enhancer-tdb2").toAbsolutePath();
+        try {
+            String assemblerStr = String.join("\n",
+                    "PREFIX ja: <http://jena.hpl.hp.com/2005/11/Assembler#>",
+                    "PREFIX se: <http://jena.apache.org/service-enhancer#>",
+                    "PREFIX tdb2: <http://jena.apache.org/2016/tdb#>",
+                    "<urn:example:root> a se:DatasetServiceEnhancer ; ja:baseDataset <urn:example:base> .",
+                    "<urn:example:root> se:cacheMaxEntryCount 5 ; se:cachePageSize 1000 ; se:cacheMaxPageCount 10 .",
+                    "<urn:example:root> se:bulkMaxSize 20 ; se:bulkSize 10 ; se:bulkMaxOutOfBandSize 5 .",
+                    "<urn:example:base> a tdb2:DatasetTDB2 .",
+                    "<urn:example:base> tdb2:unionDefaultGraph true ."
+                );
+
+            String dataStr = String.join("\n",
+                    "PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>",
+                    "<urn:example:g1> { <urn:example:s1> a <urn:example:class1> ; rdfs:label \"s1\" }",
+                    "<urn:example:g2> { <urn:example:s2> a <urn:example:class2> ; rdfs:label \"s2\" }");
+
+            String queryStr = "SELECT * { ?s a [] BIND(NOW() AS ?outer) SERVICE <loop:> { ?s ?p ?o BIND(NOW() AS ?inner) } }";
+
+            Model confModel = ModelFactory.createDefaultModel();
+            RDFDataMgr.read(confModel, new StringReader(assemblerStr), null, Lang.TURTLE);
+            Resource tdb2Conf = confModel.getResource("urn:example:base");
+            tdb2Conf.addProperty(VocabTDB2.pLocation, tdb2TmpFolder.toString());
+
+            Dataset tdb2Ds = DatasetFactory.assemble(confModel.getResource("urn:example:root"));
+
+            Txn.executeWrite(tdb2Ds, () -> RDFDataMgr.read(tdb2Ds, new StringReader(dataStr), null, Lang.TRIG));
+
+            int actualRowCount = Txn.calculateRead(tdb2Ds, () -> {
+                try (QueryExecution qe = QueryExecution.create(queryStr, tdb2Ds)) {
+                    ResultSet rs = qe.execSelect();
+                    // System.out.println(ResultSetFormatter.asText(rs));
+                    return ResultSetFormatter.consume(rs);
+                }
+            });
+
+            Assert.assertEquals(4, actualRowCount);
+        } finally {
+            MoreFiles.deleteRecursively(tdb2TmpFolder);
+        }
+    }
 }
diff --git a/jena-extras/jena-serviceenhancer/src/test/java/org/apache/jena/sparql/service/enhancer/impl/AbstractTestServiceEnhancerResultSetLimits.java b/jena-extras/jena-serviceenhancer/src/test/java/org/apache/jena/sparql/service/enhancer/impl/AbstractTestServiceEnhancerResultSetLimits.java
index a9511b2f4f..0a949a953b 100644
--- a/jena-extras/jena-serviceenhancer/src/test/java/org/apache/jena/sparql/service/enhancer/impl/AbstractTestServiceEnhancerResultSetLimits.java
+++ b/jena-extras/jena-serviceenhancer/src/test/java/org/apache/jena/sparql/service/enhancer/impl/AbstractTestServiceEnhancerResultSetLimits.java
@@ -163,51 +163,12 @@ public abstract class AbstractTestServiceEnhancerResultSetLimits {
     }
 
     public static int testCore(Dataset dataset, String queryStr, int hiddenLimit) {
-
-
         Query query = QueryFactory.create(queryStr);
 
-        // Register a service plugin that slices the result
+        // Set up a custom service executor registry that "secretly" slices
+        // (i.e. cuts off) result sets after a certain number of rows
+        // This simulates result set limits which one typically encounters with public SPARQL endpoints
         ServiceExecutorRegistry reg = ServiceExecutorRegistry.get().copy();
-/*
-        reg.addBulkLink(
-                (op, iter, execCxt, chain) -> {
-                    // apply the slice to each input binding
-                    QueryIterator x = new QueryIterRepeatApply(iter, execCxt) {
-                        @Override
-                        protected QueryIterator nextStage(Binding binding) {
-                            QueryIterator qi = QueryIterPlainWrapper.create(Collections.singleton(binding).iterator(), execCxt);
-                            return new QueryIterSlice(chain.createExecution(op, qi, execCxt), 0, hiddenLimit, execCxt);
-                        }
-                    };
-
-                    QueryIterator r;
-                    boolean failOnPurpose = false;
-                    if (failOnPurpose) {
-                        QueryIterator y = QueryIterPlainWrapper.create(new AbstractIterator<Binding>() {
-                            @Override
-                            protected Binding computeNext() {
-                                throw new RuntimeException("Synthetic error to test for resource leaks on failure");
-                            }
-                        }, execCxt);
-
-                        QueryIterConcat z = new QueryIterConcat(execCxt);
-                        z.add(x);
-                        z.add(y);
-                        r = z;
-                    } else {
-                        r = x;
-                    }
-
-                    return r;
-                });
-*/
-
-//        reg.addBulkLink(
-//                (op, iter, execCxt, chain) -> {
-//                        return new QueryIterSlice(chain.createExecution(op, iter, execCxt), 0, hiddenLimit, execCxt);
-//                    }
-//                );
 
         reg.addSingleLink((opExec, opOrig, binding, execCxt, chain) -> {
             return new QueryIterSlice(chain.createExecution(opExec, opOrig, binding, execCxt), 0, hiddenLimit, execCxt);
diff --git a/jena-extras/jena-serviceenhancer/src/test/java/org/apache/jena/sparql/service/enhancer/impl/TestServiceEnhancerBatchQueryRewriter.java b/jena-extras/jena-serviceenhancer/src/test/java/org/apache/jena/sparql/service/enhancer/impl/TestServiceEnhancerBatchQueryRewriter.java
new file mode 100644
index 0000000000..11512e904d
--- /dev/null
+++ b/jena-extras/jena-serviceenhancer/src/test/java/org/apache/jena/sparql/service/enhancer/impl/TestServiceEnhancerBatchQueryRewriter.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.jena.sparql.service.enhancer.impl;
+
+import org.apache.jena.graph.NodeFactory;
+import org.apache.jena.query.Query;
+import org.apache.jena.query.QueryFactory;
+import org.apache.jena.query.Syntax;
+import org.apache.jena.sparql.algebra.Algebra;
+import org.apache.jena.sparql.algebra.Op;
+import org.apache.jena.sparql.algebra.OpAsQuery;
+import org.apache.jena.sparql.algebra.op.OpService;
+import org.apache.jena.sparql.core.Var;
+import org.apache.jena.sparql.engine.binding.Binding;
+import org.apache.jena.sparql.engine.binding.BindingFactory;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestServiceEnhancerBatchQueryRewriter {
+
+    /**
+     * Test that the generation of a bulk request from a query that uses limit and order by uses the correct rewrite.
+     * Specifically, ORDER BY must be retained on the union members even if they appear without slice because
+     * it might make a difference when the result set is cut off by the service's result set limit.
+     */
+    @Test
+    public void testOrderBy_01() {
+        OpService op = (OpService)Algebra.compile(QueryFactory.create(String.join("\n",
+            "SELECT * {",
+            "  SERVICE <http://example.org/> { SELECT * { ?s ?p ?o } ORDER BY ?s LIMIT 1 }",
+            "}"), Syntax.syntaxARQ).getQueryPattern());
+
+        Batch<Integer, PartitionRequest<Binding>> batch = BatchImpl.forInteger();
+        Var o = Var.alloc("o");
+        batch.put(0, new PartitionRequest<>(0, BindingFactory.binding(o, NodeFactory.createLiteral("x1")), 1, 5));
+        batch.put(1, new PartitionRequest<>(1, BindingFactory.binding(o, NodeFactory.createLiteral("x2")), 2, 6));
+
+        BatchQueryRewriter rewriter = new BatchQueryRewriter(new OpServiceInfo(op), Var.alloc("idx"), false, false, false);
+        BatchQueryRewriteResult rewrite = rewriter.rewrite(batch);
+        Op resultOp = rewrite.getOp();
+        Query actualQuery = OpAsQuery.asQuery(resultOp);
+
+        Query expectedQuery = QueryFactory.create(String.join("\n",
+            "SELECT  *",
+            "WHERE",
+            "   {   { { SELECT  *",
+            "          WHERE",
+            "             { ?s  ?p  \"x1\" }",
+            "          ORDER BY ?s",
+            "          OFFSET  1",
+            "          LIMIT   5",
+            "         }",
+            "        BIND(0 AS ?idx)",
+            "       }",
+            "    UNION",
+            "       {   { { SELECT  *",
+            "              WHERE",
+            "                 { ?s  ?p  \"x2\" }",
+            "              ORDER BY ?s",
+            "              OFFSET  2",
+            "              LIMIT   6",
+            "             }",
+            "            BIND(1 AS ?idx)",
+            "           }",
+            "        UNION",
+            "           { BIND(1000000000 AS ?idx) }",
+            "       }",
+            "   }",
+            "ORDER BY ASC(?idx) ?s"));
+
+        Assert.assertEquals(expectedQuery, actualQuery);
+    }
+}
diff --git a/jena-extras/jena-serviceenhancer/src/test/java/org/apache/jena/sparql/service/enhancer/impl/TestBatcher.java b/jena-extras/jena-serviceenhancer/src/test/java/org/apache/jena/sparql/service/enhancer/impl/TestServiceEnhancerBatcher.java
similarity index 99%
rename from jena-extras/jena-serviceenhancer/src/test/java/org/apache/jena/sparql/service/enhancer/impl/TestBatcher.java
rename to jena-extras/jena-serviceenhancer/src/test/java/org/apache/jena/sparql/service/enhancer/impl/TestServiceEnhancerBatcher.java
index e5e53dd60d..7950e72541 100644
--- a/jena-extras/jena-serviceenhancer/src/test/java/org/apache/jena/sparql/service/enhancer/impl/TestBatcher.java
+++ b/jena-extras/jena-serviceenhancer/src/test/java/org/apache/jena/sparql/service/enhancer/impl/TestServiceEnhancerBatcher.java
@@ -32,7 +32,7 @@ import org.apache.jena.ext.com.google.common.collect.Streams;
 import org.junit.Assert;
 import org.junit.Test;
 
-public class TestBatcher {
+public class TestServiceEnhancerBatcher {
 
     private static final List<Entry<String, Integer>> testData01 = List.<Entry<String, Integer>>of(
             Map.entry("a", 0),