You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by ti...@apache.org on 2018/06/13 15:51:42 UTC
[drill] branch master updated (63d0251 -> 2427aa0)
This is an automated email from the ASF dual-hosted git repository.
timothyfarkas pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git.
from 63d0251 DRILL-6489: Fix filter push down for Hbase & Mapr-DB binary tables when convert function is used in a view
new 7be1e01 DRILL-6477: Drillbit crashes with OOME (Heap) for a large WebUI query
new 2427aa0 DRILL-6476: Generate explain plan which shows relation between Lateral and the corresponding Unnest.
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../java/org/apache/drill/exec/ExecConstants.java | 1 -
.../exec/planner/physical/PhysicalPlanCreator.java | 5 --
.../drill/exec/planner/physical/UnnestPrel.java | 3 +
.../physical/explain/NumberingRelWriter.java | 69 ++++++++++++++++++----
.../drill/exec/server/rest/QueryWrapper.java | 48 ++++++++++++++-
.../impl/lateraljoin/TestLateralPlans.java | 21 +++++++
6 files changed, 128 insertions(+), 19 deletions(-)
--
To stop receiving notification emails like this one, please contact
timothyfarkas@apache.org.
[drill] 01/02: DRILL-6477: Drillbit crashes with OOME (Heap) for a
large WebUI query
Posted by ti...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
timothyfarkas pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 7be1e01a71245f7234fe347decf9efa4acc36e52
Author: Kunal Khatua <ku...@apache.org>
AuthorDate: Thu Jun 7 13:32:00 2018 -0700
DRILL-6477: Drillbit crashes with OOME (Heap) for a large WebUI query
For queries submitted through the WebUI and retrieving a large result-set, the Drillbit often hangs or crashes due to the (foreman) Drillbit running out of Heap memory.
This is because the Web client translates the result set into a massive object in the heap-space and tries to send that back to the browser. This results in the VM thread actively trying to perform GC if the memory is not sufficient.
The workaround is to have the active webConnection of the query periodically timeout to allow for checking the consumed heap-space. A level of 0.85 (i.e. 85%) is set as default threshold, crossing which, a query submitted through the REST API is marked and failed.
In addition, a user exception is thrown, indicting the cause of the query failing, along with alternative suggestions for re-executing the query.
closes #1309
---
.../java/org/apache/drill/exec/ExecConstants.java | 1 -
.../drill/exec/server/rest/QueryWrapper.java | 48 +++++++++++++++++++++-
2 files changed, 46 insertions(+), 3 deletions(-)
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
index 1070d76..776c469 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
@@ -204,7 +204,6 @@ public final class ExecConstants {
public static final String SERVICE_KEYTAB_LOCATION = SERVICE_LOGIN_PREFIX + ".keytab";
public static final String KERBEROS_NAME_MAPPING = SERVICE_LOGIN_PREFIX + ".auth_to_local";
-
public static final String USER_SSL_ENABLED = "drill.exec.security.user.encryption.ssl.enabled";
public static final String BIT_ENCRYPTION_SASL_ENABLED = "drill.exec.security.bit.encryption.sasl.enabled";
public static final String BIT_ENCRYPTION_SASL_MAX_WRAPPED_SIZE = "drill.exec.security.bit.encryption.sasl.max_wrapped_size";
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryWrapper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryWrapper.java
index 911ac0f..cf74937 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryWrapper.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryWrapper.java
@@ -20,7 +20,10 @@ package org.apache.drill.exec.server.rest;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.collect.Maps;
+
+import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.exec.proto.UserBitShared.QueryId;
+import org.apache.drill.exec.proto.UserBitShared.QueryResult.QueryState;
import org.apache.drill.exec.proto.UserBitShared.QueryType;
import org.apache.drill.exec.proto.UserProtos.RunQuery;
import org.apache.drill.exec.proto.helper.QueryIdHelper;
@@ -28,18 +31,26 @@ import org.apache.drill.exec.proto.UserProtos.QueryResultsMode;
import org.apache.drill.exec.work.WorkManager;
import javax.xml.bind.annotation.XmlRootElement;
+
+import java.lang.management.ManagementFactory;
+import java.lang.management.MemoryMXBean;
import java.util.Collection;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.TimeUnit;
@XmlRootElement
public class QueryWrapper {
private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(QueryWrapper.class);
+ // Heap usage threshold/trigger to provide resiliency on web server for queries submitted via HTTP
+ private static final double HEAP_MEMORY_FAILURE_THRESHOLD = 0.85;
private final String query;
private final String queryType;
+ private static MemoryMXBean memMXBean = ManagementFactory.getMemoryMXBean();
+
@JsonCreator
public QueryWrapper(@JsonProperty("query") String query, @JsonProperty("queryType") String queryType) {
this.query = query;
@@ -59,7 +70,6 @@ public class QueryWrapper {
}
public QueryResult run(final WorkManager workManager, final WebUserConnection webUserConnection) throws Exception {
-
final RunQuery runQuery = RunQuery.newBuilder().setType(getType())
.setPlan(getQuery())
.setResultsMode(QueryResultsMode.STREAM_FULL)
@@ -68,8 +78,37 @@ public class QueryWrapper {
// Submit user query to Drillbit work queue.
final QueryId queryId = workManager.getUserWorker().submitWork(webUserConnection, runQuery);
+ boolean isComplete = false;
+ boolean nearlyOutOfHeapSpace = false;
+ float usagePercent = getHeapUsage();
+
// Wait until the query execution is complete or there is error submitting the query
- webUserConnection.await();
+ logger.debug("Wait until the query execution is complete or there is error submitting the query");
+ do {
+ try {
+ isComplete = webUserConnection.await(TimeUnit.SECONDS.toMillis(1)); /*periodically timeout to check heap*/
+ } catch (Exception e) { }
+
+ usagePercent = getHeapUsage();
+ if (usagePercent > HEAP_MEMORY_FAILURE_THRESHOLD) {
+ nearlyOutOfHeapSpace = true;
+ }
+ } while (!isComplete && !nearlyOutOfHeapSpace);
+
+ //Fail if nearly out of heap space
+ if (nearlyOutOfHeapSpace) {
+ workManager.getBee().getForemanForQueryId(queryId)
+ .addToEventQueue(QueryState.FAILED,
+ UserException.resourceError(
+ new Throwable(
+ "There is not enough heap memory to run this query using the web interface. "
+ + "Please try a query with fewer columns or with a filter or limit condition to limit the data returned. "
+ + "You can also try an ODBC/JDBC client. "
+ )
+ )
+ .build(logger)
+ );
+ }
if (logger.isTraceEnabled()) {
logger.trace("Query {} is completed ", queryId);
@@ -83,6 +122,11 @@ public class QueryWrapper {
return new QueryResult(queryId, webUserConnection.columns, webUserConnection.results);
}
+ //Detect possible excess heap
+ private float getHeapUsage() {
+ return (float) memMXBean.getHeapMemoryUsage().getUsed() / memMXBean.getHeapMemoryUsage().getMax();
+ }
+
public static class QueryResult {
private final String queryId;
public final Collection<String> columns;
--
To stop receiving notification emails like this one, please contact
timothyfarkas@apache.org.
[drill] 02/02: DRILL-6476: Generate explain plan which shows
relation between Lateral and the corresponding Unnest.
Posted by ti...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
timothyfarkas pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 2427aa0e6d4c371def099500131dc0eadf01ecac
Author: Hanumath Rao Maduri <ha...@gmail.com>
AuthorDate: Wed Jun 6 20:48:12 2018 -0700
DRILL-6476: Generate explain plan which shows relation between Lateral and the corresponding Unnest.
closes #1308
---
.../exec/planner/physical/PhysicalPlanCreator.java | 5 --
.../drill/exec/planner/physical/UnnestPrel.java | 3 +
.../physical/explain/NumberingRelWriter.java | 69 ++++++++++++++++++----
.../impl/lateraljoin/TestLateralPlans.java | 21 +++++++
4 files changed, 82 insertions(+), 16 deletions(-)
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PhysicalPlanCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PhysicalPlanCreator.java
index 6a94662..220add6 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PhysicalPlanCreator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PhysicalPlanCreator.java
@@ -52,11 +52,6 @@ public class PhysicalPlanCreator {
return context;
}
-// public int getOperatorId(Prel prel){
-// OpId id = opIdMap.get(prel);
-// return id.getAsSingleInt();
-// }
-
public PhysicalOperator addMetadata(Prel originalPrel, PhysicalOperator op){
op.setOperatorId(opIdMap.get(originalPrel).getAsSingleInt());
op.setCost(originalPrel.estimateRowCount(originalPrel.getCluster().getMetadataQuery()));
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnnestPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnnestPrel.java
index cd598eb..a22beea 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnnestPrel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnnestPrel.java
@@ -75,4 +75,7 @@ public class UnnestPrel extends DrillUnnestRelBase implements Prel {
return true;
}
+ public Class<?> getParentClass() {
+ return CorrelatePrel.class;
+ }
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/explain/NumberingRelWriter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/explain/NumberingRelWriter.java
index 045dba9..38b97b6 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/explain/NumberingRelWriter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/explain/NumberingRelWriter.java
@@ -19,6 +19,7 @@ package org.apache.drill.exec.planner.physical.explain;
import java.io.PrintWriter;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -31,8 +32,10 @@ import org.apache.calcite.rex.RexNode;
import org.apache.calcite.runtime.FlatLists;
import org.apache.calcite.sql.SqlExplainLevel;
import org.apache.calcite.util.Pair;
+import org.apache.drill.exec.planner.physical.CorrelatePrel;
import org.apache.drill.exec.planner.physical.HashJoinPrel;
import org.apache.drill.exec.planner.physical.Prel;
+import org.apache.drill.exec.planner.physical.UnnestPrel;
import org.apache.drill.exec.planner.physical.explain.PrelSequencer.OpId;
import com.google.common.collect.ImmutableList;
@@ -47,6 +50,7 @@ class NumberingRelWriter implements RelWriter {
private final SqlExplainLevel detailLevel;
protected final Spacer spacer = new Spacer();
private final List<Pair<String, Object>> values = new ArrayList<>();
+ private final Map<String, Prel> sourceOperatorRegistry;
private final Map<Prel, OpId> ids;
//~ Constructors -----------------------------------------------------------
@@ -55,6 +59,7 @@ class NumberingRelWriter implements RelWriter {
this.pw = pw;
this.ids = ids;
this.detailLevel = detailLevel;
+ this.sourceOperatorRegistry = new HashMap<>();
}
//~ Methods ----------------------------------------------------------------
@@ -62,16 +67,10 @@ class NumberingRelWriter implements RelWriter {
protected void explain_(
RelNode rel,
List<Pair<String, Object>> values) {
- List<RelNode> inputs = rel.getInputs();
- if (rel instanceof HashJoinPrel && ((HashJoinPrel) rel).isSwapped()) {
- HashJoinPrel joinPrel = (HashJoinPrel) rel;
- inputs = FlatLists.of(joinPrel.getRight(), joinPrel.getLeft());
- }
-
RelMetadataQuery mq = RelMetadataQuery.instance();
if (!mq.isVisibleInExplain(rel, detailLevel)) {
// render children in place of this, at same level
- explainInputs(inputs);
+ explainInputs(rel);
return;
}
@@ -95,6 +94,7 @@ class NumberingRelWriter implements RelWriter {
s.append(rel.getRelTypeName().replace("Prel", ""));
if (detailLevel != SqlExplainLevel.NO_ATTRIBUTES) {
int j = 0;
+ s.append(getDependentSrcOp(rel));
for (Pair<String, Object> value : values) {
if (value.right instanceof RelNode) {
continue;
@@ -125,14 +125,61 @@ class NumberingRelWriter implements RelWriter {
}
pw.println(s);
spacer.add(2);
- explainInputs(inputs);
+ explainInputs(rel);
spacer.subtract(2);
}
- private void explainInputs(List<RelNode> inputs) {
- for (RelNode input : inputs) {
- input.explain(this);
+ private String getDependentSrcOp(RelNode rel) {
+ if (rel instanceof UnnestPrel) {
+ return this.getDependentSrcOp((UnnestPrel) rel);
+ }
+ return "";
+ }
+
+ private String getDependentSrcOp(UnnestPrel unnest) {
+ Prel parent = this.getRegisteredPrel(unnest.getParentClass());
+ if (parent != null && parent instanceof CorrelatePrel) {
+ OpId id = ids.get(parent);
+ return String.format(" [srcOp=%02d-%02d] ", id.fragmentId, id.opId);
}
+ return "";
+ }
+
+ public void register(Prel toRegister) {
+ this.sourceOperatorRegistry.put(toRegister.getClass().getSimpleName(), toRegister);
+ }
+
+ public Prel getRegisteredPrel(Class<?> classname) {
+ return this.sourceOperatorRegistry.get(classname.getSimpleName());
+ }
+
+ public void unRegister(Prel unregister) {
+ this.sourceOperatorRegistry.remove(unregister.getClass().getSimpleName());
+ }
+
+
+ private void explainInputs(RelNode rel) {
+ if (rel instanceof CorrelatePrel) {
+ this.explainInputs((CorrelatePrel) rel);
+ } else {
+ List<RelNode> inputs = rel.getInputs();
+ if (rel instanceof HashJoinPrel && ((HashJoinPrel) rel).isSwapped()) {
+ HashJoinPrel joinPrel = (HashJoinPrel) rel;
+ inputs = FlatLists.of(joinPrel.getRight(), joinPrel.getLeft());
+ }
+ for (RelNode input : inputs) {
+ input.explain(this);
+ }
+ }
+ }
+
+ //Correlate is handled differently because explain plan
+ //needs to show relation between Lateral and Unnest operators.
+ private void explainInputs(CorrelatePrel correlate) {
+ correlate.getInput(0).explain(this);
+ this.register(correlate);
+ correlate.getInput(1).explain(this);
+ this.unRegister(correlate);
}
public final void explain(RelNode rel, List<Pair<String, Object>> valueList) {
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/lateraljoin/TestLateralPlans.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/lateraljoin/TestLateralPlans.java
index 00ab971..d027e77 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/lateraljoin/TestLateralPlans.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/lateraljoin/TestLateralPlans.java
@@ -435,4 +435,25 @@ public class TestLateralPlans extends BaseTestQuery {
String CorrelateUnnest = matcher.group(0);
return CorrelateUnnest.substring(CorrelateUnnest.lastIndexOf("Scan"));
}
+
+
+ //The following test is for testing the explain plan contains relation between lateral and corresponding unnest.
+ @Test
+ public void testLateralAndUnnestExplainPlan() throws Exception {
+ String Sql = "select c.* from cp.`lateraljoin/nested-customer.json` c, unnest(c.orders) Orders(ord)";
+ ClusterFixtureBuilder builder = ClusterFixture.builder(dirTestWatcher)
+ .setOptionDefault(ExecConstants.ENABLE_UNNEST_LATERAL_KEY, true)
+ .setOptionDefault(ExecConstants.SLICE_TARGET, 1);
+
+ try (ClusterFixture cluster = builder.build();
+ ClientFixture client = cluster.clientFixture()) {
+ String explain = client.queryBuilder().sql(Sql).explainText();
+ String srcOp = explain.substring(explain.indexOf("srcOp"));
+ assertTrue(srcOp != null && srcOp.length() > 0);
+ String correlateFragmentPattern = srcOp.substring(srcOp.indexOf("=")+1, srcOp.indexOf("]"));
+ assertTrue(correlateFragmentPattern != null && correlateFragmentPattern.length() > 0);
+ Matcher matcher = Pattern.compile(correlateFragmentPattern + ".*Correlate", Pattern.MULTILINE | Pattern.DOTALL).matcher(explain);
+ assertTrue(matcher.find());
+ }
+ }
}
--
To stop receiving notification emails like this one, please contact
timothyfarkas@apache.org.