You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by ti...@apache.org on 2018/06/16 05:45:54 UTC
[drill] branch master updated (ac8e698 -> 4baf769)
This is an automated email from the ASF dual-hosted git repository.
timothyfarkas pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git.
from ac8e698 DRILL-6353: Upgrade Parquet MR dependencies
new ca39cb6 DRILL-6487: Limit estimateRowCount should not return negative rowcount
new 54c88a3 DRILL-6468: CatastrophicFailures should not do a graceful shutdown of drill when terminating the JVM.
new 4baf769 DRILL-6455: Add missing JDBC Scan Operator for profiles
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../apache/drill/common/CatastrophicFailure.java | 2 +-
.../native/client/src/protobuf/UserBitShared.pb.cc | 13 ++---
.../native/client/src/protobuf/UserBitShared.pb.h | 5 +-
.../apache/drill/exec/store/jdbc/JdbcSubScan.java | 3 +-
.../exec/planner/common/DrillLimitRelBase.java | 3 +-
.../org/apache/drill/exec/server/Drillbit.java | 6 ++-
.../org/apache/drill/exec/server/FailureUtils.java | 60 ++++++++++++++++++++++
.../apache/drill/exec/work/foreman/Foreman.java | 20 ++++----
.../drill/exec/work/fragment/FragmentExecutor.java | 10 ++--
.../physical/impl/limit/TestLimitPlanning.java | 7 +++
.../apache/drill/exec/server/TestFailureUtils.java | 46 +++++++++++++++++
.../drill/exec/exception/OutOfMemoryException.java | 4 +-
.../org/apache/drill/exec/proto/UserBitShared.java | 21 +++++---
.../drill/exec/proto/beans/CoreOperatorType.java | 4 +-
protocol/src/main/protobuf/UserBitShared.proto | 1 +
15 files changed, 171 insertions(+), 34 deletions(-)
create mode 100644 exec/java-exec/src/main/java/org/apache/drill/exec/server/FailureUtils.java
create mode 100644 exec/java-exec/src/test/java/org/apache/drill/exec/server/TestFailureUtils.java
--
To stop receiving notification emails like this one, please contact
timothyfarkas@apache.org.
[drill] 02/03: DRILL-6468: CatastrophicFailures should not do a
graceful shutdown of drill when terminating the JVM.
Posted by ti...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
timothyfarkas pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 54c88a3f56cce20ceb900a3a83bf19c04d93ea1e
Author: Timothy Farkas <ti...@apache.org>
AuthorDate: Tue Jun 5 13:50:35 2018 -0700
DRILL-6468: CatastrophicFailures should not do a graceful shutdown of drill when terminating the JVM.
closes #1306
---
.../apache/drill/common/CatastrophicFailure.java | 2 +-
.../org/apache/drill/exec/server/Drillbit.java | 6 ++-
.../org/apache/drill/exec/server/FailureUtils.java | 60 ++++++++++++++++++++++
.../apache/drill/exec/work/foreman/Foreman.java | 20 ++++----
.../drill/exec/work/fragment/FragmentExecutor.java | 10 ++--
.../apache/drill/exec/server/TestFailureUtils.java | 46 +++++++++++++++++
.../drill/exec/exception/OutOfMemoryException.java | 4 +-
7 files changed, 131 insertions(+), 17 deletions(-)
diff --git a/common/src/main/java/org/apache/drill/common/CatastrophicFailure.java b/common/src/main/java/org/apache/drill/common/CatastrophicFailure.java
index 18e5747..6a45daa 100644
--- a/common/src/main/java/org/apache/drill/common/CatastrophicFailure.java
+++ b/common/src/main/java/org/apache/drill/common/CatastrophicFailure.java
@@ -30,7 +30,7 @@ public class CatastrophicFailure {
* Exit the VM as we hit a catastrophic failure.
* @param e
* The Throwable that occurred
- * @param name
+ * @param message
* A descriptive message
* @param code
* An error code to exit the JVM with.
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/Drillbit.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/Drillbit.java
index 7b1d617..de11ddb 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/Drillbit.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/Drillbit.java
@@ -360,6 +360,11 @@ public class Drillbit implements AutoCloseable {
@Override
public void run() {
+ if (FailureUtils.hadUnrecoverableFailure()) {
+ // We cannot close drill cleanly in this case.
+ return;
+ }
+
logger.info("Received shutdown request.");
try {
/*
@@ -453,5 +458,4 @@ public class Drillbit implements AutoCloseable {
// return as-is
return s;
}
-
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/FailureUtils.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/FailureUtils.java
new file mode 100644
index 0000000..4d2fa4f
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/FailureUtils.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.server;
+
+import io.netty.util.internal.OutOfDirectMemoryError;
+import org.apache.drill.common.CatastrophicFailure;
+import org.apache.drill.exec.exception.OutOfMemoryException;
+
+public final class FailureUtils {
+ private static volatile boolean unrecoverableFailure;
+
+ public static final int EXIT_CODE_HEAP_OOM = -1;
+ public static final int EXIT_CODE_JAVA_ERROR = -2;
+
+ /**
+ * This message is used to distinguish between direct memory and heap memory {@link OutOfMemoryError}s.
+ */
+ public static final String DIRECT_MEMORY_OOM_MESSAGE = "Direct buffer memory";
+
+ private FailureUtils() {
+ // Don't instantiate
+ }
+
+ public static boolean isDirectMemoryOOM(Throwable e) {
+ if (e instanceof OutOfDirectMemoryError || e instanceof OutOfMemoryException) {
+ // These are always direct memory errors
+ return true;
+ }
+
+ return (e instanceof OutOfMemoryError) && DIRECT_MEMORY_OOM_MESSAGE.equals(e.getMessage());
+ }
+
+ public static boolean isHeapOOM(Throwable e) {
+ return (e instanceof OutOfMemoryError) && !DIRECT_MEMORY_OOM_MESSAGE.equals(e.getMessage());
+ }
+
+ public static void unrecoverableFailure(Throwable e, String message, int exitCode) {
+ unrecoverableFailure = true;
+ CatastrophicFailure.exit(e, message, exitCode);
+ }
+
+ public static boolean hadUnrecoverableFailure() {
+ return unrecoverableFailure;
+ }
+}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
index 5633022..16e8eb0 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
@@ -23,7 +23,6 @@ import com.google.protobuf.InvalidProtocolBufferException;
import io.netty.channel.ChannelFuture;
import io.netty.util.concurrent.Future;
import io.netty.util.concurrent.GenericFutureListener;
-import org.apache.drill.common.CatastrophicFailure;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.common.logical.LogicalPlan;
@@ -55,6 +54,7 @@ import org.apache.drill.exec.rpc.BaseRpcOutcomeListener;
import org.apache.drill.exec.rpc.RpcException;
import org.apache.drill.exec.rpc.UserClientConnection;
import org.apache.drill.exec.server.DrillbitContext;
+import org.apache.drill.exec.server.FailureUtils;
import org.apache.drill.exec.server.options.OptionManager;
import org.apache.drill.exec.testing.ControlsInjector;
import org.apache.drill.exec.testing.ControlsInjectorFactory;
@@ -70,6 +70,8 @@ import java.io.IOException;
import java.util.Date;
import java.util.List;
+import static org.apache.drill.exec.server.FailureUtils.EXIT_CODE_HEAP_OOM;
+
/**
* Foreman manages all the fragments (local and remote) for a single query where this
* is the driving/root node.
@@ -273,25 +275,23 @@ public class Foreman implements Runnable {
throw new IllegalStateException();
}
injector.injectChecked(queryContext.getExecutionControls(), "run-try-end", ForemanException.class);
- } catch (final OutOfMemoryException e) {
- queryStateProcessor.moveToState(QueryState.FAILED, UserException.memoryError(e).build(logger));
} catch (final ForemanException e) {
queryStateProcessor.moveToState(QueryState.FAILED, e);
- } catch (AssertionError | Exception ex) {
- queryStateProcessor.moveToState(QueryState.FAILED,
- new ForemanException("Unexpected exception during fragment initialization: " + ex.getMessage(), ex));
- } catch (final OutOfMemoryError e) {
- if ("Direct buffer memory".equals(e.getMessage())) {
- queryStateProcessor.moveToState(QueryState.FAILED, UserException.resourceError(e).message("One or more nodes ran out of memory while executing the query.").build(logger));
+ } catch (final OutOfMemoryError | OutOfMemoryException e) {
+ if (FailureUtils.isDirectMemoryOOM(e)) {
+ queryStateProcessor.moveToState(QueryState.FAILED, UserException.memoryError(e).build(logger));
} else {
/*
* FragmentExecutors use a DrillbitStatusListener to watch out for the death of their query's Foreman. So, if we
* die here, they should get notified about that, and cancel themselves; we don't have to attempt to notify
* them, which might not work under these conditions.
*/
- CatastrophicFailure.exit(e, "Unable to handle out of memory condition in Foreman.", -1);
+ FailureUtils.unrecoverableFailure(e, "Unable to handle out of memory condition in Foreman.", EXIT_CODE_HEAP_OOM);
}
+ } catch (AssertionError | Exception ex) {
+ queryStateProcessor.moveToState(QueryState.FAILED,
+ new ForemanException("Unexpected exception during fragment initialization: " + ex.getMessage(), ex));
} finally {
// restore the thread's original name
currentThread.setName(originalName);
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/FragmentExecutor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/FragmentExecutor.java
index 48d8a05..a745bf3 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/FragmentExecutor.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/FragmentExecutor.java
@@ -25,7 +25,6 @@ import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
-import org.apache.drill.common.CatastrophicFailure;
import org.apache.drill.common.DeferredException;
import org.apache.drill.common.EventProcessor;
import org.apache.drill.common.exceptions.UserException;
@@ -43,12 +42,15 @@ import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
import org.apache.drill.exec.proto.ExecProtos.FragmentHandle;
import org.apache.drill.exec.proto.UserBitShared.FragmentState;
import org.apache.drill.exec.proto.helper.QueryIdHelper;
+import org.apache.drill.exec.server.FailureUtils;
import org.apache.drill.exec.testing.ControlsInjector;
import org.apache.drill.exec.testing.ControlsInjectorFactory;
import org.apache.drill.exec.util.ImpersonationUtil;
import org.apache.drill.exec.work.foreman.DrillbitStatusListener;
import org.apache.hadoop.security.UserGroupInformation;
+import static org.apache.drill.exec.server.FailureUtils.EXIT_CODE_HEAP_OOM;
+
/**
* <h2>Overview</h2>
* <p>
@@ -300,11 +302,11 @@ public class FragmentExecutor implements Runnable {
});
} catch (OutOfMemoryError | OutOfMemoryException e) {
- if (!(e instanceof OutOfMemoryError) || "Direct buffer memory".equals(e.getMessage())) {
+ if (FailureUtils.isDirectMemoryOOM(e)) {
fail(UserException.memoryError(e).build(logger));
} else {
- // we have a heap out of memory error. The JVM in unstable, exit.
- CatastrophicFailure.exit(e, "Unable to handle out of memory condition in FragmentExecutor.", -2);
+ // we have a heap out of memory error. The JVM is unstable, exit.
+ FailureUtils.unrecoverableFailure(e, "Unable to handle out of memory condition in FragmentExecutor.", EXIT_CODE_HEAP_OOM);
}
} catch (InterruptedException e) {
// Swallow interrupted exceptions since we intentionally interrupt the root when cancelling a query
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestFailureUtils.java b/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestFailureUtils.java
new file mode 100644
index 0000000..6736b46
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestFailureUtils.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.server;
+
+import org.apache.drill.exec.exception.OutOfMemoryException;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static org.apache.drill.exec.server.FailureUtils.DIRECT_MEMORY_OOM_MESSAGE;
+
+public class TestFailureUtils {
+ @Test
+ public void testIsDirectMemoryOOM() {
+ Assert.assertTrue(FailureUtils.isDirectMemoryOOM(new OutOfMemoryException()));
+ Assert.assertFalse(FailureUtils.isDirectMemoryOOM(new OutOfMemoryError()));
+ Assert.assertFalse(FailureUtils.isDirectMemoryOOM(new OutOfMemoryError("Heap went boom.")));
+ Assert.assertTrue(FailureUtils.isDirectMemoryOOM(new OutOfMemoryError(DIRECT_MEMORY_OOM_MESSAGE)));
+ }
+
+ @Test
+ public void testIsHeapOOM() {
+ Assert.assertTrue(FailureUtils.isHeapOOM(new OutOfMemoryError()));
+ Assert.assertTrue(FailureUtils.isHeapOOM(new OutOfMemoryError("Heap went boom.")));
+ Assert.assertFalse(FailureUtils.isHeapOOM(new OutOfMemoryError(DIRECT_MEMORY_OOM_MESSAGE)));
+ Assert.assertFalse(FailureUtils.isHeapOOM(new IOException()));
+ Assert.assertFalse(FailureUtils.isHeapOOM(new NullPointerException()));
+ Assert.assertFalse(FailureUtils.isHeapOOM(new OutOfMemoryException()));
+ }
+}
diff --git a/exec/memory/base/src/main/java/org/apache/drill/exec/exception/OutOfMemoryException.java b/exec/memory/base/src/main/java/org/apache/drill/exec/exception/OutOfMemoryException.java
index 01cf6de..f19a0d1 100644
--- a/exec/memory/base/src/main/java/org/apache/drill/exec/exception/OutOfMemoryException.java
+++ b/exec/memory/base/src/main/java/org/apache/drill/exec/exception/OutOfMemoryException.java
@@ -17,7 +17,9 @@
*/
package org.apache.drill.exec.exception;
-
+/**
+ * This is thrown in various cases when Drill cannot allocate Direct Memory. <b>Note: </b> This does <b>NOT</b> get thrown when we run out of heap memory.
+ */
public class OutOfMemoryException extends RuntimeException {
private static final long serialVersionUID = -6858052345185793382L;
--
To stop receiving notification emails like this one, please contact
timothyfarkas@apache.org.
[drill] 03/03: DRILL-6455: Add missing JDBC Scan Operator for
profiles
Posted by ti...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
timothyfarkas pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 4baf769e5e9df810b3b01bad1757b27a423b7842
Author: Kunal Khatua <ku...@apache.org>
AuthorDate: Sun Jun 10 23:36:10 2018 -0700
DRILL-6455: Add missing JDBC Scan Operator for profiles
The operator is missing in the profile protobuf. This commit introduces that.
1. Added protobuf files (incl generated C++ and Java)
2. Updated JdbcSubScan's getOperatorType API
closes #1297
---
.../native/client/src/protobuf/UserBitShared.pb.cc | 13 +++++++------
.../native/client/src/protobuf/UserBitShared.pb.h | 5 +++--
.../apache/drill/exec/store/jdbc/JdbcSubScan.java | 3 ++-
.../org/apache/drill/exec/proto/UserBitShared.java | 21 +++++++++++++++------
.../drill/exec/proto/beans/CoreOperatorType.java | 4 +++-
protocol/src/main/protobuf/UserBitShared.proto | 1 +
6 files changed, 31 insertions(+), 16 deletions(-)
diff --git a/contrib/native/client/src/protobuf/UserBitShared.pb.cc b/contrib/native/client/src/protobuf/UserBitShared.pb.cc
index e237669..c99030d 100644
--- a/contrib/native/client/src/protobuf/UserBitShared.pb.cc
+++ b/contrib/native/client/src/protobuf/UserBitShared.pb.cc
@@ -750,7 +750,7 @@ void protobuf_AddDesc_UserBitShared_2eproto() {
"TATEMENT\020\005*\207\001\n\rFragmentState\022\013\n\007SENDING\020"
"\000\022\027\n\023AWAITING_ALLOCATION\020\001\022\013\n\007RUNNING\020\002\022"
"\014\n\010FINISHED\020\003\022\r\n\tCANCELLED\020\004\022\n\n\006FAILED\020\005"
- "\022\032\n\026CANCELLATION_REQUESTED\020\006*\360\006\n\020CoreOpe"
+ "\022\032\n\026CANCELLATION_REQUESTED\020\006*\377\006\n\020CoreOpe"
"ratorType\022\021\n\rSINGLE_SENDER\020\000\022\024\n\020BROADCAS"
"T_SENDER\020\001\022\n\n\006FILTER\020\002\022\022\n\016HASH_AGGREGATE"
"\020\003\022\r\n\tHASH_JOIN\020\004\022\016\n\nMERGE_JOIN\020\005\022\031\n\025HAS"
@@ -772,11 +772,11 @@ void protobuf_AddDesc_UserBitShared_2eproto() {
"AN\020$\022\021\n\rPCAP_SUB_SCAN\020%\022\022\n\016KAFKA_SUB_SCA"
"N\020&\022\021\n\rKUDU_SUB_SCAN\020\'\022\013\n\007FLATTEN\020(\022\020\n\014L"
"ATERAL_JOIN\020)\022\n\n\006UNNEST\020*\022,\n(HIVE_DRILL_"
- "NATIVE_PARQUET_ROW_GROUP_SCAN\020+*g\n\nSaslS"
- "tatus\022\020\n\014SASL_UNKNOWN\020\000\022\016\n\nSASL_START\020\001\022"
- "\024\n\020SASL_IN_PROGRESS\020\002\022\020\n\014SASL_SUCCESS\020\003\022"
- "\017\n\013SASL_FAILED\020\004B.\n\033org.apache.drill.exe"
- "c.protoB\rUserBitSharedH\001", 5184);
+ "NATIVE_PARQUET_ROW_GROUP_SCAN\020+\022\r\n\tJDBC_"
+ "SCAN\020,*g\n\nSaslStatus\022\020\n\014SASL_UNKNOWN\020\000\022\016"
+ "\n\nSASL_START\020\001\022\024\n\020SASL_IN_PROGRESS\020\002\022\020\n\014"
+ "SASL_SUCCESS\020\003\022\017\n\013SASL_FAILED\020\004B.\n\033org.a"
+ "pache.drill.exec.protoB\rUserBitSharedH\001", 5199);
::google::protobuf::MessageFactory::InternalRegisterGeneratedFile(
"UserBitShared.proto", &protobuf_RegisterTypes);
UserCredentials::default_instance_ = new UserCredentials();
@@ -941,6 +941,7 @@ bool CoreOperatorType_IsValid(int value) {
case 41:
case 42:
case 43:
+ case 44:
return true;
default:
return false;
diff --git a/contrib/native/client/src/protobuf/UserBitShared.pb.h b/contrib/native/client/src/protobuf/UserBitShared.pb.h
index c4d846f..a78258a 100644
--- a/contrib/native/client/src/protobuf/UserBitShared.pb.h
+++ b/contrib/native/client/src/protobuf/UserBitShared.pb.h
@@ -247,11 +247,12 @@ enum CoreOperatorType {
FLATTEN = 40,
LATERAL_JOIN = 41,
UNNEST = 42,
- HIVE_DRILL_NATIVE_PARQUET_ROW_GROUP_SCAN = 43
+ HIVE_DRILL_NATIVE_PARQUET_ROW_GROUP_SCAN = 43,
+ JDBC_SCAN = 44
};
bool CoreOperatorType_IsValid(int value);
const CoreOperatorType CoreOperatorType_MIN = SINGLE_SENDER;
-const CoreOperatorType CoreOperatorType_MAX = HIVE_DRILL_NATIVE_PARQUET_ROW_GROUP_SCAN;
+const CoreOperatorType CoreOperatorType_MAX = JDBC_SCAN;
const int CoreOperatorType_ARRAYSIZE = CoreOperatorType_MAX + 1;
const ::google::protobuf::EnumDescriptor* CoreOperatorType_descriptor();
diff --git a/contrib/storage-jdbc/src/main/java/org/apache/drill/exec/store/jdbc/JdbcSubScan.java b/contrib/storage-jdbc/src/main/java/org/apache/drill/exec/store/jdbc/JdbcSubScan.java
index 65fa4fd..34d1827 100755
--- a/contrib/storage-jdbc/src/main/java/org/apache/drill/exec/store/jdbc/JdbcSubScan.java
+++ b/contrib/storage-jdbc/src/main/java/org/apache/drill/exec/store/jdbc/JdbcSubScan.java
@@ -20,6 +20,7 @@ package org.apache.drill.exec.store.jdbc;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.logical.StoragePluginConfig;
import org.apache.drill.exec.physical.base.AbstractSubScan;
+import org.apache.drill.exec.proto.beans.CoreOperatorType;
import org.apache.drill.exec.store.StoragePluginRegistry;
import com.fasterxml.jackson.annotation.JacksonInject;
@@ -52,7 +53,7 @@ public class JdbcSubScan extends AbstractSubScan {
@Override
public int getOperatorType() {
- return -1;
+ return CoreOperatorType.JDBC_SCAN.getNumber();
}
public String getSql() {
diff --git a/protocol/src/main/java/org/apache/drill/exec/proto/UserBitShared.java b/protocol/src/main/java/org/apache/drill/exec/proto/UserBitShared.java
index 6770cd9..7162ead 100644
--- a/protocol/src/main/java/org/apache/drill/exec/proto/UserBitShared.java
+++ b/protocol/src/main/java/org/apache/drill/exec/proto/UserBitShared.java
@@ -541,6 +541,10 @@ public final class UserBitShared {
* <code>HIVE_DRILL_NATIVE_PARQUET_ROW_GROUP_SCAN = 43;</code>
*/
HIVE_DRILL_NATIVE_PARQUET_ROW_GROUP_SCAN(43, 43),
+ /**
+ * <code>JDBC_SCAN = 44;</code>
+ */
+ JDBC_SCAN(44, 44),
;
/**
@@ -719,6 +723,10 @@ public final class UserBitShared {
* <code>HIVE_DRILL_NATIVE_PARQUET_ROW_GROUP_SCAN = 43;</code>
*/
public static final int HIVE_DRILL_NATIVE_PARQUET_ROW_GROUP_SCAN_VALUE = 43;
+ /**
+ * <code>JDBC_SCAN = 44;</code>
+ */
+ public static final int JDBC_SCAN_VALUE = 44;
public final int getNumber() { return value; }
@@ -769,6 +777,7 @@ public final class UserBitShared {
case 41: return LATERAL_JOIN;
case 42: return UNNEST;
case 43: return HIVE_DRILL_NATIVE_PARQUET_ROW_GROUP_SCAN;
+ case 44: return JDBC_SCAN;
default: return null;
}
}
@@ -24305,7 +24314,7 @@ public final class UserBitShared {
"TATEMENT\020\005*\207\001\n\rFragmentState\022\013\n\007SENDING\020" +
"\000\022\027\n\023AWAITING_ALLOCATION\020\001\022\013\n\007RUNNING\020\002\022" +
"\014\n\010FINISHED\020\003\022\r\n\tCANCELLED\020\004\022\n\n\006FAILED\020\005" +
- "\022\032\n\026CANCELLATION_REQUESTED\020\006*\360\006\n\020CoreOpe" +
+ "\022\032\n\026CANCELLATION_REQUESTED\020\006*\377\006\n\020CoreOpe" +
"ratorType\022\021\n\rSINGLE_SENDER\020\000\022\024\n\020BROADCAS" +
"T_SENDER\020\001\022\n\n\006FILTER\020\002\022\022\n\016HASH_AGGREGATE" +
"\020\003\022\r\n\tHASH_JOIN\020\004\022\016\n\nMERGE_JOIN\020\005\022\031\n\025HAS" +
@@ -24327,11 +24336,11 @@ public final class UserBitShared {
"AN\020$\022\021\n\rPCAP_SUB_SCAN\020%\022\022\n\016KAFKA_SUB_SCA" +
"N\020&\022\021\n\rKUDU_SUB_SCAN\020\'\022\013\n\007FLATTEN\020(\022\020\n\014L" +
"ATERAL_JOIN\020)\022\n\n\006UNNEST\020*\022,\n(HIVE_DRILL_" +
- "NATIVE_PARQUET_ROW_GROUP_SCAN\020+*g\n\nSaslS" +
- "tatus\022\020\n\014SASL_UNKNOWN\020\000\022\016\n\nSASL_START\020\001\022" +
- "\024\n\020SASL_IN_PROGRESS\020\002\022\020\n\014SASL_SUCCESS\020\003\022" +
- "\017\n\013SASL_FAILED\020\004B.\n\033org.apache.drill.exe" +
- "c.protoB\rUserBitSharedH\001"
+ "NATIVE_PARQUET_ROW_GROUP_SCAN\020+\022\r\n\tJDBC_" +
+ "SCAN\020,*g\n\nSaslStatus\022\020\n\014SASL_UNKNOWN\020\000\022\016" +
+ "\n\nSASL_START\020\001\022\024\n\020SASL_IN_PROGRESS\020\002\022\020\n\014" +
+ "SASL_SUCCESS\020\003\022\017\n\013SASL_FAILED\020\004B.\n\033org.a" +
+ "pache.drill.exec.protoB\rUserBitSharedH\001"
};
com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
diff --git a/protocol/src/main/java/org/apache/drill/exec/proto/beans/CoreOperatorType.java b/protocol/src/main/java/org/apache/drill/exec/proto/beans/CoreOperatorType.java
index fdf29b4..f14f44a 100644
--- a/protocol/src/main/java/org/apache/drill/exec/proto/beans/CoreOperatorType.java
+++ b/protocol/src/main/java/org/apache/drill/exec/proto/beans/CoreOperatorType.java
@@ -65,7 +65,8 @@ public enum CoreOperatorType implements com.dyuproject.protostuff.EnumLite<CoreO
FLATTEN(40),
LATERAL_JOIN(41),
UNNEST(42),
- HIVE_DRILL_NATIVE_PARQUET_ROW_GROUP_SCAN(43);
+ HIVE_DRILL_NATIVE_PARQUET_ROW_GROUP_SCAN(43),
+ JDBC_SCAN(44);
public final int number;
@@ -127,6 +128,7 @@ public enum CoreOperatorType implements com.dyuproject.protostuff.EnumLite<CoreO
case 41: return LATERAL_JOIN;
case 42: return UNNEST;
case 43: return HIVE_DRILL_NATIVE_PARQUET_ROW_GROUP_SCAN;
+ case 44: return JDBC_SCAN;
default: return null;
}
}
diff --git a/protocol/src/main/protobuf/UserBitShared.proto b/protocol/src/main/protobuf/UserBitShared.proto
index 1b0f968..302e3b1 100644
--- a/protocol/src/main/protobuf/UserBitShared.proto
+++ b/protocol/src/main/protobuf/UserBitShared.proto
@@ -332,6 +332,7 @@ enum CoreOperatorType {
LATERAL_JOIN = 41;
UNNEST = 42;
HIVE_DRILL_NATIVE_PARQUET_ROW_GROUP_SCAN = 43;
+ JDBC_SCAN = 44;
}
/* Registry that contains list of jars, each jar contains its name and list of function signatures.
--
To stop receiving notification emails like this one, please contact
timothyfarkas@apache.org.
[drill] 01/03: DRILL-6487: Limit estimateRowCount should not return
negative rowcount
Posted by ti...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
timothyfarkas pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit ca39cb67987a0efff2fe889ca4c5bf3a6fe5273a
Author: Gautam Parai <gp...@maprtech.com>
AuthorDate: Thu Jun 14 17:26:32 2018 -0700
DRILL-6487: Limit estimateRowCount should not return negative rowcount
closes #1322
---
.../org/apache/drill/exec/planner/common/DrillLimitRelBase.java | 3 ++-
.../apache/drill/exec/physical/impl/limit/TestLimitPlanning.java | 7 +++++++
2 files changed, 9 insertions(+), 1 deletion(-)
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillLimitRelBase.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillLimitRelBase.java
index afe5dad..7d070b6 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillLimitRelBase.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillLimitRelBase.java
@@ -87,7 +87,8 @@ public abstract class DrillLimitRelBase extends SingleRel implements DrillRelNod
int off = offset != null ? RexLiteral.intValue(offset) : 0 ;
if (fetch == null) {
- return getInput().estimateRowCount(mq) - off;
+ // If estimated rowcount is less than offset return 0
+ return Math.max(0, getInput().estimateRowCount(mq) - off);
} else {
int f = RexLiteral.intValue(fetch);
return off + f;
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/limit/TestLimitPlanning.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/limit/TestLimitPlanning.java
index 3f5fee2..087191a 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/limit/TestLimitPlanning.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/limit/TestLimitPlanning.java
@@ -29,4 +29,11 @@ public class TestLimitPlanning extends PlanTestBase {
PlanTestBase.testPlanMatchingPatterns(query, new String[]{".*Sort\\(.*"}, new String[]{".*TopN\\(.*"});
}
+
+ @Test
+ public void offsetMoreThanTotalRowsWithoutFetch() throws Exception {
+ String query = "select full_name from cp.`employee.json` offset 1156";
+ // Should not raise an assert
+ PlanTestBase.testPlanMatchingPatterns(query, new String[]{".*Limit\\(offset=\\[1156\\]\\).*"});
+ }
}
--
To stop receiving notification emails like this one, please contact
timothyfarkas@apache.org.