You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by pr...@apache.org on 2017/10/11 17:55:31 UTC
[1/5] drill git commit: DRILL-5645: negation of expression causes
null pointer exception
Repository: drill
Updated Branches:
refs/heads/master bbc422404 -> cf3a5eef2
DRILL-5645: negation of expression causes null pointer exception
closes #892
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/a7b25714
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/a7b25714
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/a7b25714
Branch: refs/heads/master
Commit: a7b25714d824c7447a73251314383eccb9b7b18b
Parents: bbc4224
Author: Josiah Yan <jo...@gmail.com>
Authored: Wed Aug 2 22:50:54 2017 +0800
Committer: Paul Rogers <pr...@maprtech.com>
Committed: Wed Oct 11 10:19:29 2017 -0700
----------------------------------------------------------------------
.../src/main/codegen/data/MathFunc.tdd | 13 ++++++++++
.../main/codegen/templates/MathFunctions.java | 4 +++
.../drill/exec/expr/fn/impl/MathFunctions.java | 15 -----------
.../drill/exec/planner/logical/DrillOptiq.java | 11 +++------
.../org/apache/drill/TestFunctionsQuery.java | 26 +++++++++++++++++++-
5 files changed, 45 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/a7b25714/exec/java-exec/src/main/codegen/data/MathFunc.tdd
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/data/MathFunc.tdd b/exec/java-exec/src/main/codegen/data/MathFunc.tdd
index f37310b..feda49f 100644
--- a/exec/java-exec/src/main/codegen/data/MathFunc.tdd
+++ b/exec/java-exec/src/main/codegen/data/MathFunc.tdd
@@ -56,6 +56,19 @@ unaryMathFunctions : [
{input: "UInt8", outputType: "UInt8", castType: "long"}
]
},
+ {className: "Negative", funcName: "negative", javaFunc : " - ", types: [
+ {input: "Int", outputType: "Int", castType: "int"},
+ {input: "BigInt", outputType: "BigInt", castType: "long"},
+ {input: "Float4", outputType: "Float4", castType: "float"},
+ {input: "Float8", outputType: "Float8", castType: "double"},
+ {input: "SmallInt", outputType: "SmallInt", castType: "short"},
+ {input: "TinyInt", outputType: "TinyInt", castType: "byte"},
+ {input: "UInt1", outputType: "UInt1", castType: "byte"},
+ {input: "UInt2", outputType: "UInt2", castType: "char"},
+ {input: "UInt4", outputType: "UInt4", castType: "int"},
+ {input: "UInt8", outputType: "UInt8", castType: "long"}
+ ]
+ },
{className: "Sqrt", funcName: "sqrt", javaFunc : "java.lang.Math.sqrt", types: [
{input: "Int", outputType: "Int", castType: "int"},
{input: "BigInt", outputType: "BigInt", castType: "long"},
http://git-wip-us.apache.org/repos/asf/drill/blob/a7b25714/exec/java-exec/src/main/codegen/templates/MathFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/MathFunctions.java b/exec/java-exec/src/main/codegen/templates/MathFunctions.java
index 4144fcf..50ee6e6 100644
--- a/exec/java-exec/src/main/codegen/templates/MathFunctions.java
+++ b/exec/java-exec/src/main/codegen/templates/MathFunctions.java
@@ -59,7 +59,11 @@ public class GMathFunctions{
<#list func.types as type>
+ <#if func.funcName=="negative">
+ @FunctionTemplate(names = {"negative", "u-", "-"}, scope = FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL)
+ <#else>
@FunctionTemplate(name = "${func.funcName}", scope = FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL)
+ </#if>
public static class ${func.className}${type.input} implements DrillSimpleFunc {
@Param ${type.input}Holder in;
http://git-wip-us.apache.org/repos/asf/drill/blob/a7b25714/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/MathFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/MathFunctions.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/MathFunctions.java
index 0d5cd92..52d1a64 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/MathFunctions.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/MathFunctions.java
@@ -35,21 +35,6 @@ public class MathFunctions{
private MathFunctions(){}
- @FunctionTemplate(names = {"negative", "u-", "-"}, scope = FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL)
- public static class Negative implements DrillSimpleFunc{
-
- @Param BigIntHolder input;
- @Output BigIntHolder out;
-
- public void setup(){}
-
- public void eval(){
- out.value = -input.value;
- return;
- }
-
- }
-
@FunctionTemplate(name = "power", scope = FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL)
public static class Power implements DrillSimpleFunc{
http://git-wip-us.apache.org/repos/asf/drill/blob/a7b25714/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java
index db0cfbd..0d8efd5 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java
@@ -167,14 +167,9 @@ public class DrillOptiq {
return FunctionCallFactory.createExpression(call.getOperator().getName().toLowerCase(),
ExpressionPosition.UNKNOWN, arg);
case MINUS_PREFIX:
- final RexBuilder builder = inputs.get(0).getCluster().getRexBuilder();
- final List<RexNode> operands = Lists.newArrayList();
- operands.add(builder.makeExactLiteral(new BigDecimal(-1)));
- operands.add(call.getOperands().get(0));
-
- return visitCall((RexCall) builder.makeCall(
- SqlStdOperatorTable.MULTIPLY,
- operands));
+ List<LogicalExpression> operands = Lists.newArrayList();
+ operands.add(call.getOperands().get(0).accept(this));
+ return FunctionCallFactory.createExpression("u-", operands);
}
throw new AssertionError("todo: implement syntax " + syntax + "(" + call + ")");
case SPECIAL:
http://git-wip-us.apache.org/repos/asf/drill/blob/a7b25714/exec/java-exec/src/test/java/org/apache/drill/TestFunctionsQuery.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestFunctionsQuery.java b/exec/java-exec/src/test/java/org/apache/drill/TestFunctionsQuery.java
index f565665..1e8fc44 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestFunctionsQuery.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestFunctionsQuery.java
@@ -728,7 +728,7 @@ public class TestFunctionsQuery extends BaseTestQuery {
@Test
public void testNegative() throws Exception {
- String query = "select negative(2) as NEG " +
+ String query = "select negative(cast(2 as bigint)) as NEG " +
"from cp.`employee.json` where employee_id = 1";
testBuilder()
@@ -926,4 +926,28 @@ public class TestFunctionsQuery extends BaseTestQuery {
.baselineValues(false)
.go();
}
+
+ /**
+ * Test for DRILL-5645, where negation of expressions that do not contain
+ * a RelNode input results in a NullPointerException
+ */
+ @Test
+ public void testNegate() throws Exception {
+ String query = "select -(2 * 2) as col1 from ( values ( 1 ) ) T ( C1 )";
+ testBuilder()
+ .sqlQuery(query)
+ .unOrdered()
+ .baselineColumns("col1")
+ .baselineValues(-4)
+ .go();
+
+ // Test float
+ query = "select -(1.1 * 1) as col1 from ( values ( 1 ) ) T ( C1 )";
+ testBuilder()
+ .sqlQuery(query)
+ .unOrdered()
+ .baselineColumns("col1")
+ .baselineValues(-1.1)
+ .go();
+ }
}
[5/5] drill git commit: DRILL-5859: Fixed queued time calculation
Posted by pr...@apache.org.
DRILL-5859: Fixed queued time calculation
closes #982
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/cf3a5eef
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/cf3a5eef
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/cf3a5eef
Branch: refs/heads/master
Commit: cf3a5eef23ffe0cdc33366941e60cee4e593f684
Parents: e14cb7a
Author: Prasad Nagaraj Subramanya <pr...@gmail.com>
Authored: Tue Oct 10 11:12:03 2017 -0700
Committer: Paul Rogers <pr...@maprtech.com>
Committed: Wed Oct 11 10:25:20 2017 -0700
----------------------------------------------------------------------
.../java/org/apache/drill/exec/work/foreman/Foreman.java | 3 ++-
.../drill/exec/work/foreman/rm/DistributedQueryQueue.java | 9 +++++----
.../org/apache/drill/exec/work/foreman/rm/QueryQueue.java | 4 ++--
3 files changed, 9 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/cf3a5eef/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
index a1f9d9f..10d8537 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
@@ -455,9 +455,10 @@ public class Foreman implements Runnable {
.build(logger);
} catch (QueryQueueException e) {
throw new ForemanSetupException(e.getMessage(), e);
+ } finally {
+ queryManager.markQueueWaitEndTime();
}
moveToState(QueryState.STARTING, null);
- queryManager.markQueueWaitEndTime();
}
/**
http://git-wip-us.apache.org/repos/asf/drill/blob/cf3a5eef/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/rm/DistributedQueryQueue.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/rm/DistributedQueryQueue.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/rm/DistributedQueryQueue.java
index 9a4c78d..f4c5536 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/rm/DistributedQueryQueue.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/rm/DistributedQueryQueue.java
@@ -140,7 +140,7 @@ public class DistributedQueryQueue implements QueryQueue {
private static class ConfigSet {
private final long queueThreshold;
- private final long queueTimeout;
+ private final int queueTimeout;
private final int largeQueueSize;
private final int smallQueueSize;
private final double largeToSmallRatio;
@@ -149,7 +149,7 @@ public class DistributedQueryQueue implements QueryQueue {
public ConfigSet(SystemOptionManager optionManager) {
queueThreshold = optionManager.getOption(ExecConstants.QUEUE_THRESHOLD_SIZE);
- queueTimeout = optionManager.getOption(ExecConstants.QUEUE_TIMEOUT);
+ queueTimeout = (int) optionManager.getOption(ExecConstants.QUEUE_TIMEOUT);
// Option manager supports only long values, but we do not expect
// more than 2 billion active queries, so queue size is stored as
@@ -274,8 +274,9 @@ public class DistributedQueryQueue implements QueryQueue {
if (lease == null) {
int timeoutSecs = (int) Math.round(configSet.queueTimeout/1000.0);
- logger.warn("Queue timeout: {} after {} seconds.", queueName, timeoutSecs);
- throw new QueueTimeoutException(queryId, queueName, timeoutSecs);
+ logger.warn("Queue timeout: {} after {} ms. ({} seconds)", queueName,
+ String.format("%,d", configSet.queueTimeout), timeoutSecs);
+ throw new QueueTimeoutException(queryId, queueName, configSet.queueTimeout);
}
return new DistributedQueueLease(queryId, queueName, lease, queryMemory);
}
http://git-wip-us.apache.org/repos/asf/drill/blob/cf3a5eef/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/rm/QueryQueue.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/rm/QueryQueue.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/rm/QueryQueue.java
index 72dc2d6..67a8b96 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/rm/QueryQueue.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/rm/QueryQueue.java
@@ -65,8 +65,8 @@ public interface QueryQueue {
public QueueTimeoutException(QueryId queryId, String queueName, int timeoutMs) {
super( String.format(
- "Query timed out of the %s queue after %d ms.",
- queueName, timeoutMs ));
+ "Query timed out of the %s queue after %,d ms. (%d seconds)",
+ queueName, timeoutMs, (int) Math.round(timeoutMs/1000.0) ));
this.queryId = queryId;
this.queueName = queueName;
this.timeoutMs = timeoutMs;
[3/5] drill git commit: DRILL-5857: Fix NumberFormatException in Hive
unit tests
Posted by pr...@apache.org.
DRILL-5857: Fix NumberFormatException in Hive unit tests
closes #980
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/0049f173
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/0049f173
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/0049f173
Branch: refs/heads/master
Commit: 0049f17331462e0c7940bb5b5b79eab8e75d2a86
Parents: 27b6605
Author: Volodymyr Vysotskyi <vv...@gmail.com>
Authored: Mon Oct 9 14:28:42 2017 +0000
Committer: Paul Rogers <pr...@maprtech.com>
Committed: Wed Oct 11 10:23:17 2017 -0700
----------------------------------------------------------------------
.../apache/drill/exec/store/hive/HiveMetadataProvider.java | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/0049f173/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveMetadataProvider.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveMetadataProvider.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveMetadataProvider.java
index e80b37b..8abc79a 100644
--- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveMetadataProvider.java
+++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveMetadataProvider.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -194,8 +194,9 @@ public class HiveMetadataProvider {
* Get the stats from table properties. If not found -1 is returned for each stats field.
* CAUTION: stats may not be up-to-date with the underlying data. It is always good to run the ANALYZE command on
* Hive table to have up-to-date stats.
- * @param properties
- * @return
+ *
+ * @param properties the source of table stats
+ * @return {@link HiveStats} instance with rows number and size in bytes from specified properties
*/
private HiveStats getStatsFromProps(final Properties properties) {
long numRows = -1;
@@ -208,7 +209,7 @@ public class HiveMetadataProvider {
final String sizeInBytesProp = properties.getProperty(StatsSetupConst.TOTAL_SIZE);
if (sizeInBytesProp != null) {
- sizeInBytes = Long.valueOf(numRowsProp);
+ sizeInBytes = Long.valueOf(sizeInBytesProp);
}
} catch (final NumberFormatException e) {
logger.error("Failed to parse Hive stats in metastore.", e);
[2/5] drill git commit: DRILL-5849: Add freemarker lib to
dependencyManagement to ensure proper version is used when resolving
dependency version conflicts
Posted by pr...@apache.org.
DRILL-5849: Add freemarker lib to dependencyManagement to ensure proper version is used when resolving dependency version conflicts
closes #977
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/27b6605f
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/27b6605f
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/27b6605f
Branch: refs/heads/master
Commit: 27b6605fab5a52526c2abb5a90c649febe725905
Parents: a7b2571
Author: Arina Ielchiieva <ar...@gmail.com>
Authored: Fri Oct 6 16:58:28 2017 +0300
Committer: Paul Rogers <pr...@maprtech.com>
Committed: Wed Oct 11 10:21:38 2017 -0700
----------------------------------------------------------------------
exec/java-exec/pom.xml | 1 -
pom.xml | 5 +++++
tools/fmpp/pom.xml | 1 -
3 files changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/27b6605f/exec/java-exec/pom.xml
----------------------------------------------------------------------
diff --git a/exec/java-exec/pom.xml b/exec/java-exec/pom.xml
index 06340d5..a16fa6b 100644
--- a/exec/java-exec/pom.xml
+++ b/exec/java-exec/pom.xml
@@ -224,7 +224,6 @@
<dependency>
<groupId>org.freemarker</groupId>
<artifactId>freemarker</artifactId>
- <version>${freemarker.version}</version>
</dependency>
<dependency>
<groupId>net.hydromatic</groupId>
http://git-wip-us.apache.org/repos/asf/drill/blob/27b6605f/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index f394507..4f7c2e2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -679,6 +679,11 @@
<dependencyManagement>
<dependencies>
<dependency>
+ <groupId>org.freemarker</groupId>
+ <artifactId>freemarker</artifactId>
+ <version>${freemarker.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-aws</artifactId>
<version>${hadoop.version}</version>
http://git-wip-us.apache.org/repos/asf/drill/blob/27b6605f/tools/fmpp/pom.xml
----------------------------------------------------------------------
diff --git a/tools/fmpp/pom.xml b/tools/fmpp/pom.xml
index 27eca82..6fe7925 100644
--- a/tools/fmpp/pom.xml
+++ b/tools/fmpp/pom.xml
@@ -52,7 +52,6 @@
<dependency>
<groupId>org.freemarker</groupId>
<artifactId>freemarker</artifactId>
- <version>${freemarker.version}</version>
</dependency>
</dependencies>
<build>
[4/5] drill git commit: DRILL-5854: IllegalStateException when empty
batch with valid schema is received
Posted by pr...@apache.org.
DRILL-5854: IllegalStateException when empty batch with valid schema is received
closes #981
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/e14cb7ac
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/e14cb7ac
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/e14cb7ac
Branch: refs/heads/master
Commit: e14cb7ac2dacd618dc217dc3c136da8ba2fe9367
Parents: 0049f17
Author: Padma Penumarthy <pp...@yahoo.com>
Authored: Mon Oct 9 15:05:17 2017 -0700
Committer: Paul Rogers <pr...@maprtech.com>
Committed: Wed Oct 11 10:24:23 2017 -0700
----------------------------------------------------------------------
.../drill/exec/physical/impl/mergereceiver/MergingRecordBatch.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/e14cb7ac/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/mergereceiver/MergingRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/mergereceiver/MergingRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/mergereceiver/MergingRecordBatch.java
index 35c1f81..eff1ae9 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/mergereceiver/MergingRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/mergereceiver/MergingRecordBatch.java
@@ -224,6 +224,7 @@ public class MergingRecordBatch extends AbstractRecordBatch<MergingReceiverPOP>
if (rawBatch == null) {
createDummyBatch = true;
rawBatches.add(rawBatch);
+ p++; // move to next sender
continue;
}