You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@drill.apache.org by ja...@apache.org on 2014/05/21 22:28:39 UTC
[1/5] git commit: Fix DRILL-791: In Phase 1of2 use the agg expr
ordinal and create new AggregateCall for non-COUNT functions.
Repository: incubator-drill
Updated Branches:
refs/heads/master 5d7e3d3ab -> ae570aad8
Fix DRILL-791: In Phase 1of2 use the agg expr ordinal and create new AggregateCall for non-COUNT functions.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/37c1d067
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/37c1d067
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/37c1d067
Branch: refs/heads/master
Commit: 37c1d067436f03347ccc7a69eada0bc5fd6a8558
Parents: 5d7e3d3
Author: Aman Sinha <as...@maprtech.com>
Authored: Tue May 20 13:40:39 2014 -0700
Committer: Aman Sinha <as...@maprtech.com>
Committed: Tue May 20 13:40:39 2014 -0700
----------------------------------------------------------------------
.../apache/drill/exec/planner/physical/AggPrelBase.java | 10 +++++++++-
1 file changed, 9 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/37c1d067/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/AggPrelBase.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/AggPrelBase.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/AggPrelBase.java
index c3b1188..fb60885 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/AggPrelBase.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/AggPrelBase.java
@@ -149,7 +149,15 @@ public abstract class AggPrelBase extends AggregateRelBase implements Prel{
phase2AggCallList.add(newAggCall);
} else {
- phase2AggCallList.add(aggCall.e);
+ AggregateCall newAggCall =
+ new AggregateCall(
+ aggCall.e.getAggregation(),
+ aggCall.e.isDistinct(),
+ Collections.singletonList(aggExprOrdinal),
+ aggCall.e.getType(),
+ aggCall.e.getName());
+
+ phase2AggCallList.add(newAggCall);
}
}
}
[3/5] git commit: Fix ExprLexer to correctly deserialize decimal data
type.
Posted by ja...@apache.org.
Fix ExprLexer to correctly deserialize decimal data type.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/935095ef
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/935095ef
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/935095ef
Branch: refs/heads/master
Commit: 935095ef15ecef13eac194bdcc48314f1d59a270
Parents: 68f4bdb
Author: Mehant Baid <me...@gmail.com>
Authored: Tue May 20 15:26:01 2014 -0700
Committer: Mehant Baid <me...@gmail.com>
Committed: Tue May 20 16:56:38 2014 -0700
----------------------------------------------------------------------
.../apache/drill/common/expression/parser/ExprLexer.g | 12 ++++++------
.../src/test/resources/decimal/cast_float_decimal.json | 2 +-
.../src/test/resources/decimal/cast_simple_decimal.json | 10 +++++-----
.../decimal/test_decimal_sparse_dense_dense.json | 10 +++++-----
4 files changed, 17 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/935095ef/common/src/main/antlr3/org/apache/drill/common/expression/parser/ExprLexer.g
----------------------------------------------------------------------
diff --git a/common/src/main/antlr3/org/apache/drill/common/expression/parser/ExprLexer.g b/common/src/main/antlr3/org/apache/drill/common/expression/parser/ExprLexer.g
index 58ed215..e3a0e34 100644
--- a/common/src/main/antlr3/org/apache/drill/common/expression/parser/ExprLexer.g
+++ b/common/src/main/antlr3/org/apache/drill/common/expression/parser/ExprLexer.g
@@ -55,12 +55,12 @@ INTERVAL : 'interval' | 'INTERVAL';
INTERVALYEAR : 'intervalyear' | 'INTERVALYEAR';
INTERVALDAY : 'intervalday' | 'INTERVALDAY';
Period : '.';
-DECIMAL9 : 'decimal9';
-DECIMAL18 : 'decimal18';
-DECIMAL28DENSE : 'decimal28dense';
-DECIMAL28SPARSE : 'decimal28sparse';
-DECIMAL38DENSE : 'decimal38dense';
-DECIMAL38SPARSE : 'decimal38sparse';
+DECIMAL9 : 'decimal9' | 'DECIMAL9';
+DECIMAL18 : 'decimal18' | 'DECIMAL18';
+DECIMAL28DENSE : 'decimal28dense' | 'DECIMAL28DENSE';
+DECIMAL28SPARSE : 'decimal28sparse' | 'DECIMAL28SPARSE';
+DECIMAL38DENSE : 'decimal38dense' | 'DECIMAL38DENSE';
+DECIMAL38SPARSE : 'decimal38sparse' | 'DECIMAL38SPARSE';
Or : '||' | 'or' | 'OR' | 'Or';
And : '&&' | 'and' | 'AND' ;
Equals : '==' | '=';
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/935095ef/exec/java-exec/src/test/resources/decimal/cast_float_decimal.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/decimal/cast_float_decimal.json b/exec/java-exec/src/test/resources/decimal/cast_float_decimal.json
index 33732ae..b9e05d3 100644
--- a/exec/java-exec/src/test/resources/decimal/cast_float_decimal.json
+++ b/exec/java-exec/src/test/resources/decimal/cast_float_decimal.json
@@ -31,7 +31,7 @@
"pop" : "project",
"@id" : 4,
"exprs" : [ {
- "ref" : "DECIMAL9",
+ "ref" : "DECIMAL_9",
"expr" : " cast(F4 as decimal9(9, 4)) "
},
{"ref": "DECIMAL38", "expr" : "cast(F8 as decimal38sparse(38, 4))"}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/935095ef/exec/java-exec/src/test/resources/decimal/cast_simple_decimal.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/decimal/cast_simple_decimal.json b/exec/java-exec/src/test/resources/decimal/cast_simple_decimal.json
index 279de3c..0291724 100644
--- a/exec/java-exec/src/test/resources/decimal/cast_simple_decimal.json
+++ b/exec/java-exec/src/test/resources/decimal/cast_simple_decimal.json
@@ -19,10 +19,10 @@
"pop" : "project",
"@id" : 2,
"exprs" : [ {
- "ref" : "DECIMAL9",
+ "ref" : "DECIMAL_9",
"expr" : " (cast(DEC9 as decimal9(9, 4))) "
},
- { "ref" : "DECIMAL18", "expr": "(cast(DEC18 as decimal18(18, 9)))" }
+ { "ref" : "DECIMAL_18", "expr": "(cast(DEC18 as decimal18(18, 9)))" }
],
"child" : 1
@@ -31,10 +31,10 @@
"pop" : "project",
"@id" : 4,
"exprs" : [ {
- "ref" : "DECIMAL9",
- "expr" : " cast(DECIMAL9 as varchar(100)) "
+ "ref" : "DECIMAL_9",
+ "expr" : " cast(DECIMAL_9 as varchar(100)) "
},
- {"ref": "DECIMAL18", "expr" : "cast(DECIMAL18 as varchar(100))"}
+ {"ref": "DECIMAL_18", "expr" : "cast(DECIMAL_18 as varchar(100))"}
],
"child" : 2
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/935095ef/exec/java-exec/src/test/resources/decimal/test_decimal_sparse_dense_dense.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/decimal/test_decimal_sparse_dense_dense.json b/exec/java-exec/src/test/resources/decimal/test_decimal_sparse_dense_dense.json
index 03add9a..b1a2c43 100644
--- a/exec/java-exec/src/test/resources/decimal/test_decimal_sparse_dense_dense.json
+++ b/exec/java-exec/src/test/resources/decimal/test_decimal_sparse_dense_dense.json
@@ -19,7 +19,7 @@
"pop" : "project",
"@id" : 2,
"exprs" : [ {
- "ref" : "DECIMAL28SPARSE",
+ "ref" : "DECIMAL28SPARSE_1",
"expr" : " (cast(DEC9 as decimal28sparse(28, 4))) "
}
],
@@ -30,8 +30,8 @@
"pop" : "project",
"@id" : 3,
"exprs" : [ {
- "ref" : "DECIMAL28DENSE",
- "expr" : " cast(DECIMAL28SPARSE as decimal28dense(28, 4)) "
+ "ref" : "DECIMAL28DENSE_1",
+ "expr" : " cast(DECIMAL28SPARSE_1 as decimal28dense(28, 4)) "
}
],
@@ -41,8 +41,8 @@
"pop" : "project",
"@id" : 4,
"exprs" : [ {
- "ref" : "DECIMAL38DENSE",
- "expr" : " cast(DECIMAL28DENSE as decimal38dense(38, 4)) "
+ "ref" : "DECIMAL38DENSE_1",
+ "expr" : " cast(DECIMAL28DENSE_1 as decimal38dense(38, 4)) "
}
],
[2/5] git commit: DRILL-784: Implement min, max,
count for VarChar and VarBinary data type. Re-enable aggregate
function tests.
Posted by ja...@apache.org.
DRILL-784: Implement min, max, count for VarChar and VarBinary data type. Re-enable aggregate function tests.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/68f4bdb4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/68f4bdb4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/68f4bdb4
Branch: refs/heads/master
Commit: 68f4bdb403225d58a5d9d68b4de361d18d3a8bc3
Parents: 37c1d06
Author: Mehant Baid <me...@gmail.com>
Authored: Mon May 19 17:23:51 2014 -0700
Committer: Mehant Baid <me...@gmail.com>
Committed: Tue May 20 13:43:03 2014 -0700
----------------------------------------------------------------------
.../src/main/codegen/data/AggrTypes1.tdd | 18 ++-
.../templates/VarCharAggrFunctions1.java | 158 +++++++++++++++++++
.../jdbc/test/TestAggregateFunctionsQuery.java | 32 +++-
3 files changed, 198 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/68f4bdb4/exec/java-exec/src/main/codegen/data/AggrTypes1.tdd
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/data/AggrTypes1.tdd b/exec/java-exec/src/main/codegen/data/AggrTypes1.tdd
index 71931df..d76d913 100644
--- a/exec/java-exec/src/main/codegen/data/AggrTypes1.tdd
+++ b/exec/java-exec/src/main/codegen/data/AggrTypes1.tdd
@@ -40,7 +40,11 @@
{inputType: "IntervalYear", outputType: "IntervalYear", runningType: "IntervalYear", major: "Date", initialValue: "Integer.MAX_VALUE"},
{inputType: "NullableIntervalYear", outputType: "IntervalYear", runningType: "IntervalYear", major: "Date", initialValue: "Integer.MAX_VALUE"},
{inputType: "Interval", outputType: "Interval", runningType: "BigInt", major: "Date", initialValue: "Long.MAX_VALUE"},
- {inputType: "NullableInterval", outputType: "Interval", runningType: "BigInt", major: "Date", initialValue: "Long.MAX_VALUE"}
+ {inputType: "NullableInterval", outputType: "Interval", runningType: "BigInt", major: "Date", initialValue: "Long.MAX_VALUE"},
+ {inputType: "VarChar", outputType: "VarChar", runningType: "VarChar", major: "VarBytes", initialValue: "0xFF", bufferEnd: "65536"},
+ {inputType: "NullableVarChar", outputType: "VarChar", runningType: "VarChar", major: "VarBytes", initialValue: "0xFF",bufferEnd: "65536"},
+ {inputType: "VarBinary", outputType: "VarBinary", runningType: "VarBinary", major: "VarBytes", initialValue: "0xFF", bufferEnd: "65536"},
+ {inputType: "NullableVarBinary", outputType: "VarBinary", runningType: "VarBinary", major: "VarBytes", initialValue: "0xFF", bufferEnd: "65536"}
]
},
{className: "Max", funcName: "max", types: [
@@ -67,7 +71,11 @@
{inputType: "IntervalYear", outputType: "IntervalYear", runningType: "IntervalYear", major: "Date", initialValue: "Integer.MIN_VALUE"},
{inputType: "NullableIntervalYear", outputType: "IntervalYear", runningType: "IntervalYear", major: "Date", initialValue: "Integer.MIN_VALUE"},
{inputType: "Interval", outputType: "Interval", runningType: "BigInt", major: "Date", initialValue: "Long.MIN_VALUE"},
- {inputType: "NullableInterval", outputType: "Interval", runningType: "BigInt", major: "Date", initialValue: "Long.MIN_VALUE"}
+ {inputType: "NullableInterval", outputType: "Interval", runningType: "BigInt", major: "Date", initialValue: "Long.MIN_VALUE"},
+ {inputType: "VarChar", outputType: "VarChar", runningType: "VarChar", major: "VarBytes", initialValue: "0x00", bufferEnd: "0"},
+ {inputType: "NullableVarChar", outputType: "VarChar", runningType: "VarChar", major: "VarBytes", initialValue: "0x00", bufferEnd: "0"},
+ {inputType: "VarBinary", outputType: "VarBinary", runningType: "VarBinary", major: "VarBytes", initialValue: "0x00", bufferEnd: "0"},
+ {inputType: "NullableVarBinary", outputType: "VarBinary", runningType: "VarBinary", major: "VarBytes", initialValue: "0x00", bufferEnd: "0"}
]
},
{className: "Sum", funcName: "sum", types: [
@@ -113,7 +121,11 @@
{inputType: "IntervalYear", outputType: "BigInt", runningType: "BigInt", major: "Date", initialValue: "0"},
{inputType: "NullableIntervalYear", outputType: "BigInt", runningType: "BigInt", major: "Date", initialValue: "0"},
{inputType: "Interval", outputType: "BigInt", runningType: "BigInt", major: "Date", initialValue: "0"},
- {inputType: "NullableInterval", outputType: "BigInt", runningType: "BigInt", major: "Date", initialValue: "0"}
+ {inputType: "NullableInterval", outputType: "BigInt", runningType: "BigInt", major: "Date", initialValue: "0"},
+ {inputType: "VarChar", outputType: "BigInt", runningType: "BigInt", major: "VarBytes"},
+ {inputType: "NullableVarChar", outputType: "BigInt", runningType: "BigInt", major: "VarBytes"},
+ {inputType: "VarBinary", outputType: "BigInt", runningType: "BigInt", major: "VarBytes"},
+ {inputType: "NullableVarBinary", outputType: "BigInt", runningType: "BigInt", major: "VarBytes"}
]
}
]
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/68f4bdb4/exec/java-exec/src/main/codegen/templates/VarCharAggrFunctions1.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/VarCharAggrFunctions1.java b/exec/java-exec/src/main/codegen/templates/VarCharAggrFunctions1.java
new file mode 100644
index 0000000..b5b5010
--- /dev/null
+++ b/exec/java-exec/src/main/codegen/templates/VarCharAggrFunctions1.java
@@ -0,0 +1,158 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+<@pp.dropOutputFile />
+
+
+
+<#list aggrtypes1.aggrtypes as aggrtype>
+<@pp.changeOutputFile name="/org/apache/drill/exec/expr/fn/impl/gaggr/${aggrtype.className}VarBytesFunctions.java" />
+
+<#include "/@includes/license.ftl" />
+
+<#-- A utility class that is used to generate java code for aggr functions that maintain a single -->
+<#-- running counter to hold the result. This includes: MIN, MAX, COUNT. -->
+
+/*
+ * This class is automatically generated from VarCharAggrFunctions1.java using FreeMarker.
+ */
+
+package org.apache.drill.exec.expr.fn.impl.gaggr;
+
+import org.apache.drill.exec.expr.DrillAggFunc;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope;
+import org.apache.drill.exec.expr.annotations.Output;
+import org.apache.drill.exec.expr.annotations.Param;
+import org.apache.drill.exec.expr.annotations.Workspace;
+import org.apache.drill.exec.expr.holders.*;
+import org.apache.drill.exec.record.RecordBatch;
+import io.netty.buffer.ByteBuf;
+
+@SuppressWarnings("unused")
+
+public class ${aggrtype.className}VarBytesFunctions {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${aggrtype.className}Functions.class);
+
+<#list aggrtype.types as type>
+<#if type.major == "VarBytes">
+
+@FunctionTemplate(name = "${aggrtype.funcName}", scope = FunctionTemplate.FunctionScope.POINT_AGGREGATE)
+public static class ${type.inputType}${aggrtype.className} implements DrillAggFunc{
+
+ @Param ${type.inputType}Holder in;
+ @Workspace ${type.runningType}Holder value;
+ @Output ${type.outputType}Holder out;
+
+ public void setup(RecordBatch b) {
+ value = new ${type.runningType}Holder();
+ <#if aggrtype.funcName == "max" || aggrtype.funcName == "min">
+ value.start = 0;
+ value.end = ${type.bufferEnd};
+ io.netty.buffer.ByteBuf buffer = io.netty.buffer.Unpooled.wrappedBuffer(new byte [65536]);
+ for (int i = 0; i < 65536; i++) {
+ buffer.setByte(i, ${type.initialValue});
+ }
+ value.buffer = buffer;
+
+ <#else>
+ value.value = 0;
+ </#if>
+ }
+
+ @Override
+ public void add() {
+ <#if type.inputType?starts_with("Nullable")>
+ sout: {
+ if (in.isSet == 0) {
+ // processing nullable input and the value is null, so don't do anything...
+ break sout;
+ }
+ </#if>
+ <#if aggrtype.funcName == "max" || aggrtype.funcName == "min">
+ int cmp = 0;
+ boolean swap = false;
+
+ // Compare the bytes
+ for (int l = in.start, r = value.start; l < in.end && r < value.end; l++, r++) {
+ byte leftByte = in.buffer.getByte(l);
+ byte rightByte = value.buffer.getByte(r);
+ if (leftByte != rightByte) {
+ cmp = ((leftByte & 0xFF) - (rightByte & 0xFF)) > 0 ? 1 : -1;
+ break;
+ }
+ }
+
+ if (cmp == 0) {
+ int l = (in.end - in.start) - (value.end - value.start);
+ if (l > 0) {
+ cmp = 1;
+ } else {
+ cmp = -1;
+ }
+ }
+
+ <#if aggrtype.className == "Min">
+ swap = (cmp == -1);
+ <#elseif aggrtype.className == "Max">
+ swap = (cmp == 1);
+ </#if>
+
+ if (swap) {
+ int length = in.end - in.start;
+ in.buffer.getBytes(in.start, value.buffer, 0, length);
+ value.end = length;
+ }
+ <#else>
+ value.value++;
+ </#if>
+ <#if type.inputType?starts_with("Nullable")>
+ } // end of sout block
+ </#if>
+ }
+
+ @Override
+ public void output() {
+ <#if aggrtype.funcName == "max" || aggrtype.funcName == "min">
+ out.start = value.start;
+ out.end = value.end;
+ out.buffer = value.buffer;
+ <#else>
+ out.value = value.value;
+ </#if>
+ }
+
+ @Override
+ public void reset() {
+ value = new ${type.runningType}Holder();
+ <#if aggrtype.funcName == "max" || aggrtype.funcName == "min">
+ value.start = 0;
+ value.end = ${type.bufferEnd};
+ io.netty.buffer.ByteBuf buffer = io.netty.buffer.Unpooled.wrappedBuffer(new byte [65536]);
+ for (int i = 0; i < 65536; i++) {
+ buffer.setByte(i, ${type.initialValue});
+ }
+ value.buffer = buffer;
+ <#else>
+ value.value = 0;
+ </#if>
+ }
+}
+</#if>
+</#list>
+}
+</#list>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/68f4bdb4/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestAggregateFunctionsQuery.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestAggregateFunctionsQuery.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestAggregateFunctionsQuery.java
index 4a57ffb..5c5a218 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestAggregateFunctionsQuery.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestAggregateFunctionsQuery.java
@@ -36,10 +36,11 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TestRule;
+import org.joda.time.DateTime;
+
import com.google.common.base.Function;
import com.google.common.base.Stopwatch;
-@Ignore // until stream agg changing schema is fixed.
public class TestAggregateFunctionsQuery {
public static final String WORKING_PATH;
@@ -53,12 +54,15 @@ public class TestAggregateFunctionsQuery {
String query = new String("SELECT max(cast(HIRE_DATE as date)) as MAX_DATE, min(cast(HIRE_DATE as date)) as MIN_DATE" +
" FROM `employee.json`");
+
+ String t = new DateTime(1998, 1, 1, 0, 0, 0, 0).toString();
+ String t1 = new DateTime(1993, 5, 1, 0, 0, 0, 0).toString();
+
+ String result = String.format("MAX_DATE="+ t + "; " + "MIN_DATE=" + t1 + "\n");
+
JdbcAssert.withFull("cp")
.sql(query)
- .returns(
- "MAX_DATE=1998-01-01; " +
- "MIN_DATE=1993-05-01\n"
- );
+ .returns(result);
}
@Test
@@ -69,8 +73,8 @@ public class TestAggregateFunctionsQuery {
JdbcAssert.withFull("cp")
.sql(query)
.returns(
- "MAX_DAYS=7671 days 0:0:0.0; " +
- "MIN_DAYS=5965 days 0:0:0.0\n"
+ "MAX_DAYS=P7671D; " +
+ "MIN_DAYS=P5965D\n"
);
}
@@ -96,4 +100,18 @@ public class TestAggregateFunctionsQuery {
"MIN_DEC38=1.00000000000\n "
);
}
+
+
+ @Test
+ public void testVarCharAggFunction() throws Exception{
+ String query = new String("select max(full_name) as MAX_NAME, min(full_name) as MIN_NAME" +
+ " FROM `employee.json`");
+
+ JdbcAssert.withFull("cp")
+ .sql(query)
+ .returns(
+ "MAX_NAME=Zach Lovell; " +
+ "MIN_NAME=A. Joyce Jarvis\n"
+ );
+ }
}
[4/5] git commit: DRILL-763: Handle SubSchemaWrapper correctly in
ShowFileHandler.
Posted by ja...@apache.org.
DRILL-763: Handle SubSchemaWrapper correctly in ShowFileHandler.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/ea1ab62b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/ea1ab62b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/ea1ab62b
Branch: refs/heads/master
Commit: ea1ab62bc5b60fad4c306062b92f91eb429954a5
Parents: 935095e
Author: Mehant Baid <me...@gmail.com>
Authored: Wed May 21 01:22:39 2014 -0700
Committer: Mehant Baid <me...@gmail.com>
Committed: Wed May 21 01:39:15 2014 -0700
----------------------------------------------------------------------
.../planner/sql/handlers/ShowFileHandler.java | 50 ++++++++++----------
.../exec/store/dfs/FileSystemSchemaFactory.java | 6 +--
.../exec/store/dfs/HasFileSystemSchema.java | 26 ----------
.../exec/store/dfs/WorkspaceSchemaFactory.java | 7 ++-
.../apache/drill/jdbc/test/TestJdbcQuery.java | 3 +-
5 files changed, 33 insertions(+), 59 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea1ab62b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ShowFileHandler.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ShowFileHandler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ShowFileHandler.java
index 4f7c424..8ce0463 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ShowFileHandler.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ShowFileHandler.java
@@ -26,6 +26,10 @@ import net.hydromatic.optiq.tools.Planner;
import net.hydromatic.optiq.tools.RelConversionException;
import net.hydromatic.optiq.tools.ValidationException;
+import org.apache.drill.exec.store.AbstractSchema;
+import org.apache.drill.exec.store.SubSchemaWrapper;
+import org.apache.drill.exec.store.dfs.WorkspaceSchemaFactory;
+import org.apache.drill.exec.store.dfs.WorkspaceSchemaFactory.WorkspaceSchema;
import org.eigenbase.sql.SqlIdentifier;
import org.eigenbase.sql.SqlNode;
@@ -33,12 +37,12 @@ import org.apache.drill.exec.ops.QueryContext;
import org.apache.drill.exec.physical.PhysicalPlan;
import org.apache.drill.exec.planner.sql.DirectPlan;
import org.apache.drill.exec.planner.sql.parser.SqlShowFiles;
-import org.apache.drill.exec.store.dfs.HasFileSystemSchema;
import org.apache.drill.exec.store.dfs.shim.DrillFileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.FileStatus;
+
public class ShowFileHandler extends DefaultSqlHandler {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SetOptionHandler.class);
@@ -50,40 +54,36 @@ public class ShowFileHandler extends DefaultSqlHandler {
public PhysicalPlan getPlan(SqlNode sqlNode) throws ValidationException, RelConversionException, IOException {
SqlIdentifier from = ((SqlShowFiles) sqlNode).getDb();
- String fromDir = from.names.get((from.names.size() - 1));
- // Get the correct subschema
- SchemaPlus schema = context.getNewDefaultSchema().getParentSchema();
- for (int i = 0; i < from.names.size() - 1 && schema != null; i++) {
- schema = schema.getSubSchema(from.names.get(i));
- }
+ DrillFileSystem fs = null;
+ String defaultLocation = null;
- // Traverse from the root schema if current schema is null
- if (schema == null) {
- schema = context.getRootSchema();
-
- for (int i = 0; i < from.names.size() - 1 && schema != null; i++) {
- schema = schema.getSubSchema(from.names.get(i));
- }
+ try {
- if (schema == null) {
- throw new ValidationException("Invalid schema");
+ // Traverse and find the schema
+ SchemaPlus drillSchema = findSchema(context.getRootSchema(), context.getNewDefaultSchema(), from.names.subList(0, from.names.size() - 1));
+ AbstractSchema tempSchema = getDrillSchema(drillSchema);
+ WorkspaceSchema schema = null;
+ if (tempSchema instanceof WorkspaceSchema) {
+ schema = ((WorkspaceSchema)tempSchema);
+ } else {
+ throw new ValidationException("Unsupported schema");
}
- }
- DrillFileSystem fs;
+ // Get the file system object
+ fs = schema.getFS();
- // Get the DrillFileSystem object
- try {
- HasFileSystemSchema fsSchema = schema.unwrap(HasFileSystemSchema.class);
- fs = fsSchema.getFS();
- } catch (ClassCastException e) {
- throw new ValidationException("Schema not an instance of file system schema");
+ // Get the default path
+ defaultLocation = schema.getDefaultLocation();
+ } catch (Exception e) {
+ return DirectPlan.createDirectPlan(context, false, String.format("Current schema '%s' is not a file system schema. " +
+ "Can't execute show files on this schema.", from.toString()));
}
List<ShowFilesCommandResult> rows = new ArrayList<>();
+ String fromDir = from.names.get((from.names.size() - 1));
- for (FileStatus fileStatus : fs.list(false, new Path(fromDir))) {
+ for (FileStatus fileStatus : fs.list(false, new Path(defaultLocation, fromDir))) {
ShowFilesCommandResult result = new ShowFilesCommandResult(fileStatus.getPath().getName(), fileStatus.isDir(),
!fileStatus.isDir(), fileStatus.getLen(),
fileStatus.getOwner(), fileStatus.getGroup(),
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea1ab62b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemSchemaFactory.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemSchemaFactory.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemSchemaFactory.java
index 5d291f9..3b74b72 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemSchemaFactory.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemSchemaFactory.java
@@ -66,7 +66,7 @@ public class FileSystemSchemaFactory implements SchemaFactory{
schema.setPlus(plusOfThis);
}
- public class FileSystemSchema extends AbstractSchema implements HasFileSystemSchema {
+ public class FileSystemSchema extends AbstractSchema {
private final WorkspaceSchema defaultSchema;
private final Map<String, WorkspaceSchema> schemaMap = Maps.newHashMap();
@@ -132,10 +132,6 @@ public class FileSystemSchemaFactory implements SchemaFactory{
return defaultSchema.isMutable();
}
- public DrillFileSystem getFS() {
- return defaultSchema.getFS();
- }
-
@Override
public CreateTableEntry createNewTable(String tableName) {
return defaultSchema.createNewTable(tableName);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea1ab62b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/HasFileSystemSchema.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/HasFileSystemSchema.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/HasFileSystemSchema.java
deleted file mode 100644
index c11887a..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/HasFileSystemSchema.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.store.dfs;
-
-import net.hydromatic.optiq.SchemaPlus;
-import org.apache.drill.exec.store.dfs.shim.DrillFileSystem;
-
-public interface HasFileSystemSchema {
-
- public DrillFileSystem getFS();
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea1ab62b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/WorkspaceSchemaFactory.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/WorkspaceSchemaFactory.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/WorkspaceSchemaFactory.java
index b17779b..cf85861 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/WorkspaceSchemaFactory.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/WorkspaceSchemaFactory.java
@@ -113,7 +113,7 @@ public class WorkspaceSchemaFactory implements ExpandingConcurrentMap.MapValueFa
public void destroy(DrillTable value) {
}
- public class WorkspaceSchema extends AbstractSchema implements HasFileSystemSchema {
+ public class WorkspaceSchema extends AbstractSchema {
private ExpandingConcurrentMap<String, DrillTable> tables = new ExpandingConcurrentMap<String, DrillTable>(WorkspaceSchemaFactory.this);
private UserSession session;
@@ -141,11 +141,14 @@ public class WorkspaceSchemaFactory implements ExpandingConcurrentMap.MapValueFa
return config.isWritable();
}
- @Override
public DrillFileSystem getFS() {
return fs;
}
+ public String getDefaultLocation() {
+ return config.getLocation();
+ }
+
@Override
public CreateTableEntry createNewTable(String tableName) {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea1ab62b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
index 191aa0a..9ef5cc4 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
@@ -46,7 +46,7 @@ import com.google.common.base.Stopwatch;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
-public class TestJdbcQuery extends JdbcTest{
+ public class TestJdbcQuery extends JdbcTest{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestJdbcQuery.class);
@@ -787,6 +787,7 @@ public class TestJdbcQuery extends JdbcTest{
@Test
public void testShowFiles() throws Exception {
testQuery("show files from dfs.`/tmp`");
+ testQuery("show files from `dfs.default`.`/tmp`");
}
[5/5] git commit: DRILL-680: INFORMATION_SCHEMA.COLUMNS does not
display HBase column families
Posted by ja...@apache.org.
DRILL-680: INFORMATION_SCHEMA.COLUMNS does not display HBase column families
+ Enhanced result layout with option to set output width on per column basis.
+ Pretty print plan fragments.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/ae570aad
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/ae570aad
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/ae570aad
Branch: refs/heads/master
Commit: ae570aad8b572a5c1fe612f79fe2524cb5e4e879
Parents: ea1ab62
Author: Aditya Kishore <ad...@maprtech.com>
Authored: Tue May 20 06:34:48 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Wed May 21 08:41:57 2014 -0700
----------------------------------------------------------------------
.../drill/exec/store/hbase/DrillHBaseTable.java | 62 ++++++++++++++++++++
.../exec/store/hbase/HBaseSchemaFactory.java | 6 +-
.../org/apache/drill/hbase/BaseHBaseTest.java | 12 ++--
.../drill/hbase/TestHBaseFilterPushDown.java | 6 ++
.../drill/hbase/TestHBaseProjectPushDown.java | 8 ++-
.../planner/fragment/SimpleParallelizer.java | 8 +--
.../org/apache/drill/exec/rpc/RpcException.java | 10 ++--
.../org/apache/drill/exec/util/VectorUtil.java | 52 ++++++++++------
8 files changed, 126 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ae570aad/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/DrillHBaseTable.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/DrillHBaseTable.java b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/DrillHBaseTable.java
new file mode 100644
index 0000000..ff6991f
--- /dev/null
+++ b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/DrillHBaseTable.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.hbase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Set;
+
+import org.apache.drill.exec.planner.logical.DrillTable;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.eigenbase.reltype.RelDataType;
+import org.eigenbase.reltype.RelDataTypeFactory;
+import org.eigenbase.sql.type.SqlTypeName;
+
+public class DrillHBaseTable extends DrillTable implements DrillHBaseConstants {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillHBaseTable.class);
+
+ private HTableDescriptor table;
+
+ public DrillHBaseTable(String storageEngineName, HBaseStoragePlugin plugin, HBaseScanSpec scanSpec) {
+ super(storageEngineName, plugin, scanSpec);
+ try(HBaseAdmin admin = new HBaseAdmin(plugin.getConfig().getHBaseConf())) {
+ table = admin.getTableDescriptor(HBaseUtils.getBytes(scanSpec.getTableName()));
+ } catch (IOException e) {
+ logger.warn("Failure while loading table names for database '{}'.", storageEngineName, e);
+ }
+ }
+
+ @Override
+ public RelDataType getRowType(RelDataTypeFactory typeFactory) {
+ ArrayList<RelDataType> typeList = new ArrayList<>();
+ ArrayList<String> fieldNameList = new ArrayList<>();
+
+ fieldNameList.add(ROW_KEY);
+ typeList.add(typeFactory.createSqlType(SqlTypeName.ANY));
+
+ Set<byte[]> families = table.getFamiliesKeys();
+ for (byte[] family : families) {
+ fieldNameList.add(Bytes.toString(family));
+ typeList.add(typeFactory.createMapType(typeFactory.createSqlType(SqlTypeName.VARCHAR), typeFactory.createSqlType(SqlTypeName.ANY)));
+ }
+ return typeFactory.createStructType(typeList, fieldNameList);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ae570aad/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseSchemaFactory.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseSchemaFactory.java b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseSchemaFactory.java
index 84f363b..7b76092 100644
--- a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseSchemaFactory.java
+++ b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseSchemaFactory.java
@@ -25,7 +25,6 @@ import net.hydromatic.optiq.Schema;
import net.hydromatic.optiq.SchemaPlus;
import net.hydromatic.optiq.Table;
-import org.apache.drill.exec.planner.logical.DynamicDrillTable;
import org.apache.drill.exec.rpc.user.UserSession;
import org.apache.drill.exec.store.AbstractSchema;
import org.apache.drill.exec.store.SchemaFactory;
@@ -74,8 +73,8 @@ public class HBaseSchemaFactory implements SchemaFactory {
@Override
public Table getTable(String name) {
- Object selection = new HBaseScanSpec(name);
- return new DynamicDrillTable(plugin, schemaName, selection);
+ HBaseScanSpec scanSpec = new HBaseScanSpec(name);
+ return new DrillHBaseTable(schemaName, plugin, scanSpec);
}
@Override
@@ -99,4 +98,5 @@ public class HBaseSchemaFactory implements SchemaFactory {
}
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ae570aad/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/BaseHBaseTest.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/BaseHBaseTest.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/BaseHBaseTest.java
index 9e07d9f..96f0c4a 100644
--- a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/BaseHBaseTest.java
+++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/BaseHBaseTest.java
@@ -45,7 +45,7 @@ public class BaseHBaseTest extends BaseTestQuery {
@Rule public TestName TEST_NAME = new TestName();
- private int columnWidth = 8;
+ private int[] columnWidths = new int[] { 8 };
@Before
public void printID() throws Exception {
@@ -58,7 +58,7 @@ public class BaseHBaseTest extends BaseTestQuery {
* Change the following to HBaseTestsSuite.configure(false, true)
* if you want to test against an externally running HBase cluster.
*/
- HBaseTestsSuite.configure(true, true);
+ HBaseTestsSuite.configure(false, true);
HBaseTestsSuite.initCluster();
HBaseStoragePlugin plugin = (HBaseStoragePlugin) bit.getContext().getStorage().getPlugin("hbase");
@@ -71,9 +71,13 @@ public class BaseHBaseTest extends BaseTestQuery {
}
protected void setColumnWidth(int columnWidth) {
- this.columnWidth = columnWidth;
+ this.columnWidths = new int[] { columnWidth };
}
+ protected void setColumnWidths(int[] columnWidths) {
+ this.columnWidths = columnWidths;
+ }
+
protected String getPlanText(String planFile, String tableName) throws IOException {
return Files.toString(FileUtils.getResourceAsFile(planFile), Charsets.UTF_8)
.replaceFirst("\"hbase\\.zookeeper\\.property\\.clientPort\".*:.*\\d+", "\"hbase.zookeeper.property.clientPort\" : " + HBaseTestsSuite.getZookeeperPort())
@@ -102,7 +106,7 @@ public class BaseHBaseTest extends BaseTestQuery {
if (loader.getRecordCount() <= 0) {
break;
}
- VectorUtil.showVectorAccessibleContent(loader, columnWidth);
+ VectorUtil.showVectorAccessibleContent(loader, columnWidths);
loader.clear();
result.release();
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ae570aad/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseFilterPushDown.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseFilterPushDown.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseFilterPushDown.java
index 90404b7..2071e90 100644
--- a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseFilterPushDown.java
+++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseFilterPushDown.java
@@ -23,6 +23,7 @@ public class TestHBaseFilterPushDown extends BaseHBaseTest {
@Test
public void testFilterPushDownRowKeyEqual() throws Exception {
+ setColumnWidths(new int[] {8, 38, 38});
runSQLVerifyCount("SELECT\n"
+ " *\n"
+ "FROM\n"
@@ -34,6 +35,7 @@ public class TestHBaseFilterPushDown extends BaseHBaseTest {
@Test
public void testFilterPushDownRowKeyGreaterThan() throws Exception {
+ setColumnWidths(new int[] {8, 38, 38});
runSQLVerifyCount("SELECT\n"
+ " *\n"
+ "FROM\n"
@@ -45,6 +47,7 @@ public class TestHBaseFilterPushDown extends BaseHBaseTest {
@Test
public void testFilterPushDownRowKeyBetween() throws Exception {
+ setColumnWidths(new int[] {8, 74, 38});
runSQLVerifyCount("SELECT\n"
+ " *\n"
+ "FROM\n"
@@ -56,6 +59,7 @@ public class TestHBaseFilterPushDown extends BaseHBaseTest {
@Test
public void testFilterPushDownMultiColumns() throws Exception {
+ setColumnWidths(new int[] {8, 74, 38});
runSQLVerifyCount("SELECT\n"
+ " *\n"
+ "FROM\n"
@@ -67,6 +71,7 @@ public class TestHBaseFilterPushDown extends BaseHBaseTest {
@Test
public void testFilterPushDownConvertExpression() throws Exception {
+ setColumnWidths(new int[] {8, 38, 38});
runSQLVerifyCount("SELECT\n"
+ " *\n"
+ "FROM\n"
@@ -78,6 +83,7 @@ public class TestHBaseFilterPushDown extends BaseHBaseTest {
@Test
public void testFilterPushDownRowKeyLessThanOrEqualTo() throws Exception {
+ setColumnWidths(new int[] {8, 74, 38});
runSQLVerifyCount("SELECT\n"
+ " *\n"
+ "FROM\n"
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ae570aad/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseProjectPushDown.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseProjectPushDown.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseProjectPushDown.java
index b66d2ed..7037b9a 100644
--- a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseProjectPushDown.java
+++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseProjectPushDown.java
@@ -23,6 +23,7 @@ public class TestHBaseProjectPushDown extends BaseHBaseTest {
@Test
public void testRowKeyPushDown() throws Exception{
+ setColumnWidth(8);
runSQLVerifyCount("SELECT\n"
+ "row_key\n"
+ "FROM\n"
@@ -32,6 +33,7 @@ public class TestHBaseProjectPushDown extends BaseHBaseTest {
@Test
public void testColumnWith1RowPushDown() throws Exception{
+ setColumnWidth(6);
runSQLVerifyCount("SELECT\n"
+ "f2['c7'] as `f[c7]`\n"
+ "FROM\n"
@@ -41,7 +43,7 @@ public class TestHBaseProjectPushDown extends BaseHBaseTest {
@Test
public void testRowKeyAndColumnPushDown() throws Exception{
- setColumnWidth(9);
+ setColumnWidths(new int[] {8, 9, 6, 2, 6});
runSQLVerifyCount("SELECT\n"
+ "row_key, f['c1']*31 as `f[c1]*31`, f['c2'] as `f[c2]`, 5 as `5`, 'abc' as `'abc'`\n"
+ "FROM\n"
@@ -51,9 +53,9 @@ public class TestHBaseProjectPushDown extends BaseHBaseTest {
@Test
public void testColumnFamilyPushDown() throws Exception{
- setColumnWidth(74);
+ setColumnWidths(new int[] {8, 74, 38});
runSQLVerifyCount("SELECT\n"
- + "f, f2\n"
+ + "row_key, f, f2\n"
+ "FROM\n"
+ " hbase.`[TABLE_NAME]` tableName"
, 6);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ae570aad/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/SimpleParallelizer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/SimpleParallelizer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/SimpleParallelizer.java
index c34869d..313a81d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/SimpleParallelizer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/SimpleParallelizer.java
@@ -20,6 +20,7 @@ package org.apache.drill.exec.planner.fragment;
import java.util.Collection;
import java.util.List;
+import org.apache.commons.lang3.StringEscapeUtils;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.exceptions.PhysicalOperatorSetupException;
import org.apache.drill.exec.exception.FragmentSetupException;
@@ -38,9 +39,6 @@ import org.apache.drill.exec.work.QueryWorkUnit;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
-import org.apache.drill.exec.expr.fn.impl.DateUtility;
-import org.joda.time.DateTime;
-import org.joda.time.DateTimeZone;
/**
* The simple parallelizer determines the level of parallelization of a plan based on the cost of the underlying
@@ -143,11 +141,11 @@ public class SimpleParallelizer {
.build();
if (isRootNode) {
- logger.debug("Root fragment:\n {}", fragment);
+ logger.debug("Root fragment:\n {}", StringEscapeUtils.unescapeJava(fragment.toString()));
rootFragment = fragment;
rootOperator = root;
} else {
- logger.debug("Remote fragment:\n {}", fragment);
+ logger.debug("Remote fragment:\n {}", StringEscapeUtils.unescapeJava(fragment.toString()));
fragments.add(fragment);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ae570aad/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcException.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcException.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcException.java
index dba72a1..9b5eb1d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcException.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcException.java
@@ -19,13 +19,14 @@ package org.apache.drill.exec.rpc;
import java.util.concurrent.ExecutionException;
-import org.apache.commons.lang.StringEscapeUtils;
+import org.apache.commons.lang3.StringEscapeUtils;
import org.apache.drill.common.exceptions.DrillIOException;
/**
* Parent class for all rpc exceptions.
*/
public class RpcException extends DrillIOException{
+ private static final long serialVersionUID = -5964230316010502319L;
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(RpcException.class);
public RpcException() {
@@ -47,17 +48,16 @@ public class RpcException extends DrillIOException{
public RpcException(Throwable cause) {
super(cause);
}
-
+
public static RpcException mapException(Throwable t){
while(t instanceof ExecutionException) t = ((ExecutionException)t).getCause();
if(t instanceof RpcException) return ((RpcException) t);
return new RpcException(t);
}
-
+
public static RpcException mapException(String message, Throwable t){
while(t instanceof ExecutionException) t = ((ExecutionException)t).getCause();
return new RpcException(message, t);
}
-
-
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ae570aad/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java
index a43225e..bdba1d8 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java
@@ -17,14 +17,11 @@
*/
package org.apache.drill.exec.util;
-import java.util.Arrays;
import java.util.List;
import org.apache.commons.lang.StringUtils;
-import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.record.VectorAccessible;
import org.apache.drill.exec.record.VectorWrapper;
-import org.apache.drill.exec.rpc.RpcException;
import com.beust.jcommander.internal.Lists;
@@ -37,7 +34,7 @@ public class VectorUtil {
int rows = va.getRecordCount();
List<String> columns = Lists.newArrayList();
for (VectorWrapper<?> vw : va) {
- columns.add(vw.getValueVector().getField().getAsSchemaPath().getAsUnescapedPath());
+ columns.add(vw.getValueVector().getField().getPath().getAsUnescapedPath());
}
int width = columns.size();
@@ -71,55 +68,74 @@ public class VectorUtil {
}
public static void showVectorAccessibleContent(VectorAccessible va) {
- showVectorAccessibleContent(va, DEFAULT_COLUMN_WIDTH);
+ showVectorAccessibleContent(va, DEFAULT_COLUMN_WIDTH);
}
+
public static void showVectorAccessibleContent(VectorAccessible va, int columnWidth) {
+ showVectorAccessibleContent(va, new int[]{ columnWidth });
+ }
- int rows = va.getRecordCount();
+ public static void showVectorAccessibleContent(VectorAccessible va, int[] columnWidths) {
+ int width = 0;
+ int columnIndex = 0;
List<String> columns = Lists.newArrayList();
+ List<String> formats = Lists.newArrayList();
for (VectorWrapper<?> vw : va) {
- columns.add(vw.getValueVector().getField().getAsSchemaPath().getAsUnescapedPath());
+ int columnWidth = getColumnWidth(columnWidths, columnIndex);
+ width += columnWidth + 2;
+ formats.add("| %-" + columnWidth + "s");
+ columns.add(vw.getValueVector().getField().getPath().getAsUnescapedPath());
+ columnIndex++;
}
- int width = columns.size() * (columnWidth + 2);
-
- String format = ("| %-" + columnWidth + "s");
-
+ int rows = va.getRecordCount();
for (int row = 0; row < rows; row++) {
+ // header, every 50 rows.
if (row%50 == 0) {
System.out.println(StringUtils.repeat("-", width + 1));
+ columnIndex = 0;
for (String column : columns) {
- System.out.printf(format, column.length() <= columnWidth ? column : column.substring(0, columnWidth - 1));
+ int columnWidth = getColumnWidth(columnWidths, columnIndex);
+ System.out.printf(formats.get(columnIndex), column.length() <= columnWidth ? column : column.substring(0, columnWidth - 1));
+ columnIndex++;
}
System.out.printf("|\n");
System.out.println(StringUtils.repeat("-", width + 1));
}
+ // column values
+ columnIndex = 0;
for (VectorWrapper<?> vw : va) {
+ int columnWidth = getColumnWidth(columnWidths, columnIndex);
Object o = vw.getValueVector().getAccessor().getObject(row);
if (o == null) {
//null value
- System.out.printf(format, "");
+ System.out.printf(formats.get(columnIndex), "");
}
else if (o instanceof byte[]) {
String value = new String((byte[]) o);
- System.out.printf(format, value.length() <= columnWidth ? value : value.substring(0, columnWidth - 1));
+ System.out.printf(formats.get(columnIndex), value.length() <= columnWidth ? value : value.substring(0, columnWidth - 1));
} else if (o instanceof List) {
System.out.printf("| %s", o);
} else {
String value = o.toString();
- System.out.printf(format, value.length() <= columnWidth ? value : value.substring(0,columnWidth - 1));
+ System.out.printf(formats.get(columnIndex), value.length() <= columnWidth ? value : value.substring(0,columnWidth - 1));
}
+ columnIndex++;
}
System.out.printf("|\n");
}
+ if (rows > 0) {
+ System.out.println(StringUtils.repeat("-", width + 1));
+ }
for (VectorWrapper<?> vw : va) {
vw.clear();
}
-
- if (rows > 0 )
- System.out.println(StringUtils.repeat("-", width + 1));
}
+ private static int getColumnWidth(int[] columnWidths, int columnIndex) {
+ return (columnWidths == null) ? DEFAULT_COLUMN_WIDTH
+ : (columnWidths.length > columnIndex) ? columnWidths[columnIndex] : columnWidths[0];
+ }
}