You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2016/10/05 20:10:34 UTC
[1/6] incubator-impala git commit: IMPALA-4196: Cross compile
bit-byte-functions
Repository: incubator-impala
Updated Branches:
refs/heads/master 757c68b29 -> 3be113cb9
IMPALA-4196: Cross compile bit-byte-functions
Change-Id: I5a1291bfd202b500405a884e4a62f0ca2447244a
Reviewed-on: http://gerrit.cloudera.org:8080/4557
Reviewed-by: Bharath Vissapragada <bh...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/64c39482
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/64c39482
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/64c39482
Branch: refs/heads/master
Commit: 64c394827ab6f88e39545dc6c7b4ab1e5bb00c91
Parents: 757c68b
Author: Bharath Vissapragada <bh...@cloudera.com>
Authored: Wed Sep 28 16:32:38 2016 -0700
Committer: Internal Jenkins <cl...@gerrit.cloudera.org>
Committed: Sat Oct 1 01:42:21 2016 +0000
----------------------------------------------------------------------
be/src/codegen/impala-ir.cc | 1 +
be/src/exprs/CMakeLists.txt | 2 +-
be/src/exprs/bit-byte-functions-ir.cc | 204 +++++++++++++++++++
be/src/exprs/bit-byte-functions.cc | 204 -------------------
.../queries/QueryTest/exprs.test | 8 +
5 files changed, 214 insertions(+), 205 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/64c39482/be/src/codegen/impala-ir.cc
----------------------------------------------------------------------
diff --git a/be/src/codegen/impala-ir.cc b/be/src/codegen/impala-ir.cc
index 9b14058..eac17a6 100644
--- a/be/src/codegen/impala-ir.cc
+++ b/be/src/codegen/impala-ir.cc
@@ -33,6 +33,7 @@
#include "exec/partitioned-hash-join-node-ir.cc"
#include "exec/topn-node-ir.cc"
#include "exprs/aggregate-functions-ir.cc"
+#include "exprs/bit-byte-functions-ir.cc"
#include "exprs/cast-functions-ir.cc"
#include "exprs/compound-predicates-ir.cc"
#include "exprs/conditional-functions-ir.cc"
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/64c39482/be/src/exprs/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/be/src/exprs/CMakeLists.txt b/be/src/exprs/CMakeLists.txt
index feec660..87a9ae0 100644
--- a/be/src/exprs/CMakeLists.txt
+++ b/be/src/exprs/CMakeLists.txt
@@ -26,7 +26,7 @@ add_library(Exprs
agg-fn-evaluator.cc
aggregate-functions-ir.cc
anyval-util.cc
- bit-byte-functions.cc
+ bit-byte-functions-ir.cc
case-expr.cc
cast-functions-ir.cc
compound-predicates.cc
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/64c39482/be/src/exprs/bit-byte-functions-ir.cc
----------------------------------------------------------------------
diff --git a/be/src/exprs/bit-byte-functions-ir.cc b/be/src/exprs/bit-byte-functions-ir.cc
new file mode 100644
index 0000000..c2208ad
--- /dev/null
+++ b/be/src/exprs/bit-byte-functions-ir.cc
@@ -0,0 +1,204 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "exprs/bit-byte-functions.h"
+
+#include <boost/type_traits/make_unsigned.hpp>
+
+#include "gutil/strings/substitute.h"
+
+#include "util/bit-util.h"
+
+#include "common/names.h"
+
+using namespace impala_udf;
+
+using boost::make_unsigned;
+
+using impala::BitUtil;
+
+namespace impala {
+
+// Generic algorithm for shifting and rotating signed integers
+// Declare here to resolve mutual recursion
+template<typename T>
+static T RotateLeftImpl(T v, int32_t shift);
+template<typename T>
+static T RotateRightImpl(T v, int32_t shift);
+template<typename T>
+static T ShiftLeftImpl(T v, int32_t shift);
+template<typename T>
+static T ShiftRightLogicalImpl(T v, int32_t shift);
+
+template <typename T>
+IntVal BitByteFunctions::CountSet(FunctionContext* ctx, const T& v) {
+ if (v.is_null) return IntVal::null();
+ return IntVal(BitUtil::PopcountSigned(v.val));
+}
+
+template IntVal BitByteFunctions::CountSet(FunctionContext*, const TinyIntVal&);
+template IntVal BitByteFunctions::CountSet(FunctionContext*, const SmallIntVal&);
+template IntVal BitByteFunctions::CountSet(FunctionContext*, const IntVal&);
+template IntVal BitByteFunctions::CountSet(FunctionContext*, const BigIntVal&);
+
+template <typename T>
+IntVal BitByteFunctions::CountSet(FunctionContext* ctx, const T& v, const IntVal &bitval) {
+ if (v.is_null || bitval.is_null) return IntVal::null();
+ if (bitval.val == 0) {
+ return IntVal(sizeof(v.val) * 8 - BitUtil::PopcountSigned(v.val));
+ } else if (bitval.val == 1) {
+ return IntVal(BitUtil::PopcountSigned(v.val));
+ }
+
+ ctx->SetError(Substitute("Invalid bit val: $0", bitval.val).c_str());
+ return IntVal::null();
+}
+
+template IntVal BitByteFunctions::CountSet(FunctionContext*, const TinyIntVal&,
+ const IntVal&);
+template IntVal BitByteFunctions::CountSet(FunctionContext*, const SmallIntVal&,
+ const IntVal&);
+template IntVal BitByteFunctions::CountSet(FunctionContext*, const IntVal&,
+ const IntVal&);
+template IntVal BitByteFunctions::CountSet(FunctionContext*, const BigIntVal&,
+ const IntVal&);
+
+template<typename T>
+TinyIntVal BitByteFunctions::GetBit(FunctionContext* ctx, const T& v,
+ const IntVal& bitpos) {
+ if (v.is_null || bitpos.is_null) return TinyIntVal::null();
+ if (bitpos.val < 0 || bitpos.val >= sizeof(v.val) * 8) {
+ ctx->SetError(Substitute("Invalid bit position: $0", bitpos.val).c_str());
+ return TinyIntVal::null();
+ }
+ return TinyIntVal(BitUtil::GetBit(v.val, bitpos.val));
+}
+
+template TinyIntVal BitByteFunctions::GetBit(FunctionContext*, const TinyIntVal&,
+ const IntVal&);
+template TinyIntVal BitByteFunctions::GetBit(FunctionContext*, const SmallIntVal&,
+ const IntVal&);
+template TinyIntVal BitByteFunctions::GetBit(FunctionContext*, const IntVal&,
+ const IntVal&);
+template TinyIntVal BitByteFunctions::GetBit(FunctionContext*, const BigIntVal&,
+ const IntVal&);
+
+template<typename T>
+T BitByteFunctions::SetBit(FunctionContext* ctx, const T& v,
+ const IntVal& bitpos) {
+ if (v.is_null || bitpos.is_null) return T::null();
+ if (bitpos.val < 0 || bitpos.val >= sizeof(v.val) * 8) {
+ ctx->SetError(Substitute("Invalid bit position: $0", bitpos.val).c_str());
+ return T::null();
+ }
+ return T(BitUtil::SetBit(v.val, bitpos.val));
+}
+
+template<typename T>
+T BitByteFunctions::SetBit(FunctionContext* ctx, const T& v,
+ const IntVal& bitpos, const IntVal& bitval) {
+ if (v.is_null || bitpos.is_null || bitval.is_null) return T::null();
+ if (bitpos.val < 0 || bitpos.val >= sizeof(v.val) * 8) {
+ ctx->SetError(Substitute("Invalid bit position: $0", bitpos.val).c_str());
+ return T::null();
+ }
+ if (bitval.val == 0) {
+ return T(BitUtil::UnsetBit(v.val, bitpos.val));
+ } else if (bitval.val == 1) {
+ return T(BitUtil::SetBit(v.val, bitpos.val));
+ }
+ ctx->SetError(Substitute("Invalid bit val: $0", bitval.val).c_str());
+ return T::null();
+}
+
+template TinyIntVal BitByteFunctions::SetBit(FunctionContext*, const TinyIntVal&,
+ const IntVal&);
+template SmallIntVal BitByteFunctions::SetBit(FunctionContext*, const SmallIntVal&,
+ const IntVal&);
+template IntVal BitByteFunctions::SetBit(FunctionContext*, const IntVal&, const IntVal&);
+template BigIntVal BitByteFunctions::SetBit(FunctionContext*, const BigIntVal&,
+ const IntVal&);
+template TinyIntVal BitByteFunctions::SetBit(FunctionContext*, const TinyIntVal&,
+ const IntVal&, const IntVal&);
+template SmallIntVal BitByteFunctions::SetBit(FunctionContext*, const SmallIntVal&,
+ const IntVal&, const IntVal&);
+template IntVal BitByteFunctions::SetBit(FunctionContext*, const IntVal&, const IntVal&,
+ const IntVal&);
+template BigIntVal BitByteFunctions::SetBit(FunctionContext*, const BigIntVal&,
+ const IntVal&, const IntVal&);
+
+template<typename T>
+static T RotateLeftImpl(T v, int32_t shift) {
+ // Handle negative shifts
+ if (shift < 0) return RotateRightImpl(v, -shift);
+
+ // Handle wrapping around multiple times
+ shift = shift % (sizeof(T) * 8);
+ return (v << shift) | BitUtil::ShiftRightLogical(v, sizeof(T) * 8 - shift);
+}
+
+template<typename T>
+static T RotateRightImpl(T v, int32_t shift) {
+ // Handle negative shifts
+ if (shift < 0) return RotateLeftImpl(v, -shift);
+
+ // Handle wrapping around multiple times
+ shift = shift % (sizeof(T) * 8);
+ return BitUtil::ShiftRightLogical(v, shift) | (v << (sizeof(T) * 8 - shift));
+}
+
+template<typename T>
+static T ShiftLeftImpl(T v, int32_t shift) {
+ if (shift < 0) return ShiftRightLogicalImpl(v, -shift);
+ return v << shift;
+}
+
+// Logical right shift rather than arithmetic right shift
+template<typename T>
+static T ShiftRightLogicalImpl(T v, int32_t shift) {
+ if (shift < 0) return ShiftLeftImpl(v, -shift);
+ // Conversion to unsigned ensures most significant bits always filled with 0's
+ return BitUtil::ShiftRightLogical(v, shift);
+}
+
+// Generates a shift/rotate function for Impala integer value type based on the shift
+// algorithm implemented by ALGO
+#define SHIFT_FN(NAME, INPUT_TYPE, ALGO) \
+ INPUT_TYPE BitByteFunctions::NAME(FunctionContext* ctx, const INPUT_TYPE& v, \
+ const IntVal& shift) { \
+ if (v.is_null || shift.is_null) return INPUT_TYPE::null(); \
+ return INPUT_TYPE(ALGO(v.val, shift.val)); \
+ }
+
+SHIFT_FN(RotateLeft, TinyIntVal, RotateLeftImpl);
+SHIFT_FN(RotateLeft, SmallIntVal, RotateLeftImpl);
+SHIFT_FN(RotateLeft, IntVal, RotateLeftImpl);
+SHIFT_FN(RotateLeft, BigIntVal, RotateLeftImpl);
+SHIFT_FN(RotateRight, TinyIntVal, RotateRightImpl);
+SHIFT_FN(RotateRight, SmallIntVal, RotateRightImpl);
+SHIFT_FN(RotateRight, IntVal, RotateRightImpl);
+SHIFT_FN(RotateRight, BigIntVal, RotateRightImpl);
+SHIFT_FN(ShiftLeft, TinyIntVal, ShiftLeftImpl);
+SHIFT_FN(ShiftLeft, SmallIntVal, ShiftLeftImpl);
+SHIFT_FN(ShiftLeft, IntVal, ShiftLeftImpl);
+SHIFT_FN(ShiftLeft, BigIntVal, ShiftLeftImpl);
+SHIFT_FN(ShiftRight, TinyIntVal, ShiftRightLogicalImpl);
+SHIFT_FN(ShiftRight, SmallIntVal, ShiftRightLogicalImpl);
+SHIFT_FN(ShiftRight, IntVal, ShiftRightLogicalImpl);
+SHIFT_FN(ShiftRight, BigIntVal, ShiftRightLogicalImpl);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/64c39482/be/src/exprs/bit-byte-functions.cc
----------------------------------------------------------------------
diff --git a/be/src/exprs/bit-byte-functions.cc b/be/src/exprs/bit-byte-functions.cc
deleted file mode 100644
index c2208ad..0000000
--- a/be/src/exprs/bit-byte-functions.cc
+++ /dev/null
@@ -1,204 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "exprs/bit-byte-functions.h"
-
-#include <boost/type_traits/make_unsigned.hpp>
-
-#include "gutil/strings/substitute.h"
-
-#include "util/bit-util.h"
-
-#include "common/names.h"
-
-using namespace impala_udf;
-
-using boost::make_unsigned;
-
-using impala::BitUtil;
-
-namespace impala {
-
-// Generic algorithm for shifting and rotating signed integers
-// Declare here to resolve mutual recursion
-template<typename T>
-static T RotateLeftImpl(T v, int32_t shift);
-template<typename T>
-static T RotateRightImpl(T v, int32_t shift);
-template<typename T>
-static T ShiftLeftImpl(T v, int32_t shift);
-template<typename T>
-static T ShiftRightLogicalImpl(T v, int32_t shift);
-
-template <typename T>
-IntVal BitByteFunctions::CountSet(FunctionContext* ctx, const T& v) {
- if (v.is_null) return IntVal::null();
- return IntVal(BitUtil::PopcountSigned(v.val));
-}
-
-template IntVal BitByteFunctions::CountSet(FunctionContext*, const TinyIntVal&);
-template IntVal BitByteFunctions::CountSet(FunctionContext*, const SmallIntVal&);
-template IntVal BitByteFunctions::CountSet(FunctionContext*, const IntVal&);
-template IntVal BitByteFunctions::CountSet(FunctionContext*, const BigIntVal&);
-
-template <typename T>
-IntVal BitByteFunctions::CountSet(FunctionContext* ctx, const T& v, const IntVal &bitval) {
- if (v.is_null || bitval.is_null) return IntVal::null();
- if (bitval.val == 0) {
- return IntVal(sizeof(v.val) * 8 - BitUtil::PopcountSigned(v.val));
- } else if (bitval.val == 1) {
- return IntVal(BitUtil::PopcountSigned(v.val));
- }
-
- ctx->SetError(Substitute("Invalid bit val: $0", bitval.val).c_str());
- return IntVal::null();
-}
-
-template IntVal BitByteFunctions::CountSet(FunctionContext*, const TinyIntVal&,
- const IntVal&);
-template IntVal BitByteFunctions::CountSet(FunctionContext*, const SmallIntVal&,
- const IntVal&);
-template IntVal BitByteFunctions::CountSet(FunctionContext*, const IntVal&,
- const IntVal&);
-template IntVal BitByteFunctions::CountSet(FunctionContext*, const BigIntVal&,
- const IntVal&);
-
-template<typename T>
-TinyIntVal BitByteFunctions::GetBit(FunctionContext* ctx, const T& v,
- const IntVal& bitpos) {
- if (v.is_null || bitpos.is_null) return TinyIntVal::null();
- if (bitpos.val < 0 || bitpos.val >= sizeof(v.val) * 8) {
- ctx->SetError(Substitute("Invalid bit position: $0", bitpos.val).c_str());
- return TinyIntVal::null();
- }
- return TinyIntVal(BitUtil::GetBit(v.val, bitpos.val));
-}
-
-template TinyIntVal BitByteFunctions::GetBit(FunctionContext*, const TinyIntVal&,
- const IntVal&);
-template TinyIntVal BitByteFunctions::GetBit(FunctionContext*, const SmallIntVal&,
- const IntVal&);
-template TinyIntVal BitByteFunctions::GetBit(FunctionContext*, const IntVal&,
- const IntVal&);
-template TinyIntVal BitByteFunctions::GetBit(FunctionContext*, const BigIntVal&,
- const IntVal&);
-
-template<typename T>
-T BitByteFunctions::SetBit(FunctionContext* ctx, const T& v,
- const IntVal& bitpos) {
- if (v.is_null || bitpos.is_null) return T::null();
- if (bitpos.val < 0 || bitpos.val >= sizeof(v.val) * 8) {
- ctx->SetError(Substitute("Invalid bit position: $0", bitpos.val).c_str());
- return T::null();
- }
- return T(BitUtil::SetBit(v.val, bitpos.val));
-}
-
-template<typename T>
-T BitByteFunctions::SetBit(FunctionContext* ctx, const T& v,
- const IntVal& bitpos, const IntVal& bitval) {
- if (v.is_null || bitpos.is_null || bitval.is_null) return T::null();
- if (bitpos.val < 0 || bitpos.val >= sizeof(v.val) * 8) {
- ctx->SetError(Substitute("Invalid bit position: $0", bitpos.val).c_str());
- return T::null();
- }
- if (bitval.val == 0) {
- return T(BitUtil::UnsetBit(v.val, bitpos.val));
- } else if (bitval.val == 1) {
- return T(BitUtil::SetBit(v.val, bitpos.val));
- }
- ctx->SetError(Substitute("Invalid bit val: $0", bitval.val).c_str());
- return T::null();
-}
-
-template TinyIntVal BitByteFunctions::SetBit(FunctionContext*, const TinyIntVal&,
- const IntVal&);
-template SmallIntVal BitByteFunctions::SetBit(FunctionContext*, const SmallIntVal&,
- const IntVal&);
-template IntVal BitByteFunctions::SetBit(FunctionContext*, const IntVal&, const IntVal&);
-template BigIntVal BitByteFunctions::SetBit(FunctionContext*, const BigIntVal&,
- const IntVal&);
-template TinyIntVal BitByteFunctions::SetBit(FunctionContext*, const TinyIntVal&,
- const IntVal&, const IntVal&);
-template SmallIntVal BitByteFunctions::SetBit(FunctionContext*, const SmallIntVal&,
- const IntVal&, const IntVal&);
-template IntVal BitByteFunctions::SetBit(FunctionContext*, const IntVal&, const IntVal&,
- const IntVal&);
-template BigIntVal BitByteFunctions::SetBit(FunctionContext*, const BigIntVal&,
- const IntVal&, const IntVal&);
-
-template<typename T>
-static T RotateLeftImpl(T v, int32_t shift) {
- // Handle negative shifts
- if (shift < 0) return RotateRightImpl(v, -shift);
-
- // Handle wrapping around multiple times
- shift = shift % (sizeof(T) * 8);
- return (v << shift) | BitUtil::ShiftRightLogical(v, sizeof(T) * 8 - shift);
-}
-
-template<typename T>
-static T RotateRightImpl(T v, int32_t shift) {
- // Handle negative shifts
- if (shift < 0) return RotateLeftImpl(v, -shift);
-
- // Handle wrapping around multiple times
- shift = shift % (sizeof(T) * 8);
- return BitUtil::ShiftRightLogical(v, shift) | (v << (sizeof(T) * 8 - shift));
-}
-
-template<typename T>
-static T ShiftLeftImpl(T v, int32_t shift) {
- if (shift < 0) return ShiftRightLogicalImpl(v, -shift);
- return v << shift;
-}
-
-// Logical right shift rather than arithmetic right shift
-template<typename T>
-static T ShiftRightLogicalImpl(T v, int32_t shift) {
- if (shift < 0) return ShiftLeftImpl(v, -shift);
- // Conversion to unsigned ensures most significant bits always filled with 0's
- return BitUtil::ShiftRightLogical(v, shift);
-}
-
-// Generates a shift/rotate function for Impala integer value type based on the shift
-// algorithm implemented by ALGO
-#define SHIFT_FN(NAME, INPUT_TYPE, ALGO) \
- INPUT_TYPE BitByteFunctions::NAME(FunctionContext* ctx, const INPUT_TYPE& v, \
- const IntVal& shift) { \
- if (v.is_null || shift.is_null) return INPUT_TYPE::null(); \
- return INPUT_TYPE(ALGO(v.val, shift.val)); \
- }
-
-SHIFT_FN(RotateLeft, TinyIntVal, RotateLeftImpl);
-SHIFT_FN(RotateLeft, SmallIntVal, RotateLeftImpl);
-SHIFT_FN(RotateLeft, IntVal, RotateLeftImpl);
-SHIFT_FN(RotateLeft, BigIntVal, RotateLeftImpl);
-SHIFT_FN(RotateRight, TinyIntVal, RotateRightImpl);
-SHIFT_FN(RotateRight, SmallIntVal, RotateRightImpl);
-SHIFT_FN(RotateRight, IntVal, RotateRightImpl);
-SHIFT_FN(RotateRight, BigIntVal, RotateRightImpl);
-SHIFT_FN(ShiftLeft, TinyIntVal, ShiftLeftImpl);
-SHIFT_FN(ShiftLeft, SmallIntVal, ShiftLeftImpl);
-SHIFT_FN(ShiftLeft, IntVal, ShiftLeftImpl);
-SHIFT_FN(ShiftLeft, BigIntVal, ShiftLeftImpl);
-SHIFT_FN(ShiftRight, TinyIntVal, ShiftRightLogicalImpl);
-SHIFT_FN(ShiftRight, SmallIntVal, ShiftRightLogicalImpl);
-SHIFT_FN(ShiftRight, IntVal, ShiftRightLogicalImpl);
-SHIFT_FN(ShiftRight, BigIntVal, ShiftRightLogicalImpl);
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/64c39482/testdata/workloads/functional-query/queries/QueryTest/exprs.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/exprs.test b/testdata/workloads/functional-query/queries/QueryTest/exprs.test
index 5cf4c43..3c7f5d5 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/exprs.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/exprs.test
@@ -2494,3 +2494,11 @@ select reverse('123456789abcdef0'), reverse('123456789abcdef01'),
---- TYPES
STRING,STRING,STRING,STRING,STRING
====
+---- QUERY
+# IMPALA-4196: Regression test for built-in bit-byte-functions with codegen
+select count(shiftleft(int_col, 1)) from functional_parquet.alltypes
+---- RESULTS
+7300
+---- TYPES
+bigint
+====
[3/6] incubator-impala git commit: Revert
org.apache.impala.hive.serde.ParquetOutputFormat
Posted by ta...@apache.org.
Revert org.apache.impala.hive.serde.ParquetOutputFormat
The patch for IMPALA-3786 replaced most instances of
com.cloudera.impala.* with org.apache.impala.*
This patch rolls back one such replacement:
com.cloudera.impala.hive.serde.ParquetOutputFormat
which is necessary for backwards compatability with previously
created tables.
Change-Id: I27738dfc3d44bb5fabd3a066ffbba99527160bce
Reviewed-on: http://gerrit.cloudera.org:8080/4611
Reviewed-by: Alex Behm <al...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/8a08850f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/8a08850f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/8a08850f
Branch: refs/heads/master
Commit: 8a08850fd42d516c5527899f295222f7c75ad721
Parents: 886e26d
Author: Thomas Tauber-Marshall <tm...@cloudera.com>
Authored: Mon Oct 3 16:31:45 2016 -0700
Committer: Internal Jenkins <cl...@gerrit.cloudera.org>
Committed: Tue Oct 4 03:52:48 2016 +0000
----------------------------------------------------------------------
.../java/org/apache/impala/catalog/HdfsStorageDescriptorTest.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/8a08850f/fe/src/test/java/org/apache/impala/catalog/HdfsStorageDescriptorTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/catalog/HdfsStorageDescriptorTest.java b/fe/src/test/java/org/apache/impala/catalog/HdfsStorageDescriptorTest.java
index 0d8af1c..0934fed 100644
--- a/fe/src/test/java/org/apache/impala/catalog/HdfsStorageDescriptorTest.java
+++ b/fe/src/test/java/org/apache/impala/catalog/HdfsStorageDescriptorTest.java
@@ -60,7 +60,7 @@ public class HdfsStorageDescriptorTest {
"parquet.hive.MapredParquetInputFormat",
"org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat"};
String [] outputFormats = new String [] {
- "org.apache.impala.hive.serde.ParquetOutputFormat",
+ "com.cloudera.impala.hive.serde.ParquetOutputFormat",
"parquet.hive.DeprecatedParquetOutputFormat",
"parquet.hive.MapredParquetOutputFormat",
"org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat"};
[2/6] incubator-impala git commit: IMPALA-4234: Remove astyle config
file, looks outdated.
Posted by ta...@apache.org.
IMPALA-4234: Remove astyle config file, looks outdated.
Change-Id: Ibdeba67fcfa538a49d335d42cfdc799753fe7e48
Reviewed-on: http://gerrit.cloudera.org:8080/4578
Reviewed-by: Alex Behm <al...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/886e26d5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/886e26d5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/886e26d5
Branch: refs/heads/master
Commit: 886e26d58fd0b7ac3d3ffb277b330cc061835c29
Parents: 64c3948
Author: Lars Volker <lv...@cloudera.com>
Authored: Fri Sep 30 23:48:12 2016 +0200
Committer: Internal Jenkins <cl...@gerrit.cloudera.org>
Committed: Tue Oct 4 01:19:01 2016 +0000
----------------------------------------------------------------------
be/.astylerc | 95 -------------------------------------------------------
1 file changed, 95 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/886e26d5/be/.astylerc
----------------------------------------------------------------------
diff --git a/be/.astylerc b/be/.astylerc
deleted file mode 100644
index 1692559..0000000
--- a/be/.astylerc
+++ /dev/null
@@ -1,95 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# Reference: http://astyle.sourceforge.net/astyle.html
-
-# Indent using 2 spaces
---indent=spaces=2
-
-# Indent using attached brackets, e.g.
-# int Foo(bool isBar) {
-# if (isBar) {
-# bar();
-# return 1;
-# } else
-# return 0;
-# }
-#
---brackets=attach
-
-# Indent 'switch' blocks so that the 'case X:' statements are indented in
-# the switch block. The entire case block is indented:
-#
-# switch (foo)
-# {
-# case 1:
-# a += 1;
-# break;
-#
-# case 2:
-# {
-# a += 2;
-# break;
-# }
-# }
-#
-#becomes:
-#
-# switch (foo)
-# {
-# case 1:
-# a += 1;
-# break;
-#
-# case 2:
-# {
-# a += 2;
-# break;
-# }
-# }
---indent-switches
-
-# Insert space padding around operators. Any end of line comments will
-# remain in the original column, if possible. Note that there is no option
-# to unpad. Once padded, they stay padded.
-#
-# if (foo==2)
-# a=bar((b-c)*a,d--);
-#
-#becomes:
-#
-# if (foo == 2)
-# a = bar((b - c) * a, d--);
---pad-oper
-
-# Attach a pointer or reference operator (* or &) to the variable type (left):
-#
-# char *foo1;
-# char &foo2;
-#
-# becomes (with align-pointer=type):
-#
-# char* foo1;
-# char& foo2;
---align-pointer=type
-
-# Don't break one-line blocks.
-#
-# if (isFoo)
-# { isFoo = false; cout << isFoo << endl; }
-# remains unchanged.
---keep-one-line-blocks
[5/6] incubator-impala git commit: IMPALA-4042: Preserve root types
when substituting grouping exprs
Posted by ta...@apache.org.
IMPALA-4042: Preserve root types when substituting grouping exprs
In case of count(distinct), FunctionCallExpr.analyze() changes type
for "NULL" into "BOOLEAN" to make sure that BE doesn't see any
"NULL_TYPE" exprs. In the meantime, Expr substitution, happening in
Expr.substituteImpl() reverts this change back to original type,
"NULL_TYPE".
This causes an issue when AggregateInfo.checkConsistency() performs
precondition check where slot types from
AggregateInfo.outputTupleDesc_ should be matched with the types from
AggregateInfo.groupingExpr_. The slot type shows "BOOLEAN" while type
from groupingExpr_ is "NULL_TYPE", which makes the precondition fail
and throws an exception.
To resolve the issue, preserveRootType is set to true when
Expr.substituteList() gets called in AggregateInfo.substitute()
Change-Id: Icf3b4511234e473e5b9548fbf3e97f333c9980f1
(cherry picked from commit b17785b4890bedd1c825140ce3c48cd7d9734295)
Reviewed-on: http://gerrit.cloudera.org:8080/4600
Reviewed-by: Alex Behm <al...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/112ff68e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/112ff68e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/112ff68e
Branch: refs/heads/master
Commit: 112ff68eddde5e29bacf31170741ee7746c0cbb8
Parents: e659337
Author: Yonghyun Hwang <yo...@cloudera.com>
Authored: Fri Sep 30 16:12:46 2016 -0700
Committer: Internal Jenkins <cl...@gerrit.cloudera.org>
Committed: Wed Oct 5 03:04:17 2016 +0000
----------------------------------------------------------------------
.../apache/impala/analysis/AggregateInfo.java | 4 +++-
.../queries/PlannerTest/distinct.test | 22 ++++++++++++++++++++
2 files changed, 25 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/112ff68e/fe/src/main/java/org/apache/impala/analysis/AggregateInfo.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/AggregateInfo.java b/fe/src/main/java/org/apache/impala/analysis/AggregateInfo.java
index 29b80ef..8590872 100644
--- a/fe/src/main/java/org/apache/impala/analysis/AggregateInfo.java
+++ b/fe/src/main/java/org/apache/impala/analysis/AggregateInfo.java
@@ -335,7 +335,9 @@ public class AggregateInfo extends AggregateInfoBase {
*/
public void substitute(ExprSubstitutionMap smap, Analyzer analyzer)
throws InternalException {
- groupingExprs_ = Expr.substituteList(groupingExprs_, smap, analyzer, false);
+
+ // Preserve the root type for NULL literals.
+ groupingExprs_ = Expr.substituteList(groupingExprs_, smap, analyzer, true);
LOG.trace("AggInfo: grouping_exprs=" + Expr.debugString(groupingExprs_));
// The smap in this case should not substitute the aggs themselves, only
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/112ff68e/testdata/workloads/functional-planner/queries/PlannerTest/distinct.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/distinct.test b/testdata/workloads/functional-planner/queries/PlannerTest/distinct.test
index 15f1d59..e5f3bce 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/distinct.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/distinct.test
@@ -473,3 +473,25 @@ select * from (select count(distinct int_col) cd from functional.alltypes group
00:SCAN HDFS [functional.alltypes]
partitions=24/24 files=24 size=478.45KB
====
+# IMPALA-4042: count(distinct NULL) fails on a view
+select count(distinct null) from functional.alltypes_view
+---- DISTRIBUTEDPLAN
+06:AGGREGATE [FINALIZE]
+| output: count:merge(NULL)
+|
+05:EXCHANGE [UNPARTITIONED]
+|
+02:AGGREGATE
+| output: count(NULL)
+|
+04:AGGREGATE
+| group by: NULL
+|
+03:EXCHANGE [HASH(NULL)]
+|
+01:AGGREGATE [STREAMING]
+| group by: NULL
+|
+00:SCAN HDFS [functional.alltypes]
+ partitions=24/24 files=24 size=478.45KB
+====
[4/6] incubator-impala git commit: Add vim-specific files to
.gitignore
Posted by ta...@apache.org.
Add vim-specific files to .gitignore
Change-Id: I1abcd8ca0e18178684c916ef6f7d55c25c0814a4
Reviewed-on: http://gerrit.cloudera.org:8080/4562
Reviewed-by: Tim Armstrong <ta...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/e6593378
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/e6593378
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/e6593378
Branch: refs/heads/master
Commit: e6593378716138583328f64afb27f926223aad5e
Parents: 8a08850
Author: Lars Volker <lv...@cloudera.com>
Authored: Thu Sep 29 17:21:24 2016 +0200
Committer: Internal Jenkins <cl...@gerrit.cloudera.org>
Committed: Tue Oct 4 18:49:50 2016 +0000
----------------------------------------------------------------------
.gitignore | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/e6593378/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index 68a43cb..4a7f11a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,5 @@
*~
*.pyc
-.*.swp
derby.log
thirdparty
compile_commands.json
@@ -54,4 +53,9 @@ core.*
hs_err_pid*.log
# Binaries disallowed by ASF rules
-*.jar
\ No newline at end of file
+*.jar
+
+# Vim related files
+.*.swp
+.vimrc
+be/.ycm_extra_conf.py
[6/6] incubator-impala git commit: IMPALA-4246: SleepForMs() utility
function has undefined behavior for > 1s
Posted by ta...@apache.org.
IMPALA-4246: SleepForMs() utility function has undefined behavior for > 1s
Our SleepForMs() function relied on usleep() which sleeps for 'n'
microseconds. However, the manpage for usleep() specifies that this
may not work for values > 1000000 us (or 1s).
This patch removes the use of usleep() and uses
std::this_thread::sleep_for() instead, which was introduced with C++11.
Change-Id: I06c55b1be287b264e7601c9c89788ae5929571cf
Reviewed-on: http://gerrit.cloudera.org:8080/4622
Reviewed-by: Sailesh Mukil <sa...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/3be113cb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/3be113cb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/3be113cb
Branch: refs/heads/master
Commit: 3be113cb9fd6460a80b8198a50f3619c4b9539a2
Parents: 112ff68
Author: Sailesh Mukil <sa...@cloudera.com>
Authored: Tue Oct 4 15:31:58 2016 -0700
Committer: Internal Jenkins <cl...@gerrit.cloudera.org>
Committed: Wed Oct 5 03:29:03 2016 +0000
----------------------------------------------------------------------
be/src/util/time.cc | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3be113cb/be/src/util/time.cc
----------------------------------------------------------------------
diff --git a/be/src/util/time.cc b/be/src/util/time.cc
index b7d31ad..e6530de 100644
--- a/be/src/util/time.cc
+++ b/be/src/util/time.cc
@@ -15,13 +15,14 @@
// specific language governing permissions and limitations
// under the License.
-#include <unistd.h>
+#include <chrono>
+#include <thread>
#include "util/time.h"
using namespace impala;
+using namespace std;
void impala::SleepForMs(const int64_t duration_ms) {
- // TODO: Replace with sleep_for when we upgrade to recent boost / C++11?
- usleep(duration_ms * 1000L);
+ this_thread::sleep_for(chrono::milliseconds(duration_ms));
}