You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by kw...@apache.org on 2017/02/13 19:23:26 UTC
[3/4] incubator-impala git commit: IMPALA-2020,
IMPALA-4809: Codegen support for DECIMAL_V2
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f982c3f7/be/src/exprs/expr.h
----------------------------------------------------------------------
diff --git a/be/src/exprs/expr.h b/be/src/exprs/expr.h
index 13ba312..3a92c21 100644
--- a/be/src/exprs/expr.h
+++ b/be/src/exprs/expr.h
@@ -260,55 +260,8 @@ class Expr {
/// not strip these symbols.
static void InitBuiltinsDummy();
- // Any additions to this enum must be reflected in both GetConstant*() and
- // GetIrConstant().
- enum ExprConstant {
- // RETURN_TYPE_*: properties of FunctionContext::GetReturnType().
- RETURN_TYPE_SIZE, // int
- RETURN_TYPE_PRECISION, // int
- RETURN_TYPE_SCALE, // int
- // ARG_TYPE_* with parameter i: properties of FunctionContext::GetArgType(i).
- ARG_TYPE_SIZE, // int[]
- ARG_TYPE_PRECISION, // int[]
- ARG_TYPE_SCALE, // int[]
- };
-
- // Static function for obtaining a runtime constant. Expr compute functions and
- // builtins implementing the UDF interface should use this function, rather than
- // accessing runtime constants directly, so any recognized constants can be inlined via
- // InlineConstants() in the codegen path. In the interpreted path, this function will
- // work as-is.
- //
- // 'c' determines which constant is returned. The type of the constant is annotated in
- // the ExprConstant enum above. If the constant is an array, 'i' must be specified and
- // indicates which element to return. 'i' must always be an immediate integer value so
- // InlineConstants() can resolve the index, e.g., it cannot be a variable or an
- // expression like "1 + 1". For example, if 'c' = ARG_TYPE_SIZE, then 'T' = int and
- // 0 <= i < children_.size().
- //
- // InlineConstants() can be run on the function to replace recognized constants. The
- // constants are only replaced in the function itself, so any callee functions with
- // constants to be replaced must be inlined into the function that InlineConstants()
- // is run on (e.g. by annotating them with IR_ALWAYS_INLINE).
- //
- // TODO: implement a loop unroller (or use LLVM's) so we can use GetConstantInt() in
- // loops
- static int GetConstantInt(const FunctionContext& ctx, ExprConstant c, int i = -1);
-
- /// Finds all calls to Expr::GetConstantInt() in 'fn' and replaces them with the
- /// appropriate runtime constants based on the arguments. 'return_type' is the
- /// return type of the UDF or UDAF, i.e. the value of FunctionContext::GetReturnType().
- /// 'arg_types' are the argument types of the UDF or UDAF, i.e. the values of
- /// FunctionContext::GetArgType().
- static int InlineConstants(const FunctionContext::TypeDesc& return_type,
- const std::vector<FunctionContext::TypeDesc>& arg_types, LlvmCodeGen* codegen,
- llvm::Function* fn);
-
static const char* LLVM_CLASS_NAME;
- // Expr::GetConstantInt() symbol prefix.
- static const char* GET_CONSTANT_INT_SYMBOL_PREFIX;
-
protected:
friend class AggFnEvaluator;
friend class CastExpr;
@@ -409,11 +362,6 @@ class Expr {
/// functions (e.g. in ScalarFnCall() when codegen is disabled).
llvm::Function* GetStaticGetValWrapper(ColumnType type, LlvmCodeGen* codegen);
- /// Replace all calls to Expr::GetConstant() in 'fn' based on the types of the
- /// expr and its children. This is a convenience method that invokes the static
- /// InlineConstants() function with the correct arguments for the expr.
- int InlineConstants(LlvmCodeGen* codegen, llvm::Function* fn);
-
/// Simple debug string that provides no expr subclass-specific information
std::string DebugString(const std::string& expr_name) const;
@@ -457,13 +405,6 @@ class Expr {
static StringVal GetStringVal(Expr* expr, ExprContext* context, const TupleRow* row);
static TimestampVal GetTimestampVal(Expr* expr, ExprContext* context, const TupleRow* row);
static DecimalVal GetDecimalVal(Expr* expr, ExprContext* context, const TupleRow* row);
-
-
- // Helper function for GetConstantInt() and InlineConstants(): return the constant value
- // given the specific argument and return types.
- static int GetConstantInt(const FunctionContext::TypeDesc& return_type,
- const std::vector<FunctionContext::TypeDesc>& arg_types, ExprConstant c,
- int i = -1);
};
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f982c3f7/be/src/exprs/math-functions-ir.cc
----------------------------------------------------------------------
diff --git a/be/src/exprs/math-functions-ir.cc b/be/src/exprs/math-functions-ir.cc
index da8df36..6a331ba 100644
--- a/be/src/exprs/math-functions-ir.cc
+++ b/be/src/exprs/math-functions-ir.cc
@@ -433,7 +433,7 @@ template <typename T> T MathFunctions::Negative(FunctionContext* ctx, const T& v
template <>
DecimalVal MathFunctions::Negative(FunctionContext* ctx, const DecimalVal& val) {
if (val.is_null) return val;
- int type_byte_size = Expr::GetConstantInt(*ctx, Expr::RETURN_TYPE_SIZE);
+ int type_byte_size = ctx->impl()->GetConstFnAttr(FunctionContextImpl::RETURN_TYPE_SIZE);
switch (type_byte_size) {
case 4:
return DecimalVal(-val.val4);
@@ -519,7 +519,7 @@ template <bool ISLEAST> DecimalVal MathFunctions::LeastGreatest(
DCHECK_GT(num_args, 0);
if (args[0].is_null) return DecimalVal::null();
DecimalVal result_val = args[0];
- int type_byte_size = Expr::GetConstantInt(*ctx, Expr::RETURN_TYPE_SIZE);
+ int type_byte_size = ctx->impl()->GetConstFnAttr(FunctionContextImpl::RETURN_TYPE_SIZE);
for (int i = 1; i < num_args; ++i) {
if (args[i].is_null) return DecimalVal::null();
switch (type_byte_size) {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f982c3f7/be/src/exprs/scalar-fn-call.cc
----------------------------------------------------------------------
diff --git a/be/src/exprs/scalar-fn-call.cc b/be/src/exprs/scalar-fn-call.cc
index 06830b7..59aac6d 100644
--- a/be/src/exprs/scalar-fn-call.cc
+++ b/be/src/exprs/scalar-fn-call.cc
@@ -316,8 +316,8 @@ Status ScalarFnCall::GetCodegendComputeFn(LlvmCodeGen* codegen, Function** fn) {
NumFixedArgs(), vararg_start_idx_ != -1, &udf, &cache_entry_));
// Inline constants into the function if it has an IR body.
if (!udf->isDeclaration()) {
- InlineConstants(AnyValUtil::ColumnTypeToTypeDesc(type_),
- AnyValUtil::ColumnTypesToTypeDescs(arg_types), codegen, udf);
+ codegen->InlineConstFnAttrs(AnyValUtil::ColumnTypeToTypeDesc(type_),
+ AnyValUtil::ColumnTypesToTypeDescs(arg_types), udf);
udf = codegen->FinalizeFunction(udf);
if (udf == NULL) {
return Status(
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f982c3f7/be/src/runtime/decimal-value.inline.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/decimal-value.inline.h b/be/src/runtime/decimal-value.inline.h
index f886787..3a65a16 100644
--- a/be/src/runtime/decimal-value.inline.h
+++ b/be/src/runtime/decimal-value.inline.h
@@ -79,7 +79,8 @@ inline const T DecimalValue<T>::fractional_part(int scale) const {
template<typename T>
inline DecimalValue<T> DecimalValue<T>::ScaleTo(int src_scale, int dst_scale,
- int dst_precision, bool* overflow) const { int delta_scale = src_scale - dst_scale;
+ int dst_precision, bool* overflow) const {
+ int delta_scale = src_scale - dst_scale;
T result = value();
T max_value = DecimalUtil::GetScaleMultiplier<T>(dst_precision);
if (delta_scale >= 0) {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f982c3f7/be/src/runtime/lib-cache.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/lib-cache.cc b/be/src/runtime/lib-cache.cc
index 00c1159..9a8d873 100644
--- a/be/src/runtime/lib-cache.cc
+++ b/be/src/runtime/lib-cache.cc
@@ -400,12 +400,9 @@ Status LibCache::GetCacheEntryInternal(const string& hdfs_lib_file, LibType type
DynamicOpen((*entry)->local_path.c_str(), &(*entry)->shared_object_handle));
} else if (type == TYPE_IR) {
// Load the module temporarily and populate all symbols.
- ObjectPool pool;
- scoped_ptr<LlvmCodeGen> codegen;
- string module_id = filesystem::path((*entry)->local_path).stem().string();
- RETURN_IF_ERROR(LlvmCodeGen::CreateFromFile(
- &pool, NULL, (*entry)->local_path, module_id, &codegen));
- codegen->GetSymbols(&(*entry)->symbols);
+ const string file = (*entry)->local_path;
+ const string module_id = filesystem::path(file).stem().string();
+ RETURN_IF_ERROR(LlvmCodeGen::GetSymbols(file, module_id, &(*entry)->symbols));
} else {
DCHECK_EQ(type, TYPE_JAR);
// Nothing to do.
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f982c3f7/be/src/runtime/runtime-state.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/runtime-state.cc b/be/src/runtime/runtime-state.cc
index de0fc97..0d7f262 100644
--- a/be/src/runtime/runtime-state.cc
+++ b/be/src/runtime/runtime-state.cc
@@ -156,7 +156,7 @@ Status RuntimeState::CreateBlockMgr() {
Status RuntimeState::CreateCodegen() {
if (codegen_.get() != NULL) return Status::OK();
// TODO: add the fragment ID to the codegen ID as well
- RETURN_IF_ERROR(LlvmCodeGen::CreateImpalaCodegen(obj_pool_.get(),
+ RETURN_IF_ERROR(LlvmCodeGen::CreateImpalaCodegen(this,
instance_mem_tracker_.get(), PrintId(fragment_instance_id()), &codegen_));
codegen_->EnableOptimizations(true);
profile_.AddChild(codegen_->runtime_profile());
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f982c3f7/be/src/runtime/runtime-state.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/runtime-state.h b/be/src/runtime/runtime-state.h
index 14f9d38..97014aa 100644
--- a/be/src/runtime/runtime-state.h
+++ b/be/src/runtime/runtime-state.h
@@ -104,6 +104,7 @@ class RuntimeState {
bool abort_on_default_limit_exceeded() const {
return query_options().abort_on_default_limit_exceeded;
}
+ bool decimal_v2() const { return query_options().decimal_v2; }
const TQueryCtx& query_ctx() const;
const TPlanFragmentInstanceCtx& instance_ctx() const { return *instance_ctx_; }
const TUniqueId& session_id() const { return query_ctx().session.session_id; }
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f982c3f7/be/src/service/frontend.cc
----------------------------------------------------------------------
diff --git a/be/src/service/frontend.cc b/be/src/service/frontend.cc
index b83d091..e48cb1e 100644
--- a/be/src/service/frontend.cc
+++ b/be/src/service/frontend.cc
@@ -82,7 +82,7 @@ Frontend::Frontend() {
{"getTableFiles", "([B)[B", &get_table_files_id_},
{"showCreateFunction", "([B)Ljava/lang/String;", &show_create_function_id_},
{"buildTestDescriptorTable", "([B)[B", &build_test_descriptor_table_id_}
-};
+ };
JNIEnv* jni_env = getJNIEnv();
// create instance of java class JniFrontend
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f982c3f7/be/src/udf/udf-internal.h
----------------------------------------------------------------------
diff --git a/be/src/udf/udf-internal.h b/be/src/udf/udf-internal.h
index 96d0fc7..907bb22 100644
--- a/be/src/udf/udf-internal.h
+++ b/be/src/udf/udf-internal.h
@@ -132,12 +132,53 @@ class FunctionContextImpl {
return arg_types_;
}
- // UDFs may manipulate DecimalVal arguments via SIMD instructions such as 'movaps'
- // that require 16-byte memory alignment.
+ RuntimeState* state() { return state_; }
+
+ /// Various static attributes of the UDF/UDA that can be injected as constants
+ /// by codegen. Note that the argument types refer to those in the UDF/UDA signature,
+ /// not the arguments of the C++ functions implementing the UDF/UDA. Any change to
+ /// this enum must be reflected in FunctionContextImpl::GetConstFnAttr().
+ enum ConstFnAttr {
+ /// RETURN_TYPE_*: properties of FunctionContext::GetReturnType()
+ RETURN_TYPE_SIZE, // int
+ RETURN_TYPE_PRECISION, // int
+ RETURN_TYPE_SCALE, // int
+ /// ARG_TYPE_* with parameter i: properties of FunctionContext::GetArgType(i)
+ ARG_TYPE_SIZE, // int[]
+ ARG_TYPE_PRECISION, // int[]
+ ARG_TYPE_SCALE, // int[]
+ /// True if decimal_v2 query option is set.
+ DECIMAL_V2,
+ };
+
+ /// This function returns the various static attributes of the UDF/UDA. Calls to this
+ /// function are replaced by constants injected by codegen. If codegen is disabled,
+ /// this function is interpreted as-is.
+ ///
+ /// 't' is the static function attribute defined in the ConstFnAttr enum above.
+ /// For function attributes of arguments, 'i' holds the argument number (0 indexed).
+ /// Please note that argument refers to the arguments in the signature of the UDF or UDA.
+ /// 'i' must always be an immediate integer value in order to utilize the constant
+ /// replacement when codegen is enabled. e.g., it cannot be a variable or an expression
+ /// like "1 + 1".
+ ///
+ int GetConstFnAttr(ConstFnAttr t, int i = -1);
+
+ /// Return the function attribute 't' defined in ConstFnAttr above.
+ static int GetConstFnAttr(const RuntimeState* state,
+ const impala_udf::FunctionContext::TypeDesc& return_type,
+ const std::vector<impala_udf::FunctionContext::TypeDesc>& arg_types,
+ ConstFnAttr t, int i = -1);
+
+ /// UDFs may manipulate DecimalVal arguments via SIMD instructions such as 'movaps'
+ /// that require 16-byte memory alignment.
static const int VARARGS_BUFFER_ALIGNMENT = 16;
+
+ /// The LLVM class name for FunctionContext. Used for handcrafted IR.
static const char* LLVM_FUNCTIONCONTEXT_NAME;
- RuntimeState* state() { return state_; }
+ /// FunctionContextImpl::GetConstFnAttr() symbol. Used for call sites replacement.
+ static const char* GET_CONST_FN_ATTR_SYMBOL;
private:
friend class impala_udf::FunctionContext;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f982c3f7/be/src/udf/udf-test-harness.cc
----------------------------------------------------------------------
diff --git a/be/src/udf/udf-test-harness.cc b/be/src/udf/udf-test-harness.cc
index 1c3737d..d34b5ee 100644
--- a/be/src/udf/udf-test-harness.cc
+++ b/be/src/udf/udf-test-harness.cc
@@ -18,6 +18,7 @@
#include "udf/udf-test-harness.h"
#include <vector>
+#include "runtime/runtime-state.h"
#include "udf/udf-internal.h"
#include "common/names.h"
@@ -27,8 +28,8 @@ using namespace impala;
FunctionContext* UdfTestHarness::CreateTestContext(
const FunctionContext::TypeDesc& return_type,
- const vector<FunctionContext::TypeDesc>& arg_types) {
- return FunctionContextImpl::CreateContext(NULL, NULL, return_type, arg_types, 0, true);
+ const vector<FunctionContext::TypeDesc>& arg_types, RuntimeState* state) {
+ return FunctionContextImpl::CreateContext(state, NULL, return_type, arg_types, 0, true);
}
void UdfTestHarness::SetConstantArgs(
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f982c3f7/be/src/udf/udf-test-harness.h
----------------------------------------------------------------------
diff --git a/be/src/udf/udf-test-harness.h b/be/src/udf/udf-test-harness.h
index 67ba8fc..668a44d 100644
--- a/be/src/udf/udf-test-harness.h
+++ b/be/src/udf/udf-test-harness.h
@@ -24,6 +24,7 @@
#include <boost/function.hpp>
#include <boost/scoped_ptr.hpp>
+#include "runtime/runtime-state.h"
#include "udf/udf.h"
#include "udf/udf-debug.h"
@@ -36,7 +37,8 @@ class UdfTestHarness {
/// argument of the UDF not including the FunctionContext*. The caller is responsible
/// for calling delete on it. This context has additional debugging validation enabled.
static FunctionContext* CreateTestContext(const FunctionContext::TypeDesc& return_type,
- const std::vector<FunctionContext::TypeDesc>& arg_types);
+ const std::vector<FunctionContext::TypeDesc>& arg_types,
+ impala::RuntimeState* state = nullptr);
/// Use with test contexts to test use of IsArgConstant() and GetConstantArg().
/// constant_args should contain an AnyVal* for each argument of the UDF not including
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f982c3f7/be/src/udf/udf.cc
----------------------------------------------------------------------
diff --git a/be/src/udf/udf.cc b/be/src/udf/udf.cc
index cf1fa0f..c2a5270 100644
--- a/be/src/udf/udf.cc
+++ b/be/src/udf/udf.cc
@@ -82,7 +82,12 @@ class RuntimeState {
assert(false);
}
- bool abort_on_error() {
+ bool abort_on_error() const {
+ assert(false);
+ return false;
+ }
+
+ bool decimal_v2() const {
assert(false);
return false;
}
@@ -99,6 +104,7 @@ class RuntimeState {
}
#else
+#include "exprs/anyval-util.h"
#include "runtime/free-pool.h"
#include "runtime/mem-tracker.h"
#include "runtime/runtime-state.h"
@@ -114,6 +120,8 @@ using std::pair;
const int FunctionContextImpl::VARARGS_BUFFER_ALIGNMENT;
const char* FunctionContextImpl::LLVM_FUNCTIONCONTEXT_NAME =
"class.impala_udf::FunctionContext";
+const char* FunctionContextImpl::GET_CONST_FN_ATTR_SYMBOL =
+ "_ZN6impala19FunctionContextImpl14GetConstFnAttrENS0_11ConstFnAttrEi";
static const int MAX_WARNINGS = 1000;
@@ -501,3 +509,53 @@ const FunctionContext::TypeDesc* FunctionContext::GetArgType(int arg_idx) const
if (arg_idx < 0 || arg_idx >= impl_->arg_types_.size()) return NULL;
return &impl_->arg_types_[arg_idx];
}
+
+static int GetTypeByteSize(const FunctionContext::TypeDesc& type) {
+#if defined(IMPALA_UDF_SDK_BUILD) && IMPALA_UDF_SDK_BUILD
+ return 0;
+#else
+ return AnyValUtil::TypeDescToColumnType(type).GetByteSize();
+#endif
+}
+
+int FunctionContextImpl::GetConstFnAttr(FunctionContextImpl::ConstFnAttr t, int i) {
+ return GetConstFnAttr(state_, return_type_, arg_types_, t, i);
+}
+
+int FunctionContextImpl::GetConstFnAttr(const RuntimeState* state,
+ const FunctionContext::TypeDesc& return_type,
+ const vector<FunctionContext::TypeDesc>& arg_types,
+ ConstFnAttr t, int i) {
+ switch (t) {
+ case RETURN_TYPE_SIZE:
+ assert(i == -1);
+ return GetTypeByteSize(return_type);
+ case RETURN_TYPE_PRECISION:
+ assert(i == -1);
+ assert(return_type.type == FunctionContext::TYPE_DECIMAL);
+ return return_type.precision;
+ case RETURN_TYPE_SCALE:
+ assert(i == -1);
+ assert(return_type.type == FunctionContext::TYPE_DECIMAL);
+ return return_type.scale;
+ case ARG_TYPE_SIZE:
+ assert(i >= 0);
+ assert(i < arg_types.size());
+ return GetTypeByteSize(arg_types[i]);
+ case ARG_TYPE_PRECISION:
+ assert(i >= 0);
+ assert(i < arg_types.size());
+ assert(arg_types[i].type == FunctionContext::TYPE_DECIMAL);
+ return arg_types[i].precision;
+ case ARG_TYPE_SCALE:
+ assert(i >= 0);
+ assert(i < arg_types.size());
+ assert(arg_types[i].type == FunctionContext::TYPE_DECIMAL);
+ return arg_types[i].scale;
+ case DECIMAL_V2:
+ return state->decimal_v2();
+ default:
+ assert(false);
+ return -1;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f982c3f7/testdata/workloads/functional-query/queries/QueryTest/decimal.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/decimal.test b/testdata/workloads/functional-query/queries/QueryTest/decimal.test
index 6e7cbae..4a1316b 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/decimal.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/decimal.test
@@ -449,3 +449,47 @@ NULL,0.0000
---- TYPES
DECIMAL, DECIMAL
====
+---- QUERY
+# Test casting behavior without decimal_v2 query option set.
+set decimal_v2=false;
+select cast(d3 as decimal(20, 3)) from functional.decimal_tbl;
+---- RESULTS
+1.234
+12.345
+123.456
+1234.567
+12345.678
+---- TYPES
+DECIMAL
+====
+---- QUERY
+# Test casting behavior with decimal_v2 query option set.
+set decimal_v2=true;
+select cast(d3 as decimal(20, 3)) from functional.decimal_tbl;
+---- RESULTS
+1.235
+12.346
+123.457
+1234.568
+12345.679
+---- TYPES
+DECIMAL
+====
+---- QUERY
+# Test casting behavior without decimal_v2 query option set.
+set decimal_v2=false;
+select sum(cast(d3 as DECIMAL(20,2)) + cast(d5 as DECIMAL(20,4))) from functional.decimal_tbl;
+---- RESULTS
+26078.2788
+---- TYPES
+DECIMAL
+====
+---- QUERY
+# Test casting behavior with decimal_v2 query option set.
+set decimal_v2=true;
+select sum(cast(d3 as DECIMAL(20,2)) + cast(d5 as DECIMAL(20,4))) from functional.decimal_tbl;
+---- RESULTS
+26078.3189
+---- TYPES
+DECIMAL
+====
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f982c3f7/tests/query_test/test_decimal_casting.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_decimal_casting.py b/tests/query_test/test_decimal_casting.py
index b23e790..fac49c5 100644
--- a/tests/query_test/test_decimal_casting.py
+++ b/tests/query_test/test_decimal_casting.py
@@ -23,6 +23,7 @@ from metacomm.combinatorics.all_pairs2 import all_pairs2 as all_pairs
from random import randint
from tests.common.impala_test_suite import ImpalaTestSuite
+from tests.common.test_dimensions import create_exec_option_dimension_from_dict
from tests.common.test_vector import ImpalaTestDimension, ImpalaTestMatrix
class TestDecimalCasting(ImpalaTestSuite):
@@ -42,8 +43,8 @@ class TestDecimalCasting(ImpalaTestSuite):
# mimics test_vectors.py and takes a subset of all decimal types
'pairwise' : all_pairs([(p, s) for p in xrange(1, 39) for s in xrange(0, p + 1)])
}
- # We can cast for numerrics or string types.
- CAST_FROM = ['string', 'number']
+ # We can cast for numerics, string or decimal types.
+ CAST_FROM = ['string', 'number', 'decimal']
# Set the default precisin to 38 to operate on decimal values.
getcontext().prec = 38
# Represents a 0 in decimal
@@ -60,6 +61,8 @@ class TestDecimalCasting(ImpalaTestSuite):
*TestDecimalCasting.DECIMAL_TYPES_MAP[cls.exploration_strategy()]))
cls.ImpalaTestMatrix.add_dimension(
ImpalaTestDimension('cast_from', *TestDecimalCasting.CAST_FROM))
+ cls.ImpalaTestMatrix.add_dimension(create_exec_option_dimension_from_dict(
+ {'decimal_v2': ['false']}))
cls.iterations = 1
def setup_method(self, method):
@@ -79,15 +82,21 @@ class TestDecimalCasting(ImpalaTestSuite):
assert actual == expected, "Cast: {0}, Expected: {1}, Actual: {2}".format(cast,\
expected, actual)
- def _normalize_cast_expr(self, decimal_val, scale, from_string=False):
+ def _normalize_cast_expr(self, decimal_val, scale, cast_from):
"""Convert the decimal value to a string litetal to avoid overflow.
If an integer literal is greater than the max bigint supported by Impala, it
overflows. This methods replaces it with a string literal.
"""
- if (scale == 0 and abs(decimal_val) > self.max_bigint) or from_string:
- return "select cast('{0}' as Decimal({1}, {2}))"
- return "select cast({0} as Decimal({1}, {2}))"
+ # Decimal({1},{2}) is the target type to cast to. If casting from decimal,
+ # Decimal({3},{4}) is the intermediate type to cast {0} to.
+ if (scale == 0 and abs(decimal_val) > self.max_bigint) or (cast_from == 'string'):
+ return "select cast('{0}' as Decimal({1},{2}))"
+ elif cast_from == 'decimal':
+ base_cast = "cast('{0}' as Decimal({3},{4}))"
+ return "select cast(" + base_cast + " as Decimal({1},{2}))"
+ else:
+ return "select cast({0} as Decimal({1},{2}))"
def test_min_max_zero_null(self, vector):
"""Sanity test at limits.
@@ -98,20 +107,21 @@ class TestDecimalCasting(ImpalaTestSuite):
- NULL is expressible in all decimal types
"""
precision, scale = vector.get_value('decimal_type')
- from_string = vector.get_value('cast_from') == 'string'
dec_max = Decimal('{0}.{1}'.format('9' * (precision - scale), '9' * scale))
# Multiplying large values eith -1 can produce an overflow.
dec_min = Decimal('-{0}'.format(str(dec_max)))
- cast = self._normalize_cast_expr(dec_max, scale, from_string=from_string)
+ cast = self._normalize_cast_expr(dec_max, scale, vector.get_value('cast_from'))
# Test max
- res = Decimal(self.execute_scalar(cast.format(dec_max, precision, scale)))
+ res = Decimal(self.execute_scalar(\
+ cast.format(dec_max, precision, scale, precision, scale)))
self._assert_decimal_result(cast, res, dec_max)
# Test Min
- res = Decimal(self.execute_scalar(cast.format(dec_min, precision, scale)))
+ res = Decimal(self.execute_scalar(\
+ cast.format(dec_min, precision, scale, precision, scale)))
self._assert_decimal_result(cast, res, dec_min)
# Test zero
- res = Decimal(self.execute_scalar(cast.format(TestDecimalCasting.DECIMAL_ZERO,
- precision, scale)))
+ res = Decimal(self.execute_scalar(\
+ cast.format(TestDecimalCasting.DECIMAL_ZERO, precision, scale, precision, scale)))
self._assert_decimal_result(cast, res, TestDecimalCasting.DECIMAL_ZERO)
# Test NULL
null_cast = "select cast(NULL as Decimal({0}, {1}))".format(precision, scale)
@@ -122,11 +132,12 @@ class TestDecimalCasting(ImpalaTestSuite):
"""Test to verify that an exact representation of the desired Decimal type is
maintained."""
precision, scale = vector.get_value('decimal_type')
- from_string = vector.get_value('cast_from') == 'string'
+ if vector.get_value('cast_from') == 'decimal':
+ pytest.skip("Casting between the same decimal type isn't interesting")
for i in xrange(self.iterations):
val = self._gen_decimal_val(precision, scale)
- cast = self._normalize_cast_expr(val, scale, from_string=from_string)\
- .format(val, precision, scale)
+ cast = self._normalize_cast_expr(val, scale, vector.get_value('cast_from'))\
+ .format(val, precision, scale, precision, scale)
res = Decimal(self.execute_scalar(cast))
self._assert_decimal_result(cast, res, val)
@@ -134,12 +145,12 @@ class TestDecimalCasting(ImpalaTestSuite):
"""Test to verify that we always return NULL when trying to cast a number with greater
precision that its intended decimal type"""
precision, scale = vector.get_value('decimal_type')
- from_string = vector.get_value('cast_from') == 'string'
for i in xrange(self.iterations):
# Generate a decimal with a larger precision than the one we're casting to.
- val = self._gen_decimal_val(randint(precision + 1, 39), scale)
- cast = self._normalize_cast_expr(val, scale, from_string=from_string)\
- .format(val, precision, scale)
+ from_precision = randint(precision + 1, 39)
+ val = self._gen_decimal_val(from_precision, scale)
+ cast = self._normalize_cast_expr(val, scale, vector.get_value('cast_from'))\
+ .format(val, precision, scale, min(38, from_precision), scale)
res = self.execute_scalar(cast)
self._assert_decimal_result(cast, res, 'NULL')
@@ -148,16 +159,17 @@ class TestDecimalCasting(ImpalaTestSuite):
than the target decimal type (with no change in precision).
"""
precision, scale = vector.get_value('decimal_type')
- from_string = vector.get_value('cast_from') == 'string'
+ is_decimal_v2 = vector.get_value('exec_option')['decimal_v2'] == 'true'
if precision == scale:
pytest.skip("Cannot underflow scale when precision and scale are equal")
for i in xrange(self.iterations):
- new_scale = randint(scale + 1, precision)
- val = self._gen_decimal_val(precision, randint(new_scale, precision))
- # We don't need to normalize the cast expr because scale will never be zero
- cast = self._normalize_cast_expr(val, scale, from_string=from_string)\
- .format(val, precision, scale)
- res = Decimal(self.execute_scalar(cast))
- # Truncate the decimal value to its target scale with quantize.
- self._assert_decimal_result(cast, res, val.quantize(Decimal('0e-%s' % scale),
- rounding=ROUND_DOWN))
+ from_scale = randint(scale + 1, precision)
+ val = self._gen_decimal_val(precision, from_scale)
+ cast = self._normalize_cast_expr(val, scale, vector.get_value('cast_from'))\
+ .format(val, precision, scale, precision, from_scale)
+ res = Decimal(self.execute_scalar(cast, vector.get_value('exec_option')))
+ if is_decimal_v2:
+ expected_val = val.quantize(Decimal('0e-%s' % scale))
+ else:
+ expected_val = val.quantize(Decimal('0e-%s' % scale), rounding=ROUND_DOWN)
+ self._assert_decimal_result(cast, res, expected_val)