You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by ar...@apache.org on 2018/05/18 18:43:19 UTC
[drill] 03/08: DRILL-6421: Refactor DecimalUtility and
CoreDecimalUtility classes
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit bdbd93fc726224ec45d06f09434c7406e444242e
Author: Volodymyr Vysotskyi <vv...@gmail.com>
AuthorDate: Wed May 16 21:34:56 2018 +0300
DRILL-6421: Refactor DecimalUtility and CoreDecimalUtility classes
closes #1267
---
.../java/org/apache/drill/common/types/Types.java | 106 ++--
.../drill/common/util/CoreDecimalUtility.java | 55 --
.../drill/exec/store/hive/HiveUtilities.java | 8 +-
.../templates/Decimal/CastVarDecimalDecimal.java | 6 +-
.../exec/expr/ExpressionTreeMaterializer.java | 43 +-
.../drill/exec/physical/impl/join/JoinUtils.java | 6 +-
.../drill/exec/planner/sql/TypeInferenceUtils.java | 4 +-
.../exec/store/parquet/ParquetRecordWriter.java | 3 +-
.../columnreaders/ParquetToDrillTypeConverter.java | 4 +-
.../main/codegen/templates/ColumnAccessors.java | 14 +-
.../main/codegen/templates/FixedValueVectors.java | 2 +-
.../org/apache/drill/exec/util/DecimalUtility.java | 672 ++++-----------------
.../drill/exec/vector/ValueHolderHelper.java | 8 +-
.../drill/common/expression/ValueExpressions.java | 21 +-
14 files changed, 185 insertions(+), 767 deletions(-)
diff --git a/common/src/main/java/org/apache/drill/common/types/Types.java b/common/src/main/java/org/apache/drill/common/types/Types.java
index 5c6e7f3..e66a340 100644
--- a/common/src/main/java/org/apache/drill/common/types/Types.java
+++ b/common/src/main/java/org/apache/drill/common/types/Types.java
@@ -27,12 +27,11 @@ import org.apache.drill.common.exceptions.DrillRuntimeException;
import org.apache.drill.common.types.TypeProtos.DataMode;
import org.apache.drill.common.types.TypeProtos.MajorType;
import org.apache.drill.common.types.TypeProtos.MinorType;
-import org.apache.drill.common.util.CoreDecimalUtility;
import com.google.protobuf.TextFormat;
+@SuppressWarnings("WeakerAccess")
public class Types {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(Types.class);
public static final int MAX_VARCHAR_LENGTH = 65535;
public static final int UNDEFINED = 0;
@@ -47,10 +46,6 @@ public class Types {
return toType.getMinorType() == MinorType.UNION;
}
- public enum Comparability {
- UNKNOWN, NONE, EQUAL, ORDERED
- }
-
public static boolean isComplex(final MajorType type) {
switch(type.getMinorType()) {
case LIST:
@@ -90,6 +85,37 @@ public class Types {
case UINT4:
case UINT8:
return true;
+ default:
+ return false;
+ }
+ }
+
+ /**
+ * Returns true if specified type is decimal data type.
+ *
+ * @param type type to check
+ * @return true if specified type is decimal data type.
+ */
+ public static boolean isDecimalType(MajorType type) {
+ return isDecimalType(type.getMinorType());
+ }
+
+ /**
+ * Returns true if specified type is decimal data type.
+ *
+ * @param minorType type to check
+ * @return true if specified type is decimal data type.
+ */
+ public static boolean isDecimalType(MinorType minorType) {
+ switch(minorType) {
+ case VARDECIMAL:
+ case DECIMAL38SPARSE:
+ case DECIMAL38DENSE:
+ case DECIMAL28SPARSE:
+ case DECIMAL28DENSE:
+ case DECIMAL18:
+ case DECIMAL9:
+ return true;
default:
return false;
}
@@ -182,11 +208,9 @@ public class Types {
* Extend decimal type with precision and scale.
*
* @param type major type
- * @param typeName type converted to a string
* @return type name augmented with precision and scale,
* if type is a decimal
*/
-
public static String getExtendedSqlTypeName(MajorType type) {
String typeName = getSqlTypeName(type);
@@ -266,7 +290,7 @@ public class Types {
/**
* Reports whether given RPC-level type is a signed type (per semantics of
- * {@link ResultSetMetaData#isSigned(int)}).
+ * {@link java.sql.ResultSetMetaData#isSigned(int)}).
*/
public static boolean isJdbcSignedType( final MajorType type ) {
final boolean isSigned;
@@ -477,41 +501,6 @@ public class Types {
}
}
- public static boolean isBytesScalarType(final MajorType type) {
- if (type.getMode() == REPEATED) {
- return false;
- }
- switch(type.getMinorType()) {
- case FIXEDBINARY:
- case VARBINARY:
- return true;
- default:
- return false;
- }
- }
-
- public static Comparability getComparability(final MajorType type) {
- if (type.getMode() == REPEATED) {
- return Comparability.NONE;
- }
- if (type.getMinorType() == MinorType.LATE) {
- return Comparability.UNKNOWN;
- }
-
- switch(type.getMinorType()) {
- case LATE:
- return Comparability.UNKNOWN;
- case MAP:
- return Comparability.NONE;
- case BIT:
- return Comparability.EQUAL;
- default:
- return Comparability.ORDERED;
- }
-
- }
-
-
public static boolean softEquals(final MajorType a, final MajorType b, final boolean allowNullSwap) {
if (a.getMinorType() != b.getMinorType()) {
return false;
@@ -532,10 +521,6 @@ public class Types {
return a.getMode() == b.getMode();
}
- public static boolean isLateBind(final MajorType type) {
- return type.getMinorType() == MinorType.LATE;
- }
-
public static boolean isUntypedNull(final MajorType type) {
return type.getMinorType() == MinorType.NULL;
}
@@ -572,19 +557,6 @@ public class Types {
return MajorType.newBuilder().setMode(DataMode.OPTIONAL).setMinorType(type).build();
}
- public static MajorType overrideMinorType(final MajorType originalMajorType, final MinorType overrideMinorType) {
- switch (originalMajorType.getMode()) {
- case REPEATED:
- return repeated(overrideMinorType);
- case OPTIONAL:
- return optional(overrideMinorType);
- case REQUIRED:
- return required(overrideMinorType);
- default:
- throw new UnsupportedOperationException();
- }
- }
-
public static MajorType overrideMode(final MajorType originalMajorType, final DataMode overrideMode) {
return withScaleAndPrecision(originalMajorType.getMinorType(), overrideMode, originalMajorType.getScale(), originalMajorType.getPrecision());
}
@@ -765,7 +737,7 @@ public class Types {
public static MajorType.Builder calculateTypePrecisionAndScale(MajorType leftType, MajorType rightType, MajorType.Builder typeBuilder) {
if (leftType.getMinorType().equals(rightType.getMinorType())) {
boolean isScalarString = Types.isScalarStringType(leftType) && Types.isScalarStringType(rightType);
- boolean isDecimal = CoreDecimalUtility.isDecimalType(leftType);
+ boolean isDecimal = isDecimalType(leftType);
if ((isScalarString || isDecimal) && leftType.hasPrecision() && rightType.hasPrecision()) {
typeBuilder.setPrecision(Math.max(leftType.getPrecision(), rightType.getPrecision()));
@@ -778,10 +750,6 @@ public class Types {
return typeBuilder;
}
- public static boolean isLaterType(MajorType type) {
- return type.getMinorType() == MinorType.LATE;
- }
-
public static boolean isEquivalent(MajorType type1, MajorType type2) {
// Requires full type equality, including fields such as precision and scale.
@@ -815,10 +783,8 @@ public class Types {
// Now it gets slow because subtype lists are not ordered.
- List<MinorType> copy1 = new ArrayList<>();
- List<MinorType> copy2 = new ArrayList<>();
- copy1.addAll(subtypes1);
- copy2.addAll(subtypes2);
+ List<MinorType> copy1 = new ArrayList<>(subtypes1);
+ List<MinorType> copy2 = new ArrayList<>(subtypes2);
Collections.sort(copy1);
Collections.sort(copy2);
return copy1.equals(copy2);
diff --git a/common/src/main/java/org/apache/drill/common/util/CoreDecimalUtility.java b/common/src/main/java/org/apache/drill/common/util/CoreDecimalUtility.java
deleted file mode 100644
index 0e8a1c7..0000000
--- a/common/src/main/java/org/apache/drill/common/util/CoreDecimalUtility.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.util;
-
-import java.math.BigDecimal;
-
-import org.apache.drill.common.types.TypeProtos;
-
-public class CoreDecimalUtility {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(CoreDecimalUtility.class);
-
- public static long getDecimal18FromBigDecimal(BigDecimal input, int scale, int precision) {
- // Truncate or pad to set the input to the correct scale
- input = input.setScale(scale, BigDecimal.ROUND_HALF_UP);
-
- return (input.unscaledValue().longValue());
- }
-
- public static int getDecimal9FromBigDecimal(BigDecimal input, int scale, int precision) {
- // Truncate/ or pad to set the input to the correct scale
- input = input.setScale(scale, BigDecimal.ROUND_HALF_UP);
-
- return (input.unscaledValue().intValue());
- }
-
- /*
- * Helper function to detect if the given data type is Decimal
- */
- public static boolean isDecimalType(TypeProtos.MajorType type) {
- return isDecimalType(type.getMinorType());
- }
-
- public static boolean isDecimalType(TypeProtos.MinorType minorType) {
- return minorType == TypeProtos.MinorType.VARDECIMAL ||
- minorType == TypeProtos.MinorType.DECIMAL9 ||
- minorType == TypeProtos.MinorType.DECIMAL18 ||
- minorType == TypeProtos.MinorType.DECIMAL28SPARSE ||
- minorType == TypeProtos.MinorType.DECIMAL38SPARSE;
- }
-}
diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveUtilities.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveUtilities.java
index 8f34584..5279f28 100644
--- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveUtilities.java
+++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveUtilities.java
@@ -233,7 +233,7 @@ public class HiveUtilities {
final Decimal9Holder holder = new Decimal9Holder();
holder.scale = v.getField().getScale();
holder.precision = v.getField().getPrecision();
- holder.value = DecimalUtility.getDecimal9FromBigDecimal(value, holder.scale, holder.precision);
+ holder.value = DecimalUtility.getDecimal9FromBigDecimal(value, holder.scale);
for (int i = start; i < end; i++) {
v.getMutator().setSafe(i, holder);
}
@@ -246,7 +246,7 @@ public class HiveUtilities {
final Decimal18Holder holder = new Decimal18Holder();
holder.scale = v.getField().getScale();
holder.precision = v.getField().getPrecision();
- holder.value = DecimalUtility.getDecimal18FromBigDecimal(value, holder.scale, holder.precision);
+ holder.value = DecimalUtility.getDecimal18FromBigDecimal(value, holder.scale);
for (int i = start; i < end; i++) {
v.getMutator().setSafe(i, holder);
}
@@ -266,7 +266,7 @@ public class HiveUtilities {
holder.precision = v.getField().getPrecision();
holder.buffer = managedBuffer;
holder.start = 0;
- DecimalUtility.getSparseFromBigDecimal(value, holder.buffer, 0, holder.scale, holder.precision,
+ DecimalUtility.getSparseFromBigDecimal(value, holder.buffer, 0, holder.scale,
Decimal28SparseHolder.nDecimalDigits);
for (int i = start; i < end; i++) {
v.getMutator().setSafe(i, holder);
@@ -286,7 +286,7 @@ public class HiveUtilities {
holder.precision = v.getField().getPrecision();
holder.buffer = managedBuffer;
holder.start = 0;
- DecimalUtility.getSparseFromBigDecimal(value, holder.buffer, 0, holder.scale, holder.precision,
+ DecimalUtility.getSparseFromBigDecimal(value, holder.buffer, 0, holder.scale,
Decimal38SparseHolder.nDecimalDigits);
for (int i = start; i < end; i++) {
v.getMutator().setSafe(i, holder);
diff --git a/exec/java-exec/src/main/codegen/templates/Decimal/CastVarDecimalDecimal.java b/exec/java-exec/src/main/codegen/templates/Decimal/CastVarDecimalDecimal.java
index 8ef2bf9..828d486 100644
--- a/exec/java-exec/src/main/codegen/templates/Decimal/CastVarDecimalDecimal.java
+++ b/exec/java-exec/src/main/codegen/templates/Decimal/CastVarDecimalDecimal.java
@@ -77,14 +77,14 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc {
.round(new java.math.MathContext(precision.value, java.math.RoundingMode.HALF_UP));
<#if type.to.endsWith("Decimal9")>
- out.value = org.apache.drill.exec.util.DecimalUtility.getDecimal9FromBigDecimal(bd, out.scale, out.precision);
+ out.value = org.apache.drill.exec.util.DecimalUtility.getDecimal9FromBigDecimal(bd, out.scale);
<#elseif type.to.endsWith("Decimal18")>
- out.value = org.apache.drill.exec.util.DecimalUtility.getDecimal18FromBigDecimal(bd, out.scale, out.precision);
+ out.value = org.apache.drill.exec.util.DecimalUtility.getDecimal18FromBigDecimal(bd, out.scale);
<#elseif type.to.endsWith("Sparse")>
out.start = 0;
out.buffer = buffer;
org.apache.drill.exec.util.DecimalUtility
- .getSparseFromBigDecimal(bd, out.buffer, out.start, out.scale, out.precision, out.nDecimalDigits);
+ .getSparseFromBigDecimal(bd, out.buffer, out.start, out.scale, out.nDecimalDigits);
</#if>
}
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
index ff38e0c..283953a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
@@ -21,15 +21,12 @@ import java.util.ArrayDeque;
import java.util.Arrays;
import java.util.Collections;
import java.util.Deque;
-import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Queue;
-import java.util.Set;
import com.google.common.base.Preconditions;
import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
import org.apache.drill.common.exceptions.DrillRuntimeException;
import org.apache.drill.common.expression.BooleanOperator;
import org.apache.drill.common.expression.CastExpression;
@@ -72,7 +69,6 @@ import org.apache.drill.common.types.TypeProtos.DataMode;
import org.apache.drill.common.types.TypeProtos.MajorType;
import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.common.types.Types;
-import org.apache.drill.common.util.CoreDecimalUtility;
import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
import org.apache.drill.exec.expr.fn.AbstractFuncHolder;
@@ -121,8 +117,7 @@ public class ExpressionTreeMaterializer {
public static LogicalExpression materializeFilterExpr(LogicalExpression expr, Map<SchemaPath, ColumnStatistics> fieldTypes, ErrorCollector errorCollector, FunctionLookupContext functionLookupContext) {
final FilterMaterializeVisitor filterMaterializeVisitor = new FilterMaterializeVisitor(fieldTypes, errorCollector);
- LogicalExpression out = expr.accept(filterMaterializeVisitor, functionLookupContext);
- return out;
+ return expr.accept(filterMaterializeVisitor, functionLookupContext);
}
/**
@@ -211,7 +206,7 @@ public class ExpressionTreeMaterializer {
return fromExpr;
}
- if (CoreDecimalUtility.isDecimalType(toType)) {
+ if (Types.isDecimalType(toType)) {
// Add the scale and precision to the arguments of the implicit cast
castArgs.add(new ValueExpressions.IntExpression(toType.getPrecision(), null));
castArgs.add(new ValueExpressions.IntExpression(toType.getScale(), null));
@@ -338,10 +333,6 @@ public class ExpressionTreeMaterializer {
* Once this code is more well tested, we will probably remove this flag
*/
private final boolean unionTypeEnabled;
- /**
- * Avoid revisiting portions of the tree that have already been materialized
- */
- private Set<LogicalExpression> materializedExpressions = Sets.newIdentityHashSet();
public AbstractMaterializeVisitor(ErrorCollector errorCollector, boolean allowComplexWriter, boolean unionTypeEnabled) {
this.errorCollector = errorCollector;
@@ -430,7 +421,7 @@ public class ExpressionTreeMaterializer {
argsWithCast.add(currentArg);
} else {
//Case 3: insert cast if param type is different from arg type.
- if (CoreDecimalUtility.isDecimalType(parmType)) {
+ if (Types.isDecimalType(parmType)) {
// We are implicitly promoting a decimal type, set the required scale and precision
parmType = MajorType.newBuilder().setMinorType(parmType.getMinorType()).setMode(parmType.getMode()).
setScale(currentArg.getMajorType().getScale()).setPrecision(computePrecision(currentArg)).build();
@@ -468,7 +459,7 @@ public class ExpressionTreeMaterializer {
extArgsWithCast.add(currentArg);
} else {
// Insert cast if param type is different from arg type.
- if (CoreDecimalUtility.isDecimalType(parmType)) {
+ if (Types.isDecimalType(parmType)) {
// We are implicitly promoting a decimal type, set the required scale and precision
parmType = MajorType.newBuilder().setMinorType(parmType.getMinorType()).setMode(parmType.getMode()).
setScale(currentArg.getMajorType().getScale()).setPrecision(computePrecision(currentArg)).build();
@@ -488,17 +479,6 @@ public class ExpressionTreeMaterializer {
return NullExpression.INSTANCE;
}
- private static final Set<String> UNION_FUNCTIONS;
-
- static {
- UNION_FUNCTIONS = new HashSet<>();
- for (MinorType t : MinorType.values()) {
- UNION_FUNCTIONS.add("assert_" + t.name().toLowerCase());
- UNION_FUNCTIONS.add("is_" + t.name().toLowerCase());
- }
- UNION_FUNCTIONS.add("typeof");
- }
-
private boolean hasUnionInput(FunctionCall call) {
for (LogicalExpression arg : call.args) {
if (arg.getMajorType().getMinorType() == MinorType.UNION) {
@@ -583,8 +563,7 @@ public class ExpressionTreeMaterializer {
QuotedString msg = new QuotedString(message, message.length(), ExpressionPosition.UNKNOWN);
List<LogicalExpression> args = Lists.newArrayList();
args.add(msg);
- FunctionCall call = new FunctionCall(ExceptionFunction.EXCEPTION_FUNCTION_NAME, args, ExpressionPosition.UNKNOWN);
- return call;
+ return new FunctionCall(ExceptionFunction.EXCEPTION_FUNCTION_NAME, args, ExpressionPosition.UNKNOWN);
}
/**
@@ -602,7 +581,6 @@ public class ExpressionTreeMaterializer {
return getExceptionFunction("Unable to cast union to " + type);
}
String castFuncName = String.format("assert_%s", type.toString());
- Collections.singletonList(arg);
return new FunctionCall(castFuncName, Collections.singletonList(arg), ExpressionPosition.UNKNOWN);
}
@@ -634,7 +612,6 @@ public class ExpressionTreeMaterializer {
if (unionTypeEnabled) {
if (thenType != elseType && !(thenType == MinorType.NULL || elseType == MinorType.NULL)) {
- MinorType leastRestrictive = MinorType.UNION;
MajorType.Builder builder = MajorType.newBuilder().setMinorType(MinorType.UNION).setMode(DataMode.OPTIONAL);
if (thenType == MinorType.UNION) {
for (MinorType subType : conditions.expression.getMajorType().getSubTypeList()) {
@@ -726,8 +703,12 @@ public class ExpressionTreeMaterializer {
}
}
- LogicalExpression expr = validateNewExpr(IfExpression.newBuilder().setElse(newElseExpr).setIfCondition(conditions).setOutputType(outputType).build());
- return expr;
+ return validateNewExpr(
+ IfExpression.newBuilder()
+ .setElse(newElseExpr)
+ .setIfCondition(conditions)
+ .setOutputType(outputType)
+ .build());
}
private LogicalExpression getConvertToNullableExpr(List<LogicalExpression> args, MinorType minorType,
@@ -880,7 +861,7 @@ public class ExpressionTreeMaterializer {
List<LogicalExpression> newArgs = Lists.newArrayList();
newArgs.add(input); //input_expr
- if (CoreDecimalUtility.isDecimalType(type)) {
+ if (Types.isDecimalType(type)) {
newArgs.add(new ValueExpressions.IntExpression(type.getPrecision(), null));
newArgs.add(new ValueExpressions.IntExpression(type.getScale(), null));
} else if (!Types.isFixedWidthType(type)) { //VarLen type
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinUtils.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinUtils.java
index b6604d2..e4dab91 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinUtils.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinUtils.java
@@ -25,6 +25,7 @@ import org.apache.drill.common.logical.data.JoinCondition;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.plan.RelOptUtil;
import org.apache.calcite.plan.volcano.RelSubset;
+import org.apache.drill.common.types.Types;
import org.apache.drill.exec.physical.impl.common.Comparator;
import org.apache.drill.exec.planner.logical.DrillAggregateRel;
import org.apache.drill.common.exceptions.DrillRuntimeException;
@@ -36,7 +37,6 @@ import org.apache.drill.exec.expr.ExpressionTreeMaterializer;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.record.VectorAccessible;
import org.apache.drill.exec.resolver.TypeCastRules;
-import org.apache.drill.exec.util.DecimalUtility;
import java.util.LinkedList;
import java.util.List;
@@ -123,8 +123,8 @@ public class JoinUtils {
// allow implicit cast if both the input types are numeric and any of them is non-decimal
// or both of them are decimal
if (TypeCastRules.isNumericType(input1) && TypeCastRules.isNumericType(input2)
- && ((!DecimalUtility.isDecimalType(input1) && !DecimalUtility.isDecimalType(input2))
- || DecimalUtility.isDecimalType(input1) && DecimalUtility.isDecimalType(input2))) {
+ && ((!Types.isDecimalType(input1) && !Types.isDecimalType(input2))
+ || Types.isDecimalType(input1) && Types.isDecimalType(input2))) {
return true;
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/TypeInferenceUtils.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/TypeInferenceUtils.java
index 529546f..4b5ac7b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/TypeInferenceUtils.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/TypeInferenceUtils.java
@@ -46,7 +46,6 @@ import org.apache.drill.common.expression.MajorTypeInLogicalExpression;
import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.common.types.Types;
-import org.apache.drill.common.util.CoreDecimalUtility;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
import org.apache.drill.exec.expr.fn.DrillFuncHolder;
import org.apache.drill.exec.planner.types.DrillRelDataTypeSystem;
@@ -57,6 +56,7 @@ import org.apache.drill.exec.resolver.TypeCastRules;
import java.util.List;
import java.util.Set;
+@SuppressWarnings("WeakerAccess")
public class TypeInferenceUtils {
private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TypeInferenceUtils.class);
@@ -906,7 +906,7 @@ public class TypeInferenceUtils {
.setMode(dataMode)
.setMinorType(minorType);
- if (CoreDecimalUtility.isDecimalType(minorType)) {
+ if (Types.isDecimalType(minorType)) {
builder
.setScale(type.getScale())
.setPrecision(type.getPrecision());
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordWriter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordWriter.java
index ec78619..114bae4 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordWriter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordWriter.java
@@ -30,6 +30,7 @@ import org.apache.drill.common.exceptions.DrillRuntimeException;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.types.TypeProtos.DataMode;
import org.apache.drill.common.types.TypeProtos.MinorType;
+import org.apache.drill.common.types.Types;
import org.apache.drill.common.util.DrillVersionInfo;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.exception.OutOfMemoryException;
@@ -251,7 +252,7 @@ public class ParquetRecordWriter extends ParquetOutputRecordWriter {
String name = field.getName();
int length = ParquetTypeHelper.getLengthForMinorType(minorType);
PrimitiveTypeName primitiveTypeName = ParquetTypeHelper.getPrimitiveTypeNameForMinorType(minorType);
- if (DecimalUtility.isDecimalType(minorType)) {
+ if (Types.isDecimalType(minorType)) {
primitiveTypeName = logicalTypeForDecimals;
if (usePrimitiveTypesForDecimals) {
if (field.getPrecision() <= ParquetTypeHelper.getMaxPrecisionForPrimitiveType(PrimitiveTypeName.INT32)) {
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetToDrillTypeConverter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetToDrillTypeConverter.java
index 309ed89..cb688fd 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetToDrillTypeConverter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetToDrillTypeConverter.java
@@ -20,7 +20,7 @@ package org.apache.drill.exec.store.parquet.columnreaders;
import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.common.types.TypeProtos.MinorType;
-import org.apache.drill.common.util.CoreDecimalUtility;
+import org.apache.drill.common.types.Types;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.server.options.OptionManager;
import org.apache.drill.exec.store.parquet.ParquetReaderUtility;
@@ -128,7 +128,7 @@ public class ParquetToDrillTypeConverter {
MinorType minorType = getMinorType(primitiveTypeName, length, convertedType, options);
TypeProtos.MajorType.Builder typeBuilder = TypeProtos.MajorType.newBuilder().setMinorType(minorType).setMode(mode);
- if (CoreDecimalUtility.isDecimalType(minorType)) {
+ if (Types.isDecimalType(minorType)) {
int precision = schemaElement.getPrecision();
int scale = schemaElement.getScale();
diff --git a/exec/vector/src/main/codegen/templates/ColumnAccessors.java b/exec/vector/src/main/codegen/templates/ColumnAccessors.java
index accf1b2..15349a2 100644
--- a/exec/vector/src/main/codegen/templates/ColumnAccessors.java
+++ b/exec/vector/src/main/codegen/templates/ColumnAccessors.java
@@ -178,13 +178,11 @@ public class ColumnAccessors {
<#elseif drillType == "Decimal9">
return DecimalUtility.getBigDecimalFromPrimitiveTypes(
buf.getInt(${getOffset}),
- type.getScale(),
- type.getPrecision());
+ type.getScale());
<#elseif drillType == "Decimal18">
return DecimalUtility.getBigDecimalFromPrimitiveTypes(
buf.getLong(${getOffset}),
- type.getScale(),
- type.getPrecision());
+ type.getScale());
<#elseif drillType == "IntervalYear">
return DateUtilities.fromIntervalYear(
buf.getInt(${getOffset}));
@@ -296,21 +294,21 @@ public class ColumnAccessors {
<#elseif drillType == "Decimal9">
drillBuf.setInt(${putOffset},
DecimalUtility.getDecimal9FromBigDecimal(value,
- type.getScale(), type.getPrecision()));
+ type.getScale()));
<#elseif drillType == "Decimal18">
drillBuf.setLong(${putOffset},
DecimalUtility.getDecimal18FromBigDecimal(value,
- type.getScale(), type.getPrecision()));
+ type.getScale()));
<#elseif drillType == "Decimal38Sparse">
<#-- Hard to optimize this case. Just use the available tools. -->
DecimalUtility.getSparseFromBigDecimal(value, drillBuf,
${putOffset},
- type.getScale(), type.getPrecision(), 6);
+ type.getScale(), 6);
<#elseif drillType == "Decimal28Sparse">
<#-- Hard to optimize this case. Just use the available tools. -->
DecimalUtility.getSparseFromBigDecimal(value, drillBuf,
${putOffset},
- type.getScale(), type.getPrecision(), 5);
+ type.getScale(), 5);
<#elseif drillType == "IntervalYear">
drillBuf.setInt(${putOffset},
value.getYears() * 12 + value.getMonths());
diff --git a/exec/vector/src/main/codegen/templates/FixedValueVectors.java b/exec/vector/src/main/codegen/templates/FixedValueVectors.java
index 4a98c26..eb2c93d 100644
--- a/exec/vector/src/main/codegen/templates/FixedValueVectors.java
+++ b/exec/vector/src/main/codegen/templates/FixedValueVectors.java
@@ -699,7 +699,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
public void set(int index, BigDecimal value) {
DecimalUtility.getSparseFromBigDecimal(value, data, index * VALUE_WIDTH,
- field.getScale(), field.getPrecision(), ${minor.nDecimalDigits});
+ field.getScale(), ${minor.nDecimalDigits});
}
public void setSafe(int index, BigDecimal value) {
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/util/DecimalUtility.java b/exec/vector/src/main/java/org/apache/drill/exec/util/DecimalUtility.java
index a451b97..f0a06d8 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/util/DecimalUtility.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/util/DecimalUtility.java
@@ -25,142 +25,40 @@ import io.netty.buffer.UnpooledByteBufAllocator;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.math.RoundingMode;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
import org.apache.drill.common.types.TypeProtos;
-import org.apache.drill.common.util.CoreDecimalUtility;
-import org.apache.drill.exec.expr.fn.impl.ByteFunctionHelpers;
-import org.apache.drill.exec.expr.holders.Decimal38SparseHolder;
-public class DecimalUtility extends CoreDecimalUtility {
+@SuppressWarnings("WeakerAccess")
+public class DecimalUtility {
public final static int MAX_DIGITS = 9;
public final static int MAX_DIGITS_INT = 10;
public final static int MAX_DIGITS_BIGINT = 19;
public final static int DIGITS_BASE = 1000000000;
- public final static int DIGITS_MAX = 999999999;
- public final static int INTEGER_SIZE = (Integer.SIZE/8);
-
- public final static String[] decimalToString = {"",
- "0",
- "00",
- "000",
- "0000",
- "00000",
- "000000",
- "0000000",
- "00000000",
- "000000000"};
-
- public final static long[] scale_long_constants = {
- 1,
- 10,
- 100,
- 1000,
- 10000,
- 100000,
- 1000000,
- 10000000,
- 100000000,
- 1000000000,
- 10000000000l,
- 100000000000l,
- 1000000000000l,
- 10000000000000l,
- 100000000000000l,
- 1000000000000000l,
- 10000000000000000l,
- 100000000000000000l,
- 1000000000000000000l};
-
- /*
- * Simple function that returns the static precomputed
- * power of ten, instead of using Math.pow
- */
- public static long getPowerOfTen(int power) {
- assert power >= 0 && power < scale_long_constants.length;
- return scale_long_constants[(power)];
- }
-
- /*
- * Math.pow returns a double and while multiplying with large digits
- * in the decimal data type we encounter noise. So instead of multiplying
- * with Math.pow we use the static constants to perform the multiplication
- */
- public static long adjustScaleMultiply(long input, int factor) {
- int index = Math.abs(factor);
- assert index >= 0 && index < scale_long_constants.length;
- if (factor >= 0) {
- return input * scale_long_constants[index];
- } else {
- return input / scale_long_constants[index];
- }
- }
-
- public static long adjustScaleDivide(long input, int factor) {
- int index = Math.abs(factor);
- assert index >= 0 && index < scale_long_constants.length;
- if (factor >= 0) {
- return input / scale_long_constants[index];
- } else {
- return input * scale_long_constants[index];
- }
- }
+ public final static int INTEGER_SIZE = Integer.SIZE / 8;
- /* Given the number of actual digits this function returns the
+ /**
+ * Given the number of actual digits this function returns the
* number of indexes it will occupy in the array of integers
* which are stored in base 1 billion
*/
public static int roundUp(int ndigits) {
- return (ndigits + MAX_DIGITS - 1)/MAX_DIGITS;
+ return (ndigits + MAX_DIGITS - 1) / MAX_DIGITS;
}
- /** Returns a string representation of the given integer
- * If the length of the given integer is less than the
- * passed length, this function will prepend zeroes to the string
+ /**
+ * Create a BigDecimal object using the data in the DrillBuf.
+ * This function assumes that data is provided in a sparse format.
*/
- public static StringBuilder toStringWithZeroes(int number, int desiredLength) {
- String value = ((Integer) number).toString();
- int length = value.length();
-
- StringBuilder str = new StringBuilder();
- str.append(decimalToString[desiredLength - length]);
- str.append(value);
-
- return str;
- }
-
- public static StringBuilder toStringWithZeroes(long number, int desiredLength) {
- String value = ((Long) number).toString();
- int length = value.length();
-
- StringBuilder str = new StringBuilder();
-
- // Desired length can be > MAX_DIGITS
- int zeroesLength = desiredLength - length;
- while (zeroesLength > MAX_DIGITS) {
- str.append(decimalToString[MAX_DIGITS]);
- zeroesLength -= MAX_DIGITS;
- }
- str.append(decimalToString[zeroesLength]);
- str.append(value);
-
- return str;
- }
-
- public static BigDecimal getBigDecimalFromIntermediate(ByteBuf data, int startIndex, int nDecimalDigits, int scale) {
-
- // In the intermediate representation we don't pad the scale with zeroes, so set truncate = false
- return getBigDecimalFromDrillBuf(data, startIndex, nDecimalDigits, scale, false);
- }
-
public static BigDecimal getBigDecimalFromSparse(DrillBuf data, int startIndex, int nDecimalDigits, int scale) {
-
// In the sparse representation we pad the scale with zeroes for ease of arithmetic, need to truncate
return getBigDecimalFromDrillBuf(data, startIndex, nDecimalDigits, scale, true);
}
+ /**
+ * Create a BigDecimal object using the data in the DrillBuf.
+ * This function assumes that data is provided in format supported by {@link BigInteger}.
+ */
public static BigDecimal getBigDecimalFromDrillBuf(DrillBuf bytebuf, int start, int length, int scale) {
byte[] value = new byte[length];
bytebuf.getBytes(start, value, 0, length);
@@ -168,14 +66,8 @@ public class DecimalUtility extends CoreDecimalUtility {
return new BigDecimal(unscaledValue, scale);
}
- public static BigDecimal getBigDecimalFromByteBuffer(ByteBuffer bytebuf, int start, int length, int scale) {
- byte[] value = new byte[length];
- bytebuf.get(value);
- BigInteger unscaledValue = new BigInteger(value);
- return new BigDecimal(unscaledValue, scale);
- }
-
- /** Create a BigDecimal object using the data in the DrillBuf.
+ /**
+ * Create a BigDecimal object using the data in the DrillBuf.
* This function assumes that data is provided in a non-dense format
* It works on both sparse and intermediate representations.
*/
@@ -183,48 +75,47 @@ public class DecimalUtility extends CoreDecimalUtility {
boolean truncateScale) {
// For sparse decimal type we have padded zeroes at the end, strip them while converting to BigDecimal.
- int actualDigits;
+ int actualDigits = scale % MAX_DIGITS;
// Initialize the BigDecimal, first digit in the DrillBuf has the sign so mask it out
- BigInteger decimalDigits = BigInteger.valueOf((data.getInt(startIndex)) & 0x7FFFFFFF);
+ BigInteger decimalDigits = BigInteger.valueOf(data.getInt(startIndex) & 0x7FFFFFFF);
BigInteger base = BigInteger.valueOf(DIGITS_BASE);
for (int i = 1; i < nDecimalDigits; i++) {
-
- BigInteger temp = BigInteger.valueOf(data.getInt(startIndex + (i * INTEGER_SIZE)));
- decimalDigits = decimalDigits.multiply(base);
- decimalDigits = decimalDigits.add(temp);
+ BigInteger temp = BigInteger.valueOf(data.getInt(startIndex + i * INTEGER_SIZE));
+ decimalDigits = decimalDigits.multiply(base);
+ decimalDigits = decimalDigits.add(temp);
}
// Truncate any additional padding we might have added
- if (truncateScale == true && scale > 0 && (actualDigits = scale % MAX_DIGITS) != 0) {
- BigInteger truncate = BigInteger.valueOf((int)Math.pow(10, (MAX_DIGITS - actualDigits)));
- decimalDigits = decimalDigits.divide(truncate);
+ if (truncateScale && scale > 0 && actualDigits != 0) {
+ BigInteger truncate = BigInteger.valueOf((int) Math.pow(10, MAX_DIGITS - actualDigits));
+ decimalDigits = decimalDigits.divide(truncate);
}
// set the sign
if ((data.getInt(startIndex) & 0x80000000) != 0) {
- decimalDigits = decimalDigits.negate();
+ decimalDigits = decimalDigits.negate();
}
- BigDecimal decimal = new BigDecimal(decimalDigits, scale);
-
- return decimal;
+ return new BigDecimal(decimalDigits, scale);
}
- /* This function returns a BigDecimal object from the dense decimal representation.
+ /**
+ * Returns a BigDecimal object from the dense decimal representation.
* First step is to convert the dense representation into an intermediate representation
* and then invoke getBigDecimalFromDrillBuf() to get the BigDecimal object
*/
- public static BigDecimal getBigDecimalFromDense(DrillBuf data, int startIndex, int nDecimalDigits, int scale, int maxPrecision, int width) {
+ public static BigDecimal getBigDecimalFromDense(DrillBuf data, int startIndex, int nDecimalDigits,
+ int scale, int maxPrecision, int width) {
/* This method converts the dense representation to
* an intermediate representation. The intermediate
* representation has one more integer than the dense
* representation.
*/
- byte[] intermediateBytes = new byte[((nDecimalDigits + 1) * INTEGER_SIZE)];
+ byte[] intermediateBytes = new byte[(nDecimalDigits + 1) * INTEGER_SIZE];
// Start storing from the least significant byte of the first integer
int intermediateIndex = 3;
@@ -236,80 +127,72 @@ public class DecimalUtility extends CoreDecimalUtility {
int shiftOrder;
byte shiftBits;
- // TODO: Some of the logic here is common with casting from Dense to Sparse types, factor out common code
if (maxPrecision == 38) {
- maskIndex = 0;
- shiftOrder = 6;
- shiftBits = 0x00;
- intermediateBytes[intermediateIndex++] = (byte) (data.getByte(startIndex) & 0x7F);
+ maskIndex = 0;
+ shiftOrder = 6;
+ shiftBits = 0x00;
+ intermediateBytes[intermediateIndex++] = (byte) (data.getByte(startIndex) & 0x7F);
} else if (maxPrecision == 28) {
- maskIndex = 1;
- shiftOrder = 4;
- shiftBits = (byte) ((data.getByte(startIndex) & 0x03) << shiftOrder);
- intermediateBytes[intermediateIndex++] = (byte) (((data.getByte(startIndex) & 0x3C) & 0xFF) >>> 2);
+ maskIndex = 1;
+ shiftOrder = 4;
+ shiftBits = (byte) ((data.getByte(startIndex) & 0x03) << shiftOrder);
+ intermediateBytes[intermediateIndex++] = (byte) (((data.getByte(startIndex) & 0x3C) & 0xFF) >>> 2);
} else {
- throw new UnsupportedOperationException("Dense types with max precision 38 and 28 are only supported");
+ throw new UnsupportedOperationException("Dense types with max precision 38 and 28 are only supported");
}
int inputIndex = 1;
boolean sign = false;
if ((data.getByte(startIndex) & 0x80) != 0) {
- sign = true;
+ sign = true;
}
while (inputIndex < width) {
-
intermediateBytes[intermediateIndex] = (byte) ((shiftBits) | (((data.getByte(startIndex + inputIndex) & reverseMask[maskIndex]) & 0xFF) >>> (8 - shiftOrder)));
-
shiftBits = (byte) ((data.getByte(startIndex + inputIndex) & mask[maskIndex]) << shiftOrder);
inputIndex++;
intermediateIndex++;
if (((inputIndex - 1) % INTEGER_SIZE) == 0) {
- shiftBits = (byte) ((shiftBits & 0xFF) >>> 2);
- maskIndex++;
- shiftOrder -= 2;
+ shiftBits = (byte) ((shiftBits & 0xFF) >>> 2);
+ maskIndex++;
+ shiftOrder -= 2;
}
-
}
/* copy the last byte */
intermediateBytes[intermediateIndex] = shiftBits;
- if (sign == true) {
- intermediateBytes[0] = (byte) (intermediateBytes[0] | 0x80);
+ if (sign) {
+ intermediateBytes[0] = (byte) (intermediateBytes[0] | 0x80);
}
final ByteBuf intermediate = UnpooledByteBufAllocator.DEFAULT.buffer(intermediateBytes.length);
try {
- intermediate.setBytes(0, intermediateBytes);
+ intermediate.setBytes(0, intermediateBytes);
- BigDecimal ret = getBigDecimalFromIntermediate(intermediate, 0, nDecimalDigits + 1, scale);
- return ret;
+ // In the intermediate representation we don't pad the scale with zeroes, so set truncate = false
+ return getBigDecimalFromDrillBuf(intermediate, 0, nDecimalDigits + 1, scale, false);
} finally {
intermediate.release();
}
-
}
/**
* Function converts the BigDecimal and stores it in out internal sparse representation
*/
- public static void getSparseFromBigDecimal(BigDecimal input, ByteBuf data, int startIndex, int scale, int precision,
- int nDecimalDigits) {
+ public static void getSparseFromBigDecimal(BigDecimal input, ByteBuf data, int startIndex, int scale, int nDecimalDigits) {
// Initialize the buffer
- for (int i = 0; i < nDecimalDigits; i++) {
- data.setInt(startIndex + (i * INTEGER_SIZE), 0);
- }
+ data.setZero(startIndex, nDecimalDigits * INTEGER_SIZE);
boolean sign = false;
if (input.signum() == -1) {
- // negative input
- sign = true;
- input = input.abs();
+ // negative input
+ sign = true;
+ input = input.abs();
}
// Truncate the input as per the scale provided
@@ -320,25 +203,25 @@ public class DecimalUtility extends CoreDecimalUtility {
int destIndex = nDecimalDigits - roundUp(scale) - 1;
- // we use base 1 billion integer digits for out integernal representation
+ // we use base 1 billion integer digits for out internal representation
BigDecimal base = new BigDecimal(DIGITS_BASE);
while (integerPart.compareTo(BigDecimal.ZERO) == 1) {
- // store the modulo as the integer value
- data.setInt(startIndex + (destIndex * INTEGER_SIZE), (integerPart.remainder(base)).intValue());
- destIndex--;
- // Divide by base 1 billion
- integerPart = (integerPart.divide(base)).setScale(0, BigDecimal.ROUND_DOWN);
+ // store the modulo as the integer value
+ data.setInt(startIndex + destIndex * INTEGER_SIZE, integerPart.remainder(base).intValue());
+ destIndex--;
+ // Divide by base 1 billion
+ integerPart = integerPart.divide(base, BigDecimal.ROUND_DOWN).setScale(0, BigDecimal.ROUND_DOWN);
}
/* Sparse representation contains padding of additional zeroes
* so each digit contains MAX_DIGITS for ease of arithmetic
*/
- int actualDigits;
- if ((actualDigits = (scale % MAX_DIGITS)) != 0) {
- // Pad additional zeroes
- scale = scale + (MAX_DIGITS - actualDigits);
- input = input.setScale(scale, BigDecimal.ROUND_DOWN);
+ int actualDigits = scale % MAX_DIGITS;
+ if (actualDigits != 0) {
+ // Pad additional zeroes
+ scale = scale + MAX_DIGITS - actualDigits;
+ input = input.setScale(scale, BigDecimal.ROUND_DOWN);
}
//separate out the fractional part
@@ -347,15 +230,15 @@ public class DecimalUtility extends CoreDecimalUtility {
destIndex = nDecimalDigits - 1;
while (scale > 0) {
- // Get next set of MAX_DIGITS (9) store it in the DrillBuf
- fractionalPart = fractionalPart.movePointLeft(MAX_DIGITS);
- BigDecimal temp = fractionalPart.remainder(BigDecimal.ONE);
+ // Get next set of MAX_DIGITS (9) store it in the DrillBuf
+ fractionalPart = fractionalPart.movePointLeft(MAX_DIGITS);
+ BigDecimal temp = fractionalPart.remainder(BigDecimal.ONE);
- data.setInt(startIndex + (destIndex * INTEGER_SIZE), (temp.unscaledValue().intValue()));
- destIndex--;
+ data.setInt(startIndex + destIndex * INTEGER_SIZE, temp.unscaledValue().intValue());
+ destIndex--;
- fractionalPart = fractionalPart.setScale(0, BigDecimal.ROUND_DOWN);
- scale -= MAX_DIGITS;
+ fractionalPart = fractionalPart.setScale(0, BigDecimal.ROUND_DOWN);
+ scale -= MAX_DIGITS;
}
// Set the negative sign
@@ -365,93 +248,55 @@ public class DecimalUtility extends CoreDecimalUtility {
}
/**
- * Converts from an input BigDecimal into varying width "VarDecimal" representation.
- * The object that manages the "data" is assumed to already have the proper scale set,
- * matching that of input.scale().
- * @param input input decimal number to be stored
- * @param data destination buffer to store the byte array representation of input
- * @param startIndex starting index in data to hold the bytes
- * @return startIndex + length of bytes stored (i.e., the next startIndex in the data buffer)
+ * Returns unsigned long value taken from specified {@link BigDecimal} input with specified scale
+ *
+ * @param input {@link BigDecimal} with desired value
+ * @param scale scale of the value
+ * @return long value taken from specified {@link BigDecimal}
*/
- public static int getVarDecimalFromBigDecimal(BigDecimal input, ByteBuf data, int startIndex) {
- byte[] bytes = input.unscaledValue().toByteArray();
- int len = bytes.length;
- data.setBytes(startIndex, bytes);
- return startIndex + len;
- }
-
- public static long getDecimal18FromBigDecimal(BigDecimal input, int scale, int precision) {
+ public static long getDecimal18FromBigDecimal(BigDecimal input, int scale) {
// Truncate or pad to set the input to the correct scale
input = input.setScale(scale, BigDecimal.ROUND_HALF_UP);
return input.unscaledValue().longValue();
}
- public static int getDecimal9FromBigDecimal(BigDecimal input, int scale, int precision) {
+ /**
+ * Returns unsigned int value taken from specified {@link BigDecimal} input with specified scale.
+ *
+ * @param input {@link BigDecimal} with desired value
+ * @param scale scale of the value
+ * @return int value taken from specified {@link BigDecimal}
+ */
+ public static int getDecimal9FromBigDecimal(BigDecimal input, int scale) {
// Truncate or pad to set the input to the correct scale
input = input.setScale(scale, BigDecimal.ROUND_HALF_UP);
return input.unscaledValue().intValue();
}
- public static BigDecimal getBigDecimalFromPrimitiveTypes(int input, int scale, int precision) {
+ /**
+ * Returns {@link BigDecimal} value created from specified integer value with specified scale.
+ *
+ * @param input integer value to use for creating of {@link BigDecimal}
+ * @param scale scale for resulting {@link BigDecimal}
+ * @return {@link BigDecimal} value
+ */
+ public static BigDecimal getBigDecimalFromPrimitiveTypes(int input, int scale) {
return BigDecimal.valueOf(input, scale);
}
- public static BigDecimal getBigDecimalFromPrimitiveTypes(long input, int scale, int precision) {
+ /**
+ * Returns {@link BigDecimal} value created from specified long value with specified scale.
+ *
+ * @param input long value to use for creating of {@link BigDecimal}
+ * @param scale scale for resulting {@link BigDecimal}
+ * @return {@link BigDecimal} value
+ */
+ public static BigDecimal getBigDecimalFromPrimitiveTypes(long input, int scale) {
return BigDecimal.valueOf(input, scale);
}
- public static int compareDenseBytes(DrillBuf left, int leftStart, boolean leftSign, DrillBuf right, int rightStart, boolean rightSign, int width) {
-
- int invert = 1;
-
- /* If signs are different then simply look at the
- * sign of the two inputs and determine which is greater
- */
- if (leftSign != rightSign) {
-
- return((leftSign == true) ? -1 : 1);
- } else if(leftSign == true) {
- /* Both inputs are negative, at the end we will
- * have to invert the comparison
- */
- invert = -1;
- }
-
- int cmp = 0;
-
- for (int i = 0; i < width; i++) {
- byte leftByte = left.getByte(leftStart + i);
- byte rightByte = right.getByte(rightStart + i);
- // Unsigned byte comparison
- if ((leftByte & 0xFF) > (rightByte & 0xFF)) {
- cmp = 1;
- break;
- } else if ((leftByte & 0xFF) < (rightByte & 0xFF)) {
- cmp = -1;
- break;
- }
- }
- cmp *= invert; // invert the comparison if both were negative values
-
- return cmp;
- }
-
- public static int getIntegerFromSparseBuffer(DrillBuf buffer, int start, int index) {
- int value = buffer.getInt(start + (index * 4));
-
- if (index == 0) {
- /* the first byte contains sign bit, return value without it */
- value = (value & 0x7FFFFFFF);
- }
- return value;
- }
-
- public static void setInteger(DrillBuf buffer, int start, int index, int value) {
- buffer.setInt(start + (index * 4), value);
- }
-
/**
* Compares two VarDecimal values, still stored in their respective Drill buffers
*
@@ -497,314 +342,6 @@ public class DecimalUtility extends CoreDecimalUtility {
return bdLeft.compareTo(bdRight);
}
- public static int compareSparseBytes(DrillBuf left, int leftStart, boolean leftSign, int leftScale, int leftPrecision, DrillBuf right, int rightStart, boolean rightSign, int rightPrecision, int rightScale, int width, int nDecimalDigits, boolean absCompare) {
-
- int invert = 1;
-
- if (absCompare == false) {
- if (leftSign != rightSign) {
- return (leftSign == true) ? -1 : 1;
- }
-
- // Both values are negative invert the outcome of the comparison
- if (leftSign == true) {
- invert = -1;
- }
- }
-
- int cmp = compareSparseBytesInner(left, leftStart, leftSign, leftScale, leftPrecision, right, rightStart, rightSign, rightPrecision, rightScale, width, nDecimalDigits);
- return cmp * invert;
- }
-
- public static int compareSparseBytesInner(DrillBuf left, int leftStart, boolean leftSign, int leftScale, int leftPrecision, DrillBuf right, int rightStart, boolean rightSign, int rightPrecision, int rightScale, int width, int nDecimalDigits) {
- /* compute the number of integer digits in each decimal */
- int leftInt = leftPrecision - leftScale;
- int rightInt = rightPrecision - rightScale;
-
- /* compute the number of indexes required for storing integer digits */
- int leftIntRoundedUp = org.apache.drill.exec.util.DecimalUtility.roundUp(leftInt);
- int rightIntRoundedUp = org.apache.drill.exec.util.DecimalUtility.roundUp(rightInt);
-
- /* compute number of indexes required for storing scale */
- int leftScaleRoundedUp = org.apache.drill.exec.util.DecimalUtility.roundUp(leftScale);
- int rightScaleRoundedUp = org.apache.drill.exec.util.DecimalUtility.roundUp(rightScale);
-
- /* compute index of the most significant integer digits */
- int leftIndex1 = nDecimalDigits - leftScaleRoundedUp - leftIntRoundedUp;
- int rightIndex1 = nDecimalDigits - rightScaleRoundedUp - rightIntRoundedUp;
-
- int leftStopIndex = nDecimalDigits - leftScaleRoundedUp;
- int rightStopIndex = nDecimalDigits - rightScaleRoundedUp;
-
- /* Discard the zeroes in the integer part */
- while (leftIndex1 < leftStopIndex) {
- if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) != 0) {
- break;
- }
-
- /* Digit in this location is zero, decrement the actual number
- * of integer digits
- */
- leftIntRoundedUp--;
- leftIndex1++;
- }
-
- /* If we reached the stop index then the number of integers is zero */
- if (leftIndex1 == leftStopIndex) {
- leftIntRoundedUp = 0;
- }
-
- while (rightIndex1 < rightStopIndex) {
- if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) != 0) {
- break;
- }
-
- /* Digit in this location is zero, decrement the actual number
- * of integer digits
- */
- rightIntRoundedUp--;
- rightIndex1++;
- }
-
- if (rightIndex1 == rightStopIndex) {
- rightIntRoundedUp = 0;
- }
-
- /* We have the accurate number of non-zero integer digits,
- * if the number of integer digits are different then we can determine
- * which decimal is larger and needn't go down to comparing individual values
- */
- if (leftIntRoundedUp > rightIntRoundedUp) {
- return 1;
- }
- else if (rightIntRoundedUp > leftIntRoundedUp) {
- return -1;
- }
-
- /* The number of integer digits are the same, set the each index
- * to the first non-zero integer and compare each digit
- */
- leftIndex1 = nDecimalDigits - leftScaleRoundedUp - leftIntRoundedUp;
- rightIndex1 = nDecimalDigits - rightScaleRoundedUp - rightIntRoundedUp;
-
- while (leftIndex1 < leftStopIndex && rightIndex1 < rightStopIndex) {
- if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) > getIntegerFromSparseBuffer(right, rightStart, rightIndex1)) {
- return 1;
- }
- else if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) > getIntegerFromSparseBuffer(left, leftStart, leftIndex1)) {
- return -1;
- }
-
- leftIndex1++;
- rightIndex1++;
- }
-
- /* The integer part of both the decimal's are equal, now compare
- * each individual fractional part. Set the index to be at the
- * beginning of the fractional part
- */
- leftIndex1 = leftStopIndex;
- rightIndex1 = rightStopIndex;
-
- /* Stop indexes will be the end of the array */
- leftStopIndex = nDecimalDigits;
- rightStopIndex = nDecimalDigits;
-
- /* compare the two fractional parts of the decimal */
- while (leftIndex1 < leftStopIndex && rightIndex1 < rightStopIndex) {
- if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) > getIntegerFromSparseBuffer(right, rightStart, rightIndex1)) {
- return 1;
- }
- else if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) > getIntegerFromSparseBuffer(left, leftStart, leftIndex1)) {
- return -1;
- }
-
- leftIndex1++;
- rightIndex1++;
- }
-
- /* Till now the fractional part of the decimals are equal, check
- * if one of the decimal has fractional part that is remaining
- * and is non-zero
- */
- while (leftIndex1 < leftStopIndex) {
- if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) != 0) {
- return 1;
- }
- leftIndex1++;
- }
-
- while(rightIndex1 < rightStopIndex) {
- if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) != 0) {
- return -1;
- }
- rightIndex1++;
- }
-
- /* Both decimal values are equal */
- return 0;
- }
-
- public static BigDecimal getBigDecimalFromByteArray(byte[] bytes, int start, int length, int scale) {
- byte[] value = Arrays.copyOfRange(bytes, start, start + length);
- BigInteger unscaledValue = new BigInteger(value);
- return new BigDecimal(unscaledValue, scale);
- }
-
- public static void roundDecimal(DrillBuf result, int start, int nDecimalDigits, int desiredScale, int currentScale) {
- int newScaleRoundedUp = org.apache.drill.exec.util.DecimalUtility.roundUp(desiredScale);
- int origScaleRoundedUp = org.apache.drill.exec.util.DecimalUtility.roundUp(currentScale);
-
- if (desiredScale < currentScale) {
-
- boolean roundUp = false;
-
- //Extract the first digit to be truncated to check if we need to round up
- int truncatedScaleIndex = desiredScale + 1;
- if (truncatedScaleIndex <= currentScale) {
- int extractDigitIndex = nDecimalDigits - origScaleRoundedUp -1;
- extractDigitIndex += org.apache.drill.exec.util.DecimalUtility.roundUp(truncatedScaleIndex);
- int extractDigit = getIntegerFromSparseBuffer(result, start, extractDigitIndex);
- int temp = org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS - (truncatedScaleIndex % org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS);
- if (temp != 0) {
- extractDigit = extractDigit / (int) (Math.pow(10, temp));
- }
- if ((extractDigit % 10) > 4) {
- roundUp = true;
- }
- }
-
- // Get the source index beyond which we will truncate
- int srcIntIndex = nDecimalDigits - origScaleRoundedUp - 1;
- int srcIndex = srcIntIndex + newScaleRoundedUp;
-
- // Truncate the remaining fractional part, move the integer part
- int destIndex = nDecimalDigits - 1;
- if (srcIndex != destIndex) {
- while (srcIndex >= 0) {
- setInteger(result, start, destIndex--, getIntegerFromSparseBuffer(result, start, srcIndex--));
- }
-
- // Set the remaining portion of the decimal to be zeroes
- while (destIndex >= 0) {
- setInteger(result, start, destIndex--, 0);
- }
- srcIndex = nDecimalDigits - 1;
- }
-
- // We truncated the decimal digit. Now we need to truncate within the base 1 billion fractional digit
- int truncateFactor = org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS - (desiredScale % org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS);
- if (truncateFactor != org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS) {
- truncateFactor = (int) Math.pow(10, truncateFactor);
- int fractionalDigits = getIntegerFromSparseBuffer(result, start, nDecimalDigits - 1);
- fractionalDigits /= truncateFactor;
- setInteger(result, start, nDecimalDigits - 1, fractionalDigits * truncateFactor);
- }
-
- // Finally round up the digit if needed
- if (roundUp == true) {
- srcIndex = nDecimalDigits - 1;
- int carry;
- if (truncateFactor != org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS) {
- carry = truncateFactor;
- } else {
- carry = 1;
- }
-
- while (srcIndex >= 0) {
- int value = getIntegerFromSparseBuffer(result, start, srcIndex);
- value += carry;
-
- if (value >= org.apache.drill.exec.util.DecimalUtility.DIGITS_BASE) {
- setInteger(result, start, srcIndex--, value % org.apache.drill.exec.util.DecimalUtility.DIGITS_BASE);
- carry = value / org.apache.drill.exec.util.DecimalUtility.DIGITS_BASE;
- } else {
- setInteger(result, start, srcIndex--, value);
- carry = 0;
- break;
- }
- }
- }
- } else if (desiredScale > currentScale) {
- // Add fractional digits to the decimal
-
- // Check if we need to shift the decimal digits to the left
- if (newScaleRoundedUp > origScaleRoundedUp) {
- int srcIndex = 0;
- int destIndex = newScaleRoundedUp - origScaleRoundedUp;
-
- // Check while extending scale, we are not overwriting integer part
- while (srcIndex < destIndex) {
- if (getIntegerFromSparseBuffer(result, start, srcIndex++) != 0) {
- throw new org.apache.drill.common.exceptions.DrillRuntimeException("Truncate resulting in loss of integer part, reduce scale specified");
- }
- }
-
- srcIndex = 0;
- while (destIndex < nDecimalDigits) {
- setInteger(result, start, srcIndex++, getIntegerFromSparseBuffer(result, start, destIndex++));
- }
-
- // Clear the remaining part
- while (srcIndex < nDecimalDigits) {
- setInteger(result, start, srcIndex++, 0);
- }
- }
- }
- }
-
- public static int getFirstFractionalDigit(int decimal, int scale) {
- if (scale == 0) {
- return 0;
- }
- int temp = (int) adjustScaleDivide(decimal, scale - 1);
- return Math.abs(temp % 10);
- }
-
- public static int getFirstFractionalDigit(long decimal, int scale) {
- if (scale == 0) {
- return 0;
- }
- long temp = adjustScaleDivide(decimal, scale - 1);
- return (int) (Math.abs(temp % 10));
- }
-
- public static int getFirstFractionalDigit(DrillBuf data, int scale, int start, int nDecimalDigits) {
- if (scale == 0) {
- return 0;
- }
-
- int index = nDecimalDigits - roundUp(scale);
- return (int) (adjustScaleDivide(data.getInt(start + (index * INTEGER_SIZE)), MAX_DIGITS - 1));
- }
-
- public static int compareSparseSamePrecScale(DrillBuf left, int lStart, byte[] right, int length) {
- // check the sign first
- boolean lSign = (left.getInt(lStart) & 0x80000000) != 0;
- boolean rSign = ByteFunctionHelpers.getSign(right);
- int cmp = 0;
-
- if (lSign != rSign) {
- return (lSign == false) ? 1 : -1;
- }
-
- // invert the comparison if we are comparing negative numbers
- int invert = (lSign == true) ? -1 : 1;
-
- // compare byte by byte
- int n = 0;
- while (n < length/4) {
- int leftInt = Decimal38SparseHolder.getInteger(n, lStart, left);
- int rightInt = ByteFunctionHelpers.getInteger(right, n);
- if (leftInt != rightInt) {
- cmp = (leftInt - rightInt ) > 0 ? 1 : -1;
- break;
- }
- n++;
- }
- return cmp * invert;
- }
-
/**
* Returns max length of byte array, required to store value with specified precision.
*
@@ -817,6 +354,13 @@ public class DecimalUtility extends CoreDecimalUtility {
return 0;
}
if (precision < 300) { // normal case, use exact heuristic formula
+ // Math.pow(10, precision) - 1 returns max integer value for the specified precision, example 999 for precision 3
+ // Math.log(Math.pow(10, precision) - 1) / Math.log(2) is just log with base 2 to calculate size
+ // in bits for max integer value mentioned above
+ // Math.log(Math.pow(10, precision) - 1) / Math.log(2) + 1 in this expression was added 1 to the count of bits to
+ // take into account case with negative value
+ // size in bits was divided by size of byte to calculate bytes count required to store value
+ // with the specified precision
return (int) Math.ceil((Math.log(Math.pow(10, precision) - 1) / Math.log(2) + 1) / Byte.SIZE);
} else {
// for values greater than 304 Math.pow(10, precision) returns Infinity, therefore 1 is neglected in Math.log()
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/ValueHolderHelper.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/ValueHolderHelper.java
index d1cc3b4..73027aa 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/ValueHolderHelper.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/ValueHolderHelper.java
@@ -127,7 +127,7 @@ public class ValueHolderHelper {
byte[] b = s.getBytes(Charsets.UTF_8);
vch.start = 0;
vch.end = b.length;
- vch.buffer = a.buffer(b.length); //
+ vch.buffer = a.buffer(b.length);
vch.buffer.setBytes(0, b);
return vch;
}
@@ -180,7 +180,7 @@ public class ValueHolderHelper {
dch.start = 0;
dch.buffer = buf.reallocIfNeeded(5 * DecimalUtility.INTEGER_SIZE);
DecimalUtility
- .getSparseFromBigDecimal(bigDecimal, dch.buffer, dch.start, dch.scale, dch.precision, dch.nDecimalDigits);
+ .getSparseFromBigDecimal(bigDecimal, dch.buffer, dch.start, dch.scale, Decimal28SparseHolder.nDecimalDigits);
return dch;
}
@@ -195,9 +195,9 @@ public class ValueHolderHelper {
dch.precision = bigDecimal.precision();
Decimal38SparseHolder.setSign(bigDecimal.signum() == -1, dch.start, dch.buffer);
dch.start = 0;
- dch.buffer = buf.reallocIfNeeded(dch.maxPrecision * DecimalUtility.INTEGER_SIZE);
+ dch.buffer = buf.reallocIfNeeded(Decimal38SparseHolder.maxPrecision * DecimalUtility.INTEGER_SIZE);
DecimalUtility
- .getSparseFromBigDecimal(bigDecimal, dch.buffer, dch.start, dch.scale, dch.precision, dch.nDecimalDigits);
+ .getSparseFromBigDecimal(bigDecimal, dch.buffer, dch.start, dch.scale, Decimal38SparseHolder.nDecimalDigits);
return dch;
}
diff --git a/logical/src/main/java/org/apache/drill/common/expression/ValueExpressions.java b/logical/src/main/java/org/apache/drill/common/expression/ValueExpressions.java
index 2ef809a..538e6c8 100644
--- a/logical/src/main/java/org/apache/drill/common/expression/ValueExpressions.java
+++ b/logical/src/main/java/org/apache/drill/common/expression/ValueExpressions.java
@@ -28,7 +28,6 @@ import org.apache.drill.common.types.TypeProtos.DataMode;
import org.apache.drill.common.types.TypeProtos.MajorType;
import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.common.types.Types;
-import org.apache.drill.common.util.CoreDecimalUtility;
import com.google.common.collect.Iterators;
@@ -94,22 +93,6 @@ public class ValueExpressions {
return new IntervalDayExpression(intervalInMillis);
}
- public static LogicalExpression getDecimal9(BigDecimal i) {
- return new Decimal9Expression(i, ExpressionPosition.UNKNOWN);
- }
-
- public static LogicalExpression getDecimal18(BigDecimal i) {
- return new Decimal18Expression(i, ExpressionPosition.UNKNOWN);
- }
-
- public static LogicalExpression getDecimal28(BigDecimal i) {
- return new Decimal28Expression(i, ExpressionPosition.UNKNOWN);
- }
-
- public static LogicalExpression getDecimal38(BigDecimal i) {
- return new Decimal38Expression(i, ExpressionPosition.UNKNOWN);
- }
-
public static LogicalExpression getVarDecimal(BigDecimal input, int precision, int scale) {
return new VarDecimalExpression(input, precision, scale, ExpressionPosition.UNKNOWN);
}
@@ -265,7 +248,7 @@ public class ValueExpressions {
super(pos);
this.scale = input.scale();
this.precision = input.precision();
- this.decimal = CoreDecimalUtility.getDecimal9FromBigDecimal(input, scale, precision);
+ this.decimal = input.setScale(scale, BigDecimal.ROUND_HALF_UP).intValue();
}
@@ -307,7 +290,7 @@ public class ValueExpressions {
super(pos);
this.scale = input.scale();
this.precision = input.precision();
- this.decimal = CoreDecimalUtility.getDecimal18FromBigDecimal(input, scale, precision);
+ this.decimal = input.setScale(scale, BigDecimal.ROUND_HALF_UP).longValue();
}
--
To stop receiving notification emails like this one, please contact
arina@apache.org.