You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by xu...@apache.org on 2015/07/31 02:43:35 UTC
[35/43] hive git commit: HIVE-11253. Move SearchArgument and
VectorizedRowBatch classes to storage-api. (omalley reviewed by prasanthj)
http://git-wip-us.apache.org/repos/asf/hive/blob/9ae70cb4/storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java b/storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java
new file mode 100644
index 0000000..d27ac16
--- /dev/null
+++ b/storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java
@@ -0,0 +1,687 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.io.sarg;
+
+import java.sql.Timestamp;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Deque;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * The implementation of SearchArguments.
+ */
+final class SearchArgumentImpl implements SearchArgument {
+ public static final Log LOG = LogFactory.getLog(SearchArgumentImpl.class);
+
+ static final class PredicateLeafImpl implements PredicateLeaf {
+ private final Operator operator;
+ private final Type type;
+ private final String columnName;
+ private final Object literal;
+ private final List<Object> literalList;
+
+ // Used by kryo
+ @SuppressWarnings("unused")
+ PredicateLeafImpl() {
+ operator = null;
+ type = null;
+ columnName = null;
+ literal = null;
+ literalList = null;
+ }
+
+ PredicateLeafImpl(Operator operator,
+ Type type,
+ String columnName,
+ Object literal,
+ List<Object> literalList) {
+ this.operator = operator;
+ this.type = type;
+ this.columnName = columnName;
+ this.literal = literal;
+ if (literal != null) {
+ if (literal.getClass() != type.getValueClass()) {
+ throw new IllegalArgumentException("Wrong value class " +
+ literal.getClass().getName() + " for " + type + "." + operator +
+ " leaf");
+ }
+ }
+ this.literalList = literalList;
+ if (literalList != null) {
+ Class valueCls = type.getValueClass();
+ for(Object lit: literalList) {
+ if (lit != null && lit.getClass() != valueCls) {
+ throw new IllegalArgumentException("Wrong value class item " +
+ lit.getClass().getName() + " for " + type + "." + operator +
+ " leaf");
+ }
+ }
+ }
+ }
+
+ @Override
+ public Operator getOperator() {
+ return operator;
+ }
+
+ @Override
+ public Type getType(){
+ return type;
+ }
+
+ @Override
+ public String getColumnName() {
+ return columnName;
+ }
+
+ @Override
+ public Object getLiteral() {
+ // To get around a kryo 2.22 bug while deserialize a Timestamp into Date
+ // (https://github.com/EsotericSoftware/kryo/issues/88)
+ // When we see a Date, convert back into Timestamp
+ if (literal instanceof java.util.Date) {
+ return new Timestamp(((java.util.Date)literal).getTime());
+ }
+ return literal;
+ }
+
+ @Override
+ public List<Object> getLiteralList() {
+ return literalList;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder buffer = new StringBuilder();
+ buffer.append('(');
+ buffer.append(operator);
+ buffer.append(' ');
+ buffer.append(columnName);
+ if (literal != null) {
+ buffer.append(' ');
+ buffer.append(literal);
+ } else if (literalList != null) {
+ for(Object lit: literalList) {
+ buffer.append(' ');
+ buffer.append(lit == null ? "null" : lit.toString());
+ }
+ }
+ buffer.append(')');
+ return buffer.toString();
+ }
+
+ private static boolean isEqual(Object left, Object right) {
+
+ return left == right ||
+ (left != null && right != null && left.equals(right));
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null || other.getClass() != getClass()) {
+ return false;
+ } else if (other == this) {
+ return true;
+ } else {
+ PredicateLeafImpl o = (PredicateLeafImpl) other;
+ return operator == o.operator &&
+ type == o.type &&
+ columnName.equals(o.columnName) &&
+ isEqual(literal, o.literal) &&
+ isEqual(literalList, o.literalList);
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ return operator.hashCode() +
+ type.hashCode() * 17 +
+ columnName.hashCode() * 3 * 17+
+ (literal == null ? 0 : literal.hashCode()) * 101 * 3 * 17 +
+ (literalList == null ? 0 : literalList.hashCode()) *
+ 103 * 101 * 3 * 17;
+ }
+ }
+
+
+ private final List<PredicateLeaf> leaves;
+ private final ExpressionTree expression;
+
+ SearchArgumentImpl(ExpressionTree expression, List<PredicateLeaf> leaves) {
+ this.expression = expression;
+ this.leaves = leaves;
+ }
+
+ // Used by kyro
+ @SuppressWarnings("unused")
+ SearchArgumentImpl() {
+ leaves = null;
+ expression = null;
+ }
+
+ @Override
+ public List<PredicateLeaf> getLeaves() {
+ return leaves;
+ }
+
+ @Override
+ public TruthValue evaluate(TruthValue[] leaves) {
+ return expression == null ? TruthValue.YES : expression.evaluate(leaves);
+ }
+
+ @Override
+ public ExpressionTree getExpression() {
+ return expression;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder buffer = new StringBuilder();
+ for(int i=0; i < leaves.size(); ++i) {
+ buffer.append("leaf-");
+ buffer.append(i);
+ buffer.append(" = ");
+ buffer.append(leaves.get(i).toString());
+ buffer.append('\n');
+ }
+ buffer.append("expr = ");
+ buffer.append(expression);
+ return buffer.toString();
+ }
+
+ static class BuilderImpl implements Builder {
+
+ // max threshold for CNF conversion. having >8 elements in andList will be
+ // converted to maybe
+ private static final int CNF_COMBINATIONS_THRESHOLD = 256;
+
+ private final Deque<ExpressionTree> currentTree =
+ new ArrayDeque<ExpressionTree>();
+ private final Map<PredicateLeaf, Integer> leaves =
+ new HashMap<PredicateLeaf, Integer>();
+ private final ExpressionTree root =
+ new ExpressionTree(ExpressionTree.Operator.AND);
+ {
+ currentTree.add(root);
+ }
+
+ @Override
+ public Builder startOr() {
+ ExpressionTree node = new ExpressionTree(ExpressionTree.Operator.OR);
+ currentTree.getFirst().getChildren().add(node);
+ currentTree.addFirst(node);
+ return this;
+ }
+
+ @Override
+ public Builder startAnd() {
+ ExpressionTree node = new ExpressionTree(ExpressionTree.Operator.AND);
+ currentTree.getFirst().getChildren().add(node);
+ currentTree.addFirst(node);
+ return this;
+ }
+
+ @Override
+ public Builder startNot() {
+ ExpressionTree node = new ExpressionTree(ExpressionTree.Operator.NOT);
+ currentTree.getFirst().getChildren().add(node);
+ currentTree.addFirst(node);
+ return this;
+ }
+
+ @Override
+ public Builder end() {
+ ExpressionTree current = currentTree.removeFirst();
+ if (current.getChildren().size() == 0) {
+ throw new IllegalArgumentException("Can't create expression " + root +
+ " with no children.");
+ }
+ if (current.getOperator() == ExpressionTree.Operator.NOT &&
+ current.getChildren().size() != 1) {
+ throw new IllegalArgumentException("Can't create not expression " +
+ current + " with more than 1 child.");
+ }
+ return this;
+ }
+
+ private int addLeaf(PredicateLeaf leaf) {
+ Integer result = leaves.get(leaf);
+ if (result == null) {
+ int id = leaves.size();
+ leaves.put(leaf, id);
+ return id;
+ } else {
+ return result;
+ }
+ }
+
+ @Override
+ public Builder lessThan(String column, PredicateLeaf.Type type,
+ Object literal) {
+ ExpressionTree parent = currentTree.getFirst();
+ if (column == null || literal == null) {
+ parent.getChildren().add(new ExpressionTree(TruthValue.YES_NO_NULL));
+ } else {
+ PredicateLeaf leaf =
+ new PredicateLeafImpl(PredicateLeaf.Operator.LESS_THAN,
+ type, column, literal, null);
+ parent.getChildren().add(new ExpressionTree(addLeaf(leaf)));
+ }
+ return this;
+ }
+
+ @Override
+ public Builder lessThanEquals(String column, PredicateLeaf.Type type,
+ Object literal) {
+ ExpressionTree parent = currentTree.getFirst();
+ if (column == null || literal == null) {
+ parent.getChildren().add(new ExpressionTree(TruthValue.YES_NO_NULL));
+ } else {
+ PredicateLeaf leaf =
+ new PredicateLeafImpl(PredicateLeaf.Operator.LESS_THAN_EQUALS,
+ type, column, literal, null);
+ parent.getChildren().add(new ExpressionTree(addLeaf(leaf)));
+ }
+ return this;
+ }
+
+ @Override
+ public Builder equals(String column, PredicateLeaf.Type type,
+ Object literal) {
+ ExpressionTree parent = currentTree.getFirst();
+ if (column == null || literal == null) {
+ parent.getChildren().add(new ExpressionTree(TruthValue.YES_NO_NULL));
+ } else {
+ PredicateLeaf leaf =
+ new PredicateLeafImpl(PredicateLeaf.Operator.EQUALS,
+ type, column, literal, null);
+ parent.getChildren().add(new ExpressionTree(addLeaf(leaf)));
+ }
+ return this;
+ }
+
+ @Override
+ public Builder nullSafeEquals(String column, PredicateLeaf.Type type,
+ Object literal) {
+ ExpressionTree parent = currentTree.getFirst();
+ if (column == null || literal == null) {
+ parent.getChildren().add(new ExpressionTree(TruthValue.YES_NO_NULL));
+ } else {
+ PredicateLeaf leaf =
+ new PredicateLeafImpl(PredicateLeaf.Operator.NULL_SAFE_EQUALS,
+ type, column, literal, null);
+ parent.getChildren().add(new ExpressionTree(addLeaf(leaf)));
+ }
+ return this;
+ }
+
+ @Override
+ public Builder in(String column, PredicateLeaf.Type type,
+ Object... literal) {
+ ExpressionTree parent = currentTree.getFirst();
+ if (column == null || literal == null) {
+ parent.getChildren().add(new ExpressionTree(TruthValue.YES_NO_NULL));
+ } else {
+ if (literal.length == 0) {
+ throw new IllegalArgumentException("Can't create in expression with "
+ + "no arguments");
+ }
+ List<Object> argList = new ArrayList<Object>();
+ argList.addAll(Arrays.asList(literal));
+
+ PredicateLeaf leaf =
+ new PredicateLeafImpl(PredicateLeaf.Operator.IN,
+ type, column, null, argList);
+ parent.getChildren().add(new ExpressionTree(addLeaf(leaf)));
+ }
+ return this;
+ }
+
+ @Override
+ public Builder isNull(String column, PredicateLeaf.Type type) {
+ ExpressionTree parent = currentTree.getFirst();
+ if (column == null) {
+ parent.getChildren().add(new ExpressionTree(TruthValue.YES_NO_NULL));
+ } else {
+ PredicateLeaf leaf =
+ new PredicateLeafImpl(PredicateLeaf.Operator.IS_NULL,
+ type, column, null, null);
+ parent.getChildren().add(new ExpressionTree(addLeaf(leaf)));
+ }
+ return this;
+ }
+
+ @Override
+ public Builder between(String column, PredicateLeaf.Type type, Object lower,
+ Object upper) {
+ ExpressionTree parent = currentTree.getFirst();
+ if (column == null || lower == null || upper == null) {
+ parent.getChildren().add(new ExpressionTree(TruthValue.YES_NO_NULL));
+ } else {
+ List<Object> argList = new ArrayList<Object>();
+ argList.add(lower);
+ argList.add(upper);
+ PredicateLeaf leaf =
+ new PredicateLeafImpl(PredicateLeaf.Operator.BETWEEN,
+ type, column, null, argList);
+ parent.getChildren().add(new ExpressionTree(addLeaf(leaf)));
+ }
+ return this;
+ }
+
+ @Override
+ public Builder literal(TruthValue truth) {
+ ExpressionTree parent = currentTree.getFirst();
+ parent.getChildren().add(new ExpressionTree(truth));
+ return this;
+ }
+
+ /**
+ * Recursively explore the tree to find the leaves that are still reachable
+ * after optimizations.
+ * @param tree the node to check next
+ * @param next the next available leaf id
+ * @param leafReorder
+ * @return the next available leaf id
+ */
+ static int compactLeaves(ExpressionTree tree, int next, int[] leafReorder) {
+ if (tree.getOperator() == ExpressionTree.Operator.LEAF) {
+ int oldLeaf = tree.getLeaf();
+ if (leafReorder[oldLeaf] == -1) {
+ leafReorder[oldLeaf] = next++;
+ }
+ } else if (tree.getChildren() != null){
+ for(ExpressionTree child: tree.getChildren()) {
+ next = compactLeaves(child, next, leafReorder);
+ }
+ }
+ return next;
+ }
+
+ /**
+ * Rewrite expression tree to update the leaves.
+ * @param root the root of the tree to fix
+ * @param leafReorder a map from old leaf ids to new leaf ids
+ * @return the fixed root
+ */
+ static ExpressionTree rewriteLeaves(ExpressionTree root,
+ int[] leafReorder) {
+ if (root.getOperator() == ExpressionTree.Operator.LEAF) {
+ return new ExpressionTree(leafReorder[root.getLeaf()]);
+ } else if (root.getChildren() != null){
+ List<ExpressionTree> children = root.getChildren();
+ for(int i=0; i < children.size(); ++i) {
+ children.set(i, rewriteLeaves(children.get(i), leafReorder));
+ }
+ }
+ return root;
+ }
+
+ @Override
+ public SearchArgument build() {
+ if (currentTree.size() != 1) {
+ throw new IllegalArgumentException("Failed to end " +
+ currentTree.size() + " operations.");
+ }
+ ExpressionTree optimized = pushDownNot(root);
+ optimized = foldMaybe(optimized);
+ optimized = flatten(optimized);
+ optimized = convertToCNF(optimized);
+ optimized = flatten(optimized);
+ int leafReorder[] = new int[leaves.size()];
+ Arrays.fill(leafReorder, -1);
+ int newLeafCount = compactLeaves(optimized, 0, leafReorder);
+ optimized = rewriteLeaves(optimized, leafReorder);
+ ArrayList<PredicateLeaf> leafList = new ArrayList<>(newLeafCount);
+ // expand list to correct size
+ for(int i=0; i < newLeafCount; ++i) {
+ leafList.add(null);
+ }
+ // build the new list
+ for(Map.Entry<PredicateLeaf, Integer> elem: leaves.entrySet()) {
+ int newLoc = leafReorder[elem.getValue()];
+ if (newLoc != -1) {
+ leafList.set(newLoc, elem.getKey());
+ }
+ }
+ return new SearchArgumentImpl(optimized, leafList);
+ }
+
+ /**
+ * Push the negations all the way to just before the leaves. Also remove
+ * double negatives.
+ * @param root the expression to normalize
+ * @return the normalized expression, which may share some or all of the
+ * nodes of the original expression.
+ */
+ static ExpressionTree pushDownNot(ExpressionTree root) {
+ if (root.getOperator() == ExpressionTree.Operator.NOT) {
+ ExpressionTree child = root.getChildren().get(0);
+ switch (child.getOperator()) {
+ case NOT:
+ return pushDownNot(child.getChildren().get(0));
+ case CONSTANT:
+ return new ExpressionTree(child.getConstant().not());
+ case AND:
+ root = new ExpressionTree(ExpressionTree.Operator.OR);
+ for(ExpressionTree kid: child.getChildren()) {
+ root.getChildren().add(pushDownNot(new
+ ExpressionTree(ExpressionTree.Operator.NOT, kid)));
+ }
+ break;
+ case OR:
+ root = new ExpressionTree(ExpressionTree.Operator.AND);
+ for(ExpressionTree kid: child.getChildren()) {
+ root.getChildren().add(pushDownNot(new ExpressionTree
+ (ExpressionTree.Operator.NOT, kid)));
+ }
+ break;
+ // for leaf, we don't do anything
+ default:
+ break;
+ }
+ } else if (root.getChildren() != null) {
+ // iterate through children and push down not for each one
+ for(int i=0; i < root.getChildren().size(); ++i) {
+ root.getChildren().set(i, pushDownNot(root.getChildren().get(i)));
+ }
+ }
+ return root;
+ }
+
+ /**
+ * Remove MAYBE values from the expression. If they are in an AND operator,
+ * they are dropped. If they are in an OR operator, they kill their parent.
+ * This assumes that pushDownNot has already been called.
+ * @param expr The expression to clean up
+ * @return The cleaned up expression
+ */
+ static ExpressionTree foldMaybe(ExpressionTree expr) {
+ if (expr.getChildren() != null) {
+ for(int i=0; i < expr.getChildren().size(); ++i) {
+ ExpressionTree child = foldMaybe(expr.getChildren().get(i));
+ if (child.getConstant() == TruthValue.YES_NO_NULL) {
+ switch (expr.getOperator()) {
+ case AND:
+ expr.getChildren().remove(i);
+ i -= 1;
+ break;
+ case OR:
+ // a maybe will kill the or condition
+ return child;
+ default:
+ throw new IllegalStateException("Got a maybe as child of " +
+ expr);
+ }
+ } else {
+ expr.getChildren().set(i, child);
+ }
+ }
+ if (expr.getChildren().isEmpty()) {
+ return new ExpressionTree(TruthValue.YES_NO_NULL);
+ }
+ }
+ return expr;
+ }
+
+ /**
+ * Converts multi-level ands and ors into single level ones.
+ * @param root the expression to flatten
+ * @return the flattened expression, which will always be root with
+ * potentially modified children.
+ */
+ static ExpressionTree flatten(ExpressionTree root) {
+ if (root.getChildren() != null) {
+ // iterate through the index, so that if we add more children,
+ // they don't get re-visited
+ for(int i=0; i < root.getChildren().size(); ++i) {
+ ExpressionTree child = flatten(root.getChildren().get(i));
+ // do we need to flatten?
+ if (child.getOperator() == root.getOperator() &&
+ child.getOperator() != ExpressionTree.Operator.NOT) {
+ boolean first = true;
+ for(ExpressionTree grandkid: child.getChildren()) {
+ // for the first grandkid replace the original parent
+ if (first) {
+ first = false;
+ root.getChildren().set(i, grandkid);
+ } else {
+ root.getChildren().add(++i, grandkid);
+ }
+ }
+ } else {
+ root.getChildren().set(i, child);
+ }
+ }
+ // if we have a singleton AND or OR, just return the child
+ if ((root.getOperator() == ExpressionTree.Operator.OR ||
+ root.getOperator() == ExpressionTree.Operator.AND) &&
+ root.getChildren().size() == 1) {
+ return root.getChildren().get(0);
+ }
+ }
+ return root;
+ }
+
+ /**
+ * Generate all combinations of items on the andList. For each item on the
+ * andList, it generates all combinations of one child from each and
+ * expression. Thus, (and a b) (and c d) will be expanded to: (or a c)
+ * (or a d) (or b c) (or b d). If there are items on the nonAndList, they
+ * are added to each or expression.
+ * @param result a list to put the results onto
+ * @param andList a list of and expressions
+ * @param nonAndList a list of non-and expressions
+ */
+ private static void generateAllCombinations(List<ExpressionTree> result,
+ List<ExpressionTree> andList,
+ List<ExpressionTree> nonAndList
+ ) {
+ List<ExpressionTree> kids = andList.get(0).getChildren();
+ if (result.isEmpty()) {
+ for(ExpressionTree kid: kids) {
+ ExpressionTree or = new ExpressionTree(ExpressionTree.Operator.OR);
+ result.add(or);
+ for(ExpressionTree node: nonAndList) {
+ or.getChildren().add(new ExpressionTree(node));
+ }
+ or.getChildren().add(kid);
+ }
+ } else {
+ List<ExpressionTree> work = new ArrayList<ExpressionTree>(result);
+ result.clear();
+ for(ExpressionTree kid: kids) {
+ for(ExpressionTree or: work) {
+ ExpressionTree copy = new ExpressionTree(or);
+ copy.getChildren().add(kid);
+ result.add(copy);
+ }
+ }
+ }
+ if (andList.size() > 1) {
+ generateAllCombinations(result, andList.subList(1, andList.size()),
+ nonAndList);
+ }
+ }
+
+ /**
+ * Convert an expression so that the top level operator is AND with OR
+ * operators under it. This routine assumes that all of the NOT operators
+ * have been pushed to the leaves via pushdDownNot.
+ * @param root the expression
+ * @return the normalized expression
+ */
+ static ExpressionTree convertToCNF(ExpressionTree root) {
+ if (root.getChildren() != null) {
+ // convert all of the children to CNF
+ int size = root.getChildren().size();
+ for(int i=0; i < size; ++i) {
+ root.getChildren().set(i, convertToCNF(root.getChildren().get(i)));
+ }
+ if (root.getOperator() == ExpressionTree.Operator.OR) {
+ // a list of leaves that weren't under AND expressions
+ List<ExpressionTree> nonAndList = new ArrayList<ExpressionTree>();
+ // a list of AND expressions that we need to distribute
+ List<ExpressionTree> andList = new ArrayList<ExpressionTree>();
+ for(ExpressionTree child: root.getChildren()) {
+ if (child.getOperator() == ExpressionTree.Operator.AND) {
+ andList.add(child);
+ } else if (child.getOperator() == ExpressionTree.Operator.OR) {
+ // pull apart the kids of the OR expression
+ for(ExpressionTree grandkid: child.getChildren()) {
+ nonAndList.add(grandkid);
+ }
+ } else {
+ nonAndList.add(child);
+ }
+ }
+ if (!andList.isEmpty()) {
+ if (checkCombinationsThreshold(andList)) {
+ root = new ExpressionTree(ExpressionTree.Operator.AND);
+ generateAllCombinations(root.getChildren(), andList, nonAndList);
+ } else {
+ root = new ExpressionTree(TruthValue.YES_NO_NULL);
+ }
+ }
+ }
+ }
+ return root;
+ }
+
+ private static boolean checkCombinationsThreshold(List<ExpressionTree> andList) {
+ int numComb = 1;
+ for (ExpressionTree tree : andList) {
+ numComb *= tree.getChildren().size();
+ if (numComb > CNF_COMBINATIONS_THRESHOLD) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9ae70cb4/storage-api/src/java/org/apache/hadoop/hive/serde2/io/HiveDecimalWritable.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hadoop/hive/serde2/io/HiveDecimalWritable.java b/storage-api/src/java/org/apache/hadoop/hive/serde2/io/HiveDecimalWritable.java
new file mode 100644
index 0000000..9890771
--- /dev/null
+++ b/storage-api/src/java/org/apache/hadoop/hive/serde2/io/HiveDecimalWritable.java
@@ -0,0 +1,174 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.serde2.io;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.math.BigInteger;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.common.type.HiveDecimal;
+
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.io.WritableUtils;
+
+public class HiveDecimalWritable implements WritableComparable<HiveDecimalWritable> {
+
+ static final private Log LOG = LogFactory.getLog(HiveDecimalWritable.class);
+
+ private byte[] internalStorage = new byte[0];
+ private int scale;
+
+ public HiveDecimalWritable() {
+ }
+
+ public HiveDecimalWritable(String value) {
+ set(HiveDecimal.create(value));
+ }
+
+ public HiveDecimalWritable(byte[] bytes, int scale) {
+ set(bytes, scale);
+ }
+
+ public HiveDecimalWritable(HiveDecimalWritable writable) {
+ set(writable.getHiveDecimal());
+ }
+
+ public HiveDecimalWritable(HiveDecimal value) {
+ set(value);
+ }
+
+ public HiveDecimalWritable(long value) {
+ set((HiveDecimal.create(value)));
+ }
+
+ public void set(HiveDecimal value) {
+ set(value.unscaledValue().toByteArray(), value.scale());
+ }
+
+ public void set(HiveDecimal value, int maxPrecision, int maxScale) {
+ set(HiveDecimal.enforcePrecisionScale(value, maxPrecision, maxScale));
+ }
+
+ public void set(HiveDecimalWritable writable) {
+ set(writable.getHiveDecimal());
+ }
+
+ public void set(byte[] bytes, int scale) {
+ this.internalStorage = bytes;
+ this.scale = scale;
+ }
+
+ public HiveDecimal getHiveDecimal() {
+ return HiveDecimal.create(new BigInteger(internalStorage), scale);
+ }
+
+ /**
+ * Get a HiveDecimal instance from the writable and constraint it with maximum precision/scale.
+ *
+ * @param maxPrecision maximum precision
+ * @param maxScale maximum scale
+ * @return HiveDecimal instance
+ */
+ public HiveDecimal getHiveDecimal(int maxPrecision, int maxScale) {
+ return HiveDecimal.enforcePrecisionScale(HiveDecimal.
+ create(new BigInteger(internalStorage), scale),
+ maxPrecision, maxScale);
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ scale = WritableUtils.readVInt(in);
+ int byteArrayLen = WritableUtils.readVInt(in);
+ if (internalStorage.length != byteArrayLen) {
+ internalStorage = new byte[byteArrayLen];
+ }
+ in.readFully(internalStorage);
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ WritableUtils.writeVInt(out, scale);
+ WritableUtils.writeVInt(out, internalStorage.length);
+ out.write(internalStorage);
+ }
+
+ @Override
+ public int compareTo(HiveDecimalWritable that) {
+ return getHiveDecimal().compareTo(that.getHiveDecimal());
+ }
+
+ @Override
+ public String toString() {
+ return getHiveDecimal().toString();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (this == other) {
+ return true;
+ }
+ if (other == null || getClass() != other.getClass()) {
+ return false;
+ }
+ HiveDecimalWritable bdw = (HiveDecimalWritable) other;
+
+ // 'equals' and 'compareTo' are not compatible with HiveDecimals. We want
+ // compareTo which returns true iff the numbers are equal (e.g.: 3.14 is
+ // the same as 3.140). 'Equals' returns true iff equal and the same scale
+ // is set in the decimals (e.g.: 3.14 is not the same as 3.140)
+ return getHiveDecimal().compareTo(bdw.getHiveDecimal()) == 0;
+ }
+
+ @Override
+ public int hashCode() {
+ return getHiveDecimal().hashCode();
+ }
+
+ /* (non-Javadoc)
+ * In order to update a Decimal128 fast (w/o allocation) we need to expose access to the
+ * internal storage bytes and scale.
+ * @return
+ */
+ public byte[] getInternalStorage() {
+ return internalStorage;
+ }
+
+ /* (non-Javadoc)
+ * In order to update a Decimal128 fast (w/o allocation) we need to expose access to the
+ * internal storage bytes and scale.
+ */
+ public int getScale() {
+ return scale;
+ }
+
+ public static
+ HiveDecimalWritable enforcePrecisionScale(HiveDecimalWritable writable,
+ int precision, int scale) {
+ if (writable == null) {
+ return null;
+ }
+
+ HiveDecimal dec =
+ HiveDecimal.enforcePrecisionScale(writable.getHiveDecimal(), precision,
+ scale);
+ return dec == null ? null : new HiveDecimalWritable(dec);
+ }
+}