You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jd...@apache.org on 2014/07/17 04:01:51 UTC

svn commit: r1611235 - in /hive/trunk/ql/src: java/org/apache/hadoop/hive/ql/exec/vector/expressions/ test/org/apache/hadoop/hive/ql/exec/vector/expressions/ test/org/apache/hadoop/hive/ql/exec/vector/util/

Author: jdere
Date: Thu Jul 17 02:01:51 2014
New Revision: 1611235

URL: http://svn.apache.org/r1611235
Log:
HIVE-7363: VectorExpressionWriterDecimal is missing null check in setValue() (Jason Dere, reviewed by Jitendra Pandey)

Modified:
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java
    hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorExpressionWriters.java
    hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/VectorizedRowGroupGenUtil.java

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java?rev=1611235&r1=1611234&r2=1611235&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java Thu Jul 17 02:01:51 2014
@@ -478,12 +478,15 @@ public final class VectorExpressionWrite
 
       @Override
       public Object setValue(Object field, Decimal128 value) {
+        if (null == field) {
+          field = initValue(null);
+        }
         return ((SettableHiveDecimalObjectInspector) this.objectInspector).set(field,
             HiveDecimal.create(value.toBigDecimal()));
       }
 
       @Override
-      public Object initValue(Object ignored) throws HiveException {
+      public Object initValue(Object ignored) {
         return ((SettableHiveDecimalObjectInspector) this.objectInspector).create(
             HiveDecimal.create(BigDecimal.ZERO));
       }

Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorExpressionWriters.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorExpressionWriters.java?rev=1611235&r1=1611234&r2=1611235&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorExpressionWriters.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorExpressionWriters.java Thu Jul 17 02:01:51 2014
@@ -25,8 +25,11 @@ import java.util.Random;
 
 import junit.framework.Assert;
 
+import org.apache.hadoop.hive.common.type.Decimal128;
+import org.apache.hadoop.hive.common.type.HiveDecimal;
 import org.apache.hadoop.hive.common.type.HiveVarchar;
 import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector;
 import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
 import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
 import org.apache.hadoop.hive.ql.exec.vector.TimestampUtils;
@@ -36,6 +39,7 @@ import org.apache.hadoop.hive.ql.plan.Ex
 import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
 import org.apache.hadoop.hive.serde2.io.ByteWritable;
 import org.apache.hadoop.hive.serde2.io.DoubleWritable;
+import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
 import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable;
 import org.apache.hadoop.hive.serde2.io.ShortWritable;
 import org.apache.hadoop.hive.serde2.io.TimestampWritable;
@@ -44,6 +48,7 @@ import org.apache.hadoop.hive.serde2.obj
 import org.apache.hadoop.hive.serde2.objectinspector.StandardStructObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo;
 import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
 import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
 import org.apache.hadoop.io.BooleanWritable;
@@ -79,6 +84,10 @@ public class TestVectorExpressionWriters
     return null;
   }
 
+  private Writable getWritableValue(TypeInfo ti, Decimal128 value) {
+    return new HiveDecimalWritable(HiveDecimal.create(value.toBigDecimal()));
+  }
+
   private Writable getWritableValue(TypeInfo ti, byte[] value) {
     if (ti.equals(TypeInfoFactory.stringTypeInfo)) {
       return new Text(value);
@@ -135,7 +144,7 @@ public class TestVectorExpressionWriters
     
     VectorExpressionWriter vew = getWriter(type);
     for (int i = 0; i < vectorSize; i++) {
-      values[i] = vew.initValue(null);
+      values[i] = null;  // setValue() should be able to handle null input
       values[i] = vew.setValue(values[i], dcv, i);
       if (values[i] != null) {
         Writable expected = getWritableValue(type, dcv.vector[i]);
@@ -146,6 +155,41 @@ public class TestVectorExpressionWriters
     }
   }  
 
+  private void testWriterDecimal(DecimalTypeInfo type) throws HiveException {
+    DecimalColumnVector dcv = VectorizedRowGroupGenUtil.generateDecimalColumnVector(type, true, false,
+        this.vectorSize, new Random(10));
+    dcv.isNull[2] = true;
+    VectorExpressionWriter vew = getWriter(type);
+    for (int i = 0; i < vectorSize; i++) {
+      Writable w = (Writable) vew.writeValue(dcv, i);
+      if (w != null) {
+        Writable expected = getWritableValue(type, dcv.vector[i]);
+        Assert.assertEquals(expected, w);
+      } else {
+        Assert.assertTrue(dcv.isNull[i]);
+      }
+    }
+  }
+
+  private void testSetterDecimal(DecimalTypeInfo type) throws HiveException {
+    DecimalColumnVector dcv = VectorizedRowGroupGenUtil.generateDecimalColumnVector(type, true, false,
+        this.vectorSize, new Random(10));
+    dcv.isNull[2] = true;
+    Object[] values = new Object[this.vectorSize];
+
+    VectorExpressionWriter vew = getWriter(type);
+    for (int i = 0; i < vectorSize; i++) {
+      values[i] = null;  // setValue() should be able to handle null input
+      values[i] = vew.setValue(values[i], dcv, i);
+      if (values[i] != null) {
+        Writable expected = getWritableValue(type, dcv.vector[i]);
+        Assert.assertEquals(expected, values[i]);
+      } else {
+        Assert.assertTrue(dcv.isNull[i]);
+      }
+    }
+  }
+
   private void testWriterLong(TypeInfo type) throws HiveException {
     LongColumnVector lcv = VectorizedRowGroupGenUtil.generateLongColumnVector(true, false,
         vectorSize, new Random(10));
@@ -178,7 +222,7 @@ public class TestVectorExpressionWriters
     
     VectorExpressionWriter vew = getWriter(type);
     for (int i = 0; i < vectorSize; i++) {
-      values[i] = vew.initValue(null);
+      values[i] = null;  // setValue() should be able to handle null input
       values[i] = vew.setValue(values[i], lcv, i);
       if (values[i] != null) {
         Writable expected = getWritableValue(type, lcv.vector[i]);
@@ -290,7 +334,7 @@ public class TestVectorExpressionWriters
     Object[] values = new Object[this.vectorSize];
     VectorExpressionWriter vew = getWriter(type);
     for (int i = 0; i < vectorSize; i++) {
-      values[i] = vew.initValue(null);
+      values[i] = null;  // setValue() should be able to handle null input
       Writable w = (Writable) vew.setValue(values[i], bcv, i);
       if (w != null) {
         byte [] val = new byte[bcv.length[i]];
@@ -327,7 +371,19 @@ public class TestVectorExpressionWriters
   public void testVectorExpressionWriterLong() throws HiveException {
     testWriterLong(TypeInfoFactory.longTypeInfo);
   }
-  
+
+  @Test
+  public void testVectorExpressionWriterDecimal() throws HiveException {
+    DecimalTypeInfo typeInfo = TypeInfoFactory.getDecimalTypeInfo(38, 18);
+    testWriterDecimal(typeInfo);
+  }
+
+  @Test
+  public void testVectorExpressionSetterDecimal() throws HiveException {
+    DecimalTypeInfo typeInfo = TypeInfoFactory.getDecimalTypeInfo(38, 18);
+    testSetterDecimal(typeInfo);
+  }
+
   @Test
   public void testVectorExpressionSetterLong() throws HiveException {
     testSetterLong(TypeInfoFactory.longTypeInfo);

Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/VectorizedRowGroupGenUtil.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/VectorizedRowGroupGenUtil.java?rev=1611235&r1=1611234&r2=1611235&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/VectorizedRowGroupGenUtil.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/VectorizedRowGroupGenUtil.java Thu Jul 17 02:01:51 2014
@@ -20,9 +20,12 @@ package org.apache.hadoop.hive.ql.exec.v
 
 import java.util.Random;
 
+import org.apache.hadoop.hive.common.type.Decimal128;
+import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector;
 import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
 import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
 import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo;
 
 public class VectorizedRowGroupGenUtil {
 
@@ -103,6 +106,42 @@ public class VectorizedRowGroupGenUtil {
     return dcv;
   }
 
+  public static DecimalColumnVector generateDecimalColumnVector(DecimalTypeInfo typeInfo, boolean nulls,
+      boolean repeating, int size, Random rand) {
+    DecimalColumnVector dcv =
+        new DecimalColumnVector(size, typeInfo.precision(), typeInfo.scale());
+
+    dcv.noNulls = !nulls;
+    dcv.isRepeating = repeating;
+
+    Decimal128 repeatingValue = new Decimal128();
+    do{
+      repeatingValue.update(rand.nextDouble(), (short)typeInfo.scale());
+    }while(repeatingValue.doubleValue() == 0);
+
+    int nullFrequency = generateNullFrequency(rand);
+
+    for(int i = 0; i < size; i++) {
+      if(nulls && (repeating || i % nullFrequency == 0)) {
+        dcv.isNull[i] = true;
+        dcv.vector[i] = null;//Decimal128.ONE;
+
+      }else {
+        dcv.isNull[i] = false;
+        if (repeating) {
+          dcv.vector[i].update(repeatingValue);
+        } else {
+          dcv.vector[i].update(rand.nextDouble(), (short) typeInfo.scale());
+        }
+
+        if(dcv.vector[i].doubleValue() == 0) {
+          i--;
+        }
+      }
+    }
+    return dcv;
+  }
+
   private static int generateNullFrequency(Random rand) {
     return 60 + rand.nextInt(20);
   }