You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by gu...@apache.org on 2013/10/05 01:35:53 UTC
svn commit: r1529347 [4/6] - in /hive/branches/tez: ./
ant/src/org/apache/hadoop/hive/ant/ beeline/src/java/org/apache/hive/beeline/
beeline/src/test/org/apache/hive/beeline/src/test/ bin/ common/
common/src/test/org/apache/hadoop/hive/common/type/ dat...
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionEvaluator.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionEvaluator.java?rev=1529347&r1=1529346&r2=1529347&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionEvaluator.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionEvaluator.java Fri Oct 4 23:35:50 2013
@@ -24,7 +24,7 @@ import org.apache.hadoop.hive.ql.exec.PT
import org.apache.hadoop.hive.ql.exec.PTFUtils;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.plan.PTFDesc;
-import org.apache.hadoop.hive.ql.plan.PTFDesc.PartitionedTableFunctionDef;
+import org.apache.hadoop.hive.ql.plan.ptf.PartitionedTableFunctionDef;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
@@ -46,8 +46,7 @@ import org.apache.hadoop.hive.serde2.obj
* </ol>
*
*/
-public abstract class TableFunctionEvaluator
-{
+public abstract class TableFunctionEvaluator {
/*
* how is this different from the OutpuShape set on the TableDef.
* This is the OI of the object coming out of the PTF.
@@ -59,52 +58,45 @@ public abstract class TableFunctionEvalu
* same comment as OI applies here.
*/
transient protected StructObjectInspector rawInputOI;
- protected PartitionedTableFunctionDef tDef;
+ protected PartitionedTableFunctionDef tableDef;
protected PTFDesc ptfDesc;
boolean transformsRawInput;
transient protected PTFPartition outputPartition;
- static{
+ static {
+ //TODO is this a bug? The field is not named outputOI it is named OI
PTFUtils.makeTransient(TableFunctionEvaluator.class, "outputOI", "rawInputOI");
}
- public StructObjectInspector getOutputOI()
- {
+ public StructObjectInspector getOutputOI() {
return OI;
}
- protected void setOutputOI(StructObjectInspector outputOI)
- {
+ protected void setOutputOI(StructObjectInspector outputOI) {
OI = outputOI;
}
- public PartitionedTableFunctionDef getTableDef()
- {
- return tDef;
+ public PartitionedTableFunctionDef getTableDef() {
+ return tableDef;
}
- public void setTableDef(PartitionedTableFunctionDef tDef)
- {
- this.tDef = tDef;
+ public void setTableDef(PartitionedTableFunctionDef tDef) {
+ this.tableDef = tDef;
}
- protected PTFDesc getQueryDef()
- {
+ protected PTFDesc getQueryDef() {
return ptfDesc;
}
- protected void setQueryDef(PTFDesc ptfDesc)
- {
+ protected void setQueryDef(PTFDesc ptfDesc) {
this.ptfDesc = ptfDesc;
}
- public StructObjectInspector getRawInputOI()
- {
+ public StructObjectInspector getRawInputOI() {
return rawInputOI;
}
- protected void setRawInputOI(StructObjectInspector rawInputOI)
- {
+ protected void setRawInputOI(StructObjectInspector rawInputOI) {
this.rawInputOI = rawInputOI;
}
@@ -117,17 +109,15 @@ public abstract class TableFunctionEvalu
}
public PTFPartition execute(PTFPartition iPart)
- throws HiveException
- {
+ throws HiveException {
PTFPartitionIterator<Object> pItr = iPart.iterator();
PTFOperator.connectLeadLagFunctionsToPartition(ptfDesc, pItr);
if ( outputPartition == null ) {
outputPartition = PTFPartition.create(ptfDesc.getCfg(),
- tDef.getOutputShape().getSerde(),
- OI, tDef.getOutputShape().getOI());
- }
- else {
+ tableDef.getOutputShape().getSerde(),
+ OI, tableDef.getOutputShape().getOI());
+ } else {
outputPartition.reset();
}
@@ -137,17 +127,14 @@ public abstract class TableFunctionEvalu
protected abstract void execute(PTFPartitionIterator<Object> pItr, PTFPartition oPart) throws HiveException;
- public PTFPartition transformRawInput(PTFPartition iPart) throws HiveException
- {
- if ( !isTransformsRawInput())
- {
- throw new HiveException(String.format("Internal Error: mapExecute called on function (%s)that has no Map Phase", tDef.getName()));
+ public PTFPartition transformRawInput(PTFPartition iPart) throws HiveException {
+ if (!isTransformsRawInput()) {
+ throw new HiveException(String.format("Internal Error: mapExecute called on function (%s)that has no Map Phase", tableDef.getName()));
}
return _transformRawInput(iPart);
}
- protected PTFPartition _transformRawInput(PTFPartition iPart) throws HiveException
- {
+ protected PTFPartition _transformRawInput(PTFPartition iPart) throws HiveException {
return null;
}
}
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionResolver.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionResolver.java?rev=1529347&r1=1529346&r2=1529347&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionResolver.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionResolver.java Fri Oct 4 23:35:50 2013
@@ -18,7 +18,7 @@
package org.apache.hadoop.hive.ql.udf.ptf;
-import java.util.ArrayList;
+import java.util.List;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator;
@@ -27,7 +27,7 @@ import org.apache.hadoop.hive.ql.metadat
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.PTFDesc;
-import org.apache.hadoop.hive.ql.plan.PTFDesc.PartitionedTableFunctionDef;
+import org.apache.hadoop.hive.ql.plan.ptf.PartitionedTableFunctionDef;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFResolver;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
@@ -53,8 +53,7 @@ import org.apache.hadoop.hive.serde2.obj
* </ol>
*/
@SuppressWarnings("deprecation")
-public abstract class TableFunctionResolver
-{
+public abstract class TableFunctionResolver {
TableFunctionEvaluator evaluator;
PTFDesc ptfDesc;
@@ -65,8 +64,7 @@ public abstract class TableFunctionResol
* the transformsRawInput boolean.
*/
public void initialize(HiveConf cfg, PTFDesc ptfDesc, PartitionedTableFunctionDef tDef)
- throws SemanticException
- {
+ throws SemanticException {
this.ptfDesc = ptfDesc;
evaluator = createEvaluator(ptfDesc, tDef);
@@ -79,16 +77,14 @@ public abstract class TableFunctionResol
* called during deserialization of a QueryDef during runtime.
*/
public void initialize(PTFDesc ptfDesc, PartitionedTableFunctionDef tDef, TableFunctionEvaluator evaluator)
- throws HiveException
- {
+ throws HiveException {
this.evaluator = evaluator;
this.ptfDesc = ptfDesc;
evaluator.setTableDef(tDef);
evaluator.setQueryDef(ptfDesc);
}
- public TableFunctionEvaluator getEvaluator()
- {
+ public TableFunctionEvaluator getEvaluator() {
return evaluator;
}
@@ -105,7 +101,7 @@ public abstract class TableFunctionResol
* A PTF Function must provide the 'external' names of the columns in its Output.
*
*/
- public abstract ArrayList<String> getOutputColumnNames() throws SemanticException;
+ public abstract List<String> getOutputColumnNames() throws SemanticException;
/**
@@ -127,10 +123,8 @@ public abstract class TableFunctionResol
* - subsequent to this call, a call to getRawInputOI call on the {@link TableFunctionEvaluator} must return the OI
* of the output of this function.
*/
- public void setupRawInputOI() throws SemanticException
- {
- if (!transformsRawInput())
- {
+ public void setupRawInputOI() throws SemanticException {
+ if (!transformsRawInput()) {
return;
}
throw new SemanticException(
@@ -141,9 +135,8 @@ public abstract class TableFunctionResol
* A PTF Function must provide the 'external' names of the columns in the transformed Raw Input.
*
*/
- public ArrayList<String> getRawInputColumnNames() throws SemanticException {
- if (!transformsRawInput())
- {
+ public List<String> getRawInputColumnNames() throws SemanticException {
+ if (!transformsRawInput()) {
return null;
}
throw new SemanticException(
@@ -153,10 +146,8 @@ public abstract class TableFunctionResol
/*
* Same responsibility as initializeOI, but for the RawInput.
*/
- public void initializeRawInputOI() throws HiveException
- {
- if (!transformsRawInput())
- {
+ public void initializeRawInputOI() throws HiveException {
+ if (!transformsRawInput()) {
return;
}
throw new HiveException(
@@ -166,21 +157,18 @@ public abstract class TableFunctionResol
/*
* callback method used by subclasses to set the RawInputOI on the Evaluator.
*/
- protected void setRawInputOI(StructObjectInspector rawInputOI)
- {
+ protected void setRawInputOI(StructObjectInspector rawInputOI) {
evaluator.setRawInputOI(rawInputOI);
}
/*
* callback method used by subclasses to set the OutputOI on the Evaluator.
*/
- protected void setOutputOI(StructObjectInspector outputOI)
- {
+ protected void setOutputOI(StructObjectInspector outputOI) {
evaluator.setOutputOI(outputOI);
}
- public PTFDesc getPtfDesc()
- {
+ public PTFDesc getPtfDesc() {
return ptfDesc;
}
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/WindowingTableFunction.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/WindowingTableFunction.java?rev=1529347&r1=1529346&r2=1529347&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/WindowingTableFunction.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/WindowingTableFunction.java Fri Oct 4 23:35:50 2013
@@ -31,13 +31,13 @@ import org.apache.hadoop.hive.ql.parse.S
import org.apache.hadoop.hive.ql.parse.WindowingSpec.BoundarySpec;
import org.apache.hadoop.hive.ql.parse.WindowingSpec.Direction;
import org.apache.hadoop.hive.ql.plan.PTFDesc;
-import org.apache.hadoop.hive.ql.plan.PTFDesc.BoundaryDef;
-import org.apache.hadoop.hive.ql.plan.PTFDesc.PTFExpressionDef;
-import org.apache.hadoop.hive.ql.plan.PTFDesc.PartitionedTableFunctionDef;
-import org.apache.hadoop.hive.ql.plan.PTFDesc.ValueBoundaryDef;
-import org.apache.hadoop.hive.ql.plan.PTFDesc.WindowFrameDef;
-import org.apache.hadoop.hive.ql.plan.PTFDesc.WindowFunctionDef;
-import org.apache.hadoop.hive.ql.plan.PTFDesc.WindowTableFunctionDef;
+import org.apache.hadoop.hive.ql.plan.ptf.BoundaryDef;
+import org.apache.hadoop.hive.ql.plan.ptf.PTFExpressionDef;
+import org.apache.hadoop.hive.ql.plan.ptf.PartitionedTableFunctionDef;
+import org.apache.hadoop.hive.ql.plan.ptf.ValueBoundaryDef;
+import org.apache.hadoop.hive.ql.plan.ptf.WindowFrameDef;
+import org.apache.hadoop.hive.ql.plan.ptf.WindowFunctionDef;
+import org.apache.hadoop.hive.ql.plan.ptf.WindowTableFunctionDef;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.AggregationBuffer;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/util/JavaDataModel.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/util/JavaDataModel.java?rev=1529347&r1=1529346&r2=1529347&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/util/JavaDataModel.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/util/JavaDataModel.java Fri Oct 4 23:35:50 2013
@@ -157,7 +157,7 @@ public enum JavaDataModel {
// ascii string
public int lengthFor(String string) {
- return object() + primitive1() * 3 + array() + string.length();
+ return lengthForStringOfLength(string.length());
}
public int lengthFor(NumericHistogram histogram) {
@@ -267,4 +267,33 @@ public enum JavaDataModel {
public int lengthForBooleanArrayOfSize(int length) {
return lengthForPrimitiveArrayOfSize(PRIMITIVE_BYTE, length);
}
+
+ public int lengthOfDecimal() {
+ // object overhead + 8 bytes for intCompact + 4 bytes for precision
+ // + 4 bytes for scale + size of BigInteger
+ return object() + 2 * primitive2() + lengthOfBigInteger();
+ }
+
+ private int lengthOfBigInteger() {
+ // object overhead + 4 bytes for bitCount + 4 bytes for bitLength
+ // + 4 bytes for firstNonzeroByteNum + 4 bytes for firstNonzeroIntNum +
+ // + 4 bytes for lowestSetBit + 5 bytes for size of magnitude (since max precision
+ // is only 38 for HiveDecimal) + 7 bytes of padding (since java memory allocations
+ // are 8 byte aligned)
+ return object() + 4 * primitive2();
+ }
+
+ public int lengthOfTimestamp() {
+ // object overhead + 4 bytes for int (nanos) + 4 bytes of padding
+ return object() + primitive2();
+ }
+
+ public int lengthOfDate() {
+ // object overhead + 8 bytes for long (fastTime) + 16 bytes for cdate
+ return object() + 3 * primitive2();
+ }
+
+ public int lengthForStringOfLength(int strLen) {
+ return object() + primitive1() * 3 + array() + strLen;
+ }
}
Modified: hive/branches/tez/ql/src/protobuf/org/apache/hadoop/hive/ql/io/orc/orc_proto.proto
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/protobuf/org/apache/hadoop/hive/ql/io/orc/orc_proto.proto?rev=1529347&r1=1529346&r2=1529347&view=diff
==============================================================================
--- hive/branches/tez/ql/src/protobuf/org/apache/hadoop/hive/ql/io/orc/orc_proto.proto (original)
+++ hive/branches/tez/ql/src/protobuf/org/apache/hadoop/hive/ql/io/orc/orc_proto.proto Fri Oct 4 23:35:50 2013
@@ -15,6 +15,8 @@ message DoubleStatistics {
message StringStatistics {
optional string minimum = 1;
optional string maximum = 2;
+ // sum will store the total length of all strings in a stripe
+ optional sint64 sum = 3;
}
message BucketStatistics {
@@ -33,6 +35,11 @@ message DateStatistics {
optional sint32 maximum = 2;
}
+message BinaryStatistics {
+ // sum will store the total binary blob length in a stripe
+ optional sint64 sum = 1;
+}
+
message ColumnStatistics {
optional uint64 numberOfValues = 1;
optional IntegerStatistics intStatistics = 2;
@@ -41,6 +48,7 @@ message ColumnStatistics {
optional BucketStatistics bucketStatistics = 5;
optional DecimalStatistics decimalStatistics = 6;
optional DateStatistics dateStatistics = 7;
+ optional BinaryStatistics binaryStatistics = 8;
}
message RowIndexEntry {
Modified: hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java?rev=1529347&r1=1529346&r2=1529347&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java (original)
+++ hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java Fri Oct 4 23:35:50 2013
@@ -315,12 +315,7 @@ public class QTestUtil {
testFiles = dataDir;
- String ow = System.getProperty("test.output.overwrite");
- if ((ow != null) && ow.equalsIgnoreCase("true")) {
- overWrite = true;
- } else {
- overWrite = false;
- }
+ overWrite = "true".equalsIgnoreCase(System.getProperty("test.output.overwrite"));
setup = new QTestSetup();
setup.preTest(conf);
@@ -1534,4 +1529,15 @@ public class QTestUtil {
+ "or try \"ant test ... -Dtest.silent=false\" to get more logs.");
System.err.flush();
}
+
+ public static String ensurePathEndsInSlash(String path) {
+ if(path == null) {
+ throw new NullPointerException("Path cannot be null");
+ }
+ if(path.endsWith(File.separator)) {
+ return path;
+ } else {
+ return path + File.separator;
+ }
+ }
}
Modified: hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorStringExpressions.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorStringExpressions.java?rev=1529347&r1=1529346&r2=1529347&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorStringExpressions.java (original)
+++ hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorStringExpressions.java Fri Oct 4 23:35:50 2013
@@ -670,28 +670,28 @@ public class TestVectorStringExpressions
// BEGIN pattern
expr = new FilterStringColLikeStringScalar(0, new Text("abc%"));
- Assert.assertEquals(FilterStringColLikeStringScalar.PatternType.BEGIN,
- expr.getType());
+ Assert.assertEquals(FilterStringColLikeStringScalar.BeginChecker.class,
+ expr.checker.getClass());
// END pattern
expr = new FilterStringColLikeStringScalar(0, new Text("%abc"));
- Assert.assertEquals(FilterStringColLikeStringScalar.PatternType.END,
- expr.getType());
+ Assert.assertEquals(FilterStringColLikeStringScalar.EndChecker.class,
+ expr.checker.getClass());
// MIDDLE pattern
expr = new FilterStringColLikeStringScalar(0, new Text("%abc%"));
- Assert.assertEquals(FilterStringColLikeStringScalar.PatternType.MIDDLE,
- expr.getType());
+ Assert.assertEquals(FilterStringColLikeStringScalar.MiddleChecker.class,
+ expr.checker.getClass());
// COMPLEX pattern
expr = new FilterStringColLikeStringScalar(0, new Text("%abc%de"));
- Assert.assertEquals(FilterStringColLikeStringScalar.PatternType.COMPLEX,
- expr.getType());
+ Assert.assertEquals(FilterStringColLikeStringScalar.ComplexChecker.class,
+ expr.checker.getClass());
// NONE pattern
expr = new FilterStringColLikeStringScalar(0, new Text("abc"));
- Assert.assertEquals(FilterStringColLikeStringScalar.PatternType.NONE,
- expr.getType());
+ Assert.assertEquals(FilterStringColLikeStringScalar.NoneChecker.class,
+ expr.checker.getClass());
}
public void testStringLikeMultiByte() {
Modified: hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java?rev=1529347&r1=1529346&r2=1529347&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java (original)
+++ hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java Fri Oct 4 23:35:50 2013
@@ -18,12 +18,28 @@
package org.apache.hadoop.hive.ql.io.orc;
+import static junit.framework.Assert.assertEquals;
+import static junit.framework.Assert.assertNotNull;
+import static junit.framework.Assert.assertNull;
+import static junit.framework.Assert.assertTrue;
+
+import java.io.File;
+import java.io.IOException;
+import java.math.BigInteger;
+import java.nio.ByteBuffer;
+import java.sql.Timestamp;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
-import org.apache.hadoop.hive.ql.io.sarg.TestSearchArgumentImpl;
import org.apache.hadoop.hive.serde2.io.ByteWritable;
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
import org.apache.hadoop.hive.serde2.io.ShortWritable;
@@ -55,26 +71,25 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TestName;
-import java.io.File;
-import java.io.IOException;
-import java.math.BigInteger;
-import java.nio.ByteBuffer;
-import java.sql.Timestamp;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-
-import static junit.framework.Assert.*;
-import static junit.framework.Assert.assertEquals;
-
/**
* Tests for the top level reader/streamFactory of ORC files.
*/
public class TestOrcFile {
+ public static class SimpleStruct {
+ BytesWritable bytes1;
+ Text string1;
+
+ SimpleStruct(BytesWritable b1, String s1) {
+ this.bytes1 = b1;
+ if(s1 == null) {
+ this.string1 = null;
+ } else {
+ this.string1 = new Text(s1);
+ }
+ }
+ }
+
public static class InnerStruct {
int int1;
Text string1 = new Text();
@@ -132,48 +147,6 @@ public class TestOrcFile {
}
}
- public static class AllTypesRow {
- Boolean boolean1;
- Byte byte1;
- Short short1;
- Integer int1;
- Long long1;
- Float float1;
- Double double1;
- BytesWritable bytes1;
- Text string1;
- MiddleStruct middle;
- List<InnerStruct> list = new ArrayList<InnerStruct>();
- Map<Text, InnerStruct> map = new HashMap<Text, InnerStruct>();
- Timestamp ts;
- HiveDecimal decimal1;
-
- AllTypesRow(Boolean b1, Byte b2, Short s1, Integer i1, Long l1, Float f1,
- Double d1,
- BytesWritable b3, String s2, MiddleStruct m1,
- List<InnerStruct> l2, Map<Text, InnerStruct> m2,
- Timestamp ts1, HiveDecimal decimal) {
- this.boolean1 = b1;
- this.byte1 = b2;
- this.short1 = s1;
- this.int1 = i1;
- this.long1 = l1;
- this.float1 = f1;
- this.double1 = d1;
- this.bytes1 = b3;
- if (s2 == null) {
- this.string1 = null;
- } else {
- this.string1 = new Text(s2);
- }
- this.middle = m1;
- this.list = l2;
- this.map = m2;
- this.ts = ts1;
- this.decimal1 = decimal;
- }
- }
-
private static InnerStruct inner(int i, String s) {
return new InnerStruct(i, s);
}
@@ -231,39 +204,6 @@ public class TestOrcFile {
}
@Test
- public void testWriteFormat_0_11() throws Exception {
- ObjectInspector inspector;
- synchronized (TestOrcFile.class) {
- inspector = ObjectInspectorFactory
- .getReflectionObjectInspector(AllTypesRow.class,
- ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
- }
- conf.set("hive.exec.orc.write.format", "0.11");
- Writer writer = OrcFile.createWriter(fs, testFilePath, conf, inspector,
- 100000, CompressionKind.NONE, 10000, 10000);
- for(int i = 0; i < 7500; i++) {
- if (i % 2 == 0) {
- writer.addRow(new AllTypesRow(false, (byte) 1, (short) 1024, 65536,
- Long.MAX_VALUE, (float) 1.0, -15.0, bytes(0, 1, 2, 3, 4), "hi",
- new MiddleStruct(inner(1, "bye"), inner(2, "sigh")), list(
- inner(3, "good"), inner(4, "bad")), map(), Timestamp
- .valueOf("2000-03-12 15:00:00"), new HiveDecimal(
- "12345678.6547456")));
- } else {
- writer.addRow(new AllTypesRow(true, (byte) 100, (short) 2048, 65536,
- Long.MAX_VALUE, (float) 2.0, -5.0, bytes(), "bye",
- new MiddleStruct(inner(1, "bye"), inner(2, "sigh")), list(
- inner(100000000, "cat"), inner(-100000, "in"),
- inner(1234, "hat")),
- map(inner(5, "chani"), inner(1, "mauddib")), Timestamp
- .valueOf("2000-03-12 15:00:01"), new HiveDecimal(
- "12345678.6547457")));
- }
- }
- writer.close();
- }
-
- @Test
public void testReadFormat_0_11() throws Exception {
Path resourceDir = new Path(System.getProperty("test.build.resources", "ql"
+ File.separator + "src" + File.separator + "test" + File.separator
@@ -319,7 +259,7 @@ public class TestOrcFile {
assertEquals("count: 7500 min: -15.0 max: -5.0 sum: -75000.0",
stats[7].toString());
- assertEquals("count: 7500 min: bye max: hi", stats[9].toString());
+ assertEquals("count: 7500 min: bye max: hi sum: 0", stats[9].toString());
// check the inspectors
StructObjectInspector readerInspector = (StructObjectInspector) reader
@@ -515,6 +455,93 @@ public class TestOrcFile {
}
@Test
+ public void testStringAndBinaryStatistics() throws Exception {
+
+ ObjectInspector inspector;
+ synchronized (TestOrcFile.class) {
+ inspector = ObjectInspectorFactory.getReflectionObjectInspector
+ (SimpleStruct.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
+ }
+ Writer writer = OrcFile.createWriter(testFilePath,
+ OrcFile.writerOptions(conf)
+ .inspector(inspector)
+ .stripeSize(100000)
+ .bufferSize(10000));
+ writer.addRow(new SimpleStruct(bytes(0,1,2,3,4), "foo"));
+ writer.addRow(new SimpleStruct(bytes(0,1,2,3), "bar"));
+ writer.addRow(new SimpleStruct(bytes(0,1,2,3,4,5), null));
+ writer.addRow(new SimpleStruct(null, "hi"));
+ writer.close();
+ Reader reader = OrcFile.createReader(fs, testFilePath);
+
+ // check the stats
+ ColumnStatistics[] stats = reader.getStatistics();
+ assertEquals(4, stats[0].getNumberOfValues());
+ assertEquals("count: 4", stats[0].toString());
+
+ assertEquals(3, stats[1].getNumberOfValues());
+ assertEquals(15, ((BinaryColumnStatistics) stats[1]).getSum());
+ assertEquals("count: 3 sum: 15", stats[1].toString());
+
+ assertEquals(3, stats[2].getNumberOfValues());
+ assertEquals("bar", ((StringColumnStatistics) stats[2]).getMinimum());
+ assertEquals("hi", ((StringColumnStatistics) stats[2]).getMaximum());
+ assertEquals(8, ((StringColumnStatistics) stats[2]).getSum());
+ assertEquals("count: 3 min: bar max: hi sum: 8",
+ stats[2].toString());
+
+ // check the inspectors
+ StructObjectInspector readerInspector =
+ (StructObjectInspector) reader.getObjectInspector();
+ assertEquals(ObjectInspector.Category.STRUCT,
+ readerInspector.getCategory());
+ assertEquals("struct<bytes1:binary,string1:string>",
+ readerInspector.getTypeName());
+ List<? extends StructField> fields =
+ readerInspector.getAllStructFieldRefs();
+ BinaryObjectInspector bi = (BinaryObjectInspector) readerInspector.
+ getStructFieldRef("bytes1").getFieldObjectInspector();
+ StringObjectInspector st = (StringObjectInspector) readerInspector.
+ getStructFieldRef("string1").getFieldObjectInspector();
+ RecordReader rows = reader.rows(null);
+ Object row = rows.next(null);
+ assertNotNull(row);
+ // check the contents of the first row
+ assertEquals(bytes(0,1,2,3,4), bi.getPrimitiveWritableObject(
+ readerInspector.getStructFieldData(row, fields.get(0))));
+ assertEquals("foo", st.getPrimitiveJavaObject(readerInspector.
+ getStructFieldData(row, fields.get(1))));
+
+ // check the contents of second row
+ assertEquals(true, rows.hasNext());
+ row = rows.next(row);
+ assertEquals(bytes(0,1,2,3), bi.getPrimitiveWritableObject(
+ readerInspector.getStructFieldData(row, fields.get(0))));
+ assertEquals("bar", st.getPrimitiveJavaObject(readerInspector.
+ getStructFieldData(row, fields.get(1))));
+
+ // check the contents of second row
+ assertEquals(true, rows.hasNext());
+ row = rows.next(row);
+ assertEquals(bytes(0,1,2,3,4,5), bi.getPrimitiveWritableObject(
+ readerInspector.getStructFieldData(row, fields.get(0))));
+ assertNull(st.getPrimitiveJavaObject(readerInspector.
+ getStructFieldData(row, fields.get(1))));
+
+ // check the contents of second row
+ assertEquals(true, rows.hasNext());
+ row = rows.next(row);
+ assertNull(bi.getPrimitiveWritableObject(
+ readerInspector.getStructFieldData(row, fields.get(0))));
+ assertEquals("hi", st.getPrimitiveJavaObject(readerInspector.
+ getStructFieldData(row, fields.get(1))));
+
+ // handle the close up
+ assertEquals(false, rows.hasNext());
+ rows.close();
+ }
+
+ @Test
public void test1() throws Exception {
ObjectInspector inspector;
synchronized (TestOrcFile.class) {
@@ -567,7 +594,7 @@ public class TestOrcFile {
assertEquals("count: 2 min: -15.0 max: -5.0 sum: -20.0",
stats[7].toString());
- assertEquals("count: 2 min: bye max: hi", stats[9].toString());
+ assertEquals("count: 2 min: bye max: hi sum: 5", stats[9].toString());
// check the inspectors
StructObjectInspector readerInspector =
Modified: hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcNullOptimization.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcNullOptimization.java?rev=1529347&r1=1529346&r2=1529347&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcNullOptimization.java (original)
+++ hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcNullOptimization.java Fri Oct 4 23:35:50 2013
@@ -125,7 +125,7 @@ public class TestOrcNullOptimization {
assertEquals("a", ((StringColumnStatistics) stats[2]).getMinimum());
assertEquals(19998,
((StringColumnStatistics) stats[2]).getNumberOfValues());
- assertEquals("count: 19998 min: a max: a",
+ assertEquals("count: 19998 min: a max: a sum: 19998",
stats[2].toString());
// check the inspectors
@@ -229,7 +229,7 @@ public class TestOrcNullOptimization {
assertEquals("a", ((StringColumnStatistics) stats[2]).getMinimum());
assertEquals(20000,
((StringColumnStatistics) stats[2]).getNumberOfValues());
- assertEquals("count: 20000 min: a max: b",
+ assertEquals("count: 20000 min: a max: b sum: 20000",
stats[2].toString());
// check the inspectors
@@ -329,7 +329,7 @@ public class TestOrcNullOptimization {
assertEquals("h", ((StringColumnStatistics) stats[2]).getMaximum());
assertEquals("a", ((StringColumnStatistics) stats[2]).getMinimum());
assertEquals(7, ((StringColumnStatistics) stats[2]).getNumberOfValues());
- assertEquals("count: 7 min: a max: h",
+ assertEquals("count: 7 min: a max: h sum: 7",
stats[2].toString());
// check the inspectors
Modified: hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveMetaStoreChecker.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveMetaStoreChecker.java?rev=1529347&r1=1529346&r2=1529347&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveMetaStoreChecker.java (original)
+++ hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveMetaStoreChecker.java Fri Oct 4 23:35:50 2013
@@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.metada
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -47,8 +48,8 @@ public class TestHiveMetaStoreChecker ex
private FileSystem fs;
private HiveMetaStoreChecker checker = null;
- private final String dbName = "dbname";
- private final String tableName = "tablename";
+ private final String dbName = "testhivemetastorechecker_db";
+ private final String tableName = "testhivemetastorechecker_table";
private final String partDateName = "partdate";
private final String partCityName = "partcity";
@@ -76,17 +77,25 @@ public class TestHiveMetaStoreChecker ex
part2.put(partCityName, "stockholm");
parts.add(part2);
+ //cleanup just in case something is left over from previous run
+ dropDbTable();
+ }
+
+ private void dropDbTable() {
// cleanup
- hive.dropTable(dbName, tableName, true, true);
try {
- hive.dropDatabase(dbName);
+ hive.dropTable(dbName, tableName, true, true);
+ hive.dropDatabase(dbName, true, true, true);
} catch (NoSuchObjectException e) {
// ignore
+ } catch (HiveException e) {
+ // ignore
}
}
@Override
protected void tearDown() throws Exception {
+ dropDbTable();
super.tearDown();
Hive.closeCurrent();
}
@@ -97,19 +106,19 @@ public class TestHiveMetaStoreChecker ex
CheckResult result = new CheckResult();
checker.checkMetastore(dbName, null, null, result);
// we haven't added anything so should return an all ok
- assertTrue(result.getTablesNotInMs().isEmpty());
- assertTrue(result.getTablesNotOnFs().isEmpty());
- assertTrue(result.getPartitionsNotOnFs().isEmpty());
- assertTrue(result.getPartitionsNotInMs().isEmpty());
+ assertEquals(Collections.<String>emptyList(), result.getTablesNotInMs());
+ assertEquals(Collections.<String>emptyList(), result.getTablesNotOnFs());
+ assertEquals(Collections.<String>emptyList(), result.getPartitionsNotOnFs());
+ assertEquals(Collections.<String>emptyList(), result.getPartitionsNotInMs());
// check table only, should not exist in ms
result = new CheckResult();
checker.checkMetastore(dbName, tableName, null, result);
assertEquals(1, result.getTablesNotInMs().size());
assertEquals(tableName, result.getTablesNotInMs().get(0));
- assertTrue(result.getTablesNotOnFs().isEmpty());
- assertTrue(result.getPartitionsNotOnFs().isEmpty());
- assertTrue(result.getPartitionsNotInMs().isEmpty());
+ assertEquals(Collections.<String>emptyList(), result.getTablesNotOnFs());
+ assertEquals(Collections.<String>emptyList(), result.getPartitionsNotOnFs());
+ assertEquals(Collections.<String>emptyList(), result.getPartitionsNotInMs());
Database db = new Database();
db.setName(dbName);
@@ -125,18 +134,18 @@ public class TestHiveMetaStoreChecker ex
// first check all (1) tables
result = new CheckResult();
checker.checkMetastore(dbName, null, null, result);
- assertTrue(result.getTablesNotInMs().isEmpty());
- assertTrue(result.getTablesNotOnFs().isEmpty());
- assertTrue(result.getPartitionsNotOnFs().isEmpty());
- assertTrue(result.getPartitionsNotInMs().isEmpty());
+ assertEquals(Collections.<String>emptyList(), result.getTablesNotInMs());
+ assertEquals(Collections.<String>emptyList(), result.getTablesNotOnFs());
+ assertEquals(Collections.<String>emptyList(), result.getPartitionsNotOnFs());
+ assertEquals(Collections.<String>emptyList(), result.getPartitionsNotInMs());
// then let's check the one we know about
result = new CheckResult();
checker.checkMetastore(dbName, tableName, null, result);
- assertTrue(result.getTablesNotInMs().isEmpty());
- assertTrue(result.getTablesNotOnFs().isEmpty());
- assertTrue(result.getPartitionsNotOnFs().isEmpty());
- assertTrue(result.getPartitionsNotInMs().isEmpty());
+ assertEquals(Collections.<String>emptyList(), result.getTablesNotInMs());
+ assertEquals(Collections.<String>emptyList(), result.getTablesNotOnFs());
+ assertEquals(Collections.<String>emptyList(), result.getPartitionsNotOnFs());
+ assertEquals(Collections.<String>emptyList(), result.getPartitionsNotInMs());
// remove the table folder
fs = table.getPath().getFileSystem(hive.getConf());
@@ -145,26 +154,27 @@ public class TestHiveMetaStoreChecker ex
// now this shouldn't find the path on the fs
result = new CheckResult();
checker.checkMetastore(dbName, tableName, null, result);
- assertTrue(result.getTablesNotInMs().isEmpty());
+ assertEquals(Collections.<String>emptyList(), result.getTablesNotInMs());;
assertEquals(1, result.getTablesNotOnFs().size());
assertEquals(tableName, result.getTablesNotOnFs().get(0));
- assertTrue(result.getPartitionsNotOnFs().isEmpty());
- assertTrue(result.getPartitionsNotInMs().isEmpty());
+ assertEquals(Collections.<String>emptyList(), result.getPartitionsNotOnFs());
+ assertEquals(Collections.<String>emptyList(), result.getPartitionsNotInMs());
// put it back and one additional table
fs.mkdirs(table.getPath());
Path fakeTable = table.getPath().getParent().suffix(
Path.SEPARATOR + "faketable");
fs.mkdirs(fakeTable);
+ fs.deleteOnExit(fakeTable);
// find the extra table
result = new CheckResult();
checker.checkMetastore(dbName, null, null, result);
assertEquals(1, result.getTablesNotInMs().size());
assertEquals(fakeTable.getName(), result.getTablesNotInMs().get(0));
- assertTrue(result.getTablesNotOnFs().isEmpty());
- assertTrue(result.getPartitionsNotOnFs().isEmpty());
- assertTrue(result.getPartitionsNotInMs().isEmpty());
+ assertEquals(Collections.<String>emptyList(), result.getTablesNotOnFs());
+ assertEquals(Collections.<String>emptyList(), result.getPartitionsNotOnFs());
+ assertEquals(Collections.<String>emptyList(), result.getPartitionsNotInMs());
// create a new external table
hive.dropTable(dbName, tableName);
@@ -174,10 +184,10 @@ public class TestHiveMetaStoreChecker ex
// should return all ok
result = new CheckResult();
checker.checkMetastore(dbName, null, null, result);
- assertTrue(result.getTablesNotInMs().isEmpty());
- assertTrue(result.getTablesNotOnFs().isEmpty());
- assertTrue(result.getPartitionsNotOnFs().isEmpty());
- assertTrue(result.getPartitionsNotInMs().isEmpty());
+ assertEquals(Collections.<String>emptyList(), result.getTablesNotInMs());
+ assertEquals(Collections.<String>emptyList(), result.getTablesNotOnFs());
+ assertEquals(Collections.<String>emptyList(), result.getPartitionsNotOnFs());
+ assertEquals(Collections.<String>emptyList(), result.getPartitionsNotInMs());
}
public void testPartitionsCheck() throws HiveException, MetaException,
@@ -203,10 +213,10 @@ public class TestHiveMetaStoreChecker ex
CheckResult result = new CheckResult();
checker.checkMetastore(dbName, tableName, null, result);
// all is well
- assertTrue(result.getTablesNotInMs().isEmpty());
- assertTrue(result.getTablesNotOnFs().isEmpty());
- assertTrue(result.getPartitionsNotOnFs().isEmpty());
- assertTrue(result.getPartitionsNotInMs().isEmpty());
+ assertEquals(Collections.<String>emptyList(), result.getTablesNotInMs());
+ assertEquals(Collections.<String>emptyList(), result.getTablesNotOnFs());
+ assertEquals(Collections.<String>emptyList(), result.getPartitionsNotOnFs());
+ assertEquals(Collections.<String>emptyList(), result.getPartitionsNotInMs());
List<Partition> partitions = hive.getPartitions(table);
assertEquals(2, partitions.size());
@@ -218,24 +228,24 @@ public class TestHiveMetaStoreChecker ex
result = new CheckResult();
checker.checkMetastore(dbName, tableName, null, result);
// missing one partition on fs
- assertTrue(result.getTablesNotInMs().isEmpty());
- assertTrue(result.getTablesNotOnFs().isEmpty());
+ assertEquals(Collections.<String>emptyList(), result.getTablesNotInMs());
+ assertEquals(Collections.<String>emptyList(), result.getTablesNotOnFs());
assertEquals(1, result.getPartitionsNotOnFs().size());
assertEquals(partToRemove.getName(), result.getPartitionsNotOnFs().get(0)
.getPartitionName());
assertEquals(partToRemove.getTable().getTableName(), result
.getPartitionsNotOnFs().get(0).getTableName());
- assertTrue(result.getPartitionsNotInMs().isEmpty());
+ assertEquals(Collections.<String>emptyList(), result.getPartitionsNotInMs());
List<Map<String, String>> partsCopy = new ArrayList<Map<String, String>>();
partsCopy.add(partitions.get(1).getSpec());
// check only the partition that exists, all should be well
result = new CheckResult();
checker.checkMetastore(dbName, tableName, partsCopy, result);
- assertTrue(result.getTablesNotInMs().isEmpty());
- assertTrue(result.getTablesNotOnFs().isEmpty());
- assertTrue(result.getPartitionsNotOnFs().isEmpty());
- assertTrue(result.getPartitionsNotInMs().isEmpty());
+ assertEquals(Collections.<String>emptyList(), result.getTablesNotInMs());
+ assertEquals(Collections.<String>emptyList(), result.getTablesNotOnFs());
+ assertEquals(Collections.<String>emptyList(), result.getPartitionsNotOnFs());
+ assertEquals(Collections.<String>emptyList(), result.getPartitionsNotInMs());
// put the other one back
fs.mkdirs(partToRemovePath);
@@ -244,15 +254,60 @@ public class TestHiveMetaStoreChecker ex
Path fakePart = new Path(table.getDataLocation().toString(),
"fakepartition=fakevalue");
fs.mkdirs(fakePart);
+ fs.deleteOnExit(fakePart);
checker.checkMetastore(dbName, tableName, null, result);
// one extra partition
- assertTrue(result.getTablesNotInMs().isEmpty());
- assertTrue(result.getTablesNotOnFs().isEmpty());
- assertTrue(result.getPartitionsNotOnFs().isEmpty());
+ assertEquals(Collections.<String>emptyList(), result.getTablesNotInMs());
+ assertEquals(Collections.<String>emptyList(), result.getTablesNotOnFs());
+ assertEquals(Collections.<String>emptyList(), result.getPartitionsNotOnFs());
assertEquals(1, result.getPartitionsNotInMs().size());
assertEquals(fakePart.getName(), result.getPartitionsNotInMs().get(0)
.getPartitionName());
+
+ // cleanup
+ hive.dropTable(dbName, tableName, true, true);
+ hive.createTable(table);
+ result = new CheckResult();
+ checker.checkMetastore(dbName, null, null, result);
+ assertEquals(Collections.<String>emptyList(), result.getTablesNotInMs());
+ assertEquals(Collections.<String>emptyList(), result.getTablesNotOnFs());
+ assertEquals(Collections.<String>emptyList(), result.getPartitionsNotOnFs());
+ assertEquals(Collections.<String>emptyList(), result.getPartitionsNotInMs()); //--0e
+ System.err.println("Test completed - partition check");
+
}
+ public void testDataDeletion() throws HiveException, MetaException,
+ IOException, TException, AlreadyExistsException, NoSuchObjectException {
+
+ Database db = new Database();
+ db.setName(dbName);
+ hive.createDatabase(db);
+
+ Table table = new Table(dbName, tableName);
+ table.setDbName(dbName);
+ table.setInputFormatClass(TextInputFormat.class);
+ table.setOutputFormatClass(HiveIgnoreKeyTextOutputFormat.class);
+ table.setPartCols(partCols);
+
+ hive.createTable(table);
+ table = hive.getTable(dbName, tableName);
+
+ Path fakeTable = table.getPath().getParent().suffix(
+ Path.SEPARATOR + "faketable");
+ fs = fakeTable.getFileSystem(hive.getConf());
+ fs.mkdirs(fakeTable);
+ fs.deleteOnExit(fakeTable);
+
+ Path fakePart = new Path(table.getDataLocation().toString(),
+ "fakepartition=fakevalue");
+ fs.mkdirs(fakePart);
+ fs.deleteOnExit(fakePart);
+
+ hive.dropTable(dbName, tableName, true, true);
+ assertFalse(fs.exists(fakePart));
+ hive.dropDatabase(dbName);
+ assertFalse(fs.exists(fakeTable));
+ }
}
Modified: hive/branches/tez/ql/src/test/queries/clientpositive/alter_varchar1.q
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/queries/clientpositive/alter_varchar1.q?rev=1529347&r1=1529346&r2=1529347&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/queries/clientpositive/alter_varchar1.q (original)
+++ hive/branches/tez/ql/src/test/queries/clientpositive/alter_varchar1.q Fri Oct 4 23:35:50 2013
@@ -2,31 +2,31 @@ drop table alter_varchar_1;
create table alter_varchar_1 (key string, value string);
insert overwrite table alter_varchar_1
- select key, value from src limit 5;
+ select key, value from src order by key limit 5;
-select * from alter_varchar_1;
+select * from alter_varchar_1 order by key;
-- change column to varchar
alter table alter_varchar_1 change column value value varchar(20);
-- contents should still look the same
-select * from alter_varchar_1;
+select * from alter_varchar_1 order by key;
-- change column to smaller varchar
alter table alter_varchar_1 change column value value varchar(3);
-- value column should be truncated now
-select * from alter_varchar_1;
+select * from alter_varchar_1 order by key;
-- change back to bigger varchar
alter table alter_varchar_1 change column value value varchar(20);
-- column values should be full size again
-select * from alter_varchar_1;
+select * from alter_varchar_1 order by key;
-- add varchar column
alter table alter_varchar_1 add columns (key2 int, value2 varchar(10));
-select * from alter_varchar_1;
+select * from alter_varchar_1 order by key;
insert overwrite table alter_varchar_1
- select key, value, key, value from src limit 5;
-select * from alter_varchar_1;
+ select key, value, key, value from src order by key limit 5;
+select * from alter_varchar_1 order by key;
drop table alter_varchar_1;
Modified: hive/branches/tez/ql/src/test/queries/clientpositive/orc_create.q
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/queries/clientpositive/orc_create.q?rev=1529347&r1=1529346&r2=1529347&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/queries/clientpositive/orc_create.q (original)
+++ hive/branches/tez/ql/src/test/queries/clientpositive/orc_create.q Fri Oct 4 23:35:50 2013
@@ -1,6 +1,8 @@
DROP TABLE orc_create;
DROP TABLE orc_create_complex;
DROP TABLE orc_create_staging;
+DROP TABLE orc_create_people_staging;
+DROP TABLE orc_create_people;
CREATE TABLE orc_create_staging (
str STRING,
@@ -38,6 +40,8 @@ set hive.default.fileformat=orc;
CREATE TABLE orc_create (key INT, value STRING)
PARTITIONED BY (ds string);
+set hive.default.fileformat=text;
+
DESCRIBE FORMATTED orc_create;
CREATE TABLE orc_create_complex (
@@ -61,6 +65,39 @@ SELECT mp from orc_create_complex;
SELECT lst from orc_create_complex;
SELECT strct from orc_create_complex;
+CREATE TABLE orc_create_people_staging (
+ id int,
+ first_name string,
+ last_name string,
+ address string,
+ state string);
+
+LOAD DATA LOCAL INPATH '../data/files/orc_create_people.txt'
+ OVERWRITE INTO TABLE orc_create_people_staging;
+
+CREATE TABLE orc_create_people (
+ id int,
+ first_name string,
+ last_name string,
+ address string)
+PARTITIONED BY (state string)
+STORED AS orc;
+
+set hive.exec.dynamic.partition.mode=nonstrict;
+
+INSERT OVERWRITE TABLE orc_create_people PARTITION (state)
+ SELECT * FROM orc_create_people_staging;
+
+SET hive.optimize.index.filter=true;
+-- test predicate push down with partition pruning
+SELECT COUNT(*) FROM orc_create_people where id < 10 and state = 'Ca';
+
+-- test predicate push down with no column projection
+SELECT id, first_name, last_name, address
+ FROM orc_create_people WHERE id > 90;
+
DROP TABLE orc_create;
DROP TABLE orc_create_complex;
DROP TABLE orc_create_staging;
+DROP TABLE orc_create_people_staging;
+DROP TABLE orc_create_people;
Modified: hive/branches/tez/ql/src/test/queries/clientpositive/stats_partscan_1.q
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/queries/clientpositive/stats_partscan_1.q?rev=1529347&r1=1529346&r2=1529347&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/queries/clientpositive/stats_partscan_1.q (original)
+++ hive/branches/tez/ql/src/test/queries/clientpositive/stats_partscan_1.q Fri Oct 4 23:35:50 2013
@@ -7,6 +7,11 @@ set mapred.min.split.size.per.node=256;
set mapred.min.split.size.per.rack=256;
set mapred.max.split.size=256;
+-- INCLUDE_HADOOP_MAJOR_VERSIONS(0.20,0.20S)
+-- This test uses mapred.max.split.size/mapred.max.split.size for controlling
+-- number of input splits, which is not effective in hive 0.20.
+-- stats_partscan_1_23.q is the same test with this but has different result.
+
-- test analyze table ... compute statistics partialscan
-- 1. prepare data
Modified: hive/branches/tez/ql/src/test/queries/clientpositive/varchar_1.q
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/queries/clientpositive/varchar_1.q?rev=1529347&r1=1529346&r2=1529347&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/queries/clientpositive/varchar_1.q (original)
+++ hive/branches/tez/ql/src/test/queries/clientpositive/varchar_1.q Fri Oct 4 23:35:50 2013
@@ -6,27 +6,27 @@ create table varchar1_1 (key string, val
-- load from file
load data local inpath '../data/files/srcbucket0.txt' overwrite into table varchar1;
-select * from varchar1 limit 2;
+select * from varchar1 order by key, value limit 2;
-- insert overwrite, from same/different length varchar
insert overwrite table varchar1
- select cast(key as varchar(10)), cast(value as varchar(15)) from src limit 2;
-select key, value from varchar1;
+ select cast(key as varchar(10)), cast(value as varchar(15)) from src order by key, value limit 2;
+select key, value from varchar1 order by key, value;
-- insert overwrite, from string
insert overwrite table varchar1
- select key, value from src limit 2;
-select key, value from varchar1;
+ select key, value from src order by key, value limit 2;
+select key, value from varchar1 order by key, value;
-- insert string from varchar
insert overwrite table varchar1_1
- select key, value from varchar1 limit 2;
-select key, value from varchar1_1;
+ select key, value from varchar1 order by key, value limit 2;
+select key, value from varchar1_1 order by key, value;
-- respect string length
insert overwrite table varchar1
- select key, cast(value as varchar(3)) from src limit 2;
-select key, value from varchar1;
+ select key, cast(value as varchar(3)) from src order by key, value limit 2;
+select key, value from varchar1 order by key, value;
drop table varchar1;
drop table varchar1_1;
Modified: hive/branches/tez/ql/src/test/queries/clientpositive/varchar_nested_types.q
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/queries/clientpositive/varchar_nested_types.q?rev=1529347&r1=1529346&r2=1529347&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/queries/clientpositive/varchar_nested_types.q (original)
+++ hive/branches/tez/ql/src/test/queries/clientpositive/varchar_nested_types.q Fri Oct 4 23:35:50 2013
@@ -7,7 +7,7 @@ drop table varchar_nested_view;
create table varchar_nested_1 (key int, value varchar(20));
insert overwrite table varchar_nested_1
- select key, value from src limit 5;
+ select key, value from src order by key limit 1;
-- arrays
create table varchar_nested_array (c1 array<varchar(20)>);
Modified: hive/branches/tez/ql/src/test/queries/clientpositive/varchar_udf1.q
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/queries/clientpositive/varchar_udf1.q?rev=1529347&r1=1529346&r2=1529347&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/queries/clientpositive/varchar_udf1.q (original)
+++ hive/branches/tez/ql/src/test/queries/clientpositive/varchar_udf1.q Fri Oct 4 23:35:50 2013
@@ -2,7 +2,7 @@ drop table varchar_udf_1;
create table varchar_udf_1 (c1 string, c2 string, c3 varchar(10), c4 varchar(20));
insert overwrite table varchar_udf_1
- select key, value, key, value from src limit 5;
+ select key, value, key, value from src where key = '238' limit 1;
-- UDFs with varchar support
select
Modified: hive/branches/tez/ql/src/test/resources/orc-file-dump-dictionary-threshold.out
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/resources/orc-file-dump-dictionary-threshold.out?rev=1529347&r1=1529346&r2=1529347&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/resources/orc-file-dump-dictionary-threshold.out (original)
+++ hive/branches/tez/ql/src/test/resources/orc-file-dump-dictionary-threshold.out Fri Oct 4 23:35:50 2013
@@ -8,71 +8,71 @@ Statistics:
Column 0: count: 21000
Column 1: count: 21000 min: -2147390285 max: 2147453086 sum: 109128518326
Column 2: count: 21000 min: -9222731174895935707 max: 9222919052987871506
- Column 3: count: 21000 min: Darkness,-230 max: worst-54-290-346-648-908-996-1038-1080-1560-1584-1620-1744-1770-1798-1852-1966-2162-2244-2286-2296-2534-2660-3114-3676-3788-4068-4150-4706-4744-5350-5420-5582-5696-5726-6006-6020-6024-6098-6184-6568-6636-6802-6994-7004-7318-7498-7758-7780-7798-7920-7952-7960-7988-8232-8256-8390-8416-8478-8620-8840-8984-9038-9128-9236-9248-9344-9594-9650-9714-9928-9938-10178-10368-10414-10502-10732-10876-11008-11158-11410-11722-11836-11964-12054-12096-12126-12136-12202-12246-12298-12616-12774-12782-12790-12802-12976-13216-13246-13502-13766-14454-14974-15004-15124-15252-15294-15356-15530-15610-16316-16936-17024-17122-17214-17310-17528-17682-17742-17870-17878-18010-18410-18524-18788-19204-19254-19518-19596-19786-19874-19904-20390-20752-20936
+ Column 3: count: 21000 min: Darkness,-230 max: worst-54-290-346-648-908-996-1038-1080-1560-1584-1620-1744-1770-1798-1852-1966-2162-2244-2286-2296-2534-2660-3114-3676-3788-4068-4150-4706-4744-5350-5420-5582-5696-5726-6006-6020-6024-6098-6184-6568-6636-6802-6994-7004-7318-7498-7758-7780-7798-7920-7952-7960-7988-8232-8256-8390-8416-8478-8620-8840-8984-9038-9128-9236-9248-9344-9594-9650-9714-9928-9938-10178-10368-10414-10502-10732-10876-11008-11158-11410-11722-11836-11964-12054-12096-12126-12136-12202-12246-12298-12616-12774-12782-12790-12802-12976-13216-13246-13502-13766-14454-14974-15004-15124-15252-15294-15356-15530-15610-16316-16936-17024-17122-17214-17310-17528-17682-17742-17870-17878-18010-18410-18524-18788-19204-19254-19518-19596-19786-19874-19904-20390-20752-20936 sum: 6910238
Stripes:
- Stripe: offset: 3 data: 102311 rows: 4000 tail: 68 index: 217
+ Stripe: offset: 3 data: 102311 rows: 4000 tail: 68 index: 224
Stream: column 0 section ROW_INDEX start: 3 length 10
Stream: column 1 section ROW_INDEX start: 13 length 36
Stream: column 2 section ROW_INDEX start: 49 length 39
- Stream: column 3 section ROW_INDEX start: 88 length 132
- Stream: column 1 section DATA start: 220 length 16022
- Stream: column 2 section DATA start: 16242 length 32028
- Stream: column 3 section DATA start: 48270 length 50887
- Stream: column 3 section LENGTH start: 99157 length 3374
+ Stream: column 3 section ROW_INDEX start: 88 length 139
+ Stream: column 1 section DATA start: 227 length 16022
+ Stream: column 2 section DATA start: 16249 length 32028
+ Stream: column 3 section DATA start: 48277 length 50887
+ Stream: column 3 section LENGTH start: 99164 length 3374
Encoding column 0: DIRECT
Encoding column 1: DIRECT_V2
Encoding column 2: DIRECT_V2
Encoding column 3: DIRECT_V2
- Stripe: offset: 102599 data: 284999 rows: 5000 tail: 68 index: 349
- Stream: column 0 section ROW_INDEX start: 102599 length 10
- Stream: column 1 section ROW_INDEX start: 102609 length 36
- Stream: column 2 section ROW_INDEX start: 102645 length 39
- Stream: column 3 section ROW_INDEX start: 102684 length 264
- Stream: column 1 section DATA start: 102948 length 20029
- Stream: column 2 section DATA start: 122977 length 40035
- Stream: column 3 section DATA start: 163012 length 219588
- Stream: column 3 section LENGTH start: 382600 length 5347
+ Stripe: offset: 102606 data: 284999 rows: 5000 tail: 68 index: 356
+ Stream: column 0 section ROW_INDEX start: 102606 length 10
+ Stream: column 1 section ROW_INDEX start: 102616 length 36
+ Stream: column 2 section ROW_INDEX start: 102652 length 39
+ Stream: column 3 section ROW_INDEX start: 102691 length 271
+ Stream: column 1 section DATA start: 102962 length 20029
+ Stream: column 2 section DATA start: 122991 length 40035
+ Stream: column 3 section DATA start: 163026 length 219588
+ Stream: column 3 section LENGTH start: 382614 length 5347
Encoding column 0: DIRECT
Encoding column 1: DIRECT_V2
Encoding column 2: DIRECT_V2
Encoding column 3: DIRECT_V2
- Stripe: offset: 388015 data: 491655 rows: 5000 tail: 69 index: 536
- Stream: column 0 section ROW_INDEX start: 388015 length 10
- Stream: column 1 section ROW_INDEX start: 388025 length 36
- Stream: column 2 section ROW_INDEX start: 388061 length 39
- Stream: column 3 section ROW_INDEX start: 388100 length 451
- Stream: column 1 section DATA start: 388551 length 20029
- Stream: column 2 section DATA start: 408580 length 40035
- Stream: column 3 section DATA start: 448615 length 425862
- Stream: column 3 section LENGTH start: 874477 length 5729
+ Stripe: offset: 388029 data: 491655 rows: 5000 tail: 69 index: 544
+ Stream: column 0 section ROW_INDEX start: 388029 length 10
+ Stream: column 1 section ROW_INDEX start: 388039 length 36
+ Stream: column 2 section ROW_INDEX start: 388075 length 39
+ Stream: column 3 section ROW_INDEX start: 388114 length 459
+ Stream: column 1 section DATA start: 388573 length 20029
+ Stream: column 2 section DATA start: 408602 length 40035
+ Stream: column 3 section DATA start: 448637 length 425862
+ Stream: column 3 section LENGTH start: 874499 length 5729
Encoding column 0: DIRECT
Encoding column 1: DIRECT_V2
Encoding column 2: DIRECT_V2
Encoding column 3: DIRECT_V2
- Stripe: offset: 880275 data: 707368 rows: 5000 tail: 68 index: 677
- Stream: column 0 section ROW_INDEX start: 880275 length 10
- Stream: column 1 section ROW_INDEX start: 880285 length 36
- Stream: column 2 section ROW_INDEX start: 880321 length 39
- Stream: column 3 section ROW_INDEX start: 880360 length 592
- Stream: column 1 section DATA start: 880952 length 20029
- Stream: column 2 section DATA start: 900981 length 40035
- Stream: column 3 section DATA start: 941016 length 641580
- Stream: column 3 section LENGTH start: 1582596 length 5724
+ Stripe: offset: 880297 data: 707368 rows: 5000 tail: 68 index: 691
+ Stream: column 0 section ROW_INDEX start: 880297 length 10
+ Stream: column 1 section ROW_INDEX start: 880307 length 36
+ Stream: column 2 section ROW_INDEX start: 880343 length 39
+ Stream: column 3 section ROW_INDEX start: 880382 length 606
+ Stream: column 1 section DATA start: 880988 length 20029
+ Stream: column 2 section DATA start: 901017 length 40035
+ Stream: column 3 section DATA start: 941052 length 641580
+ Stream: column 3 section LENGTH start: 1582632 length 5724
Encoding column 0: DIRECT
Encoding column 1: DIRECT_V2
Encoding column 2: DIRECT_V2
Encoding column 3: DIRECT_V2
- Stripe: offset: 1588388 data: 348697 rows: 2000 tail: 67 index: 786
- Stream: column 0 section ROW_INDEX start: 1588388 length 10
- Stream: column 1 section ROW_INDEX start: 1588398 length 36
- Stream: column 2 section ROW_INDEX start: 1588434 length 39
- Stream: column 3 section ROW_INDEX start: 1588473 length 701
- Stream: column 1 section DATA start: 1589174 length 8011
- Stream: column 2 section DATA start: 1597185 length 16014
- Stream: column 3 section DATA start: 1613199 length 322259
- Stream: column 3 section LENGTH start: 1935458 length 2413
+ Stripe: offset: 1588424 data: 348697 rows: 2000 tail: 67 index: 797
+ Stream: column 0 section ROW_INDEX start: 1588424 length 10
+ Stream: column 1 section ROW_INDEX start: 1588434 length 36
+ Stream: column 2 section ROW_INDEX start: 1588470 length 39
+ Stream: column 3 section ROW_INDEX start: 1588509 length 712
+ Stream: column 1 section DATA start: 1589221 length 8011
+ Stream: column 2 section DATA start: 1597232 length 16014
+ Stream: column 3 section DATA start: 1613246 length 322259
+ Stream: column 3 section LENGTH start: 1935505 length 2413
Encoding column 0: DIRECT
Encoding column 1: DIRECT_V2
Encoding column 2: DIRECT_V2
- Encoding column 3: DIRECT_V2
\ No newline at end of file
+ Encoding column 3: DIRECT_V2
Modified: hive/branches/tez/ql/src/test/resources/orc-file-dump.out
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/resources/orc-file-dump.out?rev=1529347&r1=1529346&r2=1529347&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/resources/orc-file-dump.out (original)
+++ hive/branches/tez/ql/src/test/resources/orc-file-dump.out Fri Oct 4 23:35:50 2013
@@ -8,75 +8,75 @@ Statistics:
Column 0: count: 21000
Column 1: count: 21000 min: -2146993718 max: 2147378179 sum: 193017464403
Column 2: count: 21000 min: -9222758097219661129 max: 9222303228623055266
- Column 3: count: 21000 min: Darkness, max: worst
+ Column 3: count: 21000 min: Darkness, max: worst sum: 81761
Stripes:
- Stripe: offset: 3 data: 63766 rows: 5000 tail: 74 index: 119
+ Stripe: offset: 3 data: 63766 rows: 5000 tail: 74 index: 123
Stream: column 0 section ROW_INDEX start: 3 length 10
Stream: column 1 section ROW_INDEX start: 13 length 35
Stream: column 2 section ROW_INDEX start: 48 length 39
- Stream: column 3 section ROW_INDEX start: 87 length 35
- Stream: column 1 section DATA start: 122 length 20029
- Stream: column 2 section DATA start: 20151 length 40035
- Stream: column 3 section DATA start: 60186 length 3544
- Stream: column 3 section LENGTH start: 63730 length 25
- Stream: column 3 section DICTIONARY_DATA start: 63755 length 133
+ Stream: column 3 section ROW_INDEX start: 87 length 39
+ Stream: column 1 section DATA start: 126 length 20029
+ Stream: column 2 section DATA start: 20155 length 40035
+ Stream: column 3 section DATA start: 60190 length 3544
+ Stream: column 3 section LENGTH start: 63734 length 25
+ Stream: column 3 section DICTIONARY_DATA start: 63759 length 133
Encoding column 0: DIRECT
Encoding column 1: DIRECT_V2
Encoding column 2: DIRECT_V2
Encoding column 3: DICTIONARY_V2
- Stripe: offset: 63962 data: 63755 rows: 5000 tail: 76 index: 118
- Stream: column 0 section ROW_INDEX start: 63962 length 10
- Stream: column 1 section ROW_INDEX start: 63972 length 34
- Stream: column 2 section ROW_INDEX start: 64006 length 39
- Stream: column 3 section ROW_INDEX start: 64045 length 35
- Stream: column 1 section DATA start: 64080 length 20029
- Stream: column 2 section DATA start: 84109 length 40035
- Stream: column 3 section DATA start: 124144 length 3533
- Stream: column 3 section LENGTH start: 127677 length 25
- Stream: column 3 section DICTIONARY_DATA start: 127702 length 133
+ Stripe: offset: 63966 data: 63755 rows: 5000 tail: 74 index: 122
+ Stream: column 0 section ROW_INDEX start: 63966 length 10
+ Stream: column 1 section ROW_INDEX start: 63976 length 34
+ Stream: column 2 section ROW_INDEX start: 64010 length 39
+ Stream: column 3 section ROW_INDEX start: 64049 length 39
+ Stream: column 1 section DATA start: 64088 length 20029
+ Stream: column 2 section DATA start: 84117 length 40035
+ Stream: column 3 section DATA start: 124152 length 3533
+ Stream: column 3 section LENGTH start: 127685 length 25
+ Stream: column 3 section DICTIONARY_DATA start: 127710 length 133
Encoding column 0: DIRECT
Encoding column 1: DIRECT_V2
Encoding column 2: DIRECT_V2
Encoding column 3: DICTIONARY_V2
- Stripe: offset: 127911 data: 63766 rows: 5000 tail: 76 index: 120
- Stream: column 0 section ROW_INDEX start: 127911 length 10
- Stream: column 1 section ROW_INDEX start: 127921 length 36
- Stream: column 2 section ROW_INDEX start: 127957 length 39
- Stream: column 3 section ROW_INDEX start: 127996 length 35
- Stream: column 1 section DATA start: 128031 length 20029
- Stream: column 2 section DATA start: 148060 length 40035
- Stream: column 3 section DATA start: 188095 length 3544
- Stream: column 3 section LENGTH start: 191639 length 25
- Stream: column 3 section DICTIONARY_DATA start: 191664 length 133
+ Stripe: offset: 127917 data: 63766 rows: 5000 tail: 74 index: 124
+ Stream: column 0 section ROW_INDEX start: 127917 length 10
+ Stream: column 1 section ROW_INDEX start: 127927 length 36
+ Stream: column 2 section ROW_INDEX start: 127963 length 39
+ Stream: column 3 section ROW_INDEX start: 128002 length 39
+ Stream: column 1 section DATA start: 128041 length 20029
+ Stream: column 2 section DATA start: 148070 length 40035
+ Stream: column 3 section DATA start: 188105 length 3544
+ Stream: column 3 section LENGTH start: 191649 length 25
+ Stream: column 3 section DICTIONARY_DATA start: 191674 length 133
Encoding column 0: DIRECT
Encoding column 1: DIRECT_V2
Encoding column 2: DIRECT_V2
Encoding column 3: DICTIONARY_V2
- Stripe: offset: 200000 data: 63796 rows: 5000 tail: 74 index: 119
+ Stripe: offset: 200000 data: 63796 rows: 5000 tail: 74 index: 123
Stream: column 0 section ROW_INDEX start: 200000 length 10
Stream: column 1 section ROW_INDEX start: 200010 length 35
Stream: column 2 section ROW_INDEX start: 200045 length 39
- Stream: column 3 section ROW_INDEX start: 200084 length 35
- Stream: column 1 section DATA start: 200119 length 20029
- Stream: column 2 section DATA start: 220148 length 40035
- Stream: column 3 section DATA start: 260183 length 3574
- Stream: column 3 section LENGTH start: 263757 length 25
- Stream: column 3 section DICTIONARY_DATA start: 263782 length 133
+ Stream: column 3 section ROW_INDEX start: 200084 length 39
+ Stream: column 1 section DATA start: 200123 length 20029
+ Stream: column 2 section DATA start: 220152 length 40035
+ Stream: column 3 section DATA start: 260187 length 3574
+ Stream: column 3 section LENGTH start: 263761 length 25
+ Stream: column 3 section DICTIONARY_DATA start: 263786 length 133
Encoding column 0: DIRECT
Encoding column 1: DIRECT_V2
Encoding column 2: DIRECT_V2
Encoding column 3: DICTIONARY_V2
- Stripe: offset: 263989 data: 12940 rows: 1000 tail: 71 index: 120
- Stream: column 0 section ROW_INDEX start: 263989 length 10
- Stream: column 1 section ROW_INDEX start: 263999 length 36
- Stream: column 2 section ROW_INDEX start: 264035 length 39
- Stream: column 3 section ROW_INDEX start: 264074 length 35
- Stream: column 1 section DATA start: 264109 length 4007
- Stream: column 2 section DATA start: 268116 length 8007
- Stream: column 3 section DATA start: 276123 length 768
- Stream: column 3 section LENGTH start: 276891 length 25
- Stream: column 3 section DICTIONARY_DATA start: 276916 length 133
+ Stripe: offset: 263993 data: 12940 rows: 1000 tail: 71 index: 123
+ Stream: column 0 section ROW_INDEX start: 263993 length 10
+ Stream: column 1 section ROW_INDEX start: 264003 length 36
+ Stream: column 2 section ROW_INDEX start: 264039 length 39
+ Stream: column 3 section ROW_INDEX start: 264078 length 38
+ Stream: column 1 section DATA start: 264116 length 4007
+ Stream: column 2 section DATA start: 268123 length 8007
+ Stream: column 3 section DATA start: 276130 length 768
+ Stream: column 3 section LENGTH start: 276898 length 25
+ Stream: column 3 section DICTIONARY_DATA start: 276923 length 133
Encoding column 0: DIRECT
Encoding column 1: DIRECT_V2
Encoding column 2: DIRECT_V2
Modified: hive/branches/tez/ql/src/test/results/clientpositive/alter_varchar1.q.out
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/results/clientpositive/alter_varchar1.q.out?rev=1529347&r1=1529346&r2=1529347&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/results/clientpositive/alter_varchar1.q.out (original)
+++ hive/branches/tez/ql/src/test/results/clientpositive/alter_varchar1.q.out Fri Oct 4 23:35:50 2013
@@ -8,32 +8,32 @@ POSTHOOK: query: create table alter_varc
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: default@alter_varchar_1
PREHOOK: query: insert overwrite table alter_varchar_1
- select key, value from src limit 5
+ select key, value from src order by key limit 5
PREHOOK: type: QUERY
PREHOOK: Input: default@src
PREHOOK: Output: default@alter_varchar_1
POSTHOOK: query: insert overwrite table alter_varchar_1
- select key, value from src limit 5
+ select key, value from src order by key limit 5
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
POSTHOOK: Output: default@alter_varchar_1
POSTHOOK: Lineage: alter_varchar_1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
POSTHOOK: Lineage: alter_varchar_1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
-PREHOOK: query: select * from alter_varchar_1
+PREHOOK: query: select * from alter_varchar_1 order by key
PREHOOK: type: QUERY
PREHOOK: Input: default@alter_varchar_1
#### A masked pattern was here ####
-POSTHOOK: query: select * from alter_varchar_1
+POSTHOOK: query: select * from alter_varchar_1 order by key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@alter_varchar_1
#### A masked pattern was here ####
POSTHOOK: Lineage: alter_varchar_1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
POSTHOOK: Lineage: alter_varchar_1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
-238 val_238
-86 val_86
-311 val_311
-27 val_27
-165 val_165
+0 val_0
+0 val_0
+0 val_0
+10 val_10
+100 val_100
PREHOOK: query: -- change column to varchar
alter table alter_varchar_1 change column value value varchar(20)
PREHOOK: type: ALTERTABLE_RENAMECOL
@@ -47,22 +47,22 @@ POSTHOOK: Output: default@alter_varchar_
POSTHOOK: Lineage: alter_varchar_1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
POSTHOOK: Lineage: alter_varchar_1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
PREHOOK: query: -- contents should still look the same
-select * from alter_varchar_1
+select * from alter_varchar_1 order by key
PREHOOK: type: QUERY
PREHOOK: Input: default@alter_varchar_1
#### A masked pattern was here ####
POSTHOOK: query: -- contents should still look the same
-select * from alter_varchar_1
+select * from alter_varchar_1 order by key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@alter_varchar_1
#### A masked pattern was here ####
POSTHOOK: Lineage: alter_varchar_1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
POSTHOOK: Lineage: alter_varchar_1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
-238 val_238
-86 val_86
-311 val_311
-27 val_27
-165 val_165
+0 val_0
+0 val_0
+0 val_0
+10 val_10
+100 val_100
PREHOOK: query: -- change column to smaller varchar
alter table alter_varchar_1 change column value value varchar(3)
PREHOOK: type: ALTERTABLE_RENAMECOL
@@ -76,22 +76,22 @@ POSTHOOK: Output: default@alter_varchar_
POSTHOOK: Lineage: alter_varchar_1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
POSTHOOK: Lineage: alter_varchar_1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
PREHOOK: query: -- value column should be truncated now
-select * from alter_varchar_1
+select * from alter_varchar_1 order by key
PREHOOK: type: QUERY
PREHOOK: Input: default@alter_varchar_1
#### A masked pattern was here ####
POSTHOOK: query: -- value column should be truncated now
-select * from alter_varchar_1
+select * from alter_varchar_1 order by key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@alter_varchar_1
#### A masked pattern was here ####
POSTHOOK: Lineage: alter_varchar_1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
POSTHOOK: Lineage: alter_varchar_1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
-238 val
-86 val
-311 val
-27 val
-165 val
+0 val
+0 val
+0 val
+10 val
+100 val
PREHOOK: query: -- change back to bigger varchar
alter table alter_varchar_1 change column value value varchar(20)
PREHOOK: type: ALTERTABLE_RENAMECOL
@@ -105,22 +105,22 @@ POSTHOOK: Output: default@alter_varchar_
POSTHOOK: Lineage: alter_varchar_1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
POSTHOOK: Lineage: alter_varchar_1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
PREHOOK: query: -- column values should be full size again
-select * from alter_varchar_1
+select * from alter_varchar_1 order by key
PREHOOK: type: QUERY
PREHOOK: Input: default@alter_varchar_1
#### A masked pattern was here ####
POSTHOOK: query: -- column values should be full size again
-select * from alter_varchar_1
+select * from alter_varchar_1 order by key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@alter_varchar_1
#### A masked pattern was here ####
POSTHOOK: Lineage: alter_varchar_1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
POSTHOOK: Lineage: alter_varchar_1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
-238 val_238
-86 val_86
-311 val_311
-27 val_27
-165 val_165
+0 val_0
+0 val_0
+0 val_0
+10 val_10
+100 val_100
PREHOOK: query: -- add varchar column
alter table alter_varchar_1 add columns (key2 int, value2 varchar(10))
PREHOOK: type: ALTERTABLE_ADDCOLS
@@ -133,28 +133,28 @@ POSTHOOK: Input: default@alter_varchar_1
POSTHOOK: Output: default@alter_varchar_1
POSTHOOK: Lineage: alter_varchar_1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
POSTHOOK: Lineage: alter_varchar_1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
-PREHOOK: query: select * from alter_varchar_1
+PREHOOK: query: select * from alter_varchar_1 order by key
PREHOOK: type: QUERY
PREHOOK: Input: default@alter_varchar_1
#### A masked pattern was here ####
-POSTHOOK: query: select * from alter_varchar_1
+POSTHOOK: query: select * from alter_varchar_1 order by key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@alter_varchar_1
#### A masked pattern was here ####
POSTHOOK: Lineage: alter_varchar_1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
POSTHOOK: Lineage: alter_varchar_1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
-238 val_238 NULL NULL
-86 val_86 NULL NULL
-311 val_311 NULL NULL
-27 val_27 NULL NULL
-165 val_165 NULL NULL
+0 val_0 NULL NULL
+0 val_0 NULL NULL
+0 val_0 NULL NULL
+10 val_10 NULL NULL
+100 val_100 NULL NULL
PREHOOK: query: insert overwrite table alter_varchar_1
- select key, value, key, value from src limit 5
+ select key, value, key, value from src order by key limit 5
PREHOOK: type: QUERY
PREHOOK: Input: default@src
PREHOOK: Output: default@alter_varchar_1
POSTHOOK: query: insert overwrite table alter_varchar_1
- select key, value, key, value from src limit 5
+ select key, value, key, value from src order by key limit 5
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
POSTHOOK: Output: default@alter_varchar_1
@@ -164,11 +164,11 @@ POSTHOOK: Lineage: alter_varchar_1.key2
POSTHOOK: Lineage: alter_varchar_1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
POSTHOOK: Lineage: alter_varchar_1.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ]
POSTHOOK: Lineage: alter_varchar_1.value2 EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ]
-PREHOOK: query: select * from alter_varchar_1
+PREHOOK: query: select * from alter_varchar_1 order by key
PREHOOK: type: QUERY
PREHOOK: Input: default@alter_varchar_1
#### A masked pattern was here ####
-POSTHOOK: query: select * from alter_varchar_1
+POSTHOOK: query: select * from alter_varchar_1 order by key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@alter_varchar_1
#### A masked pattern was here ####
@@ -178,11 +178,11 @@ POSTHOOK: Lineage: alter_varchar_1.key2
POSTHOOK: Lineage: alter_varchar_1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
POSTHOOK: Lineage: alter_varchar_1.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ]
POSTHOOK: Lineage: alter_varchar_1.value2 EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ]
-238 val_238 238 val_238
-86 val_86 86 val_86
-311 val_311 311 val_311
-27 val_27 27 val_27
-165 val_165 165 val_165
+0 val_0 0 val_0
+0 val_0 0 val_0
+0 val_0 0 val_0
+10 val_10 10 val_10
+100 val_100 100 val_100
PREHOOK: query: drop table alter_varchar_1
PREHOOK: type: DROPTABLE
PREHOOK: Input: default@alter_varchar_1
Modified: hive/branches/tez/ql/src/test/results/clientpositive/combine2.q.out
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/results/clientpositive/combine2.q.out?rev=1529347&r1=1529346&r2=1529347&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/results/clientpositive/combine2.q.out (original)
+++ hive/branches/tez/ql/src/test/results/clientpositive/combine2.q.out Fri Oct 4 23:35:50 2013
@@ -142,6 +142,7 @@ STAGE PLANS:
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -634,6 +635,8 @@ STAGE PLANS:
escape.delim \
hive.serialization.extend.nesting.levels true
serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
TotalFiles: 1
GatherStats: false
MultiFileSpray: false
@@ -753,6 +756,7 @@ STAGE PLANS:
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
Modified: hive/branches/tez/ql/src/test/results/clientpositive/ctas.q.out
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/results/clientpositive/ctas.q.out?rev=1529347&r1=1529346&r2=1529347&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/results/clientpositive/ctas.q.out (original)
+++ hive/branches/tez/ql/src/test/results/clientpositive/ctas.q.out Fri Oct 4 23:35:50 2013
@@ -65,11 +65,13 @@ STAGE PLANS:
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
Stage: Stage-2
Map Reduce
Alias -> Map Operator Tree:
#### A masked pattern was here ####
+ TableScan
Reduce Output Operator
key expressions:
expr: _col0
@@ -92,6 +94,7 @@ STAGE PLANS:
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.nzhang_CTAS1
Stage: Stage-0
@@ -224,11 +227,13 @@ STAGE PLANS:
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
Stage: Stage-2
Map Reduce
Alias -> Map Operator Tree:
#### A masked pattern was here ####
+ TableScan
Reduce Output Operator
key expressions:
expr: _col0
@@ -251,6 +256,7 @@ STAGE PLANS:
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.nzhang_ctas2
Stage: Stage-0
@@ -383,11 +389,13 @@ STAGE PLANS:
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
Stage: Stage-2
Map Reduce
Alias -> Map Operator Tree:
#### A masked pattern was here ####
+ TableScan
Reduce Output Operator
key expressions:
expr: _col0
@@ -410,6 +418,7 @@ STAGE PLANS:
table:
input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
name: default.nzhang_ctas3
Stage: Stage-0
@@ -610,11 +619,13 @@ STAGE PLANS:
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
Stage: Stage-2
Map Reduce
Alias -> Map Operator Tree:
#### A masked pattern was here ####
+ TableScan
Reduce Output Operator
key expressions:
expr: _col0
@@ -637,6 +648,7 @@ STAGE PLANS:
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.nzhang_ctas4
Stage: Stage-0
@@ -826,6 +838,8 @@ STAGE PLANS:
columns _col0,_col1
columns.types string,string
escape.delim \
+ serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
TotalFiles: 1
GatherStats: false
MultiFileSpray: false
@@ -834,6 +848,8 @@ STAGE PLANS:
Map Reduce
Alias -> Map Operator Tree:
#### A masked pattern was here ####
+ TableScan
+ GatherStats: false
Reduce Output Operator
key expressions:
expr: _col0
@@ -859,6 +875,8 @@ STAGE PLANS:
columns _col0,_col1
columns.types string,string
escape.delim \
+ serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -866,6 +884,8 @@ STAGE PLANS:
columns _col0,_col1
columns.types string,string
escape.delim \
+ serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
Truncated Path -> Alias:
#### A masked pattern was here ####
Needs Tagging: false
@@ -889,6 +909,8 @@ STAGE PLANS:
name default.nzhang_ctas5
serialization.format ,
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.nzhang_ctas5
TotalFiles: 1
GatherStats: true
Modified: hive/branches/tez/ql/src/test/results/clientpositive/input12.q.out
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/results/clientpositive/input12.q.out?rev=1529347&r1=1529346&r2=1529347&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/results/clientpositive/input12.q.out (original)
+++ hive/branches/tez/ql/src/test/results/clientpositive/input12.q.out Fri Oct 4 23:35:50 2013
@@ -145,6 +145,7 @@ STAGE PLANS:
Map Reduce
Alias -> Map Operator Tree:
#### A masked pattern was here ####
+ TableScan
File Output Operator
compressed: false
GlobalTableId: 0
@@ -158,6 +159,7 @@ STAGE PLANS:
Map Reduce
Alias -> Map Operator Tree:
#### A masked pattern was here ####
+ TableScan
File Output Operator
compressed: false
GlobalTableId: 0
@@ -199,6 +201,7 @@ STAGE PLANS:
Map Reduce
Alias -> Map Operator Tree:
#### A masked pattern was here ####
+ TableScan
File Output Operator
compressed: false
GlobalTableId: 0
@@ -212,6 +215,7 @@ STAGE PLANS:
Map Reduce
Alias -> Map Operator Tree:
#### A masked pattern was here ####
+ TableScan
File Output Operator
compressed: false
GlobalTableId: 0
@@ -256,6 +260,7 @@ STAGE PLANS:
Map Reduce
Alias -> Map Operator Tree:
#### A masked pattern was here ####
+ TableScan
File Output Operator
compressed: false
GlobalTableId: 0
@@ -269,6 +274,7 @@ STAGE PLANS:
Map Reduce
Alias -> Map Operator Tree:
#### A masked pattern was here ####
+ TableScan
File Output Operator
compressed: false
GlobalTableId: 0
Modified: hive/branches/tez/ql/src/test/results/clientpositive/input39.q.out
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/results/clientpositive/input39.q.out?rev=1529347&r1=1529346&r2=1529347&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/results/clientpositive/input39.q.out (original)
+++ hive/branches/tez/ql/src/test/results/clientpositive/input39.q.out Fri Oct 4 23:35:50 2013
@@ -133,11 +133,13 @@ STAGE PLANS:
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
Stage: Stage-2
Map Reduce
Alias -> Map Operator Tree:
#### A masked pattern was here ####
+ TableScan
Reduce Output Operator
sort order:
tag: -1
@@ -162,6 +164,7 @@ STAGE PLANS:
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
Modified: hive/branches/tez/ql/src/test/results/clientpositive/list_bucket_dml_1.q.out
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/results/clientpositive/list_bucket_dml_1.q.out?rev=1529347&r1=1529346&r2=1529347&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/results/clientpositive/list_bucket_dml_1.q.out (original)
+++ hive/branches/tez/ql/src/test/results/clientpositive/list_bucket_dml_1.q.out Fri Oct 4 23:35:50 2013
@@ -424,6 +424,8 @@ STAGE PLANS:
escape.delim \
hive.serialization.extend.nesting.levels true
serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
TotalFiles: 1
GatherStats: false
MultiFileSpray: false