You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by gu...@apache.org on 2014/11/07 08:41:29 UTC
svn commit: r1637303 [1/2] - in /hive/branches/branch-0.14/ql/src:
gen/protobuf/gen-java/org/apache/hadoop/hive/ql/io/orc/
java/org/apache/hadoop/hive/ql/io/orc/
protobuf/org/apache/hadoop/hive/ql/io/orc/
test/org/apache/hadoop/hive/ql/io/orc/ test/res...
Author: gunther
Date: Fri Nov 7 07:41:28 2014
New Revision: 1637303
URL: http://svn.apache.org/r1637303
Log:
HIVE-8732: ORC string statistics are not merged correctly (Owen O'Malley, reviewed by Prasanth J)
Added:
hive/branches/branch-0.14/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestColumnStatistics.java
Modified:
hive/branches/branch-0.14/ql/src/gen/protobuf/gen-java/org/apache/hadoop/hive/ql/io/orc/OrcProto.java
hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ColumnStatisticsImpl.java
hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/io/orc/FileDump.java
hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java
hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcNewSplit.java
hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java
hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/io/orc/Reader.java
hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java
hive/branches/branch-0.14/ql/src/protobuf/org/apache/hadoop/hive/ql/io/orc/orc_proto.proto
hive/branches/branch-0.14/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
hive/branches/branch-0.14/ql/src/test/resources/orc-file-dump-dictionary-threshold.out
hive/branches/branch-0.14/ql/src/test/resources/orc-file-dump.out
hive/branches/branch-0.14/ql/src/test/results/clientpositive/alter_merge_orc.q.out
hive/branches/branch-0.14/ql/src/test/results/clientpositive/alter_merge_stats_orc.q.out
hive/branches/branch-0.14/ql/src/test/results/clientpositive/annotate_stats_part.q.out
hive/branches/branch-0.14/ql/src/test/results/clientpositive/annotate_stats_table.q.out
hive/branches/branch-0.14/ql/src/test/results/clientpositive/dynpart_sort_opt_vectorization.q.out
hive/branches/branch-0.14/ql/src/test/results/clientpositive/dynpart_sort_optimization2.q.out
hive/branches/branch-0.14/ql/src/test/results/clientpositive/extrapolate_part_stats_full.q.out
hive/branches/branch-0.14/ql/src/test/results/clientpositive/extrapolate_part_stats_partial.q.out
hive/branches/branch-0.14/ql/src/test/results/clientpositive/orc_analyze.q.out
hive/branches/branch-0.14/ql/src/test/results/clientpositive/tez/alter_merge_orc.q.out
hive/branches/branch-0.14/ql/src/test/results/clientpositive/tez/alter_merge_stats_orc.q.out
hive/branches/branch-0.14/ql/src/test/results/clientpositive/tez/dynpart_sort_opt_vectorization.q.out
hive/branches/branch-0.14/ql/src/test/results/clientpositive/tez/dynpart_sort_optimization2.q.out
hive/branches/branch-0.14/ql/src/test/results/clientpositive/tez/orc_analyze.q.out
hive/branches/branch-0.14/ql/src/test/results/clientpositive/tez/vector_decimal_10_0.q.out
hive/branches/branch-0.14/ql/src/test/results/clientpositive/tez/vectorized_ptf.q.out
hive/branches/branch-0.14/ql/src/test/results/clientpositive/vectorized_ptf.q.out
Modified: hive/branches/branch-0.14/ql/src/gen/protobuf/gen-java/org/apache/hadoop/hive/ql/io/orc/OrcProto.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/gen/protobuf/gen-java/org/apache/hadoop/hive/ql/io/orc/OrcProto.java?rev=1637303&r1=1637302&r2=1637303&view=diff
==============================================================================
--- hive/branches/branch-0.14/ql/src/gen/protobuf/gen-java/org/apache/hadoop/hive/ql/io/orc/OrcProto.java (original)
+++ hive/branches/branch-0.14/ql/src/gen/protobuf/gen-java/org/apache/hadoop/hive/ql/io/orc/OrcProto.java Fri Nov 7 07:41:28 2014
@@ -16333,14 +16333,32 @@ public final class OrcProto {
// repeated uint32 version = 4 [packed = true];
/**
* <code>repeated uint32 version = 4 [packed = true];</code>
+ *
+ * <pre>
+ * the version of the file format
+ * [0, 11] = Hive 0.11
+ * [0, 12] = Hive 0.12
+ * </pre>
*/
java.util.List<java.lang.Integer> getVersionList();
/**
* <code>repeated uint32 version = 4 [packed = true];</code>
+ *
+ * <pre>
+ * the version of the file format
+ * [0, 11] = Hive 0.11
+ * [0, 12] = Hive 0.12
+ * </pre>
*/
int getVersionCount();
/**
* <code>repeated uint32 version = 4 [packed = true];</code>
+ *
+ * <pre>
+ * the version of the file format
+ * [0, 11] = Hive 0.11
+ * [0, 12] = Hive 0.12
+ * </pre>
*/
int getVersion(int index);
@@ -16354,6 +16372,28 @@ public final class OrcProto {
*/
long getMetadataLength();
+ // optional uint32 writerVersion = 6;
+ /**
+ * <code>optional uint32 writerVersion = 6;</code>
+ *
+ * <pre>
+ * Version of the writer:
+ * 0 (or missing) = original
+ * 1 = HIVE-8732 fixed
+ * </pre>
+ */
+ boolean hasWriterVersion();
+ /**
+ * <code>optional uint32 writerVersion = 6;</code>
+ *
+ * <pre>
+ * Version of the writer:
+ * 0 (or missing) = original
+ * 1 = HIVE-8732 fixed
+ * </pre>
+ */
+ int getWriterVersion();
+
// optional string magic = 8000;
/**
* <code>optional string magic = 8000;</code>
@@ -16483,8 +16523,13 @@ public final class OrcProto {
metadataLength_ = input.readUInt64();
break;
}
- case 64002: {
+ case 48: {
bitField0_ |= 0x00000010;
+ writerVersion_ = input.readUInt32();
+ break;
+ }
+ case 64002: {
+ bitField0_ |= 0x00000020;
magic_ = input.readBytes();
break;
}
@@ -16584,6 +16629,12 @@ public final class OrcProto {
private java.util.List<java.lang.Integer> version_;
/**
* <code>repeated uint32 version = 4 [packed = true];</code>
+ *
+ * <pre>
+ * the version of the file format
+ * [0, 11] = Hive 0.11
+ * [0, 12] = Hive 0.12
+ * </pre>
*/
public java.util.List<java.lang.Integer>
getVersionList() {
@@ -16591,12 +16642,24 @@ public final class OrcProto {
}
/**
* <code>repeated uint32 version = 4 [packed = true];</code>
+ *
+ * <pre>
+ * the version of the file format
+ * [0, 11] = Hive 0.11
+ * [0, 12] = Hive 0.12
+ * </pre>
*/
public int getVersionCount() {
return version_.size();
}
/**
* <code>repeated uint32 version = 4 [packed = true];</code>
+ *
+ * <pre>
+ * the version of the file format
+ * [0, 11] = Hive 0.11
+ * [0, 12] = Hive 0.12
+ * </pre>
*/
public int getVersion(int index) {
return version_.get(index);
@@ -16619,6 +16682,34 @@ public final class OrcProto {
return metadataLength_;
}
+ // optional uint32 writerVersion = 6;
+ public static final int WRITERVERSION_FIELD_NUMBER = 6;
+ private int writerVersion_;
+ /**
+ * <code>optional uint32 writerVersion = 6;</code>
+ *
+ * <pre>
+ * Version of the writer:
+ * 0 (or missing) = original
+ * 1 = HIVE-8732 fixed
+ * </pre>
+ */
+ public boolean hasWriterVersion() {
+ return ((bitField0_ & 0x00000010) == 0x00000010);
+ }
+ /**
+ * <code>optional uint32 writerVersion = 6;</code>
+ *
+ * <pre>
+ * Version of the writer:
+ * 0 (or missing) = original
+ * 1 = HIVE-8732 fixed
+ * </pre>
+ */
+ public int getWriterVersion() {
+ return writerVersion_;
+ }
+
// optional string magic = 8000;
public static final int MAGIC_FIELD_NUMBER = 8000;
private java.lang.Object magic_;
@@ -16630,7 +16721,7 @@ public final class OrcProto {
* </pre>
*/
public boolean hasMagic() {
- return ((bitField0_ & 0x00000010) == 0x00000010);
+ return ((bitField0_ & 0x00000020) == 0x00000020);
}
/**
* <code>optional string magic = 8000;</code>
@@ -16680,6 +16771,7 @@ public final class OrcProto {
compressionBlockSize_ = 0L;
version_ = java.util.Collections.emptyList();
metadataLength_ = 0L;
+ writerVersion_ = 0;
magic_ = "";
}
private byte memoizedIsInitialized = -1;
@@ -16714,6 +16806,9 @@ public final class OrcProto {
output.writeUInt64(5, metadataLength_);
}
if (((bitField0_ & 0x00000010) == 0x00000010)) {
+ output.writeUInt32(6, writerVersion_);
+ }
+ if (((bitField0_ & 0x00000020) == 0x00000020)) {
output.writeBytes(8000, getMagicBytes());
}
getUnknownFields().writeTo(output);
@@ -16757,6 +16852,10 @@ public final class OrcProto {
}
if (((bitField0_ & 0x00000010) == 0x00000010)) {
size += com.google.protobuf.CodedOutputStream
+ .computeUInt32Size(6, writerVersion_);
+ }
+ if (((bitField0_ & 0x00000020) == 0x00000020)) {
+ size += com.google.protobuf.CodedOutputStream
.computeBytesSize(8000, getMagicBytes());
}
size += getUnknownFields().getSerializedSize();
@@ -16889,8 +16988,10 @@ public final class OrcProto {
bitField0_ = (bitField0_ & ~0x00000008);
metadataLength_ = 0L;
bitField0_ = (bitField0_ & ~0x00000010);
- magic_ = "";
+ writerVersion_ = 0;
bitField0_ = (bitField0_ & ~0x00000020);
+ magic_ = "";
+ bitField0_ = (bitField0_ & ~0x00000040);
return this;
}
@@ -16943,6 +17044,10 @@ public final class OrcProto {
if (((from_bitField0_ & 0x00000020) == 0x00000020)) {
to_bitField0_ |= 0x00000010;
}
+ result.writerVersion_ = writerVersion_;
+ if (((from_bitField0_ & 0x00000040) == 0x00000040)) {
+ to_bitField0_ |= 0x00000020;
+ }
result.magic_ = magic_;
result.bitField0_ = to_bitField0_;
onBuilt();
@@ -16982,8 +17087,11 @@ public final class OrcProto {
if (other.hasMetadataLength()) {
setMetadataLength(other.getMetadataLength());
}
+ if (other.hasWriterVersion()) {
+ setWriterVersion(other.getWriterVersion());
+ }
if (other.hasMagic()) {
- bitField0_ |= 0x00000020;
+ bitField0_ |= 0x00000040;
magic_ = other.magic_;
onChanged();
}
@@ -17126,6 +17234,12 @@ public final class OrcProto {
}
/**
* <code>repeated uint32 version = 4 [packed = true];</code>
+ *
+ * <pre>
+ * the version of the file format
+ * [0, 11] = Hive 0.11
+ * [0, 12] = Hive 0.12
+ * </pre>
*/
public java.util.List<java.lang.Integer>
getVersionList() {
@@ -17133,18 +17247,36 @@ public final class OrcProto {
}
/**
* <code>repeated uint32 version = 4 [packed = true];</code>
+ *
+ * <pre>
+ * the version of the file format
+ * [0, 11] = Hive 0.11
+ * [0, 12] = Hive 0.12
+ * </pre>
*/
public int getVersionCount() {
return version_.size();
}
/**
* <code>repeated uint32 version = 4 [packed = true];</code>
+ *
+ * <pre>
+ * the version of the file format
+ * [0, 11] = Hive 0.11
+ * [0, 12] = Hive 0.12
+ * </pre>
*/
public int getVersion(int index) {
return version_.get(index);
}
/**
* <code>repeated uint32 version = 4 [packed = true];</code>
+ *
+ * <pre>
+ * the version of the file format
+ * [0, 11] = Hive 0.11
+ * [0, 12] = Hive 0.12
+ * </pre>
*/
public Builder setVersion(
int index, int value) {
@@ -17155,6 +17287,12 @@ public final class OrcProto {
}
/**
* <code>repeated uint32 version = 4 [packed = true];</code>
+ *
+ * <pre>
+ * the version of the file format
+ * [0, 11] = Hive 0.11
+ * [0, 12] = Hive 0.12
+ * </pre>
*/
public Builder addVersion(int value) {
ensureVersionIsMutable();
@@ -17164,6 +17302,12 @@ public final class OrcProto {
}
/**
* <code>repeated uint32 version = 4 [packed = true];</code>
+ *
+ * <pre>
+ * the version of the file format
+ * [0, 11] = Hive 0.11
+ * [0, 12] = Hive 0.12
+ * </pre>
*/
public Builder addAllVersion(
java.lang.Iterable<? extends java.lang.Integer> values) {
@@ -17174,6 +17318,12 @@ public final class OrcProto {
}
/**
* <code>repeated uint32 version = 4 [packed = true];</code>
+ *
+ * <pre>
+ * the version of the file format
+ * [0, 11] = Hive 0.11
+ * [0, 12] = Hive 0.12
+ * </pre>
*/
public Builder clearVersion() {
version_ = java.util.Collections.emptyList();
@@ -17215,6 +17365,63 @@ public final class OrcProto {
return this;
}
+ // optional uint32 writerVersion = 6;
+ private int writerVersion_ ;
+ /**
+ * <code>optional uint32 writerVersion = 6;</code>
+ *
+ * <pre>
+ * Version of the writer:
+ * 0 (or missing) = original
+ * 1 = HIVE-8732 fixed
+ * </pre>
+ */
+ public boolean hasWriterVersion() {
+ return ((bitField0_ & 0x00000020) == 0x00000020);
+ }
+ /**
+ * <code>optional uint32 writerVersion = 6;</code>
+ *
+ * <pre>
+ * Version of the writer:
+ * 0 (or missing) = original
+ * 1 = HIVE-8732 fixed
+ * </pre>
+ */
+ public int getWriterVersion() {
+ return writerVersion_;
+ }
+ /**
+ * <code>optional uint32 writerVersion = 6;</code>
+ *
+ * <pre>
+ * Version of the writer:
+ * 0 (or missing) = original
+ * 1 = HIVE-8732 fixed
+ * </pre>
+ */
+ public Builder setWriterVersion(int value) {
+ bitField0_ |= 0x00000020;
+ writerVersion_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>optional uint32 writerVersion = 6;</code>
+ *
+ * <pre>
+ * Version of the writer:
+ * 0 (or missing) = original
+ * 1 = HIVE-8732 fixed
+ * </pre>
+ */
+ public Builder clearWriterVersion() {
+ bitField0_ = (bitField0_ & ~0x00000020);
+ writerVersion_ = 0;
+ onChanged();
+ return this;
+ }
+
// optional string magic = 8000;
private java.lang.Object magic_ = "";
/**
@@ -17225,7 +17432,7 @@ public final class OrcProto {
* </pre>
*/
public boolean hasMagic() {
- return ((bitField0_ & 0x00000020) == 0x00000020);
+ return ((bitField0_ & 0x00000040) == 0x00000040);
}
/**
* <code>optional string magic = 8000;</code>
@@ -17277,7 +17484,7 @@ public final class OrcProto {
if (value == null) {
throw new NullPointerException();
}
- bitField0_ |= 0x00000020;
+ bitField0_ |= 0x00000040;
magic_ = value;
onChanged();
return this;
@@ -17290,7 +17497,7 @@ public final class OrcProto {
* </pre>
*/
public Builder clearMagic() {
- bitField0_ = (bitField0_ & ~0x00000020);
+ bitField0_ = (bitField0_ & ~0x00000040);
magic_ = getDefaultInstance().getMagic();
onChanged();
return this;
@@ -17307,7 +17514,7 @@ public final class OrcProto {
if (value == null) {
throw new NullPointerException();
}
- bitField0_ |= 0x00000020;
+ bitField0_ |= 0x00000040;
magic_ = value;
onChanged();
return this;
@@ -17513,13 +17720,14 @@ public final class OrcProto {
"em\022\024\n\014numberOfRows\030\006 \001(\004\022F\n\nstatistics\030\007" +
" \003(\01322.org.apache.hadoop.hive.ql.io.orc." +
"ColumnStatistics\022\026\n\016rowIndexStride\030\010 \001(\r" +
- "\"\305\001\n\nPostScript\022\024\n\014footerLength\030\001 \001(\004\022F\n" +
+ "\"\334\001\n\nPostScript\022\024\n\014footerLength\030\001 \001(\004\022F\n" +
"\013compression\030\002 \001(\01621.org.apache.hadoop.h" +
"ive.ql.io.orc.CompressionKind\022\034\n\024compres" +
"sionBlockSize\030\003 \001(\004\022\023\n\007version\030\004 \003(\rB\002\020\001" +
- "\022\026\n\016metadataLength\030\005 \001(\004\022\016\n\005magic\030\300> \001(\t",
- "*:\n\017CompressionKind\022\010\n\004NONE\020\000\022\010\n\004ZLIB\020\001\022" +
- "\n\n\006SNAPPY\020\002\022\007\n\003LZO\020\003"
+ "\022\026\n\016metadataLength\030\005 \001(\004\022\025\n\rwriterVersio",
+ "n\030\006 \001(\r\022\016\n\005magic\030\300> \001(\t*:\n\017CompressionKi" +
+ "nd\022\010\n\004NONE\020\000\022\010\n\004ZLIB\020\001\022\n\n\006SNAPPY\020\002\022\007\n\003LZ" +
+ "O\020\003"
};
com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
@@ -17651,7 +17859,7 @@ public final class OrcProto {
internal_static_org_apache_hadoop_hive_ql_io_orc_PostScript_fieldAccessorTable = new
com.google.protobuf.GeneratedMessage.FieldAccessorTable(
internal_static_org_apache_hadoop_hive_ql_io_orc_PostScript_descriptor,
- new java.lang.String[] { "FooterLength", "Compression", "CompressionBlockSize", "Version", "MetadataLength", "Magic", });
+ new java.lang.String[] { "FooterLength", "Compression", "CompressionBlockSize", "Version", "MetadataLength", "WriterVersion", "Magic", });
return null;
}
};
Modified: hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ColumnStatisticsImpl.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ColumnStatisticsImpl.java?rev=1637303&r1=1637302&r2=1637303&view=diff
==============================================================================
--- hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ColumnStatisticsImpl.java (original)
+++ hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ColumnStatisticsImpl.java Fri Nov 7 07:41:28 2014
@@ -394,7 +394,8 @@ class ColumnStatisticsImpl implements Co
} else if (str.minimum != null) {
if (minimum.compareTo(str.minimum) > 0) {
minimum = new Text(str.getMinimum());
- } else if (maximum.compareTo(str.maximum) < 0) {
+ }
+ if (maximum.compareTo(str.maximum) < 0) {
maximum = new Text(str.getMaximum());
}
}
@@ -563,7 +564,8 @@ class ColumnStatisticsImpl implements Co
} else if (dec.minimum != null) {
if (minimum.compareTo(dec.minimum) > 0) {
minimum = dec.minimum;
- } else if (maximum.compareTo(dec.maximum) < 0) {
+ }
+ if (maximum.compareTo(dec.maximum) < 0) {
maximum = dec.maximum;
}
if (sum == null || dec.sum == null) {
@@ -671,7 +673,8 @@ class ColumnStatisticsImpl implements Co
} else if (dateStats.minimum != null) {
if (minimum > dateStats.minimum) {
minimum = dateStats.minimum;
- } else if (maximum < dateStats.maximum) {
+ }
+ if (maximum < dateStats.maximum) {
maximum = dateStats.maximum;
}
}
@@ -767,7 +770,8 @@ class ColumnStatisticsImpl implements Co
} else if (timestampStats.minimum != null) {
if (minimum > timestampStats.minimum) {
minimum = timestampStats.minimum;
- } else if (maximum < timestampStats.maximum) {
+ }
+ if (maximum < timestampStats.maximum) {
maximum = timestampStats.maximum;
}
}
Modified: hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/io/orc/FileDump.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/io/orc/FileDump.java?rev=1637303&r1=1637302&r2=1637303&view=diff
==============================================================================
--- hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/io/orc/FileDump.java (original)
+++ hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/io/orc/FileDump.java Fri Nov 7 07:41:28 2014
@@ -65,6 +65,8 @@ public final class FileDump {
System.out.println("Structure for " + filename);
Path path = new Path(filename);
Reader reader = OrcFile.createReader(path, OrcFile.readerOptions(conf));
+ System.out.println("File Version: " + reader.getFileVersion().getName() +
+ " with " + reader.getWriterVersion());
RecordReaderImpl rows = (RecordReaderImpl) reader.rows();
System.out.println("Rows: " + reader.getNumberOfRows());
System.out.println("Compression: " + reader.getCompression());
Modified: hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java?rev=1637303&r1=1637302&r2=1637303&view=diff
==============================================================================
--- hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java (original)
+++ hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java Fri Nov 7 07:41:28 2014
@@ -97,6 +97,26 @@ public final class OrcFile {
}
}
+ /**
+ * Records the version of the writer in terms of which bugs have been fixed.
+ * For bugs in the writer, but the old readers already read the new data
+ * correctly, bump this version instead of the Version.
+ */
+ public static enum WriterVersion {
+ ORIGINAL(0),
+ HIVE_8732(1); // corrupted stripe/file maximum column statistics
+
+ private final int id;
+
+ public int getId() {
+ return id;
+ }
+
+ private WriterVersion(int id) {
+ this.id = id;
+ }
+ }
+
public static enum EncodingStrategy {
SPEED, COMPRESSION;
}
Modified: hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java?rev=1637303&r1=1637302&r2=1637303&view=diff
==============================================================================
--- hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java (original)
+++ hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java Fri Nov 7 07:41:28 2014
@@ -630,6 +630,7 @@ public class OrcInputFormat implements
private final boolean isOriginal;
private final List<Long> deltas;
private final boolean hasBase;
+ private OrcFile.WriterVersion writerVersion;
SplitGenerator(Context context, FileSystem fs,
FileStatus file, FileInfo fileInfo,
@@ -775,7 +776,9 @@ public class OrcInputFormat implements
Reader.Options options = new Reader.Options();
setIncludedColumns(options, types, context.conf, isOriginal);
setSearchArgument(options, types, context.conf, isOriginal);
- if (options.getSearchArgument() != null) {
+ // only do split pruning if HIVE-8732 has been fixed in the writer
+ if (options.getSearchArgument() != null &&
+ writerVersion != OrcFile.WriterVersion.ORIGINAL) {
SearchArgument sarg = options.getSearchArgument();
List<PredicateLeaf> sargLeaves = sarg.getLeaves();
List<StripeStatistics> stripeStats = metadata.getStripeStatistics();
@@ -866,6 +869,7 @@ public class OrcInputFormat implements
fileMetaInfo = fileInfo.fileMetaInfo;
metadata = fileInfo.metadata;
types = fileInfo.types;
+ writerVersion = fileInfo.writerVersion;
// For multiple runs, in case sendSplitsInFooter changes
if (fileMetaInfo == null && context.footerInSplits) {
orcReader = OrcFile.createReader(file.getPath(),
@@ -873,6 +877,7 @@ public class OrcInputFormat implements
fileInfo.fileMetaInfo = ((ReaderImpl) orcReader).getFileMetaInfo();
fileInfo.metadata = orcReader.getMetadata();
fileInfo.types = orcReader.getTypes();
+ fileInfo.writerVersion = orcReader.getWriterVersion();
}
} else {
orcReader = OrcFile.createReader(file.getPath(),
@@ -880,13 +885,14 @@ public class OrcInputFormat implements
stripes = orcReader.getStripes();
metadata = orcReader.getMetadata();
types = orcReader.getTypes();
+ writerVersion = orcReader.getWriterVersion();
fileMetaInfo = context.footerInSplits ?
((ReaderImpl) orcReader).getFileMetaInfo() : null;
if (context.cacheStripeDetails) {
// Populate into cache.
Context.footerCache.put(file.getPath(),
new FileInfo(file.getModificationTime(), file.getLen(), stripes,
- metadata, types, fileMetaInfo));
+ metadata, types, fileMetaInfo, writerVersion));
}
}
} catch (Throwable th) {
@@ -981,18 +987,21 @@ public class OrcInputFormat implements
ReaderImpl.FileMetaInfo fileMetaInfo;
Metadata metadata;
List<OrcProto.Type> types;
+ private OrcFile.WriterVersion writerVersion;
FileInfo(long modificationTime, long size,
List<StripeInformation> stripeInfos,
Metadata metadata, List<OrcProto.Type> types,
- ReaderImpl.FileMetaInfo fileMetaInfo) {
+ ReaderImpl.FileMetaInfo fileMetaInfo,
+ OrcFile.WriterVersion writerVersion) {
this.modificationTime = modificationTime;
this.size = size;
this.stripeInfos = stripeInfos;
this.fileMetaInfo = fileMetaInfo;
this.metadata = metadata;
this.types = types;
+ this.writerVersion = writerVersion;
}
}
Modified: hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcNewSplit.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcNewSplit.java?rev=1637303&r1=1637302&r2=1637303&view=diff
==============================================================================
--- hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcNewSplit.java (original)
+++ hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcNewSplit.java Fri Nov 7 07:41:28 2014
@@ -38,6 +38,7 @@ public class OrcNewSplit extends FileSpl
private boolean isOriginal;
private boolean hasBase;
private final List<Long> deltas = new ArrayList<Long>();
+ private OrcFile.WriterVersion writerVersion;
protected OrcNewSplit(){
//The FileSplit() constructor in hadoop 0.20 and 1.x is package private so can't use it.
@@ -83,6 +84,7 @@ public class OrcNewSplit extends FileSpl
WritableUtils.writeVInt(out, footerBuff.limit() - footerBuff.position());
out.write(footerBuff.array(), footerBuff.position(),
footerBuff.limit() - footerBuff.position());
+ WritableUtils.writeVInt(out, fileMetaInfo.writerVersion.getId());
}
}
@@ -111,9 +113,11 @@ public class OrcNewSplit extends FileSpl
int footerBuffSize = WritableUtils.readVInt(in);
ByteBuffer footerBuff = ByteBuffer.allocate(footerBuffSize);
in.readFully(footerBuff.array(), 0, footerBuffSize);
+ OrcFile.WriterVersion writerVersion =
+ ReaderImpl.getWriterVersion(WritableUtils.readVInt(in));
fileMetaInfo = new ReaderImpl.FileMetaInfo(compressionType, bufferSize,
- metadataSize, footerBuff);
+ metadataSize, footerBuff, writerVersion);
}
}
Modified: hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java?rev=1637303&r1=1637302&r2=1637303&view=diff
==============================================================================
--- hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java (original)
+++ hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java Fri Nov 7 07:41:28 2014
@@ -42,6 +42,7 @@ public class OrcSplit extends FileSplit
private boolean isOriginal;
private boolean hasBase;
private final List<Long> deltas = new ArrayList<Long>();
+ private OrcFile.WriterVersion writerVersion;
static final int BASE_FLAG = 4;
static final int ORIGINAL_FLAG = 2;
@@ -92,6 +93,7 @@ public class OrcSplit extends FileSplit
WritableUtils.writeVInt(out, footerBuff.limit() - footerBuff.position());
out.write(footerBuff.array(), footerBuff.position(),
footerBuff.limit() - footerBuff.position());
+ WritableUtils.writeVInt(out, fileMetaInfo.writerVersion.getId());
}
}
@@ -120,9 +122,11 @@ public class OrcSplit extends FileSplit
int footerBuffSize = WritableUtils.readVInt(in);
ByteBuffer footerBuff = ByteBuffer.allocate(footerBuffSize);
in.readFully(footerBuff.array(), 0, footerBuffSize);
+ OrcFile.WriterVersion writerVersion =
+ ReaderImpl.getWriterVersion(WritableUtils.readVInt(in));
fileMetaInfo = new ReaderImpl.FileMetaInfo(compressionType, bufferSize,
- metadataSize, footerBuff);
+ metadataSize, footerBuff, writerVersion);
}
}
Modified: hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/io/orc/Reader.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/io/orc/Reader.java?rev=1637303&r1=1637302&r2=1637303&view=diff
==============================================================================
--- hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/io/orc/Reader.java (original)
+++ hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/io/orc/Reader.java Fri Nov 7 07:41:28 2014
@@ -129,6 +129,16 @@ public interface Reader {
List<OrcProto.Type> getTypes();
/**
+ * Get the file format version.
+ */
+ OrcFile.Version getFileVersion();
+
+ /**
+ * Get the version of the writer of this file.
+ */
+ OrcFile.WriterVersion getWriterVersion();
+
+ /**
* Options for creating a RecordReader.
*/
public static class Options {
Modified: hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java?rev=1637303&r1=1637302&r2=1637303&view=diff
==============================================================================
--- hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java (original)
+++ hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java Fri Nov 7 07:41:28 2014
@@ -62,6 +62,7 @@ final class ReaderImpl implements Reader
private long deserializedSize = -1;
private final Configuration conf;
private final List<Integer> versionList;
+ private final OrcFile.WriterVersion writerVersion;
//serialized footer - Keeping this around for use by getFileMetaInfo()
// will help avoid cpu cycles spend in deserializing at cost of increased
@@ -182,6 +183,22 @@ final class ReaderImpl implements Reader
}
@Override
+ public OrcFile.Version getFileVersion() {
+ for (OrcFile.Version version: OrcFile.Version.values()) {
+ if (version.getMajor() == versionList.get(0) &&
+ version.getMinor() == versionList.get(1)) {
+ return version;
+ }
+ }
+ return OrcFile.Version.V_0_11;
+ }
+
+ @Override
+ public OrcFile.WriterVersion getWriterVersion() {
+ return writerVersion;
+ }
+
+ @Override
public int getRowIndexStride() {
return footer.getRowIndexStride();
}
@@ -309,8 +326,22 @@ final class ReaderImpl implements Reader
this.footer = rInfo.footer;
this.inspector = rInfo.inspector;
this.versionList = footerMetaData.versionList;
+ this.writerVersion = footerMetaData.writerVersion;
}
+ /**
+ * Get the WriterVersion based on the ORC file postscript.
+ * @param writerVersion the integer writer version
+ * @return
+ */
+ static OrcFile.WriterVersion getWriterVersion(int writerVersion) {
+ for(OrcFile.WriterVersion version: OrcFile.WriterVersion.values()) {
+ if (version.getId() == writerVersion) {
+ return version;
+ }
+ }
+ return OrcFile.WriterVersion.ORIGINAL;
+ }
private static FileMetaInfo extractMetaInfoFromFooter(FileSystem fs,
Path path,
@@ -346,6 +377,12 @@ final class ReaderImpl implements Reader
int footerSize = (int) ps.getFooterLength();
int metadataSize = (int) ps.getMetadataLength();
+ OrcFile.WriterVersion writerVersion;
+ if (ps.hasWriterVersion()) {
+ writerVersion = getWriterVersion(ps.getWriterVersion());
+ } else {
+ writerVersion = OrcFile.WriterVersion.ORIGINAL;
+ }
//check compression codec
switch (ps.getCompression()) {
@@ -391,7 +428,8 @@ final class ReaderImpl implements Reader
(int) ps.getCompressionBlockSize(),
(int) ps.getMetadataLength(),
buffer,
- ps.getVersionList()
+ ps.getVersionList(),
+ writerVersion
);
}
@@ -451,25 +489,29 @@ final class ReaderImpl implements Reader
final int metadataSize;
final ByteBuffer footerBuffer;
final List<Integer> versionList;
+ final OrcFile.WriterVersion writerVersion;
FileMetaInfo(String compressionType, int bufferSize, int metadataSize,
- ByteBuffer footerBuffer) {
- this(compressionType, bufferSize, metadataSize, footerBuffer, null);
+ ByteBuffer footerBuffer, OrcFile.WriterVersion writerVersion) {
+ this(compressionType, bufferSize, metadataSize, footerBuffer, null,
+ writerVersion);
}
FileMetaInfo(String compressionType, int bufferSize, int metadataSize,
- ByteBuffer footerBuffer, List<Integer> versionList){
+ ByteBuffer footerBuffer, List<Integer> versionList,
+ OrcFile.WriterVersion writerVersion){
this.compressionType = compressionType;
this.bufferSize = bufferSize;
this.metadataSize = metadataSize;
this.footerBuffer = footerBuffer;
this.versionList = versionList;
+ this.writerVersion = writerVersion;
}
}
public FileMetaInfo getFileMetaInfo(){
return new FileMetaInfo(compressionKind.toString(), bufferSize,
- metadataSize, footerByteBuffer, versionList);
+ metadataSize, footerByteBuffer, versionList, writerVersion);
}
Modified: hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java?rev=1637303&r1=1637302&r2=1637303&view=diff
==============================================================================
--- hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java (original)
+++ hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java Fri Nov 7 07:41:28 2014
@@ -2230,7 +2230,8 @@ class WriterImpl implements Writer, Memo
.setMetadataLength(metadataLength)
.setMagic(OrcFile.MAGIC)
.addVersion(version.getMajor())
- .addVersion(version.getMinor());
+ .addVersion(version.getMinor())
+ .setWriterVersion(OrcFile.WriterVersion.HIVE_8732.getId());
if (compress != CompressionKind.NONE) {
builder.setCompressionBlockSize(bufferSize);
}
Modified: hive/branches/branch-0.14/ql/src/protobuf/org/apache/hadoop/hive/ql/io/orc/orc_proto.proto
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/protobuf/org/apache/hadoop/hive/ql/io/orc/orc_proto.proto?rev=1637303&r1=1637302&r2=1637303&view=diff
==============================================================================
--- hive/branches/branch-0.14/ql/src/protobuf/org/apache/hadoop/hive/ql/io/orc/orc_proto.proto (original)
+++ hive/branches/branch-0.14/ql/src/protobuf/org/apache/hadoop/hive/ql/io/orc/orc_proto.proto Fri Nov 7 07:41:28 2014
@@ -191,8 +191,15 @@ message PostScript {
optional uint64 footerLength = 1;
optional CompressionKind compression = 2;
optional uint64 compressionBlockSize = 3;
+ // the version of the file format
+ // [0, 11] = Hive 0.11
+ // [0, 12] = Hive 0.12
repeated uint32 version = 4 [packed = true];
optional uint64 metadataLength = 5;
+ // Version of the writer:
+ // 0 (or missing) = original
+ // 1 = HIVE-8732 fixed
+ optional uint32 writerVersion = 6;
// Leave this last in the record
optional string magic = 8000;
}
Added: hive/branches/branch-0.14/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestColumnStatistics.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestColumnStatistics.java?rev=1637303&view=auto
==============================================================================
--- hive/branches/branch-0.14/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestColumnStatistics.java (added)
+++ hive/branches/branch-0.14/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestColumnStatistics.java Fri Nov 7 07:41:28 2014
@@ -0,0 +1,176 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.io.orc;
+
+import org.apache.hadoop.hive.common.type.HiveDecimal;
+import org.apache.hadoop.hive.serde2.io.DateWritable;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+import java.sql.Timestamp;
+
+import static junit.framework.Assert.assertEquals;
+
+/**
+ * Test ColumnStatisticsImpl for ORC.
+ */
+public class TestColumnStatistics {
+
+ @Test
+ public void testLongMerge() throws Exception {
+ ObjectInspector inspector =
+ PrimitiveObjectInspectorFactory.javaIntObjectInspector;
+
+ ColumnStatisticsImpl stats1 = ColumnStatisticsImpl.create(inspector);
+ ColumnStatisticsImpl stats2 = ColumnStatisticsImpl.create(inspector);
+ stats1.updateInteger(10);
+ stats1.updateInteger(10);
+ stats2.updateInteger(1);
+ stats2.updateInteger(1000);
+ stats1.merge(stats2);
+ IntegerColumnStatistics typed = (IntegerColumnStatistics) stats1;
+ assertEquals(1, typed.getMinimum());
+ assertEquals(1000, typed.getMaximum());
+ stats1.reset();
+ stats1.updateInteger(-10);
+ stats1.updateInteger(10000);
+ stats1.merge(stats2);
+ assertEquals(-10, typed.getMinimum());
+ assertEquals(10000, typed.getMaximum());
+ }
+
+ @Test
+ public void testDoubleMerge() throws Exception {
+ ObjectInspector inspector =
+ PrimitiveObjectInspectorFactory.javaDoubleObjectInspector;
+
+ ColumnStatisticsImpl stats1 = ColumnStatisticsImpl.create(inspector);
+ ColumnStatisticsImpl stats2 = ColumnStatisticsImpl.create(inspector);
+ stats1.updateDouble(10.0);
+ stats1.updateDouble(100.0);
+ stats2.updateDouble(1.0);
+ stats2.updateDouble(1000.0);
+ stats1.merge(stats2);
+ DoubleColumnStatistics typed = (DoubleColumnStatistics) stats1;
+ assertEquals(1.0, typed.getMinimum(), 0.001);
+ assertEquals(1000.0, typed.getMaximum(), 0.001);
+ stats1.reset();
+ stats1.updateDouble(-10);
+ stats1.updateDouble(10000);
+ stats1.merge(stats2);
+ assertEquals(-10, typed.getMinimum(), 0.001);
+ assertEquals(10000, typed.getMaximum(), 0.001);
+ }
+
+
+ @Test
+ public void testStringMerge() throws Exception {
+ ObjectInspector inspector =
+ PrimitiveObjectInspectorFactory.javaStringObjectInspector;
+
+ ColumnStatisticsImpl stats1 = ColumnStatisticsImpl.create(inspector);
+ ColumnStatisticsImpl stats2 = ColumnStatisticsImpl.create(inspector);
+ stats1.updateString(new Text("bob"));
+ stats1.updateString(new Text("david"));
+ stats1.updateString(new Text("charles"));
+ stats2.updateString(new Text("anne"));
+ stats2.updateString(new Text("erin"));
+ stats1.merge(stats2);
+ StringColumnStatistics typed = (StringColumnStatistics) stats1;
+ assertEquals("anne", typed.getMinimum());
+ assertEquals("erin", typed.getMaximum());
+ stats1.reset();
+ stats1.updateString(new Text("aaa"));
+ stats1.updateString(new Text("zzz"));
+ stats1.merge(stats2);
+ assertEquals("aaa", typed.getMinimum());
+ assertEquals("zzz", typed.getMaximum());
+ }
+
+ @Test
+ public void testDateMerge() throws Exception {
+ ObjectInspector inspector =
+ PrimitiveObjectInspectorFactory.javaDateObjectInspector;
+
+ ColumnStatisticsImpl stats1 = ColumnStatisticsImpl.create(inspector);
+ ColumnStatisticsImpl stats2 = ColumnStatisticsImpl.create(inspector);
+ stats1.updateDate(new DateWritable(1000));
+ stats1.updateDate(new DateWritable(100));
+ stats2.updateDate(new DateWritable(10));
+ stats2.updateDate(new DateWritable(2000));
+ stats1.merge(stats2);
+ DateColumnStatistics typed = (DateColumnStatistics) stats1;
+ assertEquals(new DateWritable(10), typed.getMinimum());
+ assertEquals(new DateWritable(2000), typed.getMaximum());
+ stats1.reset();
+ stats1.updateDate(new DateWritable(-10));
+ stats1.updateDate(new DateWritable(10000));
+ stats1.merge(stats2);
+ assertEquals(-10, typed.getMinimum().getDays());
+ assertEquals(10000, typed.getMaximum().getDays());
+ }
+
+ @Test
+ public void testTimestampMerge() throws Exception {
+ ObjectInspector inspector =
+ PrimitiveObjectInspectorFactory.javaTimestampObjectInspector;
+
+ ColumnStatisticsImpl stats1 = ColumnStatisticsImpl.create(inspector);
+ ColumnStatisticsImpl stats2 = ColumnStatisticsImpl.create(inspector);
+ stats1.updateTimestamp(new Timestamp(10));
+ stats1.updateTimestamp(new Timestamp(100));
+ stats2.updateTimestamp(new Timestamp(1));
+ stats2.updateTimestamp(new Timestamp(1000));
+ stats1.merge(stats2);
+ TimestampColumnStatistics typed = (TimestampColumnStatistics) stats1;
+ assertEquals(1, typed.getMinimum().getTime());
+ assertEquals(1000, typed.getMaximum().getTime());
+ stats1.reset();
+ stats1.updateTimestamp(new Timestamp(-10));
+ stats1.updateTimestamp(new Timestamp(10000));
+ stats1.merge(stats2);
+ assertEquals(-10, typed.getMinimum().getTime());
+ assertEquals(10000, typed.getMaximum().getTime());
+ }
+
+ @Test
+ public void testDecimalMerge() throws Exception {
+ ObjectInspector inspector =
+ PrimitiveObjectInspectorFactory.javaHiveDecimalObjectInspector;
+
+ ColumnStatisticsImpl stats1 = ColumnStatisticsImpl.create(inspector);
+ ColumnStatisticsImpl stats2 = ColumnStatisticsImpl.create(inspector);
+ stats1.updateDecimal(HiveDecimal.create(10));
+ stats1.updateDecimal(HiveDecimal.create(100));
+ stats2.updateDecimal(HiveDecimal.create(1));
+ stats2.updateDecimal(HiveDecimal.create(1000));
+ stats1.merge(stats2);
+ DecimalColumnStatistics typed = (DecimalColumnStatistics) stats1;
+ assertEquals(1, typed.getMinimum().longValue());
+ assertEquals(1000, typed.getMaximum().longValue());
+ stats1.reset();
+ stats1.updateDecimal(HiveDecimal.create(-10));
+ stats1.updateDecimal(HiveDecimal.create(10000));
+ stats1.merge(stats2);
+ assertEquals(-10, typed.getMinimum().longValue());
+ assertEquals(10000, typed.getMaximum().longValue());
+ }
+}
Modified: hive/branches/branch-0.14/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java?rev=1637303&r1=1637302&r2=1637303&view=diff
==============================================================================
--- hive/branches/branch-0.14/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java (original)
+++ hive/branches/branch-0.14/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java Fri Nov 7 07:41:28 2014
@@ -1620,14 +1620,14 @@ public class TestInputOutputFormat {
assertEquals("mock:/combinationAcid/p=0/base_0000010/bucket_00000",
split.getPath().toString());
assertEquals(0, split.getStart());
- assertEquals(580, split.getLength());
+ assertEquals(582, split.getLength());
split = (HiveInputFormat.HiveInputSplit) splits[1];
assertEquals("org.apache.hadoop.hive.ql.io.orc.OrcInputFormat",
split.inputFormatClassName());
assertEquals("mock:/combinationAcid/p=0/base_0000010/bucket_00001",
split.getPath().toString());
assertEquals(0, split.getStart());
- assertEquals(601, split.getLength());
+ assertEquals(603, split.getLength());
CombineHiveInputFormat.CombineHiveInputSplit combineSplit =
(CombineHiveInputFormat.CombineHiveInputSplit) splits[2];
assertEquals(BUCKETS, combineSplit.getNumPaths());
@@ -1635,7 +1635,7 @@ public class TestInputOutputFormat {
assertEquals("mock:/combinationAcid/p=1/00000" + bucket + "_0",
combineSplit.getPath(bucket).toString());
assertEquals(0, combineSplit.getOffset(bucket));
- assertEquals(225, combineSplit.getLength(bucket));
+ assertEquals(227, combineSplit.getLength(bucket));
}
String[] hosts = combineSplit.getLocations();
assertEquals(2, hosts.length);
Modified: hive/branches/branch-0.14/ql/src/test/resources/orc-file-dump-dictionary-threshold.out
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/test/resources/orc-file-dump-dictionary-threshold.out?rev=1637303&r1=1637302&r2=1637303&view=diff
==============================================================================
--- hive/branches/branch-0.14/ql/src/test/resources/orc-file-dump-dictionary-threshold.out (original)
+++ hive/branches/branch-0.14/ql/src/test/resources/orc-file-dump-dictionary-threshold.out Fri Nov 7 07:41:28 2014
@@ -1,4 +1,5 @@
Structure for TestFileDump.testDump.orc
+File Version: 0.12 with HIVE_8732
Rows: 21000
Compression: ZLIB
Compression size: 10000
@@ -182,6 +183,6 @@ Stripes:
Row group index column 3:
Entry 0: count: 1000 min: Darkness,-230-368-488-586-862-930-1686-2044-2636-2652-2872-3108-3162-3192-3404-3442-3508-3542-3550-3712-3980-4146-4204-4336-4390-4418-4424-4490-4512-4650-4768-4924-4950-5210-5524-5630-5678-5710-5758-5952-6238-6252-6300-6366-6668-6712-6926-6942-7100-7194-7802-8030-8452-8608-8640-8862-8868-9134-9234-9412-9602-9608-9642-9678-9740-9780-10426-10510-10514-10706-10814-10870-10942-11028-11244-11326-11462-11496-11656-11830-12022-12178-12418-12832-13304-13448-13590-13618-13908-14188-14246-14340-14364-14394-14762-14850-14964-15048-15494-15674-15726-16006-16056-16180-16304-16332-16452-16598-16730-16810-16994-17210-17268-17786-17962-18214-18444-18446-18724-18912-18952-19164-19348-19400-19546-19776-19896-20084 max: worst-54-290-346-648-908-996-1038-1080-1560-1584-1620-1744-1770-1798-1852-1966-2162-2244-2286-2296-2534-2660-3114-3676-3788-4068-4150-4706-4744-5350-5420-5582-5696-5726-6006-6020-6024-6098-6184-6568-6636-6802-6994-7004-7318-7498-7758-7780-7798-7920-7952-
7960-7988-8232-8256-8390-8416-8478-8620-8840-8984-9038-9128-9236-9248-9344-9594-9650-9714-9928-9938-10178-10368-10414-10502-10732-10876-11008-11158-11410-11722-11836-11964-12054-12096-12126-12136-12202-12246-12298-12616-12774-12782-12790-12802-12976-13216-13246-13502-13766-14454-14974-15004-15124-15252-15294-15356-15530-15610-16316-16936-17024-17122-17214-17310-17528-17682-17742-17870-17878-18010-18410-18524-18788-19204-19254-19518-19596-19786-19874-19904-20390-20752-20936 positions: 0,0,0,0,0
-File length: 2033557 bytes
+File length: 2033559 bytes
Padding length: 0 bytes
Padding ratio: 0%
Modified: hive/branches/branch-0.14/ql/src/test/resources/orc-file-dump.out
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/test/resources/orc-file-dump.out?rev=1637303&r1=1637302&r2=1637303&view=diff
==============================================================================
--- hive/branches/branch-0.14/ql/src/test/resources/orc-file-dump.out (original)
+++ hive/branches/branch-0.14/ql/src/test/resources/orc-file-dump.out Fri Nov 7 07:41:28 2014
@@ -1,4 +1,5 @@
Structure for TestFileDump.testDump.orc
+File Version: 0.12 with HIVE_8732
Rows: 21000
Compression: ZLIB
Compression size: 10000
@@ -187,6 +188,6 @@ Stripes:
Row group index column 3:
Entry 0: count: 1000 min: Darkness, max: worst positions: 0,0,0
-File length: 270760 bytes
+File length: 270762 bytes
Padding length: 0 bytes
Padding ratio: 0%
Modified: hive/branches/branch-0.14/ql/src/test/results/clientpositive/alter_merge_orc.q.out
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/test/results/clientpositive/alter_merge_orc.q.out?rev=1637303&r1=1637302&r2=1637303&view=diff
==============================================================================
--- hive/branches/branch-0.14/ql/src/test/results/clientpositive/alter_merge_orc.q.out (original)
+++ hive/branches/branch-0.14/ql/src/test/results/clientpositive/alter_merge_orc.q.out Fri Nov 7 07:41:28 2014
@@ -48,9 +48,9 @@ columns:struct columns { i32 key, string
partitioned:false
partitionColumns:
totalNumberFiles:3
-totalFileSize:7488
-maxFileSize:2496
-minFileSize:2496
+totalFileSize:7494
+maxFileSize:2498
+minFileSize:2498
#### A masked pattern was here ####
PREHOOK: query: select count(1) from src_orc_merge_test
@@ -91,9 +91,9 @@ columns:struct columns { i32 key, string
partitioned:false
partitionColumns:
totalNumberFiles:1
-totalFileSize:7167
-maxFileSize:7167
-minFileSize:7167
+totalFileSize:7169
+maxFileSize:7169
+minFileSize:7169
#### A masked pattern was here ####
PREHOOK: query: select count(1) from src_orc_merge_test
@@ -171,9 +171,9 @@ columns:struct columns { i32 key, string
partitioned:true
partitionColumns:struct partition_columns { string ds}
totalNumberFiles:3
-totalFileSize:7488
-maxFileSize:2496
-minFileSize:2496
+totalFileSize:7494
+maxFileSize:2498
+minFileSize:2498
#### A masked pattern was here ####
PREHOOK: query: select count(1) from src_orc_merge_test_part
@@ -218,9 +218,9 @@ columns:struct columns { i32 key, string
partitioned:true
partitionColumns:struct partition_columns { string ds}
totalNumberFiles:1
-totalFileSize:7167
-maxFileSize:7167
-minFileSize:7167
+totalFileSize:7169
+maxFileSize:7169
+minFileSize:7169
#### A masked pattern was here ####
PREHOOK: query: select count(1) from src_orc_merge_test_part
Modified: hive/branches/branch-0.14/ql/src/test/results/clientpositive/alter_merge_stats_orc.q.out
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/test/results/clientpositive/alter_merge_stats_orc.q.out?rev=1637303&r1=1637302&r2=1637303&view=diff
==============================================================================
--- hive/branches/branch-0.14/ql/src/test/results/clientpositive/alter_merge_stats_orc.q.out (original)
+++ hive/branches/branch-0.14/ql/src/test/results/clientpositive/alter_merge_stats_orc.q.out Fri Nov 7 07:41:28 2014
@@ -48,9 +48,9 @@ columns:struct columns { i32 key, string
partitioned:false
partitionColumns:
totalNumberFiles:3
-totalFileSize:7488
-maxFileSize:2496
-minFileSize:2496
+totalFileSize:7494
+maxFileSize:2498
+minFileSize:2498
#### A masked pattern was here ####
PREHOOK: query: desc extended src_orc_merge_test_stat
@@ -94,7 +94,7 @@ Table Parameters:
numFiles 3
numRows 1500
rawDataSize 141000
- totalSize 7488
+ totalSize 7494
#### A masked pattern was here ####
# Storage Information
@@ -146,7 +146,7 @@ Table Parameters:
numFiles 1
numRows 1500
rawDataSize 141000
- totalSize 7167
+ totalSize 7169
#### A masked pattern was here ####
# Storage Information
@@ -216,9 +216,9 @@ columns:struct columns { i32 key, string
partitioned:true
partitionColumns:struct partition_columns { string ds}
totalNumberFiles:3
-totalFileSize:7488
-maxFileSize:2496
-minFileSize:2496
+totalFileSize:7494
+maxFileSize:2498
+minFileSize:2498
#### A masked pattern was here ####
PREHOOK: query: desc formatted src_orc_merge_test_part_stat partition (ds='2011')
@@ -249,7 +249,7 @@ Partition Parameters:
numFiles 3
numRows 1500
rawDataSize 141000
- totalSize 7488
+ totalSize 7494
#### A masked pattern was here ####
# Storage Information
@@ -300,7 +300,7 @@ Partition Parameters:
numFiles 3
numRows 1500
rawDataSize 141000
- totalSize 7488
+ totalSize 7494
#### A masked pattern was here ####
# Storage Information
@@ -359,7 +359,7 @@ Partition Parameters:
numFiles 1
numRows 1500
rawDataSize 141000
- totalSize 7167
+ totalSize 7169
#### A masked pattern was here ####
# Storage Information
Modified: hive/branches/branch-0.14/ql/src/test/results/clientpositive/annotate_stats_part.q.out
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/test/results/clientpositive/annotate_stats_part.q.out?rev=1637303&r1=1637302&r2=1637303&view=diff
==============================================================================
--- hive/branches/branch-0.14/ql/src/test/results/clientpositive/annotate_stats_part.q.out (original)
+++ hive/branches/branch-0.14/ql/src/test/results/clientpositive/annotate_stats_part.q.out Fri Nov 7 07:41:28 2014
@@ -98,11 +98,11 @@ STAGE PLANS:
Processor Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 5 Data size: 723 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 5 Data size: 727 Basic stats: COMPLETE Column stats: PARTIAL
Select Operator
expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 5 Data size: 723 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 5 Data size: 727 Basic stats: COMPLETE Column stats: PARTIAL
ListSink
PREHOOK: query: -- partition level analyze statistics for specific parition
@@ -135,11 +135,11 @@ STAGE PLANS:
Processor Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 2 Data size: 323 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 325 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: state (type: string), locid (type: int), zip (type: bigint), '__HIVE_DEFAULT_PARTITION__' (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 2 Data size: 323 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 325 Basic stats: COMPLETE Column stats: NONE
ListSink
PREHOOK: query: -- basicStatState: PARTIAL colStatState: NONE
@@ -158,11 +158,11 @@ STAGE PLANS:
Processor Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 9 Data size: 723 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 9 Data size: 727 Basic stats: COMPLETE Column stats: PARTIAL
Select Operator
expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 9 Data size: 723 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 9 Data size: 727 Basic stats: COMPLETE Column stats: PARTIAL
ListSink
PREHOOK: query: -- basicStatState: COMPLETE colStatState: NONE
@@ -181,11 +181,11 @@ STAGE PLANS:
Processor Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 7 Data size: 400 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 402 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: state (type: string), locid (type: int), zip (type: bigint), '2001' (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 7 Data size: 400 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 402 Basic stats: COMPLETE Column stats: NONE
ListSink
PREHOOK: query: -- partition level analyze statistics for all partitions
@@ -222,11 +222,11 @@ STAGE PLANS:
Processor Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 1 Data size: 323 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 325 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: state (type: string), locid (type: int), zip (type: bigint), '__HIVE_DEFAULT_PARTITION__' (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 323 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 325 Basic stats: COMPLETE Column stats: NONE
ListSink
PREHOOK: query: -- basicStatState: COMPLETE colStatState: NONE
@@ -245,11 +245,11 @@ STAGE PLANS:
Processor Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 8 Data size: 723 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 8 Data size: 727 Basic stats: COMPLETE Column stats: PARTIAL
Select Operator
expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 8 Data size: 723 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 8 Data size: 727 Basic stats: COMPLETE Column stats: PARTIAL
ListSink
PREHOOK: query: -- basicStatState: COMPLETE colStatState: NONE
@@ -268,11 +268,11 @@ STAGE PLANS:
Processor Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 8 Data size: 723 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 8 Data size: 727 Basic stats: COMPLETE Column stats: PARTIAL
Select Operator
expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 8 Data size: 723 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 8 Data size: 727 Basic stats: COMPLETE Column stats: PARTIAL
ListSink
PREHOOK: query: -- both partitions will be pruned
@@ -331,11 +331,11 @@ STAGE PLANS:
Processor Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 8 Data size: 723 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 8 Data size: 727 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: zip (type: bigint)
outputColumnNames: _col0
- Statistics: Num rows: 8 Data size: 723 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 8 Data size: 727 Basic stats: COMPLETE Column stats: NONE
ListSink
PREHOOK: query: -- basicStatState: COMPLETE colStatState: PARTIAL
@@ -354,7 +354,7 @@ STAGE PLANS:
Processor Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 8 Data size: 723 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 8 Data size: 727 Basic stats: COMPLETE Column stats: PARTIAL
Select Operator
expressions: state (type: string)
outputColumnNames: _col0
@@ -377,7 +377,7 @@ STAGE PLANS:
Processor Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 8 Data size: 723 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 727 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: year (type: string)
outputColumnNames: _col0
@@ -402,7 +402,7 @@ STAGE PLANS:
Processor Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 8 Data size: 723 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 8 Data size: 727 Basic stats: COMPLETE Column stats: PARTIAL
Select Operator
expressions: state (type: string), locid (type: int)
outputColumnNames: _col0, _col1
@@ -425,7 +425,7 @@ STAGE PLANS:
Processor Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 7 Data size: 400 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 7 Data size: 402 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: state (type: string), locid (type: int)
outputColumnNames: _col0, _col1
@@ -448,11 +448,11 @@ STAGE PLANS:
Processor Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 1 Data size: 323 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 325 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: state (type: string), locid (type: int)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 323 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 325 Basic stats: COMPLETE Column stats: NONE
ListSink
PREHOOK: query: -- basicStatState: COMPLETE colStatState: PARTIAL
@@ -471,11 +471,11 @@ STAGE PLANS:
Processor Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 8 Data size: 723 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 8 Data size: 727 Basic stats: COMPLETE Column stats: PARTIAL
Select Operator
expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 8 Data size: 723 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 8 Data size: 727 Basic stats: COMPLETE Column stats: PARTIAL
ListSink
PREHOOK: query: -- This is to test filter expression evaluation on partition column
@@ -496,7 +496,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 7 Data size: 400 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 7 Data size: 402 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: (locid > 0) (type: boolean)
Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
@@ -532,7 +532,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 7 Data size: 400 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 7 Data size: 402 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: (locid > 0) (type: boolean)
Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
@@ -568,7 +568,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 7 Data size: 400 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 7 Data size: 402 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: (locid > 0) (type: boolean)
Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
Modified: hive/branches/branch-0.14/ql/src/test/results/clientpositive/annotate_stats_table.q.out
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/test/results/clientpositive/annotate_stats_table.q.out?rev=1637303&r1=1637302&r2=1637303&view=diff
==============================================================================
--- hive/branches/branch-0.14/ql/src/test/results/clientpositive/annotate_stats_table.q.out (original)
+++ hive/branches/branch-0.14/ql/src/test/results/clientpositive/annotate_stats_table.q.out Fri Nov 7 07:41:28 2014
@@ -89,11 +89,11 @@ STAGE PLANS:
Processor Tree:
TableScan
alias: emp_orc
- Statistics: Num rows: 3 Data size: 364 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3 Data size: 366 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: lastname (type: string), deptid (type: int)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 3 Data size: 364 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3 Data size: 366 Basic stats: COMPLETE Column stats: NONE
ListSink
PREHOOK: query: -- table level analyze statistics
@@ -122,11 +122,11 @@ STAGE PLANS:
Processor Tree:
TableScan
alias: emp_orc
- Statistics: Num rows: 48 Data size: 364 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 48 Data size: 366 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: lastname (type: string), deptid (type: int)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 48 Data size: 364 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 48 Data size: 366 Basic stats: COMPLETE Column stats: NONE
ListSink
PREHOOK: query: -- column level partial statistics
@@ -155,11 +155,11 @@ STAGE PLANS:
Processor Tree:
TableScan
alias: emp_orc
- Statistics: Num rows: 48 Data size: 364 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 48 Data size: 366 Basic stats: COMPLETE Column stats: PARTIAL
Select Operator
expressions: lastname (type: string), deptid (type: int)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 48 Data size: 364 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 48 Data size: 366 Basic stats: COMPLETE Column stats: PARTIAL
ListSink
PREHOOK: query: -- all selected columns have statistics
@@ -180,7 +180,7 @@ STAGE PLANS:
Processor Tree:
TableScan
alias: emp_orc
- Statistics: Num rows: 48 Data size: 364 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 48 Data size: 366 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: deptid (type: int)
outputColumnNames: _col0
@@ -213,11 +213,11 @@ STAGE PLANS:
Processor Tree:
TableScan
alias: emp_orc
- Statistics: Num rows: 48 Data size: 364 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 48 Data size: 366 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: lastname (type: string), deptid (type: int)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 48 Data size: 364 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 48 Data size: 366 Basic stats: COMPLETE Column stats: COMPLETE
ListSink
PREHOOK: query: -- basicStatState: COMPLETE colStatState: COMPLETE
@@ -236,7 +236,7 @@ STAGE PLANS:
Processor Tree:
TableScan
alias: emp_orc
- Statistics: Num rows: 48 Data size: 364 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 48 Data size: 366 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: lastname (type: string)
outputColumnNames: _col0
@@ -259,7 +259,7 @@ STAGE PLANS:
Processor Tree:
TableScan
alias: emp_orc
- Statistics: Num rows: 48 Data size: 364 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 48 Data size: 366 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: deptid (type: int)
outputColumnNames: _col0
@@ -282,7 +282,7 @@ STAGE PLANS:
Processor Tree:
TableScan
alias: emp_orc
- Statistics: Num rows: 48 Data size: 364 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 48 Data size: 366 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: lastname (type: string), deptid (type: int)
outputColumnNames: _col0, _col1
Modified: hive/branches/branch-0.14/ql/src/test/results/clientpositive/dynpart_sort_opt_vectorization.q.out
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/test/results/clientpositive/dynpart_sort_opt_vectorization.q.out?rev=1637303&r1=1637302&r2=1637303&view=diff
==============================================================================
--- hive/branches/branch-0.14/ql/src/test/results/clientpositive/dynpart_sort_opt_vectorization.q.out (original)
+++ hive/branches/branch-0.14/ql/src/test/results/clientpositive/dynpart_sort_opt_vectorization.q.out Fri Nov 7 07:41:28 2014
@@ -880,7 +880,7 @@ Partition Parameters:
numFiles 2
numRows 32
rawDataSize 640
- totalSize 1348
+ totalSize 1352
#### A masked pattern was here ####
# Storage Information
@@ -924,7 +924,7 @@ Partition Parameters:
numFiles 2
numRows 6
rawDataSize 120
- totalSize 1050
+ totalSize 1054
#### A masked pattern was here ####
# Storage Information
@@ -968,7 +968,7 @@ Partition Parameters:
numFiles 2
numRows 14
rawDataSize 280
- totalSize 1166
+ totalSize 1170
#### A masked pattern was here ####
# Storage Information
@@ -1012,7 +1012,7 @@ Partition Parameters:
numFiles 2
numRows 6
rawDataSize 120
- totalSize 1050
+ totalSize 1054
#### A masked pattern was here ####
# Storage Information
@@ -1055,7 +1055,7 @@ Partition Parameters:
numFiles 8
numRows 32
rawDataSize 640
- totalSize 4340
+ totalSize 4356
#### A masked pattern was here ####
# Storage Information
@@ -1098,7 +1098,7 @@ Partition Parameters:
numFiles 8
numRows 6
rawDataSize 120
- totalSize 2094
+ totalSize 2110
#### A masked pattern was here ####
# Storage Information
@@ -1141,7 +1141,7 @@ Partition Parameters:
numFiles 8
numRows 32
rawDataSize 640
- totalSize 4326
+ totalSize 4342
#### A masked pattern was here ####
# Storage Information
@@ -1184,7 +1184,7 @@ Partition Parameters:
numFiles 8
numRows 6
rawDataSize 120
- totalSize 2094
+ totalSize 2110
#### A masked pattern was here ####
# Storage Information
Modified: hive/branches/branch-0.14/ql/src/test/results/clientpositive/dynpart_sort_optimization2.q.out
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/test/results/clientpositive/dynpart_sort_optimization2.q.out?rev=1637303&r1=1637302&r2=1637303&view=diff
==============================================================================
--- hive/branches/branch-0.14/ql/src/test/results/clientpositive/dynpart_sort_optimization2.q.out (original)
+++ hive/branches/branch-0.14/ql/src/test/results/clientpositive/dynpart_sort_optimization2.q.out Fri Nov 7 07:41:28 2014
@@ -1183,7 +1183,7 @@ Partition Parameters:
numFiles 1
numRows 11
rawDataSize 88
- totalSize 417
+ totalSize 419
#### A masked pattern was here ####
# Storage Information
@@ -1245,7 +1245,7 @@ Partition Parameters:
numFiles 1
numRows 13
rawDataSize 104
- totalSize 440
+ totalSize 442
#### A masked pattern was here ####
# Storage Information
@@ -1404,7 +1404,7 @@ Partition Parameters:
numFiles 1
numRows 11
rawDataSize 88
- totalSize 417
+ totalSize 419
#### A masked pattern was here ####
# Storage Information
@@ -1466,7 +1466,7 @@ Partition Parameters:
numFiles 1
numRows 13
rawDataSize 104
- totalSize 440
+ totalSize 442
#### A masked pattern was here ####
# Storage Information
Modified: hive/branches/branch-0.14/ql/src/test/results/clientpositive/extrapolate_part_stats_full.q.out
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/test/results/clientpositive/extrapolate_part_stats_full.q.out?rev=1637303&r1=1637302&r2=1637303&view=diff
==============================================================================
Files hive/branches/branch-0.14/ql/src/test/results/clientpositive/extrapolate_part_stats_full.q.out (original) and hive/branches/branch-0.14/ql/src/test/results/clientpositive/extrapolate_part_stats_full.q.out Fri Nov 7 07:41:28 2014 differ
Modified: hive/branches/branch-0.14/ql/src/test/results/clientpositive/extrapolate_part_stats_partial.q.out
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/test/results/clientpositive/extrapolate_part_stats_partial.q.out?rev=1637303&r1=1637302&r2=1637303&view=diff
==============================================================================
Files hive/branches/branch-0.14/ql/src/test/results/clientpositive/extrapolate_part_stats_partial.q.out (original) and hive/branches/branch-0.14/ql/src/test/results/clientpositive/extrapolate_part_stats_partial.q.out Fri Nov 7 07:41:28 2014 differ
Modified: hive/branches/branch-0.14/ql/src/test/results/clientpositive/orc_analyze.q.out
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/test/results/clientpositive/orc_analyze.q.out?rev=1637303&r1=1637302&r2=1637303&view=diff
==============================================================================
--- hive/branches/branch-0.14/ql/src/test/results/clientpositive/orc_analyze.q.out (original)
+++ hive/branches/branch-0.14/ql/src/test/results/clientpositive/orc_analyze.q.out Fri Nov 7 07:41:28 2014
@@ -107,7 +107,7 @@ Table Parameters:
numFiles 1
numRows 100
rawDataSize 52600
- totalSize 3121
+ totalSize 3123
#### A masked pattern was here ####
# Storage Information
@@ -197,7 +197,7 @@ Table Parameters:
numFiles 1
numRows 100
rawDataSize 52600
- totalSize 3121
+ totalSize 3123
#### A masked pattern was here ####
# Storage Information
@@ -313,7 +313,7 @@ Partition Parameters:
numFiles 1
numRows 50
rawDataSize 21950
- totalSize 2024
+ totalSize 2026
#### A masked pattern was here ####
# Storage Information
@@ -358,7 +358,7 @@ Partition Parameters:
numFiles 1
numRows 50
rawDataSize 22050
- totalSize 2043
+ totalSize 2045
#### A masked pattern was here ####
# Storage Information
@@ -460,7 +460,7 @@ Partition Parameters:
numFiles 1
numRows 50
rawDataSize 21950
- totalSize 2024
+ totalSize 2026
#### A masked pattern was here ####
# Storage Information
@@ -505,7 +505,7 @@ Partition Parameters:
numFiles 1
numRows 50
rawDataSize 22050
- totalSize 2043
+ totalSize 2045
#### A masked pattern was here ####
# Storage Information
@@ -627,7 +627,7 @@ Partition Parameters:
numFiles 1
numRows 50
rawDataSize 21950
- totalSize 2024
+ totalSize 2026
#### A masked pattern was here ####
# Storage Information
@@ -672,7 +672,7 @@ Partition Parameters:
numFiles 1
numRows 50
rawDataSize 22050
- totalSize 2043
+ totalSize 2045
#### A masked pattern was here ####
# Storage Information
@@ -780,7 +780,7 @@ Partition Parameters:
numFiles 1
numRows 50
rawDataSize 21950
- totalSize 2024
+ totalSize 2026
#### A masked pattern was here ####
# Storage Information
@@ -825,7 +825,7 @@ Partition Parameters:
numFiles 1
numRows 50
rawDataSize 22050
- totalSize 2043
+ totalSize 2045
#### A masked pattern was here ####
# Storage Information
@@ -992,7 +992,7 @@ Partition Parameters:
numFiles 1
numRows 50
rawDataSize 21950
- totalSize 2024
+ totalSize 2026
#### A masked pattern was here ####
# Storage Information
Modified: hive/branches/branch-0.14/ql/src/test/results/clientpositive/tez/alter_merge_orc.q.out
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/test/results/clientpositive/tez/alter_merge_orc.q.out?rev=1637303&r1=1637302&r2=1637303&view=diff
==============================================================================
--- hive/branches/branch-0.14/ql/src/test/results/clientpositive/tez/alter_merge_orc.q.out (original)
+++ hive/branches/branch-0.14/ql/src/test/results/clientpositive/tez/alter_merge_orc.q.out Fri Nov 7 07:41:28 2014
@@ -48,9 +48,9 @@ columns:struct columns { i32 key, string
partitioned:false
partitionColumns:
totalNumberFiles:3
-totalFileSize:7488
-maxFileSize:2496
-minFileSize:2496
+totalFileSize:7494
+maxFileSize:2498
+minFileSize:2498
#### A masked pattern was here ####
PREHOOK: query: select count(1) from src_orc_merge_test
@@ -91,9 +91,9 @@ columns:struct columns { i32 key, string
partitioned:false
partitionColumns:
totalNumberFiles:1
-totalFileSize:7167
-maxFileSize:7167
-minFileSize:7167
+totalFileSize:7169
+maxFileSize:7169
+minFileSize:7169
#### A masked pattern was here ####
PREHOOK: query: select count(1) from src_orc_merge_test
@@ -171,9 +171,9 @@ columns:struct columns { i32 key, string
partitioned:true
partitionColumns:struct partition_columns { string ds}
totalNumberFiles:3
-totalFileSize:7488
-maxFileSize:2496
-minFileSize:2496
+totalFileSize:7494
+maxFileSize:2498
+minFileSize:2498
#### A masked pattern was here ####
PREHOOK: query: select count(1) from src_orc_merge_test_part
@@ -218,9 +218,9 @@ columns:struct columns { i32 key, string
partitioned:true
partitionColumns:struct partition_columns { string ds}
totalNumberFiles:1
-totalFileSize:7167
-maxFileSize:7167
-minFileSize:7167
+totalFileSize:7169
+maxFileSize:7169
+minFileSize:7169
#### A masked pattern was here ####
PREHOOK: query: select count(1) from src_orc_merge_test_part
Modified: hive/branches/branch-0.14/ql/src/test/results/clientpositive/tez/alter_merge_stats_orc.q.out
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/test/results/clientpositive/tez/alter_merge_stats_orc.q.out?rev=1637303&r1=1637302&r2=1637303&view=diff
==============================================================================
--- hive/branches/branch-0.14/ql/src/test/results/clientpositive/tez/alter_merge_stats_orc.q.out (original)
+++ hive/branches/branch-0.14/ql/src/test/results/clientpositive/tez/alter_merge_stats_orc.q.out Fri Nov 7 07:41:28 2014
@@ -48,9 +48,9 @@ columns:struct columns { i32 key, string
partitioned:false
partitionColumns:
totalNumberFiles:3
-totalFileSize:7488
-maxFileSize:2496
-minFileSize:2496
+totalFileSize:7494
+maxFileSize:2498
+minFileSize:2498
#### A masked pattern was here ####
PREHOOK: query: desc extended src_orc_merge_test_stat
@@ -94,7 +94,7 @@ Table Parameters:
numFiles 3
numRows 1500
rawDataSize 141000
- totalSize 7488
+ totalSize 7494
#### A masked pattern was here ####
# Storage Information
@@ -146,7 +146,7 @@ Table Parameters:
numFiles 1
numRows 1500
rawDataSize 141000
- totalSize 7167
+ totalSize 7169
#### A masked pattern was here ####
# Storage Information
@@ -216,9 +216,9 @@ columns:struct columns { i32 key, string
partitioned:true
partitionColumns:struct partition_columns { string ds}
totalNumberFiles:3
-totalFileSize:7488
-maxFileSize:2496
-minFileSize:2496
+totalFileSize:7494
+maxFileSize:2498
+minFileSize:2498
#### A masked pattern was here ####
PREHOOK: query: desc formatted src_orc_merge_test_part_stat partition (ds='2011')
@@ -249,7 +249,7 @@ Partition Parameters:
numFiles 3
numRows 1500
rawDataSize 141000
- totalSize 7488
+ totalSize 7494
#### A masked pattern was here ####
# Storage Information
@@ -300,7 +300,7 @@ Partition Parameters:
numFiles 3
numRows 1500
rawDataSize 141000
- totalSize 7488
+ totalSize 7494
#### A masked pattern was here ####
# Storage Information
@@ -359,7 +359,7 @@ Partition Parameters:
numFiles 1
numRows 1500
rawDataSize 141000
- totalSize 7167
+ totalSize 7169
#### A masked pattern was here ####
# Storage Information