You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2012/04/12 20:46:48 UTC
svn commit: r1325442 - in /hive/trunk/ql/src:
java/org/apache/hadoop/hive/ql/io/ test/data/
test/org/apache/hadoop/hive/ql/io/ test/results/clientpositive/
Author: hashutosh
Date: Thu Apr 12 18:46:48 2012
New Revision: 1325442
URL: http://svn.apache.org/viewvc?rev=1325442&view=rev
Log:
HIVE-2711 : Make the header of RCFile unique (Owen Omalley via Ashutosh Chauhan)
Added:
hive/trunk/ql/src/test/data/
hive/trunk/ql/src/test/data/rc-file-v0.rc (with props)
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/RCFile.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/TestRCFile.java
hive/trunk/ql/src/test/results/clientpositive/alter_concatenate_indexed_table.q.out
hive/trunk/ql/src/test/results/clientpositive/alter_merge.q.out
hive/trunk/ql/src/test/results/clientpositive/alter_merge_stats.q.out
hive/trunk/ql/src/test/results/clientpositive/create_merge_compressed.q.out
hive/trunk/ql/src/test/results/clientpositive/ctas.q.out
hive/trunk/ql/src/test/results/clientpositive/partition_wise_fileformat.q.out
hive/trunk/ql/src/test/results/clientpositive/partition_wise_fileformat3.q.out
hive/trunk/ql/src/test/results/clientpositive/sample10.q.out
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/RCFile.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/RCFile.java?rev=1325442&r1=1325441&r2=1325442&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/RCFile.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/RCFile.java Thu Apr 12 18:46:48 2012
@@ -100,16 +100,12 @@ import org.apache.hadoop.util.Reflection
*
* <h5 id="Header">RC Header</h5>
* <ul>
- * <li>version - 3 bytes of magic header <b>SEQ</b>, followed by 1 byte of
- * actual version number (e.g. SEQ4 or SEQ6)</li>
- * <li>keyClassName -KeyBuffer's class name</li>
- * <li>valueClassName - ValueBuffer's class name</li>
+ * <li>version - 3 bytes of magic header <b>RCF</b>, followed by 1 byte of
+ * actual version number (e.g. RCF1)</li>
* <li>compression - A boolean which specifies if compression is turned on for
* keys/values in this file.</li>
- * <li>blockCompression - always false. this field is kept for compatible with
- * SequeceFile's format</li>
- * <li>compression codec - <code>CompressionCodec</code> class which is used for
- * compression of keys and/or values (if compression is enabled).</li>
+ * <li>compression codec - <code>CompressionCodec</code> class which is used
+ * for compression of keys and/or values (if compression is enabled).</li>
* <li>metadata - {@link Metadata} for this file.</li>
* <li>sync - A sync marker to denote end of the header.</li>
* </ul>
@@ -163,14 +159,22 @@ public class RCFile {
public static final String BLOCK_MISSING_MESSAGE =
"Could not obtain block";
- /*
- * these header and Sync are kept from SequenceFile, for compatible of
- * SequenceFile's format.
- */
- private static final byte VERSION_WITH_METADATA = (byte) 6;
- private static final byte[] VERSION = new byte[] {
- (byte) 'S', (byte) 'E', (byte) 'Q', VERSION_WITH_METADATA
- };
+ // All of the versions should be place in this list.
+ private static final int ORIGINAL_VERSION = 0; // version with SEQ
+ private static final int NEW_MAGIC_VERSION = 1; // version with RCF
+
+ private static final int CURRENT_VERSION = NEW_MAGIC_VERSION;
+
+ // The first version of RCFile used the sequence file header.
+ private static final byte[] ORIGINAL_MAGIC = new byte[] {
+ (byte) 'S', (byte) 'E', (byte) 'Q'};
+ // the version that was included with the original magic, which is mapped
+ // into ORIGINAL_VERSION
+ private static final byte ORIGINAL_MAGIC_VERSION = 6;
+
+ // The 'magic' bytes at the beginning of the RCFile
+ private static final byte[] MAGIC = new byte[] {
+ (byte) 'R', (byte) 'C', (byte) 'F'};
private static final int SYNC_ESCAPE = -1; // "length" of sync entries
private static final int SYNC_HASH_SIZE = 16; // number of bytes in hash
@@ -778,7 +782,8 @@ public class RCFile {
/** Write the initial part of file header. */
void initializeFileHeader() throws IOException {
- out.write(VERSION);
+ out.write(MAGIC);
+ out.write(CURRENT_VERSION);
}
/** Write the final part of file header. */
@@ -793,11 +798,7 @@ public class RCFile {
/** Write and flush the file header. */
void writeFileHeader() throws IOException {
- Text.writeString(out, KeyBuffer.class.getName());
- Text.writeString(out, ValueBuffer.class.getName());
-
out.writeBoolean(isCompressed());
- out.writeBoolean(false);
if (isCompressed()) {
Text.writeString(out, (codec.getClass()).getName());
@@ -1229,41 +1230,50 @@ public class RCFile {
}
private void init() throws IOException {
- byte[] versionBlock = new byte[VERSION.length];
- in.readFully(versionBlock);
+ byte[] magic = new byte[MAGIC.length];
+ in.readFully(magic);
- if ((versionBlock[0] != VERSION[0]) || (versionBlock[1] != VERSION[1])
- || (versionBlock[2] != VERSION[2])) {
- throw new IOException(file + " not a RCFile");
- }
+ if (Arrays.equals(magic, ORIGINAL_MAGIC)) {
+ byte vers = in.readByte();
+ if (vers != ORIGINAL_MAGIC_VERSION) {
+ throw new IOException(file + " is a version " + vers +
+ " SequenceFile instead of an RCFile.");
+ }
+ version = ORIGINAL_VERSION;
+ } else {
+ if (!Arrays.equals(magic, MAGIC)) {
+ throw new IOException(file + " not a RCFile and has magic of " +
+ new String(magic));
+ }
- // Set 'version'
- version = versionBlock[3];
- if (version > VERSION[3]) {
- throw new VersionMismatchException(VERSION[3], version);
+ // Set 'version'
+ version = in.readByte();
+ if (version > CURRENT_VERSION) {
+ throw new VersionMismatchException((byte) CURRENT_VERSION, version);
+ }
}
- try {
- Class<?> keyCls = conf.getClassByName(Text.readString(in));
- Class<?> valCls = conf.getClassByName(Text.readString(in));
- if (!keyCls.equals(KeyBuffer.class)
- || !valCls.equals(ValueBuffer.class)) {
- throw new IOException(file + " not a RCFile");
+ if (version == ORIGINAL_VERSION) {
+ try {
+ Class<?> keyCls = conf.getClassByName(Text.readString(in));
+ Class<?> valCls = conf.getClassByName(Text.readString(in));
+ if (!keyCls.equals(KeyBuffer.class)
+ || !valCls.equals(ValueBuffer.class)) {
+ throw new IOException(file + " not a RCFile");
+ }
+ } catch (ClassNotFoundException e) {
+ throw new IOException(file + " not a RCFile", e);
}
- } catch (ClassNotFoundException e) {
- throw new IOException(file + " not a RCFile", e);
}
- if (version > 2) { // if version > 2
- decompress = in.readBoolean(); // is compressed?
- } else {
- decompress = false;
- }
+ decompress = in.readBoolean(); // is compressed?
- // is block-compressed? it should be always false.
- boolean blkCompressed = in.readBoolean();
- if (blkCompressed) {
- throw new IOException(file + " not a RCFile.");
+ if (version == ORIGINAL_VERSION) {
+ // is block-compressed? it should be always false.
+ boolean blkCompressed = in.readBoolean();
+ if (blkCompressed) {
+ throw new IOException(file + " not a RCFile.");
+ }
}
// setup the compression codec
@@ -1282,14 +1292,10 @@ public class RCFile {
}
metadata = new Metadata();
- if (version >= VERSION_WITH_METADATA) { // if version >= 6
- metadata.readFields(in);
- }
+ metadata.readFields(in);
- if (version > 1) { // if version > 1
- in.readFully(sync); // read sync bytes
- headerEnd = in.getPos();
- }
+ in.readFully(sync); // read sync bytes
+ headerEnd = in.getPos();
}
/** Return the current byte position in the input file. */
@@ -1387,7 +1393,7 @@ public class RCFile {
return -1;
}
int length = in.readInt();
- if (version > 1 && sync != null && length == SYNC_ESCAPE) { // process
+ if (sync != null && length == SYNC_ESCAPE) { // process
// a
// sync entry
lastSeenSyncPos = in.getPos() - 4; // minus SYNC_ESCAPE's length
Added: hive/trunk/ql/src/test/data/rc-file-v0.rc
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/data/rc-file-v0.rc?rev=1325442&view=auto
==============================================================================
Binary file - no diff available.
Propchange: hive/trunk/ql/src/test/data/rc-file-v0.rc
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/TestRCFile.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/TestRCFile.java?rev=1325442&r1=1325441&r2=1325442&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/TestRCFile.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/TestRCFile.java Thu Apr 12 18:46:48 2012
@@ -271,6 +271,23 @@ public class TestRCFile extends TestCase
reader.close();
}
+ public void testReadOldFileHeader() throws IOException {
+ String[] row = new String[]{"Tester", "Bart", "333 X St.", "Reno", "NV",
+ "USA"};
+ RCFile.Reader reader =
+ new RCFile.Reader(fs, new Path("src/test/data/rc-file-v0.rc"), conf);
+ LongWritable rowID = new LongWritable();
+ BytesRefArrayWritable cols = new BytesRefArrayWritable();
+ assertTrue("old file reader first row", reader.next(rowID));
+ reader.getCurrentRow(cols);
+ assertEquals(row.length, cols.size());
+ for (int i=0; i < cols.size(); ++i) {
+ assertEquals(row[i], new String(cols.get(i).getBytesCopy()));
+ }
+ assertFalse("old file reader end", reader.next(rowID));
+ reader.close();
+ }
+
public void testWriteAndFullyRead() throws IOException, SerDeException {
writeTest(fs, 10000, file, bytesArray);
fullyReadTest(fs, 10000, file);
@@ -525,7 +542,8 @@ public class TestRCFile extends TestCase
System.out.println("The " + i + "th split read "
+ (readCount - previousReadCount));
}
- assertEquals("readCount should be equal to writeCount", readCount, writeCount);
+ assertEquals("readCount should be equal to writeCount", writeCount,
+ readCount);
}
Modified: hive/trunk/ql/src/test/results/clientpositive/alter_concatenate_indexed_table.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/alter_concatenate_indexed_table.q.out?rev=1325442&r1=1325441&r2=1325442&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/alter_concatenate_indexed_table.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/alter_concatenate_indexed_table.q.out Thu Apr 12 18:46:48 2012
@@ -86,9 +86,9 @@ columns:struct columns { i32 key, string
partitioned:false
partitionColumns:
totalNumberFiles:1
-totalFileSize:334
-maxFileSize:334
-minFileSize:334
+totalFileSize:239
+maxFileSize:239
+minFileSize:239
#### A masked pattern was here ####
PREHOOK: query: select count(1) from src_rc_concatenate_test
@@ -208,9 +208,9 @@ columns:struct columns { i32 key, string
partitioned:true
partitionColumns:struct partition_columns { string ds}
totalNumberFiles:1
-totalFileSize:334
-maxFileSize:334
-minFileSize:334
+totalFileSize:239
+maxFileSize:239
+minFileSize:239
#### A masked pattern was here ####
PREHOOK: query: select count(1) from src_rc_concatenate_test_part
Modified: hive/trunk/ql/src/test/results/clientpositive/alter_merge.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/alter_merge.q.out?rev=1325442&r1=1325441&r2=1325442&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/alter_merge.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/alter_merge.q.out Thu Apr 12 18:46:48 2012
@@ -76,9 +76,9 @@ columns:struct columns { i32 key, string
partitioned:false
partitionColumns:
totalNumberFiles:1
-totalFileSize:334
-maxFileSize:334
-minFileSize:334
+totalFileSize:239
+maxFileSize:239
+minFileSize:239
#### A masked pattern was here ####
PREHOOK: query: select count(1) from src_rc_merge_test
@@ -184,9 +184,9 @@ columns:struct columns { i32 key, string
partitioned:true
partitionColumns:struct partition_columns { string ds}
totalNumberFiles:1
-totalFileSize:334
-maxFileSize:334
-minFileSize:334
+totalFileSize:239
+maxFileSize:239
+minFileSize:239
#### A masked pattern was here ####
PREHOOK: query: select count(1) from src_rc_merge_test_part
Modified: hive/trunk/ql/src/test/results/clientpositive/alter_merge_stats.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/alter_merge_stats.q.out?rev=1325442&r1=1325441&r2=1325442&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/alter_merge_stats.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/alter_merge_stats.q.out Thu Apr 12 18:46:48 2012
@@ -82,9 +82,9 @@ columns:struct columns { i32 key, string
partitioned:false
partitionColumns:
totalNumberFiles:1
-totalFileSize:334
-maxFileSize:334
-minFileSize:334
+totalFileSize:239
+maxFileSize:239
+minFileSize:239
#### A masked pattern was here ####
PREHOOK: query: desc extended src_rc_merge_test_stat
@@ -190,9 +190,9 @@ columns:struct columns { i32 key, string
partitioned:true
partitionColumns:struct partition_columns { string ds}
totalNumberFiles:1
-totalFileSize:334
-maxFileSize:334
-minFileSize:334
+totalFileSize:239
+maxFileSize:239
+minFileSize:239
#### A masked pattern was here ####
PREHOOK: query: desc extended src_rc_merge_test_part_stat
Modified: hive/trunk/ql/src/test/results/clientpositive/create_merge_compressed.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/create_merge_compressed.q.out?rev=1325442&r1=1325441&r2=1325442&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/create_merge_compressed.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/create_merge_compressed.q.out Thu Apr 12 18:46:48 2012
@@ -52,9 +52,9 @@ columns:struct columns { i32 key, string
partitioned:false
partitionColumns:
totalNumberFiles:2
-totalFileSize:532
-maxFileSize:266
-minFileSize:266
+totalFileSize:342
+maxFileSize:171
+minFileSize:171
#### A masked pattern was here ####
PREHOOK: query: select count(1) from tgt_rc_merge_test
@@ -111,9 +111,9 @@ columns:struct columns { i32 key, string
partitioned:false
partitionColumns:
totalNumberFiles:1
-totalFileSize:338
-maxFileSize:338
-minFileSize:338
+totalFileSize:243
+maxFileSize:243
+minFileSize:243
#### A masked pattern was here ####
PREHOOK: query: select count(1) from tgt_rc_merge_test
Modified: hive/trunk/ql/src/test/results/clientpositive/ctas.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/ctas.q.out?rev=1325442&r1=1325441&r2=1325442&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/ctas.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/ctas.q.out Thu Apr 12 18:46:48 2012
@@ -479,7 +479,7 @@ Table Parameters:
numPartitions 0
numRows 10
rawDataSize 120
- totalSize 294
+ totalSize 199
#### A masked pattern was here ####
# Storage Information
@@ -546,7 +546,7 @@ Table Parameters:
numPartitions 0
numRows 10
rawDataSize 120
- totalSize 294
+ totalSize 199
#### A masked pattern was here ####
# Storage Information
Modified: hive/trunk/ql/src/test/results/clientpositive/partition_wise_fileformat.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/partition_wise_fileformat.q.out?rev=1325442&r1=1325441&r2=1325442&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/partition_wise_fileformat.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/partition_wise_fileformat.q.out Thu Apr 12 18:46:48 2012
@@ -159,8 +159,8 @@ columns:struct columns { string key, str
partitioned:true
partitionColumns:struct partition_columns { string dt}
totalNumberFiles:2
-totalFileSize:586
-maxFileSize:370
+totalFileSize:491
+maxFileSize:275
minFileSize:216
#### A masked pattern was here ####
@@ -201,9 +201,9 @@ columns:struct columns { string key, str
partitioned:true
partitionColumns:struct partition_columns { string dt}
totalNumberFiles:1
-totalFileSize:370
-maxFileSize:370
-minFileSize:370
+totalFileSize:275
+maxFileSize:275
+minFileSize:275
#### A masked pattern was here ####
PREHOOK: query: select key from partition_test_partitioned where dt=100
@@ -388,7 +388,7 @@ columns:struct columns { string key, str
partitioned:true
partitionColumns:struct partition_columns { string dt}
totalNumberFiles:3
-totalFileSize:1474
+totalFileSize:1379
maxFileSize:888
minFileSize:216
#### A masked pattern was here ####
@@ -434,9 +434,9 @@ columns:struct columns { string key, str
partitioned:true
partitionColumns:struct partition_columns { string dt}
totalNumberFiles:1
-totalFileSize:370
-maxFileSize:370
-minFileSize:370
+totalFileSize:275
+maxFileSize:275
+minFileSize:275
#### A masked pattern was here ####
PREHOOK: query: show table extended like partition_test_partitioned partition(dt=102)
Modified: hive/trunk/ql/src/test/results/clientpositive/partition_wise_fileformat3.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/partition_wise_fileformat3.q.out?rev=1325442&r1=1325441&r2=1325442&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/partition_wise_fileformat3.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/partition_wise_fileformat3.q.out Thu Apr 12 18:46:48 2012
@@ -35,9 +35,9 @@ columns:struct columns { string key, str
partitioned:true
partitionColumns:struct partition_columns { string dt}
totalNumberFiles:1
-totalFileSize:370
-maxFileSize:370
-minFileSize:370
+totalFileSize:275
+maxFileSize:275
+minFileSize:275
#### A masked pattern was here ####
PREHOOK: query: alter table partition_test_partitioned set fileformat Sequencefile
Modified: hive/trunk/ql/src/test/results/clientpositive/sample10.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/sample10.q.out?rev=1325442&r1=1325441&r2=1325442&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/sample10.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/sample10.q.out Thu Apr 12 18:46:48 2012
@@ -120,7 +120,7 @@ STAGE PLANS:
serialization.ddl struct srcpartbucket { string key, string value}
serialization.format 1
serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
- totalSize 687
+ totalSize 307
#### A masked pattern was here ####
serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
@@ -141,7 +141,7 @@ STAGE PLANS:
serialization.ddl struct srcpartbucket { string key, string value}
serialization.format 1
serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
- totalSize 2748
+ totalSize 1228
#### A masked pattern was here ####
serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
name: default.srcpartbucket
@@ -169,7 +169,7 @@ STAGE PLANS:
serialization.ddl struct srcpartbucket { string key, string value}
serialization.format 1
serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
- totalSize 687
+ totalSize 307
#### A masked pattern was here ####
serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
@@ -190,7 +190,7 @@ STAGE PLANS:
serialization.ddl struct srcpartbucket { string key, string value}
serialization.format 1
serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
- totalSize 2748
+ totalSize 1228
#### A masked pattern was here ####
serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
name: default.srcpartbucket
@@ -218,7 +218,7 @@ STAGE PLANS:
serialization.ddl struct srcpartbucket { string key, string value}
serialization.format 1
serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
- totalSize 687
+ totalSize 307
#### A masked pattern was here ####
serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
@@ -239,7 +239,7 @@ STAGE PLANS:
serialization.ddl struct srcpartbucket { string key, string value}
serialization.format 1
serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
- totalSize 2748
+ totalSize 1228
#### A masked pattern was here ####
serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
name: default.srcpartbucket
@@ -267,7 +267,7 @@ STAGE PLANS:
serialization.ddl struct srcpartbucket { string key, string value}
serialization.format 1
serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
- totalSize 687
+ totalSize 307
#### A masked pattern was here ####
serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
@@ -288,7 +288,7 @@ STAGE PLANS:
serialization.ddl struct srcpartbucket { string key, string value}
serialization.format 1
serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
- totalSize 2748
+ totalSize 1228
#### A masked pattern was here ####
serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
name: default.srcpartbucket