You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2012/03/28 18:58:22 UTC
svn commit: r1306464 - in /hive/trunk/ql/src:
java/org/apache/hadoop/hive/ql/io/RCFile.java
test/org/apache/hadoop/hive/ql/io/TestRCFile.java
Author: hashutosh
Date: Wed Mar 28 16:58:22 2012
New Revision: 1306464
URL: http://svn.apache.org/viewvc?rev=1306464&view=rev
Log:
HIVE-2898: Add nicer helper functions for adding and reading metadata from RCFiles (Owen Omalley via Ashutosh Chauhan)
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/RCFile.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/TestRCFile.java
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/RCFile.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/RCFile.java?rev=1306464&r1=1306463&r2=1306464&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/RCFile.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/RCFile.java Wed Mar 28 16:58:22 2012
@@ -539,6 +539,23 @@ public class RCFile {
}
/**
+ * Create a metadata object with alternating key-value pairs.
+ * Eg. metadata(key1, value1, key2, value2)
+ */
+ public static Metadata createMetadata(Text... values) {
+ if (values.length % 2 != 0) {
+ throw new IllegalArgumentException("Must have a matched set of " +
+ "key-value pairs. " + values.length+
+ " strings supplied.");
+ }
+ Metadata result = new Metadata();
+ for(int i=0; i < values.length; i += 2) {
+ result.set(values[i], values[i+1]);
+ }
+ return result;
+ }
+
+ /**
* Write KeyBuffer/ValueBuffer pairs to a RCFile. RCFile's format is
* compatible with SequenceFile's.
*
@@ -1187,7 +1204,7 @@ public class RCFile {
}
/**
- * Return the metadata (string to string map) that was written into the
+ * Return the metadata (Text to Text map) that was written into the
* file.
*/
public Metadata getMetadata() {
@@ -1195,6 +1212,14 @@ public class RCFile {
}
/**
+ * Return the metadata value associated with the given key.
+ * @param key the metadata key to retrieve
+ */
+ public Text getMetadataValueOf(Text key) {
+ return metadata.get(key);
+ }
+
+ /**
* Override this method to specialize the type of
* {@link FSDataInputStream} returned.
*/
Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/TestRCFile.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/TestRCFile.java?rev=1306464&r1=1306463&r2=1306464&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/TestRCFile.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/TestRCFile.java Wed Mar 28 16:58:22 2012
@@ -50,7 +50,6 @@ import org.apache.hadoop.hive.serde2.obj
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.SequenceFile.Metadata;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.compress.CompressionCodec;
@@ -149,11 +148,14 @@ public class TestRCFile extends TestCase
"123".getBytes("UTF-8"), "1000".getBytes("UTF-8"),
"5.3".getBytes("UTF-8"), "hive and hadoop".getBytes("UTF-8"),
new byte[0], "NULL".getBytes("UTF-8")};
- Metadata metadata = new Metadata();
- metadata.set(new Text("apple"), new Text("block"));
RCFileOutputFormat.setColumnNumber(conf, expectedFieldsData.length);
- RCFile.Writer writer = new RCFile.Writer(fs, conf, file, null,
- metadata, new DefaultCodec());
+ RCFile.Writer writer =
+ new RCFile.Writer(fs, conf, file, null,
+ RCFile.createMetadata(new Text("apple"),
+ new Text("block"),
+ new Text("cat"),
+ new Text("dog")),
+ new DefaultCodec());
BytesRefArrayWritable bytes = new BytesRefArrayWritable(record_1.length);
for (int i = 0; i < record_1.length; i++) {
BytesRefWritable cu = new BytesRefWritable(record_1[i], 0,
@@ -183,6 +185,10 @@ public class TestRCFile extends TestCase
RCFile.Reader reader = new RCFile.Reader(fs, file, conf);
assertEquals(new Text("block"),
reader.getMetadata().get(new Text("apple")));
+ assertEquals(new Text("block"),
+ reader.getMetadataValueOf(new Text("apple")));
+ assertEquals(new Text("dog"),
+ reader.getMetadataValueOf(new Text("cat")));
LongWritable rowID = new LongWritable();
for (int i = 0; i < 2; i++) {