You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by om...@apache.org on 2015/11/30 20:15:25 UTC
[14/27] hive git commit: HIVE-12498: ACID: Setting
OrcRecordUpdater.OrcOptions.tableProperties() has no effect (Prasanth
Jayachandran reviewed by Eugene Koifman)
HIVE-12498: ACID: Setting OrcRecordUpdater.OrcOptions.tableProperties() has no effect (Prasanth Jayachandran reviewed by Eugene Koifman)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f679a5e1
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f679a5e1
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f679a5e1
Branch: refs/heads/master-fixed
Commit: f679a5e19da55e1ef90179ea06ae999582601588
Parents: 22b6203
Author: Prasanth Jayachandran <j....@gmail.com>
Authored: Wed Nov 25 12:10:02 2015 -0600
Committer: Owen O'Malley <om...@apache.org>
Committed: Mon Nov 30 11:14:36 2015 -0800
----------------------------------------------------------------------
.../hadoop/hive/ql/io/orc/OrcRecordUpdater.java | 3 +-
.../hive/ql/io/orc/TestOrcRecordUpdater.java | 58 ++++++++++++++++++--
2 files changed, 54 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/f679a5e1/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java
index 67c5a11..ee31c23 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java
@@ -247,7 +247,8 @@ public class OrcRecordUpdater implements RecordUpdater {
writerOptions = ((OrcOptions) options).getOrcOptions();
}
if (writerOptions == null) {
- writerOptions = OrcFile.writerOptions(options.getConfiguration());
+ writerOptions = OrcFile.writerOptions(options.getTableProperties(),
+ options.getConfiguration());
}
writerOptions.fileSystem(fs).callback(indexBuilder);
if (!options.isWritingBase()) {
http://git-wip-us.apache.org/repos/asf/hive/blob/f679a5e1/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRecordUpdater.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRecordUpdater.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRecordUpdater.java
index 22030b4..973cc40 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRecordUpdater.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRecordUpdater.java
@@ -18,6 +18,15 @@
package org.apache.hadoop.hive.ql.io.orc;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.File;
+import java.io.PrintStream;
+import java.util.Properties;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -33,12 +42,6 @@ import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.Reporter;
import org.junit.Test;
-import java.io.DataInputStream;
-import java.io.File;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-
public class TestOrcRecordUpdater {
@Test
@@ -180,6 +183,49 @@ public class TestOrcRecordUpdater {
}
@Test
+ public void testWriterTblProperties() throws Exception {
+ Path root = new Path(workDir, "testWriterTblProperties");
+ Configuration conf = new Configuration();
+ // Must use raw local because the checksummer doesn't honor flushes.
+ FileSystem fs = FileSystem.getLocal(conf).getRaw();
+ ObjectInspector inspector;
+ synchronized (TestOrcFile.class) {
+ inspector = ObjectInspectorFactory.getReflectionObjectInspector
+ (MyRow.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
+ }
+ Properties tblProps = new Properties();
+ tblProps.setProperty("orc.compress", "SNAPPY");
+ AcidOutputFormat.Options options = new AcidOutputFormat.Options(conf)
+ .filesystem(fs)
+ .bucket(10)
+ .writingBase(false)
+ .minimumTransactionId(10)
+ .maximumTransactionId(19)
+ .inspector(inspector)
+ .reporter(Reporter.NULL)
+ .finalDestination(root)
+ .tableProperties(tblProps);
+ RecordUpdater updater = new OrcRecordUpdater(root, options);
+ updater.insert(11, new MyRow("first"));
+ updater.insert(11, new MyRow("second"));
+ updater.insert(11, new MyRow("third"));
+ updater.flush();
+ updater.insert(12, new MyRow("fourth"));
+ updater.insert(12, new MyRow("fifth"));
+ updater.flush();
+
+ PrintStream origOut = System.out;
+ ByteArrayOutputStream myOut = new ByteArrayOutputStream();
+ System.setOut(new PrintStream(myOut));
+ FileDump.main(new String[]{root.toUri().toString()});
+ System.out.flush();
+ String outDump = new String(myOut.toByteArray());
+ assertEquals(true, outDump.contains("Compression: SNAPPY"));
+ System.setOut(origOut);
+ updater.close(false);
+ }
+
+ @Test
public void testUpdates() throws Exception {
Path root = new Path(workDir, "testUpdates");
Configuration conf = new Configuration();