You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ek...@apache.org on 2015/10/01 03:23:25 UTC
hive git commit: HIVE-11073 ORC FileDump utility ignores errors when
writing output (backporting from 2.0.0)
Repository: hive
Updated Branches:
refs/heads/branch-1 f20a359d7 -> f953fc3cf
HIVE-11073 ORC FileDump utility ignores errors when writing output (backporting from 2.0.0)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f953fc3c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f953fc3c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f953fc3c
Branch: refs/heads/branch-1
Commit: f953fc3cf5625ed1002922aa69b7830e7e5f32d8
Parents: f20a359
Author: Eugene Koifman <ek...@hortonworks.com>
Authored: Wed Sep 30 18:23:17 2015 -0700
Committer: Eugene Koifman <ek...@hortonworks.com>
Committed: Wed Sep 30 18:23:17 2015 -0700
----------------------------------------------------------------------
.../apache/hadoop/hive/ql/io/orc/FileDump.java | 7 ++-
.../hadoop/hive/ql/io/orc/TestFileDump.java | 50 ++++++++++++++++++++
2 files changed, 56 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/f953fc3c/ql/src/java/org/apache/hadoop/hive/ql/io/orc/FileDump.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/FileDump.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/FileDump.java
index 7b883cc..76ecb33 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/FileDump.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/FileDump.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.io.orc;
import java.io.IOException;
import java.io.OutputStreamWriter;
+import java.io.PrintStream;
import java.text.DecimalFormat;
import java.util.ArrayList;
import java.util.Collection;
@@ -481,7 +482,8 @@ public final class FileDump {
String filename) throws IOException, JSONException {
Path path = new Path(filename);
Reader reader = OrcFile.createReader(path.getFileSystem(conf), path);
- OutputStreamWriter out = new OutputStreamWriter(System.out, "UTF-8");
+ PrintStream printStream = System.out;
+ OutputStreamWriter out = new OutputStreamWriter(printStream, "UTF-8");
RecordReader rows = reader.rows(null);
Object row = null;
List<OrcProto.Type> types = reader.getTypes();
@@ -491,6 +493,9 @@ public final class FileDump {
printObject(writer, row, types, 0);
out.write("\n");
out.flush();
+ if (printStream.checkError()) {
+ throw new IOException("Error encountered when writing to stdout.");
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/f953fc3c/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestFileDump.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestFileDump.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestFileDump.java
index 5b200f1..3d18d30 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestFileDump.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestFileDump.java
@@ -26,6 +26,8 @@ import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileReader;
+import java.io.IOException;
+import java.io.OutputStream;
import java.io.PrintStream;
import java.sql.Date;
import java.sql.Timestamp;
@@ -247,6 +249,54 @@ public class TestFileDump {
assertEquals("{\"b\":false,\"bt\":20,\"s\":200,\"i\":2000,\"l\":20000,\"f\":8,\"d\":40,\"de\":\"2.2222\",\"t\":\"2014-11-25 18:02:44\",\"dt\":\"2014-09-28\",\"str\":\"abcd\",\"c\":\"world \",\"vc\":\"world\",\"m\":[{\"_key\":\"k3\",\"_value\":\"v3\"}],\"a\":[200,300],\"st\":{\"i\":20,\"s\":\"bar\"}}", lines[1]);
}
+ @Test(expected = IOException.class)
+ public void testDataDumpThrowsIOException() throws Exception {
+ PrintStream origOut = System.out;
+ try {
+ ObjectInspector inspector;
+ synchronized (TestOrcFile.class) {
+ inspector = ObjectInspectorFactory.getReflectionObjectInspector
+ (AllTypesRecord.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
+ }
+ Writer writer = OrcFile.createWriter(fs, testFilePath, conf, inspector,
+ 100000, CompressionKind.NONE, 10000, 1000);
+ Map<String, String> m = new HashMap<String, String>(2);
+ m.put("k1", "v1");
+ writer.addRow(new AllTypesRecord(
+ true,
+ (byte) 10,
+ (short) 100,
+ 1000,
+ 10000L,
+ 4.0f,
+ 20.0,
+ HiveDecimal.create("4.2222"),
+ new Timestamp(1416967764000L),
+ new Date(1416967764000L),
+ "string",
+ new HiveChar("hello", 5),
+ new HiveVarchar("hello", 10),
+ m,
+ Arrays.asList(100, 200),
+ new AllTypesRecord.Struct(10, "foo")));
+
+ writer.close();
+
+ OutputStream myOut = new OutputStream() {
+ @Override
+ public void write(int b) throws IOException {
+ throw new IOException();
+ }
+ };
+
+ // replace stdout and run command
+ System.setOut(new PrintStream(myOut));
+ FileDump.main(new String[]{testFilePath.toString(), "-d"});
+ } finally {
+ System.setOut(origOut);
+ }
+ }
+
// Test that if the fraction of rows that have distinct strings is greater than the configured
// threshold dictionary encoding is turned off. If dictionary encoding is turned off the length
// of the dictionary stream for the column will be 0 in the ORC file dump.