You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jd...@apache.org on 2016/07/05 17:54:18 UTC
hive git commit: HIVE-14119: LLAP external recordreader not returning
non-ascii string properly (Jason Dere, reviewed by Sergey Shelukhin)
Repository: hive
Updated Branches:
refs/heads/master 59d00c7c1 -> 23fd2ae1c
HIVE-14119: LLAP external recordreader not returning non-ascii string properly (Jason Dere, reviewed by Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/23fd2ae1
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/23fd2ae1
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/23fd2ae1
Branch: refs/heads/master
Commit: 23fd2ae1c0bf27cedca043607decc263c9d3fe91
Parents: 59d00c7
Author: Jason Dere <jd...@hortonworks.com>
Authored: Tue Jul 5 10:54:29 2016 -0700
Committer: Jason Dere <jd...@hortonworks.com>
Committed: Tue Jul 5 10:54:29 2016 -0700
----------------------------------------------------------------------
.../apache/hive/jdbc/TestJdbcWithMiniLlap.java | 98 ++++++++++++--------
.../hadoop/hive/llap/LlapRowRecordReader.java | 1 +
2 files changed, 62 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/23fd2ae1/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniLlap.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniLlap.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniLlap.java
index 5d0da30..4897364 100644
--- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniLlap.java
+++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniLlap.java
@@ -19,6 +19,7 @@
package org.apache.hive.jdbc;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
@@ -158,9 +159,66 @@ public class TestJdbcWithMiniLlap {
stmt.close();
}
- private int getLlapIFRowCount(String query, int numSplits) throws Exception {
+ @Test(timeout = 60000)
+ public void testLlapInputFormatEndToEnd() throws Exception {
+ createTestTable("testtab1");
+
+ int rowCount;
+
+ RowCollector rowCollector = new RowCollector();
+ String query = "select * from testtab1 where under_col = 0";
+ rowCount = processQuery(query, 1, rowCollector);
+ assertEquals(3, rowCount);
+ assertArrayEquals(new String[] {"0", "val_0"}, rowCollector.rows.get(0));
+ assertArrayEquals(new String[] {"0", "val_0"}, rowCollector.rows.get(1));
+ assertArrayEquals(new String[] {"0", "val_0"}, rowCollector.rows.get(2));
+
+ // Try empty rows query
+ rowCollector.rows.clear();
+ query = "select * from testtab1 where true = false";
+ rowCount = processQuery(query, 1, rowCollector);
+ assertEquals(0, rowCount);
+ }
+
+ @Test(timeout = 60000)
+ public void testNonAsciiStrings() throws Exception {
+ createTestTable("testtab1");
+
+ RowCollector rowCollector = new RowCollector();
+ String nonAscii = "� c�t� du gar�on";
+ String query = "select value, '" + nonAscii + "' from testtab1 where under_col=0";
+ int rowCount = processQuery(query, 1, rowCollector);
+ assertEquals(3, rowCount);
- // Setup LlapInputFormat
+ assertArrayEquals(new String[] {"val_0", nonAscii}, rowCollector.rows.get(0));
+ assertArrayEquals(new String[] {"val_0", nonAscii}, rowCollector.rows.get(1));
+ assertArrayEquals(new String[] {"val_0", nonAscii}, rowCollector.rows.get(2));
+ }
+
+ private interface RowProcessor {
+ void process(Row row);
+ }
+
+ private static class RowCollector implements RowProcessor {
+ ArrayList<String[]> rows = new ArrayList<String[]>();
+ Schema schema = null;
+ int numColumns = 0;
+
+ public void process(Row row) {
+ if (schema == null) {
+ schema = row.getSchema();
+ numColumns = schema.getColumns().size();
+ }
+
+ String[] arr = new String[numColumns];
+ for (int idx = 0; idx < numColumns; ++idx) {
+ arr[idx] = row.getValue(idx).toString();
+ }
+ rows.add(arr);
+ }
+ }
+
+ private int processQuery(String query, int numSplits, RowProcessor rowProcessor) throws Exception {
String url = miniHS2.getJdbcURL();
String user = System.getProperty("user.name");
String pwd = user;
@@ -185,47 +243,13 @@ public class TestJdbcWithMiniLlap {
int numColumns = 2;
RecordReader<NullWritable, Row> reader = inputFormat.getRecordReader(split, job, null);
- if (reader instanceof LlapRowRecordReader && first) {
- Schema schema = ((LlapRowRecordReader) reader).getSchema();
- System.out.println(""+schema);
- assertEquals(numColumns, schema.getColumns().size());
- }
-
- if (first) {
- System.out.println("Results: ");
- System.out.println("");
- first = false;
- }
-
Row row = reader.createValue();
while (reader.next(NullWritable.get(), row)) {
- for (int idx = 0; idx < numColumns; idx++) {
- if (idx > 0) {
- System.out.print(", ");
- }
- System.out.print(row.getValue(idx));
- }
- System.out.println("");
+ rowProcessor.process(row);
++rowCount;
}
}
return rowCount;
}
-
- @Test(timeout = 60000)
- public void testLlapInputFormatEndToEnd() throws Exception {
- createTestTable("testtab1");
-
- int rowCount;
-
- String query = "select * from testtab1 where under_col = 0";
- rowCount = getLlapIFRowCount(query, 1);
- assertEquals(3, rowCount);
-
- // Try empty rows query
- query = "select * from testtab1 where true = false";
- rowCount = getLlapIFRowCount(query, 1);
- assertEquals(0, rowCount);
- }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/23fd2ae1/llap-client/src/java/org/apache/hadoop/hive/llap/LlapRowRecordReader.java
----------------------------------------------------------------------
diff --git a/llap-client/src/java/org/apache/hadoop/hive/llap/LlapRowRecordReader.java b/llap-client/src/java/org/apache/hadoop/hive/llap/LlapRowRecordReader.java
index 084da0a..10d7c94 100644
--- a/llap-client/src/java/org/apache/hadoop/hive/llap/LlapRowRecordReader.java
+++ b/llap-client/src/java/org/apache/hadoop/hive/llap/LlapRowRecordReader.java
@@ -165,6 +165,7 @@ public class LlapRowRecordReader implements RecordReader<NullWritable, Row> {
String types = typesBuffer.toString();
props.put(serdeConstants.LIST_COLUMNS, columns);
props.put(serdeConstants.LIST_COLUMN_TYPES, types);
+ props.put(serdeConstants.ESCAPE_CHAR, "\\");
SerDe serde = new LazySimpleSerDe();
serde.initialize(conf, props);