You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2013/05/09 06:38:39 UTC
svn commit: r1480526 [2/2] - in /hive/branches/vectorization: ./ data/files/
hcatalog/build-support/ant/
hcatalog/server-extensions/src/main/java/org/apache/hcatalog/listener/
hcatalog/src/test/ hcatalog/src/test/e2e/templeton/drivers/
hcatalog/src/tes...
Modified: hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java?rev=1480526&r1=1480525&r2=1480526&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java (original)
+++ hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java Thu May 9 04:38:38 2013
@@ -52,6 +52,7 @@ import org.junit.Test;
import org.junit.rules.TestName;
import java.io.File;
+import java.io.IOException;
import java.math.BigInteger;
import java.nio.ByteBuffer;
import java.sql.Timestamp;
@@ -639,7 +640,7 @@ public class TestOrcFile {
row.setFieldValue(0, null);
union.set((byte) 0, new IntWritable(1732050807));
row.setFieldValue(2, null);
- for(int i=0; i < 1000; ++i) {
+ for(int i=0; i < 5000; ++i) {
writer.addRow(row);
}
union.set((byte) 0, new IntWritable(0));
@@ -651,7 +652,7 @@ public class TestOrcFile {
writer.close();
Reader reader = OrcFile.createReader(fs, testFilePath);
assertEquals(false, reader.getMetadataKeys().iterator().hasNext());
- assertEquals(1309, reader.getNumberOfRows());
+ assertEquals(5309, reader.getNumberOfRows());
DecimalColumnStatistics stats =
(DecimalColumnStatistics) reader.getStatistics()[5];
assertEquals(303, stats.getNumberOfValues());
@@ -732,7 +733,7 @@ public class TestOrcFile {
assertEquals(new HiveDecimal(new BigInteger(118, rand),
rand.nextInt(36)), row.getFieldValue(2));
}
- for(int i=0; i < 1000; ++i) {
+ for(int i=0; i < 5000; ++i) {
row = (OrcStruct) rows.next(row);
assertEquals(new IntWritable(1732050807), union.getObject());
}
@@ -942,6 +943,8 @@ public class TestOrcFile {
double rate;
Path path = null;
long lastAllocation = 0;
+ int rows = 0;
+ MemoryManager.Callback callback;
MyMemoryManager(Configuration conf, long totalSpace, double rate) {
super(conf);
@@ -954,6 +957,7 @@ public class TestOrcFile {
MemoryManager.Callback callback) {
this.path = path;
this.lastAllocation = requestedAllocation;
+ this.callback = callback;
}
@Override
@@ -971,6 +975,13 @@ public class TestOrcFile {
double getAllocationScale() {
return rate;
}
+
+ @Override
+ void addedRow() throws IOException {
+ if (++rows % 100 == 0) {
+ callback.checkMemory(rate);
+ }
+ }
}
@Test
@@ -995,9 +1006,9 @@ public class TestOrcFile {
for(StripeInformation stripe: reader.getStripes()) {
i += 1;
assertTrue("stripe " + i + " is too long at " + stripe.getDataLength(),
- stripe.getDataLength() < 10000);
+ stripe.getDataLength() < 5000);
}
- assertEquals(3, i);
+ assertEquals(25, i);
assertEquals(2500, reader.getNumberOfRows());
}
}
Modified: hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestStringRedBlackTree.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestStringRedBlackTree.java?rev=1480526&r1=1480525&r2=1480526&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestStringRedBlackTree.java (original)
+++ hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestStringRedBlackTree.java Thu May 9 04:38:38 2013
@@ -104,7 +104,7 @@ public class TestStringRedBlackTree {
System.err.println(indent + "NULL");
} else {
System.err.println(indent + "Node " + node + " color " +
- (tree.isRed(node) ? "red" : "black") + " count " + tree.getCount(node));
+ (tree.isRed(node) ? "red" : "black"));
printTree(tree, indent + " ", tree.getLeft(node));
printTree(tree, indent + " ", tree.getRight(node));
}
@@ -112,14 +112,12 @@ public class TestStringRedBlackTree {
private static class MyVisitor implements StringRedBlackTree.Visitor {
private final String[] words;
- private final int[] counts;
private final int[] order;
private final DataOutputBuffer buffer = new DataOutputBuffer();
int current = 0;
- MyVisitor(String[] args, int[] counts, int[] order) {
+ MyVisitor(String[] args, int[] order) {
words = args;
- this.counts = counts;
this.order = order;
}
@@ -128,7 +126,6 @@ public class TestStringRedBlackTree {
) throws IOException {
String word = context.getText().toString();
assertEquals("in word " + current, words[current], word);
- assertEquals("in word " + current, counts[current], context.getCount());
assertEquals("in word " + current, order[current],
context.getOriginalPosition());
buffer.reset();
@@ -138,14 +135,14 @@ public class TestStringRedBlackTree {
}
}
- void checkContents(StringRedBlackTree tree, int[] counts, int[] order,
+ void checkContents(StringRedBlackTree tree, int[] order,
String... params
) throws IOException {
- tree.visit(new MyVisitor(params, counts, order));
+ tree.visit(new MyVisitor(params, order));
}
StringRedBlackTree buildTree(String... params) throws IOException {
- StringRedBlackTree result = new StringRedBlackTree();
+ StringRedBlackTree result = new StringRedBlackTree(1000);
for(String word: params) {
result.add(word);
checkTree(result);
@@ -156,7 +153,7 @@ public class TestStringRedBlackTree {
@Test
public void test1() throws Exception {
StringRedBlackTree tree = new StringRedBlackTree(5);
- assertEquals(0, tree.getByteSize());
+ assertEquals(0, tree.getSizeInBytes());
checkTree(tree);
assertEquals(0, tree.add("owen"));
checkTree(tree);
@@ -186,15 +183,12 @@ public class TestStringRedBlackTree {
checkTree(tree);
assertEquals(9, tree.add("z"));
checkTree(tree);
- checkContents(tree, new int[]{2,1,2,1,1,1,1,2,1,1},
- new int[]{2,5,1,4,6,3,7,0,9,8},
+ checkContents(tree, new int[]{2,5,1,4,6,3,7,0,9,8},
"alan", "arun", "ashutosh", "eric", "eric14", "greg",
"o", "owen", "z", "ziggy");
- assertEquals(10*5*4 + 8 + 6 + 5 + 5 * 4 + 2 * 1, tree.getByteSize());
+ assertEquals(32888, tree.getSizeInBytes());
// check that adding greg again bumps the count
- assertEquals(1, tree.getCount(3));
assertEquals(3, tree.add("greg"));
- assertEquals(2, tree.getCount(3));
assertEquals(41, tree.getCharacterSize());
// add some more strings to test the different branches of the
// rebalancing
@@ -210,7 +204,7 @@ public class TestStringRedBlackTree {
checkTree(tree);
tree.clear();
checkTree(tree);
- assertEquals(0, tree.getByteSize());
+ assertEquals(0, tree.getSizeInBytes());
assertEquals(0, tree.getCharacterSize());
}
@@ -220,8 +214,7 @@ public class TestStringRedBlackTree {
buildTree("a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l",
"m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z");
assertEquals(26, tree.size());
- checkContents(tree, new int[]{1,1,1, 1,1,1, 1,1,1, 1,1,1, 1,1,1, 1,1,1,
- 1,1,1, 1,1,1, 1,1}, new int[]{0,1,2, 3,4,5, 6,7,8, 9,10,11, 12,13,14,
+ checkContents(tree, new int[]{0,1,2, 3,4,5, 6,7,8, 9,10,11, 12,13,14,
15,16,17, 18,19,20, 21,22,23, 24,25},
"a", "b", "c", "d", "e", "f", "g", "h", "i", "j","k", "l", "m", "n", "o",
"p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z");
@@ -233,8 +226,7 @@ public class TestStringRedBlackTree {
buildTree("z", "y", "x", "w", "v", "u", "t", "s", "r", "q", "p", "o", "n",
"m", "l", "k", "j", "i", "h", "g", "f", "e", "d", "c", "b", "a");
assertEquals(26, tree.size());
- checkContents(tree, new int[]{1,1,1, 1,1,1, 1,1,1, 1,1,1, 1,1,1, 1,1,1,
- 1,1,1, 1,1,1, 1,1}, new int[]{25,24,23, 22,21,20, 19,18,17, 16,15,14,
+ checkContents(tree, new int[]{25,24,23, 22,21,20, 19,18,17, 16,15,14,
13,12,11, 10,9,8, 7,6,5, 4,3,2, 1,0},
"a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o",
"p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z");
Modified: hive/branches/vectorization/ql/src/test/queries/clientpositive/column_access_stats.q
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/queries/clientpositive/column_access_stats.q?rev=1480526&r1=1480525&r2=1480526&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/queries/clientpositive/column_access_stats.q (original)
+++ hive/branches/vectorization/ql/src/test/queries/clientpositive/column_access_stats.q Thu May 9 04:38:38 2013
@@ -8,11 +8,15 @@ LOAD DATA LOCAL INPATH '../data/files/T1
CREATE TABLE T2(key STRING, val STRING) STORED AS TEXTFILE;
CREATE TABLE T3(key STRING, val STRING) STORED AS TEXTFILE;
+CREATE TABLE T4(key STRING, val STRING) PARTITIONED BY (p STRING);
-- Simple select queries
SELECT key FROM T1 ORDER BY key;
SELECT key, val FROM T1 ORDER BY key, val;
SELECT 1 FROM T1;
+SELECT key, val from T4 where p=1;
+SELECT val FROM T4 where p=1;
+SELECT p, val FROM T4 where p=1;
-- More complicated select queries
EXPLAIN SELECT key FROM (SELECT key, val FROM T1) subq1 ORDER BY key;
Modified: hive/branches/vectorization/ql/src/test/queries/clientpositive/orc_create.q
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/queries/clientpositive/orc_create.q?rev=1480526&r1=1480525&r2=1480526&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/queries/clientpositive/orc_create.q (original)
+++ hive/branches/vectorization/ql/src/test/queries/clientpositive/orc_create.q Thu May 9 04:38:38 2013
@@ -1,4 +1,18 @@
DROP TABLE orc_create;
+DROP TABLE orc_create_complex;
+DROP TABLE orc_create_staging;
+
+CREATE TABLE orc_create_staging (
+ str STRING,
+ mp MAP<STRING,STRING>,
+ lst ARRAY<STRING>,
+ strct STRUCT<A:STRING,B:STRING>
+) ROW FORMAT DELIMITED
+ FIELDS TERMINATED BY '|'
+ COLLECTION ITEMS TERMINATED BY ','
+ MAP KEYS TERMINATED BY ':';
+
+DESCRIBE FORMATTED orc_create_staging;
CREATE TABLE orc_create (key INT, value STRING)
PARTITIONED BY (ds string)
@@ -26,4 +40,27 @@ CREATE TABLE orc_create (key INT, value
DESCRIBE FORMATTED orc_create;
+CREATE TABLE orc_create_complex (
+ str STRING,
+ mp MAP<STRING,STRING>,
+ lst ARRAY<STRING>,
+ strct STRUCT<A:STRING,B:STRING>
+) STORED AS ORC;
+
+DESCRIBE FORMATTED orc_create_complex;
+
+LOAD DATA LOCAL INPATH '../data/files/orc_create.txt' OVERWRITE INTO TABLE orc_create_staging;
+
+SELECT * from orc_create_staging;
+
+INSERT OVERWRITE TABLE orc_create_complex SELECT * FROM orc_create_staging;
+
+SELECT * from orc_create_complex;
+SELECT str from orc_create_complex;
+SELECT mp from orc_create_complex;
+SELECT lst from orc_create_complex;
+SELECT strct from orc_create_complex;
+
DROP TABLE orc_create;
+DROP TABLE orc_create_complex;
+DROP TABLE orc_create_staging;
Modified: hive/branches/vectorization/ql/src/test/resources/orc-file-dump.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/resources/orc-file-dump.out?rev=1480526&r1=1480525&r2=1480526&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/resources/orc-file-dump.out (original)
+++ hive/branches/vectorization/ql/src/test/resources/orc-file-dump.out Thu May 9 04:38:38 2013
@@ -11,74 +11,87 @@ Statistics:
Column 3: count: 21000 min: Darkness, max: worst
Stripes:
- Stripe: offset: 3 data: 83505 rows: 6000 tail: 91 index: 179
+ Stripe: offset: 3 data: 69638 rows: 5000 tail: 85 index: 126
Stream: column 0 section ROW_INDEX start: 3 length 10
Stream: column 1 section ROW_INDEX start: 13 length 38
Stream: column 2 section ROW_INDEX start: 51 length 42
- Stream: column 3 section DICTIONARY_COUNT start: 93 length 53
- Stream: column 3 section ROW_INDEX start: 146 length 36
- Stream: column 1 section PRESENT start: 182 length 11
- Stream: column 1 section DATA start: 193 length 27086
- Stream: column 2 section PRESENT start: 27279 length 11
- Stream: column 2 section DATA start: 27290 length 52124
- Stream: column 3 section PRESENT start: 79414 length 11
- Stream: column 3 section DATA start: 79425 length 4091
- Stream: column 3 section LENGTH start: 83516 length 38
- Stream: column 3 section DICTIONARY_DATA start: 83554 length 133
+ Stream: column 3 section ROW_INDEX start: 93 length 36
+ Stream: column 1 section PRESENT start: 129 length 11
+ Stream: column 1 section DATA start: 140 length 22605
+ Stream: column 2 section PRESENT start: 22745 length 11
+ Stream: column 2 section DATA start: 22756 length 43426
+ Stream: column 3 section PRESENT start: 66182 length 11
+ Stream: column 3 section DATA start: 66193 length 3403
+ Stream: column 3 section LENGTH start: 69596 length 38
+ Stream: column 3 section DICTIONARY_DATA start: 69634 length 133
Encoding column 0: DIRECT
Encoding column 1: DIRECT
Encoding column 2: DIRECT
Encoding column 3: DICTIONARY[35]
- Stripe: offset: 83778 data: 83453 rows: 6000 tail: 91 index: 180
- Stream: column 0 section ROW_INDEX start: 83778 length 10
- Stream: column 1 section ROW_INDEX start: 83788 length 39
- Stream: column 2 section ROW_INDEX start: 83827 length 42
- Stream: column 3 section DICTIONARY_COUNT start: 83869 length 53
- Stream: column 3 section ROW_INDEX start: 83922 length 36
- Stream: column 1 section PRESENT start: 83958 length 11
- Stream: column 1 section DATA start: 83969 length 27093
- Stream: column 2 section PRESENT start: 111062 length 11
- Stream: column 2 section DATA start: 111073 length 52119
- Stream: column 3 section PRESENT start: 163192 length 11
- Stream: column 3 section DATA start: 163203 length 4037
- Stream: column 3 section LENGTH start: 167240 length 38
- Stream: column 3 section DICTIONARY_DATA start: 167278 length 133
+ Stripe: offset: 69852 data: 69617 rows: 5000 tail: 83 index: 124
+ Stream: column 0 section ROW_INDEX start: 69852 length 10
+ Stream: column 1 section ROW_INDEX start: 69862 length 36
+ Stream: column 2 section ROW_INDEX start: 69898 length 42
+ Stream: column 3 section ROW_INDEX start: 69940 length 36
+ Stream: column 1 section PRESENT start: 69976 length 11
+ Stream: column 1 section DATA start: 69987 length 22597
+ Stream: column 2 section PRESENT start: 92584 length 11
+ Stream: column 2 section DATA start: 92595 length 43439
+ Stream: column 3 section PRESENT start: 136034 length 11
+ Stream: column 3 section DATA start: 136045 length 3377
+ Stream: column 3 section LENGTH start: 139422 length 38
+ Stream: column 3 section DICTIONARY_DATA start: 139460 length 133
Encoding column 0: DIRECT
Encoding column 1: DIRECT
Encoding column 2: DIRECT
Encoding column 3: DICTIONARY[35]
- Stripe: offset: 167502 data: 83456 rows: 6000 tail: 92 index: 182
- Stream: column 0 section ROW_INDEX start: 167502 length 10
- Stream: column 1 section ROW_INDEX start: 167512 length 39
- Stream: column 2 section ROW_INDEX start: 167551 length 42
- Stream: column 3 section DICTIONARY_COUNT start: 167593 length 55
- Stream: column 3 section ROW_INDEX start: 167648 length 36
- Stream: column 1 section PRESENT start: 167684 length 11
- Stream: column 1 section DATA start: 167695 length 27080
- Stream: column 2 section PRESENT start: 194775 length 11
- Stream: column 2 section DATA start: 194786 length 52093
- Stream: column 3 section PRESENT start: 246879 length 11
- Stream: column 3 section DATA start: 246890 length 4079
- Stream: column 3 section LENGTH start: 250969 length 38
- Stream: column 3 section DICTIONARY_DATA start: 251007 length 133
+ Stripe: offset: 139676 data: 69603 rows: 5000 tail: 85 index: 127
+ Stream: column 0 section ROW_INDEX start: 139676 length 10
+ Stream: column 1 section ROW_INDEX start: 139686 length 39
+ Stream: column 2 section ROW_INDEX start: 139725 length 42
+ Stream: column 3 section ROW_INDEX start: 139767 length 36
+ Stream: column 1 section PRESENT start: 139803 length 11
+ Stream: column 1 section DATA start: 139814 length 22594
+ Stream: column 2 section PRESENT start: 162408 length 11
+ Stream: column 2 section DATA start: 162419 length 43415
+ Stream: column 3 section PRESENT start: 205834 length 11
+ Stream: column 3 section DATA start: 205845 length 3390
+ Stream: column 3 section LENGTH start: 209235 length 38
+ Stream: column 3 section DICTIONARY_DATA start: 209273 length 133
Encoding column 0: DIRECT
Encoding column 1: DIRECT
Encoding column 2: DIRECT
Encoding column 3: DICTIONARY[35]
- Stripe: offset: 251232 data: 41842 rows: 3000 tail: 90 index: 172
- Stream: column 0 section ROW_INDEX start: 251232 length 10
- Stream: column 1 section ROW_INDEX start: 251242 length 39
- Stream: column 2 section ROW_INDEX start: 251281 length 43
- Stream: column 3 section DICTIONARY_COUNT start: 251324 length 44
- Stream: column 3 section ROW_INDEX start: 251368 length 36
- Stream: column 1 section PRESENT start: 251404 length 9
- Stream: column 1 section DATA start: 251413 length 13544
- Stream: column 2 section PRESENT start: 264957 length 9
- Stream: column 2 section DATA start: 264966 length 26072
- Stream: column 3 section PRESENT start: 291038 length 9
- Stream: column 3 section DATA start: 291047 length 2028
- Stream: column 3 section LENGTH start: 293075 length 38
- Stream: column 3 section DICTIONARY_DATA start: 293113 length 133
+ Stripe: offset: 209491 data: 69584 rows: 5000 tail: 84 index: 126
+ Stream: column 0 section ROW_INDEX start: 209491 length 10
+ Stream: column 1 section ROW_INDEX start: 209501 length 38
+ Stream: column 2 section ROW_INDEX start: 209539 length 42
+ Stream: column 3 section ROW_INDEX start: 209581 length 36
+ Stream: column 1 section PRESENT start: 209617 length 11
+ Stream: column 1 section DATA start: 209628 length 22575
+ Stream: column 2 section PRESENT start: 232203 length 11
+ Stream: column 2 section DATA start: 232214 length 43426
+ Stream: column 3 section PRESENT start: 275640 length 11
+ Stream: column 3 section DATA start: 275651 length 3379
+ Stream: column 3 section LENGTH start: 279030 length 38
+ Stream: column 3 section DICTIONARY_DATA start: 279068 length 133
+ Encoding column 0: DIRECT
+ Encoding column 1: DIRECT
+ Encoding column 2: DIRECT
+ Encoding column 3: DICTIONARY[35]
+ Stripe: offset: 279285 data: 14111 rows: 1000 tail: 80 index: 127
+ Stream: column 0 section ROW_INDEX start: 279285 length 10
+ Stream: column 1 section ROW_INDEX start: 279295 length 39
+ Stream: column 2 section ROW_INDEX start: 279334 length 42
+ Stream: column 3 section ROW_INDEX start: 279376 length 36
+ Stream: column 1 section PRESENT start: 279412 length 5
+ Stream: column 1 section DATA start: 279417 length 4529
+ Stream: column 2 section PRESENT start: 283946 length 5
+ Stream: column 2 section DATA start: 283951 length 8690
+ Stream: column 3 section PRESENT start: 292641 length 5
+ Stream: column 3 section DATA start: 292646 length 706
+ Stream: column 3 section LENGTH start: 293352 length 38
+ Stream: column 3 section DICTIONARY_DATA start: 293390 length 133
Encoding column 0: DIRECT
Encoding column 1: DIRECT
Encoding column 2: DIRECT
Modified: hive/branches/vectorization/ql/src/test/results/clientpositive/column_access_stats.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientpositive/column_access_stats.q.out?rev=1480526&r1=1480525&r2=1480526&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientpositive/column_access_stats.q.out (original)
+++ hive/branches/vectorization/ql/src/test/results/clientpositive/column_access_stats.q.out Thu May 9 04:38:38 2013
@@ -9,6 +9,8 @@ PREHOOK: query: CREATE TABLE T2(key STRI
PREHOOK: type: CREATETABLE
PREHOOK: query: CREATE TABLE T3(key STRING, val STRING) STORED AS TEXTFILE
PREHOOK: type: CREATETABLE
+PREHOOK: query: CREATE TABLE T4(key STRING, val STRING) PARTITIONED BY (p STRING)
+PREHOOK: type: CREATETABLE
PREHOOK: query: -- Simple select queries
SELECT key FROM T1 ORDER BY key
PREHOOK: type: QUERY
@@ -46,6 +48,27 @@ PREHOOK: Input: default@t1
1
1
1
+PREHOOK: query: SELECT key, val from T4 where p=1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t4
+#### A masked pattern was here ####
+Table:default@t4
+Columns:key,val
+
+PREHOOK: query: SELECT val FROM T4 where p=1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t4
+#### A masked pattern was here ####
+Table:default@t4
+Columns:val
+
+PREHOOK: query: SELECT p, val FROM T4 where p=1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t4
+#### A masked pattern was here ####
+Table:default@t4
+Columns:val
+
PREHOOK: query: -- More complicated select queries
EXPLAIN SELECT key FROM (SELECT key, val FROM T1) subq1 ORDER BY key
PREHOOK: type: QUERY
Modified: hive/branches/vectorization/ql/src/test/results/clientpositive/orc_create.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientpositive/orc_create.q.out?rev=1480526&r1=1480525&r2=1480526&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientpositive/orc_create.q.out (original)
+++ hive/branches/vectorization/ql/src/test/results/clientpositive/orc_create.q.out Thu May 9 04:38:38 2013
@@ -2,6 +2,69 @@ PREHOOK: query: DROP TABLE orc_create
PREHOOK: type: DROPTABLE
POSTHOOK: query: DROP TABLE orc_create
POSTHOOK: type: DROPTABLE
+PREHOOK: query: DROP TABLE orc_create_complex
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE orc_create_complex
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: DROP TABLE orc_create_staging
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE orc_create_staging
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE orc_create_staging (
+ str STRING,
+ mp MAP<STRING,STRING>,
+ lst ARRAY<STRING>,
+ strct STRUCT<A:STRING,B:STRING>
+) ROW FORMAT DELIMITED
+ FIELDS TERMINATED BY '|'
+ COLLECTION ITEMS TERMINATED BY ','
+ MAP KEYS TERMINATED BY ':'
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE TABLE orc_create_staging (
+ str STRING,
+ mp MAP<STRING,STRING>,
+ lst ARRAY<STRING>,
+ strct STRUCT<A:STRING,B:STRING>
+) ROW FORMAT DELIMITED
+ FIELDS TERMINATED BY '|'
+ COLLECTION ITEMS TERMINATED BY ','
+ MAP KEYS TERMINATED BY ':'
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@orc_create_staging
+PREHOOK: query: DESCRIBE FORMATTED orc_create_staging
+PREHOOK: type: DESCTABLE
+POSTHOOK: query: DESCRIBE FORMATTED orc_create_staging
+POSTHOOK: type: DESCTABLE
+# col_name data_type comment
+
+str string None
+mp map<string,string> None
+lst array<string> None
+strct struct<A:string,B:string> None
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Protect Mode: None
+Retention: 0
+#### A masked pattern was here ####
+Table Type: MANAGED_TABLE
+Table Parameters:
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+InputFormat: org.apache.hadoop.mapred.TextInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+Storage Desc Params:
+ colelction.delim ,
+ field.delim |
+ mapkey.delim :
+ serialization.format |
PREHOOK: query: CREATE TABLE orc_create (key INT, value STRING)
PARTITIONED BY (ds string)
STORED AS ORC
@@ -185,6 +248,156 @@ Bucket Columns: []
Sort Columns: []
Storage Desc Params:
serialization.format 1
+PREHOOK: query: CREATE TABLE orc_create_complex (
+ str STRING,
+ mp MAP<STRING,STRING>,
+ lst ARRAY<STRING>,
+ strct STRUCT<A:STRING,B:STRING>
+) STORED AS ORC
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE TABLE orc_create_complex (
+ str STRING,
+ mp MAP<STRING,STRING>,
+ lst ARRAY<STRING>,
+ strct STRUCT<A:STRING,B:STRING>
+) STORED AS ORC
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@orc_create_complex
+PREHOOK: query: DESCRIBE FORMATTED orc_create_complex
+PREHOOK: type: DESCTABLE
+POSTHOOK: query: DESCRIBE FORMATTED orc_create_complex
+POSTHOOK: type: DESCTABLE
+# col_name data_type comment
+
+str string from deserializer
+mp map<string,string> from deserializer
+lst array<string> from deserializer
+strct struct<A:string,B:string> from deserializer
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Protect Mode: None
+Retention: 0
+#### A masked pattern was here ####
+Table Type: MANAGED_TABLE
+Table Parameters:
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+Storage Desc Params:
+ serialization.format 1
+PREHOOK: query: LOAD DATA LOCAL INPATH '../data/files/orc_create.txt' OVERWRITE INTO TABLE orc_create_staging
+PREHOOK: type: LOAD
+PREHOOK: Output: default@orc_create_staging
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../data/files/orc_create.txt' OVERWRITE INTO TABLE orc_create_staging
+POSTHOOK: type: LOAD
+POSTHOOK: Output: default@orc_create_staging
+PREHOOK: query: SELECT * from orc_create_staging
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orc_create_staging
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * from orc_create_staging
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orc_create_staging
+#### A masked pattern was here ####
+line1 {"key11":"value11","key12":"value12","key13":"value13"} ["a","b","c"] {"a":"one","b":"two"}
+line2 {"key21":"value21","key22":"value22","key23":"value23"} ["d","e","f"] {"a":"three","b":"four"}
+line3 {"key31":"value31","key32":"value32","key33":"value33"} ["g","h","i"] {"a":"five","b":"six"}
+PREHOOK: query: INSERT OVERWRITE TABLE orc_create_complex SELECT * FROM orc_create_staging
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orc_create_staging
+PREHOOK: Output: default@orc_create_complex
+POSTHOOK: query: INSERT OVERWRITE TABLE orc_create_complex SELECT * FROM orc_create_staging
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orc_create_staging
+POSTHOOK: Output: default@orc_create_complex
+POSTHOOK: Lineage: orc_create_complex.lst SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:lst, type:array<string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.mp SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:mp, type:map<string,string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.str SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:str, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.strct SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:strct, type:struct<A:string,B:string>, comment:null), ]
+PREHOOK: query: SELECT * from orc_create_complex
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orc_create_complex
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * from orc_create_complex
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orc_create_complex
+#### A masked pattern was here ####
+POSTHOOK: Lineage: orc_create_complex.lst SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:lst, type:array<string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.mp SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:mp, type:map<string,string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.str SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:str, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.strct SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:strct, type:struct<A:string,B:string>, comment:null), ]
+line1 {"key12":"value12","key11":"value11","key13":"value13"} ["a","b","c"] {"A":"one","B":"two"}
+line2 {"key21":"value21","key23":"value23","key22":"value22"} ["d","e","f"] {"A":"three","B":"four"}
+line3 {"key33":"value33","key31":"value31","key32":"value32"} ["g","h","i"] {"A":"five","B":"six"}
+PREHOOK: query: SELECT str from orc_create_complex
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orc_create_complex
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT str from orc_create_complex
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orc_create_complex
+#### A masked pattern was here ####
+POSTHOOK: Lineage: orc_create_complex.lst SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:lst, type:array<string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.mp SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:mp, type:map<string,string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.str SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:str, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.strct SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:strct, type:struct<A:string,B:string>, comment:null), ]
+line1
+line2
+line3
+PREHOOK: query: SELECT mp from orc_create_complex
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orc_create_complex
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT mp from orc_create_complex
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orc_create_complex
+#### A masked pattern was here ####
+POSTHOOK: Lineage: orc_create_complex.lst SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:lst, type:array<string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.mp SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:mp, type:map<string,string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.str SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:str, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.strct SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:strct, type:struct<A:string,B:string>, comment:null), ]
+{"key12":"value12","key11":"value11","key13":"value13"}
+{"key21":"value21","key23":"value23","key22":"value22"}
+{"key33":"value33","key31":"value31","key32":"value32"}
+PREHOOK: query: SELECT lst from orc_create_complex
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orc_create_complex
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT lst from orc_create_complex
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orc_create_complex
+#### A masked pattern was here ####
+POSTHOOK: Lineage: orc_create_complex.lst SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:lst, type:array<string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.mp SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:mp, type:map<string,string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.str SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:str, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.strct SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:strct, type:struct<A:string,B:string>, comment:null), ]
+["a","b","c"]
+["d","e","f"]
+["g","h","i"]
+PREHOOK: query: SELECT strct from orc_create_complex
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orc_create_complex
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT strct from orc_create_complex
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orc_create_complex
+#### A masked pattern was here ####
+POSTHOOK: Lineage: orc_create_complex.lst SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:lst, type:array<string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.mp SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:mp, type:map<string,string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.str SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:str, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.strct SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:strct, type:struct<A:string,B:string>, comment:null), ]
+{"a":"one","b":"two"}
+{"a":"three","b":"four"}
+{"a":"five","b":"six"}
PREHOOK: query: DROP TABLE orc_create
PREHOOK: type: DROPTABLE
PREHOOK: Input: default@orc_create
@@ -193,3 +406,31 @@ POSTHOOK: query: DROP TABLE orc_create
POSTHOOK: type: DROPTABLE
POSTHOOK: Input: default@orc_create
POSTHOOK: Output: default@orc_create
+POSTHOOK: Lineage: orc_create_complex.lst SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:lst, type:array<string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.mp SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:mp, type:map<string,string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.str SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:str, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.strct SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:strct, type:struct<A:string,B:string>, comment:null), ]
+PREHOOK: query: DROP TABLE orc_create_complex
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@orc_create_complex
+PREHOOK: Output: default@orc_create_complex
+POSTHOOK: query: DROP TABLE orc_create_complex
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@orc_create_complex
+POSTHOOK: Output: default@orc_create_complex
+POSTHOOK: Lineage: orc_create_complex.lst SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:lst, type:array<string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.mp SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:mp, type:map<string,string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.str SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:str, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.strct SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:strct, type:struct<A:string,B:string>, comment:null), ]
+PREHOOK: query: DROP TABLE orc_create_staging
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@orc_create_staging
+PREHOOK: Output: default@orc_create_staging
+POSTHOOK: query: DROP TABLE orc_create_staging
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@orc_create_staging
+POSTHOOK: Output: default@orc_create_staging
+POSTHOOK: Lineage: orc_create_complex.lst SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:lst, type:array<string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.mp SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:mp, type:map<string,string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.str SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:str, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.strct SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:strct, type:struct<A:string,B:string>, comment:null), ]
Modified: hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/operation/HiveCommandOperation.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/operation/HiveCommandOperation.java?rev=1480526&r1=1480525&r2=1480526&view=diff
==============================================================================
--- hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/operation/HiveCommandOperation.java (original)
+++ hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/operation/HiveCommandOperation.java Thu May 9 04:38:38 2013
@@ -34,6 +34,7 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.ql.processors.CommandProcessor;
import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.io.IOUtils;
import org.apache.hive.service.cli.FetchOrientation;
import org.apache.hive.service.cli.HiveSQLException;
import org.apache.hive.service.cli.OperationState;
@@ -85,6 +86,12 @@ public abstract class HiveCommandOperati
}
}
+
+ private void tearDownSessionIO() {
+ IOUtils.cleanup(LOG, parentSession.getSessionState().out);
+ IOUtils.cleanup(LOG, parentSession.getSessionState().err);
+ }
+
/* (non-Javadoc)
* @see org.apache.hive.service.cli.operation.Operation#run()
*/
@@ -121,6 +128,7 @@ public abstract class HiveCommandOperati
@Override
public void close() throws HiveSQLException {
setState(OperationState.CLOSED);
+ tearDownSessionIO();
cleanTmpFile();
}
@@ -187,6 +195,7 @@ public abstract class HiveCommandOperati
if (resultReader != null) {
SessionState sessionState = getParentSession().getSessionState();
File tmp = sessionState.getTmpOutputFile();
+ IOUtils.cleanup(LOG, resultReader);
tmp.delete();
resultReader = null;
}
Modified: hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/operation/Operation.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/operation/Operation.java?rev=1480526&r1=1480525&r2=1480526&view=diff
==============================================================================
--- hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/operation/Operation.java (original)
+++ hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/operation/Operation.java Thu May 9 04:38:38 2013
@@ -32,7 +32,7 @@ import org.apache.hive.service.cli.sessi
public abstract class Operation {
- private final HiveSession parentSession;
+ protected final HiveSession parentSession;
private OperationState state = OperationState.INITIALIZED;
private final OperationHandle opHandle;
private HiveConf configuration;
Modified: hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java?rev=1480526&r1=1480525&r2=1480526&view=diff
==============================================================================
--- hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java (original)
+++ hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java Thu May 9 04:38:38 2013
@@ -28,6 +28,7 @@ import org.apache.hadoop.hive.conf.HiveC
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.ql.history.HiveHistory;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hive.common.util.HiveVersionInfo;
import org.apache.hive.service.cli.FetchOrientation;
@@ -295,6 +296,10 @@ public class HiveSessionImpl implements
operationManager.closeOperation(opHandle);
}
opHandleSet.clear();
+ HiveHistory hiveHist = sessionState.getHiveHistory();
+ if (null != hiveHist) {
+ hiveHist.closeStream();
+ }
} finally {
release();
}
@@ -367,4 +372,4 @@ public class HiveSessionImpl implements
protected HiveSession getSession() {
return this;
}
-}
\ No newline at end of file
+}