You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by na...@apache.org on 2012/01/25 19:42:06 UTC
svn commit: r1235864 [1/2] - in /hive/trunk/ql/src:
java/org/apache/hadoop/hive/ql/index/
java/org/apache/hadoop/hive/ql/index/compact/
java/org/apache/hadoop/hive/ql/optimizer/ test/org/apache/hadoop/hive/ql/
test/queries/clientpositive/ test/results/...
Author: namit
Date: Wed Jan 25 18:42:06 2012
New Revision: 1235864
URL: http://svn.apache.org/viewvc?rev=1235864&view=rev
Log:
HIVE-2727 add a testcase for partitioned view on union and base tables have
index (He Yongqiang via namit)
Added:
hive/trunk/ql/src/test/queries/clientpositive/union_view.q
hive/trunk/ql/src/test/results/clientpositive/union_view.q.out
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/IndexMetadataChangeTask.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/compact/CompactIndexHandler.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/IndexUtils.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/IndexMetadataChangeTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/IndexMetadataChangeTask.java?rev=1235864&r1=1235863&r2=1235864&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/IndexMetadataChangeTask.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/IndexMetadataChangeTask.java Wed Jan 25 18:42:06 2012
@@ -67,14 +67,14 @@ public class IndexMetadataChangeTask ext
return 1;
}
- Path url = new Path(part.getDataLocation().toString());
+ Path url = new Path(part.getPartitionPath().toString());
FileSystem fs = url.getFileSystem(conf);
FileStatus fstat = fs.getFileStatus(url);
part.getParameters().put(HiveIndex.INDEX_TABLE_CREATETIME, Long.toString(fstat.getModificationTime()));
db.alterPartition(tbl.getTableName(), part);
} else {
- Path url = new Path(tbl.getDataLocation().toString());
+ Path url = new Path(tbl.getPath().toString());
FileSystem fs = url.getFileSystem(conf);
FileStatus fstat = fs.getFileStatus(url);
tbl.getParameters().put(HiveIndex.INDEX_TABLE_CREATETIME, Long.toString(fstat.getModificationTime()));
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/compact/CompactIndexHandler.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/compact/CompactIndexHandler.java?rev=1235864&r1=1235863&r2=1235864&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/compact/CompactIndexHandler.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/compact/CompactIndexHandler.java Wed Jan 25 18:42:06 2012
@@ -63,6 +63,8 @@ import org.apache.hadoop.hive.ql.udf.gen
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualOrLessThan;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPGreaterThan;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPLessThan;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNotNull;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNull;
public class CompactIndexHandler extends TableBasedIndexHandler {
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/IndexUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/IndexUtils.java?rev=1235864&r1=1235863&r2=1235864&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/IndexUtils.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/IndexUtils.java Wed Jan 25 18:42:06 2012
@@ -85,6 +85,10 @@ public final class IndexUtils {
queryPartitions = queryPartitionList.getUnknownPartns();
}
+ if(queryPartitions == null) {
+ return null;
+ }
+
for (Partition part : queryPartitions) {
List<Table> sourceIndexTables = getIndexTables(hive, part, indexes);
if (!containsPartition(hive, part, indexes)) {
Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java?rev=1235864&r1=1235863&r2=1235864&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java Wed Jan 25 18:42:06 2012
@@ -905,6 +905,7 @@ public class QTestUtil {
".*invalidscheme:.*",
".*lastUpdateTime.*",
".*lastAccessTime.*",
+ ".*lastModifiedTime.*",
".*[Oo]wner.*",
".*CreateTime.*",
".*LastAccessTime.*",
Added: hive/trunk/ql/src/test/queries/clientpositive/union_view.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/union_view.q?rev=1235864&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/union_view.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/union_view.q Wed Jan 25 18:42:06 2012
@@ -0,0 +1,84 @@
+CREATE TABLE src_union_1 (key int, value string) PARTITIONED BY (ds string);
+CREATE INDEX src_union_1_key_idx ON TABLE src_union_1(key) AS 'COMPACT' WITH DEFERRED REBUILD;
+
+CREATE TABLE src_union_2 (key int, value string) PARTITIONED BY (ds string, part_1 string);
+CREATE INDEX src_union_2_key_idx ON TABLE src_union_2(key) AS 'COMPACT' WITH DEFERRED REBUILD;
+
+CREATE TABLE src_union_3(key int, value string) PARTITIONED BY (ds string, part_1 string, part_2 string);
+CREATE INDEX src_union_3_key_idx ON TABLE src_union_3(key) AS 'COMPACT' WITH DEFERRED REBUILD;
+
+SET hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat;
+
+SET hive.optimize.index.filter=true;
+SET hive.optimize.index.filter.compact.minsize=0;
+
+SET hive.exec.pre.hooks=;
+SET hive.exec.post.hooks=;
+SET hive.semantic.analyzer.hook=;
+SET hive.merge.mapfiles=false;
+SET hive.merge.mapredfiles=false;
+
+INSERT OVERWRITE TABLE src_union_1 PARTITION (ds='1') SELECT * FROM src;
+ALTER INDEX src_union_1_key_idx ON src_union_1 PARTITION (ds='1') REBUILD;
+
+INSERT OVERWRITE TABLE src_union_2 PARTITION (ds='2', part_1='1') SELECT * FROM src;
+INSERT OVERWRITE TABLE src_union_2 PARTITION (ds='2', part_1='2') SELECT * FROM src;
+ALTER INDEX src_union_2_key_idx ON src_union_2 PARTITION (ds='2', part_1='1') REBUILD;
+ALTER INDEX src_union_2_key_idx ON src_union_2 PARTITION (ds='2', part_1='2') REBUILD;
+
+INSERT OVERWRITE TABLE src_union_3 PARTITION (ds='3', part_1='1', part_2='2:3+4') SELECT * FROM src;
+INSERT OVERWRITE TABLE src_union_3 PARTITION (ds='3', part_1='2', part_2='2:3+4') SELECT * FROM src;
+ALTER INDEX src_union_3_key_idx ON src_union_3 PARTITION (ds='3', part_1='1', part_2='2:3+4') REBUILD;
+ALTER INDEX src_union_3_key_idx ON src_union_3 PARTITION (ds='3', part_1='2', part_2='2:3+4') REBUILD;
+
+EXPLAIN SELECT key, value, ds FROM src_union_1 WHERE key=86 and ds='1';
+EXPLAIN SELECT key, value, ds FROM src_union_2 WHERE key=86 and ds='2';
+EXPLAIN SELECT key, value, ds FROM src_union_3 WHERE key=86 and ds='3';
+
+SELECT key, value, ds FROM src_union_1 WHERE key=86 AND ds ='1';
+SELECT key, value, ds FROM src_union_2 WHERE key=86 AND ds ='2';
+SELECT key, value, ds FROM src_union_3 WHERE key=86 AND ds ='3';
+
+EXPLAIN SELECT count(1) from src_union_1 WHERE ds ='1';
+EXPLAIN SELECT count(1) from src_union_2 WHERE ds ='2';
+EXPLAIN SELECT count(1) from src_union_3 WHERE ds ='3';
+
+SELECT count(1) from src_union_1 WHERE ds ='1';
+SELECT count(1) from src_union_2 WHERE ds ='2';
+SELECT count(1) from src_union_3 WHERE ds ='3';
+
+CREATE VIEW src_union_view PARTITIONED ON (ds) as
+SELECT key, value, ds FROM (
+SELECT key, value, ds FROM src_union_1
+UNION ALL
+SELECT key, value, ds FROM src_union_2
+UNION ALL
+SELECT key, value, ds FROM src_union_3
+) subq;
+
+EXPLAIN SELECT key, value, ds FROM src_union_view WHERE key=86 AND ds ='1';
+EXPLAIN SELECT key, value, ds FROM src_union_view WHERE key=86 AND ds ='2';
+EXPLAIN SELECT key, value, ds FROM src_union_view WHERE key=86 AND ds ='3';
+EXPLAIN SELECT key, value, ds FROM src_union_view WHERE key=86 AND ds IS NOT NULL order by ds;
+
+SELECT key, value, ds FROM src_union_view WHERE key=86 AND ds ='1';
+SELECT key, value, ds FROM src_union_view WHERE key=86 AND ds ='2';
+SELECT key, value, ds FROM src_union_view WHERE key=86 AND ds ='3';
+SELECT key, value, ds FROM src_union_view WHERE key=86 AND ds IS NOT NULL order by ds;
+
+EXPLAIN SELECT count(1) from src_union_view WHERE ds ='1';
+EXPLAIN SELECT count(1) from src_union_view WHERE ds ='2';
+EXPLAIN SELECT count(1) from src_union_view WHERE ds ='3';
+
+SELECT count(1) from src_union_view WHERE ds ='1';
+SELECT count(1) from src_union_view WHERE ds ='2';
+SELECT count(1) from src_union_view WHERE ds ='3';
+
+INSERT OVERWRITE TABLE src_union_3 PARTITION (ds='4', part_1='1', part_2='2:3+4') SELECT * FROM src;
+ALTER INDEX src_union_3_key_idx ON src_union_3 PARTITION (ds='4', part_1='1', part_2='2:3+4') REBUILD;
+
+EXPLAIN SELECT key, value, ds FROM src_union_view WHERE key=86 AND ds ='4';
+SELECT key, value, ds FROM src_union_view WHERE key=86 AND ds ='4';
+
+EXPLAIN SELECT count(1) from src_union_view WHERE ds ='4';
+SELECT count(1) from src_union_view WHERE ds ='4';