You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by pr...@apache.org on 2014/04/29 03:10:35 UTC
svn commit: r1590853 - in /hive/trunk:
itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/history/
ql/src/java/org/apache/hadoop/hive/ql/exec/
ql/src/java/org/apache/hadoop/hive/ql/metadata/
ql/src/java/org/apache/hadoop/hive/ql/optimizer/listbuc...
Author: prasanthj
Date: Tue Apr 29 01:10:34 2014
New Revision: 1590853
URL: http://svn.apache.org/r1590853
Log:
HIVE-6968: list bucketing feature does not update the location map for unpartitioned tables
Added:
hive/trunk/ql/src/test/queries/clientpositive/list_bucket_dml_14.q
hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_14.q.out
Modified:
hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/history/TestHiveHistory.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/listbucketingpruner/LBPartitionProcFactory.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java
Modified: hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/history/TestHiveHistory.java
URL: http://svn.apache.org/viewvc/hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/history/TestHiveHistory.java?rev=1590853&r1=1590852&r2=1590853&view=diff
==============================================================================
--- hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/history/TestHiveHistory.java (original)
+++ hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/history/TestHiveHistory.java Tue Apr 29 01:10:34 2014
@@ -103,7 +103,7 @@ public class TestHiveHistory extends Tes
db.dropTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, src, true, true);
db.createTable(src, cols, null, TextInputFormat.class,
IgnoreKeyTextOutputFormat.class);
- db.loadTable(hadoopDataFile[i], src, false, false, false);
+ db.loadTable(hadoopDataFile[i], src, false, false, false, false);
i++;
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java?rev=1590853&r1=1590852&r2=1590853&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java Tue Apr 29 01:10:34 2014
@@ -273,7 +273,8 @@ public class MoveTask extends Task<MoveW
if (tbd.getPartitionSpec().size() == 0) {
dc = new DataContainer(table.getTTable());
db.loadTable(tbd.getSourcePath(), tbd.getTable()
- .getTableName(), tbd.getReplace(), tbd.getHoldDDLTime(), work.isSrcLocal());
+ .getTableName(), tbd.getReplace(), tbd.getHoldDDLTime(), work.isSrcLocal(),
+ isSkewedStoredAsDirs(tbd));
if (work.getOutputs() != null) {
work.getOutputs().add(new WriteEntity(table,
(tbd.getReplace() ? WriteEntity.WriteType.INSERT_OVERWRITE :
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java?rev=1590853&r1=1590852&r2=1590853&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java Tue Apr 29 01:10:34 2014
@@ -1468,9 +1468,11 @@ private void constructOneLBLocationMap(F
* @param holdDDLTime
* @param isSrcLocal
* If the source directory is LOCAL
+ * @param isSkewedStoreAsSubdir
+ * if list bucketing enabled
*/
public void loadTable(Path loadPath, String tableName, boolean replace,
- boolean holdDDLTime, boolean isSrcLocal) throws HiveException {
+ boolean holdDDLTime, boolean isSrcLocal, boolean isSkewedStoreAsSubdir) throws HiveException {
Table tbl = getTable(tableName);
if (replace) {
tbl.replaceFiles(loadPath, isSrcLocal);
@@ -1478,6 +1480,20 @@ private void constructOneLBLocationMap(F
tbl.copyFiles(loadPath, isSrcLocal);
}
+ try {
+ if (isSkewedStoreAsSubdir) {
+ SkewedInfo skewedInfo = tbl.getSkewedInfo();
+ // Construct list bucketing location mappings from sub-directory name.
+ Map<List<String>, String> skewedColValueLocationMaps = constructListBucketingLocationMap(
+ tbl.getPath(), skewedInfo);
+ // Add list bucketing location mappings.
+ skewedInfo.setSkewedColValueLocationMaps(skewedColValueLocationMaps);
+ }
+ } catch (IOException e) {
+ LOG.error(StringUtils.stringifyException(e));
+ throw new HiveException(e);
+ }
+
if (!holdDDLTime) {
try {
alterTable(tableName, tbl);
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/listbucketingpruner/LBPartitionProcFactory.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/listbucketingpruner/LBPartitionProcFactory.java?rev=1590853&r1=1590852&r2=1590853&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/listbucketingpruner/LBPartitionProcFactory.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/listbucketingpruner/LBPartitionProcFactory.java Tue Apr 29 01:10:34 2014
@@ -25,6 +25,7 @@ import org.apache.hadoop.hive.ql.exec.UD
import org.apache.hadoop.hive.ql.lib.NodeProcessor;
import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.optimizer.PrunerOperatorFactory;
import org.apache.hadoop.hive.ql.optimizer.pcr.PcrOpProcFactory;
import org.apache.hadoop.hive.ql.optimizer.ppr.PartitionPruner;
@@ -51,23 +52,24 @@ public class LBPartitionProcFactory exte
protected void generatePredicate(NodeProcessorCtx procCtx, FilterOperator fop,
TableScanOperator top) throws SemanticException, UDFArgumentException {
LBOpPartitionWalkerCtx owc = (LBOpPartitionWalkerCtx) procCtx;
-
- //Run partition pruner to get partitions
- ParseContext parseCtx = owc.getParseContext();
- PrunedPartitionList prunedPartList;
- try {
- String alias = (String) parseCtx.getTopOps().keySet().toArray()[0];
- prunedPartList = PartitionPruner.prune(top, parseCtx, alias);
- } catch (HiveException e) {
- // Has to use full name to make sure it does not conflict with
- // org.apache.commons.lang.StringUtils
- throw new SemanticException(e.getMessage(), e);
- }
-
- if (prunedPartList != null) {
- owc.setPartitions(prunedPartList);
+ Table tbl = owc.getParseContext().getTopToTable().get(top);
+ if (tbl.isPartitioned()) {
+ // Run partition pruner to get partitions
+ ParseContext parseCtx = owc.getParseContext();
+ PrunedPartitionList prunedPartList;
+ try {
+ String alias = (String) parseCtx.getTopOps().keySet().toArray()[0];
+ prunedPartList = PartitionPruner.prune(top, parseCtx, alias);
+ } catch (HiveException e) {
+ // Has to use full name to make sure it does not conflict with
+ // org.apache.commons.lang.StringUtils
+ throw new SemanticException(e.getMessage(), e);
+ }
+
+ if (prunedPartList != null) {
+ owc.setPartitions(prunedPartList);
+ }
}
-
}
}
Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java?rev=1590853&r1=1590852&r2=1590853&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java Tue Apr 29 01:10:34 2014
@@ -132,7 +132,7 @@ public class TestExecDriver extends Test
db.dropTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, src, true, true);
db.createTable(src, cols, null, TextInputFormat.class,
IgnoreKeyTextOutputFormat.class);
- db.loadTable(hadoopDataFile[i], src, false, false, true);
+ db.loadTable(hadoopDataFile[i], src, false, false, true, false);
i++;
}
Added: hive/trunk/ql/src/test/queries/clientpositive/list_bucket_dml_14.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/list_bucket_dml_14.q?rev=1590853&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/list_bucket_dml_14.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/list_bucket_dml_14.q Tue Apr 29 01:10:34 2014
@@ -0,0 +1,38 @@
+set hive.mapred.supports.subdirectories=true;
+set hive.exec.dynamic.partition=true;
+set hive.exec.dynamic.partition.mode=nonstrict;
+set hive.input.format=org.apache.hadoop.hive.ql.io.BucketizedHiveInputFormat;
+set hive.merge.mapfiles=false;
+set hive.merge.mapredfiles=false;
+set mapred.input.dir.recursive=true;
+
+-- list bucketing DML : unpartitioned table and 2 stage query plan.
+
+-- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
+
+-- create a skewed table
+create table list_bucketing (key String, value String)
+skewed by (key) on ("484")
+stored as DIRECTORIES
+;
+
+-- list bucketing DML
+explain extended
+insert overwrite table list_bucketing select * from src;
+insert overwrite table list_bucketing select * from src;
+
+-- check DML result
+desc formatted list_bucketing;
+
+select count(1) from src;
+select count(1) from list_bucketing;
+
+select key, value from src where key = "484";
+set hive.optimize.listbucketing=true;
+explain extended
+select key, value from list_bucketing where key = "484";
+select key, value from list_bucketing where key = "484";
+
+-- clean up resources
+drop table list_bucketing;
+
Added: hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_14.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_14.q.out?rev=1590853&view=auto
==============================================================================
Files hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_14.q.out (added) and hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_14.q.out Tue Apr 29 01:10:34 2014 differ