You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by gu...@apache.org on 2014/01/28 06:48:10 UTC
svn commit: r1561947 [11/17] - in /hive/branches/tez: ./ ant/
ant/src/org/apache/hadoop/hive/ant/ beeline/ cli/
cli/src/java/org/apache/hadoop/hive/cli/ common/ common/src/
common/src/java/org/apache/hadoop/hive/common/type/
common/src/java/org/apache/...
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java Tue Jan 28 05:48:03 2014
@@ -242,7 +242,7 @@ public class SimpleFetchOptimizer implem
for (Partition partition : partsList.getNotDeniedPartns()) {
inputs.add(new ReadEntity(partition));
- listP.add(partition.getPartitionPath());
+ listP.add(partition.getDataLocation());
partP.add(Utilities.getPartitionDesc(partition));
}
Table sourceTable = partsList.getSourceTable();
@@ -296,7 +296,7 @@ public class SimpleFetchOptimizer implem
}
long total = 0;
for (Partition partition : partsList.getNotDeniedPartns()) {
- Path path = partition.getPartitionPath();
+ Path path = partition.getDataLocation();
total += getFileLength(jobConf, path, partition.getInputFormatClass());
}
return total;
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SizeBasedBigTableSelectorForAutoSMJ.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SizeBasedBigTableSelectorForAutoSMJ.java?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SizeBasedBigTableSelectorForAutoSMJ.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SizeBasedBigTableSelectorForAutoSMJ.java Tue Jan 28 05:48:03 2014
@@ -78,7 +78,7 @@ public abstract class SizeBasedBigTableS
}
protected long getSize(HiveConf conf, Partition partition) {
- Path path = partition.getPartitionPath();
+ Path path = partition.getDataLocation();
String size = partition.getParameters().get("totalSize");
return getSize(conf, size, path);
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java Tue Jan 28 05:48:03 2014
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.hadoop.hive.ql.optimizer;
import java.util.ArrayList;
@@ -10,6 +27,7 @@ import java.util.Stack;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.common.StatsSetupConst;
+import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
import org.apache.hadoop.hive.ql.exec.ColumnInfo;
import org.apache.hadoop.hive.ql.exec.Description;
@@ -30,6 +48,7 @@ import org.apache.hadoop.hive.ql.lib.Nod
import org.apache.hadoop.hive.ql.lib.Rule;
import org.apache.hadoop.hive.ql.lib.RuleRegExp;
import org.apache.hadoop.hive.ql.metadata.Hive;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.Partition;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.parse.ParseContext;
@@ -42,12 +61,15 @@ import org.apache.hadoop.hive.ql.plan.Fe
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFCount;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFMax;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFMin;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFSum;
import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
import org.apache.hadoop.hive.serde2.objectinspector.StandardStructObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
/** There is a set of queries which can be answered entirely from statistics stored in metastore.
* Examples of such queries are count(*), count(a), max(a), min(b) etc. Hive already collects
@@ -181,30 +203,29 @@ public class StatsOptimizer implements T
Hive hive = Hive.get(pctx.getConf());
for (AggregationDesc aggr : aggrs) {
- if (aggr.getGenericUDAFName().equals(GenericUDAFCount.class.getAnnotation(
+ if (aggr.getGenericUDAFName().equals(GenericUDAFSum.class.getAnnotation(
+ Description.class).name())) {
+ if(!(aggr.getParameters().get(0) instanceof ExprNodeConstantDesc)){
+ return null;
+ }
+ Long rowCnt = getRowCnt(pctx, tsOp, tbl);
+ if(rowCnt == null) {
+ return null;
+ }
+ oneRow.add(HiveDecimal.create(((ExprNodeConstantDesc) aggr.getParameters().get(0))
+ .getValue().toString()).multiply(HiveDecimal.create(rowCnt)));
+ ois.add(PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
+ PrimitiveCategory.DECIMAL));
+ }
+ else if (aggr.getGenericUDAFName().equals(GenericUDAFCount.class.getAnnotation(
Description.class).name())) {
- long rowCnt = 0;
+ Long rowCnt = 0L;
if ((aggr.getParameters().isEmpty() || aggr.getParameters().get(0) instanceof
ExprNodeConstantDesc)) {
// Its either count (*) or count(1) case
- if(tbl.isPartitioned()) {
- for (Partition part : hive.getAllPartitionsOf(tbl)) {
- long partRowCnt = Long.parseLong(part.getParameters()
- .get(StatsSetupConst.ROW_COUNT));
- if (partRowCnt < 1) {
- Log.debug("Partition doesn't have upto date stats " + part.getSpec());
- return null;
- }
- rowCnt += partRowCnt;
- }
- } else { // unpartitioned table
- rowCnt = Long.parseLong(tbl.getProperty(StatsSetupConst.ROW_COUNT));
- if (rowCnt < 1) {
- // if rowCnt < 1 than its either empty table or table on which stats are not
- // computed We assume the worse and don't attempt to optimize.
- Log.debug("Table doesn't have upto date stats " + tbl.getTableName());
- return null;
- }
+ rowCnt = getRowCnt(pctx, tsOp, tbl);
+ if(rowCnt == null) {
+ return null;
}
} else {
// Its count(col) case
@@ -221,7 +242,7 @@ public class StatsOptimizer implements T
Log.debug("Stats for table : " + tbl.getTableName() + " are not upto date.");
return null;
}
- rowCnt = Long.parseLong(tbl.getProperty(StatsSetupConst.ROW_COUNT));
+ rowCnt = Long.parseLong(tbl.getProperty(StatsSetupConst.ROW_COUNT));
if (rowCnt < 1) {
Log.debug("Table doesn't have upto date stats " + tbl.getTableName());
return null;
@@ -238,13 +259,13 @@ public class StatsOptimizer implements T
rowCnt -= nullCnt;
}
} else {
- for (Partition part : hive.getAllPartitionsOf(tbl)) {
+ for (Partition part : pctx.getPrunedPartitions(tsOp.getConf().getAlias(), tsOp).getPartitions()) {
if (!StatsSetupConst.areStatsUptoDate(part.getParameters())) {
Log.debug("Stats for part : " + part.getSpec() + " are not upto date.");
return null;
}
- Long partRowCnt = Long.parseLong(part.getParameters()
- .get(StatsSetupConst.ROW_COUNT));
+ Long partRowCnt = Long.parseLong(part.getParameters()
+ .get(StatsSetupConst.ROW_COUNT));
if (partRowCnt < 1) {
Log.debug("Partition doesn't have upto date stats " + part.getSpec());
return null;
@@ -298,7 +319,7 @@ public class StatsOptimizer implements T
return null;
}
} else {
- Set<Partition> parts = hive.getAllPartitionsOf(tbl);
+ Set<Partition> parts = pctx.getPrunedPartitions(tsOp.getConf().getAlias(), tsOp).getPartitions();
switch(type) {
case Integeral: {
long maxVal = Long.MIN_VALUE;
@@ -370,7 +391,7 @@ public class StatsOptimizer implements T
return null;
}
} else {
- Set<Partition> parts = hive.getAllPartitionsOf(tbl);
+ Set<Partition> parts = pctx.getPrunedPartitions(tsOp.getConf().getAlias(), tsOp).getPartitions();
switch(type) {
case Integeral: {
long minVal = Long.MAX_VALUE;
@@ -442,5 +463,28 @@ public class StatsOptimizer implements T
return null;
}
+
+ private Long getRowCnt (ParseContext pCtx, TableScanOperator tsOp, Table tbl) throws HiveException {
+ Long rowCnt = 0L;
+ if(tbl.isPartitioned()) {
+ for (Partition part : pctx.getPrunedPartitions(tsOp.getConf().getAlias(), tsOp).getPartitions()) {
+ long partRowCnt = Long.parseLong(part.getParameters().get(StatsSetupConst.ROW_COUNT));
+ if (partRowCnt < 1) {
+ Log.debug("Partition doesn't have upto date stats " + part.getSpec());
+ return null;
+ }
+ rowCnt += partRowCnt;
+ }
+ } else { // unpartitioned table
+ rowCnt = Long.parseLong(tbl.getProperty(StatsSetupConst.ROW_COUNT));
+ if (rowCnt < 1) {
+ // if rowCnt < 1 than its either empty table or table on which stats are not
+ // computed We assume the worse and don't attempt to optimize.
+ Log.debug("Table doesn't have upto date stats " + tbl.getTableName());
+ rowCnt = null;
+ }
+ }
+ return rowCnt;
+ }
}
}
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/BucketingSortingOpProcFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/BucketingSortingOpProcFactory.java?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/BucketingSortingOpProcFactory.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/BucketingSortingOpProcFactory.java Tue Jan 28 05:48:03 2014
@@ -462,7 +462,7 @@ public class BucketingSortingOpProcFacto
// Set the inferred bucket columns for the file this FileSink produces
if (bucketCols != null) {
List<BucketCol> newBucketCols = getNewBucketCols(bucketCols, colInfos);
- bctx.getBucketedColsByDirectory().put(fop.getConf().getDirName(), newBucketCols);
+ bctx.getBucketedColsByDirectory().put(fop.getConf().getDirName().toString(), newBucketCols);
bctx.setBucketedCols(fop, newBucketCols);
}
@@ -471,7 +471,7 @@ public class BucketingSortingOpProcFacto
// Set the inferred sort columns for the file this FileSink produces
if (sortCols != null) {
List<SortCol> newSortCols = getNewSortCols(sortCols, colInfos);
- bctx.getSortedColsByDirectory().put(fop.getConf().getDirName(), newSortCols);
+ bctx.getSortedColsByDirectory().put(fop.getConf().getDirName().toString(), newSortCols);
bctx.setSortedCols(fop, newSortCols);
}
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CommonJoinTaskDispatcher.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CommonJoinTaskDispatcher.java?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CommonJoinTaskDispatcher.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CommonJoinTaskDispatcher.java Tue Jan 28 05:48:03 2014
@@ -246,7 +246,7 @@ public class CommonJoinTaskDispatcher ex
}
// The mapJoinTaskFileSinkOperator writes to a different directory
- String childMRPath = mapJoinTaskFileSinkOperator.getConf().getDirName();
+ String childMRPath = mapJoinTaskFileSinkOperator.getConf().getDirName().toString();
List<String> childMRAliases = childMapWork.getPathToAliases().get(childMRPath);
if (childMRAliases == null || childMRAliases.size() != 1) {
return;
@@ -441,7 +441,7 @@ public class CommonJoinTaskDispatcher ex
Configuration conf = context.getConf();
- // If sizes of atleast n-1 tables in a n-way join is known, and their sum is smaller than
+ // If sizes of at least n-1 tables in a n-way join is known, and their sum is smaller than
// the threshold size, convert the join into map-join and don't create a conditional task
boolean convertJoinMapJoin = HiveConf.getBoolVar(conf,
HiveConf.ConfVars.HIVECONVERTJOINNOCONDITIONALTASK);
@@ -451,47 +451,32 @@ public class CommonJoinTaskDispatcher ex
long mapJoinSize = HiveConf.getLongVar(conf,
HiveConf.ConfVars.HIVECONVERTJOINNOCONDITIONALTASKTHRESHOLD);
- boolean bigTableFound = false;
- long largestBigTableCandidateSize = -1;
- long sumTableSizes = 0;
- for (String alias : aliasToWork.keySet()) {
+ Long bigTableSize = null;
+ Set<String> aliases = aliasToWork.keySet();
+ for (String alias : aliases) {
int tablePosition = getPosition(currWork, joinOp, alias);
- boolean bigTableCandidate = bigTableCandidates.contains(tablePosition);
- Long size = aliasToSize.get(alias);
- // The size is not available at compile time if the input is a sub-query.
- // If the size of atleast n-1 inputs for a n-way join are available at compile time,
- // and the sum of them is less than the specified threshold, then convert the join
- // into a map-join without the conditional task.
- if ((size == null) || (size > mapJoinSize)) {
- sumTableSizes += largestBigTableCandidateSize;
- if (bigTableFound || (sumTableSizes > mapJoinSize) || !bigTableCandidate) {
- convertJoinMapJoin = false;
- break;
- }
- bigTableFound = true;
+ if (!bigTableCandidates.contains(tablePosition)) {
+ continue;
+ }
+ long sumOfOthers = Utilities.sumOfExcept(aliasToSize, aliases, alias);
+ if (sumOfOthers < 0 || sumOfOthers > mapJoinSize) {
+ continue; // some small alias is not known or too big
+ }
+ if (bigTableSize == null && bigTablePosition >= 0 && tablePosition < bigTablePosition) {
+ continue; // prefer right most alias
+ }
+ Long aliasSize = aliasToSize.get(alias);
+ if (bigTableSize == null || (aliasSize != null && aliasSize > bigTableSize)) {
bigTablePosition = tablePosition;
- largestBigTableCandidateSize = mapJoinSize + 1;
- } else {
- if (bigTableCandidate && size > largestBigTableCandidateSize) {
- bigTablePosition = tablePosition;
- sumTableSizes += largestBigTableCandidateSize;
- largestBigTableCandidateSize = size;
- } else {
- sumTableSizes += size;
- }
- if (sumTableSizes > mapJoinSize) {
- convertJoinMapJoin = false;
- break;
- }
+ bigTableSize = aliasSize;
}
}
}
- String bigTableAlias = null;
currWork.setOpParseCtxMap(parseCtx.getOpParseCtx());
currWork.setJoinTree(joinTree);
- if (convertJoinMapJoin) {
+ if (bigTablePosition >= 0) {
// create map join task and set big table as bigTablePosition
MapRedTask newTask = convertTaskToMapJoinTask(currTask.getWork(), bigTablePosition).getFirst();
@@ -521,7 +506,7 @@ public class CommonJoinTaskDispatcher ex
// create map join task and set big table as i
ObjectPair<MapRedTask, String> newTaskAlias = convertTaskToMapJoinTask(newWork, i);
MapRedTask newTask = newTaskAlias.getFirst();
- bigTableAlias = newTaskAlias.getSecond();
+ String bigTableAlias = newTaskAlias.getSecond();
if (cannotConvert(bigTableAlias, aliasToSize,
aliasTotalKnownInputSize, ThresholdOfSmallTblSizeSum)) {
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenMRSkewJoinProcessor.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenMRSkewJoinProcessor.java?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenMRSkewJoinProcessor.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenMRSkewJoinProcessor.java Tue Jan 28 05:48:03 2014
@@ -123,21 +123,20 @@ public final class GenMRSkewJoinProcesso
Task<? extends Serializable> child =
children != null && children.size() == 1 ? children.get(0) : null;
- String baseTmpDir = parseCtx.getContext().getMRTmpFileURI();
+ Path baseTmpDir = parseCtx.getContext().getMRTmpPath();
JoinDesc joinDescriptor = joinOp.getConf();
Map<Byte, List<ExprNodeDesc>> joinValues = joinDescriptor.getExprs();
int numAliases = joinValues.size();
- Map<Byte, String> bigKeysDirMap = new HashMap<Byte, String>();
- Map<Byte, Map<Byte, String>> smallKeysDirMap = new HashMap<Byte, Map<Byte, String>>();
- Map<Byte, String> skewJoinJobResultsDir = new HashMap<Byte, String>();
+ Map<Byte, Path> bigKeysDirMap = new HashMap<Byte, Path>();
+ Map<Byte, Map<Byte, Path>> smallKeysDirMap = new HashMap<Byte, Map<Byte, Path>>();
+ Map<Byte, Path> skewJoinJobResultsDir = new HashMap<Byte, Path>();
Byte[] tags = joinDescriptor.getTagOrder();
for (int i = 0; i < numAliases; i++) {
Byte alias = tags[i];
- String bigKeysDir = getBigKeysDir(baseTmpDir, alias);
- bigKeysDirMap.put(alias, bigKeysDir);
- Map<Byte, String> smallKeysMap = new HashMap<Byte, String>();
+ bigKeysDirMap.put(alias, getBigKeysDir(baseTmpDir, alias));
+ Map<Byte, Path> smallKeysMap = new HashMap<Byte, Path>();
smallKeysDirMap.put(alias, smallKeysMap);
for (Byte src2 : tags) {
if (!src2.equals(alias)) {
@@ -154,8 +153,8 @@ public final class GenMRSkewJoinProcesso
joinDescriptor.setSkewKeyDefinition(HiveConf.getIntVar(parseCtx.getConf(),
HiveConf.ConfVars.HIVESKEWJOINKEY));
- HashMap<String, Task<? extends Serializable>> bigKeysDirToTaskMap =
- new HashMap<String, Task<? extends Serializable>>();
+ HashMap<Path, Task<? extends Serializable>> bigKeysDirToTaskMap =
+ new HashMap<Path, Task<? extends Serializable>>();
List<Serializable> listWorks = new ArrayList<Serializable>();
List<Task<? extends Serializable>> listTasks = new ArrayList<Task<? extends Serializable>>();
MapredWork currPlan = (MapredWork) currTask.getWork();
@@ -272,13 +271,13 @@ public final class GenMRSkewJoinProcesso
ArrayList<String> aliases = new ArrayList<String>();
String alias = src.toString();
aliases.add(alias);
- String bigKeyDirPath = bigKeysDirMap.get(src);
- newPlan.getPathToAliases().put(bigKeyDirPath, aliases);
+ Path bigKeyDirPath = bigKeysDirMap.get(src);
+ newPlan.getPathToAliases().put(bigKeyDirPath.toString(), aliases);
newPlan.getAliasToWork().put(alias, tblScan_op);
PartitionDesc part = new PartitionDesc(tableDescList.get(src), null);
- newPlan.getPathToPartitionInfo().put(bigKeyDirPath, part);
+ newPlan.getPathToPartitionInfo().put(bigKeyDirPath.toString(), part);
newPlan.getAliasToPartnInfo().put(alias, part);
Operator<? extends OperatorDesc> reducer = clonePlan.getReduceWork().getReducer();
@@ -297,7 +296,7 @@ public final class GenMRSkewJoinProcesso
MapredLocalWork localPlan = new MapredLocalWork(
new LinkedHashMap<String, Operator<? extends OperatorDesc>>(),
new LinkedHashMap<String, FetchWork>());
- Map<Byte, String> smallTblDirs = smallKeysDirMap.get(src);
+ Map<Byte, Path> smallTblDirs = smallKeysDirMap.get(src);
for (int j = 0; j < numAliases; j++) {
if (j == i) {
@@ -306,7 +305,7 @@ public final class GenMRSkewJoinProcesso
Byte small_alias = tags[j];
Operator<? extends OperatorDesc> tblScan_op2 = parentOps[j];
localPlan.getAliasToWork().put(small_alias.toString(), tblScan_op2);
- Path tblDir = new Path(smallTblDirs.get(small_alias));
+ Path tblDir = smallTblDirs.get(small_alias);
localPlan.getAliasToFetchWork().put(small_alias.toString(),
new FetchWork(tblDir, tableDescList.get(small_alias)));
}
@@ -393,20 +392,19 @@ public final class GenMRSkewJoinProcesso
private static String SMALLKEYS = "smallkeys";
private static String RESULTS = "results";
- static String getBigKeysDir(String baseDir, Byte srcTbl) {
- return baseDir + Path.SEPARATOR + skewJoinPrefix + UNDERLINE + BIGKEYS
- + UNDERLINE + srcTbl;
+ static Path getBigKeysDir(Path baseDir, Byte srcTbl) {
+ return new Path(baseDir, skewJoinPrefix + UNDERLINE + BIGKEYS + UNDERLINE + srcTbl);
}
- static String getBigKeysSkewJoinResultDir(String baseDir, Byte srcTbl) {
- return baseDir + Path.SEPARATOR + skewJoinPrefix + UNDERLINE + BIGKEYS
- + UNDERLINE + RESULTS + UNDERLINE + srcTbl;
+ static Path getBigKeysSkewJoinResultDir(Path baseDir, Byte srcTbl) {
+ return new Path(baseDir, skewJoinPrefix + UNDERLINE + BIGKEYS
+ + UNDERLINE + RESULTS + UNDERLINE + srcTbl);
}
- static String getSmallKeysDir(String baseDir, Byte srcTblBigTbl,
+ static Path getSmallKeysDir(Path baseDir, Byte srcTblBigTbl,
Byte srcTblSmallTbl) {
- return baseDir + Path.SEPARATOR + skewJoinPrefix + UNDERLINE + SMALLKEYS
- + UNDERLINE + srcTblBigTbl + UNDERLINE + srcTblSmallTbl;
+ return new Path(baseDir, skewJoinPrefix + UNDERLINE + SMALLKEYS
+ + UNDERLINE + srcTblBigTbl + UNDERLINE + srcTblSmallTbl);
}
}
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MapJoinResolver.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MapJoinResolver.java?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MapJoinResolver.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MapJoinResolver.java Tue Jan 28 05:48:03 2014
@@ -26,6 +26,7 @@ import java.util.List;
import java.util.Map;
import java.util.Stack;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.ql.Context;
import org.apache.hadoop.hive.ql.exec.ConditionalTask;
import org.apache.hadoop.hive.ql.exec.MapJoinOperator;
@@ -101,10 +102,10 @@ public class MapJoinResolver implements
if (localwork != null) {
// get the context info and set up the shared tmp URI
Context ctx = physicalContext.getContext();
- String tmpFileURI = Utilities.generateTmpURI(ctx.getLocalTmpFileURI(), currTask.getId());
- localwork.setTmpFileURI(tmpFileURI);
- String hdfsTmpURI = Utilities.generateTmpURI(ctx.getMRTmpFileURI(), currTask.getId());
- mapredWork.getMapWork().setTmpHDFSFileURI(hdfsTmpURI);
+ Path tmpPath = Utilities.generateTmpPath(ctx.getLocalTmpPath(), currTask.getId());
+ localwork.setTmpPath(tmpPath);
+ mapredWork.getMapWork().setTmpHDFSPath(Utilities.generateTmpPath(
+ ctx.getMRTmpPath(), currTask.getId()));
// create a task for this local work; right now, this local work is shared
// by the original MapredTask and this new generated MapredLocalTask.
MapredLocalTask localTask = (MapredLocalTask) TaskFactory.get(localwork, physicalContext
@@ -130,7 +131,7 @@ public class MapJoinResolver implements
// create new local work and setup the dummy ops
MapredLocalWork newLocalWork = new MapredLocalWork();
newLocalWork.setDummyParentOp(dummyOps);
- newLocalWork.setTmpFileURI(tmpFileURI);
+ newLocalWork.setTmpPath(tmpPath);
newLocalWork.setInputFileChangeSensitive(localwork.getInputFileChangeSensitive());
newLocalWork.setBucketMapjoinContext(localwork.copyPartSpecMappingOnly());
mapredWork.getMapWork().setMapLocalWork(newLocalWork);
@@ -166,15 +167,15 @@ public class MapJoinResolver implements
// get bigKeysDirToTaskMap
ConditionalResolverSkewJoinCtx context = (ConditionalResolverSkewJoinCtx) conditionalTask
.getResolverCtx();
- HashMap<String, Task<? extends Serializable>> bigKeysDirToTaskMap = context
+ HashMap<Path, Task<? extends Serializable>> bigKeysDirToTaskMap = context
.getDirToTaskMap();
// to avoid concurrent modify the hashmap
- HashMap<String, Task<? extends Serializable>> newbigKeysDirToTaskMap = new HashMap<String, Task<? extends Serializable>>();
+ HashMap<Path, Task<? extends Serializable>> newbigKeysDirToTaskMap = new HashMap<Path, Task<? extends Serializable>>();
// reset the resolver
- for (Map.Entry<String, Task<? extends Serializable>> entry : bigKeysDirToTaskMap
+ for (Map.Entry<Path, Task<? extends Serializable>> entry : bigKeysDirToTaskMap
.entrySet()) {
Task<? extends Serializable> task = entry.getValue();
- String key = entry.getKey();
+ Path key = entry.getKey();
if (task.equals(currTask)) {
newbigKeysDirToTaskMap.put(key, localTask);
} else {
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MetadataOnlyOptimizer.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MetadataOnlyOptimizer.java?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MetadataOnlyOptimizer.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MetadataOnlyOptimizer.java Tue Jan 28 05:48:03 2014
@@ -238,7 +238,7 @@ public class MetadataOnlyOptimizer imple
for (String path : paths) {
PartitionDesc partDesc = work.getPathToPartitionInfo().get(path);
PartitionDesc newPartition = changePartitionToMetadataOnly(partDesc);
- Path fakePath = new Path(physicalContext.getContext().getMRTmpFileURI()
+ Path fakePath = new Path(physicalContext.getContext().getMRTmpPath()
+ newPartition.getTableName()
+ encode(newPartition.getPartSpec()));
work.getPathToPartitionInfo().remove(path);
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java Tue Jan 28 05:48:03 2014
@@ -43,6 +43,7 @@ import org.apache.hadoop.hive.ql.exec.Ma
import org.apache.hadoop.hive.ql.exec.Operator;
import org.apache.hadoop.hive.ql.exec.OperatorFactory;
import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator;
+import org.apache.hadoop.hive.ql.exec.SMBMapJoinOperator;
import org.apache.hadoop.hive.ql.exec.SelectOperator;
import org.apache.hadoop.hive.ql.exec.TableScanOperator;
import org.apache.hadoop.hive.ql.exec.Task;
@@ -78,6 +79,7 @@ import org.apache.hadoop.hive.ql.plan.Ma
import org.apache.hadoop.hive.ql.plan.MapWork;
import org.apache.hadoop.hive.ql.plan.OperatorDesc;
import org.apache.hadoop.hive.ql.plan.PartitionDesc;
+import org.apache.hadoop.hive.ql.plan.SMBJoinDesc;
import org.apache.hadoop.hive.ql.plan.TableScanDesc;
import org.apache.hadoop.hive.ql.plan.TezWork;
import org.apache.hadoop.hive.ql.plan.api.OperatorType;
@@ -555,6 +557,8 @@ public class Vectorizer implements Physi
case MAPJOIN:
if (op instanceof MapJoinOperator) {
ret = validateMapJoinOperator((MapJoinOperator) op);
+ } else if (op instanceof SMBMapJoinOperator) {
+ ret = validateSMBMapJoinOperator((SMBMapJoinOperator) op);
}
break;
case GROUPBY:
@@ -583,6 +587,12 @@ public class Vectorizer implements Physi
return ret;
}
+ private boolean validateSMBMapJoinOperator(SMBMapJoinOperator op) {
+ SMBJoinDesc desc = op.getConf();
+ // Validation is the same as for map join, since the 'small' tables are not vectorized
+ return validateMapJoinDesc(desc);
+ }
+
private boolean validateTableScanOperator(TableScanOperator op) {
TableScanDesc desc = op.getConf();
return !desc.isGatherStats();
@@ -590,6 +600,10 @@ public class Vectorizer implements Physi
private boolean validateMapJoinOperator(MapJoinOperator op) {
MapJoinDesc desc = op.getConf();
+ return validateMapJoinDesc(desc);
+ }
+
+ private boolean validateMapJoinDesc(MapJoinDesc desc) {
byte posBigTable = (byte) desc.getPosBigTable();
List<ExprNodeDesc> filterExprs = desc.getFilters().get(posBigTable);
List<ExprNodeDesc> keyExprs = desc.getKeys().get(posBigTable);
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/UnionProcFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/UnionProcFactory.java?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/UnionProcFactory.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/UnionProcFactory.java Tue Jan 28 05:48:03 2014
@@ -212,7 +212,7 @@ public final class UnionProcFactory {
// for each sub-query. Also, these different filesinks need to be linked to each other
FileSinkOperator fileSinkOp = (FileSinkOperator)stack.get(pos);
// For file sink operator, change the directory name
- String parentDirName = fileSinkOp.getConf().getDirName();
+ Path parentDirName = fileSinkOp.getConf().getDirName();
// Clone the fileSinkDesc of the final fileSink and create similar fileSinks at
// each parent
@@ -220,9 +220,7 @@ public final class UnionProcFactory {
for (Operator<? extends OperatorDesc> parent : parents) {
FileSinkDesc fileSinkDesc = (FileSinkDesc) fileSinkOp.getConf().clone();
-
- String dirName = parentDirName + Path.SEPARATOR + parent.getIdentifier() ;
- fileSinkDesc.setDirName(dirName);
+ fileSinkDesc.setDirName(new Path(parentDirName, parent.getIdentifier()));
fileSinkDesc.setLinkedFileSink(true);
fileSinkDesc.setParentDir(parentDirName);
parent.setChildOperators(null);
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java Tue Jan 28 05:48:03 2014
@@ -39,6 +39,7 @@ import org.antlr.runtime.tree.Tree;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.HiveMetaStore;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Order;
@@ -785,7 +786,7 @@ public abstract class BaseSemanticAnalyz
}
// check if the columns, as well as value types in the partition() clause are valid
- validatePartSpec(tableHandle, tmpPartSpec, ast, conf);
+ validatePartSpec(tableHandle, tmpPartSpec, ast, conf, false);
List<FieldSchema> parts = tableHandle.getPartitionKeys();
partSpec = new LinkedHashMap<String, String>(partspec.getChildCount());
@@ -1187,8 +1188,8 @@ public abstract class BaseSemanticAnalyz
}
public static void validatePartSpec(Table tbl, Map<String, String> partSpec,
- ASTNode astNode, HiveConf conf) throws SemanticException {
- Utilities.validatePartSpecColumnNames(tbl, partSpec);
+ ASTNode astNode, HiveConf conf, boolean shouldBeFull) throws SemanticException {
+ tbl.validatePartColumnNames(partSpec, shouldBeFull);
if (!HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_TYPE_CHECK_ON_INSERT)) {
return;
@@ -1256,9 +1257,6 @@ public abstract class BaseSemanticAnalyz
}
}
- /** A fixed date format to be used for hive partition column values. */
- private static final DateFormat partitionDateFormat = new SimpleDateFormat("yyyy-MM-dd");
-
private static String normalizeDateCol(
Object colValue, String originalColSpec) throws SemanticException {
Date value;
@@ -1269,7 +1267,7 @@ public abstract class BaseSemanticAnalyz
} else {
throw new SemanticException("Unexpected date type " + colValue.getClass());
}
- return partitionDateFormat.format(value);
+ return HiveMetaStore.PARTITION_DATE_FORMAT.format(value);
}
protected Database getDatabase(String dbName) throws SemanticException {
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java Tue Jan 28 05:48:03 2014
@@ -233,11 +233,11 @@ public class DDLSemanticAnalyzer extends
ast = (ASTNode) ast.getChild(1);
if (ast.getToken().getType() == HiveParser.TOK_ALTERTABLE_FILEFORMAT) {
analyzeAlterTableFileFormat(ast, tableName, partSpec);
- } else if (ast.getToken().getType() == HiveParser.TOK_ALTERTABLE_ALTERPARTS_PROTECTMODE) {
+ } else if (ast.getToken().getType() == HiveParser.TOK_ALTERTABLE_PROTECTMODE) {
analyzeAlterTableProtectMode(ast, tableName, partSpec);
} else if (ast.getToken().getType() == HiveParser.TOK_ALTERTABLE_LOCATION) {
analyzeAlterTableLocation(ast, tableName, partSpec);
- } else if (ast.getToken().getType() == HiveParser.TOK_ALTERTABLE_ALTERPARTS_MERGEFILES) {
+ } else if (ast.getToken().getType() == HiveParser.TOK_ALTERTABLE_MERGEFILES) {
analyzeAlterTablePartMergeFiles(tablePart, ast, tableName, partSpec);
} else if (ast.getToken().getType() == HiveParser.TOK_ALTERTABLE_SERIALIZER) {
analyzeAlterTableSerde(ast, tableName, partSpec);
@@ -267,51 +267,51 @@ public class DDLSemanticAnalyzer extends
analyzeDropIndex(ast);
break;
case HiveParser.TOK_DESCTABLE:
- ctx.setResFile(new Path(ctx.getLocalTmpFileURI()));
+ ctx.setResFile(ctx.getLocalTmpPath());
analyzeDescribeTable(ast);
break;
case HiveParser.TOK_SHOWDATABASES:
- ctx.setResFile(new Path(ctx.getLocalTmpFileURI()));
+ ctx.setResFile(ctx.getLocalTmpPath());
analyzeShowDatabases(ast);
break;
case HiveParser.TOK_SHOWTABLES:
- ctx.setResFile(new Path(ctx.getLocalTmpFileURI()));
+ ctx.setResFile(ctx.getLocalTmpPath());
analyzeShowTables(ast);
break;
case HiveParser.TOK_SHOWCOLUMNS:
- ctx.setResFile(new Path(ctx.getLocalTmpFileURI()));
+ ctx.setResFile(ctx.getLocalTmpPath());
analyzeShowColumns(ast);
break;
case HiveParser.TOK_SHOW_TABLESTATUS:
- ctx.setResFile(new Path(ctx.getLocalTmpFileURI()));
+ ctx.setResFile(ctx.getLocalTmpPath());
analyzeShowTableStatus(ast);
break;
case HiveParser.TOK_SHOW_TBLPROPERTIES:
- ctx.setResFile(new Path(ctx.getLocalTmpFileURI()));
+ ctx.setResFile(ctx.getLocalTmpPath());
analyzeShowTableProperties(ast);
break;
case HiveParser.TOK_SHOWFUNCTIONS:
- ctx.setResFile(new Path(ctx.getLocalTmpFileURI()));
+ ctx.setResFile(ctx.getLocalTmpPath());
analyzeShowFunctions(ast);
break;
case HiveParser.TOK_SHOWLOCKS:
- ctx.setResFile(new Path(ctx.getLocalTmpFileURI()));
+ ctx.setResFile(ctx.getLocalTmpPath());
analyzeShowLocks(ast);
break;
case HiveParser.TOK_SHOWDBLOCKS:
- ctx.setResFile(new Path(ctx.getLocalTmpFileURI()));
+ ctx.setResFile(ctx.getLocalTmpPath());
analyzeShowDbLocks(ast);
break;
case HiveParser.TOK_DESCFUNCTION:
- ctx.setResFile(new Path(ctx.getLocalTmpFileURI()));
+ ctx.setResFile(ctx.getLocalTmpPath());
analyzeDescFunction(ast);
break;
case HiveParser.TOK_DESCDATABASE:
- ctx.setResFile(new Path(ctx.getLocalTmpFileURI()));
+ ctx.setResFile(ctx.getLocalTmpPath());
analyzeDescDatabase(ast);
break;
case HiveParser.TOK_MSCK:
- ctx.setResFile(new Path(ctx.getLocalTmpFileURI()));
+ ctx.setResFile(ctx.getLocalTmpPath());
analyzeMetastoreCheck(ast);
break;
case HiveParser.TOK_DROPVIEW:
@@ -365,8 +365,8 @@ public class DDLSemanticAnalyzer extends
case HiveParser.TOK_ALTERTABLE_DROPPARTS:
analyzeAlterTableDropParts(ast, false);
break;
- case HiveParser.TOK_ALTERTABLE_ALTERPARTS:
- analyzeAlterTableAlterParts(ast);
+ case HiveParser.TOK_ALTERTABLE_PARTCOLTYPE:
+ analyzeAlterTablePartColType(ast);
break;
case HiveParser.TOK_ALTERTABLE_PROPERTIES:
analyzeAlterTableProps(ast, false, false);
@@ -381,15 +381,15 @@ public class DDLSemanticAnalyzer extends
analyzeAlterIndexProps(ast);
break;
case HiveParser.TOK_SHOWPARTITIONS:
- ctx.setResFile(new Path(ctx.getLocalTmpFileURI()));
+ ctx.setResFile(ctx.getLocalTmpPath());
analyzeShowPartitions(ast);
break;
case HiveParser.TOK_SHOW_CREATETABLE:
- ctx.setResFile(new Path(ctx.getLocalTmpFileURI()));
+ ctx.setResFile(ctx.getLocalTmpPath());
analyzeShowCreateTable(ast);
break;
case HiveParser.TOK_SHOWINDEXES:
- ctx.setResFile(new Path(ctx.getLocalTmpFileURI()));
+ ctx.setResFile(ctx.getLocalTmpPath());
analyzeShowIndexes(ast);
break;
case HiveParser.TOK_LOCKTABLE:
@@ -423,9 +423,13 @@ public class DDLSemanticAnalyzer extends
analyzeDropRole(ast);
break;
case HiveParser.TOK_SHOW_ROLE_GRANT:
- ctx.setResFile(new Path(ctx.getLocalTmpFileURI()));
+ ctx.setResFile(ctx.getLocalTmpPath());
analyzeShowRoleGrant(ast);
break;
+ case HiveParser.TOK_SHOW_ROLES:
+ ctx.setResFile(ctx.getLocalTmpPath());
+ analyzeShowRoles(ast);
+ break;
case HiveParser.TOK_GRANT_ROLE:
analyzeGrantRevokeRole(true, ast);
break;
@@ -436,7 +440,7 @@ public class DDLSemanticAnalyzer extends
analyzeGrant(ast);
break;
case HiveParser.TOK_SHOW_GRANT:
- ctx.setResFile(new Path(ctx.getLocalTmpFileURI()));
+ ctx.setResFile(ctx.getLocalTmpPath());
analyzeShowGrant(ast);
break;
case HiveParser.TOK_REVOKE:
@@ -515,6 +519,15 @@ public class DDLSemanticAnalyzer extends
}
}
+ private void analyzeShowRoles(ASTNode ast) {
+ RoleDDLDesc showRolesDesc = new RoleDDLDesc(null, null,
+ RoleDDLDesc.RoleOperation.SHOW_ROLES, null);
+ showRolesDesc.setResFile(ctx.getResFile().toString());
+ rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(),
+ showRolesDesc), conf));
+ setFetchTask(createFetchTask(RoleDDLDesc.getSchema()));
+ }
+
private void analyzeAlterDatabase(ASTNode ast) throws SemanticException {
String dbName = unescapeIdentifier(ast.getChild(0).getText());
@@ -758,7 +771,7 @@ public class DDLSemanticAnalyzer extends
Partition part = db.getPartition(table, partSpec, false);
Path tabPath = table.getPath();
- Path partPath = part.getPartitionPath();
+ Path partPath = part.getDataLocation();
// if the table is in a different dfs than the partition,
// replace the partition's dfs with the table's dfs.
@@ -840,9 +853,9 @@ public class DDLSemanticAnalyzer extends
TableDesc tblDesc = Utilities.getTableDesc(table);
// Write the output to temporary directory and move it to the final location at the end
// so the operation is atomic.
- String queryTmpdir = ctx.getExternalTmpFileURI(newTblPartLoc.toUri());
- truncateTblDesc.setOutputDir(new Path(queryTmpdir));
- LoadTableDesc ltd = new LoadTableDesc(new Path(queryTmpdir), tblDesc,
+ Path queryTmpdir = ctx.getExternalTmpPath(newTblPartLoc.toUri());
+ truncateTblDesc.setOutputDir(queryTmpdir);
+ LoadTableDesc ltd = new LoadTableDesc(queryTmpdir, tblDesc,
partSpec == null ? new HashMap<String, String>() : partSpec);
ltd.setLbCtx(lbCtx);
Task<MoveWork> moveTsk = TaskFactory.get(new MoveWork(null, null, ltd, null, false),
@@ -1402,7 +1415,7 @@ public class DDLSemanticAnalyzer extends
isArchived = ArchiveUtils.isArchived(part);
Path tabPath = tblObj.getPath();
- Path partPath = part.getPartitionPath();
+ Path partPath = part.getDataLocation();
// if the table is in a different dfs than the partition,
// replace the partition's dfs with the table's dfs.
@@ -1455,9 +1468,9 @@ public class DDLSemanticAnalyzer extends
ddlWork.setNeedLock(true);
Task<? extends Serializable> mergeTask = TaskFactory.get(ddlWork, conf);
TableDesc tblDesc = Utilities.getTableDesc(tblObj);
- String queryTmpdir = ctx.getExternalTmpFileURI(newTblPartLoc.toUri());
- mergeDesc.setOutputDir(new Path(queryTmpdir));
- LoadTableDesc ltd = new LoadTableDesc(new Path(queryTmpdir), tblDesc,
+ Path queryTmpdir = ctx.getExternalTmpPath(newTblPartLoc.toUri());
+ mergeDesc.setOutputDir(queryTmpdir);
+ LoadTableDesc ltd = new LoadTableDesc(queryTmpdir, tblDesc,
partSpec == null ? new HashMap<String, String>() : partSpec);
ltd.setLbCtx(lbCtx);
Task<MoveWork> moveTsk = TaskFactory.get(new MoveWork(null, null, ltd, null, false),
@@ -2420,7 +2433,7 @@ public class DDLSemanticAnalyzer extends
dropTblDesc), conf));
}
- private void analyzeAlterTableAlterParts(ASTNode ast)
+ private void analyzeAlterTablePartColType(ASTNode ast)
throws SemanticException {
// get table name
String tblName = getUnescapedName((ASTNode)ast.getChild(0));
@@ -2498,32 +2511,30 @@ public class DDLSemanticAnalyzer extends
validateAlterTableType(tab, AlterTableTypes.ADDPARTITION, expectView);
inputs.add(new ReadEntity(tab));
- List<AddPartitionDesc> partitionDescs = new ArrayList<AddPartitionDesc>();
-
int numCh = ast.getChildCount();
int start = ifNotExists ? 2 : 1;
String currentLocation = null;
Map<String, String> currentPart = null;
+ // Parser has done some verification, so the order of tokens doesn't need to be verified here.
+ AddPartitionDesc addPartitionDesc = new AddPartitionDesc(tab.getDbName(), tblName, ifNotExists);
for (int num = start; num < numCh; num++) {
ASTNode child = (ASTNode) ast.getChild(num);
switch (child.getToken().getType()) {
case HiveParser.TOK_PARTSPEC:
if (currentPart != null) {
- Partition partition = getPartitionForOutput(tab, currentPart);
- if (partition == null || !ifNotExists) {
- AddPartitionDesc addPartitionDesc = new AddPartitionDesc(
- tab.getDbName(), tblName, currentPart,
- currentLocation, ifNotExists, expectView);
- partitionDescs.add(addPartitionDesc);
- }
+ addPartitionDesc.addPartition(currentPart, currentLocation);
currentLocation = null;
}
currentPart = getPartSpec(child);
- validatePartSpec(tab, currentPart, (ASTNode)child, conf);
+ validatePartitionValues(currentPart); // validate reserved values
+ validatePartSpec(tab, currentPart, (ASTNode)child, conf, true);
break;
case HiveParser.TOK_PARTITIONLOCATION:
// if location specified, set in partition
+ if (isView) {
+ throw new SemanticException("LOCATION clause illegal for view partition");
+ }
currentLocation = unescapeSQLString(child.getChild(0).getText());
break;
default:
@@ -2533,47 +2544,25 @@ public class DDLSemanticAnalyzer extends
// add the last one
if (currentPart != null) {
- Partition partition = getPartitionForOutput(tab, currentPart);
- if (partition == null || !ifNotExists) {
- AddPartitionDesc addPartitionDesc = new AddPartitionDesc(
- tab.getDbName(), tblName, currentPart,
- currentLocation, ifNotExists, expectView);
- partitionDescs.add(addPartitionDesc);
- }
+ addPartitionDesc.addPartition(currentPart, currentLocation);
}
- if (partitionDescs.isEmpty()) {
+ if (addPartitionDesc.getPartitionCount() == 0) {
// nothing to do
return;
}
- for (AddPartitionDesc addPartitionDesc : partitionDescs) {
- try {
- tab.isValidSpec(addPartitionDesc.getPartSpec());
- } catch (HiveException ex) {
- throw new SemanticException(ErrorMsg.INVALID_PARTITION_SPEC.getMsg(ex.getMessage()));
- }
- rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(),
- addPartitionDesc), conf));
- }
+ rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), addPartitionDesc), conf));
if (isView) {
- // Compile internal query to capture underlying table partition
- // dependencies
+ // Compile internal query to capture underlying table partition dependencies
StringBuilder cmd = new StringBuilder();
cmd.append("SELECT * FROM ");
cmd.append(HiveUtils.unparseIdentifier(tblName));
cmd.append(" WHERE ");
boolean firstOr = true;
- for (AddPartitionDesc partitionDesc : partitionDescs) {
- // Perform this check early so that we get a better error message.
- try {
- // Note that isValidSpec throws an exception (it never
- // actually returns false).
- tab.isValidSpec(partitionDesc.getPartSpec());
- } catch (HiveException ex) {
- throw new SemanticException(ErrorMsg.INVALID_PARTITION_SPEC.getMsg(ex.getMessage()));
- }
+ for (int i = 0; i < addPartitionDesc.getPartitionCount(); ++i) {
+ AddPartitionDesc.OnePartitionDesc partitionDesc = addPartitionDesc.getPartition(i);
if (firstOr) {
firstOr = false;
} else {
@@ -2581,8 +2570,7 @@ public class DDLSemanticAnalyzer extends
}
boolean firstAnd = true;
cmd.append("(");
- for (Map.Entry<String, String> entry : partitionDesc.getPartSpec().entrySet())
- {
+ for (Map.Entry<String, String> entry : partitionDesc.getPartSpec().entrySet()) {
if (firstAnd) {
firstAnd = false;
} else {
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java Tue Jan 28 05:48:03 2014
@@ -22,7 +22,6 @@ import java.io.Serializable;
import java.util.Collections;
import java.util.List;
-import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.ql.exec.Task;
@@ -65,7 +64,7 @@ public class ExplainSemanticAnalyzer ext
sem.analyze(input, ctx);
sem.validate();
- ctx.setResFile(new Path(ctx.getLocalTmpFileURI()));
+ ctx.setResFile(ctx.getLocalTmpPath());
List<Task<? extends Serializable>> tasks = sem.getRootTasks();
Task<? extends Serializable> fetchTask = sem.getFetchTask();
if (tasks == null) {
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/ExportSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/ExportSemanticAnalyzer.java?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/ExportSemanticAnalyzer.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/ExportSemanticAnalyzer.java Tue Jan 28 05:48:03 2014
@@ -87,8 +87,7 @@ public class ExportSemanticAnalyzer exte
if (ts.tableHandle.isPartitioned()) {
partitions = (ts.partitions != null) ? ts.partitions : db.getPartitions(ts.tableHandle);
}
- String tmpfile = ctx.getLocalTmpFileURI();
- Path path = new Path(tmpfile, "_metadata");
+ Path path = new Path(ctx.getLocalTmpPath(), "_metadata");
EximUtil.createExportDump(FileSystem.getLocal(conf), path, ts.tableHandle, partitions);
Task<? extends Serializable> rTask = TaskFactory.get(new CopyWork(
path, new Path(toURI), false), conf);
@@ -105,19 +104,19 @@ public class ExportSemanticAnalyzer exte
if (ts.tableHandle.isPartitioned()) {
for (Partition partition : partitions) {
- URI fromURI = partition.getDataLocation();
+ Path fromPath = partition.getDataLocation();
Path toPartPath = new Path(parentPath, partition.getName());
Task<? extends Serializable> rTask = TaskFactory.get(
- new CopyWork(new Path(fromURI), toPartPath, false),
+ new CopyWork(fromPath, toPartPath, false),
conf);
rootTasks.add(rTask);
inputs.add(new ReadEntity(partition));
}
} else {
- URI fromURI = ts.tableHandle.getDataLocation();
+ Path fromPath = ts.tableHandle.getDataLocation();
Path toDataPath = new Path(parentPath, "data");
Task<? extends Serializable> rTask = TaskFactory.get(new CopyWork(
- new Path(fromURI), toDataPath, false), conf);
+ fromPath, toDataPath, false), conf);
rootTasks.add(rTask);
inputs.add(new ReadEntity(ts.tableHandle));
}
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g Tue Jan 28 05:48:03 2014
@@ -204,7 +204,7 @@ subQuerySource
@init { gParent.msgs.push("subquery source"); }
@after { gParent.msgs.pop(); }
:
- LPAREN queryStatementExpression RPAREN identifier -> ^(TOK_SUBQUERY queryStatementExpression identifier)
+ LPAREN queryStatementExpression[false] RPAREN identifier -> ^(TOK_SUBQUERY queryStatementExpression identifier)
;
//---------------------- Rules for parsing PTF clauses -----------------------------
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g Tue Jan 28 05:48:03 2014
@@ -280,6 +280,7 @@ KW_NOSCAN: 'NOSCAN';
KW_PARTIALSCAN: 'PARTIALSCAN';
KW_USER: 'USER';
KW_ROLE: 'ROLE';
+KW_ROLES: 'ROLES';
KW_INNER: 'INNER';
KW_EXCHANGE: 'EXCHANGE';
KW_ADMIN: 'ADMIN';
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g Tue Jan 28 05:48:03 2014
@@ -138,8 +138,9 @@ TOK_ALTERTABLE_RENAMEPART;
TOK_ALTERTABLE_REPLACECOLS;
TOK_ALTERTABLE_ADDPARTS;
TOK_ALTERTABLE_DROPPARTS;
-TOK_ALTERTABLE_ALTERPARTS;
-TOK_ALTERTABLE_ALTERPARTS_PROTECTMODE;
+TOK_ALTERTABLE_PARTCOLTYPE;
+TOK_ALTERTABLE_PROTECTMODE;
+TOK_ALTERTABLE_MERGEFILES;
TOK_ALTERTABLE_TOUCH;
TOK_ALTERTABLE_ARCHIVE;
TOK_ALTERTABLE_UNARCHIVE;
@@ -276,6 +277,7 @@ TOK_PRIV_OBJECT_COL;
TOK_GRANT_ROLE;
TOK_REVOKE_ROLE;
TOK_SHOW_ROLE_GRANT;
+TOK_SHOW_ROLES;
TOK_SHOWINDEXES;
TOK_SHOWDBLOCKS;
TOK_INDEXCOMMENT;
@@ -284,7 +286,6 @@ TOK_DATABASEPROPERTIES;
TOK_DATABASELOCATION;
TOK_DBPROPLIST;
TOK_ALTERDATABASE_PROPERTIES;
-TOK_ALTERTABLE_ALTERPARTS_MERGEFILES;
TOK_TABNAME;
TOK_TABSRC;
TOK_RESTRICT;
@@ -385,7 +386,7 @@ import java.util.HashMap;
xlateMap.put("KW_ALTER", "ALTER");
xlateMap.put("KW_DESCRIBE", "DESCRIBE");
xlateMap.put("KW_DROP", "DROP");
- xlateMap.put("KW_REANME", "REANME");
+ xlateMap.put("KW_RENAME", "RENAME");
xlateMap.put("KW_TO", "TO");
xlateMap.put("KW_COMMENT", "COMMENT");
xlateMap.put("KW_BOOLEAN", "BOOLEAN");
@@ -555,6 +556,13 @@ import java.util.HashMap;
}
return msg;
}
+
+ // counter to generate unique union aliases
+ private int aliasCounter;
+
+ private String generateUnionAlias() {
+ return "_u" + (++aliasCounter);
+ }
}
@rulecatch {
@@ -580,7 +588,7 @@ explainStatement
execStatement
@init { msgs.push("statement"); }
@after { msgs.pop(); }
- : queryStatementExpression
+ : queryStatementExpression[true]
| loadStatement
| exportStatement
| importStatement
@@ -640,6 +648,7 @@ ddlStatement
| revokePrivileges
| showGrants
| showRoleGrants
+ | showRoles
| grantRole
| revokeRole
;
@@ -759,7 +768,7 @@ createTableStatement
tableFileFormat?
tableLocation?
tablePropertiesPrefixed?
- (KW_AS selectStatement)?
+ (KW_AS selectStatement[true])?
)
-> ^(TOK_CREATETABLE $name $ext? ifNotExists?
^(TOK_LIKETABLE $likeName?)
@@ -891,8 +900,16 @@ alterTableStatementSuffix
| alterTblPartitionStatement
| alterStatementSuffixSkewedby
| alterStatementSuffixExchangePartition
+ | alterStatementPartitionKeyType
;
+alterStatementPartitionKeyType
+@init {msgs.push("alter partition key type"); }
+@after {msgs.pop();}
+ : identifier KW_PARTITION KW_COLUMN LPAREN columnNameType RPAREN
+ -> ^(TOK_ALTERTABLE_PARTCOLTYPE identifier columnNameType)
+ ;
+
alterViewStatementSuffix
@init { msgs.push("alter view statement"); }
@after { msgs.pop(); }
@@ -903,7 +920,7 @@ alterViewStatementSuffix
-> ^(TOK_ALTERVIEW_ADDPARTS alterStatementSuffixAddPartitions)
| alterStatementSuffixDropPartitions
-> ^(TOK_ALTERVIEW_DROPPARTS alterStatementSuffixDropPartitions)
- | name=tableName KW_AS selectStatement
+ | name=tableName KW_AS selectStatement[true]
-> ^(TOK_ALTERVIEW_AS $name selectStatement)
;
@@ -1049,8 +1066,6 @@ alterTblPartitionStatement
@after {msgs.pop();}
: tablePartitionPrefix alterTblPartitionStatementSuffix
-> ^(TOK_ALTERTABLE_PARTITION tablePartitionPrefix alterTblPartitionStatementSuffix)
- |Identifier KW_PARTITION KW_COLUMN LPAREN columnNameType RPAREN
- -> ^(TOK_ALTERTABLE_ALTERPARTS Identifier columnNameType)
;
alterTblPartitionStatementSuffix
@@ -1142,7 +1157,7 @@ alterStatementSuffixProtectMode
@init { msgs.push("alter partition protect mode statement"); }
@after { msgs.pop(); }
: alterProtectMode
- -> ^(TOK_ALTERTABLE_ALTERPARTS_PROTECTMODE alterProtectMode)
+ -> ^(TOK_ALTERTABLE_PROTECTMODE alterProtectMode)
;
alterStatementSuffixRenamePart
@@ -1156,7 +1171,7 @@ alterStatementSuffixMergeFiles
@init { msgs.push(""); }
@after { msgs.pop(); }
: KW_CONCATENATE
- -> ^(TOK_ALTERTABLE_ALTERPARTS_MERGEFILES)
+ -> ^(TOK_ALTERTABLE_MERGEFILES)
;
alterProtectMode
@@ -1334,6 +1349,13 @@ showRoleGrants
-> ^(TOK_SHOW_ROLE_GRANT principalName)
;
+showRoles
+@init {msgs.push("show roles");}
+@after {msgs.pop();}
+ : KW_SHOW KW_ROLES
+ -> ^(TOK_SHOW_ROLES)
+ ;
+
showGrants
@init {msgs.push("show grants");}
@after {msgs.pop();}
@@ -1361,6 +1383,7 @@ privObjectType
@init {msgs.push("privilege object type type");}
@after {msgs.pop();}
: KW_DATABASE -> ^(TOK_DB_TYPE)
+ | KW_VIEW -> ^(TOK_TABLE_TYPE)
| KW_TABLE? -> ^(TOK_TABLE_TYPE)
;
@@ -1466,7 +1489,7 @@ createViewStatement
(LPAREN columnNameCommentList RPAREN)? tableComment? viewPartition?
tablePropertiesPrefixed?
KW_AS
- selectStatement
+ selectStatement[true]
-> ^(TOK_CREATEVIEW $name orReplace?
ifNotExists?
columnNameCommentList?
@@ -1857,45 +1880,75 @@ unionType
: KW_UNIONTYPE LESSTHAN colTypeList GREATERTHAN -> ^(TOK_UNIONTYPE colTypeList)
;
-queryOperator
-@init { msgs.push("query operator"); }
+setOperator
+@init { msgs.push("set operator"); }
@after { msgs.pop(); }
: KW_UNION KW_ALL -> ^(TOK_UNION)
;
-// select statement select ... from ... where ... group by ... order by ...
-queryStatementExpression
- : queryStatement (queryOperator^ queryStatement)*
+queryStatementExpression[boolean topLevel]
+ :
+ fromStatement[topLevel]
+ | regularBody[topLevel]
;
+
+fromStatement[boolean topLevel]
+: (singleFromStatement -> singleFromStatement)
+ (u=setOperator r=singleFromStatement
+ -> ^($u {$fromStatement.tree} $r)
+ )*
+ -> {u != null && topLevel}? ^(TOK_QUERY
+ ^(TOK_FROM
+ ^(TOK_SUBQUERY
+ {$fromStatement.tree}
+ {adaptor.create(Identifier, generateUnionAlias())}
+ )
+ )
+ ^(TOK_INSERT
+ ^(TOK_DESTINATION ^(TOK_DIR TOK_TMP_FILE))
+ ^(TOK_SELECT ^(TOK_SELEXPR TOK_ALLCOLREF))
+ )
+ )
+ -> {$fromStatement.tree}
+ ;
-queryStatement
+
+singleFromStatement
:
fromClause
( b+=body )+ -> ^(TOK_QUERY fromClause body+)
- | regular_body
;
-regular_body
+regularBody[boolean topLevel]
:
- insertClause
- selectClause
- fromClause
- whereClause?
- groupByClause?
- havingClause?
- orderByClause?
- clusterByClause?
- distributeByClause?
- sortByClause?
- window_clause?
- limitClause? -> ^(TOK_QUERY fromClause ^(TOK_INSERT insertClause
- selectClause whereClause? groupByClause? havingClause? orderByClause? clusterByClause?
- distributeByClause? sortByClause? window_clause? limitClause?))
+ i=insertClause
+ s=selectStatement[topLevel]
+ {$s.tree.getChild(1).replaceChildren(0, 0, $i.tree);} -> {$s.tree}
|
- selectStatement
+ selectStatement[topLevel]
;
-selectStatement
+ selectStatement[boolean topLevel]
+ : (singleSelectStatement -> singleSelectStatement)
+ (u=setOperator b=singleSelectStatement
+ -> ^($u {$selectStatement.tree} $b)
+ )*
+ -> {u != null && topLevel}? ^(TOK_QUERY
+ ^(TOK_FROM
+ ^(TOK_SUBQUERY
+ {$selectStatement.tree}
+ {adaptor.create(Identifier, generateUnionAlias())}
+ )
+ )
+ ^(TOK_INSERT
+ ^(TOK_DESTINATION ^(TOK_DIR TOK_TMP_FILE))
+ ^(TOK_SELECT ^(TOK_SELEXPR TOK_ALLCOLREF))
+ )
+ )
+ -> {$selectStatement.tree}
+ ;
+
+singleSelectStatement
:
selectClause
fromClause
@@ -1912,7 +1965,6 @@ selectStatement
distributeByClause? sortByClause? window_clause? limitClause?))
;
-
body
:
insertClause
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g Tue Jan 28 05:48:03 2014
@@ -373,7 +373,7 @@ precedenceEqualOperator
subQueryExpression
:
- LPAREN! selectStatement RPAREN!
+ LPAREN! selectStatement[true] RPAREN!
;
precedenceEqualExpression
@@ -535,5 +535,5 @@ identifier
nonReserved
:
- KW_TRUE | KW_FALSE | KW_LIKE | KW_EXISTS | KW_ASC | KW_DESC | KW_ORDER | KW_GROUP | KW_BY | KW_AS | KW_INSERT | KW_OVERWRITE | KW_OUTER | KW_LEFT | KW_RIGHT | KW_FULL | KW_PARTITION | KW_PARTITIONS | KW_TABLE | KW_TABLES | KW_COLUMNS | KW_INDEX | KW_INDEXES | KW_REBUILD | KW_FUNCTIONS | KW_SHOW | KW_MSCK | KW_REPAIR | KW_DIRECTORY | KW_LOCAL | KW_USING | KW_CLUSTER | KW_DISTRIBUTE | KW_SORT | KW_UNION | KW_LOAD | KW_EXPORT | KW_IMPORT | KW_DATA | KW_INPATH | KW_IS | KW_NULL | KW_CREATE | KW_EXTERNAL | KW_ALTER | KW_CHANGE | KW_FIRST | KW_AFTER | KW_DESCRIBE | KW_DROP | KW_RENAME | KW_IGNORE | KW_PROTECTION | KW_TO | KW_COMMENT | KW_BOOLEAN | KW_TINYINT | KW_SMALLINT | KW_INT | KW_BIGINT | KW_FLOAT | KW_DOUBLE | KW_DATE | KW_DATETIME | KW_TIMESTAMP | KW_DECIMAL | KW_STRING | KW_ARRAY | KW_STRUCT | KW_UNIONTYPE | KW_PARTITIONED | KW_CLUSTERED | KW_SORTED | KW_INTO | KW_BUCKETS | KW_ROW | KW_ROWS | KW_FORMAT | KW_DELIMITED | KW_FIELDS | KW_TERMINATED | KW_ESCAPED | KW_COLLECTION |
KW_ITEMS | KW_KEYS | KW_KEY_TYPE | KW_LINES | KW_STORED | KW_FILEFORMAT | KW_SEQUENCEFILE | KW_TEXTFILE | KW_RCFILE | KW_ORCFILE | KW_INPUTFORMAT | KW_OUTPUTFORMAT | KW_INPUTDRIVER | KW_OUTPUTDRIVER | KW_OFFLINE | KW_ENABLE | KW_DISABLE | KW_READONLY | KW_NO_DROP | KW_LOCATION | KW_BUCKET | KW_OUT | KW_OF | KW_PERCENT | KW_ADD | KW_REPLACE | KW_RLIKE | KW_REGEXP | KW_TEMPORARY | KW_EXPLAIN | KW_FORMATTED | KW_PRETTY | KW_DEPENDENCY | KW_LOGICAL | KW_SERDE | KW_WITH | KW_DEFERRED | KW_SERDEPROPERTIES | KW_DBPROPERTIES | KW_LIMIT | KW_SET | KW_UNSET | KW_TBLPROPERTIES | KW_IDXPROPERTIES | KW_VALUE_TYPE | KW_ELEM_TYPE | KW_MAPJOIN | KW_STREAMTABLE | KW_HOLD_DDLTIME | KW_CLUSTERSTATUS | KW_UTC | KW_UTCTIMESTAMP | KW_LONG | KW_DELETE | KW_PLUS | KW_MINUS | KW_FETCH | KW_INTERSECT | KW_VIEW | KW_IN | KW_DATABASES | KW_MATERIALIZED | KW_SCHEMA | KW_SCHEMAS | KW_GRANT | KW_REVOKE | KW_SSL | KW_UNDO | KW_LOCK | KW_LOCKS | KW_UNLOCK | KW_SHARED | KW_EXCLUSIVE | KW_PROCEDURE | KW_UNSIGNED | KW
_WHILE | KW_READ | KW_READS | KW_PURGE | KW_RANGE | KW_ANALYZE | KW_BEFORE | KW_BETWEEN | KW_BOTH | KW_BINARY | KW_CONTINUE | KW_CURSOR | KW_TRIGGER | KW_RECORDREADER | KW_RECORDWRITER | KW_SEMI | KW_LATERAL | KW_TOUCH | KW_ARCHIVE | KW_UNARCHIVE | KW_COMPUTE | KW_STATISTICS | KW_USE | KW_OPTION | KW_CONCATENATE | KW_SHOW_DATABASE | KW_UPDATE | KW_RESTRICT | KW_CASCADE | KW_SKEWED | KW_ROLLUP | KW_CUBE | KW_DIRECTORIES | KW_FOR | KW_GROUPING | KW_SETS | KW_TRUNCATE | KW_NOSCAN | KW_USER | KW_ROLE | KW_INNER | KW_DEFINED | KW_ADMIN
+ KW_TRUE | KW_FALSE | KW_LIKE | KW_EXISTS | KW_ASC | KW_DESC | KW_ORDER | KW_GROUP | KW_BY | KW_AS | KW_INSERT | KW_OVERWRITE | KW_OUTER | KW_LEFT | KW_RIGHT | KW_FULL | KW_PARTITION | KW_PARTITIONS | KW_TABLE | KW_TABLES | KW_COLUMNS | KW_INDEX | KW_INDEXES | KW_REBUILD | KW_FUNCTIONS | KW_SHOW | KW_MSCK | KW_REPAIR | KW_DIRECTORY | KW_LOCAL | KW_USING | KW_CLUSTER | KW_DISTRIBUTE | KW_SORT | KW_UNION | KW_LOAD | KW_EXPORT | KW_IMPORT | KW_DATA | KW_INPATH | KW_IS | KW_NULL | KW_CREATE | KW_EXTERNAL | KW_ALTER | KW_CHANGE | KW_FIRST | KW_AFTER | KW_DESCRIBE | KW_DROP | KW_RENAME | KW_IGNORE | KW_PROTECTION | KW_TO | KW_COMMENT | KW_BOOLEAN | KW_TINYINT | KW_SMALLINT | KW_INT | KW_BIGINT | KW_FLOAT | KW_DOUBLE | KW_DATE | KW_DATETIME | KW_TIMESTAMP | KW_DECIMAL | KW_STRING | KW_ARRAY | KW_STRUCT | KW_UNIONTYPE | KW_PARTITIONED | KW_CLUSTERED | KW_SORTED | KW_INTO | KW_BUCKETS | KW_ROW | KW_ROWS | KW_FORMAT | KW_DELIMITED | KW_FIELDS | KW_TERMINATED | KW_ESCAPED | KW_COLLECTION |
KW_ITEMS | KW_KEYS | KW_KEY_TYPE | KW_LINES | KW_STORED | KW_FILEFORMAT | KW_SEQUENCEFILE | KW_TEXTFILE | KW_RCFILE | KW_ORCFILE | KW_INPUTFORMAT | KW_OUTPUTFORMAT | KW_INPUTDRIVER | KW_OUTPUTDRIVER | KW_OFFLINE | KW_ENABLE | KW_DISABLE | KW_READONLY | KW_NO_DROP | KW_LOCATION | KW_BUCKET | KW_OUT | KW_OF | KW_PERCENT | KW_ADD | KW_REPLACE | KW_RLIKE | KW_REGEXP | KW_TEMPORARY | KW_EXPLAIN | KW_FORMATTED | KW_PRETTY | KW_DEPENDENCY | KW_LOGICAL | KW_SERDE | KW_WITH | KW_DEFERRED | KW_SERDEPROPERTIES | KW_DBPROPERTIES | KW_LIMIT | KW_SET | KW_UNSET | KW_TBLPROPERTIES | KW_IDXPROPERTIES | KW_VALUE_TYPE | KW_ELEM_TYPE | KW_MAPJOIN | KW_STREAMTABLE | KW_HOLD_DDLTIME | KW_CLUSTERSTATUS | KW_UTC | KW_UTCTIMESTAMP | KW_LONG | KW_DELETE | KW_PLUS | KW_MINUS | KW_FETCH | KW_INTERSECT | KW_VIEW | KW_IN | KW_DATABASES | KW_MATERIALIZED | KW_SCHEMA | KW_SCHEMAS | KW_GRANT | KW_REVOKE | KW_SSL | KW_UNDO | KW_LOCK | KW_LOCKS | KW_UNLOCK | KW_SHARED | KW_EXCLUSIVE | KW_PROCEDURE | KW_UNSIGNED | KW
_WHILE | KW_READ | KW_READS | KW_PURGE | KW_RANGE | KW_ANALYZE | KW_BEFORE | KW_BETWEEN | KW_BOTH | KW_BINARY | KW_CONTINUE | KW_CURSOR | KW_TRIGGER | KW_RECORDREADER | KW_RECORDWRITER | KW_SEMI | KW_LATERAL | KW_TOUCH | KW_ARCHIVE | KW_UNARCHIVE | KW_COMPUTE | KW_STATISTICS | KW_USE | KW_OPTION | KW_CONCATENATE | KW_SHOW_DATABASE | KW_UPDATE | KW_RESTRICT | KW_CASCADE | KW_SKEWED | KW_ROLLUP | KW_CUBE | KW_DIRECTORIES | KW_FOR | KW_GROUPING | KW_SETS | KW_TRUNCATE | KW_NOSCAN | KW_USER | KW_ROLE | KW_ROLES | KW_INNER | KW_DEFINED | KW_ADMIN
;
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java Tue Jan 28 05:48:03 2014
@@ -129,9 +129,11 @@ public class ImportSemanticAnalyzer exte
}
List<Partition> partitions = rv.getValue();
for (Partition partition : partitions) {
- AddPartitionDesc partDesc = new AddPartitionDesc(dbname, tblDesc.getTableName(),
+ // TODO: this should not create AddPartitionDesc per partition
+ AddPartitionDesc partsDesc = new AddPartitionDesc(dbname, tblDesc.getTableName(),
EximUtil.makePartSpec(tblDesc.getPartCols(), partition.getValues()),
partition.getSd().getLocation(), partition.getParameters());
+ AddPartitionDesc.OnePartitionDesc partDesc = partsDesc.getPartition(0);
partDesc.setInputFormat(partition.getSd().getInputFormat());
partDesc.setOutputFormat(partition.getSd().getOutputFormat());
partDesc.setNumBuckets(partition.getSd().getNumBuckets());
@@ -142,7 +144,7 @@ public class ImportSemanticAnalyzer exte
partDesc.setSortCols(partition.getSd().getSortCols());
partDesc.setLocation(new Path(fromPath,
Warehouse.makePartName(tblDesc.getPartCols(), partition.getValues())).toString());
- partitionDescs.add(partDesc);
+ partitionDescs.add(partsDesc);
}
} catch (IOException e) {
throw new SemanticException(ErrorMsg.INVALID_PATH.getMsg(), e);
@@ -188,7 +190,7 @@ public class ImportSemanticAnalyzer exte
for (Iterator<AddPartitionDesc> partnIter = partitionDescs
.listIterator(); partnIter.hasNext();) {
AddPartitionDesc addPartitionDesc = partnIter.next();
- if (!found && addPartitionDesc.getPartSpec().equals(partSpec)) {
+ if (!found && addPartitionDesc.getPartition(0).getPartSpec().equals(partSpec)) {
found = true;
} else {
partnIter.remove();
@@ -221,12 +223,12 @@ public class ImportSemanticAnalyzer exte
if (table.isPartitioned()) {
LOG.debug("table partitioned");
for (AddPartitionDesc addPartitionDesc : partitionDescs) {
- if (db.getPartition(table, addPartitionDesc.getPartSpec(), false) == null) {
+ Map<String, String> partSpec = addPartitionDesc.getPartition(0).getPartSpec();
+ if (db.getPartition(table, partSpec, false) == null) {
rootTasks.add(addSinglePartition(fromURI, fs, tblDesc, table, wh, addPartitionDesc));
} else {
throw new SemanticException(
- ErrorMsg.PARTITION_EXISTS
- .getMsg(partSpecToString(addPartitionDesc.getPartSpec())));
+ ErrorMsg.PARTITION_EXISTS.getMsg(partSpecToString(partSpec)));
}
}
} else {
@@ -281,7 +283,7 @@ public class ImportSemanticAnalyzer exte
private Task<?> loadTable(URI fromURI, Table table) {
Path dataPath = new Path(fromURI.toString(), "data");
- Path tmpPath = new Path(ctx.getExternalTmpFileURI(fromURI));
+ Path tmpPath = ctx.getExternalTmpPath(fromURI);
Task<?> copyTask = TaskFactory.get(new CopyWork(dataPath,
tmpPath, false), conf);
LoadTableDesc loadTableWork = new LoadTableDesc(tmpPath,
@@ -297,42 +299,43 @@ public class ImportSemanticAnalyzer exte
private Task<?> addSinglePartition(URI fromURI, FileSystem fs, CreateTableDesc tblDesc,
Table table, Warehouse wh,
AddPartitionDesc addPartitionDesc) throws MetaException, IOException, HiveException {
+ AddPartitionDesc.OnePartitionDesc partSpec = addPartitionDesc.getPartition(0);
if (tblDesc.isExternal() && tblDesc.getLocation() == null) {
LOG.debug("Importing in-place: adding AddPart for partition "
- + partSpecToString(addPartitionDesc.getPartSpec()));
+ + partSpecToString(partSpec.getPartSpec()));
// addPartitionDesc already has the right partition location
Task<?> addPartTask = TaskFactory.get(new DDLWork(getInputs(),
getOutputs(), addPartitionDesc), conf);
return addPartTask;
} else {
- String srcLocation = addPartitionDesc.getLocation();
+ String srcLocation = partSpec.getLocation();
Path tgtPath = null;
if (tblDesc.getLocation() == null) {
if (table.getDataLocation() != null) {
tgtPath = new Path(table.getDataLocation().toString(),
- Warehouse.makePartPath(addPartitionDesc.getPartSpec()));
+ Warehouse.makePartPath(partSpec.getPartSpec()));
} else {
tgtPath = new Path(wh.getTablePath(
db.getDatabaseCurrent(), tblDesc.getTableName()),
- Warehouse.makePartPath(addPartitionDesc.getPartSpec()));
+ Warehouse.makePartPath(partSpec.getPartSpec()));
}
} else {
tgtPath = new Path(tblDesc.getLocation(),
- Warehouse.makePartPath(addPartitionDesc.getPartSpec()));
+ Warehouse.makePartPath(partSpec.getPartSpec()));
}
checkTargetLocationEmpty(fs, tgtPath);
- addPartitionDesc.setLocation(tgtPath.toString());
+ partSpec.setLocation(tgtPath.toString());
LOG.debug("adding dependent CopyWork/AddPart/MoveWork for partition "
- + partSpecToString(addPartitionDesc.getPartSpec())
+ + partSpecToString(partSpec.getPartSpec())
+ " with source location: " + srcLocation);
- Path tmpPath = new Path(ctx.getExternalTmpFileURI(fromURI));
+ Path tmpPath = ctx.getExternalTmpPath(fromURI);
Task<?> copyTask = TaskFactory.get(new CopyWork(new Path(srcLocation),
tmpPath, false), conf);
Task<?> addPartTask = TaskFactory.get(new DDLWork(getInputs(),
getOutputs(), addPartitionDesc), conf);
LoadTableDesc loadTableWork = new LoadTableDesc(tmpPath,
Utilities.getTableDesc(table),
- addPartitionDesc.getPartSpec(), true);
+ partSpec.getPartSpec(), true);
loadTableWork.setInheritTableSpecs(false);
Task<?> loadPartTask = TaskFactory.get(new MoveWork(
getInputs(), getOutputs(), loadTableWork, null, false),
@@ -399,7 +402,7 @@ public class ImportSemanticAnalyzer exte
if (tableDesc.getLocation() != null) { // IMPORT statement specified
// location
if (!table.getDataLocation()
- .equals(new URI(tableDesc.getLocation()))) {
+ .equals(new Path(tableDesc.getLocation()))) {
throw new SemanticException(
ErrorMsg.INCOMPATIBLE_SCHEMA.getMsg(" Location does not match"));
}
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java Tue Jan 28 05:48:03 2014
@@ -206,8 +206,8 @@ public class LoadSemanticAnalyzer extend
throw new SemanticException(ErrorMsg.LOAD_INTO_STORED_AS_DIR.getMsg());
}
- URI toURI = (ts.partHandle != null) ? ts.partHandle.getDataLocation()
- : ts.tableHandle.getDataLocation();
+ URI toURI = ((ts.partHandle != null) ? ts.partHandle.getDataLocation()
+ : ts.tableHandle.getDataLocation()).toUri();
List<FieldSchema> parts = ts.tableHandle.getPartitionKeys();
if ((parts != null && parts.size() > 0)
@@ -226,7 +226,7 @@ public class LoadSemanticAnalyzer extend
// might seem redundant in the case
// that the hive warehouse is also located in the local file system - but
// that's just a test case.
- String copyURIStr = ctx.getExternalTmpFileURI(toURI);
+ String copyURIStr = ctx.getExternalTmpPath(toURI).toString();
URI copyURI = URI.create(copyURIStr);
rTask = TaskFactory.get(new CopyWork(new Path(fromURI), new Path(copyURI)), conf);
fromURI = copyURI;
@@ -234,7 +234,6 @@ public class LoadSemanticAnalyzer extend
// create final load/move work
- String loadTmpPath = ctx.getExternalTmpFileURI(toURI);
Map<String, String> partSpec = ts.getPartSpec();
if (partSpec == null) {
partSpec = new LinkedHashMap<String, String>();
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java Tue Jan 28 05:48:03 2014
@@ -345,6 +345,9 @@ public class PTFTranslator {
private WindowFunctionDef translate(WindowTableFunctionDef wdwTFnDef,
WindowFunctionSpec spec) throws SemanticException {
WindowFunctionInfo wFnInfo = FunctionRegistry.getWindowFunctionInfo(spec.getName());
+ if (wFnInfo == null) {
+ throw new SemanticException(ErrorMsg.INVALID_FUNCTION.getMsg(spec.getName()));
+ }
WindowFunctionDef def = new WindowFunctionDef();
def.setName(spec.getName());
def.setAlias(spec.getAlias());
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Tue Jan 28 05:48:03 2014
@@ -181,7 +181,9 @@ import org.apache.hadoop.hive.serde2.typ
import org.apache.hadoop.mapred.InputFormat;
/**
- * Implementation of the semantic analyzer.
+ * Implementation of the semantic analyzer. It generates the query plan.
+ * There are other specific semantic analyzers for some hive operations such as
+ * DDLSemanticAnalyzer for ddl operations.
*/
public class SemanticAnalyzer extends BaseSemanticAnalyzer {
@@ -967,10 +969,9 @@ public class SemanticAnalyzer extends Ba
break;
case HiveParser.TOK_UNION:
- // currently, we dont support subq1 union subq2 - the user has to
- // explicitly say:
- // select * from (subq1 union subq2) subqalias
if (!qbp.getIsSubQ()) {
+ // this shouldn't happen. The parser should have converted the union to be
+ // contained in a subquery. Just in case, we keep the error as a fallback.
throw new SemanticException(generateErrorMessage(ast,
ErrorMsg.UNION_NOTIN_SUBQ.getMsg()));
}
@@ -1021,7 +1022,7 @@ public class SemanticAnalyzer extends Ba
} catch (HiveException e) {
LOG.info("Error while getting metadata : ", e);
}
- validatePartSpec(table, partition, (ASTNode)tab, conf);
+ validatePartSpec(table, partition, (ASTNode)tab, conf, false);
}
skipRecursion = false;
break;
@@ -1263,8 +1264,8 @@ public class SemanticAnalyzer extends Ba
throw new SemanticException(e);
}
try {
- fname = ctx.getExternalTmpFileURI(
- FileUtils.makeQualified(location, conf).toUri());
+ fname = ctx.getExternalTmpPath(
+ FileUtils.makeQualified(location, conf).toUri()).toString();
} catch (Exception e) {
throw new SemanticException(generateErrorMessage(ast,
"Error creating temporary folder on: " + location.toString()), e);
@@ -1278,7 +1279,7 @@ public class SemanticAnalyzer extends Ba
}
} else {
qb.setIsQuery(true);
- fname = ctx.getMRTmpFileURI();
+ fname = ctx.getMRTmpPath().toString();
ctx.setResDir(new Path(fname));
}
}
@@ -5264,7 +5265,7 @@ public class SemanticAnalyzer extends Ba
Table dest_tab = null; // destination table if any
Partition dest_part = null;// destination partition if any
- String queryTmpdir = null; // the intermediate destination directory
+ Path queryTmpdir = null; // the intermediate destination directory
Path dest_path = null; // the final destination directory
TableDesc table_desc = null;
int currentTableId = 0;
@@ -5307,7 +5308,7 @@ public class SemanticAnalyzer extends Ba
}
dpCtx = qbm.getDPCtx(dest);
if (dpCtx == null) {
- Utilities.validatePartSpecColumnNames(dest_tab, partSpec);
+ dest_tab.validatePartColumnNames(partSpec, false);
dpCtx = new DynamicPartitionCtx(dest_tab, partSpec,
conf.getVar(HiveConf.ConfVars.DEFAULTPARTITIONNAME),
conf.getIntVar(HiveConf.ConfVars.DYNAMICPARTITIONMAXPARTSPERNODE));
@@ -5330,9 +5331,9 @@ public class SemanticAnalyzer extends Ba
boolean isNonNativeTable = dest_tab.isNonNative();
if (isNonNativeTable) {
- queryTmpdir = dest_path.toUri().getPath();
+ queryTmpdir = dest_path;
} else {
- queryTmpdir = ctx.getExternalTmpFileURI(dest_path.toUri());
+ queryTmpdir = ctx.getExternalTmpPath(dest_path.toUri());
}
if (dpCtx != null) {
// set the root of the temporay path where dynamic partition columns will populate
@@ -5355,7 +5356,7 @@ public class SemanticAnalyzer extends Ba
// Create the work for moving the table
// NOTE: specify Dynamic partitions in dest_tab for WriteEntity
if (!isNonNativeTable) {
- ltd = new LoadTableDesc(new Path(queryTmpdir),table_desc, dpCtx);
+ ltd = new LoadTableDesc(queryTmpdir,table_desc, dpCtx);
ltd.setReplace(!qb.getParseInfo().isInsertIntoTable(dest_tab.getDbName(),
dest_tab.getTableName()));
ltd.setLbCtx(lbCtx);
@@ -5418,14 +5419,14 @@ public class SemanticAnalyzer extends Ba
}
Path tabPath = dest_tab.getPath();
- Path partPath = dest_part.getPartitionPath();
+ Path partPath = dest_part.getDataLocation();
// if the table is in a different dfs than the partition,
// replace the partition's dfs with the table's dfs.
dest_path = new Path(tabPath.toUri().getScheme(), tabPath.toUri()
.getAuthority(), partPath.toUri().getPath());
- queryTmpdir = ctx.getExternalTmpFileURI(dest_path.toUri());
+ queryTmpdir = ctx.getExternalTmpPath(dest_path.toUri());
table_desc = Utilities.getTableDesc(dest_tab);
// Add sorting/bucketing if needed
@@ -5438,7 +5439,7 @@ public class SemanticAnalyzer extends Ba
lbCtx = constructListBucketingCtx(dest_part.getSkewedColNames(),
dest_part.getSkewedColValues(), dest_part.getSkewedColValueLocationMaps(),
dest_part.isStoredAsSubDirectories(), conf);
- ltd = new LoadTableDesc(new Path(queryTmpdir), table_desc, dest_part.getSpec());
+ ltd = new LoadTableDesc(queryTmpdir, table_desc, dest_part.getSpec());
ltd.setReplace(!qb.getParseInfo().isInsertIntoTable(dest_tab.getDbName(),
dest_tab.getTableName()));
ltd.setLbCtx(lbCtx);
@@ -5469,18 +5470,18 @@ public class SemanticAnalyzer extends Ba
// fall through
case QBMetaData.DEST_DFS_FILE: {
dest_path = new Path(qbm.getDestFileForAlias(dest));
-
+
if (isLocal) {
// for local directory - we always write to map-red intermediate
// store and then copy to local fs
- queryTmpdir = ctx.getMRTmpFileURI();
+ queryTmpdir = ctx.getMRTmpPath();
} else {
// otherwise write to the file system implied by the directory
// no copy is required. we may want to revisit this policy in future
try {
Path qPath = FileUtils.makeQualified(dest_path, conf);
- queryTmpdir = ctx.getExternalTmpFileURI(qPath.toUri());
+ queryTmpdir = ctx.getExternalTmpPath(qPath.toUri());
} catch (Exception e) {
throw new SemanticException("Error creating temporary folder on: "
+ dest_path, e);
@@ -5558,7 +5559,7 @@ public class SemanticAnalyzer extends Ba
}
boolean isDfsDir = (dest_type.intValue() == QBMetaData.DEST_DFS_FILE);
- loadFileWork.add(new LoadFileDesc(tblDesc, new Path(queryTmpdir), dest_path, isDfsDir, cols,
+ loadFileWork.add(new LoadFileDesc(tblDesc, queryTmpdir, dest_path, isDfsDir, cols,
colTypes));
if (tblDesc == null) {
@@ -5638,7 +5639,7 @@ public class SemanticAnalyzer extends Ba
// the same as directory name. The directory name
// can be changed in the optimizer but the key should not be changed
// it should be the same as the MoveWork's sourceDir.
- fileSinkDesc.setStatsAggPrefix(fileSinkDesc.getDirName());
+ fileSinkDesc.setStatsAggPrefix(fileSinkDesc.getDirName().toString());
if (dest_part != null) {
try {