You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by th...@apache.org on 2013/09/14 01:55:40 UTC
svn commit: r1523148 [2/2] - in /hive/branches/branch-0.12: metastore/if/
metastore/src/gen/thrift/gen-cpp/
metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/
metastore/src/gen/thrift/gen-php/metastore/ metastore/src/gen/thrift...
Modified: hive/branches/branch-0.12/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java?rev=1523148&r1=1523147&r2=1523148&view=diff
==============================================================================
--- hive/branches/branch-0.12/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java (original)
+++ hive/branches/branch-0.12/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java Fri Sep 13 23:55:39 2013
@@ -76,7 +76,6 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.metastore.api.Role;
import org.apache.hadoop.hive.metastore.api.SerDeInfo;
import org.apache.hadoop.hive.metastore.api.SkewedInfo;
-import org.apache.hadoop.hive.metastore.api.SkewedValueList;
import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.index.HiveIndexHandler;
@@ -1234,8 +1233,8 @@ public class Hive {
org.apache.hadoop.hive.metastore.api.Partition newCreatedTpart = newTPart.getTPartition();
SkewedInfo skewedInfo = newCreatedTpart.getSd().getSkewedInfo();
/* Construct list bucketing location mappings from sub-directory name. */
- Map<SkewedValueList, String> skewedColValueLocationMaps =
- constructListBucketingLocationMap(newPartPath, skewedInfo);
+ Map<List<String>, String> skewedColValueLocationMaps = constructListBucketingLocationMap(
+ newPartPath, skewedInfo);
/* Add list bucketing location mappings. */
skewedInfo.setSkewedColValueLocationMaps(skewedColValueLocationMaps);
newCreatedTpart.getSd().setSkewedInfo(skewedInfo);
@@ -1268,8 +1267,7 @@ public class Hive {
* @throws IOException
*/
private void walkDirTree(FileStatus fSta, FileSystem fSys,
- Map<SkewedValueList, String> skewedColValueLocationMaps,
- Path newPartPath, SkewedInfo skewedInfo)
+ Map<List<String>, String> skewedColValueLocationMaps, Path newPartPath, SkewedInfo skewedInfo)
throws IOException {
/* Base Case. It's leaf. */
if (!fSta.isDir()) {
@@ -1295,7 +1293,7 @@ private void walkDirTree(FileStatus fSta
* @param skewedInfo
*/
private void constructOneLBLocationMap(FileStatus fSta,
- Map<SkewedValueList, String> skewedColValueLocationMaps,
+ Map<List<String>, String> skewedColValueLocationMaps,
Path newPartPath, SkewedInfo skewedInfo) {
Path lbdPath = fSta.getPath().getParent();
List<String> skewedValue = new ArrayList<String>();
@@ -1318,7 +1316,7 @@ private void constructOneLBLocationMap(F
}
if ((skewedValue.size() > 0) && (skewedValue.size() == skewedInfo.getSkewedColNames().size())
&& !skewedColValueLocationMaps.containsKey(skewedValue)) {
- skewedColValueLocationMaps.put(new SkewedValueList(skewedValue), lbdPath.toString());
+ skewedColValueLocationMaps.put(skewedValue, lbdPath.toString());
}
}
@@ -1331,10 +1329,9 @@ private void constructOneLBLocationMap(F
* @throws IOException
* @throws FileNotFoundException
*/
- private Map<SkewedValueList, String> constructListBucketingLocationMap(Path newPartPath,
+ private Map<List<String>, String> constructListBucketingLocationMap(Path newPartPath,
SkewedInfo skewedInfo) throws IOException, FileNotFoundException {
- Map<SkewedValueList, String> skewedColValueLocationMaps =
- new HashMap<SkewedValueList, String>();
+ Map<List<String>, String> skewedColValueLocationMaps = new HashMap<List<String>, String>();
FileSystem fSys = newPartPath.getFileSystem(conf);
walkDirTree(fSys.getFileStatus(newPartPath), fSys, skewedColValueLocationMaps, newPartPath,
skewedInfo);
Modified: hive/branches/branch-0.12/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java?rev=1523148&r1=1523147&r2=1523148&view=diff
==============================================================================
--- hive/branches/branch-0.12/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java (original)
+++ hive/branches/branch-0.12/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java Fri Sep 13 23:55:39 2013
@@ -40,7 +40,6 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.Order;
-import org.apache.hadoop.hive.metastore.api.SkewedValueList;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils;
@@ -661,18 +660,18 @@ public class Partition implements Serial
public void setSkewedValueLocationMap(List<String> valList, String dirName)
throws HiveException {
- Map<SkewedValueList, String> mappings = tPartition.getSd().getSkewedInfo()
+ Map<List<String>, String> mappings = tPartition.getSd().getSkewedInfo()
.getSkewedColValueLocationMaps();
if (null == mappings) {
- mappings = new HashMap<SkewedValueList, String>();
+ mappings = new HashMap<List<String>, String>();
tPartition.getSd().getSkewedInfo().setSkewedColValueLocationMaps(mappings);
}
// Add or update new mapping
- mappings.put(new SkewedValueList(valList), dirName);
+ mappings.put(valList, dirName);
}
- public Map<SkewedValueList, String> getSkewedColValueLocationMaps() {
+ public Map<List<String>, String> getSkewedColValueLocationMaps() {
return tPartition.getSd().getSkewedInfo().getSkewedColValueLocationMaps();
}
}
Modified: hive/branches/branch-0.12/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java?rev=1523148&r1=1523147&r2=1523148&view=diff
==============================================================================
--- hive/branches/branch-0.12/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java (original)
+++ hive/branches/branch-0.12/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java Fri Sep 13 23:55:39 2013
@@ -45,7 +45,6 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.metastore.api.Order;
import org.apache.hadoop.hive.metastore.api.SerDeInfo;
import org.apache.hadoop.hive.metastore.api.SkewedInfo;
-import org.apache.hadoop.hive.metastore.api.SkewedValueList;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils;
import org.apache.hadoop.hive.ql.io.HiveOutputFormat;
@@ -145,7 +144,7 @@ public class Table implements Serializab
SkewedInfo skewInfo = new SkewedInfo();
skewInfo.setSkewedColNames(new ArrayList<String>());
skewInfo.setSkewedColValues(new ArrayList<List<String>>());
- skewInfo.setSkewedColValueLocationMaps(new HashMap<SkewedValueList, String>());
+ skewInfo.setSkewedColValueLocationMaps(new HashMap<List<String>, String>());
sd.setSkewedInfo(skewInfo);
}
@@ -523,20 +522,20 @@ public class Table implements Serializab
public void setSkewedValueLocationMap(List<String> valList, String dirName)
throws HiveException {
- Map<SkewedValueList, String> mappings = tTable.getSd().getSkewedInfo()
+ Map<List<String>, String> mappings = tTable.getSd().getSkewedInfo()
.getSkewedColValueLocationMaps();
if (null == mappings) {
- mappings = new HashMap<SkewedValueList, String>();
+ mappings = new HashMap<List<String>, String>();
tTable.getSd().getSkewedInfo().setSkewedColValueLocationMaps(mappings);
}
// Add or update new mapping
- mappings.put(new SkewedValueList(valList), dirName);
+ mappings.put(valList, dirName);
}
- public Map<SkewedValueList,String> getSkewedColValueLocationMaps() {
+ public Map<List<String>,String> getSkewedColValueLocationMaps() {
return (tTable.getSd().getSkewedInfo() != null) ? tTable.getSd().getSkewedInfo()
- .getSkewedColValueLocationMaps() : new HashMap<SkewedValueList, String>();
+ .getSkewedColValueLocationMaps() : new HashMap<List<String>, String>();
}
public void setSkewedColValues(List<List<String>> skewedValues) throws HiveException {
Modified: hive/branches/branch-0.12/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java?rev=1523148&r1=1523147&r2=1523148&view=diff
==============================================================================
--- hive/branches/branch-0.12/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java (original)
+++ hive/branches/branch-0.12/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java Fri Sep 13 23:55:39 2013
@@ -32,7 +32,6 @@ import org.apache.hadoop.hive.conf.HiveC
import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Index;
-import org.apache.hadoop.hive.metastore.api.SkewedValueList;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.ql.index.HiveIndex;
import org.apache.hadoop.hive.ql.index.HiveIndex.IndexType;
@@ -201,7 +200,7 @@ public final class MetaDataFormatUtils {
formatOutput("Skewed Values:", skewedColValues.toString(), tableInfo);
}
- Map<SkewedValueList, String> skewedColMap = storageDesc.getSkewedInfo()
+ Map<List<String>, String> skewedColMap = storageDesc.getSkewedInfo()
.getSkewedColValueLocationMaps();
if ((skewedColMap!=null) && (skewedColMap.size() > 0)) {
formatOutput("Skewed Value to Path:", skewedColMap.toString(),
@@ -209,8 +208,9 @@ public final class MetaDataFormatUtils {
Map<List<String>, String> truncatedSkewedColMap = new HashMap<List<String>, String>();
// walk through existing map to truncate path so that test won't mask it
// then we can verify location is right
- for (Entry<SkewedValueList, String> entry : skewedColMap.entrySet()) {
- truncatedSkewedColMap.put(entry.getKey().getSkewedValueList(),
+ Set<Entry<List<String>, String>> entries = skewedColMap.entrySet();
+ for (Entry<List<String>, String> entry : entries) {
+ truncatedSkewedColMap.put(entry.getKey(),
PlanUtils.removePrefixFromWarehouseConfig(entry.getValue()));
}
formatOutput("Skewed Value to Truncated Path:",
Modified: hive/branches/branch-0.12/ql/src/java/org/apache/hadoop/hive/ql/optimizer/listbucketingpruner/ListBucketingPruner.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/ql/src/java/org/apache/hadoop/hive/ql/optimizer/listbucketingpruner/ListBucketingPruner.java?rev=1523148&r1=1523147&r2=1523148&view=diff
==============================================================================
--- hive/branches/branch-0.12/ql/src/java/org/apache/hadoop/hive/ql/optimizer/listbucketingpruner/ListBucketingPruner.java (original)
+++ hive/branches/branch-0.12/ql/src/java/org/apache/hadoop/hive/ql/optimizer/listbucketingpruner/ListBucketingPruner.java Fri Sep 13 23:55:39 2013
@@ -27,7 +27,6 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.FileUtils;
-import org.apache.hadoop.hive.metastore.api.SkewedValueList;
import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
import org.apache.hadoop.hive.ql.metadata.Partition;
import org.apache.hadoop.hive.ql.optimizer.PrunerUtils;
@@ -289,13 +288,12 @@ public class ListBucketingPruner impleme
List<List<String>> uniqSkewedValues) throws SemanticException {
// For each entry in dynamic-multi-dimension collection.
List<String> skewedCols = part.getSkewedColNames(); // Retrieve skewed column.
- Map<SkewedValueList, String> mappings = part.getSkewedColValueLocationMaps(); // Retrieve skewed
+ Map<List<String>, String> mappings = part.getSkewedColValueLocationMaps(); // Retrieve skewed
// map.
assert ListBucketingPrunerUtils.isListBucketingPart(part) : part.getName()
+ " skewed metadata is corrupted. No skewed column and/or location mappings information.";
List<List<String>> skewedValues = part.getSkewedColValues();
List<Boolean> nonSkewedValueMatchResult = new ArrayList<Boolean>();
- SkewedValueList skewedValueList = new SkewedValueList();
for (List<String> cell : collections) {
// Walk through the tree to decide value.
// Example: skewed column: C1, C2 ;
@@ -309,9 +307,8 @@ public class ListBucketingPruner impleme
/* It's valid case if a partition: */
/* 1. is defined with skewed columns and skewed values in metadata */
/* 2. doesn't have all skewed values within its data */
- skewedValueList.setSkewedValueList(cell);
- if (mappings.get(skewedValueList) != null) {
- selectedPaths.add(new Path(mappings.get(skewedValueList)));
+ if (mappings.get(cell) != null) {
+ selectedPaths.add(new Path(mappings.get(cell)));
}
}
} else {
Modified: hive/branches/branch-0.12/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java?rev=1523148&r1=1523147&r2=1523148&view=diff
==============================================================================
--- hive/branches/branch-0.12/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java (original)
+++ hive/branches/branch-0.12/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java Fri Sep 13 23:55:39 2013
@@ -37,7 +37,6 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Order;
-import org.apache.hadoop.hive.metastore.api.SkewedValueList;
import org.apache.hadoop.hive.ql.Context;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.QueryProperties;
@@ -981,31 +980,18 @@ public abstract class BaseSemanticAnalyz
* @return
*/
protected ListBucketingCtx constructListBucketingCtx(List<String> skewedColNames,
- List<List<String>> skewedValues, Map<SkewedValueList, String> skewedColValueLocationMaps,
+ List<List<String>> skewedValues, Map<List<String>, String> skewedColValueLocationMaps,
boolean isStoredAsSubDirectories, HiveConf conf) {
ListBucketingCtx lbCtx = new ListBucketingCtx();
lbCtx.setSkewedColNames(skewedColNames);
lbCtx.setSkewedColValues(skewedValues);
- lbCtx.setLbLocationMap(convertSkewedValueListToSimpleList(skewedColValueLocationMaps));
+ lbCtx.setLbLocationMap(skewedColValueLocationMaps);
lbCtx.setStoredAsSubDirectories(isStoredAsSubDirectories);
lbCtx.setDefaultKey(ListBucketingPrunerUtils.HIVE_LIST_BUCKETING_DEFAULT_KEY);
lbCtx.setDefaultDirName(ListBucketingPrunerUtils.HIVE_LIST_BUCKETING_DEFAULT_DIR_NAME);
return lbCtx;
}
- // This is done to avoid the need of sending metastore jars to task nodes.
- private Map<List<String>, String> convertSkewedValueListToSimpleList(
- Map<SkewedValueList, String> skewedColValueLocationMaps) {
- if (skewedColValueLocationMaps == null) {
- return null;
- }
- Map<List<String>, String> converted = new HashMap<List<String>, String>();
- for (Map.Entry<SkewedValueList, String> entry : skewedColValueLocationMaps.entrySet()) {
- converted.put(entry.getKey().getSkewedValueList(), entry.getValue());
- }
- return converted;
- }
-
/**
* Given a ASTNode, return list of values.
*