You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ga...@apache.org on 2013/04/13 00:10:33 UTC
svn commit: r1467491 [2/2] - in /hive/trunk: metastore/if/
metastore/src/gen/thrift/gen-cpp/
metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/
metastore/src/gen/thrift/gen-php/metastore/
metastore/src/gen/thrift/gen-py/hive_me...
Modified: hive/trunk/metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py?rev=1467491&r1=1467490&r2=1467491&view=diff
==============================================================================
--- hive/trunk/metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py (original)
+++ hive/trunk/metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py Fri Apr 12 22:10:31 2013
@@ -1232,6 +1232,74 @@ class Order:
def __ne__(self, other):
return not (self == other)
+class SkewedValueList:
+ """
+ Attributes:
+ - skewedValueList
+ """
+
+ thrift_spec = (
+ None, # 0
+ (1, TType.LIST, 'skewedValueList', (TType.STRING,None), None, ), # 1
+ )
+
+ def __init__(self, skewedValueList=None,):
+ self.skewedValueList = skewedValueList
+
+ def read(self, iprot):
+ if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+ fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+ return
+ iprot.readStructBegin()
+ while True:
+ (fname, ftype, fid) = iprot.readFieldBegin()
+ if ftype == TType.STOP:
+ break
+ if fid == 1:
+ if ftype == TType.LIST:
+ self.skewedValueList = []
+ (_etype90, _size87) = iprot.readListBegin()
+ for _i91 in xrange(_size87):
+ _elem92 = iprot.readString();
+ self.skewedValueList.append(_elem92)
+ iprot.readListEnd()
+ else:
+ iprot.skip(ftype)
+ else:
+ iprot.skip(ftype)
+ iprot.readFieldEnd()
+ iprot.readStructEnd()
+
+ def write(self, oprot):
+ if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+ oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+ return
+ oprot.writeStructBegin('SkewedValueList')
+ if self.skewedValueList is not None:
+ oprot.writeFieldBegin('skewedValueList', TType.LIST, 1)
+ oprot.writeListBegin(TType.STRING, len(self.skewedValueList))
+ for iter93 in self.skewedValueList:
+ oprot.writeString(iter93)
+ oprot.writeListEnd()
+ oprot.writeFieldEnd()
+ oprot.writeFieldStop()
+ oprot.writeStructEnd()
+
+ def validate(self):
+ return
+
+
+ def __repr__(self):
+ L = ['%s=%r' % (key, value)
+ for key, value in self.__dict__.iteritems()]
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+
+ def __eq__(self, other):
+ return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+
+ def __ne__(self, other):
+ return not (self == other)
+
class SkewedInfo:
"""
Attributes:
@@ -1244,7 +1312,7 @@ class SkewedInfo:
None, # 0
(1, TType.LIST, 'skewedColNames', (TType.STRING,None), None, ), # 1
(2, TType.LIST, 'skewedColValues', (TType.LIST,(TType.STRING,None)), None, ), # 2
- (3, TType.MAP, 'skewedColValueLocationMaps', (TType.LIST,(TType.STRING,None),TType.STRING,None), None, ), # 3
+ (3, TType.MAP, 'skewedColValueLocationMaps', (TType.STRUCT,(SkewedValueList, SkewedValueList.thrift_spec),TType.STRING,None), None, ), # 3
)
def __init__(self, skewedColNames=None, skewedColValues=None, skewedColValueLocationMaps=None,):
@@ -1264,41 +1332,37 @@ class SkewedInfo:
if fid == 1:
if ftype == TType.LIST:
self.skewedColNames = []
- (_etype90, _size87) = iprot.readListBegin()
- for _i91 in xrange(_size87):
- _elem92 = iprot.readString();
- self.skewedColNames.append(_elem92)
+ (_etype97, _size94) = iprot.readListBegin()
+ for _i98 in xrange(_size94):
+ _elem99 = iprot.readString();
+ self.skewedColNames.append(_elem99)
iprot.readListEnd()
else:
iprot.skip(ftype)
elif fid == 2:
if ftype == TType.LIST:
self.skewedColValues = []
- (_etype96, _size93) = iprot.readListBegin()
- for _i97 in xrange(_size93):
- _elem98 = []
- (_etype102, _size99) = iprot.readListBegin()
- for _i103 in xrange(_size99):
- _elem104 = iprot.readString();
- _elem98.append(_elem104)
+ (_etype103, _size100) = iprot.readListBegin()
+ for _i104 in xrange(_size100):
+ _elem105 = []
+ (_etype109, _size106) = iprot.readListBegin()
+ for _i110 in xrange(_size106):
+ _elem111 = iprot.readString();
+ _elem105.append(_elem111)
iprot.readListEnd()
- self.skewedColValues.append(_elem98)
+ self.skewedColValues.append(_elem105)
iprot.readListEnd()
else:
iprot.skip(ftype)
elif fid == 3:
if ftype == TType.MAP:
self.skewedColValueLocationMaps = {}
- (_ktype106, _vtype107, _size105 ) = iprot.readMapBegin()
- for _i109 in xrange(_size105):
- _key110 = []
- (_etype115, _size112) = iprot.readListBegin()
- for _i116 in xrange(_size112):
- _elem117 = iprot.readString();
- _key110.append(_elem117)
- iprot.readListEnd()
- _val111 = iprot.readString();
- self.skewedColValueLocationMaps[_key110] = _val111
+ (_ktype113, _vtype114, _size112 ) = iprot.readMapBegin()
+ for _i116 in xrange(_size112):
+ _key117 = SkewedValueList()
+ _key117.read(iprot)
+ _val118 = iprot.readString();
+ self.skewedColValueLocationMaps[_key117] = _val118
iprot.readMapEnd()
else:
iprot.skip(ftype)
@@ -1315,29 +1379,26 @@ class SkewedInfo:
if self.skewedColNames is not None:
oprot.writeFieldBegin('skewedColNames', TType.LIST, 1)
oprot.writeListBegin(TType.STRING, len(self.skewedColNames))
- for iter118 in self.skewedColNames:
- oprot.writeString(iter118)
+ for iter119 in self.skewedColNames:
+ oprot.writeString(iter119)
oprot.writeListEnd()
oprot.writeFieldEnd()
if self.skewedColValues is not None:
oprot.writeFieldBegin('skewedColValues', TType.LIST, 2)
oprot.writeListBegin(TType.LIST, len(self.skewedColValues))
- for iter119 in self.skewedColValues:
- oprot.writeListBegin(TType.STRING, len(iter119))
- for iter120 in iter119:
- oprot.writeString(iter120)
+ for iter120 in self.skewedColValues:
+ oprot.writeListBegin(TType.STRING, len(iter120))
+ for iter121 in iter120:
+ oprot.writeString(iter121)
oprot.writeListEnd()
oprot.writeListEnd()
oprot.writeFieldEnd()
if self.skewedColValueLocationMaps is not None:
oprot.writeFieldBegin('skewedColValueLocationMaps', TType.MAP, 3)
- oprot.writeMapBegin(TType.LIST, TType.STRING, len(self.skewedColValueLocationMaps))
- for kiter121,viter122 in self.skewedColValueLocationMaps.items():
- oprot.writeListBegin(TType.STRING, len(kiter121))
- for iter123 in kiter121:
- oprot.writeString(iter123)
- oprot.writeListEnd()
- oprot.writeString(viter122)
+ oprot.writeMapBegin(TType.STRUCT, TType.STRING, len(self.skewedColValueLocationMaps))
+ for kiter122,viter123 in self.skewedColValueLocationMaps.items():
+ kiter122.write(oprot)
+ oprot.writeString(viter123)
oprot.writeMapEnd()
oprot.writeFieldEnd()
oprot.writeFieldStop()
Modified: hive/trunk/metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb?rev=1467491&r1=1467490&r2=1467491&view=diff
==============================================================================
--- hive/trunk/metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb (original)
+++ hive/trunk/metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb Fri Apr 12 22:10:31 2013
@@ -289,6 +289,22 @@ class Order
::Thrift::Struct.generate_accessors self
end
+class SkewedValueList
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ SKEWEDVALUELIST = 1
+
+ FIELDS = {
+ SKEWEDVALUELIST => {:type => ::Thrift::Types::LIST, :name => 'skewedValueList', :element => {:type => ::Thrift::Types::STRING}}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ end
+
+ ::Thrift::Struct.generate_accessors self
+end
+
class SkewedInfo
include ::Thrift::Struct, ::Thrift::Struct_Union
SKEWEDCOLNAMES = 1
@@ -298,7 +314,7 @@ class SkewedInfo
FIELDS = {
SKEWEDCOLNAMES => {:type => ::Thrift::Types::LIST, :name => 'skewedColNames', :element => {:type => ::Thrift::Types::STRING}},
SKEWEDCOLVALUES => {:type => ::Thrift::Types::LIST, :name => 'skewedColValues', :element => {:type => ::Thrift::Types::LIST, :element => {:type => ::Thrift::Types::STRING}}},
- SKEWEDCOLVALUELOCATIONMAPS => {:type => ::Thrift::Types::MAP, :name => 'skewedColValueLocationMaps', :key => {:type => ::Thrift::Types::LIST, :element => {:type => ::Thrift::Types::STRING}}, :value => {:type => ::Thrift::Types::STRING}}
+ SKEWEDCOLVALUELOCATIONMAPS => {:type => ::Thrift::Types::MAP, :name => 'skewedColValueLocationMaps', :key => {:type => ::Thrift::Types::STRUCT, :class => ::SkewedValueList}, :value => {:type => ::Thrift::Types::STRING}}
}
def struct_fields; FIELDS; end
Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java?rev=1467491&r1=1467490&r2=1467491&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java (original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java Fri Apr 12 22:10:31 2013
@@ -85,6 +85,7 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.metastore.api.Role;
import org.apache.hadoop.hive.metastore.api.SerDeInfo;
import org.apache.hadoop.hive.metastore.api.SkewedInfo;
+import org.apache.hadoop.hive.metastore.api.SkewedValueList;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.metastore.api.StringColumnStatsData;
import org.apache.hadoop.hive.metastore.api.Table;
@@ -1054,13 +1055,13 @@ public class ObjectStore implements RawS
* @param mMap
* @return
*/
- private Map<List<String>, String> covertToSkewedMap(Map<MStringList, String> mMap) {
- Map<List<String>, String> map = null;
+ private Map<SkewedValueList, String> covertToSkewedMap(Map<MStringList, String> mMap) {
+ Map<SkewedValueList, String> map = null;
if (mMap != null) {
- map = new HashMap<List<String>, String>(mMap.size());
+ map = new HashMap<SkewedValueList, String>(mMap.size());
Set<MStringList> keys = mMap.keySet();
for (MStringList key : keys) {
- map.put(new ArrayList<String>(key.getInternalList()), mMap.get(key));
+ map.put(new SkewedValueList(new ArrayList<String>(key.getInternalList())), mMap.get(key));
}
}
return map;
@@ -1071,13 +1072,12 @@ public class ObjectStore implements RawS
* @param mMap
* @return
*/
- private Map<MStringList, String> covertToMapMStringList(Map<List<String>, String> mMap) {
+ private Map<MStringList, String> convertToMapMStringList(Map<SkewedValueList, String> mMap) {
Map<MStringList, String> map = null;
if (mMap != null) {
map = new HashMap<MStringList, String>(mMap.size());
- Set<List<String>> keys = mMap.keySet();
- for (List<String> key : keys) {
- map.put(new MStringList(key), mMap.get(key));
+ for (Map.Entry<SkewedValueList, String> entry : mMap.entrySet()) {
+ map.put(new MStringList(entry.getKey().getSkewedValueList()), entry.getValue());
}
}
return map;
@@ -1124,7 +1124,7 @@ public class ObjectStore implements RawS
: sd.getSkewedInfo().getSkewedColNames(),
convertToMStringLists((null == sd.getSkewedInfo()) ? null : sd.getSkewedInfo()
.getSkewedColValues()),
- covertToMapMStringList((null == sd.getSkewedInfo()) ? null : sd.getSkewedInfo()
+ convertToMapMStringList((null == sd.getSkewedInfo()) ? null : sd.getSkewedInfo()
.getSkewedColValueLocationMaps()), sd.isStoredAsSubDirectories());
}
Modified: hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java?rev=1467491&r1=1467490&r2=1467491&view=diff
==============================================================================
--- hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java (original)
+++ hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java Fri Apr 12 22:10:31 2013
@@ -53,6 +53,7 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.SerDeInfo;
import org.apache.hadoop.hive.metastore.api.SkewedInfo;
+import org.apache.hadoop.hive.metastore.api.SkewedValueList;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.metastore.api.StringColumnStatsData;
import org.apache.hadoop.hive.metastore.api.Table;
@@ -185,8 +186,8 @@ public abstract class TestHiveMetaStore
skewInfor.setSkewedColNames(Arrays.asList("name"));
List<String> skv = Arrays.asList("1");
skewInfor.setSkewedColValues(Arrays.asList(skv));
- Map<List<String>, String> scvlm = new HashMap<List<String>, String>();
- scvlm.put(skv, "location1");
+ Map<SkewedValueList, String> scvlm = new HashMap<SkewedValueList, String>();
+ scvlm.put(new SkewedValueList(skv), "location1");
skewInfor.setSkewedColValueLocationMaps(scvlm);
sd.setSkewedInfo(skewInfor);
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java?rev=1467491&r1=1467490&r2=1467491&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java Fri Apr 12 22:10:31 2013
@@ -34,6 +34,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.FileUtils;
+import org.apache.hadoop.hive.metastore.api.SkewedValueList;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils;
import org.apache.hadoop.hive.ql.io.HiveKey;
@@ -708,7 +709,7 @@ public class FileSinkOperator extends Te
List<String> skewedCols = lbCtx.getSkewedColNames();
List<List<String>> allSkewedVals = lbCtx.getSkewedColValues();
List<String> skewedValsCandidate = null;
- Map<List<String>, String> locationMap = lbCtx.getLbLocationMap();
+ Map<SkewedValueList, String> locationMap = lbCtx.getLbLocationMap();
/* Convert input row to standard objects. */
ObjectInspectorUtils.copyToStandardObject(standObjs, row,
@@ -726,14 +727,14 @@ public class FileSinkOperator extends Te
if (allSkewedVals.contains(skewedValsCandidate)) {
/* matches skewed values. */
lbDirName = FileUtils.makeListBucketingDirName(skewedCols, skewedValsCandidate);
- locationMap.put(skewedValsCandidate, lbDirName);
+ locationMap.put(new SkewedValueList(skewedValsCandidate), lbDirName);
} else {
/* create default directory. */
lbDirName = FileUtils.makeDefaultListBucketingDirName(skewedCols,
lbCtx.getDefaultDirName());
List<String> defaultKey = Arrays.asList(lbCtx.getDefaultKey());
if (!locationMap.containsKey(defaultKey)) {
- locationMap.put(defaultKey, lbDirName);
+ locationMap.put(new SkewedValueList(defaultKey), lbDirName);
}
}
return lbDirName;
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java?rev=1467491&r1=1467490&r2=1467491&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java Fri Apr 12 22:10:31 2013
@@ -76,6 +76,7 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.metastore.api.Role;
import org.apache.hadoop.hive.metastore.api.SerDeInfo;
import org.apache.hadoop.hive.metastore.api.SkewedInfo;
+import org.apache.hadoop.hive.metastore.api.SkewedValueList;
import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.index.HiveIndexHandler;
@@ -1220,8 +1221,8 @@ public class Hive {
org.apache.hadoop.hive.metastore.api.Partition newCreatedTpart = newTPart.getTPartition();
SkewedInfo skewedInfo = newCreatedTpart.getSd().getSkewedInfo();
/* Construct list bucketing location mappings from sub-directory name. */
- Map<List<String>, String> skewedColValueLocationMaps = constructListBucketingLocationMap(
- newPartPath, skewedInfo);
+ Map<SkewedValueList, String> skewedColValueLocationMaps =
+ constructListBucketingLocationMap(newPartPath, skewedInfo);
/* Add list bucketing location mappings. */
skewedInfo.setSkewedColValueLocationMaps(skewedColValueLocationMaps);
newCreatedTpart.getSd().setSkewedInfo(skewedInfo);
@@ -1254,7 +1255,8 @@ public class Hive {
* @throws IOException
*/
private void walkDirTree(FileStatus fSta, FileSystem fSys,
- Map<List<String>, String> skewedColValueLocationMaps, Path newPartPath, SkewedInfo skewedInfo)
+ Map<SkewedValueList, String> skewedColValueLocationMaps,
+ Path newPartPath, SkewedInfo skewedInfo)
throws IOException {
/* Base Case. It's leaf. */
if (!fSta.isDir()) {
@@ -1280,7 +1282,7 @@ private void walkDirTree(FileStatus fSta
* @param skewedInfo
*/
private void constructOneLBLocationMap(FileStatus fSta,
- Map<List<String>, String> skewedColValueLocationMaps,
+ Map<SkewedValueList, String> skewedColValueLocationMaps,
Path newPartPath, SkewedInfo skewedInfo) {
Path lbdPath = fSta.getPath().getParent();
List<String> skewedValue = new ArrayList<String>();
@@ -1303,7 +1305,7 @@ private void constructOneLBLocationMap(F
}
if ((skewedValue.size() > 0) && (skewedValue.size() == skewedInfo.getSkewedColNames().size())
&& !skewedColValueLocationMaps.containsKey(skewedValue)) {
- skewedColValueLocationMaps.put(skewedValue, lbdPath.toString());
+ skewedColValueLocationMaps.put(new SkewedValueList(skewedValue), lbdPath.toString());
}
}
@@ -1316,9 +1318,10 @@ private void constructOneLBLocationMap(F
* @throws IOException
* @throws FileNotFoundException
*/
- private Map<List<String>, String> constructListBucketingLocationMap(Path newPartPath,
+ private Map<SkewedValueList, String> constructListBucketingLocationMap(Path newPartPath,
SkewedInfo skewedInfo) throws IOException, FileNotFoundException {
- Map<List<String>, String> skewedColValueLocationMaps = new HashMap<List<String>, String>();
+ Map<SkewedValueList, String> skewedColValueLocationMaps =
+ new HashMap<SkewedValueList, String>();
FileSystem fSys = newPartPath.getFileSystem(conf);
walkDirTree(fSys.getFileStatus(newPartPath), fSys, skewedColValueLocationMaps, newPartPath,
skewedInfo);
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java?rev=1467491&r1=1467490&r2=1467491&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java Fri Apr 12 22:10:31 2013
@@ -40,6 +40,7 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.Order;
+import org.apache.hadoop.hive.metastore.api.SkewedValueList;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils;
@@ -647,18 +648,18 @@ public class Partition implements Serial
public void setSkewedValueLocationMap(List<String> valList, String dirName)
throws HiveException {
- Map<List<String>, String> mappings = tPartition.getSd().getSkewedInfo()
+ Map<SkewedValueList, String> mappings = tPartition.getSd().getSkewedInfo()
.getSkewedColValueLocationMaps();
if (null == mappings) {
- mappings = new HashMap<List<String>, String>();
+ mappings = new HashMap<SkewedValueList, String>();
tPartition.getSd().getSkewedInfo().setSkewedColValueLocationMaps(mappings);
}
// Add or update new mapping
- mappings.put(valList, dirName);
+ mappings.put(new SkewedValueList(valList), dirName);
}
- public Map<List<String>, String> getSkewedColValueLocationMaps() {
+ public Map<SkewedValueList, String> getSkewedColValueLocationMaps() {
return tPartition.getSd().getSkewedInfo().getSkewedColValueLocationMaps();
}
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java?rev=1467491&r1=1467490&r2=1467491&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java Fri Apr 12 22:10:31 2013
@@ -45,6 +45,7 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.metastore.api.Order;
import org.apache.hadoop.hive.metastore.api.SerDeInfo;
import org.apache.hadoop.hive.metastore.api.SkewedInfo;
+import org.apache.hadoop.hive.metastore.api.SkewedValueList;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils;
import org.apache.hadoop.hive.ql.io.HiveOutputFormat;
@@ -144,7 +145,7 @@ public class Table implements Serializab
SkewedInfo skewInfo = new SkewedInfo();
skewInfo.setSkewedColNames(new ArrayList<String>());
skewInfo.setSkewedColValues(new ArrayList<List<String>>());
- skewInfo.setSkewedColValueLocationMaps(new HashMap<List<String>, String>());
+ skewInfo.setSkewedColValueLocationMaps(new HashMap<SkewedValueList, String>());
sd.setSkewedInfo(skewInfo);
}
@@ -518,20 +519,20 @@ public class Table implements Serializab
public void setSkewedValueLocationMap(List<String> valList, String dirName)
throws HiveException {
- Map<List<String>, String> mappings = tTable.getSd().getSkewedInfo()
+ Map<SkewedValueList, String> mappings = tTable.getSd().getSkewedInfo()
.getSkewedColValueLocationMaps();
if (null == mappings) {
- mappings = new HashMap<List<String>, String>();
+ mappings = new HashMap<SkewedValueList, String>();
tTable.getSd().getSkewedInfo().setSkewedColValueLocationMaps(mappings);
}
// Add or update new mapping
- mappings.put(valList, dirName);
+ mappings.put(new SkewedValueList(valList), dirName);
}
- public Map<List<String>,String> getSkewedColValueLocationMaps() {
+ public Map<SkewedValueList,String> getSkewedColValueLocationMaps() {
return (tTable.getSd().getSkewedInfo() != null) ? tTable.getSd().getSkewedInfo()
- .getSkewedColValueLocationMaps() : new HashMap<List<String>, String>();
+ .getSkewedColValueLocationMaps() : new HashMap<SkewedValueList, String>();
}
public void setSkewedColValues(List<List<String>> skewedValues) throws HiveException {
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java?rev=1467491&r1=1467490&r2=1467491&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java Fri Apr 12 22:10:31 2013
@@ -31,6 +31,7 @@ import org.apache.commons.lang.StringEsc
import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Index;
+import org.apache.hadoop.hive.metastore.api.SkewedValueList;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.ql.index.HiveIndex;
import org.apache.hadoop.hive.ql.index.HiveIndex.IndexType;
@@ -193,7 +194,7 @@ public final class MetaDataFormatUtils {
formatOutput("Skewed Values:", skewedColValues.toString(), tableInfo);
}
- Map<List<String>, String> skewedColMap = storageDesc.getSkewedInfo()
+ Map<SkewedValueList, String> skewedColMap = storageDesc.getSkewedInfo()
.getSkewedColValueLocationMaps();
if ((skewedColMap!=null) && (skewedColMap.size() > 0)) {
formatOutput("Skewed Value to Path:", skewedColMap.toString(),
@@ -201,9 +202,8 @@ public final class MetaDataFormatUtils {
Map<List<String>, String> truncatedSkewedColMap = new HashMap<List<String>, String>();
// walk through existing map to truncate path so that test won't mask it
// then we can verify location is right
- Set<Entry<List<String>, String>> entries = skewedColMap.entrySet();
- for (Entry<List<String>, String> entry : entries) {
- truncatedSkewedColMap.put(entry.getKey(),
+ for (Entry<SkewedValueList, String> entry : skewedColMap.entrySet()) {
+ truncatedSkewedColMap.put(entry.getKey().getSkewedValueList(),
PlanUtils.removePrefixFromWarehouseConfig(entry.getValue()));
}
formatOutput("Skewed Value to Truncated Path:",
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/listbucketingpruner/ListBucketingPruner.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/listbucketingpruner/ListBucketingPruner.java?rev=1467491&r1=1467490&r2=1467491&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/listbucketingpruner/ListBucketingPruner.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/listbucketingpruner/ListBucketingPruner.java Fri Apr 12 22:10:31 2013
@@ -27,6 +27,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.FileUtils;
+import org.apache.hadoop.hive.metastore.api.SkewedValueList;
import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
import org.apache.hadoop.hive.ql.metadata.Partition;
import org.apache.hadoop.hive.ql.optimizer.PrunerUtils;
@@ -290,12 +291,13 @@ public class ListBucketingPruner impleme
List<List<String>> uniqSkewedValues) throws SemanticException {
// For each entry in dynamic-multi-dimension collection.
List<String> skewedCols = part.getSkewedColNames(); // Retrieve skewed column.
- Map<List<String>, String> mappings = part.getSkewedColValueLocationMaps(); // Retrieve skewed
+ Map<SkewedValueList, String> mappings = part.getSkewedColValueLocationMaps(); // Retrieve skewed
// map.
assert ListBucketingPrunerUtils.isListBucketingPart(part) : part.getName()
+ " skewed metadata is corrupted. No skewed column and/or location mappings information.";
List<List<String>> skewedValues = part.getSkewedColValues();
List<Boolean> nonSkewedValueMatchResult = new ArrayList<Boolean>();
+ SkewedValueList skewedValueList = new SkewedValueList();
for (List<String> cell : collections) {
// Walk through the tree to decide value.
// Example: skewed column: C1, C2 ;
@@ -309,8 +311,9 @@ public class ListBucketingPruner impleme
/* It's valid case if a partition: */
/* 1. is defined with skewed columns and skewed values in metadata */
/* 2. doesn't have all skewed values within its data */
- if (mappings.get(cell) != null) {
- selectedPaths.add(new Path(mappings.get(cell)));
+ skewedValueList.setSkewedValueList(cell);
+ if (mappings.get(skewedValueList) != null) {
+ selectedPaths.add(new Path(mappings.get(skewedValueList)));
}
}
} else {
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java?rev=1467491&r1=1467490&r2=1467491&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java Fri Apr 12 22:10:31 2013
@@ -37,6 +37,7 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Order;
+import org.apache.hadoop.hive.metastore.api.SkewedValueList;
import org.apache.hadoop.hive.ql.Context;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.QueryProperties;
@@ -965,7 +966,7 @@ public abstract class BaseSemanticAnalyz
* @return
*/
protected ListBucketingCtx constructListBucketingCtx(List<String> skewedColNames,
- List<List<String>> skewedValues, Map<List<String>, String> skewedColValueLocationMaps,
+ List<List<String>> skewedValues, Map<SkewedValueList, String> skewedColValueLocationMaps,
boolean isStoredAsSubDirectories, HiveConf conf) {
ListBucketingCtx lbCtx = new ListBucketingCtx();
lbCtx.setSkewedColNames(skewedColNames);
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ListBucketingCtx.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ListBucketingCtx.java?rev=1467491&r1=1467490&r2=1467491&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ListBucketingCtx.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ListBucketingCtx.java Fri Apr 12 22:10:31 2013
@@ -23,6 +23,7 @@ import java.util.List;
import java.util.Map;
import org.apache.hadoop.hive.common.FileUtils;
+import org.apache.hadoop.hive.metastore.api.SkewedValueList;
import org.apache.hadoop.hive.ql.exec.ColumnInfo;
import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
import org.apache.hadoop.hive.ql.exec.RowSchema;
@@ -39,7 +40,7 @@ public class ListBucketingCtx implements
private static final long serialVersionUID = 1L;
private List<String> skewedColNames;
private List<List<String>> skewedColValues;
- private Map<List<String>, String> lbLocationMap;
+ private Map<SkewedValueList, String> lbLocationMap;
private List<SkewedColumnPositionPair> rowSkewedIndex;
private boolean isStoredAsSubDirectories;
private String defaultKey;
@@ -82,14 +83,14 @@ public class ListBucketingCtx implements
/**
* @return the lbLocationMap
*/
- public Map<List<String>, String> getLbLocationMap() {
+ public Map<SkewedValueList, String> getLbLocationMap() {
return lbLocationMap;
}
/**
* @param lbLocationMap the lbLocationMap to set
*/
- public void setLbLocationMap(Map<List<String>, String> lbLocationMap) {
+ public void setLbLocationMap(Map<SkewedValueList, String> lbLocationMap) {
this.lbLocationMap = lbLocationMap;
}