You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2013/09/12 03:21:29 UTC
svn commit: r1522098 [12/30] - in /hive/branches/vectorization: ./
beeline/src/test/org/apache/hive/beeline/src/test/ bin/ bin/ext/
common/src/java/org/apache/hadoop/hive/common/
common/src/java/org/apache/hadoop/hive/conf/ conf/ contrib/src/java/org/a...
Modified: hive/branches/vectorization/metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp?rev=1522098&r1=1522097&r2=1522098&view=diff
==============================================================================
--- hive/branches/vectorization/metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp (original)
+++ hive/branches/vectorization/metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp Thu Sep 12 01:21:10 2013
@@ -1404,10 +1404,10 @@ void swap(Order &a, Order &b) {
swap(a.__isset, b.__isset);
}
-const char* SkewedValueList::ascii_fingerprint = "ACE4F644F0FDD289DDC4EE5B83BC13C0";
-const uint8_t SkewedValueList::binary_fingerprint[16] = {0xAC,0xE4,0xF6,0x44,0xF0,0xFD,0xD2,0x89,0xDD,0xC4,0xEE,0x5B,0x83,0xBC,0x13,0xC0};
+const char* SkewedInfo::ascii_fingerprint = "4BF2ED84BC3C3EB297A2AE2FA8427EB1";
+const uint8_t SkewedInfo::binary_fingerprint[16] = {0x4B,0xF2,0xED,0x84,0xBC,0x3C,0x3E,0xB2,0x97,0xA2,0xAE,0x2F,0xA8,0x42,0x7E,0xB1};
-uint32_t SkewedValueList::read(::apache::thrift::protocol::TProtocol* iprot) {
+uint32_t SkewedInfo::read(::apache::thrift::protocol::TProtocol* iprot) {
uint32_t xfer = 0;
std::string fname;
@@ -1430,97 +1430,15 @@ uint32_t SkewedValueList::read(::apache:
case 1:
if (ftype == ::apache::thrift::protocol::T_LIST) {
{
- this->skewedValueList.clear();
+ this->skewedColNames.clear();
uint32_t _size79;
::apache::thrift::protocol::TType _etype82;
xfer += iprot->readListBegin(_etype82, _size79);
- this->skewedValueList.resize(_size79);
+ this->skewedColNames.resize(_size79);
uint32_t _i83;
for (_i83 = 0; _i83 < _size79; ++_i83)
{
- xfer += iprot->readString(this->skewedValueList[_i83]);
- }
- xfer += iprot->readListEnd();
- }
- this->__isset.skewedValueList = true;
- } else {
- xfer += iprot->skip(ftype);
- }
- break;
- default:
- xfer += iprot->skip(ftype);
- break;
- }
- xfer += iprot->readFieldEnd();
- }
-
- xfer += iprot->readStructEnd();
-
- return xfer;
-}
-
-uint32_t SkewedValueList::write(::apache::thrift::protocol::TProtocol* oprot) const {
- uint32_t xfer = 0;
- xfer += oprot->writeStructBegin("SkewedValueList");
-
- xfer += oprot->writeFieldBegin("skewedValueList", ::apache::thrift::protocol::T_LIST, 1);
- {
- xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast<uint32_t>(this->skewedValueList.size()));
- std::vector<std::string> ::const_iterator _iter84;
- for (_iter84 = this->skewedValueList.begin(); _iter84 != this->skewedValueList.end(); ++_iter84)
- {
- xfer += oprot->writeString((*_iter84));
- }
- xfer += oprot->writeListEnd();
- }
- xfer += oprot->writeFieldEnd();
-
- xfer += oprot->writeFieldStop();
- xfer += oprot->writeStructEnd();
- return xfer;
-}
-
-void swap(SkewedValueList &a, SkewedValueList &b) {
- using ::std::swap;
- swap(a.skewedValueList, b.skewedValueList);
- swap(a.__isset, b.__isset);
-}
-
-const char* SkewedInfo::ascii_fingerprint = "F4A067072461B8071E4F5A6BE5276FC0";
-const uint8_t SkewedInfo::binary_fingerprint[16] = {0xF4,0xA0,0x67,0x07,0x24,0x61,0xB8,0x07,0x1E,0x4F,0x5A,0x6B,0xE5,0x27,0x6F,0xC0};
-
-uint32_t SkewedInfo::read(::apache::thrift::protocol::TProtocol* iprot) {
-
- uint32_t xfer = 0;
- std::string fname;
- ::apache::thrift::protocol::TType ftype;
- int16_t fid;
-
- xfer += iprot->readStructBegin(fname);
-
- using ::apache::thrift::protocol::TProtocolException;
-
-
- while (true)
- {
- xfer += iprot->readFieldBegin(fname, ftype, fid);
- if (ftype == ::apache::thrift::protocol::T_STOP) {
- break;
- }
- switch (fid)
- {
- case 1:
- if (ftype == ::apache::thrift::protocol::T_LIST) {
- {
- this->skewedColNames.clear();
- uint32_t _size85;
- ::apache::thrift::protocol::TType _etype88;
- xfer += iprot->readListBegin(_etype88, _size85);
- this->skewedColNames.resize(_size85);
- uint32_t _i89;
- for (_i89 = 0; _i89 < _size85; ++_i89)
- {
- xfer += iprot->readString(this->skewedColNames[_i89]);
+ xfer += iprot->readString(this->skewedColNames[_i83]);
}
xfer += iprot->readListEnd();
}
@@ -1533,23 +1451,23 @@ uint32_t SkewedInfo::read(::apache::thri
if (ftype == ::apache::thrift::protocol::T_LIST) {
{
this->skewedColValues.clear();
- uint32_t _size90;
- ::apache::thrift::protocol::TType _etype93;
- xfer += iprot->readListBegin(_etype93, _size90);
- this->skewedColValues.resize(_size90);
- uint32_t _i94;
- for (_i94 = 0; _i94 < _size90; ++_i94)
+ uint32_t _size84;
+ ::apache::thrift::protocol::TType _etype87;
+ xfer += iprot->readListBegin(_etype87, _size84);
+ this->skewedColValues.resize(_size84);
+ uint32_t _i88;
+ for (_i88 = 0; _i88 < _size84; ++_i88)
{
{
- this->skewedColValues[_i94].clear();
- uint32_t _size95;
- ::apache::thrift::protocol::TType _etype98;
- xfer += iprot->readListBegin(_etype98, _size95);
- this->skewedColValues[_i94].resize(_size95);
- uint32_t _i99;
- for (_i99 = 0; _i99 < _size95; ++_i99)
+ this->skewedColValues[_i88].clear();
+ uint32_t _size89;
+ ::apache::thrift::protocol::TType _etype92;
+ xfer += iprot->readListBegin(_etype92, _size89);
+ this->skewedColValues[_i88].resize(_size89);
+ uint32_t _i93;
+ for (_i93 = 0; _i93 < _size89; ++_i93)
{
- xfer += iprot->readString(this->skewedColValues[_i94][_i99]);
+ xfer += iprot->readString(this->skewedColValues[_i88][_i93]);
}
xfer += iprot->readListEnd();
}
@@ -1565,17 +1483,29 @@ uint32_t SkewedInfo::read(::apache::thri
if (ftype == ::apache::thrift::protocol::T_MAP) {
{
this->skewedColValueLocationMaps.clear();
- uint32_t _size100;
- ::apache::thrift::protocol::TType _ktype101;
- ::apache::thrift::protocol::TType _vtype102;
- xfer += iprot->readMapBegin(_ktype101, _vtype102, _size100);
- uint32_t _i104;
- for (_i104 = 0; _i104 < _size100; ++_i104)
+ uint32_t _size94;
+ ::apache::thrift::protocol::TType _ktype95;
+ ::apache::thrift::protocol::TType _vtype96;
+ xfer += iprot->readMapBegin(_ktype95, _vtype96, _size94);
+ uint32_t _i98;
+ for (_i98 = 0; _i98 < _size94; ++_i98)
{
- SkewedValueList _key105;
- xfer += _key105.read(iprot);
- std::string& _val106 = this->skewedColValueLocationMaps[_key105];
- xfer += iprot->readString(_val106);
+ std::vector<std::string> _key99;
+ {
+ _key99.clear();
+ uint32_t _size101;
+ ::apache::thrift::protocol::TType _etype104;
+ xfer += iprot->readListBegin(_etype104, _size101);
+ _key99.resize(_size101);
+ uint32_t _i105;
+ for (_i105 = 0; _i105 < _size101; ++_i105)
+ {
+ xfer += iprot->readString(_key99[_i105]);
+ }
+ xfer += iprot->readListEnd();
+ }
+ std::string& _val100 = this->skewedColValueLocationMaps[_key99];
+ xfer += iprot->readString(_val100);
}
xfer += iprot->readMapEnd();
}
@@ -1603,10 +1533,10 @@ uint32_t SkewedInfo::write(::apache::thr
xfer += oprot->writeFieldBegin("skewedColNames", ::apache::thrift::protocol::T_LIST, 1);
{
xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast<uint32_t>(this->skewedColNames.size()));
- std::vector<std::string> ::const_iterator _iter107;
- for (_iter107 = this->skewedColNames.begin(); _iter107 != this->skewedColNames.end(); ++_iter107)
+ std::vector<std::string> ::const_iterator _iter106;
+ for (_iter106 = this->skewedColNames.begin(); _iter106 != this->skewedColNames.end(); ++_iter106)
{
- xfer += oprot->writeString((*_iter107));
+ xfer += oprot->writeString((*_iter106));
}
xfer += oprot->writeListEnd();
}
@@ -1615,15 +1545,15 @@ uint32_t SkewedInfo::write(::apache::thr
xfer += oprot->writeFieldBegin("skewedColValues", ::apache::thrift::protocol::T_LIST, 2);
{
xfer += oprot->writeListBegin(::apache::thrift::protocol::T_LIST, static_cast<uint32_t>(this->skewedColValues.size()));
- std::vector<std::vector<std::string> > ::const_iterator _iter108;
- for (_iter108 = this->skewedColValues.begin(); _iter108 != this->skewedColValues.end(); ++_iter108)
+ std::vector<std::vector<std::string> > ::const_iterator _iter107;
+ for (_iter107 = this->skewedColValues.begin(); _iter107 != this->skewedColValues.end(); ++_iter107)
{
{
- xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast<uint32_t>((*_iter108).size()));
- std::vector<std::string> ::const_iterator _iter109;
- for (_iter109 = (*_iter108).begin(); _iter109 != (*_iter108).end(); ++_iter109)
+ xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast<uint32_t>((*_iter107).size()));
+ std::vector<std::string> ::const_iterator _iter108;
+ for (_iter108 = (*_iter107).begin(); _iter108 != (*_iter107).end(); ++_iter108)
{
- xfer += oprot->writeString((*_iter109));
+ xfer += oprot->writeString((*_iter108));
}
xfer += oprot->writeListEnd();
}
@@ -1634,12 +1564,20 @@ uint32_t SkewedInfo::write(::apache::thr
xfer += oprot->writeFieldBegin("skewedColValueLocationMaps", ::apache::thrift::protocol::T_MAP, 3);
{
- xfer += oprot->writeMapBegin(::apache::thrift::protocol::T_STRUCT, ::apache::thrift::protocol::T_STRING, static_cast<uint32_t>(this->skewedColValueLocationMaps.size()));
- std::map<SkewedValueList, std::string> ::const_iterator _iter110;
- for (_iter110 = this->skewedColValueLocationMaps.begin(); _iter110 != this->skewedColValueLocationMaps.end(); ++_iter110)
+ xfer += oprot->writeMapBegin(::apache::thrift::protocol::T_LIST, ::apache::thrift::protocol::T_STRING, static_cast<uint32_t>(this->skewedColValueLocationMaps.size()));
+ std::map<std::vector<std::string> , std::string> ::const_iterator _iter109;
+ for (_iter109 = this->skewedColValueLocationMaps.begin(); _iter109 != this->skewedColValueLocationMaps.end(); ++_iter109)
{
- xfer += _iter110->first.write(oprot);
- xfer += oprot->writeString(_iter110->second);
+ {
+ xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast<uint32_t>(_iter109->first.size()));
+ std::vector<std::string> ::const_iterator _iter110;
+ for (_iter110 = _iter109->first.begin(); _iter110 != _iter109->first.end(); ++_iter110)
+ {
+ xfer += oprot->writeString((*_iter110));
+ }
+ xfer += oprot->writeListEnd();
+ }
+ xfer += oprot->writeString(_iter109->second);
}
xfer += oprot->writeMapEnd();
}
@@ -1658,8 +1596,8 @@ void swap(SkewedInfo &a, SkewedInfo &b)
swap(a.__isset, b.__isset);
}
-const char* StorageDescriptor::ascii_fingerprint = "283664579DDA891300CFDE15F5771323";
-const uint8_t StorageDescriptor::binary_fingerprint[16] = {0x28,0x36,0x64,0x57,0x9D,0xDA,0x89,0x13,0x00,0xCF,0xDE,0x15,0xF5,0x77,0x13,0x23};
+const char* StorageDescriptor::ascii_fingerprint = "CA8C9AA5FE4C32643757D8639CEF0CD7";
+const uint8_t StorageDescriptor::binary_fingerprint[16] = {0xCA,0x8C,0x9A,0xA5,0xFE,0x4C,0x32,0x64,0x37,0x57,0xD8,0x63,0x9C,0xEF,0x0C,0xD7};
uint32_t StorageDescriptor::read(::apache::thrift::protocol::TProtocol* iprot) {
@@ -1949,8 +1887,8 @@ void swap(StorageDescriptor &a, StorageD
swap(a.__isset, b.__isset);
}
-const char* Table::ascii_fingerprint = "AEBB39149C622C62B239988E19489807";
-const uint8_t Table::binary_fingerprint[16] = {0xAE,0xBB,0x39,0x14,0x9C,0x62,0x2C,0x62,0xB2,0x39,0x98,0x8E,0x19,0x48,0x98,0x07};
+const char* Table::ascii_fingerprint = "68640B4B66B355CF317429AF70D2C260";
+const uint8_t Table::binary_fingerprint[16] = {0x68,0x64,0x0B,0x4B,0x66,0xB3,0x55,0xCF,0x31,0x74,0x29,0xAF,0x70,0xD2,0xC2,0x60};
uint32_t Table::read(::apache::thrift::protocol::TProtocol* iprot) {
@@ -2212,8 +2150,8 @@ void swap(Table &a, Table &b) {
swap(a.__isset, b.__isset);
}
-const char* Partition::ascii_fingerprint = "81AD5F7E827DDC1A37C18BF73ECA603F";
-const uint8_t Partition::binary_fingerprint[16] = {0x81,0xAD,0x5F,0x7E,0x82,0x7D,0xDC,0x1A,0x37,0xC1,0x8B,0xF7,0x3E,0xCA,0x60,0x3F};
+const char* Partition::ascii_fingerprint = "31A52241B88A426C34087FE38343FF51";
+const uint8_t Partition::binary_fingerprint[16] = {0x31,0xA5,0x22,0x41,0xB8,0x8A,0x42,0x6C,0x34,0x08,0x7F,0xE3,0x83,0x43,0xFF,0x51};
uint32_t Partition::read(::apache::thrift::protocol::TProtocol* iprot) {
@@ -2410,8 +2348,8 @@ void swap(Partition &a, Partition &b) {
swap(a.__isset, b.__isset);
}
-const char* Index::ascii_fingerprint = "A2822E71566DF57594774A6CF56D11B5";
-const uint8_t Index::binary_fingerprint[16] = {0xA2,0x82,0x2E,0x71,0x56,0x6D,0xF5,0x75,0x94,0x77,0x4A,0x6C,0xF5,0x6D,0x11,0xB5};
+const char* Index::ascii_fingerprint = "09EEF655216AC81802850988D6C470A6";
+const uint8_t Index::binary_fingerprint[16] = {0x09,0xEE,0xF6,0x55,0x21,0x6A,0xC8,0x18,0x02,0x85,0x09,0x88,0xD6,0xC4,0x70,0xA6};
uint32_t Index::read(::apache::thrift::protocol::TProtocol* iprot) {
Modified: hive/branches/vectorization/metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h?rev=1522098&r1=1522097&r2=1522098&view=diff
==============================================================================
--- hive/branches/vectorization/metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h (original)
+++ hive/branches/vectorization/metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h Thu Sep 12 01:21:10 2013
@@ -799,49 +799,6 @@ class Order {
void swap(Order &a, Order &b);
-typedef struct _SkewedValueList__isset {
- _SkewedValueList__isset() : skewedValueList(false) {}
- bool skewedValueList;
-} _SkewedValueList__isset;
-
-class SkewedValueList {
- public:
-
- static const char* ascii_fingerprint; // = "ACE4F644F0FDD289DDC4EE5B83BC13C0";
- static const uint8_t binary_fingerprint[16]; // = {0xAC,0xE4,0xF6,0x44,0xF0,0xFD,0xD2,0x89,0xDD,0xC4,0xEE,0x5B,0x83,0xBC,0x13,0xC0};
-
- SkewedValueList() {
- }
-
- virtual ~SkewedValueList() throw() {}
-
- std::vector<std::string> skewedValueList;
-
- _SkewedValueList__isset __isset;
-
- void __set_skewedValueList(const std::vector<std::string> & val) {
- skewedValueList = val;
- }
-
- bool operator == (const SkewedValueList & rhs) const
- {
- if (!(skewedValueList == rhs.skewedValueList))
- return false;
- return true;
- }
- bool operator != (const SkewedValueList &rhs) const {
- return !(*this == rhs);
- }
-
- bool operator < (const SkewedValueList & ) const;
-
- uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
- uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
-
-};
-
-void swap(SkewedValueList &a, SkewedValueList &b);
-
typedef struct _SkewedInfo__isset {
_SkewedInfo__isset() : skewedColNames(false), skewedColValues(false), skewedColValueLocationMaps(false) {}
bool skewedColNames;
@@ -852,8 +809,8 @@ typedef struct _SkewedInfo__isset {
class SkewedInfo {
public:
- static const char* ascii_fingerprint; // = "F4A067072461B8071E4F5A6BE5276FC0";
- static const uint8_t binary_fingerprint[16]; // = {0xF4,0xA0,0x67,0x07,0x24,0x61,0xB8,0x07,0x1E,0x4F,0x5A,0x6B,0xE5,0x27,0x6F,0xC0};
+ static const char* ascii_fingerprint; // = "4BF2ED84BC3C3EB297A2AE2FA8427EB1";
+ static const uint8_t binary_fingerprint[16]; // = {0x4B,0xF2,0xED,0x84,0xBC,0x3C,0x3E,0xB2,0x97,0xA2,0xAE,0x2F,0xA8,0x42,0x7E,0xB1};
SkewedInfo() {
}
@@ -862,7 +819,7 @@ class SkewedInfo {
std::vector<std::string> skewedColNames;
std::vector<std::vector<std::string> > skewedColValues;
- std::map<SkewedValueList, std::string> skewedColValueLocationMaps;
+ std::map<std::vector<std::string> , std::string> skewedColValueLocationMaps;
_SkewedInfo__isset __isset;
@@ -874,7 +831,7 @@ class SkewedInfo {
skewedColValues = val;
}
- void __set_skewedColValueLocationMaps(const std::map<SkewedValueList, std::string> & val) {
+ void __set_skewedColValueLocationMaps(const std::map<std::vector<std::string> , std::string> & val) {
skewedColValueLocationMaps = val;
}
@@ -920,8 +877,8 @@ typedef struct _StorageDescriptor__isset
class StorageDescriptor {
public:
- static const char* ascii_fingerprint; // = "283664579DDA891300CFDE15F5771323";
- static const uint8_t binary_fingerprint[16]; // = {0x28,0x36,0x64,0x57,0x9D,0xDA,0x89,0x13,0x00,0xCF,0xDE,0x15,0xF5,0x77,0x13,0x23};
+ static const char* ascii_fingerprint; // = "CA8C9AA5FE4C32643757D8639CEF0CD7";
+ static const uint8_t binary_fingerprint[16]; // = {0xCA,0x8C,0x9A,0xA5,0xFE,0x4C,0x32,0x64,0x37,0x57,0xD8,0x63,0x9C,0xEF,0x0C,0xD7};
StorageDescriptor() : location(), inputFormat(), outputFormat(), compressed(0), numBuckets(0), storedAsSubDirectories(0) {
}
@@ -1058,8 +1015,8 @@ typedef struct _Table__isset {
class Table {
public:
- static const char* ascii_fingerprint; // = "AEBB39149C622C62B239988E19489807";
- static const uint8_t binary_fingerprint[16]; // = {0xAE,0xBB,0x39,0x14,0x9C,0x62,0x2C,0x62,0xB2,0x39,0x98,0x8E,0x19,0x48,0x98,0x07};
+ static const char* ascii_fingerprint; // = "68640B4B66B355CF317429AF70D2C260";
+ static const uint8_t binary_fingerprint[16]; // = {0x68,0x64,0x0B,0x4B,0x66,0xB3,0x55,0xCF,0x31,0x74,0x29,0xAF,0x70,0xD2,0xC2,0x60};
Table() : tableName(), dbName(), owner(), createTime(0), lastAccessTime(0), retention(0), viewOriginalText(), viewExpandedText(), tableType() {
}
@@ -1195,8 +1152,8 @@ typedef struct _Partition__isset {
class Partition {
public:
- static const char* ascii_fingerprint; // = "81AD5F7E827DDC1A37C18BF73ECA603F";
- static const uint8_t binary_fingerprint[16]; // = {0x81,0xAD,0x5F,0x7E,0x82,0x7D,0xDC,0x1A,0x37,0xC1,0x8B,0xF7,0x3E,0xCA,0x60,0x3F};
+ static const char* ascii_fingerprint; // = "31A52241B88A426C34087FE38343FF51";
+ static const uint8_t binary_fingerprint[16]; // = {0x31,0xA5,0x22,0x41,0xB8,0x8A,0x42,0x6C,0x34,0x08,0x7F,0xE3,0x83,0x43,0xFF,0x51};
Partition() : dbName(), tableName(), createTime(0), lastAccessTime(0) {
}
@@ -1299,8 +1256,8 @@ typedef struct _Index__isset {
class Index {
public:
- static const char* ascii_fingerprint; // = "A2822E71566DF57594774A6CF56D11B5";
- static const uint8_t binary_fingerprint[16]; // = {0xA2,0x82,0x2E,0x71,0x56,0x6D,0xF5,0x75,0x94,0x77,0x4A,0x6C,0xF5,0x6D,0x11,0xB5};
+ static const char* ascii_fingerprint; // = "09EEF655216AC81802850988D6C470A6";
+ static const uint8_t binary_fingerprint[16]; // = {0x09,0xEE,0xF6,0x55,0x21,0x6A,0xC8,0x18,0x02,0x85,0x09,0x88,0xD6,0xC4,0x70,0xA6};
Index() : indexName(), indexHandlerClass(), dbName(), origTableName(), createTime(0), lastAccessTime(0), indexTableName(), deferredRebuild(0) {
}
Modified: hive/branches/vectorization/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/SkewedInfo.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/SkewedInfo.java?rev=1522098&r1=1522097&r2=1522098&view=diff
==============================================================================
--- hive/branches/vectorization/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/SkewedInfo.java (original)
+++ hive/branches/vectorization/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/SkewedInfo.java Thu Sep 12 01:21:10 2013
@@ -46,7 +46,7 @@ public class SkewedInfo implements org.a
private List<String> skewedColNames; // required
private List<List<String>> skewedColValues; // required
- private Map<SkewedValueList,String> skewedColValueLocationMaps; // required
+ private Map<List<String>,String> skewedColValueLocationMaps; // required
/** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
public enum _Fields implements org.apache.thrift.TFieldIdEnum {
@@ -125,7 +125,8 @@ public class SkewedInfo implements org.a
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)))));
tmpMap.put(_Fields.SKEWED_COL_VALUE_LOCATION_MAPS, new org.apache.thrift.meta_data.FieldMetaData("skewedColValueLocationMaps", org.apache.thrift.TFieldRequirementType.DEFAULT,
new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, SkewedValueList.class),
+ new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)),
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))));
metaDataMap = Collections.unmodifiableMap(tmpMap);
org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(SkewedInfo.class, metaDataMap);
@@ -137,7 +138,7 @@ public class SkewedInfo implements org.a
public SkewedInfo(
List<String> skewedColNames,
List<List<String>> skewedColValues,
- Map<SkewedValueList,String> skewedColValueLocationMaps)
+ Map<List<String>,String> skewedColValueLocationMaps)
{
this();
this.skewedColNames = skewedColNames;
@@ -168,13 +169,16 @@ public class SkewedInfo implements org.a
this.skewedColValues = __this__skewedColValues;
}
if (other.isSetSkewedColValueLocationMaps()) {
- Map<SkewedValueList,String> __this__skewedColValueLocationMaps = new HashMap<SkewedValueList,String>();
- for (Map.Entry<SkewedValueList, String> other_element : other.skewedColValueLocationMaps.entrySet()) {
+ Map<List<String>,String> __this__skewedColValueLocationMaps = new HashMap<List<String>,String>();
+ for (Map.Entry<List<String>, String> other_element : other.skewedColValueLocationMaps.entrySet()) {
- SkewedValueList other_element_key = other_element.getKey();
+ List<String> other_element_key = other_element.getKey();
String other_element_value = other_element.getValue();
- SkewedValueList __this__skewedColValueLocationMaps_copy_key = new SkewedValueList(other_element_key);
+ List<String> __this__skewedColValueLocationMaps_copy_key = new ArrayList<String>();
+ for (String other_element_key_element : other_element_key) {
+ __this__skewedColValueLocationMaps_copy_key.add(other_element_key_element);
+ }
String __this__skewedColValueLocationMaps_copy_value = other_element_value;
@@ -275,18 +279,18 @@ public class SkewedInfo implements org.a
return (this.skewedColValueLocationMaps == null) ? 0 : this.skewedColValueLocationMaps.size();
}
- public void putToSkewedColValueLocationMaps(SkewedValueList key, String val) {
+ public void putToSkewedColValueLocationMaps(List<String> key, String val) {
if (this.skewedColValueLocationMaps == null) {
- this.skewedColValueLocationMaps = new HashMap<SkewedValueList,String>();
+ this.skewedColValueLocationMaps = new HashMap<List<String>,String>();
}
this.skewedColValueLocationMaps.put(key, val);
}
- public Map<SkewedValueList,String> getSkewedColValueLocationMaps() {
+ public Map<List<String>,String> getSkewedColValueLocationMaps() {
return this.skewedColValueLocationMaps;
}
- public void setSkewedColValueLocationMaps(Map<SkewedValueList,String> skewedColValueLocationMaps) {
+ public void setSkewedColValueLocationMaps(Map<List<String>,String> skewedColValueLocationMaps) {
this.skewedColValueLocationMaps = skewedColValueLocationMaps;
}
@@ -327,7 +331,7 @@ public class SkewedInfo implements org.a
if (value == null) {
unsetSkewedColValueLocationMaps();
} else {
- setSkewedColValueLocationMaps((Map<SkewedValueList,String>)value);
+ setSkewedColValueLocationMaps((Map<List<String>,String>)value);
}
break;
@@ -558,13 +562,13 @@ public class SkewedInfo implements org.a
case 1: // SKEWED_COL_NAMES
if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
{
- org.apache.thrift.protocol.TList _list106 = iprot.readListBegin();
- struct.skewedColNames = new ArrayList<String>(_list106.size);
- for (int _i107 = 0; _i107 < _list106.size; ++_i107)
+ org.apache.thrift.protocol.TList _list98 = iprot.readListBegin();
+ struct.skewedColNames = new ArrayList<String>(_list98.size);
+ for (int _i99 = 0; _i99 < _list98.size; ++_i99)
{
- String _elem108; // required
- _elem108 = iprot.readString();
- struct.skewedColNames.add(_elem108);
+ String _elem100; // required
+ _elem100 = iprot.readString();
+ struct.skewedColNames.add(_elem100);
}
iprot.readListEnd();
}
@@ -576,23 +580,23 @@ public class SkewedInfo implements org.a
case 2: // SKEWED_COL_VALUES
if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
{
- org.apache.thrift.protocol.TList _list109 = iprot.readListBegin();
- struct.skewedColValues = new ArrayList<List<String>>(_list109.size);
- for (int _i110 = 0; _i110 < _list109.size; ++_i110)
+ org.apache.thrift.protocol.TList _list101 = iprot.readListBegin();
+ struct.skewedColValues = new ArrayList<List<String>>(_list101.size);
+ for (int _i102 = 0; _i102 < _list101.size; ++_i102)
{
- List<String> _elem111; // required
+ List<String> _elem103; // required
{
- org.apache.thrift.protocol.TList _list112 = iprot.readListBegin();
- _elem111 = new ArrayList<String>(_list112.size);
- for (int _i113 = 0; _i113 < _list112.size; ++_i113)
+ org.apache.thrift.protocol.TList _list104 = iprot.readListBegin();
+ _elem103 = new ArrayList<String>(_list104.size);
+ for (int _i105 = 0; _i105 < _list104.size; ++_i105)
{
- String _elem114; // required
- _elem114 = iprot.readString();
- _elem111.add(_elem114);
+ String _elem106; // required
+ _elem106 = iprot.readString();
+ _elem103.add(_elem106);
}
iprot.readListEnd();
}
- struct.skewedColValues.add(_elem111);
+ struct.skewedColValues.add(_elem103);
}
iprot.readListEnd();
}
@@ -604,16 +608,25 @@ public class SkewedInfo implements org.a
case 3: // SKEWED_COL_VALUE_LOCATION_MAPS
if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
{
- org.apache.thrift.protocol.TMap _map115 = iprot.readMapBegin();
- struct.skewedColValueLocationMaps = new HashMap<SkewedValueList,String>(2*_map115.size);
- for (int _i116 = 0; _i116 < _map115.size; ++_i116)
+ org.apache.thrift.protocol.TMap _map107 = iprot.readMapBegin();
+ struct.skewedColValueLocationMaps = new HashMap<List<String>,String>(2*_map107.size);
+ for (int _i108 = 0; _i108 < _map107.size; ++_i108)
{
- SkewedValueList _key117; // required
- String _val118; // required
- _key117 = new SkewedValueList();
- _key117.read(iprot);
- _val118 = iprot.readString();
- struct.skewedColValueLocationMaps.put(_key117, _val118);
+ List<String> _key109; // required
+ String _val110; // required
+ {
+ org.apache.thrift.protocol.TList _list111 = iprot.readListBegin();
+ _key109 = new ArrayList<String>(_list111.size);
+ for (int _i112 = 0; _i112 < _list111.size; ++_i112)
+ {
+ String _elem113; // required
+ _elem113 = iprot.readString();
+ _key109.add(_elem113);
+ }
+ iprot.readListEnd();
+ }
+ _val110 = iprot.readString();
+ struct.skewedColValueLocationMaps.put(_key109, _val110);
}
iprot.readMapEnd();
}
@@ -639,9 +652,9 @@ public class SkewedInfo implements org.a
oprot.writeFieldBegin(SKEWED_COL_NAMES_FIELD_DESC);
{
oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.skewedColNames.size()));
- for (String _iter119 : struct.skewedColNames)
+ for (String _iter114 : struct.skewedColNames)
{
- oprot.writeString(_iter119);
+ oprot.writeString(_iter114);
}
oprot.writeListEnd();
}
@@ -651,13 +664,13 @@ public class SkewedInfo implements org.a
oprot.writeFieldBegin(SKEWED_COL_VALUES_FIELD_DESC);
{
oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.LIST, struct.skewedColValues.size()));
- for (List<String> _iter120 : struct.skewedColValues)
+ for (List<String> _iter115 : struct.skewedColValues)
{
{
- oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, _iter120.size()));
- for (String _iter121 : _iter120)
+ oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, _iter115.size()));
+ for (String _iter116 : _iter115)
{
- oprot.writeString(_iter121);
+ oprot.writeString(_iter116);
}
oprot.writeListEnd();
}
@@ -669,11 +682,18 @@ public class SkewedInfo implements org.a
if (struct.skewedColValueLocationMaps != null) {
oprot.writeFieldBegin(SKEWED_COL_VALUE_LOCATION_MAPS_FIELD_DESC);
{
- oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRUCT, org.apache.thrift.protocol.TType.STRING, struct.skewedColValueLocationMaps.size()));
- for (Map.Entry<SkewedValueList, String> _iter122 : struct.skewedColValueLocationMaps.entrySet())
+ oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.LIST, org.apache.thrift.protocol.TType.STRING, struct.skewedColValueLocationMaps.size()));
+ for (Map.Entry<List<String>, String> _iter117 : struct.skewedColValueLocationMaps.entrySet())
{
- _iter122.getKey().write(oprot);
- oprot.writeString(_iter122.getValue());
+ {
+ oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, _iter117.getKey().size()));
+ for (String _iter118 : _iter117.getKey())
+ {
+ oprot.writeString(_iter118);
+ }
+ oprot.writeListEnd();
+ }
+ oprot.writeString(_iter117.getValue());
}
oprot.writeMapEnd();
}
@@ -710,22 +730,22 @@ public class SkewedInfo implements org.a
if (struct.isSetSkewedColNames()) {
{
oprot.writeI32(struct.skewedColNames.size());
- for (String _iter123 : struct.skewedColNames)
+ for (String _iter119 : struct.skewedColNames)
{
- oprot.writeString(_iter123);
+ oprot.writeString(_iter119);
}
}
}
if (struct.isSetSkewedColValues()) {
{
oprot.writeI32(struct.skewedColValues.size());
- for (List<String> _iter124 : struct.skewedColValues)
+ for (List<String> _iter120 : struct.skewedColValues)
{
{
- oprot.writeI32(_iter124.size());
- for (String _iter125 : _iter124)
+ oprot.writeI32(_iter120.size());
+ for (String _iter121 : _iter120)
{
- oprot.writeString(_iter125);
+ oprot.writeString(_iter121);
}
}
}
@@ -734,10 +754,16 @@ public class SkewedInfo implements org.a
if (struct.isSetSkewedColValueLocationMaps()) {
{
oprot.writeI32(struct.skewedColValueLocationMaps.size());
- for (Map.Entry<SkewedValueList, String> _iter126 : struct.skewedColValueLocationMaps.entrySet())
+ for (Map.Entry<List<String>, String> _iter122 : struct.skewedColValueLocationMaps.entrySet())
{
- _iter126.getKey().write(oprot);
- oprot.writeString(_iter126.getValue());
+ {
+ oprot.writeI32(_iter122.getKey().size());
+ for (String _iter123 : _iter122.getKey())
+ {
+ oprot.writeString(_iter123);
+ }
+ }
+ oprot.writeString(_iter122.getValue());
}
}
}
@@ -749,51 +775,59 @@ public class SkewedInfo implements org.a
BitSet incoming = iprot.readBitSet(3);
if (incoming.get(0)) {
{
- org.apache.thrift.protocol.TList _list127 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
- struct.skewedColNames = new ArrayList<String>(_list127.size);
- for (int _i128 = 0; _i128 < _list127.size; ++_i128)
+ org.apache.thrift.protocol.TList _list124 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
+ struct.skewedColNames = new ArrayList<String>(_list124.size);
+ for (int _i125 = 0; _i125 < _list124.size; ++_i125)
{
- String _elem129; // required
- _elem129 = iprot.readString();
- struct.skewedColNames.add(_elem129);
+ String _elem126; // required
+ _elem126 = iprot.readString();
+ struct.skewedColNames.add(_elem126);
}
}
struct.setSkewedColNamesIsSet(true);
}
if (incoming.get(1)) {
{
- org.apache.thrift.protocol.TList _list130 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.LIST, iprot.readI32());
- struct.skewedColValues = new ArrayList<List<String>>(_list130.size);
- for (int _i131 = 0; _i131 < _list130.size; ++_i131)
+ org.apache.thrift.protocol.TList _list127 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.LIST, iprot.readI32());
+ struct.skewedColValues = new ArrayList<List<String>>(_list127.size);
+ for (int _i128 = 0; _i128 < _list127.size; ++_i128)
{
- List<String> _elem132; // required
+ List<String> _elem129; // required
{
- org.apache.thrift.protocol.TList _list133 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
- _elem132 = new ArrayList<String>(_list133.size);
- for (int _i134 = 0; _i134 < _list133.size; ++_i134)
+ org.apache.thrift.protocol.TList _list130 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
+ _elem129 = new ArrayList<String>(_list130.size);
+ for (int _i131 = 0; _i131 < _list130.size; ++_i131)
{
- String _elem135; // required
- _elem135 = iprot.readString();
- _elem132.add(_elem135);
+ String _elem132; // required
+ _elem132 = iprot.readString();
+ _elem129.add(_elem132);
}
}
- struct.skewedColValues.add(_elem132);
+ struct.skewedColValues.add(_elem129);
}
}
struct.setSkewedColValuesIsSet(true);
}
if (incoming.get(2)) {
{
- org.apache.thrift.protocol.TMap _map136 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRUCT, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
- struct.skewedColValueLocationMaps = new HashMap<SkewedValueList,String>(2*_map136.size);
- for (int _i137 = 0; _i137 < _map136.size; ++_i137)
+ org.apache.thrift.protocol.TMap _map133 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.LIST, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
+ struct.skewedColValueLocationMaps = new HashMap<List<String>,String>(2*_map133.size);
+ for (int _i134 = 0; _i134 < _map133.size; ++_i134)
{
- SkewedValueList _key138; // required
- String _val139; // required
- _key138 = new SkewedValueList();
- _key138.read(iprot);
- _val139 = iprot.readString();
- struct.skewedColValueLocationMaps.put(_key138, _val139);
+ List<String> _key135; // required
+ String _val136; // required
+ {
+ org.apache.thrift.protocol.TList _list137 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
+ _key135 = new ArrayList<String>(_list137.size);
+ for (int _i138 = 0; _i138 < _list137.size; ++_i138)
+ {
+ String _elem139; // required
+ _elem139 = iprot.readString();
+ _key135.add(_elem139);
+ }
+ }
+ _val136 = iprot.readString();
+ struct.skewedColValueLocationMaps.put(_key135, _val136);
}
}
struct.setSkewedColValueLocationMapsIsSet(true);
Modified: hive/branches/vectorization/metastore/src/gen/thrift/gen-php/metastore/Types.php
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/metastore/src/gen/thrift/gen-php/metastore/Types.php?rev=1522098&r1=1522097&r2=1522098&view=diff
==============================================================================
--- hive/branches/vectorization/metastore/src/gen/thrift/gen-php/metastore/Types.php (original)
+++ hive/branches/vectorization/metastore/src/gen/thrift/gen-php/metastore/Types.php Thu Sep 12 01:21:10 2013
@@ -1815,104 +1815,6 @@ class Order {
}
-class SkewedValueList {
- static $_TSPEC;
-
- public $skewedValueList = null;
-
- public function __construct($vals=null) {
- if (!isset(self::$_TSPEC)) {
- self::$_TSPEC = array(
- 1 => array(
- 'var' => 'skewedValueList',
- 'type' => TType::LST,
- 'etype' => TType::STRING,
- 'elem' => array(
- 'type' => TType::STRING,
- ),
- ),
- );
- }
- if (is_array($vals)) {
- if (isset($vals['skewedValueList'])) {
- $this->skewedValueList = $vals['skewedValueList'];
- }
- }
- }
-
- public function getName() {
- return 'SkewedValueList';
- }
-
- public function read($input)
- {
- $xfer = 0;
- $fname = null;
- $ftype = 0;
- $fid = 0;
- $xfer += $input->readStructBegin($fname);
- while (true)
- {
- $xfer += $input->readFieldBegin($fname, $ftype, $fid);
- if ($ftype == TType::STOP) {
- break;
- }
- switch ($fid)
- {
- case 1:
- if ($ftype == TType::LST) {
- $this->skewedValueList = array();
- $_size87 = 0;
- $_etype90 = 0;
- $xfer += $input->readListBegin($_etype90, $_size87);
- for ($_i91 = 0; $_i91 < $_size87; ++$_i91)
- {
- $elem92 = null;
- $xfer += $input->readString($elem92);
- $this->skewedValueList []= $elem92;
- }
- $xfer += $input->readListEnd();
- } else {
- $xfer += $input->skip($ftype);
- }
- break;
- default:
- $xfer += $input->skip($ftype);
- break;
- }
- $xfer += $input->readFieldEnd();
- }
- $xfer += $input->readStructEnd();
- return $xfer;
- }
-
- public function write($output) {
- $xfer = 0;
- $xfer += $output->writeStructBegin('SkewedValueList');
- if ($this->skewedValueList !== null) {
- if (!is_array($this->skewedValueList)) {
- throw new TProtocolException('Bad type in structure.', TProtocolException::INVALID_DATA);
- }
- $xfer += $output->writeFieldBegin('skewedValueList', TType::LST, 1);
- {
- $output->writeListBegin(TType::STRING, count($this->skewedValueList));
- {
- foreach ($this->skewedValueList as $iter93)
- {
- $xfer += $output->writeString($iter93);
- }
- }
- $output->writeListEnd();
- }
- $xfer += $output->writeFieldEnd();
- }
- $xfer += $output->writeFieldStop();
- $xfer += $output->writeStructEnd();
- return $xfer;
- }
-
-}
-
class SkewedInfo {
static $_TSPEC;
@@ -1946,11 +1848,14 @@ class SkewedInfo {
3 => array(
'var' => 'skewedColValueLocationMaps',
'type' => TType::MAP,
- 'ktype' => TType::STRUCT,
+ 'ktype' => TType::LST,
'vtype' => TType::STRING,
'key' => array(
- 'type' => TType::STRUCT,
- 'class' => '\metastore\SkewedValueList',
+ 'type' => TType::LST,
+ 'etype' => TType::STRING,
+ 'elem' => array(
+ 'type' => TType::STRING,
+ ),
),
'val' => array(
'type' => TType::STRING,
@@ -1993,14 +1898,14 @@ class SkewedInfo {
case 1:
if ($ftype == TType::LST) {
$this->skewedColNames = array();
- $_size94 = 0;
- $_etype97 = 0;
- $xfer += $input->readListBegin($_etype97, $_size94);
- for ($_i98 = 0; $_i98 < $_size94; ++$_i98)
+ $_size87 = 0;
+ $_etype90 = 0;
+ $xfer += $input->readListBegin($_etype90, $_size87);
+ for ($_i91 = 0; $_i91 < $_size87; ++$_i91)
{
- $elem99 = null;
- $xfer += $input->readString($elem99);
- $this->skewedColNames []= $elem99;
+ $elem92 = null;
+ $xfer += $input->readString($elem92);
+ $this->skewedColNames []= $elem92;
}
$xfer += $input->readListEnd();
} else {
@@ -2010,24 +1915,24 @@ class SkewedInfo {
case 2:
if ($ftype == TType::LST) {
$this->skewedColValues = array();
- $_size100 = 0;
- $_etype103 = 0;
- $xfer += $input->readListBegin($_etype103, $_size100);
- for ($_i104 = 0; $_i104 < $_size100; ++$_i104)
+ $_size93 = 0;
+ $_etype96 = 0;
+ $xfer += $input->readListBegin($_etype96, $_size93);
+ for ($_i97 = 0; $_i97 < $_size93; ++$_i97)
{
- $elem105 = null;
- $elem105 = array();
- $_size106 = 0;
- $_etype109 = 0;
- $xfer += $input->readListBegin($_etype109, $_size106);
- for ($_i110 = 0; $_i110 < $_size106; ++$_i110)
+ $elem98 = null;
+ $elem98 = array();
+ $_size99 = 0;
+ $_etype102 = 0;
+ $xfer += $input->readListBegin($_etype102, $_size99);
+ for ($_i103 = 0; $_i103 < $_size99; ++$_i103)
{
- $elem111 = null;
- $xfer += $input->readString($elem111);
- $elem105 []= $elem111;
+ $elem104 = null;
+ $xfer += $input->readString($elem104);
+ $elem98 []= $elem104;
}
$xfer += $input->readListEnd();
- $this->skewedColValues []= $elem105;
+ $this->skewedColValues []= $elem98;
}
$xfer += $input->readListEnd();
} else {
@@ -2037,18 +1942,27 @@ class SkewedInfo {
case 3:
if ($ftype == TType::MAP) {
$this->skewedColValueLocationMaps = array();
- $_size112 = 0;
- $_ktype113 = 0;
- $_vtype114 = 0;
- $xfer += $input->readMapBegin($_ktype113, $_vtype114, $_size112);
- for ($_i116 = 0; $_i116 < $_size112; ++$_i116)
+ $_size105 = 0;
+ $_ktype106 = 0;
+ $_vtype107 = 0;
+ $xfer += $input->readMapBegin($_ktype106, $_vtype107, $_size105);
+ for ($_i109 = 0; $_i109 < $_size105; ++$_i109)
{
- $key117 = new \metastore\SkewedValueList();
- $val118 = '';
- $key117 = new \metastore\SkewedValueList();
- $xfer += $key117->read($input);
- $xfer += $input->readString($val118);
- $this->skewedColValueLocationMaps[$key117] = $val118;
+ $key110 = array();
+ $val111 = '';
+ $key110 = array();
+ $_size112 = 0;
+ $_etype115 = 0;
+ $xfer += $input->readListBegin($_etype115, $_size112);
+ for ($_i116 = 0; $_i116 < $_size112; ++$_i116)
+ {
+ $elem117 = null;
+ $xfer += $input->readString($elem117);
+ $key110 []= $elem117;
+ }
+ $xfer += $input->readListEnd();
+ $xfer += $input->readString($val111);
+ $this->skewedColValueLocationMaps[$key110] = $val111;
}
$xfer += $input->readMapEnd();
} else {
@@ -2076,9 +1990,9 @@ class SkewedInfo {
{
$output->writeListBegin(TType::STRING, count($this->skewedColNames));
{
- foreach ($this->skewedColNames as $iter119)
+ foreach ($this->skewedColNames as $iter118)
{
- $xfer += $output->writeString($iter119);
+ $xfer += $output->writeString($iter118);
}
}
$output->writeListEnd();
@@ -2093,14 +2007,14 @@ class SkewedInfo {
{
$output->writeListBegin(TType::LST, count($this->skewedColValues));
{
- foreach ($this->skewedColValues as $iter120)
+ foreach ($this->skewedColValues as $iter119)
{
{
- $output->writeListBegin(TType::STRING, count($iter120));
+ $output->writeListBegin(TType::STRING, count($iter119));
{
- foreach ($iter120 as $iter121)
+ foreach ($iter119 as $iter120)
{
- $xfer += $output->writeString($iter121);
+ $xfer += $output->writeString($iter120);
}
}
$output->writeListEnd();
@@ -2117,12 +2031,21 @@ class SkewedInfo {
}
$xfer += $output->writeFieldBegin('skewedColValueLocationMaps', TType::MAP, 3);
{
- $output->writeMapBegin(TType::STRUCT, TType::STRING, count($this->skewedColValueLocationMaps));
+ $output->writeMapBegin(TType::LST, TType::STRING, count($this->skewedColValueLocationMaps));
{
- foreach ($this->skewedColValueLocationMaps as $kiter122 => $viter123)
+ foreach ($this->skewedColValueLocationMaps as $kiter121 => $viter122)
{
- $xfer += $kiter122->write($output);
- $xfer += $output->writeString($viter123);
+ {
+ $output->writeListBegin(TType::STRING, count($kiter121));
+ {
+ foreach ($kiter121 as $iter123)
+ {
+ $xfer += $output->writeString($iter123);
+ }
+ }
+ $output->writeListEnd();
+ }
+ $xfer += $output->writeString($viter122);
}
}
$output->writeMapEnd();
Modified: hive/branches/vectorization/metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py?rev=1522098&r1=1522097&r2=1522098&view=diff
==============================================================================
--- hive/branches/vectorization/metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py (original)
+++ hive/branches/vectorization/metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py Thu Sep 12 01:21:10 2013
@@ -1232,74 +1232,6 @@ class Order:
def __ne__(self, other):
return not (self == other)
-class SkewedValueList:
- """
- Attributes:
- - skewedValueList
- """
-
- thrift_spec = (
- None, # 0
- (1, TType.LIST, 'skewedValueList', (TType.STRING,None), None, ), # 1
- )
-
- def __init__(self, skewedValueList=None,):
- self.skewedValueList = skewedValueList
-
- def read(self, iprot):
- if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
- fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
- return
- iprot.readStructBegin()
- while True:
- (fname, ftype, fid) = iprot.readFieldBegin()
- if ftype == TType.STOP:
- break
- if fid == 1:
- if ftype == TType.LIST:
- self.skewedValueList = []
- (_etype90, _size87) = iprot.readListBegin()
- for _i91 in xrange(_size87):
- _elem92 = iprot.readString();
- self.skewedValueList.append(_elem92)
- iprot.readListEnd()
- else:
- iprot.skip(ftype)
- else:
- iprot.skip(ftype)
- iprot.readFieldEnd()
- iprot.readStructEnd()
-
- def write(self, oprot):
- if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
- oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
- return
- oprot.writeStructBegin('SkewedValueList')
- if self.skewedValueList is not None:
- oprot.writeFieldBegin('skewedValueList', TType.LIST, 1)
- oprot.writeListBegin(TType.STRING, len(self.skewedValueList))
- for iter93 in self.skewedValueList:
- oprot.writeString(iter93)
- oprot.writeListEnd()
- oprot.writeFieldEnd()
- oprot.writeFieldStop()
- oprot.writeStructEnd()
-
- def validate(self):
- return
-
-
- def __repr__(self):
- L = ['%s=%r' % (key, value)
- for key, value in self.__dict__.iteritems()]
- return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
-
- def __eq__(self, other):
- return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
-
- def __ne__(self, other):
- return not (self == other)
-
class SkewedInfo:
"""
Attributes:
@@ -1312,7 +1244,7 @@ class SkewedInfo:
None, # 0
(1, TType.LIST, 'skewedColNames', (TType.STRING,None), None, ), # 1
(2, TType.LIST, 'skewedColValues', (TType.LIST,(TType.STRING,None)), None, ), # 2
- (3, TType.MAP, 'skewedColValueLocationMaps', (TType.STRUCT,(SkewedValueList, SkewedValueList.thrift_spec),TType.STRING,None), None, ), # 3
+ (3, TType.MAP, 'skewedColValueLocationMaps', (TType.LIST,(TType.STRING,None),TType.STRING,None), None, ), # 3
)
def __init__(self, skewedColNames=None, skewedColValues=None, skewedColValueLocationMaps=None,):
@@ -1332,37 +1264,41 @@ class SkewedInfo:
if fid == 1:
if ftype == TType.LIST:
self.skewedColNames = []
- (_etype97, _size94) = iprot.readListBegin()
- for _i98 in xrange(_size94):
- _elem99 = iprot.readString();
- self.skewedColNames.append(_elem99)
+ (_etype90, _size87) = iprot.readListBegin()
+ for _i91 in xrange(_size87):
+ _elem92 = iprot.readString();
+ self.skewedColNames.append(_elem92)
iprot.readListEnd()
else:
iprot.skip(ftype)
elif fid == 2:
if ftype == TType.LIST:
self.skewedColValues = []
- (_etype103, _size100) = iprot.readListBegin()
- for _i104 in xrange(_size100):
- _elem105 = []
- (_etype109, _size106) = iprot.readListBegin()
- for _i110 in xrange(_size106):
- _elem111 = iprot.readString();
- _elem105.append(_elem111)
+ (_etype96, _size93) = iprot.readListBegin()
+ for _i97 in xrange(_size93):
+ _elem98 = []
+ (_etype102, _size99) = iprot.readListBegin()
+ for _i103 in xrange(_size99):
+ _elem104 = iprot.readString();
+ _elem98.append(_elem104)
iprot.readListEnd()
- self.skewedColValues.append(_elem105)
+ self.skewedColValues.append(_elem98)
iprot.readListEnd()
else:
iprot.skip(ftype)
elif fid == 3:
if ftype == TType.MAP:
self.skewedColValueLocationMaps = {}
- (_ktype113, _vtype114, _size112 ) = iprot.readMapBegin()
- for _i116 in xrange(_size112):
- _key117 = SkewedValueList()
- _key117.read(iprot)
- _val118 = iprot.readString();
- self.skewedColValueLocationMaps[_key117] = _val118
+ (_ktype106, _vtype107, _size105 ) = iprot.readMapBegin()
+ for _i109 in xrange(_size105):
+ _key110 = []
+ (_etype115, _size112) = iprot.readListBegin()
+ for _i116 in xrange(_size112):
+ _elem117 = iprot.readString();
+ _key110.append(_elem117)
+ iprot.readListEnd()
+ _val111 = iprot.readString();
+ self.skewedColValueLocationMaps[_key110] = _val111
iprot.readMapEnd()
else:
iprot.skip(ftype)
@@ -1379,26 +1315,29 @@ class SkewedInfo:
if self.skewedColNames is not None:
oprot.writeFieldBegin('skewedColNames', TType.LIST, 1)
oprot.writeListBegin(TType.STRING, len(self.skewedColNames))
- for iter119 in self.skewedColNames:
- oprot.writeString(iter119)
+ for iter118 in self.skewedColNames:
+ oprot.writeString(iter118)
oprot.writeListEnd()
oprot.writeFieldEnd()
if self.skewedColValues is not None:
oprot.writeFieldBegin('skewedColValues', TType.LIST, 2)
oprot.writeListBegin(TType.LIST, len(self.skewedColValues))
- for iter120 in self.skewedColValues:
- oprot.writeListBegin(TType.STRING, len(iter120))
- for iter121 in iter120:
- oprot.writeString(iter121)
+ for iter119 in self.skewedColValues:
+ oprot.writeListBegin(TType.STRING, len(iter119))
+ for iter120 in iter119:
+ oprot.writeString(iter120)
oprot.writeListEnd()
oprot.writeListEnd()
oprot.writeFieldEnd()
if self.skewedColValueLocationMaps is not None:
oprot.writeFieldBegin('skewedColValueLocationMaps', TType.MAP, 3)
- oprot.writeMapBegin(TType.STRUCT, TType.STRING, len(self.skewedColValueLocationMaps))
- for kiter122,viter123 in self.skewedColValueLocationMaps.items():
- kiter122.write(oprot)
- oprot.writeString(viter123)
+ oprot.writeMapBegin(TType.LIST, TType.STRING, len(self.skewedColValueLocationMaps))
+ for kiter121,viter122 in self.skewedColValueLocationMaps.items():
+ oprot.writeListBegin(TType.STRING, len(kiter121))
+ for iter123 in kiter121:
+ oprot.writeString(iter123)
+ oprot.writeListEnd()
+ oprot.writeString(viter122)
oprot.writeMapEnd()
oprot.writeFieldEnd()
oprot.writeFieldStop()
Modified: hive/branches/vectorization/metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb?rev=1522098&r1=1522097&r2=1522098&view=diff
==============================================================================
--- hive/branches/vectorization/metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb (original)
+++ hive/branches/vectorization/metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb Thu Sep 12 01:21:10 2013
@@ -289,22 +289,6 @@ class Order
::Thrift::Struct.generate_accessors self
end
-class SkewedValueList
- include ::Thrift::Struct, ::Thrift::Struct_Union
- SKEWEDVALUELIST = 1
-
- FIELDS = {
- SKEWEDVALUELIST => {:type => ::Thrift::Types::LIST, :name => 'skewedValueList', :element => {:type => ::Thrift::Types::STRING}}
- }
-
- def struct_fields; FIELDS; end
-
- def validate
- end
-
- ::Thrift::Struct.generate_accessors self
-end
-
class SkewedInfo
include ::Thrift::Struct, ::Thrift::Struct_Union
SKEWEDCOLNAMES = 1
@@ -314,7 +298,7 @@ class SkewedInfo
FIELDS = {
SKEWEDCOLNAMES => {:type => ::Thrift::Types::LIST, :name => 'skewedColNames', :element => {:type => ::Thrift::Types::STRING}},
SKEWEDCOLVALUES => {:type => ::Thrift::Types::LIST, :name => 'skewedColValues', :element => {:type => ::Thrift::Types::LIST, :element => {:type => ::Thrift::Types::STRING}}},
- SKEWEDCOLVALUELOCATIONMAPS => {:type => ::Thrift::Types::MAP, :name => 'skewedColValueLocationMaps', :key => {:type => ::Thrift::Types::STRUCT, :class => ::SkewedValueList}, :value => {:type => ::Thrift::Types::STRING}}
+ SKEWEDCOLVALUELOCATIONMAPS => {:type => ::Thrift::Types::MAP, :name => 'skewedColValueLocationMaps', :key => {:type => ::Thrift::Types::LIST, :element => {:type => ::Thrift::Types::STRING}}, :value => {:type => ::Thrift::Types::STRING}}
}
def struct_fields; FIELDS; end
Modified: hive/branches/vectorization/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java?rev=1522098&r1=1522097&r2=1522098&view=diff
==============================================================================
--- hive/branches/vectorization/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java (original)
+++ hive/branches/vectorization/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java Thu Sep 12 01:21:10 2013
@@ -33,13 +33,13 @@ import javax.jdo.Query;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.common.ObjectPair;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.Order;
import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.SerDeInfo;
import org.apache.hadoop.hive.metastore.api.SkewedInfo;
-import org.apache.hadoop.hive.metastore.api.SkewedValueList;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.parser.ExpressionTree;
@@ -73,28 +73,59 @@ class MetaStoreDirectSql {
* @param dbName Metastore db name.
* @param tblName Metastore table name.
* @param partNames Partition names to get.
+ * @param max The maximum number of partitions to return.
* @return List of partitions.
*/
public List<Partition> getPartitionsViaSqlFilter(
- String dbName, String tblName, List<String> partNames) throws MetaException {
+ String dbName, String tblName, List<String> partNames, Integer max) throws MetaException {
String list = repeat(",?", partNames.size()).substring(1);
- return getPartitionsViaSqlFilterInternal(dbName, tblName,
- "and PARTITIONS.PART_NAME in (" + list + ")" , partNames, new ArrayList<String>());
+ return getPartitionsViaSqlFilterInternal(dbName, tblName, null,
+ "and PARTITIONS.PART_NAME in (" + list + ")", partNames, new ArrayList<String>(), max);
}
/**
* Gets partitions by using direct SQL queries.
- * @param dbName Metastore db name.
- * @param tblName Metastore table name.
+ * @param table The table.
* @param parser The parsed filter from which the SQL filter will be generated.
+ * @param max The maximum number of partitions to return.
* @return List of partitions.
*/
- public List<Partition> getPartitionsViaSqlFilter(Table table, String dbName,
- String tblName, FilterParser parser) throws MetaException {
+ public List<Partition> getPartitionsViaSqlFilter(
+ Table table, FilterParser parser, Integer max) throws MetaException {
List<String> params = new ArrayList<String>(), joins = new ArrayList<String>();
String sqlFilter = (parser == null) ? null
: PartitionFilterGenerator.generateSqlFilter(table, parser.tree, params, joins);
- return getPartitionsViaSqlFilterInternal(dbName, tblName, sqlFilter, params, joins);
+ return getPartitionsViaSqlFilterInternal(table.getDbName(), table.getTableName(),
+ isViewTable(table), sqlFilter, params, joins, max);
+ }
+
+ /**
+ * Gets all partitions of a table by using direct SQL queries.
+ * @param dbName Metastore db name.
+ * @param tblName Metastore table name.
+ * @param max The maximum number of partitions to return.
+ * @return List of partitions.
+ */
+ public List<Partition> getPartitions(
+ String dbName, String tblName, Integer max) throws MetaException {
+ return getPartitionsViaSqlFilterInternal(dbName, tblName, null,
+ null, new ArrayList<String>(), new ArrayList<String>(), max);
+ }
+
+ private static Boolean isViewTable(Table t) {
+ return t.isSetTableType() ?
+ t.getTableType().equals(TableType.VIRTUAL_VIEW.toString()) : null;
+ }
+
+ private boolean isViewTable(String dbName, String tblName) throws MetaException {
+ String queryText = "select TBL_TYPE from TBLS" +
+ " inner join DBS on TBLS.DB_ID = DBS.DB_ID " +
+ " where TBLS.TBL_NAME = ? and DBS.NAME = ?";
+ Object[] params = new Object[] { tblName, dbName };
+ Query query = pm.newQuery("javax.jdo.query.SQL", queryText);
+ query.setUnique(true);
+ Object result = query.executeWithArray(params);
+ return (result != null) && result.toString().equals(TableType.VIRTUAL_VIEW.toString());
}
/**
@@ -102,16 +133,24 @@ class MetaStoreDirectSql {
* queries created by DN retrieving stuff for each object individually.
* @param dbName Metastore db name.
* @param tblName Metastore table name.
+ * @param isView Whether table is a view. Can be passed as null if not immediately
+ * known, then this method will get it only if necessary.
* @param sqlFilter SQL filter to use. Better be SQL92-compliant. Can be null.
* @param paramsForFilter params for ?-s in SQL filter text. Params must be in order.
* @param joinsForFilter if the filter needs additional join statement, they must be in
* this list. Better be SQL92-compliant.
+ * @param max The maximum number of partitions to return.
* @return List of partition objects. FieldSchema is currently not populated.
*/
- private List<Partition> getPartitionsViaSqlFilterInternal(String dbName,
- String tblName, String sqlFilter, List<String> paramsForFilter,
- List<String> joinsForFilter) throws MetaException {
+ private List<Partition> getPartitionsViaSqlFilterInternal(String dbName, String tblName,
+ Boolean isView, String sqlFilter, List<String> paramsForFilter,
+ List<String> joinsForFilter, Integer max) throws MetaException {
boolean doTrace = LOG.isDebugEnabled();
+ dbName = dbName.toLowerCase();
+ tblName = tblName.toLowerCase();
+ // We have to be mindful of order during filtering if we are not returning all partitions.
+ String orderForFilter = (max != null) ? " order by PART_NAME asc" : "";
+
// Get all simple fields for partitions and related objects, which we can map one-on-one.
// We will do this in 2 queries to use different existing indices for each one.
// We do not get table and DB name, assuming they are the same as we are using to filter.
@@ -125,7 +164,7 @@ class MetaStoreDirectSql {
+ " inner join TBLS on PARTITIONS.TBL_ID = TBLS.TBL_ID "
+ " inner join DBS on TBLS.DB_ID = DBS.DB_ID "
+ join(joinsForFilter, ' ') + " where TBLS.TBL_NAME = ? and DBS.NAME = ?"
- + ((sqlFilter == null) ? "" : " " + sqlFilter);
+ + ((sqlFilter == null) ? "" : " " + sqlFilter) + orderForFilter;
Object[] params = new Object[paramsForFilter.size() + 2];
params[0] = tblName;
params[1] = dbName;
@@ -135,12 +174,16 @@ class MetaStoreDirectSql {
long start = doTrace ? System.nanoTime() : 0;
Query query = pm.newQuery("javax.jdo.query.SQL", queryText);
+ if (max != null) {
+ query.setRange(0, max.shortValue());
+ }
@SuppressWarnings("unchecked")
List<Object> sqlResult = (List<Object>)query.executeWithArray(params);
+ long queryTime = doTrace ? System.nanoTime() : 0;
if (sqlResult.isEmpty()) {
+ timingTrace(doTrace, queryText, start, queryTime);
return new ArrayList<Partition>(); // no partitions, bail early.
}
- long queryTime = doTrace ? System.nanoTime() : 0;
// Prepare StringBuilder for "PART_ID in (...)" to use in future queries.
int sbCapacity = sqlResult.size() * 7; // if there are 100k things => 6 chars, plus comma
@@ -150,10 +193,7 @@ class MetaStoreDirectSql {
partSb.append((Long)partitionId).append(",");
}
String partIds = trimCommaList(partSb);
- if (doTrace) {
- LOG.debug("Direct SQL query in " + (queryTime - start) / 1000000.0 + "ms + " +
- (System.nanoTime() - queryTime) / 1000000.0 + "ms, the query is [ " + queryText + "]");
- }
+ timingTrace(doTrace, queryText, start, queryTime);
// Now get most of the other fields.
queryText =
@@ -190,9 +230,15 @@ class MetaStoreDirectSql {
Long sdId = (Long)fields[1];
Long colId = (Long)fields[2];
Long serdeId = (Long)fields[3];
+ // A partition must have either everything set, or nothing set if it's a view.
if (sdId == null || colId == null || serdeId == null) {
- throw new MetaException("Unexpected null for one of the IDs, SD " + sdId
- + ", column " + colId + ", serde " + serdeId);
+ if (isView == null) {
+ isView = isViewTable(dbName, tblName);
+ }
+ if ((sdId != null || colId != null || serdeId != null) || !isView) {
+ throw new MetaException("Unexpected null for one of the IDs, SD " + sdId + ", column "
+ + colId + ", serde " + serdeId + " for a " + (isView ? "" : "non-") + " view");
+ }
}
Partition part = new Partition();
@@ -206,6 +252,9 @@ class MetaStoreDirectSql {
if (fields[5] != null) part.setLastAccessTime((Integer)fields[5]);
partitions.put(partitionId, part);
+ if (sdId == null) continue; // Probably a view.
+ assert colId != null && serdeId != null;
+
// We assume each partition has an unique SD.
StorageDescriptor sd = new StorageDescriptor();
StorageDescriptor oldSd = sds.put(sdId, sd);
@@ -217,7 +266,7 @@ class MetaStoreDirectSql {
sd.setBucketCols(new ArrayList<String>());
sd.setParameters(new HashMap<String, String>());
sd.setSkewedInfo(new SkewedInfo(new ArrayList<String>(),
- new ArrayList<List<String>>(), new HashMap<SkewedValueList, String>()));
+ new ArrayList<List<String>>(), new HashMap<List<String>, String>()));
sd.setInputFormat((String)fields[6]);
Boolean tmpBoolean = extractSqlBoolean(fields[7]);
if (tmpBoolean != null) sd.setCompressed(tmpBoolean);
@@ -251,14 +300,7 @@ class MetaStoreDirectSql {
sd.setSerdeInfo(serde);
}
query.closeAll();
- if (doTrace) {
- LOG.debug("Direct SQL query in " + (queryTime - start) / 1000000.0 + "ms + " +
- (System.nanoTime() - queryTime) / 1000000.0 + "ms, the query is [ " + queryText + "]");
- }
-
- // Prepare IN (blah) lists for the following queries. Cut off the final ','s.
- String sdIds = trimCommaList(sdSb), serdeIds = trimCommaList(serdeSb),
- colIds = trimCommaList(colsSb);
+ timingTrace(doTrace, queryText, start, queryTime);
// Now get all the one-to-many things. Start with partitions.
queryText = "select PART_ID, PARAM_KEY, PARAM_VALUE from PARTITION_PARAMS where PART_ID in ("
@@ -275,6 +317,14 @@ class MetaStoreDirectSql {
t.addToValues((String)fields[1]);
}});
+ // Prepare IN (blah) lists for the following queries. Cut off the final ','s.
+ if (sdSb.length() == 0) {
+ assert serdeSb.length() == 0 && colsSb.length() == 0;
+ return orderedResult; // No SDs, probably a view.
+ }
+ String sdIds = trimCommaList(sdSb), serdeIds = trimCommaList(serdeSb),
+ colIds = trimCommaList(colsSb);
+
// Get all the stuff for SD. Don't do empty-list check - we expect partitions do have SDs.
queryText = "select SD_ID, PARAM_KEY, PARAM_VALUE from SD_PARAMS where SD_ID in ("
+ sdIds + ") and PARAM_KEY is not null order by SD_ID asc";
@@ -356,28 +406,35 @@ class MetaStoreDirectSql {
+ "where SKEWED_COL_VALUE_LOC_MAP.SD_ID in (" + sdIds + ")"
+ " and SKEWED_COL_VALUE_LOC_MAP.STRING_LIST_ID_KID is not null "
+ "order by SKEWED_COL_VALUE_LOC_MAP.SD_ID asc,"
+ + " SKEWED_STRING_LIST_VALUES.STRING_LIST_ID asc,"
+ " SKEWED_STRING_LIST_VALUES.INTEGER_IDX asc";
+
loopJoinOrderedResult(sds, queryText, 0, new ApplyFunc<StorageDescriptor>() {
private Long currentListId;
- private SkewedValueList currentList;
+ private List<String> currentList;
public void apply(StorageDescriptor t, Object[] fields) {
- if (!t.isSetSkewedInfo()) t.setSkewedInfo(new SkewedInfo());
+ if (!t.isSetSkewedInfo()) {
+ SkewedInfo skewedInfo = new SkewedInfo();
+ skewedInfo.setSkewedColValueLocationMaps(new HashMap<List<String>, String>());
+ t.setSkewedInfo(skewedInfo);
+ }
+ Map<List<String>, String> skewMap = t.getSkewedInfo().getSkewedColValueLocationMaps();
// Note that this is not a typical list accumulator - there's no call to finalize
// the last list. Instead we add list to SD first, as well as locally to add elements.
if (fields[1] == null) {
- currentList = null; // left outer join produced a list with no values
+ currentList = new ArrayList<String>(); // left outer join produced a list with no values
currentListId = null;
- t.getSkewedInfo().putToSkewedColValueLocationMaps(
- new SkewedValueList(), (String)fields[2]);
} else {
long fieldsListId = (Long)fields[1];
if (currentListId == null || fieldsListId != currentListId) {
- currentList = new SkewedValueList();
+ currentList = new ArrayList<String>();
currentListId = fieldsListId;
- t.getSkewedInfo().putToSkewedColValueLocationMaps(currentList, (String)fields[2]);
+ } else {
+ skewMap.remove(currentList); // value based compare.. remove first
}
- currentList.addToSkewedValueList((String)fields[3]);
+ currentList.add((String)fields[3]);
}
+ skewMap.put(currentList, (String)fields[2]);
}});
} // if (hasSkewedColumns)
@@ -403,6 +460,12 @@ class MetaStoreDirectSql {
return orderedResult;
}
+ private void timingTrace(boolean doTrace, String queryText, long start, long queryTime) {
+ if (!doTrace) return;
+ LOG.debug("Direct SQL query in " + (queryTime - start) / 1000000.0 + "ms + " +
+ (System.nanoTime() - queryTime) / 1000000.0 + "ms, the query is [ " + queryText + "]");
+ }
+
private static Boolean extractSqlBoolean(Object value) throws MetaException {
// MySQL has booleans, but e.g. Derby uses 'Y'/'N' mapping. People using derby probably
// don't care about performance anyway, but let's cover the common case.
@@ -472,10 +535,7 @@ class MetaStoreDirectSql {
}
int rv = list.size();
query.closeAll();
- if (doTrace) {
- LOG.debug("Direct SQL query in " + (queryTime - start) / 1000000.0 + "ms + " +
- (System.nanoTime() - queryTime) / 1000000.0 + "ms, the query is [" + queryText + "]");
- }
+ timingTrace(doTrace, queryText, start, queryTime);
return rv;
}
Modified: hive/branches/vectorization/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java?rev=1522098&r1=1522097&r2=1522098&view=diff
==============================================================================
--- hive/branches/vectorization/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java (original)
+++ hive/branches/vectorization/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java Thu Sep 12 01:21:10 2013
@@ -88,7 +88,6 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.metastore.api.Role;
import org.apache.hadoop.hive.metastore.api.SerDeInfo;
import org.apache.hadoop.hive.metastore.api.SkewedInfo;
-import org.apache.hadoop.hive.metastore.api.SkewedValueList;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.metastore.api.StringColumnStatsData;
import org.apache.hadoop.hive.metastore.api.Table;
@@ -1065,13 +1064,13 @@ public class ObjectStore implements RawS
* @param mMap
* @return
*/
- private Map<SkewedValueList, String> covertToSkewedMap(Map<MStringList, String> mMap) {
- Map<SkewedValueList, String> map = null;
+ private Map<List<String>, String> covertToSkewedMap(Map<MStringList, String> mMap) {
+ Map<List<String>, String> map = null;
if (mMap != null) {
- map = new HashMap<SkewedValueList, String>(mMap.size());
+ map = new HashMap<List<String>, String>(mMap.size());
Set<MStringList> keys = mMap.keySet();
for (MStringList key : keys) {
- map.put(new SkewedValueList(new ArrayList<String>(key.getInternalList())), mMap.get(key));
+ map.put(new ArrayList<String>(key.getInternalList()), mMap.get(key));
}
}
return map;
@@ -1082,12 +1081,13 @@ public class ObjectStore implements RawS
* @param mMap
* @return
*/
- private Map<MStringList, String> convertToMapMStringList(Map<SkewedValueList, String> mMap) {
+ private Map<MStringList, String> covertToMapMStringList(Map<List<String>, String> mMap) {
Map<MStringList, String> map = null;
if (mMap != null) {
map = new HashMap<MStringList, String>(mMap.size());
- for (Map.Entry<SkewedValueList, String> entry : mMap.entrySet()) {
- map.put(new MStringList(entry.getKey().getSkewedValueList()), entry.getValue());
+ Set<List<String>> keys = mMap.keySet();
+ for (List<String> key : keys) {
+ map.put(new MStringList(key), mMap.get(key));
}
}
return map;
@@ -1134,7 +1134,7 @@ public class ObjectStore implements RawS
: sd.getSkewedInfo().getSkewedColNames(),
convertToMStringLists((null == sd.getSkewedInfo()) ? null : sd.getSkewedInfo()
.getSkewedColValues()),
- convertToMapMStringList((null == sd.getSkewedInfo()) ? null : sd.getSkewedInfo()
+ covertToMapMStringList((null == sd.getSkewedInfo()) ? null : sd.getSkewedInfo()
.getSkewedColValueLocationMaps()), sd.isStoredAsSubDirectories());
}
@@ -1380,12 +1380,48 @@ public class ObjectStore implements RawS
return success;
}
- public List<Partition> getPartitions(String dbName, String tableName, int max)
- throws MetaException {
- openTransaction();
- List<Partition> parts = convertToParts(listMPartitions(dbName, tableName, max));
- commitTransaction();
- return parts;
+ public List<Partition> getPartitions(
+ String dbName, String tableName, int maxParts) throws MetaException {
+ return getPartitionsInternal(dbName, tableName, maxParts, true, true);
+ }
+
+ protected List<Partition> getPartitionsInternal(String dbName, String tableName,
+ int maxParts, boolean allowSql, boolean allowJdo) throws MetaException {
+ assert allowSql || allowJdo;
+ boolean doTrace = LOG.isDebugEnabled();
+ List<Partition> parts = null;
+ boolean doUseDirectSql = allowSql
+ && HiveConf.getBoolVar(getConf(), ConfVars.METASTORE_TRY_DIRECT_SQL);
+
+ boolean success = false;
+ try {
+ long start = doTrace ? System.nanoTime() : 0;
+ openTransaction();
+ if (doUseDirectSql) {
+ try {
+ Integer max = (maxParts < 0) ? null : maxParts;
+ parts = directSql.getPartitions(dbName, tableName, max);
+ } catch (Exception ex) {
+ handleDirectSqlError(allowJdo, ex);
+ doUseDirectSql = false;
+ start = doTrace ? System.nanoTime() : 0;
+ }
+ }
+
+ if (!doUseDirectSql) {
+ parts = convertToParts(listMPartitions(dbName, tableName, maxParts));
+ }
+ success = commitTransaction();
+ if (doTrace) {
+ LOG.debug(parts.size() + " partition retrieved using " + (doUseDirectSql ? "SQL" : "ORM")
+ + " in " + ((System.nanoTime() - start) / 1000000.0) + "ms");
+ }
+ return parts;
+ } finally {
+ if (!success) {
+ rollbackTransaction();
+ }
+ }
}
@Override
@@ -1659,9 +1695,17 @@ public class ObjectStore implements RawS
@Override
public List<Partition> getPartitionsByNames(String dbName, String tblName,
List<String> partNames) throws MetaException, NoSuchObjectException {
+ return getPartitionsByNamesInternal(dbName, tblName, partNames, true, true);
+ }
+
+ protected List<Partition> getPartitionsByNamesInternal(String dbName, String tblName,
+ List<String> partNames, boolean allowSql, boolean allowJdo)
+ throws MetaException, NoSuchObjectException {
+ assert allowSql || allowJdo;
boolean doTrace = LOG.isDebugEnabled();
List<Partition> results = null;
- boolean doUseDirectSql = HiveConf.getBoolVar(getConf(), ConfVars.METASTORE_TRY_DIRECT_SQL);
+ boolean doUseDirectSql = allowSql
+ && HiveConf.getBoolVar(getConf(), ConfVars.METASTORE_TRY_DIRECT_SQL);
boolean success = false;
try {
@@ -1669,13 +1713,11 @@ public class ObjectStore implements RawS
openTransaction();
if (doUseDirectSql) {
try {
- results = directSql.getPartitionsViaSqlFilter(dbName, tblName, partNames);
+ results = directSql.getPartitionsViaSqlFilter(dbName, tblName, partNames, null);
} catch (Exception ex) {
- LOG.error("Direct SQL failed, falling back to ORM", ex);
+ handleDirectSqlError(allowJdo, ex);
doUseDirectSql = false;
- rollbackTransaction();
start = doTrace ? System.nanoTime() : 0;
- openTransaction();
}
}
@@ -1695,6 +1737,16 @@ public class ObjectStore implements RawS
}
}
+ private void handleDirectSqlError(boolean allowJdo, Exception ex) throws MetaException {
+ LOG.error("Direct SQL failed" + (allowJdo ? ", falling back to ORM" : ""), ex);
+ if (!allowJdo) {
+ if (ex instanceof MetaException) {
+ throw (MetaException)ex;
+ }
+ throw new MetaException(ex.getMessage());
+ }
+ }
+
private List<Partition> getPartitionsViaOrm(
String dbName, String tblName, List<String> partNames) throws MetaException {
StringBuilder sb = new StringBuilder(
@@ -1734,10 +1786,17 @@ public class ObjectStore implements RawS
@Override
public List<Partition> getPartitionsByFilter(String dbName, String tblName,
String filter, short maxParts) throws MetaException, NoSuchObjectException {
+ return getPartitionsByFilterInternal(dbName, tblName, filter, maxParts, true, true);
+ }
+
+ protected List<Partition> getPartitionsByFilterInternal(String dbName, String tblName,
+ String filter, short maxParts, boolean allowSql, boolean allowJdo)
+ throws MetaException, NoSuchObjectException {
+ assert allowSql || allowJdo;
boolean doTrace = LOG.isDebugEnabled();
// There's no portable SQL limit. It doesn't make a lot of sense w/o offset anyway.
- boolean doUseDirectSql = (maxParts < 0)
- && HiveConf.getBoolVar(getConf(), ConfVars.METASTORE_TRY_DIRECT_SQL);
+ boolean doUseDirectSql = allowSql
+ && HiveConf.getBoolVar(getConf(), ConfVars.METASTORE_TRY_DIRECT_SQL);
dbName = dbName.toLowerCase();
tblName = tblName.toLowerCase();
List<Partition> results = null;
@@ -1755,14 +1814,12 @@ public class ObjectStore implements RawS
if (doUseDirectSql) {
try {
Table table = convertToTable(mtable);
- results = directSql.getPartitionsViaSqlFilter(table, dbName, tblName, parser);
+ Integer max = (maxParts < 0) ? null : (int)maxParts;
+ results = directSql.getPartitionsViaSqlFilter(table, parser, max);
} catch (Exception ex) {
- LOG.error("Direct SQL failed, falling back to ORM", ex);
+ handleDirectSqlError(allowJdo, ex);
doUseDirectSql = false;
- rollbackTransaction();
start = doTrace ? System.nanoTime() : 0;
- openTransaction();
- mtable = ensureGetMTable(dbName, tblName); // Detached on rollback, get again.
}
}
if (!doUseDirectSql) {
Modified: hive/branches/vectorization/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingRawStore.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingRawStore.java?rev=1522098&r1=1522097&r2=1522098&view=diff
==============================================================================
--- hive/branches/vectorization/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingRawStore.java (original)
+++ hive/branches/vectorization/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingRawStore.java Thu Sep 12 01:21:10 2013
@@ -23,7 +23,9 @@ import java.lang.reflect.InvocationTarge
import java.lang.reflect.Method;
import java.lang.reflect.Proxy;
import java.lang.reflect.UndeclaredThrowableException;
+import java.util.List;
+import org.apache.commons.lang.ClassUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -70,8 +72,19 @@ public class RetryingRawStore implements
RetryingRawStore handler = new RetryingRawStore(hiveConf, conf, baseClass, id);
- return (RawStore) Proxy.newProxyInstance(RetryingRawStore.class.getClassLoader()
- , baseClass.getInterfaces(), handler);
+ // Look for interfaces on both the class and all base classes.
+ return (RawStore) Proxy.newProxyInstance(RetryingRawStore.class.getClassLoader(),
+ getAllInterfaces(baseClass), handler);
+ }
+
+ private static Class<?>[] getAllInterfaces(Class<?> baseClass) {
+ List interfaces = ClassUtils.getAllInterfaces(baseClass);
+ Class<?>[] result = new Class<?>[interfaces.size()];
+ int i = 0;
+ for (Object o : interfaces) {
+ result[i++] = (Class<?>)o;
+ }
+ return result;
}
private void init() throws MetaException {
Modified: hive/branches/vectorization/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java?rev=1522098&r1=1522097&r2=1522098&view=diff
==============================================================================
--- hive/branches/vectorization/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java (original)
+++ hive/branches/vectorization/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java Thu Sep 12 01:21:10 2013
@@ -23,6 +23,7 @@ import static org.apache.hadoop.hive.met
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.util.AbstractList;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
@@ -365,6 +366,30 @@ public class Warehouse {
static final Pattern pat = Pattern.compile("([^/]+)=([^/]+)");
+ private static final Pattern slash = Pattern.compile("/");
+
+ /**
+ * Extracts values from partition name without the column names.
+ * @param name Partition name.
+ * @param result The result. Must be pre-sized to the expected number of columns.
+ */
+ public static void makeValsFromName(
+ String name, AbstractList<String> result) throws MetaException {
+ assert name != null;
+ String[] parts = slash.split(name, 0);
+ if (parts.length != result.size()) {
+ throw new MetaException(
+ "Expected " + result.size() + " components, got " + parts.length + " (" + name + ")");
+ }
+ for (int i = 0; i < parts.length; ++i) {
+ int eq = parts[i].indexOf('=');
+ if (eq <= 0) {
+ throw new MetaException("Unexpected component " + parts[i]);
+ }
+ result.set(i, unescapePathName(parts[i].substring(eq + 1)));
+ }
+ }
+
public static LinkedHashMap<String, String> makeSpecFromName(String name)
throws MetaException {
if (name == null || name.isEmpty()) {
Modified: hive/branches/vectorization/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java?rev=1522098&r1=1522097&r2=1522098&view=diff
==============================================================================
--- hive/branches/vectorization/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java (original)
+++ hive/branches/vectorization/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java Thu Sep 12 01:21:10 2013
@@ -53,7 +53,6 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.SerDeInfo;
import org.apache.hadoop.hive.metastore.api.SkewedInfo;
-import org.apache.hadoop.hive.metastore.api.SkewedValueList;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.metastore.api.StringColumnStatsData;
import org.apache.hadoop.hive.metastore.api.Table;
@@ -188,8 +187,8 @@ public abstract class TestHiveMetaStore
skewInfor.setSkewedColNames(Arrays.asList("name"));
List<String> skv = Arrays.asList("1");
skewInfor.setSkewedColValues(Arrays.asList(skv));
- Map<SkewedValueList, String> scvlm = new HashMap<SkewedValueList, String>();
- scvlm.put(new SkewedValueList(skv), "location1");
+ Map<List<String>, String> scvlm = new HashMap<List<String>, String>();
+ scvlm.put(skv, "location1");
skewInfor.setSkewedColValueLocationMaps(scvlm);
sd.setSkewedInfo(skewInfor);