You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ng...@apache.org on 2020/12/04 19:28:47 UTC
[hive] branch master updated: HIVE-24397 : Support projection
specification to the table request object(Narayanan Venkateswaran,
reviewed by Vihang K and Naveen Gangam)
This is an automated email from the ASF dual-hosted git repository.
ngangam pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 8696c82 HIVE-24397 : Support projection specification to the table request object(Narayanan Venkateswaran, reviewed by Vihang K and Naveen Gangam)
8696c82 is described below
commit 8696c82d07d303b6dbb69b4d443ab6f2b241b251
Author: Narayanan Venkateswaran <vn...@gmail.com>
AuthorDate: Tue Nov 17 15:35:54 2020 +0530
HIVE-24397 : Support projection specification to the table request object(Narayanan Venkateswaran, reviewed by Vihang K and Naveen Gangam)
---
.../hcatalog/listener/DummyRawStoreFailEvent.java | 6 +
.../gen/thrift/gen-cpp/hive_metastore_types.cpp | 22 +++
.../src/gen/thrift/gen-cpp/hive_metastore_types.h | 10 +-
.../hive/metastore/api/GetTablesRequest.java | 116 +++++++++++++-
.../thrift/gen-php/metastore/GetTablesRequest.php | 29 ++++
.../src/gen/thrift/gen-py/hive_metastore/ttypes.py | 15 +-
.../src/gen/thrift/gen-rb/hive_metastore_types.rb | 4 +-
.../hadoop/hive/metastore/HiveMetaStoreClient.java | 9 +-
.../hadoop/hive/metastore/IMetaStoreClient.java | 23 +++
.../src/main/thrift/hive_metastore.thrift | 5 +-
.../hadoop/hive/metastore/HiveMetaStore.java | 19 ++-
.../apache/hadoop/hive/metastore/ObjectStore.java | 103 +++++++++---
.../org/apache/hadoop/hive/metastore/RawStore.java | 16 ++
.../apache/hadoop/hive/metastore/TableFields.java | 123 +++++++++++++++
.../hadoop/hive/metastore/cache/CachedStore.java | 6 +
.../metastore/DummyRawStoreControlledCommit.java | 6 +
.../metastore/DummyRawStoreForJdoConnection.java | 7 +
.../metastore/HiveMetaStoreClientPreCatalog.java | 7 +
.../hive/metastore/client/TestTablesGetExists.java | 175 +++++++++++++++++++--
19 files changed, 651 insertions(+), 50 deletions(-)
diff --git a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java
index 6148ead..e23851e 100644
--- a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java
+++ b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java
@@ -383,6 +383,12 @@ public class DummyRawStoreFailEvent implements RawStore, Configurable {
}
@Override
+ public List<Table> getTableObjectsByName(String catName, String dbName, List<String> tableNames,
+ GetProjectionsSpec projectionSpec) throws MetaException, UnknownDBException {
+ return objectStore.getTableObjectsByName(catName, dbName, tableNames, projectionSpec);
+ }
+
+ @Override
public List<String> getAllTables(String catName, String dbName) throws MetaException {
return objectStore.getAllTables(catName, dbName);
}
diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/hive_metastore_types.cpp b/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
index 719d68d..abf1ef8 100644
--- a/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
+++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
@@ -30853,6 +30853,11 @@ void GetTablesRequest::__set_processorIdentifier(const std::string& val) {
this->processorIdentifier = val;
__isset.processorIdentifier = true;
}
+
+void GetTablesRequest::__set_projectionSpec(const GetProjectionsSpec& val) {
+ this->projectionSpec = val;
+__isset.projectionSpec = true;
+}
std::ostream& operator<<(std::ostream& out, const GetTablesRequest& obj)
{
obj.printTo(out);
@@ -30954,6 +30959,14 @@ uint32_t GetTablesRequest::read(::apache::thrift::protocol::TProtocol* iprot) {
xfer += iprot->skip(ftype);
}
break;
+ case 7:
+ if (ftype == ::apache::thrift::protocol::T_STRUCT) {
+ xfer += this->projectionSpec.read(iprot);
+ this->__isset.projectionSpec = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
default:
xfer += iprot->skip(ftype);
break;
@@ -31018,6 +31031,11 @@ uint32_t GetTablesRequest::write(::apache::thrift::protocol::TProtocol* oprot) c
xfer += oprot->writeString(this->processorIdentifier);
xfer += oprot->writeFieldEnd();
}
+ if (this->__isset.projectionSpec) {
+ xfer += oprot->writeFieldBegin("projectionSpec", ::apache::thrift::protocol::T_STRUCT, 7);
+ xfer += this->projectionSpec.write(oprot);
+ xfer += oprot->writeFieldEnd();
+ }
xfer += oprot->writeFieldStop();
xfer += oprot->writeStructEnd();
return xfer;
@@ -31031,6 +31049,7 @@ void swap(GetTablesRequest &a, GetTablesRequest &b) {
swap(a.catName, b.catName);
swap(a.processorCapabilities, b.processorCapabilities);
swap(a.processorIdentifier, b.processorIdentifier);
+ swap(a.projectionSpec, b.projectionSpec);
swap(a.__isset, b.__isset);
}
@@ -31041,6 +31060,7 @@ GetTablesRequest::GetTablesRequest(const GetTablesRequest& other1156) {
catName = other1156.catName;
processorCapabilities = other1156.processorCapabilities;
processorIdentifier = other1156.processorIdentifier;
+ projectionSpec = other1156.projectionSpec;
__isset = other1156.__isset;
}
GetTablesRequest& GetTablesRequest::operator=(const GetTablesRequest& other1157) {
@@ -31050,6 +31070,7 @@ GetTablesRequest& GetTablesRequest::operator=(const GetTablesRequest& other1157)
catName = other1157.catName;
processorCapabilities = other1157.processorCapabilities;
processorIdentifier = other1157.processorIdentifier;
+ projectionSpec = other1157.projectionSpec;
__isset = other1157.__isset;
return *this;
}
@@ -31062,6 +31083,7 @@ void GetTablesRequest::printTo(std::ostream& out) const {
out << ", " << "catName="; (__isset.catName ? (out << to_string(catName)) : (out << "<null>"));
out << ", " << "processorCapabilities="; (__isset.processorCapabilities ? (out << to_string(processorCapabilities)) : (out << "<null>"));
out << ", " << "processorIdentifier="; (__isset.processorIdentifier ? (out << to_string(processorIdentifier)) : (out << "<null>"));
+ out << ", " << "projectionSpec="; (__isset.projectionSpec ? (out << to_string(projectionSpec)) : (out << "<null>"));
out << ")";
}
diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/hive_metastore_types.h b/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/hive_metastore_types.h
index 7d5f5c6..397cc85 100644
--- a/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/hive_metastore_types.h
+++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/hive_metastore_types.h
@@ -11415,12 +11415,13 @@ void swap(GetTableResult &a, GetTableResult &b);
std::ostream& operator<<(std::ostream& out, const GetTableResult& obj);
typedef struct _GetTablesRequest__isset {
- _GetTablesRequest__isset() : tblNames(false), capabilities(false), catName(false), processorCapabilities(false), processorIdentifier(false) {}
+ _GetTablesRequest__isset() : tblNames(false), capabilities(false), catName(false), processorCapabilities(false), processorIdentifier(false), projectionSpec(false) {}
bool tblNames :1;
bool capabilities :1;
bool catName :1;
bool processorCapabilities :1;
bool processorIdentifier :1;
+ bool projectionSpec :1;
} _GetTablesRequest__isset;
class GetTablesRequest : public virtual ::apache::thrift::TBase {
@@ -11438,6 +11439,7 @@ class GetTablesRequest : public virtual ::apache::thrift::TBase {
std::string catName;
std::vector<std::string> processorCapabilities;
std::string processorIdentifier;
+ GetProjectionsSpec projectionSpec;
_GetTablesRequest__isset __isset;
@@ -11453,6 +11455,8 @@ class GetTablesRequest : public virtual ::apache::thrift::TBase {
void __set_processorIdentifier(const std::string& val);
+ void __set_projectionSpec(const GetProjectionsSpec& val);
+
bool operator == (const GetTablesRequest & rhs) const
{
if (!(dbName == rhs.dbName))
@@ -11477,6 +11481,10 @@ class GetTablesRequest : public virtual ::apache::thrift::TBase {
return false;
else if (__isset.processorIdentifier && !(processorIdentifier == rhs.processorIdentifier))
return false;
+ if (__isset.projectionSpec != rhs.__isset.projectionSpec)
+ return false;
+ else if (__isset.projectionSpec && !(projectionSpec == rhs.projectionSpec))
+ return false;
return true;
}
bool operator != (const GetTablesRequest &rhs) const {
diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/GetTablesRequest.java b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/GetTablesRequest.java
index c6aebd2..18bdb4f 100644
--- a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/GetTablesRequest.java
+++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/GetTablesRequest.java
@@ -17,6 +17,7 @@ package org.apache.hadoop.hive.metastore.api;
private static final org.apache.thrift.protocol.TField CAT_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("catName", org.apache.thrift.protocol.TType.STRING, (short)4);
private static final org.apache.thrift.protocol.TField PROCESSOR_CAPABILITIES_FIELD_DESC = new org.apache.thrift.protocol.TField("processorCapabilities", org.apache.thrift.protocol.TType.LIST, (short)5);
private static final org.apache.thrift.protocol.TField PROCESSOR_IDENTIFIER_FIELD_DESC = new org.apache.thrift.protocol.TField("processorIdentifier", org.apache.thrift.protocol.TType.STRING, (short)6);
+ private static final org.apache.thrift.protocol.TField PROJECTION_SPEC_FIELD_DESC = new org.apache.thrift.protocol.TField("projectionSpec", org.apache.thrift.protocol.TType.STRUCT, (short)7);
private static final org.apache.thrift.scheme.SchemeFactory STANDARD_SCHEME_FACTORY = new GetTablesRequestStandardSchemeFactory();
private static final org.apache.thrift.scheme.SchemeFactory TUPLE_SCHEME_FACTORY = new GetTablesRequestTupleSchemeFactory();
@@ -27,6 +28,7 @@ package org.apache.hadoop.hive.metastore.api;
private @org.apache.thrift.annotation.Nullable java.lang.String catName; // optional
private @org.apache.thrift.annotation.Nullable java.util.List<java.lang.String> processorCapabilities; // optional
private @org.apache.thrift.annotation.Nullable java.lang.String processorIdentifier; // optional
+ private @org.apache.thrift.annotation.Nullable GetProjectionsSpec projectionSpec; // optional
/** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
public enum _Fields implements org.apache.thrift.TFieldIdEnum {
@@ -35,7 +37,8 @@ package org.apache.hadoop.hive.metastore.api;
CAPABILITIES((short)3, "capabilities"),
CAT_NAME((short)4, "catName"),
PROCESSOR_CAPABILITIES((short)5, "processorCapabilities"),
- PROCESSOR_IDENTIFIER((short)6, "processorIdentifier");
+ PROCESSOR_IDENTIFIER((short)6, "processorIdentifier"),
+ PROJECTION_SPEC((short)7, "projectionSpec");
private static final java.util.Map<java.lang.String, _Fields> byName = new java.util.HashMap<java.lang.String, _Fields>();
@@ -63,6 +66,8 @@ package org.apache.hadoop.hive.metastore.api;
return PROCESSOR_CAPABILITIES;
case 6: // PROCESSOR_IDENTIFIER
return PROCESSOR_IDENTIFIER;
+ case 7: // PROJECTION_SPEC
+ return PROJECTION_SPEC;
default:
return null;
}
@@ -104,7 +109,7 @@ package org.apache.hadoop.hive.metastore.api;
}
// isset id assignments
- private static final _Fields optionals[] = {_Fields.TBL_NAMES,_Fields.CAPABILITIES,_Fields.CAT_NAME,_Fields.PROCESSOR_CAPABILITIES,_Fields.PROCESSOR_IDENTIFIER};
+ private static final _Fields optionals[] = {_Fields.TBL_NAMES,_Fields.CAPABILITIES,_Fields.CAT_NAME,_Fields.PROCESSOR_CAPABILITIES,_Fields.PROCESSOR_IDENTIFIER,_Fields.PROJECTION_SPEC};
public static final java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
static {
java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new java.util.EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
@@ -122,6 +127,8 @@ package org.apache.hadoop.hive.metastore.api;
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))));
tmpMap.put(_Fields.PROCESSOR_IDENTIFIER, new org.apache.thrift.meta_data.FieldMetaData("processorIdentifier", org.apache.thrift.TFieldRequirementType.OPTIONAL,
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
+ tmpMap.put(_Fields.PROJECTION_SPEC, new org.apache.thrift.meta_data.FieldMetaData("projectionSpec", org.apache.thrift.TFieldRequirementType.OPTIONAL,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT , "GetProjectionsSpec")));
metaDataMap = java.util.Collections.unmodifiableMap(tmpMap);
org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(GetTablesRequest.class, metaDataMap);
}
@@ -160,6 +167,9 @@ package org.apache.hadoop.hive.metastore.api;
if (other.isSetProcessorIdentifier()) {
this.processorIdentifier = other.processorIdentifier;
}
+ if (other.isSetProjectionSpec()) {
+ this.projectionSpec = new GetProjectionsSpec(other.projectionSpec);
+ }
}
public GetTablesRequest deepCopy() {
@@ -174,6 +184,7 @@ package org.apache.hadoop.hive.metastore.api;
this.catName = null;
this.processorCapabilities = null;
this.processorIdentifier = null;
+ this.projectionSpec = null;
}
@org.apache.thrift.annotation.Nullable
@@ -352,6 +363,30 @@ package org.apache.hadoop.hive.metastore.api;
}
}
+ @org.apache.thrift.annotation.Nullable
+ public GetProjectionsSpec getProjectionSpec() {
+ return this.projectionSpec;
+ }
+
+ public void setProjectionSpec(@org.apache.thrift.annotation.Nullable GetProjectionsSpec projectionSpec) {
+ this.projectionSpec = projectionSpec;
+ }
+
+ public void unsetProjectionSpec() {
+ this.projectionSpec = null;
+ }
+
+ /** Returns true if field projectionSpec is set (has been assigned a value) and false otherwise */
+ public boolean isSetProjectionSpec() {
+ return this.projectionSpec != null;
+ }
+
+ public void setProjectionSpecIsSet(boolean value) {
+ if (!value) {
+ this.projectionSpec = null;
+ }
+ }
+
public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable java.lang.Object value) {
switch (field) {
case DB_NAME:
@@ -402,6 +437,14 @@ package org.apache.hadoop.hive.metastore.api;
}
break;
+ case PROJECTION_SPEC:
+ if (value == null) {
+ unsetProjectionSpec();
+ } else {
+ setProjectionSpec((GetProjectionsSpec)value);
+ }
+ break;
+
}
}
@@ -426,6 +469,9 @@ package org.apache.hadoop.hive.metastore.api;
case PROCESSOR_IDENTIFIER:
return getProcessorIdentifier();
+ case PROJECTION_SPEC:
+ return getProjectionSpec();
+
}
throw new java.lang.IllegalStateException();
}
@@ -449,6 +495,8 @@ package org.apache.hadoop.hive.metastore.api;
return isSetProcessorCapabilities();
case PROCESSOR_IDENTIFIER:
return isSetProcessorIdentifier();
+ case PROJECTION_SPEC:
+ return isSetProjectionSpec();
}
throw new java.lang.IllegalStateException();
}
@@ -522,6 +570,15 @@ package org.apache.hadoop.hive.metastore.api;
return false;
}
+ boolean this_present_projectionSpec = true && this.isSetProjectionSpec();
+ boolean that_present_projectionSpec = true && that.isSetProjectionSpec();
+ if (this_present_projectionSpec || that_present_projectionSpec) {
+ if (!(this_present_projectionSpec && that_present_projectionSpec))
+ return false;
+ if (!this.projectionSpec.equals(that.projectionSpec))
+ return false;
+ }
+
return true;
}
@@ -553,6 +610,10 @@ package org.apache.hadoop.hive.metastore.api;
if (isSetProcessorIdentifier())
hashCode = hashCode * 8191 + processorIdentifier.hashCode();
+ hashCode = hashCode * 8191 + ((isSetProjectionSpec()) ? 131071 : 524287);
+ if (isSetProjectionSpec())
+ hashCode = hashCode * 8191 + projectionSpec.hashCode();
+
return hashCode;
}
@@ -624,6 +685,16 @@ package org.apache.hadoop.hive.metastore.api;
return lastComparison;
}
}
+ lastComparison = java.lang.Boolean.valueOf(isSetProjectionSpec()).compareTo(other.isSetProjectionSpec());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetProjectionSpec()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.projectionSpec, other.projectionSpec);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
return 0;
}
@@ -702,6 +773,16 @@ package org.apache.hadoop.hive.metastore.api;
}
first = false;
}
+ if (isSetProjectionSpec()) {
+ if (!first) sb.append(", ");
+ sb.append("projectionSpec:");
+ if (this.projectionSpec == null) {
+ sb.append("null");
+ } else {
+ sb.append(this.projectionSpec);
+ }
+ first = false;
+ }
sb.append(")");
return sb.toString();
}
@@ -821,6 +902,15 @@ package org.apache.hadoop.hive.metastore.api;
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
break;
+ case 7: // PROJECTION_SPEC
+ if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
+ struct.projectionSpec = new GetProjectionsSpec();
+ struct.projectionSpec.read(iprot);
+ struct.setProjectionSpecIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
default:
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
@@ -888,6 +978,13 @@ package org.apache.hadoop.hive.metastore.api;
oprot.writeFieldEnd();
}
}
+ if (struct.projectionSpec != null) {
+ if (struct.isSetProjectionSpec()) {
+ oprot.writeFieldBegin(PROJECTION_SPEC_FIELD_DESC);
+ struct.projectionSpec.write(oprot);
+ oprot.writeFieldEnd();
+ }
+ }
oprot.writeFieldStop();
oprot.writeStructEnd();
}
@@ -922,7 +1019,10 @@ package org.apache.hadoop.hive.metastore.api;
if (struct.isSetProcessorIdentifier()) {
optionals.set(4);
}
- oprot.writeBitSet(optionals, 5);
+ if (struct.isSetProjectionSpec()) {
+ optionals.set(5);
+ }
+ oprot.writeBitSet(optionals, 6);
if (struct.isSetTblNames()) {
{
oprot.writeI32(struct.tblNames.size());
@@ -950,6 +1050,9 @@ package org.apache.hadoop.hive.metastore.api;
if (struct.isSetProcessorIdentifier()) {
oprot.writeString(struct.processorIdentifier);
}
+ if (struct.isSetProjectionSpec()) {
+ struct.projectionSpec.write(oprot);
+ }
}
@Override
@@ -957,7 +1060,7 @@ package org.apache.hadoop.hive.metastore.api;
org.apache.thrift.protocol.TTupleProtocol iprot = (org.apache.thrift.protocol.TTupleProtocol) prot;
struct.dbName = iprot.readString();
struct.setDbNameIsSet(true);
- java.util.BitSet incoming = iprot.readBitSet(5);
+ java.util.BitSet incoming = iprot.readBitSet(6);
if (incoming.get(0)) {
{
org.apache.thrift.protocol.TList _list1010 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
@@ -997,6 +1100,11 @@ package org.apache.hadoop.hive.metastore.api;
struct.processorIdentifier = iprot.readString();
struct.setProcessorIdentifierIsSet(true);
}
+ if (incoming.get(5)) {
+ struct.projectionSpec = new GetProjectionsSpec();
+ struct.projectionSpec.read(iprot);
+ struct.setProjectionSpecIsSet(true);
+ }
}
}
diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/GetTablesRequest.php b/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/GetTablesRequest.php
index ba2e78d..1e84a65 100644
--- a/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/GetTablesRequest.php
+++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/GetTablesRequest.php
@@ -60,6 +60,12 @@ class GetTablesRequest
'isRequired' => false,
'type' => TType::STRING,
),
+ 7 => array(
+ 'var' => 'projectionSpec',
+ 'isRequired' => false,
+ 'type' => TType::STRUCT,
+ 'class' => '\metastore\GetProjectionsSpec',
+ ),
);
/**
@@ -86,6 +92,10 @@ class GetTablesRequest
* @var string
*/
public $processorIdentifier = null;
+ /**
+ * @var \metastore\GetProjectionsSpec
+ */
+ public $projectionSpec = null;
public function __construct($vals = null)
{
@@ -108,6 +118,9 @@ class GetTablesRequest
if (isset($vals['processorIdentifier'])) {
$this->processorIdentifier = $vals['processorIdentifier'];
}
+ if (isset($vals['projectionSpec'])) {
+ $this->projectionSpec = $vals['projectionSpec'];
+ }
}
}
@@ -191,6 +204,14 @@ class GetTablesRequest
$xfer += $input->skip($ftype);
}
break;
+ case 7:
+ if ($ftype == TType::STRUCT) {
+ $this->projectionSpec = new \metastore\GetProjectionsSpec();
+ $xfer += $this->projectionSpec->read($input);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
default:
$xfer += $input->skip($ftype);
break;
@@ -252,6 +273,14 @@ class GetTablesRequest
$xfer += $output->writeString($this->processorIdentifier);
$xfer += $output->writeFieldEnd();
}
+ if ($this->projectionSpec !== null) {
+ if (!is_object($this->projectionSpec)) {
+ throw new TProtocolException('Bad type in structure.', TProtocolException::INVALID_DATA);
+ }
+ $xfer += $output->writeFieldBegin('projectionSpec', TType::STRUCT, 7);
+ $xfer += $this->projectionSpec->write($output);
+ $xfer += $output->writeFieldEnd();
+ }
$xfer += $output->writeFieldStop();
$xfer += $output->writeStructEnd();
return $xfer;
diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py b/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py
index 8c292bf..881d402 100644
--- a/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py
+++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py
@@ -17700,17 +17700,19 @@ class GetTablesRequest(object):
- catName
- processorCapabilities
- processorIdentifier
+ - projectionSpec
"""
- def __init__(self, dbName=None, tblNames=None, capabilities=None, catName=None, processorCapabilities=None, processorIdentifier=None,):
+ def __init__(self, dbName=None, tblNames=None, capabilities=None, catName=None, processorCapabilities=None, processorIdentifier=None, projectionSpec=None,):
self.dbName = dbName
self.tblNames = tblNames
self.capabilities = capabilities
self.catName = catName
self.processorCapabilities = processorCapabilities
self.processorIdentifier = processorIdentifier
+ self.projectionSpec = projectionSpec
def read(self, iprot):
if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
@@ -17762,6 +17764,12 @@ class GetTablesRequest(object):
self.processorIdentifier = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString()
else:
iprot.skip(ftype)
+ elif fid == 7:
+ if ftype == TType.STRUCT:
+ self.projectionSpec = GetProjectionsSpec()
+ self.projectionSpec.read(iprot)
+ else:
+ iprot.skip(ftype)
else:
iprot.skip(ftype)
iprot.readFieldEnd()
@@ -17802,6 +17810,10 @@ class GetTablesRequest(object):
oprot.writeFieldBegin('processorIdentifier', TType.STRING, 6)
oprot.writeString(self.processorIdentifier.encode('utf-8') if sys.version_info[0] == 2 else self.processorIdentifier)
oprot.writeFieldEnd()
+ if self.projectionSpec is not None:
+ oprot.writeFieldBegin('projectionSpec', TType.STRUCT, 7)
+ self.projectionSpec.write(oprot)
+ oprot.writeFieldEnd()
oprot.writeFieldStop()
oprot.writeStructEnd()
@@ -28226,6 +28238,7 @@ GetTablesRequest.thrift_spec = (
(4, TType.STRING, 'catName', 'UTF8', None, ), # 4
(5, TType.LIST, 'processorCapabilities', (TType.STRING, 'UTF8', False), None, ), # 5
(6, TType.STRING, 'processorIdentifier', 'UTF8', None, ), # 6
+ (7, TType.STRUCT, 'projectionSpec', [GetProjectionsSpec, None], None, ), # 7
)
all_structs.append(GetTablesResult)
GetTablesResult.thrift_spec = (
diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb b/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb
index 08e359f..b2d6f12 100644
--- a/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb
+++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb
@@ -5040,6 +5040,7 @@ class GetTablesRequest
CATNAME = 4
PROCESSORCAPABILITIES = 5
PROCESSORIDENTIFIER = 6
+ PROJECTIONSPEC = 7
FIELDS = {
DBNAME => {:type => ::Thrift::Types::STRING, :name => 'dbName'},
@@ -5047,7 +5048,8 @@ class GetTablesRequest
CAPABILITIES => {:type => ::Thrift::Types::STRUCT, :name => 'capabilities', :class => ::ClientCapabilities, :optional => true},
CATNAME => {:type => ::Thrift::Types::STRING, :name => 'catName', :optional => true},
PROCESSORCAPABILITIES => {:type => ::Thrift::Types::LIST, :name => 'processorCapabilities', :element => {:type => ::Thrift::Types::STRING}, :optional => true},
- PROCESSORIDENTIFIER => {:type => ::Thrift::Types::STRING, :name => 'processorIdentifier', :optional => true}
+ PROCESSORIDENTIFIER => {:type => ::Thrift::Types::STRING, :name => 'processorIdentifier', :optional => true},
+ PROJECTIONSPEC => {:type => ::Thrift::Types::STRUCT, :name => 'projectionSpec', :class => ::GetProjectionsSpec, :optional => true}
}
def struct_fields; FIELDS; end
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
index 830e2e3..d3cf66a 100644
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
+++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
@@ -2349,18 +2349,25 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
@Override
public List<Table> getTableObjectsByName(String dbName, List<String> tableNames)
throws TException {
- return getTableObjectsByName(getDefaultCatalog(conf), dbName, tableNames);
+ return getTables(getDefaultCatalog(conf), dbName, tableNames, null);
}
@Override
public List<Table> getTableObjectsByName(String catName, String dbName,
List<String> tableNames) throws TException {
+ return getTables(catName, dbName, tableNames, null);
+ }
+
+ @Override
+ public List<Table> getTables(String catName, String dbName, List<String> tableNames,
+ GetProjectionsSpec projectionsSpec) throws TException {
GetTablesRequest req = new GetTablesRequest(dbName);
req.setCatName(catName);
req.setTblNames(tableNames);
req.setCapabilities(version);
if (processorCapabilities != null)
req.setProcessorCapabilities(new ArrayList<String>(Arrays.asList(processorCapabilities)));
+ req.setProjectionSpec(projectionsSpec);
List<Table> tabs = client.get_table_objects_by_name_req(req).getTables();
return deepCopyTables(FilterUtils.filterTablesIfEnabled(isClientFilterEnabled, filterHook, tabs));
}
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
index 0d9e1e9..88b971f 100644
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
+++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
@@ -737,6 +737,29 @@ public interface IMetaStoreClient {
* Get tables as objects (rather than just fetching their names). This is more expensive and
* should only be used if you actually need all the information about the tables.
* @param catName catalog name
+ * @param dbName The database the tables are located in.
+ * @param tableNames The names of the tables to fetch.
+ * @param projectionsSpec The subset of columns that need to be fetched as part of the table object.
+ * @return A list of objects representing the tables.
+ * Only the tables that can be retrieved from the database are returned. For example,
+ * if none of the requested tables could be retrieved, an empty list is returned.
+ * There is no guarantee of ordering of the returned tables.
+ * @throws InvalidOperationException
+ * The input to this operation is invalid (e.g., the list of tables names is null)
+ * @throws UnknownDBException
+ * The requested database could not be fetched.
+ * @throws TException
+ * A thrift communication error occurred
+ * @throws MetaException
+ * Any other errors
+ */
+ List<Table> getTables(String catName, String dbName, List<String> tableNames, GetProjectionsSpec projectionsSpec)
+ throws MetaException, InvalidOperationException, UnknownDBException, TException;
+
+ /**
+ * Get tables as objects (rather than just fetching their names). This is more expensive and
+ * should only be used if you actually need all the information about the tables.
+ * @param catName catalog name
* @param dbName
* The database the tables are located in.
* @param tableNames
diff --git a/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift b/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift
index 0250a8a..223787d 100644
--- a/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift
+++ b/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift
@@ -1454,7 +1454,8 @@ struct GetTablesRequest {
3: optional ClientCapabilities capabilities,
4: optional string catName,
5: optional list<string> processorCapabilities,
- 6: optional string processorIdentifier
+ 6: optional string processorIdentifier,
+ 7: optional GetProjectionsSpec projectionSpec
}
struct GetTablesResult {
@@ -1951,9 +1952,11 @@ struct GetProjectionsSpec {
1: list<string> fieldList;
// SQL-92 compliant regex pattern for param keys to be included
// _ or % wildcards are supported. '_' represent one character and '%' represents 0 or more characters
+ // Currently this is unsupported when fetching tables.
2: string includeParamKeyPattern;
// SQL-92 compliant regex pattern for param keys to be excluded
// _ or % wildcards are supported. '_' represent one character and '%' represents 0 or more characters
+ // Current this is unsupported when fetching tables.
3: string excludeParamKeyPattern;
}
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index d367fed..699864e 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -3571,24 +3571,33 @@ public class HiveMetaStore extends ThriftHiveMetastore {
public List<Table> get_table_objects_by_name(final String dbName, final List<String> tableNames)
throws MetaException, InvalidOperationException, UnknownDBException {
String[] parsedDbName = parseDbName(dbName, conf);
- return getTableObjectsInternal(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableNames, null);
+ return getTableObjectsInternal(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableNames, null, null);
}
@Override
public GetTablesResult get_table_objects_by_name_req(GetTablesRequest req) throws TException {
String catName = req.isSetCatName() ? req.getCatName() : getDefaultCatalog(conf);
return new GetTablesResult(getTableObjectsInternal(catName, req.getDbName(),
- req.getTblNames(), req.getCapabilities()));
+ req.getTblNames(), req.getCapabilities(), req.getProjectionSpec()));
}
private List<Table> getTableObjectsInternal(String catName, String dbName,
List<String> tableNames,
- ClientCapabilities capabilities)
+ ClientCapabilities capabilities,
+ GetProjectionsSpec projectionsSpec)
throws MetaException, InvalidOperationException, UnknownDBException {
if (isInTest) {
assertClientHasCapability(capabilities, ClientCapability.TEST_CAPABILITY,
"Hive tests", "get_table_objects_by_name_req");
}
+
+ if (projectionsSpec != null) {
+ if (!projectionsSpec.isSetFieldList() && (projectionsSpec.isSetIncludeParamKeyPattern() ||
+ projectionsSpec.isSetExcludeParamKeyPattern())) {
+ throw new InvalidOperationException("Include and Exclude Param key are not supported.");
+ }
+ }
+
List<Table> tables = new ArrayList<>();
startMultiTableFunction("get_multi_table", dbName, tableNames);
Exception ex = null;
@@ -3622,11 +3631,11 @@ public class HiveMetaStore extends ThriftHiveMetastore {
while (startIndex < distinctTableNames.size()) {
int endIndex = Math.min(startIndex + tableBatchSize, distinctTableNames.size());
tables.addAll(ms.getTableObjectsByName(catName, dbName, distinctTableNames.subList(
- startIndex, endIndex)));
+ startIndex, endIndex), projectionsSpec));
startIndex = endIndex;
}
for (Table t : tables) {
- if (MetaStoreUtils.isInsertOnlyTableParam(t.getParameters())) {
+ if (t.getParameters() != null && MetaStoreUtils.isInsertOnlyTableParam(t.getParameters())) {
assertClientHasCapability(capabilities, ClientCapability.INSERT_ONLY_TABLES,
"insert-only tables", "get_table_req");
}
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index 5f2cbb3..6061552 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@ -1840,12 +1840,13 @@ public class ObjectStore implements RawStore, Configurable {
}
@Override
- public List<Table> getTableObjectsByName(String catName, String db, List<String> tbl_names)
- throws MetaException, UnknownDBException {
+ public List<Table> getTableObjectsByName(String catName, String db, List<String> tbl_names,
+ GetProjectionsSpec projectionSpec) throws MetaException, UnknownDBException {
List<Table> tables = new ArrayList<>();
boolean committed = false;
- Query dbExistsQuery = null;
Query query = null;
+ List<MTable> mtables = null;
+
try {
openTransaction();
db = normalizeIdentifier(db);
@@ -1855,20 +1856,69 @@ public class ObjectStore implements RawStore, Configurable {
for (String t : tbl_names) {
lowered_tbl_names.add(normalizeIdentifier(t));
}
+
query = pm.newQuery(MTable.class);
query.setFilter("database.name == db && database.catalogName == cat && tbl_names.contains(tableName)");
query.declareParameters("java.lang.String db, java.lang.String cat, java.util.Collection tbl_names");
- Collection mtables = (Collection) query.execute(db, catName, lowered_tbl_names);
+
+ List<String> projectionFields = null;
+
+ // If a projection specification has been set, validate it and translate it to JDO columns.
+ if (projectionSpec != null) {
+ //Validate the projection fields for multi-valued fields.
+ projectionFields = TableFields.getMFieldNames(projectionSpec.getFieldList());
+ }
+
+ // If the JDO translation resulted in valid JDO columns names, use it to create a projection for the JDO query.
+ if (projectionFields != null) {
+ // fetch partially filled tables using result clause
+ query.setResult(Joiner.on(',').join(projectionFields));
+ }
+
+ if (projectionFields == null) {
+ mtables = (List<MTable>) query.execute(db, catName, lowered_tbl_names);
+ } else {
+ if (projectionFields.size() > 1) {
+ // Execute the query to fetch the partial results.
+ List<Object[]> results = (List<Object[]>) query.execute(db, catName, lowered_tbl_names);
+ // Declare the tables array to return the list of tables
+ mtables = new ArrayList<>(results.size());
+ // Iterate through each row of the result and create the MTable object.
+ for (Object[] row : results) {
+ MTable mtable = new MTable();
+ int i = 0;
+ for (Object val : row) {
+ MetaStoreServerUtils.setNestedProperty(mtable, projectionFields.get(i), val, true);
+ i++;
+ }
+ mtables.add(mtable);
+ }
+ } else if (projectionFields.size() == 1) {
+ // Execute the query to fetch the partial results.
+ List<Object> results = (List<Object>) query.execute(db, catName, lowered_tbl_names);
+ // Iterate through each row of the result and create the MTable object.
+ mtables = new ArrayList<>(results.size());
+ for (Object row : results) {
+ MTable mtable = new MTable();
+ MetaStoreServerUtils.setNestedProperty(mtable, projectionFields.get(0), row, true);
+ mtables.add(mtable);
+ }
+ }
+ }
+
if (mtables == null || mtables.isEmpty()) {
- // Need to differentiate between an unmatched pattern and a non-existent database
- dbExistsQuery = pm.newQuery(MDatabase.class, "name == db && catalogName == cat");
- dbExistsQuery.declareParameters("java.lang.String db, java.lang.String cat");
- dbExistsQuery.setUnique(true);
- dbExistsQuery.setResult("name");
- String dbNameIfExists = (String) dbExistsQuery.execute(db, catName);
- if (org.apache.commons.lang3.StringUtils.isEmpty(dbNameIfExists)) {
- throw new UnknownDBException("Could not find database " +
- DatabaseName.getQualified(catName, db));
+ Database tempDB = null;
+ NoSuchObjectException ex = null;
+ try {
+ tempDB = getDatabase(catName, db);
+ } catch(NoSuchObjectException nsoe) {
+ ex = nsoe;
+ }
+
+ if (tempDB == null) {
+ final String errorMessage = (ex == null ? "" : (": " + ex.getMessage()));
+ throw new UnknownDBException("Could not find database " + DatabaseName.getQualified(catName, db) +
+ errorMessage);
}
} else {
for (Iterator iter = mtables.iterator(); iter.hasNext(); ) {
@@ -1876,8 +1926,8 @@ public class ObjectStore implements RawStore, Configurable {
// Retrieve creation metadata if needed
if (TableType.MATERIALIZED_VIEW.toString().equals(tbl.getTableType())) {
tbl.setCreationMetadata(
- convertToCreationMetadata(
- getCreationMetadata(tbl.getCatName(), tbl.getDbName(), tbl.getTableName())));
+ convertToCreationMetadata(
+ getCreationMetadata(tbl.getCatName(), tbl.getDbName(), tbl.getTableName())));
}
tables.add(tbl);
}
@@ -1885,13 +1935,16 @@ public class ObjectStore implements RawStore, Configurable {
committed = commitTransaction();
} finally {
rollbackAndCleanup(committed, query);
- if (dbExistsQuery != null) {
- dbExistsQuery.closeAll();
- }
}
return tables;
}
+ @Override
+ public List<Table> getTableObjectsByName(String catName, String db, List<String> tbl_names)
+ throws MetaException, UnknownDBException {
+ return getTableObjectsByName(catName, db, tbl_names, null);
+ }
+
/** Makes shallow copy of a list to avoid DataNucleus mucking with our objects. */
private <T> List<T> convertList(List<T> dnList) {
return (dnList == null) ? null : Lists.newArrayList(dnList);
@@ -1912,7 +1965,7 @@ public class ObjectStore implements RawStore, Configurable {
// for backwards compatibility with old metastore persistence
if (mtbl.getViewOriginalText() != null) {
tableType = TableType.VIRTUAL_VIEW.toString();
- } else if (Boolean.parseBoolean(mtbl.getParameters().get("EXTERNAL"))) {
+ } else if (mtbl.getParameters() != null && Boolean.parseBoolean(mtbl.getParameters().get("EXTERNAL"))) {
tableType = TableType.EXTERNAL_TABLE.toString();
} else {
tableType = TableType.MANAGED_TABLE.toString();
@@ -1920,11 +1973,11 @@ public class ObjectStore implements RawStore, Configurable {
}
Map<String, String> parameters = convertMap(mtbl.getParameters());
boolean isAcidTable = TxnUtils.isAcidTable(parameters);
- final Table t = new Table(mtbl.getTableName(), mtbl.getDatabase().getName(), mtbl
- .getOwner(), mtbl.getCreateTime(), mtbl.getLastAccessTime(), mtbl
- .getRetention(), convertToStorageDescriptor(mtbl.getSd(), false, isAcidTable),
- convertToFieldSchemas(mtbl.getPartitionKeys()), parameters,
- mtbl.getViewOriginalText(), mtbl.getViewExpandedText(), tableType);
+ final Table t = new Table(mtbl.getTableName(), mtbl.getDatabase() != null ? mtbl.getDatabase().getName() : null,
+ mtbl.getOwner(), mtbl.getCreateTime(), mtbl.getLastAccessTime(), mtbl.getRetention(),
+ convertToStorageDescriptor(mtbl.getSd(), false, isAcidTable),
+ convertToFieldSchemas(mtbl.getPartitionKeys()), parameters, mtbl.getViewOriginalText(),
+ mtbl.getViewExpandedText(), tableType);
if (Strings.isNullOrEmpty(mtbl.getOwnerType())) {
// Before the ownerType exists in an old Hive schema, USER was the default type for owner.
@@ -1936,7 +1989,7 @@ public class ObjectStore implements RawStore, Configurable {
t.setId(mtbl.getId());
t.setRewriteEnabled(mtbl.isRewriteEnabled());
- t.setCatName(mtbl.getDatabase().getCatalogName());
+ t.setCatName(mtbl.getDatabase() != null ? mtbl.getDatabase().getCatalogName() : null);
t.setWriteId(mtbl.getWriteId());
return t;
}
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java
index 5681534..504a22a 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java
@@ -445,6 +445,22 @@ public interface RawStore extends Configurable {
throws MetaException, UnknownDBException;
/**
+ * @param catName catalog name
+ * @param dbname
+ * The name of the database from which to retrieve the tables
+ * @param tableNames
+ * The names of the tables to retrieve.
+ * @param projectionSpec
+ * Projection Specification containing the columns that need to be returned.
+ * @return A list of the tables retrievable from the database
+ * whose names are in the list tableNames.
+ * If there are duplicate names, only one instance of the table will be returned
+ * @throws MetaException failure in querying the RDBMS.
+ */
+ List<Table> getTableObjectsByName(String catName, String dbname, List<String> tableNames,
+ GetProjectionsSpec projectionSpec) throws MetaException, UnknownDBException;
+
+ /**
* Get all tables in a database.
* @param catName catalog name.
* @param dbName database name.
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/TableFields.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/TableFields.java
new file mode 100644
index 0000000..bda58c7
--- /dev/null
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/TableFields.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+
+import java.util.*;
+
+// This class validates the Table fields and creates a mapping to the JDO Column names. When DirectSQL is implemented
+// this class should be integrated with the PartitionProjectionEvaluator class. The PartitionProjectionEvaluator class
+// should be converted to a tempalte that works for both partition and tables.
+public class TableFields {
+ // this map stores all the single valued fields in the table class and maps them to the corresponding
+ // single-valued fields from the MTable class. This map is used to parse the given table fields.
+ private static final ImmutableMap<String, String> allSingleValuedFields = new ImmutableMap.Builder<String, String>()
+ .put("id", "id")
+ .put("tableName", "tableName")
+ .put("dbName", "database.name")
+ .put("owner", "owner")
+ .put("createTime", "createTime")
+ .put("lastAccessTime", "lastAccessTime")
+ .put("retention", "retention")
+ .put("sd.location", "sd.location")
+ .put("sd.inputFormat", "sd.inputFormat")
+ .put("sd.outputFormat", "sd.outputFormat")
+ .put("sd.compressed", "sd.isCompressed")
+ .put("sd.numBuckets", "sd.numBuckets")
+ .put("sd.serdeInfo.name", "sd.serDeInfo.name")
+ .put("sd.serdeInfo.serializationLib", "sd.serDeInfo.serializationLib")
+ .put("sd.serdeInfo.description", "sd.serDeInfo.description")
+ .put("sd.serdeInfo.serializerClass", "sd.serDeInfo.serializerClass")
+ .put("sd.serdeInfo.deserializerClass", "sd.serDeInfo.deserializerClass")
+ .put("sd.serdeInfo.serdeType", "sd.serDeInfo.serdeType")
+ .put("viewOriginalText", "viewOriginalText")
+ .put("viewExpandedText", "viewExpandedText")
+ .put("rewriteEnabled", "rewriteEnabled")
+ .put("tableType", "tableType")
+ .put("writeId", "writeId")
+ .build();
+
+ private static final ImmutableSet<String> allMultiValuedFields = new ImmutableSet.Builder<String>()
+ .add("values")
+ .add("sd.cols.name")
+ .add("sd.cols.type")
+ .add("sd.cols.comment")
+ .add("sd.serdeInfo.parameters")
+ .add("sd.bucketCols")
+ .add("sd.sortCols.col")
+ .add("sd.sortCols.order")
+ .add("sd.parameters")
+ .add("sd.skewedInfo.skewedColNames")
+ .add("sd.skewedInfo.skewedColValues")
+ .add("sd.skewedInfo.skewedColValueLocationMaps")
+ .add("partitionKeys.name")
+ .add("partitionKeys.type")
+ .add("partitionKeys.comment")
+ .add("parameters")
+ .build();
+
+ private static final ImmutableSet<String> allFields = new ImmutableSet.Builder<String>()
+ .addAll(allSingleValuedFields.keySet())
+ .addAll(allMultiValuedFields)
+ .build();
+
+ private static void validate(Collection<String> projectionFields) throws MetaException {
+ Set<String> verify = new HashSet<>(projectionFields);
+ verify.removeAll(allFields);
+ if (verify.size() > 0) {
+ throw new MetaException("Invalid table fields in the projection spec" + Arrays
+ .toString(verify.toArray(new String[verify.size()])));
+ }
+ }
+
+ /**
+ * Given a list of table fields, checks if all the fields requested are single-valued. If all
+ * the fields are single-valued returns list of equivalent MTable fieldnames
+ * which can be used in the setResult clause of a JDO query
+ *
+ * @param fields of tableFields in the projection
+ * @return List of JDO field names which can be used in setResult clause
+ * of a JDO query. Returns null if input fields cannot be used in a setResult clause
+ */
+ public static List<String> getMFieldNames(List<String> fields)
+ throws MetaException {
+ // if there are no fields requested, it means all the fields are requested which include
+ // multi-valued fields.
+ if (fields == null || fields.isEmpty()) {
+ return null;
+ }
+ // throw exception if there are invalid field names
+ validate(fields);
+ // else, check if all the fields are single-valued. In case there are multi-valued fields requested
+ // return null since setResult in JDO doesn't support multi-valued fields
+ if (!allSingleValuedFields.keySet().containsAll(fields)) {
+ return null;
+ }
+
+ List<String> jdoFields = new ArrayList<>(fields.size());
+
+ for (String field : fields) {
+ jdoFields.add(allSingleValuedFields.get(field));
+ }
+ return jdoFields;
+ }
+}
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
index f2d8f1b..9a00e35 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
@@ -1650,6 +1650,12 @@ public class CachedStore implements RawStore, Configurable {
return tables;
}
+ @Override
+ public List<Table> getTableObjectsByName(String catName, String db, List<String> tbl_names,
+ GetProjectionsSpec projectionsSpec) throws MetaException, UnknownDBException {
+ return getTableObjectsByName(catName, db, tbl_names, null);
+ }
+
@Override public List<String> getAllTables(String catName, String dbName) throws MetaException {
return rawStore.getAllTables(catName, dbName);
}
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
index a9fad00..c8f13f1 100644
--- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
@@ -351,6 +351,12 @@ public class DummyRawStoreControlledCommit implements RawStore, Configurable {
}
@Override
+ public List<Table> getTableObjectsByName(String catName, String dbname, List<String> tableNames,
+ GetProjectionsSpec projectionSpec) throws MetaException, UnknownDBException {
+ return Collections.emptyList();
+ }
+
+ @Override
public List<String> getAllTables(String catName, String dbName) throws MetaException {
return objectStore.getAllTables(catName, dbName);
}
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
index fed1a87..32172e1 100644
--- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
@@ -348,6 +348,13 @@ public class DummyRawStoreForJdoConnection implements RawStore {
}
@Override
+ public List<Table> getTableObjectsByName(String catName, String dbname, List<String> tableNames,
+ GetProjectionsSpec projectionSpec) throws MetaException, UnknownDBException {
+
+ return Collections.emptyList();
+ }
+
+ @Override
public List<String> getAllTables(String catName, String dbName) throws MetaException {
return Collections.emptyList();
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java
index 54aaf18..8d6f5b9 100644
--- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java
@@ -3273,6 +3273,13 @@ public class HiveMetaStoreClientPreCatalog implements IMetaStoreClient, AutoClos
}
@Override
+ public List<Table> getTables(String catName, String dbName, List<String> tableNames,
+ GetProjectionsSpec projectionsSpec) throws MetaException,
+ InvalidOperationException, UnknownDBException, TException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
public void updateCreationMetadata(String catName, String dbName, String tableName,
CreationMetadata cm) throws MetaException, TException {
throw new UnsupportedOperationException();
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestTablesGetExists.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestTablesGetExists.java
index e885c0a..1c65eb6 100644
--- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestTablesGetExists.java
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestTablesGetExists.java
@@ -22,13 +22,7 @@ import org.apache.hadoop.hive.metastore.ColumnType;
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
import org.apache.hadoop.hive.metastore.MetaStoreTestUtils;
import org.apache.hadoop.hive.metastore.annotation.MetastoreCheckinTest;
-import org.apache.hadoop.hive.metastore.api.Catalog;
-import org.apache.hadoop.hive.metastore.api.Database;
-import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.apache.hadoop.hive.metastore.api.UnknownDBException;
+import org.apache.hadoop.hive.metastore.api.*;
import org.apache.hadoop.hive.metastore.client.builder.CatalogBuilder;
import org.apache.hadoop.hive.metastore.client.builder.DatabaseBuilder;
import org.apache.hadoop.hive.metastore.client.builder.TableBuilder;
@@ -44,10 +38,7 @@ import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
+import java.util.*;
import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_DATABASE_NAME;
@@ -403,6 +394,168 @@ public class TestTablesGetExists extends MetaStoreClientTest {
}
@Test
+ public void testGetTableObjectsWithProjectionOfSingleField() throws Exception {
+ List<String> tableNames = new ArrayList<>();
+ tableNames.add(testTables[0].getTableName());
+ tableNames.add(testTables[1].getTableName());
+
+ List<String> projectedFields = Collections.singletonList("sd.location");
+ GetProjectionsSpec projectSpec = new GetProjectionsSpec();
+ projectSpec.setFieldList(projectedFields);
+
+ List<Table> tables = client.getTables(null, DEFAULT_DATABASE, tableNames, projectSpec);
+ Assert.assertEquals("Found tables", 2, tables.size());
+
+ for(Table table : tables) {
+ Assert.assertFalse(table.isSetDbName());
+ Assert.assertFalse(table.isSetCatName());
+ Assert.assertFalse(table.isSetTableName());
+ Assert.assertTrue(table.isSetSd());
+ }
+ }
+
+ @Test
+ public void testGetTableObjectsWithNullProjectionSpec() throws Exception {
+ List<String> tableNames = new ArrayList<>();
+ tableNames.add(testTables[0].getTableName());
+ tableNames.add(testTables[1].getTableName());
+
+ List<Table> tables = client.getTables(null, DEFAULT_DATABASE, tableNames, null);
+
+ Assert.assertEquals("Found tables", 2, tables.size());
+ }
+
+ @Test
+ public void testGetTableObjectsWithIncludePattern() throws Exception {
+ List<String> tableNames = new ArrayList<>();
+ tableNames.add(testTables[0].getTableName());
+ tableNames.add(testTables[1].getTableName());
+
+ GetProjectionsSpec projectSpec = new GetProjectionsSpec();
+ projectSpec.setExcludeParamKeyPattern("foo");
+
+ Assert.assertThrows(Exception.class, ()->client.getTables(null, DEFAULT_DATABASE, tableNames, projectSpec));
+ }
+
+ @Test
+ public void testGetTableObjectsWithNonExistentColumn() throws Exception {
+ List<String> tableNames = new ArrayList<>();
+ tableNames.add(testTables[0].getTableName());
+ tableNames.add(testTables[1].getTableName());
+
+ GetProjectionsSpec projectSpec = new GetProjectionsSpec();
+ List<String> projectedFields = Arrays.asList("Invalid1");
+ projectSpec.setFieldList(projectedFields);
+
+ Assert.assertThrows(Exception.class, ()->client.getTables(null, DEFAULT_DATABASE, tableNames, projectSpec));
+ }
+
+
+ @Test
+ public void testGetTableObjectsWithNonExistentColumns() throws Exception {
+ List<String> tableNames = new ArrayList<>();
+ tableNames.add(testTables[0].getTableName());
+ tableNames.add(testTables[1].getTableName());
+
+ GetProjectionsSpec projectSpec = new GetProjectionsSpec();
+ List<String> projectedFields = Arrays.asList("Invalid1", "Invalid2");
+ projectSpec.setFieldList(projectedFields);
+
+ Assert.assertThrows(Exception.class, ()->client.getTables(null, DEFAULT_DATABASE, tableNames, projectSpec));
+ }
+
+ @Test
+ public void testGetTableObjectsWithEmptyProjection() throws Exception {
+ List<String> tableNames = new ArrayList<>();
+ tableNames.add(testTables[0].getTableName());
+ tableNames.add(testTables[1].getTableName());
+
+ GetProjectionsSpec projectSpec = new GetProjectionsSpec();
+ List<String> projectedFields = Arrays.asList();
+ projectSpec.setFieldList(projectedFields);
+
+ List<Table> tables = client.getTables(null, DEFAULT_DATABASE, tableNames, projectSpec);
+
+ Assert.assertEquals("Found tables", 2, tables.size());
+ }
+
+ @Test
+ public void testGetTableObjectsWithProjectionOfMultipleField() throws Exception {
+ List<String> tableNames = new ArrayList<>();
+ tableNames.add(testTables[0].getTableName());
+ tableNames.add(testTables[1].getTableName());
+
+ GetProjectionsSpec projectSpec = new GetProjectionsSpec();
+ List<String> projectedFields = Arrays.asList("dbName", "tableName", "createTime", "lastAccessTime");
+ projectSpec.setFieldList(projectedFields);
+
+ List<Table> tables = client.getTables(null, DEFAULT_DATABASE, tableNames, projectSpec);
+
+ Assert.assertEquals("Found tables", 2, tables.size());
+
+ for(Table table : tables) {
+ Assert.assertTrue(table.isSetDbName());
+ Assert.assertTrue(table.isSetTableName());
+ Assert.assertTrue(table.isSetCreateTime());
+ Assert.assertFalse(table.isSetSd());
+ }
+ }
+
+ @Test
+ public void testGetTableObjectsWithProjectionOfSerDeInfoSingleValuedFields() throws Exception {
+ List<String> tableNames = new ArrayList<>();
+ tableNames.add(testTables[0].getTableName());
+ tableNames.add(testTables[1].getTableName());
+
+ GetProjectionsSpec projectSpec = new GetProjectionsSpec();
+ List<String> projectedFields = Arrays.asList("sd.serdeInfo.name", "sd.serdeInfo.serializationLib", "sd.serdeInfo.description");
+ projectSpec.setFieldList(projectedFields);
+
+ List<Table> tables = client.getTables(null, DEFAULT_DATABASE, tableNames, projectSpec);
+
+ Assert.assertEquals("Found tables", 2, tables.size());
+
+ for(Table table : tables) {
+ Assert.assertFalse(table.isSetDbName());
+ Assert.assertTrue(table.isSetSd());
+ StorageDescriptor sd = table.getSd();
+ Assert.assertFalse(sd.isSetCols());
+ Assert.assertTrue(sd.isSetSerdeInfo());
+ SerDeInfo serDeInfo = sd.getSerdeInfo();
+ Assert.assertTrue(serDeInfo.isSetSerializationLib());
+ }
+ }
+
+ @Test
+ public void testGetTableObjectsWithProjectionOfMultiValuedFields() throws Exception {
+ List<String> tableNames = new ArrayList<>();
+ tableNames.add(testTables[0].getTableName());
+ tableNames.add(testTables[1].getTableName());
+
+ GetProjectionsSpec projectSpec = new GetProjectionsSpec();
+ List<String> projectedFields = Arrays.asList("sd.cols.name", "sd.serdeInfo.name", "sd.serdeInfo.serializationLib", "sd.serdeInfo.parameters");
+ projectSpec.setFieldList(projectedFields);
+
+ List<Table> tables = client.getTables(null, DEFAULT_DATABASE, tableNames, projectSpec);
+
+ Assert.assertEquals("Found tables", 2, tables.size());
+
+ for(Table table : tables) {
+ Assert.assertTrue(table.isSetDbName());
+ Assert.assertTrue(table.isSetCatName());
+ Assert.assertTrue(table.isSetTableName());
+ Assert.assertTrue(table.isSetLastAccessTime());
+ Assert.assertTrue(table.isSetSd());
+ StorageDescriptor sd = table.getSd();
+ Assert.assertTrue(sd.isSetCols());
+ Assert.assertTrue(sd.isSetSerdeInfo());
+ Assert.assertTrue(sd.isSetBucketCols());
+ Assert.assertTrue(sd.isSetCompressed());
+ Assert.assertTrue(sd.isSetInputFormat());
+ }
+ }
+
+ @Test
public void testGetTableObjectsByNameCaseInsensitive() throws Exception {
Table table = testTables[0];