You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by cw...@apache.org on 2011/06/10 05:31:08 UTC
svn commit: r1134183 [5/5] - in /hive/trunk/metastore: if/
src/gen/thrift/gen-cpp/
src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/
src/gen/thrift/gen-php/hive_metastore/
src/gen/thrift/gen-py/hive_metastore/ src/gen/thrift/gen-rb/ src...
Modified: hive/trunk/metastore/src/gen/thrift/gen-rb/thrift_hive_metastore.rb
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/gen/thrift/gen-rb/thrift_hive_metastore.rb?rev=1134183&r1=1134182&r2=1134183&view=diff
==============================================================================
--- hive/trunk/metastore/src/gen/thrift/gen-rb/thrift_hive_metastore.rb (original)
+++ hive/trunk/metastore/src/gen/thrift/gen-rb/thrift_hive_metastore.rb Fri Jun 10 03:31:08 2011
@@ -298,6 +298,24 @@ module ThriftHiveMetastore
raise ::Thrift::ApplicationException.new(::Thrift::ApplicationException::MISSING_RESULT, 'get_table failed: unknown result')
end
+ def get_table_objects_by_name(dbname, tbl_names)
+ send_get_table_objects_by_name(dbname, tbl_names)
+ return recv_get_table_objects_by_name()
+ end
+
+ def send_get_table_objects_by_name(dbname, tbl_names)
+ send_message('get_table_objects_by_name', Get_table_objects_by_name_args, :dbname => dbname, :tbl_names => tbl_names)
+ end
+
+ def recv_get_table_objects_by_name()
+ result = receive_message(Get_table_objects_by_name_result)
+ return result.success unless result.success.nil?
+ raise result.o1 unless result.o1.nil?
+ raise result.o2 unless result.o2.nil?
+ raise result.o3 unless result.o3.nil?
+ raise ::Thrift::ApplicationException.new(::Thrift::ApplicationException::MISSING_RESULT, 'get_table_objects_by_name failed: unknown result')
+ end
+
def alter_table(dbname, tbl_name, new_tbl)
send_alter_table(dbname, tbl_name, new_tbl)
recv_alter_table()
@@ -1188,6 +1206,21 @@ module ThriftHiveMetastore
write_result(result, oprot, 'get_table', seqid)
end
+ def process_get_table_objects_by_name(seqid, iprot, oprot)
+ args = read_args(iprot, Get_table_objects_by_name_args)
+ result = Get_table_objects_by_name_result.new()
+ begin
+ result.success = @handler.get_table_objects_by_name(args.dbname, args.tbl_names)
+ rescue MetaException => o1
+ result.o1 = o1
+ rescue InvalidOperationException => o2
+ result.o2 = o2
+ rescue UnknownDBException => o3
+ result.o3 = o3
+ end
+ write_result(result, oprot, 'get_table_objects_by_name', seqid)
+ end
+
def process_alter_table(seqid, iprot, oprot)
args = read_args(iprot, Alter_table_args)
result = Alter_table_result.new()
@@ -2303,6 +2336,46 @@ module ThriftHiveMetastore
::Thrift::Struct.generate_accessors self
end
+ class Get_table_objects_by_name_args
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ DBNAME = 1
+ TBL_NAMES = 2
+
+ FIELDS = {
+ DBNAME => {:type => ::Thrift::Types::STRING, :name => 'dbname'},
+ TBL_NAMES => {:type => ::Thrift::Types::LIST, :name => 'tbl_names', :element => {:type => ::Thrift::Types::STRING}}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class Get_table_objects_by_name_result
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ SUCCESS = 0
+ O1 = 1
+ O2 = 2
+ O3 = 3
+
+ FIELDS = {
+ SUCCESS => {:type => ::Thrift::Types::LIST, :name => 'success', :element => {:type => ::Thrift::Types::STRUCT, :class => Table}},
+ O1 => {:type => ::Thrift::Types::STRUCT, :name => 'o1', :class => MetaException},
+ O2 => {:type => ::Thrift::Types::STRUCT, :name => 'o2', :class => InvalidOperationException},
+ O3 => {:type => ::Thrift::Types::STRUCT, :name => 'o3', :class => UnknownDBException}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
class Alter_table_args
include ::Thrift::Struct, ::Thrift::Struct_Union
DBNAME = 1
Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java?rev=1134183&r1=1134182&r2=1134183&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java Fri Jun 10 03:31:08 2011
@@ -516,6 +516,11 @@ public class HiveMetaStore extends Thrif
return startFunction(function, " : db=" + db + " tbl=" + tbl);
}
+ public String startMultiTableFunction(String function, String db, List<String> tbls) {
+ String tableNames = join(tbls, ",");
+ return startFunction(function, " : db=" + db + " tbls=" + tableNames);
+ }
+
public String startPartitionFunction(String function, String db, String tbl,
List<String> partVals) {
return startFunction(function, " : db=" + db + " tbl=" + tbl
@@ -1134,6 +1139,54 @@ public class HiveMetaStore extends Thrif
return t;
}
+ /**
+ * Gets multiple tables from the hive metastore.
+ * @param dbname
+ * The name of the database in which the tables reside
+ * @param names
+ * The names of the tables to get.
+ *
+ * @return A list of tables whose names are in the the list "names" and
+ * are retrievable from the database specified by "dbnames."
+ * There is no guarantee of the order of the returned tables.
+ * If there are duplicate names, only one instance of the table will be returned.
+ * @throws MetaException
+ * @throws InvalidOperationException
+ * @throws UnknownDBException
+ */
+ public List<Table> get_table_objects_by_name(final String dbname, final List<String> names)
+ throws MetaException, InvalidOperationException, UnknownDBException {
+ List<Table> tables = new ArrayList<Table>();
+ startMultiTableFunction("get_multi_table", dbname, names);
+ try {
+ tables = executeWithRetry(new Command<List<Table>>() {
+ @Override
+ public List<Table> run(RawStore ms) throws Exception {
+ if (dbname == null || dbname.isEmpty()) {
+ throw new UnknownDBException("DB name is null or empty");
+ }
+ if (names == null)
+ {
+ throw new InvalidOperationException(dbname + " cannot find null tables");
+ }
+ List<Table> foundTables = ms.getTableObjectsByName(dbname, names);
+ return foundTables;
+ }
+ });
+ } catch (MetaException e) {
+ throw e;
+ } catch (InvalidOperationException e) {
+ throw e;
+ } catch (UnknownDBException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new MetaException(e.toString());
+ } finally {
+ endFunction("get_multi_table");
+ }
+ return tables;
+ }
+
public boolean set_table_parameters(String dbname, String name,
Map<String, String> params) throws NoSuchObjectException, MetaException {
endFunction(startTableFunction("set_table_parameters", dbname, name));
Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java?rev=1134183&r1=1134182&r2=1134183&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java (original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java Fri Jun 10 03:31:08 2011
@@ -688,6 +688,12 @@ public class HiveMetaStoreClient impleme
return getTable(DEFAULT_DATABASE_NAME, tableName);
}
+ /** {@inheritDoc} */
+ public List<Table> getTableObjectsByName(String dbName, List<String> tableNames)
+ throws MetaException, InvalidOperationException, UnknownDBException, TException {
+ return deepCopyTables(client.get_table_objects_by_name(dbName, tableNames));
+ }
+
/**
* @param name
* @return the type
@@ -1064,7 +1070,7 @@ public class HiveMetaStoreClient impleme
String owner = conf.getUser();
return getDelegationToken(owner, renewerKerberosPrincipalName);
}
-
+
@Override
public String getDelegationToken(String owner, String renewerKerberosPrincipalName) throws
MetaException, TException {
Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java?rev=1134183&r1=1134182&r2=1134183&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java (original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java Fri Jun 10 03:31:08 2011
@@ -207,6 +207,28 @@ public interface IMetaStoreClient {
TException, NoSuchObjectException;
/**
+ *
+ * @param dbName
+ * The database the tables are located in.
+ * @param tableNames
+ * The names of the tables to fetch
+ * @return A list of objects representing the tables.
+ * Only the tables that can be retrieved from the database are returned. For example,
+ * if none of the requested tables could be retrieved, an empty list is returned.
+ * There is no guarantee of ordering of the returned tables.
+ * @throws InvalidOperationException
+ * The input to this operation is invalid (e.g., the list of tables names is null)
+ * @throws UnknownDBException
+ * The requested database could not be fetched.
+ * @throws TException
+ * A thrift communication error occurred
+ * @throws MetaException
+ * Any other errors
+ */
+ public List<Table> getTableObjectsByName(String dbName, List<String> tableNames)
+ throws MetaException, InvalidOperationException, UnknownDBException, TException;
+
+ /**
* @param tableName
* @param dbName
* @param partVals
Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java?rev=1134183&r1=1134182&r2=1134183&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java (original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java Fri Jun 10 03:31:08 2011
@@ -26,9 +26,9 @@ import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Properties;
import java.util.Set;
-import java.util.Map.Entry;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
@@ -71,6 +71,7 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.api.Type;
+import org.apache.hadoop.hive.metastore.api.UnknownDBException;
import org.apache.hadoop.hive.metastore.model.MDBPrivilege;
import org.apache.hadoop.hive.metastore.model.MDatabase;
import org.apache.hadoop.hive.metastore.model.MFieldSchema;
@@ -88,9 +89,9 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.metastore.model.MTableColumnPrivilege;
import org.apache.hadoop.hive.metastore.model.MTablePrivilege;
import org.apache.hadoop.hive.metastore.model.MType;
+import org.apache.hadoop.hive.metastore.parser.ExpressionTree.ANTLRNoCaseStringStream;
import org.apache.hadoop.hive.metastore.parser.FilterLexer;
import org.apache.hadoop.hive.metastore.parser.FilterParser;
-import org.apache.hadoop.hive.metastore.parser.ExpressionTree.ANTLRNoCaseStringStream;
import org.apache.hadoop.util.StringUtils;
/**
@@ -787,6 +788,43 @@ public class ObjectStore implements RawS
return mtbl;
}
+ public List<Table> getTableObjectsByName(String db, List<String> tbl_names)
+ throws MetaException, UnknownDBException {
+ List<Table> tables = new ArrayList<Table>();
+ boolean committed = false;
+ try {
+ openTransaction();
+
+ db = db.toLowerCase().trim();
+ Query dbExistsQuery = pm.newQuery(MDatabase.class, "name == db");
+ dbExistsQuery.declareParameters("java.lang.String db");
+ dbExistsQuery.setUnique(true);
+ dbExistsQuery.setResult("name");
+ String dbNameIfExists = (String) dbExistsQuery.execute(db);
+ if (dbNameIfExists == null || dbNameIfExists.isEmpty()) {
+ throw new UnknownDBException("Could not find database " + db);
+ }
+
+ List<String> lowered_tbl_names = new ArrayList<String>();
+ for (String t : tbl_names) {
+ lowered_tbl_names.add(t.toLowerCase().trim());
+ }
+ Query query = pm.newQuery(MTable.class);
+ query.setFilter("database.name == db && tbl_names.contains(tableName)");
+ query.declareParameters("java.lang.String db, java.util.Collection tbl_names");
+ Collection mtables = (Collection) query.execute(db, lowered_tbl_names);
+ for (Iterator iter = mtables.iterator(); iter.hasNext();) {
+ tables.add(convertToTable((MTable) iter.next()));
+ }
+ committed = commitTransaction();
+ } finally {
+ if (!committed) {
+ rollbackTransaction();
+ }
+ }
+ return tables;
+ }
+
private Table convertToTable(MTable mtbl) throws MetaException {
if (mtbl == null) {
return null;
Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java?rev=1134183&r1=1134182&r2=1134183&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java (original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java Fri Jun 10 03:31:08 2011
@@ -33,6 +33,7 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.metastore.api.Role;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.api.Type;
+import org.apache.hadoop.hive.metastore.api.UnknownDBException;
import org.apache.hadoop.hive.metastore.model.MDBPrivilege;
import org.apache.hadoop.hive.metastore.model.MGlobalPrivilege;
import org.apache.hadoop.hive.metastore.model.MPartitionColumnPrivilege;
@@ -114,6 +115,19 @@ public interface RawStore extends Config
public List<String> getTables(String dbName, String pattern)
throws MetaException;
+ /**
+ * @param dbname
+ * The name of the database from which to retrieve the tables
+ * @param tableNames
+ * The names of the tables to retrieve.
+ * @return A list of the tables retrievable from the database
+ * whose names are in the list tableNames.
+ * If there are duplicate names, only one instance of the table will be returned
+ * @throws MetaException
+ */
+ public List<Table> getTableObjectsByName(String dbname, List<String> tableNames)
+ throws MetaException, UnknownDBException;
+
public List<String> getAllTables(String dbName) throws MetaException;
public abstract List<String> listPartitionNames(String db_name,
Modified: hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java?rev=1134183&r1=1134182&r2=1134183&view=diff
==============================================================================
--- hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java (original)
+++ hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java Fri Jun 10 03:31:08 2011
@@ -45,6 +45,7 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.api.Type;
+import org.apache.hadoop.hive.metastore.api.UnknownDBException;
import org.apache.hadoop.hive.serde.Constants;
import org.apache.hadoop.util.StringUtils;
import org.apache.thrift.TException;
@@ -716,6 +717,59 @@ public abstract class TestHiveMetaStore
(tbl2.getPartitionKeys() == null)
|| (tbl2.getPartitionKeys().size() == 0));
+ //test get_table_objects_by_name functionality
+ ArrayList<String> tableNames = new ArrayList<String>();
+ tableNames.add(tblName2);
+ tableNames.add(tblName);
+ tableNames.add(tblName2);
+ List<Table> foundTables = client.getTableObjectsByName(dbName, tableNames);
+
+ assertEquals(foundTables.size(), 2);
+ for (Table t: foundTables) {
+ if (t.getTableName().equals(tblName2)) {
+ assertEquals(t.getSd().getLocation(), tbl2.getSd().getLocation());
+ } else {
+ assertEquals(t.getTableName(), tblName);
+ assertEquals(t.getSd().getLocation(), tbl.getSd().getLocation());
+ }
+ assertEquals(t.getSd().getCols().size(), typ1.getFields().size());
+ assertEquals(t.getSd().isCompressed(), false);
+ assertEquals(foundTables.get(0).getSd().getNumBuckets(), 1);
+ assertNotNull(t.getSd().getSerdeInfo());
+ assertEquals(t.getDbName(), dbName);
+ }
+
+ tableNames.add(1, "table_that_doesnt_exist");
+ foundTables = client.getTableObjectsByName(dbName, tableNames);
+ assertEquals(foundTables.size(), 2);
+
+ InvalidOperationException ioe = null;
+ try {
+ foundTables = client.getTableObjectsByName(dbName, null);
+ } catch (InvalidOperationException e) {
+ ioe = e;
+ }
+ assertNotNull(ioe);
+ assertTrue("Table not found", ioe.getMessage().contains("null tables"));
+
+ UnknownDBException udbe = null;
+ try {
+ foundTables = client.getTableObjectsByName("db_that_doesnt_exist", tableNames);
+ } catch (UnknownDBException e) {
+ udbe = e;
+ }
+ assertNotNull(udbe);
+ assertTrue("DB not found", udbe.getMessage().contains("not find database db_that_doesnt_exist"));
+
+ udbe = null;
+ try {
+ foundTables = client.getTableObjectsByName("", tableNames);
+ } catch (UnknownDBException e) {
+ udbe = e;
+ }
+ assertNotNull(udbe);
+ assertTrue("DB not found", udbe.getMessage().contains("is null or empty"));
+
FileSystem fs = FileSystem.get((new Path(tbl.getSd().getLocation())).toUri(), hiveConf);
client.dropTable(dbName, tblName);
assertFalse(fs.exists(new Path(tbl.getSd().getLocation())));