You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by kh...@apache.org on 2013/09/06 02:49:17 UTC
svn commit: r1520466 [17/18] - in /hive/trunk/hcatalog:
core/src/main/java/org/apache/hcatalog/cli/
core/src/main/java/org/apache/hcatalog/cli/SemanticAnalysis/
core/src/main/java/org/apache/hcatalog/common/
core/src/main/java/org/apache/hcatalog/data/...
Added: hive/trunk/hcatalog/server-extensions/src/test/java/org/apache/hcatalog/listener/TestNotificationListener.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/server-extensions/src/test/java/org/apache/hcatalog/listener/TestNotificationListener.java?rev=1520466&view=auto
==============================================================================
--- hive/trunk/hcatalog/server-extensions/src/test/java/org/apache/hcatalog/listener/TestNotificationListener.java (added)
+++ hive/trunk/hcatalog/server-extensions/src/test/java/org/apache/hcatalog/listener/TestNotificationListener.java Fri Sep 6 00:49:14 2013
@@ -0,0 +1,219 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hcatalog.listener;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import javax.jms.Connection;
+import javax.jms.ConnectionFactory;
+import javax.jms.Destination;
+import javax.jms.JMSException;
+import javax.jms.TextMessage;
+import javax.jms.Message;
+import javax.jms.MessageConsumer;
+import javax.jms.MessageListener;
+import javax.jms.Session;
+
+import org.apache.activemq.ActiveMQConnectionFactory;
+import org.apache.hadoop.hive.cli.CliSessionState;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.hadoop.hive.metastore.api.PartitionEventType;
+import org.apache.hadoop.hive.ql.Driver;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hcatalog.common.HCatConstants;
+import org.apache.hcatalog.mapreduce.HCatBaseTest;
+
+import org.apache.hcatalog.messaging.AddPartitionMessage;
+import org.apache.hcatalog.messaging.CreateDatabaseMessage;
+import org.apache.hcatalog.messaging.CreateTableMessage;
+import org.apache.hcatalog.messaging.DropDatabaseMessage;
+import org.apache.hcatalog.messaging.DropPartitionMessage;
+import org.apache.hcatalog.messaging.DropTableMessage;
+import org.apache.hcatalog.messaging.HCatEventMessage;
+import org.apache.hcatalog.messaging.MessageDeserializer;
+import org.apache.hcatalog.messaging.MessageFactory;
+import org.apache.hcatalog.messaging.jms.MessagingUtils;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestNotificationListener extends HCatBaseTest implements MessageListener {
+
+ private List<String> actualMessages = new ArrayList<String>();
+
+ @Before
+ public void setUp() throws Exception {
+ System.setProperty("java.naming.factory.initial",
+ "org.apache.activemq.jndi.ActiveMQInitialContextFactory");
+ System.setProperty("java.naming.provider.url",
+ "vm://localhost?broker.persistent=false");
+ ConnectionFactory connFac = new ActiveMQConnectionFactory(
+ "vm://localhost?broker.persistent=false");
+ Connection conn = connFac.createConnection();
+ conn.start();
+ // We want message to be sent when session commits, thus we run in
+ // transacted mode.
+ Session session = conn.createSession(true, Session.SESSION_TRANSACTED);
+ Destination hcatTopic = session
+ .createTopic(HCatConstants.HCAT_DEFAULT_TOPIC_PREFIX);
+ MessageConsumer consumer1 = session.createConsumer(hcatTopic);
+ consumer1.setMessageListener(this);
+ Destination tblTopic = session
+ .createTopic(HCatConstants.HCAT_DEFAULT_TOPIC_PREFIX + ".mydb.mytbl");
+ MessageConsumer consumer2 = session.createConsumer(tblTopic);
+ consumer2.setMessageListener(this);
+ Destination dbTopic = session
+ .createTopic(HCatConstants.HCAT_DEFAULT_TOPIC_PREFIX + ".mydb");
+ MessageConsumer consumer3 = session.createConsumer(dbTopic);
+ consumer3.setMessageListener(this);
+
+ setUpHiveConf();
+ hiveConf.set(ConfVars.METASTORE_EVENT_LISTENERS.varname,
+ NotificationListener.class.getName());
+ SessionState.start(new CliSessionState(hiveConf));
+ driver = new Driver(hiveConf);
+ client = new HiveMetaStoreClient(hiveConf);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ List<String> expectedMessages = Arrays.asList(
+ HCatConstants.HCAT_CREATE_DATABASE_EVENT,
+ HCatConstants.HCAT_CREATE_TABLE_EVENT,
+ HCatConstants.HCAT_ADD_PARTITION_EVENT,
+ HCatConstants.HCAT_DROP_PARTITION_EVENT,
+ HCatConstants.HCAT_DROP_TABLE_EVENT,
+ HCatConstants.HCAT_DROP_DATABASE_EVENT);
+ Assert.assertEquals(expectedMessages, actualMessages);
+ }
+
+ @Test
+ public void testAMQListener() throws Exception {
+ driver.run("create database mydb");
+ driver.run("use mydb");
+ driver.run("create table mytbl (a string) partitioned by (b string)");
+ driver.run("alter table mytbl add partition(b='2011')");
+ Map<String, String> kvs = new HashMap<String, String>(1);
+ kvs.put("b", "2011");
+ client.markPartitionForEvent("mydb", "mytbl", kvs,
+ PartitionEventType.LOAD_DONE);
+ driver.run("alter table mytbl drop partition(b='2011')");
+ driver.run("drop table mytbl");
+ driver.run("drop database mydb");
+ }
+
+ @Override
+ public void onMessage(Message msg) {
+ String event;
+ try {
+ event = msg.getStringProperty(HCatConstants.HCAT_EVENT);
+ String format = msg.getStringProperty(HCatConstants.HCAT_MESSAGE_FORMAT);
+ String version = msg.getStringProperty(HCatConstants.HCAT_MESSAGE_VERSION);
+ String messageBody = ((TextMessage)msg).getText();
+ actualMessages.add(event);
+ MessageDeserializer deserializer = MessageFactory.getDeserializer(format, version);
+
+ if (event.equals(HCatConstants.HCAT_CREATE_DATABASE_EVENT)) {
+
+ Assert.assertEquals("topic://" + HCatConstants.HCAT_DEFAULT_TOPIC_PREFIX, msg
+ .getJMSDestination().toString());
+ CreateDatabaseMessage message = deserializer.getCreateDatabaseMessage(messageBody);
+ Assert.assertEquals("mydb", message.getDB());
+ HCatEventMessage message2 = MessagingUtils.getMessage(msg);
+ Assert.assertTrue("Unexpected message-type.", message2 instanceof CreateDatabaseMessage);
+ Assert.assertEquals("mydb", message2.getDB());
+ } else if (event.equals(HCatConstants.HCAT_CREATE_TABLE_EVENT)) {
+
+ Assert.assertEquals("topic://hcat.mydb", msg.getJMSDestination().toString());
+ CreateTableMessage message = deserializer.getCreateTableMessage(messageBody);
+ Assert.assertEquals("mytbl", message.getTable());
+ Assert.assertEquals("mydb", message.getDB());
+ HCatEventMessage message2 = MessagingUtils.getMessage(msg);
+ Assert.assertTrue("Unexpected message-type.", message2 instanceof CreateTableMessage);
+ Assert.assertEquals("mydb", message2.getDB());
+ Assert.assertEquals("mytbl", ((CreateTableMessage) message2).getTable());
+ } else if (event.equals(HCatConstants.HCAT_ADD_PARTITION_EVENT)) {
+
+ Assert.assertEquals("topic://hcat.mydb.mytbl", msg.getJMSDestination()
+ .toString());
+ AddPartitionMessage message = deserializer.getAddPartitionMessage(messageBody);
+ Assert.assertEquals("mytbl", message.getTable());
+ Assert.assertEquals("mydb", message.getDB());
+ Assert.assertEquals(1, message.getPartitions().size());
+ Assert.assertEquals("2011", message.getPartitions().get(0).get("b"));
+ HCatEventMessage message2 = MessagingUtils.getMessage(msg);
+ Assert.assertTrue("Unexpected message-type.", message2 instanceof AddPartitionMessage);
+ Assert.assertEquals("mydb", message2.getDB());
+ Assert.assertEquals("mytbl", ((AddPartitionMessage) message2).getTable());
+ Assert.assertEquals(1, ((AddPartitionMessage) message2).getPartitions().size());
+ Assert.assertEquals("2011", ((AddPartitionMessage) message2).getPartitions().get(0).get("b"));
+ } else if (event.equals(HCatConstants.HCAT_DROP_PARTITION_EVENT)) {
+
+ Assert.assertEquals("topic://hcat.mydb.mytbl", msg.getJMSDestination()
+ .toString());
+ DropPartitionMessage message = deserializer.getDropPartitionMessage(messageBody);
+ Assert.assertEquals("mytbl", message.getTable());
+ Assert.assertEquals("mydb", message.getDB());
+ Assert.assertEquals(1, message.getPartitions().size());
+ Assert.assertEquals("2011", message.getPartitions().get(0).get("b"));
+ HCatEventMessage message2 = MessagingUtils.getMessage(msg);
+ Assert.assertTrue("Unexpected message-type.", message2 instanceof DropPartitionMessage);
+ Assert.assertEquals("mydb", message2.getDB());
+ Assert.assertEquals("mytbl", ((DropPartitionMessage) message2).getTable());
+ Assert.assertEquals(1, ((DropPartitionMessage) message2).getPartitions().size());
+ Assert.assertEquals("2011", ((DropPartitionMessage) message2).getPartitions().get(0).get("b"));
+ } else if (event.equals(HCatConstants.HCAT_DROP_TABLE_EVENT)) {
+
+ Assert.assertEquals("topic://hcat.mydb", msg.getJMSDestination().toString());
+ DropTableMessage message = deserializer.getDropTableMessage(messageBody);
+ Assert.assertEquals("mytbl", message.getTable());
+ Assert.assertEquals("mydb", message.getDB());
+ HCatEventMessage message2 = MessagingUtils.getMessage(msg);
+ Assert.assertTrue("Unexpected message-type.", message2 instanceof DropTableMessage);
+ Assert.assertEquals("mydb", message2.getDB());
+ Assert.assertEquals("mytbl", ((DropTableMessage) message2).getTable());
+ } else if (event.equals(HCatConstants.HCAT_DROP_DATABASE_EVENT)) {
+
+ Assert.assertEquals("topic://" + HCatConstants.HCAT_DEFAULT_TOPIC_PREFIX, msg
+ .getJMSDestination().toString());
+ DropDatabaseMessage message = deserializer.getDropDatabaseMessage(messageBody);
+ Assert.assertEquals("mydb", message.getDB());
+ HCatEventMessage message2 = MessagingUtils.getMessage(msg);
+ Assert.assertTrue("Unexpected message-type.", message2 instanceof DropDatabaseMessage);
+ Assert.assertEquals("mydb", message2.getDB());
+ } else if (event.equals(HCatConstants.HCAT_PARTITION_DONE_EVENT)) {
+ // TODO: Fill in when PARTITION_DONE_EVENT is supported.
+ Assert.assertTrue("Unexpected: HCAT_PARTITION_DONE_EVENT not supported (yet).", false);
+ } else {
+ Assert.assertTrue("Unexpected event-type: " + event, false);
+ }
+
+ } catch (JMSException e) {
+ e.printStackTrace(System.err);
+ assert false;
+ }
+ }
+}
Added: hive/trunk/hcatalog/webhcat/java-client/src/main/java/org/apache/hcatalog/api/ConnectionFailureException.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/java-client/src/main/java/org/apache/hcatalog/api/ConnectionFailureException.java?rev=1520466&view=auto
==============================================================================
--- hive/trunk/hcatalog/webhcat/java-client/src/main/java/org/apache/hcatalog/api/ConnectionFailureException.java (added)
+++ hive/trunk/hcatalog/webhcat/java-client/src/main/java/org/apache/hcatalog/api/ConnectionFailureException.java Fri Sep 6 00:49:14 2013
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hcatalog.api;
+
+import org.apache.hcatalog.common.HCatException;
+/**
+ * Class representing exceptions resulting from connection problems
+ * between HCat client and server.
+ */
+public class ConnectionFailureException extends HCatException {
+
+ private static final long serialVersionUID = 1L;
+
+ /**
+ * @param message Exception message.
+ * @param cause The wrapped Throwable that caused this exception.
+ */
+ public ConnectionFailureException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+}
Added: hive/trunk/hcatalog/webhcat/java-client/src/main/java/org/apache/hcatalog/api/HCatAddPartitionDesc.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/java-client/src/main/java/org/apache/hcatalog/api/HCatAddPartitionDesc.java?rev=1520466&view=auto
==============================================================================
--- hive/trunk/hcatalog/webhcat/java-client/src/main/java/org/apache/hcatalog/api/HCatAddPartitionDesc.java (added)
+++ hive/trunk/hcatalog/webhcat/java-client/src/main/java/org/apache/hcatalog/api/HCatAddPartitionDesc.java Fri Sep 6 00:49:14 2013
@@ -0,0 +1,185 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hcatalog.api;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.metastore.MetaStoreUtils;
+import org.apache.hadoop.hive.metastore.Warehouse;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hcatalog.common.HCatException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * The Class HCatAddPartitionDesc helps users in defining partition attributes.
+ */
+public class HCatAddPartitionDesc {
+
+ private static final Logger LOG = LoggerFactory.getLogger(HCatAddPartitionDesc.class);
+ private String tableName;
+ private String dbName;
+ private String location;
+ private Map<String, String> partSpec;
+
+ private HCatAddPartitionDesc(String dbName, String tbl, String loc, Map<String, String> spec) {
+ this.dbName = dbName;
+ this.tableName = tbl;
+ this.location = loc;
+ this.partSpec = spec;
+ }
+
+ /**
+ * Gets the location.
+ *
+ * @return the location
+ */
+ public String getLocation() {
+ return this.location;
+ }
+
+
+ /**
+ * Gets the partition spec.
+ *
+ * @return the partition spec
+ */
+ public Map<String, String> getPartitionSpec() {
+ return this.partSpec;
+ }
+
+ /**
+ * Gets the table name.
+ *
+ * @return the table name
+ */
+ public String getTableName() {
+ return this.tableName;
+ }
+
+ /**
+ * Gets the database name.
+ *
+ * @return the database name
+ */
+ public String getDatabaseName() {
+ return this.dbName;
+ }
+
+ @Override
+ public String toString() {
+ return "HCatAddPartitionDesc ["
+ + (tableName != null ? "tableName=" + tableName + ", " : "tableName=null")
+ + (dbName != null ? "dbName=" + dbName + ", " : "dbName=null")
+ + (location != null ? "location=" + location + ", " : "location=null")
+ + (partSpec != null ? "partSpec=" + partSpec : "partSpec=null") + "]";
+ }
+
+ /**
+ * Creates the builder for specifying attributes.
+ *
+ * @param dbName the db name
+ * @param tableName the table name
+ * @param location the location
+ * @param partSpec the part spec
+ * @return the builder
+ * @throws HCatException
+ */
+ public static Builder create(String dbName, String tableName, String location,
+ Map<String, String> partSpec) throws HCatException {
+ return new Builder(dbName, tableName, location, partSpec);
+ }
+
+ Partition toHivePartition(Table hiveTable) throws HCatException {
+ Partition hivePtn = new Partition();
+ hivePtn.setDbName(this.dbName);
+ hivePtn.setTableName(this.tableName);
+
+ List<String> pvals = new ArrayList<String>();
+ for (FieldSchema field : hiveTable.getPartitionKeys()) {
+ String val = partSpec.get(field.getName());
+ if (val == null || val.length() == 0) {
+ throw new HCatException("create partition: Value for key "
+ + field.getName() + " is null or empty");
+ }
+ pvals.add(val);
+ }
+
+ hivePtn.setValues(pvals);
+ StorageDescriptor sd = new StorageDescriptor(hiveTable.getSd());
+ hivePtn.setSd(sd);
+ hivePtn.setParameters(hiveTable.getParameters());
+ if (this.location != null) {
+ hivePtn.getSd().setLocation(this.location);
+ } else {
+ String partName;
+ try {
+ partName = Warehouse.makePartName(
+ hiveTable.getPartitionKeys(), pvals);
+ LOG.info("Setting partition location to :" + partName);
+ } catch (MetaException e) {
+ throw new HCatException("Exception while creating partition name.", e);
+ }
+ Path partPath = new Path(hiveTable.getSd().getLocation(), partName);
+ hivePtn.getSd().setLocation(partPath.toString());
+ }
+ hivePtn.setCreateTime((int) (System.currentTimeMillis() / 1000));
+ hivePtn.setLastAccessTimeIsSet(false);
+ return hivePtn;
+ }
+
+ public static class Builder {
+
+ private String tableName;
+ private String location;
+ private Map<String, String> values;
+ private String dbName;
+
+ private Builder(String dbName, String tableName, String location, Map<String, String> values) {
+ this.dbName = dbName;
+ this.tableName = tableName;
+ this.location = location;
+ this.values = values;
+ }
+
+ /**
+ * Builds the HCatAddPartitionDesc.
+ *
+ * @return the h cat add partition desc
+ * @throws HCatException
+ */
+ public HCatAddPartitionDesc build() throws HCatException {
+ if (this.dbName == null) {
+ this.dbName = MetaStoreUtils.DEFAULT_DATABASE_NAME;
+ }
+ HCatAddPartitionDesc desc = new HCatAddPartitionDesc(
+ this.dbName, this.tableName, this.location,
+ this.values);
+ return desc;
+ }
+ }
+
+}
Added: hive/trunk/hcatalog/webhcat/java-client/src/main/java/org/apache/hcatalog/api/HCatClient.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/java-client/src/main/java/org/apache/hcatalog/api/HCatClient.java?rev=1520466&view=auto
==============================================================================
--- hive/trunk/hcatalog/webhcat/java-client/src/main/java/org/apache/hcatalog/api/HCatClient.java (added)
+++ hive/trunk/hcatalog/webhcat/java-client/src/main/java/org/apache/hcatalog/api/HCatClient.java Fri Sep 6 00:49:14 2013
@@ -0,0 +1,361 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hcatalog.api;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.JavaUtils;
+import org.apache.hadoop.hive.metastore.api.PartitionEventType;
+import org.apache.hcatalog.common.HCatException;
+import org.apache.hcatalog.data.schema.HCatFieldSchema;
+
+/**
+ * The abstract class HCatClient containing APIs for HCatalog DDL commands.
+ */
+public abstract class HCatClient {
+
+ public enum DropDBMode {RESTRICT, CASCADE}
+
+ public static final String HCAT_CLIENT_IMPL_CLASS = "hcat.client.impl.class";
+
+ /**
+ * Creates an instance of HCatClient.
+ *
+ * @param conf An instance of configuration.
+ * @return An instance of HCatClient.
+ * @throws HCatException
+ */
+ public static HCatClient create(Configuration conf) throws HCatException {
+ HCatClient client = null;
+ String className = conf.get(HCAT_CLIENT_IMPL_CLASS,
+ HCatClientHMSImpl.class.getName());
+ try {
+ Class<? extends HCatClient> clientClass = Class.forName(className,
+ true, JavaUtils.getClassLoader()).asSubclass(
+ HCatClient.class);
+ client = (HCatClient) clientClass.newInstance();
+ } catch (ClassNotFoundException e) {
+ throw new HCatException(
+ "ClassNotFoundException while creating client class.", e);
+ } catch (InstantiationException e) {
+ throw new HCatException(
+ "InstantiationException while creating client class.", e);
+ } catch (IllegalAccessException e) {
+ throw new HCatException(
+ "IllegalAccessException while creating client class.", e);
+ }
+ if (client != null) {
+ client.initialize(conf);
+ }
+ return client;
+ }
+
+ abstract void initialize(Configuration conf) throws HCatException;
+
+ /**
+ * Get all existing databases that match the given
+ * pattern. The matching occurs as per Java regular expressions
+ *
+ * @param pattern java re pattern
+ * @return list of database names
+ * @throws HCatException
+ */
+ public abstract List<String> listDatabaseNamesByPattern(String pattern)
+ throws HCatException;
+
+ /**
+ * Gets the database.
+ *
+ * @param dbName The name of the database.
+ * @return An instance of HCatDatabaseInfo.
+ * @throws HCatException
+ */
+ public abstract HCatDatabase getDatabase(String dbName) throws HCatException;
+
+ /**
+ * Creates the database.
+ *
+ * @param dbInfo An instance of HCatCreateDBDesc.
+ * @throws HCatException
+ */
+ public abstract void createDatabase(HCatCreateDBDesc dbInfo)
+ throws HCatException;
+
+ /**
+ * Drops a database.
+ *
+ * @param dbName The name of the database to delete.
+ * @param ifExists Hive returns an error if the database specified does not exist,
+ * unless ifExists is set to true.
+ * @param mode This is set to either "restrict" or "cascade". Restrict will
+ * remove the schema if all the tables are empty. Cascade removes
+ * everything including data and definitions.
+ * @throws HCatException
+ */
+ public abstract void dropDatabase(String dbName, boolean ifExists,
+ DropDBMode mode) throws HCatException;
+
+ /**
+ * Returns all existing tables from the specified database which match the given
+ * pattern. The matching occurs as per Java regular expressions.
+ * @param dbName The name of the DB (to be searched)
+ * @param tablePattern The regex for the table-name
+ * @return list of table names
+ * @throws HCatException
+ */
+ public abstract List<String> listTableNamesByPattern(String dbName, String tablePattern)
+ throws HCatException;
+
+ /**
+ * Gets the table.
+ *
+ * @param dbName The name of the database.
+ * @param tableName The name of the table.
+ * @return An instance of HCatTableInfo.
+ * @throws HCatException
+ */
+ public abstract HCatTable getTable(String dbName, String tableName)
+ throws HCatException;
+
+ /**
+ * Creates the table.
+ *
+ * @param createTableDesc An instance of HCatCreateTableDesc class.
+ * @throws HCatException
+ */
+ public abstract void createTable(HCatCreateTableDesc createTableDesc) throws HCatException;
+
+ /**
+ * Updates the Table's column schema to the specified definition.
+ *
+ * @param dbName The name of the database.
+ * @param tableName The name of the table.
+ * @param columnSchema The (new) definition of the column schema (i.e. list of fields).
+ *
+ */
+ public abstract void updateTableSchema(String dbName, String tableName, List<HCatFieldSchema> columnSchema)
+ throws HCatException;
+
+ /**
+ * Creates the table like an existing table.
+ *
+ * @param dbName The name of the database.
+ * @param existingTblName The name of the existing table.
+ * @param newTableName The name of the new table.
+ * @param ifNotExists If true, then error related to already table existing is skipped.
+ * @param isExternal Set to "true", if table has be created at a different
+ * location other than default.
+ * @param location The location for the table.
+ * @throws HCatException
+ */
+ public abstract void createTableLike(String dbName, String existingTblName,
+ String newTableName, boolean ifNotExists, boolean isExternal,
+ String location) throws HCatException;
+
+ /**
+ * Drop table.
+ *
+ * @param dbName The name of the database.
+ * @param tableName The name of the table.
+ * @param ifExists Hive returns an error if the database specified does not exist,
+ * unless ifExists is set to true.
+ * @throws HCatException
+ */
+ public abstract void dropTable(String dbName, String tableName,
+ boolean ifExists) throws HCatException;
+
+ /**
+ * Renames a table.
+ *
+ * @param dbName The name of the database.
+ * @param oldName The name of the table to be renamed.
+ * @param newName The new name of the table.
+ * @throws HCatException
+ */
+ public abstract void renameTable(String dbName, String oldName,
+ String newName) throws HCatException;
+
+ /**
+ * Gets all the partitions.
+ *
+ * @param dbName The name of the database.
+ * @param tblName The name of the table.
+ * @return A list of partitions.
+ * @throws HCatException
+ */
+ public abstract List<HCatPartition> getPartitions(String dbName, String tblName)
+ throws HCatException;
+
+ /**
+ * Gets all the partitions that match the specified (and possibly partial) partition specification.
+ * A partial partition-specification is one where not all partition-keys have associated values. For example,
+ * for a table ('myDb.myTable') with 2 partition keys (dt string, region string),
+ * if for each dt ('20120101', '20120102', etc.) there can exist 3 regions ('us', 'uk', 'in'), then,
+ * 1. Complete partition spec: getPartitions('myDb', 'myTable', {dt='20120101', region='us'}) would return 1 partition.
+ * 2. Partial partition spec: getPartitions('myDb', 'myTable', {dt='20120101'}) would return all 3 partitions,
+ * with dt='20120101' (i.e. region = 'us', 'uk' and 'in').
+ * @param dbName The name of the database.
+ * @param tblName The name of the table.
+ * @param partitionSpec The partition specification. (Need not include all partition keys.)
+ * @return A list of partitions.
+ * @throws HCatException
+ */
+ public abstract List<HCatPartition> getPartitions(String dbName, String tblName, Map<String, String> partitionSpec)
+ throws HCatException;
+
+ /**
+ * Gets the partition.
+ *
+ * @param dbName The database name.
+ * @param tableName The table name.
+ * @param partitionSpec The partition specification, {[col_name,value],[col_name2,value2]}. All partition-key-values
+ * must be specified.
+ * @return An instance of HCatPartitionInfo.
+ * @throws HCatException
+ */
+ public abstract HCatPartition getPartition(String dbName, String tableName,
+ Map<String, String> partitionSpec) throws HCatException;
+
+ /**
+ * Adds the partition.
+ *
+ * @param partInfo An instance of HCatAddPartitionDesc.
+ * @throws HCatException
+ */
+ public abstract void addPartition(HCatAddPartitionDesc partInfo)
+ throws HCatException;
+
+ /**
+ * Adds a list of partitions.
+ *
+ * @param partInfoList A list of HCatAddPartitionDesc.
+ * @return The number of partitions added.
+ * @throws HCatException
+ */
+ public abstract int addPartitions(List<HCatAddPartitionDesc> partInfoList)
+ throws HCatException;
+
+ /**
+ * Drops partition(s) that match the specified (and possibly partial) partition specification.
+ * A partial partition-specification is one where not all partition-keys have associated values. For example,
+ * for a table ('myDb.myTable') with 2 partition keys (dt string, region string),
+ * if for each dt ('20120101', '20120102', etc.) there can exist 3 regions ('us', 'uk', 'in'), then,
+ * 1. Complete partition spec: dropPartitions('myDb', 'myTable', {dt='20120101', region='us'}) would drop 1 partition.
+ * 2. Partial partition spec: dropPartitions('myDb', 'myTable', {dt='20120101'}) would drop all 3 partitions,
+ * with dt='20120101' (i.e. region = 'us', 'uk' and 'in').
+ * @param dbName The database name.
+ * @param tableName The table name.
+ * @param partitionSpec The partition specification, {[col_name,value],[col_name2,value2]}.
+ * @param ifExists Hive returns an error if the partition specified does not exist, unless ifExists is set to true.
+ * @throws HCatException,ConnectionFailureException
+ */
+ public abstract void dropPartitions(String dbName, String tableName,
+ Map<String, String> partitionSpec, boolean ifExists)
+ throws HCatException;
+
+ /**
+ * List partitions by filter.
+ *
+ * @param dbName The database name.
+ * @param tblName The table name.
+ * @param filter The filter string,
+ * for example "part1 = \"p1_abc\" and part2 <= "\p2_test\"". Filtering can
+ * be done only on string partition keys.
+ * @return list of partitions
+ * @throws HCatException
+ */
+ public abstract List<HCatPartition> listPartitionsByFilter(String dbName, String tblName,
+ String filter) throws HCatException;
+
+ /**
+ * Mark partition for event.
+ *
+ * @param dbName The database name.
+ * @param tblName The table name.
+ * @param partKVs the key-values associated with the partition.
+ * @param eventType the event type
+ * @throws HCatException
+ */
+ public abstract void markPartitionForEvent(String dbName, String tblName,
+ Map<String, String> partKVs, PartitionEventType eventType)
+ throws HCatException;
+
+ /**
+ * Checks if a partition is marked for event.
+ *
+ * @param dbName the db name
+ * @param tblName the table name
+ * @param partKVs the key-values associated with the partition.
+ * @param eventType the event type
+ * @return true, if is partition marked for event
+ * @throws HCatException
+ */
+ public abstract boolean isPartitionMarkedForEvent(String dbName, String tblName,
+ Map<String, String> partKVs, PartitionEventType eventType)
+ throws HCatException;
+
+ /**
+ * Gets the delegation token.
+ *
+ * @param owner the owner
+ * @param renewerKerberosPrincipalName the renewer kerberos principal name
+ * @return the delegation token
+ * @throws HCatException,ConnectionFailureException
+ */
+ public abstract String getDelegationToken(String owner,
+ String renewerKerberosPrincipalName) throws HCatException;
+
+ /**
+ * Renew delegation token.
+ *
+ * @param tokenStrForm the token string
+ * @return the new expiration time
+ * @throws HCatException
+ */
+ public abstract long renewDelegationToken(String tokenStrForm)
+ throws HCatException;
+
+ /**
+ * Cancel delegation token.
+ *
+ * @param tokenStrForm the token string
+ * @throws HCatException
+ */
+ public abstract void cancelDelegationToken(String tokenStrForm)
+ throws HCatException;
+
+ /**
+ * Retrieve Message-bus topic for a table.
+ *
+ * @param dbName The name of the DB.
+ * @param tableName The name of the table.
+ * @return Topic-name for the message-bus on which messages will be sent for the specified table.
+ * By default, this is set to <db-name>.<table-name>. Returns null when not set.
+ */
+ public abstract String getMessageBusTopicName(String dbName, String tableName) throws HCatException;
+
+ /**
+ * Close the hcatalog client.
+ *
+ * @throws HCatException
+ */
+ public abstract void close() throws HCatException;
+}
Added: hive/trunk/hcatalog/webhcat/java-client/src/main/java/org/apache/hcatalog/api/HCatClientHMSImpl.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/java-client/src/main/java/org/apache/hcatalog/api/HCatClientHMSImpl.java?rev=1520466&view=auto
==============================================================================
--- hive/trunk/hcatalog/webhcat/java-client/src/main/java/org/apache/hcatalog/api/HCatClientHMSImpl.java (added)
+++ hive/trunk/hcatalog/webhcat/java-client/src/main/java/org/apache/hcatalog/api/HCatClientHMSImpl.java Fri Sep 6 00:49:14 2013
@@ -0,0 +1,723 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hcatalog.api;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.hadoop.hive.metastore.MetaStoreUtils;
+import org.apache.hadoop.hive.metastore.TableType;
+import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
+import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.metastore.api.InvalidObjectException;
+import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.PartitionEventType;
+import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.api.UnknownDBException;
+import org.apache.hadoop.hive.metastore.api.UnknownTableException;
+import org.apache.hcatalog.common.HCatConstants;
+import org.apache.hcatalog.common.HCatException;
+import org.apache.hcatalog.common.HCatUtil;
+import org.apache.hcatalog.data.schema.HCatFieldSchema;
+import org.apache.hcatalog.data.schema.HCatSchemaUtils;
+import org.apache.thrift.TException;
+
+/**
+ * The HCatClientHMSImpl is the Hive Metastore client based implementation of
+ * HCatClient.
+ */
+public class HCatClientHMSImpl extends HCatClient {
+
+ private HiveMetaStoreClient hmsClient;
+ private Configuration config;
+ private HiveConf hiveConfig;
+
+ @Override
+ public List<String> listDatabaseNamesByPattern(String pattern)
+ throws HCatException {
+ List<String> dbNames = null;
+ try {
+ dbNames = hmsClient.getDatabases(pattern);
+ } catch (MetaException exp) {
+ throw new HCatException("MetaException while listing db names", exp);
+ }
+ return dbNames;
+ }
+
+ @Override
+ public HCatDatabase getDatabase(String dbName) throws HCatException {
+ HCatDatabase db = null;
+ try {
+ Database hiveDB = hmsClient.getDatabase(checkDB(dbName));
+ if (hiveDB != null) {
+ db = new HCatDatabase(hiveDB);
+ }
+ } catch (NoSuchObjectException exp) {
+ throw new ObjectNotFoundException(
+ "NoSuchObjectException while fetching database", exp);
+ } catch (MetaException exp) {
+ throw new HCatException("MetaException while fetching database",
+ exp);
+ } catch (TException exp) {
+ throw new ConnectionFailureException(
+ "TException while fetching database", exp);
+ }
+ return db;
+ }
+
+ @Override
+ public void createDatabase(HCatCreateDBDesc dbInfo) throws HCatException {
+ try {
+ hmsClient.createDatabase(dbInfo.toHiveDb());
+ } catch (AlreadyExistsException exp) {
+ if (!dbInfo.getIfNotExists()) {
+ throw new HCatException(
+ "AlreadyExistsException while creating database", exp);
+ }
+ } catch (InvalidObjectException exp) {
+ throw new HCatException(
+ "InvalidObjectException while creating database", exp);
+ } catch (MetaException exp) {
+ throw new HCatException("MetaException while creating database",
+ exp);
+ } catch (TException exp) {
+ throw new ConnectionFailureException(
+ "TException while creating database", exp);
+ }
+ }
+
+ @Override
+ public void dropDatabase(String dbName, boolean ifExists, DropDBMode mode)
+ throws HCatException {
+ boolean isCascade = mode.toString().equalsIgnoreCase("cascade");
+ try {
+ hmsClient.dropDatabase(checkDB(dbName), true, ifExists, isCascade);
+ } catch (NoSuchObjectException e) {
+ if (!ifExists) {
+ throw new ObjectNotFoundException(
+ "NoSuchObjectException while dropping db.", e);
+ }
+ } catch (InvalidOperationException e) {
+ throw new HCatException(
+ "InvalidOperationException while dropping db.", e);
+ } catch (MetaException e) {
+ throw new HCatException("MetaException while dropping db.", e);
+ } catch (TException e) {
+ throw new ConnectionFailureException("TException while dropping db.",
+ e);
+ }
+ }
+
+ @Override
+ public List<String> listTableNamesByPattern(String dbName,
+ String tablePattern) throws HCatException {
+ List<String> tableNames = null;
+ try {
+ tableNames = hmsClient.getTables(checkDB(dbName), tablePattern);
+ } catch (MetaException e) {
+ throw new HCatException(
+ "MetaException while fetching table names.", e);
+ }
+ return tableNames;
+ }
+
+ @Override
+ public HCatTable getTable(String dbName, String tableName)
+ throws HCatException {
+ HCatTable table = null;
+ try {
+ Table hiveTable = hmsClient.getTable(checkDB(dbName), tableName);
+ if (hiveTable != null) {
+ table = new HCatTable(hiveTable);
+ }
+ } catch (MetaException e) {
+ throw new HCatException("MetaException while fetching table.", e);
+ } catch (NoSuchObjectException e) {
+ throw new ObjectNotFoundException(
+ "NoSuchObjectException while fetching table.", e);
+ } catch (TException e) {
+ throw new ConnectionFailureException(
+ "TException while fetching table.", e);
+ }
+ return table;
+ }
+
+ @Override
+ public void createTable(HCatCreateTableDesc createTableDesc)
+ throws HCatException {
+ try {
+ hmsClient.createTable(createTableDesc.toHiveTable(hiveConfig));
+ } catch (AlreadyExistsException e) {
+ if (!createTableDesc.getIfNotExists()) {
+ throw new HCatException(
+ "AlreadyExistsException while creating table.", e);
+ }
+ } catch (InvalidObjectException e) {
+ throw new HCatException(
+ "InvalidObjectException while creating table.", e);
+ } catch (MetaException e) {
+ throw new HCatException("MetaException while creating table.", e);
+ } catch (NoSuchObjectException e) {
+ throw new ObjectNotFoundException(
+ "NoSuchObjectException while creating table.", e);
+ } catch (TException e) {
+ throw new ConnectionFailureException(
+ "TException while creating table.", e);
+ } catch (IOException e) {
+ throw new HCatException("IOException while creating hive conf.", e);
+ }
+
+ }
+
+ @Override
+ public void updateTableSchema(String dbName, String tableName, List<HCatFieldSchema> columnSchema)
+ throws HCatException {
+ try {
+ Table table = hmsClient.getTable(dbName, tableName);
+ table.getSd().setCols(HCatSchemaUtils.getFieldSchemas(columnSchema));
+ hmsClient.alter_table(dbName, tableName, table);
+ }
+ catch (InvalidOperationException e) {
+ throw new HCatException("InvalidOperationException while updating table schema.", e);
+ }
+ catch (MetaException e) {
+ throw new HCatException("MetaException while updating table schema.", e);
+ }
+ catch (NoSuchObjectException e) {
+ throw new ObjectNotFoundException(
+ "NoSuchObjectException while updating table schema.", e);
+ }
+ catch (TException e) {
+ throw new ConnectionFailureException(
+ "TException while updating table schema.", e);
+ }
+ }
+
+ @Override
+ public void createTableLike(String dbName, String existingTblName,
+ String newTableName, boolean ifNotExists, boolean isExternal,
+ String location) throws HCatException {
+
+ Table hiveTable = getHiveTableLike(checkDB(dbName), existingTblName,
+ newTableName, ifNotExists, location);
+ if (hiveTable != null) {
+ try {
+ hmsClient.createTable(hiveTable);
+ } catch (AlreadyExistsException e) {
+ if (!ifNotExists) {
+ throw new HCatException(
+ "A table already exists with the name "
+ + newTableName, e);
+ }
+ } catch (InvalidObjectException e) {
+ throw new HCatException(
+ "InvalidObjectException in create table like command.",
+ e);
+ } catch (MetaException e) {
+ throw new HCatException(
+ "MetaException in create table like command.", e);
+ } catch (NoSuchObjectException e) {
+ throw new ObjectNotFoundException(
+ "NoSuchObjectException in create table like command.",
+ e);
+ } catch (TException e) {
+ throw new ConnectionFailureException(
+ "TException in create table like command.", e);
+ }
+ }
+ }
+
+ @Override
+ public void dropTable(String dbName, String tableName, boolean ifExists)
+ throws HCatException {
+ try {
+ hmsClient.dropTable(checkDB(dbName), tableName, true, ifExists);
+ } catch (NoSuchObjectException e) {
+ if (!ifExists) {
+ throw new ObjectNotFoundException(
+ "NoSuchObjectException while dropping table.", e);
+ }
+ } catch (MetaException e) {
+ throw new HCatException("MetaException while dropping table.", e);
+ } catch (TException e) {
+ throw new ConnectionFailureException(
+ "TException while dropping table.", e);
+ }
+ }
+
+ @Override
+ public void renameTable(String dbName, String oldName, String newName)
+ throws HCatException {
+ Table tbl;
+ try {
+ Table oldtbl = hmsClient.getTable(checkDB(dbName), oldName);
+ if (oldtbl != null) {
+ // TODO : Should be moved out.
+ if (oldtbl
+ .getParameters()
+ .get(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_STORAGE) != null) {
+ throw new HCatException(
+ "Cannot use rename command on a non-native table");
+ }
+ tbl = new Table(oldtbl);
+ tbl.setTableName(newName);
+ hmsClient.alter_table(checkDB(dbName), oldName, tbl);
+ }
+ } catch (MetaException e) {
+ throw new HCatException("MetaException while renaming table", e);
+ } catch (NoSuchObjectException e) {
+ throw new ObjectNotFoundException(
+ "NoSuchObjectException while renaming table", e);
+ } catch (InvalidOperationException e) {
+ throw new HCatException(
+ "InvalidOperationException while renaming table", e);
+ } catch (TException e) {
+ throw new ConnectionFailureException(
+ "TException while renaming table", e);
+ }
+ }
+
+ @Override
+ public List<HCatPartition> getPartitions(String dbName, String tblName)
+ throws HCatException {
+ List<HCatPartition> hcatPtns = new ArrayList<HCatPartition>();
+ try {
+ List<Partition> hivePtns = hmsClient.listPartitions(
+ checkDB(dbName), tblName, (short) -1);
+ for (Partition ptn : hivePtns) {
+ hcatPtns.add(new HCatPartition(ptn));
+ }
+ } catch (NoSuchObjectException e) {
+ throw new ObjectNotFoundException(
+ "NoSuchObjectException while retrieving partition.", e);
+ } catch (MetaException e) {
+ throw new HCatException(
+ "MetaException while retrieving partition.", e);
+ } catch (TException e) {
+ throw new ConnectionFailureException(
+ "TException while retrieving partition.", e);
+ }
+ return hcatPtns;
+ }
+
+ @Override
+ public List<HCatPartition> getPartitions(String dbName, String tblName, Map<String, String> partitionSpec) throws HCatException {
+ return listPartitionsByFilter(dbName, tblName, getFilterString(partitionSpec));
+ }
+
+ private static String getFilterString(Map<String, String> partitionSpec) {
+ final String AND = " AND ";
+
+ StringBuilder filter = new StringBuilder();
+ for (Map.Entry<String, String> entry : partitionSpec.entrySet()) {
+ filter.append(entry.getKey()).append("=").append("\"").append(entry.getValue()).append("\"").append(AND);
+ }
+
+ int length = filter.toString().length();
+ if (length > 0)
+ filter.delete(length - AND.length(), length);
+
+ return filter.toString();
+ }
+
+ @Override
+ public HCatPartition getPartition(String dbName, String tableName,
+ Map<String, String> partitionSpec) throws HCatException {
+ HCatPartition partition = null;
+ try {
+ List<HCatFieldSchema> partitionColumns = getTable(checkDB(dbName), tableName).getPartCols();
+ if (partitionColumns.size() != partitionSpec.size()) {
+ throw new HCatException("Partition-spec doesn't have the right number of partition keys.");
+ }
+
+ ArrayList<String> ptnValues = new ArrayList<String>();
+ for (HCatFieldSchema partitionColumn : partitionColumns) {
+ String partKey = partitionColumn.getName();
+ if (partitionSpec.containsKey(partKey)) {
+ ptnValues.add(partitionSpec.get(partKey)); // Partition-keys added in order.
+ }
+ else {
+ throw new HCatException("Invalid partition-key specified: " + partKey);
+ }
+ }
+ Partition hivePartition = hmsClient.getPartition(checkDB(dbName),
+ tableName, ptnValues);
+ if (hivePartition != null) {
+ partition = new HCatPartition(hivePartition);
+ }
+ } catch (MetaException e) {
+ throw new HCatException(
+ "MetaException while retrieving partition.", e);
+ } catch (NoSuchObjectException e) {
+ throw new ObjectNotFoundException(
+ "NoSuchObjectException while retrieving partition.", e);
+ } catch (TException e) {
+ throw new ConnectionFailureException(
+ "TException while retrieving partition.", e);
+ }
+ return partition;
+ }
+
+ @Override
+ public void addPartition(HCatAddPartitionDesc partInfo)
+ throws HCatException {
+ Table tbl = null;
+ try {
+ tbl = hmsClient.getTable(partInfo.getDatabaseName(),
+ partInfo.getTableName());
+ // TODO: Should be moved out.
+ if (tbl.getPartitionKeysSize() == 0) {
+ throw new HCatException("The table " + partInfo.getTableName()
+ + " is not partitioned.");
+ }
+
+ hmsClient.add_partition(partInfo.toHivePartition(tbl));
+ } catch (InvalidObjectException e) {
+ throw new HCatException(
+ "InvalidObjectException while adding partition.", e);
+ } catch (AlreadyExistsException e) {
+ throw new HCatException(
+ "AlreadyExistsException while adding partition.", e);
+ } catch (MetaException e) {
+ throw new HCatException("MetaException while adding partition.", e);
+ } catch (NoSuchObjectException e) {
+ throw new ObjectNotFoundException("The table " + partInfo.getTableName()
+ + " is could not be found.", e);
+ } catch (TException e) {
+ throw new ConnectionFailureException(
+ "TException while adding partition.", e);
+ }
+ }
+
+ @Override
+ public void dropPartitions(String dbName, String tableName,
+ Map<String, String> partitionSpec, boolean ifExists)
+ throws HCatException {
+ try {
+ dbName = checkDB(dbName);
+ List<Partition> partitions = hmsClient.listPartitionsByFilter(dbName, tableName,
+ getFilterString(partitionSpec), (short)-1);
+
+ for (Partition partition : partitions) {
+ dropPartition(partition, ifExists);
+ }
+
+ } catch (NoSuchObjectException e) {
+ throw new ObjectNotFoundException(
+ "NoSuchObjectException while dropping partition. " +
+ "Either db(" + dbName + ") or table(" + tableName + ") missing.", e);
+ } catch (MetaException e) {
+ throw new HCatException("MetaException while dropping partition.",
+ e);
+ } catch (TException e) {
+ throw new ConnectionFailureException(
+ "TException while dropping partition.", e);
+ }
+ }
+
+ private void dropPartition(Partition partition, boolean ifExists)
+ throws HCatException, MetaException, TException {
+ try {
+ hmsClient.dropPartition(partition.getDbName(), partition.getTableName(), partition.getValues());
+ } catch (NoSuchObjectException e) {
+ if (!ifExists) {
+ throw new ObjectNotFoundException(
+ "NoSuchObjectException while dropping partition: " + partition.getValues(), e);
+ }
+ }
+ }
+
+ @Override
+ public List<HCatPartition> listPartitionsByFilter(String dbName,
+ String tblName, String filter) throws HCatException {
+ List<HCatPartition> hcatPtns = new ArrayList<HCatPartition>();
+ try {
+ List<Partition> hivePtns = hmsClient.listPartitionsByFilter(
+ checkDB(dbName), tblName, filter, (short) -1);
+ for (Partition ptn : hivePtns) {
+ hcatPtns.add(new HCatPartition(ptn));
+ }
+ } catch (MetaException e) {
+ throw new HCatException("MetaException while fetching partitions.",
+ e);
+ } catch (NoSuchObjectException e) {
+ throw new ObjectNotFoundException(
+ "NoSuchObjectException while fetching partitions.", e);
+ } catch (TException e) {
+ throw new ConnectionFailureException(
+ "TException while fetching partitions.", e);
+ }
+ return hcatPtns;
+ }
+
+ @Override
+ public void markPartitionForEvent(String dbName, String tblName,
+ Map<String, String> partKVs, PartitionEventType eventType)
+ throws HCatException {
+ try {
+ hmsClient.markPartitionForEvent(checkDB(dbName), tblName, partKVs,
+ eventType);
+ } catch (MetaException e) {
+ throw new HCatException(
+ "MetaException while marking partition for event.", e);
+ } catch (NoSuchObjectException e) {
+ throw new ObjectNotFoundException(
+ "NoSuchObjectException while marking partition for event.",
+ e);
+ } catch (UnknownTableException e) {
+ throw new HCatException(
+ "UnknownTableException while marking partition for event.",
+ e);
+ } catch (UnknownDBException e) {
+ throw new HCatException(
+ "UnknownDBException while marking partition for event.", e);
+ } catch (TException e) {
+ throw new ConnectionFailureException(
+ "TException while marking partition for event.", e);
+ }
+ }
+
+ @Override
+ public boolean isPartitionMarkedForEvent(String dbName, String tblName,
+ Map<String, String> partKVs, PartitionEventType eventType)
+ throws HCatException {
+ boolean isMarked = false;
+ try {
+ isMarked = hmsClient.isPartitionMarkedForEvent(checkDB(dbName),
+ tblName, partKVs, eventType);
+ } catch (MetaException e) {
+ throw new HCatException(
+ "MetaException while checking partition for event.", e);
+ } catch (NoSuchObjectException e) {
+ throw new ObjectNotFoundException(
+ "NoSuchObjectException while checking partition for event.",
+ e);
+ } catch (UnknownTableException e) {
+ throw new HCatException(
+ "UnknownTableException while checking partition for event.",
+ e);
+ } catch (UnknownDBException e) {
+ throw new HCatException(
+ "UnknownDBException while checking partition for event.", e);
+ } catch (TException e) {
+ throw new ConnectionFailureException(
+ "TException while checking partition for event.", e);
+ }
+ return isMarked;
+ }
+
+ @Override
+ public String getDelegationToken(String owner,
+ String renewerKerberosPrincipalName) throws HCatException {
+ String token = null;
+ try {
+ token = hmsClient.getDelegationToken(owner,
+ renewerKerberosPrincipalName);
+ } catch (MetaException e) {
+ throw new HCatException(
+ "MetaException while getting delegation token.", e);
+ } catch (TException e) {
+ throw new ConnectionFailureException(
+ "TException while getting delegation token.", e);
+ }
+
+ return token;
+ }
+
+ @Override
+ public long renewDelegationToken(String tokenStrForm) throws HCatException {
+ long time = 0;
+ try {
+ time = hmsClient.renewDelegationToken(tokenStrForm);
+ } catch (MetaException e) {
+ throw new HCatException(
+ "MetaException while renewing delegation token.", e);
+ } catch (TException e) {
+ throw new ConnectionFailureException(
+ "TException while renewing delegation token.", e);
+ }
+
+ return time;
+ }
+
+ @Override
+ public void cancelDelegationToken(String tokenStrForm)
+ throws HCatException {
+ try {
+ hmsClient.cancelDelegationToken(tokenStrForm);
+ } catch (MetaException e) {
+ throw new HCatException(
+ "MetaException while canceling delegation token.", e);
+ } catch (TException e) {
+ throw new ConnectionFailureException(
+ "TException while canceling delegation token.", e);
+ }
+ }
+
+ /*
+ * @param conf /* @throws HCatException,ConnectionFailureException
+ *
+ * @see
+ * org.apache.hcatalog.api.HCatClient#initialize(org.apache.hadoop.conf.
+ * Configuration)
+ */
+ @Override
+ void initialize(Configuration conf) throws HCatException {
+ this.config = conf;
+ try {
+ hiveConfig = HCatUtil.getHiveConf(config);
+ hmsClient = HCatUtil.getHiveClient(hiveConfig);
+ } catch (MetaException exp) {
+ throw new HCatException("MetaException while creating HMS client",
+ exp);
+ } catch (IOException exp) {
+ throw new HCatException("IOException while creating HMS client",
+ exp);
+ }
+
+ }
+
+ private Table getHiveTableLike(String dbName, String existingTblName,
+ String newTableName, boolean isExternal, String location)
+ throws HCatException {
+ Table oldtbl = null;
+ Table newTable = null;
+ try {
+ oldtbl = hmsClient.getTable(checkDB(dbName), existingTblName);
+ } catch (MetaException e1) {
+ throw new HCatException(
+ "MetaException while retrieving existing table.", e1);
+ } catch (NoSuchObjectException e1) {
+ throw new ObjectNotFoundException(
+ "NoSuchObjectException while retrieving existing table.",
+ e1);
+ } catch (TException e1) {
+ throw new ConnectionFailureException(
+ "TException while retrieving existing table.", e1);
+ }
+ if (oldtbl != null) {
+ newTable = new Table();
+ newTable.setTableName(newTableName);
+ newTable.setDbName(dbName);
+ StorageDescriptor sd = new StorageDescriptor(oldtbl.getSd());
+ newTable.setSd(sd);
+ newTable.setParameters(oldtbl.getParameters());
+ if (location == null) {
+ newTable.getSd().setLocation(oldtbl.getSd().getLocation());
+ } else {
+ newTable.getSd().setLocation(location);
+ }
+ if (isExternal) {
+ newTable.putToParameters("EXTERNAL", "TRUE");
+ newTable.setTableType(TableType.EXTERNAL_TABLE.toString());
+ } else {
+ newTable.getParameters().remove("EXTERNAL");
+ }
+ // set create time
+ newTable.setCreateTime((int) (System.currentTimeMillis() / 1000));
+ newTable.setLastAccessTimeIsSet(false);
+ }
+ return newTable;
+ }
+
+ /*
+ * @throws HCatException
+ *
+ * @see org.apache.hcatalog.api.HCatClient#closeClient()
+ */
+ @Override
+ public void close() throws HCatException {
+ hmsClient.close();
+ }
+
+ private String checkDB(String name) {
+ if (StringUtils.isEmpty(name)) {
+ return MetaStoreUtils.DEFAULT_DATABASE_NAME;
+ } else {
+ return name;
+ }
+ }
+
+ /*
+ * @param partInfoList
+ * @return The size of the list of partitions.
+ * @throws HCatException,ConnectionFailureException
+ * @see org.apache.hcatalog.api.HCatClient#addPartitions(java.util.List)
+ */
+ @Override
+ public int addPartitions(List<HCatAddPartitionDesc> partInfoList)
+ throws HCatException {
+ int numPartitions = -1;
+ if ((partInfoList == null) || (partInfoList.size() == 0)) {
+ throw new HCatException("The partition list is null or empty.");
+ }
+
+ Table tbl = null;
+ try {
+ tbl = hmsClient.getTable(partInfoList.get(0).getDatabaseName(),
+ partInfoList.get(0).getTableName());
+ ArrayList<Partition> ptnList = new ArrayList<Partition>();
+ for (HCatAddPartitionDesc desc : partInfoList) {
+ ptnList.add(desc.toHivePartition(tbl));
+ }
+ numPartitions = hmsClient.add_partitions(ptnList);
+ } catch (InvalidObjectException e) {
+ throw new HCatException(
+ "InvalidObjectException while adding partition.", e);
+ } catch (AlreadyExistsException e) {
+ throw new HCatException(
+ "AlreadyExistsException while adding partition.", e);
+ } catch (MetaException e) {
+ throw new HCatException("MetaException while adding partition.", e);
+ } catch (NoSuchObjectException e) {
+ throw new ObjectNotFoundException("The table "
+ + partInfoList.get(0).getTableName()
+ + " is could not be found.", e);
+ } catch (TException e) {
+ throw new ConnectionFailureException(
+ "TException while adding partition.", e);
+ }
+ return numPartitions;
+ }
+
+ @Override
+ public String getMessageBusTopicName(String dbName, String tableName) throws HCatException {
+ try {
+ return hmsClient.getTable(dbName, tableName).getParameters().get(HCatConstants.HCAT_MSGBUS_TOPIC_NAME);
+ }
+ catch (MetaException e) {
+ throw new HCatException("MetaException while retrieving JMS Topic name.", e);
+ } catch (NoSuchObjectException e) {
+ throw new HCatException("Could not find DB:" + dbName + " or Table:" + tableName, e);
+ } catch (TException e) {
+ throw new ConnectionFailureException(
+ "TException while retrieving JMS Topic name.", e);
+ }
+ }
+}
Added: hive/trunk/hcatalog/webhcat/java-client/src/main/java/org/apache/hcatalog/api/HCatCreateDBDesc.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/java-client/src/main/java/org/apache/hcatalog/api/HCatCreateDBDesc.java?rev=1520466&view=auto
==============================================================================
--- hive/trunk/hcatalog/webhcat/java-client/src/main/java/org/apache/hcatalog/api/HCatCreateDBDesc.java (added)
+++ hive/trunk/hcatalog/webhcat/java-client/src/main/java/org/apache/hcatalog/api/HCatCreateDBDesc.java Fri Sep 6 00:49:14 2013
@@ -0,0 +1,194 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hcatalog.api;
+
+import java.util.Map;
+
+import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hcatalog.common.HCatException;
+
+/**
+ * The Class HCatCreateDBDesc for defining database attributes.
+ */
+public class HCatCreateDBDesc {
+
+ private String dbName;
+ private String locationUri;
+ private String comment;
+ private Map<String, String> dbProperties;
+ private boolean ifNotExits = false;
+
+ /**
+ * Gets the database properties.
+ *
+ * @return the database properties
+ */
+ public Map<String, String> getDatabaseProperties() {
+ return this.dbProperties;
+ }
+
+ /**
+ * Gets the if not exists.
+ *
+ * @return the if not exists
+ */
+ public boolean getIfNotExists() {
+ return this.ifNotExits;
+ }
+
+ /**
+ * Gets the comments.
+ *
+ * @return the comments
+ */
+ public String getComments() {
+ return this.comment;
+ }
+
+ /**
+ * Gets the location.
+ *
+ * @return the location
+ */
+ public String getLocation() {
+ return this.locationUri;
+ }
+
+ /**
+ * Gets the database name.
+ *
+ * @return the database name
+ */
+ public String getDatabaseName() {
+ return this.dbName;
+ }
+
+ private HCatCreateDBDesc(String dbName) {
+ this.dbName = dbName;
+ }
+
+ @Override
+ public String toString() {
+ return "HCatCreateDBDesc ["
+ + (dbName != null ? "dbName=" + dbName + ", " : "dbName=null")
+ + (locationUri != null ? "location=" + locationUri + ", "
+ : "location=null")
+ + (comment != null ? "comment=" + comment + ", " : "comment=null")
+ + (dbProperties != null ? "dbProperties=" + dbProperties + ", "
+ : "dbProperties=null") + "ifNotExits=" + ifNotExits + "]";
+ }
+
+ /**
+ * Creates the builder for defining attributes.
+ *
+ * @param dbName the db name
+ * @return the builder
+ */
+ public static Builder create(String dbName) {
+ return new Builder(dbName);
+ }
+
+ Database toHiveDb() {
+ Database hiveDB = new Database();
+ hiveDB.setDescription(this.comment);
+ hiveDB.setLocationUri(this.locationUri);
+ hiveDB.setName(this.dbName);
+ hiveDB.setParameters(this.dbProperties);
+ return hiveDB;
+ }
+
+ public static class Builder {
+
+ private String innerLoc;
+ private String innerComment;
+ private Map<String, String> innerDBProps;
+ private String dbName;
+ private boolean ifNotExists = false;
+
+ private Builder(String dbName) {
+ this.dbName = dbName;
+ }
+
+ /**
+ * Location.
+ *
+ * @param value the location of the database.
+ * @return the builder
+ */
+ public Builder location(String value) {
+ this.innerLoc = value;
+ return this;
+ }
+
+ /**
+ * Comment.
+ *
+ * @param value comments.
+ * @return the builder
+ */
+ public Builder comment(String value) {
+ this.innerComment = value;
+ return this;
+ }
+
+ /**
+ * If not exists.
+ * @param ifNotExists If set to true, hive will not throw exception, if a
+ * database with the same name already exists.
+ * @return the builder
+ */
+ public Builder ifNotExists(boolean ifNotExists) {
+ this.ifNotExists = ifNotExists;
+ return this;
+ }
+
+ /**
+ * Database properties.
+ *
+ * @param dbProps the database properties
+ * @return the builder
+ */
+ public Builder databaseProperties(Map<String, String> dbProps) {
+ this.innerDBProps = dbProps;
+ return this;
+ }
+
+
+ /**
+ * Builds the create database descriptor.
+ *
+ * @return An instance of HCatCreateDBDesc
+ * @throws HCatException
+ */
+ public HCatCreateDBDesc build() throws HCatException {
+ if (this.dbName == null) {
+ throw new HCatException("Database name cannot be null.");
+ }
+ HCatCreateDBDesc desc = new HCatCreateDBDesc(this.dbName);
+ desc.comment = this.innerComment;
+ desc.locationUri = this.innerLoc;
+ desc.dbProperties = this.innerDBProps;
+ desc.ifNotExits = this.ifNotExists;
+ return desc;
+
+ }
+
+ }
+
+}