You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by pa...@apache.org on 2011/07/08 22:47:00 UTC
svn commit: r1144504 - in /hive/trunk/metastore/src:
java/org/apache/hadoop/hive/metastore/ test/org/apache/hadoop/hive/metastore/
Author: pauly
Date: Fri Jul 8 20:47:00 2011
New Revision: 1144504
URL: http://svn.apache.org/viewvc?rev=1144504&view=rev
Log:
HIVE-2194. Add actions for alter table and alter partition events for metastore event listeners (Sohan Jain via pauly)
Modified:
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEventListener.java
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/DummyListener.java
hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java
Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java?rev=1144504&r1=1144503&r2=1144504&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java (original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java Fri Jul 8 20:47:00 2011
@@ -178,6 +178,7 @@ public class HiveAlterHandler implements
} finally {
if (!success) {
msdb.rollbackTransaction();
+ throw new MetaException("Committing the alter table transaction was not successful.");
}
if (success && moveData) {
// change the file name in hdfs
@@ -204,6 +205,5 @@ public class HiveAlterHandler implements
}
}
}
-
}
}
Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java?rev=1144504&r1=1144503&r2=1144504&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java Fri Jul 8 20:47:00 2011
@@ -73,6 +73,8 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.metastore.api.UnknownPartitionException;
import org.apache.hadoop.hive.metastore.api.UnknownTableException;
import org.apache.hadoop.hive.metastore.events.AddPartitionEvent;
+import org.apache.hadoop.hive.metastore.events.AlterPartitionEvent;
+import org.apache.hadoop.hive.metastore.events.AlterTableEvent;
import org.apache.hadoop.hive.metastore.events.CreateDatabaseEvent;
import org.apache.hadoop.hive.metastore.events.CreateTableEvent;
import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent;
@@ -708,9 +710,9 @@ public class HiveMetaStore extends Thrif
}
for (MetaStoreEventListener listener : listeners) {
listener.onDropDatabase(new DropDatabaseEvent(db, success, this));
+ }
}
}
- }
public void drop_database(final String dbName, final boolean deleteData, final boolean cascade)
throws NoSuchObjectException, InvalidOperationException, MetaException {
@@ -1717,9 +1719,16 @@ public class HiveMetaStore extends Thrif
new_part.putToParameters(Constants.DDL_TIME, Long.toString(System
.currentTimeMillis() / 1000));
}
+ Partition oldPart = ms.getPartition(db_name, tbl_name, new_part.getValues());
ms.alterPartition(db_name, tbl_name, new_part);
+ for (MetaStoreEventListener listener : listeners) {
+ listener.onAlterPartition(new AlterPartitionEvent(oldPart, new_part, true, this));
+ }
} catch (InvalidObjectException e) {
throw new InvalidOperationException("alter is not possible");
+ } catch (NoSuchObjectException e){
+ //old partition does not exist
+ throw new InvalidOperationException("alter is not possible");
}
}
@@ -1804,20 +1813,25 @@ public class HiveMetaStore extends Thrif
newTable.putToParameters(Constants.DDL_TIME, Long.toString(System
.currentTimeMillis() / 1000));
}
-
-
try {
- executeWithRetry(new Command<Boolean>() {
+ Table oldt = get_table(dbname, name);
+ boolean success = executeWithRetry(new Command<Boolean>() {
@Override
public Boolean run(RawStore ms) throws Exception {
alterHandler.alterTable(ms, wh, dbname, name, newTable);
return Boolean.TRUE;
}
});
+ for (MetaStoreEventListener listener : listeners) {
+ listener.onAlterTable(new AlterTableEvent(oldt, newTable, success, this));
+ }
} catch (MetaException e) {
throw e;
} catch (InvalidOperationException e) {
throw e;
+ } catch (NoSuchObjectException e) {
+ //thrown when the table to be altered does not exist
+ throw new InvalidOperationException(e.getMessage());
} catch (Exception e) {
assert(e instanceof RuntimeException);
throw (RuntimeException)e;
Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEventListener.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEventListener.java?rev=1144504&r1=1144503&r2=1144504&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEventListener.java (original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEventListener.java Fri Jul 8 20:47:00 2011
@@ -22,6 +22,8 @@ import org.apache.hadoop.conf.Configurab
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.events.AddPartitionEvent;
+import org.apache.hadoop.hive.metastore.events.AlterPartitionEvent;
+import org.apache.hadoop.hive.metastore.events.AlterTableEvent;
import org.apache.hadoop.hive.metastore.events.CreateDatabaseEvent;
import org.apache.hadoop.hive.metastore.events.CreateTableEvent;
import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent;
@@ -43,6 +45,7 @@ public abstract class MetaStoreEventList
public MetaStoreEventListener(Configuration config){
this.conf = config;
}
+
/**
* @param create table event.
* @throws MetaException
@@ -59,6 +62,14 @@ public abstract class MetaStoreEventList
* @param add partition event
* @throws MetaException
*/
+
+ /**
+ * @param tableEvent alter table event
+ * @throws MetaException
+ */
+ public abstract void onAlterTable (AlterTableEvent tableEvent) throws MetaException;
+
+
public abstract void onAddPartition (AddPartitionEvent partitionEvent) throws MetaException;
/**
@@ -68,6 +79,12 @@ public abstract class MetaStoreEventList
public abstract void onDropPartition (DropPartitionEvent partitionEvent) throws MetaException;
/**
+ * @param alter partition event
+ * @throws MetaException
+ */
+ public abstract void onAlterPartition (AlterPartitionEvent partitionEvent) throws MetaException;
+
+ /**
* @param create database event
* @throws MetaException
*/
@@ -94,4 +111,7 @@ public abstract class MetaStoreEventList
public void setConf(Configuration config) {
this.conf = config;
}
+
+
+
}
Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java?rev=1144504&r1=1144503&r2=1144504&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java (original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java Fri Jul 8 20:47:00 2011
@@ -1729,6 +1729,8 @@ public class ObjectStore implements RawS
} finally {
if (!success) {
rollbackTransaction();
+ throw new MetaException(
+ "The transaction for alter partition did not commit successfully.");
}
}
}
Modified: hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/DummyListener.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/DummyListener.java?rev=1144504&r1=1144503&r2=1144504&view=diff
==============================================================================
--- hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/DummyListener.java (original)
+++ hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/DummyListener.java Fri Jul 8 20:47:00 2011
@@ -24,6 +24,8 @@ import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.events.AddPartitionEvent;
+import org.apache.hadoop.hive.metastore.events.AlterPartitionEvent;
+import org.apache.hadoop.hive.metastore.events.AlterTableEvent;
import org.apache.hadoop.hive.metastore.events.CreateDatabaseEvent;
import org.apache.hadoop.hive.metastore.events.CreateTableEvent;
import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent;
@@ -75,6 +77,16 @@ public class DummyListener extends MetaS
}
@Override
+ public void onAlterTable(AlterTableEvent event) throws MetaException {
+ notifyList.add(event);
+ }
+
+ @Override
+ public void onAlterPartition(AlterPartitionEvent event) throws MetaException {
+ notifyList.add(event);
+ }
+
+ @Override
public void onLoadPartitionDone(LoadPartitionDoneEvent partEvent) throws MetaException {
notifyList.add(partEvent);
}
Modified: hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java?rev=1144504&r1=1144503&r2=1144504&view=diff
==============================================================================
--- hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java (original)
+++ hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java Fri Jul 8 20:47:00 2011
@@ -19,6 +19,7 @@
package org.apache.hadoop.hive.metastore;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -28,10 +29,13 @@ import org.apache.hadoop.hive.cli.CliSes
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.PartitionEventType;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.events.AddPartitionEvent;
+import org.apache.hadoop.hive.metastore.events.AlterPartitionEvent;
+import org.apache.hadoop.hive.metastore.events.AlterTableEvent;
import org.apache.hadoop.hive.metastore.events.CreateDatabaseEvent;
import org.apache.hadoop.hive.metastore.events.CreateTableEvent;
import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent;
@@ -47,7 +51,6 @@ import org.apache.hadoop.hive.ql.session
* {@link org.apache.hadoop.hive.metastore.hadooorg.apache.hadoop.hive.metastore.MetaStoreEventListener}
*/
public class TestMetaStoreEventListener extends TestCase {
-
private static final String msPort = "20001";
private HiveConf hiveConf;
private HiveMetaStoreClient msc;
@@ -93,64 +96,142 @@ public class TestMetaStoreEventListener
}
public void testListener() throws Exception {
+ String dbName = "tmpdb";
+ String tblName = "tmptbl";
+ String renamed = "tmptbl2";
+ int listSize = 0;
List<ListenerEvent> notifyList = DummyListener.notifyList;
- assertEquals(notifyList.size(), 0);
+ assertEquals(notifyList.size(), listSize);
- driver.run("create database tmpdb");
- Database db = msc.getDatabase("tmpdb");
- assertEquals(1, notifyList.size());
- CreateDatabaseEvent dbEvent = (CreateDatabaseEvent)(notifyList.get(0));
+ driver.run("create database " + dbName);
+ listSize++;
+ Database db = msc.getDatabase(dbName);
+ assertEquals(listSize, notifyList.size());
+ CreateDatabaseEvent dbEvent = (CreateDatabaseEvent)(notifyList.get(listSize - 1));
assert dbEvent.getStatus();
assertEquals(db.getName(), dbEvent.getDatabase().getName());
assertEquals(db.getLocationUri(), dbEvent.getDatabase().getLocationUri());
- driver.run("use tmpdb");
- driver.run("create table tmptbl (a string) partitioned by (b string)");
- Table tbl = msc.getTable("tmpdb", "tmptbl");
- assertEquals(notifyList.size(), 2);
- CreateTableEvent tblEvent = (CreateTableEvent)(notifyList.get(1));
+ driver.run("use " + dbName);
+ driver.run(String.format("create table %s (a string) partitioned by (b string)", tblName));
+ listSize++;
+ Table tbl = msc.getTable(dbName, tblName);
+ assertEquals(notifyList.size(), listSize);
+ CreateTableEvent tblEvent = (CreateTableEvent)(notifyList.get(listSize - 1));
assert tblEvent.getStatus();
assertEquals(tbl.getTableName(), tblEvent.getTable().getTableName());
assertEquals(tbl.getDbName(), tblEvent.getTable().getDbName());
assertEquals(tbl.getSd().getLocation(), tblEvent.getTable().getSd().getLocation());
driver.run("alter table tmptbl add partition (b='2011')");
+ listSize++;
Partition part = msc.getPartition("tmpdb", "tmptbl", "b=2011");
- assertEquals(notifyList.size(), 3);
- AddPartitionEvent partEvent = (AddPartitionEvent)(notifyList.get(2));
+ assertEquals(notifyList.size(), listSize);
+ AddPartitionEvent partEvent = (AddPartitionEvent)(notifyList.get(listSize-1));
assert partEvent.getStatus();
assertEquals(part, partEvent.getPartition());
+ driver.run(String.format("alter table %s touch partition (%s)", tblName, "b='2011'"));
+ listSize++;
+ assertEquals(notifyList.size(), listSize);
+ AlterPartitionEvent alterPartEvent = (AlterPartitionEvent)notifyList.get(listSize - 1);
+ assert alterPartEvent.getStatus();
+ Partition origP = msc.getPartition(dbName, tblName, "b=2011");
+ assertEquals(origP.getValues(), alterPartEvent.getOldPartition().getValues());
+ assertEquals(origP.getDbName(), alterPartEvent.getOldPartition().getDbName());
+ assertEquals(origP.getTableName(), alterPartEvent.getOldPartition().getTableName());
+ //the partition did not change,
+ // so the new partition should be similar to the original partition
+ assertEquals(origP.getValues(), alterPartEvent.getNewPartition().getValues());
+ assertEquals(origP.getDbName(), alterPartEvent.getNewPartition().getDbName());
+ assertEquals(origP.getTableName(), alterPartEvent.getNewPartition().getTableName());
+
+ driver.run(String.format("alter table %s rename to %s", tblName, renamed));
+ listSize++;
+ assertEquals(notifyList.size(), listSize);
+ Table renamedTable = msc.getTable(dbName, renamed);
+ AlterTableEvent alterTableE = (AlterTableEvent) notifyList.get(listSize-1);
+ assert alterTableE.getStatus();
+ Table oldTable = alterTableE.getOldTable();
+ Table newTable = alterTableE.getNewTable();
+ assertEquals(tbl.getDbName(), oldTable.getDbName());
+ assertEquals(tbl.getTableName(), oldTable.getTableName());
+ assertEquals(tbl.getSd().getLocation(), oldTable.getSd().getLocation());
+ assertEquals(renamedTable.getDbName(), newTable.getDbName());
+ assertEquals(renamedTable.getTableName(), newTable.getTableName());
+ assertEquals(renamedTable.getSd().getLocation(), newTable.getSd().getLocation());
+
+ //change the table name back
+ driver.run(String.format("alter table %s rename to %s", renamed, tblName));
+ listSize++;
+ assertEquals(notifyList.size(), listSize);
+
+ driver.run(String.format("alter table %s ADD COLUMNS (c int)", tblName));
+ listSize++;
+ assertEquals(notifyList.size(), listSize);
+ Table altTable = msc.getTable(dbName, tblName);
+ alterTableE = (AlterTableEvent) notifyList.get(listSize-1);
+ assert alterTableE.getStatus();
+ oldTable = alterTableE.getOldTable();
+ List<FieldSchema> origCols = tbl.getSd().getCols();
+ List<FieldSchema> oldCols = oldTable.getSd().getCols();
+ List<FieldSchema> altCols = altTable.getSd().getCols();
+ List<FieldSchema> newCols = altTable.getSd().getCols();
+
+ Iterator<FieldSchema> i = origCols.iterator();
+ Iterator<FieldSchema> j = oldCols.iterator();
+ while (i.hasNext()) {
+ assertEquals(i.next().getName(), j.next().getName());
+ }
+
+ i = altCols.iterator();
+ j = newCols.iterator();
+ while (i.hasNext()) {
+ String name = i.next().getName();
+ assertEquals(name, j.next().getName());
+ }
+
+ newTable = alterTableE.getNewTable();
+ assertEquals(tbl.getDbName(), oldTable.getDbName());
+ assertEquals(tbl.getTableName(), oldTable.getTableName());
+ assertEquals(tbl.getSd().getLocation(), oldTable.getSd().getLocation());
+ assertEquals(altTable.getDbName(), newTable.getDbName());
+ assertEquals(altTable.getTableName(), newTable.getTableName());
+ assertEquals(altTable.getSd().getLocation(), newTable.getSd().getLocation());
+
Map<String,String> kvs = new HashMap<String, String>(1);
kvs.put("b", "2011");
msc.markPartitionForEvent("tmpdb", "tmptbl", kvs, PartitionEventType.LOAD_DONE);
- assertEquals(notifyList.size(), 4);
- LoadPartitionDoneEvent partMarkEvent = (LoadPartitionDoneEvent)notifyList.get(3);
+ listSize++;
+ assertEquals(notifyList.size(), listSize);
+ LoadPartitionDoneEvent partMarkEvent = (LoadPartitionDoneEvent)notifyList.get(listSize - 1);
assert partMarkEvent.getStatus();
assertEquals(partMarkEvent.getPartitionName(), kvs);
assertEquals(partMarkEvent.getTable().getTableName(), "tmptbl");
- driver.run("alter table tmptbl drop partition (b='2011')");
- assertEquals(notifyList.size(), 5);
- DropPartitionEvent dropPart = (DropPartitionEvent)notifyList.get(4);
+ driver.run(String.format("alter table %s drop partition (b='2011')", tblName));
+ listSize++;
+ assertEquals(notifyList.size(), listSize);
+ DropPartitionEvent dropPart = (DropPartitionEvent)notifyList.get(listSize - 1);
assert dropPart.getStatus();
assertEquals(part.getValues(), dropPart.getPartition().getValues());
assertEquals(part.getDbName(), dropPart.getPartition().getDbName());
assertEquals(part.getTableName(), dropPart.getPartition().getTableName());
- driver.run("drop table tmptbl");
- assertEquals(notifyList.size(), 6);
- DropTableEvent dropTbl = (DropTableEvent)notifyList.get(5);
+ driver.run("drop table " + tblName);
+ listSize++;
+ assertEquals(notifyList.size(), listSize);
+ DropTableEvent dropTbl = (DropTableEvent)notifyList.get(listSize-1);
assert dropTbl.getStatus();
assertEquals(tbl.getTableName(), dropTbl.getTable().getTableName());
assertEquals(tbl.getDbName(), dropTbl.getTable().getDbName());
assertEquals(tbl.getSd().getLocation(), dropTbl.getTable().getSd().getLocation());
-
- driver.run("drop database tmpdb");
- assertEquals(notifyList.size(), 7);
- DropDatabaseEvent dropDB = (DropDatabaseEvent)notifyList.get(6);
+ driver.run("drop database " + dbName);
+ listSize++;
+ assertEquals(notifyList.size(), listSize);
+ DropDatabaseEvent dropDB = (DropDatabaseEvent)notifyList.get(listSize-1);
assert dropDB.getStatus();
assertEquals(db, dropDB.getDatabase());
}