You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ga...@apache.org on 2015/07/02 00:47:45 UTC
hive git commit: HIVE-11061 Table renames not propagated to partition
table in HBase metastore (gates)
Repository: hive
Updated Branches:
refs/heads/hbase-metastore 6ea5c0956 -> 1a1c0d817
HIVE-11061 Table renames not propagated to partition table in HBase metastore (gates)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1a1c0d81
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1a1c0d81
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1a1c0d81
Branch: refs/heads/hbase-metastore
Commit: 1a1c0d817da036025f1c3b953af7b52403c048ba
Parents: 6ea5c09
Author: Alan Gates <ga...@hortonworks.com>
Authored: Wed Jul 1 15:47:03 2015 -0700
Committer: Alan Gates <ga...@hortonworks.com>
Committed: Wed Jul 1 15:47:03 2015 -0700
----------------------------------------------------------------------
.../metastore/hbase/TestHBaseMetastoreSql.java | 27 +++++++++++++
.../hadoop/hive/metastore/HiveAlterHandler.java | 16 ++++----
.../hive/metastore/hbase/HBaseReadWrite.java | 40 ++++++++++++++++----
.../hadoop/hive/metastore/hbase/HBaseStore.java | 21 +++++++++-
4 files changed, 85 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/1a1c0d81/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseMetastoreSql.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseMetastoreSql.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseMetastoreSql.java
index 119e5aa..c61ebb7 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseMetastoreSql.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseMetastoreSql.java
@@ -192,5 +192,32 @@ public class TestHBaseMetastoreSql extends HBaseIntegrationTests {
Assert.assertEquals(0, rsp.getResponseCode());
}
+ @Test
+ public void alterRenamePartitioned() throws Exception {
+ driver.run("create table alterrename (c int) partitioned by (ds string)");
+ driver.run("alter table alterrename add partition (ds = 'a')");
+ CommandProcessorResponse rsp = driver.run("describe extended alterrename partition (ds='a')");
+ Assert.assertEquals(0, rsp.getResponseCode());
+ rsp = driver.run("alter table alterrename rename to alter_renamed");
+ Assert.assertEquals(0, rsp.getResponseCode());
+ rsp = driver.run("describe extended alter_renamed partition (ds='a')");
+ Assert.assertEquals(0, rsp.getResponseCode());
+ rsp = driver.run("describe extended alterrename partition (ds='a')");
+ Assert.assertEquals(10001, rsp.getResponseCode());
+ }
+
+ @Test
+ public void alterRename() throws Exception {
+ driver.run("create table alterrename1 (c int)");
+ CommandProcessorResponse rsp = driver.run("describe alterrename1");
+ Assert.assertEquals(0, rsp.getResponseCode());
+ rsp = driver.run("alter table alterrename1 rename to alter_renamed1");
+ Assert.assertEquals(0, rsp.getResponseCode());
+ rsp = driver.run("describe alter_renamed1");
+ Assert.assertEquals(0, rsp.getResponseCode());
+ rsp = driver.run("describe alterrename1");
+ Assert.assertEquals(10001, rsp.getResponseCode());
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hive/blob/1a1c0d81/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
index 99d6680..f402f73 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
@@ -17,12 +17,7 @@
*/
package org.apache.hadoop.hive.metastore;
-import java.io.IOException;
-import java.net.URI;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-
+import com.google.common.collect.Lists;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -48,7 +43,11 @@ import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hive.common.util.HiveStringUtils;
-import com.google.common.collect.Lists;
+import java.io.IOException;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
/**
* Hive specific implementation of alter
@@ -121,8 +120,7 @@ public class HiveAlterHandler implements AlterHandler {
// get old table
oldt = msdb.getTable(dbname, name);
if (oldt == null) {
- throw new InvalidOperationException("table " + newt.getDbName() + "."
- + newt.getTableName() + " doesn't exist");
+ throw new InvalidOperationException("table " + dbname + "." + name + " doesn't exist");
}
if (HiveConf.getBoolVar(hiveConf,
http://git-wip-us.apache.org/repos/asf/hive/blob/1a1c0d81/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java
index b54afb9..fd6f9f5 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java
@@ -541,6 +541,9 @@ class HBaseReadWrite {
byte[][] serialized = HBaseUtils.serializePartition(newPart, hash);
store(PART_TABLE, serialized[0], CATALOG_CF, CATALOG_COL, serialized[1]);
partCache.put(newPart.getDbName(), newPart.getTableName(), newPart);
+ if (!oldPart.getTableName().equals(newPart.getTableName())) {
+ deletePartition(oldPart.getDbName(), oldPart.getTableName(), oldPart.getValues());
+ }
}
/**
@@ -568,7 +571,7 @@ class HBaseReadWrite {
if (oldParts.size() != newParts.size()) {
throw new RuntimeException("Number of old and new partitions must match.");
}
- List<Put> puts = new ArrayList<Put>(newParts.size());
+ List<Put> puts = new ArrayList<>(newParts.size());
for (int i = 0; i < newParts.size(); i++) {
byte[] hash;
byte[] oldHash = HBaseUtils.hashStorageDescriptor(oldParts.get(i).getSd(), md);
@@ -584,6 +587,11 @@ class HBaseReadWrite {
p.add(CATALOG_CF, CATALOG_COL, serialized[1]);
puts.add(p);
partCache.put(newParts.get(i).getDbName(), newParts.get(i).getTableName(), newParts.get(i));
+ if (!newParts.get(i).getTableName().equals(oldParts.get(i).getTableName())) {
+ // We need to remove the old record as well.
+ deletePartition(oldParts.get(i).getDbName(), oldParts.get(i).getTableName(),
+ oldParts.get(i).getValues(), false);
+ }
}
HTableInterface htab = conn.getHBaseTable(PART_TABLE);
htab.put(puts);
@@ -734,10 +742,17 @@ class HBaseReadWrite {
* @throws IOException
*/
void deletePartition(String dbName, String tableName, List<String> partVals) throws IOException {
+ deletePartition(dbName, tableName, partVals, true);
+ }
+
+ private void deletePartition(String dbName, String tableName, List<String> partVals,
+ boolean decrementRefCnt) throws IOException {
// Find the partition so I can get the storage descriptor and drop it
partCache.remove(dbName, tableName, partVals);
- Partition p = getPartition(dbName, tableName, partVals, false);
- decrementStorageDescriptorRefCount(p.getSd());
+ if (decrementRefCnt) {
+ Partition p = getPartition(dbName, tableName, partVals, false);
+ decrementStorageDescriptorRefCount(p.getSd());
+ }
byte[] key = HBaseUtils.buildPartitionKey(dbName, tableName, partVals);
delete(PART_TABLE, key, null, null);
}
@@ -1287,8 +1302,10 @@ class HBaseReadWrite {
}
byte[][] serialized = HBaseUtils.serializeTable(newTable, hash);
store(TABLE_TABLE, serialized[0], CATALOG_CF, CATALOG_COL, serialized[1]);
- tableCache.put(new ObjectPair<String, String>(newTable.getDbName(), newTable.getTableName()),
- newTable);
+ tableCache.put(new ObjectPair<>(newTable.getDbName(), newTable.getTableName()), newTable);
+ if (!oldTable.getTableName().equals(newTable.getTableName())) {
+ deleteTable(oldTable.getDbName(), oldTable.getTableName());
+ }
}
/**
@@ -1298,10 +1315,17 @@ class HBaseReadWrite {
* @throws IOException
*/
void deleteTable(String dbName, String tableName) throws IOException {
+ deleteTable(dbName, tableName, true);
+ }
+
+ private void deleteTable(String dbName, String tableName, boolean decrementRefCnt)
+ throws IOException {
tableCache.remove(new ObjectPair<String, String>(dbName, tableName));
- // Find the table so I can get the storage descriptor and drop it
- Table t = getTable(dbName, tableName, false);
- decrementStorageDescriptorRefCount(t.getSd());
+ if (decrementRefCnt) {
+ // Find the table so I can get the storage descriptor and drop it
+ Table t = getTable(dbName, tableName, false);
+ decrementStorageDescriptorRefCount(t.getSd());
+ }
byte[] key = HBaseUtils.buildKey(dbName, tableName);
delete(TABLE_TABLE, key, null, null);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/1a1c0d81/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java
index 0dbdba2..4fa2ae5 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java
@@ -404,8 +404,25 @@ public class HBaseStore implements RawStore {
boolean commit = false;
openTransaction();
try {
- Table oldTable = getHBase().getTable(dbname, name);
- getHBase().replaceTable(oldTable, newTable);
+ getHBase().replaceTable(getHBase().getTable(dbname, name), newTable);
+ if (newTable.getPartitionKeys() != null && newTable.getPartitionKeys().size() > 0
+ && !name.equals(newTable.getTableName())) {
+ // They renamed the table, so we need to change each partition as well, since it changes
+ // the key.
+ try {
+ List<Partition> oldParts = getPartitions(dbname, name, -1);
+ List<Partition> newParts = new ArrayList<>(oldParts.size());
+ for (Partition oldPart : oldParts) {
+ Partition newPart = oldPart.deepCopy();
+ newPart.setTableName(newTable.getTableName());
+ newParts.add(newPart);
+ }
+ getHBase().replacePartitions(oldParts, newParts);
+ } catch (NoSuchObjectException e) {
+ LOG.debug("No partitions found for old table so not worrying about it");
+ }
+
+ }
commit = true;
} catch (IOException e) {
LOG.error("Unable to alter table " + tableNameForErrorMsg(dbname, name), e);