You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by an...@apache.org on 2015/07/13 11:07:22 UTC
hbase git commit: HBASE-12015 Not cleaning Mob data when Mob CF is
removed from table.(Pankaj Kumar)
Repository: hbase
Updated Branches:
refs/heads/hbase-11339 a1060d139 -> d1f7bcbff
HBASE-12015 Not cleaning Mob data when Mob CF is removed from table.(Pankaj Kumar)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d1f7bcbf
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d1f7bcbf
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d1f7bcbf
Branch: refs/heads/hbase-11339
Commit: d1f7bcbff7fbea1e253d3f9fb45cb98ae759916b
Parents: a1060d1
Author: anoopsjohn <an...@gmail.com>
Authored: Mon Jul 13 14:36:44 2015 +0530
Committer: anoopsjohn <an...@gmail.com>
Committed: Mon Jul 13 14:36:44 2015 +0530
----------------------------------------------------------------------
.../hadoop/hbase/master/MasterFileSystem.java | 21 +++++++-
.../procedure/AddColumnFamilyProcedure.java | 2 +-
.../procedure/DeleteColumnFamilyProcedure.java | 5 +-
.../procedure/MasterDDLOperationHelper.java | 5 +-
.../master/procedure/ModifyTableProcedure.java | 3 +-
.../org/apache/hadoop/hbase/mob/MobUtils.java | 24 +++++++++
.../hadoop/hbase/util/HFileArchiveUtil.java | 4 +-
.../hbase/regionserver/TestDeleteMobTable.java | 51 ++++++++++++++++++++
8 files changed, 105 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/d1f7bcbf/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
index de28cdc..5088901 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
@@ -46,9 +46,10 @@ import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.backup.HFileArchiver;
-import org.apache.hadoop.hbase.client.TableState;
import org.apache.hadoop.hbase.exceptions.DeserializationException;
import org.apache.hadoop.hbase.fs.HFileSystem;
+import org.apache.hadoop.hbase.mob.MobConstants;
+import org.apache.hadoop.hbase.mob.MobUtils;
import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.SplitLogTask.RecoveryMode;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.wal.DefaultWALProvider;
@@ -573,7 +574,7 @@ public class MasterFileSystem {
// @see HRegion.checkRegioninfoOnFilesystem()
}
- public void deleteFamilyFromFS(HRegionInfo region, byte[] familyName)
+ public void deleteFamilyFromFS(HRegionInfo region, byte[] familyName, boolean hasMob)
throws IOException {
// archive family store files
Path tableDir = FSUtils.getTableDir(rootdir, region.getTable());
@@ -590,6 +591,22 @@ public class MasterFileSystem {
+ ")");
}
}
+ // archive and delete mob files
+ if (hasMob) {
+ Path mobTableDir =
+ FSUtils.getTableDir(new Path(getRootDir(), MobConstants.MOB_DIR_NAME), region.getTable());
+ HRegionInfo mobRegionInfo = MobUtils.getMobRegionInfo(region.getTable());
+ Path mobFamilyDir =
+ new Path(mobTableDir,
+ new Path(mobRegionInfo.getEncodedName(), Bytes.toString(familyName)));
+ // archive mob family store files
+ MobUtils.archiveMobStoreFiles(conf, fs, mobRegionInfo, mobFamilyDir, familyName);
+ if (!fs.delete(mobFamilyDir, true)) {
+ throw new IOException("Could not delete mob store files for family "
+ + Bytes.toString(familyName) + " from FileSystem region "
+ + mobRegionInfo.getRegionNameAsString() + "(" + mobRegionInfo.getEncodedName() + ")");
+ }
+ }
}
public void stop() {
http://git-wip-us.apache.org/repos/asf/hbase/blob/d1f7bcbf/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/AddColumnFamilyProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/AddColumnFamilyProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/AddColumnFamilyProcedure.java
index 377ccb5..8e367d0 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/AddColumnFamilyProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/AddColumnFamilyProcedure.java
@@ -312,7 +312,7 @@ public class AddColumnFamilyProcedure
// Remove the column family from file system and update the table descriptor to
// the before-add-column-family-state
MasterDDLOperationHelper.deleteColumnFamilyFromFileSystem(env, tableName,
- getRegionInfoList(env), cfDescriptor.getName());
+ getRegionInfoList(env), cfDescriptor.getName(), cfDescriptor.isMobEnabled());
env.getMasterServices().getTableDescriptors().add(unmodifiedHTableDescriptor);
http://git-wip-us.apache.org/repos/asf/hbase/blob/d1f7bcbf/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteColumnFamilyProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteColumnFamilyProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteColumnFamilyProcedure.java
index 6e96910..fb9754a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteColumnFamilyProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteColumnFamilyProcedure.java
@@ -57,6 +57,7 @@ public class DeleteColumnFamilyProcedure
private HTableDescriptor unmodifiedHTableDescriptor;
private TableName tableName;
private byte [] familyName;
+ private boolean hasMob;
private UserGroupInformation user;
private List<HRegionInfo> regionInfoList;
@@ -283,6 +284,8 @@ public class DeleteColumnFamilyProcedure
throw new InvalidFamilyOperationException("Family '" + getColumnFamilyName()
+ "' does not exist, so it cannot be deleted");
}
+ // whether mob family
+ hasMob = unmodifiedHTableDescriptor.getFamily(familyName).isMobEnabled();
}
/**
@@ -334,7 +337,7 @@ public class DeleteColumnFamilyProcedure
**/
private void deleteFromFs(final MasterProcedureEnv env) throws IOException {
MasterDDLOperationHelper.deleteColumnFamilyFromFileSystem(env, tableName,
- getRegionInfoList(env), familyName);
+ getRegionInfoList(env), familyName, hasMob);
}
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/d1f7bcbf/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterDDLOperationHelper.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterDDLOperationHelper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterDDLOperationHelper.java
index c6ff1b6..abfb776 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterDDLOperationHelper.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterDDLOperationHelper.java
@@ -91,7 +91,8 @@ public final class MasterDDLOperationHelper {
final MasterProcedureEnv env,
final TableName tableName,
List<HRegionInfo> regionInfoList,
- final byte[] familyName) throws IOException {
+ final byte[] familyName,
+ boolean hasMob) throws IOException {
final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
if (LOG.isDebugEnabled()) {
LOG.debug("Removing family=" + Bytes.toString(familyName) + " from table=" + tableName);
@@ -101,7 +102,7 @@ public final class MasterDDLOperationHelper {
}
for (HRegionInfo hri : regionInfoList) {
// Delete the family directory in FS for all the regions one by one
- mfs.deleteFamilyFromFS(hri, familyName);
+ mfs.deleteFamilyFromFS(hri, familyName, hasMob);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/d1f7bcbf/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyTableProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyTableProcedure.java
index a082ea4..29846eb 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyTableProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyTableProcedure.java
@@ -368,7 +368,8 @@ public class ModifyTableProcedure
env,
getTableName(),
getRegionInfoList(env),
- familyName);
+ familyName,
+ oldHTableDescriptor.getFamily(familyName).isMobEnabled());
}
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/d1f7bcbf/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
index 53cd1a1..d283729 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
@@ -45,6 +45,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellComparator;
+import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
@@ -870,4 +871,27 @@ public class MobUtils {
return false;
}
}
+
+ /**
+ * Archive mob store files
+ * @param conf The current configuration.
+ * @param fs The current file system.
+ * @param mobRegionInfo The mob family region info.
+ * @param mobFamilyDir The mob family directory.
+ * @param family The name of the column family.
+ * @throws IOException
+ */
+ public static void archiveMobStoreFiles(Configuration conf, FileSystem fs,
+ HRegionInfo mobRegionInfo, Path mobFamilyDir, byte[] family) throws IOException {
+ // disable the block cache.
+ Configuration copyOfConf = HBaseConfiguration.create(conf);
+ copyOfConf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0f);
+ CacheConfig cacheConfig = new CacheConfig(copyOfConf);
+ FileStatus[] fileStatus = FSUtils.listStatus(fs, mobFamilyDir);
+ List<StoreFile> storeFileList = new ArrayList<StoreFile>();
+ for (FileStatus file : fileStatus) {
+ storeFileList.add(new StoreFile(fs, file.getPath(), conf, cacheConfig, BloomType.NONE));
+ }
+ HFileArchiver.archiveStoreFiles(conf, fs, mobRegionInfo, mobFamilyDir, family, storeFileList);
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/d1f7bcbf/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HFileArchiveUtil.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HFileArchiveUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HFileArchiveUtil.java
index 937e9b2..a235696 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HFileArchiveUtil.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HFileArchiveUtil.java
@@ -64,10 +64,8 @@ public class HFileArchiveUtil {
HRegionInfo region,
Path tabledir,
byte[] family) throws IOException {
- TableName tableName =
- FSUtils.getTableName(tabledir);
Path rootDir = FSUtils.getRootDir(conf);
- Path tableArchiveDir = getTableArchivePath(rootDir, tableName);
+ Path tableArchiveDir = getTableArchivePath(rootDir, region.getTable());
return HStore.getStoreHomedir(tableArchiveDir, region, family);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/d1f7bcbf/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDeleteMobTable.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDeleteMobTable.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDeleteMobTable.java
index 6dbcec0..7ad49bc 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDeleteMobTable.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDeleteMobTable.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.*;
@@ -161,6 +162,48 @@ public class TestDeleteMobTable {
}
}
}
+
+ @Test
+ public void testMobFamilyDelete() throws Exception {
+ byte[] tableName = Bytes.toBytes("testMobFamilyDelete");
+ TableName tn = TableName.valueOf(tableName);
+ HTableDescriptor htd = new HTableDescriptor(tn);
+ HColumnDescriptor hcd = new HColumnDescriptor(FAMILY);
+ hcd.setMobEnabled(true);
+ hcd.setMobThreshold(0);
+ htd.addFamily(hcd);
+ htd.addFamily(new HColumnDescriptor(Bytes.toBytes("family2")));
+ HBaseAdmin admin = null;
+ Table table = null;
+ try {
+ admin = TEST_UTIL.getHBaseAdmin();
+ admin.createTable(htd);
+ table = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration()).getTable(tn);
+ byte[] value = generateMobValue(10);
+ byte[] row = Bytes.toBytes("row");
+ Put put = new Put(row);
+ put.addColumn(FAMILY, QF, EnvironmentEdgeManager.currentTime(), value);
+ table.put(put);
+ admin.flush(tn);
+ // the mob file exists
+ Assert.assertEquals(1, countMobFiles(tn, hcd.getNameAsString()));
+ Assert.assertEquals(0, countArchiveMobFiles(tn, hcd.getNameAsString()));
+ String fileName = assertHasOneMobRow(table, tn, hcd.getNameAsString());
+ Assert.assertFalse(mobArchiveExist(tn, hcd.getNameAsString(), fileName));
+ Assert.assertTrue(mobTableDirExist(tn));
+ admin.deleteColumnFamily(tn, FAMILY);
+ Assert.assertEquals(0, countMobFiles(tn, hcd.getNameAsString()));
+ Assert.assertEquals(1, countArchiveMobFiles(tn, hcd.getNameAsString()));
+ Assert.assertTrue(mobArchiveExist(tn, hcd.getNameAsString(), fileName));
+ Assert.assertFalse(mobColumnFamilyDirExist(tn));
+ } finally {
+ table.close();
+ if (admin != null) {
+ admin.close();
+ }
+ TEST_UTIL.deleteTable(tableName);
+ }
+ }
private int countMobFiles(TableName tn, String familyName) throws IOException {
FileSystem fs = TEST_UTIL.getTestFileSystem();
@@ -189,6 +232,14 @@ public class TestDeleteMobTable {
Path tableDir = FSUtils.getTableDir(MobUtils.getMobHome(TEST_UTIL.getConfiguration()), tn);
return fs.exists(tableDir);
}
+
+ private boolean mobColumnFamilyDirExist(TableName tn) throws IOException {
+ FileSystem fs = TEST_UTIL.getTestFileSystem();
+ Path tableDir = FSUtils.getTableDir(MobUtils.getMobHome(TEST_UTIL.getConfiguration()), tn);
+ HRegionInfo mobRegionInfo = MobUtils.getMobRegionInfo(tn);
+ Path mobFamilyDir = new Path(tableDir, new Path(mobRegionInfo.getEncodedName(), Bytes.toString(FAMILY)));
+ return fs.exists(mobFamilyDir);
+ }
private boolean mobArchiveExist(TableName tn, String familyName, String fileName)
throws IOException {