You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by el...@apache.org on 2018/12/07 22:29:12 UTC
[29/51] [abbrv] hbase git commit: HBASE-21300 Fix the wrong reference
file path when restoring snapshots for tables with MOB columns
HBASE-21300 Fix the wrong reference file path when restoring snapshots for tables with MOB columns
Signed-off-by: Guanghao Zhang <zg...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f1f2b5a0
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f1f2b5a0
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f1f2b5a0
Branch: refs/heads/HBASE-20952
Commit: f1f2b5a038d730e68720503331951a3c7edb053d
Parents: d6e1d18
Author: meiyi <my...@gamil.com>
Authored: Tue Nov 27 14:32:40 2018 +0800
Committer: Guanghao Zhang <zg...@apache.org>
Committed: Thu Nov 29 11:23:45 2018 +0800
----------------------------------------------------------------------
.../procedure/CloneSnapshotProcedure.java | 31 ++++++++-
.../org/apache/hadoop/hbase/mob/MobUtils.java | 38 ++++++++++-
.../hbase/snapshot/RestoreSnapshotHelper.java | 2 +-
.../snapshot/TestMobRestoreSnapshotHelper.java | 16 ++++-
.../snapshot/TestRestoreSnapshotHelper.java | 69 +++++++++++++++++++-
5 files changed, 147 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/f1f2b5a0/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java
index 9ba3485..d351d67 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java
@@ -42,6 +42,7 @@ import org.apache.hadoop.hbase.master.MetricsSnapshot;
import org.apache.hadoop.hbase.master.RegionState;
import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
import org.apache.hadoop.hbase.master.procedure.CreateTableProcedure.CreateHdfsRegions;
+import org.apache.hadoop.hbase.mob.MobUtils;
import org.apache.hadoop.hbase.monitoring.MonitoredTask;
import org.apache.hadoop.hbase.monitoring.TaskMonitor;
import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer;
@@ -470,11 +471,39 @@ public class CloneSnapshotProcedure
// 3. Move Table temp directory to the hbase root location
CreateTableProcedure.moveTempDirectoryToHBaseRoot(env, tableDescriptor, tempTableDir);
-
+ // Move Table temp mob directory to the hbase root location
+ Path tempMobTableDir = MobUtils.getMobTableDir(tempdir, tableDescriptor.getTableName());
+ if (mfs.getFileSystem().exists(tempMobTableDir)) {
+ moveTempMobDirectoryToHBaseRoot(mfs, tableDescriptor, tempMobTableDir);
+ }
return newRegions;
}
/**
+ * Move table temp mob directory to the hbase root location
+ * @param mfs The master file system
+ * @param tableDescriptor The table to operate on
+ * @param tempMobTableDir The temp mob directory of table
+ * @throws IOException If failed to move temp mob dir to hbase root dir
+ */
+ private void moveTempMobDirectoryToHBaseRoot(final MasterFileSystem mfs,
+ final TableDescriptor tableDescriptor, final Path tempMobTableDir) throws IOException {
+ FileSystem fs = mfs.getFileSystem();
+ final Path tableMobDir =
+ MobUtils.getMobTableDir(mfs.getRootDir(), tableDescriptor.getTableName());
+ if (!fs.delete(tableMobDir, true) && fs.exists(tableMobDir)) {
+ throw new IOException("Couldn't delete mob table " + tableMobDir);
+ }
+ if (!fs.exists(tableMobDir.getParent())) {
+ fs.mkdirs(tableMobDir.getParent());
+ }
+ if (!fs.rename(tempMobTableDir, tableMobDir)) {
+ throw new IOException("Unable to move mob table from temp=" + tempMobTableDir
+ + " to hbase root=" + tableMobDir);
+ }
+ }
+
+ /**
* Add regions to hbase:meta table.
* @param env MasterProcedureEnv
* @throws IOException
http://git-wip-us.apache.org/repos/asf/hbase/blob/f1f2b5a0/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
index 9fa4e4c..304a62e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
@@ -367,7 +367,17 @@ public final class MobUtils {
*/
public static Path getMobHome(Configuration conf) {
Path hbaseDir = new Path(conf.get(HConstants.HBASE_DIR));
- return new Path(hbaseDir, MobConstants.MOB_DIR_NAME);
+ return getMobHome(hbaseDir);
+ }
+
+ /**
+ * Gets the root dir of the mob files under the qualified HBase root dir.
+ * It's {rootDir}/mobdir.
+ * @param rootDir The qualified path of HBase root directory.
+ * @return The root dir of the mob file.
+ */
+ public static Path getMobHome(Path rootDir) {
+ return new Path(rootDir, MobConstants.MOB_DIR_NAME);
}
/**
@@ -384,14 +394,36 @@ public final class MobUtils {
}
/**
+ * Gets the table dir of the mob files under the qualified HBase root dir.
+ * It's {rootDir}/mobdir/data/${namespace}/${tableName}
+ * @param rootDir The qualified path of HBase root directory.
+ * @param tableName The name of table.
+ * @return The table dir of the mob file.
+ */
+ public static Path getMobTableDir(Path rootDir, TableName tableName) {
+ return FSUtils.getTableDir(getMobHome(rootDir), tableName);
+ }
+
+ /**
* Gets the region dir of the mob files.
- * It's {HBASE_DIR}/mobdir/{namespace}/{tableName}/{regionEncodedName}.
+ * It's {HBASE_DIR}/mobdir/data/{namespace}/{tableName}/{regionEncodedName}.
* @param conf The current configuration.
* @param tableName The current table name.
* @return The region dir of the mob files.
*/
public static Path getMobRegionPath(Configuration conf, TableName tableName) {
- Path tablePath = FSUtils.getTableDir(getMobHome(conf), tableName);
+ return getMobRegionPath(new Path(conf.get(HConstants.HBASE_DIR)), tableName);
+ }
+
+ /**
+ * Gets the region dir of the mob files under the specified root dir.
+ * It's {rootDir}/mobdir/data/{namespace}/{tableName}/{regionEncodedName}.
+ * @param rootDir The qualified path of HBase root directory.
+ * @param tableName The current table name.
+ * @return The region dir of the mob files.
+ */
+ public static Path getMobRegionPath(Path rootDir, TableName tableName) {
+ Path tablePath = FSUtils.getTableDir(getMobHome(rootDir), tableName);
RegionInfo regionInfo = getMobRegionInfo(tableName);
return new Path(tablePath, regionInfo.getEncodedName());
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/f1f2b5a0/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
index 6aa378c..0acfb1a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
@@ -622,7 +622,7 @@ public class RestoreSnapshotHelper {
private void cloneHdfsMobRegion(final Map<String, SnapshotRegionManifest> regionManifests,
final RegionInfo region) throws IOException {
// clone region info (change embedded tableName with the new one)
- Path clonedRegionPath = MobUtils.getMobRegionPath(conf, tableDesc.getTableName());
+ Path clonedRegionPath = MobUtils.getMobRegionPath(rootDir, tableDesc.getTableName());
cloneRegion(clonedRegionPath, region, regionManifests.get(region.getEncodedName()));
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/f1f2b5a0/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobRestoreSnapshotHelper.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobRestoreSnapshotHelper.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobRestoreSnapshotHelper.java
index df8eb68..10c67ad 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobRestoreSnapshotHelper.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobRestoreSnapshotHelper.java
@@ -20,9 +20,12 @@ package org.apache.hadoop.hbase.snapshot;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.mob.MobConstants;
import org.apache.hadoop.hbase.snapshot.MobSnapshotTestingUtils.SnapshotMock;
-import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.util.Bytes;
import org.junit.ClassRule;
import org.junit.experimental.categories.Category;
import org.slf4j.Logger;
@@ -31,7 +34,7 @@ import org.slf4j.LoggerFactory;
/**
* Test the restore/clone operation from a file-system point of view.
*/
-@Category(SmallTests.class)
+@Category(MediumTests.class)
public class TestMobRestoreSnapshotHelper extends TestRestoreSnapshotHelper {
@ClassRule
@@ -49,4 +52,13 @@ public class TestMobRestoreSnapshotHelper extends TestRestoreSnapshotHelper {
protected SnapshotMock createSnapshotMock() throws IOException {
return new SnapshotMock(TEST_UTIL.getConfiguration(), fs, rootDir);
}
+
+ @Override
+ protected void createTableAndSnapshot(TableName tableName, String snapshotName)
+ throws IOException {
+ byte[] column = Bytes.toBytes("A");
+ Table table = MobSnapshotTestingUtils.createMobTable(TEST_UTIL, tableName, column);
+ TEST_UTIL.loadTable(table, column);
+ TEST_UTIL.getAdmin().snapshot(snapshotName, tableName);
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/f1f2b5a0/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreSnapshotHelper.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreSnapshotHelper.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreSnapshotHelper.java
index 08c5088..c1ce040 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreSnapshotHelper.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreSnapshotHelper.java
@@ -24,22 +24,32 @@ import java.io.IOException;
import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
import org.apache.hadoop.hbase.io.HFileLink;
+import org.apache.hadoop.hbase.mob.MobUtils;
import org.apache.hadoop.hbase.monitoring.MonitoredTask;
import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils.SnapshotMock;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
-import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.CommonFSUtils;
import org.apache.hadoop.hbase.util.FSTableDescriptors;
import org.apache.hadoop.hbase.util.FSUtils;
import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Assert;
import org.junit.Before;
+import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -52,7 +62,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.Snapshot
/**
* Test the restore/clone operation from a file-system point of view.
*/
-@Category({RegionServerTests.class, SmallTests.class})
+@Category({RegionServerTests.class, MediumTests.class})
public class TestRestoreSnapshotHelper {
@ClassRule
@@ -72,6 +82,16 @@ public class TestRestoreSnapshotHelper {
protected void setupConf(Configuration conf) {
}
+ @BeforeClass
+ public static void setupCluster() throws Exception {
+ TEST_UTIL.startMiniCluster();
+ }
+
+ @AfterClass
+ public static void tearDownCluster() throws Exception {
+ TEST_UTIL.shutdownMiniCluster();
+ }
+
@Before
public void setup() throws Exception {
rootDir = TEST_UTIL.getDataTestDir("testRestore");
@@ -101,6 +121,51 @@ public class TestRestoreSnapshotHelper {
restoreAndVerify("snapshot", "namespace1:testRestoreWithNamespace");
}
+ @Test
+ public void testNoHFileLinkInRootDir() throws IOException {
+ rootDir = TEST_UTIL.getDefaultRootDirPath();
+ FSUtils.setRootDir(conf, rootDir);
+ fs = rootDir.getFileSystem(conf);
+
+ TableName tableName = TableName.valueOf("testNoHFileLinkInRootDir");
+ String snapshotName = tableName.getNameAsString() + "-snapshot";
+ createTableAndSnapshot(tableName, snapshotName);
+
+ Path restoreDir = new Path("/hbase/.tmp-restore");
+ RestoreSnapshotHelper.copySnapshotForScanner(conf, fs, rootDir, restoreDir, snapshotName);
+ checkNoHFileLinkInTableDir(tableName);
+ }
+
+ protected void createTableAndSnapshot(TableName tableName, String snapshotName)
+ throws IOException {
+ byte[] column = Bytes.toBytes("A");
+ Table table = TEST_UTIL.createTable(tableName, column, 2);
+ TEST_UTIL.loadTable(table, column);
+ TEST_UTIL.getAdmin().snapshot(snapshotName, tableName);
+ }
+
+ private void checkNoHFileLinkInTableDir(TableName tableName) throws IOException {
+ Path[] tableDirs = new Path[] { CommonFSUtils.getTableDir(rootDir, tableName),
+ CommonFSUtils.getTableDir(new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY), tableName),
+ CommonFSUtils.getTableDir(MobUtils.getMobHome(rootDir), tableName) };
+ for (Path tableDir : tableDirs) {
+ Assert.assertFalse(hasHFileLink(tableDir));
+ }
+ }
+
+ private boolean hasHFileLink(Path tableDir) throws IOException {
+ if (fs.exists(tableDir)) {
+ RemoteIterator<LocatedFileStatus> iterator = fs.listFiles(tableDir, true);
+ while (iterator.hasNext()) {
+ LocatedFileStatus fileStatus = iterator.next();
+ if (fileStatus.isFile() && HFileLink.isHFileLink(fileStatus.getPath())) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
private void restoreAndVerify(final String snapshotName, final String tableName) throws IOException {
// Test Rolling-Upgrade like Snapshot.
// half machines writing using v1 and the others using v2 format.