You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by en...@apache.org on 2015/08/25 21:10:05 UTC
hbase git commit: HBASE-14302 TableSnapshotInputFormat should not
create back references when restoring snapshot
Repository: hbase
Updated Branches:
refs/heads/master d0873f5a8 -> 44caba31e
HBASE-14302 TableSnapshotInputFormat should not create back references when restoring snapshot
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/44caba31
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/44caba31
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/44caba31
Branch: refs/heads/master
Commit: 44caba31e4f259ea82d5141d2ba40a6d98f14f61
Parents: d0873f5
Author: Enis Soztutar <en...@apache.org>
Authored: Tue Aug 25 12:10:15 2015 -0700
Committer: Enis Soztutar <en...@apache.org>
Committed: Tue Aug 25 12:10:15 2015 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/io/HFileLink.java | 89 +++++++++++++++++---
.../hbase/snapshot/RestoreSnapshotHelper.java | 33 ++++++--
.../hadoop/hbase/util/ModifyRegionUtils.java | 13 +--
.../mapred/TestTableSnapshotInputFormat.java | 9 ++
.../TableSnapshotInputFormatTestBase.java | 51 ++++++++++-
.../mapreduce/TestTableSnapshotInputFormat.java | 10 +++
6 files changed, 174 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/44caba31/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java
index e4a3b58..d16970d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java
@@ -321,9 +321,30 @@ public class HFileLink extends FileLink {
public static boolean create(final Configuration conf, final FileSystem fs,
final Path dstFamilyPath, final HRegionInfo hfileRegionInfo,
final String hfileName) throws IOException {
+ return create(conf, fs, dstFamilyPath, hfileRegionInfo, hfileName, true);
+ }
+
+ /**
+ * Create a new HFileLink
+ *
+ * <p>It also adds a back-reference to the hfile back-reference directory
+ * to simplify the reference-count and the cleaning process.
+ *
+ * @param conf {@link Configuration} to read for the archive directory name
+ * @param fs {@link FileSystem} on which to write the HFileLink
+ * @param dstFamilyPath - Destination path (table/region/cf/)
+ * @param hfileRegionInfo - Linked HFile Region Info
+ * @param hfileName - Linked HFile name
+ * @param createBackRef - Whether back reference should be created. Defaults to true.
+ * @return true if the file is created, otherwise the file exists.
+ * @throws IOException on file or parent directory creation failure
+ */
+ public static boolean create(final Configuration conf, final FileSystem fs,
+ final Path dstFamilyPath, final HRegionInfo hfileRegionInfo,
+ final String hfileName, final boolean createBackRef) throws IOException {
TableName linkedTable = hfileRegionInfo.getTable();
String linkedRegion = hfileRegionInfo.getEncodedName();
- return create(conf, fs, dstFamilyPath, linkedTable, linkedRegion, hfileName);
+ return create(conf, fs, dstFamilyPath, linkedTable, linkedRegion, hfileName, createBackRef);
}
/**
@@ -344,6 +365,28 @@ public class HFileLink extends FileLink {
public static boolean create(final Configuration conf, final FileSystem fs,
final Path dstFamilyPath, final TableName linkedTable, final String linkedRegion,
final String hfileName) throws IOException {
+ return create(conf, fs, dstFamilyPath, linkedTable, linkedRegion, hfileName, true);
+ }
+
+ /**
+ * Create a new HFileLink
+ *
+ * <p>It also adds a back-reference to the hfile back-reference directory
+ * to simplify the reference-count and the cleaning process.
+ *
+ * @param conf {@link Configuration} to read for the archive directory name
+ * @param fs {@link FileSystem} on which to write the HFileLink
+ * @param dstFamilyPath - Destination path (table/region/cf/)
+ * @param linkedTable - Linked Table Name
+ * @param linkedRegion - Linked Region Name
+ * @param hfileName - Linked HFile name
+ * @param createBackRef - Whether back reference should be created. Defaults to true.
+ * @return true if the file is created, otherwise the file exists.
+ * @throws IOException on file or parent directory creation failure
+ */
+ public static boolean create(final Configuration conf, final FileSystem fs,
+ final Path dstFamilyPath, final TableName linkedTable, final String linkedRegion,
+ final String hfileName, final boolean createBackRef) throws IOException {
String familyName = dstFamilyPath.getName();
String regionName = dstFamilyPath.getParent().getName();
String tableName = FSUtils.getTableName(dstFamilyPath.getParent().getParent())
@@ -358,19 +401,24 @@ public class HFileLink extends FileLink {
// Make sure the FileLink reference directory exists
Path archiveStoreDir = HFileArchiveUtil.getStoreArchivePath(conf,
linkedTable, linkedRegion, familyName);
- Path backRefssDir = getBackReferencesDir(archiveStoreDir, hfileName);
- fs.mkdirs(backRefssDir);
-
- // Create the reference for the link
- Path backRefPath = new Path(backRefssDir, refName);
- fs.createNewFile(backRefPath);
+ Path backRefPath = null;
+ if (createBackRef) {
+ Path backRefssDir = getBackReferencesDir(archiveStoreDir, hfileName);
+ fs.mkdirs(backRefssDir);
+
+ // Create the reference for the link
+ backRefPath = new Path(backRefssDir, refName);
+ fs.createNewFile(backRefPath);
+ }
try {
// Create the link
return fs.createNewFile(new Path(dstFamilyPath, name));
} catch (IOException e) {
LOG.error("couldn't create the link=" + name + " for " + dstFamilyPath, e);
// Revert the reference if the link creation failed
- fs.delete(backRefPath, false);
+ if (createBackRef) {
+ fs.delete(backRefPath, false);
+ }
throw e;
}
}
@@ -389,13 +437,34 @@ public class HFileLink extends FileLink {
* @throws IOException on file or parent directory creation failure
*/
public static boolean createFromHFileLink(final Configuration conf, final FileSystem fs,
- final Path dstFamilyPath, final String hfileLinkName) throws IOException {
+ final Path dstFamilyPath, final String hfileLinkName)
+ throws IOException {
+ return createFromHFileLink(conf, fs, dstFamilyPath, hfileLinkName, true);
+ }
+
+ /**
+ * Create a new HFileLink starting from a hfileLink name
+ *
+ * <p>It also adds a back-reference to the hfile back-reference directory
+ * to simplify the reference-count and the cleaning process.
+ *
+ * @param conf {@link Configuration} to read for the archive directory name
+ * @param fs {@link FileSystem} on which to write the HFileLink
+ * @param dstFamilyPath - Destination path (table/region/cf/)
+ * @param hfileLinkName - HFileLink name (it contains hfile-region-table)
+ * @param createBackRef - Whether back reference should be created. Defaults to true.
+ * @return true if the file is created, otherwise the file exists.
+ * @throws IOException on file or parent directory creation failure
+ */
+ public static boolean createFromHFileLink(final Configuration conf, final FileSystem fs,
+ final Path dstFamilyPath, final String hfileLinkName, final boolean createBackRef)
+ throws IOException {
Matcher m = LINK_NAME_PATTERN.matcher(hfileLinkName);
if (!m.matches()) {
throw new IllegalArgumentException(hfileLinkName + " is not a valid HFileLink name!");
}
return create(conf, fs, dstFamilyPath, TableName.valueOf(m.group(1), m.group(2)),
- m.group(3), m.group(4));
+ m.group(3), m.group(4), createBackRef);
}
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/44caba31/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
index 8e7a222..1c684fb 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
@@ -127,6 +127,7 @@ public class RestoreSnapshotHelper {
private final Configuration conf;
private final FileSystem fs;
+ private final boolean createBackRefs;
public RestoreSnapshotHelper(final Configuration conf,
final FileSystem fs,
@@ -134,7 +135,18 @@ public class RestoreSnapshotHelper {
final HTableDescriptor tableDescriptor,
final Path rootDir,
final ForeignExceptionDispatcher monitor,
- final MonitoredTask status)
+ final MonitoredTask status) {
+ this(conf, fs, manifest, tableDescriptor, rootDir, monitor, status, true);
+ }
+
+ public RestoreSnapshotHelper(final Configuration conf,
+ final FileSystem fs,
+ final SnapshotManifest manifest,
+ final HTableDescriptor tableDescriptor,
+ final Path rootDir,
+ final ForeignExceptionDispatcher monitor,
+ final MonitoredTask status,
+ final boolean createBackRefs)
{
this.fs = fs;
this.conf = conf;
@@ -146,6 +158,7 @@ public class RestoreSnapshotHelper {
this.tableDir = FSUtils.getTableDir(rootDir, tableDesc.getTableName());
this.monitor = monitor;
this.status = status;
+ this.createBackRefs = createBackRefs;
}
/**
@@ -484,7 +497,7 @@ public class RestoreSnapshotHelper {
for (SnapshotRegionManifest.StoreFile storeFile: hfilesToAdd) {
LOG.debug("Adding HFileLink " + storeFile.getName() +
" to region=" + regionInfo.getEncodedName() + " table=" + tableName);
- restoreStoreFile(familyDir, regionInfo, storeFile);
+ restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs);
}
} else {
// Family doesn't exists in the snapshot
@@ -505,7 +518,7 @@ public class RestoreSnapshotHelper {
for (SnapshotRegionManifest.StoreFile storeFile: familyEntry.getValue()) {
LOG.trace("Adding HFileLink " + storeFile.getName() + " to table=" + tableName);
- restoreStoreFile(familyDir, regionInfo, storeFile);
+ restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs);
}
}
}
@@ -598,7 +611,7 @@ public class RestoreSnapshotHelper {
Path familyDir = new Path(regionDir, familyFiles.getFamilyName().toStringUtf8());
for (SnapshotRegionManifest.StoreFile storeFile: familyFiles.getStoreFilesList()) {
LOG.info("Adding HFileLink " + storeFile.getName() + " to table=" + tableName);
- restoreStoreFile(familyDir, snapshotRegionInfo, storeFile);
+ restoreStoreFile(familyDir, snapshotRegionInfo, storeFile, createBackRefs);
}
}
}
@@ -630,17 +643,19 @@ public class RestoreSnapshotHelper {
* </ul>
* @param familyDir destination directory for the store file
* @param regionInfo destination region info for the table
+ * @param createBackRef - Whether back reference should be created. Defaults to true.
* @param storeFile store file name (can be a Reference, HFileLink or simple HFile)
*/
private void restoreStoreFile(final Path familyDir, final HRegionInfo regionInfo,
- final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
+ final SnapshotRegionManifest.StoreFile storeFile, final boolean createBackRef)
+ throws IOException {
String hfileName = storeFile.getName();
if (HFileLink.isHFileLink(hfileName)) {
- HFileLink.createFromHFileLink(conf, fs, familyDir, hfileName);
+ HFileLink.createFromHFileLink(conf, fs, familyDir, hfileName, createBackRef);
} else if (StoreFileInfo.isReference(hfileName)) {
restoreReferenceFile(familyDir, regionInfo, storeFile);
} else {
- HFileLink.create(conf, fs, familyDir, regionInfo, hfileName);
+ HFileLink.create(conf, fs, familyDir, regionInfo, hfileName, createBackRef);
}
}
@@ -806,8 +821,10 @@ public class RestoreSnapshotHelper {
"Restoring snapshot '" + snapshotName + "' to directory " + restoreDir);
ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher();
+ // we send createBackRefs=false so that restored hfiles do not create back reference links
+ // in the base hbase root dir.
RestoreSnapshotHelper helper = new RestoreSnapshotHelper(conf, fs,
- manifest, manifest.getTableDescriptor(), restoreDir, monitor, status);
+ manifest, manifest.getTableDescriptor(), restoreDir, monitor, status, false);
helper.restoreHdfsRegions(); // TODO: parallelize.
if (LOG.isDebugEnabled()) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/44caba31/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java
index 347cad5..a936fc2 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java
@@ -23,7 +23,6 @@ import java.io.IOException;
import java.io.InterruptedIOException;
import java.util.ArrayList;
import java.util.Collection;
-import java.util.Collections;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.CompletionService;
@@ -33,7 +32,6 @@ import java.util.concurrent.ThreadFactory;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
-import org.apache.commons.lang.RandomStringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.HConstants;
@@ -44,10 +42,6 @@ import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.master.AssignmentManager;
-import org.apache.hadoop.hbase.regionserver.wal.MetricsWAL;
-import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
-import org.apache.hadoop.hbase.wal.WAL;
-import org.apache.hadoop.hbase.wal.WALFactory;
/**
* Utility methods for interacting with the regions.
@@ -176,11 +170,7 @@ public abstract class ModifyRegionUtils {
// unless I pass along via the conf.
Configuration confForWAL = new Configuration(conf);
confForWAL.set(HConstants.HBASE_DIR, rootDir.toString());
- WAL wal = (new WALFactory(confForWAL,
- Collections.<WALActionsListener>singletonList(new MetricsWAL()),
- "hregion-" + RandomStringUtils.randomNumeric(8))).
- getWAL(newRegion.getEncodedNameAsBytes());
- HRegion region = HRegion.createHRegion(newRegion, rootDir, conf, hTableDescriptor, wal, false);
+ HRegion region = HRegion.createHRegion(newRegion, rootDir, conf, hTableDescriptor, null, false);
try {
// 2. Custom user code to interact with the created region
if (task != null) {
@@ -189,7 +179,6 @@ public abstract class ModifyRegionUtils {
} finally {
// 3. Close the new region to flush to disk. Close log file too.
region.close();
- wal.close();
}
return region.getRegionInfo();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/44caba31/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableSnapshotInputFormat.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableSnapshotInputFormat.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableSnapshotInputFormat.java
index eabedec..c85cead 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableSnapshotInputFormat.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableSnapshotInputFormat.java
@@ -149,6 +149,15 @@ public class TestTableSnapshotInputFormat extends TableSnapshotInputFormatTestBa
}
@Override
+ public void testRestoreSnapshotDoesNotCreateBackRefLinksInit(TableName tableName,
+ String snapshotName, Path tmpTableDir) throws Exception {
+ JobConf job = new JobConf(UTIL.getConfiguration());
+ TableMapReduceUtil.initTableSnapshotMapJob(snapshotName,
+ COLUMNS, TestTableSnapshotMapper.class, ImmutableBytesWritable.class,
+ NullWritable.class, job, false, tmpTableDir);
+ }
+
+ @Override
protected void testWithMockedMapReduce(HBaseTestingUtility util, String snapshotName,
int numRegions, int expectedNumSplits) throws Exception {
setupCluster();
http://git-wip-us.apache.org/repos/asf/hbase/blob/44caba31/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java
index 43e29ad..d6c5d32 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.hbase.mapreduce;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Cell;
@@ -26,17 +27,21 @@ import org.apache.hadoop.hbase.CellScanner;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.io.HFileLink;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
+import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.HFileArchiveUtil;
import org.junit.Assert;
import org.junit.Test;
+import static org.junit.Assert.assertFalse;
+
import java.io.IOException;
import java.util.Arrays;
@@ -102,6 +107,50 @@ public abstract class TableSnapshotInputFormatTestBase {
testWithMapReduce(UTIL, "testWithMapReduceAndOfflineHBaseMultiRegion", 10, 8, true);
}
+ // Test that snapshot restore does not create back references in the HBase root dir.
+ @Test
+ public void testRestoreSnapshotDoesNotCreateBackRefLinks() throws Exception {
+ setupCluster();
+ TableName tableName = TableName.valueOf("testRestoreSnapshotDoesNotCreateBackRefLinks");
+ String snapshotName = "foo";
+
+ try {
+ createTableAndSnapshot(UTIL, tableName, snapshotName, getStartRow(), getEndRow(), 1);
+
+ Path tmpTableDir = UTIL.getDataTestDirOnTestFS(snapshotName);
+
+ testRestoreSnapshotDoesNotCreateBackRefLinksInit(tableName, snapshotName,tmpTableDir);
+
+ Path rootDir = FSUtils.getRootDir(UTIL.getConfiguration());
+ for (Path regionDir : FSUtils.getRegionDirs(fs, FSUtils.getTableDir(rootDir, tableName))) {
+ for (Path storeDir : FSUtils.getFamilyDirs(fs, regionDir)) {
+ for (FileStatus status : fs.listStatus(storeDir)) {
+ System.out.println(status.getPath());
+ if (StoreFileInfo.isValid(status)) {
+ Path archiveStoreDir = HFileArchiveUtil.getStoreArchivePath(UTIL.getConfiguration(),
+ tableName, regionDir.getName(), storeDir.getName());
+
+ Path path = HFileLink.getBackReferencesDir(storeDir, status.getPath().getName());
+ // assert back references directory is empty
+ assertFalse("There is a back reference in " + path, fs.exists(path));
+
+ path = HFileLink.getBackReferencesDir(archiveStoreDir, status.getPath().getName());
+ // assert back references directory is empty
+ assertFalse("There is a back reference in " + path, fs.exists(path));
+ }
+ }
+ }
+ }
+ } finally {
+ UTIL.getHBaseAdmin().deleteSnapshot(snapshotName);
+ UTIL.deleteTable(tableName);
+ tearDownCluster();
+ }
+ }
+
+ public abstract void testRestoreSnapshotDoesNotCreateBackRefLinksInit(TableName tableName,
+ String snapshotName, Path tmpTableDir) throws Exception;
+
protected void testWithMapReduce(HBaseTestingUtility util, String snapshotName,
int numRegions, int expectedNumSplits, boolean shutdownCluster) throws Exception {
setupCluster();
http://git-wip-us.apache.org/repos/asf/hbase/blob/44caba31/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java
index 8d7e2d3..da8669d 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java
@@ -178,6 +178,16 @@ public class TestTableSnapshotInputFormat extends TableSnapshotInputFormatTestBa
}
}
+ @Override
+ public void testRestoreSnapshotDoesNotCreateBackRefLinksInit(TableName tableName,
+ String snapshotName, Path tmpTableDir) throws Exception {
+ Job job = new Job(UTIL.getConfiguration());
+ TableMapReduceUtil.initTableSnapshotMapperJob(snapshotName,
+ new Scan(), TestTableSnapshotMapper.class, ImmutableBytesWritable.class,
+ NullWritable.class, job, false, tmpTableDir);
+ }
+
+ @Override
public void testWithMockedMapReduce(HBaseTestingUtility util, String snapshotName,
int numRegions, int expectedNumSplits) throws Exception {
setupCluster();