You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by jm...@apache.org on 2013/02/13 19:52:22 UTC
svn commit: r1445843 - in /hbase/branches/hbase-7290/hbase-server/src:
main/java/org/apache/hadoop/hbase/io/
main/java/org/apache/hadoop/hbase/regionserver/
main/java/org/apache/hadoop/hbase/snapshot/
test/java/org/apache/hadoop/hbase/regionserver/ tes...
Author: jmhsieh
Date: Wed Feb 13 18:52:21 2013
New Revision: 1445843
URL: http://svn.apache.org/r1445843
Log:
HBASE-7535 Fix restore reference files (Matteo Bertozzi)
Added:
hbase/branches/hbase-7290/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreSnapshotHelper.java
Modified:
hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java
hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
hbase/branches/hbase-7290/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
Modified: hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java?rev=1445843&r1=1445842&r2=1445843&view=diff
==============================================================================
--- hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java (original)
+++ hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java Wed Feb 13 18:52:21 2013
@@ -73,6 +73,14 @@ public class HFileLink extends FileLink
Pattern.compile(String.format("^(%s)=(%s)-(%s)$", HTableDescriptor.VALID_USER_TABLE_REGEX,
HRegionInfo.ENCODED_REGION_NAME_REGEX, StoreFile.HFILE_NAME_REGEX));
+ /**
+ * The link should be used for hfile and reference links
+ * that can be found in /hbase/table/region/family/
+ */
+ private static final Pattern REF_OR_HFILE_LINK_PATTERN =
+ Pattern.compile(String.format("^(%s)=(%s)-(.+)$", HTableDescriptor.VALID_USER_TABLE_REGEX,
+ HRegionInfo.ENCODED_REGION_NAME_REGEX));
+
private final Path archivePath;
private final Path originPath;
@@ -182,7 +190,7 @@ public class HFileLink extends FileLink
/**
* Convert a HFileLink path to a table relative path.
- * e.g. the link: /hbase/test/0123/cf/abcd-4567-testtb
+ * e.g. the link: /hbase/test/0123/cf/testtb=4567-abcd
* becomes: /hbase/testtb/4567/cf/abcd
*
* @param path HFileLink path
@@ -190,8 +198,8 @@ public class HFileLink extends FileLink
* @throws IOException on unexpected error.
*/
private static Path getRelativeTablePath(final Path path) {
- // hfile-region-table
- Matcher m = LINK_NAME_PATTERN.matcher(path.getName());
+ // table=region-hfile
+ Matcher m = REF_OR_HFILE_LINK_PATTERN.matcher(path.getName());
if (!m.matches()) {
throw new IllegalArgumentException(path.getName() + " is not a valid HFileLink name!");
}
@@ -211,7 +219,7 @@ public class HFileLink extends FileLink
* @return the name of the referenced HFile
*/
public static String getReferencedHFileName(final String fileName) {
- Matcher m = LINK_NAME_PATTERN.matcher(fileName);
+ Matcher m = REF_OR_HFILE_LINK_PATTERN.matcher(fileName);
if (!m.matches()) {
throw new IllegalArgumentException(fileName + " is not a valid HFileLink name!");
}
@@ -225,7 +233,7 @@ public class HFileLink extends FileLink
* @return the name of the referenced Region
*/
public static String getReferencedRegionName(final String fileName) {
- Matcher m = LINK_NAME_PATTERN.matcher(fileName);
+ Matcher m = REF_OR_HFILE_LINK_PATTERN.matcher(fileName);
if (!m.matches()) {
throw new IllegalArgumentException(fileName + " is not a valid HFileLink name!");
}
@@ -239,7 +247,7 @@ public class HFileLink extends FileLink
* @return the name of the referenced Table
*/
public static String getReferencedTableName(final String fileName) {
- Matcher m = LINK_NAME_PATTERN.matcher(fileName);
+ Matcher m = REF_OR_HFILE_LINK_PATTERN.matcher(fileName);
if (!m.matches()) {
throw new IllegalArgumentException(fileName + " is not a valid HFileLink name!");
}
Modified: hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java?rev=1445843&r1=1445842&r2=1445843&view=diff
==============================================================================
--- hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java (original)
+++ hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java Wed Feb 13 18:52:21 2013
@@ -366,6 +366,8 @@ public class StoreFile {
// Tabledir is up two directories from where Reference was written.
Path tableDir = p.getParent().getParent().getParent();
String nameStrippedOfSuffix = m.group(1);
+ LOG.debug("reference '" + p + "' to region=" + otherRegion + " hfile=" + nameStrippedOfSuffix);
+
// Build up new path with the referenced region in place of our current
// region in the reference path. Also strip regionname suffix from name.
return new Path(new Path(new Path(tableDir, otherRegion),
@@ -914,6 +916,8 @@ public class StoreFile {
public static boolean validateStoreFileName(String fileName) {
if (HFileLink.isHFileLink(fileName))
return true;
+ if (isReference(fileName))
+ return true;
return !fileName.contains("-");
}
@@ -946,7 +950,6 @@ public class StoreFile {
// Write reference with same file id only with the other region name as
// suffix and into the new region location (under same family).
Path p = new Path(splitDir, f.getPath().getName() + "." + parentRegionName);
- LOG.info("StoreFile.split(): splitDir=" + splitDir + " p=" + p);
return r.write(fs, p);
}
Modified: hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java?rev=1445843&r1=1445842&r2=1445843&view=diff
==============================================================================
--- hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java (original)
+++ hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java Wed Feb 13 18:52:21 2013
@@ -354,7 +354,12 @@ public class RestoreSnapshotHelper {
/**
* Create a new {@link HFileLink} to reference the store file.
- *
+ * <p>The store file in the snapshot can be a simple hfile, an HFileLink or a reference.
+ * <ul>
+ * <li>hfile: abc -> table=region-abc
+ * <li>reference: abc.1234 -> table=region-abc.1234
+ * <li>hfilelink: table=region-hfile -> table=region-hfile
+ * </ul>
* @param familyDir destination directory for the store file
* @param regionInfo destination region info for the table
* @param hfileName store file name (can be a Reference, HFileLink or simple HFile)
@@ -372,17 +377,46 @@ public class RestoreSnapshotHelper {
/**
* Create a new {@link Reference} as copy of the source one.
- *
+ * <p><blockquote><pre>
+ * The source table looks like:
+ * 1234/abc (original file)
+ * 5678/abc.1234 (reference file)
+ *
+ * After the clone operation looks like:
+ * wxyz/table=1234-abc
+ * stuv/table=1234-abc.wxyz
+ *
+ * NOTE that the region name in the clone change (md5 of regioninfo)
+ * and the reference should reflect that change.
+ * </pre></blockquote>
* @param familyDir destination directory for the store file
* @param regionInfo destination region info for the table
* @param hfileName reference file name
*/
private void restoreReferenceFile(final Path familyDir, final HRegionInfo regionInfo,
final String hfileName) throws IOException {
- Path inPath = new Path(new Path(new Path(snapshotDesc.getTable(),
- regionInfo.getEncodedName()), familyDir.getName()), hfileName);
- Path outPath = new Path(familyDir, StoreFile.getReferredToFile(inPath).getName());
- InputStream in = new HFileLink(conf, inPath).open(fs);
+ // Extract the referred information (hfile name and parent region)
+ String tableName = snapshotDesc.getTable();
+ Path refPath = StoreFile.getReferredToFile(new Path(new Path(new Path(tableName,
+ regionInfo.getEncodedName()), familyDir.getName()), hfileName));
+ String snapshotRegionName = refPath.getParent().getParent().getName();
+ String fileName = refPath.getName();
+
+ // The new reference should have the cloned region name as parent, if it is a clone.
+ String clonedRegionName = Bytes.toString(regionsMap.get(Bytes.toBytes(snapshotRegionName)));
+ if (clonedRegionName == null) clonedRegionName = snapshotRegionName;
+
+ // The output file should be a reference link table=snapshotRegion-fileName.clonedRegionName
+ String refLink = fileName;
+ if (!HFileLink.isHFileLink(fileName)) {
+ refLink = HFileLink.createHFileLinkName(tableName, snapshotRegionName, fileName);
+ }
+ Path outPath = new Path(familyDir, refLink + '.' + clonedRegionName);
+
+ // Create the new reference
+ Path linkPath = new Path(familyDir,
+ HFileLink.createHFileLinkName(tableName, regionInfo.getEncodedName(), hfileName));
+ InputStream in = new HFileLink(conf, linkPath).open(fs);
OutputStream out = fs.create(outPath);
IOUtils.copyBytes(in, out, conf);
}
Modified: hbase/branches/hbase-7290/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-7290/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java?rev=1445843&r1=1445842&r2=1445843&view=diff
==============================================================================
--- hbase/branches/hbase-7290/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java (original)
+++ hbase/branches/hbase-7290/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java Wed Feb 13 18:52:21 2013
@@ -225,9 +225,12 @@ public class TestStoreFile extends HBase
"MyTable_02=abc012-def345_SeqId_1_", "MyTable_02=abc012-def345_SeqId_20_" };
for (String name: legalHFileLink) {
assertTrue("should be a valid link: " + name, HFileLink.isHFileLink(name));
+ assertTrue("should be a valid StoreFile" + name, StoreFile.validateStoreFileName(name));
+ assertFalse("should not be a valid reference: " + name, StoreFile.isReference(name));
String refName = name + ".6789";
assertTrue("should be a valid link reference: " + refName, StoreFile.isReference(refName));
+ assertTrue("should be a valid StoreFile" + refName, StoreFile.validateStoreFileName(refName));
}
String[] illegalHFileLink = { ".MyTable_02=abc012-def345", "-MyTable_02.300=abc012-def345",
Added: hbase/branches/hbase-7290/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreSnapshotHelper.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-7290/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreSnapshotHelper.java?rev=1445843&view=auto
==============================================================================
--- hbase/branches/hbase-7290/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreSnapshotHelper.java (added)
+++ hbase/branches/hbase-7290/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreSnapshotHelper.java Wed Feb 13 18:52:21 2013
@@ -0,0 +1,203 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.snapshot;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.SmallTests;
+import org.apache.hadoop.hbase.catalog.CatalogTracker;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HConnectionTestingUtility;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
+import org.apache.hadoop.hbase.io.HFileLink;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.StoreFile;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSTableDescriptors;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.MD5Hash;
+import org.junit.*;
+import org.junit.experimental.categories.Category;
+import org.mockito.Mockito;
+
+/**
+ * Test the restore/clone operation from a file-system point of view.
+ */
+@Category(SmallTests.class)
+public class TestRestoreSnapshotHelper {
+ final Log LOG = LogFactory.getLog(getClass());
+
+ private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+ private final static String TEST_FAMILY = "cf";
+ private final static String TEST_HFILE = "abc";
+
+ private Configuration conf;
+ private Path archiveDir;
+ private FileSystem fs;
+ private Path rootDir;
+
+ @Before
+ public void setup() throws Exception {
+ rootDir = TEST_UTIL.getDataTestDir("testRestore");
+ archiveDir = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY);
+ fs = TEST_UTIL.getTestFileSystem();
+ conf = TEST_UTIL.getConfiguration();
+ FSUtils.setRootDir(conf, rootDir);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ fs.delete(TEST_UTIL.getDataTestDir(), true);
+ }
+
+ @Test
+ public void testRestore() throws IOException {
+ HTableDescriptor htd = createTableDescriptor("testtb");
+
+ Path snapshotDir = new Path(rootDir, "snapshot");
+ createSnapshot(rootDir, snapshotDir, htd);
+
+ // Test clone a snapshot
+ HTableDescriptor htdClone = createTableDescriptor("testtb-clone");
+ testRestore(snapshotDir, htd.getNameAsString(), htdClone);
+ verifyRestore(rootDir, htd, htdClone);
+
+ // Test clone a clone ("link to link")
+ Path cloneDir = HTableDescriptor.getTableDir(rootDir, htdClone.getName());
+ HTableDescriptor htdClone2 = createTableDescriptor("testtb-clone2");
+ testRestore(cloneDir, htdClone.getNameAsString(), htdClone2);
+ verifyRestore(rootDir, htd, htdClone2);
+ }
+
+ private void verifyRestore(final Path rootDir, final HTableDescriptor sourceHtd,
+ final HTableDescriptor htdClone) throws IOException {
+ String[] files = getHFiles(HTableDescriptor.getTableDir(rootDir, htdClone.getName()));
+ assertEquals(2, files.length);
+ assertTrue(files[0] + " should be a HFileLink", HFileLink.isHFileLink(files[0]));
+ assertTrue(files[1] + " should be a Referene", StoreFile.isReference(files[1]));
+ assertEquals(sourceHtd.getNameAsString(), HFileLink.getReferencedTableName(files[0]));
+ assertEquals(TEST_HFILE, HFileLink.getReferencedHFileName(files[0]));
+ Path refPath = getReferredToFile(files[1]);
+ assertTrue(refPath.getName() + " should be a HFileLink", HFileLink.isHFileLink(refPath.getName()));
+ assertEquals(files[0], refPath.getName());
+ }
+
+ /**
+ * Execute the restore operation
+ * @param snapshotDir The snapshot directory to use as "restore source"
+ * @param sourceTableName The name of the snapshotted table
+ * @param htdClone The HTableDescriptor of the table to restore/clone.
+ */
+ public void testRestore(final Path snapshotDir, final String sourceTableName,
+ final HTableDescriptor htdClone) throws IOException {
+ LOG.debug("pre-restore table=" + htdClone.getNameAsString() + " snapshot=" + snapshotDir);
+ FSUtils.logFileSystemState(fs, rootDir, LOG);
+
+ FSTableDescriptors.createTableDescriptor(htdClone, conf);
+ RestoreSnapshotHelper helper = getRestoreHelper(rootDir, snapshotDir, sourceTableName, htdClone);
+ helper.restore();
+
+ LOG.debug("post-restore table=" + htdClone.getNameAsString() + " snapshot=" + snapshotDir);
+ FSUtils.logFileSystemState(fs, rootDir, LOG);
+ }
+
+ /**
+ * Initialize the restore helper, based on the snapshot and table information provided.
+ */
+ private RestoreSnapshotHelper getRestoreHelper(final Path rootDir, final Path snapshotDir,
+ final String sourceTableName, final HTableDescriptor htdClone) throws IOException {
+ CatalogTracker catalogTracker = Mockito.mock(CatalogTracker.class);
+ HTableDescriptor tableDescriptor = Mockito.mock(HTableDescriptor.class);
+ ForeignExceptionDispatcher monitor = Mockito.mock(ForeignExceptionDispatcher.class);
+
+ HConnection hconnection = HConnectionTestingUtility.getMockedConnection(conf);
+ Mockito.when(catalogTracker.getConnection()).thenReturn(hconnection);
+
+ SnapshotDescription sd = SnapshotDescription.newBuilder()
+ .setName("snapshot").setTable(sourceTableName).build();
+
+ return new RestoreSnapshotHelper(conf, fs, catalogTracker, sd, snapshotDir,
+ htdClone, HTableDescriptor.getTableDir(rootDir, htdClone.getName()), monitor);
+ }
+
+ private void createSnapshot(final Path rootDir, final Path snapshotDir, final HTableDescriptor htd)
+ throws IOException {
+ // First region, simple with one plain hfile.
+ HRegion r0 = HRegion.createHRegion(new HRegionInfo(htd.getName()), archiveDir,
+ conf, htd, null, true, true);
+ Path storeFile = new Path(new Path(r0.getRegionDir(), TEST_FAMILY), TEST_HFILE);
+ fs.createNewFile(storeFile);
+ r0.close();
+
+ // Second region, used to test the split case.
+ // This region contains a reference to the hfile in the first region.
+ HRegion r1 = HRegion.createHRegion(new HRegionInfo(htd.getName()), archiveDir,
+ conf, htd, null, true, true);
+ fs.createNewFile(new Path(new Path(r1.getRegionDir(), TEST_FAMILY),
+ storeFile.getName() + '.' + r0.getRegionInfo().getEncodedName()));
+ r1.close();
+
+ Path tableDir = HTableDescriptor.getTableDir(archiveDir, htd.getName());
+ FileUtil.copy(fs, tableDir, fs, snapshotDir, false, conf);
+ }
+
+ private HTableDescriptor createTableDescriptor(final String tableName) {
+ HTableDescriptor htd = new HTableDescriptor(tableName);
+ htd.addFamily(new HColumnDescriptor(TEST_FAMILY));
+ return htd;
+ }
+
+ private Path getReferredToFile(final String referenceName) {
+ Path fakeBasePath = new Path(new Path("table", "region"), "cf");
+ return StoreFile.getReferredToFile(new Path(fakeBasePath, referenceName));
+ }
+
+ private String[] getHFiles(final Path tableDir) throws IOException {
+ List<String> files = new ArrayList<String>();
+ for (Path regionDir: FSUtils.getRegionDirs(fs, tableDir)) {
+ for (Path familyDir: FSUtils.getFamilyDirs(fs, regionDir)) {
+ for (FileStatus file: FSUtils.listStatus(fs, familyDir)) {
+ files.add(file.getPath().getName());
+ }
+ }
+ }
+ Collections.sort(files);
+ return files.toArray(new String[files.size()]);
+ }
+}
\ No newline at end of file