svn commit: r1593139 [6/6] - in /hbase/trunk: hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ hbase-protocol/src/main/protobuf/ hbase-server/src/main/java/org/apache/hadoop/hbase/client/ hbase-server/src/main/java/org/apache/ha...

[mb...@apache.org]

Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreSnapshotHelper.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreSnapshotHelper.java?rev=1593139&r1=1593138&r2=1593139&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreSnapshotHelper.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreSnapshotHelper.java Wed May  7 21:28:12 2014
@@ -21,31 +21,23 @@ import static org.junit.Assert.assertEqu
 import static org.junit.Assert.assertTrue;
 
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.SmallTests;
-import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.catalog.CatalogTracker;
 import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
 import org.apache.hadoop.hbase.io.HFileLink;
 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
-import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
 import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
+import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils.SnapshotMock;
 import org.apache.hadoop.hbase.util.FSTableDescriptors;
 import org.apache.hadoop.hbase.util.FSUtils;
 import org.junit.After;
@@ -62,7 +54,6 @@ public class TestRestoreSnapshotHelper {
   final Log LOG = LogFactory.getLog(getClass());
 
   private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
-  private final static String TEST_FAMILY = "cf";
   private final static String TEST_HFILE = "abc";
 
   private Configuration conf;
@@ -86,49 +77,65 @@ public class TestRestoreSnapshotHelper {
 
   @Test
   public void testRestore() throws IOException {
-    HTableDescriptor htd = createTableDescriptor("testtb");
-
-    Path snapshotDir = new Path(rootDir, "snapshot");
-    createSnapshot(rootDir, snapshotDir, htd);
+    // Test Rolling-Upgrade like Snapshot.
+    // half machines writing using v1 and the others using v2 format.
+    SnapshotMock snapshotMock = new SnapshotMock(TEST_UTIL.getConfiguration(), fs, rootDir);
+    SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2("snapshot");
+    builder.addRegionV1();
+    builder.addRegionV2();
+    builder.addRegionV2();
+    builder.addRegionV1();
+    Path snapshotDir = builder.commit();
+    HTableDescriptor htd = builder.getTableDescriptor();
+    SnapshotDescription desc = builder.getSnapshotDescription();
 
     // Test clone a snapshot
-    HTableDescriptor htdClone = createTableDescriptor("testtb-clone");
-    testRestore(snapshotDir, htd.getTableName().getNameAsString(), htdClone);
+    HTableDescriptor htdClone = snapshotMock.createHtd("testtb-clone");
+    testRestore(snapshotDir, desc, htdClone);
     verifyRestore(rootDir, htd, htdClone);
 
     // Test clone a clone ("link to link")
+    SnapshotDescription cloneDesc = SnapshotDescription.newBuilder()
+        .setName("cloneSnapshot")
+        .setTable("testtb-clone")
+        .build();
     Path cloneDir = FSUtils.getTableDir(rootDir, htdClone.getTableName());
-    HTableDescriptor htdClone2 = createTableDescriptor("testtb-clone2");
-    testRestore(cloneDir, htdClone.getTableName().getNameAsString(), htdClone2);
+    HTableDescriptor htdClone2 = snapshotMock.createHtd("testtb-clone2");
+    testRestore(cloneDir, cloneDesc, htdClone2);
     verifyRestore(rootDir, htd, htdClone2);
   }
 
   private void verifyRestore(final Path rootDir, final HTableDescriptor sourceHtd,
       final HTableDescriptor htdClone) throws IOException {
-    String[] files = getHFiles(FSUtils.getTableDir(rootDir, htdClone.getTableName()));
-    assertEquals(2, files.length);
-    assertTrue(files[0] + " should be a HFileLink", HFileLink.isHFileLink(files[0]));
-    assertTrue(files[1] + " should be a Referene", StoreFileInfo.isReference(files[1]));
-    assertEquals(sourceHtd.getTableName(), HFileLink.getReferencedTableName(files[0]));
-    assertEquals(TEST_HFILE, HFileLink.getReferencedHFileName(files[0]));
-    Path refPath = getReferredToFile(files[1]);
-    assertTrue(refPath.getName() + " should be a HFileLink", HFileLink.isHFileLink(refPath.getName()));
-    assertEquals(files[0], refPath.getName());
+    String[] files = SnapshotTestingUtils.listHFileNames(fs,
+      FSUtils.getTableDir(rootDir, htdClone.getTableName()));
+    assertEquals(12, files.length);
+    for (int i = 0; i < files.length; i += 2) {
+      String linkFile = files[i];
+      String refFile = files[i+1];
+      assertTrue(linkFile + " should be a HFileLink", HFileLink.isHFileLink(linkFile));
+      assertTrue(refFile + " should be a Referene", StoreFileInfo.isReference(refFile));
+      assertEquals(sourceHtd.getTableName(), HFileLink.getReferencedTableName(linkFile));
+      Path refPath = getReferredToFile(refFile);
+      LOG.debug("get reference name for file " + refFile + " = " + refPath);
+      assertTrue(refPath.getName() + " should be a HFileLink", HFileLink.isHFileLink(refPath.getName()));
+      assertEquals(linkFile, refPath.getName());
+    }
   }
 
   /**
    * Execute the restore operation
    * @param snapshotDir The snapshot directory to use as "restore source"
-   * @param sourceTableName The name of the snapshotted table
+   * @param sd The snapshot descriptor
    * @param htdClone The HTableDescriptor of the table to restore/clone.
    */
-  public void testRestore(final Path snapshotDir, final String sourceTableName,
+  public void testRestore(final Path snapshotDir, final SnapshotDescription sd,
       final HTableDescriptor htdClone) throws IOException {
     LOG.debug("pre-restore table=" + htdClone.getTableName() + " snapshot=" + snapshotDir);
     FSUtils.logFileSystemState(fs, rootDir, LOG);
 
     new FSTableDescriptors(conf).createTableDescriptor(htdClone);
-    RestoreSnapshotHelper helper = getRestoreHelper(rootDir, snapshotDir, sourceTableName, htdClone);
+    RestoreSnapshotHelper helper = getRestoreHelper(rootDir, snapshotDir, sd, htdClone);
     helper.restoreHdfsRegions();
 
     LOG.debug("post-restore table=" + htdClone.getTableName() + " snapshot=" + snapshotDir);
@@ -139,65 +146,19 @@ public class TestRestoreSnapshotHelper {
    * Initialize the restore helper, based on the snapshot and table information provided.
    */
   private RestoreSnapshotHelper getRestoreHelper(final Path rootDir, final Path snapshotDir,
-      final String sourceTableName, final HTableDescriptor htdClone) throws IOException {
+      final SnapshotDescription sd, final HTableDescriptor htdClone) throws IOException {
     CatalogTracker catalogTracker = Mockito.mock(CatalogTracker.class);
     HTableDescriptor tableDescriptor = Mockito.mock(HTableDescriptor.class);
     ForeignExceptionDispatcher monitor = Mockito.mock(ForeignExceptionDispatcher.class);
     MonitoredTask status = Mockito.mock(MonitoredTask.class);
 
-    SnapshotDescription sd = SnapshotDescription.newBuilder()
-      .setName("snapshot")
-      .setTable(sourceTableName)
-      .build();
-
-    return new RestoreSnapshotHelper(conf, fs, sd, snapshotDir,
+    SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, sd);
+    return new RestoreSnapshotHelper(conf, fs, manifest,
       htdClone, rootDir, monitor, status);
   }
 
-  private void createSnapshot(final Path rootDir, final Path snapshotDir, final HTableDescriptor htd)
-      throws IOException {
-    // First region, simple with one plain hfile.
-    HRegionInfo hri = new HRegionInfo(htd.getTableName());
-    HRegionFileSystem r0fs = HRegionFileSystem.createRegionOnFileSystem(conf,
-      fs, FSUtils.getTableDir(archiveDir, hri.getTable()), hri);
-    Path storeFile = new Path(rootDir, TEST_HFILE);
-    fs.createNewFile(storeFile);
-    r0fs.commitStoreFile(TEST_FAMILY, storeFile);
-
-    // Second region, used to test the split case.
-    // This region contains a reference to the hfile in the first region.
-    hri = new HRegionInfo(htd.getTableName());
-    HRegionFileSystem r1fs = HRegionFileSystem.createRegionOnFileSystem(conf,
-      fs, FSUtils.getTableDir(archiveDir, hri.getTable()), hri);
-    storeFile = new Path(rootDir, TEST_HFILE + '.' + r0fs.getRegionInfo().getEncodedName());
-    fs.createNewFile(storeFile);
-    r1fs.commitStoreFile(TEST_FAMILY, storeFile);
-
-    Path tableDir = FSUtils.getTableDir(archiveDir, htd.getTableName());
-    FileUtil.copy(fs, tableDir, fs, snapshotDir, false, conf);
-  }
-
-  private HTableDescriptor createTableDescriptor(final String tableName) {
-    HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
-    htd.addFamily(new HColumnDescriptor(TEST_FAMILY));
-    return htd;
-  }
-
   private Path getReferredToFile(final String referenceName) {
     Path fakeBasePath = new Path(new Path("table", "region"), "cf");
     return StoreFileInfo.getReferredToFile(new Path(fakeBasePath, referenceName));
   }
-
-  private String[] getHFiles(final Path tableDir) throws IOException {
-    List<String> files = new ArrayList<String>();
-    for (Path regionDir: FSUtils.getRegionDirs(fs, tableDir)) {
-      for (Path familyDir: FSUtils.getFamilyDirs(fs, regionDir)) {
-        for (FileStatus file: FSUtils.listStatus(fs, familyDir)) {
-          files.add(file.getPath().getName());
-        }
-      }
-    }
-    Collections.sort(files);
-    return files.toArray(new String[files.size()]);
-  }
 }

Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotDescriptionUtils.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotDescriptionUtils.java?rev=1593139&r1=1593138&r2=1593139&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotDescriptionUtils.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotDescriptionUtils.java Wed May  7 21:28:12 2014
@@ -32,8 +32,6 @@ import org.apache.hadoop.hbase.HBaseTest
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.MediumTests;
 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
-import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription.Type;
-import org.apache.hadoop.hbase.util.EnvironmentEdge;
 import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
 import org.junit.After;
 import org.junit.BeforeClass;