You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by jm...@apache.org on 2015/02/22 21:56:16 UTC
[43/50] [abbrv] hbase git commit: Merge branch 'master' (2/11/15)
into hbase-11339
http://git-wip-us.apache.org/repos/asf/hbase/blob/fe335b68/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobExportSnapshot.java
----------------------------------------------------------------------
diff --cc hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobExportSnapshot.java
index fc9d8e7,0000000..9b065e0
mode 100644,000000..100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobExportSnapshot.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobExportSnapshot.java
@@@ -1,439 -1,0 +1,439 @@@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.snapshot;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
- import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
+import org.apache.hadoop.hbase.mob.MobConstants;
+import org.apache.hadoop.hbase.mob.MobUtils;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo;
+import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
+import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils.SnapshotMock;
++import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.Pair;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Test Export Snapshot Tool
+ */
+@Category(MediumTests.class)
+public class TestMobExportSnapshot {
+ private final Log LOG = LogFactory.getLog(getClass());
+
+ protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+
+ private final static byte[] FAMILY = Bytes.toBytes("cf");
+
+ private byte[] emptySnapshotName;
+ private byte[] snapshotName;
+ private int tableNumFiles;
+ private TableName tableName;
+ private Admin admin;
+
+ public static void setUpBaseConf(Configuration conf) {
+ conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
+ conf.setInt("hbase.regionserver.msginterval", 100);
+ conf.setInt("hbase.client.pause", 250);
+ conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6);
+ conf.setBoolean("hbase.master.enabletable.roundrobin", true);
+ conf.setInt("mapreduce.map.maxattempts", 10);
+ conf.setInt(MobConstants.MOB_FILE_CACHE_SIZE_KEY, 0);
+ }
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ setUpBaseConf(TEST_UTIL.getConfiguration());
+ TEST_UTIL.startMiniCluster(3);
+ TEST_UTIL.startMiniMapReduceCluster();
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ TEST_UTIL.shutdownMiniMapReduceCluster();
+ TEST_UTIL.shutdownMiniCluster();
+ }
+
+ /**
+ * Create a table and take a snapshot of the table used by the export test.
+ */
+ @Before
+ public void setUp() throws Exception {
+ this.admin = TEST_UTIL.getHBaseAdmin();
+
+ long tid = System.currentTimeMillis();
+ tableName = TableName.valueOf("testtb-" + tid);
+ snapshotName = Bytes.toBytes("snaptb0-" + tid);
+ emptySnapshotName = Bytes.toBytes("emptySnaptb0-" + tid);
+
+ // create Table
+ MobSnapshotTestingUtils.createMobTable(TEST_UTIL, tableName, 1, FAMILY);
+
+ // Take an empty snapshot
+ admin.snapshot(emptySnapshotName, tableName);
+
+ // Add some rows
+ SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 50, FAMILY);
+ tableNumFiles = admin.getTableRegions(tableName).size();
+
+ // take a snapshot
+ admin.snapshot(snapshotName, tableName);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ TEST_UTIL.deleteTable(tableName);
+ SnapshotTestingUtils.deleteAllSnapshots(TEST_UTIL.getHBaseAdmin());
+ SnapshotTestingUtils.deleteArchiveDirectory(TEST_UTIL);
+ }
+
+ /**
+ * Verfy the result of getBalanceSplits() method.
+ * The result are groups of files, used as input list for the "export" mappers.
+ * All the groups should have similar amount of data.
+ *
+ * The input list is a pair of file path and length.
+ * The getBalanceSplits() function sort it by length,
+ * and assign to each group a file, going back and forth through the groups.
+ */
+ @Test
+ public void testBalanceSplit() throws Exception {
+ // Create a list of files
+ List<Pair<SnapshotFileInfo, Long>> files = new ArrayList<Pair<SnapshotFileInfo, Long>>();
+ for (long i = 0; i <= 20; i++) {
+ SnapshotFileInfo fileInfo = SnapshotFileInfo.newBuilder()
+ .setType(SnapshotFileInfo.Type.HFILE)
+ .setHfile("file-" + i)
+ .build();
+ files.add(new Pair<SnapshotFileInfo, Long>(fileInfo, i));
+ }
+
+ // Create 5 groups (total size 210)
+ // group 0: 20, 11, 10, 1 (total size: 42)
+ // group 1: 19, 12, 9, 2 (total size: 42)
+ // group 2: 18, 13, 8, 3 (total size: 42)
+ // group 3: 17, 12, 7, 4 (total size: 42)
+ // group 4: 16, 11, 6, 5 (total size: 42)
+ List<List<Pair<SnapshotFileInfo, Long>>> splits = ExportSnapshot.getBalancedSplits(files, 5);
+ assertEquals(5, splits.size());
+
+ String[] split0 = new String[] {"file-20", "file-11", "file-10", "file-1", "file-0"};
+ verifyBalanceSplit(splits.get(0), split0, 42);
+ String[] split1 = new String[] {"file-19", "file-12", "file-9", "file-2"};
+ verifyBalanceSplit(splits.get(1), split1, 42);
+ String[] split2 = new String[] {"file-18", "file-13", "file-8", "file-3"};
+ verifyBalanceSplit(splits.get(2), split2, 42);
+ String[] split3 = new String[] {"file-17", "file-14", "file-7", "file-4"};
+ verifyBalanceSplit(splits.get(3), split3, 42);
+ String[] split4 = new String[] {"file-16", "file-15", "file-6", "file-5"};
+ verifyBalanceSplit(splits.get(4), split4, 42);
+ }
+
+ private void verifyBalanceSplit(final List<Pair<SnapshotFileInfo, Long>> split,
+ final String[] expected, final long expectedSize) {
+ assertEquals(expected.length, split.size());
+ long totalSize = 0;
+ for (int i = 0; i < expected.length; ++i) {
+ Pair<SnapshotFileInfo, Long> fileInfo = split.get(i);
+ assertEquals(expected[i], fileInfo.getFirst().getHfile());
+ totalSize += fileInfo.getSecond();
+ }
+ assertEquals(expectedSize, totalSize);
+ }
+
+ /**
+ * Verify if exported snapshot and copied files matches the original one.
+ */
+ @Test
+ public void testExportFileSystemState() throws Exception {
+ testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles);
+ }
+
+ @Test
+ public void testExportFileSystemStateWithSkipTmp() throws Exception {
+ TEST_UTIL.getConfiguration().setBoolean(ExportSnapshot.CONF_SKIP_TMP, true);
+ testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles);
+ }
+
+ @Test
+ public void testEmptyExportFileSystemState() throws Exception {
+ testExportFileSystemState(tableName, emptySnapshotName, emptySnapshotName, 0);
+ }
+
+ @Test
+ public void testConsecutiveExports() throws Exception {
+ Path copyDir = getLocalDestinationDir();
+ testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles, copyDir, false);
+ testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles, copyDir, true);
+ removeExportDir(copyDir);
+ }
+
+ @Test
+ public void testExportWithTargetName() throws Exception {
+ final byte[] targetName = Bytes.toBytes("testExportWithTargetName");
+ testExportFileSystemState(tableName, snapshotName, targetName, tableNumFiles);
+ }
+
+ /**
+ * Mock a snapshot with files in the archive dir,
+ * two regions, and one reference file.
+ */
+ @Test
+ public void testSnapshotWithRefsExportFileSystemState() throws Exception {
+ Configuration conf = TEST_UTIL.getConfiguration();
+
+ Path rootDir = TEST_UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
+ FileSystem fs = TEST_UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
+
+ SnapshotMock snapshotMock = new SnapshotMock(TEST_UTIL.getConfiguration(), fs, rootDir);
+ SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2("tableWithRefsV1");
+ testSnapshotWithRefsExportFileSystemState(builder);
+
+ snapshotMock = new SnapshotMock(TEST_UTIL.getConfiguration(), fs, rootDir);
+ builder = snapshotMock.createSnapshotV2("tableWithRefsV2");
+ testSnapshotWithRefsExportFileSystemState(builder);
+ }
+
+ /**
+ * Generates a couple of regions for the specified SnapshotMock,
+ * and then it will run the export and verification.
+ */
+ private void testSnapshotWithRefsExportFileSystemState(SnapshotMock.SnapshotBuilder builder)
+ throws Exception {
+ Path[] r1Files = builder.addRegion();
+ Path[] r2Files = builder.addRegion();
+ builder.commit();
+ int snapshotFilesCount = r1Files.length + r2Files.length;
+
+ byte[] snapshotName = Bytes.toBytes(builder.getSnapshotDescription().getName());
+ TableName tableName = builder.getTableDescriptor().getTableName();
+ testExportFileSystemState(tableName, snapshotName, snapshotName, snapshotFilesCount);
+ }
+
+ private void testExportFileSystemState(final TableName tableName, final byte[] snapshotName,
+ final byte[] targetName, int filesExpected) throws Exception {
+ Path copyDir = getHdfsDestinationDir();
+ testExportFileSystemState(tableName, snapshotName, targetName, filesExpected, copyDir, false);
+ removeExportDir(copyDir);
+ }
+
+ /**
+ * Test ExportSnapshot
+ */
+ private void testExportFileSystemState(final TableName tableName, final byte[] snapshotName,
+ final byte[] targetName, int filesExpected, Path copyDir, boolean overwrite)
+ throws Exception {
+ URI hdfsUri = FileSystem.get(TEST_UTIL.getConfiguration()).getUri();
+ FileSystem fs = FileSystem.get(copyDir.toUri(), new Configuration());
+ copyDir = copyDir.makeQualified(fs);
+
+ List<String> opts = new ArrayList<String>();
+ opts.add("-snapshot");
+ opts.add(Bytes.toString(snapshotName));
+ opts.add("-copy-to");
+ opts.add(copyDir.toString());
+ if (targetName != snapshotName) {
+ opts.add("-target");
+ opts.add(Bytes.toString(targetName));
+ }
+ if (overwrite) opts.add("-overwrite");
+
+ // Export Snapshot
+ int res = ExportSnapshot.innerMain(TEST_UTIL.getConfiguration(),
+ opts.toArray(new String[opts.size()]));
+ assertEquals(0, res);
+
+ // Verify File-System state
+ FileStatus[] rootFiles = fs.listStatus(copyDir);
+ assertEquals(filesExpected > 0 ? 2 : 1, rootFiles.length);
+ for (FileStatus fileStatus: rootFiles) {
+ String name = fileStatus.getPath().getName();
+ assertTrue(fileStatus.isDirectory());
+ assertTrue(name.equals(HConstants.SNAPSHOT_DIR_NAME) ||
+ name.equals(HConstants.HFILE_ARCHIVE_DIRECTORY));
+ }
+
+ // compare the snapshot metadata and verify the hfiles
+ final FileSystem hdfs = FileSystem.get(hdfsUri, TEST_UTIL.getConfiguration());
+ final Path snapshotDir = new Path(HConstants.SNAPSHOT_DIR_NAME, Bytes.toString(snapshotName));
+ final Path targetDir = new Path(HConstants.SNAPSHOT_DIR_NAME, Bytes.toString(targetName));
+ verifySnapshotDir(hdfs, new Path(TEST_UTIL.getDefaultRootDirPath(), snapshotDir),
+ fs, new Path(copyDir, targetDir));
+ Set<String> snapshotFiles = verifySnapshot(fs, copyDir, tableName, Bytes.toString(targetName));
+ assertEquals(filesExpected, snapshotFiles.size());
+ }
+
+ /**
+ * Check that ExportSnapshot will return a failure if something fails.
+ */
+ @Test
+ public void testExportFailure() throws Exception {
+ assertEquals(1, runExportAndInjectFailures(snapshotName, false));
+ }
+
+ /**
+ * Check that ExportSnapshot will succede if something fails but the retry succede.
+ */
+ @Test
+ public void testExportRetry() throws Exception {
+ assertEquals(0, runExportAndInjectFailures(snapshotName, true));
+ }
+
+ /*
+ * Execute the ExportSnapshot job injecting failures
+ */
+ private int runExportAndInjectFailures(final byte[] snapshotName, boolean retry)
+ throws Exception {
+ Path copyDir = getLocalDestinationDir();
+ URI hdfsUri = FileSystem.get(TEST_UTIL.getConfiguration()).getUri();
+ FileSystem fs = FileSystem.get(copyDir.toUri(), new Configuration());
+ copyDir = copyDir.makeQualified(fs);
+
+ Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
+ conf.setBoolean(ExportSnapshot.CONF_TEST_FAILURE, true);
+ conf.setBoolean(ExportSnapshot.CONF_TEST_RETRY, retry);
+
+ // Export Snapshot
+ Path sourceDir = TEST_UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
+ int res = ExportSnapshot.innerMain(conf, new String[] {
+ "-snapshot", Bytes.toString(snapshotName),
+ "-copy-from", sourceDir.toString(),
+ "-copy-to", copyDir.toString()
+ });
+ return res;
+ }
+
+ /*
+ * verify if the snapshot folder on file-system 1 match the one on file-system 2
+ */
+ private void verifySnapshotDir(final FileSystem fs1, final Path root1,
+ final FileSystem fs2, final Path root2) throws IOException {
+ assertEquals(listFiles(fs1, root1, root1), listFiles(fs2, root2, root2));
+ }
+
+ /*
+ * Verify if the files exists
+ */
+ private Set<String> verifySnapshot(final FileSystem fs, final Path rootDir,
+ final TableName tableName, final String snapshotName) throws IOException {
+ final Path exportedSnapshot = new Path(rootDir,
+ new Path(HConstants.SNAPSHOT_DIR_NAME, snapshotName));
+ final Set<String> snapshotFiles = new HashSet<String>();
+ final Path exportedArchive = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY);
+ SnapshotReferenceUtil.visitReferencedFiles(TEST_UTIL.getConfiguration(), fs, exportedSnapshot,
+ new SnapshotReferenceUtil.SnapshotVisitor() {
+ @Override
+ public void storeFile(final HRegionInfo regionInfo, final String family,
+ final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
+ if(MobUtils.isMobRegionInfo(regionInfo))
+ return;
+ String hfile = storeFile.getName();
+ snapshotFiles.add(hfile);
+ if (storeFile.hasReference()) {
+ // Nothing to do here, we have already the reference embedded
+ } else {
+ verifyNonEmptyFile(new Path(exportedArchive,
+ new Path(FSUtils.getTableDir(new Path("./"), tableName),
+ new Path(regionInfo.getEncodedName(), new Path(family, hfile)))));
+ }
+ }
+
+ @Override
+ public void logFile (final String server, final String logfile)
+ throws IOException {
+ snapshotFiles.add(logfile);
+ verifyNonEmptyFile(new Path(exportedSnapshot, new Path(server, logfile)));
+ }
+
+ private void verifyNonEmptyFile(final Path path) throws IOException {
+ assertTrue(path + " should exists", fs.exists(path));
+ assertTrue(path + " should not be empty", fs.getFileStatus(path).getLen() > 0);
+ }
+ });
+
+ // Verify Snapshot description
+ SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, exportedSnapshot);
+ assertTrue(desc.getName().equals(snapshotName));
+ assertTrue(desc.getTable().equals(tableName.getNameAsString()));
+ return snapshotFiles;
+ }
+
+ private Set<String> listFiles(final FileSystem fs, final Path root, final Path dir)
+ throws IOException {
+ Set<String> files = new HashSet<String>();
+ int rootPrefix = root.toString().length();
+ FileStatus[] list = FSUtils.listStatus(fs, dir);
+ if (list != null) {
+ for (FileStatus fstat: list) {
+ LOG.debug(fstat.getPath());
+ if (fstat.isDirectory()) {
+ files.addAll(listFiles(fs, root, fstat.getPath()));
+ } else {
+ files.add(fstat.getPath().toString().substring(rootPrefix));
+ }
+ }
+ }
+ return files;
+ }
+
+ private Path getHdfsDestinationDir() {
+ Path rootDir = TEST_UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
+ Path path = new Path(new Path(rootDir, "export-test"), "export-" + System.currentTimeMillis());
+ LOG.info("HDFS export destination path: " + path);
+ return path;
+ }
+
+ private Path getLocalDestinationDir() {
+ Path path = TEST_UTIL.getDataTestDir("local-export-" + System.currentTimeMillis());
+ LOG.info("Local export destination path: " + path);
+ return path;
+ }
+
+ private void removeExportDir(final Path path) throws IOException {
+ FileSystem fs = FileSystem.get(path.toUri(), new Configuration());
+ fs.delete(path, true);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/fe335b68/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobFlushSnapshotFromClient.java
----------------------------------------------------------------------
diff --cc hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobFlushSnapshotFromClient.java
index 6fea4dd,0000000..5517f4a
mode 100644,000000..100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobFlushSnapshotFromClient.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobFlushSnapshotFromClient.java
@@@ -1,551 -1,0 +1,551 @@@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.snapshot;
+
+import static org.junit.Assert.*;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.commons.logging.impl.Log4JLogger;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
- import org.apache.hadoop.hbase.LargeTests;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.TableNotFoundException;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.ScannerCallable;
+import org.apache.hadoop.hbase.ipc.RpcClient;
+import org.apache.hadoop.hbase.ipc.RpcServer;
+import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
+import org.apache.hadoop.hbase.mob.MobConstants;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
++import org.apache.hadoop.hbase.testclassification.ClientTests;
++import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.log4j.Level;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Test creating/using/deleting snapshots from the client
+ * <p>
+ * This is an end-to-end test for the snapshot utility
+ *
+ * TODO This is essentially a clone of TestSnapshotFromClient. This is worth refactoring this
+ * because there will be a few more flavors of snapshots that need to run these tests.
+ */
- @Category(LargeTests.class)
++@Category({ClientTests.class, LargeTests.class})
+public class TestMobFlushSnapshotFromClient {
+ private static final Log LOG = LogFactory.getLog(TestFlushSnapshotFromClient.class);
+ private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
+ private static final int NUM_RS = 2;
+ private static final String STRING_TABLE_NAME = "test";
+ private static final byte[] TEST_FAM = Bytes.toBytes("fam");
+ private static final TableName TABLE_NAME =
+ TableName.valueOf(STRING_TABLE_NAME);
+ private final int DEFAULT_NUM_ROWS = 100;
+
+ /**
+ * Setup the config for the cluster
+ * @throws Exception on failure
+ */
+ @BeforeClass
+ public static void setupCluster() throws Exception {
+ ((Log4JLogger)RpcServer.LOG).getLogger().setLevel(Level.ALL);
- ((Log4JLogger)RpcClient.LOG).getLogger().setLevel(Level.ALL);
+ ((Log4JLogger)ScannerCallable.LOG).getLogger().setLevel(Level.ALL);
+ setupConf(UTIL.getConfiguration());
+ UTIL.startMiniCluster(NUM_RS);
+ }
+
+ private static void setupConf(Configuration conf) {
+ // disable the ui
+ conf.setInt("hbase.regionsever.info.port", -1);
+ // change the flush size to a small amount, regulating number of store files
+ conf.setInt("hbase.hregion.memstore.flush.size", 25000);
+ // so make sure we get a compaction when doing a load, but keep around some
+ // files in the store
+ conf.setInt("hbase.hstore.compaction.min", 10);
+ conf.setInt("hbase.hstore.compactionThreshold", 10);
+ // block writes if we get to 12 store files
+ conf.setInt("hbase.hstore.blockingStoreFiles", 12);
+ // Enable snapshot
+ conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
+ conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
+ ConstantSizeRegionSplitPolicy.class.getName());
+ conf.setInt(MobConstants.MOB_FILE_CACHE_SIZE_KEY, 0);
+ }
+
+ @Before
+ public void setup() throws Exception {
+ MobSnapshotTestingUtils.createMobTable(UTIL, TABLE_NAME, 1, TEST_FAM);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ UTIL.deleteTable(TABLE_NAME);
+
+ SnapshotTestingUtils.deleteAllSnapshots(UTIL.getHBaseAdmin());
+ SnapshotTestingUtils.deleteArchiveDirectory(UTIL);
+ }
+
+ @AfterClass
+ public static void cleanupTest() throws Exception {
+ try {
+ UTIL.shutdownMiniCluster();
+ } catch (Exception e) {
+ LOG.warn("failure shutting down cluster", e);
+ }
+ }
+
+ /**
+ * Test simple flush snapshotting a table that is online
+ * @throws Exception
+ */
+ @Test (timeout=300000)
+ public void testFlushTableSnapshot() throws Exception {
+ Admin admin = UTIL.getHBaseAdmin();
+ // make sure we don't fail on listing snapshots
+ SnapshotTestingUtils.assertNoSnapshots(admin);
+
+ // put some stuff in the table
+ HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
- SnapshotTestingUtils.loadData(UTIL, table, DEFAULT_NUM_ROWS, TEST_FAM);
++ SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
+
+ LOG.debug("FS state before snapshot:");
+ FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
+ FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
+
+ // take a snapshot of the enabled table
+ String snapshotString = "offlineTableSnapshot";
+ byte[] snapshot = Bytes.toBytes(snapshotString);
+ admin.snapshot(snapshotString, TABLE_NAME, SnapshotDescription.Type.FLUSH);
+ LOG.debug("Snapshot completed.");
+
+ // make sure we have the snapshot
+ List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
+ snapshot, TABLE_NAME);
+
+ // make sure its a valid snapshot
+ FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
+ Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
+ LOG.debug("FS state after snapshot:");
+ FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
+ FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
+
+ SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
+ admin, fs);
+ }
+
+ /**
+ * Test snapshotting a table that is online without flushing
+ * @throws Exception
+ */
+ @Test(timeout=30000)
+ public void testSkipFlushTableSnapshot() throws Exception {
+ Admin admin = UTIL.getHBaseAdmin();
+ // make sure we don't fail on listing snapshots
+ SnapshotTestingUtils.assertNoSnapshots(admin);
+
+ // put some stuff in the table
+ HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
+ UTIL.loadTable(table, TEST_FAM);
+
+ LOG.debug("FS state before snapshot:");
+ FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
+ FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
+
+ // take a snapshot of the enabled table
+ String snapshotString = "skipFlushTableSnapshot";
+ byte[] snapshot = Bytes.toBytes(snapshotString);
+ admin.snapshot(snapshotString, TABLE_NAME, SnapshotDescription.Type.SKIPFLUSH);
+ LOG.debug("Snapshot completed.");
+
+ // make sure we have the snapshot
+ List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
+ snapshot, TABLE_NAME);
+
+ // make sure its a valid snapshot
+ FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
+ Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
+ LOG.debug("FS state after snapshot:");
+ FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
+ FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
+
+ SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
+ admin, fs);
+
+ admin.deleteSnapshot(snapshot);
+ snapshots = admin.listSnapshots();
+ SnapshotTestingUtils.assertNoSnapshots(admin);
+ }
+
+
+ /**
+ * Test simple flush snapshotting a table that is online
+ * @throws Exception
+ */
+ @Test (timeout=300000)
+ public void testFlushTableSnapshotWithProcedure() throws Exception {
+ Admin admin = UTIL.getHBaseAdmin();
+ // make sure we don't fail on listing snapshots
+ SnapshotTestingUtils.assertNoSnapshots(admin);
+
+ // put some stuff in the table
+ HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
- SnapshotTestingUtils.loadData(UTIL, table, DEFAULT_NUM_ROWS, TEST_FAM);
++ SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
+
+ LOG.debug("FS state before snapshot:");
+ FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
+ FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
+
+ // take a snapshot of the enabled table
+ String snapshotString = "offlineTableSnapshot";
+ byte[] snapshot = Bytes.toBytes(snapshotString);
+ Map<String, String> props = new HashMap<String, String>();
+ props.put("table", TABLE_NAME.getNameAsString());
+ admin.execProcedure(SnapshotManager.ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION,
+ snapshotString, props);
+
+
+ LOG.debug("Snapshot completed.");
+
+ // make sure we have the snapshot
+ List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
+ snapshot, TABLE_NAME);
+
+ // make sure its a valid snapshot
+ FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
+ Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
+ LOG.debug("FS state after snapshot:");
+ FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
+ FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
+
+ SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
+ admin, fs);
+ }
+
+ @Test (timeout=300000)
+ public void testSnapshotFailsOnNonExistantTable() throws Exception {
+ Admin admin = UTIL.getHBaseAdmin();
+ // make sure we don't fail on listing snapshots
+ SnapshotTestingUtils.assertNoSnapshots(admin);
+ TableName tableName = TableName.valueOf("_not_a_table");
+
+ // make sure the table doesn't exist
+ boolean fail = false;
+ do {
+ try {
+ admin.getTableDescriptor(tableName);
+ fail = true;
+ LOG.error("Table:" + tableName + " already exists, checking a new name");
+ tableName = TableName.valueOf(tableName+"!");
+ } catch (TableNotFoundException e) {
+ fail = false;
+ }
+ } while (fail);
+
+ // snapshot the non-existant table
+ try {
+ admin.snapshot("fail", tableName, SnapshotDescription.Type.FLUSH);
+ fail("Snapshot succeeded even though there is not table.");
+ } catch (SnapshotCreationException e) {
+ LOG.info("Correctly failed to snapshot a non-existant table:" + e.getMessage());
+ }
+ }
+
+ @Test(timeout = 300000)
+ public void testAsyncFlushSnapshot() throws Exception {
+ Admin admin = UTIL.getHBaseAdmin();
+ SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName("asyncSnapshot")
+ .setTable(TABLE_NAME.getNameAsString())
+ .setType(SnapshotDescription.Type.FLUSH)
+ .build();
+
+ // take the snapshot async
+ admin.takeSnapshotAsync(snapshot);
+
+ // constantly loop, looking for the snapshot to complete
+ HMaster master = UTIL.getMiniHBaseCluster().getMaster();
+ SnapshotTestingUtils.waitForSnapshotToComplete(master, snapshot, 200);
+ LOG.info(" === Async Snapshot Completed ===");
+ FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
+ FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
+ // make sure we get the snapshot
+ SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshot);
+ }
+
+ @Test (timeout=300000)
+ public void testSnapshotStateAfterMerge() throws Exception {
+ int numRows = DEFAULT_NUM_ROWS;
+ Admin admin = UTIL.getHBaseAdmin();
+ // make sure we don't fail on listing snapshots
+ SnapshotTestingUtils.assertNoSnapshots(admin);
+ // load the table so we have some data
+ SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, numRows, TEST_FAM);
+
+ // Take a snapshot
+ String snapshotBeforeMergeName = "snapshotBeforeMerge";
+ admin.snapshot(snapshotBeforeMergeName, TABLE_NAME, SnapshotDescription.Type.FLUSH);
+
+ // Clone the table
+ TableName cloneBeforeMergeName = TableName.valueOf("cloneBeforeMerge");
+ admin.cloneSnapshot(snapshotBeforeMergeName, cloneBeforeMergeName);
+ SnapshotTestingUtils.waitForTableToBeOnline(UTIL, cloneBeforeMergeName);
+
+ // Merge two regions
+ List<HRegionInfo> regions = admin.getTableRegions(TABLE_NAME);
+ Collections.sort(regions, new Comparator<HRegionInfo>() {
+ public int compare(HRegionInfo r1, HRegionInfo r2) {
+ return Bytes.compareTo(r1.getStartKey(), r2.getStartKey());
+ }
+ });
+
+ int numRegions = admin.getTableRegions(TABLE_NAME).size();
+ int numRegionsAfterMerge = numRegions - 2;
+ admin.mergeRegions(regions.get(1).getEncodedNameAsBytes(),
+ regions.get(2).getEncodedNameAsBytes(), true);
+ admin.mergeRegions(regions.get(5).getEncodedNameAsBytes(),
+ regions.get(6).getEncodedNameAsBytes(), true);
+
+ // Verify that there's one region less
+ waitRegionsAfterMerge(numRegionsAfterMerge);
+ assertEquals(numRegionsAfterMerge, admin.getTableRegions(TABLE_NAME).size());
+
+ // Clone the table
+ TableName cloneAfterMergeName = TableName.valueOf("cloneAfterMerge");
+ admin.cloneSnapshot(snapshotBeforeMergeName, cloneAfterMergeName);
+ SnapshotTestingUtils.waitForTableToBeOnline(UTIL, cloneAfterMergeName);
+
+ MobSnapshotTestingUtils.verifyMobRowCount(UTIL, TABLE_NAME, numRows);
+ MobSnapshotTestingUtils.verifyMobRowCount(UTIL, cloneBeforeMergeName, numRows);
+ MobSnapshotTestingUtils.verifyMobRowCount(UTIL, cloneAfterMergeName, numRows);
+
+ // test that we can delete the snapshot
+ UTIL.deleteTable(cloneAfterMergeName);
+ UTIL.deleteTable(cloneBeforeMergeName);
+ }
+
+ @Test (timeout=300000)
+ public void testTakeSnapshotAfterMerge() throws Exception {
+ int numRows = DEFAULT_NUM_ROWS;
+ Admin admin = UTIL.getHBaseAdmin();
+ // make sure we don't fail on listing snapshots
+ SnapshotTestingUtils.assertNoSnapshots(admin);
+ // load the table so we have some data
+ SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, numRows, TEST_FAM);
+
+ // Merge two regions
+ List<HRegionInfo> regions = admin.getTableRegions(TABLE_NAME);
+ Collections.sort(regions, new Comparator<HRegionInfo>() {
+ public int compare(HRegionInfo r1, HRegionInfo r2) {
+ return Bytes.compareTo(r1.getStartKey(), r2.getStartKey());
+ }
+ });
+
+ int numRegions = admin.getTableRegions(TABLE_NAME).size();
+ int numRegionsAfterMerge = numRegions - 2;
+ admin.mergeRegions(regions.get(1).getEncodedNameAsBytes(),
+ regions.get(2).getEncodedNameAsBytes(), true);
+ admin.mergeRegions(regions.get(5).getEncodedNameAsBytes(),
+ regions.get(6).getEncodedNameAsBytes(), true);
+
+ waitRegionsAfterMerge(numRegionsAfterMerge);
+ assertEquals(numRegionsAfterMerge, admin.getTableRegions(TABLE_NAME).size());
+
+ // Take a snapshot
+ String snapshotName = "snapshotAfterMerge";
+ SnapshotTestingUtils.snapshot(admin, snapshotName, TABLE_NAME.getNameAsString(),
+ SnapshotDescription.Type.FLUSH, 3);
+
+ // Clone the table
+ TableName cloneName = TableName.valueOf("cloneMerge");
+ admin.cloneSnapshot(snapshotName, cloneName);
+ SnapshotTestingUtils.waitForTableToBeOnline(UTIL, cloneName);
+
+ MobSnapshotTestingUtils.verifyMobRowCount(UTIL, TABLE_NAME, numRows);
+ MobSnapshotTestingUtils.verifyMobRowCount(UTIL, cloneName, numRows);
+
+ // test that we can delete the snapshot
+ UTIL.deleteTable(cloneName);
+ }
+
+ /**
+ * Basic end-to-end test of simple-flush-based snapshots
+ */
+ @Test (timeout=300000)
+ public void testFlushCreateListDestroy() throws Exception {
+ LOG.debug("------- Starting Snapshot test -------------");
+ Admin admin = UTIL.getHBaseAdmin();
+ // make sure we don't fail on listing snapshots
+ SnapshotTestingUtils.assertNoSnapshots(admin);
+ // load the table so we have some data
+ SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
+
+ String snapshotName = "flushSnapshotCreateListDestroy";
+ FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
+ Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
+ SnapshotTestingUtils.createSnapshotAndValidate(admin, TABLE_NAME, Bytes.toString(TEST_FAM),
+ snapshotName, rootDir, fs, true);
+ }
+
+ /**
+ * Demonstrate that we reject snapshot requests if there is a snapshot already running on the
+ * same table currently running and that concurrent snapshots on different tables can both
+ * succeed concurretly.
+ */
+ @Test(timeout=300000)
+ public void testConcurrentSnapshottingAttempts() throws IOException, InterruptedException {
+ final String STRING_TABLE2_NAME = STRING_TABLE_NAME + "2";
+ final TableName TABLE2_NAME =
+ TableName.valueOf(STRING_TABLE2_NAME);
+
+ int ssNum = 20;
+ Admin admin = UTIL.getHBaseAdmin();
+ // make sure we don't fail on listing snapshots
+ SnapshotTestingUtils.assertNoSnapshots(admin);
+ // create second testing table
+ SnapshotTestingUtils.createTable(UTIL, TABLE2_NAME, TEST_FAM);
+ // load the table so we have some data
+ SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
+ SnapshotTestingUtils.loadData(UTIL, TABLE2_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
+
+ final CountDownLatch toBeSubmitted = new CountDownLatch(ssNum);
+ // We'll have one of these per thread
+ class SSRunnable implements Runnable {
+ SnapshotDescription ss;
+ SSRunnable(SnapshotDescription ss) {
+ this.ss = ss;
+ }
+
+ @Override
+ public void run() {
+ try {
+ Admin admin = UTIL.getHBaseAdmin();
+ LOG.info("Submitting snapshot request: " + ClientSnapshotDescriptionUtils.toString(ss));
+ admin.takeSnapshotAsync(ss);
+ } catch (Exception e) {
+ LOG.info("Exception during snapshot request: " + ClientSnapshotDescriptionUtils.toString(
+ ss)
+ + ". This is ok, we expect some", e);
+ }
+ LOG.info("Submitted snapshot request: " + ClientSnapshotDescriptionUtils.toString(ss));
+ toBeSubmitted.countDown();
+ }
+ };
+
+ // build descriptions
+ SnapshotDescription[] descs = new SnapshotDescription[ssNum];
+ for (int i = 0; i < ssNum; i++) {
+ SnapshotDescription.Builder builder = SnapshotDescription.newBuilder();
+ builder.setTable(((i % 2) == 0 ? TABLE_NAME : TABLE2_NAME).getNameAsString());
+ builder.setName("ss"+i);
+ builder.setType(SnapshotDescription.Type.FLUSH);
+ descs[i] = builder.build();
+ }
+
+ // kick each off its own thread
+ for (int i=0 ; i < ssNum; i++) {
+ new Thread(new SSRunnable(descs[i])).start();
+ }
+
+ // wait until all have been submitted
+ toBeSubmitted.await();
+
+ // loop until all are done.
+ while (true) {
+ int doneCount = 0;
+ for (SnapshotDescription ss : descs) {
+ try {
+ if (admin.isSnapshotFinished(ss)) {
+ doneCount++;
+ }
+ } catch (Exception e) {
+ LOG.warn("Got an exception when checking for snapshot " + ss.getName(), e);
+ doneCount++;
+ }
+ }
+ if (doneCount == descs.length) {
+ break;
+ }
+ Thread.sleep(100);
+ }
+
+ // dump for debugging
+ logFSTree(FSUtils.getRootDir(UTIL.getConfiguration()));
+
+ List<SnapshotDescription> taken = admin.listSnapshots();
+ int takenSize = taken.size();
+ LOG.info("Taken " + takenSize + " snapshots: " + taken);
+ assertTrue("We expect at least 1 request to be rejected because of we concurrently" +
+ " issued many requests", takenSize < ssNum && takenSize > 0);
+
+ // Verify that there's at least one snapshot per table
+ int t1SnapshotsCount = 0;
+ int t2SnapshotsCount = 0;
+ for (SnapshotDescription ss : taken) {
+ if (TableName.valueOf(ss.getTable()).equals(TABLE_NAME)) {
+ t1SnapshotsCount++;
+ } else if (TableName.valueOf(ss.getTable()).equals(TABLE2_NAME)) {
+ t2SnapshotsCount++;
+ }
+ }
+ assertTrue("We expect at least 1 snapshot of table1 ", t1SnapshotsCount > 0);
+ assertTrue("We expect at least 1 snapshot of table2 ", t2SnapshotsCount > 0);
+
+ UTIL.deleteTable(TABLE2_NAME);
+ }
+
+ private void logFSTree(Path root) throws IOException {
+ FSUtils.logFileSystemState(UTIL.getDFSCluster().getFileSystem(), root, LOG);
+ }
+
+ private void waitRegionsAfterMerge(final long numRegionsAfterMerge)
+ throws IOException, InterruptedException {
+ Admin admin = UTIL.getHBaseAdmin();
+ // Verify that there's one region less
+ long startTime = System.currentTimeMillis();
+ while (admin.getTableRegions(TABLE_NAME).size() != numRegionsAfterMerge) {
+ // This may be flaky... if after 15sec the merge is not complete give up
+ // it will fail in the assertEquals(numRegionsAfterMerge).
+ if ((System.currentTimeMillis() - startTime) > 15000)
+ break;
+ Thread.sleep(100);
+ }
+ SnapshotTestingUtils.waitForTableToBeOnline(UTIL, TABLE_NAME);
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/hbase/blob/fe335b68/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobRestoreFlushSnapshotFromClient.java
----------------------------------------------------------------------
diff --cc hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobRestoreFlushSnapshotFromClient.java
index 2838d16,0000000..d281763
mode 100644,000000..100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobRestoreFlushSnapshotFromClient.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobRestoreFlushSnapshotFromClient.java
@@@ -1,209 -1,0 +1,210 @@@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.snapshot;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
- import org.apache.hadoop.hbase.LargeTests;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.master.MasterFileSystem;
+import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
+import org.apache.hadoop.hbase.mob.MobConstants;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.regionserver.snapshot.RegionServerSnapshotManager;
++import org.apache.hadoop.hbase.testclassification.ClientTests;
++import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Test clone/restore snapshots from the client
+ *
+ * TODO This is essentially a clone of TestRestoreSnapshotFromClient. This is worth refactoring
+ * this because there will be a few more flavors of snapshots that need to run these tests.
+ */
- @Category(LargeTests.class)
++@Category({ClientTests.class,LargeTests.class})
+public class TestMobRestoreFlushSnapshotFromClient {
+ final Log LOG = LogFactory.getLog(getClass());
+
+ private final static HBaseTestingUtility UTIL = new HBaseTestingUtility();
+
+ private final byte[] FAMILY = Bytes.toBytes("cf");
+
+ private byte[] snapshotName0;
+ private byte[] snapshotName1;
+ private byte[] snapshotName2;
+ private int snapshot0Rows;
+ private int snapshot1Rows;
+ private TableName tableName;
+ private Admin admin;
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ UTIL.getConfiguration().setBoolean("hbase.online.schema.update.enable", true);
+ UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
+ UTIL.getConfiguration().setInt("hbase.client.pause", 250);
+ UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6);
+ UTIL.getConfiguration().setBoolean(
+ "hbase.master.enabletable.roundrobin", true);
+
+ // Enable snapshot
+ UTIL.getConfiguration().setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
+ UTIL.getConfiguration().setLong(RegionServerSnapshotManager.SNAPSHOT_TIMEOUT_MILLIS_KEY,
+ RegionServerSnapshotManager.SNAPSHOT_TIMEOUT_MILLIS_DEFAULT * 2);
+
+ UTIL.getConfiguration().setInt(MobConstants.MOB_FILE_CACHE_SIZE_KEY, 0);
+
+ UTIL.startMiniCluster(3);
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ UTIL.shutdownMiniCluster();
+ }
+
+ /**
+ * Initialize the tests with a table filled with some data
+ * and two snapshots (snapshotName0, snapshotName1) of different states.
+ * The tableName, snapshotNames and the number of rows in the snapshot are initialized.
+ */
+ @Before
+ public void setup() throws Exception {
+ this.admin = UTIL.getHBaseAdmin();
+
+ long tid = System.currentTimeMillis();
+ tableName = TableName.valueOf("testtb-" + tid);
+ snapshotName0 = Bytes.toBytes("snaptb0-" + tid);
+ snapshotName1 = Bytes.toBytes("snaptb1-" + tid);
+ snapshotName2 = Bytes.toBytes("snaptb2-" + tid);
+
+ // create Table
+ MobSnapshotTestingUtils.createMobTable(UTIL, tableName, 1, FAMILY);
+
+ HTable table = new HTable(UTIL.getConfiguration(), tableName);
- SnapshotTestingUtils.loadData(UTIL, table, 500, FAMILY);
++ SnapshotTestingUtils.loadData(UTIL, tableName, 500, FAMILY);
+ snapshot0Rows = MobSnapshotTestingUtils.countMobRows(table);
+ LOG.info("=== before snapshot with 500 rows");
+ logFSTree();
+
+ // take a snapshot
+ admin.snapshot(Bytes.toString(snapshotName0), tableName,
+ SnapshotDescription.Type.FLUSH);
+
+ LOG.info("=== after snapshot with 500 rows");
+ logFSTree();
+
+ // insert more data
- SnapshotTestingUtils.loadData(UTIL, table, 500, FAMILY);
++ SnapshotTestingUtils.loadData(UTIL, tableName, 500, FAMILY);
+ snapshot1Rows = MobSnapshotTestingUtils.countMobRows(table);
+ LOG.info("=== before snapshot with 1000 rows");
+ logFSTree();
+
+ // take a snapshot of the updated table
+ admin.snapshot(Bytes.toString(snapshotName1), tableName,
+ SnapshotDescription.Type.FLUSH);
+ LOG.info("=== after snapshot with 1000 rows");
+ logFSTree();
+ table.close();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ SnapshotTestingUtils.deleteAllSnapshots(UTIL.getHBaseAdmin());
+ SnapshotTestingUtils.deleteArchiveDirectory(UTIL);
+ }
+
+ @Test
+ public void testTakeFlushSnapshot() throws IOException {
+ // taking happens in setup.
+ }
+
+ @Test
+ public void testRestoreSnapshot() throws IOException {
+ MobSnapshotTestingUtils.verifyMobRowCount(UTIL, tableName, snapshot1Rows);
+
+ // Restore from snapshot-0
+ admin.disableTable(tableName);
+ admin.restoreSnapshot(snapshotName0);
+ logFSTree();
+ admin.enableTable(tableName);
+ LOG.info("=== after restore with 500 row snapshot");
+ logFSTree();
+ MobSnapshotTestingUtils.verifyMobRowCount(UTIL, tableName, snapshot0Rows);
+
+ // Restore from snapshot-1
+ admin.disableTable(tableName);
+ admin.restoreSnapshot(snapshotName1);
+ admin.enableTable(tableName);
+ MobSnapshotTestingUtils.verifyMobRowCount(UTIL, tableName, snapshot1Rows);
+ }
+
+ @Test(expected=SnapshotDoesNotExistException.class)
+ public void testCloneNonExistentSnapshot() throws IOException, InterruptedException {
+ String snapshotName = "random-snapshot-" + System.currentTimeMillis();
+ TableName tableName = TableName.valueOf("random-table-" + System.currentTimeMillis());
+ admin.cloneSnapshot(snapshotName, tableName);
+ }
+
+ @Test
+ public void testCloneSnapshot() throws IOException, InterruptedException {
+ TableName clonedTableName = TableName.valueOf("clonedtb-" + System.currentTimeMillis());
+ testCloneSnapshot(clonedTableName, snapshotName0, snapshot0Rows);
+ testCloneSnapshot(clonedTableName, snapshotName1, snapshot1Rows);
+ }
+
+ private void testCloneSnapshot(final TableName tableName, final byte[] snapshotName,
+ int snapshotRows) throws IOException, InterruptedException {
+ // create a new table from snapshot
+ admin.cloneSnapshot(snapshotName, tableName);
+ MobSnapshotTestingUtils.verifyMobRowCount(UTIL, tableName, snapshotRows);
+
+ UTIL.deleteTable(tableName);
+ }
+
+ @Test
+ public void testRestoreSnapshotOfCloned() throws IOException, InterruptedException {
+ TableName clonedTableName = TableName.valueOf("clonedtb-" + System.currentTimeMillis());
+ admin.cloneSnapshot(snapshotName0, clonedTableName);
+ MobSnapshotTestingUtils.verifyMobRowCount(UTIL, clonedTableName, snapshot0Rows);
+ admin.snapshot(Bytes.toString(snapshotName2), clonedTableName, SnapshotDescription.Type.FLUSH);
+ UTIL.deleteTable(clonedTableName);
+
+ admin.cloneSnapshot(snapshotName2, clonedTableName);
+ MobSnapshotTestingUtils.verifyMobRowCount(UTIL, clonedTableName, snapshot0Rows);
+ UTIL.deleteTable(clonedTableName);
+ }
+
+ // ==========================================================================
+ // Helpers
+ // ==========================================================================
+ private void logFSTree() throws IOException {
+ MasterFileSystem mfs = UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem();
+ FSUtils.logFileSystemState(mfs.getFileSystem(), mfs.getRootDir(), LOG);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/fe335b68/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobRestoreSnapshotHelper.java
----------------------------------------------------------------------
diff --cc hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobRestoreSnapshotHelper.java
index 3448119,0000000..1893c7a
mode 100644,000000..100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobRestoreSnapshotHelper.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobRestoreSnapshotHelper.java
@@@ -1,163 -1,0 +1,163 @@@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.snapshot;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HTableDescriptor;
- import org.apache.hadoop.hbase.SmallTests;
++import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
+import org.apache.hadoop.hbase.io.HFileLink;
+import org.apache.hadoop.hbase.mob.MobConstants;
+import org.apache.hadoop.hbase.monitoring.MonitoredTask;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
+import org.apache.hadoop.hbase.snapshot.MobSnapshotTestingUtils.SnapshotMock;
+import org.apache.hadoop.hbase.util.FSTableDescriptors;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.mockito.Mockito;
+
+/**
+ * Test the restore/clone operation from a file-system point of view.
+ */
+@Category(SmallTests.class)
+public class TestMobRestoreSnapshotHelper {
+ final Log LOG = LogFactory.getLog(getClass());
+
+ private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+ private final static String TEST_HFILE = "abc";
+
+ private Configuration conf;
+ private Path archiveDir;
+ private FileSystem fs;
+ private Path rootDir;
+
+ @Before
+ public void setup() throws Exception {
+ rootDir = TEST_UTIL.getDataTestDir("testRestore");
+ archiveDir = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY);
+ fs = TEST_UTIL.getTestFileSystem();
+ TEST_UTIL.getConfiguration().setInt(MobConstants.MOB_FILE_CACHE_SIZE_KEY, 0);
+ conf = TEST_UTIL.getConfiguration();
+ FSUtils.setRootDir(conf, rootDir);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ fs.delete(TEST_UTIL.getDataTestDir(), true);
+ }
+
+ @Test
+ public void testRestore() throws IOException {
+ // Test Rolling-Upgrade like Snapshot.
+ // half machines writing using v1 and the others using v2 format.
+ SnapshotMock snapshotMock = new SnapshotMock(TEST_UTIL.getConfiguration(), fs, rootDir);
+ SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2("snapshot");
+ builder.addRegionV1();
+ builder.addRegionV2();
+ builder.addRegionV2();
+ builder.addRegionV1();
+ Path snapshotDir = builder.commit();
+ HTableDescriptor htd = builder.getTableDescriptor();
+ SnapshotDescription desc = builder.getSnapshotDescription();
+
+ // Test clone a snapshot
+ HTableDescriptor htdClone = snapshotMock.createHtd("testtb-clone");
+ testRestore(snapshotDir, desc, htdClone);
+ verifyRestore(rootDir, htd, htdClone);
+
+ // Test clone a clone ("link to link")
+ SnapshotDescription cloneDesc = SnapshotDescription.newBuilder()
+ .setName("cloneSnapshot")
+ .setTable("testtb-clone")
+ .build();
+ Path cloneDir = FSUtils.getTableDir(rootDir, htdClone.getTableName());
+ HTableDescriptor htdClone2 = snapshotMock.createHtd("testtb-clone2");
+ testRestore(cloneDir, cloneDesc, htdClone2);
+ verifyRestore(rootDir, htd, htdClone2);
+ }
+
+ private void verifyRestore(final Path rootDir, final HTableDescriptor sourceHtd,
+ final HTableDescriptor htdClone) throws IOException {
+ String[] files = SnapshotTestingUtils.listHFileNames(fs,
+ FSUtils.getTableDir(rootDir, htdClone.getTableName()));
+ assertEquals(12, files.length);
+ for (int i = 0; i < files.length; i += 2) {
+ String linkFile = files[i];
+ String refFile = files[i+1];
+ assertTrue(linkFile + " should be a HFileLink", HFileLink.isHFileLink(linkFile));
+ assertTrue(refFile + " should be a Referene", StoreFileInfo.isReference(refFile));
+ assertEquals(sourceHtd.getTableName(), HFileLink.getReferencedTableName(linkFile));
+ Path refPath = getReferredToFile(refFile);
+ LOG.debug("get reference name for file " + refFile + " = " + refPath);
+ assertTrue(refPath.getName() + " should be a HFileLink", HFileLink.isHFileLink(refPath.getName()));
+ assertEquals(linkFile, refPath.getName());
+ }
+ }
+
+ /**
+ * Execute the restore operation
+ * @param snapshotDir The snapshot directory to use as "restore source"
+ * @param sd The snapshot descriptor
+ * @param htdClone The HTableDescriptor of the table to restore/clone.
+ */
+ public void testRestore(final Path snapshotDir, final SnapshotDescription sd,
+ final HTableDescriptor htdClone) throws IOException {
+ LOG.debug("pre-restore table=" + htdClone.getTableName() + " snapshot=" + snapshotDir);
+ FSUtils.logFileSystemState(fs, rootDir, LOG);
+
+ new FSTableDescriptors(conf).createTableDescriptor(htdClone);
+ RestoreSnapshotHelper helper = getRestoreHelper(rootDir, snapshotDir, sd, htdClone);
+ helper.restoreHdfsRegions();
+
+ LOG.debug("post-restore table=" + htdClone.getTableName() + " snapshot=" + snapshotDir);
+ FSUtils.logFileSystemState(fs, rootDir, LOG);
+ }
+
+ /**
+ * Initialize the restore helper, based on the snapshot and table information provided.
+ */
+ private RestoreSnapshotHelper getRestoreHelper(final Path rootDir, final Path snapshotDir,
+ final SnapshotDescription sd, final HTableDescriptor htdClone) throws IOException {
+ ForeignExceptionDispatcher monitor = Mockito.mock(ForeignExceptionDispatcher.class);
+ MonitoredTask status = Mockito.mock(MonitoredTask.class);
+
+ SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, sd);
+ return new RestoreSnapshotHelper(conf, fs, manifest,
+ htdClone, rootDir, monitor, status);
+ }
+
+ private Path getReferredToFile(final String referenceName) {
+ Path fakeBasePath = new Path(new Path("table", "region"), "cf");
+ return StoreFileInfo.getReferredToFile(new Path(fakeBasePath, referenceName));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/fe335b68/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobSecureExportSnapshot.java
----------------------------------------------------------------------
diff --cc hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobSecureExportSnapshot.java
index 0000000,0000000..4a89db8
new file mode 100644
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobSecureExportSnapshot.java
@@@ -1,0 -1,0 +1,53 @@@
++/**
++ * Copyright The Apache Software Foundation
++ *
++ * Licensed to the Apache Software Foundation (ASF) under one
++ * or more contributor license agreements. See the NOTICE file
++ * distributed with this work for additional information
++ * regarding copyright ownership. The ASF licenses this file
++ * to you under the Apache License, Version 2.0 (the
++ * "License"); you may not use this file except in compliance
++ * with the License. You may obtain a copy of the License at
++ *
++ * http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing, software
++ * distributed under the License is distributed on an "AS IS" BASIS,
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ * See the License for the specific language governing permissions and
++ * limitations under the License.
++ */
++package org.apache.hadoop.hbase.snapshot;
++
++import org.apache.hadoop.hbase.testclassification.LargeTests;
++import org.apache.hadoop.hbase.mapreduce.HadoopSecurityEnabledUserProviderForTesting;
++import org.apache.hadoop.hbase.security.UserProvider;
++import org.apache.hadoop.hbase.security.access.AccessControlLists;
++import org.apache.hadoop.hbase.security.access.SecureTestUtil;
++
++import org.junit.BeforeClass;
++import org.junit.experimental.categories.Category;
++
++/**
++ * Reruns TestMobExportSnapshot using MobExportSnapshot in secure mode.
++ */
++@Category(LargeTests.class)
++public class TestMobSecureExportSnapshot extends TestMobExportSnapshot {
++ @BeforeClass
++ public static void setUpBeforeClass() throws Exception {
++ setUpBaseConf(TEST_UTIL.getConfiguration());
++
++ // set the always on security provider
++ UserProvider.setUserProviderForTesting(TEST_UTIL.getConfiguration(),
++ HadoopSecurityEnabledUserProviderForTesting.class);
++
++ // setup configuration
++ SecureTestUtil.enableSecurity(TEST_UTIL.getConfiguration());
++
++ TEST_UTIL.startMiniCluster(3);
++ TEST_UTIL.startMiniMapReduceCluster();
++
++ // Wait for the ACL table to become available
++ TEST_UTIL.waitTableEnabled(AccessControlLists.ACL_TABLE_NAME);
++ }
++}
http://git-wip-us.apache.org/repos/asf/hbase/blob/fe335b68/hbase-shell/src/main/ruby/hbase/admin.rb
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/fe335b68/pom.xml
----------------------------------------------------------------------
diff --cc pom.xml
index 5c946b0,132215d..ff3d4d3
--- a/pom.xml
+++ b/pom.xml
@@@ -521,8 -516,12 +516,8 @@@
<testFailureIgnore>${surefire.testFailureIgnore}</testFailureIgnore>
<forkedProcessTimeoutInSeconds>${surefire.timeout}</forkedProcessTimeoutInSeconds>
<!--Allocate some direct memory for direct memory tests-->
- <argLine>-enableassertions -XX:MaxDirectMemorySize=1G -Xmx1900m -XX:MaxPermSize=256m -Djava.security.egd=file:/dev/./urandom -Djava.net.preferIPv4Stack=true -Djava.awt.headless=true</argLine>
+ <argLine>${hbase-surefire.argLine} ${argLine}</argLine>
<redirectTestOutputToFile>${test.output.tofile}</redirectTestOutputToFile>
- <systemPropertyVariables>
- <jacoco-agent.destfile>target/jacoco.exec</jacoco-agent.destfile>
- </systemPropertyVariables>
-
<excludes>
<!-- users can add -D option to skip particular test classes
ex: mvn test -Dtest.exclude.pattern=**/TestFoo.java,**/TestBar.java