You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2016/02/24 16:14:10 UTC
[3/3] hbase git commit: HBASE-15302 Reenable the other tests disabled
by HBASE-14678
HBASE-15302 Reenable the other tests disabled by HBASE-14678
Signed-off-by: stack <st...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/30cec72f
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/30cec72f
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/30cec72f
Branch: refs/heads/master
Commit: 30cec72f9ade972d7e9ce4bba527b0e6074cae60
Parents: 876a6ab
Author: Phil Yang <ud...@gmail.com>
Authored: Mon Feb 22 14:17:24 2016 +0800
Committer: stack <st...@apache.org>
Committed: Wed Feb 24 07:14:01 2016 -0800
----------------------------------------------------------------------
.../apache/hadoop/hbase/wal/WALSplitter.java | 11 +-
.../hbase/TestPartialResultsFromClientSide.java | 832 ++++++++
.../TestMobSnapshotCloneIndependence.java | 69 +
.../client/TestSnapshotCloneIndependence.java | 481 +++++
.../master/TestDistributedLogSplitting.java | 1799 ++++++++++++++++++
.../balancer/TestStochasticLoadBalancer2.java | 90 +
.../TestMasterFailoverWithProcedures.java | 514 +++++
.../TestMobFlushSnapshotFromClient.java | 72 +
.../apache/hadoop/hbase/wal/TestWALSplit.java | 1320 +++++++++++++
.../hbase/wal/TestWALSplitCompressed.java | 36 +
.../hbase/client/TestReplicationShell.java | 37 +
11 files changed, 5256 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/30cec72f/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java
index 54b82b2..010fd37 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java
@@ -515,13 +515,14 @@ public class WALSplitter {
* @param fs
* @param logEntry
* @param rootDir HBase root dir.
- * @param fileBeingSplit the file being split currently. Used to generate tmp file name.
+ * @param fileNameBeingSplit the file being split currently. Used to generate tmp file name.
* @return Path to file into which to dump split log edits.
* @throws IOException
*/
@SuppressWarnings("deprecation")
- private static Path getRegionSplitEditsPath(final FileSystem fs,
- final Entry logEntry, final Path rootDir, FileStatus fileBeingSplit)
+ @VisibleForTesting
+ static Path getRegionSplitEditsPath(final FileSystem fs,
+ final Entry logEntry, final Path rootDir, String fileNameBeingSplit)
throws IOException {
Path tableDir = FSUtils.getTableDir(rootDir, logEntry.getKey().getTablename());
String encodedRegionName = Bytes.toString(logEntry.getKey().getEncodedRegionName());
@@ -556,7 +557,7 @@ public class WALSplitter {
// Append file name ends with RECOVERED_LOG_TMPFILE_SUFFIX to ensure
// region's replayRecoveredEdits will not delete it
String fileName = formatRecoveredEditsFileName(logEntry.getKey().getSequenceId());
- fileName = getTmpRecoveredEditsFileName(fileName + "-" + fileBeingSplit.getPath().getName());
+ fileName = getTmpRecoveredEditsFileName(fileName + "-" + fileNameBeingSplit);
return new Path(dir, fileName);
}
@@ -1518,7 +1519,7 @@ public class WALSplitter {
* @return a path with a write for that path. caller should close.
*/
private WriterAndPath createWAP(byte[] region, Entry entry, Path rootdir) throws IOException {
- Path regionedits = getRegionSplitEditsPath(fs, entry, rootdir, fileBeingSplit);
+ Path regionedits = getRegionSplitEditsPath(fs, entry, rootdir, fileBeingSplit.getPath().getName());
if (regionedits == null) {
return null;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/30cec72f/hbase-server/src/test/java/org/apache/hadoop/hbase/TestPartialResultsFromClientSide.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestPartialResultsFromClientSide.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestPartialResultsFromClientSide.java
new file mode 100644
index 0000000..a6f8373
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestPartialResultsFromClientSide.java
@@ -0,0 +1,832 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.client.ClientScanner;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.filter.ColumnPrefixFilter;
+import org.apache.hadoop.hbase.filter.ColumnRangeFilter;
+import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter;
+import org.apache.hadoop.hbase.filter.FirstKeyValueMatchingQualifiersFilter;
+import org.apache.hadoop.hbase.filter.RandomRowFilter;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * These tests are focused on testing how partial results appear to a client. Partial results are
+ * {@link Result}s that contain only a portion of a row's complete list of cells. Partial results
+ * are formed when the server breaches its maximum result size when trying to service a client's RPC
+ * request. It is the responsibility of the scanner on the client side to recognize when partial
+ * results have been returned and to take action to form the complete results.
+ * <p>
+ * Unless the flag {@link Scan#setAllowPartialResults(boolean)} has been set to true, the caller of
+ * {@link ResultScanner#next()} should never see partial results.
+ */
+@Category(MediumTests.class)
+public class TestPartialResultsFromClientSide {
+ private static final Log LOG = LogFactory.getLog(TestPartialResultsFromClientSide.class);
+
+ private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+
+ private static Table TABLE = null;
+
+ /**
+ * Table configuration
+ */
+ private static TableName TABLE_NAME = TableName.valueOf("testTable");
+
+ private static int NUM_ROWS = 5;
+ private static byte[] ROW = Bytes.toBytes("testRow");
+ private static byte[][] ROWS = HTestConst.makeNAscii(ROW, NUM_ROWS);
+
+ // Should keep this value below 10 to keep generation of expected kv's simple. If above 10 then
+ // table/row/cf1/... will be followed by table/row/cf10/... instead of table/row/cf2/... which
+ // breaks the simple generation of expected kv's
+ private static int NUM_FAMILIES = 10;
+ private static byte[] FAMILY = Bytes.toBytes("testFamily");
+ private static byte[][] FAMILIES = HTestConst.makeNAscii(FAMILY, NUM_FAMILIES);
+
+ private static int NUM_QUALIFIERS = 10;
+ private static byte[] QUALIFIER = Bytes.toBytes("testQualifier");
+ private static byte[][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, NUM_QUALIFIERS);
+
+ private static int VALUE_SIZE = 1024;
+ private static byte[] VALUE = Bytes.createMaxByteArray(VALUE_SIZE);
+
+ private static int NUM_COLS = NUM_FAMILIES * NUM_QUALIFIERS;
+
+ // Approximation of how large the heap size of cells in our table. Should be accessed through
+ // getCellHeapSize().
+ private static long CELL_HEAP_SIZE = -1;
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ TEST_UTIL.startMiniCluster(3);
+ TABLE = createTestTable(TABLE_NAME, ROWS, FAMILIES, QUALIFIERS, VALUE);
+ }
+
+ static Table createTestTable(TableName name, byte[][] rows, byte[][] families,
+ byte[][] qualifiers, byte[] cellValue) throws IOException {
+ Table ht = TEST_UTIL.createTable(name, families);
+ List<Put> puts = createPuts(rows, families, qualifiers, cellValue);
+ ht.put(puts);
+
+ return ht;
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ TEST_UTIL.shutdownMiniCluster();
+ }
+
+ /**
+ * Ensure that the expected key values appear in a result returned from a scanner that is
+ * combining partial results into complete results
+ * @throws Exception
+ */
+ @Test
+ public void testExpectedValuesOfPartialResults() throws Exception {
+ testExpectedValuesOfPartialResults(false);
+ testExpectedValuesOfPartialResults(true);
+ }
+
+ public void testExpectedValuesOfPartialResults(boolean reversed) throws Exception {
+ Scan partialScan = new Scan();
+ partialScan.setMaxVersions();
+ // Max result size of 1 ensures that each RPC request will return a single cell. The scanner
+ // will need to reconstruct the results into a complete result before returning to the caller
+ partialScan.setMaxResultSize(1);
+ partialScan.setReversed(reversed);
+ ResultScanner partialScanner = TABLE.getScanner(partialScan);
+
+ final int startRow = reversed ? ROWS.length - 1 : 0;
+ final int endRow = reversed ? -1 : ROWS.length;
+ final int loopDelta = reversed ? -1 : 1;
+ String message;
+
+ for (int row = startRow; row != endRow; row = row + loopDelta) {
+ message = "Ensuring the expected keyValues are present for row " + row;
+ List<Cell> expectedKeyValues = createKeyValuesForRow(ROWS[row], FAMILIES, QUALIFIERS, VALUE);
+ Result result = partialScanner.next();
+ assertFalse(result.isPartial());
+ verifyResult(result, expectedKeyValues, message);
+ }
+
+ partialScanner.close();
+ }
+
+ /**
+ * Ensure that we only see Results marked as partial when the allowPartial flag is set
+ * @throws Exception
+ */
+ @Test
+ public void testAllowPartialResults() throws Exception {
+ Scan scan = new Scan();
+ scan.setAllowPartialResults(true);
+ scan.setMaxResultSize(1);
+ ResultScanner scanner = TABLE.getScanner(scan);
+ Result result = scanner.next();
+
+ assertTrue(result != null);
+ assertTrue(result.isPartial());
+ assertTrue(result.rawCells() != null);
+ assertTrue(result.rawCells().length == 1);
+
+ scanner.close();
+
+ scan.setAllowPartialResults(false);
+ scanner = TABLE.getScanner(scan);
+ result = scanner.next();
+
+ assertTrue(result != null);
+ assertTrue(!result.isPartial());
+ assertTrue(result.rawCells() != null);
+ assertTrue(result.rawCells().length == NUM_COLS);
+
+ scanner.close();
+ }
+
+ /**
+ * Ensure that the results returned from a scanner that retrieves all results in a single RPC call
+ * matches the results that are returned from a scanner that must incrementally combine partial
+ * results into complete results. A variety of scan configurations can be tested
+ * @throws Exception
+ */
+ @Test
+ public void testEquivalenceOfScanResults() throws Exception {
+ Scan oneShotScan = new Scan();
+ oneShotScan.setMaxResultSize(Long.MAX_VALUE);
+
+ Scan partialScan = new Scan(oneShotScan);
+ partialScan.setMaxResultSize(1);
+
+ testEquivalenceOfScanResults(TABLE, oneShotScan, partialScan);
+ }
+
+ public void testEquivalenceOfScanResults(Table table, Scan scan1, Scan scan2) throws Exception {
+ ResultScanner scanner1 = table.getScanner(scan1);
+ ResultScanner scanner2 = table.getScanner(scan2);
+
+ Result r1 = null;
+ Result r2 = null;
+ int count = 0;
+
+ while ((r1 = scanner1.next()) != null) {
+ r2 = scanner2.next();
+
+ assertTrue(r2 != null);
+ compareResults(r1, r2, "Comparing result #" + count);
+ count++;
+ }
+
+ r2 = scanner2.next();
+ assertTrue("r2: " + r2 + " Should be null", r2 == null);
+
+ scanner1.close();
+ scanner2.close();
+ }
+
+ /**
+ * Order of cells in partial results matches the ordering of cells from complete results
+ * @throws Exception
+ */
+ @Test
+ public void testOrderingOfCellsInPartialResults() throws Exception {
+ Scan scan = new Scan();
+
+ for (int col = 1; col <= NUM_COLS; col++) {
+ scan.setMaxResultSize(getResultSizeForNumberOfCells(col));
+ testOrderingOfCellsInPartialResults(scan);
+
+ // Test again with a reversed scanner
+ scan.setReversed(true);
+ testOrderingOfCellsInPartialResults(scan);
+ }
+ }
+
+ public void testOrderingOfCellsInPartialResults(final Scan basePartialScan) throws Exception {
+ // Scan that retrieves results in pieces (partials). By setting allowPartialResults to be true
+ // the results will NOT be reconstructed and instead the caller will see the partial results
+ // returned by the server
+ Scan partialScan = new Scan(basePartialScan);
+ partialScan.setAllowPartialResults(true);
+ ResultScanner partialScanner = TABLE.getScanner(partialScan);
+
+ // Scan that retrieves all table results in single RPC request
+ Scan oneShotScan = new Scan(basePartialScan);
+ oneShotScan.setMaxResultSize(Long.MAX_VALUE);
+ oneShotScan.setCaching(ROWS.length);
+ ResultScanner oneShotScanner = TABLE.getScanner(oneShotScan);
+
+ Result oneShotResult = oneShotScanner.next();
+ Result partialResult = null;
+ int iterationCount = 0;
+
+ while (oneShotResult != null && oneShotResult.rawCells() != null) {
+ List<Cell> aggregatePartialCells = new ArrayList<Cell>();
+ do {
+ partialResult = partialScanner.next();
+ assertTrue("Partial Result is null. iteration: " + iterationCount, partialResult != null);
+ assertTrue("Partial cells are null. iteration: " + iterationCount,
+ partialResult.rawCells() != null);
+
+ for (Cell c : partialResult.rawCells()) {
+ aggregatePartialCells.add(c);
+ }
+ } while (partialResult.isPartial());
+
+ assertTrue("Number of cells differs. iteration: " + iterationCount,
+ oneShotResult.rawCells().length == aggregatePartialCells.size());
+ final Cell[] oneShotCells = oneShotResult.rawCells();
+ for (int cell = 0; cell < oneShotCells.length; cell++) {
+ Cell oneShotCell = oneShotCells[cell];
+ Cell partialCell = aggregatePartialCells.get(cell);
+
+ assertTrue("One shot cell was null", oneShotCell != null);
+ assertTrue("Partial cell was null", partialCell != null);
+ assertTrue("Cell differs. oneShotCell:" + oneShotCell + " partialCell:" + partialCell,
+ oneShotCell.equals(partialCell));
+ }
+
+ oneShotResult = oneShotScanner.next();
+ iterationCount++;
+ }
+
+ assertTrue(partialScanner.next() == null);
+
+ partialScanner.close();
+ oneShotScanner.close();
+ }
+
+ /**
+ * Setting the max result size allows us to control how many cells we expect to see on each call
+ * to next on the scanner. Test a variety of different sizes for correctness
+ * @throws Exception
+ */
+ @Test
+ public void testExpectedNumberOfCellsPerPartialResult() throws Exception {
+ Scan scan = new Scan();
+ testExpectedNumberOfCellsPerPartialResult(scan);
+
+ scan.setReversed(true);
+ testExpectedNumberOfCellsPerPartialResult(scan);
+ }
+
+ public void testExpectedNumberOfCellsPerPartialResult(Scan baseScan) throws Exception {
+ for (int expectedCells = 1; expectedCells <= NUM_COLS; expectedCells++) {
+ testExpectedNumberOfCellsPerPartialResult(baseScan, expectedCells);
+ }
+ }
+
+ public void testExpectedNumberOfCellsPerPartialResult(Scan baseScan, int expectedNumberOfCells)
+ throws Exception {
+
+ if (LOG.isInfoEnabled()) LOG.info("groupSize:" + expectedNumberOfCells);
+
+ // Use the cellHeapSize to set maxResultSize such that we know how many cells to expect back
+ // from the call. The returned results should NOT exceed expectedNumberOfCells but may be less
+ // than it in cases where expectedNumberOfCells is not an exact multiple of the number of
+ // columns in the table.
+ Scan scan = new Scan(baseScan);
+ scan.setAllowPartialResults(true);
+ scan.setMaxResultSize(getResultSizeForNumberOfCells(expectedNumberOfCells));
+
+ ResultScanner scanner = TABLE.getScanner(scan);
+ Result result = null;
+ byte[] prevRow = null;
+ while ((result = scanner.next()) != null) {
+ assertTrue(result.rawCells() != null);
+
+ // Cases when cell count won't equal expectedNumberOfCells:
+ // 1. Returned result is the final result needed to form the complete result for that row
+ // 2. It is the first result we have seen for that row and thus may have been fetched as
+ // the last group of cells that fit inside the maxResultSize
+ assertTrue(
+ "Result's cell count differed from expected number. result: " + result,
+ result.rawCells().length == expectedNumberOfCells || !result.isPartial()
+ || !Bytes.equals(prevRow, result.getRow()));
+ prevRow = result.getRow();
+ }
+
+ scanner.close();
+ }
+
+ /**
+ * @return The approximate heap size of a cell in the test table. All cells should have
+ * approximately the same heap size, so the value is cached to avoid repeating the
+ * calculation
+ * @throws Exception
+ */
+ private long getCellHeapSize() throws Exception {
+ if (CELL_HEAP_SIZE == -1) {
+ // Do a partial scan that will return a single result with a single cell
+ Scan scan = new Scan();
+ scan.setMaxResultSize(1);
+ scan.setAllowPartialResults(true);
+ ResultScanner scanner = TABLE.getScanner(scan);
+
+ Result result = scanner.next();
+
+ assertTrue(result != null);
+ assertTrue(result.rawCells() != null);
+ assertTrue(result.rawCells().length == 1);
+
+ CELL_HEAP_SIZE = CellUtil.estimatedHeapSizeOf(result.rawCells()[0]);
+ if (LOG.isInfoEnabled()) LOG.info("Cell heap size: " + CELL_HEAP_SIZE);
+ scanner.close();
+ }
+
+ return CELL_HEAP_SIZE;
+ }
+
+ /**
+ * @param numberOfCells
+ * @return the result size that should be used in {@link Scan#setMaxResultSize(long)} if you want
+ * the server to return exactly numberOfCells cells
+ * @throws Exception
+ */
+ private long getResultSizeForNumberOfCells(int numberOfCells) throws Exception {
+ return getCellHeapSize() * numberOfCells;
+ }
+
+ /**
+ * Test various combinations of batching and partial results for correctness
+ */
+ @Test
+ public void testPartialResultsAndBatch() throws Exception {
+ for (int batch = 1; batch <= NUM_COLS / 4; batch++) {
+ for (int cellsPerPartial = 1; cellsPerPartial <= NUM_COLS / 4; cellsPerPartial++) {
+ testPartialResultsAndBatch(batch, cellsPerPartial);
+ }
+ }
+ }
+
+ public void testPartialResultsAndBatch(final int batch, final int cellsPerPartialResult)
+ throws Exception {
+ if (LOG.isInfoEnabled()) {
+ LOG.info("batch: " + batch + " cellsPerPartialResult: " + cellsPerPartialResult);
+ }
+
+ Scan scan = new Scan();
+ scan.setMaxResultSize(getResultSizeForNumberOfCells(cellsPerPartialResult));
+ scan.setBatch(batch);
+ ResultScanner scanner = TABLE.getScanner(scan);
+ Result result = scanner.next();
+ int repCount = 0;
+
+ while ((result = scanner.next()) != null) {
+ assertTrue(result.rawCells() != null);
+
+ if (result.isPartial()) {
+ final String error =
+ "Cells:" + result.rawCells().length + " Batch size:" + batch
+ + " cellsPerPartialResult:" + cellsPerPartialResult + " rep:" + repCount;
+ assertTrue(error, result.rawCells().length <= Math.min(batch, cellsPerPartialResult));
+ } else {
+ assertTrue(result.rawCells().length <= batch);
+ }
+ repCount++;
+ }
+
+ scanner.close();
+ }
+
+ /**
+ * Test the method {@link Result#createCompleteResult(List, Result)}
+ * @throws Exception
+ */
+ @Test
+ public void testPartialResultsReassembly() throws Exception {
+ Scan scan = new Scan();
+ testPartialResultsReassembly(scan);
+ scan.setReversed(true);
+ testPartialResultsReassembly(scan);
+ }
+
+ public void testPartialResultsReassembly(Scan scanBase) throws Exception {
+ Scan partialScan = new Scan(scanBase);
+ partialScan.setMaxResultSize(1);
+ partialScan.setAllowPartialResults(true);
+ ResultScanner partialScanner = TABLE.getScanner(partialScan);
+
+ Scan oneShotScan = new Scan(scanBase);
+ oneShotScan.setMaxResultSize(Long.MAX_VALUE);
+ ResultScanner oneShotScanner = TABLE.getScanner(oneShotScan);
+
+ ArrayList<Result> partials = new ArrayList<>();
+ for (int i = 0; i < NUM_ROWS; i++) {
+ Result partialResult = null;
+ Result completeResult = null;
+ Result oneShotResult = null;
+ partials.clear();
+
+ do {
+ partialResult = partialScanner.next();
+ partials.add(partialResult);
+ } while (partialResult != null && partialResult.isPartial());
+
+ completeResult = Result.createCompleteResult(partials);
+ oneShotResult = oneShotScanner.next();
+
+ compareResults(completeResult, oneShotResult, null);
+ }
+
+ assertTrue(oneShotScanner.next() == null);
+ assertTrue(partialScanner.next() == null);
+
+ oneShotScanner.close();
+ partialScanner.close();
+ }
+
+ /**
+ * When reconstructing the complete result from its partials we ensure that the row of each
+ * partial result is the same. If one of the rows differs, an exception is thrown.
+ */
+ @Test
+ public void testExceptionThrownOnMismatchedPartialResults() throws IOException {
+ assertTrue(NUM_ROWS >= 2);
+
+ ArrayList<Result> partials = new ArrayList<>();
+ Scan scan = new Scan();
+ scan.setMaxResultSize(Long.MAX_VALUE);
+ ResultScanner scanner = TABLE.getScanner(scan);
+ Result r1 = scanner.next();
+ partials.add(r1);
+ Result r2 = scanner.next();
+ partials.add(r2);
+
+ assertFalse(Bytes.equals(r1.getRow(), r2.getRow()));
+
+ try {
+ Result.createCompleteResult(partials);
+ fail("r1 and r2 are from different rows. It should not be possible to combine them into"
+ + " a single result");
+ } catch (IOException e) {
+ }
+
+ scanner.close();
+ }
+
+ /**
+ * When a scan has a filter where {@link org.apache.hadoop.hbase.filter.Filter#hasFilterRow()} is
+ * true, the scanner should not return partial results. The scanner cannot return partial results
+ * because the entire row needs to be read for the include/exclude decision to be made
+ */
+ @Test
+ public void testNoPartialResultsWhenRowFilterPresent() throws Exception {
+ Scan scan = new Scan();
+ scan.setMaxResultSize(1);
+ scan.setAllowPartialResults(true);
+ // If a filter hasFilter() is true then partial results should not be returned else filter
+ // application server side would break.
+ scan.setFilter(new RandomRowFilter(1.0f));
+ ResultScanner scanner = TABLE.getScanner(scan);
+
+ Result r = null;
+ while ((r = scanner.next()) != null) {
+ assertFalse(r.isPartial());
+ }
+
+ scanner.close();
+ }
+
+ /**
+ * Examine the interaction between the maxResultSize and caching. If the caching limit is reached
+ * before the maxResultSize limit, we should not see partial results. On the other hand, if the
+ * maxResultSize limit is reached before the caching limit, it is likely that partial results will
+ * be seen.
+ * @throws Exception
+ */
+ @Test
+ public void testPartialResultsAndCaching() throws Exception {
+ for (int caching = 1; caching <= NUM_ROWS; caching++) {
+ for (int maxResultRows = 0; maxResultRows <= NUM_ROWS; maxResultRows++) {
+ testPartialResultsAndCaching(maxResultRows, caching);
+ }
+ }
+ }
+
+ /**
+ * @param resultSizeRowLimit The row limit that will be enforced through maxResultSize
+ * @param cachingRowLimit The row limit that will be enforced through caching
+ * @throws Exception
+ */
+ public void testPartialResultsAndCaching(int resultSizeRowLimit, int cachingRowLimit)
+ throws Exception {
+ Scan scan = new Scan();
+ scan.setAllowPartialResults(true);
+
+ // The number of cells specified in the call to getResultSizeForNumberOfCells is offset to
+ // ensure that the result size we specify is not an exact multiple of the number of cells
+ // in a row. This ensures that partial results will be returned when the result size limit
+ // is reached before the caching limit.
+ int cellOffset = NUM_COLS / 3;
+ long maxResultSize = getResultSizeForNumberOfCells(resultSizeRowLimit * NUM_COLS + cellOffset);
+ scan.setMaxResultSize(maxResultSize);
+ scan.setCaching(cachingRowLimit);
+
+ ResultScanner scanner = TABLE.getScanner(scan);
+ ClientScanner clientScanner = (ClientScanner) scanner;
+ Result r = null;
+
+ // Approximate the number of rows we expect will fit into the specified max rsult size. If this
+ // approximation is less than caching, then we expect that the max result size limit will be
+ // hit before the caching limit and thus partial results may be seen
+ boolean expectToSeePartialResults = resultSizeRowLimit < cachingRowLimit;
+ while ((r = clientScanner.next()) != null) {
+ assertTrue(!r.isPartial() || expectToSeePartialResults);
+ }
+
+ scanner.close();
+ }
+
+ /**
+ * Small scans should not return partial results because it would prevent small scans from
+ * retrieving all of the necessary results in a single RPC request which is what makese small
+ * scans useful. Thus, ensure that even when {@link Scan#getAllowPartialResults()} is true, small
+ * scans do not return partial results
+ * @throws Exception
+ */
+ @Test
+ public void testSmallScansDoNotAllowPartials() throws Exception {
+ Scan scan = new Scan();
+ testSmallScansDoNotAllowPartials(scan);
+ scan.setReversed(true);
+ testSmallScansDoNotAllowPartials(scan);
+ }
+
+ public void testSmallScansDoNotAllowPartials(Scan baseScan) throws Exception {
+ Scan scan = new Scan(baseScan);
+ scan.setAllowPartialResults(true);
+ scan.setSmall(true);
+ scan.setMaxResultSize(1);
+
+ ResultScanner scanner = TABLE.getScanner(scan);
+ Result r = null;
+
+ while ((r = scanner.next()) != null) {
+ assertFalse(r.isPartial());
+ }
+
+ scanner.close();
+ }
+
+ /**
+ * Make puts to put the input value into each combination of row, family, and qualifier
+ * @param rows
+ * @param families
+ * @param qualifiers
+ * @param value
+ * @return
+ * @throws IOException
+ */
+ static ArrayList<Put> createPuts(byte[][] rows, byte[][] families, byte[][] qualifiers,
+ byte[] value) throws IOException {
+ Put put;
+ ArrayList<Put> puts = new ArrayList<>();
+
+ for (int row = 0; row < rows.length; row++) {
+ put = new Put(rows[row]);
+ for (int fam = 0; fam < families.length; fam++) {
+ for (int qual = 0; qual < qualifiers.length; qual++) {
+ KeyValue kv = new KeyValue(rows[row], families[fam], qualifiers[qual], qual, value);
+ put.add(kv);
+ }
+ }
+ puts.add(put);
+ }
+
+ return puts;
+ }
+
+ /**
+ * Make key values to represent each possible combination of family and qualifier in the specified
+ * row.
+ * @param row
+ * @param families
+ * @param qualifiers
+ * @param value
+ * @return
+ */
+ static ArrayList<Cell> createKeyValuesForRow(byte[] row, byte[][] families, byte[][] qualifiers,
+ byte[] value) {
+ ArrayList<Cell> outList = new ArrayList<>();
+ for (int fam = 0; fam < families.length; fam++) {
+ for (int qual = 0; qual < qualifiers.length; qual++) {
+ outList.add(new KeyValue(row, families[fam], qualifiers[qual], qual, value));
+ }
+ }
+ return outList;
+ }
+
+ /**
+ * Verifies that result contains all the key values within expKvList. Fails the test otherwise
+ * @param result
+ * @param expKvList
+ * @param msg
+ */
+ static void verifyResult(Result result, List<Cell> expKvList, String msg) {
+ if (LOG.isInfoEnabled()) {
+ LOG.info(msg);
+ LOG.info("Expected count: " + expKvList.size());
+ LOG.info("Actual count: " + result.size());
+ }
+
+ if (expKvList.size() == 0) return;
+
+ int i = 0;
+ for (Cell kv : result.rawCells()) {
+ if (i >= expKvList.size()) {
+ break; // we will check the size later
+ }
+
+ Cell kvExp = expKvList.get(i++);
+ assertTrue("Not equal. get kv: " + kv.toString() + " exp kv: " + kvExp.toString(),
+ kvExp.equals(kv));
+ }
+
+ assertEquals(expKvList.size(), result.size());
+ }
+
+ /**
+ * Compares two results and fails the test if the results are different
+ * @param r1
+ * @param r2
+ * @param message
+ */
+ static void compareResults(Result r1, Result r2, final String message) {
+ if (LOG.isInfoEnabled()) {
+ if (message != null) LOG.info(message);
+ LOG.info("r1: " + r1);
+ LOG.info("r2: " + r2);
+ }
+
+ final String failureMessage = "Results r1:" + r1 + " \nr2:" + r2 + " are not equivalent";
+ if (r1 == null && r2 == null) fail(failureMessage);
+ else if (r1 == null || r2 == null) fail(failureMessage);
+
+ try {
+ Result.compareResults(r1, r2);
+ } catch (Exception e) {
+ fail(failureMessage);
+ }
+ }
+
+ @Test
+ public void testReadPointAndPartialResults() throws Exception {
+ TableName testName = TableName.valueOf("testReadPointAndPartialResults");
+ int numRows = 5;
+ int numFamilies = 5;
+ int numQualifiers = 5;
+ byte[][] rows = HTestConst.makeNAscii(Bytes.toBytes("testRow"), numRows);
+ byte[][] families = HTestConst.makeNAscii(Bytes.toBytes("testFamily"), numFamilies);
+ byte[][] qualifiers = HTestConst.makeNAscii(Bytes.toBytes("testQualifier"), numQualifiers);
+ byte[] value = Bytes.createMaxByteArray(100);
+
+ Table tmpTable = createTestTable(testName, rows, families, qualifiers, value);
+
+ Scan scan = new Scan();
+ scan.setMaxResultSize(1);
+ scan.setAllowPartialResults(true);
+
+ // Open scanner before deletes
+ ResultScanner scanner = tmpTable.getScanner(scan);
+
+ Delete delete1 = new Delete(rows[0]);
+ delete1.addColumn(families[0], qualifiers[0], 0);
+ tmpTable.delete(delete1);
+
+ Delete delete2 = new Delete(rows[1]);
+ delete2.addColumn(families[1], qualifiers[1], 1);
+ tmpTable.delete(delete2);
+
+ // Should see all cells because scanner was opened prior to deletes
+ int scannerCount = countCellsFromScanner(scanner);
+ int expectedCount = numRows * numFamilies * numQualifiers;
+ assertTrue("scannerCount: " + scannerCount + " expectedCount: " + expectedCount,
+ scannerCount == expectedCount);
+
+ // Minus 2 for the two cells that were deleted
+ scanner = tmpTable.getScanner(scan);
+ scannerCount = countCellsFromScanner(scanner);
+ expectedCount = numRows * numFamilies * numQualifiers - 2;
+ assertTrue("scannerCount: " + scannerCount + " expectedCount: " + expectedCount,
+ scannerCount == expectedCount);
+
+ scanner = tmpTable.getScanner(scan);
+ // Put in 2 new rows. The timestamps differ from the deleted rows
+ Put put1 = new Put(rows[0]);
+ put1.add(new KeyValue(rows[0], families[0], qualifiers[0], 1, value));
+ tmpTable.put(put1);
+
+ Put put2 = new Put(rows[1]);
+ put2.add(new KeyValue(rows[1], families[1], qualifiers[1], 2, value));
+ tmpTable.put(put2);
+
+ // Scanner opened prior to puts. Cell count shouldn't have changed
+ scannerCount = countCellsFromScanner(scanner);
+ expectedCount = numRows * numFamilies * numQualifiers - 2;
+ assertTrue("scannerCount: " + scannerCount + " expectedCount: " + expectedCount,
+ scannerCount == expectedCount);
+
+ // Now the scanner should see the cells that were added by puts
+ scanner = tmpTable.getScanner(scan);
+ scannerCount = countCellsFromScanner(scanner);
+ expectedCount = numRows * numFamilies * numQualifiers;
+ assertTrue("scannerCount: " + scannerCount + " expectedCount: " + expectedCount,
+ scannerCount == expectedCount);
+
+ TEST_UTIL.deleteTable(testName);
+ }
+
+ /**
+ * Exhausts the scanner by calling next repetitively. Once completely exhausted, close scanner and
+ * return total cell count
+ * @param scanner
+ * @return
+ * @throws Exception
+ */
+ private int countCellsFromScanner(ResultScanner scanner) throws Exception {
+ Result result = null;
+ int numCells = 0;
+ while ((result = scanner.next()) != null) {
+ numCells += result.rawCells().length;
+ }
+
+ scanner.close();
+ return numCells;
+ }
+
+ /**
+ * Test partial Result re-assembly in the presence of different filters. The Results from the
+ * partial scanner should match the Results returned from a scanner that receives all of the
+ * results in one RPC to the server. The partial scanner is tested with a variety of different
+ * result sizes (all of which are less than the size necessary to fetch an entire row)
+ * @throws Exception
+ */
+ @Test
+ public void testPartialResultsWithColumnFilter() throws Exception {
+ testPartialResultsWithColumnFilter(new FirstKeyOnlyFilter());
+ testPartialResultsWithColumnFilter(new ColumnPrefixFilter(Bytes.toBytes("testQualifier5")));
+ testPartialResultsWithColumnFilter(new ColumnRangeFilter(Bytes.toBytes("testQualifer1"), true,
+ Bytes.toBytes("testQualifier7"), true));
+
+ Set<byte[]> qualifiers = new LinkedHashSet<>();
+ qualifiers.add(Bytes.toBytes("testQualifier5"));
+ testPartialResultsWithColumnFilter(new FirstKeyValueMatchingQualifiersFilter(qualifiers));
+ }
+
+ public void testPartialResultsWithColumnFilter(Filter filter) throws Exception {
+ assertTrue(!filter.hasFilterRow());
+
+ Scan partialScan = new Scan();
+ partialScan.setFilter(filter);
+
+ Scan oneshotScan = new Scan();
+ oneshotScan.setFilter(filter);
+ oneshotScan.setMaxResultSize(Long.MAX_VALUE);
+
+ for (int i = 1; i <= NUM_COLS; i++) {
+ partialScan.setMaxResultSize(getResultSizeForNumberOfCells(i));
+ testEquivalenceOfScanResults(TABLE, partialScan, oneshotScan);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/30cec72f/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMobSnapshotCloneIndependence.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMobSnapshotCloneIndependence.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMobSnapshotCloneIndependence.java
new file mode 100644
index 0000000..dcf20e5
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMobSnapshotCloneIndependence.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.client;
+
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.mob.MobConstants;
+import org.apache.hadoop.hbase.snapshot.MobSnapshotTestingUtils;
+import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.junit.BeforeClass;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Test to verify that the cloned table is independent of the table from which it was cloned
+ */
+@Category(LargeTests.class)
+public class TestMobSnapshotCloneIndependence extends TestSnapshotCloneIndependence {
+ private static final Log LOG = LogFactory.getLog(TestMobSnapshotCloneIndependence.class);
+
+ /**
+ * Setup the config for the cluster and start it
+ * @throws Exception on failure
+ */
+ @BeforeClass
+ public static void setupCluster() throws Exception {
+ setupConf(UTIL.getConfiguration());
+ UTIL.startMiniCluster(NUM_RS);
+ }
+
+ protected static void setupConf(Configuration conf) {
+ TestSnapshotCloneIndependence.setupConf(conf);
+ conf.setInt(MobConstants.MOB_FILE_CACHE_SIZE_KEY, 0);
+ }
+
+ @Override
+ protected Table createTable(final TableName table, byte[] family) throws Exception {
+ return MobSnapshotTestingUtils.createMobTable(UTIL, table, family);
+ }
+
+ @Override
+ public void loadData(final Table table, byte[]... families) throws Exception {
+ SnapshotTestingUtils.loadData(UTIL, table.getName(), 1000, families);
+ }
+
+ @Override
+ protected int countRows(final Table table, final byte[]... families) throws Exception {
+ return MobSnapshotTestingUtils.countMobRows(table, families);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/30cec72f/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotCloneIndependence.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotCloneIndependence.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotCloneIndependence.java
new file mode 100644
index 0000000..0a3093b
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotCloneIndependence.java
@@ -0,0 +1,481 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.client;
+
+import java.io.IOException;
+import java.util.List;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
+import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
+import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
+import org.apache.hadoop.hbase.testclassification.ClientTests;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.Ignore;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Test to verify that the cloned table is independent of the table from which it was cloned
+ */
+@Category({LargeTests.class, ClientTests.class})
+public class TestSnapshotCloneIndependence {
+ private static final Log LOG = LogFactory.getLog(TestSnapshotCloneIndependence.class);
+
+ protected static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
+
+ protected static final int NUM_RS = 2;
+ private static final String STRING_TABLE_NAME = "test";
+ private static final String TEST_FAM_STR = "fam";
+ protected static final byte[] TEST_FAM = Bytes.toBytes(TEST_FAM_STR);
+ protected static final TableName TABLE_NAME = TableName.valueOf(STRING_TABLE_NAME);
+ private static final int CLEANER_INTERVAL = 100;
+
+ /**
+ * Setup the config for the cluster and start it
+ * @throws Exception on fOailure
+ */
+ @BeforeClass
+ public static void setupCluster() throws Exception {
+ setupConf(UTIL.getConfiguration());
+ UTIL.startMiniCluster(NUM_RS);
+ }
+
+ static void setupConf(Configuration conf) {
+ // Up the handlers; this test needs more than usual.
+ conf.setInt(HConstants.REGION_SERVER_HIGH_PRIORITY_HANDLER_COUNT, 15);
+ // enable snapshot support
+ conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
+ // disable the ui
+ conf.setInt("hbase.regionsever.info.port", -1);
+ conf.setInt("hbase.master.info.port", -1);
+ // change the flush size to a small amount, regulating number of store files
+ conf.setInt("hbase.hregion.memstore.flush.size", 25000);
+ // so make sure we get a compaction when doing a load, but keep around
+ // some files in the store
+ conf.setInt("hbase.hstore.compaction.min", 10);
+ conf.setInt("hbase.hstore.compactionThreshold", 10);
+ // block writes if we get to 12 store files
+ conf.setInt("hbase.hstore.blockingStoreFiles", 12);
+ conf.setInt("hbase.regionserver.msginterval", 100);
+ conf.setBoolean("hbase.master.enabletable.roundrobin", true);
+ // Avoid potentially aggressive splitting which would cause snapshot to fail
+ conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
+ ConstantSizeRegionSplitPolicy.class.getName());
+ // Execute cleaner frequently to induce failures
+ conf.setInt("hbase.master.cleaner.interval", CLEANER_INTERVAL);
+ conf.setInt("hbase.master.hfilecleaner.plugins.snapshot.period", CLEANER_INTERVAL);
+ // Effectively disable TimeToLiveHFileCleaner. Don't want to fully disable it because that
+ // will even trigger races between creating the directory containing back references and
+ // the back reference itself.
+ conf.setInt("hbase.master.hfilecleaner.ttl", CLEANER_INTERVAL);
+ }
+
+ @Before
+ public void setup() throws Exception {
+ createTable(TABLE_NAME, TEST_FAM);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ UTIL.deleteTable(TABLE_NAME);
+ SnapshotTestingUtils.deleteAllSnapshots(UTIL.getHBaseAdmin());
+ SnapshotTestingUtils.deleteArchiveDirectory(UTIL);
+ }
+
+ @AfterClass
+ public static void cleanupTest() throws Exception {
+ try {
+ UTIL.shutdownMiniCluster();
+ } catch (Exception e) {
+ LOG.warn("failure shutting down cluster", e);
+ }
+ }
+
+ /**
+ * Verify that adding data to the cloned table will not affect the original, and vice-versa when
+ * it is taken as an online snapshot.
+ */
+ @Ignore ("Flakey. Fix") @Test (timeout=300000)
+ public void testOnlineSnapshotAppendIndependent() throws Exception {
+ runTestSnapshotAppendIndependent(true);
+ }
+
+ /**
+ * Verify that adding data to the cloned table will not affect the original, and vice-versa when
+ * it is taken as an offline snapshot.
+ */
+ @Test (timeout=300000)
+ @Ignore
+ public void testOfflineSnapshotAppendIndependent() throws Exception {
+ runTestSnapshotAppendIndependent(false);
+ }
+
+ /**
+ * Verify that adding metadata to the cloned table will not affect the original, and vice-versa
+ * when it is taken as an online snapshot.
+ */
+ @Test (timeout=300000)
+ public void testOnlineSnapshotMetadataChangesIndependent() throws Exception {
+ runTestSnapshotMetadataChangesIndependent(true);
+ }
+
+ /**
+ * Verify that adding netadata to the cloned table will not affect the original, and vice-versa
+ * when is taken as an online snapshot.
+ */
+ @Test (timeout=300000)
+ @Ignore
+ public void testOfflineSnapshotMetadataChangesIndependent() throws Exception {
+ runTestSnapshotMetadataChangesIndependent(false);
+ }
+
+ /**
+ * Verify that region operations, in this case splitting a region, are independent between the
+ * cloned table and the original.
+ */
+ @Test (timeout=300000)
+ @Ignore
+ public void testOfflineSnapshotRegionOperationsIndependent() throws Exception {
+ runTestRegionOperationsIndependent(false);
+ }
+
+ /**
+ * Verify that region operations, in this case splitting a region, are independent between the
+ * cloned table and the original.
+ */
+ @Test (timeout=300000)
+ public void testOnlineSnapshotRegionOperationsIndependent() throws Exception {
+ runTestRegionOperationsIndependent(true);
+ }
+
+ @Test (timeout=300000)
+ @Ignore
+ public void testOfflineSnapshotDeleteIndependent() throws Exception {
+ runTestSnapshotDeleteIndependent(false);
+ }
+
+ @Ignore ("Flakey test") @Test (timeout=300000)
+ public void testOnlineSnapshotDeleteIndependent() throws Exception {
+ runTestSnapshotDeleteIndependent(true);
+ }
+
+ private static void waitOnSplit(Connection c, final Table t, int originalCount) throws Exception {
+ for (int i = 0; i < 200; i++) {
+ try {
+ Thread.sleep(500);
+ } catch (InterruptedException e) {
+ // Restore the interrupted status
+ Thread.currentThread().interrupt();
+ }
+ try (RegionLocator locator = c.getRegionLocator(t.getName())) {
+ if (locator.getAllRegionLocations().size() > originalCount) {
+ return;
+ }
+ }
+ }
+ throw new Exception("Split did not increase the number of regions");
+ }
+
+ /*
+ * Take a snapshot of a table, add data, and verify that this only
+ * affects one table
+ * @param online - Whether the table is online or not during the snapshot
+ */
+ private void runTestSnapshotAppendIndependent(boolean online) throws Exception {
+ FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
+ Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
+
+ Admin admin = UTIL.getHBaseAdmin();
+ final long startTime = System.currentTimeMillis();
+ final TableName localTableName =
+ TableName.valueOf(STRING_TABLE_NAME + startTime);
+
+ try (Table original = createTable(localTableName, TEST_FAM)) {
+ loadData(original, TEST_FAM);
+ final int origTableRowCount = countRows(original);
+
+ // Take a snapshot
+ final String snapshotNameAsString = "snapshot_" + localTableName;
+ byte[] snapshotName = Bytes.toBytes(snapshotNameAsString);
+
+ SnapshotTestingUtils.createSnapshotAndValidate(admin, localTableName, TEST_FAM_STR,
+ snapshotNameAsString, rootDir, fs, online);
+
+ if (!online) {
+ tryDisable(admin, localTableName);
+ }
+ TableName cloneTableName = TableName.valueOf("test-clone-" + localTableName);
+ admin.cloneSnapshot(snapshotName, cloneTableName);
+
+ try (Table clonedTable = UTIL.getConnection().getTable(cloneTableName)) {
+
+ // Make sure that all the regions are available before starting
+ UTIL.waitUntilAllRegionsAssigned(cloneTableName);
+
+ final int clonedTableRowCount = countRows(clonedTable);
+
+ Assert.assertEquals(
+ "The line counts of original and cloned tables do not match after clone. ",
+ origTableRowCount, clonedTableRowCount);
+
+ // Attempt to add data to the test
+ final String rowKey = "new-row-" + System.currentTimeMillis();
+
+ Put p = new Put(Bytes.toBytes(rowKey));
+ p.addColumn(TEST_FAM, Bytes.toBytes("someQualifier"), Bytes.toBytes("someString"));
+ original.put(p);
+
+ // Verify that it is not present in the original table
+ Assert.assertEquals("The row count of the original table was not modified by the put",
+ origTableRowCount + 1, countRows(original));
+ Assert.assertEquals(
+ "The row count of the cloned table changed as a result of addition to the original",
+ clonedTableRowCount, countRows(clonedTable));
+
+ p = new Put(Bytes.toBytes(rowKey));
+ p.addColumn(TEST_FAM, Bytes.toBytes("someQualifier"), Bytes.toBytes("someString"));
+ clonedTable.put(p);
+
+ // Verify that the new family is not in the restored table's description
+ Assert.assertEquals(
+ "The row count of the original table was modified by the put to the clone",
+ origTableRowCount + 1, countRows(original));
+ Assert.assertEquals("The row count of the cloned table was not modified by the put",
+ clonedTableRowCount + 1, countRows(clonedTable));
+ }
+ }
+ }
+
+ /*
+ * Take a snapshot of a table, do a split, and verify that this only affects one table
+ * @param online - Whether the table is online or not during the snapshot
+ */
+ private void runTestRegionOperationsIndependent(boolean online) throws Exception {
+ FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
+ Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
+
+ // Create a table
+ Admin admin = UTIL.getHBaseAdmin();
+ final long startTime = System.currentTimeMillis();
+ final TableName localTableName =
+ TableName.valueOf(STRING_TABLE_NAME + startTime);
+ Table original = createTable(localTableName, TEST_FAM);
+ loadData(original, TEST_FAM);
+ final int loadedTableCount = countRows(original);
+ System.out.println("Original table has: " + loadedTableCount + " rows");
+
+ final String snapshotNameAsString = "snapshot_" + localTableName;
+
+ // Create a snapshot
+ SnapshotTestingUtils.createSnapshotAndValidate(admin, localTableName, TEST_FAM_STR,
+ snapshotNameAsString, rootDir, fs, online);
+
+ if (!online) {
+ tryDisable(admin, localTableName);
+ }
+
+ TableName cloneTableName = TableName.valueOf("test-clone-" + localTableName);
+
+ // Clone the snapshot
+ byte[] snapshotName = Bytes.toBytes(snapshotNameAsString);
+ admin.cloneSnapshot(snapshotName, cloneTableName);
+
+ // Verify that region information is the same pre-split
+ ((ClusterConnection) UTIL.getConnection()).clearRegionCache();
+ List<HRegionInfo> originalTableHRegions = admin.getTableRegions(localTableName);
+
+ final int originalRegionCount = originalTableHRegions.size();
+ final int cloneTableRegionCount = admin.getTableRegions(cloneTableName).size();
+ Assert.assertEquals(
+ "The number of regions in the cloned table is different than in the original table.",
+ originalRegionCount, cloneTableRegionCount);
+
+ // Split a region on the parent table
+ admin.splitRegion(originalTableHRegions.get(0).getRegionName());
+ waitOnSplit(UTIL.getConnection(), original, originalRegionCount);
+
+ // Verify that the cloned table region is not split
+ final int cloneTableRegionCount2 = admin.getTableRegions(cloneTableName).size();
+ Assert.assertEquals(
+ "The number of regions in the cloned table changed though none of its regions were split.",
+ cloneTableRegionCount, cloneTableRegionCount2);
+ }
+
+ /*
+ * Take a snapshot of a table, add metadata, and verify that this only
+ * affects one table
+ * @param online - Whether the table is online or not during the snapshot
+ */
+ private void runTestSnapshotMetadataChangesIndependent(boolean online) throws Exception {
+ FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
+ Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
+
+ // Create a table
+ Admin admin = UTIL.getHBaseAdmin();
+ final long startTime = System.currentTimeMillis();
+ final TableName localTableName =
+ TableName.valueOf(STRING_TABLE_NAME + startTime);
+ Table original = createTable(localTableName, TEST_FAM);
+ loadData(original, TEST_FAM);
+
+ final String snapshotNameAsString = "snapshot_" + localTableName;
+
+ // Create a snapshot
+ SnapshotTestingUtils.createSnapshotAndValidate(admin, localTableName, TEST_FAM_STR,
+ snapshotNameAsString, rootDir, fs, online);
+
+ if (!online) {
+ tryDisable(admin, localTableName);
+ }
+
+ TableName cloneTableName = TableName.valueOf("test-clone-" + localTableName);
+
+ // Clone the snapshot
+ byte[] snapshotName = Bytes.toBytes(snapshotNameAsString);
+ admin.cloneSnapshot(snapshotName, cloneTableName);
+
+ // Add a new column family to the original table
+ byte[] TEST_FAM_2 = Bytes.toBytes("fam2");
+ HColumnDescriptor hcd = new HColumnDescriptor(TEST_FAM_2);
+
+ tryDisable(admin, localTableName);
+ admin.addColumnFamily(localTableName, hcd);
+
+ // Verify that it is not in the snapshot
+ admin.enableTable(localTableName);
+ UTIL.waitTableAvailable(localTableName);
+
+ // get a description of the cloned table
+ // get a list of its families
+ // assert that the family is there
+ HTableDescriptor originalTableDescriptor = original.getTableDescriptor();
+ HTableDescriptor clonedTableDescriptor = admin.getTableDescriptor(cloneTableName);
+
+ Assert.assertTrue("The original family was not found. There is something wrong. ",
+ originalTableDescriptor.hasFamily(TEST_FAM));
+ Assert.assertTrue("The original family was not found in the clone. There is something wrong. ",
+ clonedTableDescriptor.hasFamily(TEST_FAM));
+
+ Assert.assertTrue("The new family was not found. ",
+ originalTableDescriptor.hasFamily(TEST_FAM_2));
+ Assert.assertTrue("The new family was not found. ",
+ !clonedTableDescriptor.hasFamily(TEST_FAM_2));
+ }
+
+ private void tryDisable(Admin admin, TableName localTableName) throws IOException {
+ int offlineRetry = 0;
+ while ( offlineRetry < 5 && admin.isTableEnabled(localTableName)) {
+ try {
+ admin.disableTable(localTableName);
+ } catch (IOException ioe) {
+ LOG.warn("Error disabling the table", ioe);
+ }
+ offlineRetry ++;
+ }
+ }
+
+ /*
+ * Take a snapshot of a table, add data, and verify that deleting the snapshot does not affect
+ * either table.
+ * @param online - Whether the table is online or not during the snapshot
+ */
+ private void runTestSnapshotDeleteIndependent(boolean online) throws Exception {
+ FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
+ Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
+
+ final Admin admin = UTIL.getHBaseAdmin();
+ final long startTime = System.currentTimeMillis();
+ final TableName localTableName =
+ TableName.valueOf(STRING_TABLE_NAME + startTime);
+
+ try (Table original = createTable(localTableName, TEST_FAM)) {
+ loadData(original, TEST_FAM);
+ }
+
+ // Take a snapshot
+ final String snapshotNameAsString = "snapshot_" + localTableName;
+ byte[] snapshotName = Bytes.toBytes(snapshotNameAsString);
+
+ SnapshotTestingUtils.createSnapshotAndValidate(admin, localTableName, TEST_FAM_STR,
+ snapshotNameAsString, rootDir, fs, online);
+
+ if (!online) {
+ tryDisable(admin, localTableName);
+ }
+
+ TableName cloneTableName = TableName.valueOf("test-clone-" + localTableName);
+ admin.cloneSnapshot(snapshotName, cloneTableName);
+
+ UTIL.waitUntilAllRegionsAssigned(cloneTableName);
+
+ // Ensure the original table does not reference the HFiles anymore
+ admin.majorCompact(localTableName);
+
+ // Deleting the snapshot used to break the cloned table by deleting in-use HFiles
+ admin.deleteSnapshot(snapshotName);
+
+ // Wait for cleaner run and DFS heartbeats so that anything that is deletable is fully deleted
+ do {
+ Thread.sleep(5000);
+ } while (!admin.listSnapshots(snapshotNameAsString).isEmpty());
+
+ try (Table original = UTIL.getConnection().getTable(localTableName)) {
+ try (Table clonedTable = UTIL.getConnection().getTable(cloneTableName)) {
+ // Verify that all regions of both tables are readable
+ final int origTableRowCount = countRows(original);
+ final int clonedTableRowCount = countRows(clonedTable);
+ Assert.assertEquals(origTableRowCount, clonedTableRowCount);
+ }
+ }
+ }
+
+ protected Table createTable(final TableName table, byte[] family) throws Exception {
+ Table t = UTIL.createTable(table, family);
+ // Wait for everything to be ready with the table
+ UTIL.waitUntilAllRegionsAssigned(table);
+
+ // At this point the table should be good to go.
+ return t;
+ }
+
+ protected void loadData(final Table table, byte[]... families) throws Exception {
+ UTIL.loadTable(table, families);
+ }
+
+ protected int countRows(final Table table, final byte[]... families) throws Exception {
+ return UTIL.countRows(table, families);
+ }
+}
\ No newline at end of file