You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by bu...@apache.org on 2017/08/27 05:33:08 UTC
[08/50] [abbrv] hbase git commit: HBASE-18640 Move mapreduce out of
hbase-server into separate module.
http://git-wip-us.apache.org/repos/asf/hbase/blob/664b6be0/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TsvImporterCustomTestMapperForOprAttr.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TsvImporterCustomTestMapperForOprAttr.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TsvImporterCustomTestMapperForOprAttr.java
deleted file mode 100644
index 9d8b8f0..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TsvImporterCustomTestMapperForOprAttr.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.mapreduce;
-
-import java.io.IOException;
-
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.mapreduce.ImportTsv.TsvParser.BadTsvLineException;
-import org.apache.hadoop.hbase.mapreduce.ImportTsv.TsvParser.ParsedLine;
-import org.apache.hadoop.hbase.util.Bytes;
-
-/**
- *
- * Just shows a simple example of how the attributes can be extracted and added
- * to the puts
- */
-public class TsvImporterCustomTestMapperForOprAttr extends TsvImporterMapper {
- @Override
- protected void populatePut(byte[] lineBytes, ParsedLine parsed, Put put, int i)
- throws BadTsvLineException, IOException {
- KeyValue kv;
- kv = new KeyValue(lineBytes, parsed.getRowKeyOffset(), parsed.getRowKeyLength(),
- parser.getFamily(i), 0, parser.getFamily(i).length, parser.getQualifier(i), 0,
- parser.getQualifier(i).length, ts, KeyValue.Type.Put, lineBytes, parsed.getColumnOffset(i),
- parsed.getColumnLength(i));
- if (parsed.getIndividualAttributes() != null) {
- String[] attributes = parsed.getIndividualAttributes();
- for (String attr : attributes) {
- String[] split = attr.split(ImportTsv.DEFAULT_ATTRIBUTES_SEPERATOR);
- if (split == null || split.length <= 1) {
- throw new BadTsvLineException("Invalid attributes seperator specified" + attributes);
- } else {
- if (split[0].length() <= 0 || split[1].length() <= 0) {
- throw new BadTsvLineException("Invalid attributes seperator specified" + attributes);
- }
- put.setAttribute(split[0], Bytes.toBytes(split[1]));
- }
- }
- }
- put.add(kv);
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/664b6be0/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java
index f641887..a81d268 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java
@@ -65,7 +65,6 @@ import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
import org.apache.hadoop.hbase.coprocessor.RegionObserver;
import org.apache.hadoop.hbase.coprocessor.RegionServerCoprocessorEnvironment;
import org.apache.hadoop.hbase.coprocessor.RegionServerObserver;
-import org.apache.hadoop.hbase.mapreduce.TableInputFormatBase;
import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
import org.apache.hadoop.hbase.master.TableNamespaceManager;
@@ -336,7 +335,7 @@ public class TestNamespaceAuditor {
byte[] columnFamily = Bytes.toBytes("info");
HTableDescriptor tableDescOne = new HTableDescriptor(tableTwo);
tableDescOne.addFamily(new HColumnDescriptor(columnFamily));
- ADMIN.createTable(tableDescOne, Bytes.toBytes("1"), Bytes.toBytes("2000"), initialRegions);
+ ADMIN.createTable(tableDescOne, Bytes.toBytes("0"), Bytes.toBytes("9"), initialRegions);
Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration());
try (Table table = connection.getTable(tableTwo)) {
UTIL.loadNumericRows(table, Bytes.toBytes("info"), 1000, 1999);
@@ -354,7 +353,7 @@ public class TestNamespaceAuditor {
hris = ADMIN.getTableRegions(tableTwo);
assertEquals(initialRegions - 1, hris.size());
Collections.sort(hris);
- ADMIN.split(tableTwo, Bytes.toBytes("500"));
+ ADMIN.split(tableTwo, Bytes.toBytes("3"));
// Not much we can do here until we have split return a Future.
Threads.sleep(5000);
hris = ADMIN.getTableRegions(tableTwo);
@@ -383,8 +382,7 @@ public class TestNamespaceAuditor {
Collections.sort(hris);
// verify that we cannot split
HRegionInfo hriToSplit2 = hris.get(1);
- ADMIN.split(tableTwo,
- TableInputFormatBase.getSplitKey(hriToSplit2.getStartKey(), hriToSplit2.getEndKey(), true));
+ ADMIN.split(tableTwo, Bytes.toBytes("6"));
Thread.sleep(2000);
assertEquals(initialRegions, ADMIN.getTableRegions(tableTwo).size());
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/664b6be0/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionFileSystem.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionFileSystem.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionFileSystem.java
index 0aa39f6..477c870 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionFileSystem.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionFileSystem.java
@@ -42,7 +42,6 @@ import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.PerformanceEvaluation;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.HTable;
@@ -62,9 +61,11 @@ import org.junit.rules.TestName;
public class TestHRegionFileSystem {
private static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
private static final Log LOG = LogFactory.getLog(TestHRegionFileSystem.class);
+
+ public static final byte[] FAMILY_NAME = Bytes.toBytes("info");
private static final byte[][] FAMILIES = {
- Bytes.add(PerformanceEvaluation.FAMILY_NAME, Bytes.toBytes("-A")),
- Bytes.add(PerformanceEvaluation.FAMILY_NAME, Bytes.toBytes("-B")) };
+ Bytes.add(FAMILY_NAME, Bytes.toBytes("-A")),
+ Bytes.add(FAMILY_NAME, Bytes.toBytes("-B")) };
private static final TableName TABLE_NAME = TableName.valueOf("TestTable");
@Rule
http://git-wip-us.apache.org/repos/asf/hbase/blob/664b6be0/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
deleted file mode 100644
index e1cb8ba..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
+++ /dev/null
@@ -1,1059 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.replication;
-
-import static org.junit.Assert.*;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.NavigableMap;
-import java.util.TreeMap;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.Waiter;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.ConnectionFactory;
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.Table;
-import org.apache.hadoop.hbase.client.replication.ReplicationAdmin;
-import org.apache.hadoop.hbase.mapreduce.replication.VerifyReplication;
-import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl;
-import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
-import org.apache.hadoop.hbase.replication.regionserver.Replication;
-import org.apache.hadoop.hbase.replication.regionserver.ReplicationSource;
-import org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceInterface;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos;
-import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
-import org.apache.hadoop.hbase.testclassification.LargeTests;
-import org.apache.hadoop.hbase.testclassification.ReplicationTests;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
-import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.util.JVMClusterUtil;
-import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
-import org.apache.hadoop.hbase.wal.WAL;
-import org.apache.hadoop.hbase.wal.WALKey;
-import org.apache.hadoop.mapreduce.Job;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-import org.junit.rules.TestName;
-
-import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists;
-
-@Category({ReplicationTests.class, LargeTests.class})
-public class TestReplicationSmallTests extends TestReplicationBase {
-
- private static final Log LOG = LogFactory.getLog(TestReplicationSmallTests.class);
- private static final String PEER_ID = "2";
-
- @Rule
- public TestName name = new TestName();
-
- /**
- * @throws java.lang.Exception
- */
- @Before
- public void setUp() throws Exception {
- // Starting and stopping replication can make us miss new logs,
- // rolling like this makes sure the most recent one gets added to the queue
- for ( JVMClusterUtil.RegionServerThread r :
- utility1.getHBaseCluster().getRegionServerThreads()) {
- utility1.getAdmin().rollWALWriter(r.getRegionServer().getServerName());
- }
- int rowCount = utility1.countRows(tableName);
- utility1.deleteTableData(tableName);
- // truncating the table will send one Delete per row to the slave cluster
- // in an async fashion, which is why we cannot just call deleteTableData on
- // utility2 since late writes could make it to the slave in some way.
- // Instead, we truncate the first table and wait for all the Deletes to
- // make it to the slave.
- Scan scan = new Scan();
- int lastCount = 0;
- for (int i = 0; i < NB_RETRIES; i++) {
- if (i==NB_RETRIES-1) {
- fail("Waited too much time for truncate");
- }
- ResultScanner scanner = htable2.getScanner(scan);
- Result[] res = scanner.next(rowCount);
- scanner.close();
- if (res.length != 0) {
- if (res.length < lastCount) {
- i--; // Don't increment timeout if we make progress
- }
- lastCount = res.length;
- LOG.info("Still got " + res.length + " rows");
- Thread.sleep(SLEEP_TIME);
- } else {
- break;
- }
- }
- }
-
- /**
- * Verify that version and column delete marker types are replicated
- * correctly.
- * @throws Exception
- */
- @Test(timeout=300000)
- public void testDeleteTypes() throws Exception {
- LOG.info("testDeleteTypes");
- final byte[] v1 = Bytes.toBytes("v1");
- final byte[] v2 = Bytes.toBytes("v2");
- final byte[] v3 = Bytes.toBytes("v3");
- htable1 = utility1.getConnection().getTable(tableName);
-
- long t = EnvironmentEdgeManager.currentTime();
- // create three versions for "row"
- Put put = new Put(row);
- put.addColumn(famName, row, t, v1);
- htable1.put(put);
-
- put = new Put(row);
- put.addColumn(famName, row, t + 1, v2);
- htable1.put(put);
-
- put = new Put(row);
- put.addColumn(famName, row, t + 2, v3);
- htable1.put(put);
-
- Get get = new Get(row);
- get.setMaxVersions();
- for (int i = 0; i < NB_RETRIES; i++) {
- if (i==NB_RETRIES-1) {
- fail("Waited too much time for put replication");
- }
- Result res = htable2.get(get);
- if (res.size() < 3) {
- LOG.info("Rows not available");
- Thread.sleep(SLEEP_TIME);
- } else {
- assertArrayEquals(CellUtil.cloneValue(res.rawCells()[0]), v3);
- assertArrayEquals(CellUtil.cloneValue(res.rawCells()[1]), v2);
- assertArrayEquals(CellUtil.cloneValue(res.rawCells()[2]), v1);
- break;
- }
- }
- // place a version delete marker (delete last version)
- Delete d = new Delete(row);
- d.addColumn(famName, row, t);
- htable1.delete(d);
-
- get = new Get(row);
- get.setMaxVersions();
- for (int i = 0; i < NB_RETRIES; i++) {
- if (i==NB_RETRIES-1) {
- fail("Waited too much time for put replication");
- }
- Result res = htable2.get(get);
- if (res.size() > 2) {
- LOG.info("Version not deleted");
- Thread.sleep(SLEEP_TIME);
- } else {
- assertArrayEquals(CellUtil.cloneValue(res.rawCells()[0]), v3);
- assertArrayEquals(CellUtil.cloneValue(res.rawCells()[1]), v2);
- break;
- }
- }
-
- // place a column delete marker
- d = new Delete(row);
- d.addColumns(famName, row, t+2);
- htable1.delete(d);
-
- // now *both* of the remaining version should be deleted
- // at the replica
- get = new Get(row);
- for (int i = 0; i < NB_RETRIES; i++) {
- if (i==NB_RETRIES-1) {
- fail("Waited too much time for del replication");
- }
- Result res = htable2.get(get);
- if (res.size() >= 1) {
- LOG.info("Rows not deleted");
- Thread.sleep(SLEEP_TIME);
- } else {
- break;
- }
- }
- }
-
- /**
- * Add a row, check it's replicated, delete it, check's gone
- * @throws Exception
- */
- @Test(timeout=300000)
- public void testSimplePutDelete() throws Exception {
- LOG.info("testSimplePutDelete");
- Put put = new Put(row);
- put.addColumn(famName, row, row);
-
- htable1 = utility1.getConnection().getTable(tableName);
- htable1.put(put);
-
- Get get = new Get(row);
- for (int i = 0; i < NB_RETRIES; i++) {
- if (i==NB_RETRIES-1) {
- fail("Waited too much time for put replication");
- }
- Result res = htable2.get(get);
- if (res.isEmpty()) {
- LOG.info("Row not available");
- Thread.sleep(SLEEP_TIME);
- } else {
- assertArrayEquals(res.value(), row);
- break;
- }
- }
-
- Delete del = new Delete(row);
- htable1.delete(del);
-
- get = new Get(row);
- for (int i = 0; i < NB_RETRIES; i++) {
- if (i==NB_RETRIES-1) {
- fail("Waited too much time for del replication");
- }
- Result res = htable2.get(get);
- if (res.size() >= 1) {
- LOG.info("Row not deleted");
- Thread.sleep(SLEEP_TIME);
- } else {
- break;
- }
- }
- }
-
- /**
- * Try a small batch upload using the write buffer, check it's replicated
- * @throws Exception
- */
- @Test(timeout=300000)
- public void testSmallBatch() throws Exception {
- LOG.info("testSmallBatch");
- // normal Batch tests
- loadData("", row);
-
- Scan scan = new Scan();
-
- ResultScanner scanner1 = htable1.getScanner(scan);
- Result[] res1 = scanner1.next(NB_ROWS_IN_BATCH);
- scanner1.close();
- assertEquals(NB_ROWS_IN_BATCH, res1.length);
-
- waitForReplication(NB_ROWS_IN_BATCH, NB_RETRIES);
- }
-
- private void waitForReplication(int expectedRows, int retries) throws IOException, InterruptedException {
- Scan scan;
- for (int i = 0; i < retries; i++) {
- scan = new Scan();
- if (i== retries -1) {
- fail("Waited too much time for normal batch replication");
- }
- ResultScanner scanner = htable2.getScanner(scan);
- Result[] res = scanner.next(expectedRows);
- scanner.close();
- if (res.length != expectedRows) {
- LOG.info("Only got " + res.length + " rows");
- Thread.sleep(SLEEP_TIME);
- } else {
- break;
- }
- }
- }
-
- private void loadData(String prefix, byte[] row) throws IOException {
- List<Put> puts = new ArrayList<>(NB_ROWS_IN_BATCH);
- for (int i = 0; i < NB_ROWS_IN_BATCH; i++) {
- Put put = new Put(Bytes.toBytes(prefix + Integer.toString(i)));
- put.addColumn(famName, row, row);
- puts.add(put);
- }
- htable1.put(puts);
- }
-
- /**
- * Test disable/enable replication, trying to insert, make sure nothing's
- * replicated, enable it, the insert should be replicated
- *
- * @throws Exception
- */
- @Test(timeout = 300000)
- public void testDisableEnable() throws Exception {
-
- // Test disabling replication
- admin.disablePeer(PEER_ID);
-
- byte[] rowkey = Bytes.toBytes("disable enable");
- Put put = new Put(rowkey);
- put.addColumn(famName, row, row);
- htable1.put(put);
-
- Get get = new Get(rowkey);
- for (int i = 0; i < NB_RETRIES; i++) {
- Result res = htable2.get(get);
- if (res.size() >= 1) {
- fail("Replication wasn't disabled");
- } else {
- LOG.info("Row not replicated, let's wait a bit more...");
- Thread.sleep(SLEEP_TIME);
- }
- }
-
- // Test enable replication
- admin.enablePeer(PEER_ID);
-
- for (int i = 0; i < NB_RETRIES; i++) {
- Result res = htable2.get(get);
- if (res.isEmpty()) {
- LOG.info("Row not available");
- Thread.sleep(SLEEP_TIME);
- } else {
- assertArrayEquals(res.value(), row);
- return;
- }
- }
- fail("Waited too much time for put replication");
- }
-
- /**
- * Integration test for TestReplicationAdmin, removes and re-add a peer
- * cluster
- *
- * @throws Exception
- */
- @Test(timeout=300000)
- public void testAddAndRemoveClusters() throws Exception {
- LOG.info("testAddAndRemoveClusters");
- admin.removePeer(PEER_ID);
- Thread.sleep(SLEEP_TIME);
- byte[] rowKey = Bytes.toBytes("Won't be replicated");
- Put put = new Put(rowKey);
- put.addColumn(famName, row, row);
- htable1.put(put);
-
- Get get = new Get(rowKey);
- for (int i = 0; i < NB_RETRIES; i++) {
- if (i == NB_RETRIES-1) {
- break;
- }
- Result res = htable2.get(get);
- if (res.size() >= 1) {
- fail("Not supposed to be replicated");
- } else {
- LOG.info("Row not replicated, let's wait a bit more...");
- Thread.sleep(SLEEP_TIME);
- }
- }
- ReplicationPeerConfig rpc = new ReplicationPeerConfig();
- rpc.setClusterKey(utility2.getClusterKey());
- admin.addPeer(PEER_ID, rpc, null);
- Thread.sleep(SLEEP_TIME);
- rowKey = Bytes.toBytes("do rep");
- put = new Put(rowKey);
- put.addColumn(famName, row, row);
- LOG.info("Adding new row");
- htable1.put(put);
-
- get = new Get(rowKey);
- for (int i = 0; i < NB_RETRIES; i++) {
- if (i==NB_RETRIES-1) {
- fail("Waited too much time for put replication");
- }
- Result res = htable2.get(get);
- if (res.isEmpty()) {
- LOG.info("Row not available");
- Thread.sleep(SLEEP_TIME*i);
- } else {
- assertArrayEquals(res.value(), row);
- break;
- }
- }
- }
-
-
- /**
- * Do a more intense version testSmallBatch, one that will trigger
- * wal rolling and other non-trivial code paths
- * @throws Exception
- */
- @Test(timeout=300000)
- public void testLoading() throws Exception {
- LOG.info("Writing out rows to table1 in testLoading");
- List<Put> puts = new ArrayList<>(NB_ROWS_IN_BIG_BATCH);
- for (int i = 0; i < NB_ROWS_IN_BIG_BATCH; i++) {
- Put put = new Put(Bytes.toBytes(i));
- put.addColumn(famName, row, row);
- puts.add(put);
- }
- // The puts will be iterated through and flushed only when the buffer
- // size is reached.
- htable1.put(puts);
-
- Scan scan = new Scan();
-
- ResultScanner scanner = htable1.getScanner(scan);
- Result[] res = scanner.next(NB_ROWS_IN_BIG_BATCH);
- scanner.close();
-
- assertEquals(NB_ROWS_IN_BIG_BATCH, res.length);
-
- LOG.info("Looking in table2 for replicated rows in testLoading");
- long start = System.currentTimeMillis();
- // Retry more than NB_RETRIES. As it was, retries were done in 5 seconds and we'd fail
- // sometimes.
- final long retries = NB_RETRIES * 10;
- for (int i = 0; i < retries; i++) {
- scan = new Scan();
- scanner = htable2.getScanner(scan);
- res = scanner.next(NB_ROWS_IN_BIG_BATCH);
- scanner.close();
- if (res.length != NB_ROWS_IN_BIG_BATCH) {
- if (i == retries - 1) {
- int lastRow = -1;
- for (Result result : res) {
- int currentRow = Bytes.toInt(result.getRow());
- for (int row = lastRow+1; row < currentRow; row++) {
- LOG.error("Row missing: " + row);
- }
- lastRow = currentRow;
- }
- LOG.error("Last row: " + lastRow);
- fail("Waited too much time for normal batch replication, " +
- res.length + " instead of " + NB_ROWS_IN_BIG_BATCH + "; waited=" +
- (System.currentTimeMillis() - start) + "ms");
- } else {
- LOG.info("Only got " + res.length + " rows... retrying");
- Thread.sleep(SLEEP_TIME);
- }
- } else {
- break;
- }
- }
- }
-
- /**
- * Do a small loading into a table, make sure the data is really the same,
- * then run the VerifyReplication job to check the results. Do a second
- * comparison where all the cells are different.
- * @throws Exception
- */
- @Test(timeout=300000)
- public void testVerifyRepJob() throws Exception {
- // Populate the tables, at the same time it guarantees that the tables are
- // identical since it does the check
- testSmallBatch();
-
- String[] args = new String[] {PEER_ID, tableName.getNameAsString()};
- runVerifyReplication(args, NB_ROWS_IN_BATCH, 0);
-
- Scan scan = new Scan();
- ResultScanner rs = htable2.getScanner(scan);
- Put put = null;
- for (Result result : rs) {
- put = new Put(result.getRow());
- Cell firstVal = result.rawCells()[0];
- put.addColumn(CellUtil.cloneFamily(firstVal), CellUtil.cloneQualifier(firstVal),
- Bytes.toBytes("diff data"));
- htable2.put(put);
- }
- Delete delete = new Delete(put.getRow());
- htable2.delete(delete);
- runVerifyReplication(args, 0, NB_ROWS_IN_BATCH);
- }
-
- /**
- * Load a row into a table, make sure the data is really the same,
- * delete the row, make sure the delete marker is replicated,
- * run verify replication with and without raw to check the results.
- * @throws Exception
- */
- @Test(timeout=300000)
- public void testVerifyRepJobWithRawOptions() throws Exception {
- LOG.info(name.getMethodName());
-
- final TableName tableName = TableName.valueOf(name.getMethodName());
- byte[] familyname = Bytes.toBytes("fam_raw");
- byte[] row = Bytes.toBytes("row_raw");
-
- Table lHtable1 = null;
- Table lHtable2 = null;
-
- try {
- HTableDescriptor table = new HTableDescriptor(tableName);
- HColumnDescriptor fam = new HColumnDescriptor(familyname);
- fam.setMaxVersions(100);
- fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
- table.addFamily(fam);
- scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
- for (HColumnDescriptor f : table.getColumnFamilies()) {
- scopes.put(f.getName(), f.getScope());
- }
-
- Connection connection1 = ConnectionFactory.createConnection(conf1);
- Connection connection2 = ConnectionFactory.createConnection(conf2);
- try (Admin admin1 = connection1.getAdmin()) {
- admin1.createTable(table, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE);
- }
- try (Admin admin2 = connection2.getAdmin()) {
- admin2.createTable(table, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE);
- }
- utility1.waitUntilAllRegionsAssigned(tableName);
- utility2.waitUntilAllRegionsAssigned(tableName);
-
- lHtable1 = utility1.getConnection().getTable(tableName);
- lHtable2 = utility2.getConnection().getTable(tableName);
-
- Put put = new Put(row);
- put.addColumn(familyname, row, row);
- lHtable1.put(put);
-
- Get get = new Get(row);
- for (int i = 0; i < NB_RETRIES; i++) {
- if (i==NB_RETRIES-1) {
- fail("Waited too much time for put replication");
- }
- Result res = lHtable2.get(get);
- if (res.isEmpty()) {
- LOG.info("Row not available");
- Thread.sleep(SLEEP_TIME);
- } else {
- assertArrayEquals(res.value(), row);
- break;
- }
- }
-
- Delete del = new Delete(row);
- lHtable1.delete(del);
-
- get = new Get(row);
- for (int i = 0; i < NB_RETRIES; i++) {
- if (i==NB_RETRIES-1) {
- fail("Waited too much time for del replication");
- }
- Result res = lHtable2.get(get);
- if (res.size() >= 1) {
- LOG.info("Row not deleted");
- Thread.sleep(SLEEP_TIME);
- } else {
- break;
- }
- }
-
- // Checking verifyReplication for the default behavior.
- String[] argsWithoutRaw = new String[] {PEER_ID, tableName.getNameAsString()};
- runVerifyReplication(argsWithoutRaw, 0, 0);
-
- // Checking verifyReplication with raw
- String[] argsWithRawAsTrue = new String[] {"--raw", PEER_ID, tableName.getNameAsString()};
- runVerifyReplication(argsWithRawAsTrue, 1, 0);
- } finally {
- if (lHtable1 != null) {
- lHtable1.close();
- }
- if (lHtable2 != null) {
- lHtable2.close();
- }
- }
- }
-
- private void runVerifyReplication(String[] args, int expectedGoodRows, int expectedBadRows)
- throws IOException, InterruptedException, ClassNotFoundException {
- Job job = new VerifyReplication().createSubmittableJob(new Configuration(conf1), args);
- if (job == null) {
- fail("Job wasn't created, see the log");
- }
- if (!job.waitForCompletion(true)) {
- fail("Job failed, see the log");
- }
- assertEquals(expectedGoodRows, job.getCounters().
- findCounter(VerifyReplication.Verifier.Counters.GOODROWS).getValue());
- assertEquals(expectedBadRows, job.getCounters().
- findCounter(VerifyReplication.Verifier.Counters.BADROWS).getValue());
- }
-
- @Test(timeout=300000)
- // VerifyReplication should honor versions option
- public void testHBase14905() throws Exception {
- // normal Batch tests
- byte[] qualifierName = Bytes.toBytes("f1");
- Put put = new Put(Bytes.toBytes("r1"));
- put.addColumn(famName, qualifierName, Bytes.toBytes("v1002"));
- htable1.put(put);
- put.addColumn(famName, qualifierName, Bytes.toBytes("v1001"));
- htable1.put(put);
- put.addColumn(famName, qualifierName, Bytes.toBytes("v1112"));
- htable1.put(put);
-
- Scan scan = new Scan();
- scan.setMaxVersions(100);
- ResultScanner scanner1 = htable1.getScanner(scan);
- Result[] res1 = scanner1.next(1);
- scanner1.close();
-
- assertEquals(1, res1.length);
- assertEquals(3, res1[0].getColumnCells(famName, qualifierName).size());
-
- for (int i = 0; i < NB_RETRIES; i++) {
- scan = new Scan();
- scan.setMaxVersions(100);
- scanner1 = htable2.getScanner(scan);
- res1 = scanner1.next(1);
- scanner1.close();
- if (res1.length != 1) {
- LOG.info("Only got " + res1.length + " rows");
- Thread.sleep(SLEEP_TIME);
- } else {
- int cellNumber = res1[0].getColumnCells(famName, Bytes.toBytes("f1")).size();
- if (cellNumber != 3) {
- LOG.info("Only got " + cellNumber + " cells");
- Thread.sleep(SLEEP_TIME);
- } else {
- break;
- }
- }
- if (i == NB_RETRIES-1) {
- fail("Waited too much time for normal batch replication");
- }
- }
-
- put.addColumn(famName, qualifierName, Bytes.toBytes("v1111"));
- htable2.put(put);
- put.addColumn(famName, qualifierName, Bytes.toBytes("v1112"));
- htable2.put(put);
-
- scan = new Scan();
- scan.setMaxVersions(100);
- scanner1 = htable2.getScanner(scan);
- res1 = scanner1.next(NB_ROWS_IN_BATCH);
- scanner1.close();
-
- assertEquals(1, res1.length);
- assertEquals(5, res1[0].getColumnCells(famName, qualifierName).size());
-
- String[] args = new String[] {"--versions=100", PEER_ID, tableName.getNameAsString()};
- runVerifyReplication(args, 0, 1);
- }
-
- @Test(timeout=300000)
- // VerifyReplication should honor versions option
- public void testVersionMismatchHBase14905() throws Exception {
- // normal Batch tests
- byte[] qualifierName = Bytes.toBytes("f1");
- Put put = new Put(Bytes.toBytes("r1"));
- long ts = System.currentTimeMillis();
- put.addColumn(famName, qualifierName, ts + 1, Bytes.toBytes("v1"));
- htable1.put(put);
- put.addColumn(famName, qualifierName, ts + 2, Bytes.toBytes("v2"));
- htable1.put(put);
- put.addColumn(famName, qualifierName, ts + 3, Bytes.toBytes("v3"));
- htable1.put(put);
-
- Scan scan = new Scan();
- scan.setMaxVersions(100);
- ResultScanner scanner1 = htable1.getScanner(scan);
- Result[] res1 = scanner1.next(1);
- scanner1.close();
-
- assertEquals(1, res1.length);
- assertEquals(3, res1[0].getColumnCells(famName, qualifierName).size());
-
- for (int i = 0; i < NB_RETRIES; i++) {
- scan = new Scan();
- scan.setMaxVersions(100);
- scanner1 = htable2.getScanner(scan);
- res1 = scanner1.next(1);
- scanner1.close();
- if (res1.length != 1) {
- LOG.info("Only got " + res1.length + " rows");
- Thread.sleep(SLEEP_TIME);
- } else {
- int cellNumber = res1[0].getColumnCells(famName, Bytes.toBytes("f1")).size();
- if (cellNumber != 3) {
- LOG.info("Only got " + cellNumber + " cells");
- Thread.sleep(SLEEP_TIME);
- } else {
- break;
- }
- }
- if (i == NB_RETRIES-1) {
- fail("Waited too much time for normal batch replication");
- }
- }
-
- try {
- // Disabling replication and modifying the particular version of the cell to validate the feature.
- admin.disablePeer(PEER_ID);
- Put put2 = new Put(Bytes.toBytes("r1"));
- put2.addColumn(famName, qualifierName, ts +2, Bytes.toBytes("v99"));
- htable2.put(put2);
-
- scan = new Scan();
- scan.setMaxVersions(100);
- scanner1 = htable2.getScanner(scan);
- res1 = scanner1.next(NB_ROWS_IN_BATCH);
- scanner1.close();
- assertEquals(1, res1.length);
- assertEquals(3, res1[0].getColumnCells(famName, qualifierName).size());
-
- String[] args = new String[] {"--versions=100", PEER_ID, tableName.getNameAsString()};
- runVerifyReplication(args, 0, 1);
- }
- finally {
- admin.enablePeer(PEER_ID);
- }
- }
-
- /**
- * Test for HBASE-9038, Replication.scopeWALEdits would NPE if it wasn't filtering out
- * the compaction WALEdit
- * @throws Exception
- */
- @Test(timeout=300000)
- public void testCompactionWALEdits() throws Exception {
- WALProtos.CompactionDescriptor compactionDescriptor =
- WALProtos.CompactionDescriptor.getDefaultInstance();
- HRegionInfo hri = new HRegionInfo(htable1.getName(),
- HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
- WALEdit edit = WALEdit.createCompaction(hri, compactionDescriptor);
- Replication.scopeWALEdits(new WALKey(), edit,
- htable1.getConfiguration(), null);
- }
-
- /**
- * Test for HBASE-8663
- * Create two new Tables with colfamilies enabled for replication then run
- * ReplicationAdmin.listReplicated(). Finally verify the table:colfamilies. Note:
- * TestReplicationAdmin is a better place for this testing but it would need mocks.
- * @throws Exception
- */
- @Test(timeout = 300000)
- public void testVerifyListReplicatedTable() throws Exception {
- LOG.info("testVerifyListReplicatedTable");
-
- final String tName = "VerifyListReplicated_";
- final String colFam = "cf1";
- final int numOfTables = 3;
-
- Admin hadmin = utility1.getAdmin();
-
- // Create Tables
- for (int i = 0; i < numOfTables; i++) {
- HTableDescriptor ht = new HTableDescriptor(TableName.valueOf(tName + i));
- HColumnDescriptor cfd = new HColumnDescriptor(colFam);
- cfd.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
- ht.addFamily(cfd);
- hadmin.createTable(ht);
- }
-
- // verify the result
- List<HashMap<String, String>> replicationColFams = admin.listReplicated();
- int[] match = new int[numOfTables]; // array of 3 with init value of zero
-
- for (int i = 0; i < replicationColFams.size(); i++) {
- HashMap<String, String> replicationEntry = replicationColFams.get(i);
- String tn = replicationEntry.get(ReplicationAdmin.TNAME);
- if ((tn.startsWith(tName)) && replicationEntry.get(ReplicationAdmin.CFNAME).equals(colFam)) {
- int m = Integer.parseInt(tn.substring(tn.length() - 1)); // get the last digit
- match[m]++; // should only increase once
- }
- }
-
- // check the matching result
- for (int i = 0; i < match.length; i++) {
- assertTrue("listReplicated() does not match table " + i, (match[i] == 1));
- }
-
- // drop tables
- for (int i = 0; i < numOfTables; i++) {
- TableName tableName = TableName.valueOf(tName + i);
- hadmin.disableTable(tableName);
- hadmin.deleteTable(tableName);
- }
-
- hadmin.close();
- }
-
- /**
- * Test for HBase-15259 WALEdits under replay will also be replicated
- * */
- @Test
- public void testReplicationInReplay() throws Exception {
- final TableName tableName = htable1.getName();
-
- HRegion region = utility1.getMiniHBaseCluster().getRegions(tableName).get(0);
- HRegionInfo hri = region.getRegionInfo();
- NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
- for (byte[] fam : htable1.getTableDescriptor().getFamiliesKeys()) {
- scopes.put(fam, 1);
- }
- final MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
- int index = utility1.getMiniHBaseCluster().getServerWith(hri.getRegionName());
- WAL wal = utility1.getMiniHBaseCluster().getRegionServer(index).getWAL(region.getRegionInfo());
- final byte[] rowName = Bytes.toBytes("testReplicationInReplay");
- final byte[] qualifier = Bytes.toBytes("q");
- final byte[] value = Bytes.toBytes("v");
- WALEdit edit = new WALEdit(true);
- long now = EnvironmentEdgeManager.currentTime();
- edit.add(new KeyValue(rowName, famName, qualifier,
- now, value));
- WALKey walKey = new WALKey(hri.getEncodedNameAsBytes(), tableName, now, mvcc, scopes);
- wal.append(hri, walKey, edit, true);
- wal.sync();
-
- Get get = new Get(rowName);
- for (int i = 0; i < NB_RETRIES; i++) {
- if (i == NB_RETRIES-1) {
- break;
- }
- Result res = htable2.get(get);
- if (res.size() >= 1) {
- fail("Not supposed to be replicated for " + Bytes.toString(res.getRow()));
- } else {
- LOG.info("Row not replicated, let's wait a bit more...");
- Thread.sleep(SLEEP_TIME);
- }
- }
- }
-
- @Test(timeout=300000)
- public void testVerifyReplicationPrefixFiltering() throws Exception {
- final byte[] prefixRow = Bytes.toBytes("prefixrow");
- final byte[] prefixRow2 = Bytes.toBytes("secondrow");
- loadData("prefixrow", prefixRow);
- loadData("secondrow", prefixRow2);
- loadData("aaa", row);
- loadData("zzz", row);
- waitForReplication(NB_ROWS_IN_BATCH * 4, NB_RETRIES * 4);
- String[] args = new String[] {"--row-prefixes=prefixrow,secondrow", PEER_ID,
- tableName.getNameAsString()};
- runVerifyReplication(args, NB_ROWS_IN_BATCH *2, 0);
- }
-
- @Test(timeout = 300000)
- public void testVerifyReplicationSnapshotArguments() {
- String[] args =
- new String[] { "--sourceSnapshotName=snapshot1", "2", tableName.getNameAsString() };
- assertFalse(Lists.newArrayList(args).toString(), new VerifyReplication().doCommandLine(args));
-
- args = new String[] { "--sourceSnapshotTmpDir=tmp", "2", tableName.getNameAsString() };
- assertFalse(Lists.newArrayList(args).toString(), new VerifyReplication().doCommandLine(args));
-
- args = new String[] { "--sourceSnapshotName=snapshot1", "--sourceSnapshotTmpDir=tmp", "2",
- tableName.getNameAsString() };
- assertTrue(Lists.newArrayList(args).toString(), new VerifyReplication().doCommandLine(args));
-
- args = new String[] { "--peerSnapshotName=snapshot1", "2", tableName.getNameAsString() };
- assertFalse(Lists.newArrayList(args).toString(), new VerifyReplication().doCommandLine(args));
-
- args = new String[] { "--peerSnapshotTmpDir=/tmp/", "2", tableName.getNameAsString() };
- assertFalse(Lists.newArrayList(args).toString(), new VerifyReplication().doCommandLine(args));
-
- args = new String[] { "--peerSnapshotName=snapshot1", "--peerSnapshotTmpDir=/tmp/",
- "--peerFSAddress=tempfs", "--peerHBaseRootAddress=hdfs://tempfs:50070/hbase/", "2",
- tableName.getNameAsString() };
- assertTrue(Lists.newArrayList(args).toString(), new VerifyReplication().doCommandLine(args));
-
- args = new String[] { "--sourceSnapshotName=snapshot1", "--sourceSnapshotTmpDir=/tmp/",
- "--peerSnapshotName=snapshot2", "--peerSnapshotTmpDir=/tmp/", "--peerFSAddress=tempfs",
- "--peerHBaseRootAddress=hdfs://tempfs:50070/hbase/", "2", tableName.getNameAsString() };
-
- assertTrue(Lists.newArrayList(args).toString(), new VerifyReplication().doCommandLine(args));
- }
-
- @Test(timeout = 300000)
- public void testVerifyReplicationWithSnapshotSupport() throws Exception {
- // Populate the tables, at the same time it guarantees that the tables are
- // identical since it does the check
- testSmallBatch();
-
- // Take source and target tables snapshot
- Path rootDir = FSUtils.getRootDir(conf1);
- FileSystem fs = rootDir.getFileSystem(conf1);
- String sourceSnapshotName = "sourceSnapshot-" + System.currentTimeMillis();
- SnapshotTestingUtils.createSnapshotAndValidate(utility1.getHBaseAdmin(), tableName,
- new String(famName), sourceSnapshotName, rootDir, fs, true);
-
- // Take target snapshot
- Path peerRootDir = FSUtils.getRootDir(conf2);
- FileSystem peerFs = peerRootDir.getFileSystem(conf2);
- String peerSnapshotName = "peerSnapshot-" + System.currentTimeMillis();
- SnapshotTestingUtils.createSnapshotAndValidate(utility2.getHBaseAdmin(), tableName,
- new String(famName), peerSnapshotName, peerRootDir, peerFs, true);
-
- String peerFSAddress = peerFs.getUri().toString();
- String temPath1 = utility1.getRandomDir().toString();
- String temPath2 = "/tmp2";
-
- String[] args = new String[] { "--sourceSnapshotName=" + sourceSnapshotName,
- "--sourceSnapshotTmpDir=" + temPath1, "--peerSnapshotName=" + peerSnapshotName,
- "--peerSnapshotTmpDir=" + temPath2, "--peerFSAddress=" + peerFSAddress,
- "--peerHBaseRootAddress=" + FSUtils.getRootDir(conf2), "2", tableName.getNameAsString() };
-
- Job job = new VerifyReplication().createSubmittableJob(conf1, args);
- if (job == null) {
- fail("Job wasn't created, see the log");
- }
- if (!job.waitForCompletion(true)) {
- fail("Job failed, see the log");
- }
- assertEquals(NB_ROWS_IN_BATCH,
- job.getCounters().findCounter(VerifyReplication.Verifier.Counters.GOODROWS).getValue());
- assertEquals(0,
- job.getCounters().findCounter(VerifyReplication.Verifier.Counters.BADROWS).getValue());
-
- Scan scan = new Scan();
- ResultScanner rs = htable2.getScanner(scan);
- Put put = null;
- for (Result result : rs) {
- put = new Put(result.getRow());
- Cell firstVal = result.rawCells()[0];
- put.addColumn(CellUtil.cloneFamily(firstVal), CellUtil.cloneQualifier(firstVal),
- Bytes.toBytes("diff data"));
- htable2.put(put);
- }
- Delete delete = new Delete(put.getRow());
- htable2.delete(delete);
-
- sourceSnapshotName = "sourceSnapshot-" + System.currentTimeMillis();
- SnapshotTestingUtils.createSnapshotAndValidate(utility1.getHBaseAdmin(), tableName,
- new String(famName), sourceSnapshotName, rootDir, fs, true);
-
- peerSnapshotName = "peerSnapshot-" + System.currentTimeMillis();
- SnapshotTestingUtils.createSnapshotAndValidate(utility2.getHBaseAdmin(), tableName,
- new String(famName), peerSnapshotName, peerRootDir, peerFs, true);
-
- args = new String[] { "--sourceSnapshotName=" + sourceSnapshotName,
- "--sourceSnapshotTmpDir=" + temPath1, "--peerSnapshotName=" + peerSnapshotName,
- "--peerSnapshotTmpDir=" + temPath2, "--peerFSAddress=" + peerFSAddress,
- "--peerHBaseRootAddress=" + FSUtils.getRootDir(conf2), "2", tableName.getNameAsString() };
-
- job = new VerifyReplication().createSubmittableJob(conf1, args);
- if (job == null) {
- fail("Job wasn't created, see the log");
- }
- if (!job.waitForCompletion(true)) {
- fail("Job failed, see the log");
- }
- assertEquals(0,
- job.getCounters().findCounter(VerifyReplication.Verifier.Counters.GOODROWS).getValue());
- assertEquals(NB_ROWS_IN_BATCH,
- job.getCounters().findCounter(VerifyReplication.Verifier.Counters.BADROWS).getValue());
- }
-
- @Test
- public void testEmptyWALRecovery() throws Exception {
- final int numRs = utility1.getHBaseCluster().getRegionServerThreads().size();
-
- // for each RS, create an empty wal with same walGroupId
- final List<Path> emptyWalPaths = new ArrayList<>();
- long ts = System.currentTimeMillis();
- for (int i = 0; i < numRs; i++) {
- HRegionInfo regionInfo =
- utility1.getHBaseCluster().getRegions(htable1.getName()).get(0).getRegionInfo();
- WAL wal = utility1.getHBaseCluster().getRegionServer(i).getWAL(regionInfo);
- Path currentWalPath = AbstractFSWALProvider.getCurrentFileName(wal);
- String walGroupId = AbstractFSWALProvider.getWALPrefixFromWALName(currentWalPath.getName());
- Path emptyWalPath = new Path(utility1.getDataTestDir(), walGroupId + "." + ts);
- utility1.getTestFileSystem().create(emptyWalPath).close();
- emptyWalPaths.add(emptyWalPath);
- }
-
- // inject our empty wal into the replication queue
- for (int i = 0; i < numRs; i++) {
- Replication replicationService =
- (Replication) utility1.getHBaseCluster().getRegionServer(i).getReplicationSourceService();
- replicationService.preLogRoll(null, emptyWalPaths.get(i));
- replicationService.postLogRoll(null, emptyWalPaths.get(i));
- }
-
- // wait for ReplicationSource to start reading from our empty wal
- waitForLogAdvance(numRs, emptyWalPaths, false);
-
- // roll the original wal, which enqueues a new wal behind our empty wal
- for (int i = 0; i < numRs; i++) {
- HRegionInfo regionInfo =
- utility1.getHBaseCluster().getRegions(htable1.getName()).get(0).getRegionInfo();
- WAL wal = utility1.getHBaseCluster().getRegionServer(i).getWAL(regionInfo);
- wal.rollWriter(true);
- }
-
- // ReplicationSource should advance past the empty wal, or else the test will fail
- waitForLogAdvance(numRs, emptyWalPaths, true);
-
- // we're now writing to the new wal
- // if everything works, the source should've stopped reading from the empty wal, and start
- // replicating from the new wal
- testSimplePutDelete();
- }
-
- /**
- * Waits for the ReplicationSource to start reading from the given paths
- * @param numRs number of regionservers
- * @param emptyWalPaths path for each regionserver
- * @param invert if true, waits until ReplicationSource is NOT reading from the given paths
- */
- private void waitForLogAdvance(final int numRs, final List<Path> emptyWalPaths,
- final boolean invert) throws Exception {
- Waiter.waitFor(conf1, 10000, new Waiter.Predicate<Exception>() {
- @Override
- public boolean evaluate() throws Exception {
- for (int i = 0; i < numRs; i++) {
- Replication replicationService = (Replication) utility1.getHBaseCluster()
- .getRegionServer(i).getReplicationSourceService();
- for (ReplicationSourceInterface rsi : replicationService.getReplicationManager()
- .getSources()) {
- ReplicationSource source = (ReplicationSource) rsi;
- if (!invert && !emptyWalPaths.get(i).equals(source.getCurrentPath())) {
- return false;
- }
- if (invert && emptyWalPaths.get(i).equals(source.getCurrentPath())) {
- return false;
- }
- }
- }
- return true;
- }
- });
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/664b6be0/hbase-server/src/test/java/org/apache/hadoop/hbase/security/HBaseKerberosUtils.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/HBaseKerberosUtils.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/HBaseKerberosUtils.java
index 07bb2b7..94991e1 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/HBaseKerberosUtils.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/HBaseKerberosUtils.java
@@ -17,15 +17,22 @@
*/
package org.apache.hadoop.hbase.security;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
-
import org.apache.hadoop.hbase.shaded.com.google.common.base.Strings;
+import org.apache.hadoop.security.UserGroupInformation;
+
+import java.io.IOException;
+import java.net.InetAddress;
@InterfaceAudience.Private
public class HBaseKerberosUtils {
+ private static final Log LOG = LogFactory.getLog(HBaseKerberosUtils.class);
+
public static final String KRB_PRINCIPAL = "hbase.regionserver.kerberos.principal";
public static final String MASTER_KRB_PRINCIPAL = "hbase.master.kerberos.principal";
public static final String KRB_KEYTAB_FILE = "hbase.regionserver.keytab.file";
@@ -81,4 +88,21 @@ public class HBaseKerberosUtils {
conf.set(KRB_PRINCIPAL, System.getProperty(KRB_PRINCIPAL));
conf.set(MASTER_KRB_PRINCIPAL, System.getProperty(KRB_PRINCIPAL));
}
+
+ public static UserGroupInformation loginAndReturnUGI(Configuration conf, String username)
+ throws IOException {
+ String hostname = InetAddress.getLocalHost().getHostName();
+ String keyTabFileConfKey = "hbase." + username + ".keytab.file";
+ String keyTabFileLocation = conf.get(keyTabFileConfKey);
+ String principalConfKey = "hbase." + username + ".kerberos.principal";
+ String principal = org.apache.hadoop.security.SecurityUtil
+ .getServerPrincipal(conf.get(principalConfKey), hostname);
+ if (keyTabFileLocation == null || principal == null) {
+ LOG.warn("Principal or key tab file null for : " + principalConfKey + ", "
+ + keyTabFileConfKey);
+ }
+ UserGroupInformation ugi =
+ UserGroupInformation.loginUserFromKeytabAndReturnUGI(principal, keyTabFileLocation);
+ return ugi;
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/664b6be0/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
deleted file mode 100644
index 2e3cb5e..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
+++ /dev/null
@@ -1,381 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.snapshot;
-
-import static org.apache.hadoop.util.ToolRunner.run;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-import java.net.URI;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.CategoryBasedTimeout;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
-import org.apache.hadoop.hbase.testclassification.LargeTests;
-import org.apache.hadoop.hbase.testclassification.VerySlowMapReduceTests;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSUtils;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.Ignore;
-import org.junit.experimental.categories.Category;
-import org.junit.rules.TestName;
-import org.junit.rules.TestRule;
-
-/**
- * Test Export Snapshot Tool
- */
-@Ignore
-@Category({VerySlowMapReduceTests.class, LargeTests.class})
-public class TestExportSnapshot {
- @Rule public final TestRule timeout = CategoryBasedTimeout.builder().
- withTimeout(this.getClass()).withLookingForStuckThread(true).build();
- private static final Log LOG = LogFactory.getLog(TestExportSnapshot.class);
-
- protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
-
- protected final static byte[] FAMILY = Bytes.toBytes("cf");
-
- @Rule
- public final TestName testName = new TestName();
-
- protected TableName tableName;
- private byte[] emptySnapshotName;
- private byte[] snapshotName;
- private int tableNumFiles;
- private Admin admin;
-
- public static void setUpBaseConf(Configuration conf) {
- conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
- conf.setInt("hbase.regionserver.msginterval", 100);
- conf.setInt("hbase.client.pause", 250);
- conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6);
- conf.setBoolean("hbase.master.enabletable.roundrobin", true);
- conf.setInt("mapreduce.map.maxattempts", 10);
- // If a single node has enough failures (default 3), resource manager will blacklist it.
- // With only 2 nodes and tests injecting faults, we don't want that.
- conf.setInt("mapreduce.job.maxtaskfailures.per.tracker", 100);
- }
-
- @BeforeClass
- public static void setUpBeforeClass() throws Exception {
- setUpBaseConf(TEST_UTIL.getConfiguration());
- TEST_UTIL.startMiniCluster(1, 3);
- TEST_UTIL.startMiniMapReduceCluster();
- }
-
- @AfterClass
- public static void tearDownAfterClass() throws Exception {
- TEST_UTIL.shutdownMiniMapReduceCluster();
- TEST_UTIL.shutdownMiniCluster();
- }
-
- /**
- * Create a table and take a snapshot of the table used by the export test.
- */
- @Before
- public void setUp() throws Exception {
- this.admin = TEST_UTIL.getAdmin();
-
- tableName = TableName.valueOf("testtb-" + testName.getMethodName());
- snapshotName = Bytes.toBytes("snaptb0-" + testName.getMethodName());
- emptySnapshotName = Bytes.toBytes("emptySnaptb0-" + testName.getMethodName());
-
- // create Table
- createTable();
-
- // Take an empty snapshot
- admin.snapshot(emptySnapshotName, tableName);
-
- // Add some rows
- SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 50, FAMILY);
- tableNumFiles = admin.getTableRegions(tableName).size();
-
- // take a snapshot
- admin.snapshot(snapshotName, tableName);
- }
-
- protected void createTable() throws Exception {
- SnapshotTestingUtils.createPreSplitTable(TEST_UTIL, tableName, 2, FAMILY);
- }
-
- protected interface RegionPredicate {
- boolean evaluate(final HRegionInfo regionInfo);
- }
-
- protected RegionPredicate getBypassRegionPredicate() {
- return null;
- }
-
- @After
- public void tearDown() throws Exception {
- TEST_UTIL.deleteTable(tableName);
- SnapshotTestingUtils.deleteAllSnapshots(TEST_UTIL.getAdmin());
- SnapshotTestingUtils.deleteArchiveDirectory(TEST_UTIL);
- }
-
- /**
- * Verify if exported snapshot and copied files matches the original one.
- */
- @Test
- public void testExportFileSystemState() throws Exception {
- testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles);
- }
-
- @Test
- public void testExportFileSystemStateWithSkipTmp() throws Exception {
- TEST_UTIL.getConfiguration().setBoolean(ExportSnapshot.CONF_SKIP_TMP, true);
- try {
- testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles);
- } finally {
- TEST_UTIL.getConfiguration().setBoolean(ExportSnapshot.CONF_SKIP_TMP, false);
- }
- }
-
- @Test
- public void testEmptyExportFileSystemState() throws Exception {
- testExportFileSystemState(tableName, emptySnapshotName, emptySnapshotName, 0);
- }
-
- @Test
- public void testConsecutiveExports() throws Exception {
- Path copyDir = getLocalDestinationDir();
- testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles, copyDir, false);
- testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles, copyDir, true);
- removeExportDir(copyDir);
- }
-
- @Test
- public void testExportWithTargetName() throws Exception {
- final byte[] targetName = Bytes.toBytes("testExportWithTargetName");
- testExportFileSystemState(tableName, snapshotName, targetName, tableNumFiles);
- }
-
- private void testExportFileSystemState(final TableName tableName, final byte[] snapshotName,
- final byte[] targetName, int filesExpected) throws Exception {
- testExportFileSystemState(tableName, snapshotName, targetName,
- filesExpected, getHdfsDestinationDir(), false);
- }
-
- protected void testExportFileSystemState(final TableName tableName,
- final byte[] snapshotName, final byte[] targetName, int filesExpected,
- Path copyDir, boolean overwrite) throws Exception {
- testExportFileSystemState(TEST_UTIL.getConfiguration(), tableName, snapshotName, targetName,
- filesExpected, TEST_UTIL.getDefaultRootDirPath(), copyDir,
- overwrite, getBypassRegionPredicate(), true);
- }
-
- /**
- * Creates destination directory, runs ExportSnapshot() tool, and runs some verifications.
- */
- protected static void testExportFileSystemState(final Configuration conf, final TableName tableName,
- final byte[] snapshotName, final byte[] targetName, final int filesExpected,
- final Path sourceDir, Path copyDir, final boolean overwrite,
- final RegionPredicate bypassregionPredicate, boolean success) throws Exception {
- URI hdfsUri = FileSystem.get(conf).getUri();
- FileSystem fs = FileSystem.get(copyDir.toUri(), new Configuration());
- copyDir = copyDir.makeQualified(fs);
-
- List<String> opts = new ArrayList<>();
- opts.add("--snapshot");
- opts.add(Bytes.toString(snapshotName));
- opts.add("--copy-to");
- opts.add(copyDir.toString());
- if (targetName != snapshotName) {
- opts.add("--target");
- opts.add(Bytes.toString(targetName));
- }
- if (overwrite) opts.add("--overwrite");
-
- // Export Snapshot
- int res = run(conf, new ExportSnapshot(), opts.toArray(new String[opts.size()]));
- assertEquals(success ? 0 : 1, res);
- if (!success) {
- final Path targetDir = new Path(HConstants.SNAPSHOT_DIR_NAME, Bytes.toString(targetName));
- assertFalse(fs.exists(new Path(copyDir, targetDir)));
- return;
- }
-
- // Verify File-System state
- FileStatus[] rootFiles = fs.listStatus(copyDir);
- assertEquals(filesExpected > 0 ? 2 : 1, rootFiles.length);
- for (FileStatus fileStatus: rootFiles) {
- String name = fileStatus.getPath().getName();
- assertTrue(fileStatus.isDirectory());
- assertTrue(name.equals(HConstants.SNAPSHOT_DIR_NAME) ||
- name.equals(HConstants.HFILE_ARCHIVE_DIRECTORY));
- }
-
- // compare the snapshot metadata and verify the hfiles
- final FileSystem hdfs = FileSystem.get(hdfsUri, conf);
- final Path snapshotDir = new Path(HConstants.SNAPSHOT_DIR_NAME, Bytes.toString(snapshotName));
- final Path targetDir = new Path(HConstants.SNAPSHOT_DIR_NAME, Bytes.toString(targetName));
- verifySnapshotDir(hdfs, new Path(sourceDir, snapshotDir),
- fs, new Path(copyDir, targetDir));
- Set<String> snapshotFiles = verifySnapshot(conf, fs, copyDir, tableName,
- Bytes.toString(targetName), bypassregionPredicate);
- assertEquals(filesExpected, snapshotFiles.size());
- }
-
- /**
- * Check that ExportSnapshot will succeed if something fails but the retry succeed.
- */
- @Test
- public void testExportRetry() throws Exception {
- Path copyDir = getLocalDestinationDir();
- FileSystem fs = FileSystem.get(copyDir.toUri(), new Configuration());
- copyDir = copyDir.makeQualified(fs);
- Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
- conf.setBoolean(ExportSnapshot.Testing.CONF_TEST_FAILURE, true);
- conf.setInt(ExportSnapshot.Testing.CONF_TEST_FAILURE_COUNT, 2);
- conf.setInt("mapreduce.map.maxattempts", 3);
- testExportFileSystemState(conf, tableName, snapshotName, snapshotName, tableNumFiles,
- TEST_UTIL.getDefaultRootDirPath(), copyDir, true, getBypassRegionPredicate(), true);
- }
-
- /**
- * Check that ExportSnapshot will fail if we inject failure more times than MR will retry.
- */
- @Test
- public void testExportFailure() throws Exception {
- Path copyDir = getLocalDestinationDir();
- FileSystem fs = FileSystem.get(copyDir.toUri(), new Configuration());
- copyDir = copyDir.makeQualified(fs);
- Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
- conf.setBoolean(ExportSnapshot.Testing.CONF_TEST_FAILURE, true);
- conf.setInt(ExportSnapshot.Testing.CONF_TEST_FAILURE_COUNT, 4);
- conf.setInt("mapreduce.map.maxattempts", 3);
- testExportFileSystemState(conf, tableName, snapshotName, snapshotName, tableNumFiles,
- TEST_UTIL.getDefaultRootDirPath(), copyDir, true, getBypassRegionPredicate(), false);
- }
-
- /*
- * verify if the snapshot folder on file-system 1 match the one on file-system 2
- */
- protected static void verifySnapshotDir(final FileSystem fs1, final Path root1,
- final FileSystem fs2, final Path root2) throws IOException {
- assertEquals(listFiles(fs1, root1, root1), listFiles(fs2, root2, root2));
- }
-
- protected Set<String> verifySnapshot(final FileSystem fs, final Path rootDir,
- final TableName tableName, final String snapshotName) throws IOException {
- return verifySnapshot(TEST_UTIL.getConfiguration(), fs, rootDir, tableName,
- snapshotName, getBypassRegionPredicate());
- }
-
- /*
- * Verify if the files exists
- */
- protected static Set<String> verifySnapshot(final Configuration conf, final FileSystem fs,
- final Path rootDir, final TableName tableName, final String snapshotName,
- final RegionPredicate bypassregionPredicate) throws IOException {
- final Path exportedSnapshot = new Path(rootDir,
- new Path(HConstants.SNAPSHOT_DIR_NAME, snapshotName));
- final Set<String> snapshotFiles = new HashSet<>();
- final Path exportedArchive = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY);
- SnapshotReferenceUtil.visitReferencedFiles(conf, fs, exportedSnapshot,
- new SnapshotReferenceUtil.SnapshotVisitor() {
- @Override
- public void storeFile(final HRegionInfo regionInfo, final String family,
- final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
- if (bypassregionPredicate != null && bypassregionPredicate.evaluate(regionInfo))
- return;
-
- String hfile = storeFile.getName();
- snapshotFiles.add(hfile);
- if (storeFile.hasReference()) {
- // Nothing to do here, we have already the reference embedded
- } else {
- verifyNonEmptyFile(new Path(exportedArchive,
- new Path(FSUtils.getTableDir(new Path("./"), tableName),
- new Path(regionInfo.getEncodedName(), new Path(family, hfile)))));
- }
- }
-
- private void verifyNonEmptyFile(final Path path) throws IOException {
- assertTrue(path + " should exists", fs.exists(path));
- assertTrue(path + " should not be empty", fs.getFileStatus(path).getLen() > 0);
- }
- });
-
- // Verify Snapshot description
- SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, exportedSnapshot);
- assertTrue(desc.getName().equals(snapshotName));
- assertTrue(desc.getTable().equals(tableName.getNameAsString()));
- return snapshotFiles;
- }
-
- private static Set<String> listFiles(final FileSystem fs, final Path root, final Path dir)
- throws IOException {
- Set<String> files = new HashSet<>();
- int rootPrefix = root.makeQualified(fs).toString().length();
- FileStatus[] list = FSUtils.listStatus(fs, dir);
- if (list != null) {
- for (FileStatus fstat: list) {
- LOG.debug(fstat.getPath());
- if (fstat.isDirectory()) {
- files.addAll(listFiles(fs, root, fstat.getPath()));
- } else {
- files.add(fstat.getPath().makeQualified(fs).toString().substring(rootPrefix));
- }
- }
- }
- return files;
- }
-
- private Path getHdfsDestinationDir() {
- Path rootDir = TEST_UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
- Path path = new Path(new Path(rootDir, "export-test"), "export-" + System.currentTimeMillis());
- LOG.info("HDFS export destination path: " + path);
- return path;
- }
-
- private Path getLocalDestinationDir() {
- Path path = TEST_UTIL.getDataTestDir("local-export-" + System.currentTimeMillis());
- LOG.info("Local export destination path: " + path);
- return path;
- }
-
- private static void removeExportDir(final Path path) throws IOException {
- FileSystem fs = FileSystem.get(path.toUri(), new Configuration());
- fs.delete(path, true);
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/664b6be0/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotHelpers.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotHelpers.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotHelpers.java
deleted file mode 100644
index e31e81e..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotHelpers.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.snapshot;
-
-import static org.junit.Assert.assertEquals;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotFileInfo;
-import org.apache.hadoop.hbase.testclassification.RegionServerTests;
-import org.apache.hadoop.hbase.testclassification.SmallTests;
-import org.apache.hadoop.hbase.util.Pair;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-/**
- * Test Export Snapshot Tool helpers
- */
-@Category({RegionServerTests.class, SmallTests.class})
-public class TestExportSnapshotHelpers {
- /**
- * Verfy the result of getBalanceSplits() method.
- * The result are groups of files, used as input list for the "export" mappers.
- * All the groups should have similar amount of data.
- *
- * The input list is a pair of file path and length.
- * The getBalanceSplits() function sort it by length,
- * and assign to each group a file, going back and forth through the groups.
- */
- @Test
- public void testBalanceSplit() throws Exception {
- // Create a list of files
- List<Pair<SnapshotFileInfo, Long>> files = new ArrayList<>(21);
- for (long i = 0; i <= 20; i++) {
- SnapshotFileInfo fileInfo = SnapshotFileInfo.newBuilder()
- .setType(SnapshotFileInfo.Type.HFILE)
- .setHfile("file-" + i)
- .build();
- files.add(new Pair<>(fileInfo, i));
- }
-
- // Create 5 groups (total size 210)
- // group 0: 20, 11, 10, 1 (total size: 42)
- // group 1: 19, 12, 9, 2 (total size: 42)
- // group 2: 18, 13, 8, 3 (total size: 42)
- // group 3: 17, 12, 7, 4 (total size: 42)
- // group 4: 16, 11, 6, 5 (total size: 42)
- List<List<Pair<SnapshotFileInfo, Long>>> splits = ExportSnapshot.getBalancedSplits(files, 5);
- assertEquals(5, splits.size());
-
- String[] split0 = new String[] {"file-20", "file-11", "file-10", "file-1", "file-0"};
- verifyBalanceSplit(splits.get(0), split0, 42);
- String[] split1 = new String[] {"file-19", "file-12", "file-9", "file-2"};
- verifyBalanceSplit(splits.get(1), split1, 42);
- String[] split2 = new String[] {"file-18", "file-13", "file-8", "file-3"};
- verifyBalanceSplit(splits.get(2), split2, 42);
- String[] split3 = new String[] {"file-17", "file-14", "file-7", "file-4"};
- verifyBalanceSplit(splits.get(3), split3, 42);
- String[] split4 = new String[] {"file-16", "file-15", "file-6", "file-5"};
- verifyBalanceSplit(splits.get(4), split4, 42);
- }
-
- private void verifyBalanceSplit(final List<Pair<SnapshotFileInfo, Long>> split,
- final String[] expected, final long expectedSize) {
- assertEquals(expected.length, split.size());
- long totalSize = 0;
- for (int i = 0; i < expected.length; ++i) {
- Pair<SnapshotFileInfo, Long> fileInfo = split.get(i);
- assertEquals(expected[i], fileInfo.getFirst().getHfile());
- totalSize += fileInfo.getSecond();
- }
- assertEquals(expectedSize, totalSize);
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/664b6be0/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotNoCluster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotNoCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotNoCluster.java
deleted file mode 100644
index 00778502..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotNoCluster.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.snapshot;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.CategoryBasedTimeout;
-import org.apache.hadoop.hbase.HBaseCommonTestingUtility;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
-import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils.SnapshotMock;
-import org.apache.hadoop.hbase.testclassification.MapReduceTests;
-import org.apache.hadoop.hbase.testclassification.MediumTests;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.junit.BeforeClass;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-import org.junit.rules.TestRule;
-
-/**
- * Test Export Snapshot Tool
- */
-@Category({MapReduceTests.class, MediumTests.class})
-public class TestExportSnapshotNoCluster {
- @Rule public final TestRule timeout = CategoryBasedTimeout.builder().
- withTimeout(this.getClass()).withLookingForStuckThread(true).build();
- private static final Log LOG = LogFactory.getLog(TestExportSnapshotNoCluster.class);
-
- protected final static HBaseCommonTestingUtility TEST_UTIL = new HBaseCommonTestingUtility();
-
- private static FileSystem fs;
- private static Path testDir;
-
- public static void setUpBaseConf(Configuration conf) {
- conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
- conf.setInt("hbase.regionserver.msginterval", 100);
- conf.setInt("hbase.client.pause", 250);
- conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6);
- conf.setBoolean("hbase.master.enabletable.roundrobin", true);
- conf.setInt("mapreduce.map.maxattempts", 10);
- conf.set(HConstants.HBASE_DIR, testDir.toString());
- }
-
- @BeforeClass
- public static void setUpBeforeClass() throws Exception {
- testDir = TEST_UTIL.getDataTestDir();
- fs = testDir.getFileSystem(TEST_UTIL.getConfiguration());
-
- setUpBaseConf(TEST_UTIL.getConfiguration());
- }
-
- /**
- * Mock a snapshot with files in the archive dir,
- * two regions, and one reference file.
- */
- @Test
- public void testSnapshotWithRefsExportFileSystemState() throws Exception {
- SnapshotMock snapshotMock = new SnapshotMock(TEST_UTIL.getConfiguration(), fs, testDir);
- SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2("tableWithRefsV1",
- "tableWithRefsV1");
- testSnapshotWithRefsExportFileSystemState(builder);
-
- snapshotMock = new SnapshotMock(TEST_UTIL.getConfiguration(), fs, testDir);
- builder = snapshotMock.createSnapshotV2("tableWithRefsV2", "tableWithRefsV2");
- testSnapshotWithRefsExportFileSystemState(builder);
- }
-
- /**
- * Generates a couple of regions for the specified SnapshotMock,
- * and then it will run the export and verification.
- */
- private void testSnapshotWithRefsExportFileSystemState(SnapshotMock.SnapshotBuilder builder)
- throws Exception {
- Path[] r1Files = builder.addRegion();
- Path[] r2Files = builder.addRegion();
- builder.commit();
- int snapshotFilesCount = r1Files.length + r2Files.length;
-
- byte[] snapshotName = Bytes.toBytes(builder.getSnapshotDescription().getName());
- TableName tableName = builder.getTableDescriptor().getTableName();
- TestExportSnapshot.testExportFileSystemState(TEST_UTIL.getConfiguration(),
- tableName, snapshotName, snapshotName, snapshotFilesCount,
- testDir, getDestinationDir(), false, null, true);
- }
-
- private Path getDestinationDir() {
- Path path = new Path(new Path(testDir, "export-test"), "export-" + System.currentTimeMillis());
- LOG.info("HDFS export destination path: " + path);
- return path;
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/664b6be0/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobExportSnapshot.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobExportSnapshot.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobExportSnapshot.java
deleted file mode 100644
index 7407a7d..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobExportSnapshot.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.snapshot;
-
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.mob.MobConstants;
-import org.apache.hadoop.hbase.mob.MobUtils;
-import org.apache.hadoop.hbase.testclassification.LargeTests;
-import org.apache.hadoop.hbase.testclassification.VerySlowRegionServerTests;
-import org.junit.BeforeClass;
-import org.junit.Ignore;
-import org.junit.experimental.categories.Category;
-
-/**
- * Test Export Snapshot Tool
- */
-@Ignore
-@Category({VerySlowRegionServerTests.class, LargeTests.class})
-public class TestMobExportSnapshot extends TestExportSnapshot {
-
- public static void setUpBaseConf(Configuration conf) {
- TestExportSnapshot.setUpBaseConf(conf);
- conf.setInt(MobConstants.MOB_FILE_CACHE_SIZE_KEY, 0);
- }
-
- @BeforeClass
- public static void setUpBeforeClass() throws Exception {
- setUpBaseConf(TEST_UTIL.getConfiguration());
- TEST_UTIL.startMiniCluster(1, 3);
- TEST_UTIL.startMiniMapReduceCluster();
- }
-
- @Override
- protected void createTable() throws Exception {
- MobSnapshotTestingUtils.createPreSplitMobTable(TEST_UTIL, tableName, 2, FAMILY);
- }
-
- @Override
- protected RegionPredicate getBypassRegionPredicate() {
- return new RegionPredicate() {
- @Override
- public boolean evaluate(final HRegionInfo regionInfo) {
- return MobUtils.isMobRegionInfo(regionInfo);
- }
- };
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/664b6be0/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobSecureExportSnapshot.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobSecureExportSnapshot.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobSecureExportSnapshot.java
deleted file mode 100644
index 98d03c0..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobSecureExportSnapshot.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/**
- * Copyright The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.snapshot;
-
-import org.apache.hadoop.hbase.testclassification.LargeTests;
-import org.apache.hadoop.hbase.testclassification.VerySlowRegionServerTests;
-import org.apache.hadoop.hbase.mapreduce.HadoopSecurityEnabledUserProviderForTesting;
-import org.apache.hadoop.hbase.security.UserProvider;
-import org.apache.hadoop.hbase.security.access.AccessControlLists;
-import org.apache.hadoop.hbase.security.access.SecureTestUtil;
-
-import org.junit.BeforeClass;
-import org.junit.Ignore;
-import org.junit.experimental.categories.Category;
-
-/**
- * Reruns TestMobExportSnapshot using MobExportSnapshot in secure mode.
- */
-@Ignore
-@Category({VerySlowRegionServerTests.class, LargeTests.class})
-public class TestMobSecureExportSnapshot extends TestMobExportSnapshot {
- @BeforeClass
- public static void setUpBeforeClass() throws Exception {
- setUpBaseConf(TEST_UTIL.getConfiguration());
- // Setup separate test-data directory for MR cluster and set corresponding configurations.
- // Otherwise, different test classes running MR cluster can step on each other.
- TEST_UTIL.getDataTestDir();
-
- // set the always on security provider
- UserProvider.setUserProviderForTesting(TEST_UTIL.getConfiguration(),
- HadoopSecurityEnabledUserProviderForTesting.class);
-
- // setup configuration
- SecureTestUtil.enableSecurity(TEST_UTIL.getConfiguration());
-
- TEST_UTIL.startMiniCluster(1, 3);
- TEST_UTIL.startMiniMapReduceCluster();
-
- // Wait for the ACL table to become available
- TEST_UTIL.waitTableEnabled(AccessControlLists.ACL_TABLE_NAME);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/664b6be0/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSecureExportSnapshot.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSecureExportSnapshot.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSecureExportSnapshot.java
deleted file mode 100644
index 7d4832c..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSecureExportSnapshot.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/**
- * Copyright The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.snapshot;
-
-import org.apache.hadoop.hbase.CategoryBasedTimeout;
-import org.apache.hadoop.hbase.mapreduce.HadoopSecurityEnabledUserProviderForTesting;
-import org.apache.hadoop.hbase.security.UserProvider;
-import org.apache.hadoop.hbase.security.access.AccessControlLists;
-import org.apache.hadoop.hbase.security.access.SecureTestUtil;
-
-import org.apache.hadoop.hbase.testclassification.LargeTests;
-import org.apache.hadoop.hbase.testclassification.VerySlowRegionServerTests;
-import org.junit.BeforeClass;
-import org.junit.Ignore;
-import org.junit.Rule;
-import org.junit.experimental.categories.Category;
-import org.junit.rules.TestRule;
-
-/**
- * Reruns TestExportSnapshot using ExportSnapshot in secure mode.
- */
-@Ignore
-@Category({VerySlowRegionServerTests.class, LargeTests.class})
-public class TestSecureExportSnapshot extends TestExportSnapshot {
- @Rule public final TestRule timeout = CategoryBasedTimeout.builder().
- withTimeout(this.getClass()).withLookingForStuckThread(true).build();
- @BeforeClass
- public static void setUpBeforeClass() throws Exception {
- setUpBaseConf(TEST_UTIL.getConfiguration());
- // Setup separate test-data directory for MR cluster and set corresponding configurations.
- // Otherwise, different test classes running MR cluster can step on each other.
- TEST_UTIL.getDataTestDir();
-
- // set the always on security provider
- UserProvider.setUserProviderForTesting(TEST_UTIL.getConfiguration(),
- HadoopSecurityEnabledUserProviderForTesting.class);
-
- // setup configuration
- SecureTestUtil.enableSecurity(TEST_UTIL.getConfiguration());
-
- TEST_UTIL.startMiniCluster(1, 3);
- TEST_UTIL.startMiniMapReduceCluster();
-
- // Wait for the ACL table to become available
- TEST_UTIL.waitTableEnabled(AccessControlLists.ACL_TABLE_NAME);
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/664b6be0/hbase-server/src/test/java/org/apache/hadoop/hbase/util/HFileTestUtil.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/HFileTestUtil.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/HFileTestUtil.java
index 236994a..0487bf4 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/HFileTestUtil.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/HFileTestUtil.java
@@ -22,6 +22,8 @@ import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.fail;
import java.io.IOException;
+import java.util.Arrays;
+import java.util.Locale;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
@@ -29,6 +31,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.ArrayBackedTag;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.Tag;
import org.apache.hadoop.hbase.TagType;
@@ -50,6 +53,17 @@ import org.apache.hadoop.hbase.regionserver.StoreFile;
*/
public class HFileTestUtil {
+ public static final String OPT_DATA_BLOCK_ENCODING_USAGE =
+ "Encoding algorithm (e.g. prefix "
+ + "compression) to use for data blocks in the test column family, "
+ + "one of " + Arrays.toString(DataBlockEncoding.values()) + ".";
+ public static final String OPT_DATA_BLOCK_ENCODING =
+ HColumnDescriptor.DATA_BLOCK_ENCODING.toLowerCase(Locale.ROOT);
+ /** Column family used by the test */
+ public static byte[] DEFAULT_COLUMN_FAMILY = Bytes.toBytes("test_cf");
+ /** Column families used by the test */
+ public static final byte[][] DEFAULT_COLUMN_FAMILIES = { DEFAULT_COLUMN_FAMILY };
+
/**
* Create an HFile with the given number of rows between a given
* start key and end key @ family:qualifier. The value will be the key value.
http://git-wip-us.apache.org/repos/asf/hbase/blob/664b6be0/hbase-server/src/test/java/org/apache/hadoop/hbase/util/LoadTestDataGeneratorWithTags.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/LoadTestDataGeneratorWithTags.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/LoadTestDataGeneratorWithTags.java
index 2ea01bb..0b3c612 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/LoadTestDataGeneratorWithTags.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/LoadTestDataGeneratorWithTags.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.hbase.ArrayBackedTag;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.util.MultiThreadedAction.DefaultDataGenerator;
+import org.apache.hadoop.hbase.util.test.LoadTestDataGenerator;
@InterfaceAudience.Private
public class LoadTestDataGeneratorWithTags extends DefaultDataGenerator {
@@ -74,7 +75,7 @@ public class LoadTestDataGeneratorWithTags extends DefaultDataGenerator {
List<Tag> tags;
for (CellScanner cellScanner = m.cellScanner(); cellScanner.advance();) {
Cell cell = cellScanner.current();
- byte[] tag = LoadTestTool.generateData(random,
+ byte[] tag = LoadTestDataGenerator.generateData(random,
minTagLength + random.nextInt(maxTagLength - minTagLength));
tags = new ArrayList<>();
for (int n = 0; n < numTags; n++) {