You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ap...@apache.org on 2016/12/03 00:06:43 UTC
[1/4] hbase git commit: HBASE-16409 Row key for bad row should be
properly delimited in VerifyReplication
Repository: hbase
Updated Branches:
refs/heads/0.98 358067eac -> 4cdd0adfb
HBASE-16409 Row key for bad row should be properly delimited in VerifyReplication
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e05e70c2
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e05e70c2
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e05e70c2
Branch: refs/heads/0.98
Commit: e05e70c24551f62fce0be3ad7d4f5bc5a26fd51d
Parents: 358067e
Author: tedyu <yu...@gmail.com>
Authored: Sat Aug 20 17:18:27 2016 -0700
Committer: Andrew Purtell <ap...@apache.org>
Committed: Thu Dec 1 16:43:26 2016 -0800
----------------------------------------------------------------------
.../hbase/mapreduce/replication/VerifyReplication.java | 13 +++++++++++--
1 file changed, 11 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/e05e70c2/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
index a5d1ddd..6c1e490 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
@@ -80,6 +80,7 @@ public class VerifyReplication extends Configured implements Tool {
static int versions = -1;
static String tableName = null;
static String families = null;
+ static String delimiter = null;
static String peerId = null;
static String rowPrefixes = null;
@@ -182,7 +183,8 @@ public class VerifyReplication extends Configured implements Tool {
private void logFailRowAndIncreaseCounter(Context context, Counters counter, Result row) {
context.getCounter(counter).increment(1);
context.getCounter(Counters.BADROWS).increment(1);
- LOG.error(counter.toString() + ", rowkey=" + Bytes.toString(row.getRow()));
+ LOG.error(counter.toString() + ", rowkey=" + delimiter + Bytes.toString(row.getRow()) +
+ delimiter);
}
@Override
@@ -387,6 +389,12 @@ public class VerifyReplication extends Configured implements Tool {
printUsage("Invalid argument '" + cmd + "'");
}
+ final String delimiterArgKey = "--delimiter=";
+ if (cmd.startsWith(delimiterArgKey)) {
+ delimiter = cmd.substring(delimiterArgKey.length());
+ continue;
+ }
+
if (i == args.length-2) {
peerId = cmd;
}
@@ -422,7 +430,7 @@ public class VerifyReplication extends Configured implements Tool {
System.err.println("ERROR: " + errorMsg);
}
System.err.println("Usage: verifyrep [--starttime=X]" +
- " [--endtime=Y] [--families=A] [--row-prefixes=B] <peerid> <tablename>");
+ " [--endtime=Y] [--families=A] [--row-prefixes=B] [--delimiter=] <peerid> <tablename>");
System.err.println();
System.err.println("Options:");
System.err.println(" starttime beginning of the time range");
@@ -431,6 +439,7 @@ public class VerifyReplication extends Configured implements Tool {
System.err.println(" versions number of cell versions to verify");
System.err.println(" families comma-separated list of families to copy");
System.err.println(" row-prefixes comma-separated list of row key prefixes to filter on ");
+ System.err.println(" delimiter the delimiter used in display around rowkey");
System.err.println();
System.err.println("Args:");
System.err.println(" peerid Id of the peer used for verification, must match the one given for replication");
[2/4] hbase git commit: HBASE-16409 Addendum fixes the case where
delimiter is not specified by user
Posted by ap...@apache.org.
HBASE-16409 Addendum fixes the case where delimiter is not specified by user
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/dd488c06
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/dd488c06
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/dd488c06
Branch: refs/heads/0.98
Commit: dd488c06f69af5383726fcc38877829de57369e6
Parents: e05e70c
Author: tedyu <yu...@gmail.com>
Authored: Thu Aug 25 11:54:42 2016 -0700
Committer: Andrew Purtell <ap...@apache.org>
Committed: Thu Dec 1 16:43:31 2016 -0800
----------------------------------------------------------------------
.../hadoop/hbase/mapreduce/replication/VerifyReplication.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/dd488c06/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
index 6c1e490..5fc3dd5 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
@@ -80,7 +80,7 @@ public class VerifyReplication extends Configured implements Tool {
static int versions = -1;
static String tableName = null;
static String families = null;
- static String delimiter = null;
+ static String delimiter = "";
static String peerId = null;
static String rowPrefixes = null;
[4/4] hbase git commit: HBASE-16946 Provide Raw scan as an option in
VerifyReplication (Sreekar Pallapothu)
Posted by ap...@apache.org.
HBASE-16946 Provide Raw scan as an option in VerifyReplication (Sreekar Pallapothu)
Conflicts:
hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
Amending-Author: Andrew Purtell <ap...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/4cdd0adf
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/4cdd0adf
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/4cdd0adf
Branch: refs/heads/0.98
Commit: 4cdd0adfbdfb8b4c28b7e226b88fb69d0e4e404f
Parents: e2df4b8
Author: tedyu <yu...@gmail.com>
Authored: Fri Nov 4 03:36:17 2016 -0700
Committer: Andrew Purtell <ap...@apache.org>
Committed: Fri Dec 2 16:04:29 2016 -0800
----------------------------------------------------------------------
.../replication/VerifyReplication.java | 13 +++
.../replication/TestReplicationSmallTests.java | 102 ++++++++++++++++++-
2 files changed, 112 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/4cdd0adf/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
index 1ec31a1..b22b11e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
@@ -84,6 +84,7 @@ public class VerifyReplication extends Configured implements Tool {
static String peerId = null;
static String rowPrefixes = null;
static boolean verbose = false;
+ static boolean includeDeletedCells = false;
/**
* Map-only comparator for 2 tables
@@ -126,6 +127,8 @@ public class VerifyReplication extends Configured implements Tool {
scan.addFamily(Bytes.toBytes(fam));
}
}
+ boolean includeDeletedCells = conf.getBoolean(NAME + ".includeDeletedCells", false);
+ scan.setRaw(includeDeletedCells);
String rowPrefixes = conf.get(NAME + ".rowPrefixes", null);
setRowPrefixFilter(scan, rowPrefixes);
scan.setTimeRange(startTime, endTime);
@@ -267,6 +270,7 @@ public class VerifyReplication extends Configured implements Tool {
conf.setLong(NAME+".startTime", startTime);
conf.setLong(NAME+".endTime", endTime);
conf.setBoolean(NAME +".verbose", verbose);
+ conf.setBoolean(NAME +".includeDeletedCells", includeDeletedCells);
if (families != null) {
conf.set(NAME+".families", families);
}
@@ -291,6 +295,7 @@ public class VerifyReplication extends Configured implements Tool {
Scan scan = new Scan();
scan.setTimeRange(startTime, endTime);
+ scan.setRaw(includeDeletedCells);
if (versions >= 0) {
scan.setMaxVersions(versions);
LOG.info("Number of versions set to " + versions);
@@ -368,6 +373,12 @@ public class VerifyReplication extends Configured implements Tool {
continue;
}
+ final String includeDeletedCellsArgKey = "--raw";
+ if (cmd.equals(includeDeletedCellsArgKey)) {
+ includeDeletedCells = true;
+ continue;
+ }
+
final String versionsArgKey = "--versions=";
if (cmd.startsWith(versionsArgKey)) {
versions = Integer.parseInt(cmd.substring(versionsArgKey.length()));
@@ -433,6 +444,7 @@ public class VerifyReplication extends Configured implements Tool {
families = null;
peerId = null;
rowPrefixes = null;
+ includeDeletedCells = false;
}
/*
@@ -451,6 +463,7 @@ public class VerifyReplication extends Configured implements Tool {
System.err.println(" without endtime means from starttime to forever");
System.err.println(" endtime end of the time range");
System.err.println(" versions number of cell versions to verify");
+ System.err.println(" raw includes raw scan if given in options");
System.err.println(" families comma-separated list of families to copy");
System.err.println(" row-prefixes comma-separated list of row key prefixes to filter on ");
System.err.println(" delimiter the delimiter used in display around rowkey");
http://git-wip-us.apache.org/repos/asf/hbase/blob/4cdd0adf/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
index 5df28a5..6ddc209 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
@@ -33,18 +33,18 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.ClusterStatus;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.ServerLoad;
-import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HConnectionManager;
import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
@@ -486,6 +486,102 @@ public class TestReplicationSmallTests extends TestReplicationBase {
runVerifyReplication(args, 0, NB_ROWS_IN_BATCH);
}
+ /**
+ * Load a row into a table, make sure the data is really the same,
+ * delete the row, make sure the delete marker is replicated,
+ * run verify replication with and without raw to check the results.
+ * @throws Exception
+ */
+ @Test(timeout=300000)
+ public void testVerifyRepJobWithRawOptions() throws Exception {
+ LOG.info("testVerifyRepJobWithRawOptions");
+
+ TableName tablename = TableName.valueOf("test_raw");
+ byte[] familyname = Bytes.toBytes("fam_raw");
+ byte[] row = Bytes.toBytes("row_raw");
+
+ HTableInterface lHtable1 = null;
+ HTableInterface lHtable2 = null;
+
+ try {
+ HTableDescriptor table = new HTableDescriptor(tablename);
+ HColumnDescriptor fam = new HColumnDescriptor(familyname);
+ fam.setMaxVersions(100);
+ fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
+ table.addFamily(fam);
+
+ HBaseAdmin admin1 = new HBaseAdmin(conf1);
+ try {
+ admin1.createTable(table, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE);
+ } finally {
+ admin1.close();
+ }
+ utility1.waitUntilAllRegionsAssigned(tablename);
+
+ HBaseAdmin admin2 = new HBaseAdmin(conf2);
+ try {
+ admin2.createTable(table, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE);
+ } finally {
+ admin2.close();
+ }
+ utility2.waitUntilAllRegionsAssigned(tablename);
+
+ lHtable1 = HConnectionManager.createConnection(conf1).getTable(tablename);
+ lHtable2 = HConnectionManager.createConnection(conf2).getTable(tablename);
+
+ Put put = new Put(row);
+ put.add(familyname, row, row);
+ lHtable1.put(put);
+
+ Get get = new Get(row);
+ for (int i = 0; i < NB_RETRIES; i++) {
+ if (i==NB_RETRIES-1) {
+ fail("Waited too much time for put replication");
+ }
+ Result res = lHtable2.get(get);
+ if (res.size() == 0) {
+ LOG.info("Row not available");
+ Thread.sleep(SLEEP_TIME);
+ } else {
+ assertArrayEquals(res.value(), row);
+ break;
+ }
+ }
+
+ Delete del = new Delete(row);
+ lHtable1.delete(del);
+
+ get = new Get(row);
+ for (int i = 0; i < NB_RETRIES; i++) {
+ if (i==NB_RETRIES-1) {
+ fail("Waited too much time for del replication");
+ }
+ Result res = lHtable2.get(get);
+ if (res.size() >= 1) {
+ LOG.info("Row not deleted");
+ Thread.sleep(SLEEP_TIME);
+ } else {
+ break;
+ }
+ }
+
+ // Checking verifyReplication for the default behavior.
+ String[] argsWithoutRaw = new String[] {PEER_ID, tablename.getNameAsString()};
+ runVerifyReplication(argsWithoutRaw, 0, 0);
+
+ // Checking verifyReplication with raw
+ String[] argsWithRawAsTrue = new String[] {"--raw", PEER_ID, tablename.getNameAsString()};
+ runVerifyReplication(argsWithRawAsTrue, 1, 0);
+ } finally {
+ if (lHtable1 != null) {
+ lHtable1.close();
+ }
+ if (lHtable2 != null) {
+ lHtable2.close();
+ }
+ }
+ }
+
private void runVerifyReplication(String[] args, int expectedGoodRows, int expectedBadRows)
throws IOException, InterruptedException, ClassNotFoundException {
Job job = VerifyReplication.createSubmittableJob(new Configuration(CONF_WITH_LOCALFS), args);
[3/4] hbase git commit: HBASE-16772 Add verbose option to
VerifyReplication for logging good rows
Posted by ap...@apache.org.
HBASE-16772 Add verbose option to VerifyReplication for logging good rows
Conflicts:
hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
Amending-Author: Andrew Purtell <ap...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e2df4b80
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e2df4b80
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e2df4b80
Branch: refs/heads/0.98
Commit: e2df4b80589aa42bd8ef77debad29ded1702a956
Parents: dd488c0
Author: Andrew Purtell <ap...@apache.org>
Authored: Fri Dec 2 16:03:57 2016 -0800
Committer: Andrew Purtell <ap...@apache.org>
Committed: Fri Dec 2 16:03:57 2016 -0800
----------------------------------------------------------------------
.../mapreduce/replication/VerifyReplication.java | 17 ++++++++++++++++-
1 file changed, 16 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/e2df4b80/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
index 5fc3dd5..1ec31a1 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
@@ -83,6 +83,7 @@ public class VerifyReplication extends Configured implements Tool {
static String delimiter = "";
static String peerId = null;
static String rowPrefixes = null;
+ static boolean verbose = false;
/**
* Map-only comparator for 2 tables
@@ -95,6 +96,7 @@ public class VerifyReplication extends Configured implements Tool {
private ResultScanner replicatedScanner;
private Result currentCompareRowInPeerTable;
+ private boolean verbose = false;
/**
* Map method that compares every scanned row with the equivalent from
@@ -110,6 +112,7 @@ public class VerifyReplication extends Configured implements Tool {
throws IOException {
if (replicatedScanner == null) {
Configuration conf = context.getConfiguration();
+ verbose = conf.getBoolean(NAME +".verbose", false);
final Scan scan = new Scan();
scan.setBatch(batch);
scan.setCacheBlocks(false);
@@ -161,6 +164,9 @@ public class VerifyReplication extends Configured implements Tool {
try {
Result.compareResults(value, currentCompareRowInPeerTable);
context.getCounter(Counters.GOODROWS).increment(1);
+ if (verbose) {
+ LOG.info("Good row key: " + delimiter + Bytes.toString(value.getRow()) + delimiter);
+ }
} catch (Exception e) {
logFailRowAndIncreaseCounter(context, Counters.CONTENT_DIFFERENT_ROWS, value);
LOG.error("Exception while comparing row : " + e);
@@ -260,6 +266,7 @@ public class VerifyReplication extends Configured implements Tool {
conf.set(NAME+".tableName", tableName);
conf.setLong(NAME+".startTime", startTime);
conf.setLong(NAME+".endTime", endTime);
+ conf.setBoolean(NAME +".verbose", verbose);
if (families != null) {
conf.set(NAME+".families", families);
}
@@ -395,6 +402,12 @@ public class VerifyReplication extends Configured implements Tool {
continue;
}
+ final String verboseKey = "--verbose";
+ if (cmd.startsWith(verboseKey)) {
+ verbose = true;
+ continue;
+ }
+
if (i == args.length-2) {
peerId = cmd;
}
@@ -430,7 +443,8 @@ public class VerifyReplication extends Configured implements Tool {
System.err.println("ERROR: " + errorMsg);
}
System.err.println("Usage: verifyrep [--starttime=X]" +
- " [--endtime=Y] [--families=A] [--row-prefixes=B] [--delimiter=] <peerid> <tablename>");
+ " [--endtime=Y] [--families=A] [--row-prefixes=B] [--delimiter=] [--recomparesleep=] " +
+ "[--verbose] <peerid> <tablename>");
System.err.println();
System.err.println("Options:");
System.err.println(" starttime beginning of the time range");
@@ -440,6 +454,7 @@ public class VerifyReplication extends Configured implements Tool {
System.err.println(" families comma-separated list of families to copy");
System.err.println(" row-prefixes comma-separated list of row key prefixes to filter on ");
System.err.println(" delimiter the delimiter used in display around rowkey");
+ System.err.println(" verbose logs row keys of good rows");
System.err.println();
System.err.println("Args:");
System.err.println(" peerid Id of the peer used for verification, must match the one given for replication");