You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by el...@apache.org on 2017/04/11 17:31:16 UTC
[11/54] [abbrv] hbase git commit: HBASE-17871 scan#setBatch(int) call
leads wrong result of VerifyReplication
HBASE-17871 scan#setBatch(int) call leads wrong result of VerifyReplication
Signed-off-by: tedyu <yu...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/ec5188df
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/ec5188df
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/ec5188df
Branch: refs/heads/HBASE-16961
Commit: ec5188df3090d42088b6f4cb8f0c2fd49425f8c1
Parents: d7e3116
Author: Tomu Tsuruhara <to...@linecorp.com>
Authored: Wed Apr 5 21:42:28 2017 +0900
Committer: tedyu <yu...@gmail.com>
Committed: Thu Apr 6 07:00:13 2017 -0700
----------------------------------------------------------------------
.../mapreduce/replication/VerifyReplication.java | 19 +++++++++++++++----
1 file changed, 15 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/ec5188df/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
index ba5966b..3f8317b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
@@ -77,7 +77,7 @@ public class VerifyReplication extends Configured implements Tool {
private final static String PEER_CONFIG_PREFIX = NAME + ".peer.";
static long startTime = 0;
static long endTime = Long.MAX_VALUE;
- static int batch = Integer.MAX_VALUE;
+ static int batch = -1;
static int versions = -1;
static String tableName = null;
static String families = null;
@@ -110,6 +110,7 @@ public class VerifyReplication extends Configured implements Tool {
private int sleepMsBeforeReCompare;
private String delimiter = "";
private boolean verbose = false;
+ private int batch = -1;
/**
* Map method that compares every scanned row with the equivalent from
@@ -128,8 +129,11 @@ public class VerifyReplication extends Configured implements Tool {
sleepMsBeforeReCompare = conf.getInt(NAME +".sleepMsBeforeReCompare", 0);
delimiter = conf.get(NAME + ".delimiter", "");
verbose = conf.getBoolean(NAME +".verbose", false);
+ batch = conf.getInt(NAME + ".batch", -1);
final Scan scan = new Scan();
- scan.setBatch(batch);
+ if (batch > 0) {
+ scan.setBatch(batch);
+ }
scan.setCacheBlocks(false);
scan.setCaching(conf.getInt(TableInputFormat.SCAN_CACHEDROWS, 1));
long startTime = conf.getLong(NAME + ".startTime", 0);
@@ -329,6 +333,7 @@ public class VerifyReplication extends Configured implements Tool {
conf.setLong(NAME+".endTime", endTime);
conf.setInt(NAME +".sleepMsBeforeReCompare", sleepMsBeforeReCompare);
conf.set(NAME + ".delimiter", delimiter);
+ conf.setInt(NAME + ".batch", batch);
conf.setBoolean(NAME +".verbose", verbose);
conf.setBoolean(NAME +".includeDeletedCells", includeDeletedCells);
if (families != null) {
@@ -356,6 +361,10 @@ public class VerifyReplication extends Configured implements Tool {
Scan scan = new Scan();
scan.setTimeRange(startTime, endTime);
scan.setRaw(includeDeletedCells);
+ scan.setCacheBlocks(false);
+ if (batch > 0) {
+ scan.setBatch(batch);
+ }
if (versions >= 0) {
scan.setMaxVersions(versions);
LOG.info("Number of versions set to " + versions);
@@ -503,7 +512,7 @@ public class VerifyReplication extends Configured implements Tool {
private static void restoreDefaults() {
startTime = 0;
endTime = Long.MAX_VALUE;
- batch = Integer.MAX_VALUE;
+ batch = -1;
versions = -1;
tableName = null;
families = null;
@@ -521,13 +530,15 @@ public class VerifyReplication extends Configured implements Tool {
}
System.err.println("Usage: verifyrep [--starttime=X]" +
" [--endtime=Y] [--families=A] [--row-prefixes=B] [--delimiter=] [--recomparesleep=] " +
- "[--verbose] <peerid> <tablename>");
+ "[--batch=] [--verbose] <peerid> <tablename>");
System.err.println();
System.err.println("Options:");
System.err.println(" starttime beginning of the time range");
System.err.println(" without endtime means from starttime to forever");
System.err.println(" endtime end of the time range");
System.err.println(" versions number of cell versions to verify");
+ System.err.println(" batch batch count for scan, " +
+ "note that result row counts will no longer be actual number of rows when you use this option");
System.err.println(" raw includes raw scan if given in options");
System.err.println(" families comma-separated list of families to copy");
System.err.println(" row-prefixes comma-separated list of row key prefixes to filter on ");