You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by sy...@apache.org on 2016/03/30 18:03:44 UTC
[42/50] [abbrv] hbase git commit: HBASE-15191 CopyTable and
VerifyReplication - Option to specify batch size, versions (Parth Shah)
HBASE-15191 CopyTable and VerifyReplication - Option to specify batch size, versions (Parth Shah)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/cbf9c1e1
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/cbf9c1e1
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/cbf9c1e1
Branch: refs/heads/hbase-12439
Commit: cbf9c1e116717972c243abedb53e7d26ef03cc85
Parents: 7793bc5
Author: tedyu <yu...@gmail.com>
Authored: Mon Mar 28 17:25:00 2016 -0700
Committer: tedyu <yu...@gmail.com>
Committed: Mon Mar 28 17:25:00 2016 -0700
----------------------------------------------------------------------
.../hadoop/hbase/mapreduce/CopyTable.java | 31 ++++++++++++++++----
.../replication/VerifyReplication.java | 9 ++++++
2 files changed, 35 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/cbf9c1e1/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java
index b7f4b66..dd8b891 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java
@@ -54,7 +54,9 @@ public class CopyTable extends Configured implements Tool {
final static String NAME = "copytable";
long startTime = 0;
- long endTime = 0;
+ long endTime = HConstants.LATEST_TIMESTAMP;
+ int batch = Integer.MAX_VALUE;
+ int cacheRow = -1;
int versions = -1;
String tableName = null;
String startRow = null;
@@ -82,15 +84,22 @@ public class CopyTable extends Configured implements Tool {
if (!doCommandLine(args)) {
return null;
}
-
+
Job job = Job.getInstance(getConf(), getConf().get(JOB_NAME_CONF_KEY, NAME + "_" + tableName));
job.setJarByClass(CopyTable.class);
Scan scan = new Scan();
+
+ scan.setBatch(batch);
scan.setCacheBlocks(false);
- if (startTime != 0) {
- scan.setTimeRange(startTime,
- endTime == 0 ? HConstants.LATEST_TIMESTAMP : endTime);
+
+ if (cacheRow > 0) {
+ scan.setCaching(cacheRow);
+ } else {
+ scan.setCaching(getConf().getInt(HConstants.HBASE_CLIENT_SCANNER_CACHING, 100));
}
+
+ scan.setTimeRange(startTime, endTime);
+
if (allCells) {
scan.setRaw(true);
}
@@ -252,6 +261,18 @@ public class CopyTable extends Configured implements Tool {
endTime = Long.parseLong(cmd.substring(endTimeArgKey.length()));
continue;
}
+
+ final String batchArgKey = "--batch=";
+ if (cmd.startsWith(batchArgKey)) {
+ batch = Integer.parseInt(cmd.substring(batchArgKey.length()));
+ continue;
+ }
+
+ final String cacheRowArgKey = "--cacheRow=";
+ if (cmd.startsWith(cacheRowArgKey)) {
+ cacheRow = Integer.parseInt(cmd.substring(cacheRowArgKey.length()));
+ continue;
+ }
final String versionsArgKey = "--versions=";
if (cmd.startsWith(versionsArgKey)) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/cbf9c1e1/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
index e6b4802..a452036 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
@@ -72,6 +72,7 @@ public class VerifyReplication extends Configured implements Tool {
private final static String PEER_CONFIG_PREFIX = NAME + ".peer.";
static long startTime = 0;
static long endTime = Long.MAX_VALUE;
+ static int batch = Integer.MAX_VALUE;
static int versions = -1;
static String tableName = null;
static String families = null;
@@ -110,6 +111,8 @@ public class VerifyReplication extends Configured implements Tool {
if (replicatedScanner == null) {
Configuration conf = context.getConfiguration();
final Scan scan = new Scan();
+ scan.setBatch(batch);
+ scan.setCacheBlocks(false);
scan.setCaching(conf.getInt(TableInputFormat.SCAN_CACHEDROWS, 1));
long startTime = conf.getLong(NAME + ".startTime", 0);
long endTime = conf.getLong(NAME + ".endTime", Long.MAX_VALUE);
@@ -338,6 +341,12 @@ public class VerifyReplication extends Configured implements Tool {
versions = Integer.parseInt(cmd.substring(versionsArgKey.length()));
continue;
}
+
+ final String batchArgKey = "--batch=";
+ if (cmd.startsWith(batchArgKey)) {
+ batch = Integer.parseInt(cmd.substring(batchArgKey.length()));
+ continue;
+ }
final String familiesArgKey = "--families=";
if (cmd.startsWith(familiesArgKey)) {