You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by zh...@apache.org on 2024/05/29 07:51:09 UTC
(hbase) branch branch-2.6 updated: HBASE-28616 Remove/Deprecated the rs.* related configuration in TableOutputFormat (#5946)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-2.6
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.6 by this push:
new 209b8116303 HBASE-28616 Remove/Deprecated the rs.* related configuration in TableOutputFormat (#5946)
209b8116303 is described below
commit 209b81163037809662d4241b8cad0cdcc345dd31
Author: Duo Zhang <zh...@apache.org>
AuthorDate: Wed May 29 15:21:18 2024 +0800
HBASE-28616 Remove/Deprecated the rs.* related configuration in TableOutputFormat (#5946)
Signed-off-by: Andrew Purtell <ap...@apache.org>
Signed-off-by: Pankaj Kumar <pa...@apache.org>
Reviewed-by: Subrat Mishra <su...@gmail.com>
(cherry picked from commit 771e1d92c0a342c4511d69d2ece4d7d58cd47d45)
---
...IntegrationTestBigLinkedListWithVisibility.java | 2 +-
.../apache/hadoop/hbase/mapreduce/CopyTable.java | 3 +-
.../apache/hadoop/hbase/mapreduce/SyncTable.java | 3 +-
.../hadoop/hbase/mapreduce/TableMapReduceUtil.java | 87 ++++++++++++++++++----
.../hadoop/hbase/mapreduce/TableOutputFormat.java | 22 +++---
5 files changed, 87 insertions(+), 30 deletions(-)
diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedListWithVisibility.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedListWithVisibility.java
index b5dee263546..5e3fd82be7b 100644
--- a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedListWithVisibility.java
+++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedListWithVisibility.java
@@ -273,7 +273,7 @@ public class IntegrationTestBigLinkedListWithVisibility extends IntegrationTestB
}
job.getConfiguration().setBoolean("mapreduce.map.speculative", false);
job.getConfiguration().setBoolean("mapreduce.reduce.speculative", false);
- TableMapReduceUtil.initTableReducerJob(COMMON_TABLE_NAME, null, job, null, null, null, null);
+ TableMapReduceUtil.initTableReducerJob(COMMON_TABLE_NAME, null, job);
TableMapReduceUtil.addDependencyJars(job);
TableMapReduceUtil.addDependencyJars(job.getConfiguration(), AbstractHBaseTool.class);
TableMapReduceUtil.initCredentials(job);
diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java
index 5453b8a333a..8c8fbfb6e74 100644
--- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java
+++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java
@@ -180,8 +180,7 @@ public class CopyTable extends Configured implements Tool {
}
} else {
initCopyTableMapperReducerJob(job, scan);
- TableMapReduceUtil.initTableReducerJob(dstTableName, null, job, null, peerAddress, null,
- null);
+ TableMapReduceUtil.initTableReducerJob(dstTableName, null, job, null, peerAddress);
}
return job;
diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/SyncTable.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/SyncTable.java
index 19e73dda904..89fca4fad4d 100644
--- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/SyncTable.java
+++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/SyncTable.java
@@ -167,8 +167,7 @@ public class SyncTable extends Configured implements Tool {
} else {
// No reducers. Just write straight to table. Call initTableReducerJob
// because it sets up the TableOutputFormat.
- TableMapReduceUtil.initTableReducerJob(targetTableName, null, job, null, targetZkCluster,
- null, null);
+ TableMapReduceUtil.initTableReducerJob(targetTableName, null, job, null, targetZkCluster);
// would be nice to add an option for bulk load instead
}
diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
index 7027b566026..f0b99b54122 100644
--- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
+++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
@@ -569,7 +569,7 @@ public class TableMapReduceUtil {
*/
public static void initTableReducerJob(String table, Class<? extends TableReducer> reducer,
Job job, Class partitioner) throws IOException {
- initTableReducerJob(table, reducer, job, partitioner, null, null, null);
+ initTableReducerJob(table, reducer, job, partitioner, null);
}
/**
@@ -589,15 +589,11 @@ public class TableMapReduceUtil {
* <code> <hbase.zookeeper.quorum>:<
* hbase.zookeeper.client.port>:<zookeeper.znode.parent>
* </code> such as <code>server,server2,server3:2181:/hbase</code>.
- * @param serverClass redefined hbase.regionserver.class
- * @param serverImpl redefined hbase.regionserver.impl
* @throws IOException When determining the region count fails.
*/
public static void initTableReducerJob(String table, Class<? extends TableReducer> reducer,
- Job job, Class partitioner, String quorumAddress, String serverClass, String serverImpl)
- throws IOException {
- initTableReducerJob(table, reducer, job, partitioner, quorumAddress, serverClass, serverImpl,
- true);
+ Job job, Class partitioner, String quorumAddress) throws IOException {
+ initTableReducerJob(table, reducer, job, partitioner, quorumAddress, true);
}
/**
@@ -617,16 +613,13 @@ public class TableMapReduceUtil {
* <code> <hbase.zookeeper.quorum>:<
* hbase.zookeeper.client.port>:<zookeeper.znode.parent>
* </code> such as <code>server,server2,server3:2181:/hbase</code>.
- * @param serverClass redefined hbase.regionserver.class
- * @param serverImpl redefined hbase.regionserver.impl
* @param addDependencyJars upload HBase jars and jars for any of the configured job classes via
* the distributed cache (tmpjars).
* @throws IOException When determining the region count fails.
*/
public static void initTableReducerJob(String table, Class<? extends TableReducer> reducer,
- Job job, Class partitioner, String quorumAddress, String serverClass, String serverImpl,
- boolean addDependencyJars) throws IOException {
-
+ Job job, Class partitioner, String quorumAddress, boolean addDependencyJars)
+ throws IOException {
Configuration conf = job.getConfiguration();
HBaseConfiguration.merge(conf, HBaseConfiguration.create(conf));
job.setOutputFormatClass(TableOutputFormat.class);
@@ -640,10 +633,6 @@ public class TableMapReduceUtil {
ZKConfig.validateClusterKey(quorumAddress);
conf.set(TableOutputFormat.QUORUM_ADDRESS, quorumAddress);
}
- if (serverClass != null && serverImpl != null) {
- conf.set(TableOutputFormat.REGION_SERVER_CLASS, serverClass);
- conf.set(TableOutputFormat.REGION_SERVER_IMPL, serverImpl);
- }
job.setOutputKeyClass(ImmutableBytesWritable.class);
job.setOutputValueClass(Writable.class);
if (partitioner == HRegionPartitioner.class) {
@@ -663,6 +652,72 @@ public class TableMapReduceUtil {
initCredentials(job);
}
+ /**
+ * Use this before submitting a TableReduce job. It will appropriately set up the JobConf.
+ * @param table The output table.
+ * @param reducer The reducer class to use.
+ * @param job The current job to adjust. Make sure the passed job is carrying all
+ * necessary HBase configuration.
+ * @param partitioner Partitioner to use. Pass <code>null</code> to use default partitioner.
+ * @param quorumAddress Distant cluster to write to; default is null for output to the cluster
+ * that is designated in <code>hbase-site.xml</code>. Set this String to the
+ * zookeeper ensemble of an alternate remote cluster when you would have the
+ * reduce write a cluster that is other than the default; e.g. copying tables
+ * between clusters, the source would be designated by
+ * <code>hbase-site.xml</code> and this param would have the ensemble address
+ * of the remote cluster. The format to pass is particular. Pass
+ * <code> <hbase.zookeeper.quorum>:<
+ * hbase.zookeeper.client.port>:<zookeeper.znode.parent>
+ * </code> such as <code>server,server2,server3:2181:/hbase</code>.
+ * @param serverClass redefined hbase.regionserver.class
+ * @param serverImpl redefined hbase.regionserver.impl
+ * @throws IOException When determining the region count fails.
+ * @deprecated Since 2.5.9, 2.6.1, 2.7.0, will be removed in 4.0.0. The {@code serverClass} and
+ * {@code serverImpl} do not take effect any more, just use
+ * {@link #initTableReducerJob(String, Class, Job, Class, String)} instead.
+ * @see #initTableReducerJob(String, Class, Job, Class, String)
+ */
+ @Deprecated
+ public static void initTableReducerJob(String table, Class<? extends TableReducer> reducer,
+ Job job, Class partitioner, String quorumAddress, String serverClass, String serverImpl)
+ throws IOException {
+ initTableReducerJob(table, reducer, job, partitioner, quorumAddress);
+ }
+
+ /**
+ * Use this before submitting a TableReduce job. It will appropriately set up the JobConf.
+ * @param table The output table.
+ * @param reducer The reducer class to use.
+ * @param job The current job to adjust. Make sure the passed job is carrying all
+ * necessary HBase configuration.
+ * @param partitioner Partitioner to use. Pass <code>null</code> to use default partitioner.
+ * @param quorumAddress Distant cluster to write to; default is null for output to the cluster
+ * that is designated in <code>hbase-site.xml</code>. Set this String to
+ * the zookeeper ensemble of an alternate remote cluster when you would
+ * have the reduce write a cluster that is other than the default; e.g.
+ * copying tables between clusters, the source would be designated by
+ * <code>hbase-site.xml</code> and this param would have the ensemble
+ * address of the remote cluster. The format to pass is particular. Pass
+ * <code> <hbase.zookeeper.quorum>:<
+ * hbase.zookeeper.client.port>:<zookeeper.znode.parent>
+ * </code> such as <code>server,server2,server3:2181:/hbase</code>.
+ * @param serverClass redefined hbase.regionserver.class
+ * @param serverImpl redefined hbase.regionserver.impl
+ * @param addDependencyJars upload HBase jars and jars for any of the configured job classes via
+ * the distributed cache (tmpjars).
+ * @throws IOException When determining the region count fails.
+ * @deprecated Since 2.5.9, 2.6.1, 2.7.0, will be removed in 4.0.0. The {@code serverClass} and
+ * {@code serverImpl} do not take effect any more, just use
+ * {@link #initTableReducerJob(String, Class, Job, Class, String, boolean)} instead.
+ * @see #initTableReducerJob(String, Class, Job, Class, String, boolean)
+ */
+ @Deprecated
+ public static void initTableReducerJob(String table, Class<? extends TableReducer> reducer,
+ Job job, Class partitioner, String quorumAddress, String serverClass, String serverImpl,
+ boolean addDependencyJars) throws IOException {
+ initTableReducerJob(table, reducer, job, partitioner, quorumAddress, addDependencyJars);
+ }
+
/**
* Ensures that the given number of reduce tasks for the given job configuration does not exceed
* the number of regions for the given table.
diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableOutputFormat.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableOutputFormat.java
index 17c6c0e4551..a8ec67c9b23 100644
--- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableOutputFormat.java
+++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableOutputFormat.java
@@ -66,16 +66,26 @@ public class TableOutputFormat<KEY> extends OutputFormat<KEY, Mutation> implemen
* Optional job parameter to specify a peer cluster. Used specifying remote cluster when copying
* between hbase clusters (the source is picked up from <code>hbase-site.xml</code>).
* @see TableMapReduceUtil#initTableReducerJob(String, Class, org.apache.hadoop.mapreduce.Job,
- * Class, String, String, String)
+ * Class, String)
*/
public static final String QUORUM_ADDRESS = OUTPUT_CONF_PREFIX + "quorum";
/** Optional job parameter to specify peer cluster's ZK client port */
public static final String QUORUM_PORT = OUTPUT_CONF_PREFIX + "quorum.port";
- /** Optional specification of the rs class name of the peer cluster */
+ /**
+ * Optional specification of the rs class name of the peer cluster.
+ * @deprecated Since 2.5.9, 2.6.1 and 2.7.0, will be removed in 4.0.0. Does not take effect from
+ * long ago, see HBASE-6044.
+ */
+ @Deprecated
public static final String REGION_SERVER_CLASS = OUTPUT_CONF_PREFIX + "rs.class";
- /** Optional specification of the rs impl name of the peer cluster */
+ /**
+ * Optional specification of the rs impl name of the peer cluster
+ * @deprecated Since 2.5.9, 2.6.1 and 2.7.0, will be removed in 4.0.0. Does not take effect from
+ * long ago, see HBASE-6044.
+ */
+ @Deprecated
public static final String REGION_SERVER_IMPL = OUTPUT_CONF_PREFIX + "rs.impl";
/** The configuration. */
@@ -208,15 +218,9 @@ public class TableOutputFormat<KEY> extends OutputFormat<KEY, Mutation> implemen
String address = otherConf.get(QUORUM_ADDRESS);
int zkClientPort = otherConf.getInt(QUORUM_PORT, 0);
- String serverClass = otherConf.get(REGION_SERVER_CLASS);
- String serverImpl = otherConf.get(REGION_SERVER_IMPL);
try {
this.conf = HBaseConfiguration.createClusterConf(otherConf, address, OUTPUT_CONF_PREFIX);
-
- if (serverClass != null) {
- this.conf.set(HConstants.REGION_SERVER_IMPL, serverImpl);
- }
if (zkClientPort != 0) {
this.conf.setInt(HConstants.ZOOKEEPER_CLIENT_PORT, zkClientPort);
}