You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by xu...@apache.org on 2022/10/24 07:40:54 UTC
[hudi] branch master updated: [HUDI-5027] Replace hardcoded hbase config keys with constant variables (#6946)
This is an automated email from the ASF dual-hosted git repository.
xushiyan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 5cf8699297 [HUDI-5027] Replace hardcoded hbase config keys with constant variables (#6946)
5cf8699297 is described below
commit 5cf8699297d453b8bf2e4b0944cbbbfffa9afb01
Author: slfan1989 <55...@users.noreply.github.com>
AuthorDate: Mon Oct 24 15:40:43 2022 +0800
[HUDI-5027] Replace hardcoded hbase config keys with constant variables (#6946)
---
.../hudi/index/hbase/SparkHoodieHBaseIndex.java | 25 +++++++++++++++-------
.../index/hbase/TestSparkHoodieHBaseIndex.java | 11 ++++++----
2 files changed, 24 insertions(+), 12 deletions(-)
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/index/hbase/SparkHoodieHBaseIndex.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/index/hbase/SparkHoodieHBaseIndex.java
index e6395b9c6d..f99bf876c9 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/index/hbase/SparkHoodieHBaseIndex.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/index/hbase/SparkHoodieHBaseIndex.java
@@ -85,6 +85,15 @@ import java.util.concurrent.TimeUnit;
import scala.Tuple2;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION;
+import static org.apache.hadoop.hbase.HConstants.ZOOKEEPER_QUORUM;
+import static org.apache.hadoop.hbase.HConstants.ZOOKEEPER_ZNODE_PARENT;
+import static org.apache.hadoop.hbase.HConstants.ZOOKEEPER_CLIENT_PORT;
+import static org.apache.hadoop.hbase.security.SecurityConstants.MASTER_KRB_PRINCIPAL;
+import static org.apache.hadoop.hbase.security.SecurityConstants.REGIONSERVER_KRB_PRINCIPAL;
+import static org.apache.hadoop.hbase.security.User.HBASE_SECURITY_AUTHORIZATION_CONF_KEY;
+import static org.apache.hadoop.hbase.security.User.HBASE_SECURITY_CONF_KEY;
+
/**
* Hoodie Index implementation backed by HBase.
*/
@@ -145,22 +154,22 @@ public class SparkHoodieHBaseIndex extends HoodieIndex<Object, Object> {
private Connection getHBaseConnection() {
Configuration hbaseConfig = HBaseConfiguration.create();
String quorum = config.getHbaseZkQuorum();
- hbaseConfig.set("hbase.zookeeper.quorum", quorum);
+ hbaseConfig.set(ZOOKEEPER_QUORUM, quorum);
String zkZnodeParent = config.getHBaseZkZnodeParent();
if (zkZnodeParent != null) {
- hbaseConfig.set("zookeeper.znode.parent", zkZnodeParent);
+ hbaseConfig.set(ZOOKEEPER_ZNODE_PARENT, zkZnodeParent);
}
String port = String.valueOf(config.getHbaseZkPort());
- hbaseConfig.set("hbase.zookeeper.property.clientPort", port);
+ hbaseConfig.set(ZOOKEEPER_CLIENT_PORT, port);
try {
String authentication = config.getHBaseIndexSecurityAuthentication();
if (authentication.equals("kerberos")) {
- hbaseConfig.set("hbase.security.authentication", "kerberos");
- hbaseConfig.set("hadoop.security.authentication", "kerberos");
- hbaseConfig.set("hbase.security.authorization", "true");
- hbaseConfig.set("hbase.regionserver.kerberos.principal", config.getHBaseIndexRegionserverPrincipal());
- hbaseConfig.set("hbase.master.kerberos.principal", config.getHBaseIndexMasterPrincipal());
+ hbaseConfig.set(HBASE_SECURITY_CONF_KEY, "kerberos");
+ hbaseConfig.set(HADOOP_SECURITY_AUTHENTICATION, "kerberos");
+ hbaseConfig.set(HBASE_SECURITY_AUTHORIZATION_CONF_KEY, "true");
+ hbaseConfig.set(REGIONSERVER_KRB_PRINCIPAL, config.getHBaseIndexRegionserverPrincipal());
+ hbaseConfig.set(MASTER_KRB_PRINCIPAL, config.getHBaseIndexMasterPrincipal());
String principal = config.getHBaseIndexKerberosUserPrincipal();
String keytab = SparkFiles.get(config.getHBaseIndexKerberosUserKeytab());
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/index/hbase/TestSparkHoodieHBaseIndex.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/index/hbase/TestSparkHoodieHBaseIndex.java
index 407fb8de0e..f22a067ad8 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/index/hbase/TestSparkHoodieHBaseIndex.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/index/hbase/TestSparkHoodieHBaseIndex.java
@@ -87,6 +87,9 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
+import static org.apache.hadoop.hbase.HConstants.ZOOKEEPER_CLIENT_PORT;
+import static org.apache.hadoop.hbase.HConstants.ZOOKEEPER_ZNODE_PARENT;
+import static org.apache.hadoop.hbase.HConstants.ZOOKEEPER_QUORUM;
/**
* Note :: HBaseTestingUtility is really flaky with issues where the HbaseMiniCluster fails to shutdown across tests,
@@ -111,7 +114,7 @@ public class TestSparkHoodieHBaseIndex extends SparkClientFunctionalTestHarness
public static void init() throws Exception {
// Initialize HbaseMiniCluster
hbaseConfig = HBaseConfiguration.create();
- hbaseConfig.set("zookeeper.znode.parent", "/hudi-hbase-test");
+ hbaseConfig.set(ZOOKEEPER_ZNODE_PARENT, "/hudi-hbase-test");
utility = new HBaseTestingUtility(hbaseConfig);
utility.startMiniCluster();
@@ -816,10 +819,10 @@ public class TestSparkHoodieHBaseIndex extends SparkClientFunctionalTestHarness
.forTable("test-trip-table")
.withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.HBASE)
.withHBaseIndexConfig(new HoodieHBaseIndexConfig.Builder()
- .hbaseZkPort(Integer.parseInt(hbaseConfig.get("hbase.zookeeper.property.clientPort")))
+ .hbaseZkPort(Integer.parseInt(hbaseConfig.get(ZOOKEEPER_CLIENT_PORT)))
.hbaseIndexPutBatchSizeAutoCompute(true)
- .hbaseZkZnodeParent(hbaseConfig.get("zookeeper.znode.parent", ""))
- .hbaseZkQuorum(hbaseConfig.get("hbase.zookeeper.quorum")).hbaseTableName(TABLE_NAME)
+ .hbaseZkZnodeParent(hbaseConfig.get(ZOOKEEPER_ZNODE_PARENT, ""))
+ .hbaseZkQuorum(hbaseConfig.get(ZOOKEEPER_QUORUM)).hbaseTableName(TABLE_NAME)
.hbaseIndexUpdatePartitionPath(updatePartitionPath)
.hbaseIndexRollbackSync(rollbackSync)
.hbaseIndexGetBatchSize(hbaseIndexBatchSize).build())