You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/08/18 23:27:41 UTC
kylin git commit: KYLIN-1910 Support Separate HBase Cluster with NN
HA and Kerberos Authentication
Repository: kylin
Updated Branches:
refs/heads/master 6c88139d4 -> af2a82714
KYLIN-1910 Support Separate HBase Cluster with NN HA and Kerberos Authentication
Signed-off-by: Yang Li <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/af2a8271
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/af2a8271
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/af2a8271
Branch: refs/heads/master
Commit: af2a82714d54a640c8e3941b4d79e9fb2bbe77a2
Parents: 6c88139
Author: sunyerui <su...@gmail.com>
Authored: Thu Jul 21 17:41:32 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Fri Aug 19 07:14:41 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 4 +++
.../kylin/storage/hbase/HBaseConnection.java | 34 ++++++++++++++++++++
.../kylin/storage/hbase/steps/CubeHFileJob.java | 2 ++
3 files changed, 40 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/af2a8271/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 7a7d796..9a8e6fd 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -208,6 +208,10 @@ abstract public class KylinConfigBase implements Serializable {
return getOptional("kylin.hbase.cluster.fs", "");
}
+ public String getHBaseClusterHDFSConfigFile() {
+ return getOptional("kylin.hbase.cluster.hdfs.config.file", "");
+ }
+
public String getKylinJobLogDir() {
return getOptional("kylin.job.log.dir", "/tmp/kylin/logs");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/af2a8271/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
index d4dd3ae..e93c2bd 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
@@ -20,6 +20,7 @@ package org.apache.kylin.storage.hbase;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
+import java.util.Collection;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
@@ -43,6 +44,7 @@ import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HConnection;
import org.apache.hadoop.hbase.client.HConnectionManager;
import org.apache.hadoop.hbase.util.Threads;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.StorageException;
import org.apache.kylin.engine.mr.HadoopUtil;
@@ -140,6 +142,7 @@ public class HBaseConnection {
private static Configuration newHBaseConfiguration(String url) {
Configuration conf = HBaseConfiguration.create(HadoopUtil.getCurrentConfiguration());
+ addHBaseClusterNNHAConfiguration(conf);
// using a hbase:xxx URL is deprecated, instead hbase config is always loaded from hbase-site.xml in classpath
if (!(StringUtils.isEmpty(url) || "hbase".equals(url)))
@@ -168,6 +171,37 @@ public class HBaseConnection {
return conf;
}
+ // See YARN-3021. Copy here in case of missing in dependency MR client jars
+ public static final String JOB_NAMENODES_TOKEN_RENEWAL_EXCLUDE = "mapreduce.job.hdfs-servers.token-renewal.exclude";
+
+ public static void addHBaseClusterNNHAConfiguration(Configuration conf) {
+ String hdfsConfigFile = KylinConfig.getInstanceFromEnv().getHBaseClusterHDFSConfigFile();
+ if (hdfsConfigFile == null || hdfsConfigFile.isEmpty()) {
+ return;
+ }
+ Configuration hdfsConf = new Configuration(false);
+ hdfsConf.addResource(hdfsConfigFile);
+ Collection<String> nameServices = hdfsConf.getTrimmedStringCollection(DFSConfigKeys.DFS_NAMESERVICES);
+ Collection<String> mainNameServices = conf.getTrimmedStringCollection(DFSConfigKeys.DFS_NAMESERVICES);
+ for (String serviceId : nameServices) {
+ mainNameServices.add(serviceId);
+
+ String serviceConfKey = DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX + "." + serviceId;
+ String proxyConfKey = DFSConfigKeys.DFS_CLIENT_FAILOVER_PROXY_PROVIDER_KEY_PREFIX + "." + serviceId;
+ conf.set(serviceConfKey, hdfsConf.get(serviceConfKey, ""));
+ conf.set(proxyConfKey, hdfsConf.get(proxyConfKey, ""));
+
+ Collection<String> nameNodes = hdfsConf.getTrimmedStringCollection(serviceConfKey);
+ for (String nameNode : nameNodes) {
+ String rpcConfKey = DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY + "." + serviceId + "." + nameNode;
+ conf.set(rpcConfKey, hdfsConf.get(rpcConfKey, ""));
+ }
+ }
+ conf.setStrings(DFSConfigKeys.DFS_NAMESERVICES, mainNameServices.toArray(new String[0]));
+ // See YARN-3021, instruct RM skip renew token of hbase cluster name services
+ conf.setStrings(JOB_NAMENODES_TOKEN_RENEWAL_EXCLUDE, nameServices.toArray(new String[0]));
+ }
+
public static String makeQualifiedPathInHBaseCluster(String path) {
try {
FileSystem fs = FileSystem.get(getCurrentHBaseConfiguration());
http://git-wip-us.apache.org/repos/asf/kylin/blob/af2a8271/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java
index e4a9c1e..9145cef 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java
@@ -42,6 +42,7 @@ import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
import org.apache.kylin.engine.mr.common.BatchConstants;
+import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -75,6 +76,7 @@ public class CubeHFileJob extends AbstractHadoopJob {
job = Job.getInstance(getConf(), getOptionValue(OPTION_JOB_NAME));
setJobClasspath(job, cube.getConfig());
+ HBaseConnection.addHBaseClusterNNHAConfiguration(job.getConfiguration());
addInputDirs(getOptionValue(OPTION_INPUT_PATH), job);
FileOutputFormat.setOutputPath(job, output);