You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by vr...@apache.org on 2018/01/19 23:35:15 UTC
hadoop git commit: YARN-7537 [Atsv2] load hbase configuration from
filesystem rather than URL. Contributed by Rohith Sharma
Repository: hadoop
Updated Branches:
refs/heads/trunk cce71dcee -> ec8f47e7f
YARN-7537 [Atsv2] load hbase configuration from filesystem rather than URL. Contributed by Rohith Sharma
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ec8f47e7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ec8f47e7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ec8f47e7
Branch: refs/heads/trunk
Commit: ec8f47e7fadbe62c0c39390d0a46cefd50e98492
Parents: cce71dc
Author: Vrushali C <vr...@apache.org>
Authored: Fri Jan 19 15:34:40 2018 -0800
Committer: Vrushali C <vr...@apache.org>
Committed: Fri Jan 19 15:34:40 2018 -0800
----------------------------------------------------------------------
.../src/main/resources/yarn-default.xml | 2 +-
.../common/HBaseTimelineStorageUtils.java | 40 ++++++++++++++------
2 files changed, 29 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec8f47e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 017799a..b83673f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -2455,7 +2455,7 @@
</property>
<property>
- <description> Optional URL to an hbase-site.xml configuration file to be
+ <description> Optional FS path to an hbase-site.xml configuration file to be
used to connect to the timeline-service hbase cluster. If empty or not
specified, then the HBase configuration will be loaded from the classpath.
When specified the values in the specified configuration file will override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec8f47e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/HBaseTimelineStorageUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/HBaseTimelineStorageUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/HBaseTimelineStorageUtils.java
index c115b18..c25a0d2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/HBaseTimelineStorageUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/HBaseTimelineStorageUtils.java
@@ -18,13 +18,14 @@
package org.apache.hadoop.yarn.server.timelineservice.storage.common;
import java.io.IOException;
-import java.net.MalformedURLException;
-import java.net.URL;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HBaseConfiguration;
@@ -269,28 +270,43 @@ public final class HBaseTimelineStorageUtils {
* @return a configuration with the HBase configuration from the classpath,
* optionally overwritten by the timeline service configuration URL if
* specified.
- * @throws MalformedURLException if a timeline service HBase configuration URL
- * is specified but is a malformed URL.
+ * @throws IOException if a timeline service HBase configuration path
+ * is specified but unable to read it.
*/
public static Configuration getTimelineServiceHBaseConf(Configuration conf)
- throws MalformedURLException {
+ throws IOException {
if (conf == null) {
throw new NullPointerException();
}
Configuration hbaseConf;
- String timelineServiceHBaseConfFileURL =
+ String timelineServiceHBaseConfFilePath =
conf.get(YarnConfiguration.TIMELINE_SERVICE_HBASE_CONFIGURATION_FILE);
- if (timelineServiceHBaseConfFileURL != null
- && timelineServiceHBaseConfFileURL.length() > 0) {
+ if (timelineServiceHBaseConfFilePath != null
+ && timelineServiceHBaseConfFilePath.length() > 0) {
LOG.info("Using hbase configuration at " +
- timelineServiceHBaseConfFileURL);
+ timelineServiceHBaseConfFilePath);
// create a clone so that we don't mess with out input one
hbaseConf = new Configuration(conf);
Configuration plainHBaseConf = new Configuration(false);
- URL hbaseSiteXML = new URL(timelineServiceHBaseConfFileURL);
- plainHBaseConf.addResource(hbaseSiteXML);
- HBaseConfiguration.merge(hbaseConf, plainHBaseConf);
+ FileSystem fs = null;
+ FSDataInputStream in = null;
+ try {
+ Path hbaseConfigPath = new Path(timelineServiceHBaseConfFilePath);
+ fs = FileSystem.newInstance(conf);
+ in = fs.open(hbaseConfigPath);
+ plainHBaseConf.addResource(in);
+ HBaseConfiguration.merge(hbaseConf, plainHBaseConf);
+ } catch (IOException e) {
+ throw e;
+ } finally {
+ if (in != null) {
+ in.close();
+ }
+ if (fs != null) {
+ fs.close();
+ }
+ }
} else {
// default to what is on the classpath
hbaseConf = HBaseConfiguration.create(conf);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org