You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by av...@apache.org on 2016/11/16 20:52:40 UTC
[1/2] ambari git commit: AMBARI-17291 zookeeper.quorum in
storm-metrics2.properties is broken (Masahiro Tanaka via avijayan)
Repository: ambari
Updated Branches:
refs/heads/branch-2.5 43d30212f -> 280bb6ecf
AMBARI-17291 zookeeper.quorum in storm-metrics2.properties is broken (Masahiro Tanaka via avijayan)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9dae770a
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9dae770a
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9dae770a
Branch: refs/heads/branch-2.5
Commit: 9dae770a4b90dbc6116a0298ed322880ab732173
Parents: 43d3021
Author: Aravindan Vijayan <av...@hortonworks.com>
Authored: Wed Nov 16 12:25:51 2016 -0800
Committer: Aravindan Vijayan <av...@hortonworks.com>
Committed: Wed Nov 16 12:51:30 2016 -0800
----------------------------------------------------------------------
.../STORM/0.9.1/package/scripts/params_linux.py | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/9dae770a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py
index ff590ee..e753e98 100644
--- a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py
@@ -24,6 +24,7 @@ import ambari_simplejson as json # simplejson is much faster comparing to Python
import status_params
from ambari_commons.constants import AMBARI_SUDO_BINARY
+from ambari_commons import yaml_utils
from resource_management.libraries.functions import format
from resource_management.libraries.functions.default import default
from resource_management.libraries.functions.get_bare_principal import get_bare_principal
@@ -202,9 +203,9 @@ metric_collector_legacy_sink_jar = "/usr/lib/storm/lib/ambari-metrics-storm-sink
# Cluster Zookeeper quorum
zookeeper_quorum = ""
if storm_zookeeper_servers:
- for server in storm_zookeeper_servers:
- zookeeper_quorum += server + ':' + storm_zookeeper_port + ","
- zookeeper_quorum = zookeeper_quorum[:-1]
+ storm_zookeeper_servers_list = yaml_utils.get_values_from_yaml_array(storm_zookeeper_servers)
+ zookeeper_quorum = (":" + storm_zookeeper_port + ",").join(storm_zookeeper_servers_list)
+ zookeeper_quorum += ":" + storm_zookeeper_port
jar_jvm_opts = ''
[2/2] ambari git commit: AMBARI-18911 : Storm start is failing due to
metrics initialization error (avijayan)
Posted by av...@apache.org.
AMBARI-18911 : Storm start is failing due to metrics initialization error (avijayan)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/280bb6ec
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/280bb6ec
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/280bb6ec
Branch: refs/heads/branch-2.5
Commit: 280bb6ecfac05e32de104edc4e9532d44580dc93
Parents: 9dae770
Author: Aravindan Vijayan <av...@hortonworks.com>
Authored: Wed Nov 16 12:51:25 2016 -0800
Committer: Aravindan Vijayan <av...@hortonworks.com>
Committed: Wed Nov 16 12:51:31 2016 -0800
----------------------------------------------------------------------
ambari-metrics/ambari-metrics-common/pom.xml | 9 ++
.../timeline/AbstractTimelineMetricsSink.java | 20 ++--
.../AbstractTimelineMetricSinkTest.java | 98 ++++++++++++++++++++
.../storm/StormTimelineMetricsReporter.java | 1 +
4 files changed, 117 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/280bb6ec/ambari-metrics/ambari-metrics-common/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-common/pom.xml b/ambari-metrics/ambari-metrics-common/pom.xml
index c5d83d0..c329d7a 100644
--- a/ambari-metrics/ambari-metrics-common/pom.xml
+++ b/ambari-metrics/ambari-metrics-common/pom.xml
@@ -77,6 +77,10 @@
<shadedPattern>org.apache.hadoop.metrics2.sink.relocated.commons.io</shadedPattern>
</relocation>
<relocation>
+ <pattern>org.apache.commons.lang</pattern>
+ <shadedPattern>org.apache.hadoop.metrics2.sink.relocated.commons.lang</shadedPattern>
+ </relocation>
+ <relocation>
<pattern>org.apache.curator</pattern>
<shadedPattern>org.apache.hadoop.metrics2.sink.relocated.curator</shadedPattern>
</relocation>
@@ -154,6 +158,11 @@
<version>1.9.13</version>
</dependency>
<dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ <version>2.5</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-math3</artifactId>
<version>3.1.1</version>
http://git-wip-us.apache.org/repos/asf/ambari/blob/280bb6ec/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java
index efa5cba..173e675 100644
--- a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java
+++ b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java
@@ -23,6 +23,7 @@ import com.google.common.reflect.TypeToken;
import com.google.gson.Gson;
import com.google.gson.JsonSyntaxException;
import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.metrics2.sink.timeline.availability.MetricCollectorHAHelper;
@@ -477,24 +478,21 @@ public abstract class AbstractTimelineMetricsSink {
/**
* Parses input Sting of format "['host1', 'host2']" into Collection of hostnames
*/
- protected Collection<String> parseHostsStringIntoCollection(String hostsString) {
+ public Collection<String> parseHostsStringIntoCollection(String hostsString) {
Set<String> hosts = new HashSet<>();
- if (hostsString == null) {
+ if (StringUtils.isEmpty(hostsString)) {
LOG.error("No Metric collector configured.");
return hosts;
}
- hostsString = hostsString.replace("[", "");
- hostsString = hostsString.replace("]", "");
- hostsString = hostsString.replace("'", "");
+ String[] untrimmedHosts = hostsString.split(",");
- String [] hostNamesWithApostrophes = hostsString.split(",");
-
- for (String host : hostNamesWithApostrophes) {
- host = host.trim();
- if (host.equals("")) continue;
- hosts.add(host);
+ for (String host : untrimmedHosts) {
+ host = StringUtils.substringBetween(host, "'");
+ if (StringUtils.isEmpty(host))
+ continue;
+ hosts.add(host.trim());
}
return hosts;
http://git-wip-us.apache.org/repos/asf/ambari/blob/280bb6ec/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/AbstractTimelineMetricSinkTest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/AbstractTimelineMetricSinkTest.java b/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/AbstractTimelineMetricSinkTest.java
new file mode 100644
index 0000000..5e016f8
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/AbstractTimelineMetricSinkTest.java
@@ -0,0 +1,98 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.metrics2.sink.timeline.availability;
+
+import junit.framework.Assert;
+import org.apache.hadoop.metrics2.sink.timeline.AbstractTimelineMetricsSink;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.Collection;
+
+public class AbstractTimelineMetricSinkTest {
+
+ @Test
+ public void testParseHostsStringIntoCollection() {
+ AbstractTimelineMetricsSink sink = new TestTimelineMetricsSink();
+ Collection<String> hosts;
+
+ hosts = sink.parseHostsStringIntoCollection("[]");
+ Assert.assertTrue(hosts.isEmpty());
+
+ hosts = sink.parseHostsStringIntoCollection("[u'test1.123.abc.def.local']");
+ Assert.assertTrue(hosts.size() == 1);
+ Assert.assertTrue(hosts.contains("test1.123.abc.def.local"));
+
+ hosts = sink.parseHostsStringIntoCollection("['test1.123.abc.def.local']");
+ Assert.assertTrue(hosts.size() == 1);
+ Assert.assertTrue(hosts.contains("test1.123.abc.def.local"));
+
+ hosts = sink.parseHostsStringIntoCollection("[u'test1.123.abc.def.local', u'test1.456.abc.def.local']");
+ Assert.assertTrue(hosts.size() == 2);
+
+ hosts = sink.parseHostsStringIntoCollection("['test1.123.abc.def.local', 'test1.456.abc.def.local']");
+ Assert.assertTrue(hosts.size() == 2);
+ Assert.assertTrue(hosts.contains("test1.123.abc.def.local"));
+ Assert.assertTrue(hosts.contains("test1.456.abc.def.local"));
+
+ }
+
+ private class TestTimelineMetricsSink extends AbstractTimelineMetricsSink {
+ @Override
+ protected String getCollectorUri(String host) {
+ return "";
+ }
+
+ @Override
+ protected String getCollectorProtocol() {
+ return "http";
+ }
+
+ @Override
+ protected String getCollectorPort() {
+ return "2181";
+ }
+
+ @Override
+ protected int getTimeoutSeconds() {
+ return 10;
+ }
+
+ @Override
+ protected String getZookeeperQuorum() {
+ return "localhost:2181";
+ }
+
+ @Override
+ protected Collection<String> getConfiguredCollectorHosts() {
+ return Arrays.asList("localhost");
+ }
+
+ @Override
+ protected String getHostname() {
+ return "h1";
+ }
+
+ @Override
+ public boolean emitMetrics(TimelineMetrics metrics) {
+ super.init();
+ return super.emitMetrics(metrics);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/280bb6ec/ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java b/ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java
index 5c61e03..9329aae 100644
--- a/ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java
+++ b/ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java
@@ -105,6 +105,7 @@ public class StormTimelineMetricsReporter extends AbstractTimelineMetricsSink
collectorHosts = parseHostsStringIntoCollection(conf.getProperty(COLLECTOR_HOSTS_PROPERTY).toString());
port = conf.getProperty(COLLECTOR_PORT) != null ? conf.getProperty(COLLECTOR_PORT) : "6188";
+ protocol = conf.getProperty(COLLECTOR_PROTOCOL, "http");
zkQuorum = conf.getProperty(ZOOKEEPER_QUORUM) != null ? conf.getProperty(ZOOKEEPER_QUORUM) : null;
timeoutSeconds = conf.getProperty(METRICS_POST_TIMEOUT_SECONDS) != null ?