You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by mp...@apache.org on 2014/12/05 23:11:12 UTC
ambari git commit: AMBARI-8566. Remaining changes to Hadoop Sink
implementation. (mpapirkovskyy)
Repository: ambari
Updated Branches:
refs/heads/trunk 1e94bc1ab -> 9e052eed1
AMBARI-8566. Remaining changes to Hadoop Sink implementation. (mpapirkovskyy)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9e052eed
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9e052eed
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9e052eed
Branch: refs/heads/trunk
Commit: 9e052eed1a582072a448e923f50107e00716c04f
Parents: 1e94bc1
Author: Myroslav Papirkovskyy <mp...@hortonworks.com>
Authored: Fri Dec 5 21:43:21 2014 +0200
Committer: Myroslav Papirkovskyy <mp...@hortonworks.com>
Committed: Sat Dec 6 00:11:06 2014 +0200
----------------------------------------------------------------------
ambari-metrics/ambari-metrics-common/pom.xml | 29 +----
.../flume/FlumeTimelineMetricsSinkTest.java | 117 +++++++++++++++++++
.../flume/FlumeTimelineMetricsSinkTest.java | 117 -------------------
.../ambari-metrics-hadoop-sink/pom.xml | 47 ++++++--
.../src/main/assemblies/sink-jar.xml | 37 ++++++
.../timeline/HadoopTimelineMetricsSinkTest.java | 115 ++++++++++++++++++
.../conf/unix/ambari-metrics-monitor | 1 +
.../ambari-metrics-host-monitoring/pom.xml | 7 +-
ambari-metrics/pom.xml | 27 -----
.../templates/hadoop-metrics2.properties.j2 | 4 +-
...-metrics2-hbase.properties-GANGLIA-MASTER.j2 | 10 +-
...doop-metrics2-hbase.properties-GANGLIA-RS.j2 | 8 +-
.../hadoop-metrics2-hbase.properties.j2 | 8 +-
pom.xml | 6 +
14 files changed, 333 insertions(+), 200 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/9e052eed/ambari-metrics/ambari-metrics-common/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-common/pom.xml b/ambari-metrics/ambari-metrics-common/pom.xml
index 786ad93..7bfd9f1 100644
--- a/ambari-metrics/ambari-metrics-common/pom.xml
+++ b/ambari-metrics/ambari-metrics-common/pom.xml
@@ -31,33 +31,6 @@
<artifactId>maven-compiler-plugin</artifactId>
<version>3.0</version>
</plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <version>1.8</version>
- <executions>
- <execution>
- <id>parse-version</id>
- <phase>validate</phase>
- <goals>
- <goal>parse-version</goal>
- </goals>
- </execution>
- <execution>
- <id>regex-property</id>
- <goals>
- <goal>regex-property</goal>
- </goals>
- <configuration>
- <name>ambariVersion</name>
- <value>${project.version}</value>
- <regex>^([0-9]+)\.([0-9]+)\.([0-9]+)(\.|-).*</regex>
- <replacement>$1.$2.$3</replacement>
- <failIfNoMatch>false</failIfNoMatch>
- </configuration>
- </execution>
- </executions>
- </plugin>
</plugins>
</build>
@@ -78,4 +51,4 @@
<version>1.8.0</version>
</dependency>
</dependencies>
-</project>
\ No newline at end of file
+</project>
http://git-wip-us.apache.org/repos/asf/ambari/blob/9e052eed/ambari-metrics/ambari-metrics-flume-sink/src/test/java/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSinkTest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-flume-sink/src/test/java/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSinkTest.java b/ambari-metrics/ambari-metrics-flume-sink/src/test/java/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSinkTest.java
new file mode 100644
index 0000000..0275db6
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-flume-sink/src/test/java/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSinkTest.java
@@ -0,0 +1,117 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.metrics2.sink.flume;
+
+import org.apache.commons.httpclient.HttpClient;
+import org.apache.flume.instrumentation.util.JMXPollUtil;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
+import org.apache.hadoop.metrics2.sink.timeline.cache.TimelineMetricsCache;
+import org.easymock.EasyMock;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+import java.util.Collections;
+
+import static org.powermock.api.easymock.PowerMock.*;
+
+@RunWith(PowerMockRunner.class)
+@PrepareForTest(JMXPollUtil.class)
+public class FlumeTimelineMetricsSinkTest {
+ @Test
+ public void testNonNumericMetricMetricExclusion() throws InterruptedException {
+ FlumeTimelineMetricsSink flumeTimelineMetricsSink = new FlumeTimelineMetricsSink();
+ TimelineMetricsCache timelineMetricsCache = getTimelineMetricsCache(flumeTimelineMetricsSink);
+ flumeTimelineMetricsSink.setPollFrequency(1);
+ HttpClient httpClient = EasyMock.createNiceMock(HttpClient.class);
+ flumeTimelineMetricsSink.setHttpClient(httpClient);
+ mockStatic(JMXPollUtil.class);
+ EasyMock.expect(JMXPollUtil.getAllMBeans()).andReturn(
+ Collections.singletonMap("component1", Collections.singletonMap("key1", "value1"))).once();
+ replay(JMXPollUtil.class, timelineMetricsCache, httpClient);
+ flumeTimelineMetricsSink.start();
+ Thread.sleep(5);
+ flumeTimelineMetricsSink.stop();
+ verifyAll();
+ }
+
+ @Test
+ public void testNumericMetricMetricSubmission() throws InterruptedException {
+ FlumeTimelineMetricsSink flumeTimelineMetricsSink = new FlumeTimelineMetricsSink();
+ TimelineMetricsCache timelineMetricsCache = getTimelineMetricsCache(flumeTimelineMetricsSink);
+ flumeTimelineMetricsSink.setPollFrequency(1);
+ HttpClient httpClient = EasyMock.createNiceMock(HttpClient.class);
+ flumeTimelineMetricsSink.setHttpClient(httpClient);
+ mockStatic(JMXPollUtil.class);
+ EasyMock.expect(JMXPollUtil.getAllMBeans()).andReturn(
+ Collections.singletonMap("component1", Collections.singletonMap("key1", "42"))).once();
+ replay(JMXPollUtil.class, timelineMetricsCache, httpClient);
+ flumeTimelineMetricsSink.start();
+ Thread.sleep(5);
+ flumeTimelineMetricsSink.stop();
+ verifyAll();
+ }
+
+ private TimelineMetricsCache getTimelineMetricsCache(FlumeTimelineMetricsSink flumeTimelineMetricsSink) {
+ TimelineMetricsCache timelineMetricsCache = EasyMock.createNiceMock(TimelineMetricsCache.class);
+ flumeTimelineMetricsSink.setMetricsCache(timelineMetricsCache);
+ EasyMock.expect(timelineMetricsCache.getTimelineMetric("key1"))
+ .andReturn(new TimelineMetric()).once();
+ timelineMetricsCache.putTimelineMetric(EasyMock.anyObject(TimelineMetric.class));
+ EasyMock.expectLastCall().once();
+ return timelineMetricsCache;
+ }
+
+ @Test
+ public void testMonitorRestart() throws InterruptedException {
+ FlumeTimelineMetricsSink flumeTimelineMetricsSink = new FlumeTimelineMetricsSink();
+ TimelineMetricsCache timelineMetricsCache = getTimelineMetricsCache(flumeTimelineMetricsSink);
+ flumeTimelineMetricsSink.setPollFrequency(1);
+ HttpClient httpClient = EasyMock.createNiceMock(HttpClient.class);
+ flumeTimelineMetricsSink.setHttpClient(httpClient);
+ mockStatic(JMXPollUtil.class);
+ EasyMock.expect(JMXPollUtil.getAllMBeans()).andReturn(
+ Collections.singletonMap("component1", Collections.singletonMap("key1", "42"))).once();
+ flumeTimelineMetricsSink.start();
+ flumeTimelineMetricsSink.stop();
+ replay(JMXPollUtil.class, timelineMetricsCache, httpClient);
+ flumeTimelineMetricsSink.start();
+ Thread.sleep(5);
+ flumeTimelineMetricsSink.stop();
+ verifyAll();
+ }
+
+ @Test
+ public void testMetricsRetrievalExceptionTolerance() throws InterruptedException {
+ FlumeTimelineMetricsSink flumeTimelineMetricsSink = new FlumeTimelineMetricsSink();
+ TimelineMetricsCache timelineMetricsCache = getTimelineMetricsCache(flumeTimelineMetricsSink);
+ flumeTimelineMetricsSink.setPollFrequency(1);
+ HttpClient httpClient = EasyMock.createNiceMock(HttpClient.class);
+ flumeTimelineMetricsSink.setHttpClient(httpClient);
+ mockStatic(JMXPollUtil.class);
+ EasyMock.expect(JMXPollUtil.getAllMBeans()).
+ andThrow(new RuntimeException("Failed to retrieve Flume Properties")).once();
+ replay(JMXPollUtil.class, timelineMetricsCache, httpClient);
+ flumeTimelineMetricsSink.start();
+ Thread.sleep(5);
+ flumeTimelineMetricsSink.stop();
+ verifyAll();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/9e052eed/ambari-metrics/ambari-metrics-flume-sink/src/test/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSinkTest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-flume-sink/src/test/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSinkTest.java b/ambari-metrics/ambari-metrics-flume-sink/src/test/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSinkTest.java
deleted file mode 100644
index 0275db6..0000000
--- a/ambari-metrics/ambari-metrics-flume-sink/src/test/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSinkTest.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metrics2.sink.flume;
-
-import org.apache.commons.httpclient.HttpClient;
-import org.apache.flume.instrumentation.util.JMXPollUtil;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
-import org.apache.hadoop.metrics2.sink.timeline.cache.TimelineMetricsCache;
-import org.easymock.EasyMock;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.powermock.core.classloader.annotations.PrepareForTest;
-import org.powermock.modules.junit4.PowerMockRunner;
-
-import java.util.Collections;
-
-import static org.powermock.api.easymock.PowerMock.*;
-
-@RunWith(PowerMockRunner.class)
-@PrepareForTest(JMXPollUtil.class)
-public class FlumeTimelineMetricsSinkTest {
- @Test
- public void testNonNumericMetricMetricExclusion() throws InterruptedException {
- FlumeTimelineMetricsSink flumeTimelineMetricsSink = new FlumeTimelineMetricsSink();
- TimelineMetricsCache timelineMetricsCache = getTimelineMetricsCache(flumeTimelineMetricsSink);
- flumeTimelineMetricsSink.setPollFrequency(1);
- HttpClient httpClient = EasyMock.createNiceMock(HttpClient.class);
- flumeTimelineMetricsSink.setHttpClient(httpClient);
- mockStatic(JMXPollUtil.class);
- EasyMock.expect(JMXPollUtil.getAllMBeans()).andReturn(
- Collections.singletonMap("component1", Collections.singletonMap("key1", "value1"))).once();
- replay(JMXPollUtil.class, timelineMetricsCache, httpClient);
- flumeTimelineMetricsSink.start();
- Thread.sleep(5);
- flumeTimelineMetricsSink.stop();
- verifyAll();
- }
-
- @Test
- public void testNumericMetricMetricSubmission() throws InterruptedException {
- FlumeTimelineMetricsSink flumeTimelineMetricsSink = new FlumeTimelineMetricsSink();
- TimelineMetricsCache timelineMetricsCache = getTimelineMetricsCache(flumeTimelineMetricsSink);
- flumeTimelineMetricsSink.setPollFrequency(1);
- HttpClient httpClient = EasyMock.createNiceMock(HttpClient.class);
- flumeTimelineMetricsSink.setHttpClient(httpClient);
- mockStatic(JMXPollUtil.class);
- EasyMock.expect(JMXPollUtil.getAllMBeans()).andReturn(
- Collections.singletonMap("component1", Collections.singletonMap("key1", "42"))).once();
- replay(JMXPollUtil.class, timelineMetricsCache, httpClient);
- flumeTimelineMetricsSink.start();
- Thread.sleep(5);
- flumeTimelineMetricsSink.stop();
- verifyAll();
- }
-
- private TimelineMetricsCache getTimelineMetricsCache(FlumeTimelineMetricsSink flumeTimelineMetricsSink) {
- TimelineMetricsCache timelineMetricsCache = EasyMock.createNiceMock(TimelineMetricsCache.class);
- flumeTimelineMetricsSink.setMetricsCache(timelineMetricsCache);
- EasyMock.expect(timelineMetricsCache.getTimelineMetric("key1"))
- .andReturn(new TimelineMetric()).once();
- timelineMetricsCache.putTimelineMetric(EasyMock.anyObject(TimelineMetric.class));
- EasyMock.expectLastCall().once();
- return timelineMetricsCache;
- }
-
- @Test
- public void testMonitorRestart() throws InterruptedException {
- FlumeTimelineMetricsSink flumeTimelineMetricsSink = new FlumeTimelineMetricsSink();
- TimelineMetricsCache timelineMetricsCache = getTimelineMetricsCache(flumeTimelineMetricsSink);
- flumeTimelineMetricsSink.setPollFrequency(1);
- HttpClient httpClient = EasyMock.createNiceMock(HttpClient.class);
- flumeTimelineMetricsSink.setHttpClient(httpClient);
- mockStatic(JMXPollUtil.class);
- EasyMock.expect(JMXPollUtil.getAllMBeans()).andReturn(
- Collections.singletonMap("component1", Collections.singletonMap("key1", "42"))).once();
- flumeTimelineMetricsSink.start();
- flumeTimelineMetricsSink.stop();
- replay(JMXPollUtil.class, timelineMetricsCache, httpClient);
- flumeTimelineMetricsSink.start();
- Thread.sleep(5);
- flumeTimelineMetricsSink.stop();
- verifyAll();
- }
-
- @Test
- public void testMetricsRetrievalExceptionTolerance() throws InterruptedException {
- FlumeTimelineMetricsSink flumeTimelineMetricsSink = new FlumeTimelineMetricsSink();
- TimelineMetricsCache timelineMetricsCache = getTimelineMetricsCache(flumeTimelineMetricsSink);
- flumeTimelineMetricsSink.setPollFrequency(1);
- HttpClient httpClient = EasyMock.createNiceMock(HttpClient.class);
- flumeTimelineMetricsSink.setHttpClient(httpClient);
- mockStatic(JMXPollUtil.class);
- EasyMock.expect(JMXPollUtil.getAllMBeans()).
- andThrow(new RuntimeException("Failed to retrieve Flume Properties")).once();
- replay(JMXPollUtil.class, timelineMetricsCache, httpClient);
- flumeTimelineMetricsSink.start();
- Thread.sleep(5);
- flumeTimelineMetricsSink.stop();
- verifyAll();
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/9e052eed/ambari-metrics/ambari-metrics-hadoop-sink/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-hadoop-sink/pom.xml b/ambari-metrics/ambari-metrics-hadoop-sink/pom.xml
index 1e854e2..76f9fce 100644
--- a/ambari-metrics/ambari-metrics-hadoop-sink/pom.xml
+++ b/ambari-metrics/ambari-metrics-hadoop-sink/pom.xml
@@ -32,20 +32,36 @@ limitations under the License.
<plugins>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
- <configuration>
- <descriptors>
- <descriptor>src/main/assemblies/sink.xml</descriptor>
- </descriptors>
- <tarLongFileMode>gnu</tarLongFileMode>
- </configuration>
<executions>
<execution>
+ <configuration>
+ <descriptors>
+ <descriptor>src/main/assemblies/sink.xml</descriptor>
+ </descriptors>
+ <tarLongFileMode>gnu</tarLongFileMode>
+ </configuration>
<id>build-tarball</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
+
+ <execution>
+ <configuration>
+ <descriptors>
+ <descriptor>src/main/assemblies/sink-jar.xml</descriptor>
+ </descriptors>
+ <tarLongFileMode>gnu</tarLongFileMode>
+ <appendAssemblyId>false</appendAssemblyId>
+ <finalName>${project.artifactId}-with-common-${project.version}</finalName>
+ </configuration>
+ <id>build-jar</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
</executions>
</plugin>
<plugin>
@@ -115,11 +131,11 @@ limitations under the License.
<groupname>root</groupname>
<sources>
<source>
- <location>target/${project.artifactId}-${project.version}.jar</location>
+ <location>target/${project.artifactId}-with-common-${project.version}.jar</location>
</source>
<softlinkSource>
<destination>ambari-metrics-hadoop-sink.jar</destination>
- <location>/usr/lib/ambari-metrics-hadoop-sink/${project.artifactId}-${project.version}.jar</location>
+ <location>/usr/lib/ambari-metrics-hadoop-sink/${project.artifactId}-with-common-${project.version}.jar</location>
</softlinkSource>
</sources>
@@ -135,7 +151,7 @@ limitations under the License.
<dependency>
<groupId>org.apache.ambari</groupId>
<artifactId>ambari-metrics-common</artifactId>
- <version>0.1.0-SNAPSHOT</version>
+ <version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -179,9 +195,16 @@ limitations under the License.
<scope>compile</scope>
</dependency>
<dependency>
- <groupId>org.codehaus.jackson</groupId>
- <artifactId>jackson-core-asl</artifactId>
- <version>1.8.0</version>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ <version>4.10</version>
+ </dependency>
+ <dependency>
+ <groupId>org.easymock</groupId>
+ <artifactId>easymock</artifactId>
+ <version>3.2</version>
+ <scope>test</scope>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/ambari/blob/9e052eed/ambari-metrics/ambari-metrics-hadoop-sink/src/main/assemblies/sink-jar.xml
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-hadoop-sink/src/main/assemblies/sink-jar.xml b/ambari-metrics/ambari-metrics-hadoop-sink/src/main/assemblies/sink-jar.xml
new file mode 100644
index 0000000..83c3099
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-hadoop-sink/src/main/assemblies/sink-jar.xml
@@ -0,0 +1,37 @@
+<?xml version='1.0'?>
+
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<assembly>
+ <id>hadoop-sink-jar</id>
+ <formats>
+ <format>jar</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <dependencySets>
+ <dependencySet>
+ <outputDirectory>/</outputDirectory>
+ <unpack>true</unpack>
+ <includes>
+ <include>org.apache.ambari:ambari-metrics-common</include>
+ <include>org.apache.ambari:ambari-metrics-hadoop-sink</include>
+ </includes>
+ </dependencySet>
+ </dependencySets>
+</assembly>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/9e052eed/ambari-metrics/ambari-metrics-hadoop-sink/src/test/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSinkTest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-hadoop-sink/src/test/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSinkTest.java b/ambari-metrics/ambari-metrics-hadoop-sink/src/test/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSinkTest.java
new file mode 100644
index 0000000..2421a73
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-hadoop-sink/src/test/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSinkTest.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.metrics2.sink.timeline;
+
+import org.apache.commons.configuration.SubsetConfiguration;
+import org.apache.commons.httpclient.HttpClient;
+import org.apache.commons.httpclient.methods.PostMethod;
+import org.apache.hadoop.metrics2.AbstractMetric;
+import org.apache.hadoop.metrics2.MetricsRecord;
+import org.apache.hadoop.metrics2.sink.timeline.base.AbstractTimelineMetricsSink;
+import org.easymock.IAnswer;
+
+import java.util.Arrays;
+import java.util.Iterator;
+
+import static org.apache.hadoop.metrics2.sink.timeline.base.AbstractTimelineMetricsSink.*;
+import static org.easymock.EasyMock.anyInt;
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.createNiceMock;
+import static org.easymock.EasyMock.eq;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.expectLastCall;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.verify;
+
+public class HadoopTimelineMetricsSinkTest {
+
+ @org.junit.Test
+ public void testPutMetrics() throws Exception {
+ HadoopTimelineMetricsSink sink = new HadoopTimelineMetricsSink();
+
+ SubsetConfiguration conf = createNiceMock(SubsetConfiguration.class);
+ expect(conf.getString(eq("slave.host.name"))).andReturn("testhost").anyTimes();
+ expect(conf.getParent()).andReturn(null).anyTimes();
+ expect(conf.getPrefix()).andReturn("service").anyTimes();
+ expect(conf.getString(eq(COLLECTOR_HOST_PROPERTY))).andReturn("localhost:63188").anyTimes();
+
+ expect(conf.getInt(eq(MAX_METRIC_ROW_CACHE_SIZE), anyInt())).andReturn(10).anyTimes();
+ expect(conf.getInt(eq(METRICS_SEND_INTERVAL), anyInt())).andReturn(1000).anyTimes();
+
+ conf.setListDelimiter(eq(','));
+ expectLastCall().anyTimes();
+
+ expect(conf.getKeys()).andReturn(new Iterator() {
+ @Override
+ public boolean hasNext() {
+ return false;
+ }
+
+ @Override
+ public Object next() {
+ return null;
+ }
+
+ @Override
+ public void remove() {
+
+ }
+ }).once();
+
+
+ HttpClient httpClient = createNiceMock(HttpClient.class);
+
+ expect(httpClient.executeMethod(anyObject(PostMethod.class))).andReturn(200).once(); //metrics send only once due to caching
+
+ AbstractMetric metric = createNiceMock(AbstractMetric.class);
+ expect(metric.name()).andReturn("metricName").anyTimes();
+ expect(metric.value()).andReturn(9.5687).anyTimes();
+ //TODO currently only numeric metrics are supported
+
+ MetricsRecord record = createNiceMock(MetricsRecord.class);
+ expect(record.name()).andReturn("testName").anyTimes();
+ expect(record.context()).andReturn("testContext").anyTimes();
+ expect(record.timestamp()).andAnswer(new IAnswer<Long>() {
+ @Override
+ public Long answer() throws Throwable {
+ return System.currentTimeMillis();
+ }
+ }).anyTimes();
+
+ expect(record.metrics()).andReturn(Arrays.asList(metric)).anyTimes();
+
+
+ replay(conf, httpClient, record, metric);
+
+ sink.setHttpClient(httpClient);
+ sink.init(conf);
+
+ sink.putMetrics(record);
+
+ Thread.sleep(1500L);
+
+ sink.putMetrics(record);
+
+ verify(conf, httpClient, record, metric);
+
+
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/9e052eed/ambari-metrics/ambari-metrics-host-monitoring/conf/unix/ambari-metrics-monitor
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-host-monitoring/conf/unix/ambari-metrics-monitor b/ambari-metrics/ambari-metrics-host-monitoring/conf/unix/ambari-metrics-monitor
index fd086d9..9198ce9 100644
--- a/ambari-metrics/ambari-metrics-host-monitoring/conf/unix/ambari-metrics-monitor
+++ b/ambari-metrics/ambari-metrics-host-monitoring/conf/unix/ambari-metrics-monitor
@@ -67,6 +67,7 @@ function write_pidfile
fi
}
+#TODO decide if rebuild on each start (pretty quickly) to tolerate major node changes (like kernel update)
#build psutil
if [ ! "$(ls -A ${RESOURCE_MONITORING_DIR}/psutil/build)" ]; then
echo "Building psutil..."
http://git-wip-us.apache.org/repos/asf/ambari/blob/9e052eed/ambari-metrics/ambari-metrics-host-monitoring/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-host-monitoring/pom.xml b/ambari-metrics/ambari-metrics-host-monitoring/pom.xml
index b354461..ce396df 100644
--- a/ambari-metrics/ambari-metrics-host-monitoring/pom.xml
+++ b/ambari-metrics/ambari-metrics-host-monitoring/pom.xml
@@ -120,6 +120,7 @@
<autoRequires>false</autoRequires>
<requires>
<require>ambari-metrics-hadoop-sink</require>
+ <require>ambari-metrics-flume-sink</require>
<require>${python.ver}</require>
<require>gcc</require>
<require>python-devel</require>
@@ -172,6 +173,10 @@
<location>
${project.basedir}/src/main/python/psutil
</location>
+ <excludes>
+ <exclude>build/**</exclude>
+ <exclude>build/*</exclude>
+ </excludes>
</source>
</sources>
</mapping>
@@ -208,7 +213,7 @@
<executions>
<execution>
<id>psutils-compile</id>
- <phase>process-classes</phase>
+ <phase>process-test-classes</phase>
<goals>
<goal>run</goal>
</goals>
http://git-wip-us.apache.org/repos/asf/ambari/blob/9e052eed/ambari-metrics/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-metrics/pom.xml b/ambari-metrics/pom.xml
index b090205..12c571b 100644
--- a/ambari-metrics/pom.xml
+++ b/ambari-metrics/pom.xml
@@ -67,33 +67,6 @@
</executions>
</plugin>
<plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <version>1.8</version>
- <executions>
- <execution>
- <id>parse-version</id>
- <phase>validate</phase>
- <goals>
- <goal>parse-version</goal>
- </goals>
- </execution>
- <execution>
- <id>regex-property</id>
- <goals>
- <goal>regex-property</goal>
- </goals>
- <configuration>
- <name>ambariVersion</name>
- <value>${project.version}</value>
- <regex>^([0-9]+)\.([0-9]+)\.([0-9]+)(\.|-).*</regex>
- <replacement>$1.$2.$3</replacement>
- <failIfNoMatch>false</failIfNoMatch>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.0</version>
</plugin>
http://git-wip-us.apache.org/repos/asf/ambari/blob/9e052eed/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/templates/hadoop-metrics2.properties.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/templates/hadoop-metrics2.properties.j2 b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/templates/hadoop-metrics2.properties.j2
index 31ca5c3..d7a66ab 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/templates/hadoop-metrics2.properties.j2
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/templates/hadoop-metrics2.properties.j2
@@ -67,7 +67,7 @@ resourcemanager.sink.ganglia.tagsForPrefix.yarn=Queue
{% if has_metric_collector %}
*.period=60
-*.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.TimelineMetricsSink
+*.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink
*.sink.timeline.period=10
datanode.sink.timeline.collector={{metric_collector_host}}:8188
namenode.sink.timeline.collector={{metric_collector_host}}:8188
@@ -80,4 +80,4 @@ supervisor.sink.timeline.collector={{metric_collector_host}}:8188
maptask.sink.timeline.collector={{metric_collector_host}}:8188
reducetask.sink.timeline.collector={{metric_collector_host}}:8188
-{% endif %}
\ No newline at end of file
+{% endif %}
http://git-wip-us.apache.org/repos/asf/ambari/blob/9e052eed/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2 b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2
index fc033a7..2f13b28 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2
@@ -86,20 +86,20 @@ hbase.sink.ganglia.servers={{ganglia_server_host}}:8663
# If this variable is left out, then the default is no expiration.
hbase.extendedperiod = 3600
-hbase.class=org.apache.hadoop.metrics2.sink.timeline.TimelineMetricsSink
+hbase.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink
hbase.period=10
hbase.collector={{metric_collector_host}}:8188
-jvm.class=org.apache.hadoop.metrics2.sink.timeline.TimelineMetricsSink
+jvm.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink
jvm.period=10
jvm.collector={{metric_collector_host}}:8188
-rpc.class=org.apache.hadoop.metrics2.sink.timeline.TimelineMetricsSink
+rpc.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink
rpc.period=10
rpc.collector={{metric_collector_host}}:8188
-hbase.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.TimelineMetricsSink
+hbase.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink
hbase.sink.timeline.period=10
hbase.sink.timeline.collector={{metric_collector_host}}:8188
-{% endif %}
\ No newline at end of file
+{% endif %}
http://git-wip-us.apache.org/repos/asf/ambari/blob/9e052eed/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2 b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2
index 97e76d8..1ca3a98 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2
@@ -85,19 +85,19 @@ hbase.sink.ganglia.servers={{ganglia_server_host}}:8656
# If this variable is left out, then the default is no expiration.
hbase.extendedperiod = 3600
-hbase.class=org.apache.hadoop.metrics2.sink.timeline.TimelineMetricsSink
+hbase.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink
hbase.period=10
hbase.collector={{metric_collector_host}}:8188
-jvm.class=org.apache.hadoop.metrics2.sink.timeline.TimelineMetricsSink
+jvm.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink
jvm.period=10
jvm.collector={{metric_collector_host}}:8188
-rpc.class=org.apache.hadoop.metrics2.sink.timeline.TimelineMetricsSink
+rpc.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink
rpc.period=10
rpc.collector={{metric_collector_host}}:8188
-hbase.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.TimelineMetricsSink
+hbase.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink
hbase.sink.timeline.period=10
hbase.sink.timeline.collector={{metric_collector_host}}:8188
http://git-wip-us.apache.org/repos/asf/ambari/blob/9e052eed/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/templates/hadoop-metrics2-hbase.properties.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/templates/hadoop-metrics2-hbase.properties.j2 b/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/templates/hadoop-metrics2-hbase.properties.j2
index 8dc4bea..0d3ce44 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/templates/hadoop-metrics2-hbase.properties.j2
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/templates/hadoop-metrics2-hbase.properties.j2
@@ -38,18 +38,18 @@
# If this variable is left out, then the default is no expiration.
hbase.extendedperiod = 3600
-hbase.class=org.apache.hadoop.metrics2.sink.timeline.TimelineMetricsSink
+hbase.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink
hbase.period=10
hbase.collector={{ams_collector_host_single}}:8188
-jvm.class=org.apache.hadoop.metrics2.sink.timeline.TimelineMetricsSink
+jvm.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink
jvm.period=10
jvm.collector={{ams_collector_host_single}}:8188
-rpc.class=org.apache.hadoop.metrics2.sink.timeline.TimelineMetricsSink
+rpc.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink
rpc.period=10
rpc.collector={{ams_collector_host_single}}:8188
-hbase.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.TimelineMetricsSink
+hbase.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink
hbase.sink.timeline.period=10
hbase.sink.timeline.collector={{ams_collector_host_single}}:8188
http://git-wip-us.apache.org/repos/asf/ambari/blob/9e052eed/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 87fb4b8..79f951b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -361,6 +361,12 @@
</modules>
</profile>
<profile>
+ <id>ambari-metrics</id>
+ <modules>
+ <module>ambari-metrics</module>
+ </modules>
+ </profile>
+ <profile>
<id>windows</id>
<activation>
<os>