You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@eagle.apache.org by ha...@apache.org on 2016/10/24 11:38:12 UTC
incubator-eagle git commit: [EAGLE-650] Hadoop JMX Metric Monitoring
Application
Repository: incubator-eagle
Updated Branches:
refs/heads/master 3ff3e55ac -> 698b30282
[EAGLE-650] Hadoop JMX Metric Monitoring Application
Hadoop JMX Metric Monitoring Application
https://issues.apache.org/jira/browse/EAGLE-650
Monitored Services:
* hdfs name node
* yarn resource manager node
* hbase master node
Author: Hao Chen <ha...@apache.org>
Closes #552 from haoch/EAGLE-650.
Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/698b3028
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/698b3028
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/698b3028
Branch: refs/heads/master
Commit: 698b302821c7a89cee1172dbf8d2cdb296506b54
Parents: 3ff3e55
Author: Hao Chen <ha...@apache.org>
Authored: Mon Oct 24 19:37:58 2016 +0800
Committer: Hao Chen <ha...@apache.org>
Committed: Mon Oct 24 19:37:58 2016 +0800
----------------------------------------------------------------------
...e.alert.app.AlertUnitTopologyAppProvider.xml | 3 +-
.../app/environment/impl/StaticEnvironment.java | 19 +---
.../eagle/app/service/ApplicationAction.java | 14 +--
.../apache/eagle/app/sink/KafkaStreamSink.java | 39 ++++++--
.../app/environment/StaticEnvironmentTest.java | 10 ++-
.../apache/eagle/metadata/model/StreamDesc.java | 3 +-
.../metadata/utils/StreamIdConversions.java | 46 ++++++++++
.../metadata/utils/StreamIdConversionsTest.java | 37 ++++++++
...e.app.example.ExampleApplicationProvider.xml | 3 +-
...apache.eagle.gc.GCLogApplicationProvider.xml | 5 +-
eagle-hadoop-metric/pom.xml | 6 +-
.../metric/HadoopMetricMonitorAppProdiver.java | 23 +++++
...le.metric.HadoopMetricMonitorAppProdiver.xml | 94 ++++++++++++++++++++
...org.apache.eagle.app.spi.ApplicationProvider | 16 ++++
.../HadoopMetricMonitorAppProdiverTest.java | 84 +++++++++++++++++
...doop.queue.HadoopQueueRunningAppProvider.xml | 2 +-
...gregation.AggregationApplicationProvider.xml | 2 +-
....security.hbase.HBaseAuditLogAppProvider.xml | 4 +-
...ecurity.auditlog.HdfsAuditLogAppProvider.xml | 2 +-
eagle-server/pom.xml | 7 ++
20 files changed, 370 insertions(+), 49 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/698b3028/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/providers/org.apache.eagle.alert.app.AlertUnitTopologyAppProvider.xml
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/providers/org.apache.eagle.alert.app.AlertUnitTopologyAppProvider.xml b/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/providers/org.apache.eagle.alert.app.AlertUnitTopologyAppProvider.xml
index 28f7db4..7b12917 100644
--- a/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/providers/org.apache.eagle.alert.app.AlertUnitTopologyAppProvider.xml
+++ b/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/providers/org.apache.eagle.alert.app.AlertUnitTopologyAppProvider.xml
@@ -18,7 +18,8 @@
<application>
<type>AlertUnitTopologyApp</type>
- <name>Alert Unit Topology Application</name>
+ <name>Alert Engine</name>
+ <description>Real-time Alert Engine</description>
<version>0.5.0-incubating</version>
<configuration>
<!-- alert topology sizing parameters -->
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/698b3028/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/impl/StaticEnvironment.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/impl/StaticEnvironment.java b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/impl/StaticEnvironment.java
index 1c297fb..1c17b1f 100644
--- a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/impl/StaticEnvironment.java
+++ b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/impl/StaticEnvironment.java
@@ -17,23 +17,10 @@
package org.apache.eagle.app.environment.impl;
import com.typesafe.config.Config;
-import org.apache.eagle.app.environment.Environment;
-import org.apache.eagle.app.sink.StreamSinkProvider;
-
-public class StaticEnvironment implements Environment {
- private final Config config;
+import org.apache.eagle.app.environment.AbstractEnvironment;
+public class StaticEnvironment extends AbstractEnvironment {
public StaticEnvironment(Config config) {
- this.config = config;
- }
-
- @Override
- public Config config() {
- return this.config;
- }
-
- @Override
- public StreamSinkProvider streamSink() {
- throw new IllegalStateException("streamSink() is not supported in " + StaticEnvironment.class.getSimpleName());
+ super(config);
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/698b3028/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationAction.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationAction.java b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationAction.java
index 557737f..b7258d3 100644
--- a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationAction.java
+++ b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationAction.java
@@ -31,6 +31,7 @@ import org.apache.eagle.app.environment.ExecutionRuntime;
import org.apache.eagle.app.environment.ExecutionRuntimeManager;
import org.apache.eagle.app.sink.KafkaStreamSinkConfig;
import org.apache.eagle.metadata.model.ApplicationEntity;
+import org.apache.eagle.metadata.utils.StreamIdConversions;
import org.apache.eagle.metadata.model.StreamDesc;
import org.apache.eagle.metadata.model.StreamSinkConfig;
import org.slf4j.Logger;
@@ -86,21 +87,14 @@ public class ApplicationAction implements Serializable {
this.alertMetadataService = alertMetadataService;
}
- /**
- * Generate global unique streamId to install.
- * TODO refactor with streamId and siteId
- */
- private static String generateUniqueStreamId(String siteId, String streamTypeId) {
- return String.format("%s_%s", streamTypeId, siteId).toUpperCase();
- }
-
public void doInstall() {
if (metadata.getDescriptor().getStreams() != null) {
List<StreamDesc> streamDescToInstall = metadata.getDescriptor().getStreams().stream().map((streamDefinition -> {
StreamDefinition copied = streamDefinition.copy();
copied.setSiteId(metadata.getSite().getSiteId());
- copied.setStreamId(generateUniqueStreamId(metadata.getSite().getSiteId(), copied.getStreamId()));
- StreamSinkConfig streamSinkConfig = this.runtime.environment().streamSink().getSinkConfig(copied.getStreamId(), this.effectiveConfig);
+ copied.setStreamId(StreamIdConversions.formatSiteStreamId(metadata.getSite().getSiteId(), copied.getStreamId()));
+ StreamSinkConfig streamSinkConfig = this.runtime.environment()
+ .streamSink().getSinkConfig(StreamIdConversions.parseStreamTypeId(copied.getSiteId(), copied.getStreamId()), this.effectiveConfig);
StreamDesc streamDesc = new StreamDesc();
streamDesc.setSchema(copied);
streamDesc.setSink(streamSinkConfig);
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/698b3028/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/sink/KafkaStreamSink.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/sink/KafkaStreamSink.java b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/sink/KafkaStreamSink.java
index cf5351b..d6d80f4 100644
--- a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/sink/KafkaStreamSink.java
+++ b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/sink/KafkaStreamSink.java
@@ -24,6 +24,7 @@ import com.typesafe.config.Config;
import kafka.javaapi.producer.Producer;
import kafka.producer.KeyedMessage;
import kafka.producer.ProducerConfig;
+import org.apache.eagle.metadata.utils.StreamIdConversions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -96,18 +97,42 @@ public class KafkaStreamSink extends StormStreamSink<KafkaStreamSinkConfig> {
}
public static class Provider implements StreamSinkProvider<KafkaStreamSink, KafkaStreamSinkConfig> {
+ private static final Logger LOG = LoggerFactory.getLogger(Provider.class);
+ private static final String DEAULT_SHARED_TOPIC_CONF_KEY = "dataSinkConfig.topic";
+
+ private String getStreamSpecificTopicConfigKey(String streamId) {
+ return String.format("dataSinkConfig.%s.topic",streamId);
+ }
+
@Override
public KafkaStreamSinkConfig getSinkConfig(String streamId, Config config) {
KafkaStreamSinkConfig desc = new KafkaStreamSinkConfig();
- desc.setTopicId(config.getString("dataSinkConfig.topic"));
+ String streamSpecificTopicConfigKey = getStreamSpecificTopicConfigKey(streamId);
+ if (config.hasPath(streamSpecificTopicConfigKey)) {
+ desc.setTopicId(streamSpecificTopicConfigKey);
+ } else if (config.hasPath(DEAULT_SHARED_TOPIC_CONF_KEY)) {
+ desc.setTopicId(config.getString(DEAULT_SHARED_TOPIC_CONF_KEY));
+ LOG.warn("Using default shared topic {}: {}", DEAULT_SHARED_TOPIC_CONF_KEY, desc.getTopicId());
+ } else {
+ LOG.error("Neither stream specific topic: {} nor default shared topic: {} found in config", streamSpecificTopicConfigKey, DEAULT_SHARED_TOPIC_CONF_KEY);
+ throw new IllegalArgumentException("Neither stream specific topic: "
+ + streamSpecificTopicConfigKey + " nor default shared topic: " + DEAULT_SHARED_TOPIC_CONF_KEY + " found in config");
+ }
desc.setBrokerList(config.getString("dataSinkConfig.brokerList"));
- desc.setSerializerClass(config.getString("dataSinkConfig.serializerClass"));
- desc.setKeySerializerClass(config.getString("dataSinkConfig.keySerializerClass"));
+ desc.setSerializerClass(config.hasPath("dataSinkConfig.serializerClass")
+ ? config.getString("dataSinkConfig.serializerClass") : "kafka.serializer.StringEncoder");
+ desc.setKeySerializerClass(config.hasPath("dataSinkConfig.keySerializerClass")
+ ? config.getString("dataSinkConfig.keySerializerClass") : "kafka.serializer.StringEncoder");
+
// new added properties for async producer
- desc.setNumBatchMessages(config.getString("dataSinkConfig.numBatchMessages"));
- desc.setProducerType(config.getString("dataSinkConfig.producerType"));
- desc.setMaxQueueBufferMs(config.getString("dataSinkConfig.maxQueueBufferMs"));
- desc.setRequestRequiredAcks(config.getString("dataSinkConfig.requestRequiredAcks"));
+ desc.setNumBatchMessages(config.hasPath("dataSinkConfig.numBatchMessages")
+ ? config.getString("dataSinkConfig.numBatchMessages") : "1024");
+ desc.setProducerType(config.hasPath("dataSinkConfig.producerType")
+ ? config.getString("dataSinkConfig.producerType") : "async");
+ desc.setMaxQueueBufferMs(config.hasPath("dataSinkConfig.maxQueueBufferMs")
+ ? config.getString("dataSinkConfig.maxQueueBufferMs") : "3000");
+ desc.setRequestRequiredAcks(config.hasPath("dataSinkConfig.requestRequiredAcks")
+ ? config.getString("dataSinkConfig.requestRequiredAcks") : "1");
return desc;
}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/698b3028/eagle-core/eagle-app/eagle-app-base/src/test/java/org/apache/eagle/app/environment/StaticEnvironmentTest.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-app/eagle-app-base/src/test/java/org/apache/eagle/app/environment/StaticEnvironmentTest.java b/eagle-core/eagle-app/eagle-app-base/src/test/java/org/apache/eagle/app/environment/StaticEnvironmentTest.java
index 797eba8..de64e6a 100644
--- a/eagle-core/eagle-app/eagle-app-base/src/test/java/org/apache/eagle/app/environment/StaticEnvironmentTest.java
+++ b/eagle-core/eagle-app/eagle-app-base/src/test/java/org/apache/eagle/app/environment/StaticEnvironmentTest.java
@@ -16,6 +16,7 @@
*/
package org.apache.eagle.app.environment;
+import com.typesafe.config.ConfigFactory;
import org.apache.eagle.app.environment.impl.StaticEnvironment;
import org.junit.Before;
import org.junit.Test;
@@ -25,11 +26,14 @@ public class StaticEnvironmentTest {
@Before
public void setUp(){
- environment = new StaticEnvironment(null);
+ environment = new StaticEnvironment(ConfigFactory.load());
}
- @Test(expected = IllegalStateException.class)
- public void testNotSupport(){
+ /**
+ * StaticEnvironment stream sink will be usefully for collection only app
+ */
+ @Test
+ public void testStreamSinkSupport(){
environment.streamSink();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/698b3028/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/StreamDesc.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/StreamDesc.java b/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/StreamDesc.java
index abf6a81..47d16fd 100644
--- a/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/StreamDesc.java
+++ b/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/StreamDesc.java
@@ -17,6 +17,7 @@
package org.apache.eagle.metadata.model;
import org.apache.eagle.alert.engine.coordinator.StreamDefinition;
+import org.apache.eagle.metadata.utils.StreamIdConversions;
public class StreamDesc {
private String streamId;
@@ -28,7 +29,7 @@ public class StreamDesc {
}
public void setStreamId(String streamId) {
- this.streamId = streamId;
+ this.streamId = StreamIdConversions.formStreamTypeId(streamId);
}
public StreamDefinition getSchema() {
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/698b3028/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/utils/StreamIdConversions.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/utils/StreamIdConversions.java b/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/utils/StreamIdConversions.java
new file mode 100644
index 0000000..b5f5631
--- /dev/null
+++ b/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/utils/StreamIdConversions.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.metadata.utils;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * The name conversion about streamId:
+ * 1. Case insensitive.
+ * 2. All in upper case.
+ */
+public class StreamIdConversions {
+
+ public static String formStreamTypeId(String streamTypeId) {
+ return streamTypeId.toUpperCase();
+ }
+
+ public static String formatSiteStreamId(String siteId, String streamTypeId) {
+ return String.format("%s_%s", streamTypeId, siteId).toUpperCase();
+ }
+
+ public static String parseStreamTypeId(String siteId, String generatedUniqueStreamId) {
+ String subffix = String.format("_%s", siteId).toUpperCase();
+ if (generatedUniqueStreamId.endsWith(subffix)) {
+ int streamTypeIdLength = generatedUniqueStreamId.length() - subffix.length();
+ Preconditions.checkArgument(streamTypeIdLength > 0, "Invalid streamId: " + generatedUniqueStreamId + ", streamTypeId is empty");
+ return generatedUniqueStreamId.substring(0, streamTypeIdLength).toUpperCase();
+ } else {
+ throw new IllegalArgumentException("Invalid streamId: " + generatedUniqueStreamId + ", not end with \"" + subffix + "\"");
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/698b3028/eagle-core/eagle-metadata/eagle-metadata-base/src/test/java/org/apache/eagle/metadata/utils/StreamIdConversionsTest.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-metadata/eagle-metadata-base/src/test/java/org/apache/eagle/metadata/utils/StreamIdConversionsTest.java b/eagle-core/eagle-metadata/eagle-metadata-base/src/test/java/org/apache/eagle/metadata/utils/StreamIdConversionsTest.java
new file mode 100644
index 0000000..f0018bf
--- /dev/null
+++ b/eagle-core/eagle-metadata/eagle-metadata-base/src/test/java/org/apache/eagle/metadata/utils/StreamIdConversionsTest.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.metadata.utils;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class StreamIdConversionsTest {
+ @Test
+ public void testGenerateUniqueStreamId() {
+ Assert.assertEquals("MOCK_STREAM_TEST", StreamIdConversions.formatSiteStreamId("test","mock_stream"));
+ }
+
+ @Test
+ public void testFormatStreamId() {
+ Assert.assertEquals("MOCK_STREAM", StreamIdConversions.formStreamTypeId("mock_stream"));
+ }
+
+ @Test
+ public void testParseStreamTypeId() {
+ Assert.assertEquals("MOCK_STREAM", StreamIdConversions.parseStreamTypeId("test","MOCK_STREAM_TEST"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/698b3028/eagle-examples/eagle-app-example/src/main/resources/META-INF/providers/org.apache.eagle.app.example.ExampleApplicationProvider.xml
----------------------------------------------------------------------
diff --git a/eagle-examples/eagle-app-example/src/main/resources/META-INF/providers/org.apache.eagle.app.example.ExampleApplicationProvider.xml b/eagle-examples/eagle-app-example/src/main/resources/META-INF/providers/org.apache.eagle.app.example.ExampleApplicationProvider.xml
index 432b92f..c1e8711 100644
--- a/eagle-examples/eagle-app-example/src/main/resources/META-INF/providers/org.apache.eagle.app.example.ExampleApplicationProvider.xml
+++ b/eagle-examples/eagle-app-example/src/main/resources/META-INF/providers/org.apache.eagle.app.example.ExampleApplicationProvider.xml
@@ -18,9 +18,8 @@
<application>
<type>EXAMPLE_APPLICATION</type>
- <name>Example Monitoring Application</name>
+ <name>Example Monitor APP</name>
<version>0.5.0-incubating</version>
- <appClass>org.apache.eagle.app.example.ExampleStormApplication</appClass>
<viewPath>/apps/example</viewPath>
<configuration>
<property>
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/698b3028/eagle-gc/src/main/resources/META-INF/providers/org.apache.eagle.gc.GCLogApplicationProvider.xml
----------------------------------------------------------------------
diff --git a/eagle-gc/src/main/resources/META-INF/providers/org.apache.eagle.gc.GCLogApplicationProvider.xml b/eagle-gc/src/main/resources/META-INF/providers/org.apache.eagle.gc.GCLogApplicationProvider.xml
index a0477a2..c5f9732 100644
--- a/eagle-gc/src/main/resources/META-INF/providers/org.apache.eagle.gc.GCLogApplicationProvider.xml
+++ b/eagle-gc/src/main/resources/META-INF/providers/org.apache.eagle.gc.GCLogApplicationProvider.xml
@@ -21,10 +21,9 @@
-->
<application>
- <type>GCLogApplication</type>
- <name>GC Log Monitoring Application</name>
+ <type>GC_LOG_MONITOR</type>
+ <name>GC Log Monitor</name>
<version>0.5.0-incubating</version>
- <appClass>org.apache.eagle.gc.GCLogApplication</appClass>
<configuration>
<!-- topology level configurations -->
<property>
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/698b3028/eagle-hadoop-metric/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-hadoop-metric/pom.xml b/eagle-hadoop-metric/pom.xml
index b397e37..33a7b90 100644
--- a/eagle-hadoop-metric/pom.xml
+++ b/eagle-hadoop-metric/pom.xml
@@ -26,6 +26,10 @@
<artifactId>eagle-hadoop-metric</artifactId>
<dependencies>
+ <dependency>
+ <groupId>org.apache.eagle</groupId>
+ <artifactId>eagle-app-base</artifactId>
+ <version>${project.version}</version>
+ </dependency>
</dependencies>
-
</project>
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/698b3028/eagle-hadoop-metric/src/main/java/org/apache/eagle/metric/HadoopMetricMonitorAppProdiver.java
----------------------------------------------------------------------
diff --git a/eagle-hadoop-metric/src/main/java/org/apache/eagle/metric/HadoopMetricMonitorAppProdiver.java b/eagle-hadoop-metric/src/main/java/org/apache/eagle/metric/HadoopMetricMonitorAppProdiver.java
new file mode 100644
index 0000000..e6ebde1
--- /dev/null
+++ b/eagle-hadoop-metric/src/main/java/org/apache/eagle/metric/HadoopMetricMonitorAppProdiver.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.metric;
+
+import org.apache.eagle.app.StaticApplicationProvider;
+
+public class HadoopMetricMonitorAppProdiver extends StaticApplicationProvider {
+ // Metadata: META-INF/providers/org.apache.eagle.metric.HadoopMetricMonitorAppProdiver.xml
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/698b3028/eagle-hadoop-metric/src/main/resources/META-INF/providers/org.apache.eagle.metric.HadoopMetricMonitorAppProdiver.xml
----------------------------------------------------------------------
diff --git a/eagle-hadoop-metric/src/main/resources/META-INF/providers/org.apache.eagle.metric.HadoopMetricMonitorAppProdiver.xml b/eagle-hadoop-metric/src/main/resources/META-INF/providers/org.apache.eagle.metric.HadoopMetricMonitorAppProdiver.xml
new file mode 100644
index 0000000..752c0cb
--- /dev/null
+++ b/eagle-hadoop-metric/src/main/resources/META-INF/providers/org.apache.eagle.metric.HadoopMetricMonitorAppProdiver.xml
@@ -0,0 +1,94 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<application>
+ <type>HADOOP_JMX_METRIC_MONITOR</type>
+ <name>Hadoop JMX Metric Monitor</name>
+ <version>0.5.0-incubating</version>
+ <configuration>
+ <!-- data source configurations -->
+ <property>
+ <name>dataSinkConfig.HADOOP_JMX_METRIC_STREAM.topic</name>
+ <displayName>JMX Metric Kafka Topic</displayName>
+ <value>hadoop_jmx_metric_{SITE_ID}</value>
+ <description>Hadoop JMX metric kafka topic name for stream: HADOOP_JMX_METRIC_STREAM</description>
+ <required>true</required>
+ </property>
+ <property>
+ <name>dataSinkConfig.brokerList</name>
+ <displayName>Kafka Brokers</displayName>
+ <value>localhost:6667</value>
+ <description>Sourced Kafka Brokers like broker1,broker2,...</description>
+ <required>true</required>
+ </property>
+ </configuration>
+ <streams>
+ <stream>
+ <streamId>HADOOP_JMX_METRIC_STREAM</streamId>
+ <description>Hadoop JMX Metric Stream including name node, resource manager, etc.</description>
+ <validate>true</validate>
+ <timeseries>true</timeseries>
+ <columns>
+ <column>
+ <name>host</name>
+ <type>string</type>
+ </column>
+ <column>
+ <name>timestamp</name>
+ <type>long</type>
+ </column>
+ <column>
+ <name>metric</name>
+ <type>string</type>
+ </column>
+ <column>
+ <name>component</name>
+ <type>string</type>
+ </column>
+ <column>
+ <name>site</name>
+ <type>string</type>
+ </column>
+ <column>
+ <name>value</name>
+ <type>double</type>
+ <defaultValue>0.0</defaultValue>
+ </column>
+ </columns>
+ </stream>
+ </streams>
+ <docs>
+ <install>
+ <b>How to Install</b>
+ <ol>
+ <li>Create a kafka topic named <code>hadoop_jmx_metric_{SITE_ID}</code></li>
+ <li>Setup metric collector for monitored Hadoop/HBase using <a target="_blank" href="https://github.com/apache/incubator-eagle/tree/master/eagle-external/hadoop_jmx_collector">hadoop_jmx_collector</a></li>
+ <li>Setup <code>Kafka Topic Name</code> and <code>Kafka Broker List </code> in <code>Settings</code> tab</li>
+ <li>Click "Install" button then you will see the <code>HADOOP_JMX_METRIC_STREAM_{SITE_ID}</code> in <a href="/#/alert/streamList">Streams</a></li>
+ </ol>
+ </install>
+ <uninstall>
+ <b>How to Uninstall</b>
+ <ol>
+ <li>Stop and remove installed <code>hadoop_jmx_collector</code></li>
+ <li>Remove kafka topic <code>hadoop_jmx_metric_{SITE_ID}</code></li>
+ <li>Click "Uninstall" button which will remove stream <code>HADOOP_JMX_METRIC_STREAM_{SITE_ID}</code></li>
+ </ol>
+ </uninstall>
+ </docs>
+</application>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/698b3028/eagle-hadoop-metric/src/main/resources/META-INF/services/org.apache.eagle.app.spi.ApplicationProvider
----------------------------------------------------------------------
diff --git a/eagle-hadoop-metric/src/main/resources/META-INF/services/org.apache.eagle.app.spi.ApplicationProvider b/eagle-hadoop-metric/src/main/resources/META-INF/services/org.apache.eagle.app.spi.ApplicationProvider
new file mode 100644
index 0000000..25d911d
--- /dev/null
+++ b/eagle-hadoop-metric/src/main/resources/META-INF/services/org.apache.eagle.app.spi.ApplicationProvider
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+org.apache.eagle.metric.HadoopMetricMonitorAppProdiver
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/698b3028/eagle-hadoop-metric/src/main/test/java/org/apache/eagle/metric/HadoopMetricMonitorAppProdiverTest.java
----------------------------------------------------------------------
diff --git a/eagle-hadoop-metric/src/main/test/java/org/apache/eagle/metric/HadoopMetricMonitorAppProdiverTest.java b/eagle-hadoop-metric/src/main/test/java/org/apache/eagle/metric/HadoopMetricMonitorAppProdiverTest.java
new file mode 100644
index 0000000..ee0b3c0
--- /dev/null
+++ b/eagle-hadoop-metric/src/main/test/java/org/apache/eagle/metric/HadoopMetricMonitorAppProdiverTest.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.metric;
+
+
+import com.google.inject.Inject;
+import org.apache.eagle.app.resource.ApplicationResource;
+import org.apache.eagle.app.service.ApplicationOperations;
+import org.apache.eagle.app.test.ApplicationSimulator;
+import org.apache.eagle.app.test.ApplicationTestBase;
+import org.apache.eagle.metadata.model.ApplicationEntity;
+import org.apache.eagle.metadata.model.SiteEntity;
+import org.apache.eagle.metadata.resource.SiteResource;
+import org.apache.eagle.metadata.service.ApplicationStatusUpdateService;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class HadoopMetricMonitorAppProdiverTest extends ApplicationTestBase {
+
+ @Inject
+ private SiteResource siteResource;
+ @Inject
+ private ApplicationResource applicationResource;
+ @Inject
+ private ApplicationSimulator simulator;
+ @Inject
+ ApplicationStatusUpdateService statusUpdateService;
+
+ @Test
+ public void testApplicationLifecycle() throws InterruptedException {
+ // Create local site
+ SiteEntity siteEntity = new SiteEntity();
+ siteEntity.setSiteId("test_site");
+ siteEntity.setSiteName("Test Site");
+ siteEntity.setDescription("Test Site for HADOOP_JMX_METRIC_MONITOR");
+ siteResource.createSite(siteEntity);
+ Assert.assertNotNull(siteEntity.getUuid());
+
+ ApplicationOperations.InstallOperation installOperation = new ApplicationOperations.InstallOperation("test_site", "HADOOP_JMX_METRIC_MONITOR", ApplicationEntity.Mode.LOCAL);
+ installOperation.setConfiguration(getConf());
+ // Install application
+ ApplicationEntity applicationEntity = applicationResource.installApplication(installOperation).getData();
+ // Uninstall application
+ applicationResource.uninstallApplication(new ApplicationOperations.UninstallOperation(applicationEntity.getUuid()));
+ try {
+ applicationResource.getApplicationEntityByUUID(applicationEntity.getUuid());
+ Assert.fail("Application instance (UUID: " + applicationEntity.getUuid() + ") should have been uninstalled");
+ } catch (Exception ex) {
+ // Expected exception
+ }
+ }
+
+ private Map<String, Object> getConf() {
+ Map<String, Object> conf = new HashMap<>();
+ conf.put("dataSinkConfig.topic", "testTopic");
+ conf.put("dataSinkConfig.brokerList", "broker");
+ conf.put("dataSinkConfig.serializerClass", "serializerClass");
+ conf.put("dataSinkConfig.keySerializerClass", "keySerializerClass");
+ conf.put("dataSinkConfig.producerType", "async");
+ conf.put("dataSinkConfig.numBatchMessages", 4096);
+ conf.put("dataSinkConfig.maxQueueBufferMs", 5000);
+ conf.put("dataSinkConfig.requestRequiredAcks", 0);
+ conf.put("spoutNum", 2);
+ conf.put("mode", "LOCAL");
+ return conf;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/698b3028/eagle-jpm/eagle-hadoop-queue/src/main/resources/META-INF/providers/org.apache.eagle.hadoop.queue.HadoopQueueRunningAppProvider.xml
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-hadoop-queue/src/main/resources/META-INF/providers/org.apache.eagle.hadoop.queue.HadoopQueueRunningAppProvider.xml b/eagle-jpm/eagle-hadoop-queue/src/main/resources/META-INF/providers/org.apache.eagle.hadoop.queue.HadoopQueueRunningAppProvider.xml
index 02b60ef..c5e9ff9 100644
--- a/eagle-jpm/eagle-hadoop-queue/src/main/resources/META-INF/providers/org.apache.eagle.hadoop.queue.HadoopQueueRunningAppProvider.xml
+++ b/eagle-jpm/eagle-hadoop-queue/src/main/resources/META-INF/providers/org.apache.eagle.hadoop.queue.HadoopQueueRunningAppProvider.xml
@@ -18,7 +18,7 @@
<application>
<type>HADOOP_QUEUE_RUNNING_APP</type>
- <name>Hadoop Queue Running Monitoring</name>
+ <name>Hadoop Queue Monitor</name>
<version>0.5.0-incubating</version>
<configuration>
<!-- org.apache.eagle.hadoop.queue.HadoopQueueRunningAppConfig -->
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/698b3028/eagle-jpm/eagle-jpm-aggregation/src/main/resources/META-INF/providers/org.apache.eagle.jpm.aggregation.AggregationApplicationProvider.xml
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-aggregation/src/main/resources/META-INF/providers/org.apache.eagle.jpm.aggregation.AggregationApplicationProvider.xml b/eagle-jpm/eagle-jpm-aggregation/src/main/resources/META-INF/providers/org.apache.eagle.jpm.aggregation.AggregationApplicationProvider.xml
index 29ca98a..2c8d60c 100644
--- a/eagle-jpm/eagle-jpm-aggregation/src/main/resources/META-INF/providers/org.apache.eagle.jpm.aggregation.AggregationApplicationProvider.xml
+++ b/eagle-jpm/eagle-jpm-aggregation/src/main/resources/META-INF/providers/org.apache.eagle.jpm.aggregation.AggregationApplicationProvider.xml
@@ -18,7 +18,7 @@
<application>
<type>MR_JPM_AGGREGATION_APP</type>
- <name>Map Reduce Job Metrics Aggregation</name>
+ <name>MR Metrics Aggregation</name>
<version>0.5.0-incubating</version>
<configuration>
<property>
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/698b3028/eagle-security/eagle-security-hbase-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.hbase.HBaseAuditLogAppProvider.xml
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.hbase.HBaseAuditLogAppProvider.xml b/eagle-security/eagle-security-hbase-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.hbase.HBaseAuditLogAppProvider.xml
index 8f648a1..1cfd068 100644
--- a/eagle-security/eagle-security-hbase-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.hbase.HBaseAuditLogAppProvider.xml
+++ b/eagle-security/eagle-security-hbase-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.hbase.HBaseAuditLogAppProvider.xml
@@ -17,8 +17,8 @@
-->
<application>
- <type>HBaseAuditLogApplication</type>
- <name>Hbase Audit Log Monitoring Application</name>
+ <type>HBASE_AUDIT_LOG_MONITOR</type>
+ <name>Hbase Audit Log Monitor</name>
<version>0.5.0-incubating</version>
<appClass>org.apache.eagle.security.hbase.HBaseAuditLogApplication</appClass>
<configuration>
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/698b3028/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.auditlog.HdfsAuditLogAppProvider.xml
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.auditlog.HdfsAuditLogAppProvider.xml b/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.auditlog.HdfsAuditLogAppProvider.xml
index 2c02a8f..bb302c1 100644
--- a/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.auditlog.HdfsAuditLogAppProvider.xml
+++ b/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.auditlog.HdfsAuditLogAppProvider.xml
@@ -22,7 +22,7 @@
<application>
<type>HdfsAuditLogApplication</type>
- <name>Hdfs Audit Log Monitoring Application</name>
+ <name>Hdfs Audit Log Monitor</name>
<version>0.5.0-incubating</version>
<configuration>
<!-- topology related configurations -->
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/698b3028/eagle-server/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-server/pom.xml b/eagle-server/pom.xml
index 1a3e7c9..f58a166 100644
--- a/eagle-server/pom.xml
+++ b/eagle-server/pom.xml
@@ -328,6 +328,13 @@
<artifactId>eagle-topology-app</artifactId>
<version>${project.version}</version>
</dependency>
+
+ <!--- App: HadoopMetricMonitorApp -->
+ <dependency>
+ <groupId>org.apache.eagle</groupId>
+ <artifactId>eagle-hadoop-metric</artifactId>
+ <version>${project.version}</version>
+ </dependency>
</dependencies>
</profile>
</profiles>