You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@eagle.apache.org by qi...@apache.org on 2016/11/25 09:33:27 UTC
incubator-eagle git commit: [EAGLE-798] Add a new alert publisher to
store alerts into hbase
Repository: incubator-eagle
Updated Branches:
refs/heads/master 52dc7a5d9 -> 6a0529e96
[EAGLE-798] Add a new alert publisher to store alerts into hbase
https://issues.apache.org/jira/browse/EAGLE-798
Author: Zhao, Qingwen <qi...@apache.org>
Closes #683 from qingwen220/EAGLE-798.
Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/6a0529e9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/6a0529e9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/6a0529e9
Branch: refs/heads/master
Commit: 6a0529e96e1f581c4fa476a82e5e897fc2509828
Parents: 52dc7a5
Author: Zhao, Qingwen <qi...@apache.org>
Authored: Fri Nov 25 17:33:13 2016 +0800
Committer: Zhao, Qingwen <qi...@apache.org>
Committed: Fri Nov 25 17:33:13 2016 +0800
----------------------------------------------------------------------
.../eagle-alert-parent/eagle-alert-app/pom.xml | 5 ++
.../eagle/alert/app/AlertEagleStorePlugin.java | 93 ++++++++++++++++++++
.../alert/engine/model/AlertPublishEvent.java | 2 +
.../eagle/alert/utils/AlertConstants.java | 2 +
.../publisher/impl/AlertEagleStorePlugin.java | 2 +-
.../eagle-metadata/eagle-metadata-base/pom.xml | 5 ++
.../eagle/metadata/model/AlertEntity.java | 74 ++++++++++++++++
.../metadata/model/AlertEntityRepository.java | 27 ++++++
.../app/dev/public/js/services/policySrv.js | 5 --
.../hdfs/HdfsTopologyEntityParser.java | 14 +--
10 files changed, 213 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/6a0529e9/eagle-core/eagle-alert-parent/eagle-alert-app/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert-app/pom.xml b/eagle-core/eagle-alert-parent/eagle-alert-app/pom.xml
index d0760d5..400afaa 100644
--- a/eagle-core/eagle-alert-parent/eagle-alert-app/pom.xml
+++ b/eagle-core/eagle-alert-parent/eagle-alert-app/pom.xml
@@ -34,6 +34,11 @@
</dependency>
<dependency>
<groupId>org.apache.eagle</groupId>
+ <artifactId>eagle-client-base</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.eagle</groupId>
<artifactId>alert-coordinator</artifactId>
<version>${project.version}</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/6a0529e9/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/java/org/apache/eagle/alert/app/AlertEagleStorePlugin.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/java/org/apache/eagle/alert/app/AlertEagleStorePlugin.java b/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/java/org/apache/eagle/alert/app/AlertEagleStorePlugin.java
new file mode 100644
index 0000000..a534012
--- /dev/null
+++ b/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/java/org/apache/eagle/alert/app/AlertEagleStorePlugin.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.app;
+
+import com.google.common.base.Preconditions;
+import com.typesafe.config.Config;
+import org.apache.eagle.alert.engine.coordinator.Publishment;
+import org.apache.eagle.alert.engine.model.AlertStreamEvent;
+import org.apache.eagle.alert.engine.publisher.impl.AbstractPublishPlugin;
+import org.apache.eagle.alert.utils.AlertConstants;
+import org.apache.eagle.metadata.model.AlertEntity;
+import org.apache.eagle.service.client.IEagleServiceClient;
+import org.apache.eagle.service.client.impl.EagleServiceClientImpl;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.eagle.alert.engine.model.AlertPublishEvent.*;
+
+public class AlertEagleStorePlugin extends AbstractPublishPlugin {
+ private static final Logger LOG = LoggerFactory.getLogger(AlertEagleStorePlugin.class);
+ private IEagleServiceClient client;
+
+ @Override
+ public void init(Config config, Publishment publishment, Map conf) throws Exception {
+ super.init(config, publishment, conf);
+ client = new EagleServiceClientImpl(config.getString("service.host"), config.getInt("service.port"),
+ config.getString("service.username"), config.getString("service.password"));
+ }
+
+ @Override
+ public void close() {
+ try {
+ client.close();
+ } catch (IOException e) {
+ LOG.error(e.getMessage(), e);
+ }
+ }
+
+ @Override
+ public void onAlert(AlertStreamEvent event) throws Exception {
+ List<AlertStreamEvent> eventList = this.dedup(event);
+ if (eventList == null || eventList.isEmpty()) {
+ return;
+ }
+ List<AlertEntity> alertEvents = new ArrayList<>();
+ for (AlertStreamEvent e : eventList) {
+ alertEvents.add(convertAlertEvent(e));
+ }
+ client.create(alertEvents, AlertConstants.ALERT_SERVICE_ENDPOINT_NAME);
+ }
+
+ @Override
+ protected Logger getLogger() {
+ return LOG;
+ }
+
+ public AlertEntity convertAlertEvent(AlertStreamEvent event) {
+ Preconditions.checkNotNull(event.getAlertId(), "alertId is not initialized before being published: " + event.toString());
+ AlertEntity alertEvent = new AlertEntity();
+ Map<String, String> tags = new HashMap<>();
+ tags.put(POLICY_ID_KEY, event.getPolicyId());
+ tags.put(ALERT_ID_KEY, event.getAlertId());
+ if (event.getExtraData() != null && !event.getExtraData().isEmpty()) {
+ tags.put(SITE_ID_KEY, event.getExtraData().get(SITE_ID_KEY).toString());
+ alertEvent.setPolicyValue(event.getExtraData().get(POLICY_VALUE_KEY).toString());
+ alertEvent.setAppIds((List<String>) event.getExtraData().get(APP_IDS_KEY));
+ }
+ alertEvent.setTimestamp(event.getCreatedTime());
+ alertEvent.setAlertData(event.getDataMap());
+ alertEvent.setTags(tags);
+ return alertEvent;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/6a0529e9/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/model/AlertPublishEvent.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/model/AlertPublishEvent.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/model/AlertPublishEvent.java
index 462e0fb..aee0ba0 100644
--- a/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/model/AlertPublishEvent.java
+++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/model/AlertPublishEvent.java
@@ -33,8 +33,10 @@ public class AlertPublishEvent {
private long alertTimestamp;
private Map<String, Object> alertData;
+ public static final String ALERT_ID_KEY = "alertId";
public static final String SITE_ID_KEY = "siteId";
public static final String APP_IDS_KEY = "appIds";
+ public static final String POLICY_ID_KEY = "policyId";
public static final String POLICY_VALUE_KEY = "policyValue";
public String getAlertId() {
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/6a0529e9/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/utils/AlertConstants.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/utils/AlertConstants.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/utils/AlertConstants.java
index 2cbc977..566944f 100644
--- a/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/utils/AlertConstants.java
+++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/utils/AlertConstants.java
@@ -24,4 +24,6 @@ public class AlertConstants {
public static final String DEFAULT_SPOUT_NAME = "alertEngineSpout";
public static final String DEFAULT_ROUTERBOLT_NAME = "streamRouterBolt";
+
+ public static final String ALERT_SERVICE_ENDPOINT_NAME = "AlertService";
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/6a0529e9/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/impl/AlertEagleStorePlugin.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/impl/AlertEagleStorePlugin.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/impl/AlertEagleStorePlugin.java
index f31d850..48c3663 100644
--- a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/impl/AlertEagleStorePlugin.java
+++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/impl/AlertEagleStorePlugin.java
@@ -50,7 +50,7 @@ public class AlertEagleStorePlugin extends AbstractPublishPlugin {
try {
client.close();
} catch (IOException e) {
- e.printStackTrace();
+ LOG.error(e.getMessage(), e);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/6a0529e9/eagle-core/eagle-metadata/eagle-metadata-base/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-metadata/eagle-metadata-base/pom.xml b/eagle-core/eagle-metadata/eagle-metadata-base/pom.xml
index b5a0b27..8df0674 100644
--- a/eagle-core/eagle-metadata/eagle-metadata-base/pom.xml
+++ b/eagle-core/eagle-metadata/eagle-metadata-base/pom.xml
@@ -60,5 +60,10 @@
<artifactId>eagle-common</artifactId>
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.eagle</groupId>
+ <artifactId>eagle-entity-base</artifactId>
+ <version>${project.version}</version>
+ </dependency>
</dependencies>
</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/6a0529e9/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/AlertEntity.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/AlertEntity.java b/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/AlertEntity.java
new file mode 100644
index 0000000..c350dd9
--- /dev/null
+++ b/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/AlertEntity.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eagle.metadata.model;
+
+import org.apache.eagle.alert.utils.AlertConstants;
+import org.apache.eagle.log.base.taggedlog.TaggedLogAPIEntity;
+import org.apache.eagle.log.entity.meta.*;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import java.util.List;
+import java.util.Map;
+
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+@Table("alert_detail")
+@ColumnFamily("f")
+@Prefix("alert_detail")
+@Service(AlertConstants.ALERT_SERVICE_ENDPOINT_NAME)
+@TimeSeries(true)
+@Tags({"alertId", "siteId", "policyId"})
+@Partition({"siteId"})
+@Indexes({
+ @Index(name = "Index_1_policyId", columns = { "policyId" }, unique = true)
+ })
+public class AlertEntity extends TaggedLogAPIEntity {
+ @Column("a")
+ private List<String> appIds;
+ @Column("b")
+ private String policyValue;
+ @Column("c")
+ private Map<String, Object> alertData;
+
+ public List<String> getAppIds() {
+ return appIds;
+ }
+
+ public void setAppIds(List<String> appIds) {
+ this.appIds = appIds;
+ valueChanged("appIds");
+ }
+
+ public String getPolicyValue() {
+ return policyValue;
+ }
+
+ public void setPolicyValue(String policyValue) {
+ this.policyValue = policyValue;
+ valueChanged("policyValue");
+ }
+
+ public Map<String, Object> getAlertData() {
+ return alertData;
+ }
+
+ public void setAlertData(Map<String, Object> alertData) {
+ this.alertData = alertData;
+ valueChanged("alertData");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/6a0529e9/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/AlertEntityRepository.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/AlertEntityRepository.java b/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/AlertEntityRepository.java
new file mode 100644
index 0000000..c8219b6
--- /dev/null
+++ b/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/AlertEntityRepository.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eagle.metadata.model;
+
+import org.apache.eagle.log.entity.repo.EntityRepository;
+
+public class AlertEntityRepository extends EntityRepository {
+
+ public AlertEntityRepository() {
+ registerEntity(AlertEntity.class);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/6a0529e9/eagle-server/src/main/webapp/app/dev/public/js/services/policySrv.js
----------------------------------------------------------------------
diff --git a/eagle-server/src/main/webapp/app/dev/public/js/services/policySrv.js b/eagle-server/src/main/webapp/app/dev/public/js/services/policySrv.js
index e3268f0..c2f3d8c 100644
--- a/eagle-server/src/main/webapp/app/dev/public/js/services/policySrv.js
+++ b/eagle-server/src/main/webapp/app/dev/public/js/services/policySrv.js
@@ -43,11 +43,6 @@
name: "Storage",
displayFields: [],
fields: []
- },
- 'org.apache.eagle.alert.engine.publisher.impl.AlertFilePublisher': {
- name: "LocalFile",
- displayFields: ["fileName"],
- fields: ["fileName", "rotate_every_kb", "number_of_files"]
}
},
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/6a0529e9/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hdfs/HdfsTopologyEntityParser.java
----------------------------------------------------------------------
diff --git a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hdfs/HdfsTopologyEntityParser.java b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hdfs/HdfsTopologyEntityParser.java
index 72dce80..3cb55e9 100644
--- a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hdfs/HdfsTopologyEntityParser.java
+++ b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hdfs/HdfsTopologyEntityParser.java
@@ -100,7 +100,8 @@ public class HdfsTopologyEntityParser implements TopologyEntityParser {
result.getMasterNodes().add(namenodeEntity);
numNamenode++;
if (namenodeEntity.getStatus().equalsIgnoreCase(NAME_NODE_ACTIVE_STATUS)) {
- createSlaveNodeEntities(url, timestamp, result);
+ String namenodeVersion = createSlaveNodeEntities(url, timestamp, result);
+ namenodeEntity.setVersion(namenodeVersion);
}
} catch (RuntimeException ex) {
ex.printStackTrace();
@@ -118,16 +119,9 @@ public class HdfsTopologyEntityParser implements TopologyEntityParser {
final Map<String, JMXBean> jmxBeanMap = JMXQueryHelper.query(urlString);
final JMXBean bean = jmxBeanMap.get(JMX_FS_NAME_SYSTEM_BEAN_NAME);
- final String nameNodeUrlString = buildNamenodeInfo(url);
- final Map<String, JMXBean> nameNodeMap = JMXQueryHelper.query(nameNodeUrlString);
- final JMXBean nameNodeBean = nameNodeMap.get(JMX_NAMENODE_INFO);
-
if (bean == null || bean.getPropertyMap() == null) {
throw new ServiceNotResponseException("Invalid JMX format, FSNamesystem bean is null!");
}
- if (nameNodeBean == null || nameNodeBean.getPropertyMap() == null) {
- throw new ServiceNotResponseException("Invalid JMX format, NameNode bean is null!");
- }
final String hostname = (String) bean.getPropertyMap().get(HA_NAME);
HdfsServiceTopologyAPIEntity result = createHdfsServiceEntity(TopologyConstants.NAME_NODE_ROLE, hostname, updateTime);
@@ -139,11 +133,10 @@ public class HdfsTopologyEntityParser implements TopologyEntityParser {
result.setUsedCapacityTB(Double.toString(capacityUsedGB / 1024));
final Integer blocksTotal = (Integer) bean.getPropertyMap().get(BLOCKS_TOTAL);
result.setNumBlocks(Integer.toString(blocksTotal));
- result.setVersion((String) nameNodeBean.getPropertyMap().get(NAME_NODE_VERSION));
return result;
}
- private void createSlaveNodeEntities(String url, long updateTime, TopologyEntityParserResult result) throws IOException {
+ private String createSlaveNodeEntities(String url, long updateTime, TopologyEntityParserResult result) throws IOException {
final String urlString = buildNamenodeInfo(url);
final Map<String, JMXBean> jmxBeanMap = JMXQueryHelper.query(urlString);
final JMXBean bean = jmxBeanMap.get(JMX_NAMENODE_INFO);
@@ -152,6 +145,7 @@ public class HdfsTopologyEntityParser implements TopologyEntityParser {
}
createAllDataNodeEntities(bean, updateTime, result);
createAllJournalNodeEntities(bean, updateTime, result);
+ return (String) bean.getPropertyMap().get(NAME_NODE_VERSION);
}
private void createAllJournalNodeEntities(JMXBean bean, long updateTime, TopologyEntityParserResult result) throws UnknownHostException {