You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@eagle.apache.org by yo...@apache.org on 2016/08/26 04:35:45 UTC
incubator-eagle git commit: wrapper alert engine as one application
of eagle server hdfs audit log + alert engine end-2-end
Repository: incubator-eagle
Updated Branches:
refs/heads/develop 36f9a8481 -> 000b74604
wrapper alert engine as one application of eagle server
hdfs audit log + alert engine end-2-end
https://issues.apache.org/jira/browse/EAGLE-481
Author: @yonzhang <yo...@apache.org>
Closes: #392
Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/000b7460
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/000b7460
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/000b7460
Branch: refs/heads/develop
Commit: 000b746041ec07774e4ed5d5562f9c66307f1c20
Parents: 36f9a84
Author: yonzhang <yo...@gmail.com>
Authored: Thu Aug 25 21:39:53 2016 -0700
Committer: yonzhang <yo...@gmail.com>
Committed: Thu Aug 25 21:39:53 2016 -0700
----------------------------------------------------------------------
.../eagle-alert-parent/eagle-alert-app/pom.xml | 35 ++++
.../eagle/alert/app/AlertUnitTopologyApp.java | 39 ++++
.../alert/app/AlertUnitTopologyAppProvider.java | 29 +++
...e.alert.app.AlertUnitTopologyAppProvider.xml | 135 +++++++++++++
...org.apache.eagle.app.spi.ApplicationProvider | 18 ++
.../src/main/resources/application.conf | 60 ++++++
.../alert/AlertPolicyValidateProvider.java | 36 ----
.../eagle/alert/config/ZKConfigBuilder.java | 8 +-
.../scheme/JsonStringStreamNameSelector.java | 6 +-
.../alert/engine/spout/CorrelationSpout.java | 4 +-
.../src/main/resources/application.conf | 19 +-
.../alert/engine/e2e/CoordinatorClient.java | 6 +-
eagle-core/eagle-alert-parent/pom.xml | 9 +-
.../eagle/app/service/ApplicationContext.java | 6 +-
.../src/main/resources/log4j.properties | 2 +-
.../eagle-security-hdfs-auditlog/README.md | 191 +++++++++++++++++++
...ecurity.auditlog.HdfsAuditLogAppProvider.xml | 26 ++-
.../src/main/resources/scripts.txt | 22 +++
eagle-server/pom.xml | 7 +
19 files changed, 581 insertions(+), 77 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/000b7460/eagle-core/eagle-alert-parent/eagle-alert-app/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert-app/pom.xml b/eagle-core/eagle-alert-parent/eagle-alert-app/pom.xml
new file mode 100644
index 0000000..768709f
--- /dev/null
+++ b/eagle-core/eagle-alert-parent/eagle-alert-app/pom.xml
@@ -0,0 +1,35 @@
+<?xml version="1.0"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.eagle</groupId>
+ <artifactId>eagle-alert-parent</artifactId>
+ <version>0.5.0-incubating-SNAPSHOT</version>
+ </parent>
+ <artifactId>eagle-alert-app</artifactId>
+ <packaging>jar</packaging>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.eagle</groupId>
+ <artifactId>eagle-app-base</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/000b7460/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/java/org/apache/eagle/alert/app/AlertUnitTopologyApp.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/java/org/apache/eagle/alert/app/AlertUnitTopologyApp.java b/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/java/org/apache/eagle/alert/app/AlertUnitTopologyApp.java
new file mode 100644
index 0000000..a80c3b8
--- /dev/null
+++ b/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/java/org/apache/eagle/alert/app/AlertUnitTopologyApp.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eagle.alert.app;import backtype.storm.generated.StormTopology;
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+import org.apache.eagle.alert.engine.UnitTopologyMain;
+import org.apache.eagle.app.StormApplication;
+import org.apache.eagle.app.environment.impl.StormEnvironment;
+
+/**
+ * since 8/25/16.
+ */
+public class AlertUnitTopologyApp extends StormApplication {
+ @Override
+ public StormTopology execute(Config config, StormEnvironment environment) {
+ return UnitTopologyMain.createTopology(config);
+ }
+
+ public static void main(String[] args){
+ Config config = ConfigFactory.load();
+ AlertUnitTopologyApp app = new AlertUnitTopologyApp();
+ app.run(config);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/000b7460/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/java/org/apache/eagle/alert/app/AlertUnitTopologyAppProvider.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/java/org/apache/eagle/alert/app/AlertUnitTopologyAppProvider.java b/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/java/org/apache/eagle/alert/app/AlertUnitTopologyAppProvider.java
new file mode 100644
index 0000000..39a4583
--- /dev/null
+++ b/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/java/org/apache/eagle/alert/app/AlertUnitTopologyAppProvider.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eagle.alert.app;
+import org.apache.eagle.app.spi.AbstractApplicationProvider;
+
+/**
+ * since 8/25/16.
+ */
+public class AlertUnitTopologyAppProvider extends AbstractApplicationProvider<AlertUnitTopologyApp> {
+ @Override
+ public AlertUnitTopologyApp getApplication() {
+ return new AlertUnitTopologyApp();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/000b7460/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/providers/org.apache.eagle.alert.app.AlertUnitTopologyAppProvider.xml
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/providers/org.apache.eagle.alert.app.AlertUnitTopologyAppProvider.xml b/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/providers/org.apache.eagle.alert.app.AlertUnitTopologyAppProvider.xml
new file mode 100644
index 0000000..498cb8d
--- /dev/null
+++ b/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/providers/org.apache.eagle.alert.app.AlertUnitTopologyAppProvider.xml
@@ -0,0 +1,135 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<application>
+ <type>AlertUnitTopologyApp</type>
+ <name>Alert Unit Topology Application</name>
+ <version>0.5.0-incubating</version>
+ <appClass>org.apache.eagle.alert.app.AlertUnitTopologyApp</appClass>
+ <viewPath>/apps/alert</viewPath>
+ <configuration>
+ <!-- alert topology sizing parameters -->
+ <property>
+ <name>topology.numOfTotalWorkers</name>
+ <displayName>topology.numOfTotalWorkers</displayName>
+ <value>2</value>
+ <description>number of total storm workers</description>
+ </property>
+ <property>
+ <name>topology.numOfSpoutTasks</name>
+ <displayName>topology.numOfSpoutTasks</displayName>
+ <value>1</value>
+ <description>number of spout tasks</description>
+ </property>
+ <property>
+ <name>topology.numOfRouterBolts</name>
+ <displayName>topology.numOfRouterBolts</displayName>
+ <value>4</value>
+ <description>number of router tasks</description>
+ </property>
+ <property>
+ <name>topology.numOfAlertBolts</name>
+ <displayName>topology.numOfAlertBolts</displayName>
+ <value>10</value>
+ <description>number of alert tasks</description>
+ </property>
+ <property>
+ <name>topology.numOfPublishTasks</name>
+ <displayName>topology.numOfPublishTasks</displayName>
+ <value>1</value>
+ <description>number of publish tasks</description>
+ </property>
+ <property>
+ <name>topology.messageTimeoutSecs</name>
+ <displayName>topology.messageTimeoutSecs</displayName>
+ <value>3600</value>
+ <description>number of tuple timeout in seconds</description>
+ </property>
+
+ <!-- alert spout configuration -->
+ <property>
+ <name>spout.kafkaBrokerZkQuorum</name>
+ <displayName>spout.kafkaBrokerZkQuorum</displayName>
+ <value>server.eagle.apache.org:2181</value>
+ <description>zookeeper quorum for spout to consume data</description>
+ </property>
+ <property>
+ <name>spout.kafkaBrokerZkBasePath</name>
+ <displayName>spout.kafkaBrokerZkBasePath</displayName>
+ <value>/brokers</value>
+ <description>zk znode path for kafka brokers</description>
+ </property>
+ <property>
+ <name>spout.stormKafkaUseSameZkQuorumWithKafkaBroker</name>
+ <displayName>spout.stormKafkaUseSameZkQuorumWithKafkaBroker</displayName>
+ <value>true</value>
+ <description>same zookeeper for kafka server and kafka consumer(Storm-Kafka)</description>
+ </property>
+ <property>
+ <name>spout.stormKafkaTransactionZkPath</name>
+ <displayName>spout.stormKafkaTransactionZkPath</displayName>
+ <value>/consumers</value>
+ <description>zk path for storm kafka transaction</description>
+ </property>
+ <property>
+ <name>spout.stormKafkaEagleConsumer</name>
+ <displayName>spout.stormKafkaEagleConsumer</displayName>
+ <value>eagle_consumer</value>
+ <description>zookeeper quorum for spout to consume data</description>
+ </property>
+
+ <!-- zk config for alert engine -->
+ <property>
+ <name>zkConfig.zkQuorum</name>
+ <displayName>zkConfig.zkQuorum</displayName>
+ <value>server.eagle.apache.org:2181</value>
+ <description>zk quorum for alert engine</description>
+ </property>
+ <property>
+ <name>zkConfig.zkRoot</name>
+ <displayName>zkConfig.zkRoot</displayName>
+ <value>/alert</value>
+ <description>zk znode path for alert engine</description>
+ </property>
+
+ <property>
+ <name>metadataService.context</name>
+ <displayName>metadataService.context</displayName>
+ <value>/rest</value>
+ <description>metadata service context path</description>
+ </property>
+ <property>
+ <name>metadataService.host</name>
+ <displayName>metadataService.host</displayName>
+ <value>localhost</value>
+ <description>metadata service host</description>
+ </property>
+ <property>
+ <name>metadataService.port</name>
+ <displayName>metadataService.port</displayName>
+ <value>9090</value>
+ <description>metadata service port</description>
+ </property>
+ </configuration>
+ <docs>
+ <install>
+ </install>
+ <uninstall>
+ </uninstall>
+ </docs>
+</application>
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/000b7460/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/services/org.apache.eagle.app.spi.ApplicationProvider
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/services/org.apache.eagle.app.spi.ApplicationProvider b/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/services/org.apache.eagle.app.spi.ApplicationProvider
new file mode 100644
index 0000000..ceb079d
--- /dev/null
+++ b/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/services/org.apache.eagle.app.spi.ApplicationProvider
@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+org.apache.eagle.alert.app.AlertUnitTopologyAppProvider
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/000b7460/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/application.conf
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/application.conf b/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/application.conf
new file mode 100644
index 0000000..1a25cfa
--- /dev/null
+++ b/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/application.conf
@@ -0,0 +1,60 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+{
+ "appId" : "HBaseAuditLogApp",
+ "mode" : "LOCAL",
+ "siteId" : "testsite",
+ "topology" : {
+ "name" : "alertUnitTopology_1",
+ "numOfTotalWorkers" : 2,
+ "numOfSpoutTasks" : 1,
+ "numOfRouterBolts" : 4,
+ "numOfAlertBolts" : 10,
+ "numOfPublishTasks" : 1,
+ "messageTimeoutSecs": 3600,
+ "localMode" : "true"
+ },
+ "spout" : {
+ "kafkaBrokerZkQuorum": "server.eagle.apache.org:2181",
+ "kafkaBrokerZkBasePath": "/kafka",
+ "stormKafkaUseSameZkQuorumWithKafkaBroker": true,
+ "stormKafkaTransactionZkQuorum": "",
+ "stormKafkaTransactionZkPath": "/consumers",
+ "stormKafkaEagleConsumer": "eagle_consumer"
+ },
+ "zkConfig" : {
+ "zkQuorum" : "server.eagle.apache.org:2181",
+ "zkRoot" : "/alert"
+ },
+ "metadataService": {
+ "context" : "/rest",
+ "host" : "localhost",
+ "port" : 9090
+ },
+ "metric":{
+ "sink": {
+ // "kafka": {
+ // "topic": "alert_metric"
+ // "bootstrap.servers": "localhost:6667"
+ // }
+ // "stdout": {}
+ // "elasticsearch": {
+ // "hosts": ["localhost:9200"]
+ // "index": "alert_metric"
+ // "timestampField": "timestamp"
+ // }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/000b7460/eagle-core/eagle-alert-parent/eagle-alert-service/src/main/java/org/apache/eagle/service/alert/AlertPolicyValidateProvider.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert-service/src/main/java/org/apache/eagle/service/alert/AlertPolicyValidateProvider.java b/eagle-core/eagle-alert-parent/eagle-alert-service/src/main/java/org/apache/eagle/service/alert/AlertPolicyValidateProvider.java
deleted file mode 100644
index 47ddcb9..0000000
--- a/eagle-core/eagle-alert-parent/eagle-alert-service/src/main/java/org/apache/eagle/service/alert/AlertPolicyValidateProvider.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.service.alert;
-
-import java.util.List;
-
-import org.apache.eagle.log.entity.GenericServiceAPIResponseEntity;
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-import com.fasterxml.jackson.annotation.JsonTypeInfo;
-import com.fasterxml.jackson.databind.Module;
-
-@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", visible=true)
-@JsonIgnoreProperties(ignoreUnknown = true)
-public abstract class AlertPolicyValidateProvider {
- public String type;
-
- public abstract GenericServiceAPIResponseEntity<String> validate();
-
- public abstract String PolicyType();
-
- public abstract List<Module> BindingModules();
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/000b7460/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/config/ZKConfigBuilder.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/config/ZKConfigBuilder.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/config/ZKConfigBuilder.java
index 9d77a58..d7b3232 100644
--- a/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/config/ZKConfigBuilder.java
+++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/config/ZKConfigBuilder.java
@@ -29,10 +29,10 @@ public class ZKConfigBuilder {
ZKConfig zkConfig = new ZKConfig();
zkConfig.zkQuorum = config.getString("zkConfig.zkQuorum");
zkConfig.zkRoot = config.getString("zkConfig.zkRoot");
- zkConfig.zkSessionTimeoutMs = config.getInt("zkConfig.zkSessionTimeoutMs");
- zkConfig.connectionTimeoutMs = config.getInt("zkConfig.connectionTimeoutMs");
- zkConfig.zkRetryTimes = config.getInt("zkConfig.zkRetryTimes");
- zkConfig.zkRetryInterval = config.getInt("zkConfig.zkRetryInterval");
+ zkConfig.zkSessionTimeoutMs = config.hasPath("zkConfig.zkSessionTimeoutMs") ? config.getInt("zkConfig.zkSessionTimeoutMs") : 10000;
+ zkConfig.connectionTimeoutMs = config.hasPath("zkConfig.connectionTimeoutMs") ? config.getInt("zkConfig.connectionTimeoutMs") : 10000;
+ zkConfig.zkRetryTimes = config.hasPath("zkConfig.zkRetryTimes") ? config.getInt("zkConfig.zkRetryTimes") : 3;
+ zkConfig.zkRetryInterval = config.hasPath("zkConfig.zkRetryInterval") ? config.getInt("zkConfig.zkRetryInterval") : 3000;
return zkConfig;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/000b7460/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/scheme/JsonStringStreamNameSelector.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/scheme/JsonStringStreamNameSelector.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/scheme/JsonStringStreamNameSelector.java
index 1182e3f..226dd84 100644
--- a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/scheme/JsonStringStreamNameSelector.java
+++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/scheme/JsonStringStreamNameSelector.java
@@ -33,9 +33,9 @@ import org.slf4j.LoggerFactory;
*/
public class JsonStringStreamNameSelector implements StreamNameSelector {
private final static Logger LOG = LoggerFactory.getLogger(JsonStringStreamNameSelector.class);
- private final static String USER_PROVIDED_STREAM_NAME_PROPERTY = "userProvidedStreamName";
- private final static String FIELD_NAMES_TO_INFER_STREAM_NAME_PROPERTY = "fieldNamesToInferStreamName";
- private final static String STREAM_NAME_FORMAT = "streamNameFormat";
+ public final static String USER_PROVIDED_STREAM_NAME_PROPERTY = "userProvidedStreamName";
+ public final static String FIELD_NAMES_TO_INFER_STREAM_NAME_PROPERTY = "fieldNamesToInferStreamName";
+ public final static String STREAM_NAME_FORMAT = "streamNameFormat";
private String userProvidedStreamName;
private String[] fieldNamesToInferStreamName;
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/000b7460/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/spout/CorrelationSpout.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/spout/CorrelationSpout.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/spout/CorrelationSpout.java
index f54d5cd..99c1fed 100644
--- a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/spout/CorrelationSpout.java
+++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/spout/CorrelationSpout.java
@@ -332,9 +332,9 @@ public class CorrelationSpout extends BaseRichSpout implements SpoutSpecListener
spoutConfig.zkPort = utils.getZkPort();
}
// transaction update interval
- spoutConfig.stateUpdateIntervalMs = config.getLong("spout.stormKafkaStateUpdateIntervalMs");
+ spoutConfig.stateUpdateIntervalMs = config.hasPath("spout.stormKafkaStateUpdateIntervalMs") ? config.getInt("spout.stormKafkaStateUpdateIntervalMs") : 2000;
// Kafka fetch size
- spoutConfig.fetchSizeBytes = config.getInt("spout.stormKafkaFetchSizeBytes");
+ spoutConfig.fetchSizeBytes = config.hasPath("spout.stormKafkaFetchSizeBytes") ? config.getInt("spout.stormKafkaFetchSizeBytes") : 1048586;
// "startOffsetTime" is for test usage, prod should not use this
if (config.hasPath("spout.stormKafkaStartOffsetTime")) {
spoutConfig.startOffsetTime = config.getInt("spout.stormKafkaStartOffsetTime");
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/000b7460/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/resources/application.conf
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/resources/application.conf b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/resources/application.conf
index 7030e45..dac2f07 100644
--- a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/resources/application.conf
+++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/resources/application.conf
@@ -29,32 +29,17 @@
"stormKafkaUseSameZkQuorumWithKafkaBroker": true,
"stormKafkaTransactionZkQuorum": "",
"stormKafkaTransactionZkPath": "/consumers",
- "stormKafkaEagleConsumer": "eagle_consumer",
- "stormKafkaStateUpdateIntervalMs": 2000,
- "stormKafkaFetchSizeBytes": 1048586,
+ "stormKafkaEagleConsumer": "eagle_consumer"
},
"zkConfig" : {
"zkQuorum" : "server.eagle.apache.org:2181",
- "zkRoot" : "/alert",
- "zkSessionTimeoutMs" : 10000,
- "connectionTimeoutMs" : 10000,
- "zkRetryTimes" : 3,
- "zkRetryInterval" : 3000
- },
- "dynamicConfigSource" : {
- "initDelayMillis": 3000,
- "delayMillis" : 10000
+ "zkRoot" : "/alert"
},
"metadataService": {
"context" : "/rest",
"host" : "localhost",
"port" : 9090
},
- "coordinatorService": {
- "host": "localhost",
- "port": 9090,
- "context" : "/rest"
- }
"metric":{
"sink": {
// "kafka": {
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/000b7460/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/test/java/org/apache/eagle/alert/engine/e2e/CoordinatorClient.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/test/java/org/apache/eagle/alert/engine/e2e/CoordinatorClient.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/test/java/org/apache/eagle/alert/engine/e2e/CoordinatorClient.java
index aebf3b5..0cf5115 100644
--- a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/test/java/org/apache/eagle/alert/engine/e2e/CoordinatorClient.java
+++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/test/java/org/apache/eagle/alert/engine/e2e/CoordinatorClient.java
@@ -41,9 +41,9 @@ public class CoordinatorClient implements Closeable {
@SuppressWarnings("unused")
private static final Logger LOG = LoggerFactory.getLogger(CoordinatorClient.class);
- private static final String EAGLE_COORDINATOR_SERVICE_CONTEXT = "coordinatorService.context";
- private static final String EAGLE_COORDINATOR_SERVICE_PORT = "coordinatorService.port";
- private static final String EAGLE_COORDINATOR_SERVICE_HOST = "coordinatorService.host";
+ private static final String EAGLE_COORDINATOR_SERVICE_CONTEXT = "metadataService.context";
+ private static final String EAGLE_COORDINATOR_SERVICE_PORT = "metadataService.port";
+ private static final String EAGLE_COORDINATOR_SERVICE_HOST = "metadataService.host";
private static final String COORDINATOR_SCHEDULE_API = "/coordinator/build";
private String host;
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/000b7460/eagle-core/eagle-alert-parent/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/pom.xml b/eagle-core/eagle-alert-parent/pom.xml
index d5166cb..e2ba4d8 100644
--- a/eagle-core/eagle-alert-parent/pom.xml
+++ b/eagle-core/eagle-alert-parent/pom.xml
@@ -15,9 +15,7 @@
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.eagle</groupId>
@@ -33,5 +31,6 @@
<modules>
<module>eagle-alert</module>
<module>eagle-alert-service</module>
- </modules>
-</project>
+ <module>eagle-alert-app</module>
+ </modules>
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/000b7460/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationContext.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationContext.java b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationContext.java
index 52eb628..1a80b78 100644
--- a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationContext.java
+++ b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationContext.java
@@ -102,9 +102,9 @@ public class ApplicationContext implements Serializable, ApplicationLifecycle {
datasource.setTopic(kafkaCfg.getTopicId());
datasource.setSchemeCls(JsonScheme.class.getCanonicalName());
Tuple2StreamMetadata tuple2Stream = new Tuple2StreamMetadata();
- Set<String> activeStreamNames = new HashSet<>();
- activeStreamNames.add(streamDesc.getSchema().getStreamId());
- tuple2Stream.setActiveStreamNames(activeStreamNames);
+ Properties prop = new Properties();
+ prop.put(JsonStringStreamNameSelector.USER_PROVIDED_STREAM_NAME_PROPERTY, streamDesc.getStreamId());
+ tuple2Stream.setStreamNameSelectorProp(prop);
tuple2Stream.setTimestampColumn("timestamp");
tuple2Stream.setStreamNameSelectorCls(JsonStringStreamNameSelector.class.getCanonicalName());
datasource.setCodec(tuple2Stream);
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/000b7460/eagle-core/eagle-app/eagle-app-base/src/main/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-app/eagle-app-base/src/main/resources/log4j.properties b/eagle-core/eagle-app/eagle-app-base/src/main/resources/log4j.properties
index fb13ad5..d59ded6 100644
--- a/eagle-core/eagle-app/eagle-app-base/src/main/resources/log4j.properties
+++ b/eagle-core/eagle-app/eagle-app-base/src/main/resources/log4j.properties
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-log4j.rootLogger=DEBUG, stdout
+log4j.rootLogger=INFO, stdout
# standard output
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/000b7460/eagle-security/eagle-security-hdfs-auditlog/README.md
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hdfs-auditlog/README.md b/eagle-security/eagle-security-hdfs-auditlog/README.md
new file mode 100644
index 0000000..8d2e97a
--- /dev/null
+++ b/eagle-security/eagle-security-hdfs-auditlog/README.md
@@ -0,0 +1,191 @@
+<!--
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% endcomment %}
+-->
+
+Development in IDE
+
+## 1. Start eagle-server
+In IDE, configure the following main class and program arguments
+
+org.apache.eagle.server.ServerMain server src/main/resources/configuration.yml
+
+## 2. Start alert engine
+
+### 2.1 Create new site
+
+http://localhost:9090/rest/sites POST
+```
+{
+"siteId" : "testsite",
+"siteName" :"testsite",
+"description" : "test description",
+"context" : {}
+}
+```
+
+###n2.2 Create logic alert engine topology
+
+http://localhost:9090/rest/metadata/topologies POST
+```
+{
+ "name": "alertUnitTopology_1",
+ "numOfSpout": 1,
+ "numOfAlertBolt": 10,
+ "numOfGroupBolt": 4,
+ "spoutId": "alertEngineSpout",
+ "groupNodeIds": [
+ "streamRouterBolt0",
+ "streamRouterBolt1",
+ "streamRouterBolt2",
+ "streamRouterBolt3"
+ ],
+ "alertBoltIds": [
+ "alertBolt0",
+ "alertBolt1",
+ "alertBolt2",
+ "alertBolt3",
+ "alertBolt4",
+ "alertBolt5",
+ "alertBolt6",
+ "alertBolt7",
+ "alertBolt8",
+ "alertBolt9"
+ ],
+ "pubBoltId": "alertPublishBolt",
+ "spoutParallelism": 1,
+ "groupParallelism": 1,
+ "alertParallelism": 1
+}
+```
+
+### 2.3 Install alert engine application
+Please reference eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/providers/org.apache.eagle.alert.app.AlertUnitTopologyAppProvider.xml for
+complete configuration.
+
+http://localhost:9090/rest/apps/install POST
+```
+{
+"siteId" : "testsite",
+"appType" : "AlertUnitTopologyApp",
+"mode" : "LOCAL",
+"configuration" : {
+ }
+}
+```
+
+### 2.4 Run alert engine
+Please use correct uuid
+
+http://localhost:9090/rest/apps/start POST
+```
+{
+"uuid": "dc61c4b8-f60d-4d95-bfd7-f6b07382a3f3",
+"appId": "AlertUnitTopologyApp\uff0dtestsite"
+}
+```
+
+## 3 Start Hdfs audit log monitoring application
+
+### 3.1 Install HdfsAuditLog app
+
+http://localhost:9090/rest/apps/install POST
+```
+{
+"siteId" : "testsite",
+"appType" : "HdfsAuditLogApplication",
+"mode" : "LOCAL",
+"configuration" : {
+ "dataSourceConfig.topic" :"hdfs_audit_log"}
+}
+```
+
+### 3.2 Start HdfsAuditLog app
+Please use correct uuid
+
+http://localhost:9090/rest/apps/start POST
+```
+{
+"uuid": "dc61c4b8-f60d-4d95-bfd7-f6b07382a3f3",
+"appId": "HdfsAuditLogApplication\uff0dtestsite"
+}
+```
+
+## 4 Check
+### 4.1 Check if alert data source is created
+http://localhost:9090/rest/metadata/datasources GET
+
+### 4.2 Check if alert stream is creatd
+http://localhost:9090/rest/metadata/streams GET
+
+## 5 Create alert policy and verify alert
+### 5.1 create one policy
+
+http://localhost:9090/rest/metadata/policies POST
+```
+{
+ "name": "hdfsPolicy",
+ "description": "hdfsPolicy",
+ "inputStreams": [
+ "hdfs_audit_log_enriched_stream"
+ ],
+ "outputStreams": [
+ "hdfs_audit_log_enriched_stream_out"
+ ],
+ "definition": {
+ "type": "siddhi",
+ "value": "from hdfs_audit_log_enriched_stream[user=='hadoop'] select * insert into hdfs_audit_log_enriched_stream_out"
+ },
+ "partitionSpec": [
+ {
+ "streamId": "hdfs_audit_log_enriched_stream",
+ "type": "GROUPBY",
+ "columns" : [
+ "user"
+ ]
+ }
+ ],
+ "parallelismHint": 2
+}
+```
+
+### 5.2 Create alert publishment
+```
+{
+ "name":"hdfs_audit_log_enriched_stream_out",
+ "type":"org.apache.eagle.alert.engine.publisher.impl.AlertEmailPublisher",
+ "policyIds": [
+ "hdfsPolicy"
+ ],
+ "properties": {
+ "subject":"alert when user is hadoop",
+ "template":"",
+ "sender": "eagle@apache.org",
+ "recipients": "eagle@apache.org",
+ "mail.smtp.host":"",
+ "connection": "plaintext",
+ "mail.smtp.port": "25"
+ },
+ "dedupIntervalMin" : "PT1M",
+ "serializer" : "org.apache.eagle.alert.engine.publisher.impl.StringEventSerializer"
+}
+```
+
+### 5.3 Send message and verify alert
+./kafka-console-producer.sh --topic hdfs_audit_log --broker-list sandbox.hortonworks.com:6667
+
+2015-04-24 12:51:31,798 INFO FSNamesystem.audit: allowed=true ugi=hdfs (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/apps/hbase/data dst=null perm=null proto=rpc
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/000b7460/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.auditlog.HdfsAuditLogAppProvider.xml
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.auditlog.HdfsAuditLogAppProvider.xml b/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.auditlog.HdfsAuditLogAppProvider.xml
index 2a8ff0f..2fb7274 100644
--- a/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.auditlog.HdfsAuditLogAppProvider.xml
+++ b/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.auditlog.HdfsAuditLogAppProvider.xml
@@ -155,21 +155,41 @@
<timeseries>true</timeseries>
<columns>
<column>
- <name>action</name>
+ <name>src</name>
<type>string</type>
</column>
<column>
- <name>host</name>
+ <name>dst</name>
<type>string</type>
</column>
<column>
- <name>status</name>
+ <name>host</name>
<type>string</type>
</column>
<column>
<name>timestamp</name>
<type>long</type>
</column>
+ <column>
+ <name>allowed</name>
+ <type>bool</type>
+ </column>
+ <column>
+ <name>user</name>
+ <type>string</type>
+ </column>
+ <column>
+ <name>cmd</name>
+ <type>string</type>
+ </column>
+ <column>
+ <name>sensitivityType</name>
+ <type>string</type>
+ </column>
+ <column>
+ <name>securityZone</name>
+ <type>string</type>
+ </column>
</columns>
</stream>
</streams>
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/000b7460/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/scripts.txt
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/scripts.txt b/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/scripts.txt
new file mode 100644
index 0000000..c5f7c9b
--- /dev/null
+++ b/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/scripts.txt
@@ -0,0 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+./kafka-topics.sh --topic hdfs_audit_log --zookeeper localhost:2181 --create --replication-factor 1 --partitions 2
+
+./kafka-topics.sh --topic hdfs_audit_log --zookeeper localhost:2181 --create --replication-factor 1 --partitions 2
+
+./kafka-console-producer.sh --topic hdfs_audit_log --broker-list sandbox.hortonworks.com:6667
+
+./kafka-console-consumer.sh --topic hdfs_audit_log_parsed --zookeeper sandbox.hortonworks.com:2181 --from-beginning
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/000b7460/eagle-server/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-server/pom.xml b/eagle-server/pom.xml
index bc9a919..5e07a7d 100644
--- a/eagle-server/pom.xml
+++ b/eagle-server/pom.xml
@@ -164,6 +164,13 @@
<artifactId>eagle-gc</artifactId>
<version>${project.version}</version>
</dependency>
+
+ <!-- Alert unit topology as an application -->
+ <dependency>
+ <groupId>org.apache.eagle</groupId>
+ <artifactId>eagle-alert-app</artifactId>
+ <version>${project.version}</version>
+ </dependency>
</dependencies>
<build>
<resources>