You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@eagle.apache.org by ha...@apache.org on 2015/11/30 03:48:58 UTC
[1/5] incubator-eagle git commit: change package name
Repository: incubator-eagle
Updated Branches:
refs/heads/master 5bffdfd3a -> dfcf14098
change package name
Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/4329ef12
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/4329ef12
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/4329ef12
Branch: refs/heads/master
Commit: 4329ef12365fbf163fc91126271ddad0d54198b9
Parents: 5bffdfd
Author: Zhao, Qingwen <qi...@ebay.com>
Authored: Wed Nov 25 19:38:30 2015 +0800
Committer: Zhao, Qingwen <qi...@ebay.com>
Committed: Wed Nov 25 19:41:32 2015 +0800
----------------------------------------------------------------------
eagle-assembly/src/main/bin/eagle-topology.sh | 3 +-
.../bin/hdfs-securitylog-metadata-create.sh | 2 +-
.../sandbox-hdfsSecurityLog-application.conf | 2 +-
.../HDFSSecurityLogKafkaDeserializer.java | 63 ---------------
.../HDFSSecurityLogProcessorMain.java | 50 ------------
.../parse/HDFSSecurityLogObject.java | 27 -------
.../parse/HDFSSecurityLogParser.java | 82 --------------------
.../HDFSSecurityLogKafkaDeserializer.java | 63 +++++++++++++++
.../HDFSSecurityLogProcessorMain.java | 50 ++++++++++++
.../parse/HDFSSecurityLogObject.java | 27 +++++++
.../parse/HDFSSecurityLogParser.java | 82 ++++++++++++++++++++
.../src/main/resources/application.conf | 2 +-
.../security/TestHDFSSecuritylogParser.java | 41 ----------
.../security/TestHDFSSecuritylogParser.java | 41 ++++++++++
14 files changed, 268 insertions(+), 267 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/4329ef12/eagle-assembly/src/main/bin/eagle-topology.sh
----------------------------------------------------------------------
diff --git a/eagle-assembly/src/main/bin/eagle-topology.sh b/eagle-assembly/src/main/bin/eagle-topology.sh
index c8625e0..ebcef9c 100755
--- a/eagle-assembly/src/main/bin/eagle-topology.sh
+++ b/eagle-assembly/src/main/bin/eagle-topology.sh
@@ -111,7 +111,8 @@ done
if [ -z "$jarName" ]; then
- jarName="${EAGLE_HOME}/lib/topology/eagle-topology-0.1.0-assembly.jar"
+ echo "Error: jar file is not found"
+ exit 1
fi
if [ -z "$mainClass" ]; then
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/4329ef12/eagle-assembly/src/main/bin/hdfs-securitylog-metadata-create.sh
----------------------------------------------------------------------
diff --git a/eagle-assembly/src/main/bin/hdfs-securitylog-metadata-create.sh b/eagle-assembly/src/main/bin/hdfs-securitylog-metadata-create.sh
index 113bc66..7aa09a9 100644
--- a/eagle-assembly/src/main/bin/hdfs-securitylog-metadata-create.sh
+++ b/eagle-assembly/src/main/bin/hdfs-securitylog-metadata-create.sh
@@ -31,7 +31,7 @@ curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:a
#### AlertStreamSchemaService: schema for event from alert stream
-curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=AlertStreamSchemaService" -d '[{"prefix":"alertStreamSchema","tags":{"dataSource":"hdfsSecurityLog","streamName":"hdfsSecurityLogEventStream","attrName":"timestamp"},"attrDescription":"milliseconds of the datetime","attrType":"long","category":"","attrValueResolver":""},{"prefix":"alertStreamSchema","tags":{"dataSource":"hdfsSecurityLog","streamName":"hdfsSecurityLogEventStream","attrName":"user"},"attrDescription":"","attrType":"string","category":"","attrValueResolver":"eagle.service.security.hbase.resolver.HbaseRequestResolver"},{"prefix":"alertStreamSchema","tags":{"dataSource":"hdfsSecurityLog","streamName":"hdfsSecurityLogEventStream","attrName":"allowed"},"attrDescription":"true, false or none","attrType":"bool","category":"","attrValueResolver":""}]'
+curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=AlertStreamSchemaService" -d '[{"prefix":"alertStreamSchema","tags":{"dataSource":"hdfsSecurityLog","streamName":"hdfsSecurityLogEventStream","attrName":"timestamp"},"attrDescription":"milliseconds of the datetime","attrType":"long","category":"","attrValueResolver":""},{"prefix":"alertStreamSchema","tags":{"dataSource":"hdfsSecurityLog","streamName":"hdfsSecurityLogEventStream","attrName":"user"},"attrDescription":"","attrType":"string","category":"","attrValueResolver":""},{"prefix":"alertStreamSchema","tags":{"dataSource":"hdfsSecurityLog","streamName":"hdfsSecurityLogEventStream","attrName":"allowed"},"attrDescription":"true, false or none","attrType":"bool","category":"","attrValueResolver":""}]'
curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=AlertDataSourceService" -d '[{"prefix":"alertDataSource","tags":{"site":"sandbox","dataSource":"hdfsSecurityLog"},"enabled":"true","desc":"HDFS Security"}]'
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/4329ef12/eagle-assembly/src/main/conf/sandbox-hdfsSecurityLog-application.conf
----------------------------------------------------------------------
diff --git a/eagle-assembly/src/main/conf/sandbox-hdfsSecurityLog-application.conf b/eagle-assembly/src/main/conf/sandbox-hdfsSecurityLog-application.conf
index bc2baf8..c70d811 100644
--- a/eagle-assembly/src/main/conf/sandbox-hdfsSecurityLog-application.conf
+++ b/eagle-assembly/src/main/conf/sandbox-hdfsSecurityLog-application.conf
@@ -30,7 +30,7 @@
"brokerZkPath" : "/brokers",
"zkConnectionTimeoutMS" : 15000,
"fetchSize" : 1048586,
- "deserializerClass" : "eagle.security.securitylog.HDFSSecurityLogKafkaDeserializer",
+ "deserializerClass" : "org.apache.eagle.security.securitylog.HDFSSecurityLogKafkaDeserializer",
"transactionZKServers" : "127.0.0.1",
"transactionZKPort" : 2181,
"transactionZKRoot" : "/consumers",
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/4329ef12/eagle-security/eagle-security-hdfs-securitylog/src/main/java/eagle/security/securitylog/HDFSSecurityLogKafkaDeserializer.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hdfs-securitylog/src/main/java/eagle/security/securitylog/HDFSSecurityLogKafkaDeserializer.java b/eagle-security/eagle-security-hdfs-securitylog/src/main/java/eagle/security/securitylog/HDFSSecurityLogKafkaDeserializer.java
deleted file mode 100644
index 2c54f9a..0000000
--- a/eagle-security/eagle-security-hdfs-securitylog/src/main/java/eagle/security/securitylog/HDFSSecurityLogKafkaDeserializer.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package eagle.security.securitylog;
-
-
-import eagle.security.securitylog.parse.HDFSSecurityLogObject;
-import org.apache.eagle.dataproc.impl.storm.kafka.SpoutKafkaMessageDeserializer;
-import eagle.security.securitylog.parse.HDFSSecurityLogParser;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Map;
-import java.util.Properties;
-import java.util.TreeMap;
-
-public class HDFSSecurityLogKafkaDeserializer implements SpoutKafkaMessageDeserializer {
- Logger LOG = LoggerFactory.getLogger(HDFSSecurityLogKafkaDeserializer.class);
-
- private Properties props;
-
- public HDFSSecurityLogKafkaDeserializer(Properties props){
- this.props = props;
- }
-
- @Override
- public Object deserialize(byte[] arg0) {
- String logLine = new String(arg0);
-
- HDFSSecurityLogParser parser = new HDFSSecurityLogParser();
- HDFSSecurityLogObject entity = null;
- try{
- entity = parser.parse(logLine);
- }catch(Exception ex){
- LOG.error("Failing parse audit log message", ex);
- }
- if(entity == null){
- LOG.warn("Event ignored as it can't be correctly parsed, the log is ", logLine);
- return null;
- }
- Map<String, Object> map = new TreeMap<String, Object>();
- map.put("timestamp", entity.timestamp);
- map.put("allowed", entity.allowed);
- map.put("user", entity.user);
-
- return map;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/4329ef12/eagle-security/eagle-security-hdfs-securitylog/src/main/java/eagle/security/securitylog/HDFSSecurityLogProcessorMain.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hdfs-securitylog/src/main/java/eagle/security/securitylog/HDFSSecurityLogProcessorMain.java b/eagle-security/eagle-security-hdfs-securitylog/src/main/java/eagle/security/securitylog/HDFSSecurityLogProcessorMain.java
deleted file mode 100644
index c87e1bc..0000000
--- a/eagle-security/eagle-security-hdfs-securitylog/src/main/java/eagle/security/securitylog/HDFSSecurityLogProcessorMain.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package eagle.security.securitylog;
-
-
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigFactory;
-import com.typesafe.config.ConfigRenderOptions;
-import org.apache.eagle.dataproc.impl.storm.kafka.KafkaSourcedSpoutProvider;
-import org.apache.eagle.dataproc.util.ConfigOptionParser;
-import org.apache.eagle.datastream.ExecutionEnvironmentFactory;
-import org.apache.eagle.datastream.StormExecutionEnvironment;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class HDFSSecurityLogProcessorMain {
- private static final Logger LOG = LoggerFactory.getLogger(HDFSSecurityLogProcessorMain.class);
-
- public static void main(String[] args) throws Exception{
- new ConfigOptionParser().load(args);
- System.setProperty("config.trace", "loads");
- Config config = ConfigFactory.load();
-
- LOG.info("Config class: " + config.getClass().getCanonicalName());
-
- if(LOG.isDebugEnabled()) LOG.debug("Config content:"+config.root().render(ConfigRenderOptions.concise()));
-
- StormExecutionEnvironment env = ExecutionEnvironmentFactory.getStorm(config);
-
- env.newSource(new KafkaSourcedSpoutProvider().getSpout(config)).renameOutputFields(1).withName("kafkaMsgConsumer")
- .alertWithConsumer("hdfsSecurityLogEventStream", "hdfsSecurityLogAlertExecutor");
- env.execute();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/4329ef12/eagle-security/eagle-security-hdfs-securitylog/src/main/java/eagle/security/securitylog/parse/HDFSSecurityLogObject.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hdfs-securitylog/src/main/java/eagle/security/securitylog/parse/HDFSSecurityLogObject.java b/eagle-security/eagle-security-hdfs-securitylog/src/main/java/eagle/security/securitylog/parse/HDFSSecurityLogObject.java
deleted file mode 100644
index 5e3e8ba..0000000
--- a/eagle-security/eagle-security-hdfs-securitylog/src/main/java/eagle/security/securitylog/parse/HDFSSecurityLogObject.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package eagle.security.securitylog.parse;
-
-
-public class HDFSSecurityLogObject {
- public long timestamp;
- public Boolean allowed;
- public String user;
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/4329ef12/eagle-security/eagle-security-hdfs-securitylog/src/main/java/eagle/security/securitylog/parse/HDFSSecurityLogParser.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hdfs-securitylog/src/main/java/eagle/security/securitylog/parse/HDFSSecurityLogParser.java b/eagle-security/eagle-security-hdfs-securitylog/src/main/java/eagle/security/securitylog/parse/HDFSSecurityLogParser.java
deleted file mode 100644
index 144fb94..0000000
--- a/eagle-security/eagle-security-hdfs-securitylog/src/main/java/eagle/security/securitylog/parse/HDFSSecurityLogParser.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package eagle.security.securitylog.parse;
-
-
-import org.apache.eagle.security.util.LogParseUtil;
-import org.apache.eagle.common.DateTimeUtil;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.Serializable;
-import java.text.ParseException;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-public class HDFSSecurityLogParser implements Serializable {
- private final static Logger LOG = LoggerFactory.getLogger(HDFSSecurityLogParser.class);
-
- private final static int LOGDATE_INDEX = 1;
- private final static int LOGATTRS_INDEX = 4;
- private final static String SUCCESSFUL = "successful";
- private final static String USER = "user";
- private final static String ALLOWED = "allowed";
- private final static Pattern loggerPattern = Pattern.compile("^([\\d\\s\\-:,]+)\\s+(\\w+)\\s+(.*):\\s+(.*)");
-
- public HDFSSecurityLogObject parse(String logLine) throws ParseException {
- Matcher logMatcher = loggerPattern.matcher(logLine);
- if(!logMatcher.find()) return null;
-
- HDFSSecurityLogObject ret = new HDFSSecurityLogObject();
- String date = logMatcher.group(LOGDATE_INDEX);
- ret.timestamp = DateTimeUtil.humanDateToMilliseconds(logMatcher.group(LOGDATE_INDEX));
-
- Map<String, String> attrMap = parseAttr(logMatcher.group(LOGATTRS_INDEX));
-
- if(attrMap == null) return null;
-
- ret.user = attrMap.get(USER);
- if(attrMap.get(ALLOWED).equals(ALLOWED)) {
- ret.allowed = true;
- } else {
- ret.allowed = false;
- }
-
- return ret;
- }
-
- private Map<String, String> parseAttr(String logLine) {
- if(logLine == null) return null;
-
- Map<String, String> ret = new HashMap<>();
- String [] fields = logLine.split("\\s+");
-
- if(fields[1].equalsIgnoreCase(SUCCESSFUL)) {
- ret.put(ALLOWED, "allowed");
- } else {
- ret.put(ALLOWED, "denied");
- }
-
- String user = LogParseUtil.parseUserFromUGI(fields[3]);
- ret.put(USER, user);
- return ret;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/4329ef12/eagle-security/eagle-security-hdfs-securitylog/src/main/java/org/apache/eagle/security/securitylog/HDFSSecurityLogKafkaDeserializer.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hdfs-securitylog/src/main/java/org/apache/eagle/security/securitylog/HDFSSecurityLogKafkaDeserializer.java b/eagle-security/eagle-security-hdfs-securitylog/src/main/java/org/apache/eagle/security/securitylog/HDFSSecurityLogKafkaDeserializer.java
new file mode 100644
index 0000000..5dada1e
--- /dev/null
+++ b/eagle-security/eagle-security-hdfs-securitylog/src/main/java/org/apache/eagle/security/securitylog/HDFSSecurityLogKafkaDeserializer.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.security.securitylog;
+
+
+import org.apache.eagle.security.securitylog.parse.HDFSSecurityLogObject;
+import org.apache.eagle.dataproc.impl.storm.kafka.SpoutKafkaMessageDeserializer;
+import org.apache.eagle.security.securitylog.parse.HDFSSecurityLogParser;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Map;
+import java.util.Properties;
+import java.util.TreeMap;
+
+public class HDFSSecurityLogKafkaDeserializer implements SpoutKafkaMessageDeserializer {
+ Logger LOG = LoggerFactory.getLogger(HDFSSecurityLogKafkaDeserializer.class);
+
+ private Properties props;
+
+ public HDFSSecurityLogKafkaDeserializer(Properties props){
+ this.props = props;
+ }
+
+ @Override
+ public Object deserialize(byte[] arg0) {
+ String logLine = new String(arg0);
+
+ HDFSSecurityLogParser parser = new HDFSSecurityLogParser();
+ HDFSSecurityLogObject entity = null;
+ try{
+ entity = parser.parse(logLine);
+ }catch(Exception ex){
+ LOG.error("Failing parse audit log message", ex);
+ }
+ if(entity == null){
+ LOG.warn("Event ignored as it can't be correctly parsed, the log is ", logLine);
+ return null;
+ }
+ Map<String, Object> map = new TreeMap<String, Object>();
+ map.put("timestamp", entity.timestamp);
+ map.put("allowed", entity.allowed);
+ map.put("user", entity.user);
+
+ return map;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/4329ef12/eagle-security/eagle-security-hdfs-securitylog/src/main/java/org/apache/eagle/security/securitylog/HDFSSecurityLogProcessorMain.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hdfs-securitylog/src/main/java/org/apache/eagle/security/securitylog/HDFSSecurityLogProcessorMain.java b/eagle-security/eagle-security-hdfs-securitylog/src/main/java/org/apache/eagle/security/securitylog/HDFSSecurityLogProcessorMain.java
new file mode 100644
index 0000000..b514a7c
--- /dev/null
+++ b/eagle-security/eagle-security-hdfs-securitylog/src/main/java/org/apache/eagle/security/securitylog/HDFSSecurityLogProcessorMain.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.security.securitylog;
+
+
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+import com.typesafe.config.ConfigRenderOptions;
+import org.apache.eagle.dataproc.impl.storm.kafka.KafkaSourcedSpoutProvider;
+import org.apache.eagle.dataproc.util.ConfigOptionParser;
+import org.apache.eagle.datastream.ExecutionEnvironmentFactory;
+import org.apache.eagle.datastream.StormExecutionEnvironment;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class HDFSSecurityLogProcessorMain {
+ private static final Logger LOG = LoggerFactory.getLogger(HDFSSecurityLogProcessorMain.class);
+
+ public static void main(String[] args) throws Exception{
+ new ConfigOptionParser().load(args);
+ System.setProperty("config.trace", "loads");
+ Config config = ConfigFactory.load();
+
+ LOG.info("Config class: " + config.getClass().getCanonicalName());
+
+ if(LOG.isDebugEnabled()) LOG.debug("Config content:"+config.root().render(ConfigRenderOptions.concise()));
+
+ StormExecutionEnvironment env = ExecutionEnvironmentFactory.getStorm(config);
+
+ env.newSource(new KafkaSourcedSpoutProvider().getSpout(config)).renameOutputFields(1).withName("kafkaMsgConsumer")
+ .alertWithConsumer("hdfsSecurityLogEventStream", "hdfsSecurityLogAlertExecutor");
+ env.execute();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/4329ef12/eagle-security/eagle-security-hdfs-securitylog/src/main/java/org/apache/eagle/security/securitylog/parse/HDFSSecurityLogObject.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hdfs-securitylog/src/main/java/org/apache/eagle/security/securitylog/parse/HDFSSecurityLogObject.java b/eagle-security/eagle-security-hdfs-securitylog/src/main/java/org/apache/eagle/security/securitylog/parse/HDFSSecurityLogObject.java
new file mode 100644
index 0000000..fe19ff2
--- /dev/null
+++ b/eagle-security/eagle-security-hdfs-securitylog/src/main/java/org/apache/eagle/security/securitylog/parse/HDFSSecurityLogObject.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.security.securitylog.parse;
+
+
+public class HDFSSecurityLogObject {
+ public long timestamp;
+ public Boolean allowed;
+ public String user;
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/4329ef12/eagle-security/eagle-security-hdfs-securitylog/src/main/java/org/apache/eagle/security/securitylog/parse/HDFSSecurityLogParser.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hdfs-securitylog/src/main/java/org/apache/eagle/security/securitylog/parse/HDFSSecurityLogParser.java b/eagle-security/eagle-security-hdfs-securitylog/src/main/java/org/apache/eagle/security/securitylog/parse/HDFSSecurityLogParser.java
new file mode 100644
index 0000000..efd2371
--- /dev/null
+++ b/eagle-security/eagle-security-hdfs-securitylog/src/main/java/org/apache/eagle/security/securitylog/parse/HDFSSecurityLogParser.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.security.securitylog.parse;
+
+
+import org.apache.eagle.security.util.LogParseUtil;
+import org.apache.eagle.common.DateTimeUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Serializable;
+import java.text.ParseException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class HDFSSecurityLogParser implements Serializable {
+ private final static Logger LOG = LoggerFactory.getLogger(HDFSSecurityLogParser.class);
+
+ private final static int LOGDATE_INDEX = 1;
+ private final static int LOGATTRS_INDEX = 4;
+ private final static String SUCCESSFUL = "successful";
+ private final static String USER = "user";
+ private final static String ALLOWED = "allowed";
+ private final static Pattern loggerPattern = Pattern.compile("^([\\d\\s\\-:,]+)\\s+(\\w+)\\s+(.*):\\s+(.*)");
+
+ public HDFSSecurityLogObject parse(String logLine) throws ParseException {
+ Matcher logMatcher = loggerPattern.matcher(logLine);
+ if(!logMatcher.find()) return null;
+
+ HDFSSecurityLogObject ret = new HDFSSecurityLogObject();
+ String date = logMatcher.group(LOGDATE_INDEX);
+ ret.timestamp = DateTimeUtil.humanDateToMilliseconds(logMatcher.group(LOGDATE_INDEX));
+
+ Map<String, String> attrMap = parseAttr(logMatcher.group(LOGATTRS_INDEX));
+
+ if(attrMap == null) return null;
+
+ ret.user = attrMap.get(USER);
+ if(attrMap.get(ALLOWED).equals(ALLOWED)) {
+ ret.allowed = true;
+ } else {
+ ret.allowed = false;
+ }
+
+ return ret;
+ }
+
+ private Map<String, String> parseAttr(String logLine) {
+ if(logLine == null) return null;
+
+ Map<String, String> ret = new HashMap<>();
+ String [] fields = logLine.split("\\s+");
+
+ if(fields[1].equalsIgnoreCase(SUCCESSFUL)) {
+ ret.put(ALLOWED, "allowed");
+ } else {
+ ret.put(ALLOWED, "denied");
+ }
+
+ String user = LogParseUtil.parseUserFromUGI(fields[3]);
+ ret.put(USER, user);
+ return ret;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/4329ef12/eagle-security/eagle-security-hdfs-securitylog/src/main/resources/application.conf
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hdfs-securitylog/src/main/resources/application.conf b/eagle-security/eagle-security-hdfs-securitylog/src/main/resources/application.conf
index fa970d1..9900ac8 100644
--- a/eagle-security/eagle-security-hdfs-securitylog/src/main/resources/application.conf
+++ b/eagle-security/eagle-security-hdfs-securitylog/src/main/resources/application.conf
@@ -29,7 +29,7 @@
"zkConnection" : "sandbox.hortonworks.com:2181",
"zkConnectionTimeoutMS" : 15000,
"fetchSize" : 1048586,
- "deserializerClass" : "eagle.security.securitylog.HDFSSecurityLogKafkaDeserializer",
+ "deserializerClass" : "org.apache.eagle.security.securitylog.HDFSSecurityLogKafkaDeserializer",
"transactionZKServers" : "sandbox.hortonworks.com",
"transactionZKPort" : 2181,
"transactionZKRoot" : "/consumers",
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/4329ef12/eagle-security/eagle-security-hdfs-securitylog/src/test/java/eagle/security/TestHDFSSecuritylogParser.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hdfs-securitylog/src/test/java/eagle/security/TestHDFSSecuritylogParser.java b/eagle-security/eagle-security-hdfs-securitylog/src/test/java/eagle/security/TestHDFSSecuritylogParser.java
deleted file mode 100644
index 4c79f86..0000000
--- a/eagle-security/eagle-security-hdfs-securitylog/src/test/java/eagle/security/TestHDFSSecuritylogParser.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package eagle.security;
-
-
-import eagle.security.securitylog.parse.HDFSSecurityLogObject;
-import eagle.security.securitylog.parse.HDFSSecurityLogParser;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.text.ParseException;
-
-public class TestHDFSSecuritylogParser {
-
- @Test
- public void test() throws ParseException {
- String msg = "2015-11-18 08:41:10,200 INFO SecurityLogger.org.apache.hadoop.security.authorize.ServiceAuthorizationManager: Authorization successful for hbase (auth:SIMPLE) for protocol=interface org.apache.hadoop.hdfs.protocol.ClientProtocol";
-
- HDFSSecurityLogParser parser = new HDFSSecurityLogParser();
- HDFSSecurityLogObject obj = parser.parse(msg);
-
- Assert.assertEquals("hbase", obj.user);
- Assert.assertEquals(true, obj.allowed);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/4329ef12/eagle-security/eagle-security-hdfs-securitylog/src/test/java/org/apache/eagle/security/TestHDFSSecuritylogParser.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hdfs-securitylog/src/test/java/org/apache/eagle/security/TestHDFSSecuritylogParser.java b/eagle-security/eagle-security-hdfs-securitylog/src/test/java/org/apache/eagle/security/TestHDFSSecuritylogParser.java
new file mode 100644
index 0000000..b72d05e
--- /dev/null
+++ b/eagle-security/eagle-security-hdfs-securitylog/src/test/java/org/apache/eagle/security/TestHDFSSecuritylogParser.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.security;
+
+
+import org.apache.eagle.security.securitylog.parse.HDFSSecurityLogObject;
+import org.apache.eagle.security.securitylog.parse.HDFSSecurityLogParser;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.text.ParseException;
+
+public class TestHDFSSecuritylogParser {
+
+ @Test
+ public void test() throws ParseException {
+ String msg = "2015-11-18 08:41:10,200 INFO SecurityLogger.org.apache.hadoop.security.authorize.ServiceAuthorizationManager: Authorization successful for hbase (auth:SIMPLE) for protocol=interface org.apache.hadoop.hdfs.protocol.ClientProtocol";
+
+ HDFSSecurityLogParser parser = new HDFSSecurityLogParser();
+ HDFSSecurityLogObject obj = parser.parse(msg);
+
+ Assert.assertEquals("hbase", obj.user);
+ Assert.assertEquals(true, obj.allowed);
+ }
+}
[5/5] incubator-eagle git commit: add exception processing
Posted by ha...@apache.org.
add exception processing
Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/dfcf1409
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/dfcf1409
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/dfcf1409
Branch: refs/heads/master
Commit: dfcf1409856521925fdb326c086da58f318fd334
Parents: 4046666
Author: Zhao, Qingwen <qi...@ebay.com>
Authored: Fri Nov 27 21:37:12 2015 +0800
Committer: Zhao, Qingwen <qi...@ebay.com>
Committed: Fri Nov 27 21:37:12 2015 +0800
----------------------------------------------------------------------
.../eagle/datastream/StormTopologyExecutorImpl.scala | 11 ++++++++---
1 file changed, 8 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/dfcf1409/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StormTopologyExecutorImpl.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StormTopologyExecutorImpl.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StormTopologyExecutorImpl.scala
index 48ee9c4..df763b9 100644
--- a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StormTopologyExecutorImpl.scala
+++ b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StormTopologyExecutorImpl.scala
@@ -43,9 +43,14 @@ case class StormTopologyExecutorImpl(topology: StormTopology, config: com.typesa
if(file.exists()) {
val inputFileStream = new FileInputStream(file)
val yaml = new Yaml()
- val stormConf = yaml.load(inputFileStream).asInstanceOf[java.util.LinkedHashMap[String, Object]]
- conf.putAll(stormConf)
- inputFileStream.close()
+ try {
+ val stormConf = yaml.load(inputFileStream).asInstanceOf[java.util.LinkedHashMap[String, Object]]
+ if(stormConf != null) conf.putAll(stormConf)
+ } catch {
+ case _ => ()
+ } finally {
+ if(inputFileStream != null) inputFileStream.close()
+ }
}
}
[4/5] incubator-eagle git commit: remove --jar in eagle-topology.sh
Posted by ha...@apache.org.
remove --jar in eagle-topology.sh
Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/4046666b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/4046666b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/4046666b
Branch: refs/heads/master
Commit: 4046666b170d4b3d8b3fcbcf3fc1b24e55f0d2dc
Parents: c38b922
Author: Zhao, Qingwen <qi...@ebay.com>
Authored: Thu Nov 26 15:44:02 2015 +0800
Committer: Zhao, Qingwen <qi...@ebay.com>
Committed: Thu Nov 26 15:44:02 2015 +0800
----------------------------------------------------------------------
eagle-assembly/src/main/bin/eagle-topology.sh | 15 ++++-----------
1 file changed, 4 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/4046666b/eagle-assembly/src/main/bin/eagle-topology.sh
----------------------------------------------------------------------
diff --git a/eagle-assembly/src/main/bin/eagle-topology.sh b/eagle-assembly/src/main/bin/eagle-topology.sh
index ebcef9c..e1104ac 100755
--- a/eagle-assembly/src/main/bin/eagle-topology.sh
+++ b/eagle-assembly/src/main/bin/eagle-topology.sh
@@ -22,14 +22,13 @@ TOPOLOGY_NAME_SET=0
function print_help() {
echo "Usage: $0 options {start | stop | status}"
echo "Options: Description:"
- echo " --jar <jar path> Default is $EAGLE_HOME/lib/topology/eagle-topology-0.1.0-assembly.jar"
echo " --main <main class> Default is org.apache.eagle.security.auditlog.HdfsAuditLogProcessorMain"
echo " --topology <topology name> Default is sandbox-hdfsAuditLog-topology"
echo " --config <file path> Default is $EAGLE_HOME/conf/sandbox-hdfsAuditLog-application.conf"
echo " --storm-ui <storm ui url> Execute through storm UI API, default: http://localhost:8744"
echo "Command Examples:"
- echo " $0 [--jar <jarPath>] --main <mainClass> [--topology <topologyName>] --config <filePath> start"
+ echo " $0 --main <mainClass> [--topology <topologyName>] --config <filePath> start"
echo " $0 --topology <topologyName> stop"
echo " $0 --topology <topologyName> status"
}
@@ -64,14 +63,6 @@ case $1 in
cmd=$1
shift
;;
- --jar)
- if [ $# -lt 3 ]; then
- print_help
- exit 1
- fi
- jarName=$2
- shift 2
- ;;
--main)
if [ $# -lt 3 ]; then
print_help
@@ -110,6 +101,8 @@ case $1 in
done
+jarName=$(ls ${EAGLE_HOME}/lib/topology/eagle-topology-*-assembly.jar)
+
if [ -z "$jarName" ]; then
echo "Error: jar file is not found"
exit 1
@@ -163,7 +156,7 @@ case $cmd in
echo "Checking topology $topologyName status ..."
output=`storm list -c nimbus.host=$EAGLE_NIMBUS_HOST | grep $topologyName`
if [ $? != 0 ];then
- echo "Topolog is not alive: $topologyName is not found"
+ echo "Topology is not alive: $topologyName is not found"
exit 1
fi
[3/5] incubator-eagle git commit: Fix quartz job have already exists
error
Posted by ha...@apache.org.
Fix quartz job have already exists error
Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/c38b9221
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/c38b9221
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/c38b9221
Branch: refs/heads/master
Commit: c38b9221dc7671fc87b781374fee4b44f1c32e03
Parents: 86e3d6d
Author: Zhao, Qingwen <qi...@ebay.com>
Authored: Wed Nov 25 20:57:31 2015 +0800
Committer: Zhao, Qingwen <qi...@ebay.com>
Committed: Wed Nov 25 20:57:31 2015 +0800
----------------------------------------------------------------------
.../java/org/apache/eagle/security/util/ExternalDataJoiner.java | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/c38b9221/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/util/ExternalDataJoiner.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/util/ExternalDataJoiner.java b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/util/ExternalDataJoiner.java
index 79c1a75..79a44c0 100644
--- a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/util/ExternalDataJoiner.java
+++ b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/util/ExternalDataJoiner.java
@@ -77,8 +77,9 @@ public class ExternalDataJoiner {
public void start(){
// for job
- JobDetail job = JobBuilder.newJob(jobCls)
- .withIdentity(jobCls.getName() + ".job", QUARTZ_GROUP_NAME)
+ String group = String.format("%s.%s.%s", QUARTZ_GROUP_NAME, jobDataMap.getString("site"), jobDataMap.getString("dataSource"));
+ JobDetail job = JobBuilder.newJob(jobCls)
+ .withIdentity(jobCls.getName() + ".job", group)
.setJobData(jobDataMap)
.build();
[2/5] incubator-eagle git commit: enable storm configuration for Eagle
Posted by ha...@apache.org.
enable storm configuration for Eagle
Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/86e3d6d0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/86e3d6d0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/86e3d6d0
Branch: refs/heads/master
Commit: 86e3d6d0d9684c96f5f9a145e96514eb65bc79e3
Parents: 4329ef1
Author: Zhao, Qingwen <qi...@ebay.com>
Authored: Wed Nov 25 20:55:51 2015 +0800
Committer: Zhao, Qingwen <qi...@ebay.com>
Committed: Wed Nov 25 20:55:51 2015 +0800
----------------------------------------------------------------------
.../datastream/StormTopologyExecutorImpl.scala | 17 +++++++++++++++++
.../security/hbase/HbaseAuditLogProcessorMain.java | 4 +---
.../auditlog/HdfsAuditLogProcessorMain.java | 4 +---
.../securitylog/HDFSSecurityLogProcessorMain.java | 4 +---
.../jobrunning/HiveJobRunningMonitoringMain.java | 6 +++---
.../src/assembly/eagle-topology-assembly.xml | 4 ++--
6 files changed, 25 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/86e3d6d0/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StormTopologyExecutorImpl.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StormTopologyExecutorImpl.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StormTopologyExecutorImpl.scala
index 737fe44..48ee9c4 100644
--- a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StormTopologyExecutorImpl.scala
+++ b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StormTopologyExecutorImpl.scala
@@ -16,9 +16,12 @@
*/
package org.apache.eagle.datastream
+import java.io.{FileInputStream, File}
+
import backtype.storm.generated.StormTopology
import backtype.storm.utils.Utils
import backtype.storm.{Config, LocalCluster, StormSubmitter}
+import org.yaml.snakeyaml.Yaml
import storm.trident.spout.RichSpoutBatchExecutor
case class StormTopologyExecutorImpl(topology: StormTopology, config: com.typesafe.config.Config) extends AbstractTopologyExecutor {
@@ -32,6 +35,20 @@ case class StormTopologyExecutorImpl(topology: StormTopology, config: com.typesa
conf.put(Config.TOPOLOGY_EXECUTOR_RECEIVE_BUFFER_SIZE, Int.box(16384))
conf.put(Config.TOPOLOGY_EXECUTOR_SEND_BUFFER_SIZE, Int.box(16384))
+ if(config.hasPath("envContextConfig.stormConfigFile")) {
+ //val inputFileStream = {
+ // StormTopologyExecutorImpl.getClass.getClassLoader.getResourceAsStream(config.getString("envContextConfig.stormConfigFile"))
+ //}
+ val file = new File(config.getString("envContextConfig.stormConfigFile"))
+ if(file.exists()) {
+ val inputFileStream = new FileInputStream(file)
+ val yaml = new Yaml()
+ val stormConf = yaml.load(inputFileStream).asInstanceOf[java.util.LinkedHashMap[String, Object]]
+ conf.putAll(stormConf)
+ inputFileStream.close()
+ }
+ }
+
val topologyName = config.getString("envContextConfig.topologyName")
if (!localMode) {
StormSubmitter.submitTopologyWithProgressBar(topologyName, conf, topology)
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/86e3d6d0/eagle-security/eagle-security-hbase-securitylog/src/main/java/org/apache/eagle/security/hbase/HbaseAuditLogProcessorMain.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-securitylog/src/main/java/org/apache/eagle/security/hbase/HbaseAuditLogProcessorMain.java b/eagle-security/eagle-security-hbase-securitylog/src/main/java/org/apache/eagle/security/hbase/HbaseAuditLogProcessorMain.java
index 5d6d9f3..1a90cbd 100644
--- a/eagle-security/eagle-security-hbase-securitylog/src/main/java/org/apache/eagle/security/hbase/HbaseAuditLogProcessorMain.java
+++ b/eagle-security/eagle-security-hbase-securitylog/src/main/java/org/apache/eagle/security/hbase/HbaseAuditLogProcessorMain.java
@@ -33,9 +33,7 @@ public class HbaseAuditLogProcessorMain {
private static final Logger LOG = LoggerFactory.getLogger(HbaseAuditLogProcessorMain.class);
public static void main(String[] args) throws Exception{
- new ConfigOptionParser().load(args);
- System.setProperty("config.trace", "loads");
- Config config = ConfigFactory.load();
+ Config config = new ConfigOptionParser().load(args);
LOG.info("Config class: " + config.getClass().getCanonicalName());
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/86e3d6d0/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/HdfsAuditLogProcessorMain.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/HdfsAuditLogProcessorMain.java b/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/HdfsAuditLogProcessorMain.java
index ce0c6be..fa712d4 100644
--- a/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/HdfsAuditLogProcessorMain.java
+++ b/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/HdfsAuditLogProcessorMain.java
@@ -38,9 +38,7 @@ public class HdfsAuditLogProcessorMain {
private static final Logger LOG = LoggerFactory.getLogger(HdfsAuditLogProcessorMain.class);
public static void main(String[] args) throws Exception{
- new ConfigOptionParser().load(args);
- System.setProperty("config.trace", "loads");
- Config config = ConfigFactory.load();
+ Config config = new ConfigOptionParser().load(args);
LOG.info("Config class: " + config.getClass().getCanonicalName());
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/86e3d6d0/eagle-security/eagle-security-hdfs-securitylog/src/main/java/org/apache/eagle/security/securitylog/HDFSSecurityLogProcessorMain.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hdfs-securitylog/src/main/java/org/apache/eagle/security/securitylog/HDFSSecurityLogProcessorMain.java b/eagle-security/eagle-security-hdfs-securitylog/src/main/java/org/apache/eagle/security/securitylog/HDFSSecurityLogProcessorMain.java
index b514a7c..9238cad 100644
--- a/eagle-security/eagle-security-hdfs-securitylog/src/main/java/org/apache/eagle/security/securitylog/HDFSSecurityLogProcessorMain.java
+++ b/eagle-security/eagle-security-hdfs-securitylog/src/main/java/org/apache/eagle/security/securitylog/HDFSSecurityLogProcessorMain.java
@@ -33,9 +33,7 @@ public class HDFSSecurityLogProcessorMain {
private static final Logger LOG = LoggerFactory.getLogger(HDFSSecurityLogProcessorMain.class);
public static void main(String[] args) throws Exception{
- new ConfigOptionParser().load(args);
- System.setProperty("config.trace", "loads");
- Config config = ConfigFactory.load();
+ Config config = new ConfigOptionParser().load(args);
LOG.info("Config class: " + config.getClass().getCanonicalName());
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/86e3d6d0/eagle-security/eagle-security-hive/src/main/java/org/apache/eagle/security/hive/jobrunning/HiveJobRunningMonitoringMain.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hive/src/main/java/org/apache/eagle/security/hive/jobrunning/HiveJobRunningMonitoringMain.java b/eagle-security/eagle-security-hive/src/main/java/org/apache/eagle/security/hive/jobrunning/HiveJobRunningMonitoringMain.java
index 983e871..c3e192a 100644
--- a/eagle-security/eagle-security-hive/src/main/java/org/apache/eagle/security/hive/jobrunning/HiveJobRunningMonitoringMain.java
+++ b/eagle-security/eagle-security-hive/src/main/java/org/apache/eagle/security/hive/jobrunning/HiveJobRunningMonitoringMain.java
@@ -33,9 +33,9 @@ public class HiveJobRunningMonitoringMain {
private static final Logger LOG = LoggerFactory.getLogger(HiveJobRunningMonitoringMain.class);
public static void main(String[] args) throws Exception{
- new ConfigOptionParser().load(args);
- System.setProperty("config.trace", "loads");
- Config config = ConfigFactory.load();
+ Config config = new ConfigOptionParser().load(args);
+ //System.setProperty("config.trace", "loads");
+ //Config config = ConfigFactory.load();
LOG.info("Config class: " + config.getClass().getCanonicalName());
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/86e3d6d0/eagle-topology-assembly/src/assembly/eagle-topology-assembly.xml
----------------------------------------------------------------------
diff --git a/eagle-topology-assembly/src/assembly/eagle-topology-assembly.xml b/eagle-topology-assembly/src/assembly/eagle-topology-assembly.xml
index 2e34c9b..db8563f 100644
--- a/eagle-topology-assembly/src/assembly/eagle-topology-assembly.xml
+++ b/eagle-topology-assembly/src/assembly/eagle-topology-assembly.xml
@@ -16,8 +16,8 @@
<excludes>
<exclude>**/application.conf</exclude>
<exclude>**/defaults.yaml</exclude>
- <exclude>**/storm.yaml</exclude>
- <exclude>**/storm.yaml.1</exclude>
+ <exclude>**/*storm.yaml</exclude>
+ <exclude>**/*storm.yaml.1</exclude>
<exclude>**/log4j.properties</exclude>
</excludes>
</unpackOptions>