You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@eagle.apache.org by ha...@apache.org on 2015/11/30 03:48:59 UTC
[2/5] incubator-eagle git commit: enable storm configuration for Eagle
enable storm configuration for Eagle
Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/86e3d6d0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/86e3d6d0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/86e3d6d0
Branch: refs/heads/master
Commit: 86e3d6d0d9684c96f5f9a145e96514eb65bc79e3
Parents: 4329ef1
Author: Zhao, Qingwen <qi...@ebay.com>
Authored: Wed Nov 25 20:55:51 2015 +0800
Committer: Zhao, Qingwen <qi...@ebay.com>
Committed: Wed Nov 25 20:55:51 2015 +0800
----------------------------------------------------------------------
.../datastream/StormTopologyExecutorImpl.scala | 17 +++++++++++++++++
.../security/hbase/HbaseAuditLogProcessorMain.java | 4 +---
.../auditlog/HdfsAuditLogProcessorMain.java | 4 +---
.../securitylog/HDFSSecurityLogProcessorMain.java | 4 +---
.../jobrunning/HiveJobRunningMonitoringMain.java | 6 +++---
.../src/assembly/eagle-topology-assembly.xml | 4 ++--
6 files changed, 25 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/86e3d6d0/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StormTopologyExecutorImpl.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StormTopologyExecutorImpl.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StormTopologyExecutorImpl.scala
index 737fe44..48ee9c4 100644
--- a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StormTopologyExecutorImpl.scala
+++ b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StormTopologyExecutorImpl.scala
@@ -16,9 +16,12 @@
*/
package org.apache.eagle.datastream
+import java.io.{FileInputStream, File}
+
import backtype.storm.generated.StormTopology
import backtype.storm.utils.Utils
import backtype.storm.{Config, LocalCluster, StormSubmitter}
+import org.yaml.snakeyaml.Yaml
import storm.trident.spout.RichSpoutBatchExecutor
case class StormTopologyExecutorImpl(topology: StormTopology, config: com.typesafe.config.Config) extends AbstractTopologyExecutor {
@@ -32,6 +35,20 @@ case class StormTopologyExecutorImpl(topology: StormTopology, config: com.typesa
conf.put(Config.TOPOLOGY_EXECUTOR_RECEIVE_BUFFER_SIZE, Int.box(16384))
conf.put(Config.TOPOLOGY_EXECUTOR_SEND_BUFFER_SIZE, Int.box(16384))
+ if(config.hasPath("envContextConfig.stormConfigFile")) {
+ //val inputFileStream = {
+ // StormTopologyExecutorImpl.getClass.getClassLoader.getResourceAsStream(config.getString("envContextConfig.stormConfigFile"))
+ //}
+ val file = new File(config.getString("envContextConfig.stormConfigFile"))
+ if(file.exists()) {
+ val inputFileStream = new FileInputStream(file)
+ val yaml = new Yaml()
+ val stormConf = yaml.load(inputFileStream).asInstanceOf[java.util.LinkedHashMap[String, Object]]
+ conf.putAll(stormConf)
+ inputFileStream.close()
+ }
+ }
+
val topologyName = config.getString("envContextConfig.topologyName")
if (!localMode) {
StormSubmitter.submitTopologyWithProgressBar(topologyName, conf, topology)
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/86e3d6d0/eagle-security/eagle-security-hbase-securitylog/src/main/java/org/apache/eagle/security/hbase/HbaseAuditLogProcessorMain.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-securitylog/src/main/java/org/apache/eagle/security/hbase/HbaseAuditLogProcessorMain.java b/eagle-security/eagle-security-hbase-securitylog/src/main/java/org/apache/eagle/security/hbase/HbaseAuditLogProcessorMain.java
index 5d6d9f3..1a90cbd 100644
--- a/eagle-security/eagle-security-hbase-securitylog/src/main/java/org/apache/eagle/security/hbase/HbaseAuditLogProcessorMain.java
+++ b/eagle-security/eagle-security-hbase-securitylog/src/main/java/org/apache/eagle/security/hbase/HbaseAuditLogProcessorMain.java
@@ -33,9 +33,7 @@ public class HbaseAuditLogProcessorMain {
private static final Logger LOG = LoggerFactory.getLogger(HbaseAuditLogProcessorMain.class);
public static void main(String[] args) throws Exception{
- new ConfigOptionParser().load(args);
- System.setProperty("config.trace", "loads");
- Config config = ConfigFactory.load();
+ Config config = new ConfigOptionParser().load(args);
LOG.info("Config class: " + config.getClass().getCanonicalName());
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/86e3d6d0/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/HdfsAuditLogProcessorMain.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/HdfsAuditLogProcessorMain.java b/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/HdfsAuditLogProcessorMain.java
index ce0c6be..fa712d4 100644
--- a/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/HdfsAuditLogProcessorMain.java
+++ b/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/HdfsAuditLogProcessorMain.java
@@ -38,9 +38,7 @@ public class HdfsAuditLogProcessorMain {
private static final Logger LOG = LoggerFactory.getLogger(HdfsAuditLogProcessorMain.class);
public static void main(String[] args) throws Exception{
- new ConfigOptionParser().load(args);
- System.setProperty("config.trace", "loads");
- Config config = ConfigFactory.load();
+ Config config = new ConfigOptionParser().load(args);
LOG.info("Config class: " + config.getClass().getCanonicalName());
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/86e3d6d0/eagle-security/eagle-security-hdfs-securitylog/src/main/java/org/apache/eagle/security/securitylog/HDFSSecurityLogProcessorMain.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hdfs-securitylog/src/main/java/org/apache/eagle/security/securitylog/HDFSSecurityLogProcessorMain.java b/eagle-security/eagle-security-hdfs-securitylog/src/main/java/org/apache/eagle/security/securitylog/HDFSSecurityLogProcessorMain.java
index b514a7c..9238cad 100644
--- a/eagle-security/eagle-security-hdfs-securitylog/src/main/java/org/apache/eagle/security/securitylog/HDFSSecurityLogProcessorMain.java
+++ b/eagle-security/eagle-security-hdfs-securitylog/src/main/java/org/apache/eagle/security/securitylog/HDFSSecurityLogProcessorMain.java
@@ -33,9 +33,7 @@ public class HDFSSecurityLogProcessorMain {
private static final Logger LOG = LoggerFactory.getLogger(HDFSSecurityLogProcessorMain.class);
public static void main(String[] args) throws Exception{
- new ConfigOptionParser().load(args);
- System.setProperty("config.trace", "loads");
- Config config = ConfigFactory.load();
+ Config config = new ConfigOptionParser().load(args);
LOG.info("Config class: " + config.getClass().getCanonicalName());
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/86e3d6d0/eagle-security/eagle-security-hive/src/main/java/org/apache/eagle/security/hive/jobrunning/HiveJobRunningMonitoringMain.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hive/src/main/java/org/apache/eagle/security/hive/jobrunning/HiveJobRunningMonitoringMain.java b/eagle-security/eagle-security-hive/src/main/java/org/apache/eagle/security/hive/jobrunning/HiveJobRunningMonitoringMain.java
index 983e871..c3e192a 100644
--- a/eagle-security/eagle-security-hive/src/main/java/org/apache/eagle/security/hive/jobrunning/HiveJobRunningMonitoringMain.java
+++ b/eagle-security/eagle-security-hive/src/main/java/org/apache/eagle/security/hive/jobrunning/HiveJobRunningMonitoringMain.java
@@ -33,9 +33,9 @@ public class HiveJobRunningMonitoringMain {
private static final Logger LOG = LoggerFactory.getLogger(HiveJobRunningMonitoringMain.class);
public static void main(String[] args) throws Exception{
- new ConfigOptionParser().load(args);
- System.setProperty("config.trace", "loads");
- Config config = ConfigFactory.load();
+ Config config = new ConfigOptionParser().load(args);
+ //System.setProperty("config.trace", "loads");
+ //Config config = ConfigFactory.load();
LOG.info("Config class: " + config.getClass().getCanonicalName());
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/86e3d6d0/eagle-topology-assembly/src/assembly/eagle-topology-assembly.xml
----------------------------------------------------------------------
diff --git a/eagle-topology-assembly/src/assembly/eagle-topology-assembly.xml b/eagle-topology-assembly/src/assembly/eagle-topology-assembly.xml
index 2e34c9b..db8563f 100644
--- a/eagle-topology-assembly/src/assembly/eagle-topology-assembly.xml
+++ b/eagle-topology-assembly/src/assembly/eagle-topology-assembly.xml
@@ -16,8 +16,8 @@
<excludes>
<exclude>**/application.conf</exclude>
<exclude>**/defaults.yaml</exclude>
- <exclude>**/storm.yaml</exclude>
- <exclude>**/storm.yaml.1</exclude>
+ <exclude>**/*storm.yaml</exclude>
+ <exclude>**/*storm.yaml.1</exclude>
<exclude>**/log4j.properties</exclude>
</excludes>
</unpackOptions>