You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@eagle.apache.org by ha...@apache.org on 2015/12/16 07:01:43 UTC
[1/5] incubator-eagle git commit: EAGLE-66 Typesafe Streaming DSL and
KeyValue based Grouping
Repository: incubator-eagle
Updated Branches:
refs/heads/master 2734b4222 -> 52b8e58b1
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-security/eagle-security-hive/src/main/java/org/apache/eagle/security/hive/jobrunning/HiveJobRunningMonitoringMain.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hive/src/main/java/org/apache/eagle/security/hive/jobrunning/HiveJobRunningMonitoringMain.java b/eagle-security/eagle-security-hive/src/main/java/org/apache/eagle/security/hive/jobrunning/HiveJobRunningMonitoringMain.java
index 09e7475..5b1bee6 100644
--- a/eagle-security/eagle-security-hive/src/main/java/org/apache/eagle/security/hive/jobrunning/HiveJobRunningMonitoringMain.java
+++ b/eagle-security/eagle-security-hive/src/main/java/org/apache/eagle/security/hive/jobrunning/HiveJobRunningMonitoringMain.java
@@ -18,12 +18,8 @@
*/
package org.apache.eagle.security.hive.jobrunning;
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigFactory;
-import com.typesafe.config.ConfigRenderOptions;
-import org.apache.eagle.dataproc.util.ConfigOptionParser;
-import org.apache.eagle.datastream.ExecutionEnvironmentFactory;
-import org.apache.eagle.datastream.StormExecutionEnvironment;
+import org.apache.eagle.datastream.ExecutionEnvironments;
+import org.apache.eagle.datastream.storm.StormExecutionEnvironment;
import org.apache.eagle.security.hive.sensitivity.HiveResourceSensitivityDataJoinExecutor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -33,20 +29,15 @@ public class HiveJobRunningMonitoringMain {
private static final Logger LOG = LoggerFactory.getLogger(HiveJobRunningMonitoringMain.class);
public static void main(String[] args) throws Exception{
- Config config = new ConfigOptionParser().load(args);
-
- LOG.info("Config class: " + config.getClass().getCanonicalName());
-
- if(LOG.isDebugEnabled()) LOG.debug("Config content:"+config.root().render(ConfigRenderOptions.concise()));
-
+ StormExecutionEnvironment env = ExecutionEnvironments.getStorm(args);
String spoutName = "msgConsumer";
- int parallelism = config.getInt("envContextConfig.parallelismConfig." + spoutName);
- StormExecutionEnvironment env = ExecutionEnvironmentFactory.getStorm(config);
- env.newSource(new HiveJobRunningSourcedStormSpoutProvider().getSpout(config, parallelism)).renameOutputFields(4).withName(spoutName).groupBy(Arrays.asList(0))
+ int parallelism = env.getConfig().getInt("envContextConfig.parallelismConfig." + spoutName);
+ env.fromSpout(new HiveJobRunningSourcedStormSpoutProvider().getSpout(env.getConfig(), parallelism))
+ .withOutputFields(4).nameAs(spoutName).groupBy(Arrays.asList(0))
.flatMap(new JobConfigurationAdaptorExecutor()).groupBy(Arrays.asList(0))
.flatMap(new HiveQueryParserExecutor()).groupBy(Arrays.asList(0))
.flatMap(new HiveResourceSensitivityDataJoinExecutor())
.alertWithConsumer("hiveAccessLogStream", "hiveAccessAlertByRunningJob");
env.execute();
}
-};
\ No newline at end of file
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/UserProfileDetectionBatchMain.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/UserProfileDetectionBatchMain.java b/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/UserProfileDetectionBatchMain.java
index c95dfe0..1fdb3bd 100644
--- a/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/UserProfileDetectionBatchMain.java
+++ b/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/UserProfileDetectionBatchMain.java
@@ -19,11 +19,9 @@
package org.apache.eagle.security.userprofile;
import com.typesafe.config.Config;
-import com.typesafe.config.ConfigFactory;
-import com.typesafe.config.ConfigRenderOptions;
import org.apache.eagle.dataproc.impl.storm.kafka.KafkaSourcedSpoutProvider;
-import org.apache.eagle.dataproc.util.ConfigOptionParser;
import org.apache.eagle.datastream.*;
+import org.apache.eagle.datastream.storm.StormExecutionEnvironment;
import org.apache.eagle.security.userprofile.model.UserActivityAggModelEntity;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -31,19 +29,9 @@ import org.slf4j.LoggerFactory;
import java.util.Map;
public class UserProfileDetectionBatchMain {
- private final static Logger LOG = LoggerFactory.getLogger(UserProfileDetectionBatchMain.class);
-
public static void main(String[] args) throws Exception{
- new ConfigOptionParser().load(args);
- System.setProperty("config.trace", "loads");
- Config config = ConfigFactory.load();
-
- LOG.info("Config class: " + config.getClass().getCanonicalName());
-
- if(LOG.isDebugEnabled()) LOG.debug("Config content:"+config.root().render(ConfigRenderOptions.concise()));
-
- StormExecutionEnvironment env = ExecutionEnvironmentFactory.getStorm(config);
- env.newSource(new KafkaSourcedSpoutProvider().getSpout(config)).renameOutputFields(1)
+ StormExecutionEnvironment env = ExecutionEnvironments.getStorm(args);
+ env.fromSpout(new KafkaSourcedSpoutProvider()).withOutputFields(1)
.flatMap(new UserActivityPartitionExecutor())
.alertWithConsumer(UserProfileDetectionConstants.USER_ACTIVITY_AGGREGATION_STREAM,
UserProfileDetectionConstants.USER_PROFILE_ANOMALY_DETECTION_EXECUTOR);
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/UserProfileDetectionStreamMain.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/UserProfileDetectionStreamMain.java b/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/UserProfileDetectionStreamMain.java
index caea5e9..1048bdd 100644
--- a/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/UserProfileDetectionStreamMain.java
+++ b/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/UserProfileDetectionStreamMain.java
@@ -19,35 +19,23 @@
package org.apache.eagle.security.userprofile;
import com.typesafe.config.Config;
-import com.typesafe.config.ConfigRenderOptions;
import org.apache.eagle.dataproc.impl.storm.kafka.KafkaSourcedSpoutProvider;
-import org.apache.eagle.dataproc.util.ConfigOptionParser;
import org.apache.eagle.datastream.*;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.apache.eagle.datastream.storm.StormExecutionEnvironment;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
public class UserProfileDetectionStreamMain {
- private final static Logger LOG = LoggerFactory.getLogger(UserProfileDetectionStreamMain.class);
-
public static void main(String[] args) throws Exception{
- Config config = new ConfigOptionParser().load(args);
-
- LOG.info("Config class: " + config.getClass().getCanonicalName());
-
- if(LOG.isDebugEnabled()) LOG.debug("Config content:"+config.root().render(ConfigRenderOptions.concise()));
-
- StormExecutionEnvironment env = ExecutionEnvironmentFactory.getStorm(config);
- env.newSource(new KafkaSourcedSpoutProvider().getSpout(config)).renameOutputFields(1).withName("kafkaMsgConsumer")
- .flatMap(new AuditLogTransformer()).withName("transformer") // [user,map]
+ StormExecutionEnvironment env = ExecutionEnvironments.getStorm(args);
+ env.fromSpout(new KafkaSourcedSpoutProvider()).withOutputFields(1).nameAs("kafkaMsgConsumer")
+ .flatMap(new AuditLogTransformer()).nameAs("transformer") // [user,map]
.groupBy(Arrays.asList(0)) // group by [user]
- .flatMap(new UserProfileAggregatorExecutor()).withName("aggregator")
- .alertWithConsumer(Arrays.asList(UserProfileDetectionConstants.USER_ACTIVITY_AGGREGATION_STREAM),
+ .flatMap(new UserProfileAggregatorExecutor()).nameAs("aggregator")
+ .alertWithConsumer(UserProfileDetectionConstants.USER_ACTIVITY_AGGREGATION_STREAM,
UserProfileDetectionConstants.USER_PROFILE_ANOMALY_DETECTION_EXECUTOR); // alert
- ;
env.execute();
}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/test.txt
----------------------------------------------------------------------
diff --git a/test.txt b/test.txt
deleted file mode 100644
index 6623e06..0000000
--- a/test.txt
+++ /dev/null
@@ -1 +0,0 @@
-test for keep file for user command reassembler
[3/5] incubator-eagle git commit: EAGLE-66 Typesafe Streaming DSL and
KeyValue based Grouping
Posted by ha...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/StreamProducer.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/StreamProducer.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/StreamProducer.scala
new file mode 100644
index 0000000..4d7dcd1
--- /dev/null
+++ b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/StreamProducer.scala
@@ -0,0 +1,294 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+package org.apache.eagle.datastream.core
+
+import java.util
+import java.util.concurrent.atomic.AtomicInteger
+
+import backtype.storm.topology.base.BaseRichSpout
+import com.typesafe.config.Config
+import org.apache.eagle.alert.entity.AlertAPIEntity
+import org.apache.eagle.datastream._
+import org.apache.eagle.partition.PartitionStrategy
+import org.jgrapht.experimental.dag.DirectedAcyclicGraph
+import org.slf4j.LoggerFactory
+
+import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
+/**
+ * StreamProducer = StreamInfo + StreamProtocol
+ *
+ * StreamProducer is processing logic element, used the base class for all other concrete StreamProducer
+ * Stream Producer can be treated as logical processing element but physical
+ * It defines high level type-safe API for user to organize data stream flow
+ *
+ * StreamProducer is independent of execution environment
+ *
+ * @tparam T processed elements type
+ */
+abstract class StreamProducer[+T <: Any] extends StreamInfo with StreamProtocol[T] {
+ private var graph: DirectedAcyclicGraph[StreamProducer[Any], StreamConnector[Any,Any]] = null
+ private val LOG = LoggerFactory.getLogger(classOf[StreamProducer[T]])
+
+ /**
+ * Should not modify graph when setting it
+ */
+ def initWith(graph:DirectedAcyclicGraph[StreamProducer[Any], StreamConnector[Any,Any]],config:Config, hook:Boolean = true):StreamProducer[T]={
+ this.config = config
+ this.graph = graph
+ if(hook && ! this.graph.containsVertex(this)) {
+ this.graph.addVertex(this)
+ }
+ LOG.debug(this.graph.toString)
+ this
+ }
+
+ /**
+ * Get stream pure metadata info
+ * @return
+ */
+ override def getInfo:StreamInfo = this.asInstanceOf[StreamInfo]
+
+ override def stream(streamId:String):StreamProducer[T] = {
+ this.streamId = streamId
+ this
+ }
+
+ override def filter(fn : T => Boolean): StreamProducer[T] ={
+ val ret = FilterProducer[T](fn)
+ hookup(this, ret)
+ ret
+ }
+
+ override def flatMap[R](flatMapper:FlatMapper[R]): StreamProducer[R] = {
+ val ret = FlatMapProducer[T,R](flatMapper)
+ hookup(this, ret)
+ ret
+ }
+
+ override def foreach(fn : T => Unit) : Unit = {
+ val ret = ForeachProducer[T](fn)
+ hookup(this, ret)
+ }
+
+ override def map[R](fn : T => R) : StreamProducer[R] = {
+ val ret = MapperProducer[T,R](0,fn)
+ hookup(this, ret)
+ ret
+ }
+
+ override def map1[R](fn : T => R): StreamProducer[R] = {
+ val ret = MapperProducer[T,R](1, fn)
+ hookup(this, ret)
+ ret
+ }
+
+ override def map2[R](fn : T => R): StreamProducer[R] = {
+ val ret = MapperProducer[T,R](2, fn)
+ hookup(this, ret)
+ ret
+ }
+
+ override def map3[R](fn : T => R): StreamProducer[R] = {
+ val ret = MapperProducer(3, fn)
+ hookup(this, ret)
+ ret
+ }
+
+ override def map4[R](fn : T => R): StreamProducer[R] = {
+ val ret = MapperProducer(4, fn)
+ hookup(this, ret)
+ ret
+ }
+
+ /**
+ * starting from 0, groupby operator would be upon edge of the graph
+ */
+ override def groupBy(fields : Int*) : StreamProducer[T] = {
+ // validate each field index is greater or equal to 0
+ fields.foreach(n => if(n<0) throw new IllegalArgumentException("field index should be always >= 0"))
+ val ret = GroupByFieldProducer[T](fields)
+ hookup(this, ret)
+ ret
+ }
+
+ //groupBy java version, starting from 1
+ override def groupBy(fields : java.util.List[Integer]) : StreamProducer[T] = {
+ // validate each field index is greater or equal to 0
+ fields.foreach(n => if(n<0) throw new IllegalArgumentException("field index should be always >= 0"))
+ val ret = GroupByFieldProducer[T](fields.asScala.toSeq.asInstanceOf[Seq[Int]])
+ hookup(this, ret)
+ ret
+ }
+
+ override def groupByKey(keySelector: T=> Any):StreamProducer[T] = {
+ val ret = GroupByKeyProducer(keySelector)
+ hookup(this,ret)
+ ret
+ }
+
+ override def streamUnion[T2,T3](others : Seq[StreamProducer[T2]]) : StreamProducer[T3] = {
+ val ret = StreamUnionProducer[T, T2, T3](others)
+ hookup(this, ret)
+ ret
+ }
+
+ def streamUnion[T2,T3](other : StreamProducer[T2]) : StreamProducer[T3] = {
+ streamUnion(Seq(other))
+ }
+
+ def streamUnion[T2,T3](others : util.List[StreamProducer[T2]]) : StreamProducer[T3] = streamUnion(others.asScala)
+
+ override def groupBy(strategy : PartitionStrategy) : StreamProducer[T] = {
+ val ret = GroupByStrategyProducer(strategy)
+ hookup(this, ret)
+ ret
+ }
+
+ /**
+ * alert is always sink of data flow
+ */
+ def alertWithConsumer(upStreamNames: util.List[String], alertExecutorId : String) = {
+ alert(upStreamNames.asScala, alertExecutorId,consume = true)
+ }
+
+ def alertWithoutConsumer(upStreamNames: util.List[String], alertExecutorId : String) = {
+ alert(upStreamNames.asScala, alertExecutorId, consume = false)
+ }
+
+ def alert(upStreamNames: Seq[String], alertExecutorId : String, consume: Boolean=true, strategy : PartitionStrategy=null) = {
+ val ret = AlertStreamSink(upStreamNames, alertExecutorId, consume, strategy)
+ hookup(this, ret)
+ }
+
+ def alertWithConsumer(upStreamName: String, alertExecutorId : String, strategy: PartitionStrategy): Unit ={
+ alert(util.Arrays.asList(upStreamName), alertExecutorId, consume = true, strategy = strategy)
+ }
+
+ def alertWithConsumer(upStreamName: String, alertExecutorId : String): Unit ={
+ alert(util.Arrays.asList(upStreamName), alertExecutorId, consume = true)
+ }
+
+ def alertWithoutConsumer(upStreamName: String, alertExecutorId : String, strategy: PartitionStrategy): Unit ={
+ alert(util.Arrays.asList(upStreamName), alertExecutorId, consume = false, strategy)
+ }
+
+ def alertWithoutConsumer(upStreamName: String, alertExecutorId : String): Unit ={
+ alert(util.Arrays.asList(upStreamName), alertExecutorId, consume = false)
+ }
+
+ protected def hookup[T1,T2](current: StreamProducer[T1], next: StreamProducer[T2]) = {
+ current.graph.addVertex(next)
+ current.graph.addEdge(current, next, StreamConnector(current, next))
+ passOnContext[T1,T2](current, next)
+ }
+
+ private def passOnContext[T1 ,T2](current: StreamProducer[T1], next: StreamProducer[T2]): Unit ={
+ next.initWith(current.graph,current.config)
+ }
+
+ /**
+ * can be set by programatically or by configuration
+ */
+ override def parallelism(parallelism : Int) : StreamProducer[T] = {
+ this.parallelismNum = parallelism
+ this
+ }
+
+ override def parallelism : Int = this.parallelismNum
+ override def stream:String = this.streamId
+
+ /**
+ * Component name
+ *
+ * @param componentName component name
+ * @return
+ */
+ override def nameAs(componentName : String) : StreamProducer[T] = {
+ this.name = componentName
+ this
+ }
+}
+
+case class FilterProducer[T](fn : T => Boolean) extends StreamProducer[T]{
+ override def toString: String = s"FilterProducer"
+}
+
+case class FlatMapProducer[T, R](var mapper: FlatMapper[R]) extends StreamProducer[R]{
+ override def toString: String = mapper.toString
+}
+
+case class MapperProducer[T,R](numOutputFields : Int, var fn : T => R) extends StreamProducer[R]{
+ override def toString: String = s"MapperProducer"
+}
+
+case class ForeachProducer[T](var fn : T => Unit) extends StreamProducer[T]
+
+abstract class GroupByProducer[T] extends StreamProducer[T]
+case class GroupByFieldProducer[T](fields : Seq[Int]) extends GroupByProducer[T]
+case class GroupByStrategyProducer[T](partitionStrategy: PartitionStrategy) extends GroupByProducer[T]
+case class GroupByKeyProducer[T](keySelectorFunc:T => Any) extends GroupByProducer[T]{
+ override def toString: String = s"GroupByKey"
+}
+
+object GroupByProducer {
+ def apply[T](fields: Seq[Int]) = new GroupByFieldProducer[T](fields)
+ def apply[T](partitionStrategy : PartitionStrategy) = new GroupByStrategyProducer[T](partitionStrategy)
+ def apply[T](keySelector:T => Any) = new GroupByKeyProducer[T](keySelector)
+}
+
+case class StreamUnionProducer[T1,T2,T3](others: Seq[StreamProducer[T2]]) extends StreamProducer[T3]
+
+case class StormSourceProducer[T](source: BaseRichSpout) extends StreamProducer[T]{
+ var numFields : Int = 0
+
+ /**
+ * rename outputfields to f0, f1, f2, ...
+ * if one spout declare some field names, those fields names will be modified
+ * @param n
+ */
+ def withOutputFields(n : Int): StormSourceProducer[T] ={
+ this.numFields = n
+ this
+ }
+}
+
+case class IterableStreamProducer[T](iterable: Iterable[T],recycle:Boolean = false) extends StreamProducer[T]
+
+case class AlertStreamSink(upStreamNames: util.List[String], alertExecutorId : String, var consume: Boolean=true, strategy: PartitionStrategy=null) extends StreamProducer[AlertAPIEntity] {
+ def consume(consume: Boolean): AlertStreamSink = {
+ this.consume = consume
+ this
+ }
+}
+
+object UniqueId{
+ val id : AtomicInteger = new AtomicInteger(0);
+ def incrementAndGetId() : Int = {
+ id.incrementAndGet()
+ }
+}
+
+trait KeySelector extends Serializable{
+ def key(t:Any):Any
+}
+
+case class KeySelectorWrapper[T](fn:T => Any) extends KeySelector{
+ override def key(t: Any): Any = fn(t.asInstanceOf[T])
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/StreamProducerGraph.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/StreamProducerGraph.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/StreamProducerGraph.scala
new file mode 100644
index 0000000..f3fcc4d
--- /dev/null
+++ b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/StreamProducerGraph.scala
@@ -0,0 +1,29 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+package org.apache.eagle.datastream.core
+
+trait StreamProducerGraph {
+ def addEdge(from: StreamProducer[Any], to: StreamProducer[Any], streamConnector: StreamConnector[Any,Any])
+ def addVertex(producer: StreamProducer[Any])
+ def iterator() : Iterator[StreamProducer[Any]]
+ def isSource(v : StreamProducer[Any]) : Boolean
+ def outgoingEdgesOf(v : StreamProducer[Any]) : scala.collection.mutable.Set[StreamConnector[Any,Any]]
+ def getNodeByName(name : String) : Option[StreamProducer[Any]]
+ def incomingVertexOf(v: StreamProducer[Any]) : scala.collection.mutable.Set[StreamProducer[Any]]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/StreamProtocol.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/StreamProtocol.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/StreamProtocol.scala
new file mode 100644
index 0000000..346f728
--- /dev/null
+++ b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/StreamProtocol.scala
@@ -0,0 +1,176 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eagle.datastream.core
+
+import com.typesafe.config.Config
+import org.apache.commons.lang3.builder.HashCodeBuilder
+import org.apache.eagle.datastream.FlatMapper
+import org.apache.eagle.partition.PartitionStrategy
+import org.jgrapht.experimental.dag.DirectedAcyclicGraph
+
+import scala.reflect.runtime.{universe => ru}
+
+/**
+ * StreamInfo should be fully serializable and having not runtime type information
+ */
+class StreamInfo extends Serializable{
+ /**
+ * Processing Element Id
+ */
+ val id:Int = UniqueId.incrementAndGetId()
+
+ /**
+ * Processing Element Name
+ */
+ var name: String = null
+
+ var streamId:String=null
+ var parallelismNum: Int = 1
+
+ /**
+ * Keyed input stream
+ */
+ var inKeyed:Boolean = false
+ /**
+ * Keyed output stream
+ */
+ var outKeyed:Boolean = false
+ /**
+ * Output key selector
+ */
+ var keySelector:KeySelector = null
+
+// Type Information
+// ================
+//
+// /**
+// * Entity class type of T
+// */
+// var typeClass:Class[_] = null
+//
+// /**
+// * Type Class Simple Name
+// * @return
+// */
+// def typeClassName = if(typeClass == null) null else typeClass.getSimpleName
+//
+// @transient private var _typeTag[_] = null
+//
+// def typeTag[_] = {
+// if(_typeTag == null) _typeTag = Reflections.typeTag(this.typeClass)
+// _typeTag
+// }
+
+ var config: Config = null
+
+ def getInfo = this
+
+ override def hashCode(): Int = new HashCodeBuilder().append(this.id).append(this.getClass).toHashCode
+}
+
+/**
+ * Stream interaction protocol interface
+ *
+ * @tparam T processed elements type
+ */
+trait StreamProtocol[+T <: Any]{
+ /**
+ * Initialize the stream metadata info
+ */
+ def initWith(graph:DirectedAcyclicGraph[StreamProducer[Any], StreamConnector[Any,Any]],config:Config, hook:Boolean = true):StreamProducer[T]
+
+ /**
+ * Support Java API
+ *
+ * @param flatMapper
+ * @tparam R
+ * @return
+ */
+ def flatMap[R](flatMapper:FlatMapper[R]): StreamProducer[R]
+
+ /**
+ *
+ * @param fn
+ * @return
+ */
+ def filter(fn : T => Boolean): StreamProducer[T]
+
+ /**
+ *
+ * @param fn
+ */
+ def foreach(fn : T => Unit) : Unit
+
+ /**
+ * Type safe mapper
+ * @param fn
+ * @tparam R
+ * @return
+ */
+ def map[R](fn : T => R): StreamProducer[R]
+
+ /**
+ * Field base mapper
+ * @param fn
+ * @tparam R
+ * @return
+ */
+ def map1[R](fn : T => R) : StreamProducer[R]
+ def map2[R](fn : T => R) : StreamProducer[R]
+ def map3[R](fn : T => R) : StreamProducer[R]
+ def map4[R](fn : T => R) : StreamProducer[R]
+
+ def groupBy(fields : Int*) : StreamProducer[T]
+ def groupBy(fields : java.util.List[Integer]) : StreamProducer[T]
+ def groupBy(strategy : PartitionStrategy) : StreamProducer[T]
+
+ /**
+ * @param keyer key selector function
+ * @return
+ */
+ def groupByKey(keyer:T => Any):StreamProducer[T]
+ def streamUnion[T2,T3](otherStreams : Seq[StreamProducer[T2]]) : StreamProducer[T3]
+ def alert(upStreamNames: Seq[String], alertExecutorId : String, consume: Boolean,strategy : PartitionStrategy)
+ /**
+ * Set processing element parallelism setting
+ * @param parallelismNum parallelism value
+ * @return
+ */
+ def parallelism(parallelismNum : Int) : StreamProducer[T]
+ def parallelism : Int
+ /**
+ * Set component name
+ *
+ * @param componentName
+ * @return
+ */
+ def nameAs(componentName : String) : StreamProducer[T]
+
+ /**
+ * Set stream name
+ * @param streamId stream ID
+ * @return
+ */
+ def stream(streamId: String): StreamProducer[T]
+ def stream: String
+
+ def ? (fn:T => Boolean):StreamProducer[T] = this.filter(fn)
+ def ~>[R](flatMapper : FlatMapper[R]) = this.flatMap[R](flatMapper)
+ def ! (upStreamNames: Seq[String], alertExecutorId : String, consume: Boolean = true,strategy: PartitionStrategy = null) = alert(upStreamNames, alertExecutorId, consume,strategy)
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/StreamSourceBuilder.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/StreamSourceBuilder.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/StreamSourceBuilder.scala
new file mode 100644
index 0000000..9884e88
--- /dev/null
+++ b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/StreamSourceBuilder.scala
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.datastream.core
+
+import org.jgrapht.experimental.dag.DirectedAcyclicGraph
+
+import scala.reflect.runtime.{universe => ru}
+
+/**
+ * @since 12/7/15
+ */
+trait StreamSourceBuilder {
+ def config:Configuration
+
+ /**
+ * Business logic DAG
+ * @return
+ */
+ def dag:DirectedAcyclicGraph[StreamProducer[Any], StreamConnector[Any,Any]]
+
+ /**
+ *
+ * @param iterable top level Iterable interface
+ * @param recycle
+ * @tparam T
+ * @return
+ */
+ def from[T:ru.TypeTag](iterable: Iterable[T],recycle:Boolean = false):IterableStreamProducer[T]={
+ val p = IterableStreamProducer[T](iterable,recycle)
+ p.initWith(dag,config.get)
+ p
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/StreamUnionExpansion.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/StreamUnionExpansion.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/StreamUnionExpansion.scala
new file mode 100644
index 0000000..351782b
--- /dev/null
+++ b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/StreamUnionExpansion.scala
@@ -0,0 +1,71 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.datastream.core
+
+import com.typesafe.config.Config
+import org.jgrapht.experimental.dag.DirectedAcyclicGraph
+import org.slf4j.LoggerFactory
+
+import scala.collection.JavaConversions._
+import scala.collection.mutable.ListBuffer
+
+/**
+ * union operator should be expanded
+ */
+case class StreamUnionExpansion(config: Config) extends StreamDAGExpansion(config){
+ val LOG = LoggerFactory.getLogger(classOf[StreamUnionExpansion])
+
+ override def expand(dag: DirectedAcyclicGraph[StreamProducer[Any], StreamConnector[Any,Any]]) = {
+ val iter = dag.iterator()
+ var toBeAddedEdges = new ListBuffer[StreamConnector[Any,Any]]
+ var toBeRemovedVertex = new ListBuffer[StreamProducer[Any]]
+ while(iter.hasNext) {
+ val current = iter.next()
+ dag.outgoingEdgesOf(current).foreach(edge => {
+ val child = edge.to
+ child match {
+ case StreamUnionProducer(others) => {
+ dag.outgoingEdgesOf(child).foreach(c2 => {
+ toBeAddedEdges += StreamConnector(current, c2.to, edge)
+ others.foreach(o => {
+ toBeAddedEdges += StreamConnector(o, c2.to, edge)
+ })
+ })
+ toBeRemovedVertex += child
+ }
+ case _ =>
+ }
+ })
+ }
+
+ // add back edges
+ toBeAddedEdges.foreach(e => dag.addEdge(e.from, e.to, e))
+ toBeRemovedVertex.foreach(v => dag.removeVertex(v))
+ }
+}
+
+object StreamUnionExpansion{
+ def apply()(implicit config:Config, dag: DirectedAcyclicGraph[StreamProducer[Any], StreamConnector[Any,Any]]): StreamUnionExpansion ={
+ val e = StreamUnionExpansion(config)
+ e.expand(dag)
+ e
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/kafka/JsonMessageDeserializer.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/kafka/JsonMessageDeserializer.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/kafka/JsonMessageDeserializer.scala
deleted file mode 100644
index 754abe0..0000000
--- a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/kafka/JsonMessageDeserializer.scala
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.datastream.kafka
-
-import java.io.IOException
-import java.util
-import java.util.Properties
-
-import com.fasterxml.jackson.databind.ObjectMapper
-import org.apache.eagle.dataproc.impl.storm.kafka.SpoutKafkaMessageDeserializer
-import org.slf4j.{Logger, LoggerFactory}
-
-/**
- * @since 11/6/15
- */
-case class JsonMessageDeserializer(props:Properties) extends SpoutKafkaMessageDeserializer{
- private val objectMapper: ObjectMapper = new ObjectMapper
- private val LOG: Logger = LoggerFactory.getLogger(classOf[JsonMessageDeserializer])
-
- override def deserialize(bytes: Array[Byte]): AnyRef = {
- var map: util.Map[String, _] = null
- try {
- map = objectMapper.readValue(bytes, classOf[util.TreeMap[String, _]])
- } catch {
- case e: IOException => {
- LOG.error("Failed to deserialize json from: " + new String(bytes), e)
- }
- }
- map
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/kafka/KafkaStreamMonitor.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/kafka/KafkaStreamMonitor.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/kafka/KafkaStreamMonitor.scala
deleted file mode 100644
index eff35fc..0000000
--- a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/kafka/KafkaStreamMonitor.scala
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.datastream.kafka
-
-import org.apache.eagle.datastream.StormStreamApp
-import org.apache.eagle.dataproc.impl.storm.kafka.KafkaSourcedSpoutProvider
-
-/**
- * @since 11/6/15
- */
-class KafkaStreamMonitorApp extends StormStreamApp{
- val streamName = get[String]("eagle.stream.name","eventStream")
- val streamExecutorId = get[String]("eagle.stream.executor",s"${streamName}Executor")
-
- set("dataSourceConfig.deserializerClass",classOf[JsonMessageDeserializer].getCanonicalName)
-
- source(new KafkaSourcedSpoutProvider).renameOutputFields(1).withName(streamName)
- .alertWithConsumer(streamName, streamExecutorId)
-}
-
-object KafkaStreamMonitor extends KafkaStreamMonitorApp
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/storm/AbstractStreamBolt.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/storm/AbstractStreamBolt.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/storm/AbstractStreamBolt.scala
new file mode 100644
index 0000000..e109118
--- /dev/null
+++ b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/storm/AbstractStreamBolt.scala
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.datastream.storm
+
+import java.util
+
+import backtype.storm.task.{OutputCollector, TopologyContext}
+import backtype.storm.topology.OutputFieldsDeclarer
+import backtype.storm.topology.base.BaseRichBolt
+import backtype.storm.tuple.{Fields, Tuple}
+import org.apache.eagle.datastream.core.StreamInfo
+import org.apache.eagle.datastream.utils.NameConstants
+import org.slf4j.LoggerFactory
+
+/**
+ *
+ * @param fieldsNum zero-fieldsNum may means something different
+ * @param ack
+ * @param streamInfo
+ * @tparam T
+ */
+abstract class AbstractStreamBolt[T](val fieldsNum:Int=0, val ack:Boolean = true)(implicit streamInfo:StreamInfo) extends BaseRichBolt{
+ private var _collector: OutputCollector = null
+ private val LOG = LoggerFactory.getLogger(classOf[AbstractStreamBolt[T]])
+
+ /**
+ * If outKeyed then
+ * Fields = ("key","value"]
+ * elsif num > 0
+ * Fields = ["f0","f1",..,"fn"]
+ * elsif num == 0
+ * Fields = ["f0"]
+ * end
+ *
+ * @param declarer
+ */
+ override def declareOutputFields(declarer: OutputFieldsDeclarer): Unit = {
+ if(streamInfo.outKeyed) {
+ declarer.declare(new Fields(NameConstants.FIELD_KEY,NameConstants.FIELD_VALUE))
+ }else{
+ if(fieldsNum > 0) {
+ val fields = new util.ArrayList[String]()
+ var i: Int = 0
+ while (i < fieldsNum) {
+ fields.add(NameConstants.FIELD_PREFIX + i)
+ i += 1
+ }
+ declarer.declare(new Fields(fields))
+ }else if(fieldsNum == 0){
+ declarer.declare(new Fields(NameConstants.FIELD_PREFIX + 0))
+ }
+ }
+ }
+
+ def emit(values:util.List[AnyRef])(implicit input:Tuple){
+ if (streamInfo.outKeyed) {
+ _collector.emit(input, util.Arrays.asList(streamInfo.keySelector.key(values).asInstanceOf[AnyRef], values))
+ } else {
+ _collector.emit(input, values)
+ }
+ }
+
+ def emit(value:Any)(implicit input:Tuple){
+ if(streamInfo.outKeyed) {
+ _collector.emit(input, util.Arrays.asList(streamInfo.keySelector.key(value).asInstanceOf[AnyRef],value.asInstanceOf[AnyRef]))
+ }else{
+ _collector.emit(input,util.Arrays.asList(value.asInstanceOf[AnyRef]))
+ }
+ }
+
+ override def execute(input: Tuple): Unit = {
+ try {
+ implicit val _input = input
+ if (streamInfo.inKeyed) {
+ val key = input.getValueByField(NameConstants.FIELD_KEY)
+ val value = input.getValueByField(NameConstants.FIELD_VALUE).asInstanceOf[T]
+ onKeyValue(key, value)
+ } else {
+ onValues(input.getValues)
+ }
+ if(ack) _collector.ack(input)
+ }catch {
+ case t: Throwable => {
+ LOG.error(s"Got exception when processing $input",t)
+ _collector.fail(input)
+ }
+ }
+ }
+
+ /**
+ * Handle keyed stream value
+ */
+ def onKeyValue(key:Any,value:T)(implicit input:Tuple)
+
+ /**
+ * Handle general stream values list
+ *
+ * @param values
+ */
+ def onValues(values:util.List[AnyRef])(implicit input:Tuple)
+
+ override def prepare(stormConf: util.Map[_, _], context: TopologyContext, collector: OutputCollector): Unit = {
+ _collector = collector
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/storm/FilterBoltWrapper.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/storm/FilterBoltWrapper.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/storm/FilterBoltWrapper.scala
new file mode 100644
index 0000000..b175b41
--- /dev/null
+++ b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/storm/FilterBoltWrapper.scala
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.datastream.storm
+
+import java.util
+
+import backtype.storm.tuple.Tuple
+import org.apache.eagle.datastream.core.StreamInfo
+
+case class FilterBoltWrapper(fn:Any => Boolean)(implicit info:StreamInfo) extends AbstractStreamBolt[Any](fieldsNum = 1){
+ /**
+ * Handle keyed stream value
+ */
+ override def onKeyValue(key: Any, value: Any)(implicit input:Tuple): Unit = {
+ if(fn(value)) emit(value)
+ }
+
+ /**
+ * Handle general stream values list
+ *
+ * @param values
+ */
+ override def onValues(values: util.List[AnyRef])(implicit input:Tuple): Unit = {
+ val value = values.get(0)
+ if(fn(value)) emit(value)
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/storm/ForeachBoltWrapper.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/storm/ForeachBoltWrapper.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/storm/ForeachBoltWrapper.scala
new file mode 100644
index 0000000..4c105b2
--- /dev/null
+++ b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/storm/ForeachBoltWrapper.scala
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.datastream.storm
+
+import java.util
+
+import backtype.storm.tuple.Tuple
+import org.apache.eagle.datastream.core.StreamInfo
+
+/**
+ * @since 12/6/15
+ */
+case class ForeachBoltWrapper(fn:Any=>Unit)(implicit info:StreamInfo) extends AbstractStreamBolt[Any] {
+ /**
+ * Handle keyed stream value
+ * @param value
+ */
+ override def onKeyValue(key:Any,value: Any)(implicit input:Tuple): Unit = {
+ fn(value)
+ }
+
+ /**
+ * Handle non-keyed stream values list
+ *
+ * @param values
+ */
+ override def onValues(values: util.List[AnyRef])(implicit input:Tuple): Unit = {
+ fn(values)
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/storm/IterableStreamSpout.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/storm/IterableStreamSpout.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/storm/IterableStreamSpout.scala
new file mode 100644
index 0000000..e3b6e25
--- /dev/null
+++ b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/storm/IterableStreamSpout.scala
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.datastream.storm
+
+import java.util
+
+import backtype.storm.spout.SpoutOutputCollector
+import backtype.storm.task.TopologyContext
+import backtype.storm.topology.OutputFieldsDeclarer
+import backtype.storm.topology.base.BaseRichSpout
+import backtype.storm.tuple.Fields
+import backtype.storm.utils.Utils
+import org.apache.eagle.datastream.core.StreamInfo
+import org.apache.eagle.datastream.utils.NameConstants
+import org.slf4j.LoggerFactory
+
+import scala.collection.JavaConverters._
+
+/**
+ * @since 12/6/15
+ */
+case class IterableStreamSpout(iterable: Iterable[Any],recycle:Boolean = true)(implicit info:StreamInfo) extends BaseRichSpout {
+ val LOG = LoggerFactory.getLogger(classOf[IterableStreamSpout])
+ var _collector:SpoutOutputCollector=null
+ var _iterator:Iterator[Any] = null
+
+ override def open(conf: util.Map[_, _], context: TopologyContext, collector: SpoutOutputCollector): Unit = {
+ this._collector = collector
+ this._iterator = iterable.iterator
+ }
+
+ override def nextTuple(): Unit = {
+ if(_iterator.hasNext){
+ val current = _iterator.next().asInstanceOf[AnyRef]
+ if(info.outKeyed) {
+ _collector.emit(List(info.keySelector.key(current),current).asJava.asInstanceOf[util.List[AnyRef]])
+ }else{
+ _collector.emit(List(current).asJava)
+ }
+ }else if(recycle){
+ LOG.info("Recycling the iterator")
+ _iterator = iterable.iterator
+ }else{
+ LOG.info("No tuple left, sleep forever")
+ this.deactivate()
+ Utils.sleep(Long.MaxValue)
+ }
+ }
+
+ override def declareOutputFields(declarer: OutputFieldsDeclarer): Unit = {
+ if(info.outKeyed) {
+ declarer.declare(new Fields(NameConstants.FIELD_KEY,NameConstants.FIELD_VALUE))
+ }else{
+ declarer.declare(new Fields(NameConstants.FIELD_PREFIX))
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/storm/JavaStormBoltWrapper.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/storm/JavaStormBoltWrapper.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/storm/JavaStormBoltWrapper.scala
new file mode 100644
index 0000000..1c0d42e
--- /dev/null
+++ b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/storm/JavaStormBoltWrapper.scala
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.datastream.storm
+
+import java.util
+
+import backtype.storm.task.{OutputCollector, TopologyContext}
+import backtype.storm.topology.OutputFieldsDeclarer
+import backtype.storm.topology.base.BaseRichBolt
+import backtype.storm.tuple.{Fields, Tuple}
+import org.apache.eagle.datastream.{Collector, EagleTuple, JavaStormStreamExecutor}
+import org.slf4j.LoggerFactory
+
+import scala.collection.JavaConverters._
+
+case class JavaStormBoltWrapper(worker : JavaStormStreamExecutor[EagleTuple]) extends BaseRichBolt{
+ val LOG = LoggerFactory.getLogger(StormBoltWrapper.getClass)
+ var _collector : OutputCollector = null
+
+ override def prepare(stormConf: util.Map[_, _], context: TopologyContext, collector: OutputCollector): Unit = {
+ _collector = collector
+ worker.init
+ }
+
+ override def execute(input : Tuple): Unit ={
+ worker.flatMap(input.getValues, new Collector[EagleTuple](){
+ def collect(t: EagleTuple): Unit ={
+ _collector.emit(input, t.getList.asJava)
+ }
+ })
+ _collector.ack(input)
+ }
+
+ override def declareOutputFields(declarer : OutputFieldsDeclarer): Unit ={
+ val fields = worker.fields
+ LOG.info("output fields for worker " + worker + " : " + fields.toList)
+ declarer.declare(new Fields(fields:_*))
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/storm/JsonMessageDeserializer.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/storm/JsonMessageDeserializer.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/storm/JsonMessageDeserializer.scala
new file mode 100644
index 0000000..0001d2f
--- /dev/null
+++ b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/storm/JsonMessageDeserializer.scala
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.datastream.storm
+
+import java.io.IOException
+import java.util
+import java.util.Properties
+
+import com.fasterxml.jackson.databind.ObjectMapper
+import org.apache.eagle.dataproc.impl.storm.kafka.SpoutKafkaMessageDeserializer
+import org.slf4j.{Logger, LoggerFactory}
+
+/**
+ * @since 11/6/15
+ */
+case class JsonMessageDeserializer(props:Properties) extends SpoutKafkaMessageDeserializer{
+ private val objectMapper: ObjectMapper = new ObjectMapper
+ private val LOG: Logger = LoggerFactory.getLogger(classOf[JsonMessageDeserializer])
+
+ override def deserialize(bytes: Array[Byte]): AnyRef = {
+ var map: util.Map[String, _] = null
+ try {
+ map = objectMapper.readValue(bytes, classOf[util.TreeMap[String, _]])
+ } catch {
+ case e: IOException => {
+ LOG.error("Failed to deserialize json from: " + new String(bytes), e)
+ }
+ }
+ map
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/storm/KafkaStreamMonitor.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/storm/KafkaStreamMonitor.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/storm/KafkaStreamMonitor.scala
new file mode 100644
index 0000000..8c92590
--- /dev/null
+++ b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/storm/KafkaStreamMonitor.scala
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.datastream.storm
+
+import org.apache.eagle.dataproc.impl.storm.kafka.KafkaSourcedSpoutProvider
+import org.apache.eagle.datastream.ExecutionEnvironments
+
+class KafkaStreamMonitorApp extends App {
+ val env = ExecutionEnvironments.get[StormExecutionEnvironment](args)
+ val streamName = env.config.get[String]("eagle.stream.name","eventStream")
+ val streamExecutorId = env.config.get[String]("eagle.stream.executor",s"${streamName}Executor")
+ env.config.set("dataSourceConfig.deserializerClass",classOf[JsonMessageDeserializer].getCanonicalName)
+ env.fromSpout(new KafkaSourcedSpoutProvider()).parallelism(1).nameAs(streamName) ! (Seq(streamName),streamExecutorId)
+ env.execute()
+}
+
+object KafkaStreamMonitor extends KafkaStreamMonitorApp
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/storm/MapBoltWrapper.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/storm/MapBoltWrapper.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/storm/MapBoltWrapper.scala
new file mode 100644
index 0000000..1119e08
--- /dev/null
+++ b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/storm/MapBoltWrapper.scala
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.datastream.storm
+
+import java.util
+
+import backtype.storm.tuple.Tuple
+import org.apache.eagle.datastream.core.StreamInfo
+
+/**
+ * @param num if num is zero, then means that it's using type-safe way, because to map operation, it must require at least one output field
+ * @param fn
+ * @param streamInfo
+ */
+case class MapBoltWrapper(num: Int, fn: Any => Any)(implicit streamInfo: StreamInfo) extends AbstractStreamBolt[Any](fieldsNum = num){
+ /**
+ * Handle keyed stream value
+ */
+ override def onKeyValue(key: Any, value: Any)(implicit input:Tuple): Unit = {
+ emit(fn(value))
+ }
+
+ /**
+ * Handle general stream values list
+ *
+ * @param values
+ */
+ override def onValues(values: util.List[AnyRef])(implicit input:Tuple): Unit = {
+ val size = values.size()
+ if(size == 0) return
+ if(num == 0) {
+ emit(fn(values.get(0)))
+ } else {
+ var tuple: AnyRef = null
+ size match {
+ case 1 => tuple = scala.Tuple1[AnyRef](values.get(0))
+ case 2 => tuple = scala.Tuple2(values.get(0), values.get(1))
+ case 3 => tuple = scala.Tuple3(values.get(0), values.get(1), values.get(2))
+ case 4 => tuple = scala.Tuple4(values.get(0), values.get(1), values.get(2), values.get(3))
+ case _ => throw new IllegalArgumentException(s"Exceed max supported tuple size $size > 4")
+ }
+ val output = fn(tuple)
+ output match {
+ case scala.Tuple1(a) => emit(util.Arrays.asList(a.asInstanceOf[AnyRef]))
+ case scala.Tuple2(a, b) => emit(util.Arrays.asList(a.asInstanceOf[AnyRef], b.asInstanceOf[AnyRef]))
+ case scala.Tuple3(a, b, c) => emit(util.Arrays.asList(a.asInstanceOf[AnyRef], b.asInstanceOf[AnyRef], c.asInstanceOf[AnyRef]))
+ case scala.Tuple4(a, b, c, d) => emit(util.Arrays.asList(a.asInstanceOf[AnyRef], b.asInstanceOf[AnyRef], c.asInstanceOf[AnyRef], d.asInstanceOf[AnyRef]))
+ case a => emit(util.Arrays.asList(a.asInstanceOf[AnyRef]))
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/storm/SpoutProxy.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/storm/SpoutProxy.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/storm/SpoutProxy.scala
new file mode 100644
index 0000000..37622e4
--- /dev/null
+++ b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/storm/SpoutProxy.scala
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.datastream.storm
+
+import java.util
+
+import backtype.storm.spout.{ISpoutOutputCollector, SpoutOutputCollector}
+import backtype.storm.task.TopologyContext
+import backtype.storm.topology.OutputFieldsDeclarer
+import backtype.storm.topology.base.BaseRichSpout
+import backtype.storm.tuple.Fields
+import org.apache.eagle.datastream.core.{KeySelector, StreamInfo}
+import org.apache.eagle.datastream.utils.NameConstants
+import java.util.{List => JList}
+
+/**
+ * Declare delegated BaseRichSpout with given field names
+ *
+ * @param delegate delegated BaseRichSpout
+ * @param outputFields given field names
+ */
+case class SpoutProxy(delegate: BaseRichSpout, outputFields: java.util.List[String]) extends BaseRichSpout{
+ def open(conf: java.util.Map[_, _], context: TopologyContext, collector: SpoutOutputCollector) {
+ this.delegate.open(conf, context, collector)
+ }
+
+ def nextTuple {
+ this.delegate.nextTuple
+ }
+
+ override def ack(msgId: AnyRef) {
+ this.delegate.ack(msgId)
+ }
+
+ override def fail(msgId: AnyRef) {
+ this.delegate.fail(msgId)
+ }
+
+ override def deactivate {
+ this.delegate.deactivate
+ }
+
+ override def declareOutputFields(declarer: OutputFieldsDeclarer) {
+ declarer.declare(new Fields(outputFields))
+ }
+
+ override def close {
+ this.delegate.close
+ }
+}
+
+case class KeyedSpoutOutputCollector(delegate: ISpoutOutputCollector,keyer:KeySelector) extends SpoutOutputCollector(delegate) {
+ override def emitDirect (taskId: Int, streamId: String, tuple: JList[AnyRef], messageId: AnyRef):Unit = {
+ val kv = toKeyValue(tuple)
+ delegate.emitDirect(taskId, streamId,kv, messageId)
+ }
+
+ override def emit(streamId: String, tuple: JList[AnyRef], messageId: AnyRef):JList[Integer] = {
+ val kv = toKeyValue(tuple)
+ delegate.emit(streamId,kv, messageId)
+ }
+
+ def toKeyValue(tuple: JList[AnyRef]) = util.Arrays.asList(keyer.key(tuple.get(0)),tuple.get(0)).asInstanceOf[JList[AnyRef]]
+}
+
+case class KeyedSpoutProxy(delegate: BaseRichSpout)(implicit streamInfo:StreamInfo) extends BaseRichSpout{
+ override def open(conf: java.util.Map[_, _], context: TopologyContext, collector: SpoutOutputCollector) {
+ if(!streamInfo.outKeyed) throw new IllegalArgumentException(s"$streamInfo is not key-based")
+ if(streamInfo.keySelector == null) throw new NullPointerException(s"KeySelector $streamInfo is null")
+
+ this.delegate.open(conf, context, KeyedSpoutOutputCollector(collector,streamInfo.keySelector))
+ }
+
+ override def nextTuple {
+ this.delegate.nextTuple
+ }
+
+ override def ack(msgId: AnyRef) {
+ this.delegate.ack(msgId)
+ }
+
+ override def fail(msgId: AnyRef) {
+ this.delegate.fail(msgId)
+ }
+
+ override def deactivate {
+ this.delegate.deactivate
+ }
+
+ override def declareOutputFields(declarer: OutputFieldsDeclarer) {
+ declarer.declare(new Fields(NameConstants.FIELD_KEY,NameConstants.FIELD_VALUE))
+ }
+
+ override def close {
+ this.delegate.close
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/storm/StormBoltFactory.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/storm/StormBoltFactory.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/storm/StormBoltFactory.scala
new file mode 100644
index 0000000..b048b90
--- /dev/null
+++ b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/storm/StormBoltFactory.scala
@@ -0,0 +1,53 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+package org.apache.eagle.datastream.storm
+
+import backtype.storm.topology.base.BaseRichBolt
+import com.typesafe.config.Config
+import org.apache.eagle.datastream._
+import org.apache.eagle.datastream.core._
+
+object StormBoltFactory {
+ def getBoltWrapper(graph: StreamProducerGraph, producer : StreamProducer[Any], config : Config) : BaseRichBolt = {
+ implicit val streamInfo = producer.getInfo
+ producer match{
+ case FlatMapProducer(worker) => {
+ if(worker.isInstanceOf[JavaStormStreamExecutor[EagleTuple]]){
+ worker.asInstanceOf[JavaStormStreamExecutor[EagleTuple]].prepareConfig(config)
+ JavaStormBoltWrapper(worker.asInstanceOf[JavaStormStreamExecutor[EagleTuple]])
+ }else if(worker.isInstanceOf[StormStreamExecutor[EagleTuple]]){
+ worker.asInstanceOf[StormStreamExecutor[EagleTuple]].prepareConfig(config)
+ StormBoltWrapper(worker.asInstanceOf[StormStreamExecutor[EagleTuple]])
+ }else {
+ throw new UnsupportedOperationException
+ }
+ }
+ case filter:FilterProducer[Any] => {
+ FilterBoltWrapper(filter.fn)
+ }
+ case mapper:MapperProducer[Any,Any] => {
+ MapBoltWrapper(mapper.numOutputFields, mapper.fn)
+ }
+ case foreach:ForeachProducer[Any] => {
+ ForeachBoltWrapper(foreach.fn)
+ }
+ case _ => throw new UnsupportedOperationException(s"Unsupported producer: ${producer.toString}")
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/storm/StormBoltWrapper.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/storm/StormBoltWrapper.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/storm/StormBoltWrapper.scala
new file mode 100644
index 0000000..3b91e77
--- /dev/null
+++ b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/storm/StormBoltWrapper.scala
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.datastream.storm
+
+import java.util
+
+import backtype.storm.task.{OutputCollector, TopologyContext}
+import backtype.storm.topology.OutputFieldsDeclarer
+import backtype.storm.topology.base.BaseRichBolt
+import backtype.storm.tuple.{Fields, Tuple}
+import org.apache.eagle.datastream.{Collector, EagleTuple, StormStreamExecutor}
+import org.slf4j.LoggerFactory
+
+import scala.collection.JavaConverters._
+
+case class StormBoltWrapper(worker : StormStreamExecutor[EagleTuple]) extends BaseRichBolt{
+ val LOG = LoggerFactory.getLogger(StormBoltWrapper.getClass)
+ var _collector : OutputCollector = null
+
+ override def prepare(stormConf: util.Map[_, _], context: TopologyContext, collector: OutputCollector): Unit = {
+ _collector = collector
+ worker.init
+ }
+
+ override def execute(input : Tuple): Unit = {
+ try {
+ worker.flatMap(input.getValues.asScala, new Collector[EagleTuple] {
+ override def collect(t: EagleTuple): Unit = {
+ _collector.emit(input, t.getList.asJava)
+ }
+ })
+ }catch{
+ case ex: Exception => {
+ LOG.error("fail executing", ex)
+ _collector.fail(input)
+ throw new RuntimeException(ex)
+ }
+ }
+ _collector.ack(input)
+ }
+
+ override def declareOutputFields(declarer : OutputFieldsDeclarer): Unit ={
+ val fields = worker.fields
+ LOG.info("Output fields for worker " + worker + " : " + fields.toList)
+ declarer.declare(new Fields(fields:_*))
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/storm/StormExecutionEnvironment.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/storm/StormExecutionEnvironment.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/storm/StormExecutionEnvironment.scala
new file mode 100644
index 0000000..4e7d743
--- /dev/null
+++ b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/storm/StormExecutionEnvironment.scala
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.datastream.storm
+
+import backtype.storm.topology.base.BaseRichSpout
+import com.typesafe.config.Config
+import org.apache.eagle.dataproc.impl.storm.StormSpoutProvider
+import org.apache.eagle.datastream.core.{ExecutionEnvironmentBase, StormSourceProducer, StreamDAG}
+
+
+/**
+ * @since 12/7/15
+ */
+case class StormExecutionEnvironment(private val conf:Config) extends ExecutionEnvironmentBase(conf){
+
+
+ override def execute(dag: StreamDAG) : Unit = {
+ StormTopologyCompiler(config.get, dag).buildTopology.execute
+ }
+
+ def fromSpout[T](source: BaseRichSpout): StormSourceProducer[T] = {
+ val ret = StormSourceProducer[T](source)
+ ret.initWith(dag,config.get)
+ ret
+ }
+
+ def fromSpout[T](sourceProvider: StormSpoutProvider):StormSourceProducer[T] = fromSpout(sourceProvider.getSpout(config.get))
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/storm/StormExecutorForAlertWrapper.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/storm/StormExecutorForAlertWrapper.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/storm/StormExecutorForAlertWrapper.scala
new file mode 100644
index 0000000..1b0d133
--- /dev/null
+++ b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/storm/StormExecutorForAlertWrapper.scala
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.datastream.storm
+
+import java.util
+
+import com.typesafe.config.Config
+import org.apache.eagle.datastream._
+
+case class StormExecutorForAlertWrapper(delegate: StormStreamExecutor[Tuple2[String, util.SortedMap[AnyRef, AnyRef]]], streamName: String)
+ extends StormStreamExecutor3[String, String, util.SortedMap[Object, Object]]{
+ override def prepareConfig(config: Config): Unit = {
+ delegate.prepareConfig(config)
+ }
+
+ override def init: Unit = {
+ delegate.init
+ }
+
+ override def flatMap(input: Seq[AnyRef], collector: Collector[Tuple3[String, String, util.SortedMap[Object, Object]]]): Unit = {
+ delegate.flatMap(input, new Collector[Tuple2[String, util.SortedMap[AnyRef, AnyRef]]] {
+ override def collect(r: Tuple2[String, util.SortedMap[AnyRef, AnyRef]]): Unit = {
+ collector.collect(Tuple3(r.f0, streamName, r.f1))
+ }
+ })
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/storm/StormSpoutFactory.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/storm/StormSpoutFactory.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/storm/StormSpoutFactory.scala
new file mode 100644
index 0000000..74ed11d
--- /dev/null
+++ b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/storm/StormSpoutFactory.scala
@@ -0,0 +1,67 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+package org.apache.eagle.datastream.storm
+
+import java.util
+
+import backtype.storm.topology.base.BaseRichSpout
+import com.typesafe.config.Config
+import org.apache.eagle.datastream.core.{IterableStreamProducer, StormSourceProducer, StreamProducer}
+import org.apache.eagle.datastream.utils.NameConstants
+
+object StormSpoutFactory {
+ def createSpout(config: Config, from: StreamProducer[Any]): BaseRichSpout = {
+ implicit val streamInfo = from.getInfo
+ from match {
+ case p@StormSourceProducer(source) =>
+ if(p.outKeyed) {
+ createKeyedProxySpout(p)
+ }else {
+ createProxySpout(p)
+ }
+ case p@IterableStreamProducer(iterable,recycle) =>
+ IterableStreamSpout(iterable,recycle)
+ case _ =>
+ throw new IllegalArgumentException(s"Cannot compile unknown $from to a Storm Spout")
+ }
+ }
+
+ /**
+ * @param sourceProducer source producer
+ * @return
+ */
+ def createProxySpout(sourceProducer: StormSourceProducer[Any]): BaseRichSpout = {
+ val numFields = sourceProducer.numFields
+ if (numFields <= 0) {
+ sourceProducer.source
+ } else {
+ var i = 0
+ val ret = new util.ArrayList[String]
+ while (i < numFields) {
+ ret.add(NameConstants.FIELD_PREFIX + i)
+ i += 1
+ }
+ SpoutProxy(sourceProducer.source, ret)
+ }
+ }
+
+ def createKeyedProxySpout(sourceProducer: StormSourceProducer[Any]):BaseRichSpout = {
+ KeyedSpoutProxy(sourceProducer.source)(sourceProducer.getInfo)
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/storm/StormTopologyCompiler.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/storm/StormTopologyCompiler.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/storm/StormTopologyCompiler.scala
new file mode 100644
index 0000000..18f52cb
--- /dev/null
+++ b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/storm/StormTopologyCompiler.scala
@@ -0,0 +1,118 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+package org.apache.eagle.datastream.storm
+
+import java.util
+
+import backtype.storm.topology.base.BaseRichBolt
+import backtype.storm.topology.{BoltDeclarer, TopologyBuilder}
+import backtype.storm.tuple.Fields
+import com.typesafe.config.Config
+import org.apache.eagle.dataproc.impl.storm.partition.CustomPartitionGrouping
+import org.apache.eagle.datastream.core._
+import org.apache.eagle.datastream.utils.NameConstants
+import org.slf4j.LoggerFactory
+
+import scala.collection.mutable.ListBuffer
+
+case class StormTopologyCompiler(config: Config, graph: StreamProducerGraph) extends AbstractTopologyCompiler{
+ val LOG = LoggerFactory.getLogger(StormTopologyCompiler.getClass)
+ val boltCache = scala.collection.mutable.Map[StreamProducer[Any], StormBoltWrapper]()
+
+ override def buildTopology: AbstractTopologyExecutor ={
+ val builder = new TopologyBuilder();
+ val iter = graph.iterator()
+ val boltDeclarerCache = scala.collection.mutable.Map[String, BoltDeclarer]()
+ val stormTopologyGraph = ListBuffer[String]()
+ while(iter.hasNext){
+ val from = iter.next()
+ val fromName = from.name
+ if(graph.isSource(from)){
+ val spout = StormSpoutFactory.createSpout(config, from)
+ builder.setSpout(fromName, spout, from.parallelism)
+ LOG.info("Spout: " + fromName + " with parallelism " + from.parallelism)
+ } else {
+ LOG.debug("Bolt" + fromName)
+ }
+ val edges = graph.outgoingEdgesOf(from)
+ edges.foreach(sc => {
+ val toName = sc.to.name
+ var boltDeclarer: BoltDeclarer = null
+ val toBolt = createBoltIfAbsent(toName)
+ boltDeclarerCache.get(toName) match {
+ case None => {
+ var finalParallelism = 1
+ graph.getNodeByName(toName) match {
+ case Some(p) => finalParallelism = p.parallelism
+ case None => finalParallelism = 1
+ }
+ boltDeclarer = builder.setBolt(toName, toBolt, finalParallelism)
+ LOG.info("Bolt: " + toName + " with parallelism " + finalParallelism)
+ boltDeclarerCache.put(toName, boltDeclarer)
+ }
+ case Some(bt) => boltDeclarer = bt
+ }
+
+ sc match {
+ case GroupbyFieldsConnector(_, _, groupByFields) =>
+ boltDeclarer.fieldsGrouping(fromName, new Fields(fields(groupByFields)))
+ case GroupbyStrategyConnector(_, _, strategy) =>
+ boltDeclarer.customGrouping(fromName, new CustomPartitionGrouping(strategy));
+ case GroupbyKeyConnector(_, _, keySelector) =>
+ boltDeclarer.fieldsGrouping(fromName, new Fields(NameConstants.FIELD_KEY));
+ case ShuffleConnector(_, _) => {
+ boltDeclarer.shuffleGrouping(fromName)
+ }
+ case _ => throw new UnsupportedOperationException(s"Supported stream connector $sc")
+ }
+
+ if (graph.isSource(from)) {
+ stormTopologyGraph += s"Spout{$fromName}{${from.parallelism}) ~> Bolt{$toName}{${from.parallelism}} in $sc"
+ } else {
+ stormTopologyGraph += s"Bolt{$fromName}{${from.parallelism}) ~> Bolt{$toName}{${from.parallelism}} in $sc"
+ }
+ })
+ }
+ LOG.info(s"Storm topology DAG\n{\n \t${stormTopologyGraph.mkString("\n\t")} \n}")
+ new StormTopologyExecutorImpl(builder.createTopology, config)
+ }
+
+ def fields(fields : Seq[Int]): java.util.List[String] ={
+ val ret = new util.ArrayList[String]
+ fields.map(n => ret.add(NameConstants.FIELD_PREFIX + n))
+ ret
+ }
+
+ def createBoltIfAbsent(name : String) : BaseRichBolt = {
+ val producer = graph.getNodeByName(name)
+ producer match{
+ case Some(p) => createBoltIfAbsent(graph, p)
+ case None => throw new IllegalArgumentException("please check bolt name " + name)
+ }
+ }
+
+ def createBoltIfAbsent(graph: StreamProducerGraph, producer : StreamProducer[Any]): BaseRichBolt ={
+ boltCache.get(producer) match{
+ case Some(bolt) => bolt
+ case None => {
+ StormBoltFactory.getBoltWrapper(graph, producer, config)
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/storm/StormTopologyExecutorImpl.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/storm/StormTopologyExecutorImpl.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/storm/StormTopologyExecutorImpl.scala
new file mode 100644
index 0000000..11ec46d
--- /dev/null
+++ b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/storm/StormTopologyExecutorImpl.scala
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.datastream.storm
+
+import java.io.{File, FileInputStream}
+
+import _root_.storm.trident.spout.RichSpoutBatchExecutor
+import backtype.storm.generated.StormTopology
+import backtype.storm.utils.Utils
+import backtype.storm.{Config, LocalCluster, StormSubmitter}
+import org.apache.eagle.datastream.core.AbstractTopologyExecutor
+import org.yaml.snakeyaml.Yaml
+
+case class StormTopologyExecutorImpl(topology: StormTopology, config: com.typesafe.config.Config) extends AbstractTopologyExecutor {
+ @throws(classOf[Exception])
+ def execute {
+ val localMode: Boolean = config.getString("envContextConfig.mode").equalsIgnoreCase("local")
+ val conf: Config = new Config
+ conf.put(RichSpoutBatchExecutor.MAX_BATCH_SIZE_CONF, Int.box(64 * 1024))
+ conf.put(Config.TOPOLOGY_RECEIVER_BUFFER_SIZE, Int.box(8))
+ conf.put(Config.TOPOLOGY_TRANSFER_BUFFER_SIZE, Int.box(32))
+ conf.put(Config.TOPOLOGY_EXECUTOR_RECEIVE_BUFFER_SIZE, Int.box(16384))
+ conf.put(Config.TOPOLOGY_EXECUTOR_SEND_BUFFER_SIZE, Int.box(16384))
+
+ if(config.hasPath("envContextConfig.stormConfigFile")) {
+ val file = new File(config.getString("envContextConfig.stormConfigFile"))
+ if(file.exists()) {
+ val inputFileStream = new FileInputStream(file)
+ val yaml = new Yaml()
+ try {
+ val stormConf = yaml.load(inputFileStream).asInstanceOf[java.util.LinkedHashMap[String, Object]]
+ if(stormConf != null) conf.putAll(stormConf)
+ } catch {
+ case _: Throwable => ()
+ } finally {
+ if(inputFileStream != null) inputFileStream.close()
+ }
+ }
+ }
+
+ val topologyName = config.getString("envContextConfig.topologyName")
+ if (!localMode) {
+ StormSubmitter.submitTopologyWithProgressBar(topologyName, conf, topology)
+ } else {
+ val cluster: LocalCluster = new LocalCluster
+ cluster.submitTopology(topologyName, conf, topology)
+ while(true) {
+ try {
+ Utils.sleep(Integer.MAX_VALUE)
+ }
+ catch {
+ case _: Throwable => () // Do nothing
+ }
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/utils/AlertExecutorConsumerUtils.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/utils/AlertExecutorConsumerUtils.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/utils/AlertExecutorConsumerUtils.scala
new file mode 100644
index 0000000..67818e1
--- /dev/null
+++ b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/utils/AlertExecutorConsumerUtils.scala
@@ -0,0 +1,76 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.datastream.utils
+
+import java.util
+
+import org.apache.eagle.alert.dedup.{AlertEmailDeduplicationExecutor, AlertEntityDeduplicationExecutor}
+import org.apache.eagle.alert.notification.AlertNotificationExecutor
+import org.apache.eagle.alert.persist.AlertPersistExecutor
+import org.apache.eagle.datastream.core.{StreamConnector, FlatMapProducer, StreamProducer}
+import org.apache.eagle.executor.AlertExecutor
+import org.slf4j.{Logger, LoggerFactory}
+
+import scala.collection.mutable.ListBuffer
+
+/**
+ * Create alert executors and provide callback for programmer to link alert executor to immediate parent executors
+ *
+ * <br/><br/>
+ * Explanations for programId, alertExecutorId and policy<br/><br/>
+ * - programId - distributed or single-process program for example one storm topology<br/>
+ * - alertExecutorId - one process/thread which executes multiple policies<br/>
+ * - policy - some rules to be evaluated<br/>
+ *
+ * <br/>
+ *
+ * Normally the mapping is like following:
+ * <pre>
+ * programId (1:N) alertExecutorId
+ * alertExecutorId (1:N) policy
+ * </pre>
+ */
+
+object AlertExecutorConsumerUtils {
+ private val LOG: Logger = LoggerFactory.getLogger(AlertExecutorConsumerUtils.getClass)
+
+ def setupAlertConsumers(toBeAddedEdges: ListBuffer[StreamConnector[Any,Any]], alertStreamProducers: List[StreamProducer[Any]]): Unit = {
+ val alertExecutorIdList: java.util.List[String] = new util.ArrayList[String]()
+ alertStreamProducers.map(x =>
+ alertExecutorIdList.add(x.asInstanceOf[FlatMapProducer[AnyRef, AnyRef]].mapper.asInstanceOf[AlertExecutor].getAlertExecutorId));
+ val alertDefDao = alertStreamProducers.head.asInstanceOf[FlatMapProducer[AnyRef, AnyRef]].mapper.asInstanceOf[AlertExecutor].getAlertDefinitionDao
+ val entityDedupExecutor: AlertEntityDeduplicationExecutor = new AlertEntityDeduplicationExecutor(alertExecutorIdList, alertDefDao)
+ val emailDedupExecutor: AlertEmailDeduplicationExecutor = new AlertEmailDeduplicationExecutor(alertExecutorIdList, alertDefDao)
+ val notificationExecutor: AlertNotificationExecutor = new AlertNotificationExecutor(alertExecutorIdList, alertDefDao)
+ val persistExecutor: AlertPersistExecutor = new AlertPersistExecutor
+
+ val entityDedupStreamProducer = FlatMapProducer(entityDedupExecutor)
+ val persistStreamProducer = FlatMapProducer(persistExecutor)
+ val emailDedupStreamProducer = FlatMapProducer(emailDedupExecutor)
+ val notificationStreamProducer = FlatMapProducer(notificationExecutor)
+ toBeAddedEdges += StreamConnector(entityDedupStreamProducer, persistStreamProducer)
+ toBeAddedEdges += StreamConnector(emailDedupStreamProducer, notificationStreamProducer)
+
+ alertStreamProducers.foreach(sp => {
+ toBeAddedEdges += StreamConnector(sp, entityDedupStreamProducer)
+ toBeAddedEdges += StreamConnector(sp, emailDedupStreamProducer)
+ })
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/utils/GraphPrinter.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/utils/GraphPrinter.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/utils/GraphPrinter.scala
new file mode 100644
index 0000000..33a805c
--- /dev/null
+++ b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/utils/GraphPrinter.scala
@@ -0,0 +1,59 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.datastream.utils
+
+import org.apache.eagle.datastream.core.{StreamConnector, StreamProducer}
+import org.jgrapht.experimental.dag.DirectedAcyclicGraph
+import org.slf4j.LoggerFactory
+
+import scala.collection.JavaConversions._
+import scala.collection.mutable.ListBuffer
+
+/**
+ * @see scala.tools.nsc.DotDiagramGenerator
+ */
+object GraphPrinter {
+ private val LOG = LoggerFactory.getLogger(GraphPrinter.getClass)
+ def print(dag: DirectedAcyclicGraph[StreamProducer[Any], StreamConnector[Any, Any]], message:String = "Print stream DAG graph"): Unit = {
+ val graphStr = ListBuffer[String]()
+ val iter = dag.iterator()
+ while (iter.hasNext) {
+ val current = iter.next()
+ dag.outgoingEdgesOf(current).foreach(edge => {
+ graphStr += s"${edge.from.name}{${edge.from.parallelism}} ~> ${edge.to.name}{${edge.to.parallelism}} in ${edge.toString}"
+ })
+ }
+ LOG.info(message+"\n{ \n\t" + graphStr.mkString("\n\t") + "\n}")
+ }
+
+ def printDotDigraph(dag: DirectedAcyclicGraph[StreamProducer[Any], StreamConnector[Any, Any]], title:String = "dag", message:String = "Print DOT digraph (copy and visualize with http://www.webgraphviz.com/)"): String = {
+ val graphStr = ListBuffer[String]()
+ val iter = dag.iterator()
+ while (iter.hasNext) {
+ val current = iter.next()
+ dag.outgoingEdgesOf(current).foreach(edge => {
+ graphStr += s""""${edge.from.name} x ${edge.from.parallelismNum}" -> "${edge.to.name} x ${edge.from.parallelismNum}" [label = "$edge"];"""
+ })
+ }
+ val dotDigraph = s"""digraph $title { \n\t${graphStr.mkString("\n\t")} \n}"""
+ LOG.info(s"""$message\n\n$dotDigraph\n""")
+ dotDigraph
+ }
+}
\ No newline at end of file
[4/5] incubator-eagle git commit: EAGLE-66 Typesafe Streaming DSL and
KeyValue based Grouping
Posted by ha...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StormTopologyExecutorImpl.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StormTopologyExecutorImpl.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StormTopologyExecutorImpl.scala
deleted file mode 100644
index df763b9..0000000
--- a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StormTopologyExecutorImpl.scala
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.datastream
-
-import java.io.{FileInputStream, File}
-
-import backtype.storm.generated.StormTopology
-import backtype.storm.utils.Utils
-import backtype.storm.{Config, LocalCluster, StormSubmitter}
-import org.yaml.snakeyaml.Yaml
-import storm.trident.spout.RichSpoutBatchExecutor
-
-case class StormTopologyExecutorImpl(topology: StormTopology, config: com.typesafe.config.Config) extends AbstractTopologyExecutor {
- @throws(classOf[Exception])
- def execute {
- val localMode: Boolean = config.getString("envContextConfig.mode").equalsIgnoreCase("local")
- val conf: Config = new Config
- conf.put(RichSpoutBatchExecutor.MAX_BATCH_SIZE_CONF, Int.box(64 * 1024))
- conf.put(Config.TOPOLOGY_RECEIVER_BUFFER_SIZE, Int.box(8))
- conf.put(Config.TOPOLOGY_TRANSFER_BUFFER_SIZE, Int.box(32))
- conf.put(Config.TOPOLOGY_EXECUTOR_RECEIVE_BUFFER_SIZE, Int.box(16384))
- conf.put(Config.TOPOLOGY_EXECUTOR_SEND_BUFFER_SIZE, Int.box(16384))
-
- if(config.hasPath("envContextConfig.stormConfigFile")) {
- //val inputFileStream = {
- // StormTopologyExecutorImpl.getClass.getClassLoader.getResourceAsStream(config.getString("envContextConfig.stormConfigFile"))
- //}
- val file = new File(config.getString("envContextConfig.stormConfigFile"))
- if(file.exists()) {
- val inputFileStream = new FileInputStream(file)
- val yaml = new Yaml()
- try {
- val stormConf = yaml.load(inputFileStream).asInstanceOf[java.util.LinkedHashMap[String, Object]]
- if(stormConf != null) conf.putAll(stormConf)
- } catch {
- case _ => ()
- } finally {
- if(inputFileStream != null) inputFileStream.close()
- }
- }
- }
-
- val topologyName = config.getString("envContextConfig.topologyName")
- if (!localMode) {
- StormSubmitter.submitTopologyWithProgressBar(topologyName, conf, topology)
- }
- else {
- val cluster: LocalCluster = new LocalCluster
- cluster.submitTopology(topologyName, conf, topology)
- while(true) {
- try {
- Utils.sleep(Integer.MAX_VALUE)
- }
- catch {
- case _ => () // Do nothing
- }
- }
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StreamAlertExpansion.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StreamAlertExpansion.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StreamAlertExpansion.scala
deleted file mode 100644
index 913939a..0000000
--- a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StreamAlertExpansion.scala
+++ /dev/null
@@ -1,198 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. See the NOTICE file distributed with
- * * this work for additional information regarding copyright ownership.
- * * The ASF licenses this file to You under the Apache License, Version 2.0
- * * (the "License"); you may not use this file except in compliance with
- * * the License. You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License.
- *
- */
-
-package org.apache.eagle.datastream
-
-import java.util
-
-import com.typesafe.config.Config
-import org.apache.eagle.alert.dao.AlertDefinitionDAOImpl
-import org.apache.eagle.executor.AlertExecutorCreationUtils
-import org.apache.eagle.service.client.EagleServiceConnector
-import org.jgrapht.experimental.dag.DirectedAcyclicGraph
-import org.slf4j.LoggerFactory
-
-import scala.collection.JavaConversions._
-import scala.collection.mutable.ListBuffer
-
-/**
- * The constraints for alert is:
- * 1. only 3 StreamProducers can be put immediately before MapProducer, FlatMapProducer, StreamUnionProducer
- * 2. For StreamUnionProducer, the only supported unioned producers are MapProducer and FlatMapProducer
- * 3. the output for MapProducer and FlatMapProducer is 2-field tuple, key and value, key is string, value has to be SortedMap
- * 4. the framework will wrapper original MapProducer and FlatMapProducer to emit 3-field tuple, {key, streamName and value}
- * 5. the framework will automatically partition traffic with first field
- *
- *
- * 2 steps
- * step 1: wrapper previous StreamProducer with one more field "streamName"
- * step 2: partition alert executor by policy partitioner class
- */
-
-class StreamAlertExpansion(config: Config) extends StreamDAGExpansion(config) {
- val LOG = LoggerFactory.getLogger(classOf[StreamAlertExpansion])
-
- override def expand(dag: DirectedAcyclicGraph[StreamProducer, StreamConnector]): Unit ={
- val iter = dag.iterator()
- val toBeAddedEdges = new ListBuffer[StreamConnector]
- val toBeRemovedVertex = new ListBuffer[StreamProducer]
- while(iter.hasNext) {
- val current = iter.next()
- dag.outgoingEdgesOf(current).foreach(edge => {
- val child = edge.to
- onIteration(toBeAddedEdges, toBeRemovedVertex, dag, current, child)
- })
- }
- // add back edges
- toBeAddedEdges.foreach(e => {
- dag.addVertex(e.from)
- dag.addVertex(e.to)
- dag.addEdge(e.from, e.to, e)
- })
- toBeRemovedVertex.foreach(v => dag.removeVertex(v))
- }
-
- def onIteration(toBeAddedEdges: ListBuffer[StreamConnector], toBeRemovedVertex: ListBuffer[StreamProducer],
- dag: DirectedAcyclicGraph[StreamProducer, StreamConnector], current: StreamProducer, child: StreamProducer): Unit = {
- child match {
- case AlertStreamSink(id, upStreamNames, alertExecutorId, withConsumer, strategy) => {
- /**
- * step 1: wrapper previous StreamProducer with one more field "streamName"
- * for AlertStreamSink, we check previous StreamProducer and replace that
- */
- val newStreamProducers = new ListBuffer[StreamProducer]
- current match {
- case StreamUnionProducer(id, others) => {
- val incomingEdges = dag.incomingEdgesOf(current)
- incomingEdges.foreach(e => newStreamProducers += replace(toBeAddedEdges, toBeRemovedVertex, dag, e.from, upStreamNames.get(0)))
- var i: Int = 1
- others.foreach(o => {
- newStreamProducers += replace(toBeAddedEdges, toBeRemovedVertex, dag, o, upStreamNames.get(i))
- i += 1
- })
- }
- case _: FlatMapProducer[AnyRef, AnyRef] => {
- newStreamProducers += replace(toBeAddedEdges, toBeRemovedVertex, dag, current, upStreamNames.get(0))
- }
- case _: MapProducer => {
- newStreamProducers += replace(toBeAddedEdges, toBeRemovedVertex, dag, current, upStreamNames.get(0))
- }
- case s: StreamProducer if dag.inDegreeOf(s) == 0 => {
- newStreamProducers += replace(toBeAddedEdges, toBeRemovedVertex, dag, current, upStreamNames.get(0))
- }
- case p@_ => throw new IllegalStateException(s"$p can not be put before AlertStreamSink, only StreamUnionProducer,FlatMapProducer and MapProducer are supported")
- }
-
- /**
- * step 2: partition alert executor by policy partitioner class
- */
- val alertExecutors = AlertExecutorCreationUtils.createAlertExecutors(config, new AlertDefinitionDAOImpl(new EagleServiceConnector(config)), upStreamNames, alertExecutorId)
- var alertProducers = new scala.collection.mutable.MutableList[StreamProducer]
- alertExecutors.foreach(exec => {
- val t = FlatMapProducer(UniqueId.incrementAndGetId(), exec).withName(exec.getAlertExecutorId() + "_" + exec.getPartitionSeq())
- t.setConfig(config)
- t.setGraph(dag)
- alertProducers += t
- if (strategy == null) {
- newStreamProducers.foreach(newsp => toBeAddedEdges += StreamConnector(newsp, t).groupBy(Seq(0)))
- }
- else {
- newStreamProducers.foreach(newsp => toBeAddedEdges += StreamConnector(newsp, t).customGroupBy(strategy))
- }
- })
-
- // remove AlertStreamSink
- toBeRemovedVertex += child
-
- // add alert consumer if necessary
- if (withConsumer) {
- AlertExecutorConsumerUtils.setupAlertConsumers(toBeAddedEdges, alertProducers.toList)
- }
- }
- case _ =>
- }
- }
-
- private def replace(toBeAddedEdges: ListBuffer[StreamConnector], toBeRemovedVertex: ListBuffer[StreamProducer],
- dag: DirectedAcyclicGraph[StreamProducer, StreamConnector], current: StreamProducer, upStreamName: String) : StreamProducer= {
- var newsp: StreamProducer = null
- current match {
- case _: FlatMapProducer[AnyRef, AnyRef] => {
- val mapper = current.asInstanceOf[FlatMapProducer[_, _]].mapper
- mapper match {
- case a: JavaStormStreamExecutor[EagleTuple] => {
- val newmapper = new JavaStormExecutorForAlertWrapper(a.asInstanceOf[JavaStormStreamExecutor[Tuple2[String, util.SortedMap[AnyRef, AnyRef]]]], upStreamName)
- newsp = FlatMapProducer(UniqueId.incrementAndGetId(), newmapper)
- newsp.setGraph(dag)
- newsp.setConfig(config)
- }
- case b: StormStreamExecutor[EagleTuple] => {
- val newmapper = StormExecutorForAlertWrapper(b.asInstanceOf[StormStreamExecutor[Tuple2[String, util.SortedMap[AnyRef, AnyRef]]]], upStreamName)
- newsp = FlatMapProducer(UniqueId.incrementAndGetId(), newmapper)
- newsp.setGraph(dag)
- newsp.setConfig(config)
- }
- case _ => throw new IllegalArgumentException
- }
- // remove old StreamProducer and replace that with new StreamProducer
- val incomingEdges = dag.incomingEdgesOf(current)
- incomingEdges.foreach(e => toBeAddedEdges += StreamConnector(e.from, newsp))
- val outgoingEdges = dag.outgoingEdgesOf(current)
- outgoingEdges.foreach(e => toBeAddedEdges += StreamConnector(newsp, e.to))
- toBeRemovedVertex += current
- }
- case _: MapProducer => {
- val mapper = current.asInstanceOf[MapProducer].fn
- val newfun: (AnyRef => AnyRef) = {
- a => mapper(a) match {
- case scala.Tuple2(x1, x2) => (x1, upStreamName, x2)
- case _ => throw new IllegalArgumentException
- }
- }
- current match {
- case MapProducer(id, 2, fn) => newsp = MapProducer(UniqueId.incrementAndGetId(), 3, newfun)
- case _ => throw new IllegalArgumentException
- }
- val incomingEdges = dag.incomingEdgesOf(current)
- incomingEdges.foreach(e => toBeAddedEdges += StreamConnector(e.from, newsp))
- val outgoingEdges = dag.outgoingEdgesOf(current)
- outgoingEdges.foreach(e => toBeAddedEdges += StreamConnector(newsp, e.to))
- toBeRemovedVertex += current
- }
- case s: StreamProducer if dag.inDegreeOf(s) == 0 => {
- val fn:(AnyRef => AnyRef) = {
- n => {
- n match {
- case scala.Tuple3 => n
- case scala.Tuple2(x1,x2) => (x1,upStreamName,x2)
- case scala.Tuple1(x1) => (if(x1 == null) null else x1.hashCode(),upStreamName,x1)
- case _ => (if(n == null) null else n.hashCode(),upStreamName,n)
- }
- }
- }
- newsp = MapProducer(UniqueId.incrementAndGetId(),3,fn)
- toBeAddedEdges += StreamConnector(current,newsp)
- val outgoingEdges = dag.outgoingEdgesOf(current)
- outgoingEdges.foreach(e => toBeAddedEdges += StreamConnector(newsp,e.to))
- }
- case _ => throw new IllegalArgumentException("Only FlatMapProducer and MapProducer can be replaced before AlertStreamSink")
- }
- newsp
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StreamAppDSL.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StreamAppDSL.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StreamAppDSL.scala
deleted file mode 100644
index c6a3d3d..0000000
--- a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StreamAppDSL.scala
+++ /dev/null
@@ -1,120 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.datastream
-
-import backtype.storm.topology.base.BaseRichSpout
-import com.typesafe.config._
-import org.apache.eagle.dataproc.impl.storm.AbstractStormSpoutProvider
-import org.apache.eagle.dataproc.util.ConfigOptionParser
-
-import scala.reflect.runtime.universe._
-
-/**
- * @since 11/6/15
- */
-trait ConfigContext{
- def set(config:Config)
- def config:Config
-
- def set[T<:AnyRef](key:String,value:T): Unit = {
- set(config.withValue(key,ConfigValueFactory.fromAnyRef(value)))
- }
-
- /**
- *
- * @param key config key
- * @param default default value
- * @tparam T return type
- * @return
- */
- def get[T](key:String,default:T=null)(implicit tag:TypeTag[T]):T = {
- if(config.hasPath(key)) {
- get(key)
- } else default
- }
-
- def get[T](key:String)(implicit tag: TypeTag[T]):T = tag.tpe match {
- case STRING_TYPE => config.getString(key).asInstanceOf[T]
- case TypeTag.Double => config.getDouble(key).asInstanceOf[T]
- case TypeTag.Long => config.getLong(key).asInstanceOf[T]
- case TypeTag.Int => config.getInt(key).asInstanceOf[T]
- case TypeTag.Byte => config.getBytes(key).asInstanceOf[T]
- case TypeTag.Boolean => config.getBoolean(key).asInstanceOf[T]
- case NUMBER_TYPE => config.getNumber(key).asInstanceOf[T]
- case OBJECT_TYPE => config.getObject(key).asInstanceOf[T]
- case VALUE_TYPE => config.getValue(key).asInstanceOf[T]
- case ANY_REF_TYPE => config.getAnyRef(key).asInstanceOf[T]
- case INT_LIST_TYPE => config.getIntList(key).asInstanceOf[T]
- case DOUBLE_LIST_TYPE => config.getDoubleList(key).asInstanceOf[T]
- case BOOL_LIST_TYPE => config.getBooleanList(key).asInstanceOf[T]
- case LONG_LIST_TYPE => config.getLongList(key).asInstanceOf[T]
- case _ => throw new UnsupportedOperationException(s"$tag is not supported yet")
- }
-
- val STRING_TYPE = typeOf[String]
- val NUMBER_TYPE = typeOf[Number]
- val INT_LIST_TYPE = typeOf[List[Int]]
- val BOOL_LIST_TYPE = typeOf[List[Boolean]]
- val DOUBLE_LIST_TYPE = typeOf[List[Double]]
- val LONG_LIST_TYPE = typeOf[List[Double]]
- val OBJECT_TYPE = typeOf[ConfigObject]
- val VALUE_TYPE = typeOf[ConfigValue]
- val ANY_REF_TYPE = typeOf[AnyRef]
-}
-
-/**
- * Stream APP DSL
- * @tparam E
- */
-trait StreamApp[+E<:ExecutionEnvironment] extends App with ConfigContext{
- private var _executed = false
- private var _config:Config = null
-
- override def config:Config = _config
-
- override def set(config:Config) = _config = config
-
- def env:E
-
- def execute() {
- env.execute()
- _executed = true
- }
-
- override def main(args: Array[String]): Unit = {
- _config = new ConfigOptionParser().load(args)
- super.main(args)
- if(!_executed) execute()
- }
-}
-
-trait StormStreamApp extends StreamApp[StormExecutionEnvironment]{
- private var _env:StormExecutionEnvironment = null
- def source(sourceProvider: AbstractStormSpoutProvider) = {
- val spout = sourceProvider.getSpout(config)
- env.newSource(spout)
- }
-
- def source(spout:BaseRichSpout) = env.newSource(spout)
-
- override def env:StormExecutionEnvironment = {
- if(_env == null){
- _env = ExecutionEnvironmentFactory.getStorm(config)
- }
- _env
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StreamConnector.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StreamConnector.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StreamConnector.scala
deleted file mode 100644
index 083a5af..0000000
--- a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StreamConnector.scala
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. See the NOTICE file distributed with
- * * this work for additional information regarding copyright ownership.
- * * The ASF licenses this file to You under the Apache License, Version 2.0
- * * (the "License"); you may not use this file except in compliance with
- * * the License. You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License.
- *
- */
-package org.apache.eagle.datastream
-
-import org.apache.eagle.partition.PartitionStrategy
-
-case class StreamConnector(from: StreamProducer, to: StreamProducer) {
- var groupByFields : Seq[Int] = Nil
- var customGroupBy : PartitionStrategy = null
-
- def groupBy(fields : Seq[Int]) : StreamConnector = {
- groupByFields = fields
- this
- }
-
- def customGroupBy(custom : PartitionStrategy) : StreamConnector = {
- customGroupBy = custom
- this
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StreamDAGExpansion.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StreamDAGExpansion.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StreamDAGExpansion.scala
deleted file mode 100644
index 7e15233..0000000
--- a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StreamDAGExpansion.scala
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. See the NOTICE file distributed with
- * * this work for additional information regarding copyright ownership.
- * * The ASF licenses this file to You under the Apache License, Version 2.0
- * * (the "License"); you may not use this file except in compliance with
- * * the License. You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License.
- *
- */
-
-package org.apache.eagle.datastream
-
-import com.typesafe.config.Config
-import org.jgrapht.experimental.dag.DirectedAcyclicGraph
-
-abstract class StreamDAGExpansion(config: Config) {
- def expand(dag: DirectedAcyclicGraph[StreamProducer, StreamConnector])
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StreamGroupbyExpansion.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StreamGroupbyExpansion.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StreamGroupbyExpansion.scala
deleted file mode 100644
index caf71e3..0000000
--- a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StreamGroupbyExpansion.scala
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. See the NOTICE file distributed with
- * * this work for additional information regarding copyright ownership.
- * * The ASF licenses this file to You under the Apache License, Version 2.0
- * * (the "License"); you may not use this file except in compliance with
- * * the License. You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License.
- *
- */
-
-package org.apache.eagle.datastream
-
-import com.typesafe.config.Config
-import org.jgrapht.experimental.dag.DirectedAcyclicGraph
-import org.slf4j.LoggerFactory
-import scala.collection.JavaConversions._
-import scala.collection.mutable.ListBuffer
-
-/**
- * Replace GroupByProducer(Vertex) with StreamConnector (Edge)
- * @param config context configuration
- */
-class StreamGroupbyExpansion(config: Config) extends StreamDAGExpansion(config){
- val LOG = LoggerFactory.getLogger(classOf[StreamGroupbyExpansion])
-
- override def expand(dag: DirectedAcyclicGraph[StreamProducer, StreamConnector]) = {
- val iter = dag.iterator()
- var toBeAddedEdges = new ListBuffer[StreamConnector]
- var toBeRemovedVertex = new ListBuffer[StreamProducer]
- while(iter.hasNext) {
- val current = iter.next()
- dag.outgoingEdgesOf(current).foreach(edge => {
- val child = edge.to
- child match {
- case p : GroupByProducer => {
- dag.outgoingEdgesOf(p).foreach(c2 => {
- if (p.fields != Nil) {
- toBeAddedEdges += StreamConnector(current, c2.to).groupBy(p.fields)
- }
- else if (p.partitionStrategy != null) {
- toBeAddedEdges += StreamConnector(current, c2.to).customGroupBy(p.partitionStrategy)
- }
- else toBeAddedEdges += StreamConnector(current, c2.to);
- })
- toBeRemovedVertex += p
- }
- case _ =>
- }
- })
- }
-
- // add back edges
- toBeAddedEdges.foreach(e => dag.addEdge(e.from, e.to, e))
- toBeRemovedVertex.foreach(v => dag.removeVertex(v))
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StreamNameExpansion.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StreamNameExpansion.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StreamNameExpansion.scala
deleted file mode 100644
index 5208a97..0000000
--- a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StreamNameExpansion.scala
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. See the NOTICE file distributed with
- * * this work for additional information regarding copyright ownership.
- * * The ASF licenses this file to You under the Apache License, Version 2.0
- * * (the "License"); you may not use this file except in compliance with
- * * the License. You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License.
- *
- */
-
-package org.apache.eagle.datastream
-
-import com.typesafe.config.Config
-import org.jgrapht.experimental.dag.DirectedAcyclicGraph
-import org.slf4j.LoggerFactory
-
-/**
- * to set name for each StreamProducer
- * 1. if name is given programatically, then use this name
- * 2. otherwise use name generated by scala internally
- */
-class StreamNameExpansion(config: Config) extends StreamDAGExpansion(config){
- val LOG = LoggerFactory.getLogger(classOf[StreamNameExpansion])
-
- override def expand(dag: DirectedAcyclicGraph[StreamProducer, StreamConnector]) = {
- val iter = dag.iterator()
- while(iter.hasNext){
- val sp = iter.next()
- sp.name = NodeNameSelector(sp).getName
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StreamParallelismConfigExpansion.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StreamParallelismConfigExpansion.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StreamParallelismConfigExpansion.scala
deleted file mode 100644
index 0264ca6..0000000
--- a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StreamParallelismConfigExpansion.scala
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. See the NOTICE file distributed with
- * * this work for additional information regarding copyright ownership.
- * * The ASF licenses this file to You under the Apache License, Version 2.0
- * * (the "License"); you may not use this file except in compliance with
- * * the License. You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License.
- *
- */
-
-package org.apache.eagle.datastream
-
-import java.util.regex.Pattern
-
-import com.typesafe.config.{ConfigValue, ConfigObject, Config}
-import org.jgrapht.experimental.dag.DirectedAcyclicGraph
-import org.slf4j.LoggerFactory
-import scala.collection.JavaConverters._
-
-class StreamParallelismConfigExpansion(config: Config) extends StreamDAGExpansion(config){
- val LOG = LoggerFactory.getLogger(classOf[StreamParallelismConfigExpansion])
-
- override def expand(dag: DirectedAcyclicGraph[StreamProducer, StreamConnector]) = {
- val map = getParallelismMap(config)
- val iter = dag.iterator()
- while(iter.hasNext){
- val streamProducer = iter.next()
- if(streamProducer.name != null) {
- map.foreach(tuple => {
- tuple._1.matcher(streamProducer.name).find() match {
- case true => streamProducer.parallelism = tuple._2
- case false =>
- }
- })
- }
- }
- }
-
- private def getParallelismMap(config: Config) : Map[Pattern, Int]= {
- val parallelismConfig: ConfigObject = config.getObject("envContextConfig.parallelismConfig")
- LOG.info("Found parallelismConfig ? " + (if (parallelismConfig == null) "no" else "yes"))
- parallelismConfig.asScala.toMap map {
- case (name, value) => (Pattern.compile(name), value.asInstanceOf[ConfigValue].unwrapped().asInstanceOf[Int])
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StreamProducer.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StreamProducer.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StreamProducer.scala
deleted file mode 100644
index 796e486..0000000
--- a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StreamProducer.scala
+++ /dev/null
@@ -1,221 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. See the NOTICE file distributed with
- * * this work for additional information regarding copyright ownership.
- * * The ASF licenses this file to You under the Apache License, Version 2.0
- * * (the "License"); you may not use this file except in compliance with
- * * the License. You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License.
- *
- */
-package org.apache.eagle.datastream
-
-import java.util
-import java.util.concurrent.atomic.AtomicInteger
-
-import backtype.storm.topology.base.BaseRichSpout
-import com.typesafe.config.Config
-import org.apache.eagle.partition.PartitionStrategy
-import org.jgrapht.experimental.dag.DirectedAcyclicGraph
-import scala.collection.JavaConversions._
-import scala.collection.JavaConverters._
-
-/**
- * StreamProducer is the base class for all other concrete StreamProducer
- * It defines high level API for user to organize data stream flow
- *
- * StreamProducer is independent of execution environment
- */
-
-trait StreamProducer{
- var name: String = null
- var parallelism: Int = 1
- var graph: DirectedAcyclicGraph[StreamProducer, StreamConnector] = null
- var config: Config = null
-
- private def incrementAndGetId() = UniqueId.incrementAndGetId()
-
- def setGraph(graph: DirectedAcyclicGraph[StreamProducer, StreamConnector]): Unit = this.graph = graph
- def getGraph: DirectedAcyclicGraph[StreamProducer, StreamConnector] = graph
- def setConfig(config: Config) : Unit = this.config = config
- def getConfig: Config = config
-
- def filter(fn : AnyRef => Boolean): StreamProducer ={
- val ret = FilterProducer(incrementAndGetId(), fn)
- hookupDAG(graph, this, ret)
- ret
- }
-
- def flatMap[T, R](mapper : FlatMapper[T, R]) : StreamProducer = {
- val ret = FlatMapProducer(incrementAndGetId(), mapper)
- hookupDAG(graph, this, ret)
- ret
- }
-
- def map1(fn : AnyRef => AnyRef) : StreamProducer = {
- val ret = MapProducer(incrementAndGetId(), 1, fn)
- hookupDAG(graph, this, ret)
- ret
- }
-
- def map2(fn : AnyRef => AnyRef) : StreamProducer = {
- val ret = MapProducer(incrementAndGetId(), 2, fn)
- hookupDAG(graph, this, ret)
- ret
- }
-
- def map3(fn : AnyRef => AnyRef) : StreamProducer = {
- val ret = MapProducer(incrementAndGetId(), 3, fn)
- hookupDAG(graph, this, ret)
- ret
- }
-
- def map4(fn : AnyRef => AnyRef) : StreamProducer = {
- val ret = MapProducer(incrementAndGetId(), 4, fn)
- hookupDAG(graph, this, ret)
- ret
- }
-
- /**
- * starting from 0, groupby operator would be upon edge of the graph
- */
- def groupBy(fields : Int*) : StreamProducer = {
- // validate each field index is greater or equal to 0
- fields.foreach(n => if(n<0) throw new IllegalArgumentException("field index should be always >= 0"))
- val ret = GroupByProducer(incrementAndGetId(), fields)
- hookupDAG(graph, this, ret)
- ret
- }
-
- //groupBy java version, starting from 1
- def groupBy(fields : java.util.List[Integer]) : StreamProducer = {
- // validate each field index is greater or equal to 0
- fields.foreach(n => if(n<0) throw new IllegalArgumentException("field index should be always >= 0"))
- val ret = GroupByProducer(incrementAndGetId(), fields.asScala.toSeq.asInstanceOf[Seq[Int]])
- hookupDAG(graph, this, ret)
- ret
- }
-
- def customGroupBy(strategy : PartitionStrategy) : StreamProducer = {
- val ret = GroupByProducer(incrementAndGetId(), strategy)
- hookupDAG(graph, this, ret)
- ret
- }
-
- def streamUnion(others : util.List[StreamProducer]) : StreamProducer = {
- streamUnion(others);
- }
-
- def streamUnion(others : Seq[StreamProducer]) : StreamProducer = {
- val ret = StreamUnionProducer(incrementAndGetId(), others)
- hookupDAG(graph, this, ret)
- ret
- }
-
- def streamUnion(other : StreamProducer) : StreamProducer = {
- streamUnion(Seq(other))
- }
-
- /**
- * alert is always sink of data flow
- */
- def alertWithConsumer(upStreamNames: util.List[String], alertExecutorId : String) = {
- alert(upStreamNames, alertExecutorId, true)
- }
-
- def alertWithoutConsumer(upStreamNames: util.List[String], alertExecutorId : String) = {
- alert(upStreamNames, alertExecutorId, false)
- }
-
- def alert(upStreamNames: util.List[String], alertExecutorId : String, withConsumer: Boolean=true, strategy : PartitionStrategy=null ) = {
- val ret = AlertStreamSink(incrementAndGetId(), upStreamNames, alertExecutorId, withConsumer, strategy)
- hookupDAG(graph, this, ret)
- }
-
- def alertWithConsumer(upStreamName: String, alertExecutorId : String, strategy: PartitionStrategy): Unit ={
- alert(util.Arrays.asList(upStreamName), alertExecutorId, true, strategy)
- }
-
- def alertWithConsumer(upStreamName: String, alertExecutorId : String): Unit ={
- alert(util.Arrays.asList(upStreamName), alertExecutorId, true)
- }
-
- def alertWithoutConsumer(upStreamName: String, alertExecutorId : String, strategy: PartitionStrategy): Unit ={
- alert(util.Arrays.asList(upStreamName), alertExecutorId, false, strategy)
- }
-
- def alertWithoutConsumer(upStreamName: String, alertExecutorId : String): Unit ={
- alert(util.Arrays.asList(upStreamName), alertExecutorId, false)
- }
-
- def hookupDAG(graph: DirectedAcyclicGraph[StreamProducer, StreamConnector], current: StreamProducer, next: StreamProducer) = {
- current.getGraph.addVertex(next)
- current.getGraph.addEdge(current, next, StreamConnector(current, next))
- passOnContext(current, next)
- }
-
- private def passOnContext(current: StreamProducer, next: StreamProducer): Unit ={
- next.graph = current.graph
- next.config = current.config
- }
-
- /**
- * can be set by programatically or by configuration
- */
- def withParallelism(parallelism : Int) : StreamProducer = {
- this.parallelism = parallelism
- this
- }
-
- def withName(name : String) : StreamProducer = {
- this.name = name
- this
- }
-}
-
-case class FilterProducer(id: Int, fn : AnyRef => Boolean) extends StreamProducer
-
-case class FlatMapProducer[T, R](id: Int, var mapper: FlatMapper[T, R]) extends StreamProducer {
- override def toString() = mapper.toString + "_" + id
-}
-
-case class MapProducer(id: Int, numOutputFields : Int, var fn : AnyRef => AnyRef) extends StreamProducer
-
-case class GroupByProducer(id: Int, fields : Seq[Int], partitionStrategy: PartitionStrategy) extends StreamProducer
-
-object GroupByProducer {
- def apply(id: Int, fields: Seq[Int]) = new GroupByProducer(id, fields, null)
- def apply(id: Int, partitionStrategy : PartitionStrategy) = new GroupByProducer(id, Nil, partitionStrategy)
-}
-
-case class StreamUnionProducer(id: Int, others: Seq[StreamProducer]) extends StreamProducer
-
-case class StormSourceProducer(id: Int, source : BaseRichSpout) extends StreamProducer{
- var numFields : Int = 0
- /**
- * rename outputfields to f0, f1, f2, ...
- * if one spout declare some field names, those fields names will be modified
- * @param n
- */
- def renameOutputFields(n : Int): StormSourceProducer ={
- this.numFields = n
- this
- }
-}
-
-case class AlertStreamSink(id: Int, upStreamNames: util.List[String], alertExecutorId : String, withConsumer: Boolean=true, strategy: PartitionStrategy=null) extends StreamProducer
-
-object UniqueId{
- val id : AtomicInteger = new AtomicInteger(0);
- def incrementAndGetId() : Int = {
- id.incrementAndGet()
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StreamUnionExpansion.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StreamUnionExpansion.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StreamUnionExpansion.scala
deleted file mode 100644
index 83a83fe..0000000
--- a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StreamUnionExpansion.scala
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. See the NOTICE file distributed with
- * * this work for additional information regarding copyright ownership.
- * * The ASF licenses this file to You under the Apache License, Version 2.0
- * * (the "License"); you may not use this file except in compliance with
- * * the License. You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License.
- *
- */
-
-package org.apache.eagle.datastream
-
-import com.typesafe.config.Config
-import org.jgrapht.experimental.dag.DirectedAcyclicGraph
-import org.slf4j.LoggerFactory
-import scala.collection.JavaConversions._
-import scala.collection.mutable.ListBuffer
-
-/**
- * union operator should be expanded
- */
-class StreamUnionExpansion(config: Config) extends StreamDAGExpansion(config){
- val LOG = LoggerFactory.getLogger(classOf[StreamUnionExpansion])
-
- override def expand(dag: DirectedAcyclicGraph[StreamProducer, StreamConnector]) = {
- val iter = dag.iterator()
- var toBeAddedEdges = new ListBuffer[StreamConnector]
- var toBeRemovedVertex = new ListBuffer[StreamProducer]
- while(iter.hasNext) {
- val current = iter.next()
- dag.outgoingEdgesOf(current).foreach(edge => {
- val child = edge.to
- val groupByFields = edge.groupByFields;
- child match {
- case StreamUnionProducer(id, others) => {
- dag.outgoingEdgesOf(child).foreach(c2 => {
- toBeAddedEdges += StreamConnector(current, c2.to).groupBy(groupByFields)
- others.foreach(o => {
- toBeAddedEdges += StreamConnector(o, c2.to).groupBy(groupByFields)
- })
- })
- toBeRemovedVertex += child
- }
- case _ =>
- }
- })
- }
-
- // add back edges
- toBeAddedEdges.foreach(e => dag.addEdge(e.from, e.to, e))
- toBeRemovedVertex.foreach(v => dag.removeVertex(v))
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/UnionUtils.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/UnionUtils.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/UnionUtils.scala
deleted file mode 100644
index 02e211b..0000000
--- a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/UnionUtils.scala
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. See the NOTICE file distributed with
- * * this work for additional information regarding copyright ownership.
- * * The ASF licenses this file to You under the Apache License, Version 2.0
- * * (the "License"); you may not use this file except in compliance with
- * * the License. You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License.
- *
- */
-package org.apache.eagle.datastream
-
-import java.util
-
-import scala.collection.JavaConverters._
-
-object UnionUtils {
- def join(producers : StreamProducer*) : StreamProducer = {
- producers.head.streamUnion(producers.drop(1))
- }
-
- def join(producers : java.util.List[StreamProducer]) : StreamProducer = {
- val newList = new util.ArrayList(producers)
- val head = newList.get(0)
- newList.remove(0)
- head.streamUnion(newList.asScala);
- }
-
- def join(producers : List[StreamProducer]) : StreamProducer = {
- val head = producers.head
- head.streamUnion(producers.tail);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/AbstractTopologyCompiler.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/AbstractTopologyCompiler.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/AbstractTopologyCompiler.scala
new file mode 100644
index 0000000..46f4738
--- /dev/null
+++ b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/AbstractTopologyCompiler.scala
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.datastream.core
+
+trait AbstractTopologyCompiler{
+ def buildTopology : AbstractTopologyExecutor
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/AbstractTopologyExecutor.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/AbstractTopologyExecutor.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/AbstractTopologyExecutor.scala
new file mode 100644
index 0000000..1e1664a
--- /dev/null
+++ b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/AbstractTopologyExecutor.scala
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.datastream.core
+
+trait AbstractTopologyExecutor {
+ def execute
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/Configuration.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/Configuration.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/Configuration.scala
new file mode 100644
index 0000000..e3f3050
--- /dev/null
+++ b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/Configuration.scala
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eagle.datastream.core
+
+import com.typesafe.config.{Config, _}
+
+import scala.reflect.runtime.universe._
+
+/**
+ * @since 12/4/15
+ */
+case class Configuration(private var config:Config) extends Serializable{
+ def get:Config = config
+
+ def set[T<:AnyRef](key:String,value:T): Unit = {
+ config = config.withValue(key,ConfigValueFactory.fromAnyRef(value))
+ }
+
+ /**
+ *
+ * @param key config key
+ * @param default default value
+ * @tparam T return type
+ * @return
+ */
+ def get[T](key:String,default:T=null)(implicit tag:TypeTag[T]):T = {
+ if(get.hasPath(key)) {
+ get(key)
+ } else default
+ }
+
+ def get[T](key:String)(implicit tag: TypeTag[T]):T = tag.tpe match {
+ case STRING_TYPE => config.getString(key).asInstanceOf[T]
+ case TypeTag.Double => get.getDouble(key).asInstanceOf[T]
+ case TypeTag.Long => get.getLong(key).asInstanceOf[T]
+ case TypeTag.Int => get.getInt(key).asInstanceOf[T]
+ case TypeTag.Byte => get.getBytes(key).asInstanceOf[T]
+ case TypeTag.Boolean => get.getBoolean(key).asInstanceOf[T]
+ case NUMBER_TYPE => get.getNumber(key).asInstanceOf[T]
+ case OBJECT_TYPE => get.getObject(key).asInstanceOf[T]
+ case VALUE_TYPE => get.getValue(key).asInstanceOf[T]
+ case ANY_REF_TYPE => get.getAnyRef(key).asInstanceOf[T]
+ case INT_LIST_TYPE => get.getIntList(key).asInstanceOf[T]
+ case DOUBLE_LIST_TYPE => get.getDoubleList(key).asInstanceOf[T]
+ case BOOL_LIST_TYPE => get.getBooleanList(key).asInstanceOf[T]
+ case LONG_LIST_TYPE => get.getLongList(key).asInstanceOf[T]
+ case _ => throw new UnsupportedOperationException(s"$tag is not supported yet")
+ }
+
+ val STRING_TYPE = typeOf[String]
+ val NUMBER_TYPE = typeOf[Number]
+ val INT_LIST_TYPE = typeOf[List[Int]]
+ val BOOL_LIST_TYPE = typeOf[List[Boolean]]
+ val DOUBLE_LIST_TYPE = typeOf[List[Double]]
+ val LONG_LIST_TYPE = typeOf[List[Double]]
+ val OBJECT_TYPE = typeOf[ConfigObject]
+ val VALUE_TYPE = typeOf[ConfigValue]
+ val ANY_REF_TYPE = typeOf[AnyRef]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/ExecutionEnvironment.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/ExecutionEnvironment.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/ExecutionEnvironment.scala
new file mode 100644
index 0000000..7394b75
--- /dev/null
+++ b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/ExecutionEnvironment.scala
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eagle.datastream.core
+
+import com.typesafe.config.Config
+import org.apache.eagle.datastream.utils.GraphPrinter
+import org.jgrapht.experimental.dag.DirectedAcyclicGraph
+
+/**
+ * @since 0.3.0
+ */
+trait ExecutionEnvironment {
+ def config:Configuration
+
+ /**
+ * Business logic DAG
+ * @return
+ */
+ def dag:DirectedAcyclicGraph[StreamProducer[Any], StreamConnector[Any,Any]]
+
+ /**
+ * Start to execute
+ */
+ def execute():Unit
+
+ /**
+ * Support Java Style Config
+ *
+ * @return
+ */
+ def getConfig:Config = config.get
+}
+
+/**
+ * @todo Use Configuration instead of Config
+ *
+ * @param conf
+ */
+abstract class ExecutionEnvironmentBase(private val conf:Config) extends ExecutionEnvironment with StreamSourceBuilder {
+ implicit private val _dag = new DirectedAcyclicGraph[StreamProducer[Any], StreamConnector[Any,Any]](classOf[StreamConnector[Any,Any]])
+ private val _config:Configuration = Configuration(conf)
+
+ override def dag = _dag
+ override def config = _config
+
+ override def execute(): Unit = {
+ implicit val i_conf = _config.get
+ StreamNameExpansion()
+ GraphPrinter.print(dag,message="Before expanded DAG ")
+ StreamAlertExpansion()
+ StreamUnionExpansion()
+ StreamGroupbyExpansion()
+ StreamParallelismConfigExpansion()
+ StreamNameExpansion()
+ GraphPrinter.print(dag,message="After expanded DAG ")
+
+ GraphPrinter.printDotDigraph(dag)
+
+ val streamDAG = StreamDAGTransformer.transform(dag)
+ execute(streamDAG)
+ }
+
+ protected def execute(dag: StreamDAG)
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/StreamAlertExpansion.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/StreamAlertExpansion.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/StreamAlertExpansion.scala
new file mode 100644
index 0000000..84532dc
--- /dev/null
+++ b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/StreamAlertExpansion.scala
@@ -0,0 +1,203 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eagle.datastream.core
+
+import java.util
+
+import com.typesafe.config.Config
+import org.apache.eagle.alert.dao.AlertDefinitionDAOImpl
+import org.apache.eagle.datastream._
+import org.apache.eagle.datastream.storm.StormExecutorForAlertWrapper
+import org.apache.eagle.datastream.utils.AlertExecutorConsumerUtils
+import org.apache.eagle.executor.AlertExecutorCreationUtils
+import org.apache.eagle.service.client.EagleServiceConnector
+import org.jgrapht.experimental.dag.DirectedAcyclicGraph
+import org.slf4j.LoggerFactory
+
+import scala.collection.JavaConversions._
+import scala.collection.mutable.ListBuffer
+
+/**
+ * The constraints for alert is:
+ * 1. only 3 StreamProducers can be put immediately before MapProducer, FlatMapProducer, StreamUnionProducer
+ * 2. For StreamUnionProducer, the only supported unioned producers are MapProducer and FlatMapProducer
+ * 3. the output for MapProducer and FlatMapProducer is 2-field tuple, key and value, key is string, value has to be SortedMap
+ * 4. the framework will wrapper original MapProducer and FlatMapProducer to emit 3-field tuple, {key, streamName and value}
+ * 5. the framework will automatically partition traffic with first field
+ *
+ *
+ * 2 steps
+ * step 1: wrapper previous StreamProducer with one more field "streamName"
+ * step 2: partition alert executor by policy partitioner class
+ */
+
+case class StreamAlertExpansion(config: Config) extends StreamDAGExpansion(config) {
+ val LOG = LoggerFactory.getLogger(classOf[StreamAlertExpansion])
+
+ override def expand(dag: DirectedAcyclicGraph[StreamProducer[Any], StreamConnector[Any,Any]]): Unit ={
+ val iter = dag.iterator()
+ val toBeAddedEdges = new ListBuffer[StreamConnector[Any,Any]]
+ val toBeRemovedVertex = new ListBuffer[StreamProducer[Any]]
+ while(iter.hasNext) {
+ val current = iter.next()
+ dag.outgoingEdgesOf(current).foreach(edge => {
+ val child = edge.to
+ onIteration(toBeAddedEdges, toBeRemovedVertex, dag, current, child)
+ })
+ }
+ // add back edges
+ toBeAddedEdges.foreach(e => {
+ dag.addVertex(e.from)
+ dag.addVertex(e.to)
+ dag.addEdge(e.from, e.to, e)
+ })
+ toBeRemovedVertex.foreach(v => dag.removeVertex(v))
+ }
+
+ def onIteration(toBeAddedEdges: ListBuffer[StreamConnector[Any,Any]], toBeRemovedVertex: ListBuffer[StreamProducer[Any]],
+ dag: DirectedAcyclicGraph[StreamProducer[Any], StreamConnector[Any,Any]], current: StreamProducer[Any], child: StreamProducer[Any]): Unit = {
+ child match {
+ case AlertStreamSink(upStreamNames, alertExecutorId, withConsumer,strategy) => {
+ /**
+ * step 1: wrapper previous StreamProducer with one more field "streamName"
+ * for AlertStreamSink, we check previous StreamProducer and replace that
+ */
+ val newStreamProducers = new ListBuffer[StreamProducer[Any]]
+ current match {
+ case StreamUnionProducer(others) => {
+ val incomingEdges = dag.incomingEdgesOf(current)
+ incomingEdges.foreach(e => newStreamProducers += replace(toBeAddedEdges, toBeRemovedVertex, dag, e.from, upStreamNames.get(0)))
+ var i: Int = 1
+ others.foreach(o => {
+ newStreamProducers += replace(toBeAddedEdges, toBeRemovedVertex, dag, o, upStreamNames.get(i))
+ i += 1
+ })
+ }
+ case _: FlatMapProducer[AnyRef, AnyRef] => {
+ newStreamProducers += replace(toBeAddedEdges, toBeRemovedVertex, dag, current, upStreamNames.get(0))
+ }
+ case _: MapperProducer[AnyRef,AnyRef] => {
+ newStreamProducers += replace(toBeAddedEdges, toBeRemovedVertex, dag, current, upStreamNames.get(0))
+ }
+ case s: StreamProducer[AnyRef] if dag.inDegreeOf(s) == 0 => {
+ newStreamProducers += replace(toBeAddedEdges, toBeRemovedVertex, dag, current, upStreamNames.get(0))
+ }
+ case p@_ => throw new IllegalStateException(s"$p can not be put before AlertStreamSink, only StreamUnionProducer,FlatMapProducer and MapProducer are supported")
+ }
+
+ /**
+ * step 2: partition alert executor by policy partitioner class
+ */
+ val alertExecutors = AlertExecutorCreationUtils.createAlertExecutors(config, new AlertDefinitionDAOImpl(new EagleServiceConnector(config)), upStreamNames, alertExecutorId)
+ var alertProducers = new scala.collection.mutable.MutableList[StreamProducer[Any]]
+ alertExecutors.foreach(exec => {
+ val t = FlatMapProducer(exec).nameAs(exec.getAlertExecutorId + "_" + exec.getPartitionSeq).initWith(dag,config,hook = false)
+ alertProducers += t
+ newStreamProducers.foreach(newsp => toBeAddedEdges += StreamConnector[Any,Any](newsp, t,Seq(0)))
+ if (strategy == null) {
+ newStreamProducers.foreach(newsp => toBeAddedEdges += StreamConnector(newsp,t,Seq(0)))
+ }
+ else {
+ newStreamProducers.foreach(newsp => toBeAddedEdges += StreamConnector(newsp,t,strategy))
+ }
+ })
+
+ // remove AlertStreamSink
+ toBeRemovedVertex += child
+
+ // add alert consumer if necessary
+ if (withConsumer) {
+ AlertExecutorConsumerUtils.setupAlertConsumers(toBeAddedEdges, alertProducers.toList)
+ }
+ }
+ case _ =>
+ }
+ }
+
+ private def replace(toBeAddedEdges: ListBuffer[StreamConnector[Any,Any]], toBeRemovedVertex: ListBuffer[StreamProducer[Any]],
+ dag: DirectedAcyclicGraph[StreamProducer[Any], StreamConnector[Any,Any]], current: StreamProducer[Any], upStreamName: String) : StreamProducer[Any]= {
+ var newsp: StreamProducer[Any] = null
+ current match {
+ case _: FlatMapProducer[AnyRef, AnyRef] => {
+ val mapper = current.asInstanceOf[FlatMapProducer[_, _]].mapper
+ mapper match {
+ case a: JavaStormStreamExecutor[EagleTuple] => {
+ val newmapper = new JavaStormExecutorForAlertWrapper(a.asInstanceOf[JavaStormStreamExecutor[Tuple2[String, util.SortedMap[AnyRef, AnyRef]]]], upStreamName)
+ newsp = FlatMapProducer(newmapper).initWith(dag,config,hook = false)
+ }
+ case b: StormStreamExecutor[EagleTuple] => {
+ val newmapper = StormExecutorForAlertWrapper(b.asInstanceOf[StormStreamExecutor[Tuple2[String, util.SortedMap[AnyRef, AnyRef]]]], upStreamName)
+ newsp = FlatMapProducer(newmapper).initWith(dag,config,hook = false)
+ }
+ case _ => throw new IllegalArgumentException
+ }
+ // remove old StreamProducer and replace that with new StreamProducer
+ val incomingEdges = dag.incomingEdgesOf(current)
+ incomingEdges.foreach(e => toBeAddedEdges += StreamConnector(e.from, newsp))
+ val outgoingEdges = dag.outgoingEdgesOf(current)
+ outgoingEdges.foreach(e => toBeAddedEdges += StreamConnector(newsp, e.to))
+ toBeRemovedVertex += current
+ }
+ case _: MapperProducer[Any,Any] => {
+ val mapper = current.asInstanceOf[MapperProducer[Any,Any]].fn
+ val newfun: (Any => Any) = {
+ a => mapper(a) match {
+ case scala.Tuple2(x1, x2) => (x1, upStreamName, x2)
+ case _ => throw new IllegalArgumentException
+ }
+ }
+ current match {
+ case MapperProducer(2, fn) => newsp = MapperProducer(3, newfun)
+ case _ => throw new IllegalArgumentException
+ }
+ val incomingEdges = dag.incomingEdgesOf(current)
+ incomingEdges.foreach(e => toBeAddedEdges += StreamConnector(e.from, newsp))
+ val outgoingEdges = dag.outgoingEdgesOf(current)
+ outgoingEdges.foreach(e => toBeAddedEdges += StreamConnector(newsp, e.to))
+ toBeRemovedVertex += current
+ }
+ case s: StreamProducer[Any] if dag.inDegreeOf(s) == 0 => {
+ val fn:(AnyRef => AnyRef) = {
+ n => {
+ n match {
+ case scala.Tuple3 => n
+ case scala.Tuple2(x1,x2) => (x1,upStreamName,x2)
+ case scala.Tuple1(x1) => (if(x1 == null) null else x1.hashCode(),upStreamName,x1)
+ case _ => (if(n == null) null else n.hashCode(),upStreamName,n)
+ }
+ }
+ }
+ newsp = MapperProducer(3,fn)
+ toBeAddedEdges += StreamConnector(current,newsp)
+ val outgoingEdges = dag.outgoingEdgesOf(current)
+ outgoingEdges.foreach(e => toBeAddedEdges += StreamConnector(newsp,e.to))
+ }
+ case _ => throw new IllegalArgumentException("Only FlatMapProducer and MapProducer can be replaced before AlertStreamSink")
+ }
+ newsp
+ }
+}
+
+object StreamAlertExpansion{
+ def apply()(implicit config:Config, dag: DirectedAcyclicGraph[StreamProducer[Any], StreamConnector[Any,Any]]): StreamAlertExpansion ={
+ val e = StreamAlertExpansion(config)
+ e.expand(dag)
+ e
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/StreamConnector.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/StreamConnector.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/StreamConnector.scala
new file mode 100644
index 0000000..ce9d82c
--- /dev/null
+++ b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/StreamConnector.scala
@@ -0,0 +1,97 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+package org.apache.eagle.datastream.core
+
+import org.apache.eagle.partition.PartitionStrategy
+
+abstract class StreamConnector[+T1 <: Any,+T2 <: Any](val from: StreamProducer[T1], val to: StreamProducer[T2]) extends Serializable
+
+case class ShuffleConnector[+T1 <: Any,+T2 <: Any](override val from: StreamProducer[T1], override val to: StreamProducer[T2])
+ extends StreamConnector[T1,T2](from,to){
+ override def toString: String = "shuffleGroup"
+}
+
+case class GroupbyFieldsConnector[+T1 <: Any,+T2 <: Any](override val from: StreamProducer[T1], override val to: StreamProducer[T2],groupByFields : Seq[Int])
+ extends StreamConnector[T1,T2](from,to){
+ override def toString: String = s"groupByFields( $groupByFields )"
+}
+
+case class GroupbyKeyConnector[T1 <: Any,+T2 <: Any](override val from: StreamProducer[T1], override val to: StreamProducer[T2],keySelector: T1 => Any)
+ extends StreamConnector[T1,T2](from,to){
+ override def toString: String = s"groupByKey($keySelector)"
+}
+
+case class GroupbyStrategyConnector[+T1 <: Any,+T2 <: Any](override val from: StreamProducer[T1], override val to: StreamProducer[T2],customGroupBy:PartitionStrategy)
+ extends StreamConnector[T1,T2](from,to){
+ override def toString: String = s"groupByStrategy( $customGroupBy )"
+}
+
+object StreamConnector{
+ /**
+ *
+ * @param from
+ * @param to
+ * @tparam T1
+ * @tparam T2
+ * @return
+ */
+ def apply[T1 <: Any,T2 <: Any](from: StreamProducer[T1], to: StreamProducer[T2]):ShuffleConnector[T1,T2] = ShuffleConnector(from,to)
+
+ /**
+ * Clone connector from old connector to apply to new processing element, return ShuffleConnector by default
+ *
+ * @param from
+ * @param to
+ * @param connector
+ * @tparam T1
+ * @tparam T2
+ * @return
+ */
+ def apply[T1 <: Any,T2 <: Any](from: StreamProducer[T1], to: StreamProducer[T2],connector: StreamConnector[Any,Any]):StreamConnector[T1,T2] = connector match {
+ case GroupbyFieldsConnector(_,_,fields) => GroupbyFieldsConnector[T1,T2](from,to,fields)
+ case GroupbyKeyConnector(_,_,keySelector) => GroupbyKeyConnector[T1,T2](from,to,keySelector)
+ case GroupbyStrategyConnector(_,_,strategy) => GroupbyStrategyConnector[T1,T2](from,to,strategy)
+ case null | ShuffleConnector(_,_) => ShuffleConnector[T1,T2](from,to)
+ case c@_ => throw new IllegalArgumentException(s"Unknown type of stream connector $c")
+ }
+
+ /**
+ *
+ * @param from
+ * @param to
+ * @param groupByFields
+ * @tparam T1
+ * @tparam T2
+ * @return
+ */
+ def apply[T1 <: Any,T2 <: Any](from: StreamProducer[T1], to: StreamProducer[T2],groupByFields : Seq[Int]):GroupbyFieldsConnector[T1,T2] = GroupbyFieldsConnector(from,to,groupByFields)
+
+ /**
+ *
+ * @param from
+ * @param to
+ * @param customGroupBy
+ * @tparam T1
+ * @tparam T2
+ * @return
+ */
+ def apply[T1 <: Any,T2 <: Any](from: StreamProducer[T1], to: StreamProducer[T2],customGroupBy: PartitionStrategy):GroupbyStrategyConnector[T1,T2] = GroupbyStrategyConnector(from,to,customGroupBy)
+
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/StreamDAG.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/StreamDAG.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/StreamDAG.scala
new file mode 100644
index 0000000..255f031
--- /dev/null
+++ b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/StreamDAG.scala
@@ -0,0 +1,68 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+package org.apache.eagle.datastream.core
+
+import org.jgrapht.experimental.dag.DirectedAcyclicGraph
+
+import scala.collection.JavaConverters._
+import scala.collection.{JavaConversions, mutable}
+
+/**
+ * wrapper of DAG, used for storm topology compiler
+ */
+class StreamDAG(val graph: DirectedAcyclicGraph[StreamProducer[Any], StreamConnector[Any,Any]]) extends StreamProducerGraph {
+ var nodeMap: mutable.Map[String, StreamProducer[Any]] = null
+
+ override def addEdge(from: StreamProducer[Any], to: StreamProducer[Any], streamConnector: StreamConnector[Any,Any]): Unit = {
+ graph.addEdge(from, to, streamConnector)
+ }
+
+ override def addVertex(producer: StreamProducer[Any]): Unit = {
+ graph.addVertex(producer)
+ }
+
+ override def iterator(): Iterator[StreamProducer[Any]] = {
+ JavaConversions.asScalaIterator(graph.iterator())
+ }
+
+ override def isSource(v: StreamProducer[Any]): Boolean = {
+ graph.inDegreeOf(v) match {
+ case 0 => true
+ case _ => false
+ }
+ }
+
+ override def outgoingEdgesOf(v: StreamProducer[Any]): scala.collection.mutable.Set[StreamConnector[Any,Any]] = {
+ JavaConversions.asScalaSet(graph.outgoingEdgesOf(v))
+ }
+
+ override def getNodeByName(name: String): Option[StreamProducer[Any]] = {
+ nodeMap.get(name)
+ }
+
+ def setNodeMap(nodeMap: mutable.Map[String, StreamProducer[Any]]): Unit = {
+ this.nodeMap = nodeMap
+ }
+
+ override def incomingVertexOf(v: StreamProducer[Any]): scala.collection.mutable.Set[StreamProducer[Any]] = {
+ val set = mutable.Set[StreamProducer[Any]]()
+ graph.incomingEdgesOf(v).asScala.foreach(e => set += graph.getEdgeSource(e))
+ set
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/StreamDAGExpansion.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/StreamDAGExpansion.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/StreamDAGExpansion.scala
new file mode 100644
index 0000000..fcff639
--- /dev/null
+++ b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/StreamDAGExpansion.scala
@@ -0,0 +1,27 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.datastream.core
+
+import com.typesafe.config.Config
+import org.jgrapht.experimental.dag.DirectedAcyclicGraph
+
+abstract class StreamDAGExpansion(config: Config) {
+ def expand(dag: DirectedAcyclicGraph[StreamProducer[Any], StreamConnector[Any,Any]])
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/StreamDAGTransformer.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/StreamDAGTransformer.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/StreamDAGTransformer.scala
new file mode 100644
index 0000000..32947b9
--- /dev/null
+++ b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/StreamDAGTransformer.scala
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eagle.datastream.core
+
+import org.jgrapht.experimental.dag.DirectedAcyclicGraph
+
+import scala.collection.mutable
+
+/**
+ * convert generic DAG data structure to Storm specific DAG data structure for easy topology compiler
+ */
+object StreamDAGTransformer {
+ /**
+ * Transform DirectedAcyclicGraph[StreamProducer, StreamConnector] into StormStreamDAG
+ *
+ * @param dag DirectedAcyclicGraph[StreamProducer, StreamConnector]
+ * @return StormStreamDAG
+ */
+ def transform(dag: DirectedAcyclicGraph[StreamProducer[Any], StreamConnector[Any,Any]]) : StreamDAG = {
+ val stormDAG = new StreamDAG(dag)
+ val nodeMap = mutable.HashMap[String, StreamProducer[Any]]()
+ val iter = dag.iterator()
+ while(iter.hasNext){
+ val sp = iter.next()
+ nodeMap.put(sp.name, sp)
+ }
+ stormDAG.setNodeMap(nodeMap)
+ stormDAG
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/StreamGroupbyExpansion.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/StreamGroupbyExpansion.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/StreamGroupbyExpansion.scala
new file mode 100644
index 0000000..1a07e3f
--- /dev/null
+++ b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/StreamGroupbyExpansion.scala
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eagle.datastream.core
+
+import com.typesafe.config.Config
+import org.jgrapht.experimental.dag.DirectedAcyclicGraph
+
+import scala.collection.JavaConversions._
+import scala.collection.mutable.ListBuffer
+
+/**
+ * Replace GroupByProducer(Vertex) with StreamConnector (Edge)
+ *
+ * For example as to Storm, it's mainly for grouping method
+ *
+ * @param config context configuration
+ */
+case class StreamGroupbyExpansion(config: Config) extends StreamDAGExpansion(config){
+ override def expand(dag: DirectedAcyclicGraph[StreamProducer[Any], StreamConnector[Any,Any]]) = {
+ val iter = dag.iterator()
+ var toBeAddedEdges = new ListBuffer[StreamConnector[Any,Any]]
+ var toBeRemovedVertex = new ListBuffer[StreamProducer[Any]]
+ while(iter.hasNext) {
+ val current = iter.next()
+ dag.outgoingEdgesOf(current).foreach(edge => {
+ val child = edge.to
+ child match {
+ case p : GroupByProducer[Any] => {
+ dag.outgoingEdgesOf(p).foreach(c2 => {
+ p match {
+ case GroupByFieldProducer(fields) =>
+ toBeAddedEdges += GroupbyFieldsConnector(current, c2.to,fields)
+ case GroupByStrategyProducer(strategy) =>
+ toBeAddedEdges += GroupbyStrategyConnector(current, c2.to,strategy)
+ case GroupByKeyProducer(keySelector) =>
+ current.outKeyed = true
+ current.keySelector = KeySelectorWrapper(keySelector)
+ c2.to.inKeyed = true
+ toBeAddedEdges += GroupbyKeyConnector(current, c2.to,keySelector)
+ case _ => toBeAddedEdges += ShuffleConnector(current, c2.to)
+ }
+ })
+ toBeRemovedVertex += p
+ }
+ case _ =>
+ }
+ })
+ }
+
+ // add back edges
+ toBeAddedEdges.foreach(e => dag.addEdge(e.from, e.to, e))
+ toBeRemovedVertex.foreach(v => dag.removeVertex(v))
+ }
+}
+
+object StreamGroupbyExpansion{
+ def apply()(implicit config:Config, dag: DirectedAcyclicGraph[StreamProducer[Any], StreamConnector[Any,Any]]): StreamGroupbyExpansion ={
+ val e = StreamGroupbyExpansion(config)
+ e.expand(dag)
+ e
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/StreamNameExpansion.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/StreamNameExpansion.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/StreamNameExpansion.scala
new file mode 100644
index 0000000..4bc1812
--- /dev/null
+++ b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/StreamNameExpansion.scala
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eagle.datastream.core
+
+import com.typesafe.config.Config
+import org.apache.eagle.datastream.utils.NodeNameSelector
+import org.jgrapht.experimental.dag.DirectedAcyclicGraph
+import org.slf4j.LoggerFactory
+
+/**
+ * to set name for each StreamProducer
+ * 1. if name is given programatically, then use this name
+ * 2. otherwise use name generated by scala internally
+ */
+case class StreamNameExpansion(config: Config) extends StreamDAGExpansion(config){
+ val LOG = LoggerFactory.getLogger(classOf[StreamNameExpansion])
+
+ override def expand(dag: DirectedAcyclicGraph[StreamProducer[Any], StreamConnector[Any,Any]]) = {
+ val iter = dag.iterator()
+ while(iter.hasNext){
+ val sp = iter.next()
+ sp.name = NodeNameSelector(sp).getName
+ }
+ }
+}
+
+
+object StreamNameExpansion{
+ def apply()(implicit config:Config, dag: DirectedAcyclicGraph[StreamProducer[Any], StreamConnector[Any,Any]]): StreamNameExpansion ={
+ val e = StreamNameExpansion(config)
+ e.expand(dag)
+ e
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/StreamParallelismConfigExpansion.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/StreamParallelismConfigExpansion.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/StreamParallelismConfigExpansion.scala
new file mode 100644
index 0000000..e01ffa4
--- /dev/null
+++ b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/core/StreamParallelismConfigExpansion.scala
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eagle.datastream.core
+
+import java.util.regex.Pattern
+
+import com.typesafe.config.{Config, ConfigObject, ConfigValue}
+import org.jgrapht.experimental.dag.DirectedAcyclicGraph
+import org.slf4j.LoggerFactory
+
+import scala.collection.JavaConverters._
+
+case class StreamParallelismConfigExpansion(config: Config) extends StreamDAGExpansion(config){
+ val LOG = LoggerFactory.getLogger(classOf[StreamParallelismConfigExpansion])
+
+ override def expand(dag: DirectedAcyclicGraph[StreamProducer[Any], StreamConnector[Any,Any]]) = {
+ val map = getParallelismMap(config)
+ val iter = dag.iterator()
+ while(iter.hasNext){
+ val streamProducer = iter.next()
+ if(streamProducer.name != null) {
+ map.foreach(tuple => {
+ tuple._1.matcher(streamProducer.name).find() match {
+ case true => streamProducer.parallelism(tuple._2)
+ case false =>
+ }
+ })
+ }
+ }
+ }
+
+ private def getParallelismMap(config: Config) : Map[Pattern, Int]= {
+ val parallelismConfig: ConfigObject = config.getObject("envContextConfig.parallelismConfig")
+ parallelismConfig.asScala.toMap map {
+ case (name, value) => (Pattern.compile(name), value.asInstanceOf[ConfigValue].unwrapped().asInstanceOf[Int])
+ }
+ }
+}
+
+object StreamParallelismConfigExpansion{
+ def apply()(implicit config:Config, dag: DirectedAcyclicGraph[StreamProducer[Any], StreamConnector[Any,Any]]): StreamParallelismConfigExpansion ={
+ val e = StreamParallelismConfigExpansion(config)
+ e.expand(dag)
+ e
+ }
+}
\ No newline at end of file
[5/5] incubator-eagle git commit: EAGLE-66 Typesafe Streaming DSL and
KeyValue based Grouping
Posted by ha...@apache.org.
EAGLE-66 Typesafe Streaming DSL and KeyValue based Grouping
- Decouple StreamProducer = StreamInfo + StreamProtocol
- Support typesafe DSL for StreamProducer
- Support KeyedStream and groupByKey
- Decouple ExecutionEnvironment
https://issues.apache.org/jira/browse/EAGLE-66
Author: @haoch <ha...@apache.org>
Reviewer: @RalphSu <su...@gmail.com>
Closes #26 #17
Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/52b8e58b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/52b8e58b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/52b8e58b
Branch: refs/heads/master
Commit: 52b8e58b1af53273782454f52e61b4f4700626c9
Parents: 2734b42
Author: Hao Chen <ha...@apache.org>
Authored: Wed Dec 16 13:58:10 2015 +0800
Committer: Hao Chen <ha...@apache.org>
Committed: Wed Dec 16 13:58:10 2015 +0800
----------------------------------------------------------------------
.../alert/config/EmailNotificationConfig.java | 2 +-
.../apache/eagle/executor/AlertExecutor.java | 2 +-
.../impl/storm/AbstractStormSpoutProvider.java | 29 --
.../dataproc/impl/storm/StormSpoutProvider.java | 29 ++
.../hdfs/HDFSSourcedStormSpoutProvider.java | 5 +-
.../storm/kafka/KafkaSourcedSpoutProvider.java | 7 +-
.../storm/kafka/KafkaSourcedSpoutScheme.java | 16 +-
.../partition/CustomPartitionGrouping.java | 51 ++++
.../storm/partition/EagleCustomGrouping.java | 51 ----
.../org/apache/eagle/datastream/JavaMapper.java | 26 --
.../datastream/JavaMapperStormExecutor.java | 3 +-
.../JavaStormExecutorForAlertWrapper.java | 1 +
.../eagle/datastream/utils/JavaReflections.java | 31 ++
.../AbstractStreamProducerGraph.scala | 29 --
.../datastream/AbstractTopologyCompiler.scala | 21 --
.../datastream/AbstractTopologyExecutor.scala | 21 --
.../datastream/AlertExecutorConsumerUtils.scala | 75 -----
.../eagle/datastream/ExecutionEnvironment.scala | 73 -----
.../datastream/ExecutionEnvironments.scala | 134 +++++++++
.../eagle/datastream/FilterBoltWrapper.scala | 48 ---
.../apache/eagle/datastream/GraphPrinter.scala | 37 ---
.../eagle/datastream/JavaStormBoltWrapper.scala | 52 ----
.../eagle/datastream/MapBoltWrapper.scala | 68 -----
.../eagle/datastream/NodeNameSelector.scala | 26 --
.../eagle/datastream/OutputFieldNameConst.scala | 21 --
.../apache/eagle/datastream/SpoutProxy.scala | 59 ----
.../eagle/datastream/StormBoltFactory.scala | 47 ---
.../eagle/datastream/StormBoltWrapper.scala | 60 ----
.../StormExecutorForAlertWrapper.scala | 43 ---
.../eagle/datastream/StormSpoutFactory.scala | 46 ---
.../eagle/datastream/StormStreamDAG.scala | 68 -----
.../datastream/StormStreamDAGTransformer.scala | 46 ---
.../datastream/StormTopologyCompiler.scala | 108 -------
.../datastream/StormTopologyExecutorImpl.scala | 74 -----
.../eagle/datastream/StreamAlertExpansion.scala | 198 -------------
.../apache/eagle/datastream/StreamAppDSL.scala | 120 --------
.../eagle/datastream/StreamConnector.scala | 36 ---
.../eagle/datastream/StreamDAGExpansion.scala | 27 --
.../datastream/StreamGroupbyExpansion.scala | 65 ----
.../eagle/datastream/StreamNameExpansion.scala | 41 ---
.../StreamParallelismConfigExpansion.scala | 55 ----
.../eagle/datastream/StreamProducer.scala | 221 --------------
.../eagle/datastream/StreamUnionExpansion.scala | 62 ----
.../apache/eagle/datastream/UnionUtils.scala | 41 ---
.../core/AbstractTopologyCompiler.scala | 21 ++
.../core/AbstractTopologyExecutor.scala | 21 ++
.../eagle/datastream/core/Configuration.scala | 74 +++++
.../datastream/core/ExecutionEnvironment.scala | 80 +++++
.../datastream/core/StreamAlertExpansion.scala | 203 +++++++++++++
.../eagle/datastream/core/StreamConnector.scala | 97 ++++++
.../eagle/datastream/core/StreamDAG.scala | 68 +++++
.../datastream/core/StreamDAGExpansion.scala | 27 ++
.../datastream/core/StreamDAGTransformer.scala | 45 +++
.../core/StreamGroupbyExpansion.scala | 77 +++++
.../datastream/core/StreamNameExpansion.scala | 49 ++++
.../core/StreamParallelismConfigExpansion.scala | 61 ++++
.../eagle/datastream/core/StreamProducer.scala | 294 +++++++++++++++++++
.../datastream/core/StreamProducerGraph.scala | 29 ++
.../eagle/datastream/core/StreamProtocol.scala | 176 +++++++++++
.../datastream/core/StreamSourceBuilder.scala | 47 +++
.../datastream/core/StreamUnionExpansion.scala | 71 +++++
.../kafka/JsonMessageDeserializer.scala | 45 ---
.../datastream/kafka/KafkaStreamMonitor.scala | 35 ---
.../datastream/storm/AbstractStreamBolt.scala | 119 ++++++++
.../datastream/storm/FilterBoltWrapper.scala | 41 +++
.../datastream/storm/ForeachBoltWrapper.scala | 44 +++
.../datastream/storm/IterableStreamSpout.scala | 71 +++++
.../datastream/storm/JavaStormBoltWrapper.scala | 53 ++++
.../storm/JsonMessageDeserializer.scala | 45 +++
.../datastream/storm/KafkaStreamMonitor.scala | 31 ++
.../eagle/datastream/storm/MapBoltWrapper.scala | 66 +++++
.../eagle/datastream/storm/SpoutProxy.scala | 111 +++++++
.../datastream/storm/StormBoltFactory.scala | 53 ++++
.../datastream/storm/StormBoltWrapper.scala | 61 ++++
.../storm/StormExecutionEnvironment.scala | 42 +++
.../storm/StormExecutorForAlertWrapper.scala | 41 +++
.../datastream/storm/StormSpoutFactory.scala | 67 +++++
.../storm/StormTopologyCompiler.scala | 118 ++++++++
.../storm/StormTopologyExecutorImpl.scala | 71 +++++
.../utils/AlertExecutorConsumerUtils.scala | 76 +++++
.../eagle/datastream/utils/GraphPrinter.scala | 59 ++++
.../eagle/datastream/utils/NameConstants.scala | 24 ++
.../datastream/utils/NodeNameSelector.scala | 28 ++
.../eagle/datastream/utils/ReflectionS.scala | 55 ++++
.../eagle/datastream/utils/UnionUtils.scala | 43 +++
.../TestExecutionEnvironmentJava.java | 41 +++
.../apache/eagle/datastream/TestJavaMain.java | 30 +-
.../datastream/TestJavaReflectionUtils.java | 36 +++
.../datastream/TestKafkaStreamMonitor.java | 3 +-
.../eagle/datastream/TestDAGExpansion.scala | 40 ++-
.../datastream/TestExecutionEnvironment.scala | 34 +++
.../eagle/datastream/TestStormRunner.scala | 62 ++--
.../eagle/datastream/TestTypeSafedDSL.scala | 87 ++++++
.../org/apache/eagle/datastream/JavaMapper.java | 26 ++
.../eagle/datastream/JavaTypeCompatible.java | 24 ++
.../apache/eagle/datastream/EagleTuple.scala | 20 +-
.../apache/eagle/datastream/FlatMapper.scala | 4 +-
.../eagle/datastream/StormStreamExecutor.scala | 9 +-
.../service/client/EagleServiceConnector.java | 4 +-
.../metric/kafka/EagleMetricCollectorMain.java | 23 +-
.../kafka/KafkaOffsetSourceSpoutProvider.java | 7 +-
.../hdfs/entity/FileSensitivityAPIEntity.java | 2 +-
.../partition/DataDistributionDaoImpl.java | 1 -
.../hbase/HbaseAuditLogProcessorMain.java | 25 +-
...baseResourceSensitivityDataJoinExecutor.java | 1 -
.../run_auditlog_topology.sh | 15 +
.../run_hostname_lookkup.sh | 15 +
.../run_message_producer.sh | 15 +
.../run_message_producer_in_assembly.sh | 15 +
.../assembly/eagle-dam-auditlog-assembly.xml | 17 ++
.../auditlog/HdfsAuditLogProcessorMain.java | 44 ++-
.../auditlog/HdfsUserCommandReassembler.java | 15 +-
.../src/main/resources/application.conf | 2 +-
.../src/main/resources/auditlog/auditlog.1 | 16 -
.../main/resources/security-auditlog-storm.yaml | 15 +
.../HDFSSecurityLogProcessorMain.java | 27 +-
.../service/security/hdfs/HDFSFileSystem.java | 17 +-
.../HiveJobRunningMonitoringMain.java | 23 +-
.../UserProfileDetectionBatchMain.java | 18 +-
.../UserProfileDetectionStreamMain.java | 24 +-
test.txt | 1 -
121 files changed, 3522 insertions(+), 2350 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-alert/eagle-alert-process/src/main/java/org/apache/eagle/alert/config/EmailNotificationConfig.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/eagle-alert-process/src/main/java/org/apache/eagle/alert/config/EmailNotificationConfig.java b/eagle-core/eagle-alert/eagle-alert-process/src/main/java/org/apache/eagle/alert/config/EmailNotificationConfig.java
index dd18af7..4816bc9 100644
--- a/eagle-core/eagle-alert/eagle-alert-process/src/main/java/org/apache/eagle/alert/config/EmailNotificationConfig.java
+++ b/eagle-core/eagle-alert/eagle-alert-process/src/main/java/org/apache/eagle/alert/config/EmailNotificationConfig.java
@@ -46,4 +46,4 @@ public class EmailNotificationConfig extends NotificationConfig{
public void setTplFileName(String tplFileName) {
this.tplFileName = tplFileName;
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-alert/eagle-alert-process/src/main/java/org/apache/eagle/executor/AlertExecutor.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/eagle-alert-process/src/main/java/org/apache/eagle/executor/AlertExecutor.java b/eagle-core/eagle-alert/eagle-alert-process/src/main/java/org/apache/eagle/executor/AlertExecutor.java
index d86a846..7e4372c 100644
--- a/eagle-core/eagle-alert/eagle-alert-process/src/main/java/org/apache/eagle/executor/AlertExecutor.java
+++ b/eagle-core/eagle-alert/eagle-alert-process/src/main/java/org/apache/eagle/executor/AlertExecutor.java
@@ -409,4 +409,4 @@ public class AlertExecutor extends JavaStormStreamExecutor2<String, AlertAPIEnti
}
}
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/dataproc/impl/storm/AbstractStormSpoutProvider.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/dataproc/impl/storm/AbstractStormSpoutProvider.java b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/dataproc/impl/storm/AbstractStormSpoutProvider.java
deleted file mode 100644
index 1332887..0000000
--- a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/dataproc/impl/storm/AbstractStormSpoutProvider.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.dataproc.impl.storm;
-
-import backtype.storm.topology.base.BaseRichSpout;
-
-import com.typesafe.config.Config;
-
-/**
- * Normally storm spout is a special part of storm topology and it is implemented in underlying spout implementation
- * which can be retrieved from getSpout method.
- */
-public abstract class AbstractStormSpoutProvider{
- public abstract BaseRichSpout getSpout(Config context);
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/dataproc/impl/storm/StormSpoutProvider.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/dataproc/impl/storm/StormSpoutProvider.java b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/dataproc/impl/storm/StormSpoutProvider.java
new file mode 100644
index 0000000..ab90ad7
--- /dev/null
+++ b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/dataproc/impl/storm/StormSpoutProvider.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.dataproc.impl.storm;
+
+import backtype.storm.topology.base.BaseRichSpout;
+
+import com.typesafe.config.Config;
+
+/**
+ * Normally storm spout is a special part of storm topology and it is implemented in underlying spout implementation
+ * which can be retrieved from getSpout method.
+ */
+public interface StormSpoutProvider {
+ public BaseRichSpout getSpout(Config context);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/dataproc/impl/storm/hdfs/HDFSSourcedStormSpoutProvider.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/dataproc/impl/storm/hdfs/HDFSSourcedStormSpoutProvider.java b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/dataproc/impl/storm/hdfs/HDFSSourcedStormSpoutProvider.java
index 75305f1..2323f39 100644
--- a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/dataproc/impl/storm/hdfs/HDFSSourcedStormSpoutProvider.java
+++ b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/dataproc/impl/storm/hdfs/HDFSSourcedStormSpoutProvider.java
@@ -14,16 +14,17 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.eagle.dataproc.impl.storm.hdfs;
import com.typesafe.config.Config;
-import org.apache.eagle.dataproc.impl.storm.AbstractStormSpoutProvider;
+import org.apache.eagle.dataproc.impl.storm.StormSpoutProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import backtype.storm.topology.base.BaseRichSpout;
-public class HDFSSourcedStormSpoutProvider extends AbstractStormSpoutProvider {
+public class HDFSSourcedStormSpoutProvider implements StormSpoutProvider {
private static final Logger LOG = LoggerFactory.getLogger(HDFSSourcedStormSpoutProvider.class);
public abstract static class HDFSSpout extends BaseRichSpout{
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/dataproc/impl/storm/kafka/KafkaSourcedSpoutProvider.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/dataproc/impl/storm/kafka/KafkaSourcedSpoutProvider.java b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/dataproc/impl/storm/kafka/KafkaSourcedSpoutProvider.java
index 373b3ca..53537d7 100644
--- a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/dataproc/impl/storm/kafka/KafkaSourcedSpoutProvider.java
+++ b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/dataproc/impl/storm/kafka/KafkaSourcedSpoutProvider.java
@@ -28,9 +28,9 @@ import storm.kafka.ZkHosts;
import backtype.storm.spout.SchemeAsMultiScheme;
import backtype.storm.topology.base.BaseRichSpout;
-import org.apache.eagle.dataproc.impl.storm.AbstractStormSpoutProvider;
+import org.apache.eagle.dataproc.impl.storm.StormSpoutProvider;
-public class KafkaSourcedSpoutProvider extends AbstractStormSpoutProvider{
+public class KafkaSourcedSpoutProvider implements StormSpoutProvider {
private final static Logger LOG = LoggerFactory.getLogger(KafkaSourcedSpoutProvider.class);
public SchemeAsMultiScheme getStreamScheme(String deserClsName, Config context) {
@@ -89,7 +89,6 @@ public class KafkaSourcedSpoutProvider extends AbstractStormSpoutProvider{
}
spoutConfig.scheme = getStreamScheme(deserClsName, context);
- KafkaSpout kafkaSpout = new KafkaSpout(spoutConfig);
- return kafkaSpout;
+ return new KafkaSpout(spoutConfig);
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/dataproc/impl/storm/kafka/KafkaSourcedSpoutScheme.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/dataproc/impl/storm/kafka/KafkaSourcedSpoutScheme.java b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/dataproc/impl/storm/kafka/KafkaSourcedSpoutScheme.java
index 8b65c1f..faeb7c3 100644
--- a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/dataproc/impl/storm/kafka/KafkaSourcedSpoutScheme.java
+++ b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/dataproc/impl/storm/kafka/KafkaSourcedSpoutScheme.java
@@ -19,12 +19,12 @@ package org.apache.eagle.dataproc.impl.storm.kafka;
import backtype.storm.spout.Scheme;
import backtype.storm.tuple.Fields;
import com.typesafe.config.Config;
+import org.apache.eagle.datastream.utils.NameConstants;
import java.lang.reflect.Constructor;
import java.util.Arrays;
import java.util.List;
import java.util.Properties;
-import java.util.Map;
/**
* This scheme defines how a kafka message is deserialized and the output field name for storm stream
@@ -56,10 +56,16 @@ public class KafkaSourcedSpoutScheme implements Scheme {
// the following tasks are executed within the same process of kafka spout
return Arrays.asList(tmp);
}
-
+
+ /**
+ * Default only f0, but it requires to be overrode if different
+ *
+ * TODO: Handle the schema with KeyValue based structure
+ *
+ * @return Fields
+ */
@Override
public Fields getOutputFields() {
-// return new Fields(deserializer.getOutputFields());
- throw new UnsupportedOperationException("output fields should be declared in sub class of KafkaSourcedSpoutProvider");
+ return new Fields(NameConstants.FIELD_PREFIX()+"0");
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/dataproc/impl/storm/partition/CustomPartitionGrouping.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/dataproc/impl/storm/partition/CustomPartitionGrouping.java b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/dataproc/impl/storm/partition/CustomPartitionGrouping.java
new file mode 100644
index 0000000..9c1ab68
--- /dev/null
+++ b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/dataproc/impl/storm/partition/CustomPartitionGrouping.java
@@ -0,0 +1,51 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.dataproc.impl.storm.partition;
+
+import backtype.storm.generated.GlobalStreamId;
+import backtype.storm.grouping.CustomStreamGrouping;
+import backtype.storm.task.WorkerTopologyContext;
+import org.apache.eagle.partition.PartitionStrategy;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+public class CustomPartitionGrouping implements CustomStreamGrouping {
+
+ public List<Integer> targetTasks;
+ public PartitionStrategy strategy;
+
+ public CustomPartitionGrouping(PartitionStrategy strategy) {
+ this.strategy = strategy;
+ }
+
+ @Override
+ public void prepare(WorkerTopologyContext context, GlobalStreamId stream, List<Integer> targetTasks) {
+ this.targetTasks = new ArrayList<>(targetTasks);
+ }
+
+ @Override
+ public List<Integer> chooseTasks(int taskId, List<Object> values) {
+ int numTasks = targetTasks.size();
+ int targetTaskIndex = strategy.balance((String)values.get(0), numTasks);
+ return Arrays.asList(targetTasks.get(targetTaskIndex));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/dataproc/impl/storm/partition/EagleCustomGrouping.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/dataproc/impl/storm/partition/EagleCustomGrouping.java b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/dataproc/impl/storm/partition/EagleCustomGrouping.java
deleted file mode 100644
index 96e42b7..0000000
--- a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/dataproc/impl/storm/partition/EagleCustomGrouping.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.eagle.dataproc.impl.storm.partition;
-
-import backtype.storm.generated.GlobalStreamId;
-import backtype.storm.grouping.CustomStreamGrouping;
-import backtype.storm.task.WorkerTopologyContext;
-import org.apache.eagle.partition.PartitionStrategy;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-public class EagleCustomGrouping implements CustomStreamGrouping {
-
- public List<Integer> targetTasks;
- public PartitionStrategy strategy;
-
- public EagleCustomGrouping(PartitionStrategy strategy) {
- this.strategy = strategy;
- }
-
- @Override
- public void prepare(WorkerTopologyContext context, GlobalStreamId stream, List<Integer> targetTasks) {
- this.targetTasks = new ArrayList<>(targetTasks);
- }
-
- @Override
- public List<Integer> chooseTasks(int taskId, List<Object> values) {
- int numTasks = targetTasks.size();
- int targetTaskIndex = strategy.balance((String)values.get(0), numTasks);
- return Arrays.asList(targetTasks.get(targetTaskIndex));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/datastream/JavaMapper.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/datastream/JavaMapper.java b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/datastream/JavaMapper.java
deleted file mode 100644
index 7e66478..0000000
--- a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/datastream/JavaMapper.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. See the NOTICE file distributed with
- * * this work for additional information regarding copyright ownership.
- * * The ASF licenses this file to You under the Apache License, Version 2.0
- * * (the "License"); you may not use this file except in compliance with
- * * the License. You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License.
- *
- */
-
-package org.apache.eagle.datastream;
-
-import java.util.List;
-
-public interface JavaMapper {
- List<Object> map(List<Object> input);
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/datastream/JavaMapperStormExecutor.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/datastream/JavaMapperStormExecutor.java b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/datastream/JavaMapperStormExecutor.java
index 04a80e5..993a4a2 100644
--- a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/datastream/JavaMapperStormExecutor.java
+++ b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/datastream/JavaMapperStormExecutor.java
@@ -29,6 +29,7 @@ import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.topology.base.BaseRichBolt;
import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Tuple;
+import org.apache.eagle.datastream.utils.NameConstants;
public class JavaMapperStormExecutor extends BaseRichBolt{
public static class e1 extends JavaMapperStormExecutor {
@@ -75,7 +76,7 @@ public class JavaMapperStormExecutor extends BaseRichBolt{
public void declareOutputFields(OutputFieldsDeclarer declarer) {
List<String> fields = new ArrayList<String>();
for(int i=0; i<numOutputFields; i++){
- fields.add(OutputFieldNameConst.FIELD_PREFIX() + i);
+ fields.add(NameConstants.FIELD_PREFIX() + i);
}
declarer.declare(new Fields(fields));
}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/datastream/JavaStormExecutorForAlertWrapper.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/datastream/JavaStormExecutorForAlertWrapper.java b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/datastream/JavaStormExecutorForAlertWrapper.java
index 3aacb32..331e51a 100644
--- a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/datastream/JavaStormExecutorForAlertWrapper.java
+++ b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/datastream/JavaStormExecutorForAlertWrapper.java
@@ -52,4 +52,5 @@ public class JavaStormExecutorForAlertWrapper extends JavaStormStreamExecutor3<S
};
delegate.flatMap(input, delegateCollector);
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/datastream/utils/JavaReflections.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/datastream/utils/JavaReflections.java b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/datastream/utils/JavaReflections.java
new file mode 100644
index 0000000..04b4bed
--- /dev/null
+++ b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/datastream/utils/JavaReflections.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.datastream.utils;
+
+import java.lang.reflect.ParameterizedType;
+
+/**
+ * @since 12/7/15
+ */
+class JavaReflections {
+ @SuppressWarnings("unchecked")
+ public static Class<?> getGenericTypeClass(final Object obj,int index) {
+ return (Class<?>) ((ParameterizedType) obj
+ .getClass()
+ .getGenericSuperclass()).getActualTypeArguments()[index];
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/AbstractStreamProducerGraph.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/AbstractStreamProducerGraph.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/AbstractStreamProducerGraph.scala
deleted file mode 100644
index dc2c198..0000000
--- a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/AbstractStreamProducerGraph.scala
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. See the NOTICE file distributed with
- * * this work for additional information regarding copyright ownership.
- * * The ASF licenses this file to You under the Apache License, Version 2.0
- * * (the "License"); you may not use this file except in compliance with
- * * the License. You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License.
- *
- */
-package org.apache.eagle.datastream
-
-trait AbstractStreamProducerGraph {
- def addEdge(from: StreamProducer, to: StreamProducer, streamConnector: StreamConnector)
- def addVertex(producer: StreamProducer)
- def iterator() : Iterator[StreamProducer]
- def isSource(v : StreamProducer) : Boolean
- def outgoingEdgesOf(v : StreamProducer) : scala.collection.mutable.Set[StreamConnector]
- def getNodeByName(name : String) : Option[StreamProducer]
- def incomingVertexOf(v: StreamProducer) : scala.collection.mutable.Set[StreamProducer]
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/AbstractTopologyCompiler.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/AbstractTopologyCompiler.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/AbstractTopologyCompiler.scala
deleted file mode 100644
index 8c53ed5..0000000
--- a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/AbstractTopologyCompiler.scala
+++ /dev/null
@@ -1,21 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.datastream
-
-trait AbstractTopologyCompiler{
- def buildTopology : AbstractTopologyExecutor
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/AbstractTopologyExecutor.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/AbstractTopologyExecutor.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/AbstractTopologyExecutor.scala
deleted file mode 100644
index 9ac3cc9..0000000
--- a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/AbstractTopologyExecutor.scala
+++ /dev/null
@@ -1,21 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.datastream
-
-trait AbstractTopologyExecutor {
- def execute
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/AlertExecutorConsumerUtils.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/AlertExecutorConsumerUtils.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/AlertExecutorConsumerUtils.scala
deleted file mode 100644
index cb6ac4f..0000000
--- a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/AlertExecutorConsumerUtils.scala
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. See the NOTICE file distributed with
- * * this work for additional information regarding copyright ownership.
- * * The ASF licenses this file to You under the Apache License, Version 2.0
- * * (the "License"); you may not use this file except in compliance with
- * * the License. You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License.
- *
- */
-
-package org.apache.eagle.datastream
-
-import java.util
-
-import org.apache.eagle.alert.dedup.{AlertEmailDeduplicationExecutor, AlertEntityDeduplicationExecutor}
-import org.apache.eagle.alert.notification.AlertNotificationExecutor
-import org.apache.eagle.alert.persist.AlertPersistExecutor
-import org.apache.eagle.executor.AlertExecutor
-import org.slf4j.{Logger, LoggerFactory}
-
-import scala.collection.mutable.ListBuffer
-
-/**
- * Create alert executors and provide callback for programmer to link alert executor to immediate parent executors
- *
- * <br/><br/>
- * Explanations for programId, alertExecutorId and policy<br/><br/>
- * - programId - distributed or single-process program for example one storm topology<br/>
- * - alertExecutorId - one process/thread which executes multiple policies<br/>
- * - policy - some rules to be evaluated<br/>
- *
- * <br/>
- *
- * Normally the mapping is like following:
- * <pre>
- * programId (1:N) alertExecutorId
- * alertExecutorId (1:N) policy
- * </pre>
- */
-
-object AlertExecutorConsumerUtils {
- private val LOG: Logger = LoggerFactory.getLogger(AlertExecutorConsumerUtils.getClass)
-
- def setupAlertConsumers(toBeAddedEdges: ListBuffer[StreamConnector], alertStreamProducers: List[StreamProducer]): Unit = {
- var alertExecutorIdList : java.util.List[String] = new util.ArrayList[String]()
- alertStreamProducers.map(x =>
- alertExecutorIdList.add(x.asInstanceOf[FlatMapProducer[AnyRef, AnyRef]].mapper.asInstanceOf[AlertExecutor].getAlertExecutorId));
- val alertDefDao = alertStreamProducers.head.asInstanceOf[FlatMapProducer[AnyRef, AnyRef]].mapper.asInstanceOf[AlertExecutor].getAlertDefinitionDao
- val entityDedupExecutor: AlertEntityDeduplicationExecutor = new AlertEntityDeduplicationExecutor(alertExecutorIdList, alertDefDao)
- val emailDedupExecutor: AlertEmailDeduplicationExecutor = new AlertEmailDeduplicationExecutor(alertExecutorIdList, alertDefDao)
- val notificationExecutor: AlertNotificationExecutor = new AlertNotificationExecutor(alertExecutorIdList, alertDefDao)
- val persistExecutor: AlertPersistExecutor = new AlertPersistExecutor
-
- val entityDedupStreamProducer = FlatMapProducer(UniqueId.incrementAndGetId(),entityDedupExecutor)
- val persistStreamProducer = FlatMapProducer(UniqueId.incrementAndGetId(),persistExecutor)
- val emailDedupStreamProducer = FlatMapProducer(UniqueId.incrementAndGetId(),emailDedupExecutor)
- val notificationStreamProducer = FlatMapProducer(UniqueId.incrementAndGetId(),notificationExecutor)
- toBeAddedEdges += StreamConnector(entityDedupStreamProducer, persistStreamProducer)
- toBeAddedEdges += StreamConnector(emailDedupStreamProducer, notificationStreamProducer)
-
- alertStreamProducers.foreach(sp => {
- toBeAddedEdges += StreamConnector(sp, entityDedupStreamProducer)
- toBeAddedEdges += StreamConnector(sp, emailDedupStreamProducer)
- })
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/ExecutionEnvironment.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/ExecutionEnvironment.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/ExecutionEnvironment.scala
deleted file mode 100644
index 488d52a..0000000
--- a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/ExecutionEnvironment.scala
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.datastream
-
-import backtype.storm.topology.base.BaseRichSpout
-import com.typesafe.config.{Config, ConfigFactory}
-import org.apache.eagle.dataproc.impl.storm.AbstractStormSpoutProvider
-import org.jgrapht.experimental.dag.DirectedAcyclicGraph
-import org.slf4j.LoggerFactory
-
-object ExecutionEnvironmentFactory{
-
- def getStorm(config : Config) = new StormExecutionEnvironment(config)
- def getStorm:StormExecutionEnvironment = {
- val config = ConfigFactory.load()
- getStorm(config)
- }
-}
-
-abstract class ExecutionEnvironment(config : Config){
- def execute()
-}
-
-class StormExecutionEnvironment(config: Config) extends ExecutionEnvironment(config){
- val LOG = LoggerFactory.getLogger(classOf[StormExecutionEnvironment])
- val dag = new DirectedAcyclicGraph[StreamProducer, StreamConnector](classOf[StreamConnector])
-
- override def execute() : Unit = {
- LOG.info("initial graph:\n")
- GraphPrinter.print(dag)
- new StreamAlertExpansion(config).expand(dag)
- LOG.info("after StreamAlertExpansion graph:")
- GraphPrinter.print(dag)
- new StreamUnionExpansion(config).expand(dag)
- LOG.info("after StreamUnionExpansion graph:")
- GraphPrinter.print(dag)
- new StreamGroupbyExpansion(config).expand(dag)
- LOG.info("after StreamGroupbyExpansion graph:")
- GraphPrinter.print(dag)
- new StreamNameExpansion(config).expand(dag)
- LOG.info("after StreamNameExpansion graph:")
- GraphPrinter.print(dag)
- new StreamParallelismConfigExpansion(config).expand(dag)
- LOG.info("after StreamParallelismConfigExpansion graph:")
- GraphPrinter.print(dag)
- val stormDag = StormStreamDAGTransformer.transform(dag)
- StormTopologyCompiler(config, stormDag).buildTopology.execute
- }
-
- def newSource(source: BaseRichSpout): StormSourceProducer ={
- val ret = StormSourceProducer(UniqueId.incrementAndGetId(), source)
- ret.config = config
- ret.graph = dag
- dag.addVertex(ret)
- ret
- }
-
- def newSource(sourceProvider: AbstractStormSpoutProvider):StormSourceProducer = newSource(sourceProvider.getSpout(config))
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/ExecutionEnvironments.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/ExecutionEnvironments.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/ExecutionEnvironments.scala
new file mode 100644
index 0000000..7e5f271
--- /dev/null
+++ b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/ExecutionEnvironments.scala
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.datastream
+
+import com.typesafe.config.{Config, ConfigFactory}
+import org.apache.eagle.dataproc.util.ConfigOptionParser
+import org.apache.eagle.datastream.core._
+import org.apache.eagle.datastream.storm.StormExecutionEnvironment
+
+import scala.reflect.runtime.universe._
+
+/**
+ * Execution environment factory
+ *
+ * The factory is mainly used for create or manage execution environment,
+ * and also handles the shared works like configuration, arguments for execution environment
+ *
+ * Notice: this factory class should not know any implementation like storm or spark
+ *
+ * @since 0.3.0
+ */
+object ExecutionEnvironments{
+ /**
+ * Use `'''get[StormExecutionEnvironment](config)'''` instead
+ *
+ * @param config
+ * @return
+ */
+ @deprecated("Execution environment should not know implementation of Storm")
+ def getStorm(config : Config) = new StormExecutionEnvironment(config)
+
+ /**
+ * Use `'''get[StormExecutionEnvironment]'''` instead
+ *
+ * @return
+ */
+ @deprecated("Execution environment should not know implementation of Storm")
+ def getStorm:StormExecutionEnvironment = {
+ val config = ConfigFactory.load()
+ getStorm(config)
+ }
+
+ /**
+ * Use `'''get[StormExecutionEnvironment](args)'''` instead
+ *
+ * @see get[StormExecutionEnvironment](args)
+ *
+ * @param args
+ * @return
+ */
+ @deprecated("Execution environment should not know implementation of Storm")
+ def getStorm(args:Array[String]):StormExecutionEnvironment = {
+ getStorm(new ConfigOptionParser().load(args))
+ }
+
+ /**
+ * @param typeTag
+ * @tparam T
+ * @return
+ */
+ def get[T<:ExecutionEnvironment](implicit typeTag: TypeTag[T]): T ={
+ get[T](ConfigFactory.load())
+ }
+
+ /**
+ *
+ * @param config
+ * @param typeTag
+ * @tparam T
+ * @return
+ */
+ def get[T<:ExecutionEnvironment](config:Config)(implicit typeTag: TypeTag[T]): T ={
+ typeTag.mirror.runtimeClass(typeOf[T]).getConstructor(classOf[Config]).newInstance(config).asInstanceOf[T]
+ }
+
+ /**
+ *
+ * @param args
+ * @param typeTag
+ * @tparam T
+ * @return
+ */
+ def get[T<:ExecutionEnvironment](args:Array[String])(implicit typeTag: TypeTag[T]): T ={
+ get[T](new ConfigOptionParser().load(args))
+ }
+
+ /**
+ * Support java style for default config
+ *
+ * @param clazz execution environment class
+ * @tparam T execution environment type
+ * @return
+ */
+ def get[T<:ExecutionEnvironment](clazz:Class[T]):T ={
+ get[T](ConfigFactory.load(),clazz)
+ }
+
+ /**
+ * Support java style
+ * @param config command config
+ * @param clazz execution environment class
+ * @tparam T execution environment type
+ * @return
+ */
+ def get[T<:ExecutionEnvironment](config:Config,clazz:Class[T]):T ={
+ clazz.getConstructor(classOf[Config]).newInstance(config)
+ }
+
+ /**
+ * Support java style
+ *
+ * @param args command arguments in string array
+ * @param clazz execution environment class
+ * @tparam T execution environment type
+ * @return
+ */
+ def get[T<:ExecutionEnvironment](args:Array[String],clazz:Class[T]):T ={
+ clazz.getConstructor(classOf[Config]).newInstance(new ConfigOptionParser().load(args))
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/FilterBoltWrapper.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/FilterBoltWrapper.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/FilterBoltWrapper.scala
deleted file mode 100644
index d31759b..0000000
--- a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/FilterBoltWrapper.scala
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.datastream
-
-import java.util
-
-import backtype.storm.task.{OutputCollector, TopologyContext}
-import backtype.storm.topology.OutputFieldsDeclarer
-import backtype.storm.topology.base.BaseRichBolt
-import backtype.storm.tuple.{Fields, Tuple}
-import org.slf4j.LoggerFactory
-
-case class FilterBoltWrapper[T](fn : T => Boolean) extends BaseRichBolt{
- val LOG = LoggerFactory.getLogger(FilterBoltWrapper.getClass)
- var _collector : OutputCollector = null
-
- override def prepare(stormConf: util.Map[_, _], context: TopologyContext, collector: OutputCollector): Unit = {
- _collector = collector
- }
-
- override def execute(input : Tuple): Unit = {
- input.getValue(0) match {
- case v:T =>
- if(fn(v)){
- _collector.emit(input, input.getValues)
- _collector.ack(input)
- }
- }
- }
-
- override def declareOutputFields(declarer : OutputFieldsDeclarer): Unit ={
- declarer.declare(new Fields(OutputFieldNameConst.FIELD_PREFIX + "0"))
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/GraphPrinter.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/GraphPrinter.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/GraphPrinter.scala
deleted file mode 100644
index 1925a89..0000000
--- a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/GraphPrinter.scala
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. See the NOTICE file distributed with
- * * this work for additional information regarding copyright ownership.
- * * The ASF licenses this file to You under the Apache License, Version 2.0
- * * (the "License"); you may not use this file except in compliance with
- * * the License. You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License.
- *
- */
-
-package org.apache.eagle.datastream
-
-import org.jgrapht.experimental.dag.DirectedAcyclicGraph
-import org.slf4j.LoggerFactory
-import scala.collection.JavaConversions._
-
-object GraphPrinter {
- private val LOG = LoggerFactory.getLogger(GraphPrinter.getClass)
- def print(dag: DirectedAcyclicGraph[StreamProducer, StreamConnector]): Unit ={
- val iter = dag.iterator()
- while(iter.hasNext) {
- val current = iter.next()
- dag.outgoingEdgesOf(current).foreach(edge => {
- LOG.info(edge.from + "{" + edge.from.parallelism + "}" +" => " + edge.to + "{" + edge.to.parallelism + "}" + " with groupByFields " + edge.groupByFields)
- })
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/JavaStormBoltWrapper.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/JavaStormBoltWrapper.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/JavaStormBoltWrapper.scala
deleted file mode 100644
index 8ebfd7b..0000000
--- a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/JavaStormBoltWrapper.scala
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.datastream
-
-import java.util
-
-import backtype.storm.task.{OutputCollector, TopologyContext}
-import backtype.storm.topology.OutputFieldsDeclarer
-import backtype.storm.topology.base.BaseRichBolt
-import backtype.storm.tuple.{Fields, Tuple}
-import org.slf4j.LoggerFactory
-
-import scala.collection.JavaConverters._
-
-case class JavaStormBoltWrapper(worker : JavaStormStreamExecutor[EagleTuple]) extends BaseRichBolt{
- val LOG = LoggerFactory.getLogger(StormBoltWrapper.getClass)
- var _collector : OutputCollector = null
-
- override def prepare(stormConf: util.Map[_, _], context: TopologyContext, collector: OutputCollector): Unit = {
- _collector = collector
- worker.init
- }
-
- override def execute(input : Tuple): Unit ={
- worker.flatMap(input.getValues, new Collector[EagleTuple](){
- def collect(t: EagleTuple): Unit ={
- _collector.emit(input, t.getList.asJava)
- }
- })
- _collector.ack(input)
- }
-
- override def declareOutputFields(declarer : OutputFieldsDeclarer): Unit ={
- val fields = worker.fields
- LOG.info("output fields for worker " + worker + " : " + fields.toList)
- declarer.declare(new Fields(fields:_*))
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/MapBoltWrapper.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/MapBoltWrapper.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/MapBoltWrapper.scala
deleted file mode 100644
index 99fa32a..0000000
--- a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/MapBoltWrapper.scala
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.datastream
-
-import java.util
-
-import backtype.storm.task.{OutputCollector, TopologyContext}
-import backtype.storm.topology.OutputFieldsDeclarer
-import backtype.storm.topology.base.BaseRichBolt
-import backtype.storm.tuple.{Fields, Tuple}
-import org.slf4j.LoggerFactory
-
-/**
- * @since 9/29/15
- */
-case class MapBoltWrapper[T,R](num: Int, fn: T => R) extends BaseRichBolt {
- val LOG = LoggerFactory.getLogger(FilterBoltWrapper.getClass)
- var _collector : OutputCollector = null
-
- override def declareOutputFields(declarer: OutputFieldsDeclarer): Unit = {
- var fields = new util.ArrayList[String]()
- var i : Int = 0;
- while(i < num){
- fields.add(OutputFieldNameConst.FIELD_PREFIX + i)
- i += 1
- }
- declarer.declare(new Fields(fields))
- }
-
- override def execute(input: Tuple): Unit = {
- val size = input.size()
- var values : AnyRef = null
- size match {
- case 1 => values = scala.Tuple1(input.getValue(0))
- case 2 => values = scala.Tuple2(input.getValue(0), input.getValue(1))
- case 3 => values = scala.Tuple3(input.getValue(0), input.getValue(1), input.getValue(2))
- case 4 => values = scala.Tuple4(input.getValue(0), input.getValue(1), input.getValue(2), input.getValue(3))
- case _ => throw new IllegalArgumentException
- }
- val output = fn(values.asInstanceOf[T])
- output match {
- case scala.Tuple1(a) => _collector.emit(input, util.Arrays.asList(a.asInstanceOf[AnyRef]))
- case scala.Tuple2(a, b) => _collector.emit(input, util.Arrays.asList(a.asInstanceOf[AnyRef], b.asInstanceOf[AnyRef]))
- case scala.Tuple3(a, b, c) => _collector.emit(input, util.Arrays.asList(a.asInstanceOf[AnyRef], b.asInstanceOf[AnyRef], c.asInstanceOf[AnyRef]))
- case scala.Tuple4(a, b, c, d) => _collector.emit(input, util.Arrays.asList(a.asInstanceOf[AnyRef], b.asInstanceOf[AnyRef], c.asInstanceOf[AnyRef], d.asInstanceOf[AnyRef]))
- case a => _collector.emit(input, util.Arrays.asList(a.asInstanceOf[AnyRef]))
- }
- _collector.ack(input)
- }
-
- override def prepare(stormConf: util.Map[_, _], context: TopologyContext, collector: OutputCollector): Unit = {
- _collector = collector
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/NodeNameSelector.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/NodeNameSelector.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/NodeNameSelector.scala
deleted file mode 100644
index 8b06322..0000000
--- a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/NodeNameSelector.scala
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.datastream
-
-case class NodeNameSelector(producer : StreamProducer) {
- def getName : String = {
- producer.name match {
- case null => producer.toString
- case _ => producer.name
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/OutputFieldNameConst.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/OutputFieldNameConst.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/OutputFieldNameConst.scala
deleted file mode 100644
index 64659b7..0000000
--- a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/OutputFieldNameConst.scala
+++ /dev/null
@@ -1,21 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.datastream
-
-object OutputFieldNameConst {
- val FIELD_PREFIX = "f"
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/SpoutProxy.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/SpoutProxy.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/SpoutProxy.scala
deleted file mode 100644
index aca3b5b..0000000
--- a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/SpoutProxy.scala
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.datastream
-
-import backtype.storm.spout.SpoutOutputCollector
-import backtype.storm.task.TopologyContext
-import backtype.storm.topology.OutputFieldsDeclarer
-import backtype.storm.topology.base.BaseRichSpout
-import backtype.storm.tuple.Fields
-
-/**
- * Declare delegated BaseRichSpout with given field names
- *
- * @param delegate delegated BaseRichSpout
- * @param outputFields given field names
- */
-case class SpoutProxy(delegate: BaseRichSpout, outputFields: java.util.List[String]) extends BaseRichSpout{
- def open(conf: java.util.Map[_, _], context: TopologyContext, collector: SpoutOutputCollector) {
- this.delegate.open(conf, context, collector)
- }
-
- def nextTuple {
- this.delegate.nextTuple
- }
-
- override def ack(msgId: AnyRef) {
- this.delegate.ack(msgId)
- }
-
- override def fail(msgId: AnyRef) {
- this.delegate.fail(msgId)
- }
-
- override def deactivate {
- this.delegate.deactivate
- }
-
- override def declareOutputFields(declarer: OutputFieldsDeclarer) {
- declarer.declare(new Fields(outputFields))
- }
-
- override def close {
- this.delegate.close
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StormBoltFactory.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StormBoltFactory.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StormBoltFactory.scala
deleted file mode 100644
index 2a2e268..0000000
--- a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StormBoltFactory.scala
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. See the NOTICE file distributed with
- * * this work for additional information regarding copyright ownership.
- * * The ASF licenses this file to You under the Apache License, Version 2.0
- * * (the "License"); you may not use this file except in compliance with
- * * the License. You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License.
- *
- */
-package org.apache.eagle.datastream
-
-import backtype.storm.topology.base.BaseRichBolt
-import com.typesafe.config.Config
-
-object StormBoltFactory {
- def getBoltWrapper(graph: AbstractStreamProducerGraph, producer : StreamProducer, config : Config) : BaseRichBolt = {
- producer match{
- case FlatMapProducer(id, worker) => {
- if(worker.isInstanceOf[JavaStormStreamExecutor[EagleTuple]]){
- worker.asInstanceOf[JavaStormStreamExecutor[EagleTuple]].prepareConfig(config)
- JavaStormBoltWrapper(worker.asInstanceOf[JavaStormStreamExecutor[EagleTuple]])
- }else if(worker.isInstanceOf[StormStreamExecutor[EagleTuple]]){
- worker.asInstanceOf[StormStreamExecutor[EagleTuple]].prepareConfig(config)
- StormBoltWrapper(worker.asInstanceOf[StormStreamExecutor[EagleTuple]])
- }else {
- throw new UnsupportedOperationException
- }
- }
- case FilterProducer(id, fn) => {
- FilterBoltWrapper(fn)
- }
- case MapProducer(id, n, fn) => {
- MapBoltWrapper(n, fn)
- }
- case _ => throw new UnsupportedOperationException
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StormBoltWrapper.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StormBoltWrapper.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StormBoltWrapper.scala
deleted file mode 100644
index 7f27483..0000000
--- a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StormBoltWrapper.scala
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.datastream
-
-import java.util
-
-import backtype.storm.task.{OutputCollector, TopologyContext}
-import backtype.storm.topology.OutputFieldsDeclarer
-import backtype.storm.topology.base.BaseRichBolt
-import backtype.storm.tuple.{Fields, Tuple}
-import org.slf4j.LoggerFactory
-
-import scala.collection.JavaConverters._
-
-case class StormBoltWrapper(worker : StormStreamExecutor[EagleTuple]) extends BaseRichBolt{
- val LOG = LoggerFactory.getLogger(StormBoltWrapper.getClass)
- var _collector : OutputCollector = null
-
- override def prepare(stormConf: util.Map[_, _], context: TopologyContext, collector: OutputCollector): Unit = {
- _collector = collector
- worker.init
- }
-
- override def execute(input : Tuple): Unit = {
- try {
- worker.flatMap(input.getValues.asScala, new Collector[EagleTuple] {
- override def collect(t: EagleTuple): Unit = {
- _collector.emit(input, t.getList.asJava)
- }
- })
- }catch{
- case ex: Exception => {
- LOG.error("fail executing", ex)
- _collector.fail(input)
- throw new RuntimeException(ex)
- }
- }
- _collector.ack(input)
- }
-
- override def declareOutputFields(declarer : OutputFieldsDeclarer): Unit ={
- val fields = worker.fields
- LOG.info("output fields for worker " + worker + " : " + fields.toList)
- declarer.declare(new Fields(fields:_*))
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StormExecutorForAlertWrapper.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StormExecutorForAlertWrapper.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StormExecutorForAlertWrapper.scala
deleted file mode 100644
index 6ff1d52..0000000
--- a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StormExecutorForAlertWrapper.scala
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. See the NOTICE file distributed with
- * * this work for additional information regarding copyright ownership.
- * * The ASF licenses this file to You under the Apache License, Version 2.0
- * * (the "License"); you may not use this file except in compliance with
- * * the License. You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License.
- *
- */
-
-package org.apache.eagle.datastream
-
-import java.util
-
-import com.typesafe.config.Config
-
-case class StormExecutorForAlertWrapper(delegate: StormStreamExecutor[Tuple2[String, util.SortedMap[AnyRef, AnyRef]]], streamName: String)
- extends StormStreamExecutor3[String, String, util.SortedMap[Object, Object]]{
- override def prepareConfig(config: Config): Unit = {
- delegate.prepareConfig(config)
- }
-
- override def init: Unit = {
- delegate.init
- }
-
- override def flatMap(input: Seq[AnyRef], collector: Collector[Tuple3[String, String, util.SortedMap[Object, Object]]]): Unit = {
- delegate.flatMap(input, new Collector[Tuple2[String, util.SortedMap[AnyRef, AnyRef]]] {
- override def collect(r: Tuple2[String, util.SortedMap[AnyRef, AnyRef]]): Unit = {
- collector.collect(Tuple3(r.f0, streamName, r.f1))
- }
- })
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StormSpoutFactory.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StormSpoutFactory.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StormSpoutFactory.scala
deleted file mode 100644
index dcb51fd..0000000
--- a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StormSpoutFactory.scala
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. See the NOTICE file distributed with
- * * this work for additional information regarding copyright ownership.
- * * The ASF licenses this file to You under the Apache License, Version 2.0
- * * (the "License"); you may not use this file except in compliance with
- * * the License. You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License.
- *
- */
-package org.apache.eagle.datastream
-
-import java.util
-
-import backtype.storm.topology.base.BaseRichSpout
-import com.typesafe.config.Config
-
-object StormSpoutFactory {
- /**
- * @param config context configuration
- * @param sourceProducer source producer
- * @return
- */
- def createSpout(config: Config, sourceProducer: StormSourceProducer) : BaseRichSpout = {
- val numFields = sourceProducer.numFields
- if(numFields <= 0) {
- sourceProducer.source
- }else{
- var i = 0
- val ret = new util.ArrayList[String]
- while(i < numFields){
- ret.add(OutputFieldNameConst.FIELD_PREFIX + i)
- i += 1
- }
- SpoutProxy(sourceProducer.source, ret)
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StormStreamDAG.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StormStreamDAG.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StormStreamDAG.scala
deleted file mode 100644
index f4129ae..0000000
--- a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StormStreamDAG.scala
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. See the NOTICE file distributed with
- * * this work for additional information regarding copyright ownership.
- * * The ASF licenses this file to You under the Apache License, Version 2.0
- * * (the "License"); you may not use this file except in compliance with
- * * the License. You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License.
- *
- */
-package org.apache.eagle.datastream
-
-import org.jgrapht.experimental.dag.DirectedAcyclicGraph
-
-import scala.collection.JavaConverters._
-import scala.collection.{JavaConversions, mutable}
-
-/**
- * wrapper of DAG, used for storm topology compiler
- */
-class StormStreamDAG(graph: DirectedAcyclicGraph[StreamProducer, StreamConnector]) extends AbstractStreamProducerGraph {
- var nodeMap: mutable.Map[String, StreamProducer] = null
-
- override def addEdge(from: StreamProducer, to: StreamProducer, streamConnector: StreamConnector): Unit = {
- graph.addEdge(from, to, streamConnector)
- }
-
- override def addVertex(producer: StreamProducer): Unit = {
- graph.addVertex(producer)
- }
-
- override def iterator(): Iterator[StreamProducer] = {
- JavaConversions.asScalaIterator(graph.iterator())
- }
-
- override def isSource(v: StreamProducer): Boolean = {
- graph.inDegreeOf(v) match {
- case 0 => true
- case _ => false
- }
- }
-
- override def outgoingEdgesOf(v: StreamProducer): scala.collection.mutable.Set[StreamConnector] = {
- JavaConversions.asScalaSet(graph.outgoingEdgesOf(v))
- }
-
- override def getNodeByName(name: String): Option[StreamProducer] = {
- nodeMap.get(name)
- }
-
- def setNodeMap(nodeMap: mutable.Map[String, StreamProducer]): Unit = {
- this.nodeMap = nodeMap
- }
-
- override def incomingVertexOf(v: StreamProducer): scala.collection.mutable.Set[StreamProducer] = {
- val set = mutable.Set[StreamProducer]()
- graph.incomingEdgesOf(v).asScala.foreach(e => set += graph.getEdgeSource(e))
- set
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StormStreamDAGTransformer.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StormStreamDAGTransformer.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StormStreamDAGTransformer.scala
deleted file mode 100644
index 254d84b..0000000
--- a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StormStreamDAGTransformer.scala
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. See the NOTICE file distributed with
- * * this work for additional information regarding copyright ownership.
- * * The ASF licenses this file to You under the Apache License, Version 2.0
- * * (the "License"); you may not use this file except in compliance with
- * * the License. You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License.
- *
- */
-
-package org.apache.eagle.datastream
-
-import org.jgrapht.experimental.dag.DirectedAcyclicGraph
-import scala.collection.mutable
-
-/**
- * convert generic DAG data structure to Storm specific DAG data structure for easy topology compiler
- */
-object StormStreamDAGTransformer {
- /**
- * Transform DirectedAcyclicGraph[StreamProducer, StreamConnector] into StormStreamDAG
- *
- * @param dag DirectedAcyclicGraph[StreamProducer, StreamConnector]
- * @return StormStreamDAG
- */
- def transform(dag: DirectedAcyclicGraph[StreamProducer, StreamConnector]) : StormStreamDAG = {
- val stormDAG = new StormStreamDAG(dag)
- val nodeMap = mutable.HashMap[String, StreamProducer]()
- val iter = dag.iterator()
- while(iter.hasNext){
- val sp = iter.next()
- nodeMap.put(sp.name, sp)
- }
- stormDAG.setNodeMap(nodeMap)
- stormDAG
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StormTopologyCompiler.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StormTopologyCompiler.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StormTopologyCompiler.scala
deleted file mode 100644
index 4f9fccc..0000000
--- a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StormTopologyCompiler.scala
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. See the NOTICE file distributed with
- * * this work for additional information regarding copyright ownership.
- * * The ASF licenses this file to You under the Apache License, Version 2.0
- * * (the "License"); you may not use this file except in compliance with
- * * the License. You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License.
- *
- */
-package org.apache.eagle.datastream
-
-import java.util
-
-import backtype.storm.topology.base.BaseRichBolt
-import backtype.storm.topology.{BoltDeclarer, TopologyBuilder}
-import backtype.storm.tuple.Fields
-import com.typesafe.config.Config
-import org.apache.eagle.dataproc.impl.storm.partition.EagleCustomGrouping
-import org.slf4j.LoggerFactory
-
-case class StormTopologyCompiler(config: Config, graph: AbstractStreamProducerGraph) extends AbstractTopologyCompiler{
- val LOG = LoggerFactory.getLogger(StormTopologyCompiler.getClass)
- val boltCache = scala.collection.mutable.Map[StreamProducer, StormBoltWrapper]()
-
- override def buildTopology: AbstractTopologyExecutor ={
- val builder = new TopologyBuilder();
- val iter = graph.iterator()
- val boltDeclarerCache = scala.collection.mutable.Map[String, BoltDeclarer]()
- while(iter.hasNext){
- val from = iter.next()
- val fromName = from.name
- if(graph.isSource(from)){
- val spout = StormSpoutFactory.createSpout(config, from.asInstanceOf[StormSourceProducer])
- builder.setSpout(fromName, spout, from.parallelism)
- LOG.info("Spout name : " + fromName + " with parallelism " + from.parallelism)
- } else {
- LOG.info("Bolt name:" + fromName)
- }
-
- val edges = graph.outgoingEdgesOf(from)
- edges.foreach(sc => {
- val toName = sc.to.name
- var boltDeclarer : BoltDeclarer = null
- val toBolt = createBoltIfAbsent(toName)
- boltDeclarerCache.get(toName) match{
- case None => {
- var finalParallelism = 1
- graph.getNodeByName(toName) match {
- case Some(p) => finalParallelism = p.parallelism
- case None => finalParallelism = 1
- }
- boltDeclarer = builder.setBolt(toName, toBolt, finalParallelism);
- LOG.info("created bolt " + toName + " with parallelism " + finalParallelism)
- boltDeclarerCache.put(toName, boltDeclarer)
- }
- case Some(bt) => boltDeclarer = bt
- }
- if (sc.groupByFields != Nil) {
- boltDeclarer.fieldsGrouping(fromName, new Fields(fields(sc.groupByFields)))
- }
- else if (sc.customGroupBy != null) {
- boltDeclarer.customGrouping(fromName, new EagleCustomGrouping(sc.customGroupBy));
- }
- else {
- boltDeclarer.shuffleGrouping(fromName);
- }
-/* sc.groupByFields match{
- case Nil => boltDeclarer.shuffleGrouping(fromName)
- case p => boltDeclarer.fieldsGrouping(fromName, new Fields(fields(p)))
- }*/
- LOG.info("bolt connected " + fromName + "->" + toName + " with groupby fields " + sc.groupByFields)
- })
- }
- new StormTopologyExecutorImpl(builder.createTopology, config)
- }
-
- def fields(fields : Seq[Int]): java.util.List[String] ={
- val ret = new util.ArrayList[String]
- fields.map(n => ret.add(OutputFieldNameConst.FIELD_PREFIX + n))
- ret
- }
-
- def createBoltIfAbsent(name : String) : BaseRichBolt = {
- val producer = graph.getNodeByName(name)
- producer match{
- case Some(p) => createBoltIfAbsent(graph, p)
- case None => throw new IllegalArgumentException("please check bolt name " + name)
- }
- }
-
- def createBoltIfAbsent(graph: AbstractStreamProducerGraph, producer : StreamProducer): BaseRichBolt ={
- boltCache.get(producer) match{
- case Some(bolt) => bolt
- case None => {
- StormBoltFactory.getBoltWrapper(graph, producer, config)
- }
- }
- }
-}
[2/5] incubator-eagle git commit: EAGLE-66 Typesafe Streaming DSL and
KeyValue based Grouping
Posted by ha...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/utils/NameConstants.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/utils/NameConstants.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/utils/NameConstants.scala
new file mode 100644
index 0000000..d4836e4
--- /dev/null
+++ b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/utils/NameConstants.scala
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.datastream.utils
+
+object NameConstants {
+ val FIELD_PREFIX = "f"
+ val FIELD_KEY = "key"
+ val FIELD_VALUE = "value"
+ val FIELD_SEPARATOR = "_"
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/utils/NodeNameSelector.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/utils/NodeNameSelector.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/utils/NodeNameSelector.scala
new file mode 100644
index 0000000..331cf7c
--- /dev/null
+++ b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/utils/NodeNameSelector.scala
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.datastream.utils
+
+import org.apache.eagle.datastream.core.StreamInfo
+
+case class NodeNameSelector(producer : StreamInfo) {
+ def getName : String = {
+ producer.name match {
+ case null => producer.toString+NameConstants.FIELD_SEPARATOR+producer.id
+ case _ => producer.name
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/utils/ReflectionS.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/utils/ReflectionS.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/utils/ReflectionS.scala
new file mode 100644
index 0000000..1d48752
--- /dev/null
+++ b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/utils/ReflectionS.scala
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eagle.datastream.utils
+
+import scala.reflect.api
+import scala.reflect.runtime.{universe => ru}
+
+/**
+ * @since 12/7/15
+ */
+object Reflections{
+ private val UNIT_CLASS = classOf[Unit]
+ private val UNIT_TYPE_TAG = ru.typeTag[Unit]
+
+ /**
+ * Class to TypeTag
+ * @param clazz class
+ * @tparam T Type T
+ * @return
+ */
+ def typeTag[T](clazz:Class[T]):ru.TypeTag[T]={
+ if(clazz == null){
+ null
+ }else if(clazz == UNIT_CLASS) {
+ UNIT_TYPE_TAG.asInstanceOf[ru.TypeTag[T]]
+ } else {
+ val mirror = ru.runtimeMirror(clazz.getClassLoader)
+ val sym = mirror.staticClass(clazz.getCanonicalName)
+ val tpe = sym.selfType
+ ru.TypeTag(mirror, new api.TypeCreator {
+ def apply[U <: api.Universe with Singleton](m: api.Mirror[U]) =
+ if (m eq mirror) tpe.asInstanceOf[U#Type]
+ else throw new IllegalArgumentException(s"Type tag defined in $mirror cannot be migrated to other mirrors.")
+ })
+ }
+ }
+
+ def javaTypeClass[T](obj: AnyRef, index: Int = 0):Class[T] = JavaReflections.getGenericTypeClass(obj,index).asInstanceOf[Class[T]]
+ def javaTypeTag[T](obj: AnyRef, index: Int = 0):ru.TypeTag[T] = typeTag(JavaReflections.getGenericTypeClass(obj,index)).asInstanceOf[ru.TypeTag[T]]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/utils/UnionUtils.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/utils/UnionUtils.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/utils/UnionUtils.scala
new file mode 100644
index 0000000..4ac0cdc
--- /dev/null
+++ b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/utils/UnionUtils.scala
@@ -0,0 +1,43 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+package org.apache.eagle.datastream.utils
+
+import java.util
+
+import org.apache.eagle.datastream.core.StreamProducer
+
+import scala.collection.JavaConverters._
+
+object UnionUtils {
+ def join[T1,T2](producers : StreamProducer[T1]*) : StreamProducer[T2] = {
+ producers.head.streamUnion(producers.drop(1))
+ }
+
+ def join[T1,T2](producers : java.util.List[StreamProducer[T1]]) : StreamProducer[T2] = {
+ val newList = new util.ArrayList(producers)
+ val head = newList.get(0)
+ newList.remove(0)
+ head.streamUnion(newList.asScala);
+ }
+
+ def join[T1,T2](producers : List[StreamProducer[T1]]) : StreamProducer[T2] = {
+ val head = producers.head
+ head.streamUnion(producers.tail);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/test/java/org/apache/eagle/datastream/TestExecutionEnvironmentJava.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/test/java/org/apache/eagle/datastream/TestExecutionEnvironmentJava.java b/eagle-core/eagle-data-process/eagle-stream-process-api/src/test/java/org/apache/eagle/datastream/TestExecutionEnvironmentJava.java
new file mode 100644
index 0000000..c5f9045
--- /dev/null
+++ b/eagle-core/eagle-data-process/eagle-stream-process-api/src/test/java/org/apache/eagle/datastream/TestExecutionEnvironmentJava.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.datastream;
+
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+import junit.framework.Assert;
+import org.apache.eagle.datastream.storm.StormExecutionEnvironment;
+import org.junit.Test;
+
+/**
+ * @since 12/5/15
+ */
+public class TestExecutionEnvironmentJava {
+
+ @Test
+ public void testGetEnvInJava() {
+ StormExecutionEnvironment env0 = ExecutionEnvironments.get(StormExecutionEnvironment.class);
+ Assert.assertNotNull(env0);
+
+ StormExecutionEnvironment env1 = ExecutionEnvironments.get(new String[]{}, StormExecutionEnvironment.class);
+ Assert.assertNotNull(env1);
+ Config config = ConfigFactory.load();
+ StormExecutionEnvironment env2 = ExecutionEnvironments.get(config, StormExecutionEnvironment.class);
+ Assert.assertNotNull(env2);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/test/java/org/apache/eagle/datastream/TestJavaMain.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/test/java/org/apache/eagle/datastream/TestJavaMain.java b/eagle-core/eagle-data-process/eagle-stream-process-api/src/test/java/org/apache/eagle/datastream/TestJavaMain.java
index 07c19de..979b09d 100644
--- a/eagle-core/eagle-data-process/eagle-stream-process-api/src/test/java/org/apache/eagle/datastream/TestJavaMain.java
+++ b/eagle-core/eagle-data-process/eagle-stream-process-api/src/test/java/org/apache/eagle/datastream/TestJavaMain.java
@@ -33,22 +33,22 @@ public class TestJavaMain {
}
//@Test
- public void testGeneral(){
- Config config = ConfigFactory.load();
- StormExecutionEnvironment env = ExecutionEnvironmentFactory.getStorm(config);
- env.newSource(new TestKeyValueSpout()).renameOutputFields(2).groupBy(Arrays.asList(0)).flatMap(new GroupedEchoExecutor()).withParallelism(2);
- env.execute();
- }
+// public void testGeneral(){
+// Config config = ConfigFactory.load();
+// StormExecutionEnvironment env = ExecutionEnvironments.getStorm(config);
+// env.fromSpout(new TestKeyValueSpout()).withOutputFields(2).groupBy(Arrays.asList(0)).flatMap(new GroupedEchoExecutor()).parallelism(2);
+// env.execute();
+// }
- //@Test
- public void testMap(){
- Config config = ConfigFactory.load();
- StormExecutionEnvironment env = ExecutionEnvironmentFactory.getStorm(config);
- SerializableFunction1 f1 = new SerializableFunction1<Object, Object>();
- env.newSource(new TestKeyValueSpout()).renameOutputFields(2).
- map1(f1);
- env.execute();
- }
+// //@Test
+// public void testMap(){
+// Config config = ConfigFactory.load();
+// StormExecutionEnvironment env = ExecutionEnvironments.getStorm(config);
+// SerializableFunction1 f1 = new SerializableFunction1<Object, Object>();
+// env.fromSpout(new TestKeyValueSpout()).withOutputFields(2).
+// map1(f1);
+// env.execute();
+// }
@Test
public void test() {
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/test/java/org/apache/eagle/datastream/TestJavaReflectionUtils.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/test/java/org/apache/eagle/datastream/TestJavaReflectionUtils.java b/eagle-core/eagle-data-process/eagle-stream-process-api/src/test/java/org/apache/eagle/datastream/TestJavaReflectionUtils.java
new file mode 100644
index 0000000..6d2d36f
--- /dev/null
+++ b/eagle-core/eagle-data-process/eagle-stream-process-api/src/test/java/org/apache/eagle/datastream/TestJavaReflectionUtils.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.datastream;
+
+import junit.framework.Assert;
+import org.apache.eagle.datastream.utils.Reflections;
+import org.junit.Test;
+import scala.reflect.api.TypeTags;
+
+/**
+ * @since 12/8/15
+ */
+public class TestJavaReflectionUtils {
+ @Test
+ public void testJavaFlatMapper(){
+ Class<String> clazz = Reflections.javaTypeClass(new JavaEchoExecutor(), 0);
+ Assert.assertEquals(String.class,clazz);
+ TypeTags.TypeTag typeTag = Reflections.typeTag(clazz);
+ Assert.assertNotNull(typeTag);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/test/java/org/apache/eagle/datastream/TestKafkaStreamMonitor.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/test/java/org/apache/eagle/datastream/TestKafkaStreamMonitor.java b/eagle-core/eagle-data-process/eagle-stream-process-api/src/test/java/org/apache/eagle/datastream/TestKafkaStreamMonitor.java
index 5651e74..2a5043c 100644
--- a/eagle-core/eagle-data-process/eagle-stream-process-api/src/test/java/org/apache/eagle/datastream/TestKafkaStreamMonitor.java
+++ b/eagle-core/eagle-data-process/eagle-stream-process-api/src/test/java/org/apache/eagle/datastream/TestKafkaStreamMonitor.java
@@ -16,8 +16,7 @@
*/
package org.apache.eagle.datastream;
-import org.apache.eagle.datastream.kafka.KafkaStreamMonitorApp;
-import org.apache.eagle.datastream.kafka.KafkaStreamMonitorApp;
+import org.apache.eagle.datastream.storm.KafkaStreamMonitorApp;
/**
* @since 11/7/15
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/test/scala/org/apache/eagle/datastream/TestDAGExpansion.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/test/scala/org/apache/eagle/datastream/TestDAGExpansion.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/test/scala/org/apache/eagle/datastream/TestDAGExpansion.scala
index 5463079..4339e5a 100644
--- a/eagle-core/eagle-data-process/eagle-stream-process-api/src/test/scala/org/apache/eagle/datastream/TestDAGExpansion.scala
+++ b/eagle-core/eagle-data-process/eagle-stream-process-api/src/test/scala/org/apache/eagle/datastream/TestDAGExpansion.scala
@@ -19,46 +19,44 @@
package org.apache.eagle.datastream
-import java.util
-
-import com.typesafe.config.{ConfigFactory, Config}
+import com.typesafe.config.{Config, ConfigFactory}
+import org.apache.eagle.datastream.storm.StormExecutionEnvironment
object testStreamUnionExpansion extends App{
- val config : Config = ConfigFactory.load;
+ val config : Config = ConfigFactory.load
val env = new StormExecutionEnvironment(config)
- val tail1 = env.newSource(TestSpout()).flatMap(WordPrependForAlertExecutor("test")).map2(a => ("key1",a))
- val tail2 = env.newSource(TestSpout()).flatMap(WordAppendForAlertExecutor("test")).map2(a => ("key1",a))
+ val tail1 = env.fromSpout(TestSpout()).flatMap(WordPrependForAlertExecutor("test")).map2(a => ("key1",a))
+ val tail2 = env.fromSpout(TestSpout()).flatMap(WordAppendForAlertExecutor("test")).map2(a => ("key1",a))
tail1.streamUnion(List(tail2)).map1(a => "xyz")
-// env.execute
}
object testStreamGroupbyExpansion extends App{
val config : Config = ConfigFactory.load;
val env = new StormExecutionEnvironment(config)
- env.newSource(TestSpout()).flatMap(WordPrependForAlertExecutor("test")).groupBy(1).map2(a => ("key1",a))
+ env.fromSpout(TestSpout()).flatMap(WordPrependForAlertExecutor("test")).groupBy(1).map2(a => ("key1",a))
//env.execute
}
object testStreamUnionAndGroupbyExpansion extends App{
val config : Config = ConfigFactory.load;
val env = new StormExecutionEnvironment(config)
- val tail1 = env.newSource(TestSpout()).flatMap(WordPrependForAlertExecutor("test")).map2(a => ("key1",a)).groupBy(1)
- val tail2 = env.newSource(TestSpout()).flatMap(WordAppendForAlertExecutor("test")).map2(a => ("key1",a)).groupBy(0)
+ val tail1 = env.fromSpout(TestSpout()).flatMap(WordPrependForAlertExecutor("test")).map2(a => ("key1",a)).groupBy(1)
+ val tail2 = env.fromSpout(TestSpout()).flatMap(WordAppendForAlertExecutor("test")).map2(a => ("key1",a)).groupBy(0)
tail1.streamUnion(List(tail2)).map1(a => "xyz")
- //env.execute
+// env.execute()
}
/**
* 1. stream schema
* curl -X POST -H 'Content-Type:application/json' "http://localhost:38080/eagle-service/rest/entities?serviceName=AlertStreamSchemaService" -d '[{"prefix":"alertStreamSchema","tags":{"dataSource":"ds1","streamName":"s1","attrName":"word"},"attrDescription":"word","attrType":"string","category":"","attrValueResolver":""}]'
* 2. policy
- * curl -X POST -H 'Content-Type:application/json' "http://localhost:38080/eagle-service/rest/entities?serviceName=AlertDefinitionService" -d '[{"tags":{"site":"sandbox","dataSource":"ds1","alertExecutorId":"alert1","policyId":"testAlert","policyType":"siddhiCEPEngine"},"desc":"test alert","policyDef":"{\"type\":\"siddhiCEPEngine\",\"expression\":\"from s1 [(str:regexp(word,'\'.*test.*\'')==true)] select * insert into outputStream ;\"}","dedupeDef":"","notificationDef":"","remediationDef":"","enabled":"true"}]'
+ * curl -X POST -H 'Content-Type:application/json' "http://localhost:38080/eagle-service/rest/entities?serviceName=AlertDefinitionService" -d '[{"tags":{"site":"sandbox","dataSource":"ds1","alertExecutorId":"alert1","policyId":"testAlert","policyType":"siddhiCEPEngine"},"desc":"test alert","policyDef":"{\"type\":\"siddhiCEPEngine\",\"expression\":\"from s1 [(str:regexp(word,'\'.*test.*==true)] select * insert into outputStream ;\"}","dedupeDef":"","notificationDef":"","remediationDef":"","enabled":"true"}]'
*/
object testAlertExpansion extends App{
val config : Config = ConfigFactory.load;
val env = new StormExecutionEnvironment(config)
- val tail1 = env.newSource(TestSpout()).withName("testSpout1")
- .flatMap(WordPrependForAlertExecutor("test")).withName("prepend")
+ val tail1 = env.fromSpout(TestSpout()).nameAs("testSpout1")
+ .flatMap(WordPrependForAlertExecutor("test")).nameAs("prepend")
.alertWithConsumer("s1", "alert1")
//env.execute
}
@@ -68,14 +66,14 @@ object testAlertExpansion extends App{
* curl -X POST -H 'Content-Type:application/json' "http://localhost:38080/eagle-service/rest/entities?serviceName=AlertStreamSchemaService" -d '[{"prefix":"alertStreamSchema","tags":{"dataSource":"ds1","streamName":"s1","attrName":"word"},"attrDescription":"word","attrType":"string","category":"","attrValueResolver":""}]'
* curl -X POST -H 'Content-Type:application/json' "http://localhost:38080/eagle-service/rest/entities?serviceName=AlertStreamSchemaService" -d '[{"prefix":"alertStreamSchema","tags":{"dataSource":"ds1","streamName":"s2","attrName":"word"},"attrDescription":"word","attrType":"string","category":"","attrValueResolver":""}]'
* 2. policy
- * curl -X POST -H 'Content-Type:application/json' "http://localhost:38080/eagle-service/rest/entities?serviceName=AlertDefinitionService" -d '[{"tags":{"site":"sandbox","dataSource":"ds1","alertExecutorId":"alert1","policyId":"testAlert","policyType":"siddhiCEPEngine"},"desc":"test alert","policyDef":"{\"type\":\"siddhiCEPEngine\",\"expression\":\"from s1 [(str:regexp(word,'\'.*test.*\'')==true)] select * insert into outputStream ;\"}","dedupeDef":"","notificationDef":"","remediationDef":"","enabled":"true"}]'
+ * curl -X POST -H 'Content-Type:application/json' "http://localhost:38080/eagle-service/rest/entities?serviceName=AlertDefinitionService" -d '[{"tags":{"site":"sandbox","dataSource":"ds1","alertExecutorId":"alert1","policyId":"testAlert","policyType":"siddhiCEPEngine"},"desc":"test alert","policyDef":"{\"type\":\"siddhiCEPEngine\",\"expression\":\"from s1 [(str:regexp(word,'\'.*test.*\)==true)] select * insert into outputStream ;\"}","dedupeDef":"","notificationDef":"","remediationDef":"","enabled":"true"}]'
*/
object testAlertExpansionWithUnion extends App{
val config : Config = ConfigFactory.load;
val env = new StormExecutionEnvironment(config)
- val tail1 = env.newSource(TestSpout()).withName("testSpout1").flatMap(WordPrependForAlertExecutor("test")).withName("prepend") //.map2(a => ("key1",a))
- val tail2 = env.newSource(TestSpout()).flatMap(WordAppendForAlertExecutor("test")) //.map2(a => ("key1",a))
- tail1.streamUnion(List(tail2)).alert(util.Arrays.asList("s1","s2"), "alert1", true)
+ val tail1 = env.fromSpout(TestSpout()).nameAs("testSpout1").flatMap(WordPrependForAlertExecutor("test")).nameAs("prepend") //.map2(a => ("key1",a))
+ val tail2 = env.fromSpout(TestSpout()).flatMap(WordAppendForAlertExecutor("test")) //.map2(a => ("key1",a))
+ tail1.streamUnion(List(tail2)).alert(Seq("s1","s2"), "alert1", consume = true)
//env.execute
}
@@ -83,7 +81,7 @@ object testAlertExpansionWithUnion extends App{
object testStreamUnionExpansionWithSharedSpout extends App{
val config : Config = ConfigFactory.load;
val env = new StormExecutionEnvironment(config)
- val source = env.newSource(TestSpout())
+ val source = env.fromSpout(TestSpout())
val tail1 = source.flatMap(WordPrependForAlertExecutor("test")).map2(a => ("key1",a))
val tail2 = source.flatMap(WordAppendForAlertExecutor("test")).map2(a => ("key1",a))
tail1.streamUnion(List(tail2)).map1(a => {
@@ -96,11 +94,11 @@ object testStreamUnionExpansionWithSharedSpout extends App{
object testStreamUnionExpansionWithSharedSpout_2 extends App{
val config : Config = ConfigFactory.load;
val env = new StormExecutionEnvironment(config)
- val source = env.newSource(TestSpout())
+ val source = env.fromSpout(TestSpout())
val tail1 = source.flatMap(WordPrependForAlertExecutor("test")).map2(a => ("key1",a))
source.streamUnion(List(tail1)).map1(a => {
println(a)
"xyz"
})
// env.execute
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/test/scala/org/apache/eagle/datastream/TestExecutionEnvironment.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/test/scala/org/apache/eagle/datastream/TestExecutionEnvironment.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/test/scala/org/apache/eagle/datastream/TestExecutionEnvironment.scala
new file mode 100644
index 0000000..b43f42e
--- /dev/null
+++ b/eagle-core/eagle-data-process/eagle-stream-process-api/src/test/scala/org/apache/eagle/datastream/TestExecutionEnvironment.scala
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.datastream
+
+import com.typesafe.config.ConfigFactory
+import org.apache.eagle.datastream.storm.StormExecutionEnvironment
+
+/**
+ * @since 12/5/15
+ */
+object TestExecutionEnvironment extends App{
+ val env0 = ExecutionEnvironments.get[StormExecutionEnvironment]
+ println(env0)
+ val config = ConfigFactory.load()
+ val env1 = ExecutionEnvironments.get[StormExecutionEnvironment](config)
+ println(env1)
+ val env2 = ExecutionEnvironments.get[StormExecutionEnvironment](Array[String]("-D","key=value"))
+ println(env2)
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/test/scala/org/apache/eagle/datastream/TestStormRunner.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/test/scala/org/apache/eagle/datastream/TestStormRunner.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/test/scala/org/apache/eagle/datastream/TestStormRunner.scala
index c550182..27fdd88 100644
--- a/eagle-core/eagle-data-process/eagle-stream-process-api/src/test/scala/org/apache/eagle/datastream/TestStormRunner.scala
+++ b/eagle-core/eagle-data-process/eagle-stream-process-api/src/test/scala/org/apache/eagle/datastream/TestStormRunner.scala
@@ -16,9 +16,7 @@
*/
package org.apache.eagle.datastream
-import java.util
-
-import com.typesafe.config.{ConfigFactory, Config}
+import com.typesafe.config.{Config, ConfigFactory}
/**
* explicit union
@@ -27,14 +25,14 @@ import com.typesafe.config.{ConfigFactory, Config}
* 1. stream schema
* curl -X POST -H 'Content-Type:application/json' "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=AlertStreamSchemaService" -d '[{"prefix":"alertStreamSchema","tags":{"dataSource":"ds1","streamName":"s1","attrName":"word"},"attrDescription":"word","attrType":"string","category":"","attrValueResolver":""}]'
* 2. policy
- * curl -X POST -H 'Content-Type:application/json' "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=AlertDefinitionService" -d '[{"tags":{"site":"sandbox","dataSource":"ds1","alertExecutorId":"alert1","policyId":"testAlert","policyType":"siddhiCEPEngine"},"desc":"test alert","policyDef":"{\"type\":\"siddhiCEPEngine\",\"expression\":\"from s1 [(str:regexp(word,'\'.*test.*\'')==true)] select * insert into outputStream ;\"}","dedupeDef":"","notificationDef":"","remediationDef":"","enabled":"true"}]'
+ * curl -X POST -H 'Content-Type:application/json' "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=AlertDefinitionService" -d '[{"tags":{"site":"sandbox","dataSource":"ds1","alertExecutorId":"alert1","policyId":"testAlert","policyType":"siddhiCEPEngine"},"desc":"test alert","policyDef":"{\"type\":\"siddhiCEPEngine\",\"expression\":\"from s1 [(str:regexp(word,'\'.*test.*\)==true)] select * insert into outputStream ;\"}","dedupeDef":"","notificationDef":"","remediationDef":"","enabled":"true"}]'
*/
object UnionForAlert extends App{
val config : Config = ConfigFactory.load;
- val env = ExecutionEnvironmentFactory.getStorm(config)
- val tail1 = env.newSource(TestSpout()).flatMap(WordPrependForAlertExecutor("test")).map2(a => ("key1",a))
- val tail2 = env.newSource(TestSpout()).flatMap(WordAppendForAlertExecutor("test")).map2(a => ("key2",a))
- tail1.streamUnion(List(tail2)).alert(util.Arrays.asList("s1","s2"), "alert1", false)
+ val env = ExecutionEnvironments.getStorm(config)
+ val tail1 = env.fromSpout(TestSpout()).flatMap(WordPrependForAlertExecutor("test")).map2(a => ("key1",a))
+ val tail2 = env.fromSpout(TestSpout()).flatMap(WordAppendForAlertExecutor("test")).map2(a => ("key2",a))
+ tail1.streamUnion(List(tail2)).alert(Seq("s1","s2"), "alert1", consume = false)
//env.execute
}
@@ -43,10 +41,10 @@ object UnionForAlert extends App{
*/
object TestAlertAfterFlatMap extends App{
val config : Config = ConfigFactory.load;
- val env = ExecutionEnvironmentFactory.getStorm(config)
- val tail1 = env.newSource(TestSpout())
+ val env = ExecutionEnvironments.getStorm(config)
+ val tail1 = env.fromSpout(TestSpout())
.flatMap(WordPrependForAlertExecutor("test"))
- .alert(util.Arrays.asList("s1"), "alert1", false)
+ .alert(Seq("s1"), "alert1", consume = false)
//env.execute
}
@@ -55,44 +53,44 @@ object TestAlertAfterFlatMap extends App{
*/
object TestAlertAfterMap extends App{
val config : Config = ConfigFactory.load;
- val env = ExecutionEnvironmentFactory.getStorm(config)
- val tail1 = env.newSource(TestSpout())
+ val env = ExecutionEnvironments.getStorm(config)
+ val tail1 = env.fromSpout(TestSpout())
.flatMap(WordPrependForAlertExecutor2("test"))
.map2(a => ("key", a))
- .alert(util.Arrays.asList("s1"), "alert1", false)
+ .alert(Seq("s1"), "alert1", false)
//env.execute
}
object StormRunnerWithoutSplitOrJoin extends Application{
val config : Config = ConfigFactory.load;
- val env = ExecutionEnvironmentFactory.getStorm(config)
- env.newSource(TestSpout()).flatMap(EchoExecutor()).flatMap(WordPrependExecutor("test"))
+ val env = ExecutionEnvironments.getStorm(config)
+ env.fromSpout(TestSpout()).flatMap(EchoExecutor()).flatMap(WordPrependExecutor("test"))
.flatMap(PatternAlertExecutor("test.*"))
- //env.execute
+// env.execute()
}
object StormRunnerWithSplit extends Application{
val config : Config = ConfigFactory.load;
- val env = ExecutionEnvironmentFactory.getStorm(config)
- val toBeSplit = env.newSource(TestSpout()).flatMap(EchoExecutor())
+ val env = ExecutionEnvironments.getStorm(config)
+ val toBeSplit = env.fromSpout(TestSpout()).flatMap(EchoExecutor())
toBeSplit.flatMap(WordPrependExecutor("test")).flatMap(PatternAlertExecutor("test.*"))
toBeSplit.flatMap(WordAppendExecutor("test"))
- //env.execute
+// env.execute()
}
object StormRunnerWithUnion extends Application{
val config : Config = ConfigFactory.load;
- val env = ExecutionEnvironmentFactory.getStorm(config)
- val tail1 = env.newSource(TestSpout()).flatMap(WordPrependExecutor("test"))
- val tail2 = env.newSource(TestSpout()).flatMap(WordAppendExecutor("test"))
+ val env = ExecutionEnvironments.getStorm(config)
+ val tail1 = env.fromSpout(TestSpout()).flatMap(WordPrependExecutor("test"))
+ val tail2 = env.fromSpout(TestSpout()).flatMap(WordAppendExecutor("test"))
tail1.streamUnion(List(tail2)).flatMap(PatternAlertExecutor(".*test.*"))
- //env.execute
+ env.execute()
}
object StormRunnerWithFilter extends Application{
val config : Config = ConfigFactory.load;
- val env = ExecutionEnvironmentFactory.getStorm(config)
- env.newSource(TestSpout()).flatMap(EchoExecutor()).flatMap(WordPrependExecutor("test")).
+ val env = ExecutionEnvironments.getStorm(config)
+ env.fromSpout(TestSpout()).flatMap(EchoExecutor()).flatMap(WordPrependExecutor("test")).
filter(_=>false).
flatMap(PatternAlertExecutor("test.*"))
//env.execute
@@ -100,8 +98,8 @@ object StormRunnerWithFilter extends Application{
object StormRunnerWithJavaExecutor extends Application{
val config : Config = ConfigFactory.load;
- val env = ExecutionEnvironmentFactory.getStorm(config)
- env.newSource(TestSpout()).flatMap(new JavaEchoExecutor()).flatMap(WordPrependExecutor("test")).
+ val env = ExecutionEnvironments.getStorm(config)
+ env.fromSpout(TestSpout()).flatMap(new JavaEchoExecutor()).flatMap(WordPrependExecutor("test")).
filter(_=>false).
flatMap(PatternAlertExecutor("test.*"))
//env.execute
@@ -109,14 +107,14 @@ object StormRunnerWithJavaExecutor extends Application{
object StormRunnerWithKeyValueSpout extends Application{
val config : Config = ConfigFactory.load;
- val env = ExecutionEnvironmentFactory.getStorm(config)
- env.newSource(TestKeyValueSpout()).groupBy(1).flatMap(new GroupedEchoExecutor()).withParallelism(2)
+ val env = ExecutionEnvironments.getStorm(config)
+ env.fromSpout(TestKeyValueSpout()).groupBy(1).flatMap(new GroupedEchoExecutor()).parallelism(2)
//env.execute
}
object StormRunnerWithKeyValueSpoutRenameOutputFields extends Application{
val config : Config = ConfigFactory.load;
- val env = ExecutionEnvironmentFactory.getStorm(config)
- env.newSource(TestKeyValueSpout()).renameOutputFields(2).groupBy(0).flatMap(new GroupedEchoExecutor()).withParallelism(2)
+ val env = ExecutionEnvironments.getStorm(config)
+ env.fromSpout(TestKeyValueSpout()).withOutputFields(2).groupBy(0).flatMap(new GroupedEchoExecutor()).parallelism(2)
//env.execute
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-api/src/test/scala/org/apache/eagle/datastream/TestTypeSafedDSL.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/test/scala/org/apache/eagle/datastream/TestTypeSafedDSL.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/test/scala/org/apache/eagle/datastream/TestTypeSafedDSL.scala
new file mode 100644
index 0000000..cf95304
--- /dev/null
+++ b/eagle-core/eagle-data-process/eagle-stream-process-api/src/test/scala/org/apache/eagle/datastream/TestTypeSafedDSL.scala
@@ -0,0 +1,87 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eagle.datastream
+
+import org.apache.eagle.datastream.storm.StormExecutionEnvironment
+
+/**
+ * @since 12/4/15
+ */
+case class Entity(name:String,value:Double,var inc:Int=0)
+
+object TestIterableWithGroupBy extends App {
+
+ val env = ExecutionEnvironments.get[StormExecutionEnvironment](args)
+
+ val tuples = Seq(
+ Entity("a", 1),
+ Entity("a", 2),
+ Entity("a", 3),
+ Entity("b", 2),
+ Entity("c", 3),
+ Entity("d", 3)
+ )
+
+ env.from(tuples)
+ .groupByKey(_.name)
+ .map(o => {o.inc += 2;o})
+ .filter(_.name != "b")
+ .filter(_.name != "c")
+ .groupByKey(o=>(o.name,o.value))
+ .map(o => (o.name,o))
+ .map(o => (o._1,o._2.value,o._2.inc))
+ .foreach(println)
+
+ env.execute()
+}
+
+object TestIterableWithGroupByCircularly extends App{
+ val env = ExecutionEnvironments.get[StormExecutionEnvironment](args)
+ val tuples = Seq(
+ Entity("a", 1),
+ Entity("a", 2),
+ Entity("a", 3),
+ Entity("b", 2),
+ Entity("c", 3),
+ Entity("d", 3)
+ )
+ env.from(tuples,recycle = true)
+ .map(o => {o.inc += 2;o})
+ .groupByKey(_.name)
+ .foreach(println)
+ env.execute()
+}
+
+object TestGroupByKeyOnSpoutproxy extends App{
+ val env = ExecutionEnvironments.get[StormExecutionEnvironment](args)
+
+ val tuples = Seq(
+ Entity("a", 1),
+ Entity("a", 2),
+ Entity("a", 3),
+ Entity("b", 2),
+ Entity("c", 3),
+ Entity("d", 3)
+ )
+
+ env.fromSpout[String](TestSpout())
+ .groupByKey(_.charAt(0))
+ .foreach(println)
+ env.execute()
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/java/org/apache/eagle/datastream/JavaMapper.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/java/org/apache/eagle/datastream/JavaMapper.java b/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/java/org/apache/eagle/datastream/JavaMapper.java
new file mode 100644
index 0000000..7e66478
--- /dev/null
+++ b/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/java/org/apache/eagle/datastream/JavaMapper.java
@@ -0,0 +1,26 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.datastream;
+
+import java.util.List;
+
+public interface JavaMapper {
+ List<Object> map(List<Object> input);
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/java/org/apache/eagle/datastream/JavaTypeCompatible.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/java/org/apache/eagle/datastream/JavaTypeCompatible.java b/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/java/org/apache/eagle/datastream/JavaTypeCompatible.java
new file mode 100644
index 0000000..39ce6c2
--- /dev/null
+++ b/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/java/org/apache/eagle/datastream/JavaTypeCompatible.java
@@ -0,0 +1,24 @@
+package org.apache.eagle.datastream;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * @since 12/8/15
+ */
+public interface JavaTypeCompatible {
+ Class<?> getType();
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/scala/org/apache/eagle/datastream/EagleTuple.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/scala/org/apache/eagle/datastream/EagleTuple.scala b/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/scala/org/apache/eagle/datastream/EagleTuple.scala
index 5c89a41..63aa4fb 100644
--- a/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/scala/org/apache/eagle/datastream/EagleTuple.scala
+++ b/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/scala/org/apache/eagle/datastream/EagleTuple.scala
@@ -21,26 +21,18 @@ trait EagleTuple extends Serializable{
}
case class Tuple1[T0](f0 : T0) extends EagleTuple{
- override def getList : List[AnyRef] = {
- return List(f0.asInstanceOf[AnyRef])
- }
+ override def getList : List[AnyRef] = List(f0.asInstanceOf[AnyRef])
+
}
case class Tuple2[T0, T1](f0 : T0, f1: T1) extends EagleTuple{
- override def getList : List[AnyRef] = {
- return List(f0.asInstanceOf[AnyRef], f1.asInstanceOf[AnyRef])
- }
+ override def getList : List[AnyRef] = List(f0.asInstanceOf[AnyRef], f1.asInstanceOf[AnyRef])
}
case class Tuple3[T0, T1, T2](f0 : T0, f1: T1, f2: T2) extends EagleTuple{
- override def getList : List[AnyRef] = {
- return List(f0.asInstanceOf[AnyRef], f1.asInstanceOf[AnyRef], f2.asInstanceOf[AnyRef])
- }
+ override def getList : List[AnyRef] = List(f0.asInstanceOf[AnyRef], f1.asInstanceOf[AnyRef], f2.asInstanceOf[AnyRef])
}
case class Tuple4[T0, T1, T2, T3](f0 : T0, f1: T1, f2: T2, f3 : T3) extends EagleTuple{
- override def getList : List[AnyRef] = {
- return List(f0.asInstanceOf[AnyRef], f1.asInstanceOf[AnyRef], f2.asInstanceOf[AnyRef], f3.asInstanceOf[AnyRef])
- }
-}
-
+ override def getList : List[AnyRef] = List(f0.asInstanceOf[AnyRef], f1.asInstanceOf[AnyRef], f2.asInstanceOf[AnyRef], f3.asInstanceOf[AnyRef])
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/scala/org/apache/eagle/datastream/FlatMapper.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/scala/org/apache/eagle/datastream/FlatMapper.scala b/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/scala/org/apache/eagle/datastream/FlatMapper.scala
index 22e063d..8569ae5 100644
--- a/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/scala/org/apache/eagle/datastream/FlatMapper.scala
+++ b/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/scala/org/apache/eagle/datastream/FlatMapper.scala
@@ -16,6 +16,6 @@
*/
package org.apache.eagle.datastream
-trait FlatMapper[T, R] extends Serializable{
- def flatMap(input : T, collector : Collector[R])
+trait FlatMapper[T] extends Serializable {
+ def flatMap(input : Seq[AnyRef], collector : Collector[T])
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/scala/org/apache/eagle/datastream/StormStreamExecutor.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/scala/org/apache/eagle/datastream/StormStreamExecutor.scala b/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/scala/org/apache/eagle/datastream/StormStreamExecutor.scala
index b3d77e9..22ae213 100644
--- a/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/scala/org/apache/eagle/datastream/StormStreamExecutor.scala
+++ b/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/scala/org/apache/eagle/datastream/StormStreamExecutor.scala
@@ -17,18 +17,23 @@
package org.apache.eagle.datastream
import com.typesafe.config.Config
+import scala.collection.JavaConverters._
-trait StormStreamExecutor[R <: EagleTuple] extends FlatMapper[Seq[AnyRef], R] {
+trait StormStreamExecutor[R <: EagleTuple] extends FlatMapper[R] {
def prepareConfig(config : Config)
def init
def fields : Array[String]
}
-trait JavaStormStreamExecutor[R <: EagleTuple] extends FlatMapper[java.util.List[AnyRef], R] {
+trait JavaStormStreamExecutor[R <: EagleTuple] extends FlatMapper[R] {
def prepareConfig(config : Config)
def init
def fields : Array[String]
override def toString() = this.getClass.getSimpleName
+
+ override def flatMap(input : Seq[AnyRef], collector : Collector[R]) = flatMap(input.asJava,collector)
+
+ def flatMap(input : java.util.List[AnyRef], collector : Collector[R])
}
abstract class StormStreamExecutor1[T0] extends StormStreamExecutor[Tuple1[T0]] {
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-core/eagle-query/eagle-client-base/src/main/java/org/apache/eagle/service/client/EagleServiceConnector.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/eagle-client-base/src/main/java/org/apache/eagle/service/client/EagleServiceConnector.java b/eagle-core/eagle-query/eagle-client-base/src/main/java/org/apache/eagle/service/client/EagleServiceConnector.java
index d9363e8..2b2b1ac 100644
--- a/eagle-core/eagle-query/eagle-client-base/src/main/java/org/apache/eagle/service/client/EagleServiceConnector.java
+++ b/eagle-core/eagle-query/eagle-client-base/src/main/java/org/apache/eagle/service/client/EagleServiceConnector.java
@@ -20,10 +20,12 @@ package org.apache.eagle.service.client;
import com.typesafe.config.Config;
import org.apache.eagle.common.config.EagleConfigConstants;
+import java.io.Serializable;
+
/**
* Some common codes to enable DAO through eagle service including service host/post, credential population etc.
*/
-public class EagleServiceConnector {
+public class EagleServiceConnector implements Serializable{
private final String eagleServiceHost;
private final Integer eagleServicePort;
private String username;
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-security/eagle-metric-collection/src/main/java/org/apache/eagle/metric/kafka/EagleMetricCollectorMain.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-metric-collection/src/main/java/org/apache/eagle/metric/kafka/EagleMetricCollectorMain.java b/eagle-security/eagle-metric-collection/src/main/java/org/apache/eagle/metric/kafka/EagleMetricCollectorMain.java
index 218b812..4eeda05 100644
--- a/eagle-security/eagle-metric-collection/src/main/java/org/apache/eagle/metric/kafka/EagleMetricCollectorMain.java
+++ b/eagle-security/eagle-metric-collection/src/main/java/org/apache/eagle/metric/kafka/EagleMetricCollectorMain.java
@@ -19,12 +19,10 @@ package org.apache.eagle.metric.kafka;
import backtype.storm.spout.SchemeAsMultiScheme;
import backtype.storm.topology.base.BaseRichSpout;
import com.typesafe.config.Config;
-import com.typesafe.config.ConfigFactory;
import org.apache.eagle.dataproc.impl.storm.kafka.KafkaSourcedSpoutProvider;
import org.apache.eagle.dataproc.impl.storm.kafka.KafkaSourcedSpoutScheme;
-import org.apache.eagle.dataproc.util.ConfigOptionParser;
-import org.apache.eagle.datastream.ExecutionEnvironmentFactory;
-import org.apache.eagle.datastream.StormExecutionEnvironment;
+import org.apache.eagle.datastream.ExecutionEnvironments;
+import org.apache.eagle.datastream.storm.StormExecutionEnvironment;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import storm.kafka.BrokerHosts;
@@ -42,13 +40,8 @@ public class EagleMetricCollectorMain {
private static final Logger LOG = LoggerFactory.getLogger(EagleMetricCollectorMain.class);
public static void main(String[] args) throws Exception {
- new ConfigOptionParser().load(args);
- //System.setProperty("config.resource", "/application.local.conf");
-
- Config config = ConfigFactory.load();
-
- StormExecutionEnvironment env = ExecutionEnvironmentFactory.getStorm(config);
-
+ StormExecutionEnvironment env = ExecutionEnvironments.getStorm(args);
+ Config config = env.getConfig();
String deserClsName = config.getString("dataSourceConfig.deserializerClass");
final KafkaSourcedSpoutScheme scheme = new KafkaSourcedSpoutScheme(deserClsName, config) {
@Override
@@ -60,6 +53,8 @@ public class EagleMetricCollectorMain {
}
};
+
+ // TODO: Refactored the anonymous in to independen class file, avoiding too complex logic in main method
KafkaSourcedSpoutProvider kafkaMessageSpoutProvider = new KafkaSourcedSpoutProvider() {
@Override
public BaseRichSpout getSpout(Config context) {
@@ -119,9 +114,9 @@ public class EagleMetricCollectorMain {
}
};
- env.newSource(new KafkaOffsetSourceSpoutProvider().getSpout(config)).renameOutputFields(0).withName("kafkaLogLagChecker");
- env.newSource(kafkaMessageSpoutProvider.getSpout(config)).renameOutputFields(2).withName("kafkaMessageFetcher").groupBy(Arrays.asList(0))
+ env.fromSpout(new KafkaOffsetSourceSpoutProvider()).withOutputFields(0).nameAs("kafkaLogLagChecker");
+ env.fromSpout(kafkaMessageSpoutProvider).withOutputFields(2).nameAs("kafkaMessageFetcher").groupBy(Arrays.asList(0))
.flatMap(new KafkaMessageDistributionExecutor());
env.execute();
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-security/eagle-metric-collection/src/main/java/org/apache/eagle/metric/kafka/KafkaOffsetSourceSpoutProvider.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-metric-collection/src/main/java/org/apache/eagle/metric/kafka/KafkaOffsetSourceSpoutProvider.java b/eagle-security/eagle-metric-collection/src/main/java/org/apache/eagle/metric/kafka/KafkaOffsetSourceSpoutProvider.java
index 5fd02fd..8f25564 100644
--- a/eagle-security/eagle-metric-collection/src/main/java/org/apache/eagle/metric/kafka/KafkaOffsetSourceSpoutProvider.java
+++ b/eagle-security/eagle-metric-collection/src/main/java/org/apache/eagle/metric/kafka/KafkaOffsetSourceSpoutProvider.java
@@ -19,14 +19,11 @@ package org.apache.eagle.metric.kafka;
import backtype.storm.topology.base.BaseRichSpout;
import com.typesafe.config.Config;
import org.apache.eagle.common.config.EagleConfigConstants;
+import org.apache.eagle.dataproc.impl.storm.StormSpoutProvider;
import org.apache.eagle.dataproc.impl.storm.zookeeper.ZKStateConfig;
import org.apache.eagle.service.client.ServiceConfig;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-public class KafkaOffsetSourceSpoutProvider {
- private static final Logger LOG = LoggerFactory.getLogger(KafkaOffsetSourceSpoutProvider.class);
-
+public class KafkaOffsetSourceSpoutProvider implements StormSpoutProvider {
public BaseRichSpout getSpout(Config config){
ZKStateConfig zkStateConfig = new ZKStateConfig();
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/hdfs/entity/FileSensitivityAPIEntity.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/hdfs/entity/FileSensitivityAPIEntity.java b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/hdfs/entity/FileSensitivityAPIEntity.java
index e4e63c9..00b6ca7 100644
--- a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/hdfs/entity/FileSensitivityAPIEntity.java
+++ b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/hdfs/entity/FileSensitivityAPIEntity.java
@@ -49,4 +49,4 @@ public class FileSensitivityAPIEntity extends TaggedLogAPIEntity{
this.sensitivityType = sensitivityType;
valueChanged("sensitivityType");
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/partition/DataDistributionDaoImpl.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/partition/DataDistributionDaoImpl.java b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/partition/DataDistributionDaoImpl.java
index e808502..7d32091 100644
--- a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/partition/DataDistributionDaoImpl.java
+++ b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/partition/DataDistributionDaoImpl.java
@@ -20,7 +20,6 @@
package org.apache.eagle.security.partition;
import com.sun.jersey.api.client.WebResource;
-import org.apache.commons.lang.time.DateUtils;
import org.apache.eagle.log.entity.GenericServiceAPIResponseEntity;
import org.apache.eagle.metric.MetricConstants;
import org.apache.eagle.partition.DataDistributionDao;
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-security/eagle-security-hbase-securitylog/src/main/java/org/apache/eagle/security/hbase/HbaseAuditLogProcessorMain.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-securitylog/src/main/java/org/apache/eagle/security/hbase/HbaseAuditLogProcessorMain.java b/eagle-security/eagle-security-hbase-securitylog/src/main/java/org/apache/eagle/security/hbase/HbaseAuditLogProcessorMain.java
index 1a90cbd..b00f39c 100644
--- a/eagle-security/eagle-security-hbase-securitylog/src/main/java/org/apache/eagle/security/hbase/HbaseAuditLogProcessorMain.java
+++ b/eagle-security/eagle-security-hbase-securitylog/src/main/java/org/apache/eagle/security/hbase/HbaseAuditLogProcessorMain.java
@@ -17,32 +17,17 @@
*/
package org.apache.eagle.security.hbase;
-
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigFactory;
-import com.typesafe.config.ConfigRenderOptions;
import org.apache.eagle.dataproc.impl.storm.kafka.KafkaSourcedSpoutProvider;
-import org.apache.eagle.dataproc.util.ConfigOptionParser;
-import org.apache.eagle.datastream.ExecutionEnvironmentFactory;
-import org.apache.eagle.datastream.StormExecutionEnvironment;
+import org.apache.eagle.datastream.ExecutionEnvironments;
+import org.apache.eagle.datastream.storm.StormExecutionEnvironment;
import org.apache.eagle.security.hbase.sensitivity.HbaseResourceSensitivityDataJoinExecutor;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
public class HbaseAuditLogProcessorMain {
- private static final Logger LOG = LoggerFactory.getLogger(HbaseAuditLogProcessorMain.class);
-
public static void main(String[] args) throws Exception{
- Config config = new ConfigOptionParser().load(args);
-
- LOG.info("Config class: " + config.getClass().getCanonicalName());
-
- if(LOG.isDebugEnabled()) LOG.debug("Config content:"+config.root().render(ConfigRenderOptions.concise()));
-
- StormExecutionEnvironment env = ExecutionEnvironmentFactory.getStorm(config);
- env.newSource(new KafkaSourcedSpoutProvider().getSpout(config)).renameOutputFields(1).withName("kafkaMsgConsumer")
+ StormExecutionEnvironment env = ExecutionEnvironments.getStorm(args);
+ env.fromSpout(new KafkaSourcedSpoutProvider()).withOutputFields(1).nameAs("kafkaMsgConsumer")
.flatMap(new HbaseResourceSensitivityDataJoinExecutor())
.alertWithConsumer("hbaseSecurityLogEventStream", "hbaseSecurityLogAlertExecutor");
env.execute();
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-security/eagle-security-hbase-securitylog/src/main/java/org/apache/eagle/security/hbase/sensitivity/HbaseResourceSensitivityDataJoinExecutor.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-securitylog/src/main/java/org/apache/eagle/security/hbase/sensitivity/HbaseResourceSensitivityDataJoinExecutor.java b/eagle-security/eagle-security-hbase-securitylog/src/main/java/org/apache/eagle/security/hbase/sensitivity/HbaseResourceSensitivityDataJoinExecutor.java
index 2e8b852..5bb2aff 100644
--- a/eagle-security/eagle-security-hbase-securitylog/src/main/java/org/apache/eagle/security/hbase/sensitivity/HbaseResourceSensitivityDataJoinExecutor.java
+++ b/eagle-security/eagle-security-hbase-securitylog/src/main/java/org/apache/eagle/security/hbase/sensitivity/HbaseResourceSensitivityDataJoinExecutor.java
@@ -55,7 +55,6 @@ public class HbaseResourceSensitivityDataJoinExecutor extends JavaStormStreamExe
}
}
-
@Override
public void flatMap(List<Object> input, Collector<Tuple2<String, Map>> outputCollector){
@SuppressWarnings("unchecked")
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-security/eagle-security-hdfs-auditlog/run_auditlog_topology.sh
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hdfs-auditlog/run_auditlog_topology.sh b/eagle-security/eagle-security-hdfs-auditlog/run_auditlog_topology.sh
index 8e8d969..38d8ce9 100755
--- a/eagle-security/eagle-security-hdfs-auditlog/run_auditlog_topology.sh
+++ b/eagle-security/eagle-security-hdfs-auditlog/run_auditlog_topology.sh
@@ -1,2 +1,17 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
export JAVA_HOME=$(/usr/libexec/java_home -v 1.7)
mvn -X exec:java -Dexec.mainClass="eagle.security.auditlog.HdfsAuditLogProcessorMain"
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-security/eagle-security-hdfs-auditlog/run_hostname_lookkup.sh
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hdfs-auditlog/run_hostname_lookkup.sh b/eagle-security/eagle-security-hdfs-auditlog/run_hostname_lookkup.sh
index ba761de..fffd8c0 100755
--- a/eagle-security/eagle-security-hdfs-auditlog/run_hostname_lookkup.sh
+++ b/eagle-security/eagle-security-hdfs-auditlog/run_hostname_lookkup.sh
@@ -1,3 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
### refer to http://mojo.codehaus.org/exec-maven-plugin/usage.html
### java goal is to execute program within maven JVM
export JAVA_HOME=$(/usr/libexec/java_home -v 1.7)
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-security/eagle-security-hdfs-auditlog/run_message_producer.sh
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hdfs-auditlog/run_message_producer.sh b/eagle-security/eagle-security-hdfs-auditlog/run_message_producer.sh
index 9ecaa26..dbe50f3 100755
--- a/eagle-security/eagle-security-hdfs-auditlog/run_message_producer.sh
+++ b/eagle-security/eagle-security-hdfs-auditlog/run_message_producer.sh
@@ -1,3 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
export JAVA_HOME=$(/usr/libexec/java_home -v 1.7)
### mvn -X exec:java -Dexec.args="-input /Users/user1/Downloads/hdfs-audit.log.2015-01-22-22 -maxNum 10000000 -topic hdfs_audit_log" -Pproducer
mvn exec:java -Dexec.mainClass="eagle.security.auditlog.util.AuditLogKafkaProducer" -Dexec.args="-input /Users/user1/Downloads/hdfs-audit.log.2015-01-22-22 -maxNum 20000 -topic hdfs_audit_log"
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-security/eagle-security-hdfs-auditlog/run_message_producer_in_assembly.sh
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hdfs-auditlog/run_message_producer_in_assembly.sh b/eagle-security/eagle-security-hdfs-auditlog/run_message_producer_in_assembly.sh
index 20d7cae..128926d 100644
--- a/eagle-security/eagle-security-hdfs-auditlog/run_message_producer_in_assembly.sh
+++ b/eagle-security/eagle-security-hdfs-auditlog/run_message_producer_in_assembly.sh
@@ -1 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
java -cp target/eagle-app-security-activity-monitoring-dataproc-1.0.0-jar-with-dependencies.jar:/Users/user1/.m2/repository/org/slf4j/slf4j-api/1.4.3/slf4j-api-1.4.3.jar:/Users/user1/.m2/repository/org/slf4j/slf4j-log4j12/1.4.3/slf4j-log4j12-1.4.3.jar eagle.app.security.dataproc.util.AuditLogKafkaProducer -input /Users/user1/projects/eagle-app-security-activity-monitoring/eagle-app-security-activity-monitoring-dataproc/src/main/resources/auditlog/auditlog.1 --topic hdfs_audit_log
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-security/eagle-security-hdfs-auditlog/src/assembly/eagle-dam-auditlog-assembly.xml
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hdfs-auditlog/src/assembly/eagle-dam-auditlog-assembly.xml b/eagle-security/eagle-security-hdfs-auditlog/src/assembly/eagle-dam-auditlog-assembly.xml
index f3b17d4..0acf619 100644
--- a/eagle-security/eagle-security-hdfs-auditlog/src/assembly/eagle-dam-auditlog-assembly.xml
+++ b/eagle-security/eagle-security-hdfs-auditlog/src/assembly/eagle-dam-auditlog-assembly.xml
@@ -1,3 +1,20 @@
+<?xml version="1.0"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/HdfsAuditLogProcessorMain.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/HdfsAuditLogProcessorMain.java b/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/HdfsAuditLogProcessorMain.java
index eb84fd3..f63a9be 100644
--- a/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/HdfsAuditLogProcessorMain.java
+++ b/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/HdfsAuditLogProcessorMain.java
@@ -20,32 +20,27 @@ package org.apache.eagle.security.auditlog;
import backtype.storm.spout.SchemeAsMultiScheme;
import com.typesafe.config.Config;
-import com.typesafe.config.ConfigRenderOptions;
import org.apache.commons.lang3.time.DateUtils;
import org.apache.eagle.common.config.EagleConfigConstants;
import org.apache.eagle.dataproc.impl.storm.kafka.KafkaSourcedSpoutProvider;
import org.apache.eagle.dataproc.impl.storm.kafka.KafkaSourcedSpoutScheme;
-import org.apache.eagle.dataproc.util.ConfigOptionParser;
-import org.apache.eagle.datastream.ExecutionEnvironmentFactory;
-import org.apache.eagle.datastream.StormExecutionEnvironment;
+import org.apache.eagle.datastream.ExecutionEnvironments;
+import org.apache.eagle.datastream.core.StreamProducer;
+import org.apache.eagle.datastream.storm.StormExecutionEnvironment;
import org.apache.eagle.partition.DataDistributionDao;
import org.apache.eagle.partition.PartitionAlgorithm;
import org.apache.eagle.partition.PartitionStrategy;
import org.apache.eagle.partition.PartitionStrategyImpl;
import org.apache.eagle.security.partition.DataDistributionDaoImpl;
import org.apache.eagle.security.partition.GreedyPartitionAlgorithm;
-import org.apache.eagle.datastream.StreamProducer;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
public class HdfsAuditLogProcessorMain {
- private static final Logger LOG = LoggerFactory.getLogger(HdfsAuditLogProcessorMain.class);
-
public static PartitionStrategy createStrategy(Config config) {
+ // TODO: Refactor configuration structure to avoid repeated config processing configure ~ hao
String host = config.getString(EagleConfigConstants.EAGLE_PROPS + "." + EagleConfigConstants.EAGLE_SERVICE + "." + EagleConfigConstants.HOST);
Integer port = config.getInt(EagleConfigConstants.EAGLE_PROPS + "." + EagleConfigConstants.EAGLE_SERVICE + "." + EagleConfigConstants.PORT);
String username = config.getString(EagleConfigConstants.EAGLE_PROPS + "." + EagleConfigConstants.EAGLE_SERVICE + "." + EagleConfigConstants.USERNAME);
@@ -72,6 +67,7 @@ public class HdfsAuditLogProcessorMain {
return Arrays.asList(map.get("user"), tmp);
}
};
+
KafkaSourcedSpoutProvider provider = new KafkaSourcedSpoutProvider() {
@Override
public SchemeAsMultiScheme getStreamScheme(String deserClsName, Config context) {
@@ -81,8 +77,9 @@ public class HdfsAuditLogProcessorMain {
return provider;
}
- public static void execWithDefaultPartition(Config config, StormExecutionEnvironment env, KafkaSourcedSpoutProvider provider) {
- StreamProducer source = env.newSource(provider.getSpout(config)).renameOutputFields(2).withName("kafkaMsgConsumer").groupBy(Arrays.asList(0));
+ @SuppressWarnings("unchecked")
+ public static void execWithDefaultPartition(StormExecutionEnvironment env, KafkaSourcedSpoutProvider provider) {
+ StreamProducer source = env.fromSpout(provider).withOutputFields(2).nameAs("kafkaMsgConsumer").groupBy(Arrays.asList(0));
StreamProducer reassembler = source.flatMap(new HdfsUserCommandReassembler()).groupBy(Arrays.asList(0));
source.streamUnion(reassembler)
.flatMap(new FileSensitivityDataJoinExecutor()).groupBy(Arrays.asList(0))
@@ -91,9 +88,10 @@ public class HdfsAuditLogProcessorMain {
env.execute();
}
- public static void execWithBalancedPartition(Config config, StormExecutionEnvironment env, KafkaSourcedSpoutProvider provider) {
- PartitionStrategy strategy = createStrategy(config);
- StreamProducer source = env.newSource(provider.getSpout(config)).renameOutputFields(2).withName("kafkaMsgConsumer").customGroupBy(strategy);
+ @SuppressWarnings("unchecked")
+ public static void execWithBalancedPartition(StormExecutionEnvironment env, KafkaSourcedSpoutProvider provider) {
+ PartitionStrategy strategy = createStrategy(env.getConfig());
+ StreamProducer source = env.fromSpout(provider).withOutputFields(2).nameAs("kafkaMsgConsumer").groupBy(strategy);
StreamProducer reassembler = source.flatMap(new HdfsUserCommandReassembler()).groupBy(Arrays.asList(0));
source.streamUnion(reassembler)
.flatMap(new FileSensitivityDataJoinExecutor()).groupBy(Arrays.asList(0))
@@ -103,18 +101,14 @@ public class HdfsAuditLogProcessorMain {
}
public static void main(String[] args) throws Exception{
- Config config = new ConfigOptionParser().load(args);
- LOG.info("Config class: " + config.getClass().getCanonicalName());
- if(LOG.isDebugEnabled()) LOG.debug("Config content:"+config.root().render(ConfigRenderOptions.concise()));
-
- StormExecutionEnvironment env = ExecutionEnvironmentFactory.getStorm(config);
- KafkaSourcedSpoutProvider provider = createProvider(config);
- Boolean balancePartition = config.hasPath("eagleProps.balancePartitionEnabled") ? config.getBoolean("eagleProps.balancePartitionEnabled") : false;
+ StormExecutionEnvironment env = ExecutionEnvironments.getStorm(args);
+ Config config = env.getConfig();
+ KafkaSourcedSpoutProvider provider = createProvider(env.getConfig());
+ Boolean balancePartition = config.hasPath("eagleProps.balancePartitionEnabled") && config.getBoolean("eagleProps.balancePartitionEnabled");
if (balancePartition) {
- execWithBalancedPartition(config, env, provider);
- }
- else {
- execWithDefaultPartition(config, env, provider);
+ execWithBalancedPartition(env, provider);
+ } else {
+ execWithDefaultPartition(env, provider);
}
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/HdfsUserCommandReassembler.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/HdfsUserCommandReassembler.java b/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/HdfsUserCommandReassembler.java
index c6894dc..c2d50db 100644
--- a/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/HdfsUserCommandReassembler.java
+++ b/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/HdfsUserCommandReassembler.java
@@ -36,9 +36,6 @@ import org.wso2.siddhi.core.stream.input.InputHandler;
import java.util.*;
-/**
- * Created by yonzhang on 11/20/15.
- */
public class HdfsUserCommandReassembler extends JavaStormStreamExecutor2<String, Map> {
private static final Logger LOG = LoggerFactory.getLogger(HdfsUserCommandReassembler.class);
private Config config;
@@ -142,14 +139,14 @@ public class HdfsUserCommandReassembler extends JavaStormStreamExecutor2<String,
@Override
public void flatMap(List<Object> input, Collector<Tuple2<String, Map>> collector) {
- LOG.debug("incoming event:" + input.get(1));
- SortedMap<String, Object> toBeCopied = (SortedMap<String, Object>)input.get(1);
- SortedMap<String, Object> event = new TreeMap<String, Object>(toBeCopied);
+ if(LOG.isDebugEnabled()) LOG.debug("incoming event:" + input.get(1));
+ SortedMap<String, Object> toBeCopied = (SortedMap<String, Object>) input.get(1);
+ SortedMap<String, Object> event = new TreeMap<>(toBeCopied);
Object[] siddhiEvent = convertToSiddhiEvent(collector, event);
try {
inputHandler.send(siddhiEvent);
- }catch(Exception ex){
- LOG.error("fail sending event to Siddhi pattern engine", ex);
+ } catch (Exception ex){
+ LOG.error("Fail sending event to Siddhi pattern engine", ex);
throw new IllegalStateException(ex);
}
}
@@ -163,4 +160,4 @@ public class HdfsUserCommandReassembler extends JavaStormStreamExecutor2<String,
}
return siddhiEvent;
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/application.conf
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/application.conf b/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/application.conf
index 1d13082..5a82fda 100644
--- a/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/application.conf
+++ b/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/application.conf
@@ -55,7 +55,7 @@
#"kafkaStatisticRangeInMin" : 60,
"eagleService": {
"host": "localhost",
- "port": 9099,
+ "port": 38080,
"username": "admin",
"password": "secret"
},
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/auditlog/auditlog.1
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/auditlog/auditlog.1 b/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/auditlog/auditlog.1
deleted file mode 100644
index 7d470f4..0000000
--- a/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/auditlog/auditlog.1
+++ /dev/null
@@ -1,16 +0,0 @@
-2015-09-21 21:36:52,172 INFO FSNamesystem.audit: allowed=true ugi=ambari-qa (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/tmp/hive dst=null perm=null proto=rpc
-2015-09-21 21:36:52,268 INFO FSNamesystem.audit: allowed=true ugi=ambari-qa (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/tmp/hive dst=null perm=null proto=rpc
-2015-09-21 21:36:52,274 INFO FSNamesystem.audit: allowed=true ugi=ambari-qa (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/tmp/hive/ambari-qa dst=null perm=null proto=rpc
-2015-09-21 21:36:52,430 INFO FSNamesystem.audit: allowed=true ugi=ambari-qa (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/tmp/hive/ambari-qa/e7b21624-afca-473d-99ef-847614a46929 dst=null perm=null proto=rpc
-2015-09-21 21:36:52,437 INFO FSNamesystem.audit: allowed=true ugi=ambari-qa (auth:SIMPLE) ip=/10.0.2.15 cmd=mkdirs src=/tmp/hive/ambari-qa/e7b21624-afca-473d-99ef-847614a46929 dst=null perm=ambari-qa:hdfs:rwx------ proto=rpc
-2015-09-21 21:36:52,443 INFO FSNamesystem.audit: allowed=true ugi=ambari-qa (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/tmp/hive/ambari-qa/e7b21624-afca-473d-99ef-847614a46929 dst=null perm=null proto=rpc
-2015-09-21 21:36:52,448 INFO FSNamesystem.audit: allowed=true ugi=ambari-qa (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/tmp/hive/ambari-qa/e7b21624-afca-473d-99ef-847614a46929/_tmp_space.db dst=null perm=null proto=rpc
-2015-09-21 21:36:52,451 INFO FSNamesystem.audit: allowed=true ugi=ambari-qa (auth:SIMPLE) ip=/10.0.2.15 cmd=mkdirs src=/tmp/hive/ambari-qa/e7b21624-afca-473d-99ef-847614a46929/_tmp_space.db dst=null perm=ambari-qa:hdfs:rwx------ proto=rpc
-2015-09-21 21:36:52,453 INFO FSNamesystem.audit: allowed=true ugi=ambari-qa (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/tmp/hive/ambari-qa/e7b21624-afca-473d-99ef-847614a46929/_tmp_space.db dst=null perm=null proto=rpc
-2015-09-21 21:36:55,455 INFO FSNamesystem.audit: allowed=true ugi=ambari-qa (auth:SIMPLE) ip=/10.0.2.15 cmd=delete src=/tmp/hive/ambari-qa/e7b21624-afca-473d-99ef-847614a46929 dst=null perm=null proto=rpc
-2015-09-21 21:36:55,477 INFO FSNamesystem.audit: allowed=true ugi=ambari-qa (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/tmp/hive/ambari-qa/e7b21624-afca-473d-99ef-847614a46929 dst=null perm=null proto=rpc
-2015-09-21 21:36:55,480 INFO FSNamesystem.audit: allowed=true ugi=ambari-qa (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/tmp/hive/ambari-qa/e7b21624-afca-473d-99ef-847614a46929/_tmp_space.db dst=null perm=null proto=rpc
-2015-09-21 21:37:20,809 INFO FSNamesystem.audit: allowed=true ugi=oozie (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/oozie/share/lib dst=null perm=null proto=rpc
-2015-09-21 21:37:24,330 INFO FSNamesystem.audit: allowed=false ugi=hive (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/ranger/audit/hiveServer2/20150921/sandbox.hortonworks.com-audit.log dst=null perm=null proto=rpc
-2015-09-21 21:37:24,334 INFO FSNamesystem.audit: allowed=false ugi=hive (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/ranger/audit/hiveServer2/20150921/sandbox.hortonworks.com-audit-1.log dst=null perm=null proto=rpc
-2015-09-21 21:37:24,338 INFO FSNamesystem.audit: allowed=false ugi=hive (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/ranger/audit/hiveServer2/20150921 dst=null perm=null proto=rpc
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/security-auditlog-storm.yaml
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/security-auditlog-storm.yaml b/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/security-auditlog-storm.yaml
index 16368cd..a68a323 100644
--- a/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/security-auditlog-storm.yaml
+++ b/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/security-auditlog-storm.yaml
@@ -1,3 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
topology.workers: 1
topology.acker.executors: 1
topology.tasks: 1
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-security/eagle-security-hdfs-securitylog/src/main/java/org/apache/eagle/security/securitylog/HDFSSecurityLogProcessorMain.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hdfs-securitylog/src/main/java/org/apache/eagle/security/securitylog/HDFSSecurityLogProcessorMain.java b/eagle-security/eagle-security-hdfs-securitylog/src/main/java/org/apache/eagle/security/securitylog/HDFSSecurityLogProcessorMain.java
index 9238cad..07bdd3b 100644
--- a/eagle-security/eagle-security-hdfs-securitylog/src/main/java/org/apache/eagle/security/securitylog/HDFSSecurityLogProcessorMain.java
+++ b/eagle-security/eagle-security-hdfs-securitylog/src/main/java/org/apache/eagle/security/securitylog/HDFSSecurityLogProcessorMain.java
@@ -17,32 +17,15 @@
*/
package org.apache.eagle.security.securitylog;
-
-
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigFactory;
-import com.typesafe.config.ConfigRenderOptions;
import org.apache.eagle.dataproc.impl.storm.kafka.KafkaSourcedSpoutProvider;
-import org.apache.eagle.dataproc.util.ConfigOptionParser;
-import org.apache.eagle.datastream.ExecutionEnvironmentFactory;
-import org.apache.eagle.datastream.StormExecutionEnvironment;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.apache.eagle.datastream.ExecutionEnvironments;
+import org.apache.eagle.datastream.storm.StormExecutionEnvironment;
public class HDFSSecurityLogProcessorMain {
- private static final Logger LOG = LoggerFactory.getLogger(HDFSSecurityLogProcessorMain.class);
-
public static void main(String[] args) throws Exception{
- Config config = new ConfigOptionParser().load(args);
-
- LOG.info("Config class: " + config.getClass().getCanonicalName());
-
- if(LOG.isDebugEnabled()) LOG.debug("Config content:"+config.root().render(ConfigRenderOptions.concise()));
-
- StormExecutionEnvironment env = ExecutionEnvironmentFactory.getStorm(config);
-
- env.newSource(new KafkaSourcedSpoutProvider().getSpout(config)).renameOutputFields(1).withName("kafkaMsgConsumer")
+ StormExecutionEnvironment env = ExecutionEnvironments.getStorm(args);
+ env.fromSpout(new KafkaSourcedSpoutProvider()).withOutputFields(1).nameAs("kafkaMsgConsumer")
.alertWithConsumer("hdfsSecurityLogEventStream", "hdfsSecurityLogAlertExecutor");
env.execute();
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/52b8e58b/eagle-security/eagle-security-hdfs-web/src/main/java/org/apache/eagle/service/security/hdfs/HDFSFileSystem.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hdfs-web/src/main/java/org/apache/eagle/service/security/hdfs/HDFSFileSystem.java b/eagle-security/eagle-security-hdfs-web/src/main/java/org/apache/eagle/service/security/hdfs/HDFSFileSystem.java
index 9a17075..67e4092 100644
--- a/eagle-security/eagle-security-hdfs-web/src/main/java/org/apache/eagle/service/security/hdfs/HDFSFileSystem.java
+++ b/eagle-security/eagle-security-hdfs-web/src/main/java/org/apache/eagle/service/security/hdfs/HDFSFileSystem.java
@@ -1,12 +1,13 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,