You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@eagle.apache.org by ra...@apache.org on 2016/06/01 05:56:16 UTC
[01/19] incubator-eagle git commit: remove alert
Repository: incubator-eagle
Updated Branches:
refs/heads/branch-0.5 911ad2576 -> 75a8265c6
remove alert
Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/fb00fa36
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/fb00fa36
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/fb00fa36
Branch: refs/heads/branch-0.5
Commit: fb00fa3650bde3484a77c2fff8d0c15e70bfa9b7
Parents: 911ad25
Author: Ralph, Su <su...@gmail.com>
Authored: Wed Jun 1 13:45:51 2016 +0800
Committer: Ralph, Su <su...@gmail.com>
Committed: Wed Jun 1 13:45:51 2016 +0800
----------------------------------------------------------------------
eagle-core/eagle-alert/alert | 1 -
1 file changed, 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fb00fa36/eagle-core/eagle-alert/alert
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert b/eagle-core/eagle-alert/alert
deleted file mode 160000
index 092942c..0000000
--- a/eagle-core/eagle-alert/alert
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 092942cdd19d9224f2bca9e58d2052b7bdf8fbd5
[12/19] incubator-eagle git commit: EAGLE-324: Init branch-v0.5
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/evaluator/impl/AlertBoltOutputCollectorWrapper.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/evaluator/impl/AlertBoltOutputCollectorWrapper.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/evaluator/impl/AlertBoltOutputCollectorWrapper.java
new file mode 100755
index 0000000..8d8a4d2
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/evaluator/impl/AlertBoltOutputCollectorWrapper.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.evaluator.impl;
+
+import java.util.Arrays;
+
+import org.apache.eagle.alert.engine.AlertStreamCollector;
+import org.apache.eagle.alert.engine.StreamContext;
+import org.apache.eagle.alert.engine.model.AlertStreamEvent;
+
+import backtype.storm.task.OutputCollector;
+
+public class AlertBoltOutputCollectorWrapper implements AlertStreamCollector {
+ private final OutputCollector delegate;
+ private final Object outputLock;
+ private final StreamContext streamContext;
+
+ public AlertBoltOutputCollectorWrapper(OutputCollector outputCollector, Object outputLock, StreamContext streamContext){
+ this.delegate = outputCollector;
+ this.outputLock = outputLock;
+ this.streamContext = streamContext;
+ }
+
+ @Override
+ public void emit(AlertStreamEvent event) {
+ synchronized (outputLock) {
+ streamContext.counter().scope("alert_count").incr();
+ delegate.emit(Arrays.asList(event.getStreamId(), event));
+ }
+ }
+
+ @Override
+ public void flush() {
+ // do nothing
+ }
+
+ @Override
+ public void close() {
+
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/evaluator/impl/PolicyGroupEvaluatorImpl.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/evaluator/impl/PolicyGroupEvaluatorImpl.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/evaluator/impl/PolicyGroupEvaluatorImpl.java
new file mode 100644
index 0000000..f063618
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/evaluator/impl/PolicyGroupEvaluatorImpl.java
@@ -0,0 +1,172 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.evaluator.impl;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.eagle.alert.engine.AlertStreamCollector;
+import org.apache.eagle.alert.engine.StreamContext;
+import org.apache.eagle.alert.engine.coordinator.PolicyDefinition;
+import org.apache.eagle.alert.engine.coordinator.StreamDefinition;
+import org.apache.eagle.alert.engine.evaluator.PolicyGroupEvaluator;
+import org.apache.eagle.alert.engine.evaluator.PolicyHandlerContext;
+import org.apache.eagle.alert.engine.evaluator.PolicyStreamHandler;
+import org.apache.eagle.alert.engine.evaluator.PolicyStreamHandlers;
+import org.apache.eagle.alert.engine.model.PartitionedEvent;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class PolicyGroupEvaluatorImpl implements PolicyGroupEvaluator {
+ private static final long serialVersionUID = -5499413193675985288L;
+
+ private final static Logger LOG = LoggerFactory.getLogger(PolicyGroupEvaluatorImpl.class);
+
+ private AlertStreamCollector collector;
+ // mapping from policy name to PolicyDefinition
+ private volatile Map<String,PolicyDefinition> policyDefinitionMap = new HashMap<>();
+ // mapping from policy name to PolicyStreamHandler
+ private volatile Map<String,PolicyStreamHandler> policyStreamHandlerMap = new HashMap<>();
+ private String policyEvaluatorId;
+ private StreamContext context;
+
+ public PolicyGroupEvaluatorImpl(String policyEvaluatorId){
+ this.policyEvaluatorId = policyEvaluatorId;
+ }
+
+ public void init(StreamContext context, AlertStreamCollector collector) {
+ this.collector = collector;
+ this.policyStreamHandlerMap = new HashMap<>();
+ this.context = context;
+ Thread.currentThread().setName(policyEvaluatorId);
+ }
+
+ public void nextEvent(PartitionedEvent event) {
+ this.context.counter().scope("receive_count").incr();
+ dispatch(event);
+ }
+
+ @Override
+ public String getName() {
+ return this.policyEvaluatorId;
+ }
+
+ public void close() {
+ for(PolicyStreamHandler handler: policyStreamHandlerMap.values()){
+ try {
+ handler.close();
+ } catch (Exception e) {
+ LOG.error("Failed to close handler {}",handler.toString(),e);
+ }
+ }
+ }
+
+ /**
+ * fixme make selection of PolicyStreamHandler to be more efficient
+ * @param partitionedEvent PartitionedEvent
+ */
+ private void dispatch(PartitionedEvent partitionedEvent){
+ boolean handled = false;
+ for(Map.Entry<String,PolicyStreamHandler> policyStreamHandler: policyStreamHandlerMap.entrySet()){
+ if(isAcceptedByPolicy(partitionedEvent,policyDefinitionMap.get(policyStreamHandler.getKey()))){
+ try {
+ handled = true;
+ this.context.counter().scope("eval_count").incr();
+ policyStreamHandler.getValue().send(partitionedEvent.getEvent());
+ } catch (Exception e) {
+ this.context.counter().scope("fail_count").incr();
+ LOG.error("{} failed to handle {}",policyStreamHandler.getValue(), partitionedEvent.getEvent());
+ }
+ }
+ }
+ if(!handled){
+ this.context.counter().scope("drop_count").incr();
+ LOG.warn("Drop stream non-matched event {}, which should not be sent to evaluator", partitionedEvent);
+ } else {
+ this.context.counter().scope("accept_count").incr();
+ }
+ }
+
+ private static boolean isAcceptedByPolicy(PartitionedEvent event, PolicyDefinition policy){
+ return policy.getInputStreams() != null
+ && policy.getInputStreams().contains(event.getEvent().getStreamId())
+ && policy.getPartitionSpec().contains(event.getPartition());
+ }
+
+ @Override
+ public void onPolicyChange(List<PolicyDefinition> added, List<PolicyDefinition> removed, List<PolicyDefinition> modified, Map<String, StreamDefinition> sds) {
+ Map<String,PolicyDefinition> copyPolicies = new HashMap<>(policyDefinitionMap);
+ Map<String,PolicyStreamHandler> copyHandlers = new HashMap<>(policyStreamHandlerMap);
+ for(PolicyDefinition pd : added){
+ inplaceAdd(copyPolicies, copyHandlers, pd, sds);
+ }
+ for(PolicyDefinition pd : removed){
+ inplaceRemove(copyPolicies, copyHandlers, pd);
+ }
+ for(PolicyDefinition pd : modified){
+ inplaceRemove(copyPolicies, copyHandlers, pd);
+ inplaceAdd(copyPolicies, copyHandlers, pd, sds);
+ }
+
+ // logging
+ LOG.info("Policy metadata updated with added={}, removed={}, modified={}", added, removed, modified);
+
+ // switch reference
+ this.policyDefinitionMap = copyPolicies;
+ this.policyStreamHandlerMap = copyHandlers;
+ }
+
+ private void inplaceAdd(Map<String, PolicyDefinition> policies, Map<String, PolicyStreamHandler> handlers, PolicyDefinition policy, Map<String, StreamDefinition> sds) {
+ if(handlers.containsKey(policy.getName())){
+ LOG.error("metadata calculation error, try to add existing PolicyDefinition " + policy);
+ }else {
+ policies.put(policy.getName(), policy);
+ PolicyStreamHandler handler = PolicyStreamHandlers.createHandler(policy.getDefinition().type, sds);
+ try {
+ PolicyHandlerContext context = new PolicyHandlerContext();
+ context.setPolicyCounter(this.context.counter());
+ context.setPolicyDefinition(policy);
+ context.setParentEvaluator(this);
+ context.setPolicyEvaluatorId(policyEvaluatorId);
+ handler.prepare(collector, context);
+ handlers.put(policy.getName(), handler);
+ } catch (Exception e) {
+ LOG.error(e.getMessage(), e);
+ policies.remove(policy.getName());
+ handlers.remove(policy.getName());
+ }
+ }
+ }
+
+ private void inplaceRemove(Map<String, PolicyDefinition> policies, Map<String, PolicyStreamHandler> handlers, PolicyDefinition policy) {
+ if(handlers.containsKey(policy.getName())) {
+ PolicyStreamHandler handler = handlers.get(policy.getName());
+ try {
+ handler.close();
+ } catch (Exception e) {
+ LOG.error("Failed to close handler {}",handler,e);
+ }finally {
+ policies.remove(policy.getName());
+ handlers.remove(policy.getName());
+ LOG.info("Removed policy: {}",policy);
+ }
+ } else {
+ LOG.error("metadata calculation error, try to remove nonexisting PolicyDefinition: "+policy);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/evaluator/impl/SiddhiDefinitionAdapter.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/evaluator/impl/SiddhiDefinitionAdapter.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/evaluator/impl/SiddhiDefinitionAdapter.java
new file mode 100644
index 0000000..5b83abb
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/evaluator/impl/SiddhiDefinitionAdapter.java
@@ -0,0 +1,142 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.evaluator.impl;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.eagle.alert.engine.coordinator.StreamColumn;
+import org.apache.eagle.alert.engine.coordinator.StreamDefinition;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.wso2.siddhi.query.api.definition.AbstractDefinition;
+import org.wso2.siddhi.query.api.definition.Attribute;
+
+import com.google.common.base.Preconditions;
+
+public class SiddhiDefinitionAdapter {
+ private final static Logger LOG = LoggerFactory.getLogger(SiddhiDefinitionAdapter.class);
+ public final static String DEFINE_STREAM_TEMPLATE = "define stream %s ( %s );";
+
+ public static String buildStreamDefinition(StreamDefinition streamDefinition){
+ List<String> columns = new ArrayList<>();
+ Preconditions.checkNotNull(streamDefinition,"StreamDefinition is null");
+ if(streamDefinition.getColumns()!=null) {
+ for (StreamColumn column : streamDefinition.getColumns()) {
+ columns.add(String.format("%s %s", column.getName(), convertToSiddhiAttributeType(column.getType()).toString().toLowerCase()));
+ }
+ }else{
+ LOG.warn("No columns found for stream {}"+streamDefinition.getStreamId());
+ }
+ return String.format(DEFINE_STREAM_TEMPLATE, streamDefinition.getStreamId(),StringUtils.join(columns,","));
+ }
+
+ public static Attribute.Type convertToSiddhiAttributeType(StreamColumn.Type type){
+ if(_EAGLE_SIDDHI_TYPE_MAPPING.containsKey(type)){
+ return _EAGLE_SIDDHI_TYPE_MAPPING.get(type);
+ }
+
+ throw new IllegalArgumentException("Unknown stream type: "+type);
+ }
+
+ public static Class<?> convertToJavaAttributeType(StreamColumn.Type type){
+ if(_EAGLE_JAVA_TYPE_MAPPING.containsKey(type)){
+ return _EAGLE_JAVA_TYPE_MAPPING.get(type);
+ }
+
+ throw new IllegalArgumentException("Unknown stream type: "+type);
+ }
+
+ public static StreamColumn.Type convertFromJavaAttributeType(Class<?> type){
+ if(_JAVA_EAGLE_TYPE_MAPPING.containsKey(type)){
+ return _JAVA_EAGLE_TYPE_MAPPING.get(type);
+ }
+
+ throw new IllegalArgumentException("Unknown stream type: "+type);
+ }
+
+ public static StreamColumn.Type convertFromSiddhiAttributeType(Attribute.Type type){
+ if(_SIDDHI_EAGLE_TYPE_MAPPING.containsKey(type)){
+ return _SIDDHI_EAGLE_TYPE_MAPPING.get(type);
+ }
+
+ throw new IllegalArgumentException("Unknown siddhi type: "+type);
+ }
+
+ /**
+ * public enum Type {
+ * STRING, INT, LONG, FLOAT, DOUBLE, BOOL, OBJECT
+ * }
+ */
+ private final static Map<StreamColumn.Type,Attribute.Type> _EAGLE_SIDDHI_TYPE_MAPPING = new HashMap<>();
+ private final static Map<StreamColumn.Type,Class<?>> _EAGLE_JAVA_TYPE_MAPPING = new HashMap<>();
+ private final static Map<Class<?>,StreamColumn.Type> _JAVA_EAGLE_TYPE_MAPPING = new HashMap<>();
+ private final static Map<Attribute.Type,StreamColumn.Type> _SIDDHI_EAGLE_TYPE_MAPPING = new HashMap<>();
+
+ static {
+ _EAGLE_SIDDHI_TYPE_MAPPING.put(StreamColumn.Type.STRING,Attribute.Type.STRING);
+ _EAGLE_SIDDHI_TYPE_MAPPING.put(StreamColumn.Type.INT,Attribute.Type.INT);
+ _EAGLE_SIDDHI_TYPE_MAPPING.put(StreamColumn.Type.LONG,Attribute.Type.LONG);
+ _EAGLE_SIDDHI_TYPE_MAPPING.put(StreamColumn.Type.FLOAT,Attribute.Type.FLOAT);
+ _EAGLE_SIDDHI_TYPE_MAPPING.put(StreamColumn.Type.DOUBLE,Attribute.Type.DOUBLE);
+ _EAGLE_SIDDHI_TYPE_MAPPING.put(StreamColumn.Type.BOOL,Attribute.Type.BOOL);
+ _EAGLE_SIDDHI_TYPE_MAPPING.put(StreamColumn.Type.OBJECT,Attribute.Type.OBJECT);
+
+ _EAGLE_JAVA_TYPE_MAPPING.put(StreamColumn.Type.STRING,String.class);
+ _EAGLE_JAVA_TYPE_MAPPING.put(StreamColumn.Type.INT,Integer.class);
+ _EAGLE_JAVA_TYPE_MAPPING.put(StreamColumn.Type.LONG,Long.class);
+ _EAGLE_JAVA_TYPE_MAPPING.put(StreamColumn.Type.FLOAT,Float.class);
+ _EAGLE_JAVA_TYPE_MAPPING.put(StreamColumn.Type.DOUBLE,Double.class);
+ _EAGLE_JAVA_TYPE_MAPPING.put(StreamColumn.Type.BOOL,Boolean.class);
+ _EAGLE_JAVA_TYPE_MAPPING.put(StreamColumn.Type.OBJECT,Object.class);
+
+ _JAVA_EAGLE_TYPE_MAPPING.put(String.class,StreamColumn.Type.STRING);
+ _JAVA_EAGLE_TYPE_MAPPING.put(Integer.class,StreamColumn.Type.INT);
+ _JAVA_EAGLE_TYPE_MAPPING.put(Long.class,StreamColumn.Type.LONG);
+ _JAVA_EAGLE_TYPE_MAPPING.put(Float.class,StreamColumn.Type.FLOAT);
+ _JAVA_EAGLE_TYPE_MAPPING.put(Double.class,StreamColumn.Type.DOUBLE);
+ _JAVA_EAGLE_TYPE_MAPPING.put(Boolean.class,StreamColumn.Type.BOOL);
+ _JAVA_EAGLE_TYPE_MAPPING.put(Object.class,StreamColumn.Type.OBJECT);
+
+ _SIDDHI_EAGLE_TYPE_MAPPING.put(Attribute.Type.STRING,StreamColumn.Type.STRING);
+ _SIDDHI_EAGLE_TYPE_MAPPING.put(Attribute.Type.INT,StreamColumn.Type.INT);
+ _SIDDHI_EAGLE_TYPE_MAPPING.put(Attribute.Type.LONG,StreamColumn.Type.LONG);
+ _SIDDHI_EAGLE_TYPE_MAPPING.put(Attribute.Type.FLOAT,StreamColumn.Type.FLOAT);
+ _SIDDHI_EAGLE_TYPE_MAPPING.put(Attribute.Type.DOUBLE,StreamColumn.Type.DOUBLE);
+ _SIDDHI_EAGLE_TYPE_MAPPING.put(Attribute.Type.BOOL,StreamColumn.Type.BOOL);
+ _SIDDHI_EAGLE_TYPE_MAPPING.put(Attribute.Type.OBJECT,StreamColumn.Type.OBJECT);
+ }
+
+ public static StreamDefinition convertFromSiddiDefinition(AbstractDefinition siddhiDefinition){
+ StreamDefinition streamDefinition = new StreamDefinition();
+ streamDefinition.setStreamId(siddhiDefinition.getId());
+ List<StreamColumn> columns = new ArrayList<>(siddhiDefinition.getAttributeNameArray().length);
+ for(Attribute attribute:siddhiDefinition.getAttributeList()){
+ StreamColumn column = new StreamColumn();
+ column.setType(convertFromSiddhiAttributeType(attribute.getType()));
+ column.setName(attribute.getName());
+ columns.add(column);
+ }
+ streamDefinition.setColumns(columns);
+ streamDefinition.setTimeseries(true);
+ streamDefinition.setDescription("Auto-generated stream schema from siddhi for "+siddhiDefinition.getId());
+ return streamDefinition;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/evaluator/impl/SiddhiPolicyHandler.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/evaluator/impl/SiddhiPolicyHandler.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/evaluator/impl/SiddhiPolicyHandler.java
new file mode 100755
index 0000000..dfa5612
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/evaluator/impl/SiddhiPolicyHandler.java
@@ -0,0 +1,148 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.evaluator.impl;
+
+import java.util.Map;
+
+import org.apache.eagle.alert.engine.Collector;
+import org.apache.eagle.alert.engine.coordinator.PolicyDefinition;
+import org.apache.eagle.alert.engine.coordinator.StreamDefinition;
+import org.apache.eagle.alert.engine.coordinator.StreamDefinitionNotFoundException;
+import org.apache.eagle.alert.engine.evaluator.PolicyHandlerContext;
+import org.apache.eagle.alert.engine.evaluator.PolicyStreamHandler;
+import org.apache.eagle.alert.engine.model.AlertStreamEvent;
+import org.apache.eagle.alert.engine.model.StreamEvent;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.wso2.siddhi.core.ExecutionPlanRuntime;
+import org.wso2.siddhi.core.SiddhiManager;
+import org.wso2.siddhi.core.event.Event;
+import org.wso2.siddhi.core.stream.input.InputHandler;
+import org.wso2.siddhi.core.stream.output.StreamCallback;
+
+public class SiddhiPolicyHandler implements PolicyStreamHandler {
+ private final static Logger LOG = LoggerFactory.getLogger(SiddhiPolicyHandler.class);
+ private ExecutionPlanRuntime executionRuntime;
+ private SiddhiManager siddhiManager;
+ private Map<String, StreamDefinition> sds;
+ private PolicyDefinition policy;
+ private PolicyHandlerContext context;
+
+ public SiddhiPolicyHandler(Map<String, StreamDefinition> sds){
+ this.sds = sds;
+ }
+
+ private static String generateExecutionPlan(PolicyDefinition policyDefinition, Map<String, StreamDefinition> sds) throws StreamDefinitionNotFoundException {
+ StringBuilder builder = new StringBuilder();
+ for(String inputStream:policyDefinition.getInputStreams()) {
+ builder.append(SiddhiDefinitionAdapter.buildStreamDefinition(sds.get(inputStream)));
+ builder.append("\n");
+ }
+ builder.append(policyDefinition.getDefinition().value);
+ if(LOG.isDebugEnabled()) LOG.debug("Generated siddhi execution plan: {} from policy: {}", builder.toString(),policyDefinition);
+ return builder.toString();
+ }
+
+ private static class AlertStreamCallback extends StreamCallback{
+ private final String outputStream;
+ private final Collector<AlertStreamEvent> collector;
+ private final PolicyHandlerContext context;
+ private final StreamDefinition definition;
+
+ public AlertStreamCallback(String outputStream, StreamDefinition streamDefinition, Collector<AlertStreamEvent> collector, PolicyHandlerContext context){
+ this.outputStream = outputStream;
+ this.collector = collector;
+ this.context = context;
+ this.definition = streamDefinition;
+ }
+
+ /**
+ * Possibly more than one event will be triggered for alerting
+ * @param events
+ */
+ @Override
+ public void receive(Event[] events) {
+ LOG.info("Generated {} alerts from policy '{}' in {}", events.length,context.getPolicyDefinition().getName(), context.getPolicyEvaluatorId());
+ for(Event e : events) {
+ AlertStreamEvent event = new AlertStreamEvent();
+ event.setTimestamp(e.getTimestamp());
+ event.setData(e.getData());
+ event.setStreamId(outputStream);
+ event.setPolicy(context.getPolicyDefinition());
+ if (this.context.getParentEvaluator() != null) {
+ event.setCreatedBy(context.getParentEvaluator().getName());
+ }
+ event.setCreatedTime(System.currentTimeMillis());
+ event.setSchema(definition);
+ if(LOG.isDebugEnabled())
+ LOG.debug("Generate new alert event: {}", event);
+ collector.emit(event);
+ }
+ context.getPolicyCounter().scope(String.format("%s.%s",this.context.getPolicyDefinition().getName(),"alert_count")).incrBy(events.length);
+ }
+ }
+
+ @Override
+ public void prepare(final Collector<AlertStreamEvent> collector, PolicyHandlerContext context) throws Exception {
+ LOG.info("Initializing handler for policy {}: {}",context.getPolicyDefinition(),this);
+ this.policy = context.getPolicyDefinition();
+ this.siddhiManager = new SiddhiManager();
+ String plan = generateExecutionPlan(policy, sds);
+ try {
+ this.executionRuntime = siddhiManager.createExecutionPlanRuntime(plan);
+ LOG.info("Created siddhi runtime {}",executionRuntime.getName());
+ }catch (Exception parserException){
+ LOG.error("Failed to create siddhi runtime for policy: {}, siddhi plan: \n\n{}\n",context.getPolicyDefinition().getName(),plan,parserException);
+ throw parserException;
+ }
+ for(final String outputStream:policy.getOutputStreams()){
+ if(executionRuntime.getStreamDefinitionMap().containsKey(outputStream)) {
+ this.executionRuntime.addCallback(outputStream,
+ new AlertStreamCallback(
+ outputStream, SiddhiDefinitionAdapter.convertFromSiddiDefinition(executionRuntime.getStreamDefinitionMap().get(outputStream))
+ ,collector, context));
+ } else {
+ throw new IllegalStateException("Undefined output stream "+outputStream);
+ }
+ }
+ this.executionRuntime.start();
+ this.context = context;
+ LOG.info("Initialized policy handler for policy: {}",policy.getName());
+ }
+
+ public void send(StreamEvent event) throws Exception {
+ context.getPolicyCounter().scope(String.format("%s.%s",this.context.getPolicyDefinition().getName(),"receive_count")).incr();
+ String streamId = event.getStreamId();
+ InputHandler inputHandler = executionRuntime.getInputHandler(streamId);
+ if(inputHandler != null){
+ context.getPolicyCounter().scope(String.format("%s.%s",this.context.getPolicyDefinition().getName(),"eval_count")).incr();
+ inputHandler.send(event.getTimestamp(),event.getData());
+ }else{
+ context.getPolicyCounter().scope(String.format("%s.%s",this.context.getPolicyDefinition().getName(),"drop_count")).incr();
+ LOG.warn("No input handler found for stream {}",streamId);
+ }
+ }
+
+ public void close() {
+ LOG.info("Closing handler for policy {}",this.policy.getName());
+ this.executionRuntime.shutdown();
+ LOG.info("Shutdown siddhi runtime {}",this.executionRuntime.getName());
+ this.siddhiManager.shutdown();
+ LOG.info("Shutdown siddhi manager {}",this.siddhiManager);
+ LOG.info("Closed handler for policy {}",this.policy.getName());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/AlertDeduplicator.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/AlertDeduplicator.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/AlertDeduplicator.java
new file mode 100644
index 0000000..b894cc0
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/AlertDeduplicator.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.eagle.alert.engine.publisher;
+
+import org.apache.eagle.alert.engine.model.AlertStreamEvent;
+
+/**
+ * Dedup Eagle entities.
+ */
+public interface AlertDeduplicator {
+
+ AlertStreamEvent dedup(AlertStreamEvent event);
+
+ void setDedupIntervalMin(String intervalMin);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/AlertPublishListener.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/AlertPublishListener.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/AlertPublishListener.java
new file mode 100644
index 0000000..8f1c248
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/AlertPublishListener.java
@@ -0,0 +1,28 @@
+package org.apache.eagle.alert.engine.publisher;
+
+import java.util.List;
+
+import org.apache.eagle.alert.engine.coordinator.Publishment;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+public interface AlertPublishListener {
+ void onPublishChange(List<Publishment> added,
+ List<Publishment> removed,
+ List<Publishment> afterModified,
+ List<Publishment> beforeModified);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/AlertPublishPlugin.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/AlertPublishPlugin.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/AlertPublishPlugin.java
new file mode 100644
index 0000000..644fe2b
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/AlertPublishPlugin.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.eagle.alert.engine.publisher;
+
+import org.apache.eagle.alert.engine.coordinator.Publishment;
+import org.apache.eagle.alert.engine.model.AlertStreamEvent;
+import org.apache.eagle.alert.engine.publisher.impl.PublishStatus;
+
+import com.typesafe.config.Config;
+
+import java.io.Closeable;
+import java.util.Map;
+
+/**
+ * Created on 2/10/16.
+ * Notification Plug-in interface which provide abstraction layer to notify to different system
+ */
+public interface AlertPublishPlugin extends Closeable {
+ /**
+ * for initialization
+ * @throws Exception
+ */
+ void init(Config config, Publishment publishment) throws Exception;
+
+ void update(String dedupIntervalMin, Map<String, String> pluginProperties);
+
+ void close();
+
+ void onAlert(AlertStreamEvent event) throws Exception;
+
+ AlertStreamEvent dedup(AlertStreamEvent event);
+
+ PublishStatus getStatus();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/AlertPublishSpecListener.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/AlertPublishSpecListener.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/AlertPublishSpecListener.java
new file mode 100644
index 0000000..fc1fc28
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/AlertPublishSpecListener.java
@@ -0,0 +1,26 @@
+package org.apache.eagle.alert.engine.publisher;
+
+import java.util.Map;
+
+import org.apache.eagle.alert.coordination.model.PublishSpec;
+import org.apache.eagle.alert.engine.coordinator.StreamDefinition;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+public interface AlertPublishSpecListener {
+ void onAlertPublishSpecChange(PublishSpec spec, Map<String, StreamDefinition> sds);
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/AlertPublisher.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/AlertPublisher.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/AlertPublisher.java
new file mode 100644
index 0000000..7a44009
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/AlertPublisher.java
@@ -0,0 +1,31 @@
+package org.apache.eagle.alert.engine.publisher;
+
+
+import java.io.Serializable;
+
+import org.apache.eagle.alert.engine.model.AlertStreamEvent;
+
+import com.typesafe.config.Config;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+public interface AlertPublisher extends AlertPublishListener, Serializable {
+ void init(Config config);
+ String getName();
+ void nextEvent(AlertStreamEvent event);
+ void close();
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/AlertSink.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/AlertSink.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/AlertSink.java
new file mode 100644
index 0000000..91c9296
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/AlertSink.java
@@ -0,0 +1,33 @@
+package org.apache.eagle.alert.engine.publisher;
+
+import java.io.Serializable;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+public interface AlertSink extends Serializable {
+ /**
+ *
+ * @throws Exception
+ */
+ void open() throws Exception;
+
+ /**
+ *
+ * @throws Exception
+ */
+ void close() throws Exception;
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/PublishConstants.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/PublishConstants.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/PublishConstants.java
new file mode 100644
index 0000000..54a9afa
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/PublishConstants.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.alert.engine.publisher;
+
+public class PublishConstants {
+ public static final String NOTIFICATION_TYPE = "type";
+ public static final String EMAIL_NOTIFICATION = "email";
+ public static final String KAFKA_STORE = "kafka";
+ public static final String EAGLE_STORE = "eagleStore";
+
+ // email specific constants
+ public static final String SUBJECT = "subject";
+ public static final String SENDER = "sender";
+ public static final String RECIPIENTS = "recipients";
+ public static final String TEMPLATE = "template";
+
+ // kafka specific constants
+ public static final String TOPIC = "topic";
+ public static final String BROKER_LIST = "kafka_broker";
+
+ public static final String ALERT_EMAIL_TIME_PROPERTY = "timestamp";
+ public static final String ALERT_EMAIL_COUNT_PROPERTY = "count";
+ public static final String ALERT_EMAIL_ALERTLIST_PROPERTY = "alertList";
+ public static final String ALERT_EMAIL_ORIGIN_PROPERTY = "alertEmailOrigin";
+
+ public static final String ALERT_EMAIL_MESSAGE = "alertMessage";
+ public static final String ALERT_EMAIL_STREAM = "streamId";
+ public static final String ALERT_EMAIL_TIMESTAMP = "alertTime";
+ public static final String ALERT_EMAIL_POLICY = "policyId";
+ public static final String ALERT_EMAIL_CREATOR = "creator";
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/email/AlertEmailConstants.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/email/AlertEmailConstants.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/email/AlertEmailConstants.java
new file mode 100644
index 0000000..7d31bb5
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/email/AlertEmailConstants.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.alert.engine.publisher.email;
+
+public class AlertEmailConstants {
+
+ public static final String MAIL_AUTH = "mail.smtp.auth";
+ public static final String MAIL_HOST = "mail.smtp.host";
+ public static final String MAIL_PORT = "mail.smtp.port";
+
+ public static final String CONN_PLAINTEXT = "plaintext";
+ public static final String CONN_TLS = "tls";
+ public static final String CONN_SSL = "ssl";
+
+ public static final String CONF_MAIL_SERVER = "smtp.server";
+ public static final String CONF_MAIL_PORT = "smtp.port";
+ public static final String CONF_MAIL_CONN = "connection";
+ public static final String CONF_MAIL_DEBUG = "mailDebug";
+ public static final String CONF_MAIL_AUTH = "smtp.auth.enable";
+ public static final String CONF_AUTH_USER = "auth.username";
+ public static final String CONF_AUTH_PASSWORD = "auth.password";
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/email/AlertEmailContext.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/email/AlertEmailContext.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/email/AlertEmailContext.java
new file mode 100644
index 0000000..8723283
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/email/AlertEmailContext.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.eagle.alert.engine.publisher.email;
+
+import java.util.Map;
+
+/**
+ * alert email bean
+ * one email consists of a list of email component
+ */
+public class AlertEmailContext {
+ private Map<String, String> alertContext;
+ private String sender;
+ private String subject;
+ private String recipients;
+ private String velocityTplFile;
+ private String cc;
+
+ public Map<String, String> getAlertContext() {
+ return alertContext;
+ }
+
+ public void setAlertContext(Map<String, String> alertContext) {
+ this.alertContext = alertContext;
+ }
+ public String getVelocityTplFile() {
+ return velocityTplFile;
+ }
+ public void setVelocityTplFile(String velocityTplFile) {
+ this.velocityTplFile = velocityTplFile;
+ }
+ public String getRecipients() {
+ return recipients;
+ }
+ public void setRecipients(String recipients) {
+ this.recipients = recipients;
+ }
+ public String getSender() {
+ return sender;
+ }
+ public void setSender(String sender) {
+ this.sender = sender;
+ }
+ public String getSubject() {
+ return subject;
+ }
+ public void setSubject(String subject) {
+ this.subject = subject;
+ }
+ public String getCc() {
+ return cc;
+ }
+ public void setCc(String cc) {
+ this.cc = cc;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/email/AlertEmailGenerator.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/email/AlertEmailGenerator.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/email/AlertEmailGenerator.java
new file mode 100644
index 0000000..0c68629
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/email/AlertEmailGenerator.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+/**
+ *
+ */
+package org.apache.eagle.alert.engine.publisher.email;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import org.apache.eagle.alert.engine.model.AlertStreamEvent;
+import org.apache.eagle.alert.engine.publisher.PublishConstants;
+import org.apache.eagle.alert.utils.DateTimeUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class AlertEmailGenerator {
+ private String tplFile;
+ private String sender;
+ private String recipients;
+ private String subject;
+ private Map<String, String> properties;
+
+ private ThreadPoolExecutor executorPool;
+
+ private final static Logger LOG = LoggerFactory.getLogger(AlertEmailGenerator.class);
+
+ private final static long MAX_TIMEOUT_MS =60000;
+
+ public boolean sendAlertEmail(AlertStreamEvent entity) {
+ return sendAlertEmail(entity, recipients, null);
+ }
+
+ public boolean sendAlertEmail(AlertStreamEvent entity, String recipients) {
+ return sendAlertEmail(entity, recipients, null);
+ }
+
+ public boolean sendAlertEmail(AlertStreamEvent event, String recipients, String cc) {
+ AlertEmailContext email = new AlertEmailContext();
+ Map<String, String> alertContext = buildAlertContext(event);
+ email.setAlertContext(alertContext);
+ email.setVelocityTplFile(tplFile);
+ email.setSubject(subject);
+ email.setSender(sender);
+ email.setRecipients(recipients);
+ email.setCc(cc);
+
+ /** asynchronized email sending */
+ AlertEmailSender thread = new AlertEmailSender(email, properties);
+
+ if(this.executorPool == null) throw new IllegalStateException("Invoking thread executor pool but it's is not set yet");
+
+ LOG.info("Sending email in asynchronous to: " + recipients + ", cc: " + cc);
+ Future<?> future = this.executorPool.submit(thread);
+ Boolean status;
+ try {
+ future.get(MAX_TIMEOUT_MS, TimeUnit.MILLISECONDS);
+ status = true;
+ //LOG.info(String.format("Successfully send email to %s", recipients));
+ } catch (InterruptedException | ExecutionException e) {
+ status = false;
+ LOG.error(String.format("Failed to send email to %s, due to:%s", recipients, e),e);
+ } catch (TimeoutException e) {
+ status = false;
+ LOG.error(String.format("Failed to send email to %s due to timeout exception, max timeout: %s ms ", recipients, MAX_TIMEOUT_MS),e);
+ }
+ return status;
+ }
+
+ private Map<String, String> buildAlertContext(AlertStreamEvent event) {
+ Map<String, String> alertContext = new HashMap<>();
+ alertContext.put(PublishConstants.ALERT_EMAIL_MESSAGE, event.toString());
+ alertContext.put(PublishConstants.ALERT_EMAIL_POLICY, event.getPolicyId());
+ alertContext.put(PublishConstants.ALERT_EMAIL_TIMESTAMP, DateTimeUtil.millisecondsToHumanDateWithSeconds(event.getCreatedTime()));
+ alertContext.put(PublishConstants.ALERT_EMAIL_STREAM, event.getStreamId());
+ alertContext.put(PublishConstants.ALERT_EMAIL_CREATOR, event.getCreatedBy());
+ return alertContext;
+ }
+
+ public String getTplFile() {
+ return tplFile;
+ }
+
+ public void setTplFile(String tplFile) {
+ this.tplFile = tplFile;
+ }
+
+ public String getSender() {
+ return sender;
+ }
+
+ public void setSender(String sender) {
+ this.sender = sender;
+ }
+
+ public String getRecipients() {
+ return recipients;
+ }
+
+ public void setRecipients(String recipients) {
+ this.recipients = recipients;
+ }
+
+ public String getSubject() {
+ return subject;
+ }
+
+ public void setSubject(String subject) {
+ this.subject = subject;
+ }
+
+ public Map<String, String> getProperties() {
+ return properties;
+ }
+
+ public void setProperties(Map<String, String> properties) {
+ this.properties = properties;
+ }
+
+ public void setExecutorPool(ThreadPoolExecutor executorPool) {
+ this.executorPool = executorPool;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/email/AlertEmailGeneratorBuilder.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/email/AlertEmailGeneratorBuilder.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/email/AlertEmailGeneratorBuilder.java
new file mode 100644
index 0000000..e9635dd
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/email/AlertEmailGeneratorBuilder.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.eagle.alert.engine.publisher.email;
+
+import java.util.Map;
+import java.util.concurrent.ThreadPoolExecutor;
+
+public class AlertEmailGeneratorBuilder {
+ private AlertEmailGenerator generator;
+ private AlertEmailGeneratorBuilder(){
+ generator = new AlertEmailGenerator();
+ }
+ public static AlertEmailGeneratorBuilder newBuilder(){
+ return new AlertEmailGeneratorBuilder();
+ }
+ public AlertEmailGeneratorBuilder withSubject(String subject){
+ generator.setSubject(subject);
+ return this;
+ }
+ public AlertEmailGeneratorBuilder withSender(String sender){
+ generator.setSender(sender);
+ return this;
+ }
+ public AlertEmailGeneratorBuilder withRecipients(String recipients){
+ generator.setRecipients(recipients);
+ return this;
+ }
+ public AlertEmailGeneratorBuilder withTplFile(String tplFile){
+ generator.setTplFile(tplFile);
+ return this;
+ }
+ public AlertEmailGeneratorBuilder withMailProps(Map<String, String> mailProps) {
+ generator.setProperties(mailProps);
+ return this;
+ }
+ public AlertEmailGeneratorBuilder withExecutorPool(ThreadPoolExecutor threadPoolExecutor) {
+ generator.setExecutorPool(threadPoolExecutor);
+ return this;
+ }
+
+ public AlertEmailGenerator build(){
+ return this.generator;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/email/AlertEmailSender.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/email/AlertEmailSender.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/email/AlertEmailSender.java
new file mode 100644
index 0000000..d0e5cf6
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/email/AlertEmailSender.java
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.eagle.alert.engine.publisher.email;
+
+import java.lang.management.ManagementFactory;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.eagle.alert.engine.publisher.PublishConstants;
+import org.apache.eagle.alert.utils.DateTimeUtil;
+import org.apache.velocity.VelocityContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class AlertEmailSender implements Runnable {
+
+ protected final List<Map<String, String>> alertContexts = new ArrayList<Map<String, String>>();
+ protected final String configFileName;
+ protected final String subject;
+ protected final String sender;
+ protected final String recipients;
+ protected final String cc;
+ protected final String origin;
+ protected boolean sentSuccessfully = false;
+
+ private final static Logger LOG = LoggerFactory.getLogger(AlertEmailSender.class);
+ private final static int MAX_RETRY_COUNT = 3;
+
+
+
+ private Map<String, String> mailProps;
+
+
+ private String threadName;
+ /**
+ * Derived class may have some additional context properties to add
+ * @param context velocity context
+ * @param env environment
+ */
+ protected void additionalContext(VelocityContext context, String env) {
+ // By default there's no additional context added
+ }
+
+ public AlertEmailSender(AlertEmailContext alertEmail){
+ this.recipients = alertEmail.getRecipients();
+ this.configFileName = alertEmail.getVelocityTplFile();
+ this.subject = alertEmail.getSubject();
+ this.sender = alertEmail.getSender();
+ this.cc = alertEmail.getCc();
+
+ this.alertContexts.add(alertEmail.getAlertContext());
+ String tmp = ManagementFactory.getRuntimeMXBean().getName();
+ this.origin = tmp.split("@")[1] + "(pid:" + tmp.split("@")[0] + ")";
+ threadName = Thread.currentThread().getName();
+ LOG.info("Initialized "+threadName+": origin is : " + this.origin+", recipient of the email: " + this.recipients +", velocity TPL file: " + this.configFileName);
+ }
+
+ public AlertEmailSender(AlertEmailContext alertEmail, Map<String, String> mailProps){
+ this(alertEmail);
+ this.mailProps = mailProps;
+ }
+
+ private Properties parseMailClientConfig(Map<String, String> mailProps) {
+ if (mailProps == null) return null;
+ Properties props = new Properties();
+ String mailHost = mailProps.get(AlertEmailConstants.CONF_MAIL_SERVER);
+ String mailPort = mailProps.get(AlertEmailConstants.CONF_MAIL_PORT);
+ if (mailHost == null || mailPort == null || mailHost.isEmpty()) {
+ LOG.warn("SMTP server is unset, will exit");
+ return null;
+ }
+ props.put(AlertEmailConstants.MAIL_HOST, mailHost);
+ props.put(AlertEmailConstants.MAIL_PORT, mailPort);
+
+ String smtpAuth = mailProps.getOrDefault(AlertEmailConstants.CONF_MAIL_AUTH, "false");
+ props.put(AlertEmailConstants.MAIL_AUTH, smtpAuth);
+ if (Boolean.parseBoolean(smtpAuth)) {
+ props.put(AlertEmailConstants.CONF_AUTH_USER, mailProps.get(AlertEmailConstants.CONF_AUTH_USER));
+ props.put(AlertEmailConstants.CONF_AUTH_PASSWORD, mailProps.get(AlertEmailConstants.CONF_AUTH_PASSWORD));
+ }
+
+ String smtpConn = mailProps.getOrDefault(AlertEmailConstants.CONF_MAIL_CONN, AlertEmailConstants.CONN_PLAINTEXT);
+ if (smtpConn.equalsIgnoreCase(AlertEmailConstants.CONN_TLS)) {
+ props.put("mail.smtp.starttls.enable", "true");
+ }
+ if (smtpConn.equalsIgnoreCase(AlertEmailConstants.CONN_SSL)) {
+ props.put("mail.smtp.socketFactory.port", "465");
+ props.put("mail.smtp.socketFactory.class",
+ "javax.net.ssl.SSLSocketFactory");
+ }
+ props.put(AlertEmailConstants.CONF_MAIL_DEBUG, mailProps.getOrDefault(AlertEmailConstants.CONF_MAIL_DEBUG, "false"));
+ return props;
+ }
+
+ @Override
+ public void run() {
+ int count = 0;
+ boolean success = false;
+ while(count++ < MAX_RETRY_COUNT && !success){
+ LOG.info("Sending email, tried: " + count+", max: " + MAX_RETRY_COUNT);
+ try {
+ final EagleMailClient client;
+ if (mailProps != null) {
+ Properties props = parseMailClientConfig(mailProps);
+ client = new EagleMailClient(props);
+ }
+ else {
+ client = new EagleMailClient();
+ }
+
+ final VelocityContext context = new VelocityContext();
+ generateCommonContext(context);
+ LOG.info("After calling generateCommonContext...");
+
+ if (recipients == null || recipients.equals("")) {
+ LOG.error("Recipients is null, skip sending emails ");
+ return;
+ }
+ String title = subject;
+
+ success = client.send(sender, recipients, cc, title, configFileName, context, null);
+ LOG.info("Success of sending email: " + success);
+ if(!success && count < MAX_RETRY_COUNT) {
+ LOG.info("Sleep for a while before retrying");
+ Thread.sleep(10 * 1000);
+ }
+ }
+ catch (Exception e){
+ LOG.warn("Sending mail exception", e);
+ }
+ }
+ if (success) {
+ sentSuccessfully = true;
+ LOG.info(String.format("Successfully send email, thread: %s", threadName));
+ } else{
+ LOG.warn(String.format("Fail sending email after tries %s times, thread: %s", MAX_RETRY_COUNT, threadName));
+ }
+ }
+
+ private void generateCommonContext(VelocityContext context) {
+ context.put(PublishConstants.ALERT_EMAIL_TIME_PROPERTY, DateTimeUtil.millisecondsToHumanDateWithSeconds(System.currentTimeMillis()));
+ context.put(PublishConstants.ALERT_EMAIL_COUNT_PROPERTY, alertContexts.size());
+ context.put(PublishConstants.ALERT_EMAIL_ALERTLIST_PROPERTY, alertContexts);
+ context.put(PublishConstants.ALERT_EMAIL_ORIGIN_PROPERTY, origin);
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/email/EagleMailClient.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/email/EagleMailClient.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/email/EagleMailClient.java
new file mode 100755
index 0000000..61194a2
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/email/EagleMailClient.java
@@ -0,0 +1,240 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.eagle.alert.engine.publisher.email;
+
+import java.io.File;
+import java.io.StringWriter;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+import javax.activation.DataHandler;
+import javax.activation.DataSource;
+import javax.activation.FileDataSource;
+import javax.mail.Authenticator;
+import javax.mail.Message;
+import javax.mail.MessagingException;
+import javax.mail.Multipart;
+import javax.mail.PasswordAuthentication;
+import javax.mail.Session;
+import javax.mail.Transport;
+import javax.mail.internet.AddressException;
+import javax.mail.internet.InternetAddress;
+import javax.mail.internet.MimeBodyPart;
+import javax.mail.internet.MimeMessage;
+import javax.mail.internet.MimeMultipart;
+
+import org.apache.velocity.Template;
+import org.apache.velocity.VelocityContext;
+import org.apache.velocity.app.VelocityEngine;
+import org.apache.velocity.exception.ResourceNotFoundException;
+import org.apache.velocity.runtime.RuntimeConstants;
+import org.apache.velocity.runtime.resource.loader.ClasspathResourceLoader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class EagleMailClient {
+// private static final String CONFIG_FILE = "config.properties";
+ private static final String BASE_PATH = "templates/";
+
+ private VelocityEngine velocityEngine;
+ private Session session;
+ private static final Logger LOG = LoggerFactory.getLogger(EagleMailClient.class);
+
+ public EagleMailClient() {
+ this(new Properties());
+ }
+
+ public EagleMailClient(final Properties config) {
+ try {
+ velocityEngine = new VelocityEngine();
+ velocityEngine.setProperty(RuntimeConstants.RESOURCE_LOADER, "classpath");
+ velocityEngine.setProperty("classpath.resource.loader.class", ClasspathResourceLoader.class.getName());
+ velocityEngine.init();
+
+ config.put("mail.transport.protocol", "smtp");
+ if(Boolean.parseBoolean(config.getProperty(AlertEmailConstants.CONF_MAIL_AUTH))){
+ session = Session.getDefaultInstance(config, new Authenticator() {
+ protected PasswordAuthentication getPasswordAuthentication() {
+ return new PasswordAuthentication(config.getProperty(AlertEmailConstants.CONF_AUTH_USER), config.getProperty(AlertEmailConstants.CONF_AUTH_PASSWORD));
+ }
+ });
+ }
+ else session = Session.getDefaultInstance(config, new Authenticator() {});
+ final String debugMode = config.getProperty(AlertEmailConstants.CONF_MAIL_DEBUG, "false");
+ final boolean debug = Boolean.parseBoolean(debugMode);
+ session.setDebug(debug);
+ } catch (Exception e) {
+ LOG.error("Failed connect to smtp server",e);
+ }
+ }
+
+ private boolean _send(String from, String to, String cc, String title,
+ String content) {
+ Message msg = new MimeMessage(session);
+ try {
+ msg.setFrom(new InternetAddress(from));
+ msg.setSubject(title);
+ if (to != null) {
+ msg.setRecipients(Message.RecipientType.TO,
+ InternetAddress.parse(to));
+ }
+ if (cc != null) {
+ msg.setRecipients(Message.RecipientType.CC,
+ InternetAddress.parse(cc));
+ }
+ //msg.setRecipients(Message.RecipientType.BCC, InternetAddress.parse(DEFAULT_BCC_ADDRESS));
+ msg.setContent(content, "text/html;charset=utf-8");
+ LOG.info(String.format("Going to send mail: from[%s], to[%s], cc[%s], title[%s]", from, to, cc, title));
+ Transport.send(msg);
+ return true;
+ } catch (AddressException e) {
+ LOG.info("Send mail failed, got an AddressException: " + e.getMessage(), e);
+ return false;
+ } catch (MessagingException e) {
+ LOG.info("Send mail failed, got an AddressException: " + e.getMessage(), e);
+ return false;
+ }
+ }
+
+ private boolean _send(String from,String to,String cc,String title,String content,List<MimeBodyPart> attachments){
+ MimeMessage mail = new MimeMessage(session);
+ try {
+ mail.setFrom(new InternetAddress(from));
+ mail.setSubject(title);
+ if (to != null) {
+ mail.setRecipients(Message.RecipientType.TO,
+ InternetAddress.parse(to));
+ }
+ if (cc != null) {
+ mail.setRecipients(Message.RecipientType.CC,
+ InternetAddress.parse(cc));
+ }
+
+ //mail.setRecipients(Message.RecipientType.BCC, InternetAddress.parse(DEFAULT_BCC_ADDRESS));
+
+ MimeBodyPart mimeBodyPart = new MimeBodyPart();
+ mimeBodyPart.setContent(content,"text/html;charset=utf-8");
+
+ Multipart multipart = new MimeMultipart();
+ multipart.addBodyPart(mimeBodyPart);
+
+ for(MimeBodyPart attachment:attachments){
+ multipart.addBodyPart(attachment);
+ }
+
+ mail.setContent(multipart);
+// mail.setContent(content, "text/html;charset=utf-8");
+ LOG.info(String.format("Going to send mail: from[%s], to[%s], cc[%s], title[%s]", from, to, cc, title));
+ Transport.send(mail);
+ return true;
+ } catch (AddressException e) {
+ LOG.info("Send mail failed, got an AddressException: " + e.getMessage(), e);
+ return false;
+ } catch (MessagingException e) {
+ LOG.info("Send mail failed, got an AddressException: " + e.getMessage(), e);
+ return false;
+ }
+ }
+
+ public boolean send(String from, String to, String cc, String title,
+ String content) {
+ return this._send(from, to, cc, title, content);
+ }
+
+ public boolean send(String from, String to, String cc, String title,
+ String templatePath, VelocityContext context) {
+ Template t = null;
+ try {
+ t = velocityEngine.getTemplate(BASE_PATH + templatePath);
+ } catch (ResourceNotFoundException ex) {
+ }
+ if (t == null) {
+ try {
+ t = velocityEngine.getTemplate(templatePath);
+ } catch (ResourceNotFoundException e) {
+ t = velocityEngine.getTemplate("/" + templatePath);
+ }
+ }
+ final StringWriter writer = new StringWriter();
+ t.merge(context, writer);
+ if(LOG.isDebugEnabled()) LOG.debug(writer.toString());
+ return this.send(from, to, cc, title, writer.toString());
+ }
+
+ public boolean send(String from, String to, String cc, String title,
+ String templatePath, VelocityContext context, Map<String,File> attachments) {
+ if (attachments == null || attachments.isEmpty()) {
+ return send(from, to, cc, title, templatePath, context);
+ }
+ Template t = null;
+
+ List<MimeBodyPart> mimeBodyParts = new ArrayList<MimeBodyPart>();
+ Map<String,String> cid = new HashMap<String,String>();
+
+ for (Map.Entry<String,File> entry : attachments.entrySet()) {
+ final String attachment = entry.getKey();
+ final File attachmentFile = entry.getValue();
+ final MimeBodyPart mimeBodyPart = new MimeBodyPart();
+ if(attachmentFile !=null && attachmentFile.exists()){
+ DataSource source = new FileDataSource(attachmentFile);
+ try {
+ mimeBodyPart.setDataHandler(new DataHandler(source));
+ mimeBodyPart.setFileName(attachment);
+ mimeBodyPart.setDisposition(MimeBodyPart.ATTACHMENT);
+ mimeBodyPart.setContentID(attachment);
+ cid.put(attachment,mimeBodyPart.getContentID());
+ mimeBodyParts.add(mimeBodyPart);
+ } catch (MessagingException e) {
+ LOG.error("Generate mail failed, got exception while attaching files: " + e.getMessage(), e);
+ }
+ }else{
+ LOG.error("Attachment: " + attachment + " is null or not exists");
+ }
+ }
+ //TODO remove cid, because not used at all
+ if(LOG.isDebugEnabled()) LOG.debug("Cid maps: "+cid);
+ context.put("cid", cid);
+
+ try {
+ t = velocityEngine.getTemplate(BASE_PATH + templatePath);
+ } catch (ResourceNotFoundException ex) {
+// LOGGER.error("Template not found:"+BASE_PATH + templatePath, ex);
+ }
+
+ if (t == null) {
+ try {
+ t = velocityEngine.getTemplate(templatePath);
+ } catch (ResourceNotFoundException e) {
+ try {
+ t = velocityEngine.getTemplate("/" + templatePath);
+ }
+ catch (Exception ex) {
+ LOG.error("Template not found:"+ "/" + templatePath, ex);
+ }
+ }
+ }
+
+ final StringWriter writer = new StringWriter();
+ t.merge(context, writer);
+ if(LOG.isDebugEnabled()) LOG.debug(writer.toString());
+ return this._send(from, to, cc, title, writer.toString(), mimeBodyParts);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/impl/AlertEagleStorePersister.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/impl/AlertEagleStorePersister.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/impl/AlertEagleStorePersister.java
new file mode 100644
index 0000000..2a4e332
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/impl/AlertEagleStorePersister.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.alert.engine.publisher.impl;
+
+import java.util.List;
+
+import org.apache.eagle.alert.engine.model.StreamEvent;
+import org.apache.eagle.alert.service.IMetadataServiceClient;
+import org.apache.eagle.alert.service.MetadataServiceClientImpl;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.typesafe.config.Config;
+
+/**
+ * Alert API entity Persistor
+ */
+public class AlertEagleStorePersister {
+ private static Logger LOG = LoggerFactory.getLogger(AlertEagleStorePersister.class);
+ private Config config;
+
+ public AlertEagleStorePersister(Config config) {
+ this.config = config;
+ }
+
+ /**
+ * Persist passes list of Entities
+ * @param list
+ * @return
+ */
+ public boolean doPersist(List<? extends StreamEvent> list) {
+ if (list.isEmpty()) return false;
+ LOG.info("Going to persist entities, type: " + " " + list.get(0).getClass().getSimpleName() + ", list size: " + list.size());
+ try {
+ IMetadataServiceClient client = new MetadataServiceClientImpl(config);
+ // TODO: metadata service support
+ }
+ catch (Exception ex) {
+ LOG.error("Got an exception in persisting entities", ex);
+ return false;
+ }
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/impl/AlertEagleStorePublisher.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/impl/AlertEagleStorePublisher.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/impl/AlertEagleStorePublisher.java
new file mode 100644
index 0000000..807aacc
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/impl/AlertEagleStorePublisher.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.alert.engine.publisher.impl;
+
+import java.util.Arrays;
+import java.util.Map;
+
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+import org.apache.eagle.alert.engine.coordinator.Publishment;
+import org.apache.eagle.alert.engine.model.AlertStreamEvent;
+import org.apache.eagle.alert.engine.publisher.AlertDeduplicator;
+import org.apache.eagle.alert.engine.publisher.AlertPublishPlugin;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.typesafe.config.Config;
+
+/**
+ * Plugin to persist alerts to Eagle Storage
+ */
+public class AlertEagleStorePublisher implements AlertPublishPlugin {
+
+ private static final Logger LOG = LoggerFactory.getLogger(AlertEagleStorePublisher.class);
+ private PublishStatus status;
+ private AlertEagleStorePersister persist;
+ private AlertDeduplicator deduplicator;
+
+ @Override
+ public void init(Config config, Publishment publishment) throws Exception {
+ this.persist = new AlertEagleStorePersister(config);
+ deduplicator = new DefaultDeduplicator(publishment.getDedupIntervalMin());
+ LOG.info("initialized plugin for EagleStorePlugin");
+ }
+
+ @Override
+ public void update(String dedupIntervalMin, Map<String, String> pluginProperties) {
+ deduplicator.setDedupIntervalMin(dedupIntervalMin);
+ }
+
+ @Override
+ public PublishStatus getStatus() {
+ return this.status;
+ }
+
+ @Override
+ public AlertStreamEvent dedup(AlertStreamEvent event) {
+ return deduplicator.dedup(event);
+ }
+
+ /**
+ * Persist AlertEntity to alert_details table
+ * @param event
+ */
+ @Override
+ public void onAlert(AlertStreamEvent event) {
+ LOG.info("write alert to eagle storage " + event);
+ event = dedup(event);
+ if(event == null) {
+ return;
+ }
+ PublishStatus status = new PublishStatus();
+ try{
+ boolean result = persist.doPersist(Arrays.asList(event));
+ if(result) {
+ status.successful = true;
+ status.errorMessage = "";
+ }else{
+ status.successful = false;
+ status.errorMessage = "";
+ }
+ }catch (Exception ex ){
+ status.successful = false;
+ status.errorMessage = ex.getMessage();
+ LOG.error("Fail writing alert entity to Eagle Store", ex);
+ }
+ this.status = status;
+ }
+
+ @Override
+ public void close() {
+
+ }
+
+ @Override
+ public int hashCode(){
+ return new HashCodeBuilder().append(getClass().getCanonicalName()).toHashCode();
+ }
+
+ @Override
+ public boolean equals(Object o){
+ if(o == this)
+ return true;
+ if(!(o instanceof AlertEagleStorePublisher))
+ return false;
+ return true;
+ }
+}
\ No newline at end of file
[09/19] incubator-eagle git commit: EAGLE-324: Init branch-v0.5
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/serialization/Serializers.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/serialization/Serializers.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/serialization/Serializers.java
new file mode 100644
index 0000000..a94604c
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/serialization/Serializers.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.serialization;
+
+import org.apache.eagle.alert.engine.coordinator.StreamColumn;
+import org.apache.eagle.alert.engine.serialization.impl.*;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class Serializers {
+ private final static Map<StreamColumn.Type,Serializer<?>> COLUMN_TYPE_SER_MAPPING = new HashMap<>();
+
+ public static <T> void register(StreamColumn.Type type,Serializer<T> serializer){
+ if(COLUMN_TYPE_SER_MAPPING.containsKey(type)){
+ throw new IllegalArgumentException("Duplicated column type: "+type);
+ }
+ COLUMN_TYPE_SER_MAPPING.put(type,serializer);
+ }
+
+ public static <T> Serializer<T> getColumnSerializer(StreamColumn.Type type){
+ if(COLUMN_TYPE_SER_MAPPING.containsKey(type)){
+ return (Serializer<T>) COLUMN_TYPE_SER_MAPPING.get(type);
+ }else{
+ throw new IllegalArgumentException("Serializer of type: "+type+" not found");
+ }
+ }
+
+ public static PartitionedEventSerializer newPartitionedEventSerializer(SerializationMetadataProvider metadataProvider){
+ return new PartitionedEventSerializerImpl(metadataProvider);
+ }
+
+ static {
+ register(StreamColumn.Type.STRING,new StringSerializer());
+ register(StreamColumn.Type.INT,new IntegerSerializer());
+ register(StreamColumn.Type.LONG,new LongSerializer());
+ register(StreamColumn.Type.FLOAT,new FloatSerializer());
+ register(StreamColumn.Type.DOUBLE,new DoubleSerializer());
+ register(StreamColumn.Type.BOOL,new BooleanSerializer());
+ register(StreamColumn.Type.OBJECT,new JavaObjectSerializer());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/serialization/impl/BooleanSerializer.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/serialization/impl/BooleanSerializer.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/serialization/impl/BooleanSerializer.java
new file mode 100644
index 0000000..1e90569
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/serialization/impl/BooleanSerializer.java
@@ -0,0 +1,35 @@
+package org.apache.eagle.alert.engine.serialization.impl;
+
+import org.apache.eagle.alert.engine.serialization.Serializer;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+public class BooleanSerializer implements Serializer<Boolean> {
+ @Override
+ public void serialize(Boolean value, DataOutput dataOutput) throws IOException {
+ dataOutput.writeBoolean(value);
+ }
+
+ @Override
+ public Boolean deserialize(DataInput dataInput) throws IOException {
+ return dataInput.readBoolean();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/serialization/impl/DoubleSerializer.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/serialization/impl/DoubleSerializer.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/serialization/impl/DoubleSerializer.java
new file mode 100644
index 0000000..ad5f53c
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/serialization/impl/DoubleSerializer.java
@@ -0,0 +1,35 @@
+package org.apache.eagle.alert.engine.serialization.impl;
+
+import org.apache.eagle.alert.engine.serialization.Serializer;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+public class DoubleSerializer implements Serializer<Double> {
+ @Override
+ public void serialize(Double value, DataOutput dataOutput) throws IOException {
+ dataOutput.writeDouble(value);
+ }
+
+ @Override
+ public Double deserialize(DataInput dataInput) throws IOException {
+ return dataInput.readDouble();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/serialization/impl/FloatSerializer.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/serialization/impl/FloatSerializer.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/serialization/impl/FloatSerializer.java
new file mode 100644
index 0000000..18089a9
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/serialization/impl/FloatSerializer.java
@@ -0,0 +1,35 @@
+package org.apache.eagle.alert.engine.serialization.impl;
+
+import org.apache.eagle.alert.engine.serialization.Serializer;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+public class FloatSerializer implements Serializer<Float> {
+ @Override
+ public void serialize(Float value, DataOutput dataOutput) throws IOException {
+ dataOutput.writeFloat(value);
+ }
+
+ @Override
+ public Float deserialize(DataInput dataInput) throws IOException {
+ return dataInput.readFloat();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/serialization/impl/IntegerSerializer.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/serialization/impl/IntegerSerializer.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/serialization/impl/IntegerSerializer.java
new file mode 100644
index 0000000..d2473a9
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/serialization/impl/IntegerSerializer.java
@@ -0,0 +1,35 @@
+package org.apache.eagle.alert.engine.serialization.impl;
+
+import org.apache.eagle.alert.engine.serialization.Serializer;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+public class IntegerSerializer implements Serializer<Integer> {
+ @Override
+ public void serialize(Integer value, DataOutput dataOutput) throws IOException {
+ dataOutput.writeInt(value);
+ }
+
+ @Override
+ public Integer deserialize(DataInput dataInput) throws IOException {
+ return dataInput.readInt();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/serialization/impl/JavaObjectSerializer.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/serialization/impl/JavaObjectSerializer.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/serialization/impl/JavaObjectSerializer.java
new file mode 100644
index 0000000..76d2294
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/serialization/impl/JavaObjectSerializer.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.serialization.impl;
+
+import org.apache.commons.lang3.SerializationUtils;
+import org.apache.eagle.alert.engine.serialization.Serializer;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.io.Serializable;
+
+public class JavaObjectSerializer implements Serializer<Object> {
+ @Override
+ public void serialize(Object value, DataOutput dataOutput) throws IOException {
+ byte[] bytes = SerializationUtils.serialize((Serializable) value);
+ dataOutput.writeInt(bytes.length);
+ dataOutput.write(bytes);
+ }
+
+ @Override
+ public Object deserialize(DataInput dataInput) throws IOException {
+ int len = dataInput.readInt();
+ byte[] bytes = new byte[len];
+ dataInput.readFully(bytes);
+ return SerializationUtils.deserialize(bytes);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/serialization/impl/LongSerializer.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/serialization/impl/LongSerializer.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/serialization/impl/LongSerializer.java
new file mode 100644
index 0000000..8d85c76
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/serialization/impl/LongSerializer.java
@@ -0,0 +1,35 @@
+package org.apache.eagle.alert.engine.serialization.impl;
+
+import org.apache.eagle.alert.engine.serialization.Serializer;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+public class LongSerializer implements Serializer<Long> {
+ @Override
+ public void serialize(Long value, DataOutput dataOutput) throws IOException {
+ dataOutput.writeLong(value);
+ }
+
+ @Override
+ public Long deserialize(DataInput dataInput) throws IOException {
+ return dataInput.readLong();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/serialization/impl/PartitionedEventSerializerImpl.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/serialization/impl/PartitionedEventSerializerImpl.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/serialization/impl/PartitionedEventSerializerImpl.java
new file mode 100644
index 0000000..714920e
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/serialization/impl/PartitionedEventSerializerImpl.java
@@ -0,0 +1,98 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.serialization.impl;
+
+import com.google.common.io.ByteArrayDataOutput;
+import com.google.common.io.ByteStreams;
+import org.apache.eagle.alert.engine.coordinator.StreamPartition;
+import org.apache.eagle.alert.engine.model.PartitionedEvent;
+import org.apache.eagle.alert.engine.model.StreamEvent;
+import org.apache.eagle.alert.engine.serialization.PartitionedEventSerializer;
+import org.apache.eagle.alert.engine.serialization.SerializationMetadataProvider;
+import org.apache.eagle.alert.engine.serialization.Serializer;
+import org.apache.eagle.alert.engine.utils.CompressionUtils;
+
+import java.io.*;
+
+/**
+ * Stream Metadata Cached Serializer
+ *
+ * Performance:
+ *
+ * 1) VS Kryo Direct: reduce 73.4% space (bytes) and 42.5 % time (ms).
+ * 2) VS Java Native: reduce 92.5% space (bytes) and 94.2% time (ms)
+ *
+ * Tips:
+ *
+ * 1) Without-compression performs better than with compression for small event
+ *
+ * TODO: Cache Partition would save little space but almost half of serialization time, how to balance the performance?
+ *
+ * @see PartitionedEvent
+ */
+public class PartitionedEventSerializerImpl implements Serializer<PartitionedEvent>, PartitionedEventSerializer {
+ private final StreamEventSerializer streamEventSerializer;
+ private final Serializer<StreamPartition> streamPartitionSerializer;
+ private final boolean compress;
+
+ /**
+ * @param serializationMetadataProvider metadata provider
+ * @param compress false by default
+ */
+ public PartitionedEventSerializerImpl(SerializationMetadataProvider serializationMetadataProvider,boolean compress) {
+ this.streamEventSerializer = new StreamEventSerializer(serializationMetadataProvider);
+ this.streamPartitionSerializer = StreamPartitionSerializer.INSTANCE;
+ this.compress = compress;
+ }
+
+ public PartitionedEventSerializerImpl(SerializationMetadataProvider serializationMetadataProvider) {
+ this.streamEventSerializer = new StreamEventSerializer(serializationMetadataProvider);
+ this.streamPartitionSerializer = StreamPartitionSerializer.INSTANCE;
+ this.compress = false;
+ }
+
+ @Override
+ public void serialize(PartitionedEvent entity, DataOutput dataOutput) throws IOException {
+ dataOutput.writeLong(entity.getPartitionKey());
+ streamEventSerializer.serialize(entity.getEvent(), dataOutput);
+ streamPartitionSerializer.serialize(entity.getPartition(), dataOutput);
+ }
+
+ @Override
+ public PartitionedEvent deserialize(DataInput dataInput) throws IOException {
+ PartitionedEvent event = new PartitionedEvent();
+ event.setPartitionKey(dataInput.readLong());
+ StreamEvent streamEvent = streamEventSerializer.deserialize(dataInput);
+ StreamPartition partition = streamPartitionSerializer.deserialize(dataInput);
+ event.setEvent(streamEvent);
+ partition.setStreamId(streamEvent.getStreamId());
+ event.setPartition(partition);
+ return event;
+ }
+
+ @Override
+ public byte[] serialize(PartitionedEvent entity) throws IOException {
+ ByteArrayDataOutput dataOutput = ByteStreams.newDataOutput();
+ this.serialize(entity,dataOutput);
+ return compress ? CompressionUtils.compress(dataOutput.toByteArray()):dataOutput.toByteArray();
+ }
+
+ @Override
+ public PartitionedEvent deserialize(byte[] bytes) throws IOException {
+ return this.deserialize(ByteStreams.newDataInput(compress ? CompressionUtils.decompress(bytes):bytes));
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/serialization/impl/StreamEventSerializer.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/serialization/impl/StreamEventSerializer.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/serialization/impl/StreamEventSerializer.java
new file mode 100644
index 0000000..d166ced
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/serialization/impl/StreamEventSerializer.java
@@ -0,0 +1,87 @@
+package org.apache.eagle.alert.engine.serialization.impl;
+
+import org.apache.eagle.alert.engine.coordinator.StreamColumn;
+import org.apache.eagle.alert.engine.coordinator.StreamDefinition;
+import org.apache.eagle.alert.engine.model.StreamEvent;
+import org.apache.eagle.alert.engine.serialization.SerializationMetadataProvider;
+import org.apache.eagle.alert.engine.serialization.Serializer;
+import org.apache.eagle.alert.engine.serialization.Serializers;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.BitSet;
+
+/**
+ * @see StreamEvent
+ */
+public class StreamEventSerializer implements Serializer<StreamEvent> {
+ private final SerializationMetadataProvider serializationMetadataProvider;
+
+ public StreamEventSerializer(SerializationMetadataProvider serializationMetadataProvider){
+ this.serializationMetadataProvider = serializationMetadataProvider;
+ }
+
+ /**
+ *
+ * @param objects
+ * @return
+ */
+ private BitSet isNullBitSet(Object[] objects){
+ BitSet bitSet = new BitSet();
+ int i = 0;
+ for(Object obj:objects){
+ bitSet.set(i,obj == null);
+ i++;
+ }
+ return bitSet;
+ }
+
+ @Override
+ public void serialize(StreamEvent event, DataOutput dataOutput) throws IOException {
+ dataOutput.writeUTF(event.getStreamId());
+ dataOutput.writeLong(event.getTimestamp());
+ if(event.getData() == null || event.getData().length == 0){
+ dataOutput.writeInt(0);
+ }else{
+ BitSet isNullIndex = isNullBitSet(event.getData());
+ byte[] isNullBytes = isNullIndex.toByteArray();
+ dataOutput.writeInt(isNullBytes.length);
+ dataOutput.write(isNullBytes);
+ int i =0;
+ StreamDefinition definition = serializationMetadataProvider.getStreamDefinition(event.getStreamId());
+ if(definition == null) throw new IOException("StreamDefinition not found: "+event.getStreamId());
+ if(event.getData().length != definition.getColumns().size()){
+ throw new IOException("Event :"+event+" doesn't match with schema: "+definition);
+ }
+ for(StreamColumn column:definition.getColumns()){
+ if(!isNullIndex.get(i)) {
+ Serializers.getColumnSerializer(column.getType()).serialize(event.getData()[i],dataOutput);
+ }
+ i ++;
+ }
+ }
+ }
+
+ @Override
+ public StreamEvent deserialize(DataInput dataInput) throws IOException {
+ StreamEvent event = new StreamEvent();
+ event.setStreamId(dataInput.readUTF());
+ StreamDefinition definition = serializationMetadataProvider.getStreamDefinition(event.getStreamId());
+ event.setTimestamp(dataInput.readLong());
+ int isNullBytesLen = dataInput.readInt();
+ byte[] isNullBytes = new byte[isNullBytesLen];
+ dataInput.readFully(isNullBytes);
+ BitSet isNullIndex = BitSet.valueOf(isNullBytes);
+ Object[] attributes = new Object[definition.getColumns().size()];
+ int i = 0;
+ for(StreamColumn column:definition.getColumns()){
+ if(!isNullIndex.get(i)) {
+ attributes[i] = Serializers.getColumnSerializer(column.getType()).deserialize(dataInput);
+ }
+ i ++;
+ }
+ event.setData(attributes);
+ return event;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/serialization/impl/StreamPartitionDigestSerializer.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/serialization/impl/StreamPartitionDigestSerializer.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/serialization/impl/StreamPartitionDigestSerializer.java
new file mode 100644
index 0000000..4aa25b1
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/serialization/impl/StreamPartitionDigestSerializer.java
@@ -0,0 +1,83 @@
+package org.apache.eagle.alert.engine.serialization.impl;
+
+import org.apache.eagle.alert.engine.coordinator.StreamPartition;
+import org.apache.eagle.alert.engine.serialization.Serializer;
+
+import java.io.*;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.*;
+
+/**
+ * Don't serialize streamId
+ *
+ * @see StreamPartition
+ */
+public class StreamPartitionDigestSerializer implements Serializer<StreamPartition> {
+ public final static StreamPartitionDigestSerializer INSTANCE = new StreamPartitionDigestSerializer();
+
+ private final Map<DigestBytes,StreamPartition> checkSumPartitionMap = new HashMap<>();
+ private final Map<StreamPartition,DigestBytes> partitionCheckSumMap = new HashMap<>();
+
+ @Override
+ public void serialize(StreamPartition partition, DataOutput dataOutput) throws IOException {
+ DigestBytes checkSum = partitionCheckSumMap.get(partition);
+ if(checkSum == null){
+ try {
+ checkSum = digestCheckSum(partition);
+ partitionCheckSumMap.put(partition,checkSum);
+ checkSumPartitionMap.put(checkSum,partition);
+ } catch (NoSuchAlgorithmException e) {
+ throw new IOException(e);
+ }
+ }
+ dataOutput.writeInt(checkSum.size());
+ dataOutput.write(checkSum.toByteArray());
+ }
+
+ @Override
+ public StreamPartition deserialize(DataInput dataInput) throws IOException {
+ int checkSumLen = dataInput.readInt();
+ byte[] checksum = new byte[checkSumLen];
+ dataInput.readFully(checksum);
+ StreamPartition partition = checkSumPartitionMap.get(new DigestBytes(checksum));
+ if(partition == null){
+ throw new IOException("Illegal partition checksum: "+checksum);
+ }
+ return partition;
+ }
+
+ private class DigestBytes {
+ private final byte[] data;
+
+ public DigestBytes(byte[] bytes){
+ this.data = bytes;
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ return other instanceof DigestBytes && Arrays.equals(data, ((DigestBytes) other).data);
+ }
+
+ @Override
+ public int hashCode() {
+ return Arrays.hashCode(data);
+ }
+ public int size(){
+ return data.length;
+ }
+ public byte[] toByteArray(){
+ return data;
+ }
+ }
+
+ private DigestBytes digestCheckSum(Object obj) throws IOException, NoSuchAlgorithmException {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ ObjectOutputStream oos = new ObjectOutputStream(baos);
+ oos.writeObject(obj);
+ oos.close();
+ MessageDigest m = MessageDigest.getInstance("SHA1");
+ m.update(baos.toByteArray());
+ return new DigestBytes(m.digest());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/serialization/impl/StreamPartitionSerializer.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/serialization/impl/StreamPartitionSerializer.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/serialization/impl/StreamPartitionSerializer.java
new file mode 100644
index 0000000..b92c301
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/serialization/impl/StreamPartitionSerializer.java
@@ -0,0 +1,64 @@
+package org.apache.eagle.alert.engine.serialization.impl;
+
+import org.apache.eagle.alert.engine.coordinator.StreamPartition;
+import org.apache.eagle.alert.engine.coordinator.StreamSortSpec;
+import org.apache.eagle.alert.engine.serialization.Serializer;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Don't serialize streamId
+ *
+ * @see StreamPartition
+ */
+public class StreamPartitionSerializer implements Serializer<StreamPartition> {
+ public final static StreamPartitionSerializer INSTANCE = new StreamPartitionSerializer();
+
+ @Override
+ public void serialize(StreamPartition partition, DataOutput dataOutput) throws IOException {
+ dataOutput.writeUTF(partition.getType().toString());
+ if(partition.getColumns() == null || partition.getColumns().size() == 0){
+ dataOutput.writeInt(0);
+ } else {
+ dataOutput.writeInt(partition.getColumns().size());
+ for(String column:partition.getColumns()){
+ dataOutput.writeUTF(column);
+ }
+ }
+ if(partition.getSortSpec() == null){
+ dataOutput.writeByte(0);
+ }else {
+ dataOutput.writeByte(1);
+ dataOutput.writeUTF(partition.getSortSpec().getWindowPeriod());
+ dataOutput.writeInt(partition.getSortSpec().getWindowMargin());
+ }
+ }
+
+ @Override
+ public StreamPartition deserialize(DataInput dataInput) throws IOException {
+ StreamPartition partition = new StreamPartition();
+ partition.setType(StreamPartition.Type.locate(dataInput.readUTF()));
+ int colSize = dataInput.readInt();
+ if(colSize>0){
+ List<String> columns = new ArrayList<>(colSize);
+ for(int i=0;i<colSize;i++){
+ columns.add(dataInput.readUTF());
+ }
+ partition.setColumns(columns);
+ }
+ if(dataInput.readByte() == 1){
+ String period = dataInput.readUTF();
+ int margin = dataInput.readInt();
+
+ StreamSortSpec sortSpec = new StreamSortSpec();
+ sortSpec.setWindowPeriod(period);
+ sortSpec.setWindowMargin(margin);
+ partition.setSortSpec(sortSpec);
+ }
+ return partition;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/serialization/impl/StringSerializer.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/serialization/impl/StringSerializer.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/serialization/impl/StringSerializer.java
new file mode 100644
index 0000000..2a1541a
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/serialization/impl/StringSerializer.java
@@ -0,0 +1,35 @@
+package org.apache.eagle.alert.engine.serialization.impl;
+
+import org.apache.eagle.alert.engine.serialization.Serializer;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+public class StringSerializer implements Serializer<String> {
+ @Override
+ public void serialize(String value, DataOutput dataOutput) throws IOException {
+ dataOutput.writeUTF(value);
+ }
+
+ @Override
+ public String deserialize(DataInput dataInput) throws IOException {
+ return dataInput.readUTF();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/siddhi/extension/AttributeCollectAggregator.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/siddhi/extension/AttributeCollectAggregator.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/siddhi/extension/AttributeCollectAggregator.java
new file mode 100644
index 0000000..b705564
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/siddhi/extension/AttributeCollectAggregator.java
@@ -0,0 +1,122 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+package org.apache.eagle.alert.engine.siddhi.extension;
+
+import java.util.LinkedList;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.wso2.siddhi.core.config.ExecutionPlanContext;
+import org.wso2.siddhi.core.executor.ExpressionExecutor;
+import org.wso2.siddhi.core.query.selector.attribute.aggregator.AttributeAggregator;
+import org.wso2.siddhi.query.api.definition.Attribute;
+import org.wso2.siddhi.query.api.definition.Attribute.Type;
+
+import com.google.common.collect.ImmutableList;
+
+/**
+ * @since Apr 1, 2016
+ *
+ */
+public class AttributeCollectAggregator extends AttributeAggregator {
+
+ private static final Logger LOG = LoggerFactory.getLogger(AttributeCollectAggregator.class);
+
+ private LinkedList<Object> value;
+
+ public AttributeCollectAggregator() {
+ value = new LinkedList<Object>();
+ }
+
+ @Override
+ public void start() {
+ }
+
+ @Override
+ public void stop() {
+ }
+
+ @Override
+ public Object[] currentState() {
+ return value.toArray();
+ }
+
+ @Override
+ public void restoreState(Object[] arg0) {
+ value = new LinkedList<Object>();
+ if (arg0 != null) {
+ for (Object o : arg0) {
+ value.add(o);
+ }
+ }
+ }
+
+ @Override
+ public Type getReturnType() {
+ return Attribute.Type.OBJECT;
+ }
+
+ @Override
+ protected void init(ExpressionExecutor[] arg0, ExecutionPlanContext arg1) {
+ // TODO: Support max of elements?
+ }
+
+ @Override
+ public Object processAdd(Object arg0) {
+ value.add(arg0);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("processAdd: current values are : " + value);
+ }
+ return ImmutableList.copyOf(value);
+ }
+
+ @Override
+ public Object processAdd(Object[] arg0) {
+ value.add(arg0);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("processAdd: current values are : " + value);
+ }
+ return ImmutableList.copyOf(value);
+ }
+
+ // / NOTICE: non O(1)
+ @Override
+ public Object processRemove(Object arg0) {
+ value.remove(arg0);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("processRemove: current values are : " + value);
+ }
+ return ImmutableList.copyOf(value);
+ }
+
+ // / NOTICE: non O(1)
+ @Override
+ public Object processRemove(Object[] arg0) {
+ value.remove(arg0);
+ LOG.info("processRemove: current values are : " + value);
+ return ImmutableList.copyOf(value);
+ }
+
+ @Override
+ public Object reset() {
+ value.clear();
+ return value;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/BaseStreamWindow.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/BaseStreamWindow.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/BaseStreamWindow.java
new file mode 100644
index 0000000..38c5c30
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/BaseStreamWindow.java
@@ -0,0 +1,183 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.sorter;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+import org.apache.eagle.alert.engine.PartitionedEventCollector;
+import org.apache.eagle.alert.utils.DateTimeUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * TODO: Make sure thread-safe
+ * TODO: Leverage Off-Heap Memory to persist append-only events collection
+ */
+public abstract class BaseStreamWindow implements StreamWindow {
+ private final long endTime;
+ private final long startTime;
+ private final long margin;
+ private final AtomicBoolean expired;
+ private final long createdTime;
+ private final static Logger LOG = LoggerFactory.getLogger(BaseStreamWindow.class);
+ private PartitionedEventCollector collector;
+ private final AtomicLong lastFlushedStreamTime;
+ private final AtomicLong lastFlushedSystemTime;
+
+ /**
+ * @param startTime
+ * @param endTime
+ * @param marginTime
+ */
+ public BaseStreamWindow(long startTime, long endTime, long marginTime){
+ if(startTime >= endTime){
+ throw new IllegalArgumentException("startTime: "+startTime+" >= endTime: "+endTime+", expected: startTime < endTime");
+ }
+ if(marginTime > endTime - startTime){
+ throw new IllegalArgumentException("marginTime: "+marginTime+" > endTime: "+endTime+" - startTime "+startTime+", expected: marginTime < endTime - startTime");
+ }
+ this.startTime = startTime;
+ this.endTime = endTime;
+ this.margin = marginTime;
+ this.expired = new AtomicBoolean(false);
+ this.createdTime = System.currentTimeMillis();
+ this.lastFlushedStreamTime = new AtomicLong(0);
+ this.lastFlushedSystemTime = new AtomicLong(this.createdTime);
+ }
+
+ @Override
+ public void register(PartitionedEventCollector collector) {
+ if(this.collector!=null)
+ throw new IllegalArgumentException("Duplicated collector error");
+ this.collector = collector;
+ }
+
+ @Override
+ public long createdTime() {
+ return createdTime;
+ }
+
+ public long startTime() {
+ return this.startTime;
+ }
+
+ @Override
+ public long rejectTime(){
+ return this.lastFlushedStreamTime.get();
+ }
+
+ @Override
+ public long margin() {
+ return this.margin;
+ }
+
+ public long endTime() {
+ return this.endTime;
+ }
+
+ public boolean accept(final long eventTime) {
+ return !expired() && eventTime >= startTime && eventTime < endTime
+ && eventTime >= lastFlushedStreamTime.get(); // dropped
+ }
+
+ public boolean expired() {
+ return expired.get();
+ }
+
+ @Override
+ public boolean alive() {
+ return !expired.get();
+ }
+
+ /**
+ *
+ *
+ * Expire when
+ * 1) If stream time >= endTime + marginTime, then flush and expire
+ * 2) If systemTime - flushedTime > endTime - startTime + marginTime && streamTime >= endTime, then flush and expire.
+ * 3) If systemTime - flushedTime > endTime - startTime + marginTime && streamTime < endTime, then flush but not expire.
+ * 4) else do nothing
+ *
+ * @param clock stream time clock
+ * @param globalSystemTime system time clock
+ */
+ @Override
+ public synchronized void onTick(StreamTimeClock clock,long globalSystemTime) {
+ if(!expired()) {
+ if(clock.getTime() >= endTime + margin){
+ LOG.info("Expiring {} at stream time:{}, latency:{}, window: {}",clock.getStreamId(),DateTimeUtil.millisecondsToHumanDateWithMilliseconds(clock.getTime()),globalSystemTime - lastFlushedSystemTime.get(),this);
+ lastFlushedStreamTime.set(clock.getTime());
+ lastFlushedSystemTime.set(globalSystemTime);
+ flush();
+ expired.set(true);
+ } else if(globalSystemTime - lastFlushedSystemTime.get() >= endTime + margin - startTime && size() > 0){
+ LOG.info("Flushing {} at system time: {}, stream time: {}, latency: {}, window: {}",clock.getStreamId(),DateTimeUtil.millisecondsToHumanDateWithMilliseconds(globalSystemTime),DateTimeUtil.millisecondsToHumanDateWithMilliseconds(clock.getTime()),globalSystemTime - lastFlushedSystemTime.get(),this);
+ lastFlushedStreamTime.set(clock.getTime());
+ lastFlushedSystemTime.set(globalSystemTime);
+ flush();
+ if(lastFlushedStreamTime.get()>=this.endTime) expired.set(true);
+ }
+ } else {
+ LOG.warn("Window has already expired, should not tick: {}",this.toString());
+ }
+ }
+
+ public void close() {
+ flush();
+ expired.set(true);
+ }
+
+ @Override
+ public int hashCode() {
+ return new HashCodeBuilder().append(startTime).append(endTime).append(margin).build();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if(obj !=null && obj instanceof BaseStreamWindow) {
+ BaseStreamWindow another = (BaseStreamWindow) obj;
+ return another.startTime == this.startTime && another.endTime == this.endTime && another.margin == this.margin;
+ }
+ return false;
+ }
+
+ @Override
+ public void flush(){
+ if(this.collector == null) throw new NullPointerException("Collector is not given before window flush");
+ this.flush(collector);
+ }
+
+ /**
+ *
+ * @param collector
+ * @return max timestamp
+ */
+ protected abstract void flush(PartitionedEventCollector collector);
+
+ @Override
+ public String toString() {
+ return String.format("StreamWindow[period=[%s,%s), margin=%s ms, size=%s, reject=%s]",
+ DateTimeUtil.millisecondsToHumanDateWithMilliseconds(this.startTime),
+ DateTimeUtil.millisecondsToHumanDateWithMilliseconds(this.endTime),
+ this.margin,
+ size(),
+ this.rejectTime() == 0 ? DateTimeUtil.millisecondsToHumanDateWithMilliseconds(this.startTime): DateTimeUtil.millisecondsToHumanDateWithMilliseconds(this.rejectTime())
+ );
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/StreamTimeClock.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/StreamTimeClock.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/StreamTimeClock.java
new file mode 100644
index 0000000..0881e35
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/StreamTimeClock.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.sorter;
+
+/**
+ * The time clock per stream
+ *
+ * Should be thread-safe between getTime and moveForward
+ *
+ * By default, we currently simple support event timestamp now
+ */
+public interface StreamTimeClock {
+ /**
+ * Get stream id
+ *
+ * @return stream id
+ */
+ String getStreamId();
+
+ /**
+ * Get current time
+ *
+ * @return current timestamp value
+ */
+ long getTime();
+
+ /**
+ * @param timestamp move forward current time to given timestamp
+ */
+ void moveForward(long timestamp);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/StreamTimeClockListener.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/StreamTimeClockListener.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/StreamTimeClockListener.java
new file mode 100644
index 0000000..f7e463d
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/StreamTimeClockListener.java
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.sorter;
+
+public interface StreamTimeClockListener {
+ /**
+ * @see StreamWindow
+ *
+ * @param streamTime
+ * @param globalSystemTime
+ */
+ void onTick(StreamTimeClock streamTime, long globalSystemTime);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/StreamTimeClockManager.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/StreamTimeClockManager.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/StreamTimeClockManager.java
new file mode 100644
index 0000000..29b13eb
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/StreamTimeClockManager.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.sorter;
+
+import java.io.Serializable;
+
+/**
+ *
+ * By default, we could keep the current time clock in memory,
+ * Eventually we may need to consider the global time synchronization across all nodes
+ *
+ * TODO: maybe need to synchronize time clock globally
+ *
+ * 1) When to initialize window according to start time
+ * 2) When to close expired window according to current time
+ * 3) Automatically tick periodically as the single place for control lock
+ *
+ */
+public interface StreamTimeClockManager extends StreamTimeClockTrigger, Serializable{
+ /**
+ * @return StreamTimeClock instance
+ */
+ StreamTimeClock createStreamTimeClock(String streamId);
+
+ StreamTimeClock getStreamTimeClock(String streamId);
+
+ /**
+ * @param streamId
+ */
+ void removeStreamTimeClock(String streamId);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/StreamTimeClockTrigger.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/StreamTimeClockTrigger.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/StreamTimeClockTrigger.java
new file mode 100644
index 0000000..a0cd184
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/StreamTimeClockTrigger.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.sorter;
+
+
+/**
+ * Possible implementation:
+ *
+ * 1) EventTimeClockTrigger (by default)
+ * 2) SystemTimeClockTrigger
+ */
+public interface StreamTimeClockTrigger {
+ /**
+ * @param streamId stream id to listen to
+ * @param listener to watch on streamId
+ */
+ void registerListener(String streamId, StreamTimeClockListener listener);
+
+ /**
+ *
+ * @param streamClock
+ * @param listener
+ */
+ void registerListener(StreamTimeClock streamClock, StreamTimeClockListener listener);
+
+ /**
+ * @param listener listener to remove
+ */
+ void removeListener(StreamTimeClockListener listener);
+
+ /**
+ * Trigger tick of all listeners on certain stream
+ *
+ * @param streamId stream id
+ */
+ void triggerTickOn(String streamId);
+
+ /**
+ * Update time per new event time on stream
+ *
+ * @param streamId
+ * @param timestamp
+ */
+ void onTimeUpdate(String streamId, long timestamp);
+
+ void close();
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/StreamWindow.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/StreamWindow.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/StreamWindow.java
new file mode 100644
index 0000000..2500122
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/StreamWindow.java
@@ -0,0 +1,113 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.sorter;
+
+import org.apache.eagle.alert.engine.PartitionedEventCollector;
+import org.apache.eagle.alert.engine.model.PartitionedEvent;
+
+/**
+ * <h2>Tumbling Window instead Sliding Window</h2>
+ * We could have time overlap to sort out-of-ordered stream,
+ * but each window should never have events overlap, otherwise will have logic problem.
+ *
+ *
+ * <h2>Ingestion Time Policy</h2>
+ * Different notions of time, namely processing time, event time, and ingestion time.
+ *
+ * <ol>
+ * <li>
+ * In processing time, windows are defined with respect to the wall clock of the machine that builds and processes a window, i.e., a one minute processing time window collects elements for exactly one minute.
+ * </li>
+ * <li>
+ * In event time, windows are defined with respect to timestamps that are attached to each event record. This is common for many types of events, such as log entries, sensor data, etc, where the timestamp usually represents the time at which the event occurred. Event time has several benefits over processing time. First of all, it decouples the program semantics from the actual serving speed of the source and the processing performance of system. Hence you can process historic data, which is served at maximum speed, and continuously produced data with the same program. It also prevents semantically incorrect results in case of backpressure or delays due to failure recovery. Second, event time windows compute correct results, even if events arrive out-of-order of their timestamp which is common if a data stream gathers events from distributed sources.
+ * </li>
+ * <li>
+ * Ingestion time is a hybrid of processing and event time. It assigns wall clock timestamps to records as soon as they arrive in the system (at the source) and continues processing with event time semantics based on the attached timestamps.
+ * </li>
+ * </ol>
+ */
+public interface StreamWindow extends StreamTimeClockListener {
+ /**
+ * @return Created timestamp
+ */
+ long createdTime();
+
+ /**
+ * Get start time
+ *
+ * @return
+ */
+ long startTime();
+
+ long margin();
+
+ /**
+ * @return reject timestamp < rejectTime()
+ */
+ long rejectTime();
+
+ /**
+ * Get end time
+ *
+ * @return
+ */
+ long endTime();
+
+ /**
+ * @param timestamp event time
+ * @return true/false in boolean
+ */
+ boolean accept(long timestamp);
+
+ /**
+ * Window is expired
+ *
+ * @return whether window is expired
+ */
+ boolean expired();
+
+ /**
+ * @return whether window is alive
+ */
+ boolean alive();
+
+ /**
+ *
+ * @param event
+ */
+ boolean add(PartitionedEvent event);
+
+// /**
+// * @param collector Drain to output collector
+// */
+// void flush(PartitionedEventCollector collector);
+
+ void flush();
+
+ /**
+ * Close window
+ */
+ void close();
+
+ void register(PartitionedEventCollector collector);
+
+ /**
+ *
+ * @return
+ */
+ int size();
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/StreamWindowManager.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/StreamWindowManager.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/StreamWindowManager.java
new file mode 100644
index 0000000..a5f43aa
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/StreamWindowManager.java
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.sorter;
+
+import java.io.Closeable;
+import java.util.Collection;
+
+/**
+ * TODO: Reuse existing expired window to avoid recreating new windows again and again
+ *
+ * Single stream window manager
+ */
+public interface StreamWindowManager extends StreamTimeClockListener, Closeable {
+
+ /**
+ * @param initialTime
+ * @return
+ */
+ StreamWindow addNewWindow(long initialTime);
+
+ /**
+ * @param window
+ */
+ void removeWindow(StreamWindow window);
+
+ /**
+ * @param window
+ * @return
+ */
+ boolean hasWindow(StreamWindow window);
+
+ /**
+ * @param timestamp time
+ * @return whether window exists for time
+ */
+ boolean hasWindowFor(long timestamp);
+
+ /**
+ * @return Internal collection for performance optimization
+ */
+ Collection<StreamWindow> getWindows();
+
+ /**
+ * @param timestamp
+ * @return
+ */
+ StreamWindow getWindowFor(long timestamp);
+
+ boolean reject(long timestamp);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/StreamWindowRepository.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/StreamWindowRepository.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/StreamWindowRepository.java
new file mode 100755
index 0000000..3e035c5
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/StreamWindowRepository.java
@@ -0,0 +1,257 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.sorter;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+
+import org.apache.eagle.alert.engine.sorter.impl.StreamSortedWindowInMapDB;
+import org.apache.eagle.alert.engine.sorter.impl.StreamSortedWindowOnHeap;
+import org.mapdb.DB;
+import org.mapdb.DBMaker;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Preconditions;
+
+/**
+ *
+ * ===== Benchmark Result Report =====<br/><br/>
+ *
+ * Num. Operation Type Time<br/>
+ * ---- --------- ---- ----<br/>
+ * 1000 FlushTime DIRECT_MEMORY : 55<br/>
+ * 1000 FlushTime FILE_RAF : 63<br/>
+ * 1000 FlushTime MEMORY : 146<br/>
+ * 1000 FlushTime ONHEAP : 17<br/>
+ * 1000 InsertTime DIRECT_MEMORY : 68<br/>
+ * 1000 InsertTime FILE_RAF : 223<br/>
+ * 1000 InsertTime MEMORY : 273<br/>
+ * 1000 InsertTime ONHEAP : 20<br/>
+ * 10000 FlushTime DIRECT_MEMORY : 551<br/>
+ * 10000 FlushTime FILE_RAF : 668<br/>
+ * 10000 FlushTime MEMORY : 643<br/>
+ * 10000 FlushTime ONHEAP : 5<br/>
+ * 10000 InsertTime DIRECT_MEMORY : 446<br/>
+ * 10000 InsertTime FILE_RAF : 2095<br/>
+ * 10000 InsertTime MEMORY : 784<br/>
+ * 10000 InsertTime ONHEAP : 29<br/>
+ * 100000 FlushTime DIRECT_MEMORY : 6139<br/>
+ * 100000 FlushTime FILE_RAF : 6237<br/>
+ * 100000 FlushTime MEMORY : 6238<br/>
+ * 100000 FlushTime ONHEAP : 18<br/>
+ * 100000 InsertTime DIRECT_MEMORY : 4499<br/>
+ * 100000 InsertTime FILE_RAF : 22343<br/>
+ * 100000 InsertTime MEMORY : 4962<br/>
+ * 100000 InsertTime ONHEAP : 107<br/>
+ * 1000000 FlushTime DIRECT_MEMORY : 61356<br/>
+ * 1000000 FlushTime FILE_RAF : 63025<br/>
+ * 1000000 FlushTime MEMORY : 61380<br/>
+ * 1000000 FlushTime ONHEAP : 47<br/>
+ * 1000000 InsertTime DIRECT_MEMORY : 43637<br/>
+ * 1000000 InsertTime FILE_RAF : 464481<br/>
+ * 1000000 InsertTime MEMORY : 44367<br/>
+ * 1000000 InsertTime ONHEAP : 2040<br/>
+ *
+ * @see StreamSortedWindowOnHeap
+ * @see org.mapdb.DBMaker
+ */
+public class StreamWindowRepository {
+ public enum StorageType {
+ /**
+ * Creates new in-memory database which stores all data on heap without serialization.
+ * This mode should be very fast, but data will affect Garbage PartitionedEventCollector the same way as traditional Java Collections.
+ */
+ ONHEAP,
+
+ /**
+ * Creates new in-memory database. Changes are lost after JVM exits.
+ * This option serializes data into {@code byte[]},
+ * so they are not affected by Garbage PartitionedEventCollector.
+ */
+ MEMORY,
+
+ /**
+ * <p>
+ * Creates new in-memory database. Changes are lost after JVM exits.
+ * </p><p>
+ * This will use {@code DirectByteBuffer} outside of HEAP, so Garbage Collector is not affected
+ * You should increase ammount of direct memory with
+ * {@code -XX:MaxDirectMemorySize=10G} JVM param
+ * </p>
+ */
+ DIRECT_MEMORY,
+
+ /**
+ * By default use File.createTempFile("streamwindows","temp")
+ */
+ FILE_RAF
+ }
+
+ private final static Logger LOG = LoggerFactory.getLogger(StreamWindowRepository.class);
+ private final Map<StorageType,DB> dbPool;
+ private StreamWindowRepository(){
+ dbPool = new HashMap<>();
+ }
+
+ private static StreamWindowRepository repository;
+
+ /**
+ * Close automatically when JVM exists
+ *
+ * @return StreamWindowRepository singletonInstance
+ */
+ public static StreamWindowRepository getSingletonInstance(){
+ synchronized (StreamWindowRepository.class) {
+ if (repository == null) {
+ repository = new StreamWindowRepository();
+ Runtime.getRuntime().addShutdownHook(new Thread() {
+ @Override
+ public void run() {
+ repository.close();
+ }
+ });
+ }
+ return repository;
+ }
+ }
+
+ private DB createMapDB(StorageType storageType){
+ synchronized (dbPool) {
+ if(!dbPool.containsKey(storageType)){
+ DB db;
+ switch (storageType){
+ case ONHEAP:
+ db = DBMaker.heapDB().closeOnJvmShutdown().make();
+ LOG.info("Create ONHEAP mapdb");
+ break;
+ case MEMORY:
+ db = DBMaker.memoryDB().closeOnJvmShutdown().make();
+ LOG.info("Create MEMORY mapdb");
+ break;
+ case DIRECT_MEMORY:
+ db = DBMaker.memoryDirectDB().closeOnJvmShutdown().make();
+ LOG.info("Create DIRECT_MEMORY mapdb");
+ break;
+ case FILE_RAF:
+ try {
+ File file = File.createTempFile("window-", ".map");
+ file.delete();
+ file.deleteOnExit();
+ Preconditions.checkNotNull(file, "file is null");
+ db = DBMaker.fileDB(file).deleteFilesAfterClose().make();
+ LOG.info("Created FILE_RAF map file at {}",file.getAbsolutePath());
+ } catch (IOException e) {
+ throw new IllegalStateException(e);
+ }
+ break;
+ default:
+ throw new IllegalArgumentException("Illegal storage type: "+storageType);
+ }
+ dbPool.put(storageType,db);
+ return db;
+ }
+ return dbPool.get(storageType);
+ }
+ }
+
+ public StreamWindow createWindow(long start,long end, long margin, StorageType type){
+ StreamWindow ret;
+ switch (type) {
+ case ONHEAP:
+ ret = new StreamSortedWindowOnHeap(start, end, margin);
+ break;
+ default:
+ ret = new StreamSortedWindowInMapDB(
+ start,end,margin,
+ createMapDB(type),
+ UUID.randomUUID().toString()
+ );
+ break;
+ }
+
+ if(LOG.isDebugEnabled()) LOG.debug("Created new {}, type: {}",ret,type);
+ return ret;
+ }
+
+ public StreamWindow createWindow(long start,long end, long margin, StreamWindowStrategy strategy){
+ return strategy.createWindow(start,end,margin,this);
+ }
+
+ public StreamWindow createWindow(long start,long end, long margin){
+ return OnHeapStrategy.INSTANCE.createWindow(start,end,margin,this);
+ }
+
+ public void close(){
+ for(Map.Entry<StorageType,DB> entry:dbPool.entrySet()){
+ entry.getValue().close();
+ }
+ dbPool.clear();
+ }
+
+ public interface StreamWindowStrategy {
+ /**
+ *
+ * @param start
+ * @param end
+ * @param margin
+ * @return
+ */
+ StreamWindow createWindow(long start,long end, long margin,StreamWindowRepository repository);
+ }
+
+ public static class OnHeapStrategy implements StreamWindowStrategy{
+ public static final OnHeapStrategy INSTANCE = new OnHeapStrategy();
+ @Override
+ public StreamWindow createWindow(long start, long end, long margin,StreamWindowRepository repository) {
+ return repository.createWindow(start,end,margin,StorageType.ONHEAP);
+ }
+ }
+
+ public static class WindowSizeStrategy implements StreamWindowStrategy {
+ private final static long ONE_HOUR = 3600 * 1000;
+ private final static long FIVE_HOURS = 5* 3600 * 1000;
+ private final long onheapWindowSizeLimit;
+ private final long offheapWindowSizeLimit;
+
+ public static WindowSizeStrategy INSTANCE = new WindowSizeStrategy(ONE_HOUR,FIVE_HOURS);
+
+ public WindowSizeStrategy(long onheapWindowSizeLimit, long offheapWindowSizeLimit){
+ this.offheapWindowSizeLimit = offheapWindowSizeLimit;
+ this.onheapWindowSizeLimit = onheapWindowSizeLimit;
+
+ if(this.offheapWindowSizeLimit <this.onheapWindowSizeLimit){
+ throw new IllegalStateException("offheapWindowSizeLimit "+this.offheapWindowSizeLimit +" < onheapWindowSizeLimit "+this.onheapWindowSizeLimit);
+ }
+ }
+
+ @Override
+ public StreamWindow createWindow(long start, long end, long margin,StreamWindowRepository repository) {
+ long windowLength = end - start;
+ if(windowLength <= onheapWindowSizeLimit){
+ return repository.createWindow(start,end,margin, StreamWindowRepository.StorageType.ONHEAP);
+ }else if(windowLength > onheapWindowSizeLimit & windowLength <= offheapWindowSizeLimit){
+ return repository.createWindow(start,end,margin, StreamWindowRepository.StorageType.DIRECT_MEMORY);
+ }else {
+ return repository.createWindow(start,end,margin, StreamWindowRepository.StorageType.FILE_RAF);
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/impl/CachedEventGroupSerializer.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/impl/CachedEventGroupSerializer.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/impl/CachedEventGroupSerializer.java
new file mode 100644
index 0000000..6cdf8a0
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/impl/CachedEventGroupSerializer.java
@@ -0,0 +1,114 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.sorter.impl;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.lang.builder.HashCodeBuilder;
+import org.apache.eagle.alert.engine.coordinator.StreamPartition;
+import org.apache.eagle.alert.engine.model.PartitionedEvent;
+import org.apache.eagle.alert.engine.model.StreamEvent;
+import org.apache.eagle.alert.engine.utils.SerializableUtils;
+import org.jetbrains.annotations.NotNull;
+import org.mapdb.DataInput2;
+import org.mapdb.DataOutput2;
+import org.mapdb.serializer.GroupSerializerObjectArray;
+
+/**
+ * @deprecated performance is worse, should investigate
+ */
+public class CachedEventGroupSerializer extends GroupSerializerObjectArray<PartitionedEvent[]> {
+ private Map<Integer,StreamPartition> hashCodePartitionDict = new HashMap<>();
+ private void writePartitionedEvent(DataOutput2 out,PartitionedEvent event) throws IOException {
+ out.packLong(event.getPartitionKey());
+ int partitionHashCode = 0;
+ if(event.getPartition()!=null) {
+ partitionHashCode = event.getPartition().hashCode();
+ if (!hashCodePartitionDict.containsKey(partitionHashCode)) {
+ hashCodePartitionDict.put(partitionHashCode, event.getPartition());
+ }
+ }
+ out.packInt(partitionHashCode);
+ if(event.getEvent()!=null) {
+ byte[] eventBytes = SerializableUtils.serializeToCompressedByteArray(event.getEvent());
+ out.packInt(eventBytes.length);
+ out.write(eventBytes);
+ } else {
+ out.packInt(0);
+ }
+ }
+
+ private PartitionedEvent readPartitionedEvent(DataInput2 in) throws IOException {
+ PartitionedEvent event = new PartitionedEvent();
+ event.setPartitionKey(in.unpackLong());
+ int partitionHashCode = in.unpackInt();
+ if(partitionHashCode!=0 && hashCodePartitionDict.containsKey(partitionHashCode)) {
+ event.setPartition(hashCodePartitionDict.get(partitionHashCode));
+ }
+ int eventBytesLen = in.unpackInt();
+ if(eventBytesLen > 0) {
+ byte[] eventBytes = new byte[eventBytesLen];
+ in.readFully(eventBytes);
+ event.setEvent((StreamEvent) SerializableUtils.deserializeFromCompressedByteArray(eventBytes, "Deserialize event from bytes"));
+ }
+ return event;
+ }
+
+ @Override
+ public void serialize(DataOutput2 out, PartitionedEvent[] value) throws IOException {
+ out.packInt(value.length);
+ for (PartitionedEvent event : value) {
+ writePartitionedEvent(out,event);
+ }
+ }
+
+ @Override
+ public PartitionedEvent[] deserialize(DataInput2 in, int available) throws IOException {
+ final int size = in.unpackInt();
+ PartitionedEvent[] ret = new PartitionedEvent[size];
+ for (int i = 0; i < size; i++) {
+ ret[i] = readPartitionedEvent(in);
+ }
+ return ret;
+ }
+
+ @Override
+ public boolean isTrusted() {
+ return true;
+ }
+
+ @Override
+ public boolean equals(PartitionedEvent[] a1, PartitionedEvent[] a2) {
+ return a1[0].getTimestamp() == a2[0].getTimestamp();
+ }
+
+ @Override
+ public int hashCode(@NotNull PartitionedEvent[] events, int seed) {
+ return new HashCodeBuilder().append(events).toHashCode();
+ }
+
+ @Override
+ public int compare(PartitionedEvent[] o1, PartitionedEvent[] o2) {
+ if(o1.length>0 && o2.length>0) {
+ return (int) (o1[0].getTimestamp() - o2[0].getTimestamp());
+ }else{
+ return 0;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/impl/PartitionedEventGroupSerializer.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/impl/PartitionedEventGroupSerializer.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/impl/PartitionedEventGroupSerializer.java
new file mode 100644
index 0000000..caa291e
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/impl/PartitionedEventGroupSerializer.java
@@ -0,0 +1,112 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.sorter.impl;
+
+import java.io.IOException;
+import java.util.Comparator;
+
+import org.apache.eagle.alert.engine.model.PartitionedEvent;
+import org.apache.eagle.alert.engine.utils.SerializableUtils;
+import org.jetbrains.annotations.NotNull;
+import org.mapdb.DataInput2;
+import org.mapdb.DataOutput2;
+import org.mapdb.Serializer;
+import org.mapdb.serializer.GroupSerializer;
+
+
+public class PartitionedEventGroupSerializer implements GroupSerializer<PartitionedEvent[]> {
+ private static final GroupSerializer<byte[]> delegate = Serializer.BYTE_ARRAY;
+
+ private static PartitionedEvent[] deserialize(byte[] bytes){
+ return (PartitionedEvent[]) SerializableUtils.deserializeFromCompressedByteArray(bytes,"deserialize as stream event");
+ }
+
+ private static byte[] serialize(PartitionedEvent[] events){
+ return SerializableUtils.serializeToCompressedByteArray(events);
+ }
+
+ @Override
+ public int valueArraySearch(Object keys, PartitionedEvent[] key) {
+ return delegate.valueArraySearch(keys,serialize(key));
+ }
+
+ @SuppressWarnings("rawtypes")
+ @Override
+ public int valueArraySearch(Object keys, PartitionedEvent[] key, Comparator comparator) {
+ return delegate.valueArraySearch(keys,serialize(key),comparator);
+ }
+
+ @Override
+ public void valueArraySerialize(DataOutput2 out, Object vals) throws IOException {
+ delegate.valueArraySerialize(out,vals);
+ }
+
+ @Override
+ public Object valueArrayDeserialize(DataInput2 in, int size) throws IOException {
+ return delegate.valueArrayDeserialize(in,size);
+ }
+
+ @Override
+ public PartitionedEvent[] valueArrayGet(Object vals, int pos) {
+ return deserialize(delegate.valueArrayGet(vals,pos));
+ }
+
+ @Override
+ public int valueArraySize(Object vals) {
+ return delegate.valueArraySize(vals);
+ }
+
+ @Override
+ public Object valueArrayEmpty() {
+ return delegate.valueArrayEmpty();
+ }
+
+ @Override
+ public Object valueArrayPut(Object vals, int pos, PartitionedEvent[] newValue) {
+ return delegate.valueArrayPut(vals,pos,serialize(newValue));
+ }
+
+ @Override
+ public Object valueArrayUpdateVal(Object vals, int pos, PartitionedEvent[] newValue) {
+ return delegate.valueArrayUpdateVal(vals,pos,serialize(newValue));
+ }
+
+ @Override
+ public Object valueArrayFromArray(Object[] objects) {
+ return delegate.valueArrayFromArray(objects);
+ }
+
+ @Override
+ public Object valueArrayCopyOfRange(Object vals, int from, int to) {
+ return delegate.valueArrayCopyOfRange(vals,from,to);
+ }
+
+ @Override
+ public Object valueArrayDeleteValue(Object vals, int pos) {
+ return delegate.valueArrayDeleteValue(vals,pos);
+ }
+
+ @Override
+ public void serialize(@NotNull DataOutput2 out, @NotNull PartitionedEvent[] value) throws IOException {
+ delegate.serialize(out,serialize(value));
+ }
+
+ @Override
+ public PartitionedEvent[] deserialize(@NotNull DataInput2 input, int available) throws IOException {
+ return deserialize(delegate.deserialize(input,available));
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/impl/PartitionedEventTimeOrderingComparator.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/impl/PartitionedEventTimeOrderingComparator.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/impl/PartitionedEventTimeOrderingComparator.java
new file mode 100644
index 0000000..a9f9f39
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/impl/PartitionedEventTimeOrderingComparator.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.sorter.impl;
+
+import java.util.Comparator;
+import java.util.Objects;
+
+import org.apache.eagle.alert.engine.model.PartitionedEvent;
+
+/**
+ * TODO: Stable sorting algorithm for better performance to avoid event resorting with same timestamp?
+ */
+public class PartitionedEventTimeOrderingComparator implements Comparator<PartitionedEvent> {
+ public static final PartitionedEventTimeOrderingComparator INSTANCE = new PartitionedEventTimeOrderingComparator();
+
+ @Override
+ public int compare(PartitionedEvent o1, PartitionedEvent o2) {
+ if(Objects.equals(o1,o2)){
+ return 0;
+ }else {
+ if(o1 == null && o2 == null){
+ return 0;
+ }else if(o1 != null && o2 == null){
+ return 1;
+ }else if(o1 == null){
+ return -1;
+ }
+ // Unstable Sorting Algorithm
+ if(o1.getTimestamp() <= o2.getTimestamp()){
+ return -1;
+ } else {
+ return 1;
+ }
+ }
+ }
+}
\ No newline at end of file
[11/19] incubator-eagle git commit: EAGLE-324: Init branch-v0.5
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/impl/AlertEmailPublisher.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/impl/AlertEmailPublisher.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/impl/AlertEmailPublisher.java
new file mode 100644
index 0000000..efe29bc
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/impl/AlertEmailPublisher.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.alert.engine.publisher.impl;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+import org.apache.eagle.alert.engine.coordinator.Publishment;
+import org.apache.eagle.alert.engine.model.AlertStreamEvent;
+import org.apache.eagle.alert.engine.publisher.AlertDeduplicator;
+import org.apache.eagle.alert.engine.publisher.AlertPublishPlugin;
+import org.apache.eagle.alert.engine.publisher.PublishConstants;
+import org.apache.eagle.alert.engine.publisher.email.AlertEmailGenerator;
+import org.apache.eagle.alert.engine.publisher.email.AlertEmailGeneratorBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.typesafe.config.Config;
+
+public class AlertEmailPublisher implements AlertPublishPlugin {
+
+ private static final Logger LOG = LoggerFactory.getLogger(AlertEmailPublisher.class);
+ private AlertEmailGenerator emailGenerator;
+ private AlertDeduplicator deduplicator;
+ private Map<String, String> emailConfig;
+ private final static int DEFAULT_THREAD_POOL_CORE_SIZE = 4;
+ private final static int DEFAULT_THREAD_POOL_MAX_SIZE = 8;
+ private final static long DEFAULT_THREAD_POOL_SHRINK_TIME = 60000L; // 1 minute
+ private transient ThreadPoolExecutor executorPool;
+ private PublishStatus status;
+
+ @Override
+ public void init(Config config, Publishment publishment) throws Exception {
+ executorPool = new ThreadPoolExecutor(DEFAULT_THREAD_POOL_CORE_SIZE, DEFAULT_THREAD_POOL_MAX_SIZE, DEFAULT_THREAD_POOL_SHRINK_TIME, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<Runnable>());
+ LOG.info(" Creating Email Generator... ");
+ if (publishment.getProperties() != null) {
+ emailConfig = new HashMap<>(publishment.getProperties());
+ emailGenerator = createEmailGenerator(emailConfig);
+ }
+ deduplicator = new DefaultDeduplicator(publishment.getDedupIntervalMin());
+ }
+
+ @Override
+ public void onAlert(AlertStreamEvent event) throws Exception {
+ if(emailGenerator == null) {
+ LOG.warn("emailGenerator is null due to the incorrect configurations");
+ return;
+ }
+ event = dedup(event);
+ if(event == null) {
+ return;
+ }
+ boolean isSuccess = emailGenerator.sendAlertEmail(event);
+ PublishStatus status = new PublishStatus();
+ if(!isSuccess) {
+ status.errorMessage = "Failed to send email";
+ status.successful = false;
+ } else {
+ status.errorMessage = "";
+ status.successful = true;
+ }
+ this.status = status;
+ }
+
+ @Override
+ public void update(String dedupIntervalMin, Map<String, String> pluginProperties) {
+ deduplicator.setDedupIntervalMin(dedupIntervalMin);
+ if (pluginProperties != null && ! emailConfig.equals(pluginProperties)) {
+ emailConfig = new HashMap<>(pluginProperties);
+ emailGenerator = createEmailGenerator(pluginProperties);
+ }
+ }
+
+ @Override
+ public void close() {
+ this.executorPool.shutdown();
+ }
+
+ @Override
+ public PublishStatus getStatus() {
+ return this.status;
+ }
+
+ @Override
+ public AlertStreamEvent dedup(AlertStreamEvent event) {
+ return deduplicator.dedup(event);
+ }
+
+ /**
+ * @param notificationConfig
+ * @return
+ */
+ private AlertEmailGenerator createEmailGenerator(Map<String, String> notificationConfig) {
+ String tplFileName = notificationConfig.get(PublishConstants.TEMPLATE);
+ if (tplFileName == null || tplFileName.equals("")) {
+ tplFileName = "ALERT_DEFAULT.vm";
+ }
+ String subject = notificationConfig.get(PublishConstants.SUBJECT);
+ if (subject == null) {
+ subject = "No subject";
+ }
+ String sender = notificationConfig.get(PublishConstants.SENDER);
+ String recipients = notificationConfig.get(PublishConstants.RECIPIENTS);
+ if(sender == null || recipients == null) {
+ LOG.warn("email sender or recipients is null");
+ return null;
+ }
+ AlertEmailGenerator gen = AlertEmailGeneratorBuilder.newBuilder().
+ withMailProps(notificationConfig).
+ withSubject(subject).
+ withSender(sender).
+ withRecipients(recipients).
+ withTplFile(tplFileName).
+ withExecutorPool(this.executorPool).build();
+ return gen;
+ }
+
+ @Override
+ public int hashCode(){
+ return new HashCodeBuilder().append(getClass().getCanonicalName()).toHashCode();
+ }
+
+ @Override
+ public boolean equals(Object o){
+ if(o == this)
+ return true;
+ if(!(o instanceof AlertEmailPublisher))
+ return false;
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/impl/AlertKafkaPublisher.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/impl/AlertKafkaPublisher.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/impl/AlertKafkaPublisher.java
new file mode 100644
index 0000000..ea65298
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/impl/AlertKafkaPublisher.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.alert.engine.publisher.impl;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.eagle.alert.engine.coordinator.Publishment;
+import org.apache.eagle.alert.engine.model.AlertStreamEvent;
+import org.apache.eagle.alert.engine.publisher.AlertDeduplicator;
+import org.apache.eagle.alert.engine.publisher.AlertPublishPlugin;
+import org.apache.eagle.alert.engine.publisher.PublishConstants;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.typesafe.config.Config;
+
+public class AlertKafkaPublisher implements AlertPublishPlugin {
+
+ private static final Logger LOG = LoggerFactory.getLogger(AlertKafkaPublisher.class);
+ private AlertDeduplicator deduplicator;
+ private PublishStatus status;
+ @SuppressWarnings("rawtypes")
+ private KafkaProducer producer;
+ private String brokerList;
+ private String topic;
+
+ private final static long MAX_TIMEOUT_MS =60000;
+
+ @Override
+ public void init(Config config, Publishment publishment) throws Exception {
+ deduplicator = new DefaultDeduplicator(publishment.getDedupIntervalMin());
+ if (publishment.getProperties() != null) {
+ Map<String, String> kafkaConfig = new HashMap<>(publishment.getProperties());
+ brokerList = kafkaConfig.get(PublishConstants.BROKER_LIST).trim();
+ producer = KafkaProducerManager.INSTANCE.getProducer(brokerList);
+ topic = kafkaConfig.get(PublishConstants.TOPIC).trim();
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public void onAlert(AlertStreamEvent event) throws Exception {
+ if (producer == null) {
+ LOG.warn("KafkaProducer is null due to the incorrect configurations");
+ return;
+ }
+ event = dedup(event);
+ if(event == null) {
+ return;
+ }
+ PublishStatus status = new PublishStatus();
+ try {
+ Future<?> future = producer.send(createRecord(event, topic));
+ future.get(MAX_TIMEOUT_MS, TimeUnit.MILLISECONDS);
+ status.successful = true;
+ status.errorMessage = "";
+ LOG.info("Successfully send message to Kafka: " + brokerList);
+ } catch (InterruptedException | ExecutionException e) {
+ status.successful = false;
+ status.errorMessage = String.format("Failed to send message to %s, due to:%s", brokerList, e);
+ LOG.error(status.errorMessage, e);
+ } catch (Exception ex ) {
+ LOG.error("fail writing alert to Kafka bus", ex);
+ status.successful = false;
+ status.errorMessage = ex.getMessage();
+ }
+ this.status = status;
+ }
+
+ @Override
+ public void update(String dedupIntervalMin, Map<String, String> pluginProperties) {
+ deduplicator.setDedupIntervalMin(dedupIntervalMin);
+ String newBrokerList = pluginProperties.get(PublishConstants.BROKER_LIST).trim();
+ String newTopic = pluginProperties.get(PublishConstants.TOPIC).trim();
+ if (!newBrokerList.equals(this.brokerList)) {
+ producer.close();
+ brokerList = newBrokerList;
+ KafkaProducer newProducer = null;
+ try {
+ newProducer = KafkaProducerManager.INSTANCE.getProducer(brokerList);
+ } catch (Exception e) {
+ LOG.error("Create KafkaProducer failed with configurations: {}", pluginProperties);
+ }
+ producer = newProducer;
+ }
+ topic = newTopic;
+ }
+
+ @Override
+ public void close() {
+ producer.close();
+ }
+
+ /**
+ * To Create KafkaProducer Record
+ * @param event
+ * @return ProducerRecord
+ * @throws Exception
+ */
+ private ProducerRecord<String, String> createRecord(AlertStreamEvent event, String topic) throws Exception {
+ ProducerRecord<String, String> record = new ProducerRecord<>(topic, event.toString());
+ return record;
+ }
+
+ @Override
+ public PublishStatus getStatus() {
+ return this.status;
+ }
+
+ @Override
+ public AlertStreamEvent dedup(AlertStreamEvent event) {
+ return this.deduplicator.dedup(event);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/impl/AlertPublishPluginsFactory.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/impl/AlertPublishPluginsFactory.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/impl/AlertPublishPluginsFactory.java
new file mode 100644
index 0000000..f538088
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/impl/AlertPublishPluginsFactory.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.alert.engine.publisher.impl;
+
+import org.apache.eagle.alert.engine.coordinator.Publishment;
+import org.apache.eagle.alert.engine.publisher.AlertPublishPlugin;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.typesafe.config.Config;
+
+/**
+ * @Since on 5/11/16.
+ */
+public class AlertPublishPluginsFactory {
+
+ private static final Logger LOG = LoggerFactory.getLogger(AlertPublishPluginsFactory.class);
+
+ public static AlertPublishPlugin createNotificationPlugin(Publishment publishment, Config config) {
+ AlertPublishPlugin plugin = null;
+ String publisherType = publishment.getType();
+ try {
+ plugin = (AlertPublishPlugin) Class.forName(publisherType).newInstance();
+ plugin.init(config, publishment);
+ } catch (Exception ex) {
+ LOG.error("Error in loading AlertPublisherPlugin class: ", ex);
+ //throw new IllegalStateException(ex);
+ }
+ return plugin;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/impl/AlertPublisherImpl.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/impl/AlertPublisherImpl.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/impl/AlertPublisherImpl.java
new file mode 100644
index 0000000..fce22f1
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/impl/AlertPublisherImpl.java
@@ -0,0 +1,154 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eagle.alert.engine.publisher.impl;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.commons.collections.ListUtils;
+import org.apache.eagle.alert.engine.coordinator.Publishment;
+import org.apache.eagle.alert.engine.model.AlertStreamEvent;
+import org.apache.eagle.alert.engine.publisher.AlertPublishPlugin;
+import org.apache.eagle.alert.engine.publisher.AlertPublisher;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.typesafe.config.Config;
+
+public class AlertPublisherImpl implements AlertPublisher {
+ private static final long serialVersionUID = 4809983246198138865L;
+ private final static Logger LOG = LoggerFactory.getLogger(AlertPublisherImpl.class);
+ private final String name;
+
+ private volatile Map<String, List<String>> policyPublishPluginMapping = new ConcurrentHashMap<>(1);
+ private volatile Map<String, AlertPublishPlugin> publishPluginMapping = new ConcurrentHashMap<>(1);
+ private Config config;
+
+ public AlertPublisherImpl(String name) {
+ this.name = name;
+ }
+
+ @Override
+ public void init(Config config) {
+ this.config = config;
+ }
+
+ @Override
+ public String getName() {
+ return name;
+ }
+
+ @Override
+ public void nextEvent(AlertStreamEvent event) {
+ if(LOG.isDebugEnabled())
+ LOG.debug(event.toString());
+ notifyAlert(event);
+ }
+
+ private void notifyAlert(AlertStreamEvent event) {
+ String policyId = event.getPolicyId();
+ if(policyId == null || !policyPublishPluginMapping.containsKey(policyId)) {
+ LOG.warn("Policy {} does NOT subscribe any publishments", policyId);
+ return;
+ }
+ for(String id: policyPublishPluginMapping.get(policyId)) {
+ AlertPublishPlugin plugin = publishPluginMapping.get(id);
+ try {
+ if(LOG.isDebugEnabled()) LOG.debug("Execute alert publisher " + plugin.getClass().getCanonicalName());
+ plugin.onAlert(event);
+ } catch (Exception ex) {
+ LOG.error("Fail invoking publisher's onAlert, continue ", ex);
+ }
+ }
+ }
+
+ @Override
+ public void close() {
+ publishPluginMapping.values().forEach(plugin -> plugin.close());
+ }
+
+ @Override
+ public void onPublishChange(List<Publishment> added,
+ List<Publishment> removed,
+ List<Publishment> afterModified,
+ List<Publishment> beforeModified) {
+ if (added == null) added = new ArrayList<>();
+ if (removed == null) removed = new ArrayList<>();
+ if (afterModified == null) afterModified = new ArrayList<>();
+ if (beforeModified == null) beforeModified = new ArrayList<>();
+
+ if (afterModified.size() != beforeModified.size()) {
+ LOG.warn("beforeModified size != afterModified size");
+ return;
+ }
+
+ for (Publishment publishment : added) {
+ AlertPublishPlugin plugin = AlertPublishPluginsFactory.createNotificationPlugin(publishment, config);
+ if(plugin != null) {
+ publishPluginMapping.put(publishment.getName(), plugin);
+ onPolicyAdded(publishment.getPolicyIds(), publishment.getName());
+ } else {
+ LOG.error("Initialized alertPublisher {} failed due to invalid format", publishment);
+ }
+ }
+ for (Publishment publishment : removed) {
+ String pubName = publishment.getName();
+ onPolicyDeleted(publishment.getPolicyIds(), pubName);
+ publishPluginMapping.get(pubName).close();
+ publishPluginMapping.remove(publishment.getName());
+ }
+ for (int i = 0; i < afterModified.size(); i++) {
+ String pubName = afterModified.get(i).getName();
+ List<String> newPolicies = afterModified.get(i).getPolicyIds();
+ List<String> oldPolicies = beforeModified.get(i).getPolicyIds();
+
+ if (! newPolicies.equals(oldPolicies)) {
+ List<String> deletedPolicies = ListUtils.subtract(oldPolicies, newPolicies);
+ onPolicyDeleted(deletedPolicies, pubName);
+ List<String> addedPolicies = ListUtils.subtract(newPolicies, oldPolicies);
+ onPolicyAdded(addedPolicies, pubName);
+ }
+ Publishment newPub = afterModified.get(i);
+ publishPluginMapping.get(pubName).update(newPub.getDedupIntervalMin(), newPub.getProperties());
+ }
+ }
+
+ private synchronized void onPolicyAdded(List<String> addedPolicyIds, String pubName) {
+ if (addedPolicyIds == null || pubName == null) return;
+
+ for (String policyId : addedPolicyIds) {
+ if (policyPublishPluginMapping.get(policyId) == null) {
+ policyPublishPluginMapping.put(policyId, new ArrayList<>());
+ }
+ List<String> publishIds = policyPublishPluginMapping.get(policyId);
+ publishIds.add(pubName);
+ }
+ }
+
+ private synchronized void onPolicyDeleted(List<String> deletedPolicyIds, String pubName) {
+ if (deletedPolicyIds == null || pubName == null) return;
+
+ for (String policyId : deletedPolicyIds) {
+ List<String> publishIds = policyPublishPluginMapping.get(policyId);
+ publishIds.remove(pubName);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/impl/DefaultDeduplicator.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/impl/DefaultDeduplicator.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/impl/DefaultDeduplicator.java
new file mode 100644
index 0000000..054d679
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/impl/DefaultDeduplicator.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.eagle.alert.engine.publisher.impl;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.commons.lang.time.DateUtils;
+import org.apache.eagle.alert.engine.model.AlertStreamEvent;
+import org.apache.eagle.alert.engine.publisher.AlertDeduplicator;
+import org.joda.time.Period;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class DefaultDeduplicator implements AlertDeduplicator {
+ private long dedupIntervalMin;
+ private volatile Map<EventUniq, Long> events = new HashMap<>();
+ private static Logger LOG = LoggerFactory.getLogger(DefaultDeduplicator.class);
+
+ public enum AlertDeduplicationStatus{
+ NEW,
+ DUPLICATED,
+ IGNORED
+ }
+
+ public DefaultDeduplicator() {
+ this.dedupIntervalMin = 0;
+ }
+
+ public DefaultDeduplicator(String intervalMin) {
+ setDedupIntervalMin(intervalMin);
+ }
+
+ public DefaultDeduplicator(long intervalMin) {
+ this.dedupIntervalMin = intervalMin;
+ }
+
+ public void clearOldCache() {
+ List<EventUniq> removedkeys = new ArrayList<>();
+ for (Entry<EventUniq, Long> entry : events.entrySet()) {
+ EventUniq entity = entry.getKey();
+ if (System.currentTimeMillis() - 7 * DateUtils.MILLIS_PER_DAY > entity.createdTime) {
+ removedkeys.add(entry.getKey());
+ }
+ }
+ for (EventUniq alertKey : removedkeys) {
+ events.remove(alertKey);
+ }
+ }
+
+ public AlertDeduplicationStatus checkDedup(EventUniq key) {
+ long current = key.timestamp;
+ if(!events.containsKey(key)) {
+ events.put(key, current);
+ return AlertDeduplicationStatus.NEW;
+ }
+
+ long last = events.get(key);
+ if(current - last >= dedupIntervalMin * DateUtils.MILLIS_PER_MINUTE) {
+ events.put(key, current);
+ return AlertDeduplicationStatus.IGNORED;
+ }
+
+ return AlertDeduplicationStatus.DUPLICATED;
+ }
+
+ public AlertStreamEvent dedup(AlertStreamEvent event) {
+ if (event == null) return null;
+ clearOldCache();
+ AlertStreamEvent result = null;
+ AlertDeduplicationStatus status = checkDedup(new EventUniq(event.getStreamId(), event.getPolicyId(), event.getCreatedTime()));
+ if (!status.equals(AlertDeduplicationStatus.DUPLICATED)) {
+ result = event;
+ } else if(LOG.isDebugEnabled()){
+ LOG.debug("Alert event is skipped because it's duplicated: {}", event.toString());
+ }
+ return result;
+ }
+
+ @Override
+ public void setDedupIntervalMin(String newDedupIntervalMin) {
+ if (newDedupIntervalMin == null || newDedupIntervalMin.isEmpty()) {
+ dedupIntervalMin = 0;
+ return;
+ }
+ try {
+ Period period = Period.parse(newDedupIntervalMin);
+ this.dedupIntervalMin = period.toStandardMinutes().getMinutes();
+ } catch (Exception e) {
+ LOG.warn("Fail to pares deDupIntervalMin, will disable deduplication instead", e);
+ this.dedupIntervalMin = 0;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/impl/EventUniq.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/impl/EventUniq.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/impl/EventUniq.java
new file mode 100644
index 0000000..df472d0
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/impl/EventUniq.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+/**
+ *
+ */
+package org.apache.eagle.alert.engine.publisher.impl;
+
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+
+/**
+ * @since Mar 19, 2015
+ */
+public class EventUniq {
+ public String streamId;
+ public String policyId;
+ public Long timestamp; // event's createTimestamp
+ public long createdTime; // created time, for cache removal;
+
+ public EventUniq(String streamId, String policyId, long timestamp) {
+ this.streamId = streamId;
+ this.timestamp = timestamp;
+ this.policyId = policyId;
+ this.createdTime = System.currentTimeMillis();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj instanceof EventUniq) {
+ EventUniq au = (EventUniq) obj;
+ return (this.streamId.equalsIgnoreCase(au.streamId) & this.policyId.equalsIgnoreCase(au.policyId));
+ }
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return new HashCodeBuilder()
+ .append(streamId)
+ .append(policyId)
+ .build();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/impl/KafkaProducerManager.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/impl/KafkaProducerManager.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/impl/KafkaProducerManager.java
new file mode 100644
index 0000000..e8964a8
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/impl/KafkaProducerManager.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.eagle.alert.engine.publisher.impl;
+
+import java.util.Properties;
+
+import org.apache.kafka.clients.producer.KafkaProducer;
+
+/**
+ * The producer is thread safe and sharing a single producer instance across threads will generally be faster than having multiple instances.
+ */
+public class KafkaProducerManager {
+ public static final KafkaProducerManager INSTANCE = new KafkaProducerManager();
+
+ public KafkaProducer<String, Object> getProducer(String brokerList) {
+ Properties configMap = new Properties();
+ configMap.put("bootstrap.servers", brokerList);
+ configMap.put("metadata.broker.list", brokerList);
+ configMap.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
+ configMap.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
+ configMap.put("request.required.acks", "1");
+ configMap.put("key.deserializer","org.apache.kafka.common.serialization.StringDeserializer");
+ configMap.put("value.deserializer","org.apache.kafka.common.serialization.StringDeserializer");
+ KafkaProducer<String, Object> producer = new KafkaProducer<>(configMap);
+ return producer;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/impl/PublishStatus.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/impl/PublishStatus.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/impl/PublishStatus.java
new file mode 100644
index 0000000..c165686
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/publisher/impl/PublishStatus.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.alert.engine.publisher.impl;
+
+/**
+ * Object that holds the status of Notification Posted to Notification Plugin
+ */
+public class PublishStatus {
+ public boolean successful;
+ public String errorMessage;
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/router/AlertBoltSpecListener.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/router/AlertBoltSpecListener.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/router/AlertBoltSpecListener.java
new file mode 100644
index 0000000..e1f3e9c
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/router/AlertBoltSpecListener.java
@@ -0,0 +1,32 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.alert.engine.router;
+
+import java.util.Map;
+
+import org.apache.eagle.alert.coordination.model.AlertBoltSpec;
+import org.apache.eagle.alert.engine.coordinator.StreamDefinition;
+
+/**
+ * Since 5/2/16.
+ */
+public interface AlertBoltSpecListener {
+ void onAlertBoltSpecChange(AlertBoltSpec spec, Map<String, StreamDefinition> sds);
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/router/SpoutSpecListener.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/router/SpoutSpecListener.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/router/SpoutSpecListener.java
new file mode 100644
index 0000000..aa40dc5
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/router/SpoutSpecListener.java
@@ -0,0 +1,32 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.alert.engine.router;
+
+import java.util.Map;
+
+import org.apache.eagle.alert.coordination.model.SpoutSpec;
+import org.apache.eagle.alert.engine.coordinator.StreamDefinition;
+
+/**
+ * Since 5/3/16.
+ */
+public interface SpoutSpecListener {
+ void onSpoutSpecChange(SpoutSpec spec, Map<String, StreamDefinition> sds);
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/router/StreamRoute.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/router/StreamRoute.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/router/StreamRoute.java
new file mode 100644
index 0000000..07518d9
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/router/StreamRoute.java
@@ -0,0 +1,85 @@
+package org.apache.eagle.alert.engine.router;
+
+import java.io.Serializable;
+
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+import org.apache.eagle.alert.engine.coordinator.StreamPartition;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * <b></b>
+ * 1. Group by SingleStream[stream_1.col1]
+ *
+ * Shuffle(stream_1,[col1])
+ *
+ * <b></b>
+ * 2. Group by SingleStream[stream_1.col1,stream_1.col2]
+ *
+ * Shuffle(stream_1,[col1,col2])
+ *
+ * <b></b>
+ * 3. Group by JoinedStream[stream_1.col1,stream_1.col2,stream_2.col3]
+ *
+ * Shuffle(stream_1.col1,stream_1.col2) + Global(stream_2.col3)
+ */
+public class StreamRoute implements Serializable{
+ private static final long serialVersionUID = 4649184902196034940L;
+
+ private String targetComponentId;
+ private int partitionKey;
+ private String partitionType;
+
+ public String getTargetComponentId() {
+ return targetComponentId;
+ }
+
+ public void setTargetComponentId(String targetComponentId) {
+ this.targetComponentId = targetComponentId;
+ }
+
+ public StreamRoute(String targetComponentId, int partitionKey, StreamPartition.Type type){
+ this.setTargetComponentId(targetComponentId);
+ this.setPartitionKey(partitionKey);
+ this.setPartitionType(type);
+ }
+
+ public int getPartitionKey() {
+ return partitionKey;
+ }
+
+ public void setPartitionKey(int partitionKey) {
+ this.partitionKey = partitionKey;
+ }
+
+ public StreamPartition.Type getPartitionType() {
+ return StreamPartition.Type.valueOf(partitionType);
+ }
+
+ public void setPartitionType(StreamPartition.Type partitionType) {
+ this.partitionType = partitionType.name();
+ }
+
+ @Override
+ public int hashCode() {
+ return new HashCodeBuilder().append(partitionKey).append(partitionType).append(targetComponentId).build();
+ }
+
+ @Override
+ public String toString() {
+ return String.format("Route[target=%s, key=%s, type=%s]", this.targetComponentId, this.partitionKey, this.partitionType);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/router/StreamRoutePartitionFactory.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/router/StreamRoutePartitionFactory.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/router/StreamRoutePartitionFactory.java
new file mode 100644
index 0000000..6dcd312
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/router/StreamRoutePartitionFactory.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.eagle.alert.engine.router;
+
+import java.util.List;
+
+import org.apache.eagle.alert.engine.coordinator.StreamDefinition;
+import org.apache.eagle.alert.engine.coordinator.StreamPartition;
+import org.apache.eagle.alert.engine.router.impl.BasicStreamRoutePartitioner;
+
+public class StreamRoutePartitionFactory {
+ /**
+ * TODO: Decouple different StreamRoutePartitioner implementation from BasicStreamRoutePartitioner
+ *
+ * @param outputComponentIds
+ * @param streamDefinition
+ * @param partition
+ * @return
+ */
+ public static StreamRoutePartitioner createRoutePartitioner(List<String> outputComponentIds, StreamDefinition streamDefinition, StreamPartition partition) {
+ return new BasicStreamRoutePartitioner(outputComponentIds, streamDefinition, partition);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/router/StreamRoutePartitioner.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/router/StreamRoutePartitioner.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/router/StreamRoutePartitioner.java
new file mode 100644
index 0000000..213abef
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/router/StreamRoutePartitioner.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.router;
+
+import java.util.List;
+
+import org.apache.eagle.alert.engine.model.StreamEvent;
+
+public interface StreamRoutePartitioner {
+ /**
+ * @param event
+ * @return
+ */
+ List<StreamRoute> partition(StreamEvent event);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/router/StreamRouteSpecListener.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/router/StreamRouteSpecListener.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/router/StreamRouteSpecListener.java
new file mode 100644
index 0000000..abc465d
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/router/StreamRouteSpecListener.java
@@ -0,0 +1,30 @@
+package org.apache.eagle.alert.engine.router;
+
+import java.util.Collection;
+import java.util.Map;
+
+import org.apache.eagle.alert.coordination.model.StreamRouterSpec;
+import org.apache.eagle.alert.engine.coordinator.StreamDefinition;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+public interface StreamRouteSpecListener {
+ void onStreamRouterSpecChange(Collection<StreamRouterSpec> added,
+ Collection<StreamRouterSpec> removed,
+ Collection<StreamRouterSpec> modified,
+ Map<String, StreamDefinition> sds);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/router/StreamRouter.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/router/StreamRouter.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/router/StreamRouter.java
new file mode 100644
index 0000000..da7fb7e
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/router/StreamRouter.java
@@ -0,0 +1,30 @@
+package org.apache.eagle.alert.engine.router;
+
+import java.io.Serializable;
+
+import org.apache.eagle.alert.engine.PartitionedEventCollector;
+import org.apache.eagle.alert.engine.StreamContext;
+import org.apache.eagle.alert.engine.model.PartitionedEvent;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+public interface StreamRouter extends StreamSortSpecListener, Serializable {
+ void prepare(StreamContext context, PartitionedEventCollector outputCollector);
+ void nextEvent(PartitionedEvent event);
+ String getName();
+ void close();
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/router/StreamRouterBoltSpecListener.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/router/StreamRouterBoltSpecListener.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/router/StreamRouterBoltSpecListener.java
new file mode 100644
index 0000000..cd4bfd3
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/router/StreamRouterBoltSpecListener.java
@@ -0,0 +1,33 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.alert.engine.router;
+
+import java.util.Map;
+
+import org.apache.eagle.alert.coordination.model.RouterSpec;
+import org.apache.eagle.alert.engine.coordinator.StreamDefinition;
+
+/**
+ * Since 5/1/16.
+ * Listen to change on StreamRouterBoltSpec
+ */
+public interface StreamRouterBoltSpecListener {
+ void onStreamRouteBoltSpecChange(RouterSpec spec, Map<String, StreamDefinition> sds);
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/router/StreamSortHandler.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/router/StreamSortHandler.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/router/StreamSortHandler.java
new file mode 100644
index 0000000..2229099
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/router/StreamSortHandler.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.router;
+
+import org.apache.eagle.alert.engine.PartitionedEventCollector;
+import org.apache.eagle.alert.engine.coordinator.StreamSortSpec;
+import org.apache.eagle.alert.engine.model.PartitionedEvent;
+import org.apache.eagle.alert.engine.sorter.StreamTimeClockListener;
+
+public interface StreamSortHandler extends StreamTimeClockListener {
+ /**
+ *
+ * @param streamId
+ * @param streamSortSpecSpec
+ * @param outputCollector
+ */
+ void prepare(String streamId, StreamSortSpec streamSortSpecSpec, PartitionedEventCollector outputCollector);
+
+ /**
+ * @param event StreamEvent
+ */
+ void nextEvent(PartitionedEvent event);
+
+ /**
+ *
+ */
+ void close();
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/router/StreamSortSpecListener.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/router/StreamSortSpecListener.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/router/StreamSortSpecListener.java
new file mode 100644
index 0000000..affa979
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/router/StreamSortSpecListener.java
@@ -0,0 +1,28 @@
+package org.apache.eagle.alert.engine.router;
+
+import java.util.Map;
+
+import org.apache.eagle.alert.engine.coordinator.StreamPartition;
+import org.apache.eagle.alert.engine.coordinator.StreamSortSpec;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+public interface StreamSortSpecListener {
+ void onStreamSortSpecChange(Map<StreamPartition, StreamSortSpec> added,
+ Map<StreamPartition, StreamSortSpec> removed,
+ Map<StreamPartition, StreamSortSpec> changed);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/router/impl/BasicStreamRoutePartitioner.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/router/impl/BasicStreamRoutePartitioner.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/router/impl/BasicStreamRoutePartitioner.java
new file mode 100644
index 0000000..afb9a6f
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/router/impl/BasicStreamRoutePartitioner.java
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.router.impl;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+import org.apache.eagle.alert.engine.coordinator.StreamDefinition;
+import org.apache.eagle.alert.engine.coordinator.StreamPartition;
+import org.apache.eagle.alert.engine.model.StreamEvent;
+import org.apache.eagle.alert.engine.router.StreamRoute;
+import org.apache.eagle.alert.engine.router.StreamRoutePartitioner;
+
+public class BasicStreamRoutePartitioner implements StreamRoutePartitioner {
+ private final List<String> outputComponentIds;
+ private final StreamDefinition streamDefinition;
+ private final StreamPartition streamPartition;
+
+ public BasicStreamRoutePartitioner(List<String> outputComponentIds, StreamDefinition streamDefinition, StreamPartition partition) {
+ this.outputComponentIds = outputComponentIds;
+ this.streamDefinition = streamDefinition;
+ this.streamPartition = partition;
+ }
+
+ @Override
+ public List<StreamRoute> partition(StreamEvent event) {
+ switch (this.streamPartition.getType()){
+ case GLOBAL:
+ return routeToAll(event);
+ case GROUPBY:
+ return routeByGroupByKey(event);
+ default:
+ return routeByShuffle(event);
+ }
+ }
+
+ protected List<StreamRoute> routeByGroupByKey(StreamEvent event) {
+ int partitionKey = new HashCodeBuilder().append(event.getData(streamDefinition,this.streamPartition.getColumns())).build();
+ String selectedOutputStream = outputComponentIds.get(Math.abs(partitionKey) % this.outputComponentIds.size());
+ return Collections.singletonList(new StreamRoute(selectedOutputStream, partitionKey, StreamPartition.Type.GROUPBY));
+ }
+
+ protected List<StreamRoute> routeByShuffle(StreamEvent event) {
+ long random = System.currentTimeMillis();
+ int hash = Math.abs((int)random);
+ return Arrays.asList(new StreamRoute(outputComponentIds.get(hash % outputComponentIds.size()),-1,StreamPartition.Type.SHUFFLE));
+ }
+
+ protected List<StreamRoute> routeToAll(StreamEvent event) {
+ if(_globalRoutingKeys!=null) {
+ _globalRoutingKeys = new ArrayList<>();
+ for (String targetId : outputComponentIds) {
+ _globalRoutingKeys.add(new StreamRoute(targetId, -1, StreamPartition.Type.GLOBAL));
+ }
+ }
+ return _globalRoutingKeys;
+ }
+
+ private List<StreamRoute> _globalRoutingKeys = null;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/router/impl/RoutePhysicalGrouping.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/router/impl/RoutePhysicalGrouping.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/router/impl/RoutePhysicalGrouping.java
new file mode 100644
index 0000000..d0bf012
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/router/impl/RoutePhysicalGrouping.java
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.router.impl;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.eagle.alert.engine.coordinator.StreamPartition;
+import org.apache.eagle.alert.engine.model.PartitionedEvent;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import storm.trident.partition.GlobalGrouping;
+import backtype.storm.generated.GlobalStreamId;
+import backtype.storm.grouping.CustomStreamGrouping;
+import backtype.storm.task.WorkerTopologyContext;
+
+public class RoutePhysicalGrouping implements CustomStreamGrouping {
+ private static final long serialVersionUID = -511915083994148362L;
+ private final static Logger LOG = LoggerFactory.getLogger(RoutePhysicalGrouping.class);
+ private List<Integer> outdegreeTasks;
+ private ShuffleGrouping shuffleGroupingDelegate;
+ private GlobalGrouping globalGroupingDelegate;
+ private Map<String,Integer> connectedTargetIds;
+
+ @Override
+ public void prepare(WorkerTopologyContext context, GlobalStreamId stream, List<Integer> targetTasks) {
+ this.outdegreeTasks = new ArrayList<>(targetTasks);
+ shuffleGroupingDelegate = new ShuffleGrouping();
+ shuffleGroupingDelegate.prepare(context,stream,targetTasks);
+ globalGroupingDelegate = new GlobalGrouping();
+ globalGroupingDelegate.prepare(context,stream,targetTasks);
+ connectedTargetIds = new HashMap<>();
+ for(Integer targetId:targetTasks){
+ String targetComponentId = context.getComponentId(targetId);
+ connectedTargetIds.put(targetComponentId,targetId);
+ }
+ LOG.info("OutDegree components: [{}]", StringUtils.join(connectedTargetIds.values(),","));
+ }
+
+ @Override
+ public List<Integer> chooseTasks(int taskId, List<Object> values) {
+ Object routingKeyObj = values.get(0);
+ if(routingKeyObj!=null){
+ PartitionedEvent partitionedEvent = (PartitionedEvent) routingKeyObj;
+ if (partitionedEvent.getPartition().getType() == StreamPartition.Type.GLOBAL) {
+ return globalGroupingDelegate.chooseTasks(taskId,values);
+ } else if (partitionedEvent.getPartition().getType() == StreamPartition.Type.GROUPBY) {
+ return Collections.singletonList(outdegreeTasks.get((int)(partitionedEvent.getPartitionKey() % this.outdegreeTasks.size())));
+ }
+ // Shuffle by defaults
+ return shuffleGroupingDelegate.chooseTasks(taskId, values);
+ }
+
+ LOG.warn("Illegal null StreamRoute, throw event");
+ return Collections.emptyList();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/router/impl/ShuffleGrouping.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/router/impl/ShuffleGrouping.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/router/impl/ShuffleGrouping.java
new file mode 100644
index 0000000..c73854a
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/router/impl/ShuffleGrouping.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.router.impl;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import backtype.storm.generated.GlobalStreamId;
+import backtype.storm.grouping.CustomStreamGrouping;
+import backtype.storm.task.WorkerTopologyContext;
+
+/**
+ * NOTE: This is copy from storm 1.0.0 code. DON'T modify it.
+ *
+ * @since May 4, 2016
+ *
+ */
+public class ShuffleGrouping implements CustomStreamGrouping, Serializable {
+ private static final long serialVersionUID = 5035497345182141765L;
+ private Random random;
+ private ArrayList<List<Integer>> choices;
+ private AtomicInteger current;
+
+ @Override
+ public void prepare(WorkerTopologyContext context, GlobalStreamId stream, List<Integer> targetTasks) {
+ random = new Random();
+ choices = new ArrayList<List<Integer>>(targetTasks.size());
+ for (Integer i: targetTasks) {
+ choices.add(Arrays.asList(i));
+ }
+ Collections.shuffle(choices, random);
+ current = new AtomicInteger(0);
+ }
+
+ @Override
+ public List<Integer> chooseTasks(int taskId, List<Object> values) {
+ int rightNow;
+ int size = choices.size();
+ while (true) {
+ rightNow = current.incrementAndGet();
+ if (rightNow < size) {
+ return choices.get(rightNow);
+ } else if (rightNow == size) {
+ current.set(0);
+ //This should be thread safe so long as ArrayList does not have any internal state that can be messed up by multi-treaded access.
+ Collections.shuffle(choices, random);
+ return choices.get(0);
+ }
+ //race condition with another thread, and we lost
+ // try again
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/router/impl/StreamRouterBoltOutputCollector.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/router/impl/StreamRouterBoltOutputCollector.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/router/impl/StreamRouterBoltOutputCollector.java
new file mode 100644
index 0000000..db235a7
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/router/impl/StreamRouterBoltOutputCollector.java
@@ -0,0 +1,229 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+package org.apache.eagle.alert.engine.router.impl;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.eagle.alert.coordination.model.PolicyWorkerQueue;
+import org.apache.eagle.alert.coordination.model.StreamRouterSpec;
+import org.apache.eagle.alert.coordination.model.WorkSlot;
+import org.apache.eagle.alert.engine.PartitionedEventCollector;
+import org.apache.eagle.alert.engine.StreamContext;
+import org.apache.eagle.alert.engine.coordinator.StreamDefinition;
+import org.apache.eagle.alert.engine.coordinator.StreamPartition;
+import org.apache.eagle.alert.engine.model.PartitionedEvent;
+import org.apache.eagle.alert.engine.model.StreamEvent;
+import org.apache.eagle.alert.engine.router.StreamRoute;
+import org.apache.eagle.alert.engine.router.StreamRoutePartitionFactory;
+import org.apache.eagle.alert.engine.router.StreamRoutePartitioner;
+import org.apache.eagle.alert.engine.router.StreamRouteSpecListener;
+import org.apache.eagle.alert.engine.serialization.PartitionedEventSerializer;
+import org.apache.eagle.alert.utils.StreamIdConversion;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import backtype.storm.task.OutputCollector;
+
+import com.google.common.collect.Lists;
+
+/**
+ * After sorting, one stream's message will be routed based on its StreamPartition
+ * One stream may have multiple StreamPartitions based on how this stream is grouped by
+ *
+ * TODO: Add metric statistics
+ */
+public class StreamRouterBoltOutputCollector implements PartitionedEventCollector, StreamRouteSpecListener {
+ private final static Logger LOG = LoggerFactory.getLogger(StreamRouterBoltOutputCollector.class);
+ private final OutputCollector outputCollector;
+ private final Object outputLock = new Object();
+ private final List<String> outputStreamIds;
+ private final StreamContext streamContext;
+ private final PartitionedEventSerializer serializer;
+ private volatile Map<StreamPartition,StreamRouterSpec> routeSpecMap;
+ private volatile Map<StreamPartition, List<StreamRoutePartitioner>> routePartitionerMap;
+ private final String sourceId;
+
+ public StreamRouterBoltOutputCollector(String sourceId, OutputCollector outputCollector, List<String> outputStreamIds, StreamContext streamContext, PartitionedEventSerializer serializer){
+ this.sourceId = sourceId;
+ this.outputCollector = outputCollector;
+ this.routeSpecMap = new HashMap<>();
+ this.routePartitionerMap = new HashMap<>();
+ this.outputStreamIds = outputStreamIds;
+ this.streamContext = streamContext;
+ this.serializer = serializer;
+ }
+
+ public void emit(PartitionedEvent event) {
+ try {
+ this.streamContext.counter().scope("send_count").incr();
+ StreamPartition partition = event.getPartition();
+ StreamRouterSpec routerSpec = routeSpecMap.get(partition);
+ if (routerSpec == null) {
+ if (LOG.isDebugEnabled()) {
+ // Don't know how to route stream, if it's correct, it's better to filter useless stream in spout side
+ LOG.debug("Drop event {} as StreamPartition {} is not pointed to any router metadata {}", event, event.getPartition(), routeSpecMap);
+ }
+ this.drop(event);
+ return;
+ }
+
+ if (routePartitionerMap.get(routerSpec.getPartition()) == null) {
+ LOG.error("Partitioner for " + routerSpec + " is null");
+ synchronized (outputLock) {
+ this.streamContext.counter().scope("fail_count").incr();
+ this.outputCollector.fail(event.getAnchor());
+ }
+ return;
+ }
+
+ StreamEvent newEvent = event.getEvent().copy();
+
+ // Get handler for the partition
+ List<StreamRoutePartitioner> queuePartitioners = routePartitionerMap.get(partition);
+
+ synchronized (outputLock) {
+ for (StreamRoutePartitioner queuePartitioner : queuePartitioners) {
+ List<StreamRoute> streamRoutes = queuePartitioner.partition(newEvent);
+ // it is possible that one event can be sent to multiple slots in one slotqueue if that is All grouping
+ for (StreamRoute streamRoute : streamRoutes) {
+ String targetStreamId = StreamIdConversion.generateStreamIdBetween(sourceId, streamRoute.getTargetComponentId());
+ try {
+ PartitionedEvent emittedEvent = new PartitionedEvent(newEvent, routerSpec.getPartition(), streamRoute.getPartitionKey());
+ // Route Target Stream id instead of component id
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Emitted to stream {} with message {}", targetStreamId, emittedEvent);
+ }
+ if(this.serializer == null) {
+ outputCollector.emit(targetStreamId, event.getAnchor(), Collections.singletonList(emittedEvent));
+ } else {
+ outputCollector.emit(targetStreamId, event.getAnchor(), Collections.singletonList(serializer.serialize(emittedEvent)));
+ }
+ this.streamContext.counter().scope("emit_count").incr();
+ } catch (RuntimeException ex) {
+ this.streamContext.counter().scope("fail_count").incr();
+ LOG.error("Failed to emit to {} with {}", targetStreamId, newEvent, ex);
+ throw ex;
+ }
+ }
+ }
+ outputCollector.ack(event.getAnchor());
+ }
+ } catch (Exception ex){
+ LOG.error(ex.getMessage(),ex);
+ synchronized (outputLock) {
+ this.streamContext.counter().scope("fail_count").incr();
+ this.outputCollector.fail(event.getAnchor());
+ }
+ }
+ }
+
+ @Override
+ public void onStreamRouterSpecChange(Collection<StreamRouterSpec> added,
+ Collection<StreamRouterSpec> removed,
+ Collection<StreamRouterSpec> modified,
+ Map<String, StreamDefinition> sds){
+ Map<StreamPartition,StreamRouterSpec> copyRouteSpecMap = new HashMap<>(routeSpecMap);
+ Map<StreamPartition, List<StreamRoutePartitioner>> copyRoutePartitionerMap = new HashMap<>(routePartitionerMap);
+
+ // added StreamRouterSpec i.e. there is a new StreamPartition
+ for(StreamRouterSpec spec : added){
+ if(copyRouteSpecMap.containsKey(spec.getPartition())){
+ LOG.error("Metadata calculation error: add existing StreamRouterSpec " + spec);
+ }else{
+ inplaceAdd(copyRouteSpecMap, copyRoutePartitionerMap, spec, sds);
+ }
+ }
+
+ // removed StreamRouterSpec i.e. there is a deleted StreamPartition
+ for(StreamRouterSpec spec : removed){
+ if(!copyRouteSpecMap.containsKey(spec.getPartition())){
+ LOG.error("Metadata calculation error: remove non-existing StreamRouterSpec " + spec);
+ }else{
+ inplaceRemove(copyRouteSpecMap, copyRoutePartitionerMap, spec);
+ }
+ }
+
+ // modified StreamRouterSpec, i.e. there is modified StreamPartition, for example WorkSlotQueue assignment is changed
+ for(StreamRouterSpec spec : modified){
+ if(!copyRouteSpecMap.containsKey(spec.getPartition())){
+ LOG.error("Metadata calculation error: modify nonexisting StreamRouterSpec " + spec);
+ }else{
+ inplaceRemove(copyRouteSpecMap, copyRoutePartitionerMap, spec);
+ inplaceAdd(copyRouteSpecMap, copyRoutePartitionerMap, spec, sds);
+ }
+ }
+
+ // switch
+ routeSpecMap = copyRouteSpecMap;
+ routePartitionerMap = copyRoutePartitionerMap;
+ }
+
+ private void inplaceRemove(Map<StreamPartition,StreamRouterSpec> routeSpecMap,
+ Map<StreamPartition, List<StreamRoutePartitioner>> routePartitionerMap,
+ StreamRouterSpec toBeRemoved){
+ routeSpecMap.remove(toBeRemoved.getPartition());
+ routePartitionerMap.remove(toBeRemoved.getPartition());
+ }
+
+ private void inplaceAdd(Map<StreamPartition,StreamRouterSpec> routeSpecMap,
+ Map<StreamPartition, List<StreamRoutePartitioner>> routePartitionerMap,
+ StreamRouterSpec toBeAdded, Map<String, StreamDefinition> sds){
+ routeSpecMap.put(toBeAdded.getPartition(), toBeAdded);
+ try {
+ List<StreamRoutePartitioner> routePartitioners = calculatePartitioner(toBeAdded, sds);
+ routePartitionerMap.put(toBeAdded.getPartition(), routePartitioners);
+ }catch(Exception e){
+ LOG.error("ignore this failure StreamRouterSpec " + toBeAdded + ", with error" + e.getMessage(),e);
+ routeSpecMap.remove(toBeAdded.getPartition());
+ routePartitionerMap.remove(toBeAdded.getPartition());
+ }
+ }
+
+ private List<StreamRoutePartitioner> calculatePartitioner(StreamRouterSpec streamRouterSpec, Map<String, StreamDefinition> sds) throws Exception{
+ List<StreamRoutePartitioner> routePartitioners = new ArrayList<>();
+ for (PolicyWorkerQueue pwq : streamRouterSpec.getTargetQueue()) {
+ routePartitioners.add(StreamRoutePartitionFactory.createRoutePartitioner(
+ Lists.transform(pwq.getWorkers(), WorkSlot::getBoltId),
+ sds.get(streamRouterSpec.getPartition().getStreamId()),
+ streamRouterSpec.getPartition()));
+ }
+ return routePartitioners;
+ }
+
+ @Override
+ public void drop(PartitionedEvent event) {
+ synchronized (outputLock) {
+ this.streamContext.counter().scope("drop_count").incr();
+ if (event.getAnchor() != null) {
+ this.outputCollector.ack(event.getAnchor());
+ }else{
+ throw new IllegalStateException(event.toString()+" was not acked as anchor is null");
+ }
+ }
+ }
+
+ public void flush() {
+
+ }
+}
\ No newline at end of file
[13/19] incubator-eagle git commit: EAGLE-324: Init branch-v0.5
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-devtools/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-devtools/pom.xml b/eagle-core/eagle-alert/alert/alert-devtools/pom.xml
new file mode 100644
index 0000000..365445c
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-devtools/pom.xml
@@ -0,0 +1,96 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.eagle</groupId>
+ <artifactId>alert-parent</artifactId>
+ <version>0.0.1-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>alert-devtools</artifactId>
+
+ <properties>
+ <maven-scala.version>2.15.0</maven-scala.version>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.eagle</groupId>
+ <artifactId>alert-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.storm</groupId>
+ <artifactId>storm-kafka</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.storm</groupId>
+ <artifactId>storm-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>${kafka.artifact.id}</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <sourceDirectory>src/main/java</sourceDirectory>
+
+ <plugins>
+ <plugin>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>${maven-compiler.version}</version>
+ </plugin>
+ <!-- <plugin>
+ <groupId>org.scala-tools</groupId>
+ <artifactId>maven-scala-plugin</artifactId>
+ <version>${maven-scala.version}</version>
+ <executions>
+ <execution>
+ <goals>
+ <goal>compile</goal>
+ <goal>testCompile</goal>
+ </goals>
+ <configuration>
+ <args>
+ <arg>-make:transitive</arg>
+ <arg>-dependencyfile</arg>
+ <arg>${project.build.directory}/.scala_dependencies</arg>
+ </args>
+ <sourceDir>src/main/scala</sourceDir>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin> -->
+
+ <plugin>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <phase>install</phase>
+ <goals>
+ <goal>copy-dependencies</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>${project.build.directory}/lib</outputDirectory>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-devtools/src/main/java/org/apache/eagle/alert/tools/KafkaConsumerOffset.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-devtools/src/main/java/org/apache/eagle/alert/tools/KafkaConsumerOffset.java b/eagle-core/eagle-alert/alert/alert-devtools/src/main/java/org/apache/eagle/alert/tools/KafkaConsumerOffset.java
new file mode 100644
index 0000000..8f98d71
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-devtools/src/main/java/org/apache/eagle/alert/tools/KafkaConsumerOffset.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.tools;
+
+import java.util.Map;
+
+public class KafkaConsumerOffset {
+ public Map<String, String> topology;
+ public Long offset;
+ public Long partition;
+ public Map<String, String> broker;
+ public String topic;
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-devtools/src/main/java/org/apache/eagle/alert/tools/KafkaConsumerOffsetFetcher.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-devtools/src/main/java/org/apache/eagle/alert/tools/KafkaConsumerOffsetFetcher.java b/eagle-core/eagle-alert/alert/alert-devtools/src/main/java/org/apache/eagle/alert/tools/KafkaConsumerOffsetFetcher.java
new file mode 100644
index 0000000..4fe471c
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-devtools/src/main/java/org/apache/eagle/alert/tools/KafkaConsumerOffsetFetcher.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.tools;
+
+import com.fasterxml.jackson.databind.Module;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.jsontype.NamedType;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.retry.RetryNTimes;
+import org.apache.eagle.alert.config.ZKConfig;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class KafkaConsumerOffsetFetcher {
+ public CuratorFramework curator;
+ public String zkRoot;
+ public ObjectMapper mapper;
+ public String zkPathToPartition;
+
+ public KafkaConsumerOffsetFetcher(ZKConfig config, String ... parameters) {
+ try {
+ this.curator = CuratorFrameworkFactory.newClient(config.zkQuorum, config.zkSessionTimeoutMs, 15000,
+ new RetryNTimes(config.zkRetryTimes, config.zkRetryInterval));
+ curator.start();
+ this.zkRoot = config.zkRoot;
+ mapper = new ObjectMapper();
+ Module module = new SimpleModule("offset").registerSubtypes(new NamedType(KafkaConsumerOffset.class));
+ mapper.registerModule(module);
+ zkPathToPartition = String.format(config.zkRoot, parameters);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public Map<String, Long> fetch() throws Exception {
+ Map<String, Long> map = new HashMap<String, Long>();
+ if (curator.checkExists().forPath(zkPathToPartition) != null) {
+ List<String> partitions = curator.getChildren().forPath(zkPathToPartition);
+ for (String partition : partitions) {
+ String partitionPath = zkPathToPartition + "/" + partition;
+ String data = new String(curator.getData().forPath(partitionPath));
+ KafkaConsumerOffset offset = mapper.readValue(data, KafkaConsumerOffset.class);
+ map.put(partition, offset.offset);
+ }
+ }
+ return map;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-devtools/src/main/java/org/apache/eagle/alert/tools/KafkaLatestOffsetFetcher.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-devtools/src/main/java/org/apache/eagle/alert/tools/KafkaLatestOffsetFetcher.java b/eagle-core/eagle-alert/alert/alert-devtools/src/main/java/org/apache/eagle/alert/tools/KafkaLatestOffsetFetcher.java
new file mode 100644
index 0000000..b685e5c
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-devtools/src/main/java/org/apache/eagle/alert/tools/KafkaLatestOffsetFetcher.java
@@ -0,0 +1,97 @@
+package org.apache.eagle.alert.tools;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.*;
+import kafka.api.PartitionOffsetRequestInfo;
+import kafka.common.TopicAndPartition;
+import kafka.javaapi.*;
+import kafka.javaapi.consumer.SimpleConsumer;
+
+public class KafkaLatestOffsetFetcher {
+
+ private List<String> brokerList;
+ private int port;
+
+ public KafkaLatestOffsetFetcher(String brokerList) {
+ this.brokerList = new ArrayList<>();
+ String[] brokers = brokerList.split(",");
+ for (String broker : brokers) {
+ this.brokerList.add(broker.split(":")[0]);
+ }
+ this.port = Integer.valueOf(brokers[0].split(":")[1]);
+ }
+
+ public Map<Integer, Long> fetch(String topic, int partitionCount) {
+ Map<Integer, PartitionMetadata> metadatas = fetchPartitionMetadata(brokerList, port, topic, partitionCount);
+ Map<Integer, Long> ret = new HashMap<>();
+ for (int partition = 0; partition < partitionCount; partition++) {
+ PartitionMetadata metadata = metadatas.get(partition);
+ if (metadata == null || metadata.leader() == null) {
+ ret.put(partition, -1L);
+ //throw new RuntimeException("Can't find Leader for Topic and Partition. Exiting");
+ }
+ String leadBroker = metadata.leader().host();
+ String clientName = "Client_" + topic + "_" + partition;
+ SimpleConsumer consumer = new SimpleConsumer(leadBroker, port, 100000, 64 * 1024, clientName);
+ long latestOffset = getLatestOffset(consumer, topic, partition, clientName);
+ if (consumer != null) consumer.close();
+ ret.put(partition, latestOffset);
+ }
+ return ret;
+ }
+
+ public long getLatestOffset(SimpleConsumer consumer, String topic, int partition, String clientName) {
+ TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
+ Map<TopicAndPartition, kafka.api.PartitionOffsetRequestInfo> requestInfo = new HashMap<>();
+ requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.LatestTime(), 1));
+ kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(requestInfo, kafka.api.OffsetRequest.CurrentVersion(), clientName);
+ OffsetResponse response = consumer.getOffsetsBefore(request);
+ if (response.hasError()) {
+ throw new RuntimeException("Error fetching data offset from the broker. Reason: " + response.errorCode(topic, partition) );
+ }
+ long[] offsets = response.offsets(topic, partition);
+ return offsets[0];
+ }
+
+ private Map<Integer, PartitionMetadata> fetchPartitionMetadata(List<String> brokerList, int port, String topic, int partitionCount) {
+ Map<Integer, PartitionMetadata> partitionMetadata = new HashMap<>();
+ for (String broker : brokerList) {
+ SimpleConsumer consumer = null;
+ try {
+ consumer = new SimpleConsumer(broker, port, 100000, 64 * 1024, "leaderLookup");
+ List<String> topics = Collections.singletonList(topic);
+ TopicMetadataRequest req = new TopicMetadataRequest(topics);
+ kafka.javaapi.TopicMetadataResponse resp = consumer.send(req);
+
+ List<TopicMetadata> metaData = resp.topicsMetadata();
+ for (TopicMetadata item : metaData) {
+ for (PartitionMetadata part : item.partitionsMetadata()) {
+ partitionMetadata.put(part.partitionId(), part);
+ }
+ }
+ if (partitionMetadata.size() == partitionCount) break;
+ } catch (Exception e) {
+ throw new RuntimeException("Error communicating with Broker [" + broker + "] " + "to find Leader for [" + topic + "] Reason: ", e);
+ } finally {
+ if (consumer != null) consumer.close();
+ }
+ }
+ return partitionMetadata;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-devtools/src/main/scala/org/apache/eagle/contrib/kafka/ProducerTool.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-devtools/src/main/scala/org/apache/eagle/contrib/kafka/ProducerTool.scala b/eagle-core/eagle-alert/alert/alert-devtools/src/main/scala/org/apache/eagle/contrib/kafka/ProducerTool.scala
new file mode 100644
index 0000000..830e9ac
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-devtools/src/main/scala/org/apache/eagle/contrib/kafka/ProducerTool.scala
@@ -0,0 +1,230 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.contrib.kafka
+
+import java.io._
+import java.nio.charset.StandardCharsets
+import java.util.Properties
+import java.util.concurrent.Executors
+
+import joptsimple._
+import kafka.message._
+import kafka.producer.ConsoleProducer.{LineMessageReader, MessageReader}
+import kafka.producer.{KeyedMessage, Producer, ProducerConfig}
+import kafka.serializer._
+import org.apache.commons.io.FileUtils
+
+import scala.collection.JavaConversions._
+
+object ProducerTool {
+ def main(args: Array[String]) {
+ val parser = new OptionParser
+ val topicOpt = parser.accepts("topic", "REQUIRED: The topic id to produce messages to.")
+ .withRequiredArg
+ .describedAs("topic")
+ .ofType(classOf[String])
+ val brokerListOpt = parser.accepts("broker-list", "REQUIRED: The broker list string in the form HOST1:PORT1,HOST2:PORT2.")
+ .withRequiredArg
+ .describedAs("broker-list")
+ .ofType(classOf[String])
+ val syncOpt = parser.accepts("sync", "If set message send requests to the brokers are synchronously, one at a time as they arrive.")
+ val compressOpt = parser.accepts("compress", "If set, messages batches are sent compressed")
+ val batchSizeOpt = parser.accepts("batch-size", "Number of messages to send in a single batch if they are not being sent synchronously.")
+ .withRequiredArg
+ .describedAs("size")
+ .ofType(classOf[java.lang.Integer])
+ .defaultsTo(200)
+ val messageSendMaxRetriesOpt = parser.accepts("message-send-max-retries", "Brokers can fail receiving the message for multiple reasons, and being unavailable transiently is just one of them. This property specifies the number of retires before the producer give up and drop this message.")
+ .withRequiredArg
+ .ofType(classOf[java.lang.Integer])
+ .defaultsTo(3)
+ val retryBackoffMsOpt = parser.accepts("retry-backoff-ms", "Before each retry, the producer refreshes the metadata of relevant topics. Since leader election takes a bit of time, this property specifies the amount of time that the producer waits before refreshing the metadata.")
+ .withRequiredArg
+ .ofType(classOf[java.lang.Long])
+ .defaultsTo(100l)
+ val sendTimeoutOpt = parser.accepts("timeout", "If set and the producer is running in asynchronous mode, this gives the maximum amount of time" +
+ " a message will queue awaiting suffient batch size. The value is given in ms.")
+ .withRequiredArg
+ .describedAs("timeout_ms")
+ .ofType(classOf[java.lang.Long])
+ .defaultsTo(1000l)
+ val queueSizeOpt = parser.accepts("queue-size", "If set and the producer is running in asynchronous mode, this gives the maximum amount of " +
+ " messages will queue awaiting suffient batch size.")
+ .withRequiredArg
+ .describedAs("queue_size")
+ .ofType(classOf[java.lang.Long])
+ .defaultsTo(10000l)
+ val queueEnqueueTimeoutMsOpt = parser.accepts("queue-enqueuetimeout-ms", "Timeout for event enqueue")
+ .withRequiredArg
+ .describedAs("queue enqueuetimeout ms")
+ .ofType(classOf[java.lang.Long])
+ .defaultsTo(Int.MaxValue.toLong)
+ val requestRequiredAcksOpt = parser.accepts("request-required-acks", "The required acks of the producer requests")
+ .withRequiredArg
+ .describedAs("request required acks")
+ .ofType(classOf[java.lang.Integer])
+ .defaultsTo(0)
+ val requestTimeoutMsOpt = parser.accepts("request-timeout-ms", "The ack timeout of the producer requests. Value must be non-negative and non-zero")
+ .withRequiredArg
+ .describedAs("request timeout ms")
+ .ofType(classOf[java.lang.Integer])
+ .defaultsTo(1500)
+ val valueEncoderOpt = parser.accepts("value-serializer", "The class name of the message encoder implementation to use for serializing values.")
+ .withRequiredArg
+ .describedAs("encoder_class")
+ .ofType(classOf[java.lang.String])
+ .defaultsTo(classOf[StringEncoder].getName)
+ val keyEncoderOpt = parser.accepts("key-serializer", "The class name of the message encoder implementation to use for serializing keys.")
+ .withRequiredArg
+ .describedAs("encoder_class")
+ .ofType(classOf[java.lang.String])
+ .defaultsTo(classOf[StringEncoder].getName)
+ val messageReaderOpt = parser.accepts("line-reader", "The class name of the class to use for reading lines from standard in. " +
+ "By default each line is read as a separate message.")
+ .withRequiredArg
+ .describedAs("reader_class")
+ .ofType(classOf[java.lang.String])
+ .defaultsTo(classOf[LineMessageReader].getName)
+ val socketBufferSizeOpt = parser.accepts("socket-buffer-size", "The size of the tcp RECV size.")
+ .withRequiredArg
+ .describedAs("size")
+ .ofType(classOf[java.lang.Integer])
+ .defaultsTo(1024 * 100)
+ val propertyOpt = parser.accepts("property", "A mechanism to pass user-defined properties in the form key=value to the message reader. " +
+ "This allows custom configuration for a user-defined message reader.")
+ .withRequiredArg
+ .describedAs("prop")
+ .ofType(classOf[String])
+
+ val dataOpt = parser.accepts("data", "Input message data content")
+ .withRequiredArg()
+ .describedAs("input message data")
+ .ofType(classOf[String])
+
+ val fileOpt = parser.accepts("file", "Input message file name")
+ .withRequiredArg()
+ .describedAs("input message file")
+ .ofType(classOf[String])
+
+ val replicationOpt = parser.accepts("replication", "Message data replication count")
+ .withRequiredArg()
+ .describedAs("message replication count, default: 1")
+ .ofType(classOf[java.lang.Integer])
+ .defaultsTo(1)
+
+ val options = parser.parse(args: _*)
+ for (arg <- List(topicOpt, brokerListOpt)) {
+ if (!options.has(arg)) {
+ System.err.println("Missing required argument \"" + arg + "\"")
+ parser.printHelpOn(System.err)
+ System.exit(1)
+ }
+ }
+
+ val topic = options.valueOf(topicOpt)
+ val brokerList = options.valueOf(brokerListOpt)
+ val sync = options.has(syncOpt)
+ val compress = options.has(compressOpt)
+ val batchSize = options.valueOf(batchSizeOpt)
+ val sendTimeout = options.valueOf(sendTimeoutOpt)
+ val queueSize = options.valueOf(queueSizeOpt)
+ val queueEnqueueTimeoutMs = options.valueOf(queueEnqueueTimeoutMsOpt)
+ val requestRequiredAcks = options.valueOf(requestRequiredAcksOpt)
+ val requestTimeoutMs = options.valueOf(requestTimeoutMsOpt)
+ val keyEncoderClass = options.valueOf(keyEncoderOpt)
+ val valueEncoderClass = options.valueOf(valueEncoderOpt)
+ val readerClass = options.valueOf(messageReaderOpt)
+ val socketBuffer = options.valueOf(socketBufferSizeOpt)
+ val cmdLineProps = parseLineReaderArgs(options.valuesOf(propertyOpt))
+ val messageData = options.valuesOf(dataOpt)
+ val messageFile = options.valuesOf(fileOpt)
+ val messageReplication = options.valueOf(replicationOpt)
+
+ cmdLineProps.put("topic", topic)
+ val props = new Properties()
+ props.put("metadata.broker.list", brokerList)
+ val codec = if (compress) DefaultCompressionCodec.codec else NoCompressionCodec.codec
+ props.put("compression.codec", codec.toString)
+ props.put("producer.type", if (sync) "sync" else "async")
+ if (options.has(batchSizeOpt))
+ props.put("batch.num.messages", batchSize.toString)
+
+ props.put("message.send.max.retries", options.valueOf(messageSendMaxRetriesOpt).toString)
+ props.put("retry.backoff.ms", options.valueOf(retryBackoffMsOpt).toString)
+ props.put("queue.buffering.max.ms", sendTimeout.toString)
+ props.put("queue.buffering.max.messages", queueSize.toString)
+ props.put("queue.enqueue.timeout.ms", queueEnqueueTimeoutMs.toString)
+ props.put("request.required.acks", requestRequiredAcks.toString)
+ props.put("request.timeout.ms", requestTimeoutMs.toString)
+ props.put("key.serializer.class", keyEncoderClass)
+ props.put("serializer.class", valueEncoderClass)
+ props.put("send.buffer.bytes", socketBuffer.toString)
+
+ val reader = Class.forName(readerClass).newInstance().asInstanceOf[MessageReader[AnyRef, AnyRef]]
+
+ if (messageData.size()>0) {
+ reader.init(new ByteArrayInputStream(messageData.get(0).getBytes(StandardCharsets.UTF_8)), cmdLineProps)
+ } else if (messageFile.size()>0) {
+ reader.init(FileUtils.openInputStream(new File(messageFile.get(0))), cmdLineProps)
+ } else {
+ reader.init(System.in, cmdLineProps)
+ }
+
+ try {
+ val executor = Executors.newCachedThreadPool()
+ val producer = new Producer[AnyRef, AnyRef](new ProducerConfig(props))
+ Runtime.getRuntime.addShutdownHook(new Thread() {
+ override def run(): Unit = {
+ producer.close()
+ executor.shutdown()
+ }
+ })
+
+ var message: KeyedMessage[AnyRef, AnyRef] = null
+ do {
+ message = reader.readMessage()
+ if (message != null) {
+ var i = 0
+ while (i < messageReplication) {
+ executor.submit(new Runnable {
+ override def run(): Unit = producer.send(message)
+ }).get()
+ i += 1
+ }
+ System.out.println("Produced %d messages".format(messageReplication))
+ }
+ } while (message != null)
+ } catch {
+ case e: Exception =>
+ e.printStackTrace()
+ System.exit(1)
+ }
+ System.exit(0)
+ }
+
+ def parseLineReaderArgs(args: Iterable[String]): Properties = {
+ val splits = args.map(_ split "=").filterNot(_ == null).filterNot(_.length == 0)
+ if (!splits.forall(_.length == 2)) {
+ System.err.println("Invalid line reader properties: " + args.mkString(" "))
+ System.exit(1)
+ }
+ val props = new Properties
+ for (a <- splits)
+ props.put(a(0), a(1))
+ props
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-devtools/src/test/java/org/apache/eagle/alert/tools/TestKafkaOffset.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-devtools/src/test/java/org/apache/eagle/alert/tools/TestKafkaOffset.java b/eagle-core/eagle-alert/alert/alert-devtools/src/test/java/org/apache/eagle/alert/tools/TestKafkaOffset.java
new file mode 100644
index 0000000..e4d9f8f
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-devtools/src/test/java/org/apache/eagle/alert/tools/TestKafkaOffset.java
@@ -0,0 +1,69 @@
+package org.apache.eagle.alert.tools;
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+import org.apache.eagle.alert.config.ZKConfig;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import java.util.Map;
+
+public class TestKafkaOffset {
+ @Ignore
+ @Test
+ public void test() throws Exception {
+ System.setProperty("config.resource", "/kafka-offset-test.application.conf");
+ Config config = ConfigFactory.load();
+ ZKConfig zkConfig = new ZKConfig();
+ zkConfig.zkQuorum = config.getString("dataSourceConfig.zkQuorum");
+ zkConfig.zkRoot = config.getString("dataSourceConfig.transactionZKRoot");
+ zkConfig.zkSessionTimeoutMs = config.getInt("dataSourceConfig.zkSessionTimeoutMs");
+ zkConfig.connectionTimeoutMs = config.getInt("dataSourceConfig.connectionTimeoutMs");
+ zkConfig.zkRetryTimes = config.getInt("dataSourceConfig.zkRetryTimes");
+ zkConfig.zkRetryInterval = config.getInt("dataSourceConfig.zkRetryInterval");
+
+ String topic = "testTopic1";
+ String topology = "alertUnitTopology_1";
+
+ while(true) {
+ KafkaConsumerOffsetFetcher consumerOffsetFetcher = new KafkaConsumerOffsetFetcher(zkConfig, topic, topology);
+ String kafkaBrokerList = config.getString("dataSourceConfig.kafkaBrokerList");
+ KafkaLatestOffsetFetcher latestOffsetFetcher = new KafkaLatestOffsetFetcher(kafkaBrokerList);
+
+ Map<String, Long> consumedOffset = consumerOffsetFetcher.fetch();
+ if(consumedOffset.size() == 0){
+ System.out.println("no any consumer offset found for this topic " + topic);
+ }
+ Map<Integer, Long> latestOffset = latestOffsetFetcher.fetch(topic, consumedOffset.size());
+ if(latestOffset.size() == 0){
+ System.out.println("no any latest offset found for this topic " + topic);
+ }
+ for (Map.Entry<String, Long> entry : consumedOffset.entrySet()) {
+ String partition = entry.getKey();
+ Integer partitionNumber = Integer.valueOf(partition.split("_")[1]);
+ Long lag = latestOffset.get(partitionNumber) - entry.getValue();
+ System.out.println(String.format("parition %s, total: %d, consumed: %d, lag: %d",
+ partition, latestOffset.get(partitionNumber), entry.getValue(), lag));
+ }
+ Thread.sleep(10000);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-devtools/src/test/resources/kafka-offset-test.application.conf
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-devtools/src/test/resources/kafka-offset-test.application.conf b/eagle-core/eagle-alert/alert/alert-devtools/src/test/resources/kafka-offset-test.application.conf
new file mode 100644
index 0000000..ca4d68a
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-devtools/src/test/resources/kafka-offset-test.application.conf
@@ -0,0 +1,9 @@
+"dataSourceConfig":{
+ "kafkaBrokerList" : "kafka-broker:9092",
+ "zkQuorum" : "zk-admin:2181",
+ "transactionZKRoot" : "/consumers/eagle_consumer/%s/%s",
+ "zkSessionTimeoutMs" : 10000,
+ "connectionTimeoutMs" : 10000,
+ "zkRetryTimes" : 3,
+ "zkRetryInterval" : 3000
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-devtools/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-devtools/src/test/resources/log4j.properties b/eagle-core/eagle-alert/alert/alert-devtools/src/test/resources/log4j.properties
new file mode 100644
index 0000000..d59ded6
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-devtools/src/test/resources/log4j.properties
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+log4j.rootLogger=INFO, stdout
+
+# standard output
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %p [%t] %c{2}[%L]: %m%n
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/.gitignore
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/.gitignore b/eagle-core/eagle-alert/alert/alert-engine/.gitignore
new file mode 100644
index 0000000..b83d222
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/.gitignore
@@ -0,0 +1 @@
+/target/
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/pom.xml b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/pom.xml
new file mode 100644
index 0000000..9f4d39f
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/pom.xml
@@ -0,0 +1,65 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.eagle</groupId>
+ <artifactId>alert-engine-parent</artifactId>
+ <version>0.0.1-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>alert-engine-base</artifactId>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-jvm</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.mapdb</groupId>
+ <artifactId>mapdb</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>joda-time</groupId>
+ <artifactId>joda-time</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.velocity</groupId>
+ <artifactId>velocity</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>javax.mail</groupId>
+ <artifactId>mail</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka-clients</artifactId>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <version>2.6</version>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/AlertStreamCollector.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/AlertStreamCollector.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/AlertStreamCollector.java
new file mode 100755
index 0000000..7219271
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/AlertStreamCollector.java
@@ -0,0 +1,27 @@
+package org.apache.eagle.alert.engine;
+
+import org.apache.eagle.alert.engine.model.AlertStreamEvent;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+public interface AlertStreamCollector extends Collector<AlertStreamEvent> {
+ /**
+ * No need to be thread-safe, but should be called on in synchronous like in Storm bolt execute method
+ */
+ void flush();
+ void close();
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/Collector.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/Collector.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/Collector.java
new file mode 100755
index 0000000..9a6489c
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/Collector.java
@@ -0,0 +1,27 @@
+package org.apache.eagle.alert.engine;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+@FunctionalInterface
+public interface Collector<T> {
+ /**
+ * Must make sure thread-safe
+ *
+ * @param t
+ */
+ void emit(T t);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/PartitionedEventCollector.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/PartitionedEventCollector.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/PartitionedEventCollector.java
new file mode 100755
index 0000000..3a99e98
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/PartitionedEventCollector.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine;
+
+import org.apache.eagle.alert.engine.model.PartitionedEvent;
+
+/**
+ * Executed in thread-safe trace
+ */
+public interface PartitionedEventCollector extends Collector<PartitionedEvent> {
+ /**
+ * @param event to be dropped
+ */
+ void drop(PartitionedEvent event);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/StreamContext.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/StreamContext.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/StreamContext.java
new file mode 100644
index 0000000..609378b
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/StreamContext.java
@@ -0,0 +1,27 @@
+package org.apache.eagle.alert.engine;
+
+import backtype.storm.metric.api.MultiCountMetric;
+
+import com.typesafe.config.Config;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+public interface StreamContext {
+ MultiCountMetric counter();
+
+ Config config();
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/StreamContextImpl.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/StreamContextImpl.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/StreamContextImpl.java
new file mode 100644
index 0000000..f339cd8
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/StreamContextImpl.java
@@ -0,0 +1,26 @@
+package org.apache.eagle.alert.engine;
+
+import backtype.storm.metric.api.MultiCountMetric;
+import backtype.storm.task.TopologyContext;
+
+import com.typesafe.config.Config;
+
+public class StreamContextImpl implements StreamContext {
+ private final Config config;
+ private final MultiCountMetric counter;
+
+ public StreamContextImpl(Config config, MultiCountMetric counter, TopologyContext context) {
+ this.counter=counter;
+ this.config = config;
+ }
+
+ @Override
+ public MultiCountMetric counter() {
+ return this.counter;
+ }
+
+ @Override
+ public Config config() {
+ return this.config;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/UnitTopologyMain.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/UnitTopologyMain.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/UnitTopologyMain.java
new file mode 100644
index 0000000..2bca329
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/UnitTopologyMain.java
@@ -0,0 +1,59 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.alert.engine;
+
+import org.apache.eagle.alert.config.ZKConfig;
+import org.apache.eagle.alert.config.ZKConfigBuilder;
+import org.apache.eagle.alert.engine.coordinator.impl.ZKMetadataChangeNotifyService;
+import org.apache.eagle.alert.engine.runner.UnitTopologyRunner;
+
+import backtype.storm.generated.StormTopology;
+
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+
+/**
+ * Since 5/3/16. Make sure unit topology can be started either from command line
+ * or from remote A few parameters for starting unit topology 1. number of spout
+ * tasks 2. number of router bolts 3. number of alert bolts 4. number of publish
+ * bolts
+ *
+ * Connections 1. spout and router bolt 2. router bolt and alert bolt 3. alert
+ * bolt and publish bolt
+ */
+public class UnitTopologyMain {
+
+ public static void main(String[] args) {
+ Config config = ConfigFactory.load();
+ ZKConfig zkConfig = ZKConfigBuilder.getZKConfig(config);
+ String topologyId = config.getString("topology.name");
+ ZKMetadataChangeNotifyService changeNotifyService = new ZKMetadataChangeNotifyService(zkConfig, topologyId);
+
+ new UnitTopologyRunner(changeNotifyService).run(topologyId, config);
+ }
+
+ public static StormTopology createTopology(Config config) {
+ ZKConfig zkConfig = ZKConfigBuilder.getZKConfig(config);
+ String topologyId = config.getString("topology.name");
+ ZKMetadataChangeNotifyService changeNotifyService = new ZKMetadataChangeNotifyService(zkConfig, topologyId);
+
+ return new UnitTopologyRunner(changeNotifyService).buildTopology(topologyId, config);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/coordinator/IMetadataChangeNotifyService.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/coordinator/IMetadataChangeNotifyService.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/coordinator/IMetadataChangeNotifyService.java
new file mode 100644
index 0000000..22fe56a
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/coordinator/IMetadataChangeNotifyService.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.coordinator;
+
+import java.io.Closeable;
+import java.io.Serializable;
+
+import org.apache.eagle.alert.engine.publisher.AlertPublishSpecListener;
+import org.apache.eagle.alert.engine.router.AlertBoltSpecListener;
+import org.apache.eagle.alert.engine.router.SpoutSpecListener;
+import org.apache.eagle.alert.engine.router.StreamRouterBoltSpecListener;
+
+import com.typesafe.config.Config;
+
+/**
+ * IMetadataChangeNotifyService defines the following features
+ * 1) initialization
+ * 2) register metadata change listener
+ *
+ * In distributed environment for example storm platform,
+ * subclass implementing this interface should have the following lifecycle
+ * 1. object is created in client machine
+ * 2. object is serialized and transferred to other machine
+ * 3. object is created through deserialization
+ * 4. invoke init() method to do initialization
+ * 5. invoke various registerListener to get notified of config change
+ * 6. invoke close() to release system resource
+ */
+public interface IMetadataChangeNotifyService extends Closeable,Serializable {
+ /**
+ *
+ * @param config
+ */
+ void init(Config config, MetadataType type);
+
+ void registerListener(SpoutSpecListener listener);
+
+ void registerListener(AlertBoltSpecListener listener);
+
+ void registerListener(StreamRouterBoltSpecListener listener);
+
+ void registerListener(AlertPublishSpecListener listener);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/coordinator/MetadataType.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/coordinator/MetadataType.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/coordinator/MetadataType.java
new file mode 100644
index 0000000..ac68e26
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/coordinator/MetadataType.java
@@ -0,0 +1,30 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.alert.engine.coordinator;
+
+/**
+ * Since 5/4/16.
+ */
+public enum MetadataType {
+ SPOUT,
+ STREAM_ROUTER_BOLT,
+ ALERT_BOLT,
+ ALERT_PUBLISH_BOLT
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/coordinator/StreamDefinitionNotFoundException.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/coordinator/StreamDefinitionNotFoundException.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/coordinator/StreamDefinitionNotFoundException.java
new file mode 100644
index 0000000..927eb8c
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/coordinator/StreamDefinitionNotFoundException.java
@@ -0,0 +1,25 @@
+package org.apache.eagle.alert.engine.coordinator;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+public class StreamDefinitionNotFoundException extends Exception {
+ private static final long serialVersionUID = 6027811718016485808L;
+
+ public StreamDefinitionNotFoundException(String streamId){
+ super("Stream definition not found: "+streamId);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/coordinator/impl/AbstractMetadataChangeNotifyService.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/coordinator/impl/AbstractMetadataChangeNotifyService.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/coordinator/impl/AbstractMetadataChangeNotifyService.java
new file mode 100755
index 0000000..52fcc04
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/coordinator/impl/AbstractMetadataChangeNotifyService.java
@@ -0,0 +1,113 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.coordinator.impl;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.eagle.alert.coordination.model.AlertBoltSpec;
+import org.apache.eagle.alert.coordination.model.PublishSpec;
+import org.apache.eagle.alert.coordination.model.RouterSpec;
+import org.apache.eagle.alert.coordination.model.SpoutSpec;
+import org.apache.eagle.alert.engine.coordinator.IMetadataChangeNotifyService;
+import org.apache.eagle.alert.engine.coordinator.MetadataType;
+import org.apache.eagle.alert.engine.coordinator.StreamDefinition;
+import org.apache.eagle.alert.engine.publisher.AlertPublishSpecListener;
+import org.apache.eagle.alert.engine.router.AlertBoltSpecListener;
+import org.apache.eagle.alert.engine.router.SpoutSpecListener;
+import org.apache.eagle.alert.engine.router.StreamRouterBoltSpecListener;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Preconditions;
+import com.typesafe.config.Config;
+
+/**
+ * notify 3 components of metadata change Spout, StreamRouterBolt and AlertBolt
+ */
+@SuppressWarnings({ "serial" })
+public abstract class AbstractMetadataChangeNotifyService implements IMetadataChangeNotifyService, Closeable,
+ Serializable {
+ private final static Logger LOG = LoggerFactory.getLogger(AbstractMetadataChangeNotifyService.class);
+ private final List<StreamRouterBoltSpecListener> streamRouterBoltSpecListeners = new ArrayList<>();
+ private final List<SpoutSpecListener> spoutSpecListeners = new ArrayList<>();
+ private final List<AlertBoltSpecListener> alertBoltSpecListeners = new ArrayList<>();
+ private final List<AlertPublishSpecListener> alertPublishSpecListeners = new ArrayList<>();
+ protected MetadataType type;
+
+ /**
+ * @param config
+ */
+ @Override
+ public void init(Config config, MetadataType type) {
+ this.type = type;
+ }
+
+ @Override
+ public void registerListener(AlertPublishSpecListener listener) {
+ synchronized (alertBoltSpecListeners) {
+ Preconditions.checkNotNull(alertPublishSpecListeners, "Not initialized yet");
+ LOG.info("Register {}", listener);
+ alertPublishSpecListeners.add(listener);
+ }
+ }
+
+ @Override
+ public void registerListener(StreamRouterBoltSpecListener listener) {
+ synchronized (streamRouterBoltSpecListeners) {
+ streamRouterBoltSpecListeners.add(listener);
+ }
+ }
+
+ @Override
+ public void registerListener(AlertBoltSpecListener listener) {
+ synchronized (alertBoltSpecListeners) {
+ alertBoltSpecListeners.add(listener);
+ }
+ }
+
+ @Override
+ public void registerListener(SpoutSpecListener listener) {
+ synchronized (spoutSpecListeners) {
+ spoutSpecListeners.add(listener);
+ }
+ }
+
+ protected void notifySpout(SpoutSpec spoutSpec, Map<String, StreamDefinition> sds) {
+ spoutSpecListeners.forEach(s -> s.onSpoutSpecChange(spoutSpec, sds));
+ }
+
+ protected void notifyStreamRouterBolt(RouterSpec routerSpec, Map<String, StreamDefinition> sds) {
+ streamRouterBoltSpecListeners.forEach(s -> s.onStreamRouteBoltSpecChange(routerSpec, sds));
+ }
+
+ protected void notifyAlertBolt(AlertBoltSpec alertBoltSpec, Map<String, StreamDefinition> sds) {
+ alertBoltSpecListeners.forEach(s -> s.onAlertBoltSpecChange(alertBoltSpec, sds));
+ }
+
+ protected void notifyAlertPublishBolt(PublishSpec alertPublishSpec, Map<String, StreamDefinition> sds) {
+ alertPublishSpecListeners.forEach(s -> s.onAlertPublishSpecChange(alertPublishSpec, sds));
+ }
+
+ public void close() throws IOException {
+ LOG.info("Closed");
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/coordinator/impl/ZKMetadataChangeNotifyService.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/coordinator/impl/ZKMetadataChangeNotifyService.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/coordinator/impl/ZKMetadataChangeNotifyService.java
new file mode 100755
index 0000000..1b93072
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/coordinator/impl/ZKMetadataChangeNotifyService.java
@@ -0,0 +1,200 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.coordinator.impl;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.eagle.alert.config.ConfigBusConsumer;
+import org.apache.eagle.alert.config.ConfigChangeCallback;
+import org.apache.eagle.alert.config.ConfigValue;
+import org.apache.eagle.alert.config.ZKConfig;
+import org.apache.eagle.alert.coordination.model.AlertBoltSpec;
+import org.apache.eagle.alert.coordination.model.PublishSpec;
+import org.apache.eagle.alert.coordination.model.RouterSpec;
+import org.apache.eagle.alert.coordination.model.ScheduleState;
+import org.apache.eagle.alert.coordination.model.SpoutSpec;
+import org.apache.eagle.alert.coordination.model.VersionedPolicyDefinition;
+import org.apache.eagle.alert.coordination.model.VersionedStreamDefinition;
+import org.apache.eagle.alert.engine.coordinator.MetadataType;
+import org.apache.eagle.alert.engine.coordinator.PolicyDefinition;
+import org.apache.eagle.alert.engine.coordinator.StreamDefinition;
+import org.apache.eagle.alert.service.IMetadataServiceClient;
+import org.apache.eagle.alert.service.MetadataServiceClientImpl;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.typesafe.config.Config;
+
+/**
+ * <b>TODO</b>: performance tuning: It is not JVM level service, so it may cause
+ * zookeeper burden in case of too many listeners This does not support
+ * dynamically adding topic, all topics should be available when service object
+ * is created.
+ * <p>
+ * ZK path format is as following:
+ * <ul>
+ * <li>/alert/topology_1/spout</li>
+ * <li>/alert/topology_1/router</li>
+ * <li>/alert/topology_1/alert</li>
+ * <li>/alert/topology_1/publisher</li>
+ * </ul>
+ */
+public class ZKMetadataChangeNotifyService extends AbstractMetadataChangeNotifyService implements ConfigChangeCallback {
+ private static final long serialVersionUID = -1509237694501235144L;
+ private static final Logger LOG = LoggerFactory.getLogger(ZKMetadataChangeNotifyService.class);
+ private ZKConfig zkConfig;
+ private String topologyId;
+ private ConfigBusConsumer consumer;
+
+ private transient IMetadataServiceClient client;
+
+ public ZKMetadataChangeNotifyService(ZKConfig config, String topologyId) {
+ this.zkConfig = config;
+ this.topologyId = topologyId;
+ }
+
+ @Override
+ public void init(Config config, MetadataType type) {
+ super.init(config, type);
+ client = new MetadataServiceClientImpl(config);
+ consumer = new ConfigBusConsumer(zkConfig, topologyId + "/" + getMetadataTopicSuffix(), this);
+ LOG.info("init called for client");
+ }
+
+ /**
+ * @seeAlso Coordinator
+ * @return
+ */
+ private String getMetadataTopicSuffix() {
+ switch (type) {
+ case ALERT_BOLT:
+ return "alert";
+ case ALERT_PUBLISH_BOLT:
+ return "publisher";
+ case SPOUT:
+ return "spout";
+ case STREAM_ROUTER_BOLT:
+ return "router";
+ default:
+ throw new RuntimeException(String.format("unexpected metadata type: %s !", type));
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ consumer.close();
+ LOG.info("Config consumer closed");
+ }
+
+ @Override
+ public void onNewConfig(ConfigValue value) {
+ LOG.info("Metadata changed {}",value);
+
+ if (client == null) {
+ LOG.error("OnNewConfig trigger, but metadata service client is null. Metadata type {}", type);
+ return;
+ }
+
+ // analyze config value and notify corresponding listeners
+ String version = value.getValue().toString();
+ // brute-force load all: this might introduce load's on metadata service.
+ // FIXME : after ScheduleState persisted with better normalization, load
+ // state based on type and version
+ ScheduleState state = client.getVersionedSpec(version);
+ if (state == null) {
+ LOG.error("Failed to load schedule state of version {}, this is possibly a bug, pls check coordinator log !", version);
+ return;
+ }
+ Map<String, StreamDefinition> sds = getStreams(state.getStreamSnapshots());
+ switch (type) {
+ case ALERT_BOLT:
+ // we might query metadata service query get metadata snapshot and StreamDefinition
+ AlertBoltSpec alertSpec = state.getAlertSpecs().get(topologyId);
+ if (alertSpec == null) {
+ LOG.error(" alert spec for version {} not found for topology {} !", version, topologyId);
+ } else {
+ prePopulate(alertSpec, state.getPolicySnapshots());
+ notifyAlertBolt(alertSpec, sds);
+ }
+ break;
+ case ALERT_PUBLISH_BOLT:
+ PublishSpec pubSpec = state.getPublishSpecs().get(topologyId);
+ if (pubSpec == null) {
+ LOG.error(" alert spec for version {} not found for topology {} !", version, topologyId);
+ } else {
+ notifyAlertPublishBolt(pubSpec, sds);
+ }
+ break;
+ case SPOUT:
+ SpoutSpec spoutSpec = state.getSpoutSpecs().get(topologyId);
+ if (spoutSpec == null) {
+ LOG.error(" alert spec for version {} not found for topology {} !", version, topologyId);
+ } else {
+ notifySpout(spoutSpec, sds);
+ }
+ break;
+ case STREAM_ROUTER_BOLT:
+ RouterSpec gSpec = state.getGroupSpecs().get(topologyId);
+ if (gSpec == null) {
+ LOG.error(" alert spec for version {} not found for topology {} !", version, topologyId);
+ } else {
+ notifyStreamRouterBolt(gSpec, sds);
+ }
+ break;
+ default:
+ LOG.error("unexpected metadata type: {} ", type);
+ }
+ }
+
+ private void prePopulate(AlertBoltSpec alertSpec, List<VersionedPolicyDefinition> list) {
+ Map<String, PolicyDefinition> policyMap = listToMap(list);
+ for (Entry<String, List<String>> policyEntry : alertSpec.getBoltPolicyIdsMap().entrySet()) {
+ List<PolicyDefinition> pds = alertSpec.getBoltPoliciesMap().get(policyEntry.getKey());
+ if (pds == null) {
+ pds = new ArrayList<PolicyDefinition>();
+ alertSpec.getBoltPoliciesMap().put(policyEntry.getKey(), pds);
+ }
+ for (String policyName : policyEntry.getValue()) {
+ if (policyMap.containsKey(policyName)) {
+ pds.add(policyMap.get(policyName));
+ }
+ }
+ }
+ }
+
+ private Map<String, StreamDefinition> getStreams(List<VersionedStreamDefinition> streamSnapshots) {
+ Map<String, StreamDefinition> result = new HashMap<String, StreamDefinition>();
+ for (VersionedStreamDefinition vsd : streamSnapshots) {
+ result.put(vsd.getDefinition().getStreamId(), vsd.getDefinition());
+ }
+ return result;
+ }
+
+ private Map<String, PolicyDefinition> listToMap(List<VersionedPolicyDefinition> listStreams) {
+ Map<String, PolicyDefinition> result = new HashMap<String, PolicyDefinition>();
+ for (VersionedPolicyDefinition sd : listStreams) {
+ result.put(sd.getDefinition().getName(), sd.getDefinition());
+ }
+ return result;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/evaluator/PolicyChangeListener.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/evaluator/PolicyChangeListener.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/evaluator/PolicyChangeListener.java
new file mode 100644
index 0000000..22f1408
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/evaluator/PolicyChangeListener.java
@@ -0,0 +1,29 @@
+package org.apache.eagle.alert.engine.evaluator;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.eagle.alert.engine.coordinator.PolicyDefinition;
+import org.apache.eagle.alert.engine.coordinator.StreamDefinition;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+public interface PolicyChangeListener {
+ void onPolicyChange(List<PolicyDefinition> added,
+ List<PolicyDefinition> removed,
+ List<PolicyDefinition> modified, Map<String, StreamDefinition> sds);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/evaluator/PolicyGroupEvaluator.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/evaluator/PolicyGroupEvaluator.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/evaluator/PolicyGroupEvaluator.java
new file mode 100644
index 0000000..dd3ca24
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/evaluator/PolicyGroupEvaluator.java
@@ -0,0 +1,45 @@
+package org.apache.eagle.alert.engine.evaluator;
+
+import java.io.Serializable;
+
+import org.apache.eagle.alert.engine.AlertStreamCollector;
+import org.apache.eagle.alert.engine.StreamContext;
+import org.apache.eagle.alert.engine.model.PartitionedEvent;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * policy group refers to the policies which belong to the same MonitoredStream
+ * 3 lifecycle steps are involved in PolicyGroupEvaluator
+ * Step 1: create object. Be aware that in distributed environment, this object may be serialized and transferred across network
+ * Step 2: init. This normally is invoked only once before nextEvent is invoked
+ * Step 3: nextEvent
+ * Step 4: close
+ */
+public interface PolicyGroupEvaluator extends PolicyChangeListener, Serializable{
+ void init(StreamContext context, AlertStreamCollector collector);
+
+ /**
+ * Evaluate event
+ */
+ void nextEvent(PartitionedEvent event);
+
+ String getName();
+
+ void close();
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/evaluator/PolicyHandlerContext.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/evaluator/PolicyHandlerContext.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/evaluator/PolicyHandlerContext.java
new file mode 100644
index 0000000..2898ebc
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/evaluator/PolicyHandlerContext.java
@@ -0,0 +1,60 @@
+package org.apache.eagle.alert.engine.evaluator;
+
+import org.apache.eagle.alert.engine.coordinator.PolicyDefinition;
+
+import backtype.storm.metric.api.MultiCountMetric;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+public class PolicyHandlerContext {
+ private PolicyDefinition policyDefinition;
+ private PolicyGroupEvaluator parentEvaluator;
+ private MultiCountMetric policyCounter;
+ private String policyEvaluatorId;
+
+ public PolicyDefinition getPolicyDefinition() {
+ return policyDefinition;
+ }
+
+ public void setPolicyDefinition(PolicyDefinition policyDefinition) {
+ this.policyDefinition = policyDefinition;
+ }
+
+ public PolicyGroupEvaluator getParentEvaluator() {
+ return parentEvaluator;
+ }
+
+ public void setParentEvaluator(PolicyGroupEvaluator parentEvaluator) {
+ this.parentEvaluator = parentEvaluator;
+ }
+
+ public void setPolicyCounter(MultiCountMetric metric) {
+ this.policyCounter = metric;
+ }
+
+ public MultiCountMetric getPolicyCounter() {
+ return policyCounter;
+ }
+
+ public String getPolicyEvaluatorId() {
+ return policyEvaluatorId;
+ }
+
+ public void setPolicyEvaluatorId(String policyEvaluatorId) {
+ this.policyEvaluatorId = policyEvaluatorId;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/evaluator/PolicyStreamHandler.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/evaluator/PolicyStreamHandler.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/evaluator/PolicyStreamHandler.java
new file mode 100755
index 0000000..069d321
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/evaluator/PolicyStreamHandler.java
@@ -0,0 +1,27 @@
+package org.apache.eagle.alert.engine.evaluator;
+
+import org.apache.eagle.alert.engine.Collector;
+import org.apache.eagle.alert.engine.model.AlertStreamEvent;
+import org.apache.eagle.alert.engine.model.StreamEvent;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+public interface PolicyStreamHandler {
+ void prepare(Collector<AlertStreamEvent> collector, PolicyHandlerContext context) throws Exception;
+ void send(StreamEvent event) throws Exception;
+ void close() throws Exception;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/evaluator/PolicyStreamHandlers.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/evaluator/PolicyStreamHandlers.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/evaluator/PolicyStreamHandlers.java
new file mode 100644
index 0000000..2aa70e8
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/evaluator/PolicyStreamHandlers.java
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.evaluator;
+
+import java.util.Map;
+
+import org.apache.eagle.alert.engine.coordinator.StreamDefinition;
+import org.apache.eagle.alert.engine.evaluator.impl.SiddhiPolicyHandler;
+
+public class PolicyStreamHandlers {
+ public static final String SIDDHI_ENGINE ="siddhi";
+
+ public static PolicyStreamHandler createHandler(String type, Map<String, StreamDefinition> sds){
+ if(SIDDHI_ENGINE.equals(type)) {
+ return new SiddhiPolicyHandler(sds);
+ }
+ throw new IllegalArgumentException("Illegal policy stream handler type: "+type);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/evaluator/impl/AlertBoltOutputCollectorThreadSafeWrapper.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/evaluator/impl/AlertBoltOutputCollectorThreadSafeWrapper.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/evaluator/impl/AlertBoltOutputCollectorThreadSafeWrapper.java
new file mode 100755
index 0000000..3aa079e
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/evaluator/impl/AlertBoltOutputCollectorThreadSafeWrapper.java
@@ -0,0 +1,118 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.evaluator.impl;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.eagle.alert.engine.AlertStreamCollector;
+import org.apache.eagle.alert.engine.model.AlertStreamEvent;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import backtype.storm.task.OutputCollector;
+
+/**
+ * <h2>Thread Safe Mechanism</h2>
+ * <ul>
+ * <li>
+ * emit() method is thread-safe enough to be called anywhere asynchronously in multi-thread
+ * </li>
+ * <li>
+ * flush() method must be called synchronously, because Storm OutputCollector is not thread-safe
+ * </li>
+ * </ul>
+ */
+public class AlertBoltOutputCollectorThreadSafeWrapper implements AlertStreamCollector {
+ private final OutputCollector delegate;
+ private final LinkedBlockingQueue<AlertStreamEvent> queue;
+ private final static Logger LOG = LoggerFactory.getLogger(AlertBoltOutputCollectorThreadSafeWrapper.class);
+ private final AtomicLong lastFlushTime = new AtomicLong(System.currentTimeMillis());
+ private final AutoAlertFlusher flusher;
+ private final static int MAX_ALERT_DELAY_SECS = 10;
+
+ public AlertBoltOutputCollectorThreadSafeWrapper(OutputCollector outputCollector){
+ this.delegate = outputCollector;
+ this.queue = new LinkedBlockingQueue<>();
+ this.flusher = new AutoAlertFlusher(this);
+ this.flusher.setName(Thread.currentThread().getName()+"-alertFlusher");
+ this.flusher.start();
+ }
+
+ private static class AutoAlertFlusher extends Thread{
+ private final AlertBoltOutputCollectorThreadSafeWrapper collector;
+ private boolean stopped = false;
+ private final static Logger LOG = LoggerFactory.getLogger(AutoAlertFlusher.class);
+
+ private AutoAlertFlusher(AlertBoltOutputCollectorThreadSafeWrapper collector){
+ this.collector = collector;
+ }
+
+ @Override
+ public void run() {
+ LOG.info("Starting");
+ while(!this.stopped){
+ if(System.currentTimeMillis() - collector.lastFlushTime.get() >= MAX_ALERT_DELAY_SECS * 1000L){
+ this.collector.flush();
+ }
+ try {
+ Thread.sleep(5000);
+ } catch (InterruptedException ignored) {}
+ }
+ LOG.info("Stopped");
+ }
+ public void shutdown(){
+ LOG.info("Stopping");
+ this.stopped = true;
+ }
+ }
+
+ /**
+ * Emit method can be called in multi-thread
+ * @param event
+ */
+ @Override
+ public void emit(AlertStreamEvent event) {
+ try {
+ queue.put(event);
+ } catch (InterruptedException e) {
+ LOG.error(e.getMessage(),e);
+ }
+ }
+
+ /**
+ * Flush will be called in synchronous way like StormBolt.execute() as Storm OutputCollector is not thread-safe
+ */
+ @Override
+ public void flush() {
+ if(!queue.isEmpty()) {
+ List<AlertStreamEvent> events = new ArrayList<>();
+ queue.drainTo(events);
+ events.forEach((event) -> delegate.emit(Arrays.asList(event.getStreamId(), event)));
+ LOG.info("Flushed {} alerts", events.size());
+ }
+ lastFlushTime.set(System.currentTimeMillis());
+ }
+
+ @Override
+ public void close() {
+ this.flusher.shutdown();
+ }
+}
\ No newline at end of file
[02/19] incubator-eagle git commit: EAGLE-324: Init branch-v0.5
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata/src/test/java/org/apache/eagle/service/alert/resource/impl/MongoImplTest.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata/src/test/java/org/apache/eagle/service/alert/resource/impl/MongoImplTest.java b/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata/src/test/java/org/apache/eagle/service/alert/resource/impl/MongoImplTest.java
new file mode 100644
index 0000000..16b53d4
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata/src/test/java/org/apache/eagle/service/alert/resource/impl/MongoImplTest.java
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.service.alert.resource.impl;
+
+import java.util.Date;
+import java.util.List;
+
+import org.apache.eagle.alert.coordination.model.Kafka2TupleMetadata;
+import org.apache.eagle.alert.coordination.model.ScheduleState;
+import org.apache.eagle.alert.coordination.model.internal.PolicyAssignment;
+import org.apache.eagle.alert.coordination.model.internal.Topology;
+import org.apache.eagle.alert.engine.coordinator.PolicyDefinition;
+import org.apache.eagle.alert.engine.coordinator.Publishment;
+import org.apache.eagle.alert.engine.coordinator.PublishmentType;
+import org.apache.eagle.alert.engine.coordinator.StreamingCluster;
+import org.apache.eagle.alert.metadata.impl.MongoMetadataDaoImpl;
+import org.apache.eagle.alert.metadata.resource.IMetadataDao;
+import org.apache.eagle.alert.metadata.resource.OpResult;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+
+import de.flapdoodle.embed.mongo.MongodExecutable;
+import de.flapdoodle.embed.mongo.MongodProcess;
+import de.flapdoodle.embed.mongo.MongodStarter;
+import de.flapdoodle.embed.mongo.config.MongodConfigBuilder;
+import de.flapdoodle.embed.mongo.config.Net;
+import de.flapdoodle.embed.mongo.distribution.Version;
+import de.flapdoodle.embed.process.runtime.Network;
+
+/**
+ * @since May 1, 2016
+ *
+ */
+public class MongoImplTest {
+ private static Logger LOG = LoggerFactory.getLogger(MongoImplTest.class);
+ static IMetadataDao dao;
+
+ private static MongodExecutable mongodExe;
+ private static MongodProcess mongod;
+
+ public static void before() {
+ try {
+ MongodStarter starter = MongodStarter.getDefaultInstance();
+ mongodExe = starter.prepare(new MongodConfigBuilder().version(Version.V3_2_1)
+ .net(new Net(27017, Network.localhostIsIPv6())).build());
+ mongod = mongodExe.start();
+ } catch (Exception e) {
+ LOG.error("start embed mongod failed, assume some external mongo running. continue run test!", e);
+ }
+ }
+
+ @BeforeClass
+ public static void setup() {
+ before();
+
+ System.setProperty("config.resource", "/application-mongo.conf");
+ ConfigFactory.invalidateCaches();
+ Config config = ConfigFactory.load().getConfig("datastore");
+ dao = new MongoMetadataDaoImpl(config);
+
+ }
+
+ @AfterClass
+ public static void teardown() {
+ if (mongod != null) {
+ mongod.stop();
+ mongodExe.stop();
+ }
+ }
+
+ private String TOPO_NAME = "topoName";
+
+ @Test
+ public void test_apis() {
+ // topology
+ {
+ OpResult result = dao.addTopology(new Topology(TOPO_NAME, 3, 5));
+ System.out.println(result.message);
+ Assert.assertEquals(200, result.code);
+ List<Topology> topos = dao.listTopologies();
+ Assert.assertEquals(1, topos.size());
+
+ result = dao.addTopology(new Topology(TOPO_NAME + "-new", 3, 5));
+ topos = dao.listTopologies();
+ Assert.assertEquals(2, topos.size());
+ // add again: replace existing one
+ result = dao.addTopology(new Topology(TOPO_NAME, 4, 5));
+ topos = dao.listTopologies();
+ Assert.assertEquals(2, topos.size());
+ Assert.assertEquals(TOPO_NAME, topos.get(0).getName());
+ Assert.assertEquals(4, topos.get(0).getNumOfGroupBolt());
+ }
+ // assignment
+ {
+ PolicyAssignment assignment = new PolicyAssignment();
+ assignment.setPolicyName("policy1");
+ OpResult result = dao.addAssignment(assignment);
+ Assert.assertEquals(200, result.code);
+ List<PolicyAssignment> assigns = dao.listAssignments();
+ Assert.assertEquals(1, assigns.size());
+ }
+ // cluster
+ {
+ StreamingCluster cluster = new StreamingCluster();
+ cluster.setName("dd");
+ OpResult result = dao.addCluster(cluster);
+ Assert.assertEquals(200, result.code);
+ List<StreamingCluster> assigns = dao.listClusters();
+ Assert.assertEquals(1, assigns.size());
+ }
+ // data source
+ {
+ Kafka2TupleMetadata dataSource = new Kafka2TupleMetadata();
+ dataSource.setName("ds");
+ OpResult result = dao.addDataSource(dataSource);
+ Assert.assertEquals(200, result.code);
+ List<Kafka2TupleMetadata> assigns = dao.listDataSources();
+ Assert.assertEquals(1, assigns.size());
+ }
+ // policy
+ {
+ PolicyDefinition policy = new PolicyDefinition();
+ policy.setName("ds");
+ OpResult result = dao.addPolicy(policy);
+ Assert.assertEquals(200, result.code);
+ List<PolicyDefinition> assigns = dao.listPolicies();
+ Assert.assertEquals(1, assigns.size());
+ }
+ // publishment
+ {
+ Publishment publishment = new Publishment();
+ publishment.setName("pub-");
+ OpResult result = dao.addPublishment(publishment);
+ Assert.assertEquals(200, result.code);
+ List<Publishment> assigns = dao.listPublishment();
+ Assert.assertEquals(1, assigns.size());
+ }
+ // publishment
+ {
+ PublishmentType publishmentType = new PublishmentType();
+ publishmentType.setType("KAFKA");
+ OpResult result = dao.addPublishmentType(publishmentType);
+ Assert.assertEquals(200, result.code);
+ List<PublishmentType> assigns = dao.listPublishmentType();
+ Assert.assertEquals(1, assigns.size());
+ }
+ }
+
+ private void test_addstate() {
+ ScheduleState state = new ScheduleState();
+ state.setVersion("state-" + System.currentTimeMillis());
+ state.setGenerateTime(String.valueOf(new Date().getTime()));
+ OpResult result = dao.addScheduleState(state);
+ Assert.assertEquals(200, result.code);
+ }
+
+ @Test
+ public void test_readCurrentState() {
+ test_addstate();
+ ScheduleState state = dao.getScheduleState();
+ Assert.assertNotNull(state);
+
+ System.out.println(state.getVersion());
+ System.out.println(state.getGenerateTime());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata/src/test/resources/application-mongo.conf
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata/src/test/resources/application-mongo.conf b/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata/src/test/resources/application-mongo.conf
new file mode 100644
index 0000000..20461c8
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata/src/test/resources/application-mongo.conf
@@ -0,0 +1,6 @@
+{
+ "datastore": {
+ "metadataDao": "org.apache.eagle.service.alert.resource.impl.InMemMetadataDaoImpl",
+ "connection": "localhost:27017"
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata/src/test/resources/policy-sample.json
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata/src/test/resources/policy-sample.json b/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata/src/test/resources/policy-sample.json
new file mode 100644
index 0000000..56a53e7
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata/src/test/resources/policy-sample.json
@@ -0,0 +1,32 @@
+{
+ "name": "policy1",
+ "description": "test policy",
+ "inputStreams": [
+ "in-stream-1",
+ "in-stream-2"
+ ],
+ "outputStreams": [
+ "out-stream-1"
+ ],
+ "definition": {
+ "type": "siddhiCEPEngine",
+ "value": "select count(*) from in-stream-1"
+ },
+ "partitionSpec": [
+ {
+ "streamId": "in-stream-1",
+ "type": "GROUPBY",
+ "columns": [
+ "col-1",
+ "col-2"
+ ],
+ "sortSpec": {
+ "column": "col-1",
+ "windowPeriod": "60",
+ "order": "asc",
+ "windowMargin": 30000
+ }
+ }
+ ],
+ "parallelismHint": 2
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-metadata-parent/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-metadata-parent/pom.xml b/eagle-core/eagle-alert/alert/alert-metadata-parent/pom.xml
new file mode 100644
index 0000000..e9e3534
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-metadata-parent/pom.xml
@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with ~
+ this work for additional information regarding copyright ownership. ~ The
+ ASF licenses this file to You under the Apache License, Version 2.0 ~ (the
+ "License"); you may not use this file except in compliance with ~ the License.
+ You may obtain a copy of the License at ~ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~ ~ Unless required by applicable law or agreed to in writing, software ~
+ distributed under the License is distributed on an "AS IS" BASIS, ~ WITHOUT
+ WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ~ See the
+ License for the specific language governing permissions and ~ limitations
+ under the License. -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.eagle</groupId>
+ <artifactId>alert-parent</artifactId>
+ <version>0.0.1-SNAPSHOT</version>
+ </parent>
+ <artifactId>alert-metadata-parent</artifactId>
+ <packaging>pom</packaging>
+
+ <modules>
+ <module>alert-metadata</module>
+ <module>alert-metadata-service</module>
+ </modules>
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/pom.xml b/eagle-core/eagle-alert/alert/pom.xml
new file mode 100644
index 0000000..9d2c845
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/pom.xml
@@ -0,0 +1,387 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with ~
+ this work for additional information regarding copyright ownership. ~ The
+ ASF licenses this file to You under the Apache License, Version 2.0 ~ (the
+ "License"); you may not use this file except in compliance with ~ the License.
+ You may obtain a copy of the License at ~ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~ ~ Unless required by applicable law or agreed to in writing, software ~
+ distributed under the License is distributed on an "AS IS" BASIS, ~ WITHOUT
+ WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ~ See the
+ License for the specific language governing permissions and ~ limitations
+ under the License. -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <groupId>org.apache.eagle</groupId>
+ <artifactId>alert-parent</artifactId>
+ <version>0.0.1-SNAPSHOT</version>
+ <packaging>pom</packaging>
+
+ <modules>
+ <module>alert-common</module>
+ <module>alert-engine</module>
+ <module>alert-metadata-parent</module>
+ <module>alert-coordinator</module>
+ <module>alert-assembly</module>
+ <module>alert-devtools</module>
+ </modules>
+
+ <properties>
+ <!-- Runtime -->
+ <java.version>1.8</java.version>
+
+ <!-- Build -->
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <maven-compiler.version>2.0.2</maven-compiler.version>
+
+ <!-- Test -->
+ <junit.version>4.12</junit.version>
+
+ <!-- Utilities -->
+ <typesafe-config.version>1.2.1</typesafe-config.version>
+ <guava.version>19.0</guava.version>
+ <commons-lang3.version>3.3.2</commons-lang3.version>
+ <commons-collections.version>3.2.1</commons-collections.version>
+ <commons-lang.version>2.6</commons-lang.version>
+ <archaius.version>0.6.1</archaius.version>
+ <metrics.version>3.1.0</metrics.version>
+ <mapdb.version>3.0.0-M6</mapdb.version>
+
+ <!-- CEP -->
+ <siddhi.version>3.0.5</siddhi.version>
+
+ <!-- PUBLISH -->
+ <velocity.version>1.7</velocity.version>
+ <javax.mail.version>1.4</javax.mail.version>
+
+ <!-- JSON -->
+ <fastxml.version>2.4.5</fastxml.version>
+ <jackson.version>1.9.13</jackson.version>
+ <fasterxml-jackson.version>2.4.5</fasterxml-jackson.version>
+ <codehaus-jackson.version>1.9.13</codehaus-jackson.version>
+
+ <!-- Storm and Kafka -->
+ <storm.version>0.9.3</storm.version>
+ <storm-kafka.version>0.9.3.2.2.0.0-2041</storm-kafka.version>
+ <kafka-clients.version>0.8.2.0</kafka-clients.version>
+ <kafka.version>0.8.1.2.2.0.0-2041</kafka.version>
+ <kafka.artifact.id>kafka_2.10</kafka.artifact.id>
+ <zkclient.version>0.3</zkclient.version>
+ <slf4j-api.veresion>1.7.20</slf4j-api.veresion>
+ <slf4j-log4j12.veresion>1.7.20</slf4j-log4j12.veresion>
+ <logback.version>1.1.7</logback.version>
+
+ <!-- Web Service/Client -->
+ <jersey.version>1.19.1</jersey.version>
+ <!--<jersey-client.version>1.19.1</jersey-client.version>-->
+ <!--<jersey-server.version>1.19.1</jersey-server.version>-->
+ <!--<jersey-multipart.version>1.19.1</jersey-multipart.version>-->
+
+ <jetty.version>9.2.3.v20140905</jetty.version>
+ <tomcat-embed.version>7.0.55</tomcat-embed.version>
+ <swagger.version>1.5.8</swagger.version>
+ <curator-framework.version>2.8.0</curator-framework.version>
+ <mongodb.version>3.2.2</mongodb.version>
+ <mockito.version>1.10.19</mockito.version>
+ <joda.version>2.2</joda.version>
+ <metrics-elasticsearch-reporter.version>2.2.0</metrics-elasticsearch-reporter.version>
+ </properties>
+
+ <repositories>
+ <repository>
+ <id>Maven Central Repository</id>
+ <url>http://central.maven.org/maven2/</url>
+ </repository>
+ <repository>
+ <id>HDP Release Repository</id>
+ <url>http://repo.hortonworks.com/content/repositories/releases/</url>
+ </repository>
+ <repository>
+ <id>HDP Central Repository</id>
+ <url>http://repo.hortonworks.com/content/repositories/central/</url>
+ </repository>
+ <repository>
+ <id>WSO2 Public</id>
+ <url>http://maven.wso2.org/nexus/content/groups/wso2-public</url>
+ </repository>
+ </repositories>
+
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ <version>${fastxml.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>${fastxml.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>javax.mail</groupId>
+ <artifactId>mail</artifactId>
+ <version>${javax.mail.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.velocity</groupId>
+ <artifactId>velocity</artifactId>
+ <version>${velocity.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <version>${mockito.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-framework</artifactId>
+ <version>${curator-framework.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-recipes</artifactId>
+ <version>${curator-framework.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ <version>${fasterxml-jackson.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ <version>${commons-lang3.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-collections</groupId>
+ <artifactId>commons-collections</artifactId>
+ <version>${commons-collections.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ <version>${commons-lang.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>${junit.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>${fasterxml-jackson.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.storm</groupId>
+ <artifactId>storm-kafka</artifactId>
+ <version>${storm-kafka.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ <version>${slf4j-log4j12.veresion}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.storm</groupId>
+ <artifactId>storm-core</artifactId>
+ <version>${storm.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>log4j-over-slf4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-slf4j-impl</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-classic</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>${kafka.artifact.id}</artifactId>
+ <version>${kafka.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.sfl4j</groupId>
+ <artifactId>slf4j-log4j</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-client</artifactId>
+ <version>${jersey.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-jaxrs</artifactId>
+ <version>${codehaus-jackson.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-xc</artifactId>
+ <version>${codehaus-jackson.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-mapper-asl</artifactId>
+ <version>${codehaus-jackson.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.typesafe</groupId>
+ <artifactId>config</artifactId>
+ <version>${typesafe-config.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.netflix.archaius</groupId>
+ <artifactId>archaius-core</artifactId>
+ <version>${archaius.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-core</artifactId>
+ <version>${metrics.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-jvm</artifactId>
+ <version>${metrics.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.wso2.siddhi</groupId>
+ <artifactId>siddhi-core</artifactId>
+ <version>${siddhi.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-simple</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.wso2.siddhi</groupId>
+ <artifactId>siddhi-query-api</artifactId>
+ <version>${siddhi.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.wso2.siddhi</groupId>
+ <artifactId>siddhi-query-compiler</artifactId>
+ <version>${siddhi.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.wso2.siddhi</groupId>
+ <artifactId>siddhi-extension-regex</artifactId>
+ <version>${siddhi.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.wso2.siddhi</groupId>
+ <artifactId>siddhi-extension-string</artifactId>
+ <version>${siddhi.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka-clients</artifactId>
+ <version>${kafka-clients.version}</version>
+ <!--<scope>test</scope> -->
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>com.101tec</groupId>
+ <artifactId>zkclient</artifactId>
+ <version>${zkclient.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-server</artifactId>
+ <version>${jersey.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-servlet</artifactId>
+ <version>${jersey.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.sun.jersey.contribs</groupId>
+ <artifactId>jersey-multipart</artifactId>
+ <version>${jersey.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tomcat.embed</groupId>
+ <artifactId>tomcat-embed-core</artifactId>
+ <version>${tomcat-embed.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>${guava.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>${slf4j-api.veresion}</version>
+ </dependency>
+ <!--<dependency>-->
+ <!--<groupId>ch.qos.logback</groupId>-->
+ <!--<artifactId>logback-classic</artifactId>-->
+ <!--<version>${logback.version}</version>-->
+ <!--</dependency>-->
+ <dependency>
+ <groupId>io.swagger</groupId>
+ <artifactId>swagger-jaxrs</artifactId>
+ <version>${swagger.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.mapdb</groupId>
+ <artifactId>mapdb</artifactId>
+ <version>${mapdb.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>joda-time</groupId>
+ <artifactId>joda-time</artifactId>
+ <version>${joda.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.elasticsearch</groupId>
+ <artifactId>metrics-elasticsearch-reporter</artifactId>
+ <version>${metrics-elasticsearch-reporter.version}</version>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+
+ <build>
+ <pluginManagement>
+ <plugins>
+ <plugin>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>${maven-compiler.version}</version>
+ <configuration>
+ <source>${java.version}</source>
+ <target>${java.version}</target>
+ <encoding>UTF-8</encoding>
+ <maxmem>2048m</maxmem>
+ <fork>true</fork>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-maven-plugin</artifactId>
+ <version>${jetty.version}</version>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+ </build>
+</project>
[05/19] incubator-eagle git commit: EAGLE-324: Init branch-v0.5
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/sorter/StreamWindowBenchmarkTest.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/sorter/StreamWindowBenchmarkTest.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/sorter/StreamWindowBenchmarkTest.java
new file mode 100644
index 0000000..dc9782e
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/sorter/StreamWindowBenchmarkTest.java
@@ -0,0 +1,145 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.sorter;
+
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.lang.time.StopWatch;
+import org.apache.eagle.alert.engine.mock.MockSampleMetadataFactory;
+import org.apache.eagle.alert.engine.model.PartitionedEvent;
+import org.apache.eagle.alert.utils.DateTimeUtil;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.codahale.metrics.ConsoleReporter;
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.ScheduledReporter;
+import com.codahale.metrics.jvm.GarbageCollectorMetricSet;
+import com.codahale.metrics.jvm.MemoryUsageGaugeSet;
+
+@Ignore
+public class StreamWindowBenchmarkTest {
+ private final static Logger LOGGER = LoggerFactory.getLogger(StreamWindowBenchmarkTest.class);
+ public void sendDESCOrderedEventsToWindow(StreamWindow window, StreamWindowRepository.StorageType storageType, int num) {
+ LOGGER.info("Sending {} events to {} ({})",num,window.getClass().getSimpleName(),storageType);
+ StopWatch stopWatch = new StopWatch();
+ stopWatch.start();
+ int i=0;
+ while(i<num) {
+ PartitionedEvent event = MockSampleMetadataFactory.createPartitionedEventGroupedByName("sampleStream_1",(window.startTime()+i));
+ window.add(event);
+ i++;
+ }
+ stopWatch.stop();
+ performanceReport.put(num+"\tInsertTime\t"+storageType,stopWatch.getTime());
+ LOGGER.info("Inserted {} events in {} ms",num,stopWatch.getTime());
+ stopWatch.reset();
+ stopWatch.start();
+ window.flush();
+ stopWatch.stop();
+ performanceReport.put(num+"\tReadTime\t"+storageType,stopWatch.getTime());
+ }
+
+ private ScheduledReporter metricReporter;
+ private Map<String,Long> performanceReport;
+ @Before
+ public void setUp(){
+ final MetricRegistry metrics = new MetricRegistry();
+ metrics.registerAll(new MemoryUsageGaugeSet());
+ metrics.registerAll(new GarbageCollectorMetricSet());
+ metricReporter = ConsoleReporter.forRegistry(metrics)
+ .filter((name, metric) -> name.matches("(.*heap|total).(usage|used)"))
+// .withLoggingLevel(Slf4jReporter.LoggingLevel.DEBUG)
+ .convertRatesTo(TimeUnit.SECONDS)
+ .convertDurationsTo(TimeUnit.MILLISECONDS)
+ .build();
+ metricReporter.start(60,TimeUnit.SECONDS);
+ performanceReport = new TreeMap<>();
+ }
+
+ @After
+ public void after(){
+ StringBuilder sb = new StringBuilder();
+ for(Map.Entry<String,Long> entry:performanceReport.entrySet()){
+ sb.append(String.format("%-40s\t%s\n",entry.getKey(),entry.getValue()));
+ }
+ LOGGER.info("\n===== Benchmark Result Report =====\n\n{}",sb.toString());
+ }
+
+ private final long start = DateTimeUtil.humanDateToMillisecondsWithoutException("2016-05-04 00:00:00,000");
+ private final long stop = DateTimeUtil.humanDateToMillisecondsWithoutException("2016-05-05 00:00:00,000");
+ private final long margin = (stop - start)/3;
+
+ private void benchmarkTest(StreamWindow window, StreamWindowRepository.StorageType storageType){
+ StopWatch stopWatch = new StopWatch();
+ stopWatch.start();
+ LOGGER.info("\n===== Benchmark Test for {} ({}) =====",window.getClass().getSimpleName(),storageType);
+ metricReporter.report();
+ sendDESCOrderedEventsToWindow(window,storageType,1000);
+ metricReporter.report();
+ sendDESCOrderedEventsToWindow(window,storageType,10000);
+ metricReporter.report();
+ sendDESCOrderedEventsToWindow(window,storageType,100000);
+ metricReporter.report();
+ sendDESCOrderedEventsToWindow(window,storageType,1000000);
+ metricReporter.report();
+ stopWatch.stop();
+ LOGGER.info("\n===== Finished in total {} ms =====\n",stopWatch.getTime());
+ }
+
+ @Test @Ignore
+ public void testStreamWindowBenchmarkMain(){
+ testStreamSortedWindowOnHeap();
+ testStreamSortedWindowInSerializedMemory();
+ testStreamSortedWindowOffHeap();
+ testStreamSortedWindowFile();
+ }
+
+ @Test @Ignore
+ public void testStreamSortedWindowOnHeap() {
+ StreamWindow window = StreamWindowRepository.getSingletonInstance().createWindow(start,stop,margin,StreamWindowRepository.StorageType.ONHEAP);
+ benchmarkTest(window,StreamWindowRepository.StorageType.ONHEAP);
+ window.close();
+ }
+
+ @Test @Ignore
+ public void testStreamSortedWindowInSerializedMemory() {
+ StreamWindow window = StreamWindowRepository.getSingletonInstance().createWindow(start,stop,margin,StreamWindowRepository.StorageType.MEMORY);
+ benchmarkTest(window,StreamWindowRepository.StorageType.MEMORY);
+ window.close();
+ }
+
+ @Test @Ignore
+ public void testStreamSortedWindowOffHeap() {
+ StreamWindow window = StreamWindowRepository.getSingletonInstance().createWindow(start,stop,margin,StreamWindowRepository.StorageType.DIRECT_MEMORY);
+ benchmarkTest(window,StreamWindowRepository.StorageType.DIRECT_MEMORY);
+ window.close();
+ }
+
+ @Test @Ignore
+ public void testStreamSortedWindowFile() {
+ StreamWindow window = StreamWindowRepository.getSingletonInstance().createWindow(start,stop,margin,StreamWindowRepository.StorageType.FILE_RAF);
+ benchmarkTest(window,StreamWindowRepository.StorageType.FILE_RAF);
+ window.close();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/sorter/StreamWindowTestSuite.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/sorter/StreamWindowTestSuite.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/sorter/StreamWindowTestSuite.java
new file mode 100644
index 0000000..950aa34
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/sorter/StreamWindowTestSuite.java
@@ -0,0 +1,164 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.sorter;
+
+import java.util.List;
+
+import org.apache.eagle.alert.engine.mock.MockPartitionedCollector;
+import org.apache.eagle.alert.engine.mock.MockSampleMetadataFactory;
+import org.apache.eagle.alert.engine.model.PartitionedEvent;
+import org.apache.eagle.alert.engine.sorter.impl.PartitionedEventTimeOrderingComparator;
+import org.apache.eagle.alert.engine.sorter.impl.StreamTimeClockInLocalMemory;
+import org.apache.eagle.alert.utils.DateTimeUtil;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Ordering;
+
+@SuppressWarnings("unused")
+public class StreamWindowTestSuite {
+ private final static Logger LOGGER = LoggerFactory.getLogger(StreamWindowTestSuite.class);
+
+ private final long start = DateTimeUtil.humanDateToMillisecondsWithoutException("2016-05-04 00:00:00,000");
+ private final long stop = DateTimeUtil.humanDateToMillisecondsWithoutException("2016-05-04 00:01:00,000");
+ private final long margin = (stop - start)/3;
+
+ @Test
+ public void testStreamSortedWindowOnHeap() {
+ StreamWindow window = StreamWindowRepository.getSingletonInstance().createWindow(start,stop,margin,StreamWindowRepository.StorageType.ONHEAP);
+ streamSortedWindowMustTest(window);
+ }
+
+ @Test
+ public void testStreamSortedWindowInSerializedMemory() {
+ StreamWindow window = StreamWindowRepository.getSingletonInstance().createWindow(start,stop,margin,StreamWindowRepository.StorageType.MEMORY);
+ streamSortedWindowMustTest(window);
+ }
+
+ @Test
+ public void testStreamSortedWindowOffHeap() {
+ StreamWindow window = StreamWindowRepository.getSingletonInstance().createWindow(start,stop,margin,StreamWindowRepository.StorageType.DIRECT_MEMORY);
+ streamSortedWindowMustTest(window);
+ }
+
+ @Test
+ public void testStreamSortedWindowFile() {
+ StreamWindow window = StreamWindowRepository.getSingletonInstance().createWindow(start,stop,margin,StreamWindowRepository.StorageType.FILE_RAF);
+ streamSortedWindowMustTest(window);
+ }
+
+ @SuppressWarnings({ "unchecked", "rawtypes" })
+ private void streamSortedWindowMustTest(StreamWindow window){
+ MockPartitionedCollector collector = new MockPartitionedCollector();
+ window.register(collector);
+
+ StreamTimeClock clock = new StreamTimeClockInLocalMemory("sampleStream_1");
+ clock.moveForward(DateTimeUtil.humanDateToMillisecondsWithoutException("2016-05-04 00:00:30,000"));
+
+ // Current time is: "2016-05-04 00:00:30"
+ window.onTick(clock,System.currentTimeMillis());
+
+ Assert.assertTrue(window.alive());
+ Assert.assertFalse(window.expired());
+
+ // Accepted
+ Assert.assertTrue(window.accept(DateTimeUtil.humanDateToMillisecondsWithoutException("2016-05-04 00:00:00,000")));
+ Assert.assertTrue(window.accept(DateTimeUtil.humanDateToMillisecondsWithoutException("2016-05-04 00:00:01,000")));
+ Assert.assertTrue(window.accept(DateTimeUtil.humanDateToMillisecondsWithoutException("2016-05-04 00:00:30,000")));
+ Assert.assertTrue(window.accept(DateTimeUtil.humanDateToMillisecondsWithoutException("2016-05-04 00:00:40,000")));
+ Assert.assertTrue(window.accept(DateTimeUtil.humanDateToMillisecondsWithoutException("2016-05-04 00:00:50,000")));
+
+
+ // Rejected
+ Assert.assertFalse(window.accept(DateTimeUtil.humanDateToMillisecondsWithoutException("2016-05-03 23:59:59,000")));
+ Assert.assertFalse(window.accept(DateTimeUtil.humanDateToMillisecondsWithoutException("2016-05-04 00:01:00,000")));
+ Assert.assertFalse(window.accept(DateTimeUtil.humanDateToMillisecondsWithoutException("2016-05-04 00:01:01,000")));
+
+ // Accepted
+ Assert.assertTrue(window.add(MockSampleMetadataFactory.createRandomPartitionedEvent("sampleStream_1",DateTimeUtil.humanDateToMillisecondsWithoutException("2016-05-04 00:00:00,000"))));
+
+ Assert.assertTrue(window.add(MockSampleMetadataFactory.createRandomPartitionedEvent("sampleStream_1",DateTimeUtil.humanDateToMillisecondsWithoutException("2016-05-04 00:00:01,000"))));
+ Assert.assertTrue(window.add(MockSampleMetadataFactory.createRandomPartitionedEvent("sampleStream_1",DateTimeUtil.humanDateToMillisecondsWithoutException("2016-05-04 00:00:50,000"))));
+ Assert.assertTrue(window.add(MockSampleMetadataFactory.createRandomPartitionedEvent("sampleStream_1",DateTimeUtil.humanDateToMillisecondsWithoutException("2016-05-04 00:00:40,000"))));
+ Assert.assertTrue(window.add(MockSampleMetadataFactory.createRandomPartitionedEvent("sampleStream_1",DateTimeUtil.humanDateToMillisecondsWithoutException("2016-05-04 00:00:30,000"))));
+
+ // Should accept Duplicated
+ Assert.assertTrue("Should support duplicated timestamp",window.add(MockSampleMetadataFactory.createRandomPartitionedEvent("sampleStream_1",DateTimeUtil.humanDateToMillisecondsWithoutException("2016-05-04 00:00:00,000"))));
+
+ Assert.assertEquals(6,window.size());
+
+ // Rejected
+ Assert.assertFalse(window.add(MockSampleMetadataFactory.createRandomPartitionedEvent("sampleStream_1",DateTimeUtil.humanDateToMillisecondsWithoutException("2016-05-03 23:59:59,000"))));
+ Assert.assertFalse(window.add(MockSampleMetadataFactory.createRandomPartitionedEvent("sampleStream_1",DateTimeUtil.humanDateToMillisecondsWithoutException("2016-05-04 00:01:00,000"))));
+ Assert.assertFalse(window.add(MockSampleMetadataFactory.createRandomPartitionedEvent("sampleStream_1",DateTimeUtil.humanDateToMillisecondsWithoutException("2016-05-04 00:01:01,000"))));
+
+ Assert.assertEquals(6,window.size());
+
+ // Now is: "2016-05-04 00:00:55"
+ clock.moveForward(DateTimeUtil.humanDateToMillisecondsWithoutException("2016-05-04 00:00:55,000"));
+ window.onTick(clock,System.currentTimeMillis());
+ Assert.assertTrue(window.alive());
+ Assert.assertFalse(window.expired());
+ Assert.assertTrue(window.add(MockSampleMetadataFactory.createRandomPartitionedEvent("sampleStream_1",DateTimeUtil.humanDateToMillisecondsWithoutException("2016-05-04 00:00:35,000"))));
+ Assert.assertEquals(7,window.size());
+
+ // Flush when stream time delay too much after system time but window will still be alive
+ window.onTick(clock,System.currentTimeMillis() + 1 + stop - start + margin);
+ Assert.assertTrue(window.alive());
+ Assert.assertFalse(window.expired());
+ Assert.assertEquals(0,window.size());
+ Assert.assertEquals(7,collector.size());
+
+ Assert.assertFalse("Because window has flushed but not expired, window should reject future events < last flush stream time",
+ window.add(MockSampleMetadataFactory.createRandomPartitionedEvent("sampleStream_1",DateTimeUtil.humanDateToMillisecondsWithoutException("2016-05-04 00:00:54,000"))));
+ Assert.assertTrue("Because window has flushed but not expired, window should still accept future events >= last flush stream time",
+ window.add(MockSampleMetadataFactory.createRandomPartitionedEvent("sampleStream_1",DateTimeUtil.humanDateToMillisecondsWithoutException("2016-05-04 00:00:56,000"))));
+ Assert.assertEquals(1,window.size());
+ Assert.assertEquals(7,collector.size());
+
+ // Now is: "2016-05-04 00:01:10", not expire,
+ clock.moveForward(DateTimeUtil.humanDateToMillisecondsWithoutException("2016-05-04 00:01:10,000"));
+ window.onTick(clock,System.currentTimeMillis() + 2 * (1+ stop - start + margin));
+ Assert.assertEquals(8,collector.size());
+
+ // Now is: "2016-05-04 00:01:20", expire
+ clock.moveForward(DateTimeUtil.humanDateToMillisecondsWithoutException("2016-05-04 00:01:20,000"));
+ window.onTick(clock,System.currentTimeMillis());
+ Assert.assertFalse(window.alive());
+ Assert.assertTrue(window.expired());
+ Assert.assertFalse(window.add(MockSampleMetadataFactory.createRandomPartitionedEvent("sampleStream_1",DateTimeUtil.humanDateToMillisecondsWithoutException("2016-05-04 00:00:35,000"))));
+ Assert.assertEquals(0,window.size());
+
+ Assert.assertEquals(8,collector.size());
+
+ Ordering ordering = Ordering.from(PartitionedEventTimeOrderingComparator.INSTANCE);
+ Assert.assertTrue(ordering.isOrdered(collector.get()));
+
+ List<PartitionedEvent> list = collector.get();
+ Assert.assertEquals(8,list.size());
+ Assert.assertEquals(DateTimeUtil.humanDateToMillisecondsWithoutException("2016-05-04 00:00:00,000"),list.get(0).getTimestamp());
+ Assert.assertEquals(DateTimeUtil.humanDateToMillisecondsWithoutException("2016-05-04 00:00:00,000"),list.get(1).getTimestamp());
+ Assert.assertEquals(DateTimeUtil.humanDateToMillisecondsWithoutException("2016-05-04 00:00:01,000"),list.get(2).getTimestamp());
+ Assert.assertEquals(DateTimeUtil.humanDateToMillisecondsWithoutException("2016-05-04 00:00:30,000"),list.get(3).getTimestamp());
+ Assert.assertEquals(DateTimeUtil.humanDateToMillisecondsWithoutException("2016-05-04 00:00:35,000"),list.get(4).getTimestamp());
+ Assert.assertEquals(DateTimeUtil.humanDateToMillisecondsWithoutException("2016-05-04 00:00:40,000"),list.get(5).getTimestamp());
+ Assert.assertEquals(DateTimeUtil.humanDateToMillisecondsWithoutException("2016-05-04 00:00:50,000"),list.get(6).getTimestamp());
+ Assert.assertEquals(DateTimeUtil.humanDateToMillisecondsWithoutException("2016-05-04 00:00:56,000"),list.get(7).getTimestamp());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/sorter/TreeMultisetComparatorTest.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/sorter/TreeMultisetComparatorTest.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/sorter/TreeMultisetComparatorTest.java
new file mode 100644
index 0000000..deffed5
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/sorter/TreeMultisetComparatorTest.java
@@ -0,0 +1,94 @@
+package org.apache.eagle.alert.engine.sorter;
+
+import java.util.Arrays;
+import java.util.Iterator;
+
+import org.apache.eagle.alert.engine.coordinator.StreamPartition;
+import org.apache.eagle.alert.engine.model.PartitionedEvent;
+import org.apache.eagle.alert.engine.model.StreamEvent;
+import org.apache.eagle.alert.engine.sorter.impl.PartitionedEventTimeOrderingComparator;
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.google.common.collect.TreeMultiset;
+
+/**
+ * Since 5/10/16.
+ */
+public class TreeMultisetComparatorTest {
+ /**
+ * if 2 different events have the same timestamp, and comparator return 0 when timestamp is same,
+ * when they are added into TreeMultiset, the second event will be replaced by the first event
+ */
+ @Test
+ public void testComparator(){
+ TreeMultiset<PartitionedEvent> set = TreeMultiset.create(PartitionedEventTimeOrderingComparator.INSTANCE);
+
+ // construct PartitionEvent1
+ PartitionedEvent event1 = new PartitionedEvent();
+ StreamPartition sp = new StreamPartition();
+ sp.setColumns(Arrays.asList("host"));
+ sp.setSortSpec(null);
+ sp.setStreamId("testStreamId");
+ sp.setType(StreamPartition.Type.GROUPBY);
+ event1.setPartition(sp);
+ event1.setPartitionKey(1000);
+ StreamEvent e1 = new StreamEvent();
+ e1.setData(new Object[]{18.4});
+ e1.setStreamId("testStreamId");
+ e1.setTimestamp(1462909984000L);
+ event1.setEvent(e1);
+
+ // construct PartitionEvent2 with same timestamp but different value
+ PartitionedEvent event2 = new PartitionedEvent();
+ event2.setPartition(sp);
+ event2.setPartitionKey(1000);
+ StreamEvent e2 = new StreamEvent();
+ e2.setData(new Object[]{16.3});
+ e2.setStreamId("testStreamId");
+ e2.setTimestamp(1462909984000L);
+ event2.setEvent(e2);
+
+ // construct PartitionEvent2 with same timestamp but different value
+ PartitionedEvent event3 = new PartitionedEvent();
+ event3.setPartition(sp);
+ event3.setPartitionKey(1000);
+ StreamEvent e3 = new StreamEvent();
+ e3.setData(new Object[]{14.3});
+ e3.setStreamId("testStreamId");
+ e3.setTimestamp(1462909984001L);
+ event3.setEvent(e3);
+
+ PartitionedEvent event4 = new PartitionedEvent();
+ event4.setPartition(sp);
+ event4.setPartitionKey(1000);
+ StreamEvent e4 = new StreamEvent();
+ e4.setData(new Object[]{14.3});
+ e4.setStreamId("testStreamId");
+ e4.setTimestamp(1462909984001L);
+ event4.setEvent(e4);
+
+ Assert.assertNotEquals(event2,event3);
+ Assert.assertEquals(event3,event4);
+
+ // check content in set
+ set.add(event1);
+ set.add(event2);
+ set.add(event3);
+ set.add(event4);
+ Assert.assertEquals(4, set.size());
+ set.forEach(System.out::println);
+
+
+ Assert.assertEquals(-1,PartitionedEventTimeOrderingComparator.INSTANCE.compare(event1,event2));
+ Assert.assertEquals(-1,PartitionedEventTimeOrderingComparator.INSTANCE.compare(event1,event3));
+ Assert.assertEquals(-1,PartitionedEventTimeOrderingComparator.INSTANCE.compare(event2,event3));
+ Assert.assertEquals(0,PartitionedEventTimeOrderingComparator.INSTANCE.compare(event3,event4));
+
+ Iterator<PartitionedEvent> it = set.iterator();
+ Assert.assertEquals(16.3,it.next().getData()[0]);
+ Assert.assertEquals(18.4,it.next().getData()[0]);
+ Assert.assertEquals(14.3,it.next().getData()[0]);
+ Assert.assertEquals(14.3,it.next().getData()[0]);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/topology/AlertTopologyTest.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/topology/AlertTopologyTest.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/topology/AlertTopologyTest.java
new file mode 100644
index 0000000..62427e0
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/topology/AlertTopologyTest.java
@@ -0,0 +1,138 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.alert.engine.topology;
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Properties;
+
+import org.apache.eagle.alert.engine.spout.CorrelationSpout;
+import org.apache.eagle.alert.engine.spout.CreateTopicUtils;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.log4j.Level;
+import org.apache.log4j.LogManager;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import backtype.storm.LocalCluster;
+import backtype.storm.topology.BoltDeclarer;
+import backtype.storm.topology.SpoutDeclarer;
+import backtype.storm.topology.TopologyBuilder;
+import backtype.storm.tuple.Fields;
+
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+
+@SuppressWarnings({"serial", "unused"})
+public class AlertTopologyTest implements Serializable{
+ private static final Logger LOG = LoggerFactory.getLogger(AlertTopologyTest.class);
+ char[] alphabets = {'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z'};
+ @Ignore
+ @Test
+ public void testMultipleTopics() throws Exception {
+ final String topoId = "myTopology";
+ int numGroupbyBolts = 2;
+ int numTotalGroupbyBolts = 3;
+ System.setProperty("eagle.correlation.numGroupbyBolts", String.valueOf(numGroupbyBolts));
+ System.setProperty("eagle.correlation.topologyName", topoId);
+ System.setProperty("eagle.correlation.mode", "local");
+ System.setProperty("eagle.correlation.zkHosts", "localhost:2181");
+ final String topicName1 = "testTopic3";
+ final String topicName2 = "testTopic4";
+ // ensure topic ready
+ LogManager.getLogger(CorrelationSpout.class).setLevel(Level.DEBUG);
+ Config config = ConfigFactory.load();
+
+ CreateTopicUtils.ensureTopicReady(System.getProperty("eagle.correlation.zkHosts"), topicName1);
+ CreateTopicUtils.ensureTopicReady(System.getProperty("eagle.correlation.zkHosts"), topicName2);
+
+ TopologyBuilder topoBuilder = new TopologyBuilder();
+
+ int numBolts = config.getInt("eagle.correlation.numGroupbyBolts");
+ CorrelationSpout spout = new CorrelationSpout(config, topoId, null, numBolts);
+ String spoutId = "correlation-spout";
+ SpoutDeclarer declarer = topoBuilder.setSpout(spoutId, spout);
+ for (int i = 0; i < numBolts; i++) {
+ TestBolt bolt = new TestBolt();
+ BoltDeclarer boltDecl = topoBuilder.setBolt("engineBolt" + i, bolt);
+ boltDecl.fieldsGrouping(spoutId, "stream_" + i, new Fields());
+ }
+
+ String topoName = config.getString("eagle.correlation.topologyName");
+ LOG.info("start topology in local mode");
+ LocalCluster cluster = new LocalCluster();
+ cluster.submitTopology(topoName, new HashMap<>(), topoBuilder.createTopology());
+
+ while(true) {
+ try {
+ Thread.sleep(1000);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ }
+
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ @Ignore
+ @Test
+ public void generateRandomStringsToKafka(){
+ String topic = "testTopic3";
+ int max = 1000;
+ Properties configMap = new Properties();
+ configMap.put("bootstrap.servers", "sandbox.hortonworks.com:6667");
+ configMap.put("metadata.broker.list", "sandbox.hortonworks.com:6667");
+ configMap.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
+ configMap.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
+ configMap.put("request.required.acks", "1");
+ configMap.put("key.deserializer","org.apache.kafka.common.serialization.StringDeserializer");
+ configMap.put("value.deserializer","org.apache.kafka.common.serialization.StringDeserializer");
+ KafkaProducer<String, Object> producer = new KafkaProducer<>(configMap);
+
+ int i = 0;
+ while(i++ < max) {
+ String randomString = generateRandomString();
+ System.out.println("sending string : " + randomString);
+ ProducerRecord record = new ProducerRecord(topic, randomString);
+ producer.send(record);
+ if(i % 10 == 0){
+ try {
+ Thread.sleep(10);
+ }catch(Exception ex){
+ }
+ }
+ }
+ producer.close();
+ }
+
+ private String generateRandomString(){
+ long count = Math.round(Math.random() * 10);
+ if(count == 0)
+ count = 1;
+ StringBuilder sb = new StringBuilder();
+ while(count-- > 0) {
+ int index = (int)(Math.floor(Math.random()*26));
+ sb.append(alphabets[index]);
+ }
+ return sb.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/topology/CoordinatorSpoutIntegrationTest.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/topology/CoordinatorSpoutIntegrationTest.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/topology/CoordinatorSpoutIntegrationTest.java
new file mode 100644
index 0000000..deca9b4
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/topology/CoordinatorSpoutIntegrationTest.java
@@ -0,0 +1,101 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.alert.engine.topology;
+
+import java.io.Serializable;
+import java.util.HashMap;
+
+import org.apache.eagle.alert.engine.spout.CorrelationSpout;
+import org.apache.eagle.alert.engine.spout.CreateTopicUtils;
+import org.apache.eagle.alert.utils.AlertConstants;
+import org.apache.eagle.alert.utils.StreamIdConversion;
+import org.apache.log4j.Level;
+import org.apache.log4j.LogManager;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import backtype.storm.LocalCluster;
+import backtype.storm.generated.StormTopology;
+import backtype.storm.topology.BoltDeclarer;
+import backtype.storm.topology.SpoutDeclarer;
+import backtype.storm.topology.TopologyBuilder;
+import backtype.storm.tuple.Fields;
+import backtype.storm.utils.Utils;
+
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+
+/**
+ * Since 4/28/16.
+ */
+@SuppressWarnings({"serial", "unused", "rawtypes"})
+public class CoordinatorSpoutIntegrationTest implements Serializable {
+ private static final Logger LOG = LoggerFactory.getLogger(CoordinatorSpoutIntegrationTest.class);
+ @Ignore // this test need zookeeper
+ @Test
+ public void testConfigNotify() throws Exception{
+ final String topoId = "myTopology";
+ int numGroupbyBolts = 2;
+ int numTotalGroupbyBolts = 3;
+ System.setProperty("correlation.serviceHost", "sandbox.hortonworks.com");
+ System.setProperty("correlation.servicePort", "58080");
+ System.setProperty("withMetadataChangeNotifyService.zkConfig.zkQuorum", "sandbox.hortonworks.com:2181");
+ System.setProperty("correlation.numGroupbyBolts", String.valueOf(numGroupbyBolts));
+ System.setProperty("correlation.topologyName", topoId);
+ System.setProperty("correlation.mode", "local");
+ System.setProperty("correlation.zkHosts", "sandbox.hortonworks.com:2181");
+ final String topicName1 = "testTopic3";
+ final String topicName2 = "testTopic4";
+ // ensure topic ready
+ LogManager.getLogger(CorrelationSpout.class).setLevel(Level.DEBUG);
+ Config config = ConfigFactory.load();
+
+ CreateTopicUtils.ensureTopicReady(System.getProperty("withMetadataChangeNotifyService.zkConfig.zkQuorum"), topicName1);
+ CreateTopicUtils.ensureTopicReady(System.getProperty("withMetadataChangeNotifyService.zkConfig.zkQuorum"), topicName2);
+
+ TopologyBuilder topoBuilder = new TopologyBuilder();
+
+ int numBolts = config.getInt("correlation.numGroupbyBolts");
+ String spoutId = "correlation-spout";
+ CorrelationSpout spout = new CorrelationSpout(config, topoId,
+ new MockMetadataChangeNotifyService(topoId, spoutId), numBolts);
+ SpoutDeclarer declarer = topoBuilder.setSpout(spoutId, spout);
+ declarer.setNumTasks(2);
+ for (int i = 0; i < numBolts; i++) {
+ TestBolt bolt = new TestBolt();
+ BoltDeclarer boltDecl = topoBuilder.setBolt("engineBolt" + i, bolt);
+ boltDecl.fieldsGrouping(spoutId,
+ StreamIdConversion.generateStreamIdBetween(AlertConstants.DEFAULT_SPOUT_NAME, AlertConstants.DEFAULT_ROUTERBOLT_NAME+i), new Fields());
+ }
+
+ String topoName = config.getString("correlation.topologyName");
+ LOG.info("start topology in local mode");
+ LocalCluster cluster = new LocalCluster();
+ StormTopology topology = topoBuilder.createTopology();
+ cluster.submitTopology(topoName, new HashMap(), topology);
+
+
+ Utils.sleep(Long.MAX_VALUE);
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/topology/CorrelationSpoutTest.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/topology/CorrelationSpoutTest.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/topology/CorrelationSpoutTest.java
new file mode 100644
index 0000000..6960537
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/topology/CorrelationSpoutTest.java
@@ -0,0 +1,184 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.alert.engine.topology;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.eagle.alert.coordination.model.Kafka2TupleMetadata;
+import org.apache.eagle.alert.coordination.model.SpoutSpec;
+import org.apache.eagle.alert.coordination.model.StreamRepartitionMetadata;
+import org.apache.eagle.alert.coordination.model.Tuple2StreamMetadata;
+import org.apache.eagle.alert.engine.coordinator.StreamDefinition;
+import org.apache.eagle.alert.engine.spout.CorrelationSpout;
+import org.junit.Assert;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.slf4j.Logger;
+
+import storm.kafka.KafkaSpoutWrapper;
+import backtype.storm.spout.SpoutOutputCollector;
+import backtype.storm.task.TopologyContext;
+
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+
+@Ignore
+public class CorrelationSpoutTest {
+ @SuppressWarnings("unused")
+ private static final String TEST_SPOUT_ID = "spout-id-1";
+ private static final Logger LOG = org.slf4j.LoggerFactory.getLogger(CorrelationSpoutTest.class);
+ private String dataSourceName = "ds-name";
+
+ @SuppressWarnings({ "serial", "rawtypes" })
+ @Test
+ public void testMetadataInjestion_emptyMetadata() throws Exception{
+ String topoId = "testMetadataInjection";
+ Config config = ConfigFactory.load();
+ AtomicBoolean validated = new AtomicBoolean(false);
+ CorrelationSpout spout = new CorrelationSpout(config, topoId, null, 1) {
+ @Override
+ protected KafkaSpoutWrapper createKafkaSpout(Map conf, TopologyContext context,
+ SpoutOutputCollector collector, String topic, String schemeClsName, SpoutSpec streamMetadatas, Map<String, StreamDefinition> sds)
+ throws Exception {
+ validated.set(true);
+ return null;
+ }
+ };
+ Kafka2TupleMetadata ds = new Kafka2TupleMetadata();
+ ds.setName("ds-name");
+ ds.setType("KAFKA");
+ ds.setProperties(new HashMap<String, String>());
+ ds.setTopic("name-of-topic1");
+ ds.setSchemeCls("PlainStringScheme");
+ ds.setCodec(new Tuple2StreamMetadata());
+ Map<String,Kafka2TupleMetadata> dsMap = new HashMap<String, Kafka2TupleMetadata>();
+ dsMap.put(ds.getName(), ds);
+
+ StreamRepartitionMetadata m1 = new StreamRepartitionMetadata(ds.getName(), "s1");
+
+ Map<String, List<StreamRepartitionMetadata>> dataSources = new HashMap<String, List<StreamRepartitionMetadata>>();
+ dataSources.put(ds.getName(), Arrays.asList(m1));
+
+ SpoutSpec newMetadata = new SpoutSpec(topoId, dataSources, null, dsMap);
+
+ spout.onReload(newMetadata, null);
+ Assert.assertTrue(validated.get());
+ }
+
+ @SuppressWarnings({ "serial", "rawtypes" })
+ @Test
+ public void testMetadataInjestion_oneNewTopic2Streams() throws Exception {
+ String topoId = "testMetadataInjection";
+ final String topicName = "testTopic";
+
+ Config config = ConfigFactory.load();
+ final AtomicBoolean verified = new AtomicBoolean(false);
+ CorrelationSpout spout = new CorrelationSpout(config, topoId, null, 1) {
+ @Override
+ protected KafkaSpoutWrapper createKafkaSpout(Map conf,
+ TopologyContext context,
+ SpoutOutputCollector collector,
+ String topic,
+ String topic2SchemeClsName,
+ SpoutSpec streamMetadatas,
+ Map<String, StreamDefinition> sds) {
+ Assert.assertEquals(1, streamMetadatas.getStreamRepartitionMetadataMap().size());
+ Assert.assertTrue(streamMetadatas.getStream("s1") != null);
+ Assert.assertTrue(streamMetadatas.getStream("s2") != null);
+ Assert.assertEquals(dataSourceName, streamMetadatas.getStream("s1").getTopicName());
+ Assert.assertEquals(dataSourceName, streamMetadatas.getStream("s2").getTopicName());
+ LOG.info("successfully verified new topic and streams");
+ verified.set(true);
+ return null;
+ }
+ };
+
+ Map<String, Kafka2TupleMetadata> dsMap = createDatasource(topicName, dataSourceName);
+
+ StreamRepartitionMetadata m1 = new StreamRepartitionMetadata(dataSourceName, "s1");
+ StreamRepartitionMetadata m2 = new StreamRepartitionMetadata(dataSourceName, "s2");
+ Map<String, List<StreamRepartitionMetadata>> dataSources = new HashMap<String, List<StreamRepartitionMetadata>>();
+ dataSources.put(dataSourceName, Arrays.asList(m1, m2));
+
+ SpoutSpec newMetadata = new SpoutSpec(topoId, dataSources, null, dsMap);
+ spout.onReload(newMetadata, null);
+ Assert.assertTrue(verified.get());
+ }
+
+ private Map<String, Kafka2TupleMetadata> createDatasource(final String topicName, final String dataSourceName) {
+ Kafka2TupleMetadata ds = new Kafka2TupleMetadata();
+
+ ds.setName(dataSourceName);
+ ds.setType("KAFKA");
+ ds.setProperties(new HashMap<String, String>());
+ ds.setTopic(topicName);
+ ds.setSchemeCls("PlainStringScheme");
+ ds.setCodec(new Tuple2StreamMetadata());
+ Map<String, Kafka2TupleMetadata> dsMap = new HashMap<String, Kafka2TupleMetadata>();
+ dsMap.put(ds.getName(), ds);
+ return dsMap;
+ }
+
+ @SuppressWarnings({ "serial", "rawtypes" })
+ @Test
+ public void testMetadataInjestion_deleteOneTopic() throws Exception{
+ String topoId = "testMetadataInjection";
+ final String topicName = "testTopic";
+ Config config = ConfigFactory.load();
+ final AtomicBoolean verified = new AtomicBoolean(false);
+ CorrelationSpout spout = new CorrelationSpout(config, topoId, null, 1) {
+ @Override
+ protected KafkaSpoutWrapper createKafkaSpout(Map conf, TopologyContext context, SpoutOutputCollector collector, final String topic,
+ String schemeClsName, SpoutSpec streamMetadatas,
+ Map<String, StreamDefinition> sds){
+ return new KafkaSpoutWrapper(null, null);
+ }
+ @Override
+ protected void removeKafkaSpout(KafkaSpoutWrapper wrapper){
+ LOG.info("successfully verified removed topic and streams");
+ verified.set(true);
+ }
+ };
+
+ Map<String, Kafka2TupleMetadata> dsMap = createDatasource(topicName, dataSourceName);
+
+ StreamRepartitionMetadata m1 = new StreamRepartitionMetadata(dataSourceName, "s1");
+
+ Map<String, List<StreamRepartitionMetadata>> streamMetadatas = new HashMap<String, List<StreamRepartitionMetadata>>();
+ streamMetadatas.put(dataSourceName, Arrays.asList(m1));
+
+ SpoutSpec cachedMetadata = new SpoutSpec(topoId, streamMetadatas, null, dsMap);
+ // add new topic
+ spout.onReload(cachedMetadata, null);
+ // delete new topic
+ try {
+ spout.onReload(new SpoutSpec(topoId, new HashMap<String, List<StreamRepartitionMetadata>>(), new HashMap<>(), new HashMap<String, Kafka2TupleMetadata>()),
+ null);
+ }catch(Exception ex){
+ LOG.error("error reloading spout metadata", ex);
+ throw ex;
+ }
+ Assert.assertTrue(verified.get());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/topology/FastWordCountTopology.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/topology/FastWordCountTopology.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/topology/FastWordCountTopology.java
new file mode 100644
index 0000000..4779fac
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/topology/FastWordCountTopology.java
@@ -0,0 +1,215 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.topology;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.ThreadLocalRandom;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import backtype.storm.Config;
+import backtype.storm.LocalCluster;
+import backtype.storm.generated.ClusterSummary;
+import backtype.storm.generated.ExecutorSummary;
+import backtype.storm.generated.KillOptions;
+import backtype.storm.generated.Nimbus;
+import backtype.storm.generated.SpoutStats;
+import backtype.storm.generated.TopologyInfo;
+import backtype.storm.generated.TopologySummary;
+import backtype.storm.metric.LoggingMetricsConsumer;
+import backtype.storm.spout.SpoutOutputCollector;
+import backtype.storm.task.TopologyContext;
+import backtype.storm.topology.BasicOutputCollector;
+import backtype.storm.topology.OutputFieldsDeclarer;
+import backtype.storm.topology.TopologyBuilder;
+import backtype.storm.topology.base.BaseBasicBolt;
+import backtype.storm.topology.base.BaseRichSpout;
+import backtype.storm.tuple.Fields;
+import backtype.storm.tuple.Tuple;
+import backtype.storm.tuple.Values;
+import backtype.storm.utils.Utils;
+
+/**
+ * WordCount but the spout does not stop, and the bolts are implemented in
+ * java. This can show how fast the word count can run.
+ */
+@SuppressWarnings("serial")
+public class FastWordCountTopology {
+ public static class FastRandomSentenceSpout extends BaseRichSpout {
+ private final static Logger LOG = LoggerFactory.getLogger(FastRandomSentenceSpout.class);
+ SpoutOutputCollector _collector;
+ Random _rand;
+ private static final String[] CHOICES = {
+ "marry had a little lamb whos fleese was white as snow",
+ "and every where that marry went the lamb was sure to go",
+ "one two three four five six seven eight nine ten",
+ "this is a test of the emergency broadcast system this is only a test",
+ "peter piper picked a peck of pickeled peppers"
+ };
+
+ @SuppressWarnings("rawtypes")
+ @Override
+ public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
+ _collector = collector;
+ _rand = ThreadLocalRandom.current();
+ }
+
+ @Override
+ public void nextTuple() {
+ String sentence = CHOICES[_rand.nextInt(CHOICES.length)];
+ _collector.emit(new Values(sentence), sentence);
+ LOG.debug("Emit tuple: {}, id:{}",new Values(sentence),sentence);
+ }
+
+ @Override
+ public void ack(Object id) {
+ //Ignored
+ }
+
+ @Override
+ public void fail(Object id) {
+ _collector.emit(new Values(id), id);
+ }
+
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+ declarer.declare(new Fields("sentence"));
+ }
+ }
+
+ public static class SplitSentence extends BaseBasicBolt {
+ @Override
+ public void execute(Tuple tuple, BasicOutputCollector collector) {
+ String sentence = tuple.getString(0);
+ for (String word: sentence.split("\\s+")) {
+ collector.emit(new Values(word, 1));
+ }
+ }
+
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+ declarer.declare(new Fields("word", "count"));
+ }
+ }
+
+
+ public static class WordCount extends BaseBasicBolt {
+ Map<String, Integer> counts = new HashMap<String, Integer>();
+
+ @Override
+ public void execute(Tuple tuple, BasicOutputCollector collector) {
+ String word = tuple.getString(0);
+ Integer count = counts.get(word);
+ if (count == null)
+ count = 0;
+ count++;
+ counts.put(word, count);
+ collector.emit(new Values(word, count));
+ }
+
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+ declarer.declare(new Fields("word", "count"));
+ }
+ }
+
+ public static void printMetrics(Nimbus.Client client, String name) throws Exception {
+ ClusterSummary summary = client.getClusterInfo();
+ String id = null;
+ for (TopologySummary ts: summary.get_topologies()) {
+ if (name.equals(ts.get_name())) {
+ id = ts.get_id();
+ }
+ }
+ if (id == null) {
+ throw new Exception("Could not find a topology named "+name);
+ }
+ TopologyInfo info = client.getTopologyInfo(id);
+ int uptime = info.get_uptime_secs();
+ long acked = 0;
+ long failed = 0;
+ double weightedAvgTotal = 0.0;
+ for (ExecutorSummary exec: info.get_executors()) {
+ if ("spout".equals(exec.get_component_id())) {
+ SpoutStats stats = exec.get_stats().get_specific().get_spout();
+ Map<String, Long> failedMap = stats.get_failed().get(":all-time");
+ Map<String, Long> ackedMap = stats.get_acked().get(":all-time");
+ Map<String, Double> avgLatMap = stats.get_complete_ms_avg().get(":all-time");
+ for (String key: ackedMap.keySet()) {
+ if (failedMap != null) {
+ Long tmp = failedMap.get(key);
+ if (tmp != null) {
+ failed += tmp;
+ }
+ }
+ long ackVal = ackedMap.get(key);
+ double latVal = avgLatMap.get(key) * ackVal;
+ acked += ackVal;
+ weightedAvgTotal += latVal;
+ }
+ }
+ }
+ double avgLatency = weightedAvgTotal/acked;
+ System.out.println("uptime: "+uptime+" acked: "+acked+" avgLatency: "+avgLatency+" acked/sec: "+(((double)acked)/uptime+" failed: "+failed));
+ }
+
+ public static void kill(Nimbus.Client client, String name) throws Exception {
+ KillOptions opts = new KillOptions();
+ opts.set_wait_secs(0);
+ client.killTopologyWithOpts(name, opts);
+ }
+
+ public static void main(String[] args) throws Exception {
+
+ TopologyBuilder builder = new TopologyBuilder();
+
+ builder.setSpout("spout", new FastRandomSentenceSpout(), 4);
+
+ builder.setBolt("split", new SplitSentence(), 4).shuffleGrouping("spout");
+ builder.setBolt("count", new WordCount(), 4).fieldsGrouping("split", new Fields("word"));
+
+ Config conf = new Config();
+ conf.registerMetricsConsumer(LoggingMetricsConsumer.class);
+
+ String name = "wc-test";
+ if (args != null && args.length > 0) {
+ name = args[0];
+ }
+
+// conf.setNumWorkers(1);
+
+ LocalCluster cluster = new LocalCluster();
+ cluster.submitTopology(name, conf, builder.createTopology());
+
+ Utils.sleep(Long.MAX_VALUE);
+//
+// Map clusterConf = Utils.readStormConfig();
+// clusterConf.putAll(Utils.readCommandLineOpts());
+// Nimbus.Client client = NimbusClient.getConfiguredClient(clusterConf).getClient();
+//
+// //Sleep for 5 mins
+// for (int i = 0; i < 10; i++) {
+// Thread.sleep(30 * 1000);
+// printMetrics(client, name);
+// }
+// kill(client, name);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/topology/MockMetadataChangeNotifyService.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/topology/MockMetadataChangeNotifyService.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/topology/MockMetadataChangeNotifyService.java
new file mode 100644
index 0000000..a8fb6d0
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/topology/MockMetadataChangeNotifyService.java
@@ -0,0 +1,111 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.alert.engine.topology;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.eagle.alert.coordination.model.AlertBoltSpec;
+import org.apache.eagle.alert.coordination.model.PublishSpec;
+import org.apache.eagle.alert.coordination.model.RouterSpec;
+import org.apache.eagle.alert.coordination.model.SpoutSpec;
+import org.apache.eagle.alert.engine.coordinator.MetadataType;
+import org.apache.eagle.alert.engine.coordinator.StreamDefinition;
+import org.apache.eagle.alert.engine.coordinator.impl.AbstractMetadataChangeNotifyService;
+import org.apache.eagle.alert.engine.utils.MetadataSerDeser;
+import org.codehaus.jackson.type.TypeReference;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.typesafe.config.Config;
+
+/**
+ * Since 5/4/16.
+ */
+@SuppressWarnings({"serial"})
+public class MockMetadataChangeNotifyService extends AbstractMetadataChangeNotifyService implements Runnable {
+ private final static Logger LOG = LoggerFactory.getLogger(MockMetadataChangeNotifyService.class);
+ @SuppressWarnings("unused")
+ private static final String[] topics = new String[]{"testTopic3", "testTopic4", "testTopic5"};
+ @SuppressWarnings("unused")
+ private String topologyName;
+ @SuppressWarnings("unused")
+ private String spoutId;
+ private Map<String, StreamDefinition> sds;
+
+ public MockMetadataChangeNotifyService(String topologyName, String spoutId){
+ this.topologyName = topologyName;
+ this.spoutId = spoutId;
+ }
+
+ @Override
+ public void init(Config config, MetadataType type) {
+ super.init(config, type);
+ this.sds = defineStreamDefinitions();
+ new Thread(this).start();
+ }
+
+ @Override
+ public void run() {
+ switch (type) {
+ case SPOUT:
+ notifySpout(Arrays.asList("testTopic3", "testTopic4"), Arrays.asList("testTopic5"));
+ break;
+ case STREAM_ROUTER_BOLT:
+ populateRouterMetadata();
+ break;
+ case ALERT_BOLT:
+ populateAlertBoltSpec();
+ break;
+ case ALERT_PUBLISH_BOLT:
+ notifyAlertPublishBolt();
+ break;
+ default:
+ LOG.error("that is not possible man!");
+ }
+ }
+
+ private Map<String, StreamDefinition> defineStreamDefinitions(){
+ Map<String, StreamDefinition> sds = MetadataSerDeser.deserialize(getClass().getResourceAsStream("/testStreamDefinitionsSpec.json"),
+ new TypeReference<Map<String, StreamDefinition>>() {});
+ return sds;
+ }
+
+ private void notifySpout(List<String> plainStringTopics, List<String> jsonStringTopics){
+ SpoutSpec newSpec = MetadataSerDeser.deserialize(getClass().getResourceAsStream("/testSpoutSpec.json"), SpoutSpec.class);
+ notifySpout(newSpec, sds);
+ }
+
+ private void populateRouterMetadata(){
+ RouterSpec boltSpec = MetadataSerDeser.deserialize(getClass().getResourceAsStream("/testStreamRouterBoltSpec.json"), RouterSpec.class);
+ notifyStreamRouterBolt(boltSpec, sds);
+ }
+
+ private void populateAlertBoltSpec(){
+ AlertBoltSpec spec = MetadataSerDeser.deserialize(getClass().getResourceAsStream("/testAlertBoltSpec.json"), AlertBoltSpec.class);
+ notifyAlertBolt(spec, sds);
+ }
+
+ private void notifyAlertPublishBolt(){
+ PublishSpec spec = MetadataSerDeser.deserialize(getClass().getResourceAsStream("/testPublishSpec.json"), PublishSpec.class);
+ notifyAlertPublishBolt(spec, sds);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/topology/SendData2KafkaTest.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/topology/SendData2KafkaTest.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/topology/SendData2KafkaTest.java
new file mode 100644
index 0000000..fa2f0a6
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/topology/SendData2KafkaTest.java
@@ -0,0 +1,103 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+package org.apache.eagle.alert.engine.topology;
+
+import java.io.FileWriter;
+import java.io.PrintWriter;
+import java.io.Serializable;
+import java.nio.charset.Charset;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.Properties;
+
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.common.serialization.StringSerializer;
+import org.junit.Ignore;
+import org.junit.Test;
+
+/**
+ * Created on 3/12/16.
+ */
+@SuppressWarnings({"serial", "unchecked", "rawtypes", "resource"})
+public class SendData2KafkaTest implements Serializable{
+ /**
+ *
+ * {"timestamp": 10000, "metric": "esErrorLogEvent", "instanceUuid": "vm-InstanceId1", "host":"test-host1", "type":"nova", "stack":"NullPointException-..........."}
+ {"timestamp": 10000, "metric": "instanceFailureLogEvent", "instanceUuid": "vm-InstanceId1", "message":"instance boot failure for user liasu!"}
+ *
+ */
+ @Test
+ @Ignore
+ public void testOutput() throws Exception {
+ String s1 = "{\"timestamp\": %d, \"metric\": \"esErrorLogEvent\", \"instanceUuid\": \"vm-InstanceId%d\", \"host\":\"test-host1\", \"type\":\"nova\", \"stack\":\"NullPointException-...........\"}";
+ String s2 = "{\"timestamp\": %d, \"metric\": \"instanceFailureLogEvent\", \"instanceUuid\": \"vm-InstanceId%d\", \"message\":\"instance boot failure for user liasu!\"}";
+
+ PrintWriter espw = new PrintWriter(new FileWriter("src/test/resources/es.log"));
+ PrintWriter ifpw = new PrintWriter(new FileWriter("src/test/resources/if.log"));
+
+ long base = System.currentTimeMillis();
+ long timestamp = 10000;
+
+ for (int i = 0; i < 10; i++) {
+
+ timestamp = base + i * 10000;
+ for (int j = 0; j < 10; j++) {
+ timestamp = timestamp + j * 1000;
+
+ espw.println(String.format(s1, timestamp, i));
+ }
+
+ ifpw.println(String.format(s2, timestamp, i));
+ }
+
+ espw.flush();
+ ifpw.flush();
+ }
+
+ @Test
+ @Ignore
+ public void sendKakfa() throws Exception {
+ List<String> eslogs = Files.readAllLines(Paths.get(SendData2KafkaTest.class.getResource("/es.log").toURI()), Charset.defaultCharset());
+ List<String> iflogs = Files.readAllLines(Paths.get(SendData2KafkaTest.class.getResource("/if.log").toURI()), Charset.defaultCharset());
+
+ Properties props = new Properties();
+ props.put("bootstrap.servers", "localhost:9092");
+ props.put("value.serializer", StringSerializer.class.getCanonicalName());
+ props.put("key.serializer", StringSerializer.class.getCanonicalName());
+
+ KafkaProducer producer = new KafkaProducer<>(props);
+ while (true) {
+ for (String s : eslogs) {
+ ProducerRecord<String, String> record = new ProducerRecord<>("nn_jmx_metric_sandbox", s);
+ producer.send(record);
+ }
+
+ for (String s : iflogs) {
+ ProducerRecord<String, String> record = new ProducerRecord<>("nn_jmx_metric_sandbox", s);
+ producer.send(record);
+ }
+
+ Thread.sleep(5000);
+ }
+
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/topology/TestBolt.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/topology/TestBolt.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/topology/TestBolt.java
new file mode 100644
index 0000000..493d30e
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/topology/TestBolt.java
@@ -0,0 +1,61 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.alert.engine.topology;
+
+
+import java.util.Map;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import backtype.storm.task.OutputCollector;
+import backtype.storm.task.TopologyContext;
+import backtype.storm.topology.OutputFieldsDeclarer;
+import backtype.storm.topology.base.BaseRichBolt;
+import backtype.storm.tuple.Tuple;
+
+/**
+ * Created by yonzhang on 4/7/16.
+ */
+@SuppressWarnings({"rawtypes", "serial"})
+public class TestBolt extends BaseRichBolt {
+ private static final Logger LOG = LoggerFactory.getLogger(TestBolt.class);
+ private OutputCollector collector;
+ private long count;
+ @Override
+ public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
+ this.collector = collector;
+ }
+
+ @Override
+ public void execute(Tuple input) {
+ LOG.info("data is coming: " + input);
+ count++;
+ if(count % 10 == 0){
+ LOG.info("count = " + count);
+ }
+ collector.ack(input);
+ }
+
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/topology/TestByteBuffer.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/topology/TestByteBuffer.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/topology/TestByteBuffer.java
new file mode 100644
index 0000000..666d167
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/topology/TestByteBuffer.java
@@ -0,0 +1,53 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.alert.engine.topology;
+
+import java.nio.ByteBuffer;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Storm 1.0.0 uses ByteBuffer, we need test it
+ */
+public class TestByteBuffer {
+ @Test
+ public void testBB(){
+ ByteBuffer bb = ByteBuffer.allocate(100);
+ bb.put((byte)12);
+ Assert.assertTrue(bb.hasArray());
+ bb.rewind();
+ Assert.assertEquals(12, bb.get());
+ }
+
+ @Test
+ public void testMultipleStrings() throws Exception{
+ ByteBuffer bb = ByteBuffer.allocate(100);
+ bb.put("abc".getBytes("UTF-8"));
+ bb.put("xyz".getBytes("UTF-8"));
+ bb.put("1234".getBytes("UTF-8"));
+ bb.rewind();
+ if (bb.hasArray()) {
+ int base = bb.arrayOffset();
+ String ret = new String(bb.array(), base + bb.position(), bb.remaining());
+ System.out.println("string is: " + ret);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/topology/TestMetadataSpecSerDeser.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/topology/TestMetadataSpecSerDeser.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/topology/TestMetadataSpecSerDeser.java
new file mode 100644
index 0000000..65b16d9
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/topology/TestMetadataSpecSerDeser.java
@@ -0,0 +1,266 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.alert.engine.topology;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+
+import org.apache.eagle.alert.coordination.model.AlertBoltSpec;
+import org.apache.eagle.alert.coordination.model.Kafka2TupleMetadata;
+import org.apache.eagle.alert.coordination.model.PolicyWorkerQueue;
+import org.apache.eagle.alert.coordination.model.RouterSpec;
+import org.apache.eagle.alert.coordination.model.SpoutSpec;
+import org.apache.eagle.alert.coordination.model.StreamRepartitionMetadata;
+import org.apache.eagle.alert.coordination.model.StreamRepartitionStrategy;
+import org.apache.eagle.alert.coordination.model.StreamRouterSpec;
+import org.apache.eagle.alert.coordination.model.Tuple2StreamMetadata;
+import org.apache.eagle.alert.coordination.model.WorkSlot;
+import org.apache.eagle.alert.engine.coordinator.PolicyDefinition;
+import org.apache.eagle.alert.engine.coordinator.StreamColumn;
+import org.apache.eagle.alert.engine.coordinator.StreamDefinition;
+import org.apache.eagle.alert.engine.coordinator.StreamPartition;
+import org.apache.eagle.alert.engine.coordinator.StreamSortSpec;
+import org.apache.eagle.alert.engine.evaluator.PolicyStreamHandlers;
+import org.apache.eagle.alert.engine.utils.MetadataSerDeser;
+import org.codehaus.jackson.type.TypeReference;
+import org.joda.time.Period;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Since 5/6/16.
+ */
+public class TestMetadataSpecSerDeser {
+ private String getStreamNameByTopic(String topic){
+ return topic + "Stream";
+ }
+
+ @Test
+ public void testStreamDefinitions(){
+ Map<String, StreamDefinition> sds = new HashMap<>();
+ List<String> topics = Arrays.asList("testTopic3", "testTopic4", "testTopic5");
+ for(String topic : topics) {
+ String streamId = getStreamNameByTopic(topic);
+ if (topic.equals("testTopic3") || topic.equals("testTopic4")) {
+ StreamDefinition schema = new StreamDefinition();
+ schema.setStreamId(streamId);
+ StreamColumn column = new StreamColumn();
+ column.setName("value");
+ column.setType(StreamColumn.Type.STRING);
+ schema.setColumns(Collections.singletonList(column));
+ sds.put(schema.getStreamId(), schema);
+ }else if(topic.equals("testTopic5")){
+ StreamDefinition schema = new StreamDefinition();
+ schema.setStreamId(streamId);
+ StreamColumn column = new StreamColumn();
+ column.setName("value");
+ column.setType(StreamColumn.Type.STRING);
+ schema.setColumns(Collections.singletonList(column));
+ sds.put(schema.getStreamId(), schema);
+ }
+ }
+
+ String json = MetadataSerDeser.serialize(sds);
+ System.out.println(json);
+
+ Map<String, StreamDefinition> deserializedSpec = MetadataSerDeser.deserialize(json, new TypeReference<Map<String, StreamDefinition>>(){});
+ Assert.assertEquals(3, deserializedSpec.size());
+ }
+
+ @SuppressWarnings("unused")
+ @Test
+ public void testSpoutSpec(){
+ String topologyName = "testTopology";
+ String spoutId = "alertEngineSpout";
+ List<String> plainStringTopics = Arrays.asList("testTopic3", "testTopic4");
+ List<String> jsonStringTopics = Arrays.asList("testTopic5");
+ Map<String, Kafka2TupleMetadata> kafka2TupleMetadataMap = new HashMap<>();
+ for(String topic : plainStringTopics) {
+ Kafka2TupleMetadata kafka2TupleMetadata = new Kafka2TupleMetadata();
+ kafka2TupleMetadata.setName(topic);
+ kafka2TupleMetadata.setTopic(topic);
+ kafka2TupleMetadata.setSchemeCls("org.apache.eagle.alert.engine.scheme.PlainStringScheme");
+ kafka2TupleMetadataMap.put(topic, kafka2TupleMetadata);
+ }
+ for(String topic : jsonStringTopics){
+ Kafka2TupleMetadata kafka2TupleMetadata = new Kafka2TupleMetadata();
+ kafka2TupleMetadata.setName(topic);
+ kafka2TupleMetadata.setTopic(topic);
+ kafka2TupleMetadata.setSchemeCls("org.apache.eagle.alert.engine.scheme.JsonScheme");
+ kafka2TupleMetadataMap.put(topic, kafka2TupleMetadata);
+ }
+
+ // construct Tuple2StreamMetadata
+ Map<String, Tuple2StreamMetadata> tuple2StreamMetadataMap = new HashMap<>();
+ for(String topic : plainStringTopics) {
+ String streamId = getStreamNameByTopic(topic);
+ Tuple2StreamMetadata tuple2StreamMetadata = new Tuple2StreamMetadata();
+ Set<String> activeStreamNames = new HashSet<>();
+ activeStreamNames.add(streamId);
+ tuple2StreamMetadata.setStreamNameSelectorCls("org.apache.eagle.alert.engine.scheme.PlainStringStreamNameSelector");
+ tuple2StreamMetadata.setStreamNameSelectorProp(new Properties());
+ tuple2StreamMetadata.getStreamNameSelectorProp().put("userProvidedStreamName", streamId);
+ tuple2StreamMetadata.setActiveStreamNames(activeStreamNames);
+ tuple2StreamMetadata.setTimestampColumn("timestamp");
+ tuple2StreamMetadataMap.put(topic, tuple2StreamMetadata);
+ }
+
+ for(String topic : jsonStringTopics) {
+ String streamId = getStreamNameByTopic(topic);
+ Tuple2StreamMetadata tuple2StreamMetadata = new Tuple2StreamMetadata();
+ Set<String> activeStreamNames = new HashSet<>();
+ activeStreamNames.add(streamId);
+ tuple2StreamMetadata.setStreamNameSelectorCls("org.apache.eagle.alert.engine.scheme.JsonStringStreamNameSelector");
+ tuple2StreamMetadata.setStreamNameSelectorProp(new Properties());
+ tuple2StreamMetadata.getStreamNameSelectorProp().put("userProvidedStreamName", streamId);
+ tuple2StreamMetadata.setActiveStreamNames(activeStreamNames);
+ tuple2StreamMetadata.setTimestampColumn("timestamp");
+ tuple2StreamMetadataMap.put(topic, tuple2StreamMetadata);
+ }
+
+ // construct StreamRepartitionMetadata
+ Map<String, List<StreamRepartitionMetadata>> streamRepartitionMetadataMap = new HashMap<>();
+ for(String topic : plainStringTopics) {
+ String streamId = getStreamNameByTopic(topic);
+ StreamRepartitionMetadata streamRepartitionMetadata = new StreamRepartitionMetadata(topic, "defaultStringStream");
+ StreamRepartitionStrategy gs = new StreamRepartitionStrategy();
+ // StreamPartition, groupby col1 for stream cpuUsageStream
+ StreamPartition sp = new StreamPartition();
+ sp.setStreamId(streamId);
+ sp.setColumns(Arrays.asList("value"));
+ sp.setType(StreamPartition.Type.GROUPBY);
+ StreamSortSpec sortSpec = new StreamSortSpec();
+ sortSpec.setWindowMargin(1000);
+ sortSpec.setWindowPeriod2(Period.seconds(10));
+ sp.setSortSpec(sortSpec);
+
+ gs.partition = sp;
+ gs.numTotalParticipatingRouterBolts = 1;
+ gs.startSequence = 0;
+ streamRepartitionMetadata.addGroupStrategy(gs);
+ streamRepartitionMetadataMap.put(topic, Arrays.asList(streamRepartitionMetadata));
+ }
+
+ for(String topic : jsonStringTopics) {
+ String streamId = getStreamNameByTopic(topic);
+ StreamRepartitionMetadata streamRepartitionMetadata = new StreamRepartitionMetadata(topic, "defaultStringStream");
+ StreamRepartitionStrategy gs = new StreamRepartitionStrategy();
+ // StreamPartition, groupby col1 for stream cpuUsageStream
+ StreamPartition sp = new StreamPartition();
+ sp.setStreamId(streamId);
+ sp.setColumns(Arrays.asList("value"));
+ sp.setType(StreamPartition.Type.GROUPBY);
+ StreamSortSpec sortSpec = new StreamSortSpec();
+ sortSpec.setWindowMargin(1000);
+ sortSpec.setWindowPeriod2(Period.seconds(10));
+ sp.setSortSpec(sortSpec);
+
+ gs.partition = sp;
+ gs.numTotalParticipatingRouterBolts = 1;
+ gs.startSequence = 0;
+ streamRepartitionMetadata.addGroupStrategy(gs);
+ streamRepartitionMetadataMap.put(topic, Arrays.asList(streamRepartitionMetadata));
+ }
+
+ SpoutSpec newSpec = new SpoutSpec(topologyName, streamRepartitionMetadataMap, tuple2StreamMetadataMap, kafka2TupleMetadataMap);
+ String json = MetadataSerDeser.serialize(newSpec);
+ System.out.println(json);
+
+ SpoutSpec deserializedSpec = MetadataSerDeser.deserialize(json, SpoutSpec.class);
+ Assert.assertNotNull(deserializedSpec);
+// Assert.assertEquals(spoutId, deserializedSpec.getSpoutId());
+ }
+
+ @Test
+ public void testRouterBoltSpec(){
+ List<String> topics = Arrays.asList("testTopic3", "testTopic4", "testTopic5");
+ RouterSpec boltSpec = new RouterSpec();
+ for(String topic : topics) {
+ String streamId = getStreamNameByTopic(topic);
+ // StreamPartition, groupby col1 for stream cpuUsageStream
+ StreamPartition sp = new StreamPartition();
+ sp.setStreamId(streamId);
+ sp.setColumns(Arrays.asList("value"));
+ sp.setType(StreamPartition.Type.GROUPBY);
+
+ StreamSortSpec sortSpec = new StreamSortSpec();
+ sortSpec.setWindowMargin(1000);
+ sortSpec.setWindowPeriod2(Period.seconds(10));
+ sp.setSortSpec(sortSpec);
+
+ // set StreamRouterSpec to have 2 WorkSlot
+ StreamRouterSpec routerSpec = new StreamRouterSpec();
+ routerSpec.setPartition(sp);
+ routerSpec.setStreamId(streamId);
+ PolicyWorkerQueue queue = new PolicyWorkerQueue();
+ queue.setPartition(sp);
+ queue.setWorkers(Arrays.asList(new WorkSlot("testTopology", "alertBolt0"), new WorkSlot("testTopology", "alertBolt1")));
+ routerSpec.setTargetQueue(Arrays.asList(queue));
+ boltSpec.addRouterSpec(routerSpec);
+ }
+
+ String json = MetadataSerDeser.serialize(boltSpec);
+ System.out.println(json);
+ RouterSpec deserializedSpec = MetadataSerDeser.deserialize(json, RouterSpec.class);
+ Assert.assertEquals(3, deserializedSpec.getRouterSpecs().size());
+ }
+
+ @Test
+ public void testAlertBoltSpec(){
+ String topologyName = "testTopology";
+ AlertBoltSpec spec = new AlertBoltSpec();
+ List<String> topics = Arrays.asList("testTopic3", "testTopic4", "testTopic5");
+ for(String topic : topics) {
+ String streamId = getStreamNameByTopic(topic);
+
+ // construct StreamPartition
+ StreamPartition sp = new StreamPartition();
+ sp.setColumns(Collections.singletonList("value"));
+ sp.setStreamId(streamId);
+ sp.setType(StreamPartition.Type.GROUPBY);
+ StreamSortSpec sortSpec = new StreamSortSpec();
+ sortSpec.setWindowMargin(1000);
+ sortSpec.setWindowPeriod2(Period.seconds(10));
+ sp.setSortSpec(sortSpec);
+
+ spec.setVersion("version1");
+ spec.setTopologyName(topologyName);
+ PolicyDefinition pd = new PolicyDefinition();
+ pd.setName("policy1");
+ pd.setPartitionSpec(Collections.singletonList(sp));
+ pd.setOutputStreams(Collections.singletonList("testAlertStream"));
+ pd.setInputStreams(Collections.singletonList(streamId));
+ pd.setDefinition(new PolicyDefinition.Definition());
+ pd.getDefinition().type = PolicyStreamHandlers.SIDDHI_ENGINE;
+ pd.getDefinition().value = String.format("from %s[value=='xyz'] select value insert into testAlertStream;", streamId);
+ spec.addBoltPolicy("alertBolt0", pd.getName());
+ }
+ String json = MetadataSerDeser.serialize(spec);
+ System.out.println(json);
+ AlertBoltSpec deserializedSpec = MetadataSerDeser.deserialize(json, AlertBoltSpec.class);
+ Assert.assertEquals(topologyName, deserializedSpec.getTopologyName());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/topology/TestStormCustomGroupingRouting.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/topology/TestStormCustomGroupingRouting.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/topology/TestStormCustomGroupingRouting.java
new file mode 100644
index 0000000..0466ed3
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/topology/TestStormCustomGroupingRouting.java
@@ -0,0 +1,144 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.alert.engine.topology;
+
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.junit.Ignore;
+import org.junit.Test;
+
+import backtype.storm.Config;
+import backtype.storm.LocalCluster;
+import backtype.storm.generated.GlobalStreamId;
+import backtype.storm.grouping.CustomStreamGrouping;
+import backtype.storm.spout.SpoutOutputCollector;
+import backtype.storm.task.OutputCollector;
+import backtype.storm.task.TopologyContext;
+import backtype.storm.task.WorkerTopologyContext;
+import backtype.storm.topology.OutputFieldsDeclarer;
+import backtype.storm.topology.TopologyBuilder;
+import backtype.storm.topology.base.BaseRichBolt;
+import backtype.storm.topology.base.BaseRichSpout;
+import backtype.storm.tuple.Fields;
+import backtype.storm.tuple.Tuple;
+
+/**
+ * Since 4/29/16.
+ */
+@SuppressWarnings({ "rawtypes", "serial" })
+public class TestStormCustomGroupingRouting implements Serializable {
+ @Ignore
+ @Test
+ public void testRoutingByCustomGrouping() throws Exception{
+ Config conf = new Config();
+ conf.setNumWorkers(2); // use two worker processes
+ TopologyBuilder topologyBuilder = new TopologyBuilder();
+ topologyBuilder.setSpout("blue-spout", new BlueSpout()); // parallelism hint
+
+ topologyBuilder.setBolt("green-bolt-1", new GreenBolt(0)).setNumTasks(2)
+ .customGrouping("blue-spout", new CustomStreamGrouping(){
+ int count = 0;
+ List<Integer> targetTask;
+ @Override
+ public void prepare(WorkerTopologyContext context, GlobalStreamId stream, List<Integer> targetTasks) {
+ this.targetTask = targetTasks;
+ }
+
+ @Override
+ public List<Integer> chooseTasks(int taskId, List<Object> values) {
+ if(count % 2 == 0) {
+ count++;
+ return Arrays.asList(targetTask.get(0));
+ }else{
+ count++;
+ return Arrays.asList(targetTask.get(1));
+ }
+ }
+ });
+
+ LocalCluster cluster = new LocalCluster();
+ cluster.submitTopology("mytopology", new HashMap(), topologyBuilder.createTopology());
+
+ while(true) {
+ try {
+ Thread.sleep(1000);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ }
+
+ private static class BlueSpout extends BaseRichSpout {
+ int count = 0;
+ private SpoutOutputCollector collector;
+ public BlueSpout(){
+ }
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+ declarer.declare(new Fields("a"));
+ }
+
+ @Override
+ public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
+ this.collector = collector;
+ }
+
+ @Override
+ public void nextTuple() {
+ if(count % 2 == 0) {
+ this.collector.emit(Arrays.asList("testdata" + count));
+ count++;
+ }else{
+ this.collector.emit(Arrays.asList("testdata" + count));
+ count++;
+ }
+ try{
+ Thread.sleep(10000);
+ }catch(Exception ex){
+
+ }
+ }
+ }
+
+ private static class GreenBolt extends BaseRichBolt {
+ private int id;
+ public GreenBolt(int id){
+ this.id = id;
+ }
+ @Override
+ public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
+ }
+
+ @Override
+ public void execute(Tuple input) {
+ System.out.println("bolt " + id + " received data " + input.getString(0));
+ System.out.flush();
+ }
+
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+ declarer.declare(new Fields("a"));
+ }
+ }
+}
[17/19] incubator-eagle git commit: EAGLE-324: Init branch-v0.5
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/engine/model/StreamEvent.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/engine/model/StreamEvent.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/engine/model/StreamEvent.java
new file mode 100644
index 0000000..3e4e1df
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/engine/model/StreamEvent.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.model;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+import org.apache.eagle.alert.engine.coordinator.StreamDefinition;
+import org.apache.eagle.alert.utils.DateTimeUtil;
+
+/**
+ * @since Apr 5, 2016
+ *
+ */
+public class StreamEvent implements Serializable {
+ private static final long serialVersionUID = 2765116509856609763L;
+
+ private String streamId;
+ private Object[] data;
+ private long timestamp;
+
+ public StreamEvent(){}
+
+ public StreamEvent(String streamId,long timestamp,Object[] data){
+ this.setStreamId(streamId);
+ this.setTimestamp(timestamp);
+ this.setData(data);
+ }
+
+ public String getStreamId() {
+ return streamId;
+ }
+
+ public void setStreamId(String streamId) {
+ this.streamId = streamId;
+ }
+
+ public Object[] getData() {
+ return data;
+ }
+
+ public void setData(Object[] data) {
+ this.data = data;
+ }
+
+ public long getTimestamp() {
+ return timestamp;
+ }
+
+ public void setTimestamp(long timestamp) {
+ this.timestamp = timestamp;
+ }
+
+ @Override
+ public int hashCode() {
+ return new HashCodeBuilder().append(streamId).append(timestamp).append(data).build();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if(obj == this) return true;
+ if(obj instanceof StreamEvent){
+ StreamEvent another = (StreamEvent) obj;
+ return Objects.equals(this.streamId,another.streamId) && this.timestamp == another.timestamp && Arrays.deepEquals(this.data,another.data);
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ List<String> dataStrings = new ArrayList<>();
+ if(this.getData() != null) {
+ for (Object obj : this.getData()) {
+ if (obj != null) {
+ dataStrings.add(obj.toString());
+ } else {
+ dataStrings.add(null);
+ }
+ }
+ }
+ return String.format("StreamEvent[stream=%S,timestamp=%s,data=[%s]]",this.getStreamId(), DateTimeUtil.millisecondsToHumanDateWithMilliseconds(this.getTimestamp()), StringUtils.join(dataStrings,","));
+ }
+
+ public static StreamEventBuilder Builder(){
+ return new StreamEventBuilder();
+ }
+
+ /**
+ * @return cloned new event object
+ */
+ public StreamEvent copy(){
+ StreamEvent newEvent = new StreamEvent();
+ newEvent.setTimestamp(this.getTimestamp());
+ newEvent.setData(this.getData());
+ newEvent.setStreamId(this.getStreamId());
+ return newEvent;
+ }
+
+ public void copyFrom(StreamEvent event){
+ this.setTimestamp(event.getTimestamp());
+ this.setData(event.getData());
+ this.setStreamId(event.getStreamId());
+ }
+
+ /**
+ * @param column
+ * @param streamDefinition
+ * @return
+ */
+ public Object[] getData(StreamDefinition streamDefinition,List<String> column) {
+ ArrayList<Object> result = new ArrayList<>(column.size());
+ for (String colName : column) {
+ result.add(this.getData()[streamDefinition.getColumnIndex(colName)]);
+ }
+ return result.toArray();
+ }
+
+ public Object[] getData(StreamDefinition streamDefinition,String ... column) {
+ ArrayList<Object> result = new ArrayList<>(column.length);
+ for (String colName : column) {
+ result.add(this.getData()[streamDefinition.getColumnIndex(colName)]);
+ }
+ return result.toArray();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/engine/model/StreamEventBuilder.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/engine/model/StreamEventBuilder.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/engine/model/StreamEventBuilder.java
new file mode 100644
index 0000000..7e351bb
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/engine/model/StreamEventBuilder.java
@@ -0,0 +1,72 @@
+package org.apache.eagle.alert.engine.model;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.eagle.alert.engine.coordinator.StreamColumn;
+import org.apache.eagle.alert.engine.coordinator.StreamDefinition;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class StreamEventBuilder{
+ private final static Logger LOG = LoggerFactory.getLogger(StreamEventBuilder.class);
+
+ private StreamEvent instance;
+ private StreamDefinition streamDefinition;
+ public StreamEventBuilder(){
+ instance = new StreamEvent();
+ }
+
+ public StreamEventBuilder schema(StreamDefinition streamDefinition){
+ this.streamDefinition = streamDefinition;
+ if(instance.getStreamId() == null) instance.setStreamId(streamDefinition.getStreamId());
+ return this;
+ }
+
+ public StreamEventBuilder streamId(String streamId){
+ instance.setStreamId(streamId);
+ return this;
+ }
+
+ public StreamEventBuilder attributes(Map<String,Object> data, StreamDefinition streamDefinition){
+ this.schema(streamDefinition);
+ List<StreamColumn> columnList = streamDefinition.getColumns();
+ if(columnList!=null && columnList.size() > 0){
+ List<Object> values = new ArrayList<>(columnList.size());
+ for (StreamColumn column : columnList) {
+ values.add(data.getOrDefault(column.getName(),column.getDefaultValue()));
+ }
+ instance.setData(values.toArray());
+ } else if(LOG.isDebugEnabled()){
+ LOG.warn("All data [{}] are ignored as no columns defined in schema {}",data,streamDefinition);
+ }
+ return this;
+ }
+
+ public StreamEventBuilder attributes(Map<String,Object> data){
+ return attributes(data,this.streamDefinition);
+ }
+
+ public StreamEventBuilder attributes(Object ... data){
+ instance.setData(data);
+ return this;
+ }
+
+ public StreamEventBuilder timestamep(long timestamp){
+ instance.setTimestamp(timestamp);
+ return this;
+ }
+
+ public StreamEvent build(){
+ if(instance.getStreamId() == null){
+ throw new IllegalArgumentException("streamId is null of event: " + instance);
+ }
+ return instance;
+ }
+
+ public StreamEventBuilder copyFrom(StreamEvent event) {
+ this.instance.copyFrom(event);
+ return this;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/metric/IMetricSystem.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/metric/IMetricSystem.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/metric/IMetricSystem.java
new file mode 100644
index 0000000..06a99f4
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/metric/IMetricSystem.java
@@ -0,0 +1,64 @@
+package org.apache.eagle.alert.metric;
+
+import java.util.Map;
+
+import org.apache.eagle.alert.metric.sink.MetricSink;
+import org.apache.eagle.alert.metric.source.MetricSource;
+
+import com.codahale.metrics.MetricRegistry;
+import com.typesafe.config.Config;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+public interface IMetricSystem {
+
+ /**
+ * Initialize
+ */
+ void start();
+
+ /**
+ * Schedule reporter
+ */
+ void schedule();
+
+ /**
+ * Close and stop all resources and services
+ */
+ void stop();
+
+ /**
+ * Manual report metric
+ */
+ void report();
+
+ /**
+ *
+ * @param sink metric sink
+ */
+ void register(MetricSink sink,Config config);
+
+ /**
+ *
+ * @param source metric source
+ */
+ void register(MetricSource source);
+
+ void tags(Map<String,Object> metricTags);
+
+ MetricRegistry registry();
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/metric/MetricSystem.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/metric/MetricSystem.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/metric/MetricSystem.java
new file mode 100644
index 0000000..b91c606
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/metric/MetricSystem.java
@@ -0,0 +1,116 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.metric;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.eagle.alert.metric.sink.MetricSink;
+import org.apache.eagle.alert.metric.sink.MetricSinkRepository;
+import org.apache.eagle.alert.metric.source.MetricSource;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.codahale.metrics.MetricRegistry;
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+
+public class MetricSystem implements IMetricSystem {
+ private final Config config;
+ private Map<MetricSink,Config> sinks = new HashMap<>();
+// private Map<String,MetricSource> sources = new HashMap<>();
+ private MetricRegistry registry = new MetricRegistry();
+ private boolean running;
+ private boolean initialized;
+ private final static Logger LOG = LoggerFactory.getLogger(MetricSystem.class);
+ private final Map<String, Object> metricTags = new HashMap<>();
+
+ public MetricSystem(Config config){
+ this.config = config;
+ }
+
+ public static MetricSystem load(Config config){
+ MetricSystem instance = new MetricSystem(config);
+ instance.loadFromConfig();
+ return instance;
+ }
+
+ @Override
+ public void tags(Map<String,Object> metricTags){
+ this.metricTags.putAll(metricTags);
+ }
+
+ @Override
+ public void start() {
+ if(initialized)
+ throw new IllegalStateException("Attempting to initialize a MetricsSystem that is already intialized");
+ sinks.forEach((sink,conf) -> sink.prepare(conf.withValue("tags",ConfigFactory.parseMap(metricTags).root()),registry));
+ initialized = true;
+ }
+
+ @Override
+ public void schedule() {
+ if(running){
+ throw new IllegalStateException("Attempting to start a MetricsSystem that is already running");
+ }
+
+ sinks.keySet().forEach((sink)->sink.start(5, TimeUnit.SECONDS));
+ running = true;
+ }
+
+ public void loadFromConfig(){
+ loadSinksFromConfig();
+ }
+
+ private void loadSinksFromConfig(){
+ Config sinkCls = config.hasPath("metric.sink") ? config.getConfig("metric.sink") : null;
+ if(sinkCls == null){
+ // do nothing
+ }else{
+ for(String sinkType:sinkCls.root().unwrapped().keySet()){
+ register(MetricSinkRepository.createSink(sinkType),config.getConfig("metric.sink."+sinkType));
+ }
+ }
+ }
+
+ @Override
+ public void stop() {
+ sinks.keySet().forEach(MetricSink::stop);
+ }
+
+ @Override
+ public void report() {
+ sinks.keySet().forEach(MetricSink::report);
+ }
+
+ @Override
+ public void register(MetricSink sink,Config config) {
+ LOG.debug("Register {}",sink);
+ sinks.put(sink,config);
+ }
+
+ @Override
+ public void register(MetricSource source) {
+ registry().registerAll(source.registry());
+ }
+
+ @Override
+ public MetricRegistry registry() {
+ return registry;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/metric/entity/MetricEvent.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/metric/entity/MetricEvent.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/metric/entity/MetricEvent.java
new file mode 100644
index 0000000..b5e6c63
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/metric/entity/MetricEvent.java
@@ -0,0 +1,111 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.metric.entity;
+
+import java.util.Map;
+import java.util.TreeMap;
+
+import org.apache.eagle.alert.utils.DateTimeUtil;
+
+import com.codahale.metrics.Counter;
+import com.codahale.metrics.Gauge;
+import com.codahale.metrics.Histogram;
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.Snapshot;
+import com.codahale.metrics.Timer;
+
+public class MetricEvent extends TreeMap<String,Object>{
+
+ private static final long serialVersionUID = 6862373651636342744L;
+
+ public static Builder of(String name){
+ return new Builder(name);
+ }
+
+ /**
+ * TODO: Refactor according to ConsoleReporter
+ */
+ public static class Builder{
+ private final String name;
+ private MetricEvent instance;
+ public Builder(String name){
+ this.instance = new MetricEvent();
+ this.name = name;
+ }
+
+ public Builder from(Counter value) {
+// this.instance.put("type","counter");
+ this.instance.put("count",value.getCount());
+ return this;
+ }
+
+ public MetricEvent build(){
+ this.instance.put("name",name);
+ if(!this.instance.containsKey("timestamp")){
+ this.instance.put("timestamp", DateTimeUtil.getCurrentTimestamp());
+ }
+ return this.instance;
+ }
+
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ public Builder from(Gauge gauge) {
+ Object value = gauge.getValue();
+ if( value instanceof Map){
+ Map<? extends String, ?> map = (Map<? extends String, ?>) value;
+ this.instance.putAll(map);
+ } else {
+ this.instance.put("value", value);
+ }
+ return this;
+ }
+
+ public Builder from(Histogram value) {
+ this.instance.put("count",value.getCount());
+ Snapshot snapshot = value.getSnapshot();
+ this.instance.put("min", snapshot.getMin());
+ this.instance.put("max", snapshot.getMax());
+ this.instance.put("mean", snapshot.getMean());
+ this.instance.put("stddev", snapshot.getStdDev());
+ this.instance.put("median", snapshot.getMedian());
+ this.instance.put("75thPercentile", snapshot.get75thPercentile());
+ this.instance.put("95thPercentile", snapshot.get95thPercentile());
+ this.instance.put("98thPercentile", snapshot.get98thPercentile());
+ this.instance.put("99thPercentile", snapshot.get99thPercentile());
+ this.instance.put("999thPercentile", snapshot.get999thPercentile());
+ return this;
+ }
+
+ public Builder from(Meter value) {
+ this.instance.put("value",value.getCount());
+ this.instance.put("15MinRate",value.getFifteenMinuteRate());
+ this.instance.put("5MinRate",value.getFiveMinuteRate());
+ this.instance.put("mean",value.getMeanRate());
+ this.instance.put("1MinRate",value.getOneMinuteRate());
+ return this;
+ }
+
+ public Builder from(Timer value) {
+// this.instance.put("type","timer");
+ this.instance.put("value",value.getCount());
+ this.instance.put("15MinRate",value.getFifteenMinuteRate());
+ this.instance.put("5MinRate",value.getFiveMinuteRate());
+ this.instance.put("mean",value.getMeanRate());
+ this.instance.put("1MinRate",value.getOneMinuteRate());
+ return this;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/metric/reporter/KafkaReporter.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/metric/reporter/KafkaReporter.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/metric/reporter/KafkaReporter.java
new file mode 100644
index 0000000..0d0f8b3
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/metric/reporter/KafkaReporter.java
@@ -0,0 +1,197 @@
+package org.apache.eagle.alert.metric.reporter;
+
+import java.util.Map;
+import java.util.Properties;
+import java.util.SortedMap;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import org.apache.eagle.alert.metric.entity.MetricEvent;
+import org.apache.eagle.alert.utils.ByteUtils;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.Producer;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.codahale.metrics.ConsoleReporter;
+import com.codahale.metrics.Counter;
+import com.codahale.metrics.Gauge;
+import com.codahale.metrics.Histogram;
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.MetricFilter;
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.ScheduledReporter;
+import com.codahale.metrics.Timer;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.Preconditions;
+import com.typesafe.config.Config;
+
+public class KafkaReporter extends ScheduledReporter {
+ private final static Logger LOG = LoggerFactory.getLogger(KafkaReporter.class);
+ private final String topic;
+ private final Properties properties;
+ private final Producer<byte[], String> producer;
+ private final Map<String, Object> additionalFields;
+
+ protected KafkaReporter(MetricRegistry registry, MetricFilter filter, TimeUnit rateUnit, TimeUnit durationUnit, String topic, Properties config, Map<String, Object> additionalFields) {
+ super(registry, "kafka-reporter", filter, rateUnit, durationUnit);
+ this.topic = topic;
+ this.properties = new Properties();
+ Preconditions.checkNotNull(topic,"topic should not be null");
+// properties.put("bootstrap.servers", brokerList);
+// properties.put("metadata.broker.list", brokerList);
+ properties.put("key.serializer", "org.apache.kafka.common.serialization.ByteArraySerializer");
+ properties.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
+ properties.put("request.required.acks", "1");
+ properties.put("key.deserializer","org.apache.kafka.common.serialization.ByteArraySerializer");
+ properties.put("value.deserializer","org.apache.kafka.common.serialization.StringDeserializer");
+ if(config != null) {
+ LOG.info(config.toString());
+ properties.putAll(config);
+ }
+ this.additionalFields = additionalFields;
+ this.producer = new KafkaProducer<>(properties);
+ LOG.info("Initialized kafka-reporter");
+ }
+
+ @SuppressWarnings("rawtypes")
+ @Override
+ public void report(SortedMap<String, Gauge> gauges, SortedMap<String, Counter> counters, SortedMap<String, Histogram> histograms, SortedMap<String, Meter> meters, SortedMap<String, Timer> timers) {
+ for(SortedMap.Entry<String, Gauge> entry:gauges.entrySet()){
+ onMetricEvent(MetricEvent.of(entry.getKey()).from(entry.getValue()).build());
+ }
+ for(SortedMap.Entry<String, Counter> entry:counters.entrySet()){
+ onMetricEvent(MetricEvent.of(entry.getKey()).from(entry.getValue()).build());
+ }
+ for(SortedMap.Entry<String, Histogram> entry:histograms.entrySet()){
+ onMetricEvent(MetricEvent.of(entry.getKey()).from(entry.getValue()).build());
+ }
+ for(SortedMap.Entry<String, Meter> entry:meters.entrySet()){
+ onMetricEvent(MetricEvent.of(entry.getKey()).from(entry.getValue()).build());
+ }
+ for(SortedMap.Entry<String, Timer> entry:timers.entrySet()){
+ onMetricEvent(MetricEvent.of(entry.getKey()).from(entry.getValue()).build());
+ }
+ }
+
+ private final static ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
+ private void onMetricEvent(MetricEvent event){
+ try {
+ if(additionalFields!=null){
+ event.putAll(additionalFields);
+ }
+ // TODO: Support configurable partition key
+ byte[] key = ByteUtils.intToBytes(event.hashCode());
+ ProducerRecord<byte[],String> record = new ProducerRecord<>(topic, key, OBJECT_MAPPER.writeValueAsString(event));
+ // TODO: Support configuration timeout
+ this.producer.send(record).get(5,TimeUnit.SECONDS);
+ } catch (JsonProcessingException e) {
+ LOG.error("Failed to serialize {} as json",event,e);
+ } catch (InterruptedException | ExecutionException | TimeoutException e) {
+ LOG.error("Failed to produce message to topic {}",topic,e);
+ }
+ }
+
+ @Override
+ public void stop() {
+ this.producer.close();
+ super.stop();
+ }
+
+ @Override
+ public void close() {
+ this.producer.close();
+ super.close();
+ }
+
+ public static Builder forRegistry(MetricRegistry registry){
+ return new Builder(registry);
+ }
+
+ public static class Builder{
+ private final MetricRegistry registry;
+ private TimeUnit rateUnit;
+ private TimeUnit durationUnit;
+ private MetricFilter filter;
+ private String topic;
+ private Properties properties;
+ private Map<String, Object> additionalFields;
+
+ private Builder(MetricRegistry registry) {
+ this.registry = registry;
+ this.rateUnit = TimeUnit.SECONDS;
+ this.durationUnit = TimeUnit.MILLISECONDS;
+ this.filter = MetricFilter.ALL;
+ }
+
+ /**
+ * Convert rates to the given time unit.
+ *
+ * @param rateUnit a unit of time
+ * @return {@code this}
+ */
+ public Builder convertRatesTo(TimeUnit rateUnit) {
+ this.rateUnit = rateUnit;
+ return this;
+ }
+
+ /**
+ * Convert durations to the given time unit.
+ *
+ * @param durationUnit a unit of time
+ * @return {@code this}
+ */
+ public Builder convertDurationsTo(TimeUnit durationUnit) {
+ this.durationUnit = durationUnit;
+ return this;
+ }
+
+ /**
+ * Only report metrics which match the given filter.
+ *
+ * @param filter a {@link MetricFilter}
+ * @return {@code this}
+ */
+ public Builder filter(MetricFilter filter) {
+ this.filter = filter;
+ return this;
+ }
+
+ public Builder topic(String topic){
+ this.topic = topic;
+ return this;
+ }
+
+ public Builder config(Properties properties){
+ this.properties = properties;
+ return this;
+ }
+
+ /**
+ * Builds a {@link ConsoleReporter} with the given properties.
+ *
+ * @return a {@link ConsoleReporter}
+ */
+ public KafkaReporter build() {
+ if(topic == null && properties!=null) topic = properties.getProperty("topic");
+ return new KafkaReporter(registry,filter,rateUnit,durationUnit,topic,properties,additionalFields);
+ }
+
+ @SuppressWarnings("serial")
+ public Builder config(Config config) {
+ this.config(new Properties(){{
+ putAll(config.root().unwrapped());
+ }});
+ return this;
+ }
+
+ public Builder addFields(Map<String, Object> tags) {
+ this.additionalFields = tags;
+ return this;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/metric/sink/ConsoleSink.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/metric/sink/ConsoleSink.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/metric/sink/ConsoleSink.java
new file mode 100644
index 0000000..fd6cc41
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/metric/sink/ConsoleSink.java
@@ -0,0 +1,47 @@
+package org.apache.eagle.alert.metric.sink;
+
+import java.util.concurrent.TimeUnit;
+
+import com.codahale.metrics.ConsoleReporter;
+import com.codahale.metrics.MetricRegistry;
+import com.typesafe.config.Config;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+public class ConsoleSink implements MetricSink {
+ private ConsoleReporter reporter;
+ @Override
+ public void prepare(Config config, MetricRegistry registry) {
+ reporter = ConsoleReporter.forRegistry(registry).build();
+ }
+
+ @Override
+ public void start(long period,TimeUnit unit) {
+ reporter.start(period, unit);
+ }
+
+ @Override
+ public void stop() {
+ reporter.stop();
+ reporter.close();
+ }
+
+ @Override
+ public void report() {
+ reporter.report();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/metric/sink/ElasticSearchSink.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/metric/sink/ElasticSearchSink.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/metric/sink/ElasticSearchSink.java
new file mode 100644
index 0000000..7e30b82
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/metric/sink/ElasticSearchSink.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.metric.sink;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import org.elasticsearch.metrics.ElasticsearchReporter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.codahale.metrics.MetricRegistry;
+import com.typesafe.config.Config;
+
+public class ElasticSearchSink implements MetricSink {
+ private ElasticsearchReporter reporter = null;
+ private final static Logger LOG = LoggerFactory.getLogger(ElasticSearchSink.class);
+
+ @Override
+ public void prepare(Config config, MetricRegistry registry) {
+ LOG.debug("Preparing elasticsearch-sink");
+ try {
+ ElasticsearchReporter.Builder builder = ElasticsearchReporter.forRegistry(registry);
+ if(config.hasPath("hosts")){
+ List<String> hosts = config.getStringList("hosts");
+ builder.hosts(hosts.toArray(new String[hosts.size()]));
+ }
+ if(config.hasPath("index")){
+ builder.index(config.getString("index"));
+ }
+ builder.indexDateFormat("yyyy-MM-dd");
+ builder.timestampFieldname(config.hasPath("timestampField")?config.getString("timestampField"):"@timestamp");
+
+ if(config.hasPath("tags")) {
+ builder.additionalFields(config.getConfig("tags").root().unwrapped());
+ }
+
+ reporter = builder.build();
+ } catch (IOException e) {
+ LOG.error(e.getMessage(),e);
+ }
+ }
+
+ @Override
+ public void start(long period, TimeUnit unit) {
+ reporter.start(period, TimeUnit.SECONDS);
+ }
+
+ @Override
+ public void stop() {
+ reporter.stop();
+ reporter.close();
+ }
+
+ @Override
+ public void report() {
+ reporter.report();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/metric/sink/JmxSink.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/metric/sink/JmxSink.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/metric/sink/JmxSink.java
new file mode 100644
index 0000000..fddaf19
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/metric/sink/JmxSink.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.metric.sink;
+
+import java.util.concurrent.TimeUnit;
+
+import com.codahale.metrics.JmxReporter;
+import com.codahale.metrics.MetricRegistry;
+import com.typesafe.config.Config;
+
+public class JmxSink implements MetricSink {
+ private JmxReporter reporter;
+
+ @Override
+ public void prepare(Config config, MetricRegistry registry) {
+ reporter = JmxReporter.forRegistry(registry).build();
+ }
+
+ @Override
+ public void start(long period, TimeUnit unit) {
+ reporter.start();
+ }
+
+ @Override
+ public void stop() {
+ reporter.stop();
+ reporter.close();
+ }
+
+ @Override
+ public void report() {
+ // do nothing
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/metric/sink/KafkaSink.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/metric/sink/KafkaSink.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/metric/sink/KafkaSink.java
new file mode 100644
index 0000000..6ff000c
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/metric/sink/KafkaSink.java
@@ -0,0 +1,65 @@
+package org.apache.eagle.alert.metric.sink;
+
+import java.util.concurrent.TimeUnit;
+
+import org.apache.eagle.alert.metric.reporter.KafkaReporter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.codahale.metrics.MetricRegistry;
+import com.typesafe.config.Config;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+public class KafkaSink implements MetricSink{
+ private KafkaReporter reporter;
+ private final static Logger LOG = LoggerFactory.getLogger(KafkaSink.class);
+ @Override
+ public void prepare(Config config, MetricRegistry registry) {
+ LOG.debug("Preparing kafka-sink");
+ KafkaReporter.Builder builder = KafkaReporter.forRegistry(registry)
+ .topic(config.getString("topic"))
+ .config(config);
+
+ if(config.hasPath("tags")){
+ builder.addFields(config.getConfig("tags").root().unwrapped());
+ }
+
+ reporter = builder.build();
+ LOG.info("Prepared kafka-sink");
+ }
+
+ @Override
+ public void start(long period, TimeUnit unit) {
+ LOG.info("Starting");
+ reporter.start(period,unit);
+ }
+
+ @Override
+ public void stop() {
+ LOG.info("Stopping");
+ reporter.stop();
+
+ LOG.info("Closing");
+ reporter.close();
+ }
+
+ @Override
+ public void report() {
+ reporter.report();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/metric/sink/MetricSink.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/metric/sink/MetricSink.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/metric/sink/MetricSink.java
new file mode 100644
index 0000000..b09eda3
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/metric/sink/MetricSink.java
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.metric.sink;
+
+
+import java.util.concurrent.TimeUnit;
+
+import com.codahale.metrics.MetricRegistry;
+import com.typesafe.config.Config;
+
+public interface MetricSink {
+ void prepare(Config config, MetricRegistry registry);
+ void start(long period,TimeUnit unit);
+ void stop();
+ void report();
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/metric/sink/MetricSinkRepository.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/metric/sink/MetricSinkRepository.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/metric/sink/MetricSinkRepository.java
new file mode 100644
index 0000000..b4126f2
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/metric/sink/MetricSinkRepository.java
@@ -0,0 +1,47 @@
+package org.apache.eagle.alert.metric.sink;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+public class MetricSinkRepository {
+ private final static Map<String,Class<? extends MetricSink>> sinkTypeClassMapping = new HashMap<>();
+
+ public static void register(String sinkType,Class<? extends MetricSink> sinkClass){
+ sinkTypeClassMapping.put(sinkType,sinkClass);
+ }
+
+ public static MetricSink createSink(String sinkType){
+ if (!sinkTypeClassMapping.containsKey(sinkType)) {
+ throw new IllegalArgumentException("Unknown sink type: "+sinkType);
+ }
+ try {
+ return sinkTypeClassMapping.get(sinkType).newInstance();
+ } catch (InstantiationException | IllegalAccessException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ static {
+ register("kafka",KafkaSink.class);
+ register("jmx",JmxSink.class);
+ register("elasticsearch",ElasticSearchSink.class);
+ register("stdout",ConsoleSink.class);
+ register("logger",Slf4jSink.class);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/metric/sink/Slf4jSink.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/metric/sink/Slf4jSink.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/metric/sink/Slf4jSink.java
new file mode 100644
index 0000000..ce465fa
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/metric/sink/Slf4jSink.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.metric.sink;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import org.slf4j.LoggerFactory;
+
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.Slf4jReporter;
+import com.typesafe.config.Config;
+
+public class Slf4jSink implements MetricSink {
+ private Slf4jReporter reporter;
+
+ @SuppressWarnings("serial")
+ private final static Map<String,Slf4jReporter.LoggingLevel> LEVEL_MAPPING = new HashMap<String,Slf4jReporter.LoggingLevel>(){{
+ put("INFO",Slf4jReporter.LoggingLevel.INFO);
+ put("DEBUG",Slf4jReporter.LoggingLevel.DEBUG);
+ put("ERROR",Slf4jReporter.LoggingLevel.ERROR);
+ put("TRACE",Slf4jReporter.LoggingLevel.TRACE);
+ put("WARN",Slf4jReporter.LoggingLevel.WARN);
+ }};
+
+ private static Slf4jReporter.LoggingLevel getLoggingLevel(String level){
+ if(LEVEL_MAPPING.containsKey(level.toUpperCase())){
+ return LEVEL_MAPPING.get(level.toUpperCase());
+ } else{
+ throw new IllegalArgumentException("Illegal logging level: "+level);
+ }
+ }
+
+ @Override
+ public void prepare(Config config, MetricRegistry registry) {
+ reporter = Slf4jReporter.forRegistry(registry)
+ .outputTo(LoggerFactory.getLogger("org.apache.eagle.alert.metric"))
+ .withLoggingLevel(config.hasPath("level")? getLoggingLevel(config.getString("level")): Slf4jReporter.LoggingLevel.INFO)
+ .convertRatesTo(TimeUnit.SECONDS)
+ .convertDurationsTo(TimeUnit.MILLISECONDS)
+ .build();
+ }
+
+ @Override
+ public void start(long period,TimeUnit unit) {
+ reporter.start(period,unit);
+ }
+
+ @Override
+ public void stop() {
+ reporter.stop();
+ reporter.close();
+ }
+
+ @Override
+ public void report() {
+ reporter.report();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/metric/source/JVMMetricSource.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/metric/source/JVMMetricSource.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/metric/source/JVMMetricSource.java
new file mode 100644
index 0000000..a5bb5f4
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/metric/source/JVMMetricSource.java
@@ -0,0 +1,41 @@
+package org.apache.eagle.alert.metric.source;
+
+import com.codahale.metrics.JvmAttributeGaugeSet;
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.jvm.MemoryUsageGaugeSet;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+public class JVMMetricSource implements MetricSource{
+
+ private MetricRegistry registry = new MetricRegistry();
+
+ public JVMMetricSource(){
+ registry.registerAll(new JvmAttributeGaugeSet());
+ registry.registerAll(new MemoryUsageGaugeSet());
+ }
+
+ @Override
+ public String name() {
+ return "jvm";
+ }
+
+ @Override
+ public MetricRegistry registry() {
+ return registry;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/metric/source/MetricSource.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/metric/source/MetricSource.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/metric/source/MetricSource.java
new file mode 100644
index 0000000..59b7a02
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/metric/source/MetricSource.java
@@ -0,0 +1,24 @@
+package org.apache.eagle.alert.metric.source;
+
+import com.codahale.metrics.MetricRegistry;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+public interface MetricSource {
+ String name();
+ MetricRegistry registry();
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/metric/source/MetricSourceWrapper.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/metric/source/MetricSourceWrapper.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/metric/source/MetricSourceWrapper.java
new file mode 100644
index 0000000..137fc17
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/metric/source/MetricSourceWrapper.java
@@ -0,0 +1,39 @@
+package org.apache.eagle.alert.metric.source;
+
+import com.codahale.metrics.MetricRegistry;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+public class MetricSourceWrapper implements MetricSource {
+ private final MetricRegistry registry;
+ private final String name;
+
+ public MetricSourceWrapper(String name, MetricRegistry registry){
+ this.name = name;
+ this.registry = registry;
+ }
+
+ @Override
+ public String name() {
+ return name;
+ }
+
+ @Override
+ public MetricRegistry registry() {
+ return registry;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/resource/SimpleCORSFiler.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/resource/SimpleCORSFiler.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/resource/SimpleCORSFiler.java
new file mode 100644
index 0000000..addab44
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/resource/SimpleCORSFiler.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.resource;
+
+import java.io.IOException;
+
+import javax.servlet.Filter;
+import javax.servlet.FilterChain;
+import javax.servlet.FilterConfig;
+import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+import javax.servlet.http.HttpServletResponse;
+
+/**
+ * A simple allow all CORS filter that works with swagger UI. Tomcat CORS filter
+ * doesn't support Origin: null case which is the swagger UI request.
+ *
+ * @since Apr 15, 2016
+ *
+ */
+public class SimpleCORSFiler implements Filter {
+
+ @Override
+ public void init(FilterConfig filterConfig) throws ServletException {
+
+ }
+
+ @Override
+ public void doFilter(ServletRequest request, ServletResponse res, FilterChain chain) throws IOException,
+ ServletException {
+ HttpServletResponse response = (HttpServletResponse) res;
+ response.setHeader("Access-Control-Allow-Origin", "*");
+ response.setHeader("Access-Control-Allow-Methods", "HEAD, POST, GET, OPTIONS, DELETE");
+ response.setHeader("Access-Control-Max-Age", "3600");
+ response.setHeader("Access-Control-Allow-Headers",
+ "Origin, Accept, X-Requested-With, Content-Type, Access-Control-Request-Method, Access-Control-Request-Headers");
+ chain.doFilter(request, response);
+ }
+
+ @Override
+ public void destroy() {
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/service/IMetadataServiceClient.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/service/IMetadataServiceClient.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/service/IMetadataServiceClient.java
new file mode 100644
index 0000000..3609c0a
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/service/IMetadataServiceClient.java
@@ -0,0 +1,69 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.alert.service;
+
+import java.io.Closeable;
+import java.io.Serializable;
+import java.util.List;
+
+import org.apache.eagle.alert.coordination.model.Kafka2TupleMetadata;
+import org.apache.eagle.alert.coordination.model.ScheduleState;
+import org.apache.eagle.alert.coordination.model.SpoutSpec;
+import org.apache.eagle.alert.coordination.model.internal.Topology;
+import org.apache.eagle.alert.engine.coordinator.PolicyDefinition;
+import org.apache.eagle.alert.engine.coordinator.Publishment;
+import org.apache.eagle.alert.engine.coordinator.StreamDefinition;
+import org.apache.eagle.alert.engine.coordinator.StreamingCluster;
+
+/**
+ * service stub to get metadata from remote metadata service
+ */
+public interface IMetadataServiceClient extends Closeable, Serializable {
+
+ // user metadta
+ void addStreamingCluster(StreamingCluster cluster);
+ List<StreamingCluster> listClusters();
+
+ List<Topology> listTopologies();
+ void addTopology(Topology t);
+
+ void addPolicy(PolicyDefinition policy);
+ List<PolicyDefinition> listPolicies();
+
+ void addStreamDefinition(StreamDefinition streamDef);
+ List<StreamDefinition> listStreams();
+
+ void addDataSource(Kafka2TupleMetadata k2t);
+ List<Kafka2TupleMetadata> listDataSources();
+
+ void addPublishment(Publishment pub);
+ List<Publishment> listPublishment();
+
+ // monitor metadata
+ List<SpoutSpec> listSpoutMetadata();
+
+ ScheduleState getVersionedSpec();
+ ScheduleState getVersionedSpec(String version);
+ void addScheduleState(ScheduleState state);
+
+ void clear();
+
+ // for topology mgmt
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/service/MetadataServiceClientImpl.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/service/MetadataServiceClientImpl.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/service/MetadataServiceClientImpl.java
new file mode 100644
index 0000000..3460b77
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/service/MetadataServiceClientImpl.java
@@ -0,0 +1,235 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.alert.service;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.ws.rs.core.MediaType;
+
+import org.apache.eagle.alert.coordination.model.Kafka2TupleMetadata;
+import org.apache.eagle.alert.coordination.model.ScheduleState;
+import org.apache.eagle.alert.coordination.model.SpoutSpec;
+import org.apache.eagle.alert.coordination.model.internal.Topology;
+import org.apache.eagle.alert.engine.coordinator.PolicyDefinition;
+import org.apache.eagle.alert.engine.coordinator.Publishment;
+import org.apache.eagle.alert.engine.coordinator.StreamDefinition;
+import org.apache.eagle.alert.engine.coordinator.StreamingCluster;
+import org.codehaus.jackson.jaxrs.JacksonJsonProvider;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.sun.jersey.api.client.Client;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.GenericType;
+import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.api.client.config.ClientConfig;
+import com.sun.jersey.api.client.config.DefaultClientConfig;
+import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
+import com.typesafe.config.Config;
+
+public class MetadataServiceClientImpl implements IMetadataServiceClient {
+ private static final long serialVersionUID = 3003976065082684128L;
+
+ private static final Logger LOG = LoggerFactory.getLogger(MetadataServiceClientImpl.class);
+
+ private static final String METADATA_SCHEDULESTATES_PATH = "/metadata/schedulestates";
+ private static final String METADATA_PUBLISHMENTS_PATH = "/metadata/publishments";
+ private static final String METADATA_DATASOURCES_PATH = "/metadata/datasources";
+ private static final String METADATA_STREAMS_PATH = "/metadata/streams";
+ private static final String METADATA_POLICIES_PATH = "/metadata/policies";
+ private static final String METADATA_CLUSTERS_PATH = "/metadata/clusters";
+ private static final String METADATA_TOPOLOGY_PATH = "/metadata/topologies";
+ private static final String METADATA_CLEAR_PATH = "/metadata/clear";
+
+ private static final String EAGLE_CORRELATION_CONTEXT = "metadataService.context";
+ private static final String EAGLE_CORRELATION_SERVICE_PORT = "metadataService.port";
+ private static final String EAGLE_CORRELATION_SERVICE_HOST = "metadataService.host";
+
+ protected static final String CONTENT_TYPE = "Content-Type";
+
+ private String host;
+ private int port;
+ private String context;
+ private transient Client client;
+ private String basePath;
+
+ public MetadataServiceClientImpl(Config config) {
+ this(config.getString(EAGLE_CORRELATION_SERVICE_HOST), config.getInt(EAGLE_CORRELATION_SERVICE_PORT), config
+ .getString(EAGLE_CORRELATION_CONTEXT));
+ basePath = buildBasePath();
+ }
+
+ public MetadataServiceClientImpl(String host, int port, String context) {
+ this.host = host;
+ this.port = port;
+ this.context = context;
+ this.basePath = buildBasePath();
+ ClientConfig cc = new DefaultClientConfig();
+ cc.getProperties().put(DefaultClientConfig.PROPERTY_CONNECT_TIMEOUT, 60 * 1000);
+ cc.getProperties().put(DefaultClientConfig.PROPERTY_READ_TIMEOUT, 60 * 1000);
+ cc.getClasses().add(JacksonJsonProvider.class);
+ cc.getProperties().put(URLConnectionClientHandler.PROPERTY_HTTP_URL_CONNECTION_SET_METHOD_WORKAROUND, true);
+ this.client = Client.create(cc);
+ client.addFilter(new com.sun.jersey.api.client.filter.GZIPContentEncodingFilter());
+ }
+
+ private String buildBasePath() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("http://");
+ sb.append(host);
+ sb.append(":");
+ sb.append(port);
+ sb.append(context);
+ return sb.toString();
+ }
+
+ @Override
+ public void close() throws IOException {
+ client.destroy();
+ }
+
+ @Override
+ public List<SpoutSpec> listSpoutMetadata() {
+ ScheduleState state = getVersionedSpec();
+ return new ArrayList<>(state.getSpoutSpecs().values());
+ }
+
+ @Override
+ public List<StreamingCluster> listClusters() {
+ return list(METADATA_CLUSTERS_PATH, new GenericType<List<StreamingCluster>>() {
+ });
+ }
+
+ @Override
+ public List<PolicyDefinition> listPolicies() {
+ return list(METADATA_POLICIES_PATH, new GenericType<List<PolicyDefinition>>() {
+ });
+ }
+
+ @Override
+ public List<StreamDefinition> listStreams() {
+ return list(METADATA_STREAMS_PATH, new GenericType<List<StreamDefinition>>() {
+ });
+ }
+
+ @Override
+ public List<Kafka2TupleMetadata> listDataSources() {
+ return list(METADATA_DATASOURCES_PATH, new GenericType<List<Kafka2TupleMetadata>>() {
+ });
+ }
+
+ private <T> List<T> list(String path, GenericType<List<T>> type) {
+ WebResource r = client.resource(basePath + path);
+ LOG.info("query URL {}", basePath + path);
+ List<T> ret = r.accept(MediaType.APPLICATION_JSON_TYPE).type(MediaType.APPLICATION_JSON).get(type);
+ return ret;
+ }
+
+ @Override
+ public List<Publishment> listPublishment() {
+ return list(METADATA_PUBLISHMENTS_PATH, new GenericType<List<Publishment>>() {
+ });
+ }
+
+ @Override
+ public ScheduleState getVersionedSpec(String version) {
+ return listOne(METADATA_SCHEDULESTATES_PATH + "/" + version, ScheduleState.class);
+ }
+
+ private <T> T listOne(String path, Class<T> tClz) {
+ LOG.info("query URL {}", basePath + path);
+ WebResource r = client.resource(basePath + path);
+
+ ClientResponse resp = r.accept(MediaType.APPLICATION_JSON_TYPE).type(MediaType.APPLICATION_JSON)
+ .get(ClientResponse.class);
+ if (resp.getStatus() < 300) {
+ try {
+ return resp.getEntity(tClz);
+ } catch (Exception e) {
+ LOG.warn(" list one entity failed, ignored and continute, path {}, message {}!", path, e.getMessage());
+ }
+ }else{
+ LOG.warn("fail querying metadata service {} with http status {}", basePath + path, resp.getStatus());
+ }
+ return null;
+ }
+
+ @Override
+ public ScheduleState getVersionedSpec() {
+ return listOne(METADATA_SCHEDULESTATES_PATH, ScheduleState.class);
+ }
+
+ @Override
+ public void addScheduleState(ScheduleState state) {
+ WebResource r = client.resource(basePath + METADATA_SCHEDULESTATES_PATH);
+ r.accept(MediaType.APPLICATION_JSON_TYPE).type(MediaType.APPLICATION_JSON).post(state);
+ }
+
+ @Override
+ public List<Topology> listTopologies() {
+ return list(METADATA_TOPOLOGY_PATH, new GenericType<List<Topology>>() {
+ });
+ }
+
+ @Override
+ public void addStreamingCluster(StreamingCluster cluster) {
+ WebResource r = client.resource(basePath + METADATA_CLUSTERS_PATH);
+ r.accept(MediaType.APPLICATION_JSON_TYPE).type(MediaType.APPLICATION_JSON).post(cluster);
+ }
+
+ @Override
+ public void addTopology(Topology t) {
+ WebResource r = client.resource(basePath + METADATA_TOPOLOGY_PATH);
+ r.accept(MediaType.APPLICATION_JSON_TYPE).type(MediaType.APPLICATION_JSON).post(t);
+ }
+
+ @Override
+ public void addPolicy(PolicyDefinition policy) {
+ WebResource r = client.resource(basePath + METADATA_POLICIES_PATH);
+ r.accept(MediaType.APPLICATION_JSON_TYPE).type(MediaType.APPLICATION_JSON).post(policy);
+ }
+
+ @Override
+ public void addStreamDefinition(StreamDefinition streamDef) {
+ WebResource r = client.resource(basePath + METADATA_STREAMS_PATH);
+ r.accept(MediaType.APPLICATION_JSON_TYPE).type(MediaType.APPLICATION_JSON).post(streamDef);
+ }
+
+ @Override
+ public void addDataSource(Kafka2TupleMetadata k2t) {
+ WebResource r = client.resource(basePath + METADATA_DATASOURCES_PATH);
+ r.accept(MediaType.APPLICATION_JSON_TYPE).type(MediaType.APPLICATION_JSON).post(k2t);
+ }
+
+ @Override
+ public void addPublishment(Publishment pub) {
+ WebResource r = client.resource(basePath + METADATA_PUBLISHMENTS_PATH);
+ r.accept(MediaType.APPLICATION_JSON_TYPE).type(MediaType.APPLICATION_JSON).post(pub);
+ }
+
+ @Override
+ public void clear() {
+ WebResource r = client.resource(basePath + METADATA_CLEAR_PATH);
+ r.accept(MediaType.APPLICATION_JSON_TYPE).type(MediaType.APPLICATION_JSON).post();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/utils/AlertConstants.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/utils/AlertConstants.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/utils/AlertConstants.java
new file mode 100644
index 0000000..87c45d8
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/utils/AlertConstants.java
@@ -0,0 +1,27 @@
+package org.apache.eagle.alert.utils;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+public class AlertConstants {
+ public final static String FIELD_0 = "f0";
+ public final static String FIELD_1 = "f1";
+ public final static String FIELD_2 = "f2";
+ public final static String FIELD_3 = "f3";
+
+ public final static String DEFAULT_SPOUT_NAME = "alertEngineSpout";
+ public final static String DEFAULT_ROUTERBOLT_NAME = "streamRouterBolt";
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/utils/ByteUtils.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/utils/ByteUtils.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/utils/ByteUtils.java
new file mode 100644
index 0000000..636716d
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/utils/ByteUtils.java
@@ -0,0 +1,172 @@
+package org.apache.eagle.alert.utils;
+
+import java.io.UnsupportedEncodingException;
+
+public class ByteUtils {
+
+ public static double bytesToDouble(byte[] bytes, int offset){
+ return Double.longBitsToDouble(bytesToLong(bytes, offset));
+ }
+
+ public static double bytesToDouble(byte[] bytes){
+ return Double.longBitsToDouble(bytesToLong(bytes));
+ }
+
+ public static void doubleToBytes(double v, byte[] bytes){
+ doubleToBytes(v, bytes, 0);
+ }
+
+ public static void doubleToBytes(double v, byte[] bytes, int offset){
+ longToBytes(Double.doubleToLongBits(v), bytes, offset);
+ }
+
+ public static byte[] doubleToBytes(double v){
+ return longToBytes(Double.doubleToLongBits(v));
+ }
+
+ public static long bytesToLong(byte[] bytes){
+ return bytesToLong(bytes, 0);
+ }
+
+ public static long bytesToLong(byte[] bytes, int offset){
+ long value = 0;
+ for(int i=0; i<8; i++){
+ value <<= 8;
+ value |= (bytes[i+offset] & 0xFF);
+ }
+ return value;
+ }
+
+ public static void longToBytes(long v, byte[] bytes){
+ longToBytes(v, bytes, 0);
+ }
+
+ public static void longToBytes(long v, byte[] bytes, int offset){
+ long tmp = v;
+ for(int i=0; i<8; i++){
+ bytes[offset + 7 - i] = (byte)(tmp & 0xFF);
+ tmp >>= 8;
+ }
+ }
+
+ public static byte[] longToBytes(long v){
+ long tmp = v;
+ byte[] b = new byte[8];
+ for(int i=0; i<8; i++){
+ b[7-i] = (byte)(tmp & 0xFF);
+ tmp >>= 8;
+ }
+ return b;
+ }
+
+ public static int bytesToInt(byte[] bytes){
+ return bytesToInt(bytes, 0);
+ }
+
+ public static int bytesToInt(byte[] bytes, int offset){
+ int value = 0;
+ for(int i=0; i<4; i++){
+ value <<= 8;
+ value |= (bytes[i+offset] & 0xFF);
+ }
+ return value;
+ }
+
+ public static void intToBytes(int v, byte[] bytes){
+ intToBytes(v, bytes, 0);
+ }
+
+ public static void intToBytes(int v, byte[] bytes, int offset){
+ int tmp = v;
+ for(int i=0; i<4; i++){
+ bytes[offset + 3 - i] = (byte)(tmp & 0xFF);
+ tmp >>= 8;
+ }
+ }
+
+ public static byte[] intToBytes(int v){
+ int tmp = v;
+ byte[] b = new byte[4];
+ for(int i=0; i<4; i++){
+ b[3-i] = (byte)(tmp & 0xFF);
+ tmp >>= 8;
+ }
+ return b;
+ }
+
+ //////
+
+ public static short bytesToShort(byte[] bytes){
+ return bytesToShort(bytes, 0);
+ }
+
+ public static short bytesToShort(byte[] bytes, int offset){
+ short value = 0;
+ for(int i=0; i < 2; i++){
+ value <<= 8;
+ value |= (bytes[i+offset] & 0xFF);
+ }
+ return value;
+ }
+
+ public static void shortToBytes(short v, byte[] bytes){
+ shortToBytes(v, bytes, 0);
+ }
+
+ public static void shortToBytes(short v, byte[] bytes, int offset){
+ int tmp = v;
+ for(int i=0; i < 2; i++){
+ bytes[offset + 1 - i] = (byte)(tmp & 0xFF);
+ tmp >>= 8;
+ }
+ }
+
+ public static byte[] shortToBytes(short v){
+ int tmp = v;
+ byte[] b = new byte[2];
+ for(int i=0; i<2; i++){
+ b[1-i] = (byte)(tmp & 0xFF);
+ tmp >>= 8;
+ }
+ return b;
+ }
+
+ public static byte[] concat(byte[]... arrays) {
+ int length = 0;
+ for (byte[] array : arrays) {
+ length += array.length;
+ }
+ byte[] result = new byte[length];
+ int pos = 0;
+ for (byte[] array : arrays) {
+ System.arraycopy(array, 0, result, pos, array.length);
+ pos += array.length;
+ }
+ return result;
+ }
+
+ public static byte[] stringToBytes(String str) {
+ try {
+ return str.getBytes("UTF-8");
+ } catch (UnsupportedEncodingException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+// public static void main(String[] args){
+// int a = "ThreadName".hashCode();
+// byte[] b = intToBytes(a);
+// byte[] c = intToBytes(1676687583);
+// String s = new String(b);
+// System.out.println(s);
+
+// byte[] d = intToBytes(8652353);
+// System.out.println(bytesToInt(d));
+
+// byte[] e = longToBytes(12131513513l);
+// System.out.println(bytesToLong(e));
+// if(12131513513l == bytesToLong(e)){
+// System.out.println("yes");
+// }
+// }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/utils/ConfigUtils.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/utils/ConfigUtils.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/utils/ConfigUtils.java
new file mode 100644
index 0000000..685265f
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/utils/ConfigUtils.java
@@ -0,0 +1,31 @@
+package org.apache.eagle.alert.utils;
+
+import java.util.Properties;
+
+import com.typesafe.config.Config;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+public class ConfigUtils {
+
+ @SuppressWarnings("serial")
+ public static Properties toProperties(Config config){
+ return new Properties(){{
+ putAll(config.root().unwrapped());
+ }};
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/utils/DateTimeUtil.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/utils/DateTimeUtil.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/utils/DateTimeUtil.java
new file mode 100644
index 0000000..d611b95
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/utils/DateTimeUtil.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.utils;
+
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.TimeZone;
+
+/**
+ * be aware that SimpleDateFormat instantiation is expensive, so if that's under a tight loop, probably we need
+ * a thread local SimpleDateFormat object
+ */
+public class DateTimeUtil {
+ public static final long ONESECOND = 1L * 1000L;
+ public static final long ONEMINUTE = 1L * 60L * 1000L;
+ public static final long ONEHOUR = 1L * 60L * 60L * 1000L;
+ public static final long ONEDAY = 24L * 60L * 60L * 1000L;
+ private static TimeZone CURRENT_TIME_ZONE = TimeZone.getDefault();
+
+ public static Date humanDateToDate(String date) throws ParseException {
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ sdf.setTimeZone(CURRENT_TIME_ZONE);
+ return sdf.parse(date);
+ }
+
+ public static long getCurrentTimestamp(){
+ return System.currentTimeMillis();
+ }
+
+ public static String secondsToHumanDate(long seconds){
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ sdf.setTimeZone(CURRENT_TIME_ZONE);
+ Date t = new Date();
+ t.setTime(seconds*1000);
+ return sdf.format(t);
+ }
+
+ public static String millisecondsToHumanDateWithMilliseconds(long milliseconds){
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss,SSS");
+ sdf.setTimeZone(CURRENT_TIME_ZONE);
+ Date t = new Date();
+ t.setTime(milliseconds);
+ return sdf.format(t);
+ }
+
+ public static String millisecondsToHumanDateWithSeconds(long milliseconds){
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ sdf.setTimeZone(CURRENT_TIME_ZONE);
+ Date t = new Date();
+ t.setTime(milliseconds);
+ return sdf.format(t);
+ }
+
+ public static long humanDateToSeconds(String date) throws ParseException {
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ sdf.setTimeZone(CURRENT_TIME_ZONE);
+ Date d = sdf.parse(date);
+ return d.getTime()/1000;
+ }
+
+ public static long humanDateToMilliseconds(String date) throws ParseException {
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss,SSS");
+ sdf.setTimeZone(CURRENT_TIME_ZONE);
+ Date d = sdf.parse(date);
+ return d.getTime();
+ }
+
+
+ public static long humanDateToMillisecondsWithoutException(String date){
+ try{
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss,SSS");
+ sdf.setTimeZone(CURRENT_TIME_ZONE);
+ Date d = sdf.parse(date);
+ return d.getTime();
+ }catch(ParseException ex){
+ return 0L;
+ }
+ }
+
+ public static long humanDateToSecondsWithoutException(String date){
+ try{
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ sdf.setTimeZone(CURRENT_TIME_ZONE);
+ Date d = sdf.parse(date);
+ return (d.getTime() / 1000);
+ }catch(ParseException ex){
+ return 0L;
+ }
+ }
+ /**
+ * this could be accurate only when timezone is UTC
+ * for the timezones other than UTC, there is possibly issue, for example
+ * assume timezone is GMT+8 in China
+ * When user time is "2014-07-15 05:00:00", it will be converted to timestamp first, internally it would be "2014-07-14 21:00:00" in UTC timezone. When rounded down to day, the internal time would
+ * be changed to "2014-07-14 00:00:00", and that means the user time is "2014-07-14 08:00:00". But originally user wants to round it to "2014-07-15 00:00:00"
+ *
+ * @param field
+ * @param timeInMillis the seconds elapsed since 1970-01-01 00:00:00
+ * @return
+ */
+ public static long roundDown(int field, long timeInMillis){
+ switch(field){
+ case Calendar.DAY_OF_MONTH:
+ case Calendar.DAY_OF_WEEK:
+ case Calendar.DAY_OF_YEAR:
+ return (timeInMillis - timeInMillis % (24*60*60*1000));
+ case Calendar.HOUR:
+ return (timeInMillis - timeInMillis % (60*60*1000));
+ case Calendar.MINUTE:
+ return (timeInMillis - timeInMillis % (60*1000));
+ case Calendar.SECOND:
+ return (timeInMillis - timeInMillis % (1000));
+ default:
+ return 0L;
+ }
+ }
+
+ public static String format(long milliseconds, String format) {
+ SimpleDateFormat sdf = new SimpleDateFormat(format);
+ sdf.setTimeZone(CURRENT_TIME_ZONE);
+ Date t = new Date();
+ t.setTime(milliseconds);
+ return sdf.format(t);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/utils/HostUtils.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/utils/HostUtils.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/utils/HostUtils.java
new file mode 100644
index 0000000..3e3b6d5
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/utils/HostUtils.java
@@ -0,0 +1,70 @@
+
+package org.apache.eagle.alert.utils;
+
+import java.net.InetAddress;
+import java.net.NetworkInterface;
+import java.net.SocketException;
+import java.net.UnknownHostException;
+import java.util.Enumeration;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * http://stackoverflow.com/questions/7348711/recommended-way-to-get-hostname-in-java
+ */
+public class HostUtils {
+ private static final Logger logger = LoggerFactory
+ .getLogger(HostUtils.class);
+
+ public static String getHostName() {
+ try {
+ String hostName = InetAddress.getLocalHost().getHostName();
+ if (hostName != null && !hostName.isEmpty()) {
+ return hostName;
+ }
+ } catch (UnknownHostException e) {
+ logger.error("get hostName error!", e);
+ }
+
+ String host = System.getenv("COMPUTERNAME");
+ if (host != null)
+ return host;
+ host = System.getenv("HOSTNAME");
+ if (host != null)
+ return host;
+
+ return null;
+ }
+
+ public static String getNotLoopbackAddress() {
+ String hostName = null;
+ Enumeration<NetworkInterface> interfaces;
+ try {
+ interfaces = NetworkInterface.getNetworkInterfaces();
+ while (interfaces.hasMoreElements()) {
+ NetworkInterface nic = interfaces.nextElement();
+ Enumeration<InetAddress> addresses = nic.getInetAddresses();
+ while (hostName == null && addresses.hasMoreElements()) {
+ InetAddress address = addresses.nextElement();
+ if (!address.isLoopbackAddress()) {
+ hostName = address.getHostName();
+ }
+ }
+ }
+ } catch (SocketException e) {
+ logger.error("getNotLoopbackAddress error!", e);
+ }
+ return hostName;
+ }
+
+ public static String getHostAddress() {
+ try {
+ return InetAddress.getLocalHost().getHostAddress();
+ } catch (UnknownHostException e) {
+ logger.error("get hostAddress error!", e);
+ }
+
+ return null;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/utils/JsonUtils.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/utils/JsonUtils.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/utils/JsonUtils.java
new file mode 100644
index 0000000..de0c48b
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/utils/JsonUtils.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.utils;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+/**
+ * @since May 1, 2016
+ *
+ */
+public class JsonUtils {
+
+ public static final ObjectMapper mapper = new ObjectMapper();
+ private static final Logger LOG = LoggerFactory.getLogger(JsonUtils.class);
+
+ public static String writeValueAsString(Object o) {
+ try {
+ return mapper.writeValueAsString(o);
+ } catch (Exception e) {
+ LOG.error("write object as string failed {} !", o);
+ }
+ return "";
+ }
+}
[06/19] incubator-eagle git commit: EAGLE-324: Init branch-v0.5
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/mock/MockStreamCollector.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/mock/MockStreamCollector.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/mock/MockStreamCollector.java
new file mode 100755
index 0000000..fa07701
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/mock/MockStreamCollector.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.mock;
+
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.eagle.alert.engine.Collector;
+import org.apache.eagle.alert.engine.model.AlertStreamEvent;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class MockStreamCollector implements Collector<AlertStreamEvent> {
+ @SuppressWarnings("unused")
+ private final static Logger LOG = LoggerFactory.getLogger(MockStreamCollector.class);
+ private List<AlertStreamEvent> cache;
+ public MockStreamCollector(){
+ cache = new LinkedList<>();
+ }
+
+ public void emit(AlertStreamEvent event) {
+ cache.add(event);
+ // LOG.info("PartitionedEventCollector received: {}",event);
+ }
+
+ public void clear(){
+ cache.clear();
+ }
+
+ public List<AlertStreamEvent> get(){
+ return cache;
+ }
+
+ public int size(){
+ return cache.size();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/mock/MockStreamMetadataService.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/mock/MockStreamMetadataService.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/mock/MockStreamMetadataService.java
new file mode 100644
index 0000000..2119ecd
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/mock/MockStreamMetadataService.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.mock;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.eagle.alert.engine.coordinator.StreamDefinition;
+import org.apache.eagle.alert.engine.coordinator.StreamDefinitionNotFoundException;
+
+public class MockStreamMetadataService{
+ private final Map<String,StreamDefinition> streamSchemaMap = new HashMap<>();
+
+ public StreamDefinition getStreamDefinition(String streamId) throws StreamDefinitionNotFoundException {
+ if(streamSchemaMap.containsKey(streamId)) {
+ return streamSchemaMap.get(streamId);
+ }else {
+ throw new StreamDefinitionNotFoundException(streamId);
+ }
+ }
+
+ public void registerStream(String streamId, StreamDefinition schema){
+ streamSchemaMap.put(streamId,schema);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/mock/MockStreamReceiver.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/mock/MockStreamReceiver.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/mock/MockStreamReceiver.java
new file mode 100644
index 0000000..1e44c2a
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/mock/MockStreamReceiver.java
@@ -0,0 +1,79 @@
+package org.apache.eagle.alert.engine.mock;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.eagle.alert.engine.model.PartitionedEvent;
+import org.apache.eagle.alert.utils.AlertConstants;
+import org.apache.eagle.alert.utils.StreamIdConversion;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import backtype.storm.spout.SpoutOutputCollector;
+import backtype.storm.task.TopologyContext;
+import backtype.storm.topology.OutputFieldsDeclarer;
+import backtype.storm.topology.base.BaseRichSpout;
+import backtype.storm.tuple.Fields;
+import backtype.storm.utils.Utils;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+@SuppressWarnings("serial")
+public class MockStreamReceiver extends BaseRichSpout {
+ private final static Logger LOG = LoggerFactory.getLogger(MockStreamReceiver.class);
+ private SpoutOutputCollector collector;
+ private List<String> outputStreamIds;
+ public MockStreamReceiver(int partition){
+ outputStreamIds = new ArrayList<>(partition);
+ for(int i=0;i<partition;i++){
+ outputStreamIds.add(StreamIdConversion.generateStreamIdByPartition(i));
+ }
+ }
+
+ @SuppressWarnings("rawtypes")
+ @Override
+ public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
+ this.collector = collector;
+ }
+
+ @Override
+ public void close() {}
+
+ /**
+ * This unit test is not to mock the end2end logic of correlation spout,
+ * but simply generate some sample data for following bolts testing
+ */
+ @Override
+ public void nextTuple() {
+ PartitionedEvent event = MockSampleMetadataFactory.createRandomOutOfTimeOrderEventGroupedByName("sampleStream_1");
+ LOG.info("Receive {}",event);
+ collector.emit(outputStreamIds.get(
+ // group by the first field in event i.e. name
+ (int) (event.getPartitionKey() % outputStreamIds.size())),
+ Collections.singletonList(event));
+ Utils.sleep(500);
+ }
+
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+ for(String streamId:outputStreamIds) {
+ declarer.declareStream(streamId,new Fields(AlertConstants.FIELD_0));
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/perf/TestSerDeserPer.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/perf/TestSerDeserPer.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/perf/TestSerDeserPer.java
new file mode 100644
index 0000000..1bccbbf
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/perf/TestSerDeserPer.java
@@ -0,0 +1,301 @@
+package org.apache.eagle.alert.engine.perf;
+
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.eagle.alert.engine.coordinator.StreamPartition;
+import org.apache.eagle.alert.engine.coordinator.StreamSortSpec;
+import org.apache.eagle.alert.engine.model.PartitionedEvent;
+import org.apache.eagle.alert.engine.model.StreamEvent;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.esotericsoftware.kryo.Kryo;
+import com.esotericsoftware.kryo.io.Input;
+import com.esotericsoftware.kryo.io.Output;
+
+/**
+ * Since 5/13/16.
+ */
+public class TestSerDeserPer {
+ Object[] data = null;
+ @Before
+ public void before(){
+ int max = 100;
+ StringBuilder sb = new StringBuilder();
+ for(int i=0; i<max; i++){
+ sb.append("a");
+ }
+ data = new Object[]{sb.toString()};
+ }
+
+ @Test
+ public void testSerDeserPerf() throws Exception{
+ Kryo kryo = new Kryo();
+ Output output = new Output(new FileOutputStream("/tmp/file.bin"));
+ for(int i=0; i<1000; i++){
+ kryo.writeObject(output, constructPE());
+ }
+ output.close();
+ Input input = new Input(new FileInputStream("/tmp/file.bin"));
+ PartitionedEvent someObject = kryo.readObject(input, PartitionedEvent.class);
+ input.close();
+ Assert.assertTrue(someObject.getData().length == 1);
+ }
+
+ private PartitionedEvent constructPE(){
+ StreamEvent e = new StreamEvent();
+ e.setStreamId("testStreamId");
+ e.setTimestamp(1463159382000L);
+ e.setData(data);
+ StreamPartition sp = new StreamPartition();
+ List<String> col = new ArrayList<>();
+ col.add("host");
+ sp.setColumns(col);
+ StreamSortSpec sortSpec = new StreamSortSpec();
+ sortSpec.setWindowMargin(30000);
+ sortSpec.setWindowPeriod("PT1M");
+ sp.setSortSpec(sortSpec);
+ sp.setStreamId("testStreamId");
+ sp.setType(StreamPartition.Type.GROUPBY);
+ PartitionedEvent pe = new PartitionedEvent();
+ pe.setEvent(e);
+ pe.setPartition(sp);
+ pe.setPartitionKey(1000);
+ return pe;
+ }
+
+ @Test
+ public void testSerDeserPerf2() throws Exception{
+ Kryo kryo = new Kryo();
+ Output output = new Output(new FileOutputStream("/tmp/file2.bin"));
+ for(int i=0; i<1000; i++){
+ kryo.writeObject(output, constructNewPE());
+ }
+ output.close();
+ Input input = new Input(new FileInputStream("/tmp/file2.bin"));
+ NewPartitionedEvent someObject = kryo.readObject(input, NewPartitionedEvent.class);
+ input.close();
+ Assert.assertTrue(someObject.getData().length == 1);
+ }
+
+ private NewPartitionedEvent constructNewPE(){
+ NewPartitionedEvent pe = new NewPartitionedEvent();
+ pe.setStreamId("testStreamId");
+ pe.setTimestamp(1463159382000L);
+ pe.setData(data);
+
+ pe.setType(StreamPartition.Type.GROUPBY);
+ List<String> col = new ArrayList<>();
+ col.add("host");
+ pe.setColumns(col);
+ pe.setPartitionKey(1000);
+
+ pe.setWindowMargin(30000);
+ pe.setWindowPeriod("PT1M");
+ return pe;
+ }
+
+ @Test
+ public void testSerDeserPerf3() throws Exception{
+ Kryo kryo = new Kryo();
+ Output output = new Output(new FileOutputStream("/tmp/file3.bin"));
+ for(int i=0; i<1000; i++){
+ kryo.writeObject(output, constructNewPE2());
+ }
+ output.close();
+ Input input = new Input(new FileInputStream("/tmp/file3.bin"));
+ NewPartitionedEvent2 someObject = kryo.readObject(input, NewPartitionedEvent2.class);
+ input.close();
+ Assert.assertTrue(someObject.getData().length == 1);
+ }
+
+ private NewPartitionedEvent2 constructNewPE2(){
+ NewPartitionedEvent2 pe = new NewPartitionedEvent2();
+ pe.setStreamId(100);
+ pe.setTimestamp(1463159382000L);
+ pe.setData(data);
+
+ pe.setType(1);
+ int[] col = new int[1];
+ col[0] = 1;
+ pe.setColumns(col);
+ pe.setPartitionKey(1000);
+
+ pe.setWindowMargin(30000);
+ pe.setWindowPeriod(60);
+ return pe;
+ }
+
+ public static class NewPartitionedEvent implements Serializable {
+ private static final long serialVersionUID = -3840016190614238593L;
+ // basic
+ private String streamId;
+ private long timestamp;
+ private Object[] data;
+
+ // stream partition
+ private StreamPartition.Type type;
+ private List<String> columns = new ArrayList<>();
+ private long partitionKey;
+
+ // sort spec
+ private String windowPeriod="";
+ private long windowMargin = 30 * 1000;
+
+ public NewPartitionedEvent(){
+ }
+
+ public String getStreamId() {
+ return streamId;
+ }
+
+ public void setStreamId(String streamId) {
+ this.streamId = streamId;
+ }
+
+ public long getTimestamp() {
+ return timestamp;
+ }
+
+ public void setTimestamp(long timestamp) {
+ this.timestamp = timestamp;
+ }
+
+ public Object[] getData() {
+ return data;
+ }
+
+ public void setData(Object[] data) {
+ this.data = data;
+ }
+
+ public StreamPartition.Type getType() {
+ return type;
+ }
+
+ public void setType(StreamPartition.Type type) {
+ this.type = type;
+ }
+
+ public List<String> getColumns() {
+ return columns;
+ }
+
+ public void setColumns(List<String> columns) {
+ this.columns = columns;
+ }
+
+ public long getPartitionKey() {
+ return partitionKey;
+ }
+
+ public void setPartitionKey(long partitionKey) {
+ this.partitionKey = partitionKey;
+ }
+
+ public String getWindowPeriod() {
+ return windowPeriod;
+ }
+
+ public void setWindowPeriod(String windowPeriod) {
+ this.windowPeriod = windowPeriod;
+ }
+
+ public long getWindowMargin() {
+ return windowMargin;
+ }
+
+ public void setWindowMargin(long windowMargin) {
+ this.windowMargin = windowMargin;
+ }
+ }
+
+ public static class NewPartitionedEvent2 implements Serializable {
+ private static final long serialVersionUID = -3840016190614238593L;
+ // basic
+ private int streamId;
+ private long timestamp;
+ private Object[] data;
+
+ // stream partition
+ private int type;
+ private int[] columns;
+ private long partitionKey;
+
+ // sort spec
+ private long windowPeriod;
+ private long windowMargin = 30 * 1000;
+
+ public NewPartitionedEvent2(){
+ }
+
+ public int getStreamId() {
+ return streamId;
+ }
+
+ public void setStreamId(int streamId) {
+ this.streamId = streamId;
+ }
+
+ public int getType() {
+ return type;
+ }
+
+ public void setType(int type) {
+ this.type = type;
+ }
+
+ public int[] getColumns() {
+ return columns;
+ }
+
+ public void setColumns(int[] columns) {
+ this.columns = columns;
+ }
+
+ public long getPartitionKey() {
+ return partitionKey;
+ }
+
+ public void setPartitionKey(long partitionKey) {
+ this.partitionKey = partitionKey;
+ }
+
+ public long getWindowPeriod() {
+ return windowPeriod;
+ }
+
+ public void setWindowPeriod(long windowPeriod) {
+ this.windowPeriod = windowPeriod;
+ }
+
+ public long getTimestamp() {
+ return timestamp;
+ }
+
+ public void setTimestamp(long timestamp) {
+ this.timestamp = timestamp;
+ }
+
+ public Object[] getData() {
+ return data;
+ }
+
+ public void setData(Object[] data) {
+ this.data = data;
+ }
+
+ public long getWindowMargin() {
+ return windowMargin;
+ }
+
+ public void setWindowMargin(long windowMargin) {
+ this.windowMargin = windowMargin;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/router/TestAlertBolt.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/router/TestAlertBolt.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/router/TestAlertBolt.java
new file mode 100755
index 0000000..e322099
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/router/TestAlertBolt.java
@@ -0,0 +1,177 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.router;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.eagle.alert.coordination.model.AlertBoltSpec;
+import org.apache.eagle.alert.engine.coordinator.PolicyDefinition;
+import org.apache.eagle.alert.engine.coordinator.StreamColumn;
+import org.apache.eagle.alert.engine.coordinator.StreamDefinition;
+import org.apache.eagle.alert.engine.coordinator.StreamPartition;
+import org.apache.eagle.alert.engine.evaluator.PolicyGroupEvaluator;
+import org.apache.eagle.alert.engine.evaluator.PolicyStreamHandlers;
+import org.apache.eagle.alert.engine.evaluator.impl.PolicyGroupEvaluatorImpl;
+import org.apache.eagle.alert.engine.model.AlertStreamEvent;
+import org.apache.eagle.alert.engine.model.PartitionedEvent;
+import org.apache.eagle.alert.engine.model.StreamEvent;
+import org.apache.eagle.alert.engine.runner.AlertBolt;
+import org.apache.eagle.alert.engine.runner.TestStreamRouterBolt;
+import org.apache.eagle.alert.utils.DateTimeUtil;
+import org.junit.Assert;
+import org.junit.Test;
+
+import backtype.storm.metric.api.MultiCountMetric;
+import backtype.storm.task.GeneralTopologyContext;
+import backtype.storm.task.IOutputCollector;
+import backtype.storm.task.OutputCollector;
+import backtype.storm.task.TopologyContext;
+import backtype.storm.tuple.Fields;
+import backtype.storm.tuple.Tuple;
+import backtype.storm.tuple.TupleImpl;
+
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+
+/**
+ * Since 5/2/16.
+ */
+@SuppressWarnings({"rawtypes", "unused"})
+public class TestAlertBolt {
+ /**
+ * Following knowledge is guaranteed in
+ *
+ * @see org.apache.eagle.alert.engine.runner.AlertBolt#execute{
+ * if(!routedStreamEvent.getRoute().getTargetComponentId().equals(this.policyGroupEvaluator.getName())){
+ * throw new IllegalStateException("Got event targeted to "+ routedStreamEvent.getRoute().getTargetComponentId()+" in "+this.policyGroupEvaluator.getName());
+ * }
+ * }
+ *
+ * @throws Exception
+ *
+ * Add test case: 2 alerts should be generated even if they are very close to each other in timestamp
+ */
+ @Test
+ public void testAlertBolt() throws Exception{
+ final AtomicInteger alertCount = new AtomicInteger();
+ final Semaphore mutex = new Semaphore(0);
+ Config config = ConfigFactory.load();
+ PolicyGroupEvaluator policyGroupEvaluator = new PolicyGroupEvaluatorImpl("testPolicyGroupEvaluatorImpl");
+ TestStreamRouterBolt.MockChangeService mockChangeService = new TestStreamRouterBolt.MockChangeService();
+ AlertBolt bolt = new AlertBolt("alertBolt1", policyGroupEvaluator, config, mockChangeService);
+ OutputCollector collector = new OutputCollector(new IOutputCollector(){
+ int count = 0;
+ @Override
+ public List<Integer> emit(String streamId, Collection<Tuple> anchors, List<Object> tuple) {
+ alertCount.incrementAndGet();
+ mutex.release();
+ Assert.assertEquals("testAlertStream", tuple.get(0));
+ AlertStreamEvent event = (AlertStreamEvent) tuple.get(1);
+ System.out.println(String.format("collector received: [streamId=[%s], tuple=[%s] ", streamId, tuple));
+ return null;
+ }
+ @Override
+ public void emitDirect(int taskId, String streamId, Collection<Tuple> anchors, List<Object> tuple) { }
+ @Override
+ public void ack(Tuple input) { }
+ @Override
+ public void fail(Tuple input) { }
+ @Override
+ public void reportError(Throwable error) { }
+ });
+ Map stormConf = new HashMap<>();
+ TopologyContext topologyContext = mock(TopologyContext.class);
+ when(topologyContext.registerMetric(any(String.class), any(MultiCountMetric.class), any(int.class))).thenReturn(new MultiCountMetric());
+ bolt.prepare(stormConf, topologyContext, collector);
+
+ String streamId = "cpuUsageStream";
+
+ // construct StreamDefinition
+ StreamDefinition schema = new StreamDefinition();
+ schema.setStreamId(streamId);
+ StreamColumn column = new StreamColumn();
+ column.setName("col1");
+ column.setType(StreamColumn.Type.STRING);
+ schema.setColumns(Collections.singletonList(column));
+ Map<String, StreamDefinition> sds = new HashMap<>();
+ sds.put(schema.getStreamId(), schema);
+
+ // construct StreamPartition
+ StreamPartition sp = new StreamPartition();
+ sp.setColumns(Collections.singletonList("col1"));
+ sp.setStreamId(streamId);
+ sp.setType(StreamPartition.Type.GROUPBY);
+
+ AlertBoltSpec spec = new AlertBoltSpec();
+ spec.setVersion("version1");
+ spec.setTopologyName("testTopology");
+ PolicyDefinition pd = new PolicyDefinition();
+ pd.setName("policy1");
+ pd.setPartitionSpec(Collections.singletonList(sp));
+ pd.setOutputStreams(Collections.singletonList("testAlertStream"));
+ pd.setInputStreams(Collections.singletonList(streamId));
+ pd.setDefinition(new PolicyDefinition.Definition());
+ pd.getDefinition().type = PolicyStreamHandlers.SIDDHI_ENGINE;
+ pd.getDefinition().value = "from cpuUsageStream[col1=='value1' OR col1=='value2'] select col1 insert into testAlertStream;";
+ spec.addBoltPolicy("alertBolt1", pd.getName());
+ spec.getBoltPoliciesMap().put("alertBolt1", new ArrayList<PolicyDefinition>(Arrays.asList(pd)));
+ bolt.onAlertBoltSpecChange(spec, sds);
+
+ // contruct GeneralTopologyContext
+ GeneralTopologyContext context = mock(GeneralTopologyContext.class);
+ int taskId = 1;
+ when(context.getComponentId(taskId)).thenReturn("comp1");
+ when(context.getComponentOutputFields("comp1", "default")).thenReturn(new Fields("f0"));
+
+ // construct event with "value1"
+ StreamEvent event1 = new StreamEvent();
+ event1.setTimestamp(DateTimeUtil.humanDateToSeconds("2016-01-01 00:00:00")*1000);
+ Object[] data = new Object[]{"value1"};
+ event1.setData(data);
+ event1.setStreamId(streamId);
+ PartitionedEvent partitionedEvent1 = new PartitionedEvent(event1, sp,1001);
+
+ // construct another event with "value1"
+ StreamEvent event2 = new StreamEvent();
+ event2.setTimestamp(DateTimeUtil.humanDateToSeconds("2016-01-01 00:00:00")*1000);
+ data = new Object[]{"value2"};
+ event2.setData(data);
+ event2.setStreamId(streamId);
+ PartitionedEvent partitionedEvent2 = new PartitionedEvent(event2, sp,1001);
+
+ Tuple input = new TupleImpl(context, Collections.singletonList(partitionedEvent1), taskId, "default");
+ Tuple input2 = new TupleImpl(context, Collections.singletonList(partitionedEvent2), taskId, "default");
+ bolt.execute(input);
+ bolt.execute(input2);
+ Assert.assertTrue("Timeout to acquire mutex in 5s",mutex.tryAcquire(2, 5, TimeUnit.SECONDS));
+ Assert.assertEquals(2, alertCount.get());
+ bolt.cleanup();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/router/TestAlertPublisherBolt.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/router/TestAlertPublisherBolt.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/router/TestAlertPublisherBolt.java
new file mode 100644
index 0000000..af79f96
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/router/TestAlertPublisherBolt.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.alert.engine.router;
+
+import com.fasterxml.jackson.databind.JavaType;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.type.CollectionType;
+import com.fasterxml.jackson.databind.type.SimpleType;
+import org.apache.eagle.alert.coordination.model.PublishSpec;
+import org.apache.eagle.alert.engine.coordinator.PolicyDefinition;
+import org.apache.eagle.alert.engine.coordinator.Publishment;
+import org.apache.eagle.alert.engine.model.AlertStreamEvent;
+import org.apache.eagle.alert.engine.publisher.AlertPublisher;
+import org.apache.eagle.alert.engine.publisher.impl.AlertPublisherImpl;
+import org.apache.eagle.alert.engine.runner.AlertPublisherBolt;
+import org.apache.eagle.alert.engine.runner.MapComparator;
+import org.apache.eagle.alert.engine.utils.MetadataSerDeser;
+import org.junit.Assert;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @Since 5/14/16.
+ */
+public class TestAlertPublisherBolt {
+
+ @Ignore
+ @Test
+ public void test() {
+ Config config = ConfigFactory.load("application-test.conf");
+ AlertPublisher publisher = new AlertPublisherImpl("alertPublishBolt");
+ publisher.init(config);
+ PublishSpec spec = MetadataSerDeser.deserialize(getClass().getResourceAsStream("/testPublishSpec.json"), PublishSpec.class);
+ publisher.onPublishChange(spec.getPublishments(), null, null, null);
+ AlertStreamEvent event = create("testAlertStream");
+ publisher.nextEvent(event);
+ AlertStreamEvent event1 = create("testAlertStream");
+ publisher.nextEvent(event1);
+ }
+
+ private AlertStreamEvent create(String streamId){
+ AlertStreamEvent alert = new AlertStreamEvent();
+ PolicyDefinition policy = new PolicyDefinition();
+ policy.setName("policy1");
+ alert.setPolicy(policy);
+ alert.setCreatedTime(System.currentTimeMillis());
+ alert.setData(new Object[]{"field_1", 2, "field_3"});
+ alert.setStreamId(streamId);
+ alert.setCreatedBy(this.toString());
+ return alert;
+ }
+
+ @Test
+ public void testMapComparator() {
+ PublishSpec spec1 = MetadataSerDeser.deserialize(getClass().getResourceAsStream("/testPublishSpec.json"), PublishSpec.class);
+ PublishSpec spec2 = MetadataSerDeser.deserialize(getClass().getResourceAsStream("/testPublishSpec2.json"), PublishSpec.class);
+ Map<String, Publishment> map1 = new HashMap<>();
+ Map<String, Publishment> map2 = new HashMap<>();
+ spec1.getPublishments().forEach(p -> map1.put(p.getName(), p));
+ spec2.getPublishments().forEach(p -> map2.put(p.getName(), p));
+
+ MapComparator<String, Publishment> comparator = new MapComparator<>(map1, map2);
+ comparator.compare();
+ Assert.assertTrue(comparator.getModified().size() == 1);
+
+ AlertPublisher alertPublisher = new AlertPublisherImpl("alert-publisher-test");
+ AlertPublisherBolt publisherBolt = new AlertPublisherBolt(alertPublisher, null, null);
+ publisherBolt.onAlertPublishSpecChange(spec1, null);
+ publisherBolt.onAlertPublishSpecChange(spec2, null);
+ }
+
+ @Test
+ public void testAlertPublisher() throws Exception {
+ AlertPublisher alertPublisher = new AlertPublisherImpl("alert-publisher-test");
+ List<Publishment> oldPubs = loadEntities("/publishments.json", Publishment.class);
+ List<Publishment> newPubs = loadEntities("/publishments2.json", Publishment.class);
+ alertPublisher.onPublishChange(oldPubs, null, null, null);
+ alertPublisher.onPublishChange(null, null, newPubs, oldPubs);
+ }
+
+ private <T> List<T> loadEntities(String path, Class<T> tClz) throws Exception {
+ ObjectMapper objectMapper = new ObjectMapper();
+ JavaType type = CollectionType.construct(List.class, SimpleType.construct(tClz));
+ List<T> l = objectMapper.readValue(TestAlertPublisherBolt.class.getResourceAsStream(path), type);
+ return l;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/runner/TestStreamRouterBolt.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/runner/TestStreamRouterBolt.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/runner/TestStreamRouterBolt.java
new file mode 100644
index 0000000..8c048cb
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/runner/TestStreamRouterBolt.java
@@ -0,0 +1,259 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.runner;
+
+import backtype.storm.metric.api.MultiCountMetric;
+import backtype.storm.task.GeneralTopologyContext;
+import backtype.storm.task.IOutputCollector;
+import backtype.storm.task.OutputCollector;
+import backtype.storm.task.TopologyContext;
+import backtype.storm.tuple.Fields;
+import backtype.storm.tuple.Tuple;
+import backtype.storm.tuple.TupleImpl;
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+import org.apache.eagle.alert.coordination.model.PolicyWorkerQueue;
+import org.apache.eagle.alert.coordination.model.RouterSpec;
+import org.apache.eagle.alert.coordination.model.StreamRouterSpec;
+import org.apache.eagle.alert.coordination.model.WorkSlot;
+import org.apache.eagle.alert.engine.coordinator.StreamColumn;
+import org.apache.eagle.alert.engine.coordinator.StreamDefinition;
+import org.apache.eagle.alert.engine.coordinator.StreamPartition;
+import org.apache.eagle.alert.engine.coordinator.StreamSortSpec;
+import org.apache.eagle.alert.engine.coordinator.impl.AbstractMetadataChangeNotifyService;
+import org.apache.eagle.alert.engine.model.PartitionedEvent;
+import org.apache.eagle.alert.engine.model.StreamEvent;
+import org.apache.eagle.alert.engine.router.impl.StreamRouterImpl;
+import org.apache.eagle.alert.utils.DateTimeUtil;
+import org.joda.time.Period;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.*;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class TestStreamRouterBolt {
+ private final static Logger LOG = LoggerFactory.getLogger(TestStreamRouterBolt.class);
+
+ /**
+ * Mocked 5 Events
+ *
+ * 1. Sent in random order:
+ * "value1","value2","value3","value4","value5"
+ *
+ * 2. Received correct time order and value5 is thrown because too late: "value2","value1","value3","value4"
+ *
+ * @throws Exception
+ */
+ @SuppressWarnings("rawtypes")
+ @Test
+ public void testRouterWithSortAndRouteSpec() throws Exception{
+ Config config = ConfigFactory.load();
+ StreamRouterImpl routerImpl = new StreamRouterImpl("testStreamRouterImpl");
+ MockChangeService mockChangeService = new MockChangeService();
+ StreamRouterBolt bolt = new StreamRouterBolt(routerImpl, config, mockChangeService);
+
+ final Map<String,List<PartitionedEvent>> streamCollected = new HashMap<>();
+ final List<PartitionedEvent> orderCollected = new ArrayList<>();
+
+ OutputCollector collector = new OutputCollector(new IOutputCollector(){
+ int count = 0;
+ @Override
+ public List<Integer> emit(String streamId, Collection<Tuple> anchors, List<Object> tuple) {
+ PartitionedEvent event;
+ try {
+ event = bolt.deserialize(tuple.get(0));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ if(count == 0) {
+ count++;
+ }
+ LOG.info(String.format("Collector received: [streamId=[%s], tuple=[%s] ", streamId, tuple));
+ if(!streamCollected.containsKey(streamId)){
+ streamCollected.put(streamId,new ArrayList<>());
+ }
+ streamCollected.get(streamId).add(event);
+ orderCollected.add(event);
+ return null;
+ }
+ @Override
+ public void emitDirect(int taskId, String streamId, Collection<Tuple> anchors, List<Object> tuple) { }
+ @Override
+ public void ack(Tuple input) { }
+ @Override
+ public void fail(Tuple input) { }
+ @SuppressWarnings("unused")
+ public void resetTimeout(Tuple input) { }
+ @Override
+ public void reportError(Throwable error) { }
+ });
+
+ Map stormConf = new HashMap<>();
+ TopologyContext topologyContext = mock(TopologyContext.class);
+ when(topologyContext.registerMetric(any(String.class), any(MultiCountMetric.class), any(int.class))).thenReturn(new MultiCountMetric());
+ bolt.prepare(stormConf, topologyContext, collector);
+
+ String streamId = "cpuUsageStream";
+ // StreamPartition, groupby col1 for stream cpuUsageStream
+ StreamPartition sp = new StreamPartition();
+ sp.setStreamId(streamId);
+ sp.setColumns(Collections.singletonList("col1"));
+ sp.setType(StreamPartition.Type.GROUPBY);
+
+ StreamSortSpec sortSpec = new StreamSortSpec();
+// sortSpec.setColumn("timestamp");
+// sortSpec.setOrder("asc");
+ sortSpec.setWindowPeriod2(Period.minutes(1));
+ sortSpec.setWindowMargin(1000);
+ sp.setSortSpec(sortSpec);
+
+ RouterSpec boltSpec = new RouterSpec();
+
+ // set StreamRouterSpec to have 2 WorkSlot
+ StreamRouterSpec routerSpec = new StreamRouterSpec();
+ routerSpec.setPartition(sp);
+ routerSpec.setStreamId(streamId);
+ PolicyWorkerQueue queue = new PolicyWorkerQueue();
+ queue.setPartition(sp);
+ queue.setWorkers(Arrays.asList(new WorkSlot("testTopology","alertBolt1"), new WorkSlot("testTopology","alertBolt2")));
+ routerSpec.setTargetQueue(Collections.singletonList(queue));
+ boltSpec.addRouterSpec(routerSpec);
+
+ // construct StreamDefinition
+ StreamDefinition schema = new StreamDefinition();
+ schema.setStreamId(streamId);
+ StreamColumn column = new StreamColumn();
+ column.setName("col1");
+ column.setType(StreamColumn.Type.STRING);
+ schema.setColumns(Collections.singletonList(column));
+ Map<String, StreamDefinition> sds = new HashMap<>();
+ sds.put(schema.getStreamId(), schema);
+
+ bolt.declareOutputStreams(Arrays.asList("alertBolt1", "alertBolt2"));
+ bolt.onStreamRouteBoltSpecChange(boltSpec, sds);
+ GeneralTopologyContext context = mock(GeneralTopologyContext.class);
+ int taskId = 1;
+ when(context.getComponentId(taskId)).thenReturn("comp1");
+ when(context.getComponentOutputFields("comp1", "default")).thenReturn(new Fields("f0"));
+
+ // =======================================
+ // Mock 5 Events
+ //
+ // 1. Sent in random order:
+ // "value1","value2","value3","value4","value5"
+ //
+ // 2. Received correct time order and value5 is thrown because too:
+ // "value2","value1","value3","value4"
+ // =======================================
+
+ // construct event with "value1"
+ StreamEvent event = new StreamEvent();
+ event.setTimestamp(DateTimeUtil.humanDateToSeconds("2016-01-01 00:01:30")*1000);
+ Object[] data = new Object[]{"value1"};
+ event.setData(data);
+ event.setStreamId(streamId);
+ PartitionedEvent pEvent = new PartitionedEvent();
+ pEvent.setEvent(event);
+ pEvent.setPartition(sp);
+ Tuple input = new TupleImpl(context, Collections.singletonList(pEvent), taskId, "default");
+ bolt.execute(input);
+
+ // construct another event with "value2"
+ event = new StreamEvent();
+ event.setTimestamp(DateTimeUtil.humanDateToSeconds("2016-01-01 00:01:10")*1000);
+ data = new Object[]{"value2"};
+ event.setData(data);
+ event.setStreamId(streamId);
+ pEvent = new PartitionedEvent();
+ pEvent.setPartition(sp);
+ pEvent.setEvent(event);
+ input = new TupleImpl(context, Collections.singletonList(pEvent), taskId, "default");
+ bolt.execute(input);
+
+ // construct another event with "value3"
+ event = new StreamEvent();
+ event.setTimestamp(DateTimeUtil.humanDateToSeconds("2016-01-01 00:01:40")*1000);
+ data = new Object[]{"value3"};
+ event.setData(data);
+ event.setStreamId(streamId);
+ pEvent = new PartitionedEvent();
+ pEvent.setPartition(sp);
+ pEvent.setEvent(event);
+ input = new TupleImpl(context, Collections.singletonList(pEvent), taskId, "default");
+ bolt.execute(input);
+
+ // construct another event with "value4"
+ event = new StreamEvent();
+ event.setTimestamp(DateTimeUtil.humanDateToSeconds("2016-01-01 00:02:10")*1000);
+ data = new Object[]{"value4"};
+ event.setData(data);
+ event.setStreamId(streamId);
+ pEvent = new PartitionedEvent();
+ pEvent.setPartition(sp);
+ pEvent.setEvent(event);
+ input = new TupleImpl(context, Collections.singletonList(pEvent), taskId, "default");
+ bolt.execute(input);
+
+ // construct another event with "value5", which will be thrown because two late
+ event = new StreamEvent();
+ event.setTimestamp(DateTimeUtil.humanDateToSeconds("2016-01-01 00:00:10")*1000);
+ data = new Object[]{"value5"};
+ event.setData(data);
+ event.setStreamId(streamId);
+ pEvent = new PartitionedEvent();
+ pEvent.setPartition(sp);
+ pEvent.setEvent(event);
+ input = new TupleImpl(context, Collections.singletonList(pEvent), taskId, "default");
+ bolt.execute(input);
+
+ Assert.assertEquals("Should finally collect two streams",2,streamCollected.size());
+ Assert.assertTrue("Should collect stream stream_testStreamRouterImpl_to_alertBolt1",streamCollected.keySet().contains("stream_testStreamRouterImpl_to_alertBolt1"));
+ Assert.assertTrue("Should collect stream stream_testStreamRouterImpl_to_alertBolt2",streamCollected.keySet().contains("stream_testStreamRouterImpl_to_alertBolt2"));
+
+ Assert.assertEquals("Should finally collect 3 events",3,orderCollected.size());
+ Assert.assertArrayEquals("Should sort 3 events in ASC order",new String[]{"value2","value1","value3"},orderCollected.stream().map((d)->d.getData()[0]).toArray());
+
+ // The first 3 events are ticked automatically by window
+
+ bolt.cleanup();
+
+ // Close will flush all events in memory, so will receive the last event which is still in memory as window is not expired according to clock
+ // The 5th event will be thrown because too late and out of margin
+
+ Assert.assertEquals("Should finally collect two streams",2,streamCollected.size());
+ Assert.assertEquals("Should finally collect 3 events",4,orderCollected.size());
+ Assert.assertArrayEquals("Should sort 4 events in ASC-ordered timestamp",new String[]{"value2","value1","value3","value4"},orderCollected.stream().map((d)->d.getData()[0]).toArray());
+
+ }
+
+ @SuppressWarnings("serial")
+ public static class MockChangeService extends AbstractMetadataChangeNotifyService{
+ private final static Logger LOG = LoggerFactory.getLogger(MockChangeService.class);
+
+ @Override
+ public void close() throws IOException {
+ LOG.info("Closing");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/serialization/JavaSerializationTest.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/serialization/JavaSerializationTest.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/serialization/JavaSerializationTest.java
new file mode 100644
index 0000000..13d1015
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/serialization/JavaSerializationTest.java
@@ -0,0 +1,112 @@
+package org.apache.eagle.alert.engine.serialization;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+
+import org.apache.commons.lang3.SerializationUtils;
+import org.apache.eagle.alert.engine.coordinator.StreamColumn;
+import org.apache.eagle.alert.engine.coordinator.StreamDefinition;
+import org.apache.eagle.alert.engine.coordinator.StreamPartition;
+import org.apache.eagle.alert.engine.model.PartitionedEvent;
+import org.apache.eagle.alert.engine.model.StreamEvent;
+import org.apache.eagle.alert.utils.ByteUtils;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+public class JavaSerializationTest {
+ private final static Logger LOG = LoggerFactory.getLogger(JavaSerializationTest.class);
+
+ @Test
+ public void testJavaSerialization(){
+ PartitionedEvent partitionedEvent = new PartitionedEvent();
+ partitionedEvent.setPartitionKey(partitionedEvent.hashCode());
+ partitionedEvent.setPartition(createSampleStreamGroupbyPartition("sampleStream",Arrays.asList("name","host")));
+ StreamEvent event = new StreamEvent();
+ event.setStreamId("sampleStream");
+ event.setTimestamp(System.currentTimeMillis());
+ event.setData(new Object[]{"CPU","LOCALHOST",true,Long.MAX_VALUE,60.0});
+ partitionedEvent.setEvent(event);
+
+ int javaSerializationLength = SerializationUtils.serialize(partitionedEvent).length;
+ LOG.info("Java serialization length: {}, event: {}",javaSerializationLength,partitionedEvent);
+
+ int compactLength = 0;
+ compactLength += "sampleStream".getBytes().length;
+ compactLength += ByteUtils.intToBytes(partitionedEvent.getPartition().hashCode()).length;
+ compactLength += ByteUtils.longToBytes(partitionedEvent.getTimestamp()).length;
+ compactLength += "CPU".getBytes().length;
+ compactLength += "LOCALHOST".getBytes().length;
+ compactLength += 1;
+ compactLength += ByteUtils.longToBytes(Long.MAX_VALUE).length;
+ compactLength += ByteUtils.doubleToBytes(60.0).length;
+
+ LOG.info("Compact serialization length: {}, event: {}",compactLength,partitionedEvent);
+ Assert.assertTrue(compactLength * 20 < javaSerializationLength);
+ }
+
+
+ public static StreamDefinition createSampleStreamDefinition(String streamId){
+ StreamDefinition sampleStreamDefinition = new StreamDefinition();
+ sampleStreamDefinition.setStreamId(streamId);
+ sampleStreamDefinition.setTimeseries(true);
+ sampleStreamDefinition.setValidate(true);
+ sampleStreamDefinition.setDescription("Schema for "+streamId);
+ List<StreamColumn> streamColumns = new ArrayList<>();
+
+ streamColumns.add(new StreamColumn.Builder().name("name").type(StreamColumn.Type.STRING).build());
+ streamColumns.add(new StreamColumn.Builder().name("host").type(StreamColumn.Type.STRING).build());
+ streamColumns.add(new StreamColumn.Builder().name("flag").type(StreamColumn.Type.BOOL).build());
+ streamColumns.add(new StreamColumn.Builder().name("data").type(StreamColumn.Type.LONG).build());
+ streamColumns.add(new StreamColumn.Builder().name("value").type(StreamColumn.Type.DOUBLE).build());
+ sampleStreamDefinition.setColumns(streamColumns);
+ return sampleStreamDefinition;
+ }
+
+ public static StreamPartition createSampleStreamGroupbyPartition(String streamId, List<String> groupByField){
+ StreamPartition streamPartition = new StreamPartition();
+ streamPartition.setStreamId(streamId);
+ streamPartition.setColumns(groupByField);
+ streamPartition.setType(StreamPartition.Type.GROUPBY);
+ return streamPartition;
+ }
+
+ @SuppressWarnings("serial")
+ public static PartitionedEvent createSimpleStreamEvent() {
+ StreamEvent event = StreamEvent.Builder()
+ .schema(createSampleStreamDefinition("sampleStream_1"))
+ .streamId("sampleStream_1")
+ .timestamep(System.currentTimeMillis())
+ .attributes(new HashMap<String,Object>(){{
+ put("name","cpu");
+ put("host","localhost");
+ put("flag",true);
+ put("value",60.0);
+ put("data",Long.MAX_VALUE);
+ put("unknown","unknown column value");
+ }}).build();
+ PartitionedEvent pEvent = new PartitionedEvent();
+ pEvent.setEvent(event);
+ pEvent.setPartition(createSampleStreamGroupbyPartition("sampleStream_1", Arrays.asList("name","host")));
+ return pEvent;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/serialization/PartitionedEventSerializerTest.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/serialization/PartitionedEventSerializerTest.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/serialization/PartitionedEventSerializerTest.java
new file mode 100644
index 0000000..0347d50
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/serialization/PartitionedEventSerializerTest.java
@@ -0,0 +1,220 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.serialization;
+
+import backtype.storm.serialization.DefaultKryoFactory;
+import backtype.storm.serialization.DefaultSerializationDelegate;
+import com.esotericsoftware.kryo.*;
+import com.esotericsoftware.kryo.io.Input;
+import com.esotericsoftware.kryo.io.Output;
+import com.google.common.io.ByteArrayDataInput;
+import com.google.common.io.ByteArrayDataOutput;
+import com.google.common.io.ByteStreams;
+import org.apache.commons.lang.time.StopWatch;
+import org.apache.eagle.alert.engine.mock.MockSampleMetadataFactory;
+import org.apache.eagle.alert.engine.model.PartitionedEvent;
+import org.apache.eagle.alert.engine.serialization.impl.PartitionedEventSerializerImpl;
+import org.apache.eagle.alert.utils.TimePeriodUtils;
+import org.joda.time.Period;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.BitSet;
+
+
+public class PartitionedEventSerializerTest {
+ private final static Logger LOG = LoggerFactory.getLogger(PartitionedEventSerializerTest.class);
+ @Test
+ public void testPartitionEventSerialization() throws IOException {
+ PartitionedEvent partitionedEvent = MockSampleMetadataFactory.createPartitionedEventGroupedByName("sampleStream",System.currentTimeMillis());;
+ PartitionedEventSerializerImpl serializer = new PartitionedEventSerializerImpl(MockSampleMetadataFactory::createSampleStreamDefinition);
+
+ ByteArrayDataOutput dataOutput1 = ByteStreams.newDataOutput();
+ serializer.serialize(partitionedEvent,dataOutput1);
+ byte[] serializedBytes = dataOutput1.toByteArray();
+ PartitionedEvent deserializedEvent = serializer.deserialize(ByteStreams.newDataInput(serializedBytes));
+ Assert.assertEquals(partitionedEvent,deserializedEvent);
+
+ PartitionedEventSerializerImpl compressSerializer = new PartitionedEventSerializerImpl(MockSampleMetadataFactory::createSampleStreamDefinition,true);
+
+ byte[] serializedBytesCompressed = compressSerializer.serialize(partitionedEvent);
+ PartitionedEvent deserializedEventCompressed = compressSerializer.deserialize(serializedBytesCompressed);
+ Assert.assertEquals(partitionedEvent,deserializedEventCompressed);
+
+ PartitionedEventDigestSerializer serializer2 = new PartitionedEventDigestSerializer(MockSampleMetadataFactory::createSampleStreamDefinition);
+ ByteArrayDataOutput dataOutput2 = ByteStreams.newDataOutput();
+ serializer2.serialize(partitionedEvent,dataOutput2);
+ byte[] serializedBytes2 = dataOutput2.toByteArray();
+ ByteArrayDataInput dataInput2 = ByteStreams.newDataInput(serializedBytes2);
+ PartitionedEvent deserializedEvent2 = serializer2.deserialize(dataInput2);
+ Assert.assertEquals(partitionedEvent,deserializedEvent2);
+
+ byte[] javaSerialization = new DefaultSerializationDelegate().serialize(partitionedEvent);
+ Kryo kryo = new DefaultKryoFactory.KryoSerializableDefault();
+ Output output = new Output(10000);
+ kryo.writeClassAndObject(output,partitionedEvent);
+ byte[] kryoBytes = output.toBytes();
+ Input input = new Input(kryoBytes);
+ PartitionedEvent kryoDeserializedEvent = (PartitionedEvent) kryo.readClassAndObject(input);
+ Assert.assertEquals(partitionedEvent,kryoDeserializedEvent);
+ LOG.info("\nCached Stream:{}\nCompressed Cached Stream :{}\nCached Stream + Cached Partition: {}\nJava Native: {}\nKryo: {}\nKryo + Cached Stream: {}\nKryo + Cached Stream + Cached Partition: {}",serializedBytes.length,serializedBytesCompressed.length,serializedBytes2.length,javaSerialization.length,kryoBytes.length,kryoSerialize(serializedBytes).length,kryoSerialize(serializedBytes2).length);
+ }
+ @Test
+ public void testPartitionEventSerializationEfficiency() throws IOException {
+ PartitionedEvent partitionedEvent = MockSampleMetadataFactory.createPartitionedEventGroupedByName("sampleStream",System.currentTimeMillis());;
+ PartitionedEventSerializerImpl serializer = new PartitionedEventSerializerImpl(MockSampleMetadataFactory::createSampleStreamDefinition);
+
+ int count = 100000;
+ StopWatch stopWatch = new StopWatch();
+ stopWatch.start();
+ int i = 0;
+ while(i<count) {
+ ByteArrayDataOutput dataOutput1 = ByteStreams.newDataOutput();
+ serializer.serialize(partitionedEvent, dataOutput1);
+ byte[] serializedBytes = dataOutput1.toByteArray();
+ PartitionedEvent deserializedEvent = serializer.deserialize(ByteStreams.newDataInput(serializedBytes));
+ Assert.assertEquals(partitionedEvent, deserializedEvent);
+ i++;
+ }
+ stopWatch.stop();
+ LOG.info("Cached Stream: {} ms",stopWatch.getTime());
+ stopWatch.reset();
+ PartitionedEventSerializerImpl compressSerializer = new PartitionedEventSerializerImpl(MockSampleMetadataFactory::createSampleStreamDefinition,true);
+ i = 0;
+ stopWatch.start();
+ while(i<count) {
+ byte[] serializedBytesCompressed = compressSerializer.serialize(partitionedEvent);
+ PartitionedEvent deserializedEventCompressed = compressSerializer.deserialize(serializedBytesCompressed);
+ Assert.assertEquals(partitionedEvent, deserializedEventCompressed);
+ i++;
+ }
+ stopWatch.stop();
+ LOG.info("Compressed Cached Stream: {} ms",stopWatch.getTime());
+ stopWatch.reset();
+
+ i = 0;
+ stopWatch.start();
+ while(i<count) {
+ PartitionedEventDigestSerializer serializer2 = new PartitionedEventDigestSerializer(MockSampleMetadataFactory::createSampleStreamDefinition);
+ ByteArrayDataOutput dataOutput2 = ByteStreams.newDataOutput();
+ serializer2.serialize(partitionedEvent, dataOutput2);
+ byte[] serializedBytes2 = dataOutput2.toByteArray();
+ ByteArrayDataInput dataInput2 = ByteStreams.newDataInput(serializedBytes2);
+ PartitionedEvent deserializedEvent2 = serializer2.deserialize(dataInput2);
+ Assert.assertEquals(partitionedEvent, deserializedEvent2);
+ i++;
+ }
+ stopWatch.stop();
+ LOG.info("Cached Stream&Partition: {} ms",stopWatch.getTime());
+ stopWatch.reset();
+ i = 0;
+ stopWatch.start();
+ while(i<count) {
+ byte[] javaSerialization = new DefaultSerializationDelegate().serialize(partitionedEvent);
+ PartitionedEvent javaSerializedEvent = (PartitionedEvent) new DefaultSerializationDelegate().deserialize(javaSerialization);
+ Assert.assertEquals(partitionedEvent, javaSerializedEvent);
+ i++;
+ }
+ stopWatch.stop();
+ LOG.info("Java Native: {} ms",stopWatch.getTime());
+ stopWatch.reset();
+ i = 0;
+ stopWatch.start();
+ Kryo kryo = new DefaultKryoFactory.KryoSerializableDefault();
+ while(i<count) {
+ Output output = new Output(10000);
+ kryo.writeClassAndObject(output, partitionedEvent);
+ byte[] kryoBytes = output.toBytes();
+ Input input = new Input(kryoBytes);
+ PartitionedEvent kryoDeserializedEvent = (PartitionedEvent) kryo.readClassAndObject(input);
+ Assert.assertEquals(partitionedEvent, kryoDeserializedEvent);
+ i++;
+ }
+ stopWatch.stop();
+ LOG.info("Kryo: {} ms",stopWatch.getTime());
+ }
+
+ /**
+ * Kryo Serialization Length = Length of byte[] + 2
+ */
+ @Test
+ public void testKryoByteArraySerialization(){
+ Kryo kryo = new DefaultKryoFactory.KryoSerializableDefault();
+ byte[] bytes = new byte[]{0,1,2,3,4,5,6,7,8,9};
+ Output output = new Output(1000);
+ kryo.writeObject(output,bytes);
+ Assert.assertEquals(bytes.length + 2,output.toBytes().length);
+ }
+
+ private byte[] kryoSerialize(Object object){
+ Kryo kryo = new DefaultKryoFactory.KryoSerializableDefault();
+ Output output = new Output(100000);
+ kryo.writeClassAndObject(output,object);
+ return output.toBytes();
+ }
+
+ @Test
+ public void testBitSet(){
+ BitSet bitSet = new BitSet();
+ bitSet.set(0,true); // 1
+ bitSet.set(1,false); // 0
+ bitSet.set(2,true); // 1
+ LOG.info("Bit Set Size: {}",bitSet.size());
+ LOG.info("Bit Set Byte[]: {}",bitSet.toByteArray());
+ LOG.info("Bit Set Byte[]: {}",bitSet.toLongArray());
+ LOG.info("BitSet[0]: {}",bitSet.get(0));
+ LOG.info("BitSet[1]: {}",bitSet.get(1));
+ LOG.info("BitSet[1]: {}",bitSet.get(2));
+
+ byte[] bytes = bitSet.toByteArray();
+
+ BitSet bitSet2 = BitSet.valueOf(bytes);
+
+ LOG.info("Bit Set Size: {}",bitSet2.size());
+ LOG.info("Bit Set Byte[]: {}",bitSet2.toByteArray());
+ LOG.info("Bit Set Byte[]: {}",bitSet2.toLongArray());
+ LOG.info("BitSet[0]: {}",bitSet2.get(0));
+ LOG.info("BitSet[1]: {}",bitSet2.get(1));
+ LOG.info("BitSet[1]: {}",bitSet2.get(2));
+
+
+ BitSet bitSet3 = new BitSet();
+ bitSet3.set(0,true);
+ Assert.assertEquals(1,bitSet3.length());
+
+ BitSet bitSet4 = new BitSet();
+ bitSet4.set(0,false);
+ Assert.assertEquals(0,bitSet4.length());
+ Assert.assertFalse(bitSet4.get(1));
+ Assert.assertFalse(bitSet4.get(2));
+ }
+
+ @Test
+ public void testPeriod(){
+ Assert.assertEquals(30*60*1000, TimePeriodUtils.getMillisecondsOfPeriod(Period.parse("PT30m")));
+ Assert.assertEquals(30*60*1000, TimePeriodUtils.getMillisecondsOfPeriod(Period.millis(30*60*1000)));
+ Assert.assertEquals("PT1800S", Period.millis(30*60*1000).toString());
+ }
+
+ @Test
+ public void testPartitionType(){
+
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/siddhi/extension/AttributeCollectAggregatorTest.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/siddhi/extension/AttributeCollectAggregatorTest.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/siddhi/extension/AttributeCollectAggregatorTest.java
new file mode 100644
index 0000000..f2f3b46
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/siddhi/extension/AttributeCollectAggregatorTest.java
@@ -0,0 +1,149 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+package org.apache.eagle.alert.engine.siddhi.extension;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Random;
+
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.wso2.siddhi.core.ExecutionPlanRuntime;
+import org.wso2.siddhi.core.SiddhiManager;
+import org.wso2.siddhi.core.event.Event;
+import org.wso2.siddhi.core.stream.input.InputHandler;
+import org.wso2.siddhi.core.stream.output.StreamCallback;
+
+/**
+ * @since Apr 1, 2016
+ *
+ */
+public class AttributeCollectAggregatorTest {
+
+ private static final Logger logger = LoggerFactory.getLogger(AttributeCollectAggregatorTest.class);
+
+ @Test
+ public void test() throws Exception {
+ String ql = "define stream s1(timestamp long, host string, type string);";
+ ql += " from s1#window.externalTime(timestamp, 1 sec)";
+ ql += " select eagle:collect(timestamp) as timestamps, eagle:collect(host) as hosts, type group by type insert into output;";
+
+ SiddhiManager sm = new SiddhiManager();
+ ExecutionPlanRuntime runtime = sm.createExecutionPlanRuntime(ql);
+
+ InputHandler input = runtime.getInputHandler("s1");
+ runtime.addCallback("output", new StreamCallback() {
+
+ @Override
+ public void receive(Event[] arg0) {
+ logger.info("output event length:" + arg0.length);
+
+ for (Event e : arg0) {
+ StringBuilder sb = new StringBuilder("\t - [").append(e.getData().length).append("]");
+ for (Object o : e.getData()) {
+ sb.append("," + o);
+ }
+ logger.info(sb.toString());
+ }
+ logger.info("===end===");
+ }
+ });
+// StreamDefinition definition = (StreamDefinition) runtime.getStreamDefinitionMap().get("output");
+
+ runtime.start();
+
+ Event[] events = generateEvents();
+ for (Event e : events) {
+ input.send(e);
+ }
+
+ Thread.sleep(1000);
+
+ }
+
+ private Event[] generateEvents() {
+ List<Event> events = new LinkedList<Event>();
+
+ Random r = new Random();
+ Event e = null;
+ long base = System.currentTimeMillis();
+ {
+ e = new Event(base, new Object[] { base, "host" + r.nextInt(), "nova" });
+ base += 100;
+ events.add(e);
+ e = new Event(base, new Object[] { base, "host" + r.nextInt(), "nova" });
+ base += 100;
+ events.add(e);
+ e = new Event(base, new Object[] { base, "host" + r.nextInt(), "nova" });
+ base += 100;
+ events.add(e);
+ }
+
+ {
+ e = new Event(base, new Object[] { base, "host" + r.nextInt(), "neutron" });
+ base += 100;
+ events.add(e);
+ e = new Event(base, new Object[] { base, "host" + r.nextInt(), "neutron" });
+ base += 100;
+ events.add(e);
+ e = new Event(base, new Object[] { base, "host" + r.nextInt(), "neutron" });
+ base += 100;
+ events.add(e);
+ }
+
+ base += 10000;
+ {
+ e = new Event(base, new Object[] { base, "host" + r.nextInt(), "nova1" });
+ base += 100;
+ events.add(e);
+ e = new Event(base, new Object[] { base, "host" + r.nextInt(), "nova1" });
+ base += 100;
+ events.add(e);
+ e = new Event(base, new Object[] { base, "host" + r.nextInt(), "nova1" });
+ base += 100;
+ events.add(e);
+ }
+
+ {
+ e = new Event(base, new Object[] { base, "host" + r.nextInt(), "neutron2" });
+ base += 100;
+ events.add(e);
+ e = new Event(base, new Object[] { base, "host" + r.nextInt(), "neutron2" });
+ base += 100;
+ events.add(e);
+ e = new Event(base, new Object[] { base, "host" + r.nextInt(), "neutron2" });
+ base += 100;
+ events.add(e);
+ }
+ base += 10000;
+ e = new Event(base, new Object[] { base, "host" + r.nextInt(), "mq" });
+
+ return events.toArray(new Event[0]);
+ }
+
+ @Test
+ public void testQuery() {
+ String ql = "define stream perfmon_input_stream_cpu ( host string,timestamp long,metric string,pool string,value double,colo string );";
+ ql += "from perfmon_input_stream_cpu#window.length(3) select host, min(value) as min group by host having min>91.0 insert into perfmon_output_stream_cpu;";
+
+ SiddhiManager sm = new SiddhiManager();
+ sm.createExecutionPlanRuntime(ql);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/sorter/MapDBTestSuite.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/sorter/MapDBTestSuite.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/sorter/MapDBTestSuite.java
new file mode 100644
index 0000000..613be00
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/sorter/MapDBTestSuite.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.sorter;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.mapdb.BTreeMap;
+import org.mapdb.DB;
+import org.mapdb.DBMaker;
+import org.mapdb.Serializer;
+
+public class MapDBTestSuite {
+ @Test
+ public void testOnHeapDB(){
+ DB db = DBMaker.heapDB().make();
+ BTreeMap<Long,String> map = db.treeMap("btree").keySerializer(Serializer.LONG).valueSerializer(Serializer.STRING).create();
+ Assert.assertFalse(map.putIfAbsentBoolean(1L,"val_1"));
+ Assert.assertTrue(map.putIfAbsentBoolean(1L,"val_2"));
+ Assert.assertTrue(map.putIfAbsentBoolean(1L,"val_3"));
+ Assert.assertFalse(map.putIfAbsentBoolean(2L,"val_4"));
+
+ Assert.assertEquals("val_1",map.get(1L));
+ Assert.assertEquals("val_4",map.get(2L));
+
+ Assert.assertTrue(map.replace(2L,"val_4","val_5"));
+ Assert.assertEquals("val_5",map.get(2L));
+
+ map.close();
+ db.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/sorter/StreamSortHandlerTest.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/sorter/StreamSortHandlerTest.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/sorter/StreamSortHandlerTest.java
new file mode 100644
index 0000000..6cadba7
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/sorter/StreamSortHandlerTest.java
@@ -0,0 +1,270 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.sorter;
+
+import java.lang.management.ManagementFactory;
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Timer;
+import java.util.TimerTask;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.lang.time.StopWatch;
+import org.apache.eagle.alert.engine.mock.MockPartitionedCollector;
+import org.apache.eagle.alert.engine.mock.MockSampleMetadataFactory;
+import org.apache.eagle.alert.engine.model.PartitionedEvent;
+import org.apache.eagle.alert.engine.sorter.impl.PartitionedEventTimeOrderingComparator;
+import org.apache.eagle.alert.engine.sorter.impl.StreamSortWindowHandlerImpl;
+import org.apache.eagle.alert.engine.sorter.impl.StreamTimeClockInLocalMemory;
+import org.apache.eagle.alert.utils.DateTimeUtil;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.ScheduledReporter;
+import com.codahale.metrics.Slf4jReporter;
+import com.codahale.metrics.jvm.GarbageCollectorMetricSet;
+import com.codahale.metrics.jvm.MemoryUsageGaugeSet;
+import com.google.common.collect.Ordering;
+
+/**
+ * -XX:+UseConcMarkSweepGC -XX:+UseParNewGC -XX:+PrintGCTaskTimeStamps -XX:+PrintGCDetails -verbose:gc
+ */
+public class StreamSortHandlerTest {
+ private final static Logger LOG = LoggerFactory.getLogger(StreamSortHandlerTest.class);
+
+ static {
+ LOG.info(ManagementFactory.getRuntimeMXBean().getName());
+ }
+
+ private ScheduledReporter metricReporter;
+ @Before
+ public void setUp(){
+ final MetricRegistry metrics = new MetricRegistry();
+ metrics.registerAll(new MemoryUsageGaugeSet());
+ metrics.registerAll(new GarbageCollectorMetricSet());
+ metricReporter = Slf4jReporter.forRegistry(metrics)
+ .filter((name, metric) -> name.matches("(.*heap|pools.PS.*).usage"))
+ .withLoggingLevel(Slf4jReporter.LoggingLevel.DEBUG)
+ .convertRatesTo(TimeUnit.SECONDS)
+ .convertDurationsTo(TimeUnit.MILLISECONDS)
+ .build();
+ metricReporter.start(60,TimeUnit.SECONDS);
+ }
+
+ /**
+ * Used to debug window bucket lifecycle
+ *
+ * Window period: PT1s, margin: 5s
+ *
+ * @throws InterruptedException
+ */
+ @Test
+ public void testWithUnsortedEventsIn1MinuteWindow() throws InterruptedException {
+ MockPartitionedCollector mockCollector = new MockPartitionedCollector();
+ StreamTimeClockInLocalMemory timeClock = new StreamTimeClockInLocalMemory("sampleStream_1");
+ Ordering<PartitionedEvent> timeOrdering = Ordering.from(PartitionedEventTimeOrderingComparator.INSTANCE);
+ StreamSortWindowHandlerImpl sortHandler = new StreamSortWindowHandlerImpl();
+ sortHandler.prepare("sampleStream_1", MockSampleMetadataFactory.createSampleStreamSortSpec("sampleStream_1", "PT1m",5000),mockCollector);
+ List<PartitionedEvent> unsortedList = new LinkedList<>();
+
+ int i = 0;
+ while(i<1000) {
+ PartitionedEvent event = MockSampleMetadataFactory.createRandomOutOfTimeOrderEventGroupedByName("sampleStream_1");
+ sortHandler.nextEvent(event);
+ unsortedList.add(event);
+ if(event.getTimestamp()>timeClock.getTime()) {
+ timeClock.moveForward(event.getTimestamp());
+ }
+ sortHandler.onTick(timeClock,System.currentTimeMillis());
+ i++;
+ }
+ sortHandler.close();
+ Assert.assertFalse(timeOrdering.isOrdered(unsortedList));
+ Assert.assertTrue(timeOrdering.isOrdered(mockCollector.get()));
+ Assert.assertTrue(mockCollector.get().size() > 0);
+ }
+
+ @Test
+ public void testStreamSortHandlerWithUnsortedEventsIn1HourWindow() throws InterruptedException {
+ testWithUnsortedEventsIn1hWindow(1000000);
+ }
+
+ @Test
+ public void testSortedInPatient() throws InterruptedException {
+ MockPartitionedCollector mockCollector = new MockPartitionedCollector();
+ StreamTimeClockInLocalMemory timeClock = new StreamTimeClockInLocalMemory("sampleStream_1");
+ Ordering<PartitionedEvent> timeOrdering = Ordering.from(PartitionedEventTimeOrderingComparator.INSTANCE);
+ StreamSortWindowHandlerImpl sortHandler = new StreamSortWindowHandlerImpl();
+ sortHandler.prepare("sampleStream_1", MockSampleMetadataFactory.createSampleStreamSortSpec("sampleStream_1", "PT1h",5000),mockCollector);
+ List<PartitionedEvent> sortedList = new LinkedList<>();
+
+ int i = 0;
+ while(i<1000000) {
+ PartitionedEvent event = MockSampleMetadataFactory.createRandomPartitionedEvent("sampleStream_1",System.currentTimeMillis()+i);
+ sortHandler.nextEvent(event);
+ sortedList.add(event);
+ if(event.getTimestamp()>timeClock.getTime()) {
+ timeClock.moveForward(event.getTimestamp());
+ }
+ sortHandler.onTick(timeClock,System.currentTimeMillis());
+ i++;
+ }
+ sortHandler.close();
+ Assert.assertTrue(timeOrdering.isOrdered(sortedList));
+ Assert.assertTrue(timeOrdering.isOrdered(mockCollector.get()));
+ Assert.assertEquals(1000000,mockCollector.get().size());
+ }
+
+ /**
+ * -XX:+PrintGC
+ *
+ * @throws InterruptedException
+ */
+ @Test
+ public void testWithUnsortedEventsInLargeWindowBenchmark() throws InterruptedException {
+ metricReporter.report();
+ testWithUnsortedEventsIn1hWindow(1000);
+ metricReporter.report();
+ testWithUnsortedEventsIn1hWindow(10000);
+ metricReporter.report();
+ testWithUnsortedEventsIn1hWindow(100000);
+ metricReporter.report();
+ testWithUnsortedEventsIn1hWindow(1000000);
+ metricReporter.report();
+// testWithUnsortedEventsIn1hWindow(10000000);
+// metricReporter.report();
+ }
+
+ public void testWithUnsortedEventsIn1hWindow(int count) throws InterruptedException {
+ MockPartitionedCollector mockCollector = new MockPartitionedCollector();
+ StreamTimeClockInLocalMemory timeClock = new StreamTimeClockInLocalMemory("sampleStream_1");
+ Ordering<PartitionedEvent> timeOrdering = Ordering.from(PartitionedEventTimeOrderingComparator.INSTANCE);
+ StreamSortWindowHandlerImpl sortHandler = new StreamSortWindowHandlerImpl();
+ sortHandler.prepare("sampleStream_1", MockSampleMetadataFactory.createSampleStreamSortSpec("sampleStream_1", "PT1h",5000),mockCollector);
+ List<PartitionedEvent> unsortedList = new LinkedList<>();
+
+ StopWatch stopWatch = new StopWatch();
+ stopWatch.start();
+ int i = 0;
+ while(i<count) {
+ PartitionedEvent event = MockSampleMetadataFactory.createRandomOutOfTimeOrderEventGroupedByName("sampleStream_1");
+ sortHandler.nextEvent(event);
+ unsortedList.add(event);
+ if(event.getEvent().getTimestamp()>timeClock.getTime()) {
+ timeClock.moveForward(event.getEvent().getTimestamp());
+ }
+ sortHandler.onTick(timeClock,System.currentTimeMillis());
+ i++;
+ }
+ stopWatch.stop();
+ LOG.info("Produced {} events in {} ms",count,stopWatch.getTime());
+ sortHandler.close();
+ Assert.assertFalse(timeOrdering.isOrdered(unsortedList));
+ Assert.assertTrue(timeOrdering.isOrdered(mockCollector.get()));
+ Assert.assertTrue(mockCollector.get().size()>=0);
+ }
+
+ /**
+ * Used to debug window bucket lifecycle
+ *
+ * Window period: PT1h, margin: 5s
+ *
+ * @throws InterruptedException
+ */
+ @Test
+ public void testWithSortedEvents() throws InterruptedException {
+ MockPartitionedCollector mockCollector = new MockPartitionedCollector();
+ StreamTimeClockInLocalMemory timeClock = new StreamTimeClockInLocalMemory("sampleStream_1");
+ Ordering<PartitionedEvent> timeOrdering = Ordering.from(PartitionedEventTimeOrderingComparator.INSTANCE);
+ StreamSortWindowHandlerImpl sortHandler = new StreamSortWindowHandlerImpl();
+ sortHandler.prepare("sampleStream_1", MockSampleMetadataFactory.createSampleStreamSortSpec("sampleStream_1", "PT1h",5000),mockCollector);
+ List<PartitionedEvent> sortedList = new LinkedList<>();
+
+ int i = 0;
+ while(i<1000000) {
+ PartitionedEvent event = MockSampleMetadataFactory.createRandomPartitionedEvent("sampleStream_1",System.currentTimeMillis()+i);
+ sortHandler.nextEvent(event);
+ sortedList.add(event);
+ if(event.getTimestamp()>timeClock.getTime()) {
+ timeClock.moveForward(event.getTimestamp());
+ }
+ sortHandler.onTick(timeClock,System.currentTimeMillis());
+ i++;
+ }
+ sortHandler.close();
+ Assert.assertTrue(timeOrdering.isOrdered(sortedList));
+ Assert.assertTrue(timeOrdering.isOrdered(mockCollector.get()));
+ Assert.assertEquals(1000000,mockCollector.get().size());
+ }
+
+ /**
+ * Used to debug window bucket lifecycle
+ *
+ * Window period: PT1h, margin: 5s
+ *
+ * @throws InterruptedException
+ */
+ @Test
+ public void testWithSortedEventsAndExpireBySystemTime() throws InterruptedException {
+ MockPartitionedCollector mockCollector = new MockPartitionedCollector();
+ StreamTimeClockInLocalMemory timeClock = new StreamTimeClockInLocalMemory("sampleStream_1");
+ Ordering<PartitionedEvent> timeOrdering = Ordering.from(PartitionedEventTimeOrderingComparator.INSTANCE);
+ StreamSortWindowHandlerImpl sortHandler = new StreamSortWindowHandlerImpl();
+ sortHandler.prepare("sampleStream_1", MockSampleMetadataFactory.createSampleStreamSortSpec("sampleStream_1", "PT10s",1000),mockCollector);
+ List<PartitionedEvent> sortedList = new LinkedList<>();
+
+ PartitionedEvent event = MockSampleMetadataFactory.createRandomSortedEventGroupedByName("sampleStream_1");
+ sortHandler.nextEvent(event);
+ sortedList.add(event);
+ timeClock.moveForward(event.getTimestamp());
+ sortHandler.onTick(timeClock,System.currentTimeMillis());
+
+ // Triggered to become expired by System time
+ sortHandler.onTick(timeClock,System.currentTimeMillis()+10*1000+1000L + 1);
+
+ Assert.assertTrue(timeOrdering.isOrdered(sortedList));
+ Assert.assertTrue(timeOrdering.isOrdered(mockCollector.get()));
+ Assert.assertEquals(1,mockCollector.get().size());
+
+ sortHandler.close();
+ }
+
+// @Test
+ public void testWithTimerLock() throws InterruptedException {
+ Timer timer = new Timer();
+ List<Long> collected = new ArrayList<>();
+ timer.schedule(new TimerTask() {
+ @Override
+ public void run() {
+ synchronized (collected) {
+ LOG.info("Ticking {}", DateTimeUtil.millisecondsToHumanDateWithMilliseconds(System.currentTimeMillis()));
+ collected.add(System.currentTimeMillis());
+ try {
+ Thread.sleep(5000);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ },0,100);
+ }
+}
\ No newline at end of file
[14/19] incubator-eagle git commit: EAGLE-324: Init branch-v0.5
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-coordinator/src/test/java/org/apache/alert/coordinator/SchedulerTest.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-coordinator/src/test/java/org/apache/alert/coordinator/SchedulerTest.java b/eagle-core/eagle-alert/alert/alert-coordinator/src/test/java/org/apache/alert/coordinator/SchedulerTest.java
new file mode 100644
index 0000000..f90decc
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-coordinator/src/test/java/org/apache/alert/coordinator/SchedulerTest.java
@@ -0,0 +1,635 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.alert.coordinator;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.alert.coordinator.mock.TestTopologyMgmtService;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.eagle.alert.coordination.model.AlertBoltSpec;
+import org.apache.eagle.alert.coordination.model.Kafka2TupleMetadata;
+import org.apache.eagle.alert.coordination.model.PolicyWorkerQueue;
+import org.apache.eagle.alert.coordination.model.RouterSpec;
+import org.apache.eagle.alert.coordination.model.ScheduleState;
+import org.apache.eagle.alert.coordination.model.SpoutSpec;
+import org.apache.eagle.alert.coordination.model.StreamRepartitionMetadata;
+import org.apache.eagle.alert.coordination.model.StreamRepartitionStrategy;
+import org.apache.eagle.alert.coordination.model.StreamRouterSpec;
+import org.apache.eagle.alert.coordination.model.Tuple2StreamMetadata;
+import org.apache.eagle.alert.coordination.model.WorkSlot;
+import org.apache.eagle.alert.coordination.model.internal.MonitoredStream;
+import org.apache.eagle.alert.coordination.model.internal.PolicyAssignment;
+import org.apache.eagle.alert.coordination.model.internal.StreamWorkSlotQueue;
+import org.apache.eagle.alert.coordination.model.internal.Topology;
+import org.apache.eagle.alert.coordinator.IScheduleContext;
+import org.apache.eagle.alert.coordinator.ScheduleOption;
+import org.apache.eagle.alert.coordinator.TopologyMgmtService;
+import org.apache.eagle.alert.coordinator.impl.GreedyPolicyScheduler;
+import org.apache.eagle.alert.coordinator.model.TopologyUsage;
+import org.apache.eagle.alert.coordinator.provider.InMemScheduleConext;
+import org.apache.eagle.alert.engine.coordinator.PolicyDefinition;
+import org.apache.eagle.alert.engine.coordinator.PolicyDefinition.Definition;
+import org.apache.eagle.alert.engine.coordinator.StreamColumn;
+import org.apache.eagle.alert.engine.coordinator.StreamColumn.Type;
+import org.apache.eagle.alert.engine.coordinator.StreamDefinition;
+import org.apache.eagle.alert.engine.coordinator.StreamPartition;
+import org.apache.eagle.alert.engine.coordinator.StreamSortSpec;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.typesafe.config.ConfigFactory;
+
+/**
+ * @since Apr 22, 2016
+ *
+ */
+public class SchedulerTest {
+
+ private static final String STREAM2 = "stream2";
+ private static final String JOIN_POLICY_1 = "join-policy-1";
+ private static final String TEST_TOPIC = "test-topic";
+ private static final String TEST_POLICY_1 = "test-policy1";
+ private static final String TEST_POLICY_2 = "test-policy2";
+ private static final String TEST_POLICY_3 = "test-policy3";
+ private static final String STREAM1 = "stream1";
+ private static final String DS_NAME = "ds1";
+ private static ObjectMapper mapper = new ObjectMapper();
+ private static final Logger LOG = LoggerFactory.getLogger(SchedulerTest.class);
+
+ @BeforeClass
+ public static void setup() {
+ ConfigFactory.invalidateCaches();
+ System.setProperty("config.resource", "/application.conf");
+ }
+
+ @Test
+ public void test01_simple() throws Exception {
+ GreedyPolicyScheduler ps = new GreedyPolicyScheduler();
+ IScheduleContext context = createScheduleContext();
+ ps.init(context, createMgmtService());
+ ps.schedule(new ScheduleOption());
+
+ ScheduleState status = ps.getState();
+ context = ps.getContext(); // context updated!
+ Map<String, SpoutSpec> spec = status.getSpoutSpecs();
+
+ LOG.info(mapper.writeValueAsString(spec));
+ Assert.assertEquals(2, spec.size());
+ Assert.assertTrue(spec.containsKey("topo1"));
+ assertFirstPolicyScheduled(context, status);
+ }
+
+ private void assertFirstPolicyScheduled(IScheduleContext context, ScheduleState status) {
+ String version = status.getVersion();
+ // assert spout spec
+ {
+ Iterator<SpoutSpec> it = status.getSpoutSpecs().values().iterator();
+ {
+ // assert spout 1
+ SpoutSpec ss = it.next();
+ Assert.assertEquals(version, ss.getVersion());
+ Assert.assertEquals(1, ss.getKafka2TupleMetadataMap().size());
+ Assert.assertEquals(TEST_TOPIC, ss.getKafka2TupleMetadataMap().keySet().iterator().next());
+
+ Assert.assertEquals(1, ss.getStreamRepartitionMetadataMap().size());
+ List<StreamRepartitionMetadata> metas = ss.getStreamRepartitionMetadataMap().values().iterator().next();
+ Assert.assertEquals(1, metas.size());
+
+ StreamRepartitionMetadata streamMeta = metas.iterator().next();
+ Assert.assertEquals(STREAM1, streamMeta.getStreamId());
+ Assert.assertEquals(DS_NAME, streamMeta.getTopicName());
+ Assert.assertEquals(1, streamMeta.groupingStrategies.size());
+
+ StreamRepartitionStrategy gs = streamMeta.groupingStrategies.iterator().next();
+ Assert.assertEquals(5, gs.numTotalParticipatingRouterBolts);
+ Assert.assertEquals(5, gs.totalTargetBoltIds.size());
+ Assert.assertEquals(0, gs.startSequence);
+
+ Assert.assertTrue(context.getPolicyAssignments().containsKey(TEST_POLICY_1));
+ }
+ {
+ // assert spout 2
+ SpoutSpec ss = it.next();
+ Assert.assertEquals(version, ss.getVersion());
+ Assert.assertEquals(0, ss.getKafka2TupleMetadataMap().size());
+ }
+ }
+ // assert grp-by spec
+ {
+ Iterator<RouterSpec> gsit = status.getGroupSpecs().values().iterator();
+ {
+ Assert.assertEquals(2, status.getGroupSpecs().values().size());
+ Assert.assertTrue(gsit.hasNext());
+ RouterSpec gspec = gsit.next();
+ Assert.assertEquals(version, gspec.getVersion());
+ String topo1 = gspec.getTopologyName();
+ LOG.info("group spec topology name:", topo1);
+ List<StreamRouterSpec> routeSpecs = gspec.getRouterSpecs();
+ Assert.assertEquals(1, routeSpecs.size());
+ for (StreamRouterSpec spec : routeSpecs) {
+ StreamPartition par = spec.getPartition();
+ Assert.assertEquals(STREAM1, par.getStreamId());
+ Assert.assertEquals(Arrays.asList("col1"), par.getColumns());
+ Assert.assertEquals(STREAM1, spec.getStreamId());
+
+ Assert.assertEquals(1, spec.getTargetQueue().size());
+ List<PolicyWorkerQueue> queues = spec.getTargetQueue();
+ Assert.assertEquals(1, queues.size());
+ Assert.assertEquals(5, queues.get(0).getWorkers().size());
+ for (WorkSlot slot : queues.get(0).getWorkers()) {
+ Assert.assertEquals(topo1, slot.getTopologyName());
+ LOG.info(slot.getBoltId());
+ }
+ }
+ }
+ // grp-spec2
+ {
+ RouterSpec gs2 = gsit.next();
+ Assert.assertEquals(version, gs2.getVersion());
+ List<StreamRouterSpec> routeSpecs = gs2.getRouterSpecs();
+ Assert.assertEquals(0, routeSpecs.size());
+ }
+ }
+ // alert spec
+ {
+ Assert.assertEquals(2, status.getAlertSpecs().values().size());
+ Iterator<AlertBoltSpec> asit = status.getAlertSpecs().values().iterator();
+ // topo1
+ {
+ AlertBoltSpec alertSpec = asit.next();
+ Assert.assertEquals(version, alertSpec.getVersion());
+ String topo1 = alertSpec.getTopologyName();
+ LOG.info("alert spec topology name {}", topo1);
+ for (List<String> definitions : alertSpec.getBoltPolicyIdsMap().values()) {
+ Assert.assertEquals(1, definitions.size());
+ Assert.assertEquals(TEST_POLICY_1, definitions.get(0));
+ }
+ }
+ // topo2
+ {
+ AlertBoltSpec alertSpec = asit.next();
+ Assert.assertEquals(version, alertSpec.getVersion());
+ String topo1 = alertSpec.getTopologyName();
+ LOG.info("alert spec topology name {}", topo1);
+ Assert.assertEquals(0, alertSpec.getBoltPolicyIdsMap().size());
+ }
+ }
+ }
+
+ private TopologyMgmtService createMgmtService() {
+ TestTopologyMgmtService.BOLT_NUMBER = 5;
+ TopologyMgmtService mgmtService = new TestTopologyMgmtService();
+ return mgmtService;
+ }
+
+ private InMemScheduleConext createScheduleContext() {
+ InMemScheduleConext context = new InMemScheduleConext();
+ // topo
+ Pair<Topology, TopologyUsage> pair1 = TestTopologyMgmtService.createEmptyTopology("topo1");
+ Pair<Topology, TopologyUsage> pair2 = TestTopologyMgmtService.createEmptyTopology("topo2");
+ context.addTopology(pair1.getLeft());
+ context.addTopologyUsages(pair1.getRight());
+ context.addTopology(pair2.getLeft());
+ context.addTopologyUsages(pair2.getRight());
+
+ // policy
+ createSamplePolicy(context, TEST_POLICY_1, STREAM1);
+
+ // data source
+ Kafka2TupleMetadata ds = new Kafka2TupleMetadata();
+ ds.setName(DS_NAME);
+ ds.setTopic(TEST_TOPIC);
+ ds.setCodec(new Tuple2StreamMetadata());
+ context.addDataSource(ds);
+
+ // schema
+ {
+ StreamDefinition schema = new StreamDefinition();
+ {
+ StreamColumn c = new StreamColumn();
+ c.setName("col1");
+ c.setType(Type.STRING);
+ c.setDefaultValue("dflt");
+ schema.getColumns().add(c);
+ }
+ {
+ StreamColumn c = new StreamColumn();
+ c.setName("col2");
+ c.setType(Type.DOUBLE);
+ c.setDefaultValue("0.0");
+ schema.getColumns().add(c);
+ }
+ schema.setStreamId(STREAM1);
+ schema.setValidate(false);
+ schema.setDataSource(DS_NAME);
+ context.addSchema(schema);
+ }
+ {
+ StreamDefinition schema = new StreamDefinition();
+ {
+ StreamColumn c = new StreamColumn();
+ c.setName("col1");
+ c.setType(Type.STRING);
+ c.setDefaultValue("dflt");
+ schema.getColumns().add(c);
+ }
+ schema.setStreamId(STREAM2);
+ schema.setValidate(false);
+ schema.setDataSource(DS_NAME);
+ context.addSchema(schema);
+ }
+
+ return context;
+ }
+
+ /**
+ * Add policy after add policy
+ */
+ @Test
+ public void test_schedule_add2() {
+ IScheduleContext context = createScheduleContext();
+ GreedyPolicyScheduler ps = new GreedyPolicyScheduler();
+ TopologyMgmtService mgmtService = new TopologyMgmtService();
+ ps.init(context, mgmtService);
+
+ ScheduleOption option = new ScheduleOption();
+ ps.schedule(option);
+ ScheduleState status = ps.getState();
+ context = ps.getContext(); // context updated!
+ assertFirstPolicyScheduled(context, status);
+
+ createSamplePolicy((InMemScheduleConext) context, TEST_POLICY_2, STREAM1);
+
+ ps.init(context, mgmtService); // reinit
+ ps.schedule(option);
+ status = ps.getState();
+ context = ps.getContext(); // context updated!
+ // now assert two policy on the same queue
+ assertSecondPolicyCreated(context, status);
+
+ // add one policy on different stream of the same topic
+ createSamplePolicy((InMemScheduleConext) context, TEST_POLICY_3, STREAM2);
+
+ ps.init(context, mgmtService); // re-init
+ ps.schedule(option);
+ status = ps.getState();
+ context = ps.getContext(); // context updated!
+ assertThridPolicyScheduled(context, status);
+ }
+
+ private void assertThridPolicyScheduled(IScheduleContext context, ScheduleState status) {
+ {
+ // now assert two policy on the same queue
+ Assert.assertEquals(2, status.getSpoutSpecs().values().size());
+ Iterator<SpoutSpec> it = status.getSpoutSpecs().values().iterator();
+ {
+ // assert spout 1
+ SpoutSpec ss = it.next();
+ Assert.assertEquals(1, ss.getKafka2TupleMetadataMap().size());
+ Assert.assertEquals(TEST_TOPIC, ss.getKafka2TupleMetadataMap().keySet().iterator().next());
+
+ Assert.assertEquals(1, ss.getStreamRepartitionMetadataMap().size());
+ List<StreamRepartitionMetadata> metas = ss.getStreamRepartitionMetadataMap().values().iterator().next();
+ Assert.assertEquals(1, metas.size());
+
+ StreamRepartitionMetadata streamMeta = metas.iterator().next();
+ Assert.assertEquals(STREAM1, streamMeta.getStreamId());
+ Assert.assertEquals(DS_NAME, streamMeta.getTopicName());
+ Assert.assertEquals(1, streamMeta.groupingStrategies.size());
+
+ StreamRepartitionStrategy gs = streamMeta.groupingStrategies.iterator().next();
+ Assert.assertEquals(5, gs.numTotalParticipatingRouterBolts);
+ Assert.assertEquals(5, gs.totalTargetBoltIds.size());
+ Assert.assertEquals(0, gs.startSequence);
+
+ PolicyAssignment pa1 = context.getPolicyAssignments().get(TEST_POLICY_1);
+ PolicyAssignment pa2 = context.getPolicyAssignments().get(TEST_POLICY_2);
+ Assert.assertNotNull(pa1);
+ Assert.assertNotNull(pa2);
+ Assert.assertEquals(pa1.getQueueId(), pa2.getQueueId());
+ }
+ {
+ // assert spout 2
+ SpoutSpec ss = it.next();
+ Assert.assertEquals(1, ss.getKafka2TupleMetadataMap().size());
+
+ Assert.assertEquals(TEST_TOPIC, ss.getKafka2TupleMetadataMap().keySet().iterator().next());
+
+ Assert.assertEquals(1, ss.getStreamRepartitionMetadataMap().size());
+ List<StreamRepartitionMetadata> metas = ss.getStreamRepartitionMetadataMap().values().iterator().next();
+ Assert.assertEquals(1, metas.size());
+
+ StreamRepartitionMetadata streamMeta = metas.iterator().next();
+ Assert.assertEquals(STREAM2, streamMeta.getStreamId());
+ Assert.assertEquals(DS_NAME, streamMeta.getTopicName());
+ Assert.assertEquals(1, streamMeta.groupingStrategies.size());
+
+ StreamRepartitionStrategy gs = streamMeta.groupingStrategies.iterator().next();
+ Assert.assertEquals(5, gs.numTotalParticipatingRouterBolts);
+ Assert.assertEquals(5, gs.totalTargetBoltIds.size());
+ Assert.assertEquals(0, gs.startSequence);
+
+ // assert policy assignment for the three policies
+ PolicyAssignment pa1 = context.getPolicyAssignments().get(TEST_POLICY_1);
+ PolicyAssignment pa2 = context.getPolicyAssignments().get(TEST_POLICY_2);
+ PolicyAssignment pa3 = context.getPolicyAssignments().get(TEST_POLICY_3);
+ Assert.assertNotNull(pa1);
+ Assert.assertNotNull(pa2);
+ Assert.assertNotNull(pa3);
+ Assert.assertEquals(pa1.getQueueId(), pa2.getQueueId());
+ Assert.assertNotEquals(pa1.getQueueId(), pa3.getQueueId());
+ StreamWorkSlotQueue queue1 = getQueue(context, pa1.getQueueId()).getRight();
+ StreamWorkSlotQueue queue3 = getQueue(context, pa3.getQueueId()).getRight();
+ Assert.assertNotEquals(queue1.getWorkingSlots().get(0).getTopologyName(), queue3.getWorkingSlots().get(0).getTopologyName());
+ }
+ }
+ // group spec
+ {
+ Iterator<RouterSpec> gsit = status.getGroupSpecs().values().iterator();
+ Assert.assertEquals(2, status.getGroupSpecs().values().size());
+ {
+ // first topology's grp - spec
+ gsit.next();
+ // should be same with second policy scheduled, not assert here
+ }
+ {
+ // second topology's grp - spec
+ RouterSpec spec = gsit.next();
+ Assert.assertEquals(1, spec.getRouterSpecs().size());
+ StreamRouterSpec routeSpec = spec.getRouterSpecs().get(0);
+ Assert.assertEquals(STREAM2, routeSpec.getStreamId());
+ Assert.assertEquals(Arrays.asList("col1"), routeSpec.getPartition().getColumns());
+ }
+ }
+ // alert spec
+ {
+ Assert.assertEquals(2, status.getAlertSpecs().values().size());
+ Iterator<AlertBoltSpec> asit = status.getAlertSpecs().values().iterator();
+ {
+ // same to the two policy case, not assert here
+ asit.next();
+ }
+ {
+ // seconds topology's alert spec
+ AlertBoltSpec as = asit.next();
+ Assert.assertEquals(5, as.getBoltPolicyIdsMap().size());
+ for (List<String> pdList : as.getBoltPolicyIdsMap().values()) {
+ Assert.assertEquals(1, pdList.size());
+ Assert.assertEquals(TEST_POLICY_3, pdList.get(0));
+ }
+ }
+ }
+ }
+
+ private void assertSecondPolicyCreated(IScheduleContext context, ScheduleState status) {
+ String version = status.getVersion();
+ {
+ // spout : assert two policy on the same topology (same worker
+ // queue)
+ Iterator<SpoutSpec> it = status.getSpoutSpecs().values().iterator();
+ {
+ // assert spout 1 has two policy
+ SpoutSpec ss = it.next();
+ Assert.assertEquals(1, ss.getKafka2TupleMetadataMap().size());
+ Assert.assertEquals(TEST_TOPIC, ss.getKafka2TupleMetadataMap().keySet().iterator().next());
+
+ Assert.assertEquals(1, ss.getStreamRepartitionMetadataMap().size());
+ List<StreamRepartitionMetadata> metas = ss.getStreamRepartitionMetadataMap().values().iterator().next();
+ Assert.assertEquals(1, metas.size());
+
+ StreamRepartitionMetadata streamMeta = metas.iterator().next();
+ Assert.assertEquals(STREAM1, streamMeta.getStreamId());
+ Assert.assertEquals(DS_NAME, streamMeta.getTopicName());
+ Assert.assertEquals(1, streamMeta.groupingStrategies.size());
+
+ StreamRepartitionStrategy gs = streamMeta.groupingStrategies.iterator().next();
+ Assert.assertEquals(5, gs.numTotalParticipatingRouterBolts);
+ Assert.assertEquals(5, gs.totalTargetBoltIds.size());
+ Assert.assertEquals(0, gs.startSequence);
+
+ // assert two policy on the same queue
+ PolicyAssignment pa1 = context.getPolicyAssignments().get(TEST_POLICY_1);
+ PolicyAssignment pa2 = context.getPolicyAssignments().get(TEST_POLICY_2);
+ Assert.assertNotNull(pa1);
+ Assert.assertNotNull(pa2);
+ Assert.assertEquals(pa1.getQueueId(), pa2.getQueueId());
+ StreamWorkSlotQueue queue = getQueue(context, pa1.getQueueId()).getRight();
+ Assert.assertNotNull(queue);
+ }
+ {
+ // assert spout 2 is still empty
+ SpoutSpec ss = it.next();
+ Assert.assertEquals(0, ss.getKafka2TupleMetadataMap().size());
+ }
+ }
+
+ // assert grp-by spec. This is nothing different compare to first policy
+ {
+ Iterator<RouterSpec> gsit = status.getGroupSpecs().values().iterator();
+ {
+ Assert.assertEquals(2, status.getGroupSpecs().values().size());
+ Assert.assertTrue(gsit.hasNext());
+ RouterSpec gspec = gsit.next();
+ Assert.assertEquals(version, gspec.getVersion());
+ String topo1 = gspec.getTopologyName();
+ LOG.info("group spec topology name:", topo1);
+ List<StreamRouterSpec> routeSpecs = gspec.getRouterSpecs();
+ Assert.assertEquals(1, routeSpecs.size());
+ for (StreamRouterSpec spec : routeSpecs) {
+ StreamPartition par = spec.getPartition();
+ Assert.assertEquals(STREAM1, par.getStreamId());
+ Assert.assertEquals(Arrays.asList("col1"), par.getColumns());
+ Assert.assertEquals(STREAM1, spec.getStreamId());
+
+ Assert.assertEquals(1, spec.getTargetQueue().size());
+ List<PolicyWorkerQueue> queues = spec.getTargetQueue();
+ Assert.assertEquals(1, queues.size());
+ Assert.assertEquals(5, queues.get(0).getWorkers().size());
+ for (WorkSlot slot : queues.get(0).getWorkers()) {
+ Assert.assertEquals(topo1, slot.getTopologyName());
+ LOG.info(slot.getBoltId());
+ }
+ }
+ }
+ // grp-spec for second topology is still empty
+ {
+ RouterSpec gs2 = gsit.next();
+ Assert.assertEquals(version, gs2.getVersion());
+ List<StreamRouterSpec> routeSpecs = gs2.getRouterSpecs();
+ Assert.assertEquals(0, routeSpecs.size());
+ }
+ }
+ // alert spec
+ {
+ Assert.assertEquals(2, status.getAlertSpecs().values().size());
+ Iterator<AlertBoltSpec> asit = status.getAlertSpecs().values().iterator();
+ {
+ AlertBoltSpec alertSpec = asit.next();
+ Assert.assertEquals(version, alertSpec.getVersion());
+ String topo1 = alertSpec.getTopologyName();
+ LOG.info("alert spec topology name {}", topo1);
+ for (List<String> definitions : alertSpec.getBoltPolicyIdsMap().values()) {
+ Assert.assertEquals(2, definitions.size());
+// List<String> names = Arrays.asList(definitions.stream().map((t) -> t.getName()).toArray(String[]::new));
+ Assert.assertTrue(definitions.contains(TEST_POLICY_1));
+ Assert.assertTrue(definitions.contains(TEST_POLICY_2));
+ }
+ }
+ // second spout
+ {
+ AlertBoltSpec spec = asit.next();
+ Assert.assertEquals(0, spec.getBoltPolicyIdsMap().size());
+ }
+ }
+ }
+
+ public static Pair<MonitoredStream, StreamWorkSlotQueue> getQueue(IScheduleContext context, String queueId) {
+ for (MonitoredStream ms : context.getMonitoredStreams().values()) {
+ for (StreamWorkSlotQueue q : ms.getQueues()) {
+ if (q.getQueueId().equals(queueId)) {
+ return Pair.of(ms, q);
+ }
+ }
+ }
+ return null;
+ }
+
+ @Test
+ public void testGroupEquals() {
+ StreamRepartitionStrategy gs1 = new StreamRepartitionStrategy();
+ StreamPartition sp = new StreamPartition();
+ sp.setColumns(Arrays.asList("col1"));
+ sp.setSortSpec(new StreamSortSpec());
+ sp.setStreamId("testStream");
+ sp.setType(StreamPartition.Type.GROUPBY);
+ gs1.partition = sp;
+
+ StreamRepartitionStrategy gs2 = new StreamRepartitionStrategy();
+ sp = new StreamPartition();
+ sp.setColumns(Arrays.asList("col1"));
+ sp.setSortSpec(new StreamSortSpec());
+ sp.setStreamId("testStream");
+ sp.setType(StreamPartition.Type.GROUPBY);
+ gs2.partition = sp;
+
+ Assert.assertTrue(gs1.equals(gs2));
+ List<StreamRepartitionStrategy> list = new ArrayList<StreamRepartitionStrategy>();
+ list.add(gs1);
+ Assert.assertTrue(list.contains(gs2));
+ }
+
+ private void createSamplePolicy(InMemScheduleConext context, String policyName, String stream) {
+ PolicyDefinition pd = new PolicyDefinition();
+ pd.setParallelismHint(5);
+ Definition def = new Definition();
+ pd.setDefinition(def);
+ pd.setName(policyName);
+ pd.setInputStreams(Arrays.asList(stream));
+ pd.setOutputStreams(Arrays.asList("outputStream2"));
+ StreamPartition par = new StreamPartition();
+ par.setColumns(Arrays.asList("col1"));
+ par.setType(StreamPartition.Type.GLOBAL);
+ par.setStreamId(stream);
+ pd.setPartitionSpec(Arrays.asList(par));
+ context.addPoilcy(pd);
+ }
+
+ /**
+ * Add and remove
+ */
+ @Test
+ public void test_schedule2_remove() {
+ // TODO
+ }
+
+ @Test
+ public void test_schedule_updateParitition() {
+ // TODO
+ }
+
+ @Test
+ public void test_schedule_nogroupby() {
+ // TODO
+ }
+
+ @SuppressWarnings("unused")
+ @Test
+ public void test_schedule_multipleStream() throws Exception {
+ IScheduleContext context = createScheduleContext();
+ GreedyPolicyScheduler ps = new GreedyPolicyScheduler();
+ TopologyMgmtService mgmtService = new TopologyMgmtService();
+
+ createJoinPolicy((InMemScheduleConext) context, JOIN_POLICY_1, Arrays.asList(STREAM1, STREAM2));
+
+ ps.init(context, mgmtService);
+ ScheduleOption option = new ScheduleOption();
+ ps.schedule(option);
+ ScheduleState state = ps.getState();
+
+ context = ps.getContext(); // context updated!
+ // assert
+ Assert.assertTrue(context.getPolicyAssignments().containsKey(JOIN_POLICY_1));
+ Assert.assertTrue(context.getPolicyAssignments().containsKey(TEST_POLICY_1));
+ PolicyAssignment pa1 = context.getPolicyAssignments().get(JOIN_POLICY_1);
+ PolicyAssignment pa2 = context.getPolicyAssignments().get(TEST_POLICY_1);
+ Assert.assertNotEquals(pa1.getQueueId(), pa2.getQueueId());
+
+ StreamWorkSlotQueue joinPair = getQueue(context, pa1.getQueueId()).getRight();
+ String joinTopo = joinPair.getWorkingSlots().get(0).topologyName;
+ StreamWorkSlotQueue streamPair = getQueue(context, pa2.getQueueId()).getRight();
+ String streamTopo = streamPair.getWorkingSlots().get(0).topologyName;
+ Assert.assertNotEquals(joinTopo, streamTopo);
+
+ // TODO more assert on state
+ SpoutSpec joinSpout = state.getSpoutSpecs().get(joinTopo);
+ RouterSpec groupSpec = state.getGroupSpecs().get(joinTopo);
+ AlertBoltSpec alertSpec = state.getAlertSpecs().get(joinTopo);
+
+ Assert.assertEquals(1, joinSpout.getStreamRepartitionMetadataMap().size());
+ Assert.assertEquals(2, joinSpout.getStreamRepartitionMetadataMap().get(TEST_TOPIC).size());
+
+ Assert.assertEquals(2, groupSpec.getRouterSpecs().size());
+
+ LOG.info(new ObjectMapper().writeValueAsString(state));
+ }
+
+ private void createJoinPolicy(InMemScheduleConext context, String policyName, List<String> asList) {
+ PolicyDefinition pd = new PolicyDefinition();
+ pd.setParallelismHint(5);
+ Definition def = new Definition();
+ pd.setDefinition(def);
+ pd.setName(policyName);
+ pd.setInputStreams(asList);
+ pd.setOutputStreams(Arrays.asList("outputStream2"));
+ for (String streamId : pd.getInputStreams()) {
+ StreamPartition par = new StreamPartition();
+ par.setColumns(Arrays.asList("col1"));
+ par.setType(StreamPartition.Type.GROUPBY);
+ par.setStreamId(streamId);
+ pd.addPartition(par);
+ }
+ context.addPoilcy(pd);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-coordinator/src/test/java/org/apache/alert/coordinator/WorkSlotStrategyTest.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-coordinator/src/test/java/org/apache/alert/coordinator/WorkSlotStrategyTest.java b/eagle-core/eagle-alert/alert/alert-coordinator/src/test/java/org/apache/alert/coordinator/WorkSlotStrategyTest.java
new file mode 100644
index 0000000..e17ebab
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-coordinator/src/test/java/org/apache/alert/coordinator/WorkSlotStrategyTest.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.alert.coordinator;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.alert.coordinator.mock.TestTopologyMgmtService;
+import org.apache.eagle.alert.coordination.model.WorkSlot;
+import org.apache.eagle.alert.coordination.model.internal.MonitoredStream;
+import org.apache.eagle.alert.coordination.model.internal.StreamGroup;
+import org.apache.eagle.alert.coordination.model.internal.StreamWorkSlotQueue;
+import org.apache.eagle.alert.coordinator.impl.WorkQueueBuilder;
+import org.apache.eagle.alert.coordinator.impl.strategies.SameTopologySlotStrategy;
+import org.apache.eagle.alert.coordinator.model.AlertBoltUsage;
+import org.apache.eagle.alert.coordinator.model.TopologyUsage;
+import org.apache.eagle.alert.coordinator.provider.InMemScheduleConext;
+import org.apache.eagle.alert.engine.coordinator.StreamPartition;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ *
+ * @since Apr 27, 2016
+ *
+ */
+public class WorkSlotStrategyTest {
+
+ private static final Logger LOG = LoggerFactory.getLogger(WorkSlotStrategyTest.class);
+
+ @Test
+ public void test() {
+ InMemScheduleConext context = new InMemScheduleConext();
+
+ StreamPartition partition = new StreamPartition();
+ partition.setType(StreamPartition.Type.GLOBAL);
+ partition.setStreamId("s1");
+ partition.setColumns(Arrays.asList("f1", "f2"));
+
+ StreamGroup group = new StreamGroup();
+ group.addStreamPartition(partition);
+
+ TestTopologyMgmtService.BOLT_NUMBER = 3;
+ TestTopologyMgmtService mgmtService = new TestTopologyMgmtService();
+
+ SameTopologySlotStrategy strategy = new SameTopologySlotStrategy(context, group, mgmtService);
+ {
+ List<WorkSlot> slots = strategy.reserveWorkSlots(5, false, new HashMap<String, Object>());
+ Assert.assertEquals(0, slots.size());
+ Assert.assertEquals(1, context.getTopologies().size());
+ }
+
+ TestTopologyMgmtService.BOLT_NUMBER = 5;
+ {
+ List<WorkSlot> slots = strategy.reserveWorkSlots(5, false, new HashMap<String, Object>());
+ Assert.assertEquals(5, slots.size());
+ LOG.info(slots.get(0).getTopologyName());
+ Assert.assertEquals(2, context.getTopologies().size());
+ Assert.assertEquals(2, context.getTopologyUsages().size());
+
+ // assert all on same topology
+ for (WorkSlot ws : slots) {
+ Assert.assertEquals(slots.get(0).getTopologyName(), ws.getTopologyName());
+ }
+ Iterator<TopologyUsage> it = context.getTopologyUsages().values().iterator();
+ TopologyUsage usage = it.next();
+ for (AlertBoltUsage u : usage.getAlertUsages().values()) {
+ Assert.assertTrue(u.getPartitions().size() == 0);
+ Assert.assertTrue(u.getQueueSize() == 0);
+ }
+ // assert
+ usage = it.next();
+ for (AlertBoltUsage u : usage.getAlertUsages().values()) {
+ LOG.info(u.getBoltId());
+ Assert.assertTrue(u.getPartitions().size() == 0);
+ Assert.assertTrue(u.getBoltId(), u.getQueueSize() == 0);
+ }
+ }
+ }
+
+ @SuppressWarnings("unused")
+ @Test
+ public void test2_overlap() {
+ InMemScheduleConext context = new InMemScheduleConext();
+
+ StreamPartition partition = new StreamPartition();
+ partition.setType(StreamPartition.Type.GLOBAL);
+ partition.setStreamId("s1");
+ partition.setColumns(Arrays.asList("f1", "f2"));
+ StreamGroup sg = new StreamGroup();
+ sg.addStreamPartition(partition);
+
+ MonitoredStream ms1 = new MonitoredStream(sg);
+
+ TestTopologyMgmtService.BOLT_NUMBER = 5;
+ TestTopologyMgmtService mgmtService = new TestTopologyMgmtService();
+
+ String topo1 = null;
+ String bolt1 = null;
+ WorkQueueBuilder wrb = new WorkQueueBuilder(context, mgmtService);
+ StreamWorkSlotQueue queue = wrb.createQueue(ms1, false, 5, new HashMap<String, Object>());
+ {
+ Assert.assertEquals(5, queue.getWorkingSlots().size());
+ topo1 = queue.getWorkingSlots().get(0).getTopologyName();
+ bolt1 = queue.getWorkingSlots().get(0).getBoltId();
+ Assert.assertEquals(1, context.getTopologies().size());
+ Assert.assertEquals(1, context.getTopologyUsages().size());
+ LOG.info(queue.getWorkingSlots().get(0).getTopologyName());
+ for (WorkSlot ws : queue.getWorkingSlots()) {
+ Assert.assertEquals(topo1, ws.getTopologyName());
+ }
+
+ TopologyUsage usage = context.getTopologyUsages().values().iterator().next();
+ for (AlertBoltUsage u : usage.getAlertUsages().values()) {
+ Assert.assertTrue(u.getPartitions().size() > 0);
+ Assert.assertTrue(u.getBoltId(), u.getQueueSize() > 0);
+ }
+ }
+
+ // second partition
+ StreamPartition partition2 = new StreamPartition();
+ partition2.setType(StreamPartition.Type.GLOBAL);
+ partition2.setStreamId("s2");
+ partition2.setColumns(Arrays.asList("f1", "f2"));
+
+ StreamGroup sg2 = new StreamGroup();
+ sg2.addStreamPartition(partition2);
+ MonitoredStream ms2 = new MonitoredStream(sg2);
+ queue = wrb.createQueue(ms2, false, 5, new HashMap<String, Object>());
+ TestTopologyMgmtService.BOLT_NUMBER = 5;
+ {
+ Assert.assertEquals(5, queue.getWorkingSlots().size());
+ Assert.assertEquals(2, context.getTopologies().size());
+ Assert.assertEquals(2, context.getTopologyUsages().size());
+
+ String topo2 = queue.getWorkingSlots().get(0).getTopologyName();
+ String bolt2 = queue.getWorkingSlots().get(0).getBoltId();
+ for (WorkSlot ws : queue.getWorkingSlots()) {
+ Assert.assertEquals(topo2, ws.getTopologyName());
+ }
+ Assert.assertNotEquals(topo1, topo2);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-coordinator/src/test/java/org/apache/alert/coordinator/mock/InMemMetadataServiceClient.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-coordinator/src/test/java/org/apache/alert/coordinator/mock/InMemMetadataServiceClient.java b/eagle-core/eagle-alert/alert/alert-coordinator/src/test/java/org/apache/alert/coordinator/mock/InMemMetadataServiceClient.java
new file mode 100644
index 0000000..2fc1794
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-coordinator/src/test/java/org/apache/alert/coordinator/mock/InMemMetadataServiceClient.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.alert.coordinator.mock;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+import org.apache.eagle.alert.coordination.model.Kafka2TupleMetadata;
+import org.apache.eagle.alert.coordination.model.ScheduleState;
+import org.apache.eagle.alert.coordination.model.SpoutSpec;
+import org.apache.eagle.alert.coordination.model.internal.Topology;
+import org.apache.eagle.alert.engine.coordinator.PolicyDefinition;
+import org.apache.eagle.alert.engine.coordinator.Publishment;
+import org.apache.eagle.alert.engine.coordinator.StreamDefinition;
+import org.apache.eagle.alert.engine.coordinator.StreamingCluster;
+import org.apache.eagle.alert.service.IMetadataServiceClient;
+
+/**
+ * @since May 5, 2016
+ *
+ */
+@SuppressWarnings("serial")
+public class InMemMetadataServiceClient implements IMetadataServiceClient {
+
+ private List<StreamingCluster> clusters = new ArrayList<StreamingCluster>();
+ private List<Topology> topologies = new ArrayList<Topology>();
+ private List<PolicyDefinition> policies = new ArrayList<PolicyDefinition>();
+ private List<StreamDefinition> definitions = new ArrayList<StreamDefinition>();
+ private List<Kafka2TupleMetadata> datasources = new ArrayList<Kafka2TupleMetadata>();
+
+ private SortedMap<String, ScheduleState> scheduleStates = new TreeMap<String, ScheduleState>();
+ private List<SpoutSpec> spoutSpecs = new ArrayList<SpoutSpec>();
+ private List<Publishment> publishmetns = new ArrayList<Publishment>();
+
+ @Override
+ public void close() throws IOException {
+ }
+
+ @Override
+ public List<StreamingCluster> listClusters() {
+ return clusters;
+ }
+
+ @Override
+ public List<Topology> listTopologies() {
+ return topologies;
+ }
+
+ @Override
+ public List<PolicyDefinition> listPolicies() {
+ return policies;
+ }
+
+ @Override
+ public List<StreamDefinition> listStreams() {
+ return definitions;
+ }
+
+ @Override
+ public List<Kafka2TupleMetadata> listDataSources() {
+ return datasources;
+ }
+
+ @Override
+ public List<Publishment> listPublishment() {
+ return publishmetns;
+ }
+
+ @Override
+ public List<SpoutSpec> listSpoutMetadata() {
+ return spoutSpecs;
+ }
+
+ @Override
+ public ScheduleState getVersionedSpec() {
+ Iterator<Entry<String, ScheduleState>> it = scheduleStates.entrySet().iterator();
+ if (it.hasNext()) {
+ return it.next().getValue();
+ }
+ return null;
+ }
+
+ @Override
+ public ScheduleState getVersionedSpec(String version) {
+ return scheduleStates.get(version);
+ }
+
+ @Override
+ public void addScheduleState(ScheduleState state) {
+ scheduleStates.put(state.getVersion(), state);
+ }
+
+ @Override
+ public void addStreamingCluster(StreamingCluster cluster) {
+ clusters.add(cluster);
+ }
+
+ @Override
+ public void addTopology(Topology t) {
+ topologies.add(t);
+ }
+
+ @Override
+ public void addPolicy(PolicyDefinition policy) {
+ policies.add(policy);
+ }
+
+ @Override
+ public void addStreamDefinition(StreamDefinition streamDef) {
+ definitions.add(streamDef);
+ }
+
+ @Override
+ public void addDataSource(Kafka2TupleMetadata k2t) {
+ datasources.add(k2t);
+ }
+
+ @Override
+ public void addPublishment(Publishment pub) {
+ publishmetns.add(pub);
+ }
+
+ @Override
+ public void clear() {
+ // do nothing
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-coordinator/src/test/java/org/apache/alert/coordinator/mock/TestTopologyMgmtService.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-coordinator/src/test/java/org/apache/alert/coordinator/mock/TestTopologyMgmtService.java b/eagle-core/eagle-alert/alert/alert-coordinator/src/test/java/org/apache/alert/coordinator/mock/TestTopologyMgmtService.java
new file mode 100644
index 0000000..1a690d6
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-coordinator/src/test/java/org/apache/alert/coordinator/mock/TestTopologyMgmtService.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.alert.coordinator.mock;
+
+import java.util.List;
+
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.eagle.alert.coordination.model.internal.Topology;
+import org.apache.eagle.alert.coordinator.TopologyMgmtService;
+import org.apache.eagle.alert.coordinator.model.AlertBoltUsage;
+import org.apache.eagle.alert.coordinator.model.GroupBoltUsage;
+import org.apache.eagle.alert.coordinator.model.TopologyUsage;
+
+public class TestTopologyMgmtService extends TopologyMgmtService {
+
+ public static int BOLT_NUMBER = 5;
+
+ private int i = 0;
+
+ @Override
+ public TopologyMeta creatTopology() {
+ TopologyMeta tm = new TopologyMeta();
+ tm.topologyId = "Topoloy" + (i++);
+ tm.clusterId = "default-cluster";
+ tm.nimbusHost = "localhost";
+ tm.nimbusPort = "3000";
+ Pair<Topology, TopologyUsage> pair = TestTopologyMgmtService.createEmptyTopology(tm.topologyId);
+ tm.topology = pair.getLeft();
+ tm.usage = pair.getRight();
+
+ return tm;
+ }
+
+ @Override
+ public List<TopologyMeta> listTopologies() {
+ // TODO Auto-generated method stub
+ return super.listTopologies();
+ }
+
+ public static Pair<Topology, TopologyUsage> createEmptyTopology(String topoName) {
+ Topology t = new Topology(topoName, TestTopologyMgmtService.BOLT_NUMBER, TestTopologyMgmtService.BOLT_NUMBER);
+ for (int i = 0; i < t.getNumOfGroupBolt(); i++) {
+ t.getGroupNodeIds().add(t.getName() + "-grp-" + i);
+ }
+ for (int i = 0; i < t.getNumOfAlertBolt(); i++) {
+ t.getAlertBoltIds().add(t.getName() + "-alert-" + i);
+ }
+
+ TopologyUsage u = new TopologyUsage(topoName);
+ for (String gnid : t.getGroupNodeIds()) {
+ u.getGroupUsages().put(gnid, new GroupBoltUsage(gnid));
+ }
+ for (String anid : t.getAlertBoltIds()) {
+ u.getAlertUsages().put(anid, new AlertBoltUsage(anid));
+ }
+
+ return Pair.of(t, u);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-coordinator/src/test/resources/test-application.conf
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-coordinator/src/test/resources/test-application.conf b/eagle-core/eagle-alert/alert/alert-coordinator/src/test/resources/test-application.conf
new file mode 100644
index 0000000..deb9793
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-coordinator/src/test/resources/test-application.conf
@@ -0,0 +1,27 @@
+{
+ "coordinator" : {
+ "policiesPerBolt" : 5,
+ "boltParallelism" : 5,
+ "policyDefaultParallelism" : 5,
+ "boltLoadUpbound": 0.8,
+ "topologyLoadUpbound" : 0.8,
+ "numOfAlertBoltsPerTopology" : 5,
+ "zkConfig": {
+ "zkQuorum" : "localhost:2181",
+ "zkRoot": "/alert",
+ "zkSessionTimeoutMs" : 10000,
+ "connectionTimeoutMs" : 5000,
+ "zkRetryTimes" : 5,
+ "zkRetryInterval" : 1000
+ },
+ "metadataService": {
+ "context" : "/api",
+ "host" : "localhost",
+ "port" : 8080
+ },
+ "metadataDynamicCheck" : {
+ "initDelayMillis" : 1000,
+ "delayMillis" : 30000
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-devtools/.gitignore
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-devtools/.gitignore b/eagle-core/eagle-alert/alert/alert-devtools/.gitignore
new file mode 100644
index 0000000..415b892
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-devtools/.gitignore
@@ -0,0 +1,3 @@
+logs/
+target/
+/target/
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-devtools/bin/kafka-producer.sh
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-devtools/bin/kafka-producer.sh b/eagle-core/eagle-alert/alert/alert-devtools/bin/kafka-producer.sh
new file mode 100755
index 0000000..59068de
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-devtools/bin/kafka-producer.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ "x$EAGLE_HEAP_OPTS" = "x" ]; then
+ export EAGLE_HEAP_OPTS="-Xmx512M"
+fi
+
+exec $(dirname $0)/run-class.sh org.apache.eagle.contrib.kafka.ProducerTool "$@"
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-devtools/bin/kafka-server-start.sh
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-devtools/bin/kafka-server-start.sh b/eagle-core/eagle-alert/alert/alert-devtools/bin/kafka-server-start.sh
new file mode 100755
index 0000000..a5c0ed0
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-devtools/bin/kafka-server-start.sh
@@ -0,0 +1,51 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ $# -lt 1 ];
+then
+ echo "USAGE: $0 [-daemon] conf/kafka-server.properties [--override property=value]*"
+ exit 1
+fi
+
+base_dir=$(dirname $0)
+
+if [ "x$LOG4J_OPTS" = "x" ]; then
+ export KAFKA_LOG4J_OPTS="-Dlog4j.configuration=file:$base_dir/../conf/log4j.properties"
+fi
+
+if [ "x$HEAP_OPTS" = "x" ]; then
+ export EAGLE_HEAP_OPTS="-Xmx1G -Xms1G"
+fi
+
+EXTRA_ARGS="-name kafkaServer -loggc"
+
+COMMAND=$1
+case $COMMAND in
+ -daemon)
+ EXTRA_ARGS="-daemon "$EXTRA_ARGS
+ shift
+ ;;
+ *)
+ ;;
+esac
+
+$base_dir/kafka-server-status.sh 1>/dev/null 2>&1
+if [ "$?" == "" ];then
+ echo "Kafka is still running, please stop firstly before starting"
+ exit 0
+else
+ exec $base_dir/run-class.sh $EXTRA_ARGS kafka.Kafka "$@"
+fi
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-devtools/bin/kafka-server-status.sh
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-devtools/bin/kafka-server-status.sh b/eagle-core/eagle-alert/alert/alert-devtools/bin/kafka-server-status.sh
new file mode 100755
index 0000000..c794425
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-devtools/bin/kafka-server-status.sh
@@ -0,0 +1,24 @@
+#!/bin/sh
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+PIDS=$(ps ax | grep -i 'kafka\.Kafka' | grep java | grep -v grep | awk '{print $1}')
+
+if [ -z "$PIDS" ]; then
+ echo "No kafka server is running"
+ exit 1
+else
+ echo "Kafka server is running at $PIDS"
+ exit 0
+fi
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-devtools/bin/kafka-server-stop.sh
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-devtools/bin/kafka-server-stop.sh b/eagle-core/eagle-alert/alert/alert-devtools/bin/kafka-server-stop.sh
new file mode 100755
index 0000000..9de6d76
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-devtools/bin/kafka-server-stop.sh
@@ -0,0 +1,23 @@
+#!/bin/sh
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+PIDS=$(ps ax | grep -i 'kafka\.Kafka' | grep java | grep -v grep | awk '{print $1}')
+
+if [ -z "$PIDS" ]; then
+ echo "No kafka server to stop"
+ exit 1
+else
+ kill -s TERM $PIDS
+fi
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-devtools/bin/kafka-topics.sh
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-devtools/bin/kafka-topics.sh b/eagle-core/eagle-alert/alert/alert-devtools/bin/kafka-topics.sh
new file mode 100755
index 0000000..6f40511
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-devtools/bin/kafka-topics.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+exec $(dirname $0)/run-class.sh kafka.admin.TopicCommand "$@"
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-devtools/bin/run-class.sh
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-devtools/bin/run-class.sh b/eagle-core/eagle-alert/alert/alert-devtools/bin/run-class.sh
new file mode 100755
index 0000000..8886bae
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-devtools/bin/run-class.sh
@@ -0,0 +1,112 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ $# -lt 1 ];
+then
+ echo "USAGE: $0 [-daemon] [-name servicename] [-loggc] classname [opts]"
+ exit 1
+fi
+
+project_base=$(dirname $0)/..
+
+for file in $project_base/target/lib/*;do
+ export CLASSPATH=$file:$CLASSPATH
+done
+for file in $project_base/target/*.jar;do
+ export CLASSPATH=$file:$CLASSPATH
+done
+
+if [ -z "JMX_OPTS" ]; then
+ export JMX_OPTS="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false "
+fi
+
+# Log directory to use
+if [ "x$LOG_DIR" = "x" ]; then
+ LOG_DIR="$project_base/logs"
+fi
+
+# create logs directory
+if [ ! -d "$LOG_DIR" ]; then
+ mkdir -p "$LOG_DIR"
+fi
+
+# Log4j settings
+if [ -z "$LOG4J_OPTS" ]; then
+ # Log to console. This is a tool.
+ LOG4J_OPTS="-Dlog4j.configuration=file:$project_base/conf/cli-log4j.properties"
+else
+ # create logs directory
+ if [ ! -d "$LOG_DIR" ]; then
+ mkdir -p "$LOG_DIR"
+ fi
+fi
+
+# Which java to use
+if [ -z "$JAVA_HOME" ]; then
+ JAVA="java"
+else
+ JAVA="$JAVA_HOME/bin/java"
+fi
+
+# Memory options
+if [ -z "$HEAP_OPTS" ]; then
+ HEAP_OPTS="-Xmx256M"
+fi
+
+# JVM performance options
+if [ -z "$JVM_PERFORMANCE_OPTS" ]; then
+ JVM_PERFORMANCE_OPTS="-server -XX:+UseG1GC -XX:MaxGCPauseMillis=20 -XX:InitiatingHeapOccupancyPercent=35 -XX:+DisableExplicitGC -Djava.awt.headless=true"
+fi
+
+while [ $# -gt 0 ]; do
+ COMMAND=$1
+ case $COMMAND in
+ -name)
+ DAEMON_NAME=$2
+ CONSOLE_OUTPUT_FILE=$LOG_DIR/$DAEMON_NAME.out
+ shift 2
+ ;;
+ -loggc)
+ if [ -z "$GC_LOG_OPTS" ]; then
+ GC_LOG_ENABLED="true"
+ fi
+ shift
+ ;;
+ -daemon)
+ DAEMON_MODE="true"
+ shift
+ ;;
+ *)
+ break
+ ;;
+ esac
+done
+
+# GC options
+GC_FILE_SUFFIX='-gc.log'
+GC_LOG_FILE_NAME=''
+if [ "x$GC_LOG_ENABLED" = "xtrue" ]; then
+ GC_LOG_FILE_NAME=$DAEMON_NAME$GC_FILE_SUFFIX
+ GC_LOG_OPTS="-Xloggc:$LOG_DIR/$GC_LOG_FILE_NAME -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCTimeStamps "
+fi
+
+# Launch mode
+if [ "x$DAEMON_MODE" = "xtrue" ]; then
+ nohup $JAVA $HEAP_OPTS $JVM_PERFORMANCE_OPTS $GC_LOG_OPTS $EAGLE_JMX_OPTS $LOG4J_OPTS -cp $CLASSPATH $EAGLE_OPTS "$@" > "$CONSOLE_OUTPUT_FILE" 2>&1 < /dev/null &
+else
+ exec $JAVA $HEAP_OPTS $JVM_PERFORMANCE_OPTS $GC_LOG_OPTS $EAGLE_JMX_OPTS $LOG4J_OPTS -cp $CLASSPATH $EAGLE_OPTS "$@"
+fi
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-devtools/bin/start-coordinator.sh
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-devtools/bin/start-coordinator.sh b/eagle-core/eagle-alert/alert/alert-devtools/bin/start-coordinator.sh
new file mode 100755
index 0000000..1f5bea0
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-devtools/bin/start-coordinator.sh
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+cd $(dirname $0)/../../alert-coordinator
+
+mvn jetty:run
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-devtools/bin/start-integration1.sh
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-devtools/bin/start-integration1.sh b/eagle-core/eagle-alert/alert/alert-devtools/bin/start-integration1.sh
new file mode 100755
index 0000000..cb66149
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-devtools/bin/start-integration1.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+
+echo $(dirname $0)
+
+#start topology
+echo "starting topology..."
+cd $(dirname $0)/../../alert-engine/alert-engine-base/
+
+
+echo " as dev tests, tail -f test.log | grep AlertStreamEvent for alert stream....."
+
+mvn test -Dtest=org.apache.eagle.alert.engine.e2e.Integration1 | tee test.log
+
+# tail log output
+# tail -f test.log | grep AlertStreamEvent
+
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-devtools/bin/start-integration2.sh
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-devtools/bin/start-integration2.sh b/eagle-core/eagle-alert/alert/alert-devtools/bin/start-integration2.sh
new file mode 100755
index 0000000..b7797b0
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-devtools/bin/start-integration2.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+
+echo $(dirname $0)
+
+export MAVEN_OPTS=-Xms256M -Xmx1024M
+
+#start topology
+echo "starting topology..."
+cd $(dirname $0)/../../alert-engine/alert-engine-base/
+
+
+echo " as dev tests, tail -f test.log | grep AlertStreamEvent for alert stream....."
+
+
+mvn test -Dtest=org.apache.eagle.alert.engine.e2e.Integration2 | tee test.log
+
+# tail log output
+# tail -f test.log | grep AlertStreamEvent
+
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-devtools/bin/start-metadata.sh
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-devtools/bin/start-metadata.sh b/eagle-core/eagle-alert/alert/alert-devtools/bin/start-metadata.sh
new file mode 100755
index 0000000..1732c04
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-devtools/bin/start-metadata.sh
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+cd $(dirname $0)/../../alert-metadata-parent/alert-metadata-service
+
+mvn jetty:run
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-devtools/bin/start-sampleclient1.sh
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-devtools/bin/start-sampleclient1.sh b/eagle-core/eagle-alert/alert/alert-devtools/bin/start-sampleclient1.sh
new file mode 100755
index 0000000..df65875
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-devtools/bin/start-sampleclient1.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+cd $(dirname $0)/../../alert-assembly/
+
+java -cp ${MAVEN_REPO}/org/apache/storm/storm-core/0.9.3/storm-core-0.9.3.jar:target/alert-engine-0.0.1-SNAPSHOT-alert-assembly.jar org.apache.eagle.alert.engine.e2e.SampleClient1
+
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-devtools/bin/start-sampleclient2.sh
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-devtools/bin/start-sampleclient2.sh b/eagle-core/eagle-alert/alert/alert-devtools/bin/start-sampleclient2.sh
new file mode 100755
index 0000000..742b612
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-devtools/bin/start-sampleclient2.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+cd $(dirname $0)/../../alert-assembly/
+
+java -cp ${MAVEN_REPO}/org/apache/storm/storm-core/0.9.3/storm-core-0.9.3.jar:target/alert-engine-0.0.1-SNAPSHOT-alert-assembly.jar org.apache.eagle.alert.engine.e2e.SampleClient2
+
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-devtools/bin/start-zk-kafka.sh
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-devtools/bin/start-zk-kafka.sh b/eagle-core/eagle-alert/alert/alert-devtools/bin/start-zk-kafka.sh
new file mode 100755
index 0000000..c58de65
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-devtools/bin/start-zk-kafka.sh
@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+cd $(dirname $0)/..
+
+echo "Starting zookeeper"
+bin/zookeeper-server-start.sh -daemon conf/zookeeper-server.properties
+sleep 1
+bin/zookeeper-server-status.sh
+
+echo "Starting kafka"
+bin/kafka-server-start.sh -daemon conf/kafka-server.properties
+sleep 1
+bin/kafka-server-status.sh
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-devtools/bin/stop-zk-kafka.sh
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-devtools/bin/stop-zk-kafka.sh b/eagle-core/eagle-alert/alert/alert-devtools/bin/stop-zk-kafka.sh
new file mode 100755
index 0000000..dc8a898
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-devtools/bin/stop-zk-kafka.sh
@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+cd $(dirname $0)/..
+
+echo "Stopping zookeeper"
+bin/zookeeper-server-stop.sh -daemon conf/zookeeper-server.properties
+sleep 1
+bin/zookeeper-server-status.sh
+
+echo "Stopping kafka"
+bin/kafka-server-stop.sh -daemon conf/kafka-server.properties
+sleep 1
+bin/kafka-server-status.sh
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-devtools/bin/zookeeper-server-start.sh
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-devtools/bin/zookeeper-server-start.sh b/eagle-core/eagle-alert/alert/alert-devtools/bin/zookeeper-server-start.sh
new file mode 100755
index 0000000..a68c271
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-devtools/bin/zookeeper-server-start.sh
@@ -0,0 +1,50 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ $# -lt 1 ];
+then
+ echo "USAGE: $0 [-daemon] conf/zookeeper-server.properties"
+ exit 1
+fi
+base_dir=$(dirname $0)
+
+if [ "x$LOG4J_OPTS" = "x" ]; then
+ export LOG4J_OPTS="-Dlog4j.configuration=file:$base_dir/../conf/log4j.properties"
+fi
+
+if [ "x$EAGLE_HEAP_OPTS" = "x" ]; then
+ export EAGLE_HEAP_OPTS="-Xmx512M -Xms512M"
+fi
+
+EXTRA_ARGS="-name zookeeper -loggc"
+
+COMMAND=$1
+case $COMMAND in
+ -daemon)
+ EXTRA_ARGS="-daemon "$EXTRA_ARGS
+ shift
+ ;;
+ *)
+ ;;
+esac
+
+$base_dir/zookeeper-server-status.sh 1>/dev/null 2>&1
+if [ "$?" == "" ];then
+ echo "Zookeeper is still running, please stop firstly before starting"
+ exit 0
+else
+ exec $base_dir/run-class.sh $EXTRA_ARGS org.apache.zookeeper.server.quorum.QuorumPeerMain "$@"
+fi
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-devtools/bin/zookeeper-server-status.sh
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-devtools/bin/zookeeper-server-status.sh b/eagle-core/eagle-alert/alert/alert-devtools/bin/zookeeper-server-status.sh
new file mode 100755
index 0000000..223a310
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-devtools/bin/zookeeper-server-status.sh
@@ -0,0 +1,24 @@
+#!/bin/sh
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+PIDS=$(ps ax | grep java | grep -i QuorumPeerMain | grep -v grep | awk '{print $1}')
+
+if [ -z "$PIDS" ]; then
+ echo "No zookeeper server is running"
+ exit 1
+else
+ echo "Zookeeper server is running at $PIDS"
+ exit 0
+fi
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-devtools/bin/zookeeper-server-stop.sh
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-devtools/bin/zookeeper-server-stop.sh b/eagle-core/eagle-alert/alert/alert-devtools/bin/zookeeper-server-stop.sh
new file mode 100755
index 0000000..4155182
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-devtools/bin/zookeeper-server-stop.sh
@@ -0,0 +1,24 @@
+#!/bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+PIDS=$(ps ax | grep java | grep -i QuorumPeerMain | grep -v grep | awk '{print $1}')
+
+if [ -z "$PIDS" ]; then
+ echo "No zookeeper server to stop"
+ exit 1
+else
+ kill -s TERM $PIDS
+fi
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-devtools/conf/cli-log4j.properties
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-devtools/conf/cli-log4j.properties b/eagle-core/eagle-alert/alert/alert-devtools/conf/cli-log4j.properties
new file mode 100644
index 0000000..d59ded6
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-devtools/conf/cli-log4j.properties
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+log4j.rootLogger=INFO, stdout
+
+# standard output
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %p [%t] %c{2}[%L]: %m%n
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-devtools/conf/kafka-server.properties
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-devtools/conf/kafka-server.properties b/eagle-core/eagle-alert/alert/alert-devtools/conf/kafka-server.properties
new file mode 100644
index 0000000..a9e0010
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-devtools/conf/kafka-server.properties
@@ -0,0 +1,115 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# see kafka.server.KafkaConfig for additional details and defaults
+
+############################# Server Basics #############################
+
+# The id of the broker. This must be set to a unique integer for each broker.
+broker.id=0
+
+############################# Socket Server Settings #############################
+
+# The address the socket server listens on. It will get the value returned from
+# java.net.InetAddress.getCanonicalHostName() if not configured.
+# FORMAT:
+# listeners = security_protocol://host_name:port
+# EXAMPLE:
+# listeners = PLAINTEXT://your.host.name:9092
+#listeners=PLAINTEXT://:9092
+
+# Hostname and port the broker will advertise to producers and consumers. If not set,
+# it uses the value for "listeners" if configured. Otherwise, it will use the value
+# returned from java.net.InetAddress.getCanonicalHostName().
+#advertised.listeners=PLAINTEXT://your.host.name:9092
+
+# The number of threads handling network requests
+num.network.threads=3
+
+# The number of threads doing disk I/O
+num.io.threads=8
+
+# The send buffer (SO_SNDBUF) used by the socket server
+socket.send.buffer.bytes=102400
+
+# The receive buffer (SO_RCVBUF) used by the socket server
+socket.receive.buffer.bytes=102400
+
+# The maximum size of a request that the socket server will accept (protection against OOM)
+socket.request.max.bytes=104857600
+
+
+############################# Log Basics #############################
+
+# A comma seperated list of directories under which to store log files
+log.dirs=/tmp/dev-kafka-logs
+
+# The default number of log partitions per topic. More partitions allow greater
+# parallelism for consumption, but this will also result in more files across
+# the brokers.
+num.partitions=1
+
+# The number of threads per data directory to be used for log recovery at startup and flushing at shutdown.
+# This value is recommended to be increased for installations with data dirs located in RAID array.
+num.recovery.threads.per.data.dir=1
+
+############################# Log Flush Policy #############################
+
+# Messages are immediately written to the filesystem but by default we only fsync() to sync
+# the OS cache lazily. The following configurations control the flush of data to disk.
+# There are a few important trade-offs here:
+# 1. Durability: Unflushed data may be lost if you are not using replication.
+# 2. Latency: Very large flush intervals may lead to latency spikes when the flush does occur as there will be a lot of data to flush.
+# 3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to exceessive seeks.
+# The settings below allow one to configure the flush policy to flush data after a period of time or
+# every N messages (or both). This can be done globally and overridden on a per-topic basis.
+
+# The number of messages to accept before forcing a flush of data to disk
+#log.flush.interval.messages=10000
+
+# The maximum amount of time a message can sit in a log before we force a flush
+#log.flush.interval.ms=1000
+
+############################# Log Retention Policy #############################
+
+# The following configurations control the disposal of log segments. The policy can
+# be set to delete segments after a period of time, or after a given size has accumulated.
+# A segment will be deleted whenever *either* of these criteria are met. Deletion always happens
+# from the end of the log.
+
+# The minimum age of a log file to be eligible for deletion
+log.retention.hours=168
+
+# A size-based retention policy for logs. Segments are pruned from the log as long as the remaining
+# segments don't drop below log.retention.bytes.
+#log.retention.bytes=1073741824
+
+# The maximum size of a log segment file. When this size is reached a new log segment will be created.
+log.segment.bytes=1073741824
+
+# The interval at which log segments are checked to see if they can be deleted according
+# to the retention policies
+log.retention.check.interval.ms=300000
+
+############################# Zookeeper #############################
+
+# Zookeeper connection string (see zookeeper docs for details).
+# This is a comma separated host:port pairs, each corresponding to a zk
+# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002".
+# You can also append an optional chroot string to the urls to specify the
+# root directory for all kafka znodes.
+zookeeper.connect=localhost:2181
+
+# Timeout in ms for connecting to zookeeper
+zookeeper.connection.timeout.ms=6000
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-devtools/conf/log4j.properties
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-devtools/conf/log4j.properties b/eagle-core/eagle-alert/alert/alert-devtools/conf/log4j.properties
new file mode 100644
index 0000000..d59ded6
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-devtools/conf/log4j.properties
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+log4j.rootLogger=INFO, stdout
+
+# standard output
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %p [%t] %c{2}[%L]: %m%n
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-devtools/conf/zookeeper-server.properties
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-devtools/conf/zookeeper-server.properties b/eagle-core/eagle-alert/alert/alert-devtools/conf/zookeeper-server.properties
new file mode 100644
index 0000000..dd71ffc
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-devtools/conf/zookeeper-server.properties
@@ -0,0 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# the directory where the snapshot is stored.
+dataDir=/tmp/dev-zookeeper-data
+# the port at which the clients will connect
+clientPort=2181
+# disable the per-ip limit on the number of connections since this is a non-production config
+maxClientCnxns=0
\ No newline at end of file
[15/19] incubator-eagle git commit: EAGLE-324: Init branch-v0.5
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/impl/strategies/SameTopologySlotStrategy.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/impl/strategies/SameTopologySlotStrategy.java b/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/impl/strategies/SameTopologySlotStrategy.java
new file mode 100644
index 0000000..e755237
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/impl/strategies/SameTopologySlotStrategy.java
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.coordinator.impl.strategies;
+
+import static org.apache.eagle.alert.coordinator.CoordinatorConstants.CONFIG_ITEM_TOPOLOGY_LOAD_UPBOUND;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.eagle.alert.coordination.model.WorkSlot;
+import org.apache.eagle.alert.coordination.model.internal.StreamGroup;
+import org.apache.eagle.alert.coordination.model.internal.Topology;
+import org.apache.eagle.alert.coordinator.CoordinatorConstants;
+import org.apache.eagle.alert.coordinator.IScheduleContext;
+import org.apache.eagle.alert.coordinator.TopologyMgmtService;
+import org.apache.eagle.alert.coordinator.TopologyMgmtService.TopologyMeta;
+import org.apache.eagle.alert.coordinator.model.AlertBoltUsage;
+import org.apache.eagle.alert.coordinator.model.TopologyUsage;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+
+/**
+ * A simple strategy that only find the bolts in the same topology as the
+ * required work slots.
+ *
+ * Invariant:<br/>
+ * One slot queue only on the one topology.<br/>
+ * One topology doesn't contains two same partition slot queues.
+ *
+ * @since Apr 27, 2016
+ *
+ */
+public class SameTopologySlotStrategy implements IWorkSlotStrategy {
+
+ private static final Logger LOG = LoggerFactory.getLogger(SameTopologySlotStrategy.class);
+
+ private final IScheduleContext context;
+ private final StreamGroup partitionGroup;
+ private final TopologyMgmtService mgmtService;
+
+// private final int numOfPoliciesBoundPerBolt;
+ private final double topoLoadUpbound;
+
+ public SameTopologySlotStrategy(IScheduleContext context, StreamGroup streamPartitionGroup,
+ TopologyMgmtService mgmtService) {
+ this.context = context;
+ this.partitionGroup = streamPartitionGroup;
+ this.mgmtService = mgmtService;
+
+ Config config = ConfigFactory.load().getConfig(CoordinatorConstants.CONFIG_ITEM_COORDINATOR);
+// numOfPoliciesBoundPerBolt = config.getInt(CoordinatorConstants.POLICIES_PER_BOLT);
+ topoLoadUpbound = config.getDouble(CONFIG_ITEM_TOPOLOGY_LOAD_UPBOUND);
+ }
+
+ /**
+ * @param isDedicated
+ * - not used yet!
+ */
+ @Override
+ public List<WorkSlot> reserveWorkSlots(int size, boolean isDedicated, Map<String, Object> properties) {
+ Iterator<Topology> it = context.getTopologies().values().stream().filter((t) -> t.getNumOfAlertBolt() >= size)
+ .iterator();
+ // priority strategy first???
+ List<WorkSlot> slots = new ArrayList<WorkSlot>();
+ while (it.hasNext()) {
+ Topology t = it.next();
+ if (getQueueOnTopology(size, slots, t)) {
+ break;
+ }
+ }
+
+ if (slots.size() == 0) {
+ int supportedSize = mgmtService.getNumberOfAlertBoltsInTopology();
+ if (size > supportedSize) {
+ LOG.error("can not find available slots for queue, required size {}, supported size {} !", size, supportedSize);
+ return Collections.emptyList();
+ }
+ TopologyMeta topoMeta = mgmtService.creatTopology();
+ if (topoMeta == null) {
+ LOG.error("can not create topology for given queue requirement, required size {}, requried partition group {} !", size, partitionGroup);
+ return Collections.emptyList();
+ }
+
+ context.getTopologies().put(topoMeta.topologyId, topoMeta.topology);
+ context.getTopologyUsages().put(topoMeta.topologyId, topoMeta.usage);
+ boolean placed = getQueueOnTopology(size, slots, topoMeta.topology);
+ if (!placed) {
+ LOG.error("can not find available slots from new created topology, required size {}. This indicates an error !", size);
+ }
+ }
+ return slots;
+ }
+
+ private boolean getQueueOnTopology(int size, List<WorkSlot> slots, Topology t) {
+ TopologyUsage u = context.getTopologyUsages().get(t.getName());
+ if (!isTopologyAvailable(u)) {
+ return false;
+ }
+
+ List<String> bolts = new ArrayList<String>();
+ for (AlertBoltUsage alertUsage : u.getAlertUsages().values()) {
+ if (isBoltAvailable(alertUsage)) {
+ bolts.add(alertUsage.getBoltId());
+ }
+
+ if (bolts.size() == size) {
+ break;
+ }
+ }
+
+ if (bolts.size() == size) {
+ for (String boltId : bolts) {
+ WorkSlot slot = new WorkSlot(t.getName(), boltId);
+ slots.add(slot);
+ }
+ return true;
+ }
+ return false;
+ }
+
+ private boolean isTopologyAvailable(TopologyUsage u) {
+// for (MonitoredStream stream : u.getMonitoredStream()) {
+// if (partition.equals(stream.getStreamParitition())) {
+// return false;
+// }
+// }
+ if (u == null || u.getLoad() > topoLoadUpbound) {
+ return false;
+ }
+
+ return true;
+ }
+
+ private boolean isBoltAvailable(AlertBoltUsage alertUsage) {
+ // FIXME : more detail to compare on queue exclusion check
+ if (alertUsage.getQueueSize() > 0) {
+ return false;
+ }
+ // actually it's now 0;
+ return true;
+// return alertUsage.getPolicies().size() < numOfPoliciesBoundPerBolt;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/model/AlertBoltUsage.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/model/AlertBoltUsage.java b/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/model/AlertBoltUsage.java
new file mode 100644
index 0000000..e9148f5
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/model/AlertBoltUsage.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.coordinator.model;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.eagle.alert.coordination.model.internal.StreamGroup;
+import org.apache.eagle.alert.coordination.model.internal.StreamWorkSlotQueue;
+import org.apache.eagle.alert.engine.coordinator.PolicyDefinition;
+
+/**
+ * @since Mar 28, 2016
+ *
+ */
+public class AlertBoltUsage {
+
+ private String boltId;
+ private List<String> policies = new ArrayList<String>();
+ // the stream partitions group that scheduled for this given alert bolt
+ private List<StreamGroup> partitions = new ArrayList<StreamGroup>();
+ // the slot queue that scheduled for this given aler bolt
+ private List<StreamWorkSlotQueue> referQueues = new ArrayList<StreamWorkSlotQueue>();
+ private double load;
+
+ public AlertBoltUsage(String anid) {
+ this.boltId = anid;
+ }
+
+ public String getBoltId() {
+ return boltId;
+ }
+
+ public void setBoltId(String boltId) {
+ this.boltId = boltId;
+ }
+
+ public List<String> getPolicies() {
+ return policies;
+ }
+
+ public void addPolicies(PolicyDefinition pd) {
+ policies.add(pd.getName());
+ // add first partition
+// for (StreamPartition par : pd.getPartitionSpec()) {
+// partitions.add(par);
+// }
+ }
+
+ public double getLoad() {
+ return load;
+ }
+
+ public void setLoad(double load) {
+ this.load = load;
+ }
+
+ public List<StreamGroup> getPartitions() {
+ return partitions;
+ }
+
+ public List<StreamWorkSlotQueue> getReferQueues() {
+ return referQueues;
+ }
+
+ public int getQueueSize() {
+ return referQueues.size();
+ }
+
+ public void addQueue(StreamGroup streamPartition, StreamWorkSlotQueue queue) {
+ this.referQueues.add(queue);
+ this.partitions.add(streamPartition);
+ }
+
+ public void removeQueue(StreamWorkSlotQueue queue) {
+ this.referQueues.remove(queue);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/model/GroupBoltUsage.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/model/GroupBoltUsage.java b/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/model/GroupBoltUsage.java
new file mode 100644
index 0000000..86238d1
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/model/GroupBoltUsage.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.coordinator.model;
+
+
+/**
+ * @since Mar 28, 2016
+ *
+ */
+public class GroupBoltUsage {
+
+ private String boltId;
+ private double load;
+
+ public GroupBoltUsage(String boltId) {
+ this.boltId = boltId;
+ }
+
+// private final Set<String> streams = new HashSet<String>();
+// private final Map<String, StreamFilter> filters = new HashMap<String, StreamFilter>();
+
+// private final Map<String, List<StreamPartition>> groupByMeta;
+
+ public double getLoad() {
+ return load;
+ }
+
+ public void setLoad(double load) {
+ this.load = load;
+ }
+
+// public Set<String> getStreams() {
+// return streams;
+// }
+//
+//
+// public Map<String, StreamFilter> getFilters() {
+// return filters;
+// }
+
+// public Map<String, List<StreamPartition>> getGroupByMeta() {
+// return groupByMeta;
+// }
+
+ public String getBoltId() {
+ return boltId;
+ }
+
+ public void setBoltId(String boltId) {
+ this.boltId = boltId;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/model/TopologyUsage.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/model/TopologyUsage.java b/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/model/TopologyUsage.java
new file mode 100644
index 0000000..6eb6195
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/model/TopologyUsage.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.coordinator.model;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.eagle.alert.coordination.model.internal.MonitoredStream;
+
+/**
+ * @since Mar 27, 2016
+ *
+ */
+public class TopologyUsage {
+ // topo info
+ private String topoName;
+ private final Set<String> datasources = new HashSet<String>();
+ // usage information
+ private final Set<String> policies = new HashSet<String>();
+ private final Map<String, AlertBoltUsage> alertUsages = new HashMap<String, AlertBoltUsage>();
+ private final Map<String, GroupBoltUsage> groupUsages = new HashMap<String, GroupBoltUsage>();
+ private final List<MonitoredStream> monitoredStream = new ArrayList<MonitoredStream>();
+
+ private double load;
+
+ /**
+ * This is to be the existing/previous meta-data. <br/>
+ * Only one group meta-data for all of the group bolts in this topology.
+ */
+
+ public TopologyUsage() {
+ }
+
+ public TopologyUsage(String name) {
+ this.topoName = name;
+ }
+
+ public String getTopoName() {
+ return topoName;
+ }
+
+ public void setTopoName(String topoId) {
+ this.topoName = topoId;
+ }
+
+ public Set<String> getDataSources() {
+ return datasources;
+ }
+
+ public Set<String> getPolicies() {
+ return policies;
+ }
+
+ public Map<String, AlertBoltUsage> getAlertUsages() {
+ return alertUsages;
+ }
+
+ public AlertBoltUsage getAlertBoltUsage(String boltId) {
+ return alertUsages.get(boltId);
+ }
+
+ public Map<String, GroupBoltUsage> getGroupUsages() {
+ return groupUsages;
+ }
+
+ public List<MonitoredStream> getMonitoredStream() {
+ return monitoredStream;
+ }
+
+ public void addMonitoredStream(MonitoredStream par) {
+ if (!this.monitoredStream.contains(par)) {
+ this.monitoredStream.add(par);
+ }
+ }
+
+ public double getLoad() {
+ return load;
+ }
+
+ public void setLoad(double load) {
+ this.load = load;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/provider/InMemScheduleConext.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/provider/InMemScheduleConext.java b/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/provider/InMemScheduleConext.java
new file mode 100644
index 0000000..84a4061
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/provider/InMemScheduleConext.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.coordinator.provider;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.eagle.alert.coordination.model.Kafka2TupleMetadata;
+import org.apache.eagle.alert.coordination.model.internal.MonitoredStream;
+import org.apache.eagle.alert.coordination.model.internal.PolicyAssignment;
+import org.apache.eagle.alert.coordination.model.internal.StreamGroup;
+import org.apache.eagle.alert.coordination.model.internal.Topology;
+import org.apache.eagle.alert.coordinator.IScheduleContext;
+import org.apache.eagle.alert.coordinator.model.TopologyUsage;
+import org.apache.eagle.alert.engine.coordinator.PolicyDefinition;
+import org.apache.eagle.alert.engine.coordinator.Publishment;
+import org.apache.eagle.alert.engine.coordinator.StreamDefinition;
+
+/**
+ * @since Mar 28, 2016
+ *
+ */
+public class InMemScheduleConext implements IScheduleContext {
+
+ private Map<String, Topology> topologies = new HashMap<String, Topology>();
+ private Map<String, TopologyUsage> usages = new HashMap<String, TopologyUsage>();
+ private Map<String, PolicyDefinition> policies = new HashMap<String, PolicyDefinition>();
+ private Map<String, Kafka2TupleMetadata> datasources = new HashMap<String, Kafka2TupleMetadata>();
+ private Map<String, PolicyAssignment> policyAssignments = new HashMap<String, PolicyAssignment>();
+ private Map<String, StreamDefinition> schemas = new HashMap<String, StreamDefinition>();
+ private Map<StreamGroup, MonitoredStream> monitoredStreams = new HashMap<StreamGroup, MonitoredStream>();
+ private Map<String, Publishment> publishments = new HashMap<String, Publishment>();
+
+ public InMemScheduleConext() {
+ }
+
+ public InMemScheduleConext(IScheduleContext context) {
+ this.topologies = new HashMap<String, Topology>(context.getTopologies());
+ this.usages = new HashMap<String, TopologyUsage>(context.getTopologyUsages());
+ this.policies = new HashMap<String, PolicyDefinition>(context.getPolicies());
+ this.datasources = new HashMap<String, Kafka2TupleMetadata>(context.getDataSourceMetadata());
+ this.policyAssignments = new HashMap<String, PolicyAssignment>(context.getPolicyAssignments());
+ this.schemas = new HashMap<String, StreamDefinition>(context.getStreamSchemas());
+ this.monitoredStreams = new HashMap<StreamGroup, MonitoredStream>(context.getMonitoredStreams());
+ this.publishments = new HashMap<String, Publishment>(context.getPublishments());
+ }
+
+ public InMemScheduleConext(Map<String, Topology> topologies2, Map<String, PolicyAssignment> assignments,
+ Map<String, Kafka2TupleMetadata> kafkaSources, Map<String, PolicyDefinition> policies2,
+ Map<String, Publishment> publishments2, Map<String, StreamDefinition> streamDefinitions,
+ Map<StreamGroup, MonitoredStream> monitoredStreamMap, Map<String, TopologyUsage> usages2) {
+ this.topologies = topologies2;
+ this.policyAssignments = assignments;
+ this.datasources = kafkaSources;
+ this.policies = policies2;
+ this.publishments = publishments2;
+ this.schemas = streamDefinitions;
+ this.monitoredStreams = monitoredStreamMap;
+ this.usages = usages2;
+ }
+
+ public Map<String, Topology> getTopologies() {
+ return topologies;
+ }
+
+ public void addTopology(Topology topo) {
+ topologies.put(topo.getName(), topo);
+ }
+
+ public Map<String, TopologyUsage> getTopologyUsages() {
+ return usages;
+ }
+
+ public void addTopologyUsages(TopologyUsage usage) {
+ usages.put(usage.getTopoName(), usage);
+ }
+
+ public Map<String, PolicyDefinition> getPolicies() {
+ return policies;
+ }
+
+ public void addPoilcy(PolicyDefinition pd) {
+ this.policies.put(pd.getName(), pd);
+ }
+
+ public Map<String, Kafka2TupleMetadata> getDatasources() {
+ return datasources;
+ }
+
+ public void setDatasources(Map<String, Kafka2TupleMetadata> datasources) {
+ this.datasources = datasources;
+ }
+
+ public void addDataSource(Kafka2TupleMetadata dataSource) {
+ this.datasources.put(dataSource.getName(), dataSource);
+ }
+
+ @Override
+ public Map<String, Kafka2TupleMetadata> getDataSourceMetadata() {
+ return datasources;
+ }
+
+ public void setPolicyOrderedTopologies(Map<String, PolicyAssignment> policyAssignments) {
+ this.policyAssignments = policyAssignments;
+ }
+
+ public Map<String, PolicyAssignment> getPolicyAssignments() {
+ return this.policyAssignments;
+ }
+
+ @Override
+ public Map<String, StreamDefinition> getStreamSchemas() {
+ return schemas;
+ }
+
+ public void addSchema(StreamDefinition schema) {
+ this.schemas.put(schema.getStreamId(), schema);
+ }
+
+ public void setStreamSchemas(Map<String, StreamDefinition> schemas) {
+ this.schemas = schemas;
+ }
+
+ @Override
+ public Map<StreamGroup, MonitoredStream> getMonitoredStreams() {
+ return monitoredStreams;
+ }
+
+ @Override
+ public Map<String, Publishment> getPublishments() {
+ return publishments;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/provider/ScheduleContextBuilder.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/provider/ScheduleContextBuilder.java b/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/provider/ScheduleContextBuilder.java
new file mode 100644
index 0000000..d4d6c0c
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/provider/ScheduleContextBuilder.java
@@ -0,0 +1,400 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.coordinator.provider;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+
+import org.apache.eagle.alert.coordination.model.Kafka2TupleMetadata;
+import org.apache.eagle.alert.coordination.model.ScheduleState;
+import org.apache.eagle.alert.coordination.model.WorkSlot;
+import org.apache.eagle.alert.coordination.model.internal.MonitoredStream;
+import org.apache.eagle.alert.coordination.model.internal.PolicyAssignment;
+import org.apache.eagle.alert.coordination.model.internal.StreamGroup;
+import org.apache.eagle.alert.coordination.model.internal.StreamWorkSlotQueue;
+import org.apache.eagle.alert.coordination.model.internal.Topology;
+import org.apache.eagle.alert.coordinator.IScheduleContext;
+import org.apache.eagle.alert.coordinator.model.AlertBoltUsage;
+import org.apache.eagle.alert.coordinator.model.GroupBoltUsage;
+import org.apache.eagle.alert.coordinator.model.TopologyUsage;
+import org.apache.eagle.alert.engine.coordinator.PolicyDefinition;
+import org.apache.eagle.alert.engine.coordinator.Publishment;
+import org.apache.eagle.alert.engine.coordinator.StreamDefinition;
+import org.apache.eagle.alert.service.IMetadataServiceClient;
+import org.apache.eagle.alert.service.MetadataServiceClientImpl;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.typesafe.config.Config;
+
+/**
+ * FIXME: this class focus on correctness, not the efficiency now. There might
+ * be problem when metadata size grows too big.
+ *
+ * @since May 3, 2016
+ *
+ */
+public class ScheduleContextBuilder {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ScheduleContextBuilder.class);
+ private static final String UNIQUE_BOLT_ID = "%s-%s";// toponame-boltname
+
+ private IMetadataServiceClient client;
+
+ private Map<String, Topology> topologies;
+ private Map<String, PolicyAssignment> assignments;
+ private Map<String, Kafka2TupleMetadata> kafkaSources;
+ private Map<String, PolicyDefinition> policies;
+ private Map<String, Publishment> publishments;
+ private Map<String, StreamDefinition> streamDefinitions;
+ private Map<StreamGroup, MonitoredStream> monitoredStreamMap;
+ private Map<String, TopologyUsage> usages;
+
+ public ScheduleContextBuilder(Config config) {
+ client = new MetadataServiceClientImpl(config);
+ }
+
+ public ScheduleContextBuilder(IMetadataServiceClient client) {
+ this.client = client;
+ }
+
+ /**
+ * Built a shcedule context for metadata client service.
+ *
+ * @return
+ */
+ public IScheduleContext buildContext() {
+ topologies = listToMap(client.listTopologies());
+ kafkaSources = listToMap(client.listDataSources());
+ policies = listToMap(client.listPolicies());
+ publishments = listToMap(client.listPublishment());
+ streamDefinitions = listToMap(client.listStreams());
+
+ // TODO: See ScheduleState comments on how to improve the storage
+ ScheduleState state = client.getVersionedSpec();
+ assignments = listToMap(state == null ? new ArrayList<PolicyAssignment>() : cleanupDeprecatedAssignments(state.getAssignments()));
+
+ monitoredStreamMap = listToMap(state == null ? new ArrayList<MonitoredStream>() : cleanupDeprecatedStreamsAndAssignment(state.getMonitoredStreams()));
+
+ // build based on existing data
+ usages = buildTopologyUsage();
+
+ // copy to shedule context now
+ return new InMemScheduleConext(topologies, assignments, kafkaSources, policies, publishments,
+ streamDefinitions, monitoredStreamMap, usages);
+ }
+
+ /**
+ * 1.
+ * <pre>
+ * Check for deprecated policy stream group with its assigned monitored stream groups.
+ * If this is unmatched, we think the policy' stream group has been changed, remove the policy assignments
+ * If finally, no assignment refer to a given monitored stream, this monitored stream could be removed.
+ * Log when every time a remove happens.
+ * </pre>
+ * 2.
+ * <pre>
+ * if monitored stream's queue's is on non-existing topology, remove it.
+ * </pre>
+ * @param monitoredStreams
+ * @return
+ */
+ private List<MonitoredStream> cleanupDeprecatedStreamsAndAssignment(List<MonitoredStream> monitoredStreams) {
+ List<MonitoredStream> result = new ArrayList<MonitoredStream>(monitoredStreams);
+
+ // clear deprecated streams
+ clearMonitoredStreams(monitoredStreams);
+
+ // build queueId-> streamGroup
+ Map<String, StreamGroup> queue2StreamGroup = new HashMap<String, StreamGroup>();
+ for (MonitoredStream ms : result) {
+ for (StreamWorkSlotQueue q : ms.getQueues()) {
+ queue2StreamGroup.put(q.getQueueId(), ms.getStreamGroup());
+ }
+ }
+
+ // decide the assignment delete set
+ Set<StreamGroup> usedGroups = new HashSet<StreamGroup>();
+ Set<String> toRemove = new HashSet<String>();
+ // check if queue is still referenced by policy assignments
+ for (PolicyAssignment assignment : assignments.values()) {
+ PolicyDefinition def = policies.get(assignment.getPolicyName());
+ StreamGroup group = queue2StreamGroup.get(assignment.getQueueId());
+ if (group == null || !Objects.equals(group.getStreamPartitions(), def.getPartitionSpec())) {
+ LOG.warn(" policy assgiment {} 's policy group {} is different to the monitored stream's partition group {}, "
+ + "this indicates a policy stream partition spec change, the assignment would be removed! ",
+ assignment, def.getPartitionSpec(), group == null ? "'not found'" :group.getStreamPartitions());
+ toRemove.add(assignment.getPolicyName());
+ } else {
+ usedGroups.add(group);
+ }
+ }
+
+ // remove useless
+ assignments.keySet().removeAll(toRemove);
+ // remove non-referenced monitored streams
+ result.removeIf((t) -> {
+ boolean used = usedGroups.contains(t.getStreamGroup());
+ if (!used) {
+ LOG.warn("monitor stream with stream group {} is not referenced, "
+ + "this monitored stream and its worker queu will be removed", t.getStreamGroup());
+ }
+ return !used;
+ });
+
+ return result;
+ }
+
+ private void clearMonitoredStreams(List<MonitoredStream> monitoredStreams) {
+ Iterator<MonitoredStream> it = monitoredStreams.iterator();
+ while (it.hasNext()) {
+ MonitoredStream ms = it.next();
+ Iterator<StreamWorkSlotQueue> queueIt = ms.getQueues().iterator();
+ // clean queue that underly topology is changed(removed/down)
+ while (queueIt.hasNext()) {
+ StreamWorkSlotQueue queue = queueIt.next();
+ boolean deprecated = false;
+ for (WorkSlot ws : queue.getWorkingSlots()) {
+ // check if topology available or bolt available
+ if (!topologies.containsKey(ws.topologyName)
+ || !topologies.get(ws.topologyName).getAlertBoltIds().contains(ws.boltId)) {
+ deprecated = true;
+ break;
+ }
+ }
+ if (deprecated) {
+ queueIt.remove();
+ }
+ }
+
+ if (ms.getQueues().isEmpty()) {
+ it.remove();
+ }
+ }
+ }
+
+ private List<PolicyAssignment> cleanupDeprecatedAssignments(List<PolicyAssignment> list) {
+ List<PolicyAssignment> result = new ArrayList<PolicyAssignment>(list);
+ Iterator<PolicyAssignment> paIt = result.iterator();
+ while (paIt.hasNext()) {
+ PolicyAssignment assignment = paIt.next();
+ if (!policies.containsKey(assignment.getPolicyName())) {
+ LOG.info("Policy assignment {} 's policy not found, this assignment will be removed!", assignment);
+ paIt.remove();
+ }
+ }
+ return result;
+ }
+
+ private <T, K> Map<K, T> listToMap(List<T> collections) {
+ Map<K, T> maps = new HashMap<K, T>(collections.size());
+ for (T t : collections) {
+ maps.put(getKey(t), t);
+ }
+ return maps;
+ }
+
+ /*
+ * One drawback, once we add class, this code need to be changed!
+ */
+ @SuppressWarnings("unchecked")
+ private <T, K> K getKey(T t) {
+ if (t instanceof Topology) {
+ return (K) ((Topology) t).getName();
+ } else if (t instanceof PolicyAssignment) {
+ return (K) ((PolicyAssignment) t).getPolicyName();
+ } else if (t instanceof Kafka2TupleMetadata) {
+ return (K) ((Kafka2TupleMetadata) t).getName();
+ } else if (t instanceof PolicyDefinition) {
+ return (K) ((PolicyDefinition) t).getName();
+ } else if (t instanceof Publishment) {
+ return (K) ((Publishment) t).getName();
+ } else if (t instanceof StreamDefinition) {
+ return (K) ((StreamDefinition) t).getStreamId();
+ } else if (t instanceof MonitoredStream) {
+ return (K) ((MonitoredStream) t).getStreamGroup();
+ }
+ throw new RuntimeException("unexpected key class " + t.getClass());
+ }
+
+ private Map<String, TopologyUsage> buildTopologyUsage() {
+ Map<String, TopologyUsage> usages = new HashMap<String, TopologyUsage>();
+
+ // pre-build data structure for help
+ Map<String, Set<MonitoredStream>> topo2MonitorStream = new HashMap<String, Set<MonitoredStream>>();
+ Map<String, Set<String>> topo2Policies = new HashMap<String, Set<String>>();
+ // simply assume no bolt with the same id
+ Map<String, Set<String>> bolt2Policies = new HashMap<String, Set<String>>();
+ // simply assume no bolt with the same id
+ Map<String, Set<StreamGroup>> bolt2Partition = new HashMap<String, Set<StreamGroup>>();
+ // simply assume no bolt with the same id
+ Map<String, Set<String>> bolt2QueueIds = new HashMap<String, Set<String>>();
+ Map<String, StreamWorkSlotQueue> queueMap = new HashMap<String, StreamWorkSlotQueue>();
+
+ preBuildQueue2TopoMap(topo2MonitorStream, topo2Policies, bolt2Policies, bolt2Partition, bolt2QueueIds, queueMap);
+
+ for (Topology t : topologies.values()) {
+ TopologyUsage u = new TopologyUsage(t.getName());
+ // add group/bolt usages
+ for (String grpBolt : t.getGroupNodeIds()) {
+ GroupBoltUsage grpUsage = new GroupBoltUsage(grpBolt);
+ u.getGroupUsages().put(grpBolt, grpUsage);
+ }
+ for (String alertBolt : t.getAlertBoltIds()) {
+ String uniqueBoltId = String.format(UNIQUE_BOLT_ID, t.getName(), alertBolt);
+
+ AlertBoltUsage alertUsage = new AlertBoltUsage(alertBolt);
+ u.getAlertUsages().put(alertBolt, alertUsage);
+ // complete usage
+ addBoltUsageInfo(bolt2Policies, bolt2Partition, bolt2QueueIds, uniqueBoltId, alertUsage, queueMap);
+ }
+
+ // policy -- policy assignment
+ if (topo2Policies.containsKey(u.getTopoName())) {
+ u.getPolicies().addAll(topo2Policies.get(u.getTopoName()));
+ }
+
+ // data source
+ buildTopologyDataSource(u);
+
+ // topology usage monitored stream -- from monitored steams' queue slot item info
+ if (topo2MonitorStream.containsKey(u.getTopoName())) {
+ u.getMonitoredStream().addAll(topo2MonitorStream.get(u.getTopoName()));
+ }
+
+ usages.put(u.getTopoName(), u);
+ }
+
+ return usages;
+ }
+
+ private void addBoltUsageInfo(Map<String, Set<String>> bolt2Policies,
+ Map<String, Set<StreamGroup>> bolt2Partition, Map<String, Set<String>> bolt2QueueIds, String uniqueAlertBolt,
+ AlertBoltUsage alertUsage, Map<String, StreamWorkSlotQueue> queueMap) {
+ //
+ if (bolt2Policies.containsKey(uniqueAlertBolt)) {
+ alertUsage.getPolicies().addAll(bolt2Policies.get(uniqueAlertBolt));
+ }
+ //
+ if (bolt2Partition.containsKey(uniqueAlertBolt)) {
+ alertUsage.getPartitions().addAll(bolt2Partition.get(uniqueAlertBolt));
+ }
+ //
+ if (bolt2QueueIds.containsKey(uniqueAlertBolt)) {
+ for (String qId : bolt2QueueIds.get(uniqueAlertBolt)) {
+ if (queueMap.containsKey(qId)) {
+ alertUsage.getReferQueues().add(queueMap.get(qId));
+ } else {
+ LOG.error(" queue id {} not found in queue map !", qId);
+ }
+ }
+ }
+ }
+
+ private void buildTopologyDataSource(TopologyUsage u) {
+ for (String policyName : u.getPolicies()) {
+ PolicyDefinition def = policies.get(policyName);
+ if (def != null) {
+ u.getDataSources().addAll(findDatasource(def));
+ } else {
+ LOG.error(" policy not find {}, but reference in topology usage {} !", policyName, u.getTopoName());
+ }
+ }
+ }
+
+ private List<String> findDatasource(PolicyDefinition def) {
+ List<String> result = new ArrayList<String>();
+ List<String> inputStreams = def.getInputStreams();
+ for (String is : inputStreams) {
+ StreamDefinition ss = this.streamDefinitions.get(is);
+ if (ss == null) {
+ LOG.error("policy {} referenced stream definition {} not found in definiton !", def.getName(), is);
+ } else {
+ result.add(ss.getDataSource());
+ }
+ }
+ return result;
+ }
+
+ private void preBuildQueue2TopoMap(
+ Map<String, Set<MonitoredStream>> topo2MonitorStream,
+ Map<String, Set<String>> topo2Policies,
+ Map<String, Set<String>> bolt2Policies,
+ Map<String, Set<StreamGroup>> bolt2Partition,
+ Map<String, Set<String>> bolt2QueueIds,
+ Map<String, StreamWorkSlotQueue> queueMap) {
+ // pre-build structure
+ // why don't reuse the queue.getPolicies
+ Map<String, Set<String>> queue2Policies= new HashMap<String, Set<String>>();
+ for (PolicyAssignment pa : assignments.values()) {
+ if (!queue2Policies.containsKey(pa.getQueueId())) {
+ queue2Policies.put(pa.getQueueId(), new HashSet<String>());
+ }
+ queue2Policies.get(pa.getQueueId()).add(pa.getPolicyName());
+ }
+
+ for (MonitoredStream stream : monitoredStreamMap.values()) {
+ for (StreamWorkSlotQueue q : stream.getQueues()) {
+ queueMap.put(q.getQueueId(), q);
+ Set<String> policiesOnQ = queue2Policies.containsKey(q.getQueueId()) ? queue2Policies.get(q.getQueueId()) : new HashSet<String>();
+
+ for (WorkSlot slot : q.getWorkingSlots()) {
+ // topo2monitoredstream
+ if (!topo2MonitorStream.containsKey(slot.getTopologyName())) {
+ topo2MonitorStream.put(slot.getTopologyName(), new HashSet<MonitoredStream>());
+ }
+ topo2MonitorStream.get(slot.getTopologyName()).add(stream);
+
+ // topo2policy
+ if (!topo2Policies.containsKey(slot.getTopologyName())) {
+ topo2Policies.put(slot.getTopologyName(), new HashSet<String>());
+ }
+ topo2Policies.get(slot.getTopologyName()).addAll(policiesOnQ);
+
+ // bolt2Policy
+ if (!bolt2Policies.containsKey(getUniqueBoltId(slot))) {
+ bolt2Policies.put(getUniqueBoltId(slot), new HashSet<String>());
+ }
+ bolt2Policies.get(getUniqueBoltId(slot)).addAll(policiesOnQ);
+
+ // bolt2Queue
+ if (!bolt2QueueIds.containsKey(getUniqueBoltId(slot))) {
+ bolt2QueueIds.put(getUniqueBoltId(slot), new HashSet<String>());
+ }
+ bolt2QueueIds.get(getUniqueBoltId(slot)).add(q.getQueueId());
+
+ // bolt2Partition
+ if (!bolt2Partition.containsKey(getUniqueBoltId(slot))) {
+ bolt2Partition.put(getUniqueBoltId(slot), new HashSet<StreamGroup>());
+ }
+ bolt2Partition.get(getUniqueBoltId(slot)).add(stream.getStreamGroup());
+ }
+ }
+ }
+ }
+
+ private String getUniqueBoltId(WorkSlot slot) {
+ return String.format(UNIQUE_BOLT_ID, slot.getTopologyName(), slot.getBoltId());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/resource/CoordinatorResource.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/resource/CoordinatorResource.java b/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/resource/CoordinatorResource.java
new file mode 100644
index 0000000..6b8495e
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/resource/CoordinatorResource.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.coordinator.resource;
+
+import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+
+import org.apache.eagle.alert.coordination.model.ScheduleState;
+import org.apache.eagle.alert.coordinator.Coordinator;
+import org.apache.eagle.alert.coordinator.ScheduleOption;
+import org.apache.eagle.alert.utils.JsonUtils;
+
+/**
+ * This is to provide API access even we don't have ZK as intermediate access.
+ * FIXME : more elogant status code
+ *
+ * @since Mar 24, 2016 <br/>
+ */
+@Path("/coordinator")
+@Produces({ "application/json" })
+public class CoordinatorResource {
+
+ // sprint config here?
+ private Coordinator alertCoordinator = new Coordinator();
+
+ @GET
+ @Path("/assignments")
+ public String getAssignments() throws Exception {
+ ScheduleState state = alertCoordinator.getState();
+ return JsonUtils.writeValueAsString(state);
+ }
+
+ @POST
+ @Path("/build")
+ public String build() throws Exception {
+ ScheduleOption option = new ScheduleOption();
+ ScheduleState state = alertCoordinator.schedule(option);
+ return JsonUtils.writeValueAsString(state);
+ }
+
+ /**
+ * Manually update the topology usages, for administration
+ *
+ * @return
+ */
+ @POST
+ @Path("/refreshUsages")
+ public String refreshUsages() {
+ // TODO
+ return "";
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/trigger/DynamicPolicyLoader.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/trigger/DynamicPolicyLoader.java b/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/trigger/DynamicPolicyLoader.java
new file mode 100644
index 0000000..d7881bd
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/trigger/DynamicPolicyLoader.java
@@ -0,0 +1,81 @@
+package org.apache.eagle.alert.coordinator.trigger;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.eagle.alert.engine.coordinator.PolicyDefinition;
+import org.apache.eagle.alert.service.IMetadataServiceClient;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Poll policy change and notify listeners
+ */
+public class DynamicPolicyLoader implements Runnable{
+ private static Logger LOG = LoggerFactory.getLogger(DynamicPolicyLoader.class);
+
+ private IMetadataServiceClient client;
+ // initial cachedPolicies should be empty
+ private Map<String, PolicyDefinition> cachedPolicies = new HashMap<>();
+ private List<PolicyChangeListener> listeners = new ArrayList<>();
+
+ public DynamicPolicyLoader(IMetadataServiceClient client){
+ this.client = client;
+ }
+
+ public synchronized void addPolicyChangeListener(PolicyChangeListener listener){
+ listeners.add(listener);
+ }
+
+ /**
+ * When it is run at the first time, due to cachedPolicies being empty, all existing policies are expected
+ * to be addedPolicies
+ */
+ @SuppressWarnings("unchecked")
+ @Override
+ public void run() {
+ // we should catch every exception to avoid zombile thread
+ try {
+ List<PolicyDefinition> current = client.listPolicies();
+ Map<String, PolicyDefinition> currPolicies = new HashMap<>();
+ current.forEach(pe -> currPolicies.put(pe.getName(), pe));
+
+ Collection<String> addedPolicies = CollectionUtils.subtract(currPolicies.keySet(), cachedPolicies.keySet());
+ Collection<String> removedPolicies = CollectionUtils.subtract(cachedPolicies.keySet(), currPolicies.keySet());
+ Collection<String> potentiallyModifiedPolicies = CollectionUtils.intersection(currPolicies.keySet(), cachedPolicies.keySet());
+
+ List<String> reallyModifiedPolicies = new ArrayList<>();
+ for (String updatedPolicy : potentiallyModifiedPolicies) {
+ if (!currPolicies.get(updatedPolicy).equals(cachedPolicies.get(updatedPolicy))) {
+ reallyModifiedPolicies.add(updatedPolicy);
+ }
+ }
+
+ boolean policyChanged = false;
+ if (addedPolicies.size() != 0 ||
+ removedPolicies.size() != 0 ||
+ reallyModifiedPolicies.size() != 0) {
+ policyChanged = true;
+ }
+
+ if (!policyChanged) {
+ LOG.info("policy is not changed since last run");
+ return;
+ }
+ synchronized (this) {
+ for (PolicyChangeListener listener : listeners) {
+ listener.onPolicyChange(current, addedPolicies, removedPolicies, reallyModifiedPolicies);
+ }
+ }
+
+ // reset cached policies
+ cachedPolicies = currPolicies;
+ } catch (Throwable t) {
+ LOG.error("error loading policy, but continue to run", t);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/trigger/PolicyChangeListener.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/trigger/PolicyChangeListener.java b/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/trigger/PolicyChangeListener.java
new file mode 100644
index 0000000..8aa322f
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/trigger/PolicyChangeListener.java
@@ -0,0 +1,10 @@
+package org.apache.eagle.alert.coordinator.trigger;
+
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.eagle.alert.engine.coordinator.PolicyDefinition;
+
+public interface PolicyChangeListener {
+ void onPolicyChange(List<PolicyDefinition> allPolicies, Collection<String> addedPolicies, Collection<String> removedPolicies, Collection<String> modifiedPolicies);
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-coordinator/src/main/resources/application.conf
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-coordinator/src/main/resources/application.conf b/eagle-core/eagle-alert/alert/alert-coordinator/src/main/resources/application.conf
new file mode 100644
index 0000000..51cc315
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-coordinator/src/main/resources/application.conf
@@ -0,0 +1,27 @@
+{
+ "coordinator" : {
+ "policiesPerBolt" : 5,
+ "boltParallelism" : 5,
+ "policyDefaultParallelism" : 5,
+ "boltLoadUpbound": 0.8,
+ "topologyLoadUpbound" : 0.8,
+ "numOfAlertBoltsPerTopology" : 5,
+ "zkConfig" : {
+ "zkQuorum" : "localhost:2181",
+ "zkRoot" : "/alert",
+ "zkSessionTimeoutMs" : 10000,
+ "connectionTimeoutMs" : 10000,
+ "zkRetryTimes" : 3,
+ "zkRetryInterval" : 3000
+ },
+ "metadataService" : {
+ "host" : "localhost",
+ "port" : 8080,
+ "context" : "/api"
+ },
+ "metadataDynamicCheck" : {
+ "initDelayMillis" : 1000,
+ "delayMillis" : 30000
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-coordinator/src/main/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-coordinator/src/main/resources/log4j.properties b/eagle-core/eagle-alert/alert/alert-coordinator/src/main/resources/log4j.properties
new file mode 100644
index 0000000..d4bc126
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-coordinator/src/main/resources/log4j.properties
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+log4j.rootLogger=INFO, stdout
+
+# standard output
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %p [%t] %c{2}[%L]: %m%n
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-coordinator/src/main/webapp/WEB-INF/web.xml
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-coordinator/src/main/webapp/WEB-INF/web.xml b/eagle-core/eagle-alert/alert/alert-coordinator/src/main/webapp/WEB-INF/web.xml
new file mode 100644
index 0000000..1aa925e
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-coordinator/src/main/webapp/WEB-INF/web.xml
@@ -0,0 +1,87 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<web-app xmlns="http://java.sun.com/xml/ns/javaee"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://java.sun.com/xml/ns/javaee
+ http://java.sun.com/xml/ns/javaee/web-app_3_0.xsd"
+ version="3.0">
+ <welcome-file-list>
+ <welcome-file>index.html</welcome-file>
+ </welcome-file-list>
+ <servlet>
+ <servlet-name>Jersey Web Application</servlet-name>
+ <servlet-class>com.sun.jersey.spi.container.servlet.ServletContainer</servlet-class>
+ <init-param>
+ <param-name>com.sun.jersey.config.property.packages</param-name>
+ <param-value>io.swagger.jaxrs.json,io.swagger.jaxrs.listing,org.apache.eagle.alert.coordinator.resource,org.codehaus.jackson.jaxrs</param-value>
+ </init-param>
+ <init-param>
+ <param-name>com.sun.jersey.spi.container.ContainerRequestFilters</param-name>
+ <param-value>com.sun.jersey.api.container.filter.GZIPContentEncodingFilter;com.sun.jersey.api.container.filter.PostReplaceFilter</param-value>
+ </init-param>
+ <init-param>
+ <param-name>com.sun.jersey.spi.container.ContainerResponseFilters</param-name>
+ <param-value>com.sun.jersey.api.container.filter.GZIPContentEncodingFilter</param-value>
+ </init-param>
+ <load-on-startup>1</load-on-startup>
+ </servlet>
+ <!-- Servlet for swagger initialization only, no URL mapping. -->
+ <servlet>
+ <servlet-name>swaggerConfig</servlet-name>
+ <servlet-class>io.swagger.jaxrs.config.DefaultJaxrsConfig</servlet-class>
+ <init-param>
+ <param-name>api.version</param-name>
+ <param-value>1.0.0</param-value>
+ </init-param>
+ <init-param>
+ <param-name>swagger.api.basepath</param-name>
+ <param-value>/api</param-value>
+ </init-param>
+ <load-on-startup>2</load-on-startup>
+ </servlet>
+
+ <servlet-mapping>
+ <servlet-name>Jersey Web Application</servlet-name>
+ <url-pattern>/api/*</url-pattern>
+ </servlet-mapping>
+ <filter>
+ <filter-name>CorsFilter</filter-name>
+ <!-- this should be replaced by tomcat ones, see also metadata resource -->
+ <filter-class>org.apache.eagle.alert.resource.SimpleCORSFiler</filter-class>
+ <init-param>
+ <param-name>cors.allowed.origins</param-name>
+ <param-value>*</param-value>
+ </init-param>
+ <init-param>
+ <param-name>cors.allowed.headers</param-name>
+ <param-value>Authorization,Origin, No-Cache, X-Requested-With, If-Modified-Since, Pragma, Last-Modified, Cache-Control, Expires, Content-Type, X-E4M-With, Accept</param-value>
+ </init-param>
+ <init-param>
+ <param-name>cors.allowed.methods</param-name>
+ <param-value>GET,POST,HEAD,OPTIONS,PUT,DELETE</param-value>
+ </init-param>
+ <init-param>
+ <param-name>cors.support.credentials</param-name>
+ <param-value>true</param-value>
+ </init-param>
+ </filter>
+ <filter-mapping>
+ <filter-name>CorsFilter</filter-name>
+ <url-pattern>/*</url-pattern>
+ </filter-mapping>
+</web-app>
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-coordinator/src/main/webapp/index.html
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-coordinator/src/main/webapp/index.html b/eagle-core/eagle-alert/alert/alert-coordinator/src/main/webapp/index.html
new file mode 100644
index 0000000..1c4ea76
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-coordinator/src/main/webapp/index.html
@@ -0,0 +1,18 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+UMP Coordinator service!
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-coordinator/src/test/java/org/apache/alert/coordinator/CoordinatorTest.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-coordinator/src/test/java/org/apache/alert/coordinator/CoordinatorTest.java b/eagle-core/eagle-alert/alert/alert-coordinator/src/test/java/org/apache/alert/coordinator/CoordinatorTest.java
new file mode 100644
index 0000000..78b72b3
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-coordinator/src/test/java/org/apache/alert/coordinator/CoordinatorTest.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.alert.coordinator;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.eagle.alert.config.ConfigBusConsumer;
+import org.apache.eagle.alert.config.ConfigBusProducer;
+import org.apache.eagle.alert.config.ConfigChangeCallback;
+import org.apache.eagle.alert.config.ConfigValue;
+import org.apache.eagle.alert.config.ZKConfig;
+import org.apache.eagle.alert.config.ZKConfigBuilder;
+import org.apache.eagle.alert.coordination.model.ScheduleState;
+import org.apache.eagle.alert.coordinator.Coordinator;
+import org.apache.eagle.alert.coordinator.ScheduleOption;
+import org.apache.eagle.alert.service.IMetadataServiceClient;
+import org.apache.eagle.alert.service.MetadataServiceClientImpl;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+
+/**
+ * @since May 5, 2016
+ *
+ */
+public class CoordinatorTest {
+
+ @SuppressWarnings({ "resource", "unused" })
+ @Ignore
+ @Test
+ public void test() throws Exception {
+ before();
+ Config config = ConfigFactory.load().getConfig("coordinator");
+ ZKConfig zkConfig = ZKConfigBuilder.getZKConfig(config);
+ ConfigBusProducer producer = new ConfigBusProducer(zkConfig);
+ IMetadataServiceClient client = new MetadataServiceClientImpl(config);
+
+ Coordinator coordinator = new Coordinator(config, producer, client);
+ ScheduleOption option = new ScheduleOption();
+ ScheduleState state = coordinator.schedule(option);
+ String v = state.getVersion();
+
+ AtomicBoolean validated = new AtomicBoolean(false);
+ ConfigBusConsumer consumer = new ConfigBusConsumer(zkConfig, "topo1/spout", new ConfigChangeCallback() {
+ @Override
+ public void onNewConfig(ConfigValue value) {
+ String vId = value.getValue().toString();
+ Assert.assertEquals(v, vId);
+ validated.set(true);
+ }
+ });
+
+ Thread.sleep(1000);
+ Assert.assertTrue(validated.get());
+ }
+
+ @SuppressWarnings({ "resource", "unused" })
+ @Test
+ public void test_01() throws Exception {
+ before();
+ Config config = ConfigFactory.load().getConfig("coordinator");
+ ZKConfig zkConfig = ZKConfigBuilder.getZKConfig(config);
+ ConfigBusProducer producer = new ConfigBusProducer(zkConfig);
+ IMetadataServiceClient client = ScheduleContextBuilderTest.getSampleMetadataService();
+
+ Coordinator coordinator = new Coordinator(config, producer, client);
+ ScheduleOption option = new ScheduleOption();
+ ScheduleState state = coordinator.schedule(option);
+ String v = state.getVersion();
+
+ // TODO : assert version
+
+ CountDownLatch latch = new CountDownLatch(1);
+ AtomicBoolean validated = new AtomicBoolean(false);
+ ConfigBusConsumer consumer = new ConfigBusConsumer(zkConfig, "topo1/spout", new ConfigChangeCallback() {
+ @Override
+ public void onNewConfig(ConfigValue value) {
+ String vId = value.getValue().toString();
+ Assert.assertEquals(v, vId);
+ validated.set(true);
+ latch.countDown();
+ }
+ });
+
+ latch.await(3, TimeUnit.SECONDS);
+ Assert.assertTrue(validated.get());
+ }
+
+ @Ignore
+ @Test
+ public void test_main() throws Exception {
+ before();
+
+ Coordinator.main(null);
+ }
+
+ @Before
+ public void before() {
+ System.setProperty("config.resource", "/test-application.conf");
+ ConfigFactory.invalidateCaches();
+ ConfigFactory.load().getConfig("coordinator");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-coordinator/src/test/java/org/apache/alert/coordinator/DynamicPolicyLoaderTest.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-coordinator/src/test/java/org/apache/alert/coordinator/DynamicPolicyLoaderTest.java b/eagle-core/eagle-alert/alert/alert-coordinator/src/test/java/org/apache/alert/coordinator/DynamicPolicyLoaderTest.java
new file mode 100644
index 0000000..155a9e5
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-coordinator/src/test/java/org/apache/alert/coordinator/DynamicPolicyLoaderTest.java
@@ -0,0 +1,26 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+
+package org.apache.alert.coordinator;
+
+/**
+ * Since 4/28/16.
+ */
+public class DynamicPolicyLoaderTest {
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-coordinator/src/test/java/org/apache/alert/coordinator/MetadataServiceClientImplTest.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-coordinator/src/test/java/org/apache/alert/coordinator/MetadataServiceClientImplTest.java b/eagle-core/eagle-alert/alert/alert-coordinator/src/test/java/org/apache/alert/coordinator/MetadataServiceClientImplTest.java
new file mode 100644
index 0000000..e2ea031
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-coordinator/src/test/java/org/apache/alert/coordinator/MetadataServiceClientImplTest.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.alert.coordinator;
+
+import org.apache.eagle.alert.config.ConfigBusProducer;
+import org.apache.eagle.alert.config.ZKConfig;
+import org.apache.eagle.alert.config.ZKConfigBuilder;
+import org.apache.eagle.alert.coordination.model.ScheduleState;
+import org.apache.eagle.alert.coordinator.Coordinator;
+import org.apache.eagle.alert.service.MetadataServiceClientImpl;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+
+/**
+ * @since May 9, 2016
+ *
+ */
+public class MetadataServiceClientImplTest {
+
+ @Ignore
+ @Test
+ public void addScheduleState() throws Exception {
+ ConfigFactory.invalidateCaches();
+ System.setProperty("config.resource", "/test-application.conf");
+ Config config = ConfigFactory.load("test-application.conf").getConfig("coordinator");
+ MetadataServiceClientImpl client = new MetadataServiceClientImpl(config);
+
+ ScheduleState ss = new ScheduleState();
+ ss.setVersion("spec_version_1463764252582");
+
+ client.addScheduleState(ss);
+
+ client.close();
+
+ ss.setVersion("spec_version_1464764252582");
+ ZKConfig zkConfig = ZKConfigBuilder.getZKConfig(config);
+ ConfigBusProducer producer = new ConfigBusProducer(zkConfig);
+ Coordinator.postSchedule(client, ss, producer);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-coordinator/src/test/java/org/apache/alert/coordinator/ScheduleContextBuilderTest.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-coordinator/src/test/java/org/apache/alert/coordinator/ScheduleContextBuilderTest.java b/eagle-core/eagle-alert/alert/alert-coordinator/src/test/java/org/apache/alert/coordinator/ScheduleContextBuilderTest.java
new file mode 100644
index 0000000..f2e67de
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-coordinator/src/test/java/org/apache/alert/coordinator/ScheduleContextBuilderTest.java
@@ -0,0 +1,281 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.alert.coordinator;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.alert.coordinator.mock.InMemMetadataServiceClient;
+import org.apache.eagle.alert.coordination.model.Kafka2TupleMetadata;
+import org.apache.eagle.alert.coordination.model.ScheduleState;
+import org.apache.eagle.alert.coordination.model.Tuple2StreamMetadata;
+import org.apache.eagle.alert.coordination.model.WorkSlot;
+import org.apache.eagle.alert.coordination.model.internal.MonitoredStream;
+import org.apache.eagle.alert.coordination.model.internal.PolicyAssignment;
+import org.apache.eagle.alert.coordination.model.internal.StreamGroup;
+import org.apache.eagle.alert.coordination.model.internal.StreamWorkSlotQueue;
+import org.apache.eagle.alert.coordination.model.internal.Topology;
+import org.apache.eagle.alert.coordinator.IScheduleContext;
+import org.apache.eagle.alert.coordinator.model.AlertBoltUsage;
+import org.apache.eagle.alert.coordinator.model.TopologyUsage;
+import org.apache.eagle.alert.coordinator.provider.ScheduleContextBuilder;
+import org.apache.eagle.alert.engine.coordinator.PolicyDefinition;
+import org.apache.eagle.alert.engine.coordinator.Publishment;
+import org.apache.eagle.alert.engine.coordinator.StreamColumn;
+import org.apache.eagle.alert.engine.coordinator.StreamColumn.Type;
+import org.apache.eagle.alert.engine.coordinator.StreamDefinition;
+import org.apache.eagle.alert.engine.coordinator.StreamPartition;
+import org.apache.eagle.alert.engine.coordinator.StreamSortSpec;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * @since May 5, 2016
+ *
+ */
+public class ScheduleContextBuilderTest {
+
+ @Test
+ public void test() {
+ InMemMetadataServiceClient client = getSampleMetadataService();
+
+ ScheduleContextBuilder builder = new ScheduleContextBuilder(client);
+
+ IScheduleContext context = builder.buildContext();
+
+ // assert topology usage
+ Map<String, TopologyUsage> usages = context.getTopologyUsages();
+ Assert.assertEquals(1, usages.get(TOPO1).getMonitoredStream().size());
+ Assert.assertTrue(usages.get(TOPO1).getPolicies().contains(TEST_POLICY_1));
+
+ String alertBolt0 = TOPO1 + "-alert-" + "0";
+ String alertBolt1 = TOPO1 + "-alert-" + "1";
+ String alertBolt2 = TOPO1 + "-alert-" + "2";
+ for (AlertBoltUsage u : usages.get(TOPO1).getAlertUsages().values()) {
+ if (u.getBoltId().equals(alertBolt0) || u.getBoltId().equals(alertBolt1)
+ || u.getBoltId().equals(alertBolt2)) {
+ Assert.assertEquals(1, u.getPolicies().size());
+ Assert.assertTrue(u.getPolicies().contains(TEST_POLICY_1));
+ Assert.assertEquals(1, u.getPartitions().size());
+ Assert.assertEquals(1, u.getReferQueues().size());
+ }
+ }
+ }
+
+ @Test
+ public void test_remove_policy() {
+ InMemMetadataServiceClient client = getSampleMetadataService();
+ ScheduleContextBuilder builder = new ScheduleContextBuilder(client);
+
+ PolicyAssignment assignment1 = client.getVersionedSpec().getAssignments().get(0);
+
+ IScheduleContext context = builder.buildContext();
+ Assert.assertTrue(context.getPolicyAssignments().containsKey(TEST_POLICY_1));
+ StreamWorkSlotQueue queue = SchedulerTest.getQueue(context, assignment1.getQueueId()).getRight();
+
+ client.listPolicies().remove(0);
+ context = builder.buildContext();
+ Assert.assertFalse(context.getPolicyAssignments().containsKey(TEST_POLICY_1));
+
+ WorkSlot slot = queue.getWorkingSlots().get(0);
+ Set<String> topoPolicies = context.getTopologyUsages().get(slot.topologyName).getPolicies();
+ Assert.assertFalse(topoPolicies.contains(TEST_DATASOURCE_1));
+ Assert.assertEquals(0, topoPolicies.size());
+ }
+
+ @Test
+ public void test_changed_policy() {
+ InMemMetadataServiceClient client = getSampleMetadataService();
+ ScheduleContextBuilder builder = new ScheduleContextBuilder(client);
+ PolicyAssignment assignment1 = client.getVersionedSpec().getAssignments().get(0);
+
+ IScheduleContext context = builder.buildContext();
+ Assert.assertTrue(context.getPolicyAssignments().containsKey(TEST_POLICY_1));
+
+ StreamWorkSlotQueue queue = SchedulerTest.getQueue(context, assignment1.getQueueId()).getRight();
+
+ PolicyDefinition pd1 = client.listPolicies().get(0);
+ // add a new group by column : need to replace the partiton spec, to
+ // avoid reference same object in
+ // on jvm (no serialization and deserialization)
+ StreamPartition par = new StreamPartition(pd1.getPartitionSpec().get(0));
+ par.getColumns().add("s1");
+ pd1.getPartitionSpec().clear();
+ pd1.getPartitionSpec().add(par);
+
+ context = builder.buildContext();
+
+ // assert the policy assignment is removed
+ Assert.assertFalse(context.getPolicyAssignments().containsKey(TEST_POLICY_1));
+ // assert the monitored stream is removed as no policy on it now.
+ Assert.assertEquals(0, context.getMonitoredStreams().size());
+ // assert the topology usage doesn't contain policy
+ WorkSlot slot = queue.getWorkingSlots().get(0);
+ TopologyUsage topologyUsage = context.getTopologyUsages().get(slot.topologyName);
+ Set<String> topoPolicies = topologyUsage.getPolicies();
+ Assert.assertFalse(topoPolicies.contains(TEST_DATASOURCE_1));
+ Assert.assertEquals(0, topoPolicies.size());
+ // assert the topology usage doesn't contain the monitored stream
+ Assert.assertEquals(0, topologyUsage.getMonitoredStream().size());
+ // assert the alert bolt usage doesn't have the queue reference
+ Assert.assertEquals(0, topologyUsage.getAlertBoltUsage(slot.getBoltId()).getReferQueues().size());
+ }
+
+ @Test
+ public void test_renamed_topologies() {
+ InMemMetadataServiceClient client = getSampleMetadataService();
+ ScheduleContextBuilder builder = new ScheduleContextBuilder(client);
+
+ IScheduleContext context = builder.buildContext();
+ Assert.assertTrue(context.getPolicyAssignments().containsKey(TEST_POLICY_1));
+
+ Topology t = client.listTopologies().get(0);
+ t.setName("newName");
+
+ context = builder.buildContext();
+ Assert.assertFalse(context.getPolicyAssignments().containsKey(TEST_POLICY_1));
+ }
+
+ private static final String TOPO1 = "topo1";
+ private static final String V1 = "v1";
+ private static final String COL1 = "col1";
+ private static final String OUT_STREAM1 = "out-stream1";
+ private static final String TEST_POLICY_1 = "test-policy-1";
+ private static final String TEST_STREAM_DEF_1 = "testStreamDef";
+ private static final String TEST_DATASOURCE_1 = "test-datasource-1";
+ private static StreamPartition par;
+ private static String queueId;
+ private static StreamGroup streamGroup;
+
+ public static InMemMetadataServiceClient getSampleMetadataService() {
+ InMemMetadataServiceClient client = new InMemMetadataServiceClient();
+ client.listTopologies().add(createSampleTopology());
+ client.listDataSources().add(createKafka2TupleMetadata());
+ // client.listSpoutMetadata().add(createS)
+ client.listPolicies().add(createPolicy());
+ client.listPublishment().add(createPublishment());
+ client.listStreams().add(createStreamDefinition());
+ client.addScheduleState(createScheduleState());
+ return client;
+ }
+
+ private static ScheduleState createScheduleState() {
+ ScheduleState ss = new ScheduleState();
+ ss.setVersion(V1);
+
+ ss.getMonitoredStreams().add(createMonitoredStream());
+ ss.getAssignments().add(createAssignment());
+
+ return ss;
+ }
+
+ private static MonitoredStream createMonitoredStream() {
+ MonitoredStream ms = new MonitoredStream(streamGroup);
+ ms.setVersion(V1);
+
+ List<WorkSlot> slots = new ArrayList<WorkSlot>();
+ WorkSlot slot0 = new WorkSlot(TOPO1, TOPO1 + "-alert-" + 0);
+ WorkSlot slot1 = new WorkSlot(TOPO1, TOPO1 + "-alert-" + 1);
+ WorkSlot slot2 = new WorkSlot(TOPO1, TOPO1 + "-alert-" + 2);
+ slots.add(slot0);
+ slots.add(slot1);
+ slots.add(slot2);
+
+ StreamWorkSlotQueue q = new StreamWorkSlotQueue(streamGroup, false, new HashMap<>(), slots);
+ ms.addQueues(q);
+ queueId = q.getQueueId();
+ return ms;
+ }
+
+ private static PolicyAssignment createAssignment() {
+ PolicyAssignment pa = new PolicyAssignment(TEST_POLICY_1, queueId);
+ return pa;
+ }
+
+ private static PolicyDefinition createPolicy() {
+ PolicyDefinition def = new PolicyDefinition();
+ def.setName(TEST_POLICY_1);
+ def.setInputStreams(Arrays.asList(TEST_STREAM_DEF_1));
+ def.setOutputStreams(Arrays.asList(OUT_STREAM1));
+ def.setParallelismHint(5);
+
+ streamGroup = new StreamGroup();
+ par = new StreamPartition();
+ par.setStreamId(TEST_STREAM_DEF_1);
+ par.getColumns().add(COL1);
+ StreamSortSpec sortSpec = new StreamSortSpec();
+// sortSpec.setColumn("col1");
+ sortSpec.setWindowMargin(3);
+ sortSpec.setWindowPeriod("PT1M");
+
+ par.setSortSpec(sortSpec);
+ streamGroup.addStreamPartition(par);
+
+ List<StreamPartition> lists = new ArrayList<StreamPartition>();
+ lists.add(par);
+ def.setPartitionSpec(lists);
+ return def;
+ }
+
+ private static StreamDefinition createStreamDefinition() {
+ StreamDefinition def = new StreamDefinition();
+ def.setStreamId(TEST_STREAM_DEF_1);
+ def.setDataSource(TEST_DATASOURCE_1);
+
+ StreamColumn col = new StreamColumn();
+ col.setName(COL1);
+ col.setRequired(true);
+ col.setType(Type.STRING);
+ def.getColumns().add(col);
+
+ return def;
+ }
+
+ private static Publishment createPublishment() {
+ Publishment pub = new Publishment();
+ pub.setType("KAFKA");
+ pub.setName("test-stream-output");
+ pub.setPolicyIds(Arrays.asList(TEST_POLICY_1));
+ return pub;
+ }
+
+ private static Kafka2TupleMetadata createKafka2TupleMetadata() {
+ Kafka2TupleMetadata ktm = new Kafka2TupleMetadata();
+ ktm.setName(TEST_DATASOURCE_1);
+ ktm.setSchemeCls("SchemeClass");
+ ktm.setTopic("tupleTopic");
+ ktm.setType("KAFKA");
+ ktm.setCodec(new Tuple2StreamMetadata());
+ return ktm;
+ }
+
+ private static Topology createSampleTopology() {
+ Topology t = new Topology(TOPO1, 3, 10);
+ for (int i = 0; i < t.getNumOfGroupBolt(); i++) {
+ t.getGroupNodeIds().add(t.getName() + "-grp-" + i);
+ }
+ for (int i = 0; i < t.getNumOfAlertBolt(); i++) {
+ t.getAlertBoltIds().add(t.getName() + "-alert-" + i);
+ }
+ return t;
+ }
+
+}
[04/19] incubator-eagle git commit: EAGLE-324: Init branch-v0.5
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/topology/TestStormParallelism.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/topology/TestStormParallelism.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/topology/TestStormParallelism.java
new file mode 100644
index 0000000..279a041
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/topology/TestStormParallelism.java
@@ -0,0 +1,161 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.alert.engine.topology;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.junit.Ignore;
+import org.junit.Test;
+
+import backtype.storm.Config;
+import backtype.storm.LocalCluster;
+import backtype.storm.spout.SpoutOutputCollector;
+import backtype.storm.task.OutputCollector;
+import backtype.storm.task.TopologyContext;
+import backtype.storm.topology.OutputFieldsDeclarer;
+import backtype.storm.topology.TopologyBuilder;
+import backtype.storm.topology.base.BaseRichBolt;
+import backtype.storm.topology.base.BaseRichSpout;
+import backtype.storm.tuple.Fields;
+import backtype.storm.tuple.Tuple;
+
+/**
+ * Since 4/29/16.
+ */
+@SuppressWarnings({"serial", "rawtypes"})
+public class TestStormParallelism {
+ /**
+ * When run this test, please check the following through jstack and log
+ * 1) for blue-spout, num of executors is 2, # of tasks is 2
+ *
+ * Expected:
+ *
+ * a. 2 threads uniquely named Thread-*-blue-spout-executor[*,*]
+ * b. each thread will have single task
+ *
+ * 2) for green-bolt, num of executors is 2, # of tasks is 4
+ *
+ * Expected:
+ *
+ * a. 2 threads uniquely named Thread-*-green-bolt-executor[*,*]
+ * b. each thread will have 2 tasks
+ *
+ * 3) for yellow-bolt, num of executors is 6, # of tasks is 6
+ *
+ * Expected:
+ *
+ * a. 6 threads uniquely named Thread-*-yellow-bolt-executor[*,*]
+ * b. each thread will have 1 tasks
+ *
+ *
+ * Continue to think:
+ *
+ * For alter engine, if we use multiple tasks per component instead of one task per component,
+ * what will the parallelism mechanism affect?
+ *
+ * @throws Exception
+ */
+ @Ignore
+ @Test
+ public void testParallelism() throws Exception{
+ Config conf = new Config();
+ conf.setNumWorkers(2); // use two worker processes
+ TopologyBuilder topologyBuilder = new TopologyBuilder();
+ topologyBuilder.setSpout("blue-spout", new BlueSpout(), 2); // parallelism hint
+
+ topologyBuilder.setBolt("green-bolt", new GreenBolt(), 2)
+ .setNumTasks(4)
+ .shuffleGrouping("blue-spout");
+
+ topologyBuilder.setBolt("yellow-bolt", new YellowBolt(), 6)
+ .shuffleGrouping("green-bolt");
+
+ LocalCluster cluster = new LocalCluster();
+ cluster.submitTopology("mytopology", new HashMap(), topologyBuilder.createTopology());
+
+ while(true) {
+ try {
+ Thread.sleep(1000);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ }
+
+ private static class BlueSpout extends BaseRichSpout{
+ static int count = 0;
+ public BlueSpout(){
+ }
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+ declarer.declare(new Fields("a"));
+ }
+
+ @Override
+ public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
+ count++;
+ System.out.println("# of spout objects " + count + ", current spout " + this);
+ }
+
+ @Override
+ public void nextTuple() {
+
+ }
+ }
+
+ private static class GreenBolt extends BaseRichBolt{
+ static int count;
+ @Override
+ public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
+ count++;
+ System.out.println("# of green bolt objects " + count + ", current green bolt " + this);
+ }
+
+ @Override
+ public void execute(Tuple input) {
+
+ }
+
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+ declarer.declare(new Fields("a"));
+ }
+ }
+
+ private static class YellowBolt extends BaseRichBolt{
+ static int count;
+ @Override
+ public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
+ count++;
+ System.out.println("# of yellow bolt objects " + count + ", current yellow bolt " + this);
+ }
+
+ @Override
+ public void execute(Tuple input) {
+
+ }
+
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+ declarer.declare(new Fields("a"));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/topology/TestStormStreamIdRouting.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/topology/TestStormStreamIdRouting.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/topology/TestStormStreamIdRouting.java
new file mode 100644
index 0000000..9d3892a
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/topology/TestStormStreamIdRouting.java
@@ -0,0 +1,140 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.alert.engine.topology;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.junit.Ignore;
+import org.junit.Test;
+
+import backtype.storm.Config;
+import backtype.storm.LocalCluster;
+import backtype.storm.spout.SpoutOutputCollector;
+import backtype.storm.task.OutputCollector;
+import backtype.storm.task.TopologyContext;
+import backtype.storm.topology.OutputFieldsDeclarer;
+import backtype.storm.topology.TopologyBuilder;
+import backtype.storm.topology.base.BaseRichBolt;
+import backtype.storm.topology.base.BaseRichSpout;
+import backtype.storm.tuple.Fields;
+import backtype.storm.tuple.Tuple;
+
+
+/**
+ * Since 4/29/16.
+ */
+@SuppressWarnings({"serial", "rawtypes", "unused"})
+public class TestStormStreamIdRouting {
+ @Ignore
+ @Test
+ public void testRoutingByStreamId() throws Exception{
+ Config conf = new Config();
+ conf.setNumWorkers(2); // use two worker processes
+ TopologyBuilder topologyBuilder = new TopologyBuilder();
+ topologyBuilder.setSpout("blue-spout", new BlueSpout()); // parallelism hint
+
+ topologyBuilder.setBolt("green-bolt-1", new GreenBolt(1))
+ .shuffleGrouping("blue-spout", "green-bolt-stream-1");
+ topologyBuilder.setBolt("green-bolt-2", new GreenBolt(2))
+ .shuffleGrouping("blue-spout", "green-bolt-stream-2");
+
+ LocalCluster cluster = new LocalCluster();
+ cluster.submitTopology("mytopology", new HashMap(), topologyBuilder.createTopology());
+
+ while(true) {
+ try {
+ Thread.sleep(1000);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ }
+
+ private static class BlueSpout extends BaseRichSpout {
+ int count = 0;
+ private SpoutOutputCollector collector;
+ public BlueSpout(){
+ }
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+ declarer.declareStream("green-bolt-stream-1", new Fields("a"));
+ declarer.declareStream("green-bolt-stream-2", new Fields("a"));
+ }
+
+ @Override
+ public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
+ this.collector = collector;
+ }
+
+ @Override
+ public void nextTuple() {
+ if(count % 2 == 0) {
+ this.collector.emit("green-bolt-stream-1", Arrays.asList("testdata" + count));
+ count++;
+ }else{
+ this.collector.emit("green-bolt-stream-2", Arrays.asList("testdata" + count));
+ count++;
+ }
+ try{
+ Thread.sleep(10000);
+ }catch(Exception ex){
+
+ }
+ }
+ }
+
+ private static class GreenBolt extends BaseRichBolt {
+ private int id;
+ public GreenBolt(int id){
+ this.id = id;
+ }
+ @Override
+ public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
+ }
+
+ @Override
+ public void execute(Tuple input) {
+ System.out.println("bolt " + id + " received data " + input.getString(0));
+ }
+
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+ declarer.declare(new Fields("a"));
+ }
+ }
+
+ private static class YellowBolt extends BaseRichBolt{
+ @Override
+ public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
+ }
+
+ @Override
+ public void execute(Tuple input) {
+
+ }
+
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+ declarer.declare(new Fields("a"));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/topology/TestTuple2StreamConverter.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/topology/TestTuple2StreamConverter.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/topology/TestTuple2StreamConverter.java
new file mode 100644
index 0000000..b0e5c4a
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/topology/TestTuple2StreamConverter.java
@@ -0,0 +1,62 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.alert.engine.topology;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+
+import org.apache.eagle.alert.coordination.model.Tuple2StreamConverter;
+import org.apache.eagle.alert.coordination.model.Tuple2StreamMetadata;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Since 5/3/16.
+ */
+public class TestTuple2StreamConverter {
+ @SuppressWarnings({ "unchecked", "rawtypes" })
+ @Test
+ public void test(){
+ Tuple2StreamMetadata metadata = new Tuple2StreamMetadata();
+ Set activeStreamNames = new HashSet<>();
+ activeStreamNames.add("defaultStringStream");
+ metadata.setStreamNameSelectorCls("org.apache.eagle.alert.engine.scheme.PlainStringStreamNameSelector");
+ metadata.setStreamNameSelectorProp(new Properties());
+ metadata.getStreamNameSelectorProp().put("userProvidedStreamName", "defaultStringStream");
+ metadata.setActiveStreamNames(activeStreamNames);
+ metadata.setTimestampColumn("timestamp");
+ Tuple2StreamConverter convert = new Tuple2StreamConverter(metadata);
+ String topic = "testTopic";
+ Map m = new HashMap<>();
+ m.put("value", "IAmPlainString");
+ long t = System.currentTimeMillis();
+ m.put("timestamp", t);
+ List<Object> ret = convert.convert(Arrays.asList(topic, m));
+ Assert.assertEquals(topic, ret.get(0));
+ Assert.assertEquals("defaultStringStream", ret.get(1));
+ Assert.assertEquals(t, ret.get(2));
+ Assert.assertEquals(m, ret.get(3));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/topology/TestUnitTopologyMain.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/topology/TestUnitTopologyMain.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/topology/TestUnitTopologyMain.java
new file mode 100644
index 0000000..7132f5a
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/topology/TestUnitTopologyMain.java
@@ -0,0 +1,57 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.alert.engine.topology;
+
+import org.apache.eagle.alert.engine.runner.UnitTopologyRunner;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+
+/**
+ * Since 5/4/16.
+ */
+public class TestUnitTopologyMain {
+ @Ignore
+ @Test
+ public void testTopologyRun(){
+ testTopologyRun("/application-test.conf");
+ }
+
+ public void testTopologyRun(String configResourceName){
+ ConfigFactory.invalidateCaches();
+ System.setProperty("config.resource", configResourceName);
+ System.out.print("Set config.resource = "+configResourceName);
+ Config config = ConfigFactory.load();
+ String topologyId = config.getString("topology.name");
+ MockMetadataChangeNotifyService changeNotifyService =
+ new MockMetadataChangeNotifyService(topologyId,"alertEngineSpout");
+ new UnitTopologyRunner(changeNotifyService).run(topologyId,config);
+ }
+
+ public static void main(String[] args){
+ if(args.length>0) {
+ new TestUnitTopologyMain().testTopologyRun(args[0]);
+ } else {
+ new TestUnitTopologyMain().testTopologyRun();
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/utils/CompressionUtilsTest.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/utils/CompressionUtilsTest.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/utils/CompressionUtilsTest.java
new file mode 100644
index 0000000..0861597
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/utils/CompressionUtilsTest.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.utils;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+public class CompressionUtilsTest {
+ private final static Logger LOG = LoggerFactory.getLogger(CompressionUtilsTest.class);
+
+ @Test
+ public void testCompressAndDecompress() throws IOException {
+ String value = "http://www.apache.org/licenses/LICENSE-2.0";
+ byte[] original = value.getBytes();
+ byte[] compressed = CompressionUtils.compress(original);
+ byte[] decompressed = CompressionUtils.decompress(compressed);
+
+ LOG.info("original size: {}",original.length);
+ LOG.info("compressed size: {}",compressed.length);
+ LOG.info("decompressed size: {}",decompressed.length);
+
+ String decompressedValue = new String(decompressed);
+ Assert.assertEquals(value,decompressedValue);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/utils/TimePeriodUtilsTest.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/utils/TimePeriodUtilsTest.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/utils/TimePeriodUtilsTest.java
new file mode 100644
index 0000000..cad53d4
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/utils/TimePeriodUtilsTest.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.utils;
+
+import java.text.ParseException;
+
+import org.apache.eagle.alert.utils.DateTimeUtil;
+import org.apache.eagle.alert.utils.TimePeriodUtils;
+import org.joda.time.Period;
+import org.joda.time.Seconds;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TimePeriodUtilsTest {
+ @Test
+ public void testJodaTimePeriod() throws ParseException {
+ String periodText = "PT10m";
+ Period period = new Period(periodText);
+ int seconds = period.toStandardSeconds().getSeconds();
+ Assert.assertEquals(600, seconds);
+ Assert.assertEquals(60, period.toStandardSeconds().dividedBy(10).getSeconds());
+ }
+
+ @Test
+ public void testJodaTimePeriodBySeconds() throws ParseException {
+ String periodText = "PT10s";
+ Period period = new Period(periodText);
+ int seconds = period.toStandardSeconds().getSeconds();
+ Assert.assertEquals(10, seconds);
+ }
+
+ @Test
+ public void testFormatSecondsByPeriod15M() throws ParseException {
+
+ Period period = new Period("PT15m");
+ Seconds seconds = period.toStandardSeconds();
+ Assert.assertEquals(15*60,seconds.getSeconds());
+
+ long time = DateTimeUtil.humanDateToSeconds("2015-07-01 13:56:12");
+ long expect = DateTimeUtil.humanDateToSeconds("2015-07-01 13:45:00");
+ long result = TimePeriodUtils.formatSecondsByPeriod(time,seconds);
+ Assert.assertEquals(expect,result);
+
+ time = DateTimeUtil.humanDateToSeconds("2015-07-01 03:14:59");
+ expect = DateTimeUtil.humanDateToSeconds("2015-07-01 03:00:00");
+ result = TimePeriodUtils.formatSecondsByPeriod(time, seconds);
+ Assert.assertEquals(expect,result);
+
+ time = DateTimeUtil.humanDateToSeconds("2015-07-01 03:14:59");
+ expect = DateTimeUtil.humanDateToSeconds("2015-07-01 03:00:00");
+ result = TimePeriodUtils.formatSecondsByPeriod(time, seconds);
+ Assert.assertEquals(expect,result);
+ }
+
+ @Test
+ public void testFormatSecondsByPeriod1H() throws ParseException {
+
+ Period period = new Period("PT1h");
+ Seconds seconds = period.toStandardSeconds();
+ Assert.assertEquals(60*60,seconds.getSeconds());
+
+ long time = DateTimeUtil.humanDateToSeconds("2015-07-01 13:56:12");
+ long expect = DateTimeUtil.humanDateToSeconds("2015-07-01 13:00:00");
+ long result = TimePeriodUtils.formatSecondsByPeriod(time,seconds);
+ Assert.assertEquals(expect,result);
+
+ time = DateTimeUtil.humanDateToSeconds("2015-07-01 03:14:59");
+ expect = DateTimeUtil.humanDateToSeconds("2015-07-01 03:00:00");
+ result = TimePeriodUtils.formatSecondsByPeriod(time, seconds);
+ Assert.assertEquals(expect,result);
+
+ time = DateTimeUtil.humanDateToSeconds("2015-07-01 03:30:59");
+ expect = DateTimeUtil.humanDateToSeconds("2015-07-01 03:00:00");
+ result = TimePeriodUtils.formatSecondsByPeriod(time, seconds);
+ Assert.assertEquals(expect,result);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/application-integration.conf
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/application-integration.conf b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/application-integration.conf
new file mode 100644
index 0000000..13d4ee1
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/application-integration.conf
@@ -0,0 +1,57 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+{
+ "topology" : {
+ "name" : "alertUnitTopology_1",
+ "numOfTotalWorkers": 20,
+ "numOfSpoutTasks" : 1,
+ "numOfRouterBolts" : 4,
+ "numOfAlertBolts" : 10,
+ "numOfPublishTasks" : 1,
+ "localMode" : "true"
+ },
+ "spout" : {
+ "kafkaBrokerZkQuorum": "localhost:2181",
+ "kafkaBrokerZkBasePath": "/brokers",
+ "stormKafkaUseSameZkQuorumWithKafkaBroker": true,
+ "stormKafkaTransactionZkQuorum": "",
+ "stormKafkaTransactionZkPath": "/consumers",
+ "stormKafkaEagleConsumer": "eagle_consumer",
+ "stormKafkaStateUpdateIntervalMs": 2000,
+ "stormKafkaFetchSizeBytes": 1048586,
+ },
+ "zkConfig" : {
+ "zkQuorum" : "localhost:2181",
+ "zkRoot" : "/alert",
+ "zkSessionTimeoutMs" : 10000,
+ "connectionTimeoutMs" : 10000,
+ "zkRetryTimes" : 3,
+ "zkRetryInterval" : 3000
+ },
+ "dynamicConfigSource" : {
+ "initDelayMillis": 3000,
+ "delayMillis" : 10000
+ },
+ "metadataService": {
+ "context" : "/api",
+ "host" : "localhost",
+ "port" : 8080
+ },
+ "coordinatorService": {
+ "host": "localhost",
+ "port": "9090",
+ "context" : "/api"
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/application-test-backup.conf
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/application-test-backup.conf b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/application-test-backup.conf
new file mode 100755
index 0000000..2c99a68
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/application-test-backup.conf
@@ -0,0 +1,71 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+{
+ "topology" : {
+ "name" : "alertUnitTopology_bug",
+ "numOfSpoutTasks" : 3,
+ "numOfRouterBolts" : 6,
+ "numOfAlertBolts" : 6,
+ "numOfPublishTasks" : 1,
+ "numOfTotalWorkers":1,
+ "messageTimeoutSecs": 30, // topology.message.timeout.secs: 30 by default
+ "localMode" : true
+ },
+ "spout" : {
+ "kafkaBrokerZkQuorum": "10.64.243.71:2181",
+ "kafkaBrokerZkBasePath": "/brokers",
+ "stormKafkaUseSameZkQuorumWithKafkaBroker": true,
+ "stormKafkaTransactionZkQuorum": "",
+ "stormKafkaTransactionZkPath": "/consumers",
+ "stormKafkaEagleConsumer": "eagle_consumer",
+ "stormKafkaStateUpdateIntervalMs": 2000,
+ "stormKafkaFetchSizeBytes": 1048586,
+ },
+ "zkConfig" : {
+ "zkQuorum" : "10.64.243.71:2181",
+ "zkRoot" : "/alert",
+ "zkSessionTimeoutMs" : 10000,
+ "connectionTimeoutMs" : 10000,
+ "zkRetryTimes" : 3,
+ "zkRetryInterval" : 3000
+ },
+ "dynamicConfigSource" : {
+ "initDelayMillis": 3000,
+ "delayMillis" : 10000
+ },
+ "metadataService": {
+ "context" : "/api",
+ "host" : "localhost",
+ "port" : 8080
+ },
+ "metric":{
+ "tags":{
+ "topologyName":"alertUnitTopology_1"
+ }
+ "sink": {
+// "kafka": {
+// "topic": "alert_metric_test"
+// "bootstrap.servers": "localhost:9092"
+// }
+ "logger": {
+ "level":"DEBUG"
+ }
+ "elasticsearch": {
+ "hosts": ["10.64.223.222:9200"]
+ "index": "alert_metric_test"
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/application-test.conf
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/application-test.conf b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/application-test.conf
new file mode 100755
index 0000000..f4a797e
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/application-test.conf
@@ -0,0 +1,68 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+{
+ "topology" : {
+ "name" : "alertUnitTopology_1_test",
+ "numOfSpoutTasks" : 3,
+ "numOfRouterBolts" : 6,
+ "numOfAlertBolts" : 6,
+ "numOfPublishTasks" : 1,
+ "numOfTotalWorkers":1,
+ "messageTimeoutSecs": 30, // topology.message.timeout.secs: 30 by default
+ "localMode" : true
+ },
+ "spout" : {
+ "kafkaBrokerZkQuorum": "localhost:2181",
+ "kafkaBrokerZkBasePath": "/brokers",
+ "stormKafkaUseSameZkQuorumWithKafkaBroker": true,
+ "stormKafkaTransactionZkQuorum": "",
+ "stormKafkaTransactionZkPath": "/consumers",
+ "stormKafkaEagleConsumer": "eagle_consumer",
+ "stormKafkaStateUpdateIntervalMs": 2000,
+ "stormKafkaFetchSizeBytes": 1048586,
+ },
+ "zkConfig" : {
+ "zkQuorum" : "localhost:2181",
+ "zkRoot" : "/alert",
+ "zkSessionTimeoutMs" : 10000,
+ "connectionTimeoutMs" : 10000,
+ "zkRetryTimes" : 3,
+ "zkRetryInterval" : 3000
+ },
+ "dynamicConfigSource" : {
+ "initDelayMillis": 3000,
+ "delayMillis" : 10000
+ },
+ "metadataService": {
+ "context" : "/api",
+ "host" : "localhost",
+ "port" : 8080
+ },
+ "metric":{
+ "sink": {
+// "kafka": {
+// "topic": "alert_metric_test"
+// "bootstrap.servers": "localhost:9092"
+// }
+ "logger": {
+ "level":"INFO"
+ }
+ "elasticsearch": {
+ "hosts": ["10.64.223.222:9200"]
+ "index": "alert_metric_test"
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/correlation/application-integration-2.conf
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/correlation/application-integration-2.conf b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/correlation/application-integration-2.conf
new file mode 100644
index 0000000..bb998cd
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/correlation/application-integration-2.conf
@@ -0,0 +1,57 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+{
+ "topology" : {
+ "name" : "alertUnitTopology_2",
+ "numOfSpoutTasks" : 1,
+ "numOfRouterBolts" : 4,
+ "numOfAlertBolts" : 10,
+ "numOfPublishTasks" : 1,
+ "numOfTotalWorkers": 20,
+ "localMode" : "true"
+ },
+ "spout" : {
+ "kafkaBrokerZkQuorum": "localhost:2181",
+ "kafkaBrokerZkBasePath": "/brokers",
+ "stormKafkaUseSameZkQuorumWithKafkaBroker": true,
+ "stormKafkaTransactionZkQuorum": "",
+ "stormKafkaTransactionZkPath": "/consumers",
+ "stormKafkaEagleConsumer": "eagle_consumer",
+ "stormKafkaStateUpdateIntervalMs": 2000,
+ "stormKafkaFetchSizeBytes": 1048586,
+ },
+ "zkConfig" : {
+ "zkQuorum" : "localhost:2181",
+ "zkRoot" : "/alert",
+ "zkSessionTimeoutMs" : 10000,
+ "connectionTimeoutMs" : 10000,
+ "zkRetryTimes" : 3,
+ "zkRetryInterval" : 3000
+ },
+ "dynamicConfigSource" : {
+ "initDelayMillis": 3000,
+ "delayMillis" : 10000
+ },
+ "metadataService": {
+ "context" : "/api",
+ "host" : "localhost",
+ "port" : 8080
+ },
+ "coordinatorService": {
+ "host": "localhost",
+ "port": "9090",
+ "context" : "/api"
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/correlation/datasources.json
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/correlation/datasources.json b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/correlation/datasources.json
new file mode 100644
index 0000000..946acd6
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/correlation/datasources.json
@@ -0,0 +1,37 @@
+[
+{
+ "name": "eslog_datasource",
+ "type": "KAFKA",
+ "properties": {
+ },
+ "topic": "eslogs",
+ "schemeCls": "org.apache.eagle.alert.engine.scheme.JsonScheme",
+ "codec": {
+ "streamNameSelectorProp": {
+ "userProvidedStreamName" : "esStream",
+ "streamNameFormat":"%s"
+ },
+ "streamNameSelectorCls":"org.apache.eagle.alert.engine.scheme.JsonStringStreamNameSelector",
+ "timestampColumn": "timestamp",
+ "timestampFormat":""
+ }
+},
+{
+ "name": "bootfailure_datasource",
+ "type": "KAFKA",
+ "properties": {
+ },
+ "topic": "bootfailures",
+ "schemeCls": "org.apache.eagle.alert.engine.scheme.JsonScheme",
+ "codec": {
+ "streamNameSelectorProp": {
+ "userProvidedStreamName" : "ifStream",
+ "streamNameFormat":"%s"
+ },
+ "streamNameSelectorCls":"org.apache.eagle.alert.engine.scheme.JsonStringStreamNameSelector",
+ "timestampColumn": "timestamp",
+ "timestampFormat":""
+ }
+}
+
+]
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/correlation/policies.json
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/correlation/policies.json b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/correlation/policies.json
new file mode 100644
index 0000000..4702575
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/correlation/policies.json
@@ -0,0 +1,39 @@
+[
+{
+ "name": "logStreamJoinBootFailure",
+ "description" : "policy to check host perfmon_cpu",
+ "inputStreams": [
+ "esStream",
+ "ifStream"
+ ],
+ "outputStreams": [
+ "log_stream_join_output"
+ ],
+ "definition": {
+ "type": "siddhi",
+ "value": " from esStream#window.externalTime(timestamp, 20 min) as a join ifStream#window.externalTime(timestamp, 5 min) as b on a.instanceUuid == b.instanceUuid select logLevel, a.host as aHost, a.component, a.message as logMessage, b.message as failMessage, a.timestamp as t1, b.timestamp as t2, b.host as bHost, count(1) as errorCount insert into log_stream_join_output; "
+ },
+ "partitionSpec": [
+ {
+ "streamId" : "esStream",
+ "type" : "GROUPBY",
+ "columns" : [
+ "instanceUuid"
+ ],
+ "sortSpec": {
+ "windowPeriod" : "PT1M"
+ }
+ },
+ {
+ "streamId" : "ifStream",
+ "type" : "GROUPBY",
+ "columns" : [
+ "instanceUuid"
+ ],
+ "sortSpec": {
+ "windowPeriod" : "PT1M"
+ }
+ }
+ ]
+}
+]
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/correlation/publishments.json
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/correlation/publishments.json b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/correlation/publishments.json
new file mode 100644
index 0000000..7b531fc
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/correlation/publishments.json
@@ -0,0 +1,17 @@
+[
+{
+ "name":"log-stream-join-output",
+ "type":"org.apache.eagle.alert.engine.publisher.impl.AlertEmailPublisher",
+ "inputStreams": [
+ "log_stream_join_output"
+ ],
+ "properties": {
+ "subject":"UMP Test Alert",
+ "template":"",
+ "sender": "sender@corp.com",
+ "recipients": "receiver@corp.com",
+ "smtp.server":"mailhost.com"
+ },
+ "dedupIntervalMin" : "1"
+}
+]
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/correlation/streamdefinitions.json
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/correlation/streamdefinitions.json b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/correlation/streamdefinitions.json
new file mode 100644
index 0000000..0a26a28
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/correlation/streamdefinitions.json
@@ -0,0 +1,93 @@
+[
+{
+ "streamId": "esStream",
+ "dataSource" : "eslog_datasource",
+ "description":"the data stream for es log of different modules",
+ "validate": false,
+ "timeseries":false,
+ "columns": [
+ {
+ "name": "instanceUuid",
+ "type" : "string",
+ "defaultValue": "",
+ "required":true
+ },
+ {
+ "name": "timestamp",
+ "type" : "long",
+ "defaultValue": 0,
+ "required":true
+ },
+ {
+ "name": "logLevel",
+ "type" : "string",
+ "defaultValue": "ERROR",
+ "required": true
+ },
+ {
+ "name": "message",
+ "type" : "string",
+ "defaultValue": "",
+ "required":true
+ },
+ {
+ "name": "reqId",
+ "type" : "string",
+ "defaultValue": "",
+ "required":true
+ },
+ {
+ "name": "host",
+ "type" : "string",
+ "defaultValue": "",
+ "required":true
+ },
+ {
+ "name": "component",
+ "type" : "string",
+ "defaultValue": "nova",
+ "required":true
+ }
+ ]
+}
+,
+{
+ "streamId": "ifStream",
+ "dataSource" : "bootfailure_datasource",
+ "description":"the data stream for boot failure(instance fault)",
+ "validate": false,
+ "timeseries":false,
+ "columns": [
+ {
+ "name": "instanceUuid",
+ "type" : "string",
+ "defaultValue": "",
+ "required":true
+ },
+ {
+ "name": "timestamp",
+ "type" : "long",
+ "defaultValue": 0,
+ "required":true
+ },
+ {
+ "name": "reqId",
+ "type" : "string",
+ "defaultValue": "",
+ "required": true
+ },
+ {
+ "name": "message",
+ "type" : "string",
+ "defaultValue": "",
+ "required":true
+ },
+ {
+ "name": "host",
+ "type" : "string",
+ "defaultValue": "",
+ "required":true
+ }
+ ]
+}
+]
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/correlation/topologies.json
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/correlation/topologies.json b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/correlation/topologies.json
new file mode 100644
index 0000000..9aa8716
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/correlation/topologies.json
@@ -0,0 +1,31 @@
+[
+{
+ "name": "alertUnitTopology_2",
+ "numOfSpout":1,
+ "numOfGroupBolt": 4,
+ "numOfAlertBolt": 10,
+ "spoutId": "alertEngineSpout",
+ "groupNodeIds" : [
+ "streamRouterBolt0",
+ "streamRouterBolt1",
+ "streamRouterBolt2",
+ "streamRouterBolt3"
+ ],
+ "alertBoltIds": [
+ "alertBolt0",
+ "alertBolt1",
+ "alertBolt2",
+ "alertBolt3",
+ "alertBolt4",
+ "alertBolt5",
+ "alertBolt6",
+ "alertBolt7",
+ "alertBolt8",
+ "alertBolt9"
+ ],
+ "pubBoltId" : "alertPublishBolt",
+ "spoutParallelism": 1,
+ "groupParallelism": 1,
+ "alertParallelism": 1
+}
+]
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/datasources.json
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/datasources.json b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/datasources.json
new file mode 100644
index 0000000..77a280c
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/datasources.json
@@ -0,0 +1,19 @@
+[
+{
+ "name": "perfmon_datasource",
+ "type": "KAFKA",
+ "properties": {
+ },
+ "topic": "perfmon_metrics",
+ "schemeCls": "org.apache.eagle.alert.engine.scheme.JsonScheme",
+ "codec": {
+ "streamNameSelectorProp": {
+ "fieldNamesToInferStreamName" : "metric",
+ "streamNameFormat":"%s"
+ },
+ "streamNameSelectorCls":"org.apache.eagle.alert.engine.scheme.JsonStringStreamNameSelector",
+ "timestampColumn": "timestamp",
+ "timestampFormat":""
+ }
+}
+]
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/log4j.properties b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/log4j.properties
new file mode 100644
index 0000000..3ba587d
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/log4j.properties
@@ -0,0 +1,23 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+log4j.rootLogger=INFO, stdout
+
+# standard output
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %p [%t] %c{2}[%L]: %m%n
+
+#log4j.logger.org.apache.eagle.alert.metric=DEBUG
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/policies.json
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/policies.json b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/policies.json
new file mode 100644
index 0000000..5edece9
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/policies.json
@@ -0,0 +1,54 @@
+[
+{
+ "name": "perfmon_cpu_host_check",
+ "description" : "policy to check host perfmon_cpu",
+ "inputStreams": [
+ "perfmon_cpu_stream"
+ ],
+ "outputStreams": [
+ "perfmon_cpu_check_output"
+ ],
+ "definition": {
+ "type": "siddhi",
+ "value": "from perfmon_cpu_stream[value > 90.0] select * group by host insert into perfmon_cpu_check_output;"
+ },
+ "partitionSpec": [
+ {
+ "streamId" : "perfmon_cpu_stream",
+ "type" : "GROUPBY",
+ "columns" : [
+ "host"
+ ],
+ "sortSpec": {
+ "windowPeriod" : "PT1M"
+ }
+ }
+ ]
+},
+{
+ "name": "perfmon_cpu_pool_check",
+ "description" : "policy to check pool perfmon_cpu",
+ "inputStreams": [
+ "perfmon_cpu_stream"
+ ],
+ "outputStreams": [
+ "perfmon_cpu_check_output"
+ ],
+ "definition": {
+ "type": "siddhi",
+ "value": "from perfmon_cpu_stream[value > 75.0] select * group by pool insert into perfmon_cpu_check_output;"
+ },
+ "partitionSpec": [
+ {
+ "streamId" : "perfmon_cpu_stream",
+ "type" : "GROUPBY",
+ "columns" : [
+ "pool"
+ ],
+ "sortSpec": {
+ "windowPeriod" : "PT1M"
+ }
+ }
+ ]
+}
+]
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/publishments.json
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/publishments.json b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/publishments.json
new file mode 100644
index 0000000..b3840a5
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/publishments.json
@@ -0,0 +1,29 @@
+[
+{
+ "name":"test-stream-output",
+ "type":"org.apache.eagle.alert.engine.publisher.impl.AlertKafkaPublisher",
+ "policyIds": [
+ "perfmon_cpu_host_check", "perfmon_cpu_pool_check"
+ ],
+ "properties": {
+ "subject":"UMP Test Alert",
+ "template":"",
+ "sender": "sender@corp.com",
+ "recipients": "receiver@corp.com",
+ "smtp.server":"mailhost.com",
+ "connection": "plaintext",
+ "smtp.port": "25"
+ },
+ "dedupIntervalMin" : "PT0M"
+},
+{
+ "type": "org.apache.eagle.alert.engine.publisher.impl.AlertKafkaPublisher",
+ "name":"kafka-testAlertStream",
+ "policyIds": ["perfmon_cpu_host_check"],
+ "dedupIntervalMin": "PT1M",
+ "properties":{
+ "kafka_broker":"sandbox.hortonworks.com:6667",
+ "topic":"test_kafka"
+ }
+}
+]
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/publishments2.json
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/publishments2.json b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/publishments2.json
new file mode 100644
index 0000000..277ba75
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/publishments2.json
@@ -0,0 +1,19 @@
+[
+{
+ "name":"test-stream-output",
+ "type":"org.apache.eagle.alert.engine.publisher.impl.AlertEmailPublisher",
+ "policyIds": [
+ "policy1", "policy2"
+ ],
+ "properties": {
+ "subject":"UMP Test Alert",
+ "template":"",
+ "sender": "sender@corp.com",
+ "recipients": "receiver@corp.com",
+ "smtp.server":"mailhost.com",
+ "connection": "plaintext",
+ "smtp.port": "25"
+ },
+ "dedupIntervalMin" : "PT0M"
+}
+]
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/sample_perfmon_data.json
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/sample_perfmon_data.json b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/sample_perfmon_data.json
new file mode 100644
index 0000000..c63b9ff
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/sample_perfmon_data.json
@@ -0,0 +1,3 @@
+[
+{"host": "", "timestamp" : "", "metric" : "", "pool": "", "value": 1.0, "colo": "phx"},
+]
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/streamdefinitions.json
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/streamdefinitions.json b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/streamdefinitions.json
new file mode 100644
index 0000000..cbeae19
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/streamdefinitions.json
@@ -0,0 +1,44 @@
+[
+{
+ "streamId": "perfmon_cpu_stream",
+ "dataSource" : "perfmon_datasource",
+ "description":"the data stream for perfmon cpu metrics",
+ "validate": false,
+ "timeseries":false,
+ "columns": [
+ {
+ "name": "host",
+ "type" : "string",
+ "defaultValue": "",
+ "required":true
+ },
+ {
+ "name": "timestamp",
+ "type" : "long",
+ "defaultValue": 0,
+ "required":true
+ },{
+ "name": "metric",
+ "type" : "string",
+ "defaultValue": "perfmon_cpu",
+ "required": true
+ },{
+ "name": "pool",
+ "type" : "string",
+ "defaultValue": "raptor_general",
+ "required":true
+ },{
+ "name": "value",
+ "type" : "double",
+ "defaultValue": 0.0,
+ "required":true
+ },
+ {
+ "name": "colo",
+ "type" : "string",
+ "defaultValue": "",
+ "required":true
+ }
+ ]
+}
+]
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/testAlertBoltSpec.json
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/testAlertBoltSpec.json b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/testAlertBoltSpec.json
new file mode 100644
index 0000000..fd255cd
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/testAlertBoltSpec.json
@@ -0,0 +1,92 @@
+{
+ "version": "version1",
+ "topologyName": "testTopology",
+ "boltPoliciesMap": {
+ "alertBolt0": [
+ {
+ "name": "policy1",
+ "description": null,
+ "inputStreams": [
+ "testTopic3Stream"
+ ],
+ "outputStreams": [
+ "testAlertStream"
+ ],
+ "definition": {
+ "type": "siddhi",
+ "value": "from testTopic3Stream[value=='xyz'] select value insert into testAlertStream;"
+ },
+ "partitionSpec": [
+ {
+ "streamId": "testTopic3Stream",
+ "type": "GROUPBY",
+ "columns": [
+ "value"
+ ],
+ "sortSpec": {
+ "windowPeriod": "PT10S",
+ "windowMargin": 1000
+ }
+ }
+ ],
+ "parallelismHint": 0
+ },
+ {
+ "name": "policy2",
+ "description": null,
+ "inputStreams": [
+ "testTopic4Stream"
+ ],
+ "outputStreams": [
+ "testAlertStream"
+ ],
+ "definition": {
+ "type": "siddhi",
+ "value": "from testTopic4Stream[value=='xyz'] select value insert into testAlertStream;"
+ },
+ "partitionSpec": [
+ {
+ "streamId": "testTopic4Stream",
+ "type": "GROUPBY",
+ "columns": [
+ "value"
+ ],
+ "sortSpec": {
+ "windowPeriod": "PT10S",
+ "windowMargin": 1000
+ }
+ }
+ ],
+ "parallelismHint": 0
+ },
+ {
+ "name": "policy3",
+ "description": null,
+ "inputStreams": [
+ "testTopic5Stream"
+ ],
+ "outputStreams": [
+ "testAlertStream"
+ ],
+ "definition": {
+ "type": "siddhi",
+ "value": "from testTopic5Stream[value=='xyz'] select value insert into testAlertStream;"
+ },
+ "partitionSpec": [
+ {
+ "streamId": "testTopic5Stream",
+ "type": "GROUPBY",
+ "columns": [
+ "value"
+ ],
+ "sortSpec": {
+ "windowPeriod": "PT10S",
+ "windowMargin": 1000
+ }
+ }
+ ],
+ "parallelismHint": 0
+ }
+ ]
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/testPublishSpec.json
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/testPublishSpec.json b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/testPublishSpec.json
new file mode 100644
index 0000000..220c8fe
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/testPublishSpec.json
@@ -0,0 +1,32 @@
+{
+ "version": "version1",
+ "topologyName": "testTopology",
+ "boltId": "alertPublishBolt",
+ "publishments": [
+ {
+ "type": "org.apache.eagle.alert.engine.publisher.impl.AlertEmailPublisher",
+ "name":"email-testAlertStream",
+ "policyIds": ["policy1", "policy2", "policy3"],
+ "dedupIntervalMin": "PT1M",
+ "properties":{
+ "subject":"UMP Test Alert",
+ "template":"",
+ "sender": "sender@corp.com",
+ "recipients": "receiver@corp.com",
+ "smtp.server":"mailhost.com",
+ "connection": "plaintext",
+ "smtp.port": "25"
+ }
+ }
+/* {
+ "type": "org.apache.eagle.alert.engine.publisher.impl.AlertKafkaPublisher",
+ "name":"kafka-testAlertStream",
+ "policyIds": ["testPolicy"],
+ "dedupIntervalMin": "PT1M",
+ "properties":{
+ "kafka_broker":"sandbox.hortonworks.com:6667",
+ "topic":"test_kafka"
+ }
+ }*/
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/testPublishSpec2.json
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/testPublishSpec2.json b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/testPublishSpec2.json
new file mode 100644
index 0000000..0a16540
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/testPublishSpec2.json
@@ -0,0 +1,32 @@
+{
+ "version": "version1",
+ "topologyName": "testTopology",
+ "boltId": "alertPublishBolt",
+ "publishments": [
+ {
+ "type": "org.apache.eagle.alert.engine.publisher.impl.AlertEmailPublisher",
+ "name":"email-testAlertStream",
+ "policyIds": ["policy1"],
+ "dedupIntervalMin": "PT2M",
+ "properties":{
+ "subject":"UMP Test Alert",
+ "template":"",
+ "sender": "sender@corp.com",
+ "recipients": "receiver@corp.com",
+ "smtp.server":"mailhost.com",
+ "connection": "plaintext",
+ "smtp.port": "25"
+ }
+ }
+// {
+// "type": "org.apache.eagle.alert.engine.publisher.impl.AlertKafkaPublisher",
+// "name":"kafka-testAlertStream",
+// "policyIds": ["testPolicy"],
+// "dedupIntervalMin": "PT1M",
+// "properties":{
+// "kafka_broker":"sandbox.hortonworks.com:6667",
+// "topic":"test_kafka"
+// }
+// }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/testSpoutSpec.json
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/testSpoutSpec.json b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/testSpoutSpec.json
new file mode 100644
index 0000000..c97ec43
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/testSpoutSpec.json
@@ -0,0 +1,139 @@
+{
+ "version": null,
+ "topologyId": "testTopology",
+ "kafka2TupleMetadataMap": {
+ "testTopic5": {
+ "type": null,
+ "name": "testTopic5",
+ "properties": null,
+ "topic": "testTopic5",
+ "schemeCls": "org.apache.eagle.alert.engine.scheme.JsonScheme",
+ "codec": null
+ },
+ "testTopic3": {
+ "type": null,
+ "name": "testTopic3",
+ "properties": null,
+ "topic": "testTopic3",
+ "schemeCls": "org.apache.eagle.alert.engine.scheme.PlainStringScheme",
+ "codec": null
+ },
+ "testTopic4": {
+ "type": null,
+ "name": "testTopic4",
+ "properties": null,
+ "topic": "testTopic4",
+ "schemeCls": "org.apache.eagle.alert.engine.scheme.PlainStringScheme",
+ "codec": null
+ }
+ },
+ "tuple2StreamMetadataMap": {
+ "testTopic5": {
+ "activeStreamNames": [
+ "testTopic5Stream"
+ ],
+ "streamNameSelectorProp": {
+ "userProvidedStreamName": "testTopic5Stream"
+ },
+ "streamNameSelectorCls": "org.apache.eagle.alert.engine.scheme.JsonStringStreamNameSelector",
+ "timestampColumn": "timestamp",
+ "timestampFormat": null
+ },
+ "testTopic3": {
+ "activeStreamNames": [
+ "testTopic3Stream"
+ ],
+ "streamNameSelectorProp": {
+ "userProvidedStreamName": "testTopic3Stream"
+ },
+ "streamNameSelectorCls": "org.apache.eagle.alert.engine.scheme.PlainStringStreamNameSelector",
+ "timestampColumn": "timestamp",
+ "timestampFormat": null
+ },
+ "testTopic4": {
+ "activeStreamNames": [
+ "testTopic4Stream"
+ ],
+ "streamNameSelectorProp": {
+ "userProvidedStreamName": "testTopic4Stream"
+ },
+ "streamNameSelectorCls": "org.apache.eagle.alert.engine.scheme.PlainStringStreamNameSelector",
+ "timestampColumn": "timestamp",
+ "timestampFormat": null
+ }
+ },
+ "streamRepartitionMetadataMap": {
+ "testTopic5": [
+ {
+ "topicName": "testTopic5",
+ "streamId": "defaultStringStream",
+ "groupingStrategies": [
+ {
+ "partition": {
+ "streamId": "testTopic5Stream",
+ "type": "GROUPBY",
+ "columns": [
+ "value"
+ ],
+ "sortSpec": {
+ "windowPeriod": "PT10S",
+ "windowMargin": 1000
+ }
+ },
+ "numTotalParticipatingRouterBolts": 1,
+ "startSequence": 0,
+ "totalTargetBoltIds": []
+ }
+ ]
+ }
+ ],
+ "testTopic3": [
+ {
+ "topicName": "testTopic3",
+ "streamId": "defaultStringStream",
+ "groupingStrategies": [
+ {
+ "partition": {
+ "streamId": "testTopic3Stream",
+ "type": "GROUPBY",
+ "columns": [
+ "value"
+ ],
+ "sortSpec": {
+ "windowPeriod": "PT10S",
+ "windowMargin": 1000
+ }
+ },
+ "numTotalParticipatingRouterBolts": 1,
+ "startSequence": 0,
+ "totalTargetBoltIds": []
+ }
+ ]
+ }
+ ],
+ "testTopic4": [
+ {
+ "topicName": "testTopic4",
+ "streamId": "defaultStringStream",
+ "groupingStrategies": [
+ {
+ "partition": {
+ "streamId": "testTopic4Stream",
+ "type": "GROUPBY",
+ "columns": [
+ "value"
+ ],
+ "sortSpec": {
+ "windowPeriod": "PT10S",
+ "windowMargin": 1000
+ }
+ },
+ "numTotalParticipatingRouterBolts": 1,
+ "startSequence": 0,
+ "totalTargetBoltIds": []
+ }
+ ]
+ }
+ ]
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/testStreamDefinitionsSpec.json
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/testStreamDefinitionsSpec.json b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/testStreamDefinitionsSpec.json
new file mode 100644
index 0000000..6b3ccff
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/testStreamDefinitionsSpec.json
@@ -0,0 +1,47 @@
+{
+ "testTopic5Stream": {
+ "streamId": "testTopic5Stream",
+ "dataSource": null,
+ "description": null,
+ "validate": false,
+ "timeseries": false,
+ "columns": [
+ {
+ "name": "value",
+ "type": "STRING",
+ "defaultValue": null,
+ "required": false
+ }
+ ]
+ },
+ "testTopic4Stream": {
+ "streamId": "testTopic4Stream",
+ "dataSource": null,
+ "description": null,
+ "validate": false,
+ "timeseries": false,
+ "columns": [
+ {
+ "name": "value",
+ "type": "STRING",
+ "defaultValue": null,
+ "required": false
+ }
+ ]
+ },
+ "testTopic3Stream": {
+ "streamId": "testTopic3Stream",
+ "dataSource": null,
+ "description": null,
+ "validate": false,
+ "timeseries": false,
+ "columns": [
+ {
+ "name": "value",
+ "type": "STRING",
+ "defaultValue": null,
+ "required": false
+ }
+ ]
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/testStreamRouterBoltSpec.json
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/testStreamRouterBoltSpec.json b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/testStreamRouterBoltSpec.json
new file mode 100644
index 0000000..f4e72bf
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/testStreamRouterBoltSpec.json
@@ -0,0 +1,123 @@
+{
+ "version": null,
+ "topologyName": "testTopology",
+ "routerSpecs": [
+ {
+ "streamId": "testTopic3Stream",
+ "partition": {
+ "streamId": "testTopic3Stream",
+ "type": "GROUPBY",
+ "columns": [
+ "value"
+ ],
+ "sortSpec": {
+ "windowPeriod": "PT10S",
+ "windowMargin": 1000
+ }
+ },
+ "targetQueue": [
+ {
+ "partition": {
+ "streamId": "testTopic3Stream",
+ "type": "GROUPBY",
+ "columns": [
+ "value"
+ ],
+ "sortSpec": {
+ "windowPeriod": "PT10S",
+ "windowMargin": 1000
+ }
+ },
+ "workers": [
+ {
+ "topologyName": "testTopology",
+ "boltId": "alertBolt0"
+ },
+ {
+ "topologyName": "testTopology",
+ "boltId": "alertBolt1"
+ }
+ ]
+ }
+ ]
+ },
+ {
+ "streamId": "testTopic4Stream",
+ "partition": {
+ "streamId": "testTopic4Stream",
+ "type": "GROUPBY",
+ "columns": [
+ "value"
+ ],
+ "sortSpec": {
+ "windowPeriod": "PT10S",
+ "windowMargin": 1000
+ }
+ },
+ "targetQueue": [
+ {
+ "partition": {
+ "streamId": "testTopic4Stream",
+ "type": "GROUPBY",
+ "columns": [
+ "value"
+ ],
+ "sortSpec": {
+ "windowPeriod": "PT10S",
+ "windowMargin": 1000
+ }
+ },
+ "workers": [
+ {
+ "topologyName": "testTopology",
+ "boltId": "alertBolt0"
+ },
+ {
+ "topologyName": "testTopology",
+ "boltId": "alertBolt1"
+ }
+ ]
+ }
+ ]
+ },
+ {
+ "streamId": "testTopic5Stream",
+ "partition": {
+ "streamId": "testTopic5Stream",
+ "type": "GROUPBY",
+ "columns": [
+ "value"
+ ],
+ "sortSpec": {
+ "windowPeriod": "PT10S",
+ "windowMargin": 1000
+ }
+ },
+ "targetQueue": [
+ {
+ "partition": {
+ "streamId": "testTopic5Stream",
+ "type": "GROUPBY",
+ "columns": [
+ "value"
+ ],
+ "sortSpec": {
+ "windowPeriod": "PT10S",
+ "windowMargin": 1000
+ }
+ },
+ "workers": [
+ {
+ "topologyName": "testTopology",
+ "boltId": "alertBolt0"
+ },
+ {
+ "topologyName": "testTopology",
+ "boltId": "alertBolt1"
+ }
+ ]
+ }
+ ]
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/topic.json
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/topic.json b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/topic.json
new file mode 100644
index 0000000..b49d6ad
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/topic.json
@@ -0,0 +1 @@
+nn_jmx_metric_sandbox
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/topologies.json
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/topologies.json b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/topologies.json
new file mode 100644
index 0000000..411cc48
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/resources/topologies.json
@@ -0,0 +1,31 @@
+[
+{
+ "name": "alertUnitTopology_1",
+ "numOfSpout":1,
+ "numOfAlertBolt": 10,
+ "numOfGroupBolt": 4,
+ "spoutId": "alertEngineSpout",
+ "groupNodeIds" : [
+ "streamRouterBolt0",
+ "streamRouterBolt1",
+ "streamRouterBolt2",
+ "streamRouterBolt3"
+ ],
+ "alertBoltIds": [
+ "alertBolt0",
+ "alertBolt1",
+ "alertBolt2",
+ "alertBolt3",
+ "alertBolt4",
+ "alertBolt5",
+ "alertBolt6",
+ "alertBolt7",
+ "alertBolt8",
+ "alertBolt9"
+ ],
+ "pubBoltId" : "alertPublishBolt",
+ "spoutParallelism": 1,
+ "groupParallelism": 1,
+ "alertParallelism": 1
+}
+]
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/pom.xml b/eagle-core/eagle-alert/alert/alert-engine/pom.xml
new file mode 100644
index 0000000..4e68b4c
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/pom.xml
@@ -0,0 +1,110 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- ~ /* ~ * Licensed to the Apache Software Foundation (ASF) under one
+ or more ~ * contributor license agreements. See the NOTICE file distributed
+ with ~ * this work for additional information regarding copyright ownership.
+ ~ * The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ * (the "License"); you may not use this file except in compliance with
+ ~ * the License. You may obtain a copy of the License at ~ * ~ * http://www.apache.org/licenses/LICENSE-2.0
+ ~ * ~ * Unless required by applicable law or agreed to in writing, software
+ ~ * distributed under the License is distributed on an "AS IS" BASIS, ~ *
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ * See the License for the specific language governing permissions and ~
+ * limitations under the License. ~ */ -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>io.sherlock</groupId>
+ <artifactId>alert-parent</artifactId>
+ <version>0.0.1-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>alert-engine-parent</artifactId>
+ <packaging>pom</packaging>
+
+ <modules>
+ <module>alert-engine-base</module>
+ </modules>
+
+ <dependencies>
+ <dependency>
+ <groupId>io.sherlock</groupId>
+ <artifactId>alert-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.storm</groupId>
+ <artifactId>storm-kafka</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.storm</groupId>
+ <artifactId>storm-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>${kafka.artifact.id}</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-client</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-jaxrs</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.netflix.archaius</groupId>
+ <artifactId>archaius-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.wso2.siddhi</groupId>
+ <artifactId>siddhi-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.wso2.siddhi</groupId>
+ <artifactId>siddhi-query-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.wso2.siddhi</groupId>
+ <artifactId>siddhi-query-compiler</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.wso2.siddhi</groupId>
+ <artifactId>siddhi-extension-regex</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.wso2.siddhi</groupId>
+ <artifactId>siddhi-extension-string</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka-clients</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.101tec</groupId>
+ <artifactId>zkclient</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>joda-time</groupId>
+ <artifactId>joda-time</artifactId>
+ </dependency>
+ <!--<dependency>-->
+ <!--<groupId>org.slf4j</groupId>-->
+ <!--<artifactId>slf4j-log4j12</artifactId>-->
+ <!--</dependency>-->
+ <!--<dependency>-->
+ <!--<groupId>org.slf4j</groupId>-->
+ <!--<artifactId>slf4j-api</artifactId>-->
+ <!--</dependency>-->
+ <dependency>
+ <groupId>com.typesafe</groupId>
+ <artifactId>config</artifactId>
+ </dependency>
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata-service/.gitignore
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata-service/.gitignore b/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata-service/.gitignore
new file mode 100644
index 0000000..1dd3331
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata-service/.gitignore
@@ -0,0 +1,2 @@
+/target/
+/target/
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata-service/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata-service/pom.xml b/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata-service/pom.xml
new file mode 100644
index 0000000..83749b5
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata-service/pom.xml
@@ -0,0 +1,113 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- ~ /* ~ * Licensed to the Apache Software Foundation (ASF) under one
+ or more ~ * contributor license agreements. See the NOTICE file distributed
+ with ~ * this work for additional information regarding copyright ownership.
+ ~ * The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ * (the "License"); you may not use this file except in compliance with
+ ~ * the License. You may obtain a copy of the License at ~ * ~ * http://www.apache.org/licenses/LICENSE-2.0
+ ~ * ~ * Unless required by applicable law or agreed to in writing, software
+ ~ * distributed under the License is distributed on an "AS IS" BASIS, ~ *
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ * See the License for the specific language governing permissions and ~
+ * limitations under the License. ~ */ -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.eagle</groupId>
+ <artifactId>alert-metadata-parent</artifactId>
+ <version>0.0.1-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>alert-metadata-service</artifactId>
+ <packaging>war</packaging>
+
+ <dependencies>
+ <!-- Storm depends on org.ow2.asm:asm:4.0 -->
+ <!-- Jersey depends on asm:asm:3.0 -->
+ <dependency>
+ <groupId>org.apache.eagle</groupId>
+ <artifactId>alert-engine-base</artifactId>
+ <version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.ow2.asm</groupId>
+ <artifactId>asm</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.eagle</groupId>
+ <artifactId>alert-metadata</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-server</artifactId>
+ <!--<exclusions> -->
+ <!--<exclusion> -->
+ <!--<groupId>asm</groupId> -->
+ <!--<artifactId>asm</artifactId> -->
+ <!--</exclusion> -->
+ <!--</exclusions> -->
+ </dependency>
+ <dependency>
+ <groupId>com.sun.jersey.contribs</groupId>
+ <artifactId>jersey-multipart</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-mapper-asl</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-jaxrs</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-xc</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.typesafe</groupId>
+ <artifactId>config</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tomcat.embed</groupId>
+ <artifactId>tomcat-embed-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>io.swagger</groupId>
+ <artifactId>swagger-jaxrs</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-servlet</artifactId>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-maven-plugin</artifactId>
+ <configuration>
+ <scanIntervalSeconds>5</scanIntervalSeconds>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata-service/src/main/java/org/apache/eagle/alert/metadata/resource/MetadataResource.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata-service/src/main/java/org/apache/eagle/alert/metadata/resource/MetadataResource.java b/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata-service/src/main/java/org/apache/eagle/alert/metadata/resource/MetadataResource.java
new file mode 100644
index 0000000..94ec767
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata-service/src/main/java/org/apache/eagle/alert/metadata/resource/MetadataResource.java
@@ -0,0 +1,225 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.metadata.resource;
+
+import java.util.List;
+
+import javax.ws.rs.Consumes;
+import javax.ws.rs.DELETE;
+import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+
+import org.apache.eagle.alert.coordination.model.Kafka2TupleMetadata;
+import org.apache.eagle.alert.coordination.model.ScheduleState;
+import org.apache.eagle.alert.coordination.model.internal.PolicyAssignment;
+import org.apache.eagle.alert.coordination.model.internal.Topology;
+import org.apache.eagle.alert.engine.coordinator.PolicyDefinition;
+import org.apache.eagle.alert.engine.coordinator.Publishment;
+import org.apache.eagle.alert.engine.coordinator.PublishmentType;
+import org.apache.eagle.alert.engine.coordinator.StreamDefinition;
+import org.apache.eagle.alert.engine.coordinator.StreamingCluster;
+import org.apache.eagle.alert.metadata.impl.MetadataDaoFactory;
+
+/**
+ * @since Apr 11, 2016
+ *
+ */
+@Path("/metadata")
+@Produces("application/json")
+@Consumes("application/json")
+public class MetadataResource {
+
+ private IMetadataDao dao = MetadataDaoFactory.getInstance().getMetadataDao();
+
+ @Path("/clusters")
+ @GET
+ public List<StreamingCluster> listClusters() {
+ return dao.listClusters();
+ }
+
+ @Path("/clear")
+ @POST
+ public OpResult clear() {
+ return dao.clear();
+ }
+
+ @Path("/export")
+ @GET
+ public Models export() {
+ return dao.export();
+ }
+
+ @Path("/import")
+ @GET
+ public OpResult importModels(Models model) {
+ return dao.importModels(model);
+ }
+
+ @Path("/clusters")
+ @POST
+ public OpResult addCluster(StreamingCluster cluster) {
+ return dao.addCluster(cluster);
+ }
+
+ @Path("/clusters/{clusterId}")
+ @DELETE
+ public OpResult removeCluster(@PathParam("clusterId") String clusterId) {
+ return dao.removeCluster(clusterId);
+ }
+
+ @Path("/streams")
+ @GET
+ public List<StreamDefinition> listStreams() {
+ return dao.listStreams();
+ }
+
+ @Path("/streams")
+ @POST
+ public OpResult createStream(StreamDefinition stream) {
+ return dao.createStream(stream);
+ }
+
+ @Path("/streams/{streamId}")
+ @DELETE
+ public OpResult removeStream(@PathParam("streamId") String streamId) {
+ return dao.removeStream(streamId);
+ }
+
+ @Path("/datasources")
+ @GET
+ public List<Kafka2TupleMetadata> listDataSources() {
+ return dao.listDataSources();
+ }
+
+ @Path("/datasources")
+ @POST
+ public OpResult addDataSource(Kafka2TupleMetadata dataSource) {
+ return dao.addDataSource(dataSource);
+ }
+
+ @Path("/datasources/{datasourceId}")
+ @DELETE
+ public OpResult removeDataSource(@PathParam("datasourceId") String datasourceId) {
+ return dao.removeDataSource(datasourceId);
+ }
+
+ @Path("/policies")
+ @GET
+ public List<PolicyDefinition> listPolicies() {
+ return dao.listPolicies();
+ }
+
+ @Path("/policies")
+ @POST
+ public OpResult addPolicy(PolicyDefinition policy) {
+ return dao.addPolicy(policy);
+ }
+
+ @Path("/policies/{policyId}")
+ @DELETE
+ public OpResult removePolicy(@PathParam("policyId") String policyId) {
+ return dao.removePolicy(policyId);
+ }
+
+ @Path("/publishments")
+ @GET
+ public List<Publishment> listPublishment() {
+ return dao.listPublishment();
+ }
+
+ @Path("/publishments")
+ @POST
+ public OpResult addPublishment(Publishment publishment) {
+ return dao.addPublishment(publishment);
+ }
+
+ @Path("/publishments/{pubId}")
+ @DELETE
+ public OpResult removePublishment(@PathParam("pubId") String pubId) {
+ return dao.removePublishment(pubId);
+ }
+
+ @Path("/publishmentTypes")
+ @GET
+ public List<PublishmentType> listPublishmentType() {
+ return dao.listPublishmentType();
+ }
+
+ @Path("/publishmentTypes")
+ @POST
+ public OpResult addPublishmentType(PublishmentType publishmentType) {
+ return dao.addPublishmentType(publishmentType);
+ }
+
+ @Path("/publishmentTypes/{pubType}")
+ @DELETE
+ public OpResult removePublishmentType(@PathParam("pubType") String pubType) {
+ return dao.removePublishmentType(pubType);
+ }
+
+ @Path("/schedulestates/{versionId}")
+ @GET
+ public ScheduleState listScheduleState(@PathParam("versionId") String versionId) {
+ return dao.getScheduleState(versionId);
+ }
+
+ @Path("/schedulestates")
+ @GET
+ public ScheduleState latestScheduleState() {
+ return dao.getScheduleState();
+ }
+
+ @Path("/schedulestates")
+ @POST
+ public OpResult addScheduleState(ScheduleState state) {
+ return dao.addScheduleState(state);
+ }
+
+ @Path("/assignments")
+ @GET
+ public List<PolicyAssignment> listAssignmenets() {
+ return dao.listAssignments();
+ }
+
+ @Path("/assignments")
+ @POST
+ public OpResult addAssignmenet(PolicyAssignment pa) {
+ return dao.addAssignment(pa);
+ }
+
+ @Path("/topologies")
+ @GET
+ public List<Topology> listTopologies() {
+ return dao.listTopologies();
+ }
+
+ @Path("/topologies")
+ @POST
+ public OpResult addTopology(Topology t) {
+ return dao.addTopology(t);
+ }
+
+ @Path("/topologies/{topologyName}")
+ @DELETE
+ public OpResult removeTopology(@PathParam("topologyName") String topologyName) {
+ return dao.removeTopology(topologyName);
+ }
+
+}
[18/19] incubator-eagle git commit: EAGLE-324: Init branch-v0.5
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/StreamRouterSpec.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/StreamRouterSpec.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/StreamRouterSpec.java
new file mode 100644
index 0000000..773ae56
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/StreamRouterSpec.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.coordination.model;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+import org.apache.eagle.alert.engine.coordinator.StreamPartition;
+
+/**
+ * One RouteSpec means one rule mapping [streamId -> StreamPartition ->
+ * PolicyExecutionQueue]
+ *
+ * Key is StreamPartition
+ */
+public class StreamRouterSpec {
+ private String streamId;
+ private StreamPartition partition; // The meta-data to build
+ // StreamPartitioner
+ private List<PolicyWorkerQueue> targetQueue = new ArrayList<PolicyWorkerQueue>();
+
+ public StreamPartition getPartition() {
+ return partition;
+ }
+
+ public void setPartition(StreamPartition partition) {
+ this.partition = partition;
+ }
+
+ @Override
+ public int hashCode() {
+ return new HashCodeBuilder().append(this.streamId).append(this.partition).append(targetQueue).build();
+ }
+
+ public List<PolicyWorkerQueue> getTargetQueue() {
+ return targetQueue;
+ }
+
+ public void addQueue(PolicyWorkerQueue queue) {
+ this.targetQueue.add(queue);
+ }
+
+ public String getStreamId() {
+ return streamId;
+ }
+
+ public void setStreamId(String streamId) {
+ this.streamId = streamId;
+ }
+
+ public void setTargetQueue(List<PolicyWorkerQueue> targetQueue) {
+ this.targetQueue = targetQueue;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("StreamRouterSpec[streamId=%s,partition=%s, queue=[%s]]", this.getStreamId(),
+ this.getPartition(), this.getTargetQueue());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/Tuple2StreamConverter.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/Tuple2StreamConverter.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/Tuple2StreamConverter.java
new file mode 100644
index 0000000..4f16756
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/Tuple2StreamConverter.java
@@ -0,0 +1,65 @@
+package org.apache.eagle.alert.coordination.model;
+
+import java.text.SimpleDateFormat;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Convert incoming tuple to stream
+ * incoming tuple consists of 2 fields, topic and map of key/value
+ * output stream consists of 3 fields, stream name, timestamp, and map of key/value
+ */
+public class Tuple2StreamConverter {
+ private static final Logger LOG = LoggerFactory.getLogger(Tuple2StreamConverter.class);
+ private Tuple2StreamMetadata metadata;
+ private StreamNameSelector cachedSelector;
+ public Tuple2StreamConverter(Tuple2StreamMetadata metadata){
+ this.metadata = metadata;
+ try {
+ cachedSelector = (StreamNameSelector)Class.forName(metadata.getStreamNameSelectorCls()).
+ getConstructor(Properties.class).
+ newInstance(metadata.getStreamNameSelectorProp());
+ }catch(Exception ex){
+ LOG.error("error initializing StreamNameSelector object", ex);
+ throw new IllegalStateException(ex);
+ }
+ }
+
+ /**
+ * Assume tuple is composed of topic + map of key/value
+ * @param tuple
+ * @return
+ */
+ @SuppressWarnings({ "unchecked" })
+ public List<Object> convert(List<Object> tuple){
+ Map<String, Object> m = (Map<String, Object>)tuple.get(1);
+ String streamName = cachedSelector.getStreamName(m);
+ if(!metadata.getActiveStreamNames().contains(streamName)) {
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("streamName {} is not within activeStreamNames {}", streamName, metadata.getActiveStreamNames());
+ }
+ return null;
+ }
+
+ Object timeObject = m.get(metadata.getTimestampColumn());
+ long timestamp = 0L;
+ if(timeObject instanceof Number){
+ timestamp = ((Number) timeObject).longValue();
+ }else{
+ String timestampFieldValue = (String) m.get(metadata.getTimestampColumn());
+ try {
+ SimpleDateFormat sdf = new SimpleDateFormat(metadata.getTimestampFormat());
+ timestamp = sdf.parse(timestampFieldValue).getTime();
+ } catch (Exception ex) {
+ LOG.error("continue with current timestamp because error happens while parsing timestamp column " + metadata.getTimestampColumn() + " with format " + metadata.getTimestampFormat());
+ timestamp = System.currentTimeMillis();
+ }
+ }
+ return Arrays.asList(tuple.get(0), streamName, timestamp, tuple.get(1));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/Tuple2StreamMetadata.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/Tuple2StreamMetadata.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/Tuple2StreamMetadata.java
new file mode 100644
index 0000000..c37e785
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/Tuple2StreamMetadata.java
@@ -0,0 +1,71 @@
+package org.apache.eagle.alert.coordination.model;
+
+import java.util.HashSet;
+import java.util.Properties;
+import java.util.Set;
+
+/**
+ * @Since 4/25/16. This metadata controls how tuple is transformed to stream for
+ * example raw data consists of {"metric" : "cpuUsage", "host" :
+ * "xyz.com", "timestamp" : 1346846400, "value" : "0.9"} field "metric"
+ * is used for creating stream name, here "cpuUsage" is stream name
+ *
+ * metric could be "cpuUsage", "diskUsage", "memUsage" etc, so
+ * activeStreamNames are subset of all metric names
+ *
+ * All other messages which are not one of activeStreamNames will be
+ * filtered out
+ */
+public class Tuple2StreamMetadata {
+ /**
+ * only messages belonging to activeStreamNames will be kept while
+ * transforming tuple into stream
+ */
+ private Set<String> activeStreamNames = new HashSet<String>();
+ // the specific stream name selector
+ private Properties streamNameSelectorProp;
+ private String streamNameSelectorCls;
+ private String timestampColumn;
+ private String timestampFormat;
+
+ public Set<String> getActiveStreamNames() {
+ return activeStreamNames;
+ }
+
+ public void setActiveStreamNames(Set<String> activeStreamNames) {
+ this.activeStreamNames = activeStreamNames;
+ }
+
+ public Properties getStreamNameSelectorProp() {
+ return streamNameSelectorProp;
+ }
+
+ public void setStreamNameSelectorProp(Properties streamNameSelectorProp) {
+ this.streamNameSelectorProp = streamNameSelectorProp;
+ }
+
+ public String getStreamNameSelectorCls() {
+ return streamNameSelectorCls;
+ }
+
+ public void setStreamNameSelectorCls(String streamNameSelectorCls) {
+ this.streamNameSelectorCls = streamNameSelectorCls;
+ }
+
+ public String getTimestampColumn() {
+ return timestampColumn;
+ }
+
+ public void setTimestampColumn(String timestampColumn) {
+ this.timestampColumn = timestampColumn;
+ }
+
+ public String getTimestampFormat() {
+ return timestampFormat;
+ }
+
+ public void setTimestampFormat(String timestampFormat) {
+ this.timestampFormat = timestampFormat;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/VersionedPolicyDefinition.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/VersionedPolicyDefinition.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/VersionedPolicyDefinition.java
new file mode 100644
index 0000000..f4b8ccb
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/VersionedPolicyDefinition.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.coordination.model;
+
+import org.apache.eagle.alert.engine.coordinator.PolicyDefinition;
+
+/**
+ * @since May 25, 2016
+ *
+ */
+public class VersionedPolicyDefinition {
+ private String version;
+ private PolicyDefinition definition;
+
+ public VersionedPolicyDefinition() {
+ }
+
+ public VersionedPolicyDefinition(String version, PolicyDefinition def) {
+ this.version = version;
+ this.definition = def;
+ }
+
+ public String getVersion() {
+ return version;
+ }
+
+ public void setVersion(String version) {
+ this.version = version;
+ }
+
+ public PolicyDefinition getDefinition() {
+ return definition;
+ }
+
+ public void setDefinition(PolicyDefinition definition) {
+ this.definition = definition;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/VersionedStreamDefinition.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/VersionedStreamDefinition.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/VersionedStreamDefinition.java
new file mode 100644
index 0000000..2770aa1
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/VersionedStreamDefinition.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.coordination.model;
+
+import org.apache.eagle.alert.engine.coordinator.StreamDefinition;
+
+/**
+ * @since May 25, 2016
+ *
+ */
+public class VersionedStreamDefinition {
+ private String version;
+ private StreamDefinition definition;
+
+ public VersionedStreamDefinition() {
+ }
+
+ public VersionedStreamDefinition(String version, StreamDefinition def) {
+ this.version = version;
+ this.definition = def;
+ }
+
+ public String getVersion() {
+ return version;
+ }
+
+ public void setVersion(String version) {
+ this.version = version;
+ }
+
+ public StreamDefinition getDefinition() {
+ return definition;
+ }
+
+ public void setDefinition(StreamDefinition definition) {
+ this.definition = definition;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/WorkSlot.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/WorkSlot.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/WorkSlot.java
new file mode 100644
index 0000000..96016f4
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/WorkSlot.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.coordination.model;
+
+
+/**
+ * A slot is simply a bolt.
+ */
+public class WorkSlot {
+ public String topologyName;
+ public String boltId;
+
+ public String getTopologyName() {
+ return topologyName;
+ }
+
+ public void setTopologyName(String topologyName) {
+ this.topologyName = topologyName;
+ }
+
+ public String getBoltId() {
+ return boltId;
+ }
+
+ public void setBoltId(String boltId) {
+ this.boltId = boltId;
+ }
+
+ public WorkSlot() {
+
+ }
+
+ public WorkSlot(String topo, String boltId) {
+ this.topologyName = topo;
+ this.boltId = boltId;
+ }
+
+ public String toString() {
+ return "(" + topologyName + ":" + boltId + ")";
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/internal/MonitoredStream.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/internal/MonitoredStream.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/internal/MonitoredStream.java
new file mode 100644
index 0000000..beda896
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/internal/MonitoredStream.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.coordination.model.internal;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+
+/**
+ * A monitored stream is the unique data set in the system.
+ *
+ * It's a combination of stream and the specific grp-by on it.
+ *
+ * For correlation stream, it means multiple stream for a given monitored stream.
+ *
+ *
+ * @since Apr 27, 2016
+ *
+ */
+public class MonitoredStream {
+
+ private String version;
+
+ // the stream group that this monitored stream stands for
+ private StreamGroup streamGroup = new StreamGroup();
+ private List<StreamWorkSlotQueue> queues = new ArrayList<StreamWorkSlotQueue>();
+
+ public MonitoredStream() {
+ }
+
+ public MonitoredStream(StreamGroup par) {
+ this.streamGroup = par;
+ }
+
+ public StreamGroup getStreamGroup() {
+ return streamGroup;
+ }
+
+ public List<StreamWorkSlotQueue> getQueues() {
+ return queues;
+ }
+
+ public synchronized void addQueues(StreamWorkSlotQueue queue) {
+ queues.add(queue);
+ }
+
+ public synchronized boolean removeQueue(StreamWorkSlotQueue queue) {
+ return this.queues.remove(queue);
+ }
+
+ public int hashCode() {
+ return new HashCodeBuilder().append(streamGroup).build();
+ }
+
+ public boolean equals(Object other) {
+ if (!(other instanceof MonitoredStream)) {
+ return false;
+ }
+ MonitoredStream o = (MonitoredStream) other;
+ return Objects.equals(streamGroup, o.streamGroup);
+ }
+
+ public String getVersion() {
+ return version;
+ }
+
+ public void setVersion(String version) {
+ this.version = version;
+ }
+
+ public void setQueues(List<StreamWorkSlotQueue> queues) {
+ this.queues = queues;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/internal/PolicyAssignment.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/internal/PolicyAssignment.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/internal/PolicyAssignment.java
new file mode 100644
index 0000000..3e956ca
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/internal/PolicyAssignment.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.coordination.model.internal;
+
+/**
+ * monitor metadata
+ *
+ * @since Apr 27, 2016
+ *
+ */
+public class PolicyAssignment {
+
+ private String version;
+
+ private String policyName;
+ private String queueId;
+
+ public PolicyAssignment() {
+ }
+
+ public PolicyAssignment(String policyName, String queueId) {
+ this.policyName = policyName;
+ this.queueId = queueId;
+ }
+
+ public String getPolicyName() {
+ return policyName;
+ }
+
+ public String getQueueId() {
+ return queueId;
+ }
+
+ public void setPolicyName(String policyName) {
+ this.policyName = policyName;
+ }
+
+ public void setQueueId(String queueId) {
+ this.queueId = queueId;
+ }
+
+ public String getVersion() {
+ return version;
+ }
+
+ public void setVersion(String version) {
+ this.version = version;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("PolicyAssignment of policy %s, queueId %s, version %s !", policyName, queueId, version);
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/internal/StreamGroup.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/internal/StreamGroup.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/internal/StreamGroup.java
new file mode 100644
index 0000000..9cbd841
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/internal/StreamGroup.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.coordination.model.internal;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+import org.apache.eagle.alert.engine.coordinator.StreamPartition;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.google.common.base.Objects;
+
+/**
+ * @since May 6, 2016
+ *
+ */
+public class StreamGroup {
+
+ private List<StreamPartition> streamPartitions = new ArrayList<StreamPartition>();
+
+ public StreamGroup() {
+ }
+
+ public List<StreamPartition> getStreamPartitions() {
+ return streamPartitions;
+ }
+
+ public void addStreamPartition(StreamPartition sp) {
+ this.streamPartitions.add(sp);
+ }
+
+ public void addStreamPartitions(List<StreamPartition> sps) {
+ this.streamPartitions.addAll(sps);
+ }
+
+ @org.codehaus.jackson.annotate.JsonIgnore
+ @JsonIgnore
+ public String getStreamId() {
+ StringBuilder sb = new StringBuilder("SG[");
+ for (StreamPartition sp : streamPartitions) {
+ sb.append(sp.getStreamId()).append("-");
+ }
+ sb.append("]");
+ return sb.toString();
+ }
+
+ @Override
+ public int hashCode() {
+ // implicitly all groups in stream groups will be built for hash code
+ return new HashCodeBuilder().append(streamPartitions).build();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (!(obj instanceof StreamGroup)) {
+ return false;
+ }
+ StreamGroup o = (StreamGroup) obj;
+ return Objects.equal(this.streamPartitions, o.streamPartitions);
+ }
+
+ @Override
+ public String toString() {
+ return String.format("StreamGroup partitions=: %s ", streamPartitions);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/internal/StreamWorkSlotQueue.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/internal/StreamWorkSlotQueue.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/internal/StreamWorkSlotQueue.java
new file mode 100644
index 0000000..fab6217
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/internal/StreamWorkSlotQueue.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.coordination.model.internal;
+
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.eagle.alert.coordination.model.WorkSlot;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+
+/**
+ * A work queue for given one monitored stream.
+ *
+ * Analog to storm's "tasks for given bolt".
+ *
+ * @since Apr 27, 2016
+ *
+ */
+public class StreamWorkSlotQueue {
+ private String queueId;
+
+ private final List<WorkSlot> workingSlots = new LinkedList<WorkSlot>();
+ private boolean dedicated;
+ // some dedicated option, like dedicated userId/tenantId/policyId.
+ private Map<String, Object> dedicateOption;
+
+ private int numberOfGroupBolts;
+ private Map<String, Integer> topoGroupStartIndex = new HashMap<String, Integer>();
+
+ public StreamWorkSlotQueue() {
+ }
+
+ public StreamWorkSlotQueue(StreamGroup par, boolean isDedicated, Map<String, Object> options,
+ List<WorkSlot> slots) {
+ this.queueId = par.getStreamId() + System.currentTimeMillis();// simply generate a queue
+ this.dedicated = isDedicated;
+ dedicateOption = new HashMap<String, Object>();
+ dedicateOption.putAll(options);
+ this.workingSlots.addAll(slots);
+ }
+
+ public Map<String, Object> getDedicateOption() {
+ return dedicateOption;
+ }
+
+ public void setDedicateOption(Map<String, Object> dedicateOption) {
+ this.dedicateOption = dedicateOption;
+ }
+
+ public List<WorkSlot> getWorkingSlots() {
+ return workingSlots;
+ }
+
+ public boolean isDedicated() {
+ return dedicated;
+ }
+
+ public void setDedicated(boolean dedicated) {
+ this.dedicated = dedicated;
+ }
+
+ @org.codehaus.jackson.annotate.JsonIgnore
+ @JsonIgnore
+ public int getQueueSize() {
+ return workingSlots.size();
+ }
+
+// @org.codehaus.jackson.annotate.JsonIgnore
+// @JsonIgnore
+// public void placePolicy(PolicyDefinition pd) {
+// policies.add(pd.getName());
+// }
+
+ public int getNumberOfGroupBolts() {
+ return numberOfGroupBolts;
+ }
+
+ public void setNumberOfGroupBolts(int numberOfGroupBolts) {
+ this.numberOfGroupBolts = numberOfGroupBolts;
+ }
+
+ public Map<String, Integer> getTopoGroupStartIndex() {
+ return topoGroupStartIndex;
+ }
+
+ public void setTopoGroupStartIndex(Map<String, Integer> topoGroupStartIndex) {
+ this.topoGroupStartIndex = topoGroupStartIndex;
+ }
+
+ @org.codehaus.jackson.annotate.JsonIgnore
+ @JsonIgnore
+ public int getTopologyGroupStartIndex(String topo) {
+ if (topoGroupStartIndex.containsKey(topo)) {
+ return this.topoGroupStartIndex.get(topo);
+ }
+ return -1;
+ }
+
+ public String getQueueId() {
+ return queueId;
+ }
+
+ public void setQueueId(String queueId) {
+ this.queueId = queueId;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/internal/Topology.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/internal/Topology.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/internal/Topology.java
new file mode 100644
index 0000000..189e2a5
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/internal/Topology.java
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.coordination.model.internal;
+
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * @since Mar 24, 2016 Logically one unit topology consists of S spouts, G
+ * groupby bolts, A alertBolts normally S=1 Physically each spout is
+ * composed of s spout nodes, each groupby bolt is composed of g groupby
+ * nodes, and each alert bolt is composed of a alert nodes
+ */
+public class Topology {
+
+ private String name;
+ // number of logical nodes
+ private int numOfSpout;
+ private int numOfAlertBolt;
+ private int numOfGroupBolt;
+ private int numOfPublishBolt;
+ private String spoutId;
+ private String pubBoltId;
+ @Deprecated
+ private Set<String> groupNodeIds;
+ @Deprecated
+ private Set<String> alertBoltIds;
+
+ // number of physical nodes for each logic bolt
+ private int spoutParallelism = 1;
+ private int groupParallelism = 1;
+ private int alertParallelism = 1;
+
+ private String clusterName;
+
+ public Topology() {
+ }
+
+ public Topology(String name, int group, int alert) {
+ this.name = name;
+ this.numOfSpout = 1;
+ this.numOfGroupBolt = group;
+ this.numOfAlertBolt = alert;
+ groupNodeIds = new HashSet<String>(group);
+ alertBoltIds = new HashSet<String>(alert);
+
+ spoutParallelism = 1;
+ groupParallelism = 1;
+ alertParallelism = 1;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public int getNumOfSpout() {
+ return numOfSpout;
+ }
+
+ public void setNumOfSpout(int numOfSpout) {
+ this.numOfSpout = numOfSpout;
+ }
+
+ public int getNumOfAlertBolt() {
+ return numOfAlertBolt;
+ }
+
+ public void setNumOfAlertBolt(int numOfAlertBolt) {
+ this.numOfAlertBolt = numOfAlertBolt;
+ }
+
+ public int getNumOfGroupBolt() {
+ return numOfGroupBolt;
+ }
+
+ public void setNumOfGroupBolt(int numOfGroupBolt) {
+ this.numOfGroupBolt = numOfGroupBolt;
+ }
+
+ public String getSpoutId() {
+ return spoutId;
+ }
+
+ public void setSpoutId(String spoutId) {
+ this.spoutId = spoutId;
+ }
+
+ public String getPubBoltId() {
+ return pubBoltId;
+ }
+
+ public void setPubBoltId(String pubBoltId) {
+ this.pubBoltId = pubBoltId;
+ }
+
+ public Set<String> getGroupNodeIds() {
+ return groupNodeIds;
+ }
+
+ public void setGroupNodeIds(Set<String> groupNodeIds) {
+ this.groupNodeIds = groupNodeIds;
+ }
+
+ public Set<String> getAlertBoltIds() {
+ return alertBoltIds;
+ }
+
+ public void setAlertBoltIds(Set<String> alertBoltIds) {
+ this.alertBoltIds = alertBoltIds;
+ }
+
+ public int getNumOfPublishBolt() {
+ return numOfPublishBolt;
+ }
+
+ public void setNumOfPublishBolt(int numOfPublishBolt) {
+ this.numOfPublishBolt = numOfPublishBolt;
+ }
+
+ public int getSpoutParallelism() {
+ return spoutParallelism;
+ }
+
+ public void setSpoutParallelism(int spoutParallelism) {
+ this.spoutParallelism = spoutParallelism;
+ }
+
+ public int getGroupParallelism() {
+ return groupParallelism;
+ }
+
+ public void setGroupParallelism(int groupParallelism) {
+ this.groupParallelism = groupParallelism;
+ }
+
+ public int getAlertParallelism() {
+ return alertParallelism;
+ }
+
+ public void setAlertParallelism(int alertParallelism) {
+ this.alertParallelism = alertParallelism;
+ }
+
+ public String getClusterName() {
+ return clusterName;
+ }
+
+ public void setClusterName(String clusterName) {
+ this.clusterName = clusterName;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/engine/codec/IStreamCodec.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/engine/codec/IStreamCodec.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/engine/codec/IStreamCodec.java
new file mode 100644
index 0000000..3272d28
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/engine/codec/IStreamCodec.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.codec;
+
+import org.apache.eagle.alert.engine.model.StreamEvent;
+
+/**
+ * @since Apr 5, 2016
+ *
+ */
+public interface IStreamCodec {
+
+ StreamEvent decode(byte[] contents);
+
+ byte[] encode(StreamEvent tuple);
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/engine/codec/SherlockEventCodec.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/engine/codec/SherlockEventCodec.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/engine/codec/SherlockEventCodec.java
new file mode 100644
index 0000000..49726e9
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/engine/codec/SherlockEventCodec.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.codec;
+
+import org.apache.eagle.alert.engine.model.StreamEvent;
+
+/**
+ * This is used for event codec.
+ *
+ * @since Apr 5, 2016
+ *
+ */
+public class SherlockEventCodec implements IStreamCodec {
+
+ @Override
+ public StreamEvent decode(byte[] contents) {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public byte[] encode(StreamEvent tuple) {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/engine/codec/SherlockMetricCodec.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/engine/codec/SherlockMetricCodec.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/engine/codec/SherlockMetricCodec.java
new file mode 100644
index 0000000..0e875d8
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/engine/codec/SherlockMetricCodec.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.codec;
+
+import org.apache.eagle.alert.engine.model.StreamEvent;
+
+/**
+ * @since Apr 5, 2016
+ *
+ */
+public class SherlockMetricCodec implements IStreamCodec {
+
+ @Override
+ public StreamEvent decode(byte[] contents) {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public byte[] encode(StreamEvent tuple) {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/PolicyDefinition.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/PolicyDefinition.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/PolicyDefinition.java
new file mode 100644
index 0000000..f97eb2b
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/PolicyDefinition.java
@@ -0,0 +1,199 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.coordinator;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+
+/**
+ * @since Apr 5, 2016
+ *
+ */
+public class PolicyDefinition implements Serializable{
+ private static final long serialVersionUID = 377581499339572414L;
+ // unique identifier
+ private String name;
+ private String description;
+ private List<String> inputStreams = new ArrayList<String>();
+ private List<String> outputStreams = new ArrayList<String>();
+
+ private Definition definition;
+
+ // one stream only have one partition in one policy, since we don't support stream alias
+ private List<StreamPartition> partitionSpec = new ArrayList<StreamPartition>();
+
+ // runtime configuration for policy, these are user-invisible
+ private int parallelismHint = 1;
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public String getDescription() {
+ return description;
+ }
+
+ public void setDescription(String description) {
+ this.description = description;
+ }
+
+ public List<String> getInputStreams() {
+ return inputStreams;
+ }
+
+ public void setInputStreams(List<String> inputStreams) {
+ this.inputStreams = inputStreams;
+ }
+
+ public List<String> getOutputStreams() {
+ return outputStreams;
+ }
+
+ public void setOutputStreams(List<String> outputStreams) {
+ this.outputStreams = outputStreams;
+ }
+
+ public Definition getDefinition() {
+ return definition;
+ }
+
+ public void setDefinition(Definition definition) {
+ this.definition = definition;
+ }
+
+ public List<StreamPartition> getPartitionSpec() {
+ return partitionSpec;
+ }
+
+ public void setPartitionSpec(List<StreamPartition> partitionSpec) {
+ this.partitionSpec = partitionSpec;
+ }
+
+ public void addPartition(StreamPartition par) {
+ this.partitionSpec.add(par);
+ }
+
+ public int getParallelismHint() {
+ return parallelismHint;
+ }
+
+ public void setParallelismHint(int parallelism) {
+ this.parallelismHint = parallelism;
+ }
+
+ @Override
+ public int hashCode() {
+ return new HashCodeBuilder().
+ append(name).
+ append(description).
+ append(inputStreams).
+ append(outputStreams).
+ append(definition).
+ append(partitionSpec).
+// append(parallelismHint).
+ build();
+ }
+
+ @Override
+ public boolean equals(Object that){
+ if(that == this)
+ return true;
+ if(! (that instanceof PolicyDefinition))
+ return false;
+ PolicyDefinition another = (PolicyDefinition)that;
+ if(another.name.equals(this.name) &&
+ another.description.equals(this.description) &&
+ CollectionUtils.isEqualCollection(another.inputStreams, this.inputStreams) &&
+ CollectionUtils.isEqualCollection(another.outputStreams, this.outputStreams) &&
+ another.definition.equals(this.definition) &&
+ CollectionUtils.isEqualCollection(another.partitionSpec, this.partitionSpec)
+// && another.parallelismHint == this.parallelismHint
+ ) {
+ return true;
+ }
+ return false;
+ }
+
+ public static class Definition implements Serializable{
+ private static final long serialVersionUID = -622366527887848346L;
+
+ public String type;
+ public String value;
+
+ public Definition(String type,String value){
+ this.type = type;
+ this.value = value;
+ }
+
+ public Definition() {
+ this.type = null;
+ this.value = null;
+ }
+
+ @Override
+ public int hashCode() {
+ return new HashCodeBuilder().append(type).append(value).build();
+ }
+
+ @Override
+ public boolean equals(Object that){
+ if(that == this)
+ return true;
+ if(!(that instanceof Definition))
+ return false;
+ Definition another = (Definition)that;
+ if(another.type.equals(this.type) &&
+ another.value.equals(this.value))
+ return true;
+ return false;
+ }
+
+ public String getType() {
+ return type;
+ }
+
+ public void setType(String type) {
+ this.type = type;
+ }
+
+ public String getValue() {
+ return value;
+ }
+
+ public void setValue(String value) {
+ this.value = value;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("{type=\"%s\",value=\"%s\"",type,value);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return String.format("{name=\"%s\",definition=%s}",this.getName(),this.getDefinition().toString());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/Publishment.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/Publishment.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/Publishment.java
new file mode 100644
index 0000000..d8b4f28
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/Publishment.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.coordinator;
+
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * @since Apr 11, 2016
+ *
+ */
+public class Publishment {
+
+ private String name;
+ private String type;
+ private List<String> policyIds;
+ private String dedupIntervalMin;
+ private Map<String, String> properties;
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public String getType() {
+ return type;
+ }
+
+ public void setType(String type) {
+ this.type = type;
+ }
+
+ public List<String> getPolicyIds() {
+ return policyIds;
+ }
+
+ public void setPolicyIds(List<String> policyIds) {
+ this.policyIds = policyIds;
+ }
+
+ public String getDedupIntervalMin() {
+ return dedupIntervalMin;
+ }
+
+ public void setDedupIntervalMin(String dedupIntervalMin) {
+ this.dedupIntervalMin = dedupIntervalMin;
+ }
+
+ public Map<String, String> getProperties() {
+ return properties;
+ }
+
+ public void setProperties(Map<String, String> properties) {
+ this.properties = properties;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj instanceof Publishment) {
+ Publishment p = (Publishment) obj;
+ return (Objects.equals(name, p.getName()) &&
+ Objects.equals(type, p.getType()) &&
+ Objects.equals(dedupIntervalMin, p.getDedupIntervalMin()) &&
+ Objects.equals(policyIds, p.getPolicyIds()) &&
+ properties.equals(p.getProperties()));
+ }
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return new HashCodeBuilder()
+ .append(name)
+ .append(type)
+ .append(dedupIntervalMin)
+ .append(policyIds)
+ .append(properties)
+ .build();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/PublishmentType.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/PublishmentType.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/PublishmentType.java
new file mode 100644
index 0000000..f34b971
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/PublishmentType.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.alert.engine.coordinator;
+
+import java.util.Objects;
+
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+
+public class PublishmentType {
+
+ private String type;
+ private String className;
+ private String description;
+ private String fields;
+
+ public String getType() {
+ return type;
+ }
+
+ public void setType(String type) {
+ this.type = type;
+ }
+
+ public String getClassName(){
+ return className;
+ }
+ public void setClassName(String className){
+ this.className = className;
+ }
+
+ public String getDescription(){
+ return description;
+ }
+ public void setDescription(String description){
+ this.description = description;
+ }
+
+ public String getFields() {
+ return fields;
+ }
+
+ public void setFields(String fields) {
+ this.fields = fields;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj instanceof PublishmentType) {
+ PublishmentType p = (PublishmentType) obj;
+ return (Objects.equals(className, p.getClassName()) &&
+ Objects.equals(type, p.type) &&
+ Objects.equals(description, p.getDescription()) &&
+ Objects.equals(fields, p.getFields()));
+ }
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return new HashCodeBuilder()
+ .append(className)
+ .append(type)
+ .append(description)
+ .append(fields)
+ .build();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/StreamColumn.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/StreamColumn.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/StreamColumn.java
new file mode 100644
index 0000000..dc44571
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/StreamColumn.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.coordinator;
+
+import java.io.Serializable;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+
+public class StreamColumn implements Serializable{
+ private static final long serialVersionUID = -5457861313624389106L;
+ private String name;
+ private Type type;
+ private Object defaultValue;
+ private boolean required;
+ private String description;
+
+ public String toString(){
+ return String.format("StreamColumn=name[%s], type=[%s], defaultValue=[%s], required=[%s]", name, type, defaultValue, required);
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public Type getType() {
+ return type;
+ }
+
+ public void setType(Type type) {
+ this.type = type;
+ }
+
+ public Object getDefaultValue() {
+ return defaultValue;
+ }
+
+ public void setDefaultValue(Object defaultValue) {
+ this.defaultValue = defaultValue;
+ }
+
+ public boolean isRequired() {
+ return required;
+ }
+
+ public void setRequired(boolean required) {
+ this.required = required;
+ }
+
+ public String getDescription() {
+ return description;
+ }
+
+ public void setDescription(String description) {
+ this.description = description;
+ }
+
+
+ public enum Type implements Serializable{
+ STRING("string"), INT("int"), LONG("long"), FLOAT("float"), DOUBLE("double"), BOOL("bool"), OBJECT("object");
+
+ private final String name;
+
+ Type(String name){
+ this.name = name;
+ }
+
+ @Override
+ public String toString() {
+ return name;
+ }
+
+ @JsonCreator
+ public static Type getEnumFromValue(String value) {
+ for (Type testEnum : values()) {
+ if (testEnum.name.equals(value)) {
+ return testEnum;
+ }
+ }
+ throw new IllegalArgumentException();
+ }
+ }
+
+ public static class Builder {
+ private StreamColumn column;
+
+ public Builder(){
+ column = new StreamColumn();
+ }
+ public Builder name(String name){
+ column.setName(name);
+ return this;
+ }
+ public Builder type(Type type){
+ column.setType(type);
+ return this;
+ }
+ public Builder defaultValue(Object defaultValue){
+ column.setDefaultValue(defaultValue);
+ return this;
+ }
+ public Builder required(boolean required){
+ column.setRequired(required);
+ return this;
+ }
+
+ public StreamColumn build(){
+ return column;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/StreamDefinition.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/StreamDefinition.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/StreamDefinition.java
new file mode 100644
index 0000000..cd5773a
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/StreamDefinition.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.coordinator;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * This is actually a data source schema.
+ *
+ * @since Apr 5, 2016
+ *
+ */
+public class StreamDefinition implements Serializable{
+ private static final long serialVersionUID = 2352202882328931825L;
+ private String streamId;
+ private String dataSource;
+ private String description;
+ private boolean validate;
+ private boolean timeseries;
+
+ private List<StreamColumn> columns = new ArrayList<StreamColumn>();
+
+ public String toString(){
+ return String.format("StreamDefinition[streamId=%s, dataSource=%s, description=%s, validate=%s, timeseries=%s, columns=%s",
+ streamId,
+ dataSource,
+ description,
+ validate,
+ timeseries,
+ columns);
+ }
+
+ public String getStreamId() {
+ return streamId;
+ }
+
+ public void setStreamId(String streamId) {
+ this.streamId = streamId;
+ }
+
+ public String getDescription() {
+ return description;
+ }
+
+ public void setDescription(String description) {
+ this.description = description;
+ }
+
+ public boolean isValidate() {
+ return validate;
+ }
+
+ public void setValidate(boolean validate) {
+ this.validate = validate;
+ }
+
+ public boolean isTimeseries() {
+ return timeseries;
+ }
+
+ public void setTimeseries(boolean timeseries) {
+ this.timeseries = timeseries;
+ }
+
+ public List<StreamColumn> getColumns() {
+ return columns;
+ }
+
+ public void setColumns(List<StreamColumn> columns) {
+ this.columns = columns;
+ }
+
+ public String getDataSource() {
+ return dataSource;
+ }
+
+ public void setDataSource(String dataSource) {
+ this.dataSource = dataSource;
+ }
+
+ public int getColumnIndex(String column){
+ int i=0;
+ for(StreamColumn col:this.getColumns()){
+ if(col.getName().equals(column)) return i;
+ i++;
+ }
+ return -1;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/StreamPartition.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/StreamPartition.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/StreamPartition.java
new file mode 100644
index 0000000..7b96024
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/StreamPartition.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.coordinator;
+
+import java.io.Serializable;
+import java.util.*;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+
+/**
+ * StreamPartition defines how a data stream is partitioned and sorted
+ * streamId is used for distinguishing different streams which are spawned from the same data source
+ * type defines how to partition data among slots within one slotqueue
+ * columns are fields based on which stream is grouped
+ * sortSpec defines how data is sorted
+ */
+public class StreamPartition implements Serializable {
+ private static final long serialVersionUID = -3361648309136926040L;
+
+ private String streamId;
+ private Type type;
+ private List<String> columns = new ArrayList<>();
+ private StreamSortSpec sortSpec;
+
+ public StreamPartition() {
+ }
+
+ public StreamPartition(StreamPartition o) {
+ this.streamId = o.streamId;
+ this.type = o.type;
+ this.columns = new ArrayList<String>(o.columns);
+ this.sortSpec = o.sortSpec == null ? null : new StreamSortSpec(o.sortSpec);
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == this)
+ return true;
+ if (!(other instanceof StreamPartition)) {
+ return false;
+ }
+ StreamPartition sp = (StreamPartition) other;
+ return Objects.equals(streamId, sp.streamId) && Objects.equals(type, sp.type)
+ && CollectionUtils.isEqualCollection(columns, sp.columns) && Objects.equals(sortSpec, sp.sortSpec);
+ }
+
+ @Override
+ public int hashCode() {
+ return new HashCodeBuilder().append(streamId).append(type).append(columns).append(sortSpec).build();
+ }
+
+ public void setType(Type type) {
+ this.type = type;
+ }
+
+ public Type getType(){
+ return this.type;
+ }
+
+ public enum Type{
+ GLOBAL("GLOBAL",0), GROUPBY("GROUPBY",1), SHUFFLE("SHUFFLE",2);
+ private final String name;
+ private final int index;
+ Type(String name, int index){
+ this.name = name;
+ this.index = index;
+ }
+ @Override
+ public String toString() {
+ return this.name;
+ }
+ public static Type locate(String type){
+ Type _type = _NAME_TYPE.get(type.toUpperCase());
+ if(_type == null)
+ throw new IllegalStateException("Illegal type name: "+type);
+ return _type;
+ }
+
+ public static Type locate(int index){
+ Type _type = _INDEX_TYPE.get(index);
+ if(_type == null)
+ throw new IllegalStateException("Illegal type index: "+index);
+ return _type;
+ }
+
+ private static final Map<String,Type> _NAME_TYPE = new HashMap<>();
+ private static final Map<Integer,Type> _INDEX_TYPE = new TreeMap<>();
+ static {
+ _NAME_TYPE.put(GLOBAL.name,GLOBAL);
+ _NAME_TYPE.put(GROUPBY.name,GROUPBY);
+ _NAME_TYPE.put(SHUFFLE.name,SHUFFLE);
+
+ _INDEX_TYPE.put(GLOBAL.index,GLOBAL);
+ _INDEX_TYPE.put(GROUPBY.index,GLOBAL);
+ _INDEX_TYPE.put(SHUFFLE.index,GLOBAL);
+ }
+ }
+
+ public List<String> getColumns() {
+ return columns;
+ }
+
+ public void setColumns(List<String> columns) {
+ this.columns = columns;
+ }
+
+ public String getStreamId() {
+ return streamId;
+ }
+
+ public void setStreamId(String streamId) {
+ this.streamId = streamId;
+ }
+
+ public StreamSortSpec getSortSpec() {
+ return sortSpec;
+ }
+
+ public void setSortSpec(StreamSortSpec sortSpec) {
+ this.sortSpec = sortSpec;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("StreamPartition[streamId=%s,type=%s,columns=[%s],sortSpec=[%s]]",this.getStreamId(),this.getType(), StringUtils.join(this.getColumns(),","), sortSpec);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/StreamSortSpec.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/StreamSortSpec.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/StreamSortSpec.java
new file mode 100644
index 0000000..ee20f81
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/StreamSortSpec.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.coordinator;
+
+
+import java.io.Serializable;
+
+import org.apache.commons.lang.builder.HashCodeBuilder;
+import org.apache.eagle.alert.utils.TimePeriodUtils;
+import org.joda.time.Period;
+
+/**
+ * streamId is the key
+ */
+public class StreamSortSpec implements Serializable{
+ private static final long serialVersionUID = 3626506441441584937L;
+ private String windowPeriod="";
+ private int windowMargin = 30 * 1000; // 30 seconds by default
+
+ public StreamSortSpec() {}
+
+ public StreamSortSpec(StreamSortSpec spec) {
+ this.windowPeriod = spec.windowPeriod;
+ this.windowMargin = spec.windowMargin;
+ }
+
+ public String getWindowPeriod() {
+ return windowPeriod;
+ }
+
+ public int getWindowPeriodMillis() {
+ if(windowPeriod!=null) {
+ return TimePeriodUtils.getMillisecondsOfPeriod(Period.parse(windowPeriod));
+ } else return 0;
+ }
+
+ public void setWindowPeriod(String windowPeriod) {
+ this.windowPeriod = windowPeriod;
+ }
+ public void setWindowPeriodMillis(int windowPeriodMillis) {
+ this.windowPeriod = Period.millis(windowPeriodMillis).toString();
+ }
+
+ public void setWindowPeriod2(Period period) {
+ this.windowPeriod = period.toString();
+ }
+
+
+ public int getWindowMargin() {
+ return windowMargin;
+ }
+
+ public void setWindowMargin(int windowMargin) {
+ this.windowMargin = windowMargin;
+ }
+
+ @Override
+ public int hashCode(){
+ return new HashCodeBuilder().
+ append(windowPeriod).
+ append(windowMargin).toHashCode();
+ }
+
+ @Override
+ public boolean equals(Object that){
+ if(this == that)
+ return true;
+ if(!(that instanceof StreamSortSpec)){
+ return false;
+ }
+
+ StreamSortSpec another = (StreamSortSpec)that;
+ return
+ another.windowPeriod.equals(this.windowPeriod) &&
+ another.windowMargin == this.windowMargin;
+ }
+
+ @Override
+ public String toString(){
+ return String.format("StreamSortSpec[windowPeriod=%s,windowMargin=%d]",
+ this.getWindowPeriod(),
+ this.getWindowMargin());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/StreamingCluster.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/StreamingCluster.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/StreamingCluster.java
new file mode 100644
index 0000000..6cafb16
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/StreamingCluster.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.coordinator;
+
+import java.util.Map;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/**
+ * @since Apr 5, 2016
+ *
+ */
+public class StreamingCluster {
+ public static enum StreamingType {
+ STORM
+ }
+
+ @JsonProperty
+ private String name;
+ @JsonProperty
+ private String zone;
+ @JsonProperty
+ private StreamingType type;
+ @JsonProperty
+ private String description;
+ /**
+ * key - nimbus for storm
+ */
+ @JsonProperty
+ private Map<String, String> deployments;
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public String getZone() {
+ return zone;
+ }
+
+ public void setZone(String zone) {
+ this.zone = zone;
+ }
+
+ public StreamingType getType() {
+ return type;
+ }
+
+ public void setType(StreamingType type) {
+ this.type = type;
+ }
+
+ public String getDescription() {
+ return description;
+ }
+
+ public void setDescription(String description) {
+ this.description = description;
+ }
+
+ public Map<String, String> getDeployments() {
+ return deployments;
+ }
+
+ public void setDeployments(Map<String, String> deployments) {
+ this.deployments = deployments;
+ }
+
+ public static final String NIMBUS_HOST = "nimbusHost";
+ public static final String NIMBUS_THRIFT_PORT = "nimbusThriftPort";
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/engine/model/AlertStreamEvent.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/engine/model/AlertStreamEvent.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/engine/model/AlertStreamEvent.java
new file mode 100644
index 0000000..f36d3cb
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/engine/model/AlertStreamEvent.java
@@ -0,0 +1,89 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.model;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.eagle.alert.engine.coordinator.PolicyDefinition;
+import org.apache.eagle.alert.engine.coordinator.StreamDefinition;
+import org.apache.eagle.alert.utils.DateTimeUtil;
+
+/**
+ * streamId stands for alert type instead of source event streamId
+ */
+public class AlertStreamEvent extends StreamEvent {
+ private static final long serialVersionUID = 2392131134670106397L;
+
+ // TODO: Keep policy name only instead of policy entity
+ private PolicyDefinition policy;
+ private StreamDefinition schema;
+ private String createdBy;
+ private long createdTime;
+
+ public PolicyDefinition getPolicy() {
+ return policy;
+ }
+
+ public void setPolicy(PolicyDefinition policy) {
+ this.policy = policy;
+ }
+
+
+ public String getPolicyId() {
+ return policy.getName();
+ }
+
+ @Override
+ public String toString() {
+ List<String> dataStrings = new ArrayList<>(this.getData().length);
+ for(Object obj: this.getData()){
+ if(obj!=null) {
+ dataStrings.add(obj.toString());
+ }else{
+ dataStrings.add(null);
+ }
+ }
+ return String.format("AlertStreamEvent[stream=%S,timestamp=%s,data=[%s], policy=%s, createdBy=%s]",
+ this.getStreamId(), DateTimeUtil.millisecondsToHumanDateWithMilliseconds(this.getTimestamp()), StringUtils.join(dataStrings,","),this.getPolicy().getName(),this.getCreatedBy());
+ }
+
+ public String getCreatedBy() {
+ return createdBy;
+ }
+
+ public void setCreatedBy(String createdBy) {
+ this.createdBy = createdBy;
+ }
+
+ public StreamDefinition getSchema() {
+ return schema;
+ }
+
+ public void setSchema(StreamDefinition schema) {
+ this.schema = schema;
+ }
+
+ public long getCreatedTime() {
+ return createdTime;
+ }
+
+ public void setCreatedTime(long createdTime) {
+ this.createdTime = createdTime;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/engine/model/PartitionedEvent.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/engine/model/PartitionedEvent.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/engine/model/PartitionedEvent.java
new file mode 100644
index 0000000..cfed3e2
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/engine/model/PartitionedEvent.java
@@ -0,0 +1,149 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.model;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+import org.apache.eagle.alert.engine.coordinator.StreamPartition;
+
+import backtype.storm.tuple.Tuple;
+
+/**
+ * This is a critical data structure across spout, router bolt and alert bolt
+ * partition[StreamPartition] defines how one incoming data stream is partitioned, sorted
+ * partitionKey[long] is java hash value of groupby fields. The groupby fields are defined in StreamPartition
+ * event[StreamEvent] is actual data
+ */
+public class PartitionedEvent implements Serializable{
+ private static final long serialVersionUID = -3840016190614238593L;
+ private StreamPartition partition;
+ private long partitionKey;
+ private StreamEvent event;
+
+ /**
+ * Used for bolt-internal but not inter-bolts,
+ * will not pass across bolts
+ */
+ private transient Tuple anchor;
+
+ public PartitionedEvent(){
+ this.event = null;
+ this.partition = null;
+ this.partitionKey = 0L;
+ }
+
+ public PartitionedEvent(StreamEvent event, StreamPartition partition, int partitionKey) {
+ this.event = event;
+ this.partition = partition;
+ this.partitionKey = partitionKey;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if(obj == this) return true;
+ if(obj == null) return false;
+ if(obj instanceof PartitionedEvent){
+ PartitionedEvent another = (PartitionedEvent) obj;
+ return !(this.partitionKey != another.getPartitionKey()
+ || !Objects.equals(this.event, another.getEvent())
+ || !Objects.equals(this.partition, another.getPartition())
+ || !Objects.equals(this.anchor, another.anchor));
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ return new HashCodeBuilder()
+ .append(partitionKey)
+ .append(event)
+ .append(partition)
+ .build();
+ }
+
+ public StreamEvent getEvent() {
+ return event;
+ }
+
+ public void setEvent(StreamEvent event) {
+ this.event = event;
+ }
+
+ public StreamPartition getPartition() {
+ return partition;
+ }
+
+ public void setPartition(StreamPartition partition) {
+ this.partition = partition;
+ }
+
+ public void setPartitionKey(long partitionKey){
+ this.partitionKey = partitionKey;
+ }
+
+ public long getPartitionKey(){
+ return this.partitionKey;
+ }
+
+ public String toString(){
+ return String.format("PartitionedEvent[partition=%s,event=%s,key=%s", partition, event,partitionKey);
+ }
+
+ public long getTimestamp() {
+ return (event != null) ? event.getTimestamp() : 0L;
+ }
+
+ public String getStreamId(){
+ return (event != null) ? event.getStreamId() : null;
+ }
+
+ public Object[] getData(){
+ return event!=null ? event.getData() : null;
+ }
+
+ public boolean isSortRequired(){
+ return isPartitionRequired() && this.getPartition().getSortSpec()!=null;
+ }
+
+ public boolean isPartitionRequired(){
+ return this.getPartition()!=null;
+ }
+
+ public PartitionedEvent copy() {
+ PartitionedEvent copied = new PartitionedEvent();
+ copied.setEvent(this.getEvent());
+ copied.setPartition(this.partition);
+ copied.setPartitionKey(this.partitionKey);
+ return copied;
+ }
+
+ public Tuple getAnchor() {
+ return anchor;
+ }
+
+ public void setAnchor(Tuple anchor) {
+ this.anchor = anchor;
+ }
+
+ public PartitionedEvent withAnchor(Tuple tuple){
+ this.setAnchor(tuple);
+ return this;
+ }
+}
\ No newline at end of file
[07/19] incubator-eagle git commit: EAGLE-324: Init branch-v0.5
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/storm/kafka/KafkaSpoutWrapper.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/storm/kafka/KafkaSpoutWrapper.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/storm/kafka/KafkaSpoutWrapper.java
new file mode 100644
index 0000000..1cef187
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/storm/kafka/KafkaSpoutWrapper.java
@@ -0,0 +1,111 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+package storm.kafka;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.eagle.alert.coordination.model.SpoutSpec;
+import org.apache.eagle.alert.engine.coordinator.StreamDefinition;
+import org.apache.eagle.alert.engine.spout.ISpoutSpecLCM;
+import org.apache.eagle.alert.engine.spout.SpoutOutputCollectorWrapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import backtype.storm.Config;
+import backtype.storm.spout.SpoutOutputCollector;
+import backtype.storm.task.TopologyContext;
+
+/**
+ * NOTE!!!!! This class copy/paste some code from storm.kafka.KafkaSpout to make sure it can support one process to hold multiple
+ * KafkaSpout
+ *
+ * this collectorWrapper provides the following capabilities:
+ * 1. inject customized collector collectorWrapper, so framework can control traffic routing
+ * 2. listen to topic to stream metadata change and pass that to customized collector collectorWrapper
+ * 3. return current streams for this topic
+ */
+public class KafkaSpoutWrapper extends KafkaSpout implements ISpoutSpecLCM {
+ private static final long serialVersionUID = 5507693757424351306L;
+ @SuppressWarnings("unused")
+ private static final Logger LOG = LoggerFactory.getLogger(KafkaSpoutWrapper.class);
+ private KafkaSpoutMetric kafkaSpoutMetric;
+
+ public KafkaSpoutWrapper(SpoutConfig spoutConf, KafkaSpoutMetric kafkaSpoutMetric) {
+ super(spoutConf);
+ this.kafkaSpoutMetric = kafkaSpoutMetric;
+ }
+
+ private SpoutOutputCollectorWrapper collectorWrapper;
+
+ @SuppressWarnings({ "unchecked", "rawtypes" })
+ @Override
+ public void open(Map conf, final TopologyContext context, final SpoutOutputCollector collector) {
+ String topologyInstanceId = context.getStormId();
+ ////// !!!! begin copying code from storm.kafka.KafkaSpout to here
+ _collector = collector;
+
+ Map stateConf = new HashMap(conf);
+ List<String> zkServers = _spoutConfig.zkServers;
+ if (zkServers == null) {
+ zkServers = (List<String>) conf.get(Config.STORM_ZOOKEEPER_SERVERS);
+ }
+ Integer zkPort = _spoutConfig.zkPort;
+ if (zkPort == null) {
+ zkPort = ((Number) conf.get(Config.STORM_ZOOKEEPER_PORT)).intValue();
+ }
+ stateConf.put(Config.TRANSACTIONAL_ZOOKEEPER_SERVERS, zkServers);
+ stateConf.put(Config.TRANSACTIONAL_ZOOKEEPER_PORT, zkPort);
+ stateConf.put(Config.TRANSACTIONAL_ZOOKEEPER_ROOT, _spoutConfig.zkRoot);
+ _state = new ZkState(stateConf);
+
+ _connections = new DynamicPartitionConnections(_spoutConfig, KafkaUtils.makeBrokerReader(conf, _spoutConfig));
+
+ // using TransactionalState like this is a hack
+ int totalTasks = context.getComponentTasks(context.getThisComponentId()).size();
+ if (_spoutConfig.hosts instanceof StaticHosts) {
+ _coordinator = new StaticCoordinator(_connections, conf, _spoutConfig, _state, context.getThisTaskIndex(), totalTasks, topologyInstanceId);
+ } else {
+ _coordinator = new ZkCoordinator(_connections, conf, _spoutConfig, _state, context.getThisTaskIndex(), totalTasks, topologyInstanceId);
+ }
+
+ ////// !!!! end copying code from storm.kafka.KafkaSpout to here
+
+ // add new topic to metric
+ KafkaSpoutMetric.KafkaSpoutMetricContext metricContext = new KafkaSpoutMetric.KafkaSpoutMetricContext();
+ metricContext._connections = _connections;
+ metricContext._coordinator = _coordinator;
+ metricContext._spoutConfig = _spoutConfig;
+ kafkaSpoutMetric.addTopic(_spoutConfig.topic, metricContext);
+
+ this.collectorWrapper = (SpoutOutputCollectorWrapper)collector;
+ }
+
+ @Override
+ public void update(SpoutSpec metadata, Map<String, StreamDefinition> sds){
+ collectorWrapper.update(metadata, sds);
+ }
+
+ @Override
+ public void close(){
+ super.close();
+ kafkaSpoutMetric.removeTopic(_spoutConfig.topic);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/resources/ALERT_DEFAULT.vm
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/resources/ALERT_DEFAULT.vm b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/resources/ALERT_DEFAULT.vm
new file mode 100644
index 0000000..2da6288
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/resources/ALERT_DEFAULT.vm
@@ -0,0 +1,267 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+ <head>
+ <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+ <meta name="viewport" content="width=device-width"/>
+ <style>
+ body {
+ width:100% !important;
+ min-width: 100%;
+ -webkit-text-size-adjust:100%;
+ -ms-text-size-adjust:100%;
+ margin:0;
+ padding:0;
+ }
+
+ table {
+ border-spacing: 0;
+ border-collapse: collapse;
+ }
+
+ table th,
+ table td {
+ padding: 3px 0 3px 0;
+ }
+
+ .body {
+ width: 100%;
+ }
+
+ p,a,h1,h2,h3,ul,ol,li {
+ font-family: Helvetica, Arial, sans-serif;
+ font-weight: normal;
+ margin: 0;
+ padding: 0;
+ }
+ p {
+ font-size: 14px;
+ line-height: 19px;
+ }
+ a {
+ color: #3294b1;
+ }
+ h1 {
+ font-size: 36px;
+ margin: 15px 0 5px 0;
+ }
+ h2 {
+ font-size: 32px;
+ }
+ h3 {
+ font-size: 28px;
+ }
+
+ ul,ol {
+ margin: 0 0 0 25px;
+ padding: 0;
+ }
+
+ .btn {
+ background: #2ba6cb !important;
+ border: 1px solid #2284a1;
+ padding: 10px 20px 10px 20px;
+ text-align: center;
+ }
+ .btn:hover {
+ background: #2795b6 !important;
+ }
+ .btn a {
+ color: #FFFFFF;
+ text-decoration: none;
+ font-weight: bold;
+ padding: 10px 20px 10px 20px;
+ }
+
+ .tableBordered {
+ border-top: 1px solid #b9e5ff;
+ }
+ .tableBordered th {
+ background: #ECF8FF;
+ }
+ .tableBordered th p {
+ font-weight: bold;
+ color: #3294b1;
+ }
+ .tableBordered th,
+ .tableBordered td {
+ color: #333333;
+ border-bottom: 1px solid #b9e5ff;
+ text-align: center;
+ padding-bottom: 5px;
+ }
+
+ .panel {
+ height: 100px;
+ }
+ </style>
+ </head>
+ <body>
+ #set ( $elem = $alertList[0] )
+ #set ( $alertUrl = $elem["alertDetailUrl"] )
+ #set ( $policyUrl = $elem["policyDetailUrl"] )
+ <table class="body">
+ <tr>
+ <td align="center" valign="top" style="background: #999999; padding: 0 0 0 0;">
+ <!-- Header -->
+ <table width="580">
+ <tr>
+ <td style="padding: 0 0 0 0;" align="left" >
+ <p style="color:#FFFFFF;font-weight: bold; font-size: 22px">UMP Alerts</p>
+ </td>
+ </tr>
+ </table>
+ </td>
+ </tr>
+
+ <tr>
+ <td align="center" valign="top">
+ <!-- Eagle Body -->
+ <table width="580">
+ <tr>
+ <!-- Title -->
+ <td align="center">
+ <h1>$elem["streamId"] Alert Detected</h1>
+ </td>
+ </tr>
+ <tr>
+ <!-- Time -->
+ <td>
+ <table width="580">
+ <tr>
+ <td>
+ <p><b>Detected Time: $elem["alertTime"]</b></p>
+ </td>
+ #set ( $severity = $elem["severity"] )
+ #if (!$severity || ("$severity" == ""))
+ #set ( $elem["severity"] = "WARNING")
+ #end
+ <td align="right">
+ <p><b>
+ Severity:
+ #if ($elem["severity"] == "WARNING")
+ <span>$elem["severity"]</span>
+ #else
+ <span style="color: #FF0000;">$elem["severity"]</span>
+ #end
+ </b></p>
+ </td>
+ </tr>
+ </table>
+ </td>
+ </tr>
+ <tr>
+ <!-- Description -->
+ <td valign="top" style="background: #ECF8FF; border: 1px solid #b9e5ff; padding: 10px 10px 12px 10px;">
+ <p>$elem["alertMessage"]</p>
+ </td>
+ </tr>
+ <tr>
+ <!-- View Detail -->
+ <td align="center" style="padding: 10px 0 0 0;">
+ <table width="580">
+ <tr>
+ <td class="btn">
+ <a href="$alertUrl">View Alert Details on Eagle Web</a>
+ </td>
+ </tr>
+ </table>
+ </td>
+ </tr>
+ <tr>
+ <!-- Basic Information -->
+ <td style="padding: 20px 0 0 0;">
+ <p><b>Basic Information:</b></p>
+ </td>
+ </tr>
+ <tr>
+ <!-- Basic Information Content -->
+ <td>
+ <table class="tableBordered" width="580">
+ <tr>
+ <th>
+ <p>Policy Name</p>
+ </th>
+ <th>
+ <p>Data Source</p>
+ </th>
+ </tr>
+ <tr>
+ <td>
+ <p>$elem["policyId"]</p>
+ </td>
+ <td>
+ <p>$elem["streamId"]</p>
+ </td>
+ </tr>
+ <tr>
+
+ <th>
+ <p>Creator</p>
+ </th>
+ <th>
+ <p>Severity</p>
+ </th>
+ </tr>
+ <tr>
+ <td>
+ <p>$elem["creator"]</p>
+ </td>
+ <td>
+ <p>$elem["severity"]</p>
+ </td>
+ </tr>
+ </table>
+ </td>
+ </tr>
+ <tr>
+ <!-- View Detail -->
+ <td align="center" style="padding: 10px 0 0 0;">
+ <table width="580">
+ <tr>
+ <td class="btn">
+ <a href="$policyUrl">View Policy Details on Eagle Web</a>
+ </td>
+ </tr>
+ </table>
+ </td>
+ </tr>
+ <tr>
+ <!-- Actions Required -->
+ <td style="padding: 20px 0 0 0;">
+ <p><b>Actions Required:</b></p>
+ </td>
+ </tr>
+ <tr>
+ <!-- Possible Root Causes Content -->
+ <td class="panel" valign="top" style="background: #F4F4F4; border: 1px solid #AAAAAA; padding: 10px 10px 12px 10px;">
+ <p> $elem["streamId"] alert found, please check.</p>
+ </td>
+ </tr>
+ <tr>
+ <!-- Copyright -->
+ <td align="center">
+ <p><a href="<Eagle-Host>/alerts/alertlist.html">UMP Alert Engine</a></p>
+ </td>
+ </tr>
+ </table>
+ </td>
+ </tr>
+ </table>
+ </body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/resources/application.conf
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/resources/application.conf b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/resources/application.conf
new file mode 100644
index 0000000..b9308ef
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/resources/application.conf
@@ -0,0 +1,72 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+{
+ "topology" : {
+ "name" : "alertUnitTopology_1",
+ "numOfTotalWorkers" : 2,
+ "numOfSpoutTasks" : 1,
+ "numOfRouterBolts" : 4,
+ "numOfAlertBolts" : 10,
+ "numOfPublishTasks" : 1,
+ "messageTimeoutSecs": 3600,
+ "localMode" : "true"
+ },
+ "spout" : {
+ "kafkaBrokerZkQuorum": "localhost:2181",
+ "kafkaBrokerZkBasePath": "/brokers",
+ "stormKafkaUseSameZkQuorumWithKafkaBroker": true,
+ "stormKafkaTransactionZkQuorum": "",
+ "stormKafkaTransactionZkPath": "/consumers",
+ "stormKafkaEagleConsumer": "eagle_consumer",
+ "stormKafkaStateUpdateIntervalMs": 2000,
+ "stormKafkaFetchSizeBytes": 1048586,
+ },
+ "zkConfig" : {
+ "zkQuorum" : "localhost:2181",
+ "zkRoot" : "/alert",
+ "zkSessionTimeoutMs" : 10000,
+ "connectionTimeoutMs" : 10000,
+ "zkRetryTimes" : 3,
+ "zkRetryInterval" : 3000
+ },
+ "dynamicConfigSource" : {
+ "initDelayMillis": 3000,
+ "delayMillis" : 10000
+ },
+ "metadataService": {
+ "context" : "/api",
+ "host" : "localhost",
+ "port" : 8080
+ },
+ "coordinatorService": {
+ "host": "localhost",
+ "port": 9090,
+ "context" : "/api"
+ }
+ "metric":{
+ "sink": {
+ "kafka": {
+ "topic": "alert_metric"
+ "bootstrap.servers": "localhost:6667"
+ }
+ "stdout": {}
+ // "elasticsearch": {
+ // "hosts": ["localhost:9200"]
+ // "index": "alert_metric"
+ // "timestampField": "timestamp"
+ // }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/resources/eagle.siddhiext
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/resources/eagle.siddhiext b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/resources/eagle.siddhiext
new file mode 100644
index 0000000..506bad9
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/resources/eagle.siddhiext
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# externalTimeBatch=org.apache.eagle.policy.siddhi.extension.ExternalTimeBatchWindowProcessor
+collect=org.apache.eagle.alert.engine.siddhi.extension.AttributeCollectAggregator
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/resources/log4j.properties b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/resources/log4j.properties
new file mode 100644
index 0000000..af99e2c
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/resources/log4j.properties
@@ -0,0 +1,26 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+log4j.rootLogger=INFO, stdout
+
+# standard output
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %p [%t] %c{2}[%L]: %m%n
+
+##log4j.logger.org.apache.eagle.alert.engine.spout.CorrelationSpout=DEBUG
+log4j.logger.org.apache.eagle.alert.engine.spout.SpoutOutputCollectorWrapper=DEBUG
+log4j.logger.org.apache.eagle.alert.engine.router.impl.StreamRouterBoltOutputCollector=DEBUG
+log4j.logger.org.apache.eagle.alert.engine.sorter.impl.StreamSortWindowHandlerImpl=DEBUG
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/e2e/CoordinatorClient.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/e2e/CoordinatorClient.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/e2e/CoordinatorClient.java
new file mode 100644
index 0000000..aebf3b5
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/e2e/CoordinatorClient.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.e2e;
+
+import java.io.Closeable;
+import java.io.IOException;
+
+import javax.ws.rs.core.MediaType;
+
+import org.codehaus.jackson.jaxrs.JacksonJsonProvider;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.sun.jersey.api.client.Client;
+import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.api.client.config.ClientConfig;
+import com.sun.jersey.api.client.config.DefaultClientConfig;
+import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
+import com.typesafe.config.Config;
+
+/**
+ * @since May 9, 2016
+ *
+ */
+public class CoordinatorClient implements Closeable {
+
+ @SuppressWarnings("unused")
+ private static final Logger LOG = LoggerFactory.getLogger(CoordinatorClient.class);
+
+ private static final String EAGLE_COORDINATOR_SERVICE_CONTEXT = "coordinatorService.context";
+ private static final String EAGLE_COORDINATOR_SERVICE_PORT = "coordinatorService.port";
+ private static final String EAGLE_COORDINATOR_SERVICE_HOST = "coordinatorService.host";
+ private static final String COORDINATOR_SCHEDULE_API = "/coordinator/build";
+
+ private String host;
+ private int port;
+ private String context;
+ private transient Client client;
+ private String basePath;
+
+ public CoordinatorClient(Config config) {
+ this(config.getString(EAGLE_COORDINATOR_SERVICE_HOST), config.getInt(EAGLE_COORDINATOR_SERVICE_PORT), config
+ .getString(EAGLE_COORDINATOR_SERVICE_CONTEXT));
+ basePath = buildBasePath();
+ }
+
+ public CoordinatorClient(String host, int port, String context) {
+ this.host = host;
+ this.port = port;
+ this.context = context;
+ this.basePath = buildBasePath();
+ ClientConfig cc = new DefaultClientConfig();
+ cc.getProperties().put(DefaultClientConfig.PROPERTY_CONNECT_TIMEOUT, 60 * 1000);
+ cc.getProperties().put(DefaultClientConfig.PROPERTY_READ_TIMEOUT, 60 * 1000);
+ cc.getClasses().add(JacksonJsonProvider.class);
+ cc.getProperties().put(URLConnectionClientHandler.PROPERTY_HTTP_URL_CONNECTION_SET_METHOD_WORKAROUND, true);
+ this.client = Client.create(cc);
+ client.addFilter(new com.sun.jersey.api.client.filter.GZIPContentEncodingFilter());
+ }
+
+ private String buildBasePath() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("http://");
+ sb.append(host);
+ sb.append(":");
+ sb.append(port);
+ sb.append(context);
+ return sb.toString();
+ }
+
+ public String schedule() {
+ WebResource r = client.resource(basePath + COORDINATOR_SCHEDULE_API);
+ return r.accept(MediaType.APPLICATION_JSON_TYPE).type(MediaType.APPLICATION_JSON).post(String.class);
+ }
+
+ @Override
+ public void close() throws IOException {
+ this.client.destroy();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/e2e/Integration1.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/e2e/Integration1.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/e2e/Integration1.java
new file mode 100644
index 0000000..057aa73
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/e2e/Integration1.java
@@ -0,0 +1,236 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.e2e;
+
+import java.util.List;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+import org.apache.eagle.alert.coordination.model.Kafka2TupleMetadata;
+import org.apache.eagle.alert.coordination.model.internal.Topology;
+import org.apache.eagle.alert.engine.UnitTopologyMain;
+import org.apache.eagle.alert.engine.coordinator.PolicyDefinition;
+import org.apache.eagle.alert.engine.coordinator.Publishment;
+import org.apache.eagle.alert.engine.coordinator.StreamDefinition;
+import org.apache.eagle.alert.service.IMetadataServiceClient;
+import org.apache.eagle.alert.service.MetadataServiceClientImpl;
+import org.junit.Assert;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import backtype.storm.utils.Utils;
+
+import com.fasterxml.jackson.databind.JavaType;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.type.CollectionType;
+import com.fasterxml.jackson.databind.type.SimpleType;
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+
+
+/**
+ * Case of simple
+ *
+ * @since May 8, 2016
+ *
+ */
+public class Integration1 {
+ private static final Logger LOG = LoggerFactory.getLogger(Integration1.class);
+ private static final ObjectMapper om = new ObjectMapper();
+
+ public static void main(String[] args) throws Exception {
+ Integration1 inte = new Integration1();
+ inte.args = args;
+ inte.test_simple_threshhold();
+ }
+
+ private String[] args;
+ private ExecutorService executors = Executors.newFixedThreadPool(5);
+
+ /**
+ * Assumption:
+ * <p>
+ * start metadata service 8080, better in docker
+ * <p>
+ * start coordinator service 9090, better in docker
+ * <p>
+ * datasources : perfmon_datasource
+ * <p>
+ * stream: perfmon_cpu
+ * <p>
+ * policy : perfmon_cpu_host_check / perfmon_cpu_pool_check
+ * <p>
+ * Create topic
+ * liasu@xxx:~$ $KAFKA_HOME/bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic perfmon_metrics
+ * <p>
+ *
+ * @throws InterruptedException
+ */
+ @Ignore
+ @Test
+ public void test_simple_threshhold() throws Exception {
+ System.setProperty("config.resource", "/application-integration.conf");
+ ConfigFactory.invalidateCaches();
+ Config config = ConfigFactory.load();
+
+ System.out.println("loading metadatas...");
+ loadMetadatas("/", config);
+ System.out.println("loading metadatas done!");
+
+ executors.submit(() -> SampleClient1.main(args));
+
+ executors.submit(() -> UnitTopologyMain.main(args));
+
+ Utils.sleep(1000 * 5l);
+ while (true) {
+ proactive_schedule(config);
+
+ Utils.sleep(1000 * 60l * 5);
+ }
+ }
+
+ /**
+ * Test only run expected when there is a missed config in the config file. mark as ignored
+ * @throws InterruptedException
+ * @throws ExecutionException
+ */
+ @Ignore
+ @Test(expected = ExecutionException.class)
+ public void test_typesafe_config() throws InterruptedException, ExecutionException {
+ System.setProperty("config.resource", "/application-integration.conf");
+ ConfigFactory.invalidateCaches();
+ Future<?> f = executors.submit(() -> {
+ UnitTopologyMain.main(null);
+ });
+
+ f.get();
+ }
+
+// @Test
+// private void makeSureTopic() {
+// System.setProperty("config.resource", "/application-integration.conf");
+// ConfigFactory.invalidateCaches();
+// Config config = ConfigFactory.load();
+// ZKConfig zkconfig = ZKConfigBuilder.getZKConfig(config);
+//
+// CuratorFramework curator = CuratorFrameworkFactory.newClient(
+// zkconfig.zkQuorum,
+// zkconfig.zkSessionTimeoutMs,
+// zkconfig.connectionTimeoutMs,
+// new RetryNTimes(zkconfig.zkRetryTimes, zkconfig.zkRetryInterval)
+// );
+// }
+
+ public static void proactive_schedule(Config config) throws Exception {
+
+ try (CoordinatorClient cc = new CoordinatorClient(config)) {
+ try {
+ String resp = cc.schedule();
+ LOG.info("schedule return : {} ", resp);
+ } catch (Exception e) {
+ LOG.error("failed to call schedule!", e);
+ }
+ }
+ }
+
+ public static void loadMetadatas(String base, Config config) throws Exception {
+ IMetadataServiceClient client = new MetadataServiceClientImpl(config);
+ client.clear();
+
+ List<Kafka2TupleMetadata> metadata = loadEntities(base + "datasources.json", Kafka2TupleMetadata.class);
+ for (Kafka2TupleMetadata k : metadata) {
+ client.addDataSource(k);
+ }
+
+ List<PolicyDefinition> policies = loadEntities(base + "policies.json", PolicyDefinition.class);
+ for (PolicyDefinition p : policies) {
+ client.addPolicy(p);
+ }
+
+ List<Publishment> pubs = loadEntities(base + "publishments.json", Publishment.class);
+ for (Publishment pub : pubs) {
+ client.addPublishment(pub);
+ }
+
+ List<StreamDefinition> defs = loadEntities(base + "streamdefinitions.json", StreamDefinition.class);
+ for (StreamDefinition def : defs) {
+ client.addStreamDefinition(def);
+ }
+
+ List<Topology> topos = loadEntities(base + "topologies.json", Topology.class);
+ for (Topology t : topos) {
+ client.addTopology(t);
+ }
+
+ client.close();
+ }
+
+ public static <T> List<T> loadEntities(String path, Class<T> tClz) throws Exception {
+ JavaType type = CollectionType.construct(List.class, SimpleType.construct(tClz));
+ List<T> l = om.readValue(Integration1.class.getResourceAsStream(path), type);
+ return l;
+ }
+
+ /**
+ * <p>
+ * {"name":"xxx","numOfSpout":1,"numOfAlertBolt":3,"numOfGroupBolt":2,
+ * "spoutId"
+ * :"xxx-spout","groupNodeIds":["xxx-grp"],"alertBoltIds":["xxx-bolt"
+ * ],"pubBoltId":"xxx-pubBolt","spoutParallelism":1,"groupParallelism":1,
+ * "alertParallelism":1}
+ * <p>
+ *
+ * @throws Exception
+ */
+ @Ignore
+ @Test
+ public void testJson() throws Exception {
+ {
+ JavaType type = CollectionType.construct(List.class, SimpleType.construct(Topology.class));
+ List<Topology> l = om.readValue(Integration1.class.getResourceAsStream("/topologies.json"),
+ type);
+ Topology t = (Topology) l.get(0);
+
+ Assert.assertEquals(4, t.getGroupNodeIds().size());
+ Assert.assertEquals(10, t.getAlertBoltIds().size());
+ }
+
+ {
+ JavaType type = CollectionType.construct(List.class, SimpleType.construct(Publishment.class));
+ // publishment
+ List<Publishment> l = om.readValue(Integration1.class.getResourceAsStream("/publishments.json"), type);
+ Publishment p = l.get(0);
+ Assert.assertEquals("KAFKA", p.getType());
+ }
+
+ checkAll("/");
+ checkAll("/correlation/");
+ }
+
+ private void checkAll(String base) throws Exception {
+ loadEntities(base + "datasources.json", Kafka2TupleMetadata.class);
+ loadEntities(base + "policies.json", PolicyDefinition.class);
+ loadEntities(base + "publishments.json", Publishment.class);
+ loadEntities(base + "streamdefinitions.json", StreamDefinition.class);
+ loadEntities(base + "topologies.json", Topology.class);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/e2e/Integration2.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/e2e/Integration2.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/e2e/Integration2.java
new file mode 100644
index 0000000..7ea0e7e
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/e2e/Integration2.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.e2e;
+
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import org.apache.eagle.alert.engine.UnitTopologyMain;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.wso2.siddhi.core.ExecutionPlanRuntime;
+import org.wso2.siddhi.core.SiddhiManager;
+import org.wso2.siddhi.core.event.Event;
+import org.wso2.siddhi.core.stream.input.InputHandler;
+import org.wso2.siddhi.core.stream.output.StreamCallback;
+import org.wso2.siddhi.core.util.EventPrinter;
+
+import backtype.storm.utils.Utils;
+
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+
+/**
+ * @since May 10, 2016
+ *
+ */
+public class Integration2 {
+
+ private ExecutorService executors = Executors.newFixedThreadPool(5);
+
+ /**
+ * <pre>
+ * Create topic
+ * liasu@xxxx:~$ $KAFKA_HOME/bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic eslogs
+ * liasu@xxxx:~$ $KAFKA_HOME/bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic bootfailures
+ * </pre>
+ *
+ * @param args
+ */
+ public static void main(String[] args) throws Exception {
+ Integration2 inte = new Integration2();
+ inte.args = args;
+ inte.test_start();
+ }
+
+ private String[] args;
+
+ @Ignore
+ @Test
+ public void test_start() throws Exception {
+ System.setProperty("config.resource", "/correlation/application-integration-2.conf");
+ ConfigFactory.invalidateCaches();
+ Config config = ConfigFactory.load();
+ Integration1.loadMetadatas("/correlation/", config);
+
+ executors.submit(() -> UnitTopologyMain.main(args));
+
+ executors.submit(() -> SampleClient2.main(args));
+
+ Utils.sleep(1000 * 5l);
+ while (true) {
+ Integration1.proactive_schedule(config);
+ Utils.sleep(1000 * 60l * 5);
+ }
+ }
+
+ @Test @Ignore
+ public void test3() throws Exception {
+ SiddhiManager sm = new SiddhiManager();
+ String s1 = " define stream esStream(instanceUuid string, timestamp long, logLevel string, message string, reqId string, host string, component string); ";
+ s1 += " define stream ifStream(instanceUuid string, timestamp long, reqId string, message string, host string); ";
+ s1 += "from esStream#window.externalTime(timestamp, 20 min) as a join ifStream#window.externalTime(timestamp, 5 min) as b on a.instanceUuid == b.instanceUuid within 10 min select logLevel, a.host as aHost, a.component, a.message as logMessage, b.message as failMessage, a.timestamp as t1, b.timestamp as t2, b.host as bHost, count(1) as errorCount group by component insert into log_stream_join_output; ";
+ ExecutionPlanRuntime epr = sm.createExecutionPlanRuntime(s1);
+
+ epr.addCallback("log_stream_join_output", new StreamCallback() {
+ @Override
+ public void receive(Event[] arg0) {
+ System.out.println("join result!");
+ EventPrinter.print(arg0);
+ }
+ });
+
+ InputHandler input1 = epr.getInputHandler("esStream");
+ InputHandler input2 = epr.getInputHandler("ifStream");
+
+ epr.start();
+
+ long base = 1462880695837l;
+
+ while (true) {
+ sendEvent(input1, input2, base);
+
+ base = base + 3000;
+
+ Utils.sleep(3000);
+ }
+
+ }
+
+ private void sendEvent(InputHandler input1, InputHandler input2, long base) throws InterruptedException {
+ {
+ Event e = new Event();
+ e.setTimestamp(base);
+ e.setData(new Object[] {
+ "instance-guid-c2a1c926-b590-418e-bf57-41469d7891fa",
+ base,
+ "ERROR",
+ "NullPointException",
+ "req-id-82dab92c-9e45-4ad8-8793-96e912831f00",
+ "nova.host",
+ "NOVA"
+ });
+ input1.send(e);
+ }
+
+ {
+ Event e = new Event();
+ e.setTimestamp(base);
+ e.setData(new Object[] {"instance-guid-c2a1c926-b590-418e-bf57-41469d7891fa",
+ base,
+ "req-id-82dab92c-9e45-4ad8-8793-96e912831f00",
+ "boot failure for when try start the given vm!",
+ "boot-vm-data-center.corp.com"});
+ input2.send(e);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/e2e/SampleClient1.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/e2e/SampleClient1.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/e2e/SampleClient1.java
new file mode 100644
index 0000000..348bf78
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/e2e/SampleClient1.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.e2e;
+
+import java.util.Properties;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.eagle.alert.utils.JsonUtils;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import backtype.storm.utils.Utils;
+
+/**
+ * @since May 9, 2016
+ *
+ */
+public class SampleClient1 {
+ @SuppressWarnings("unused")
+ private static final Logger LOG = LoggerFactory.getLogger(SampleClient1.class);
+
+ private static final String PERFMON_CPU_STREAM = "perfmon_cpu_stream";
+ private static final String PERFMON_MEM_STREAM = "perfmon_mem_stream";
+
+// private static int hostIndx = 1;
+ private static String hostTemp = "host-000%d.datacenter.corp.com";
+
+ /**
+ * <pre>
+ * {"host": "", "timestamp" : "", "metric" : "", "pool": "", "value": 1.0, "colo": "phx"}
+ * </pre>
+ */
+ public static class Entity {
+ public String host;
+ public long timestamp;
+ public String metric;
+ public String pool;
+ public double value;
+ public String colo;
+ }
+
+ public static void main(String[] args) {
+ long base = System.currentTimeMillis();
+ AtomicLong msgCount = new AtomicLong();
+
+ try (KafkaProducer<String, String> proceduer = createProceduer()) {
+ while (true) {
+ int hostIndex = 6;
+ for (int i = 0; i < hostIndex; i++) {
+ base = send_metric(base, proceduer, PERFMON_CPU_STREAM, i);
+ msgCount.incrementAndGet();
+ base = send_metric(base, proceduer, PERFMON_MEM_STREAM, i);
+ msgCount.incrementAndGet();
+ }
+
+ if ((msgCount.get() % 600) == 0) {
+ System.out.println("send 600 CPU/MEM metric!");
+ }
+
+ Utils.sleep(3000);
+ }
+ }
+ }
+
+ private static long send_metric(long base, KafkaProducer<String, String> proceduer, String stream, int hostIndex) {
+
+ Pair<Long, String> pair = createEntity(base, stream, hostIndex);
+ base = pair.getKey();
+ ProducerRecord<String, String> record = new ProducerRecord<String, String>("perfmon_metrics",
+ pair.getRight());
+ proceduer.send(record);
+ return base;
+ }
+
+ private static Pair<Long, String> createEntity(long base, String stream, int hostIndex) {
+ // TODO : add randomization
+ Entity e = new Entity();
+ e.colo = "LVS";
+ e.host = String.format(hostTemp, hostIndex);
+ if (hostIndex < 3) {
+ e.pool = "hadoop-eagle-prod";
+ } else {
+ e.pool = "raptor-pool1";
+ }
+ e.timestamp = base;
+ e.metric = stream;
+ e.value = 92.0;
+
+ base = base + 1000;
+
+ return Pair.of(base, JsonUtils.writeValueAsString(e));
+ }
+
+ public static KafkaProducer<String, String> createProceduer() {
+
+ Properties configMap = new Properties();
+ // String broker_list = zkconfig.zkQuorum;
+ // TODO: replace boot strap servers with new workable server
+ configMap.put("bootstrap.servers", "localhost:9092");
+ // configMap.put("metadata.broker.list", broker_list);
+ configMap.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
+ configMap.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
+ configMap.put("request.required.acks", "1");
+ configMap.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
+ configMap.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
+ KafkaProducer<String, String> proceduer = new KafkaProducer<String, String>(configMap);
+ return proceduer;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/e2e/SampleClient2.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/e2e/SampleClient2.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/e2e/SampleClient2.java
new file mode 100644
index 0000000..06148cc
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/e2e/SampleClient2.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.e2e;
+
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.eagle.alert.utils.JsonUtils;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.ProducerRecord;
+
+import backtype.storm.utils.Utils;
+
+/**
+ * @since May 10, 2016
+ *
+ */
+public class SampleClient2 {
+
+// private static final Logger LOG = LoggerFactory.getLogger(SampleClient2.class);
+
+ public static class LogEntity {
+ public String instanceUuid;
+ public long timestamp;
+ public String logLevel;
+ public String message;
+ public String reqId;
+ public String host;
+ public String component;
+ }
+
+ public static class IfEntity {
+ public String instanceUuid;
+ public long timestamp;
+ public String reqId;
+ public String message;
+ public String host;
+ }
+
+
+ /**
+ * @param args
+ */
+ public static void main(String[] args) {
+ AtomicLong base1 = new AtomicLong(System.currentTimeMillis());
+ AtomicLong base2 = new AtomicLong(System.currentTimeMillis());
+ AtomicLong count = new AtomicLong();
+
+ try (KafkaProducer<String, String> proceduer = SampleClient1.createProceduer()) {
+ while (true) {
+ nextUuid = String.format(instanceUuidTemp, UUID.randomUUID().toString());
+ nextReqId = String.format(reqIdTemp, UUID.randomUUID().toString());
+
+ int hostIndex = 6;
+ for (int i = 0; i < hostIndex; i++) {
+ sendMetric(base1, base2, count, proceduer, i);
+ }
+
+ if (count.get() % 600 == 0) {
+ System.out.println("send 600 LOG/FAILURE metric!");
+ }
+
+ Utils.sleep(3000);
+
+ }
+ }
+ }
+
+ private static void sendMetric(AtomicLong base1, AtomicLong base2, AtomicLong count,
+ KafkaProducer<String, String> proceduer, int i) {
+ {
+ Pair<Long, String> pair = createLogEntity(base1, i);
+ ProducerRecord<String, String> logRecord = new ProducerRecord<>("eslogs", pair.getRight());
+ proceduer.send(logRecord);
+ count.incrementAndGet();
+ }
+ {
+ Pair<Long, String> pair2 = createFailureEntity(base2, i);
+ ProducerRecord<String, String> failureRecord = new ProducerRecord<>("bootfailures", pair2.getRight());
+ proceduer.send(failureRecord);
+ count.incrementAndGet();
+ }
+ }
+
+ private static String instanceUuidTemp = "instance-guid-%s";
+ private static String reqIdTemp = "req-id-%s";
+ private static String nextUuid;
+ private static String nextReqId;
+
+ private static Pair<Long, String> createLogEntity(AtomicLong base1, int hostIndex) {
+ // TODO: add randomization
+ LogEntity le = new LogEntity();
+ if (hostIndex < 3) {
+ le.component = "NOVA";
+ le.host = "nova.000-" + hostIndex + ".datacenter.corp.com";
+ le.message = "RabbitMQ Exception - MQ not connectable!";
+ } else {
+ le.component = "NEUTRON";
+ le.host = "neturon.000-" + (hostIndex - 3) + ".datacenter.corp.com";
+ le.message = "DNS Exception - Fail to connect to DNS!";
+ }
+ le.instanceUuid = nextUuid;
+ le.logLevel = "ERROR";
+ le.reqId = nextReqId;
+ le.timestamp = base1.get();
+
+ base1.addAndGet(1000);// simply some interval.
+ return Pair.of(base1.get(), JsonUtils.writeValueAsString(le));
+ }
+
+ private static Pair<Long, String> createFailureEntity(AtomicLong base, int hostIndex) {
+ // TODO: add randomization
+ IfEntity ie = new IfEntity();
+ ie.host = "boot-vm-0-" + hostIndex + ".datacenter.corp.com";
+ ie.instanceUuid = nextUuid;
+ ie.message = "boot failure for when try start the given vm!";
+ ie.reqId = nextReqId;
+ ie.timestamp = base.get();
+
+ base.addAndGet(2000);// simply some interval.
+ return Pair.of(base.get(), JsonUtils.writeValueAsString(ie));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/evaluator/AlertBoltOutputCollectorThreadSafeWrapperTest.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/evaluator/AlertBoltOutputCollectorThreadSafeWrapperTest.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/evaluator/AlertBoltOutputCollectorThreadSafeWrapperTest.java
new file mode 100755
index 0000000..518d496
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/evaluator/AlertBoltOutputCollectorThreadSafeWrapperTest.java
@@ -0,0 +1,74 @@
+package org.apache.eagle.alert.engine.evaluator;
+
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.eagle.alert.engine.evaluator.impl.AlertBoltOutputCollectorThreadSafeWrapper;
+import org.apache.eagle.alert.engine.model.AlertStreamEvent;
+import org.junit.Assert;
+import org.junit.Test;
+
+import backtype.storm.task.IOutputCollector;
+import backtype.storm.task.OutputCollector;
+
+public class AlertBoltOutputCollectorThreadSafeWrapperTest {
+ @Test
+ public void testThreadSafeAlertBoltOutputCollector(){
+ MockedStormAlertOutputCollector stormOutputCollector = new MockedStormAlertOutputCollector(null);
+ AlertBoltOutputCollectorThreadSafeWrapper alertBoltOutputCollectorWrapper = new AlertBoltOutputCollectorThreadSafeWrapper(stormOutputCollector);
+ alertBoltOutputCollectorWrapper.emit(create("mockAlert_1"));
+ alertBoltOutputCollectorWrapper.emit(create("mockAlert_2"));
+ Assert.assertEquals(0,stormOutputCollector.getCollected().size());
+ Assert.assertEquals(0,stormOutputCollector.getTupleSize());
+ alertBoltOutputCollectorWrapper.flush();
+ Assert.assertEquals(2,stormOutputCollector.getCollected().size());
+ Assert.assertEquals(2,stormOutputCollector.getTupleSize());
+ alertBoltOutputCollectorWrapper.emit(create("mockAlert_3"));
+ Assert.assertEquals(2,stormOutputCollector.getCollected().size());
+ Assert.assertEquals(2,stormOutputCollector.getTupleSize());
+ alertBoltOutputCollectorWrapper.flush();
+ alertBoltOutputCollectorWrapper.flush();
+ alertBoltOutputCollectorWrapper.flush();
+ Assert.assertEquals(3,stormOutputCollector.getCollected().size());
+ Assert.assertEquals(3,stormOutputCollector.getTupleSize());
+ }
+
+ private AlertStreamEvent create(String streamId){
+ AlertStreamEvent alert = new AlertStreamEvent();
+ alert.setCreatedBy(this.toString());
+ alert.setCreatedTime(System.currentTimeMillis());
+ alert.setData(new Object[]{"field_1",2,"field_3"});
+ alert.setStreamId(streamId);
+ return alert;
+ }
+
+ private class MockedStormAlertOutputCollector extends OutputCollector {
+ private final Map<Object,List<Object>> collected;
+ MockedStormAlertOutputCollector(IOutputCollector delegate) {
+ super(delegate);
+ collected = new HashMap<>();
+ }
+
+ @Override
+ public List<Integer> emit(String streamId, List<Object> tuple) {
+ if(!collected.containsKey(tuple.get(0))){
+ collected.put(tuple.get(0),new LinkedList<>());
+ }
+ collected.get(tuple.get(0)).add(tuple);
+ return null;
+ }
+ Map<Object,List<Object>> getCollected(){
+ return collected;
+ }
+
+ int getTupleSize(){
+ int size = 0;
+ for(List<Object> alerts:collected.values()){
+ size += alerts.size();
+ }
+ return size;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/evaluator/SiddhiCEPPolicyEventHandlerTest.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/evaluator/SiddhiCEPPolicyEventHandlerTest.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/evaluator/SiddhiCEPPolicyEventHandlerTest.java
new file mode 100755
index 0000000..ab19801
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/evaluator/SiddhiCEPPolicyEventHandlerTest.java
@@ -0,0 +1,161 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.evaluator;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.eagle.alert.engine.Collector;
+import org.apache.eagle.alert.engine.coordinator.PolicyDefinition;
+import org.apache.eagle.alert.engine.coordinator.StreamDefinition;
+import org.apache.eagle.alert.engine.evaluator.impl.SiddhiPolicyHandler;
+import org.apache.eagle.alert.engine.mock.MockSampleMetadataFactory;
+import org.apache.eagle.alert.engine.mock.MockStreamCollector;
+import org.apache.eagle.alert.engine.model.AlertStreamEvent;
+import org.apache.eagle.alert.engine.model.StreamEvent;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import backtype.storm.metric.api.MultiCountMetric;
+
+public class SiddhiCEPPolicyEventHandlerTest {
+ private final static Logger LOG = LoggerFactory.getLogger(SiddhiCEPPolicyEventHandlerTest.class);
+
+ private Map<String, StreamDefinition> createDefinition(String ... streamIds) {
+ Map<String, StreamDefinition> sds = new HashMap<>();
+ for(String streamId:streamIds) {
+ // construct StreamDefinition
+ StreamDefinition sd = MockSampleMetadataFactory.createSampleStreamDefinition(streamId);
+ sds.put(streamId, sd);
+ }
+ return sds;
+ }
+
+ @SuppressWarnings("serial")
+ @Test
+ public void testBySendSimpleEvent() throws Exception {
+ SiddhiPolicyHandler handler;
+ MockStreamCollector collector;
+
+ handler = new SiddhiPolicyHandler(createDefinition("sampleStream_1","sampleStream_2"));
+ collector = new MockStreamCollector();
+ PolicyDefinition policyDefinition = MockSampleMetadataFactory.createSingleMetricSamplePolicy();
+ PolicyHandlerContext context = new PolicyHandlerContext();
+ context.setPolicyDefinition(policyDefinition);
+ context.setPolicyCounter(new MultiCountMetric());
+ handler.prepare(collector,context);
+ StreamEvent event = StreamEvent.Builder()
+ .schema(MockSampleMetadataFactory.createSampleStreamDefinition("sampleStream_1"))
+ .streamId("sampleStream_1")
+ .timestamep(System.currentTimeMillis())
+ .attributes(new HashMap<String,Object>(){{
+ put("name","cpu");
+ put("value",60.0);
+ put("bad","bad column value");
+ }}).build();
+ handler.send(event);
+ handler.close();
+ }
+
+ @SuppressWarnings("serial")
+ @Test
+ public void testWithTwoStreamJoinPolicy() throws Exception {
+ Map<String,StreamDefinition> ssd = createDefinition("sampleStream_1","sampleStream_2");
+
+ PolicyDefinition policyDefinition = new PolicyDefinition();
+ policyDefinition.setName("SampleJoinPolicyForTest");
+ policyDefinition.setInputStreams(Arrays.asList("sampleStream_1","sampleStream_2"));
+ policyDefinition.setOutputStreams(Collections.singletonList("joinedStream"));
+ policyDefinition.setDefinition(new PolicyDefinition.Definition(PolicyStreamHandlers.SIDDHI_ENGINE,
+ "from sampleStream_1#window.length(10) as left " +
+ "join sampleStream_2#window.length(10) as right " +
+ "on left.name == right.name and left.value == right.value " +
+ "select left.timestamp,left.name,left.value "+
+ "insert into joinedStream"));
+ policyDefinition.setPartitionSpec(Collections.singletonList(MockSampleMetadataFactory.createSampleStreamGroupbyPartition("sampleStream_1", Collections.singletonList("name"))));
+ SiddhiPolicyHandler handler;
+ Semaphore mutex = new Semaphore(0);
+ List<AlertStreamEvent> alerts = new ArrayList<>(0);
+ Collector<AlertStreamEvent> collector = (event) -> {
+ LOG.info("Collected {}",event);
+ Assert.assertTrue(event != null);
+ alerts.add(event);
+ mutex.release();
+ };
+
+ handler = new SiddhiPolicyHandler(ssd);
+ PolicyHandlerContext context = new PolicyHandlerContext();
+ context.setPolicyDefinition(policyDefinition);
+ context.setPolicyCounter(new MultiCountMetric());
+ handler.prepare(collector,context);
+
+ long ts_1 = System.currentTimeMillis();
+ long ts_2 = System.currentTimeMillis()+1;
+
+ handler.send(StreamEvent.Builder()
+ .schema(ssd.get("sampleStream_1"))
+ .streamId("sampleStream_1")
+ .timestamep(ts_1)
+ .attributes(new HashMap<String,Object>(){{
+ put("name","cpu");
+ put("value",60.0);
+ put("bad","bad column value");
+ }}).build());
+
+ handler.send(StreamEvent.Builder()
+ .schema(ssd.get("sampleStream_2"))
+ .streamId("sampleStream_2")
+ .timestamep(ts_2)
+ .attributes(new HashMap<String,Object>(){{
+ put("name","cpu");
+ put("value",61.0);
+ }}).build());
+
+ handler.send(StreamEvent.Builder()
+ .schema(ssd.get("sampleStream_2"))
+ .streamId("sampleStream_2")
+ .timestamep(ts_2)
+ .attributes(new HashMap<String,Object>(){{
+ put("name","disk");
+ put("value",60.0);
+ }}).build());
+
+ handler.send(StreamEvent.Builder()
+ .schema(ssd.get("sampleStream_2"))
+ .streamId("sampleStream_2")
+ .timestamep(ts_2)
+ .attributes(new HashMap<String,Object>(){{
+ put("name","cpu");
+ put("value",60.0);
+ }}).build());
+
+ handler.close();
+
+ Assert.assertTrue("Should get result in 5 s",mutex.tryAcquire(5, TimeUnit.SECONDS));
+ Assert.assertEquals(1,alerts.size());
+ Assert.assertEquals("joinedStream",alerts.get(0).getStreamId());
+ Assert.assertEquals("cpu",alerts.get(0).getData()[1]);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/integration/MockMetadataServiceClient.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/integration/MockMetadataServiceClient.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/integration/MockMetadataServiceClient.java
new file mode 100644
index 0000000..d887451
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/integration/MockMetadataServiceClient.java
@@ -0,0 +1,124 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.integration;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.eagle.alert.coordination.model.Kafka2TupleMetadata;
+import org.apache.eagle.alert.coordination.model.ScheduleState;
+import org.apache.eagle.alert.coordination.model.SpoutSpec;
+import org.apache.eagle.alert.coordination.model.internal.Topology;
+import org.apache.eagle.alert.engine.coordinator.PolicyDefinition;
+import org.apache.eagle.alert.engine.coordinator.Publishment;
+import org.apache.eagle.alert.engine.coordinator.StreamDefinition;
+import org.apache.eagle.alert.engine.coordinator.StreamingCluster;
+import org.apache.eagle.alert.service.IMetadataServiceClient;
+
+@SuppressWarnings("serial")
+public class MockMetadataServiceClient implements IMetadataServiceClient {
+
+ @Override
+ public List<SpoutSpec> listSpoutMetadata() {
+ return null;
+ }
+
+ @Override
+ public void close() throws IOException {
+
+ }
+
+ @Override
+ public ScheduleState getVersionedSpec(String version) {
+ return null;
+ }
+
+ @Override
+ public List<StreamingCluster> listClusters() {
+ return null;
+ }
+
+ @Override
+ public List<PolicyDefinition> listPolicies() {
+ return null;
+ }
+
+ @Override
+ public List<StreamDefinition> listStreams() {
+ return null;
+ }
+
+ @Override
+ public List<Kafka2TupleMetadata> listDataSources() {
+ return null;
+ }
+
+ @Override
+ public List<Publishment> listPublishment() {
+ return null;
+ }
+
+ @Override
+ public ScheduleState getVersionedSpec() {
+ return null;
+ }
+
+ @Override
+ public void addScheduleState(ScheduleState state) {
+
+ }
+
+ @Override
+ public List<Topology> listTopologies() {
+ return null;
+ }
+
+ @Override
+ public void addStreamingCluster(StreamingCluster cluster) {
+
+ }
+
+ @Override
+ public void addTopology(Topology t) {
+
+ }
+
+ @Override
+ public void addPolicy(PolicyDefinition policy) {
+
+ }
+
+ @Override
+ public void addStreamDefinition(StreamDefinition streamDef) {
+
+ }
+
+ @Override
+ public void addDataSource(Kafka2TupleMetadata k2t) {
+
+ }
+
+ @Override
+ public void addPublishment(Publishment pub) {
+
+ }
+
+ @Override
+ public void clear() {
+
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/metric/MemoryUsageGaugeSetTest.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/metric/MemoryUsageGaugeSetTest.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/metric/MemoryUsageGaugeSetTest.java
new file mode 100644
index 0000000..76aef7e
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/metric/MemoryUsageGaugeSetTest.java
@@ -0,0 +1,47 @@
+package org.apache.eagle.alert.engine.metric;
+
+import java.util.concurrent.TimeUnit;
+
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.codahale.metrics.ConsoleReporter;
+import com.codahale.metrics.Gauge;
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.jvm.MemoryUsageGaugeSet;
+
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+public class MemoryUsageGaugeSetTest {
+ private final Logger LOG = LoggerFactory.getLogger(MemoryUsageGaugeSetTest.class);
+
+ @Test
+ public void testJVMMetrics() throws InterruptedException {
+ LOG.info("Starting testJVMMetrics");
+ final MetricRegistry metrics = new MetricRegistry();
+ ConsoleReporter reporter = ConsoleReporter.forRegistry(metrics)
+ .convertRatesTo(TimeUnit.SECONDS)
+ .convertDurationsTo(TimeUnit.MILLISECONDS)
+ .build();
+ metrics.registerAll(new MemoryUsageGaugeSet());
+ metrics.register("sample", (Gauge<Double>) () -> 0.1234);
+ reporter.start(1, TimeUnit.SECONDS);
+ Thread.sleep(5000);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/mock/MockPartitionedCollector.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/mock/MockPartitionedCollector.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/mock/MockPartitionedCollector.java
new file mode 100644
index 0000000..74d11d2
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/mock/MockPartitionedCollector.java
@@ -0,0 +1,55 @@
+package org.apache.eagle.alert.engine.mock;
+
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.eagle.alert.engine.PartitionedEventCollector;
+import org.apache.eagle.alert.engine.model.PartitionedEvent;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+public class MockPartitionedCollector implements PartitionedEventCollector {
+ @SuppressWarnings("unused")
+ private final static Logger LOG = LoggerFactory.getLogger(MockPartitionedCollector.class);
+ private List<PartitionedEvent> cache;
+ public MockPartitionedCollector(){
+ cache = new LinkedList<>();
+ }
+
+ public void emit(PartitionedEvent event) {
+ cache.add(event);
+ }
+
+ public void clear(){
+ cache.clear();
+ }
+
+ public List<PartitionedEvent> get(){
+ return cache;
+ }
+
+ public int size(){
+ return cache.size();
+ }
+
+ @Override
+ public void drop(PartitionedEvent event) {
+
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/mock/MockSampleMetadataFactory.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/mock/MockSampleMetadataFactory.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/mock/MockSampleMetadataFactory.java
new file mode 100644
index 0000000..97e6310
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/test/java/org/apache/eagle/alert/engine/mock/MockSampleMetadataFactory.java
@@ -0,0 +1,266 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.mock;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.ThreadLocalRandom;
+
+import org.apache.eagle.alert.coordination.model.PolicyWorkerQueue;
+import org.apache.eagle.alert.coordination.model.StreamRouterSpec;
+import org.apache.eagle.alert.coordination.model.WorkSlot;
+import org.apache.eagle.alert.engine.coordinator.PolicyDefinition;
+import org.apache.eagle.alert.engine.coordinator.StreamColumn;
+import org.apache.eagle.alert.engine.coordinator.StreamDefinition;
+import org.apache.eagle.alert.engine.coordinator.StreamDefinitionNotFoundException;
+import org.apache.eagle.alert.engine.coordinator.StreamPartition;
+import org.apache.eagle.alert.engine.coordinator.StreamSortSpec;
+import org.apache.eagle.alert.engine.evaluator.PolicyStreamHandlers;
+import org.apache.eagle.alert.engine.model.PartitionedEvent;
+import org.apache.eagle.alert.engine.model.StreamEvent;
+
+@SuppressWarnings("serial")
+public class MockSampleMetadataFactory {
+ private static MockStreamMetadataService mockStreamMetadataServiceInstance = null;
+ public static MockStreamMetadataService createSingletonMetadataServiceWithSample(){
+ if(mockStreamMetadataServiceInstance!=null) return mockStreamMetadataServiceInstance;
+ mockStreamMetadataServiceInstance = new MockStreamMetadataService();
+ mockStreamMetadataServiceInstance.registerStream("sampleStream",createSampleStreamDefinition("sampleStream"));
+ mockStreamMetadataServiceInstance.registerStream("sampleStream_1",createSampleStreamDefinition("sampleStream_1"));
+ mockStreamMetadataServiceInstance.registerStream("sampleStream_2",createSampleStreamDefinition("sampleStream_2"));
+ mockStreamMetadataServiceInstance.registerStream("sampleStream_3",createSampleStreamDefinition("sampleStream_3"));
+ mockStreamMetadataServiceInstance.registerStream("sampleStream_4",createSampleStreamDefinition("sampleStream_4"));
+ return mockStreamMetadataServiceInstance;
+ }
+
+ public static StreamDefinition createSampleStreamDefinition(String streamId){
+ StreamDefinition sampleStreamDefinition = new StreamDefinition();
+ sampleStreamDefinition.setStreamId(streamId);
+ sampleStreamDefinition.setTimeseries(true);
+ sampleStreamDefinition.setValidate(true);
+ sampleStreamDefinition.setDescription("Schema for "+streamId);
+ List<StreamColumn> streamColumns = new ArrayList<>();
+
+ streamColumns.add(new StreamColumn.Builder().name("name").type(StreamColumn.Type.STRING).build());
+ streamColumns.add(new StreamColumn.Builder().name("host").type(StreamColumn.Type.STRING).build());
+ streamColumns.add(new StreamColumn.Builder().name("flag").type(StreamColumn.Type.BOOL).build());
+ streamColumns.add(new StreamColumn.Builder().name("timestamp").type(StreamColumn.Type.LONG).build());
+ streamColumns.add(new StreamColumn.Builder().name("value").type(StreamColumn.Type.DOUBLE).build());
+// streamColumns.add(new StreamColumn.Builder().name("value1").type(StreamColumn.Type.DOUBLE).build());
+// streamColumns.add(new StreamColumn.Builder().name("value2").type(StreamColumn.Type.DOUBLE).build());
+// streamColumns.add(new StreamColumn.Builder().name("value3").type(StreamColumn.Type.DOUBLE).build());
+// streamColumns.add(new StreamColumn.Builder().name("value4").type(StreamColumn.Type.DOUBLE).build());
+// streamColumns.add(new StreamColumn.Builder().name("value5").type(StreamColumn.Type.DOUBLE).build());
+ sampleStreamDefinition.setColumns(streamColumns);
+ return sampleStreamDefinition;
+ }
+
+ /**
+ * By default window period is: PT1m
+ *
+ * @param streamId
+ * @return
+ */
+ public static StreamSortSpec createSampleStreamSortSpec(String streamId){
+ StreamSortSpec streamSortSpec = new StreamSortSpec();
+// streamSortSpec.setColumn("timestamp");
+// streamSortSpec.setStreamId(streamId);
+ streamSortSpec.setWindowMargin(1000);
+ streamSortSpec.setWindowPeriod("PT1m");
+ return streamSortSpec;
+ }
+
+ public static StreamSortSpec createSampleStreamSortSpec(String streamId,String period,int margin){
+ StreamSortSpec streamSortSpec = new StreamSortSpec();
+// streamSortSpec.setColumn("timestamp");
+// streamSortSpec.setStreamId(streamId);
+ streamSortSpec.setWindowMargin(margin);
+ streamSortSpec.setWindowPeriod(period);
+ return streamSortSpec;
+ }
+
+ /**
+ * Policy: from sampleStream_1[name == "cpu" and value > 50.0] select name, host, flag, value insert into outputStream;
+ *
+ * @return PolicyDefinition[from sampleStream_1[name == "cpu" and value > 50.0] select name, host, flag, value insert into outputStream;]
+ */
+ public static PolicyDefinition createSingleMetricSamplePolicy(){
+ String definePolicy = "from sampleStream_1[name == \"cpu\" and value > 50.0] select name, host, flag, value insert into outputStream;";
+ PolicyDefinition policyDefinition = new PolicyDefinition();
+ policyDefinition.setName("SamplePolicyForTest");
+ policyDefinition.setInputStreams(Arrays.asList("sampleStream_1"));
+ policyDefinition.setOutputStreams(Arrays.asList("outputStream"));
+ policyDefinition.setDefinition(new PolicyDefinition.Definition(
+ PolicyStreamHandlers.SIDDHI_ENGINE,
+ definePolicy
+ ));
+ policyDefinition.setPartitionSpec(Arrays.asList(createSampleStreamGroupbyPartition("sampleStream_1",Arrays.asList("name"))));
+ return policyDefinition;
+ }
+
+ public static StreamPartition createSampleStreamGroupbyPartition(String streamId, List<String> groupByField){
+ StreamPartition streamPartition = new StreamPartition();
+ streamPartition.setStreamId(streamId);
+ streamPartition.setColumns(new ArrayList<>(groupByField));
+ streamPartition.setType(StreamPartition.Type.GROUPBY);
+ StreamSortSpec streamSortSpec = new StreamSortSpec();
+ streamSortSpec.setWindowPeriod("PT30m");
+ streamSortSpec.setWindowMargin(10000);
+ streamPartition.setSortSpec(streamSortSpec);
+ return streamPartition;
+ }
+
+ public static StreamRouterSpec createSampleStreamRouteSpec(String streamId, String groupByField, List<String> targetEvaluatorIds){
+ List<WorkSlot> slots = Arrays.asList(targetEvaluatorIds.stream().map((t) -> {
+ return new WorkSlot("sampleTopology", t);
+ }).toArray(WorkSlot[]::new));
+ StreamRouterSpec streamRouteSpec = new StreamRouterSpec();
+ streamRouteSpec.setStreamId(streamId);
+ streamRouteSpec.setPartition(createSampleStreamGroupbyPartition(streamId,Arrays.asList(groupByField)));
+ streamRouteSpec.setTargetQueue(Arrays.asList(new PolicyWorkerQueue(slots)));
+ return streamRouteSpec;
+ }
+
+ public static StreamRouterSpec createSampleStreamRouteSpec(List<String> targetEvaluatorIds){
+ return createSampleStreamRouteSpec("sampleStream_1","name",targetEvaluatorIds);
+ }
+
+ /**
+ * GROUPBY_sampleStream_1_ON_name
+ *
+ * @param targetEvaluatorIds
+ * @return
+ */
+ public static StreamRouterSpec createRouteSpec_GROUP_sampleStream_1_BY_name(List<String> targetEvaluatorIds){
+ return createSampleStreamRouteSpec("sampleStream_1","name",targetEvaluatorIds);
+ }
+
+ public static StreamRouterSpec createRouteSpec_GROUP_sampleStream_2_BY_name(List<String> targetEvaluatorIds){
+ return createSampleStreamRouteSpec("sampleStream_2","name",targetEvaluatorIds);
+ }
+
+ public static PartitionedEvent createSimpleStreamEvent() {
+ StreamEvent event = null;
+ try {
+ event = StreamEvent.Builder()
+ .schema(MockSampleMetadataFactory.createSingletonMetadataServiceWithSample().getStreamDefinition("sampleStream_1"))
+ .streamId("sampleStream_1")
+ .timestamep(System.currentTimeMillis())
+ .attributes(new HashMap<String,Object>(){{
+ put("name","cpu");
+ put("value",60.0);
+ put("unknown","unknown column value");
+ }}).build();
+ } catch (StreamDefinitionNotFoundException e) {
+ e.printStackTrace();
+ }
+ PartitionedEvent pEvent = new PartitionedEvent();
+ pEvent.setEvent(event);
+ return pEvent;
+ }
+
+ private final static String[] SAMPLE_STREAM_NAME_OPTIONS=new String[]{
+ "cpu","memory","disk","network"
+ };
+
+ private final static String[] SAMPLE_STREAM_HOST_OPTIONS =new String[]{
+ "localhost_1","localhost_2","localhost_3","localhost_4"
+ };
+
+ private final static Boolean[] SAMPLE_STREAM_FLAG_OPTIONS=new Boolean[]{
+ true,false
+ };
+
+ private final static Double[] SAMPLE_STREAM_VALUE_OPTIONS=new Double[]{
+ -0.20, 40.4,50.5,60.6,10000.1
+ };
+ private final static String[] SAMPLE_STREAM_ID_OPTIONS=new String[]{
+ "sampleStream_1","sampleStream_2","sampleStream_3","sampleStream_4",
+ };
+ private final static Random RANDOM = ThreadLocalRandom.current();
+
+ public static StreamEvent createRandomStreamEvent() {
+ return createRandomStreamEvent(SAMPLE_STREAM_ID_OPTIONS[RANDOM.nextInt(SAMPLE_STREAM_ID_OPTIONS.length)]);
+ }
+
+ public static StreamEvent createRandomStreamEvent(String streamId) {
+ return createRandomStreamEvent(streamId,System.currentTimeMillis());
+ }
+
+ private final static Long[] TIME_DELTA_OPTIONS = new Long[]{
+ -30000L, -10000L, -5000L, -1000L, 0L, 1000L, 5000L, 10000L, 30000L
+ };
+
+ public static StreamEvent createRandomOutOfTimeOrderStreamEvent(String streamId) {
+ StreamEvent event = createRandomStreamEvent(streamId);
+ event.setTimestamp(System.currentTimeMillis()+TIME_DELTA_OPTIONS[RANDOM.nextInt(TIME_DELTA_OPTIONS.length)]);
+ return event;
+ }
+
+
+ public static PartitionedEvent createRandomOutOfTimeOrderEventGroupedByName(String streamId) {
+ StreamEvent event = createRandomStreamEvent(streamId);
+ event.setTimestamp(System.currentTimeMillis()+TIME_DELTA_OPTIONS[RANDOM.nextInt(TIME_DELTA_OPTIONS.length)]);
+ return new PartitionedEvent(event,createSampleStreamGroupbyPartition(streamId,Arrays.asList("name")),event.getData()[0].hashCode());
+ }
+
+ public static PartitionedEvent createPartitionedEventGroupedByName(String streamId,long timestamp) {
+ StreamEvent event = createRandomStreamEvent(streamId);
+ event.setTimestamp(timestamp);
+ return new PartitionedEvent(event,createSampleStreamGroupbyPartition(streamId,Arrays.asList("name")),event.getData()[0].hashCode());
+ }
+
+ public static PartitionedEvent createRandomSortedEventGroupedByName(String streamId) {
+ StreamEvent event = createRandomStreamEvent(streamId);
+ event.setTimestamp(System.currentTimeMillis());
+ return new PartitionedEvent(event,createSampleStreamGroupbyPartition(streamId,Arrays.asList("name")),event.getData()[0].hashCode());
+ }
+
+ public static StreamEvent createRandomStreamEvent(String streamId, long timestamp) {
+ StreamEvent event;
+ try {
+ event = StreamEvent.Builder()
+ .schema(MockSampleMetadataFactory.createSingletonMetadataServiceWithSample().getStreamDefinition(streamId))
+ .streamId(streamId)
+ .timestamep(timestamp)
+ .attributes(new HashMap<String,Object>(){{
+ put("name",SAMPLE_STREAM_NAME_OPTIONS[RANDOM.nextInt(SAMPLE_STREAM_NAME_OPTIONS.length)]);
+ put("value", SAMPLE_STREAM_VALUE_OPTIONS[RANDOM.nextInt(SAMPLE_STREAM_VALUE_OPTIONS.length)]);
+ put("host", SAMPLE_STREAM_HOST_OPTIONS[RANDOM.nextInt(SAMPLE_STREAM_HOST_OPTIONS.length)]);
+ put("flag",SAMPLE_STREAM_FLAG_OPTIONS[RANDOM.nextInt(SAMPLE_STREAM_FLAG_OPTIONS.length)]);
+// put("value1", SAMPLE_STREAM_VALUE_OPTIONS[RANDOM.nextInt(SAMPLE_STREAM_VALUE_OPTIONS.length)]);
+// put("value2", SAMPLE_STREAM_VALUE_OPTIONS[RANDOM.nextInt(SAMPLE_STREAM_VALUE_OPTIONS.length)]);
+// put("value3", SAMPLE_STREAM_VALUE_OPTIONS[RANDOM.nextInt(SAMPLE_STREAM_VALUE_OPTIONS.length)]);
+// put("value4", SAMPLE_STREAM_VALUE_OPTIONS[RANDOM.nextInt(SAMPLE_STREAM_VALUE_OPTIONS.length)]);
+// put("value5", SAMPLE_STREAM_VALUE_OPTIONS[RANDOM.nextInt(SAMPLE_STREAM_VALUE_OPTIONS.length)]);
+ put("unknown","unknown column value");
+ }}).build();
+ } catch (StreamDefinitionNotFoundException e) {
+ throw new IllegalStateException(e.getMessage(),e);
+ }
+ return event;
+ }
+
+ public static PartitionedEvent createRandomPartitionedEvent(String streamId, long timestamp) {
+ StreamEvent event = createRandomStreamEvent(streamId,timestamp);
+ PartitionedEvent partitionedEvent = new PartitionedEvent(event,createSampleStreamGroupbyPartition(streamId,Arrays.asList("name")),event.getData()[0].hashCode());
+ return partitionedEvent;
+ }
+}
\ No newline at end of file
[03/19] incubator-eagle git commit: EAGLE-324: Init branch-v0.5
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata-service/src/main/java/org/apache/eagle/service/topology/resource/TopologyMgmtResource.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata-service/src/main/java/org/apache/eagle/service/topology/resource/TopologyMgmtResource.java b/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata-service/src/main/java/org/apache/eagle/service/topology/resource/TopologyMgmtResource.java
new file mode 100644
index 0000000..437068f
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata-service/src/main/java/org/apache/eagle/service/topology/resource/TopologyMgmtResource.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.service.topology.resource;
+
+import org.apache.eagle.alert.metadata.resource.OpResult;
+import org.apache.eagle.service.topology.resource.impl.TopologyMgmtResourceImpl;
+import org.apache.eagle.service.topology.resource.impl.TopologyStatus;
+
+import javax.ws.rs.*;
+
+import java.util.List;
+
+/**
+ * @since May 5, 2016
+ *
+ */
+@Path("/alert")
+@Produces("application/json")
+@Consumes("application/json")
+public class TopologyMgmtResource {
+ private TopologyMgmtResourceImpl topologyManager = new TopologyMgmtResourceImpl();
+
+ @POST
+ @Path("/topologies/{topologyName}/start")
+ public OpResult startTopology(@PathParam("topologyName") String topologyName) {
+ OpResult result = new OpResult();
+ try {
+ topologyManager.startTopology(topologyName);
+ } catch (Exception ex) {
+ result.message = ex.toString();
+ }
+ return result;
+ }
+
+ @POST
+ @Path("/topologies/{topologyName}/stop")
+ public OpResult stopTopology(@PathParam("topologyName") String topologyName) {
+ OpResult result = new OpResult();
+ try {
+ topologyManager.stopTopology(topologyName);
+ } catch (Exception ex) {
+ result.message = ex.toString();
+ }
+ return result;
+ }
+
+ @GET
+ @Path("/topologies")
+ public List<TopologyStatus> getTopologies() throws Exception {
+ return topologyManager.getTopologies();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata-service/src/main/java/org/apache/eagle/service/topology/resource/impl/TopologyMgmtResourceHelper.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata-service/src/main/java/org/apache/eagle/service/topology/resource/impl/TopologyMgmtResourceHelper.java b/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata-service/src/main/java/org/apache/eagle/service/topology/resource/impl/TopologyMgmtResourceHelper.java
new file mode 100644
index 0000000..3876116
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata-service/src/main/java/org/apache/eagle/service/topology/resource/impl/TopologyMgmtResourceHelper.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.service.topology.resource.impl;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.List;
+import java.util.Optional;
+
+public class TopologyMgmtResourceHelper {
+ private static final Logger LOG = LoggerFactory.getLogger(TopologyMgmtResourceHelper.class);
+
+ public static <T> Optional<T> findById(List<T> clusters, String id) {
+ Optional<T> OptionValue = clusters.stream().filter(o -> getName(o).equalsIgnoreCase(id)).findFirst();
+ return OptionValue;
+ }
+
+ public static <T> String getName(T t) {
+ try {
+ Method m = t.getClass().getMethod("getName");
+ return (String) m.invoke(t);
+ } catch (NoSuchMethodException | SecurityException | InvocationTargetException | IllegalAccessException
+ | IllegalArgumentException e) {
+ LOG.error(" getName not found on given class :" + t.getClass().getName());
+ }
+ throw new RuntimeException(String.format("no getName() found on target class %s for matching", t.getClass()
+ .getName()));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata-service/src/main/java/org/apache/eagle/service/topology/resource/impl/TopologyMgmtResourceImpl.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata-service/src/main/java/org/apache/eagle/service/topology/resource/impl/TopologyMgmtResourceImpl.java b/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata-service/src/main/java/org/apache/eagle/service/topology/resource/impl/TopologyMgmtResourceImpl.java
new file mode 100644
index 0000000..ff9a65a
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata-service/src/main/java/org/apache/eagle/service/topology/resource/impl/TopologyMgmtResourceImpl.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.service.topology.resource.impl;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+import org.apache.eagle.alert.coordination.model.internal.Topology;
+import org.apache.eagle.alert.engine.UnitTopologyMain;
+import org.apache.eagle.alert.engine.runner.UnitTopologyRunner;
+import org.apache.eagle.alert.engine.coordinator.StreamingCluster;
+import org.apache.eagle.alert.metadata.impl.MetadataDaoFactory;
+import org.apache.eagle.alert.metadata.resource.IMetadataDao;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import backtype.storm.Config;
+import backtype.storm.StormSubmitter;
+import backtype.storm.generated.Nimbus;
+import backtype.storm.generated.StormTopology;
+import backtype.storm.generated.TopologySummary;
+import backtype.storm.utils.NimbusClient;
+import backtype.storm.utils.Utils;
+
+import com.typesafe.config.ConfigFactory;
+
+public class TopologyMgmtResourceImpl {
+ private static final IMetadataDao dao = MetadataDaoFactory.getInstance().getMetadataDao();
+ @SuppressWarnings("unused")
+ private static final Logger LOG = LoggerFactory.getLogger(TopologyMgmtResourceImpl.class);
+
+ private final String DEFAULT_NIMBUS_HOST = "sandbox.hortonworks.com";
+ private final Integer DEFAULT_NIMBUS_THRIFT_PORT = 6627;
+ private final String STORM_JAR_PATH = "topology.stormJarPath";
+
+
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ private Map getStormConf(List<StreamingCluster> clusters, String clusterId) throws Exception {
+ Map<String, Object> storm_conf = Utils.readStormConfig();
+ if(clusterId == null) {
+ storm_conf.put(Config.NIMBUS_HOST, DEFAULT_NIMBUS_HOST);
+ storm_conf.put(Config.NIMBUS_THRIFT_PORT, DEFAULT_NIMBUS_THRIFT_PORT);
+ } else {
+ if(clusters == null) clusters = dao.listClusters();
+ Optional<StreamingCluster> scOp = TopologyMgmtResourceHelper.findById(clusters, clusterId);
+ StreamingCluster cluster;
+ if(scOp.isPresent()) {
+ cluster = scOp.get();
+ } else {
+ throw new Exception("Fail to find cluster: " + clusterId);
+ }
+ storm_conf.put(Config.NIMBUS_HOST, cluster.getDeployments().getOrDefault(StreamingCluster.NIMBUS_HOST, DEFAULT_NIMBUS_HOST));
+ storm_conf.put(Config.NIMBUS_THRIFT_PORT, Integer.valueOf(cluster.getDeployments().get(StreamingCluster.NIMBUS_THRIFT_PORT)));
+ }
+ return storm_conf;
+ }
+
+ private void createTopologyHelper(Topology topologyDef, com.typesafe.config.Config config) {
+ int numOfSpoutTasks = config.getInt(UnitTopologyRunner.SPOUT_TASK_NUM);
+ int numOfRouterBolts = config.getInt(UnitTopologyRunner.ROUTER_TASK_NUM);
+ int numOfAlertBolts = config.getInt(UnitTopologyRunner.ALERT_TASK_NUM);
+ int numOfPublishTasks = config.getInt(UnitTopologyRunner.PUBLISH_TASK_NUM);
+ topologyDef.setSpoutId(UnitTopologyRunner.spoutName);
+ topologyDef.setPubBoltId(UnitTopologyRunner.alertPublishBoltName);
+ topologyDef.setNumOfSpout(numOfSpoutTasks);
+ topologyDef.setNumOfGroupBolt(numOfRouterBolts);
+ topologyDef.setNumOfAlertBolt(numOfAlertBolts);
+ topologyDef.setNumOfPublishBolt(numOfPublishTasks);
+ dao.addTopology(topologyDef);
+ }
+
+ private StormTopology createTopology(Topology topologyDef) {
+ com.typesafe.config.Config topologyConf = ConfigFactory.load("topology-sample-definition.conf");
+ String stormJarPath = "";
+ if(topologyConf.hasPath(STORM_JAR_PATH)) {
+ stormJarPath = topologyConf.getString(STORM_JAR_PATH);
+ }
+ System.setProperty("storm.jar", stormJarPath);
+ createTopologyHelper(topologyDef, topologyConf);
+ return UnitTopologyMain.createTopology(topologyConf);
+ }
+
+ public void startTopology(String topologyName) throws Exception {
+ Optional<Topology> tdop = TopologyMgmtResourceHelper.findById(dao.listTopologies(), topologyName);
+ Topology topologyDef;
+ if(tdop.isPresent()) {
+ topologyDef = tdop.get();
+ } else {
+ topologyDef = new Topology();
+ topologyDef.setName(topologyName);
+ }
+ StormSubmitter.submitTopology(topologyName, getStormConf(null, topologyDef.getClusterName()), createTopology(topologyDef));
+ }
+
+ public void stopTopology(String topologyName) throws Exception {
+ Optional<Topology> tdop = TopologyMgmtResourceHelper.findById(dao.listTopologies(), topologyName);
+ Topology topologyDef;
+ if(tdop.isPresent()) {
+ topologyDef = tdop.get();
+ } else {
+ throw new Exception("Fail to find topology " + topologyName);
+ }
+ Nimbus.Client stormClient = NimbusClient.getConfiguredClient(getStormConf(null, topologyDef.getClusterName())).getClient();
+ stormClient.killTopology(topologyName);
+ }
+
+ @SuppressWarnings({ "rawtypes", "unused" })
+ private TopologySummary getTopologySummery(List<StreamingCluster> clusters, Topology topologyDef) throws Exception {
+ Map storm_conf = getStormConf(clusters, topologyDef.getClusterName());
+ Nimbus.Client stormClient = NimbusClient.getConfiguredClient(storm_conf).getClient();
+ Optional<TopologySummary> tOp = stormClient.getClusterInfo().get_topologies().stream().filter(topology -> topology.get_name().equalsIgnoreCase(topologyDef.getName())).findFirst();
+ if(tOp.isPresent()) {
+ String id = tOp.get().get_id();
+ //StormTopology stormTopology= stormClient.getTopology(id);
+ return tOp.get();
+ } else {
+ return null;
+ }
+ }
+
+ public List<TopologyStatus> getTopologies() throws Exception {
+ List<Topology> topologyDefinitions = dao.listTopologies();
+ List<StreamingCluster> clusters = dao.listClusters();
+
+ List<TopologyStatus> topologies = new ArrayList<>();
+ for(Topology topologyDef : topologyDefinitions) {
+ TopologySummary topologySummary = getTopologySummery(clusters, topologyDef);
+ if(topologySummary != null) {
+ TopologyStatus t = new TopologyStatus();
+ t.setName(topologySummary.get_name());
+ t.setId(topologySummary.get_id());
+ t.setState(topologySummary.get_status());
+ t.setTopology(topologyDef);
+ topologies.add(t);
+ }
+ }
+ return topologies;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata-service/src/main/java/org/apache/eagle/service/topology/resource/impl/TopologyStatus.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata-service/src/main/java/org/apache/eagle/service/topology/resource/impl/TopologyStatus.java b/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata-service/src/main/java/org/apache/eagle/service/topology/resource/impl/TopologyStatus.java
new file mode 100644
index 0000000..c3381d4
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata-service/src/main/java/org/apache/eagle/service/topology/resource/impl/TopologyStatus.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.service.topology.resource.impl;
+
+import org.apache.eagle.alert.coordination.model.internal.Topology;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class TopologyStatus {
+ private String name;
+ private String id;
+ private String state;
+ private Topology topology;
+
+ private Map<String, Double> spoutLoad = new HashMap<>();
+ private Map<String, Double> boltLoad = new HashMap<>();
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public String getId() {
+ return id;
+ }
+
+ public void setId(String id) {
+ this.id = id;
+ }
+
+ public String getState() {
+ return state;
+ }
+
+ public void setState(String state) {
+ this.state = state;
+ }
+
+ public Map<String, Double> getSpoutLoad() {
+ return spoutLoad;
+ }
+
+ public void setSpoutLoad(Map<String, Double> spoutLoad) {
+ this.spoutLoad = spoutLoad;
+ }
+
+ public Map<String, Double> getBoltLoad() {
+ return boltLoad;
+ }
+
+ public void setBoltLoad(Map<String, Double> boltLoad) {
+ this.boltLoad = boltLoad;
+ }
+
+ public Topology getTopology() {
+ return topology;
+ }
+
+ public void setTopology(Topology topology) {
+ this.topology = topology;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata-service/src/main/resources/application.conf
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata-service/src/main/resources/application.conf b/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata-service/src/main/resources/application.conf
new file mode 100644
index 0000000..20461c8
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata-service/src/main/resources/application.conf
@@ -0,0 +1,6 @@
+{
+ "datastore": {
+ "metadataDao": "org.apache.eagle.service.alert.resource.impl.InMemMetadataDaoImpl",
+ "connection": "localhost:27017"
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata-service/src/main/webapp/WEB-INF/web.xml
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata-service/src/main/webapp/WEB-INF/web.xml b/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata-service/src/main/webapp/WEB-INF/web.xml
new file mode 100644
index 0000000..ab7b019
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata-service/src/main/webapp/WEB-INF/web.xml
@@ -0,0 +1,81 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with ~
+ this work for additional information regarding copyright ownership. ~ The
+ ASF licenses this file to You under the Apache License, Version 2.0 ~ (the
+ "License"); you may not use this file except in compliance with ~ the License.
+ You may obtain a copy of the License at ~ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~ ~ Unless required by applicable law or agreed to in writing, software ~
+ distributed under the License is distributed on an "AS IS" BASIS, ~ WITHOUT
+ WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ~ See the
+ License for the specific language governing permissions and ~ limitations
+ under the License. -->
+<web-app xmlns="http://java.sun.com/xml/ns/javaee" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://java.sun.com/xml/ns/javaee
+ http://java.sun.com/xml/ns/javaee/web-app_3_0.xsd"
+ version="3.0">
+ <welcome-file-list>
+ <welcome-file>index.html</welcome-file>
+ </welcome-file-list>
+ <servlet>
+ <servlet-name>Jersey Web Application</servlet-name>
+ <servlet-class>com.sun.jersey.spi.container.servlet.ServletContainer</servlet-class>
+ <init-param>
+ <param-name>com.sun.jersey.config.property.packages</param-name>
+ <param-value>io.swagger.jaxrs.json,io.swagger.jaxrs.listing,org.apache.eagle.service,org.codehaus.jackson.jaxrs</param-value>
+ </init-param>
+ <init-param>
+ <param-name>com.sun.jersey.spi.container.ContainerRequestFilters</param-name>
+ <param-value>com.sun.jersey.api.container.filter.GZIPContentEncodingFilter;com.sun.jersey.api.container.filter.PostReplaceFilter</param-value>
+ </init-param>
+ <init-param>
+ <param-name>com.sun.jersey.spi.container.ContainerResponseFilters</param-name>
+ <param-value>com.sun.jersey.api.container.filter.GZIPContentEncodingFilter</param-value>
+ </init-param>
+ <load-on-startup>1</load-on-startup>
+ </servlet>
+ <!-- Servlet for swagger initialization only, no URL mapping. -->
+ <servlet>
+ <servlet-name>swaggerConfig</servlet-name>
+ <servlet-class>io.swagger.jaxrs.config.DefaultJaxrsConfig</servlet-class>
+ <init-param>
+ <param-name>api.version</param-name>
+ <param-value>1.0.0</param-value>
+ </init-param>
+ <init-param>
+ <param-name>swagger.api.basepath</param-name>
+ <param-value>/api</param-value>
+ </init-param>
+ <load-on-startup>2</load-on-startup>
+ </servlet>
+
+ <servlet-mapping>
+ <servlet-name>Jersey Web Application</servlet-name>
+ <url-pattern>/api/*</url-pattern>
+ </servlet-mapping>
+ <filter>
+ <filter-name>CorsFilter</filter-name>
+ <!-- Ideally, should be tomcat embed core's CORSFilter. See @SimpleCORSFiler comments. -->
+ <filter-class>org.apache.eagle.alert.resource.SimpleCORSFiler</filter-class>
+ <init-param>
+ <param-name>cors.allowed.origins</param-name>
+ <param-value>*</param-value>
+ </init-param>
+ <init-param>
+ <param-name>cors.allowed.headers</param-name>
+ <param-value>Authorization,Origin, No-Cache, X-Requested-With, If-Modified-Since, Pragma, Last-Modified, Cache-Control, Expires, Content-Type, X-E4M-With, Accept</param-value>
+ </init-param>
+ <init-param>
+ <param-name>cors.allowed.methods</param-name>
+ <param-value>GET,POST,HEAD,OPTIONS,PUT,DELETE</param-value>
+ </init-param>
+ <init-param>
+ <param-name>cors.support.credentials</param-name>
+ <param-value>true</param-value>
+ </init-param>
+ </filter>
+ <filter-mapping>
+ <filter-name>CorsFilter</filter-name>
+ <url-pattern>/*</url-pattern>
+ </filter-mapping>
+</web-app>
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata-service/src/main/webapp/index.html
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata-service/src/main/webapp/index.html b/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata-service/src/main/webapp/index.html
new file mode 100644
index 0000000..5da5b32
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata-service/src/main/webapp/index.html
@@ -0,0 +1,18 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+Hello, this is UMP alert metadata service. You are welcome!
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata-service/src/test/java/org/apache/eagle/service/topology/resource/impl/TopologyMgmtResourceImplTest.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata-service/src/test/java/org/apache/eagle/service/topology/resource/impl/TopologyMgmtResourceImplTest.java b/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata-service/src/test/java/org/apache/eagle/service/topology/resource/impl/TopologyMgmtResourceImplTest.java
new file mode 100644
index 0000000..e46213e
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata-service/src/test/java/org/apache/eagle/service/topology/resource/impl/TopologyMgmtResourceImplTest.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.service.topology.resource.impl;
+
+import org.junit.Assert;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import java.util.List;
+
+
+public class TopologyMgmtResourceImplTest {
+ TopologyMgmtResourceImpl topologyManager = new TopologyMgmtResourceImpl();
+ String topologyName = "testStartTopology";
+
+ @Ignore
+ @Test
+ public void testStartTopology() throws Exception {
+ topologyManager.startTopology(topologyName);
+ Thread.sleep(10000);
+ }
+
+ @Ignore
+ @Test
+ public void testStopTopology() throws Exception {
+ topologyManager.startTopology(topologyName);
+ Thread.sleep(10000);
+ topologyManager.stopTopology(topologyName);
+ }
+
+ @Ignore
+ @Test
+ public void testGetTopologies() throws Exception {
+ topologyManager.startTopology(topologyName);
+ Thread.sleep(10000);
+ List<TopologyStatus> topologies = topologyManager.getTopologies();
+ Assert.assertTrue(topologies.size() == 1);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata/.gitignore
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata/.gitignore b/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata/.gitignore
new file mode 100644
index 0000000..1dd3331
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata/.gitignore
@@ -0,0 +1,2 @@
+/target/
+/target/
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata/pom.xml b/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata/pom.xml
new file mode 100644
index 0000000..3789dcd
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata/pom.xml
@@ -0,0 +1,52 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- ~ /* ~ * Licensed to the Apache Software Foundation (ASF) under one
+ or more ~ * contributor license agreements. See the NOTICE file distributed
+ with ~ * this work for additional information regarding copyright ownership.
+ ~ * The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ * (the "License"); you may not use this file except in compliance with
+ ~ * the License. You may obtain a copy of the License at ~ * ~ * http://www.apache.org/licenses/LICENSE-2.0
+ ~ * ~ * Unless required by applicable law or agreed to in writing, software
+ ~ * distributed under the License is distributed on an "AS IS" BASIS, ~ *
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ * See the License for the specific language governing permissions and ~
+ * limitations under the License. ~ */ -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>io.sherlock</groupId>
+ <artifactId>alert-metadata-parent</artifactId>
+ <version>0.0.1-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>alert-metadata</artifactId>
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <!-- Storm depends on org.ow2.asm:asm:4.0 -->
+ <!-- Jersey depends on asm:asm:3.0 -->
+
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>io.sherlock</groupId>
+ <artifactId>alert-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.mongodb</groupId>
+ <artifactId>mongo-java-driver</artifactId>
+ <version>${mongodb.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>de.flapdoodle.embed</groupId>
+ <artifactId>de.flapdoodle.embed.mongo</artifactId>
+ <version>1.50.5</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/impl/InMemMetadataDaoImpl.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/impl/InMemMetadataDaoImpl.java b/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/impl/InMemMetadataDaoImpl.java
new file mode 100644
index 0000000..1773042
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/impl/InMemMetadataDaoImpl.java
@@ -0,0 +1,333 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.metadata.impl;
+
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Optional;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.function.Predicate;
+
+import org.apache.eagle.alert.coordination.model.Kafka2TupleMetadata;
+import org.apache.eagle.alert.coordination.model.ScheduleState;
+import org.apache.eagle.alert.coordination.model.internal.PolicyAssignment;
+import org.apache.eagle.alert.coordination.model.internal.Topology;
+import org.apache.eagle.alert.engine.coordinator.PolicyDefinition;
+import org.apache.eagle.alert.engine.coordinator.Publishment;
+import org.apache.eagle.alert.engine.coordinator.PublishmentType;
+import org.apache.eagle.alert.engine.coordinator.StreamDefinition;
+import org.apache.eagle.alert.engine.coordinator.StreamingCluster;
+import org.apache.eagle.alert.metadata.resource.IMetadataDao;
+import org.apache.eagle.alert.metadata.resource.Models;
+import org.apache.eagle.alert.metadata.resource.OpResult;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.typesafe.config.Config;
+
+/**
+ * In memory service for simple service start. Make all service API as
+ * synchronized.
+ *
+ * @since Apr 11, 2016
+ *
+ */
+public class InMemMetadataDaoImpl implements IMetadataDao {
+
+ private static final Logger LOG = LoggerFactory.getLogger(InMemMetadataDaoImpl.class);
+
+ private List<StreamingCluster> clusters = new ArrayList<StreamingCluster>();
+ private List<StreamDefinition> schemas = new ArrayList<StreamDefinition>();
+ private List<Kafka2TupleMetadata> datasources = new ArrayList<Kafka2TupleMetadata>();
+ private List<PolicyDefinition> policies = new ArrayList<PolicyDefinition>();
+ private List<Publishment> publishments = new ArrayList<Publishment>();
+ private List<PublishmentType> publishmentTypes = new ArrayList<PublishmentType>();
+ private volatile int maxScheduleState = 100;
+ private SortedMap<String, ScheduleState> scheduleStates = new TreeMap<String, ScheduleState>();
+ private List<PolicyAssignment> assignments = new ArrayList<PolicyAssignment>();
+ private List<Topology> topologies = new ArrayList<Topology>();
+
+ public InMemMetadataDaoImpl(Config config) {
+ }
+
+ @Override
+ public synchronized List<StreamingCluster> listClusters() {
+ return clusters;
+ }
+
+ @Override
+ public OpResult addCluster(final StreamingCluster cluster) {
+ return addOrReplace(clusters, cluster);
+ }
+
+ private synchronized <T> OpResult addOrReplace(List<T> clusters, T paramT) {
+ Optional<T> scOp = clusters.stream().filter(new Predicate<T>() {
+ @Override
+ public boolean test(T t) {
+ if (getKey(t).equalsIgnoreCase(getKey(paramT))) {
+ return true;
+ }
+ return false;
+ }
+ }).findFirst();
+
+ OpResult result = new OpResult();
+ // replace
+ if (scOp.isPresent()) {
+ clusters.remove(scOp.get());
+ result.message = "replace the old one!";
+ } else {
+ result.message = "created new config!";
+ }
+ result.code = 200;
+ clusters.add(paramT);
+ return result;
+ }
+
+ public static <T> String getKey(T t) {
+ if (t instanceof StreamDefinition) {
+ return ((StreamDefinition) t).getStreamId();
+ } else if (t instanceof PublishmentType) {
+ return ((PublishmentType) t).getType();
+ }
+
+ try {
+ Method m = t.getClass().getMethod("getName");
+ return (String) m.invoke(t);
+ } catch (NoSuchMethodException | SecurityException | InvocationTargetException | IllegalAccessException
+ | IllegalArgumentException e) {
+ LOG.error(" getName not found on given class :" + t.getClass().getName());
+ }
+ throw new RuntimeException(String.format("no getName() found on target class %s for matching", t.getClass()
+ .getName()));
+ }
+
+ @SuppressWarnings("unchecked")
+ private synchronized <T> OpResult remove(List<T> clusters, String id) {
+ T[] matched = (T[]) clusters.stream().filter(new Predicate<T>() {
+
+ @Override
+ public boolean test(T t) {
+ if (getKey(t).equalsIgnoreCase(id)) {
+ return true;
+ }
+ return false;
+ }
+ }).toArray();
+
+ OpResult result = new OpResult();
+ result.code = 200;
+ if (clusters.removeAll(Arrays.asList(matched))) {
+ result.message = "removed configuration item succeed";
+ } else {
+ result.message = "no configuration item removed";
+ }
+ return result;
+ }
+
+ @Override
+ public OpResult removeCluster(final String clusterId) {
+ return remove(clusters, clusterId);
+ }
+
+ @Override
+ public synchronized List<StreamDefinition> listStreams() {
+ return schemas;
+ }
+
+ @Override
+ public OpResult createStream(StreamDefinition stream) {
+ return addOrReplace(schemas, stream);
+ }
+
+ @Override
+ public OpResult removeStream(String streamId) {
+ return remove(schemas, streamId);
+ }
+
+ @Override
+ public synchronized List<Kafka2TupleMetadata> listDataSources() {
+ return datasources;
+ }
+
+ @Override
+ public OpResult addDataSource(Kafka2TupleMetadata dataSource) {
+ return addOrReplace(datasources, dataSource);
+ }
+
+ @Override
+ public OpResult removeDataSource(String datasourceId) {
+ return remove(datasources, datasourceId);
+ }
+
+ @Override
+ public synchronized List<PolicyDefinition> listPolicies() {
+ return policies;
+ }
+
+ @Override
+ public OpResult addPolicy(PolicyDefinition policy) {
+ return addOrReplace(policies, policy);
+ }
+
+ @Override
+ public OpResult removePolicy(String policyId) {
+ return remove(policies, policyId);
+ }
+
+ @Override
+ public synchronized List<Publishment> listPublishment() {
+ return publishments;
+ }
+
+ @Override
+ public OpResult addPublishment(Publishment publishment) {
+ return addOrReplace(publishments, publishment);
+ }
+
+ @Override
+ public OpResult removePublishment(String pubId) {
+ return remove(publishments, pubId);
+ }
+
+ @Override
+ public List<PublishmentType> listPublishmentType() {
+ return publishmentTypes;
+ }
+
+ @Override
+ public OpResult addPublishmentType(PublishmentType publishmentType) {
+ return addOrReplace(publishmentTypes, publishmentType);
+ }
+
+ @Override
+ public OpResult removePublishmentType(String pubType) {
+ return remove(publishmentTypes, pubType);
+ }
+
+ @Override
+ public ScheduleState getScheduleState(String versionId) {
+ return scheduleStates.get(versionId);
+ }
+
+ @Override
+ public OpResult addScheduleState(ScheduleState state) {
+ // FIXME : might concurrent issue
+ String toRemove = null;
+ if (scheduleStates.size() > maxScheduleState) {
+ toRemove = scheduleStates.firstKey();
+ }
+ scheduleStates.put(state.getVersion(), state);
+ if (toRemove != null) {
+ scheduleStates.remove(toRemove);
+ }
+
+ OpResult result = new OpResult();
+ result.code = 200;
+ result.message = "OK";
+ return result;
+ }
+
+ @Override
+ public ScheduleState getScheduleState() {
+ if (scheduleStates.size() > 0) {
+ return scheduleStates.get(scheduleStates.lastKey());
+ }
+ return null;
+ }
+
+ @Override
+ public List<PolicyAssignment> listAssignments() {
+ return assignments;
+ }
+
+ @Override
+ public OpResult addAssignment(PolicyAssignment assignment) {
+ OpResult result = new OpResult();
+ result.code = 200;
+ result.message = "OK";
+ assignments.add(assignment);
+ return result;
+ }
+
+ @Override
+ public List<Topology> listTopologies() {
+ return topologies;
+ }
+
+ @Override
+ public OpResult addTopology(Topology t) {
+ return addOrReplace(topologies, t);
+ }
+
+ @Override
+ public OpResult removeTopology(String topologyName) {
+ return remove(topologies, topologyName);
+ }
+
+ @Override
+ public OpResult clear() {
+ this.assignments.clear();
+ this.clusters.clear();
+ this.datasources.clear();
+ this.policies.clear();
+ this.publishments.clear();
+ this.scheduleStates.clear();
+ this.schemas.clear();
+ this.topologies.clear();
+ OpResult result = new OpResult();
+ result.code = 200;
+ result.message = "OK";
+ return result;
+ }
+
+ @Override
+ public Models export() {
+ Models models = new Models();
+ models.assignments.addAll(this.assignments);
+ models.clusters.addAll(this.clusters);
+ models.datasources.addAll(this.datasources);
+ models.policies.addAll(this.policies);
+ models.publishments.addAll(this.publishments);
+ models.scheduleStates.putAll(this.scheduleStates);
+ models.schemas.addAll(this.schemas);
+ models.topologies.addAll(this.topologies);
+ return models;
+ }
+
+ @Override
+ public OpResult importModels(Models models) {
+ clear();
+ this.assignments.addAll(models.assignments);
+ this.clusters.addAll(models.clusters);
+ this.datasources.addAll(models.datasources);
+ this.policies.addAll(models.policies);
+ this.publishments.addAll(models.publishments);
+ this.scheduleStates.putAll(models.scheduleStates);
+ this.schemas.addAll(models.schemas);
+ this.topologies.addAll(models.topologies);
+ OpResult result = new OpResult();
+ result.code = 200;
+ result.message = "OK";
+ return result;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/impl/MetadataDaoFactory.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/impl/MetadataDaoFactory.java b/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/impl/MetadataDaoFactory.java
new file mode 100644
index 0000000..94717ce
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/impl/MetadataDaoFactory.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.metadata.impl;
+
+import java.lang.reflect.Constructor;
+
+import org.apache.eagle.alert.metadata.resource.IMetadataDao;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+
+/**
+ * @since Apr 12, 2016
+ *
+ */
+public class MetadataDaoFactory {
+
+ private static final MetadataDaoFactory INSTANCE = new MetadataDaoFactory();
+ private static final Logger LOG = LoggerFactory.getLogger(MetadataDaoFactory.class);
+
+ private IMetadataDao dao;
+
+ private MetadataDaoFactory() {
+ Config config = ConfigFactory.load();
+ Config datastoreConfig = config.getConfig("datastore");
+ if (datastoreConfig == null) {
+ LOG.warn("datastore is not configured, use in-memory store !!!");
+ dao = new InMemMetadataDaoImpl(datastoreConfig);
+ } else {
+ String clsName = datastoreConfig.getString("metadataDao");
+ Class<?> clz;
+ try {
+ clz = Thread.currentThread().getContextClassLoader().loadClass(clsName);
+ if (IMetadataDao.class.isAssignableFrom(clz)) {
+ Constructor<?> cotr = clz.getConstructor(Config.class);
+ dao = (IMetadataDao) cotr.newInstance(datastoreConfig);
+ } else {
+ throw new Exception("metadataDao configuration need to be implementation of IMetadataDao! ");
+ }
+ } catch (Exception e) {
+ LOG.error("error when initialize the dao, fall back to in memory mode!", e);
+ dao = new InMemMetadataDaoImpl(datastoreConfig);
+ }
+ }
+ }
+
+ public static MetadataDaoFactory getInstance() {
+ return INSTANCE;
+ }
+
+ public IMetadataDao getMetadataDao() {
+ return dao;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/impl/MongoMetadataDaoImpl.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/impl/MongoMetadataDaoImpl.java b/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/impl/MongoMetadataDaoImpl.java
new file mode 100644
index 0000000..a46f0c7
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/impl/MongoMetadataDaoImpl.java
@@ -0,0 +1,380 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.metadata.impl;
+
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.eagle.alert.coordination.model.Kafka2TupleMetadata;
+import org.apache.eagle.alert.coordination.model.ScheduleState;
+import org.apache.eagle.alert.coordination.model.internal.PolicyAssignment;
+import org.apache.eagle.alert.coordination.model.internal.Topology;
+import org.apache.eagle.alert.engine.coordinator.PolicyDefinition;
+import org.apache.eagle.alert.engine.coordinator.Publishment;
+import org.apache.eagle.alert.engine.coordinator.PublishmentType;
+import org.apache.eagle.alert.engine.coordinator.StreamDefinition;
+import org.apache.eagle.alert.engine.coordinator.StreamingCluster;
+import org.apache.eagle.alert.metadata.resource.IMetadataDao;
+import org.apache.eagle.alert.metadata.resource.Models;
+import org.apache.eagle.alert.metadata.resource.OpResult;
+import org.bson.BsonDocument;
+import org.bson.BsonInt32;
+import org.bson.BsonString;
+import org.bson.Document;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.mongodb.Function;
+import com.mongodb.MongoClient;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.client.MongoDatabase;
+import com.mongodb.client.model.IndexOptions;
+import com.mongodb.client.model.UpdateOptions;
+import com.mongodb.client.result.DeleteResult;
+import com.mongodb.client.result.UpdateResult;
+import com.typesafe.config.Config;
+
+/**
+ * @since Apr 11, 2016
+ *
+ */
+public class MongoMetadataDaoImpl implements IMetadataDao {
+
+ private static final String DB_NAME = "ump_alert_metadata";
+ private static final Logger LOG = LoggerFactory.getLogger(MongoMetadataDaoImpl.class);
+ private static final ObjectMapper mapper = new ObjectMapper();
+ static {
+ mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+ }
+
+ private final String connection;
+ private final MongoClient client;
+
+ private MongoDatabase db;
+ private MongoCollection<Document> cluster;
+ private MongoCollection<Document> schema;
+ private MongoCollection<Document> datasource;
+ private MongoCollection<Document> policy;
+ private MongoCollection<Document> publishment;
+ private MongoCollection<Document> publishmentType;
+ private MongoCollection<Document> scheduleStates;
+ private MongoCollection<Document> assignments;
+ private MongoCollection<Document> topologies;
+
+ public MongoMetadataDaoImpl(Config config) {
+ this.connection = config.getString("connection");
+ this.client = new MongoClient(connection);
+ init();
+ }
+
+ private void init() {
+ db = client.getDatabase(DB_NAME);
+ IndexOptions io = new IndexOptions().background(true).unique(true).name("nameIndex");
+ BsonDocument doc = new BsonDocument();
+ doc.append("name", new BsonInt32(1));
+ cluster = db.getCollection("clusters");
+ cluster.createIndex(doc, io);
+ {
+ BsonDocument doc2 = new BsonDocument();
+ doc2.append("streamId", new BsonInt32(1));
+ schema = db.getCollection("schemas");
+ schema.createIndex(doc2, io);
+ }
+ datasource = db.getCollection("datasources");
+ datasource.createIndex(doc, io);
+ policy = db.getCollection("policies");
+ policy.createIndex(doc, io);
+ publishment = db.getCollection("publishments");
+ publishment.createIndex(doc, io);
+ topologies = db.getCollection("topologies");
+ topologies.createIndex(doc, io);
+
+ publishmentType = db.getCollection("publishmentTypes");
+ {
+ IndexOptions io1 = new IndexOptions().background(true).unique(true).name("pubTypeIndex");
+ BsonDocument doc1 = new BsonDocument();
+ doc1.append("type", new BsonInt32(1));
+ publishmentType.createIndex(doc1, io1);
+ }
+
+ scheduleStates = db.getCollection("schedule_specs");
+ {
+ IndexOptions io1 = new IndexOptions().background(true).unique(true).name("nameIndex");
+ BsonDocument doc1 = new BsonDocument();
+ doc1.append("version", new BsonInt32(1));
+ scheduleStates.createIndex(doc1, io1);
+ }
+
+ assignments = db.getCollection("assignments");
+ {
+ IndexOptions io1 = new IndexOptions().background(true).unique(true).name("policyNameIndex");
+ BsonDocument doc1 = new BsonDocument();
+ doc1.append("policyName", new BsonInt32(1));
+ assignments.createIndex(doc1, io1);
+ }
+ }
+
+ @Override
+ public List<StreamingCluster> listClusters() {
+ return list(cluster, StreamingCluster.class);
+ }
+
+ private <T> List<T> list(MongoCollection<Document> collection, Class<T> clz) {
+ List<T> result = new LinkedList<T>();
+ collection.find().map(new Function<Document, T>() {
+ @Override
+ public T apply(Document t) {
+ String json = t.toJson();
+ try {
+ return mapper.readValue(json, clz);
+ } catch (IOException e) {
+ LOG.error("deserialize config item failed!", e);
+ }
+ return null;
+ }
+ }).into(result);
+ return result;
+ }
+
+ private <T> OpResult addOrReplace(MongoCollection<Document> collection, T t) {
+ BsonDocument filter = new BsonDocument();
+ if (t instanceof StreamDefinition) {
+ filter.append("streamId", new BsonString(InMemMetadataDaoImpl.getKey(t)));
+ } else {
+ filter.append("name", new BsonString(InMemMetadataDaoImpl.getKey(t)));
+ }
+
+ String json = "";
+ OpResult result = new OpResult();
+ try {
+ json = mapper.writeValueAsString(t);
+ UpdateOptions options = new UpdateOptions();
+ options.upsert(true);
+ UpdateResult ur = collection.replaceOne(filter, Document.parse(json), options);
+ // FIXME: could based on matched count do better matching...
+ if (ur.getModifiedCount() > 0 || ur.getUpsertedId() != null) {
+ result.code = 200;
+ result.message = String.format("update %d configuration item.", ur.getModifiedCount());
+ } else {
+ result.code = 500;
+ result.message = "no configuration item create/updated.";
+ }
+ } catch (Exception e) {
+ result.code = 500;
+ result.message = e.getMessage();
+ LOG.error("", e);
+ }
+ return result;
+ }
+
+ private <T> OpResult remove(MongoCollection<Document> collection, String name) {
+ BsonDocument filter = new BsonDocument();
+ filter.append("name", new BsonString(name));
+ DeleteResult dr = collection.deleteOne(filter);
+ OpResult result = new OpResult();
+ result.code = 200;
+ result.message = String.format(" %d config item removed!", dr.getDeletedCount());
+ return result;
+ }
+
+ @Override
+ public OpResult addCluster(StreamingCluster cluster) {
+ return addOrReplace(this.cluster, cluster);
+ }
+
+ @Override
+ public OpResult removeCluster(String clusterId) {
+ return remove(cluster, clusterId);
+ }
+
+ @Override
+ public List<StreamDefinition> listStreams() {
+ return list(schema, StreamDefinition.class);
+ }
+
+ @Override
+ public OpResult createStream(StreamDefinition stream) {
+ return addOrReplace(this.schema, stream);
+ }
+
+ @Override
+ public OpResult removeStream(String streamId) {
+ return remove(schema, streamId);
+ }
+
+ @Override
+ public List<Kafka2TupleMetadata> listDataSources() {
+ return list(datasource, Kafka2TupleMetadata.class);
+ }
+
+ @Override
+ public OpResult addDataSource(Kafka2TupleMetadata dataSource) {
+ return addOrReplace(this.datasource, dataSource);
+ }
+
+ @Override
+ public OpResult removeDataSource(String datasourceId) {
+ return remove(datasource, datasourceId);
+ }
+
+ @Override
+ public List<PolicyDefinition> listPolicies() {
+ return list(policy, PolicyDefinition.class);
+ }
+
+ @Override
+ public OpResult addPolicy(PolicyDefinition policy) {
+ return addOrReplace(this.policy, policy);
+ }
+
+ @Override
+ public OpResult removePolicy(String policyId) {
+ return remove(policy, policyId);
+ }
+
+ @Override
+ public List<Publishment> listPublishment() {
+ return list(publishment, Publishment.class);
+ }
+
+ @Override
+ public OpResult addPublishment(Publishment publishment) {
+ return addOrReplace(this.publishment, publishment);
+ }
+
+ @Override
+ public OpResult removePublishment(String pubId) {
+ return remove(publishment, pubId);
+ }
+
+ @Override
+ public List<PublishmentType> listPublishmentType() {
+ return list(publishmentType, PublishmentType.class);
+ }
+
+ @Override
+ public OpResult addPublishmentType(PublishmentType pubType) {
+ return addOrReplace(this.publishmentType, pubType);
+ }
+
+ @Override
+ public OpResult removePublishmentType(String pubType) {
+ return remove(publishmentType, pubType);
+ }
+
+ @Override
+ public ScheduleState getScheduleState(String versionId) {
+ BsonDocument doc = new BsonDocument();
+ doc.append("version", new BsonString(versionId));
+ ScheduleState state = scheduleStates.find(doc).map(new Function<Document, ScheduleState>() {
+ @Override
+ public ScheduleState apply(Document t) {
+ String json = t.toJson();
+ try {
+ return mapper.readValue(json, ScheduleState.class);
+ } catch (IOException e) {
+ LOG.error("deserialize config item failed!", e);
+ }
+ return null;
+ }
+ }).first();
+ return state;
+ }
+
+ @Override
+ public OpResult addScheduleState(ScheduleState state) {
+ return addOne(scheduleStates, state);
+ }
+
+ private <T> OpResult addOne(MongoCollection<Document> collection, T t) {
+ OpResult result = new OpResult();
+ try {
+ String json = mapper.writeValueAsString(t);
+ collection.insertOne(Document.parse(json));
+ result.code = 200;
+ result.message = "add state succeed!";
+ } catch (Exception e) {
+ result.code = 400;
+ result.message = e.getMessage();
+ LOG.error("", e);
+ }
+ return result;
+ }
+
+ @Override
+ public ScheduleState getScheduleState() {
+ BsonDocument sort = new BsonDocument();
+ sort.append("generateTime", new BsonInt32(-1));
+ ScheduleState state = scheduleStates.find().sort(sort).map(new Function<Document, ScheduleState>() {
+ @Override
+ public ScheduleState apply(Document t) {
+ String json = t.toJson();
+ try {
+ return mapper.readValue(json, ScheduleState.class);
+ } catch (IOException e) {
+ LOG.error("deserialize config item failed!", e);
+ }
+ return null;
+ }
+ }).first();
+ return state;
+ }
+
+ @Override
+ public List<PolicyAssignment> listAssignments() {
+ return list(assignments, PolicyAssignment.class);
+ }
+
+ @Override
+ public OpResult addAssignment(PolicyAssignment assignment) {
+ return addOne(assignments, assignment);
+ }
+
+ @Override
+ public List<Topology> listTopologies() {
+ return list(topologies, Topology.class);
+ }
+
+ @Override
+ public OpResult addTopology(Topology t) {
+ return addOrReplace(this.topologies, t);
+ }
+
+ @Override
+ public OpResult removeTopology(String topologyName) {
+ return remove(topologies, topologyName);
+ }
+
+ @Override
+ public OpResult clear() {
+ throw new UnsupportedOperationException("clear not support!");
+ }
+
+ @Override
+ public Models export() {
+ throw new UnsupportedOperationException("export not support!");
+ }
+
+ @Override
+ public OpResult importModels(Models models) {
+ throw new UnsupportedOperationException("importModels not support!");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/impl/jdbc/JdbcMetadataDaoImpl.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/impl/jdbc/JdbcMetadataDaoImpl.java b/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/impl/jdbc/JdbcMetadataDaoImpl.java
new file mode 100644
index 0000000..9ec119a
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/impl/jdbc/JdbcMetadataDaoImpl.java
@@ -0,0 +1,292 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.metadata.impl.jdbc;
+
+import java.util.List;
+
+import org.apache.eagle.alert.coordination.model.Kafka2TupleMetadata;
+import org.apache.eagle.alert.coordination.model.ScheduleState;
+import org.apache.eagle.alert.coordination.model.internal.PolicyAssignment;
+import org.apache.eagle.alert.coordination.model.internal.Topology;
+import org.apache.eagle.alert.engine.coordinator.PolicyDefinition;
+import org.apache.eagle.alert.engine.coordinator.Publishment;
+import org.apache.eagle.alert.engine.coordinator.PublishmentType;
+import org.apache.eagle.alert.engine.coordinator.StreamDefinition;
+import org.apache.eagle.alert.engine.coordinator.StreamingCluster;
+import org.apache.eagle.alert.metadata.resource.OpResult;
+import org.apache.eagle.alert.metadata.resource.IMetadataDao;
+import org.apache.eagle.alert.metadata.resource.Models;
+
+/**
+ * @since May 26, 2016
+ *
+ */
+public class JdbcMetadataDaoImpl implements IMetadataDao {
+
+ /* (non-Javadoc)
+ * @see org.apache.eagle.service.alert.resource.IMetadataDao#listTopologies()
+ */
+ @Override
+ public List<Topology> listTopologies() {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.eagle.service.alert.resource.IMetadataDao#addTopology(org.apache.eagle.alert.coordination.model.internal.Topology)
+ */
+ @Override
+ public OpResult addTopology(Topology t) {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.eagle.service.alert.resource.IMetadataDao#removeTopology(java.lang.String)
+ */
+ @Override
+ public OpResult removeTopology(String topologyName) {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.eagle.service.alert.resource.IMetadataDao#listClusters()
+ */
+ @Override
+ public List<StreamingCluster> listClusters() {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.eagle.service.alert.resource.IMetadataDao#addCluster(org.apache.eagle.alert.engine.coordinator.StreamingCluster)
+ */
+ @Override
+ public OpResult addCluster(StreamingCluster cluster) {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.eagle.service.alert.resource.IMetadataDao#removeCluster(java.lang.String)
+ */
+ @Override
+ public OpResult removeCluster(String clusterId) {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.eagle.service.alert.resource.IMetadataDao#listStreams()
+ */
+ @Override
+ public List<StreamDefinition> listStreams() {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.eagle.service.alert.resource.IMetadataDao#createStream(org.apache.eagle.alert.engine.coordinator.StreamDefinition)
+ */
+ @Override
+ public OpResult createStream(StreamDefinition stream) {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.eagle.service.alert.resource.IMetadataDao#removeStream(java.lang.String)
+ */
+ @Override
+ public OpResult removeStream(String streamId) {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.eagle.service.alert.resource.IMetadataDao#listDataSources()
+ */
+ @Override
+ public List<Kafka2TupleMetadata> listDataSources() {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.eagle.service.alert.resource.IMetadataDao#addDataSource(org.apache.eagle.alert.coordination.model.Kafka2TupleMetadata)
+ */
+ @Override
+ public OpResult addDataSource(Kafka2TupleMetadata dataSource) {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.eagle.service.alert.resource.IMetadataDao#removeDataSource(java.lang.String)
+ */
+ @Override
+ public OpResult removeDataSource(String datasourceId) {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.eagle.service.alert.resource.IMetadataDao#listPolicies()
+ */
+ @Override
+ public List<PolicyDefinition> listPolicies() {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.eagle.service.alert.resource.IMetadataDao#addPolicy(org.apache.eagle.alert.engine.coordinator.PolicyDefinition)
+ */
+ @Override
+ public OpResult addPolicy(PolicyDefinition policy) {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.eagle.service.alert.resource.IMetadataDao#removePolicy(java.lang.String)
+ */
+ @Override
+ public OpResult removePolicy(String policyId) {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.eagle.service.alert.resource.IMetadataDao#listPublishment()
+ */
+ @Override
+ public List<Publishment> listPublishment() {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.eagle.service.alert.resource.IMetadataDao#addPublishment(org.apache.eagle.alert.engine.coordinator.Publishment)
+ */
+ @Override
+ public OpResult addPublishment(Publishment publishment) {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.eagle.service.alert.resource.IMetadataDao#removePublishment(java.lang.String)
+ */
+ @Override
+ public OpResult removePublishment(String pubId) {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.eagle.service.alert.resource.IMetadataDao#getScheduleState(java.lang.String)
+ */
+ @Override
+ public ScheduleState getScheduleState(String versionId) {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.eagle.service.alert.resource.IMetadataDao#getScheduleState()
+ */
+ @Override
+ public ScheduleState getScheduleState() {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.eagle.service.alert.resource.IMetadataDao#addScheduleState(org.apache.eagle.alert.coordination.model.ScheduleState)
+ */
+ @Override
+ public OpResult addScheduleState(ScheduleState state) {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.eagle.service.alert.resource.IMetadataDao#listAssignments()
+ */
+ @Override
+ public List<PolicyAssignment> listAssignments() {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.eagle.service.alert.resource.IMetadataDao#addAssignment(org.apache.eagle.alert.coordination.model.internal.PolicyAssignment)
+ */
+ @Override
+ public OpResult addAssignment(PolicyAssignment assignment) {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.eagle.service.alert.resource.IMetadataDao#clear()
+ */
+ @Override
+ public OpResult clear() {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.eagle.service.alert.resource.IMetadataDao#export()
+ */
+ @Override
+ public Models export() {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.eagle.service.alert.resource.IMetadataDao#importModels(org.apache.eagle.service.alert.resource.Models)
+ */
+ @Override
+ public OpResult importModels(Models models) {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public List<PublishmentType> listPublishmentType() {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public OpResult addPublishmentType(PublishmentType publishmentType) {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public OpResult removePublishmentType(String pubType) {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/resource/IMetadataDao.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/resource/IMetadataDao.java b/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/resource/IMetadataDao.java
new file mode 100644
index 0000000..8eef985
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/resource/IMetadataDao.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.metadata.resource;
+
+import java.util.List;
+
+import org.apache.eagle.alert.coordination.model.Kafka2TupleMetadata;
+import org.apache.eagle.alert.coordination.model.ScheduleState;
+import org.apache.eagle.alert.coordination.model.internal.PolicyAssignment;
+import org.apache.eagle.alert.coordination.model.internal.Topology;
+import org.apache.eagle.alert.engine.coordinator.*;
+
+public interface IMetadataDao {
+
+ List<Topology> listTopologies();
+
+ OpResult addTopology(Topology t);
+
+ OpResult removeTopology(String topologyName);
+
+ List<StreamingCluster> listClusters();
+
+ OpResult addCluster(StreamingCluster cluster);
+
+ OpResult removeCluster(String clusterId);
+
+ List<StreamDefinition> listStreams();
+
+ OpResult createStream(StreamDefinition stream);
+
+ OpResult removeStream(String streamId);
+
+ List<Kafka2TupleMetadata> listDataSources();
+
+ OpResult addDataSource(Kafka2TupleMetadata dataSource);
+
+ OpResult removeDataSource(String datasourceId);
+
+ List<PolicyDefinition> listPolicies();
+
+ OpResult addPolicy(PolicyDefinition policy);
+
+ OpResult removePolicy(String policyId);
+
+ List<Publishment> listPublishment();
+
+ OpResult addPublishment(Publishment publishment);
+
+ OpResult removePublishment(String pubId);
+
+ List<PublishmentType> listPublishmentType();
+
+ OpResult addPublishmentType(PublishmentType publishmentType);
+
+ OpResult removePublishmentType(String pubType);
+
+ ScheduleState getScheduleState(String versionId);
+
+ ScheduleState getScheduleState();
+
+ OpResult addScheduleState(ScheduleState state);
+
+ List<PolicyAssignment> listAssignments();
+
+ OpResult addAssignment(PolicyAssignment assignment);
+
+ // APIs for test friendly
+ OpResult clear();
+ Models export();
+ OpResult importModels(Models models);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/resource/Models.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/resource/Models.java b/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/resource/Models.java
new file mode 100644
index 0000000..6bb38f1
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/resource/Models.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.metadata.resource;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+import org.apache.eagle.alert.coordination.model.Kafka2TupleMetadata;
+import org.apache.eagle.alert.coordination.model.ScheduleState;
+import org.apache.eagle.alert.coordination.model.internal.PolicyAssignment;
+import org.apache.eagle.alert.coordination.model.internal.Topology;
+import org.apache.eagle.alert.engine.coordinator.PolicyDefinition;
+import org.apache.eagle.alert.engine.coordinator.Publishment;
+import org.apache.eagle.alert.engine.coordinator.StreamDefinition;
+import org.apache.eagle.alert.engine.coordinator.StreamingCluster;
+
+/**
+ * This models used for metadata export/import to easy of test.
+ *
+ * @since May 23, 2016
+ *
+ */
+public class Models {
+ public List<StreamingCluster> clusters = new ArrayList<StreamingCluster>();
+ public List<StreamDefinition> schemas = new ArrayList<StreamDefinition>();
+ public List<Kafka2TupleMetadata> datasources = new ArrayList<Kafka2TupleMetadata>();
+ public List<PolicyDefinition> policies = new ArrayList<PolicyDefinition>();
+ public List<Publishment> publishments = new ArrayList<Publishment>();
+ public SortedMap<String, ScheduleState> scheduleStates = new TreeMap<String, ScheduleState>();
+ public List<PolicyAssignment> assignments = new ArrayList<PolicyAssignment>();
+ public List<Topology> topologies = new ArrayList<Topology>();
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/resource/OpResult.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/resource/OpResult.java b/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/resource/OpResult.java
new file mode 100644
index 0000000..44f507d
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/resource/OpResult.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.metadata.resource;
+
+/**
+ * @since Apr 11, 2016
+ *
+ */
+public class OpResult {
+
+ public int code = 200;
+ public String message = "";
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata/src/main/resources/application.conf
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata/src/main/resources/application.conf b/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata/src/main/resources/application.conf
new file mode 100644
index 0000000..efba546
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata/src/main/resources/application.conf
@@ -0,0 +1,8 @@
+{
+ "datastore": {
+ "metadataDao": "org.apache.eagle.service.alert.resource.impl.InMemMetadataDaoImpl",
+ "connection": "localhost:27017",
+ "properties" : {
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata/src/test/java/org/apache/eagle/service/alert/resource/impl/InMemoryTest.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata/src/test/java/org/apache/eagle/service/alert/resource/impl/InMemoryTest.java b/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata/src/test/java/org/apache/eagle/service/alert/resource/impl/InMemoryTest.java
new file mode 100644
index 0000000..54a2ffc
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-metadata-parent/alert-metadata/src/test/java/org/apache/eagle/service/alert/resource/impl/InMemoryTest.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.service.alert.resource.impl;
+
+import org.apache.eagle.alert.engine.coordinator.PolicyDefinition;
+import org.apache.eagle.alert.metadata.impl.InMemMetadataDaoImpl;
+import org.apache.eagle.alert.metadata.resource.IMetadataDao;
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.typesafe.config.ConfigFactory;
+
+/**
+ * @since May 1, 2016
+ *
+ */
+public class InMemoryTest {
+
+ private IMetadataDao dao = new InMemMetadataDaoImpl(ConfigFactory.load());
+
+ @Test
+ public void test_AddPolicy() {
+ PolicyDefinition pd = new PolicyDefinition();
+ pd.setName("pd1");
+ dao.addPolicy(pd);
+
+ Assert.assertEquals(1, dao.listPolicies().size());
+ }
+}
[08/19] incubator-eagle git commit: EAGLE-324: Init branch-v0.5
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/impl/StreamSortWindowHandlerImpl.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/impl/StreamSortWindowHandlerImpl.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/impl/StreamSortWindowHandlerImpl.java
new file mode 100644
index 0000000..7be69e1
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/impl/StreamSortWindowHandlerImpl.java
@@ -0,0 +1,113 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.sorter.impl;
+
+import java.io.IOException;
+
+import org.apache.eagle.alert.engine.PartitionedEventCollector;
+import org.apache.eagle.alert.engine.coordinator.StreamSortSpec;
+import org.apache.eagle.alert.engine.model.PartitionedEvent;
+import org.apache.eagle.alert.engine.router.StreamSortHandler;
+import org.apache.eagle.alert.engine.sorter.StreamTimeClock;
+import org.apache.eagle.alert.engine.sorter.StreamWindow;
+import org.apache.eagle.alert.engine.sorter.StreamWindowManager;
+import org.apache.eagle.alert.utils.DateTimeUtil;
+import org.joda.time.Period;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class StreamSortWindowHandlerImpl implements StreamSortHandler {
+ private final static Logger LOG = LoggerFactory.getLogger(StreamSortWindowHandlerImpl.class);
+ private StreamWindowManager windowManager;
+ private StreamSortSpec streamSortSpecSpec;
+ private PartitionedEventCollector outputCollector;
+ private String streamId;
+
+ public void prepare(String streamId, StreamSortSpec streamSortSpecSpec, PartitionedEventCollector outputCollector) {
+ this.windowManager = new StreamWindowManagerImpl(
+ Period.parse(streamSortSpecSpec.getWindowPeriod()),
+ streamSortSpecSpec.getWindowMargin(),
+ PartitionedEventTimeOrderingComparator.INSTANCE,
+ outputCollector);
+ this.streamSortSpecSpec = streamSortSpecSpec;
+ this.streamId = streamId;
+ this.outputCollector = outputCollector;
+ }
+
+ /**
+ * Entry point to manage window lifecycle
+ *
+ * @param event StreamEvent
+ */
+ public void nextEvent(PartitionedEvent event) {
+ final long eventTime = event.getEvent().getTimestamp();
+ boolean handled = false;
+
+ synchronized (this.windowManager){
+ for (StreamWindow window : this.windowManager.getWindows()) {
+ if (window.alive() && window.add(event)) {
+ handled = true;
+ }
+ }
+
+ // No window found for the event but not too late being rejected
+ if (!handled && !windowManager.reject(eventTime)) {
+ // later then all events, create later window
+ StreamWindow window = windowManager.addNewWindow(eventTime);
+ if (window.add(event)) {
+ LOG.info("Created {} of {} at {}", window, this.streamId, DateTimeUtil.millisecondsToHumanDateWithMilliseconds(eventTime));
+ handled = true;
+ }
+ }
+ }
+
+ if(!handled){
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("Drop expired event {}", event);
+ }
+ outputCollector.drop(event);
+ }
+ }
+
+ @Override
+ public void onTick(StreamTimeClock clock,long globalSystemTime) {
+ windowManager.onTick(clock, globalSystemTime);
+ }
+
+ @Override
+ public void close() {
+ try {
+ windowManager.close();
+ } catch (IOException e) {
+ LOG.error("Got exception while closing window manager",e);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return super.toString();
+ }
+
+ @Override
+ public int hashCode(){
+ if(streamSortSpecSpec == null){
+ throw new NullPointerException("streamSortSpec is null");
+ }else{
+ return streamSortSpecSpec.hashCode();
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/impl/StreamSortedWindowInMapDB.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/impl/StreamSortedWindowInMapDB.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/impl/StreamSortedWindowInMapDB.java
new file mode 100644
index 0000000..c1c289d
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/impl/StreamSortedWindowInMapDB.java
@@ -0,0 +1,145 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.sorter.impl;
+
+import java.util.Arrays;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.commons.lang3.time.StopWatch;
+import org.apache.eagle.alert.engine.PartitionedEventCollector;
+import org.apache.eagle.alert.engine.model.PartitionedEvent;
+import org.apache.eagle.alert.engine.model.StreamEvent;
+import org.apache.eagle.alert.engine.sorter.BaseStreamWindow;
+import org.mapdb.BTreeMap;
+import org.mapdb.DB;
+import org.mapdb.Serializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * StreamSortedWindow based on MapDB to support off-heap or disk storage.
+ *
+ * Stable sorting algorithm
+ *
+ * <br/><br/>
+ *
+ * See <a href="http://www.mapdb.org">http://www.mapdb.org</a>
+ */
+public class StreamSortedWindowInMapDB extends BaseStreamWindow {
+ private final String mapId;
+ private BTreeMap<Long, PartitionedEvent[]> bTreeMap;
+ private final static Logger LOG = LoggerFactory.getLogger(StreamSortedWindowInMapDB.class);
+ private final AtomicInteger size;
+ private long replaceOpCount = 0;
+ private final static PartitionedEventGroupSerializer STREAM_EVENT_GROUP_SERIALIZER = new PartitionedEventGroupSerializer();
+
+ /**
+ * @param start
+ * @param end
+ * @param margin
+ * @param db
+ * @param mapId physical map id, used to decide whether to reuse or not
+ */
+ @SuppressWarnings("unused")
+ public StreamSortedWindowInMapDB(long start, long end, long margin,DB db,String mapId) {
+ super(start, end, margin);
+ this.mapId = mapId;
+ try {
+ bTreeMap = db.<Long, StreamEvent>treeMap(mapId).
+ keySerializer(Serializer.LONG).
+ valueSerializer(STREAM_EVENT_GROUP_SERIALIZER).
+ createOrOpen();
+ LOG.debug("Created BTree map {}",mapId);
+ } catch (Error error){
+ LOG.info("Failed create BTree {}",mapId,error);
+ }
+ size = new AtomicInteger(0);
+ }
+
+ /**
+ * Assumed: most of adding operation will do putting only and few require replacing.
+ *
+ * <ol>
+ * <li>
+ * First of all, always try to put with created event directly
+ * </li>
+ * <li>
+ * If not absent (key already exists), then append and replace,
+ * replace operation will cause more consumption
+ * </li>
+ * </ol>
+ * @param event coming-in event
+ * @return whether success
+ */
+ @Override
+ public synchronized boolean add(PartitionedEvent event) {
+ long timestamp = event.getEvent().getTimestamp();
+ if(accept(timestamp)) {
+ boolean absent = bTreeMap.putIfAbsentBoolean(timestamp, new PartitionedEvent[]{event});
+ if (!absent) {
+ size.incrementAndGet();
+ return true;
+ } else {
+ if(LOG.isDebugEnabled()) LOG.debug("Duplicated timestamp {}, will reduce performance as replacing",timestamp);
+ PartitionedEvent[] oldValue = bTreeMap.get(timestamp);
+ PartitionedEvent[] newValue = oldValue == null ? new PartitionedEvent[1]: Arrays.copyOf(oldValue,oldValue.length+1);
+ newValue[newValue.length-1] = event;
+ PartitionedEvent[] removedValue = bTreeMap.replace(timestamp,newValue);
+ replaceOpCount ++;
+ if(replaceOpCount % 1000 == 0){
+ LOG.warn("Too many events ({}) with overlap timestamp, may reduce insertion performance",replaceOpCount);
+ }
+ if(removedValue!=null) {
+ size.incrementAndGet();
+ } else {
+ throw new IllegalStateException("Failed to replace key "+timestamp+" with "+newValue.length+" entities array to replace old "+oldValue.length+" entities array");
+ }
+ return true;
+ }
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ protected synchronized void flush(PartitionedEventCollector collector) {
+ StopWatch stopWatch = new StopWatch();
+ stopWatch.start();
+ bTreeMap.valueIterator().forEachRemaining((events)->{
+ for(PartitionedEvent event:events){
+ collector.emit(event);
+ }
+ });
+ bTreeMap.clear();
+ replaceOpCount = 0;
+ stopWatch.stop();
+ LOG.info("Flushed {} events in {} ms", size, stopWatch.getTime());
+ size.set(0);
+ }
+
+ @Override
+ public synchronized void close(){
+ super.close();
+ bTreeMap.close();
+ LOG.info("Closed {}",this.mapId);
+ }
+
+ @Override
+ public synchronized int size() {
+ return size.get();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/impl/StreamSortedWindowOnHeap.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/impl/StreamSortedWindowOnHeap.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/impl/StreamSortedWindowOnHeap.java
new file mode 100644
index 0000000..d3b1d7d
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/impl/StreamSortedWindowOnHeap.java
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.sorter.impl;
+
+import java.util.Comparator;
+
+import org.apache.commons.lang3.time.StopWatch;
+import org.apache.eagle.alert.engine.PartitionedEventCollector;
+import org.apache.eagle.alert.engine.model.PartitionedEvent;
+import org.apache.eagle.alert.engine.sorter.BaseStreamWindow;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.TreeMultiset;
+
+public class StreamSortedWindowOnHeap extends BaseStreamWindow {
+ private final static Logger LOG = LoggerFactory.getLogger(StreamSortedWindowOnHeap.class);
+ private final TreeMultiset<PartitionedEvent> treeMultisetCache;
+
+ /**
+ * @param start start time
+ * @param end end time
+ * @param margin margin time
+ */
+ public StreamSortedWindowOnHeap(long start, long end, long margin, Comparator<PartitionedEvent> comparator ){
+ super(start,end,margin);
+ treeMultisetCache = TreeMultiset.create(comparator);
+ }
+
+ public StreamSortedWindowOnHeap(long start, long end, long margin){
+ this(start,end,margin,new PartitionedEventTimeOrderingComparator());
+ }
+
+ @Override
+ public boolean add(PartitionedEvent partitionedEvent) {
+ synchronized (treeMultisetCache) {
+ if (accept(partitionedEvent.getEvent().getTimestamp())) {
+ treeMultisetCache.add(partitionedEvent);
+ return true;
+ } else {
+ if(LOG.isDebugEnabled()) LOG.debug("{} is not acceptable, ignored", partitionedEvent);
+ return false;
+ }
+ }
+ }
+
+ @Override
+ protected void flush(PartitionedEventCollector collector) {
+ synchronized (treeMultisetCache) {
+ StopWatch stopWatch = new StopWatch();
+ stopWatch.start();
+ treeMultisetCache.forEach(collector::emit);
+ int size = treeMultisetCache.size();
+ treeMultisetCache.clear();
+ stopWatch.stop();
+ LOG.info("Flushed {} events in {} ms from {}", size, stopWatch.getTime(),this.toString());
+ }
+ }
+
+ @Override
+ public int size() {
+ synchronized (treeMultisetCache) {
+ return treeMultisetCache.size();
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/impl/StreamTimeClockInLocalMemory.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/impl/StreamTimeClockInLocalMemory.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/impl/StreamTimeClockInLocalMemory.java
new file mode 100644
index 0000000..91a4a37
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/impl/StreamTimeClockInLocalMemory.java
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.sorter.impl;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.eagle.alert.engine.sorter.StreamTimeClock;
+import org.apache.eagle.alert.utils.DateTimeUtil;
+
+
+/**
+ * In memory thread-safe time clock service
+ *
+ * TODO: maybe need to synchronize time clock globally, how to?
+ */
+public class StreamTimeClockInLocalMemory implements StreamTimeClock {
+ private final AtomicLong currentTime;
+ private final String streamId;
+
+ public StreamTimeClockInLocalMemory(String streamId,long initialTime){
+ this.streamId = streamId;
+ this.currentTime = new AtomicLong(initialTime);
+ }
+ public StreamTimeClockInLocalMemory(String streamId){
+ this(streamId,0L);
+ }
+
+ @Override
+ public void moveForward(long timestamp){
+ if(timestamp < currentTime.get()){
+ throw new IllegalArgumentException(timestamp +" < "+currentTime.get()+", should not move time back");
+ }
+ this.currentTime.set(timestamp);
+ }
+
+ @Override
+ public String getStreamId() {
+ return streamId;
+ }
+
+ @Override
+ public long getTime() {
+ return currentTime.get();
+ }
+
+ @Override
+ public String toString() {
+ return String.format("StreamClock[streamId=%s, now=%s]",streamId, DateTimeUtil.millisecondsToHumanDateWithMilliseconds(currentTime.get()));
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/impl/StreamTimeClockManagerImpl.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/impl/StreamTimeClockManagerImpl.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/impl/StreamTimeClockManagerImpl.java
new file mode 100644
index 0000000..741d7f0
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/impl/StreamTimeClockManagerImpl.java
@@ -0,0 +1,171 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.sorter.impl;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Timer;
+import java.util.TimerTask;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.eagle.alert.engine.sorter.StreamTimeClock;
+import org.apache.eagle.alert.engine.sorter.StreamTimeClockListener;
+import org.apache.eagle.alert.engine.sorter.StreamTimeClockManager;
+import org.apache.eagle.alert.utils.DateTimeUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public final class StreamTimeClockManagerImpl implements StreamTimeClockManager {
+ private static final long serialVersionUID = -2770823821511195343L;
+ private final static Logger LOG = LoggerFactory.getLogger(StreamTimeClockManagerImpl.class);
+ private final Map<String,StreamTimeClock> streamIdTimeClockMap;
+ private Timer timer;
+
+ private final Map<StreamTimeClockListener,String> listenerStreamIdMap;
+ private final static AtomicInteger num = new AtomicInteger();
+
+ public StreamTimeClockManagerImpl(){
+ listenerStreamIdMap = new HashMap<>();
+ streamIdTimeClockMap = new HashMap<>();
+ timer = new Timer("StreamScheduler-"+num.getAndIncrement());
+ timer.schedule(new TimerTask() {
+ @Override
+ public void run() {
+ // Make sure the timer tick happens one by one
+ triggerTickOnAll();
+ }
+ },1000,1000);
+ }
+
+ /**
+ *
+ * By default, we could keep the current time clock in memory,
+ * Eventually we may need to consider the global time synchronization across all nodes
+ *
+ * 1) When to initialize window according to start time
+ * 2) When to close expired window according to current time
+ *
+ * @return StreamTimeClock instance
+ */
+ @Override
+ public StreamTimeClock createStreamTimeClock(String streamId){
+ synchronized (streamIdTimeClockMap) {
+ if (!streamIdTimeClockMap.containsKey(streamId)) {
+ StreamTimeClock instance = new StreamTimeClockInLocalMemory(streamId);
+ LOG.info("Created {}", instance);
+ streamIdTimeClockMap.put(streamId, instance);
+ } else {
+ LOG.warn("TimeClock for stream already existss: "+streamIdTimeClockMap.get(streamId));
+ }
+ return streamIdTimeClockMap.get(streamId);
+ }
+ }
+
+ /**
+ * @param streamId
+ * @return
+ */
+ @Override
+ public StreamTimeClock getStreamTimeClock(String streamId) {
+ synchronized (streamIdTimeClockMap) {
+ if (!streamIdTimeClockMap.containsKey(streamId)) {
+ LOG.warn("TimeClock for stream {} is not initialized before being called, create now", streamId);
+ return createStreamTimeClock(streamId);
+ }
+ return streamIdTimeClockMap.get(streamId);
+ }
+ }
+
+ /**
+ * @param streamId
+ */
+ @Override
+ public void removeStreamTimeClock(String streamId){
+ synchronized (streamIdTimeClockMap) {
+ if (streamIdTimeClockMap.containsKey(streamId)) {
+ streamIdTimeClockMap.remove(streamId);
+ LOG.info("Removed TimeClock for stream {}: {}", streamId, streamIdTimeClockMap.get(streamId));
+ } else {
+ LOG.warn("No TimeClock found for stream {}, nothing to remove", streamId);
+ }
+ }
+ }
+
+ @Override
+ public void registerListener(String streamId,StreamTimeClockListener listener) {
+ synchronized (listenerStreamIdMap) {
+ if (listenerStreamIdMap.containsKey(listener))
+ throw new IllegalArgumentException("Duplicated listener: " + listener.toString());
+ LOG.info("Register {} on {}", listener, streamId);
+ listenerStreamIdMap.put(listener, streamId);
+ }
+ }
+
+ @Override
+ public void registerListener(StreamTimeClock streamClock, StreamTimeClockListener listener) {
+ registerListener(streamClock.getStreamId(),listener);
+ }
+
+ @Override
+ public void removeListener(StreamTimeClockListener listener) {
+ listenerStreamIdMap.remove(listener);
+ }
+
+ @Override
+ public synchronized void triggerTickOn(String streamId) {
+ int count = 0;
+ for (Map.Entry<StreamTimeClockListener, String> entry : listenerStreamIdMap.entrySet()) {
+ if (entry.getValue().equals(streamId)) {
+ entry.getKey().onTick(streamIdTimeClockMap.get(streamId), getCurrentSystemTime());
+ count++;
+ }
+ }
+ if (LOG.isDebugEnabled()) LOG.debug("Triggered {} time-clock listeners on stream {}", count, streamId);
+ }
+
+ private static long getCurrentSystemTime(){
+ return System.currentTimeMillis();
+ }
+
+ @Override
+ public void onTimeUpdate(String streamId, long timestamp) {
+ StreamTimeClock timeClock = getStreamTimeClock(streamId);
+ if(timeClock == null)
+ return;
+ // Trigger time clock only when time moves forward
+ if(timestamp >= timeClock.getTime()) {
+ timeClock.moveForward(timestamp);
+ if(LOG.isDebugEnabled()) LOG.debug("Tick on stream {} with latest time {}",streamId, DateTimeUtil.millisecondsToHumanDateWithMilliseconds(timeClock.getTime()));
+ triggerTickOn(streamId);
+ }
+ }
+
+ private void triggerTickOnAll(){
+ synchronized (listenerStreamIdMap) {
+ for (Map.Entry<StreamTimeClockListener, String> entry : listenerStreamIdMap.entrySet()) {
+ triggerTickOn(entry.getValue());
+ }
+ }
+ }
+
+ @Override
+ public void close() {
+ timer.cancel();
+ triggerTickOnAll();
+ LOG.info("Closed StreamTimeClockManager {}",this);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/impl/StreamWindowManagerImpl.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/impl/StreamWindowManagerImpl.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/impl/StreamWindowManagerImpl.java
new file mode 100644
index 0000000..4e1212f
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/sorter/impl/StreamWindowManagerImpl.java
@@ -0,0 +1,173 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.sorter.impl;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.List;
+import java.util.TreeMap;
+
+import org.apache.commons.lang3.time.StopWatch;
+import org.apache.eagle.alert.engine.PartitionedEventCollector;
+import org.apache.eagle.alert.engine.model.PartitionedEvent;
+import org.apache.eagle.alert.engine.sorter.StreamTimeClock;
+import org.apache.eagle.alert.engine.sorter.StreamWindow;
+import org.apache.eagle.alert.engine.sorter.StreamWindowManager;
+import org.apache.eagle.alert.engine.sorter.StreamWindowRepository;
+import org.apache.eagle.alert.utils.DateTimeUtil;
+import org.apache.eagle.alert.utils.TimePeriodUtils;
+import org.joda.time.Period;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class StreamWindowManagerImpl implements StreamWindowManager {
+ private final static Logger LOG = LoggerFactory.getLogger(StreamWindowManagerImpl.class);
+ private final TreeMap<Long,StreamWindow> windowBuckets;
+ private final PartitionedEventCollector collector;
+ private final Period windowPeriod;
+ private final long windowMargin;
+ @SuppressWarnings("unused")
+ private final Comparator<PartitionedEvent> comparator;
+ private long rejectTime;
+
+ public StreamWindowManagerImpl(Period windowPeriod, long windowMargin, Comparator<PartitionedEvent> comparator, PartitionedEventCollector collector){
+ this.windowBuckets = new TreeMap<>();
+ this.windowPeriod = windowPeriod;
+ this.windowMargin = windowMargin;
+ this.collector = collector;
+ this.comparator = comparator;
+ }
+
+ @Override
+ public StreamWindow addNewWindow(long initialTime) {
+ synchronized (windowBuckets) {
+ if (!reject(initialTime)) {
+ Long windowStartTime = TimePeriodUtils.formatMillisecondsByPeriod(initialTime, windowPeriod);
+ Long windowEndTime = windowStartTime + TimePeriodUtils.getMillisecondsOfPeriod(windowPeriod);
+ StreamWindow window = StreamWindowRepository.getSingletonInstance().createWindow(windowStartTime, windowEndTime, windowMargin);
+ window.register(collector);
+ addWindow(window);
+ return window;
+ } else {
+ throw new IllegalStateException("Failed to create new window, as " + DateTimeUtil.millisecondsToHumanDateWithMilliseconds(initialTime) + " is too late, only allow timestamp after " + DateTimeUtil.millisecondsToHumanDateWithMilliseconds(rejectTime));
+ }
+ }
+ }
+
+ private void addWindow(StreamWindow window){
+ if (!windowBuckets.containsKey(window.startTime())) {
+ windowBuckets.put(window.startTime(), window);
+ } else {
+ throw new IllegalArgumentException("Duplicated " + window.toString());
+ }
+ }
+
+ @Override
+ public void removeWindow(StreamWindow window) {
+ synchronized (windowBuckets) {
+ windowBuckets.remove(window.startTime());
+ }
+ }
+
+ @Override
+ public boolean hasWindow(StreamWindow window) {
+ synchronized (windowBuckets) {
+ return windowBuckets.containsKey(window.startTime());
+ }
+ }
+
+ @Override
+ public boolean hasWindowFor(long timestamp) {
+ return getWindowFor(timestamp) != null;
+ }
+
+ @Override
+ public Collection<StreamWindow> getWindows() {
+ synchronized (windowBuckets) {
+ return windowBuckets.values();
+ }
+ }
+
+ @Override
+ public StreamWindow getWindowFor(long timestamp) {
+ synchronized (windowBuckets) {
+ for (StreamWindow windowBucket : windowBuckets.values()) {
+ if (timestamp >= windowBucket.startTime() && timestamp < windowBucket.endTime()) {
+ return windowBucket;
+ }
+ }
+ return null;
+ }
+ }
+
+ @Override
+ public boolean reject(long timestamp) {
+ return timestamp < rejectTime;
+ }
+
+ @Override
+ public void onTick(StreamTimeClock clock,long globalSystemTime) {
+ synchronized (windowBuckets) {
+ List<StreamWindow> toRemoved = new ArrayList<>();
+ List<StreamWindow> aliveWindow = new ArrayList<>();
+
+ for (StreamWindow windowBucket : windowBuckets.values()) {
+ windowBucket.onTick(clock, globalSystemTime);
+ if (windowBucket.rejectTime() > rejectTime) rejectTime = windowBucket.rejectTime();
+ }
+ for (StreamWindow windowBucket : windowBuckets.values()) {
+ if (windowBucket.expired() || windowBucket.endTime() <=rejectTime) {
+ toRemoved.add(windowBucket);
+ } else {
+ aliveWindow.add(windowBucket);
+ }
+ }
+ toRemoved.forEach(this::CloseAndRemoveWindow);
+ if (toRemoved.size() > 0) LOG.info("Windows: {} alive = {}, {} expired = {}", aliveWindow.size(), aliveWindow, toRemoved.size(), toRemoved);
+ }
+ }
+
+ private void CloseAndRemoveWindow(StreamWindow windowBucket){
+ StopWatch stopWatch = new StopWatch();
+ stopWatch.start();
+ CloseWindow(windowBucket);
+ removeWindow(windowBucket);
+ stopWatch.stop();
+ LOG.info("Removed {} in {} ms",windowBucket,stopWatch.getTime());
+ }
+
+ private void CloseWindow(StreamWindow windowBucket){
+ windowBucket.close();
+ }
+
+ public void close() {
+ synchronized (windowBuckets) {
+ LOG.debug("Closing");
+ StopWatch stopWatch = new StopWatch();
+ stopWatch.start();
+ int count = 0;
+ for (StreamWindow windowBucket : getWindows()) {
+ count++;
+ CloseWindow(windowBucket);
+ }
+ windowBuckets.clear();
+ stopWatch.stop();
+ LOG.info("Closed {} windows in {} ms", count, stopWatch.getTime());
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/spout/CorrelationSpout.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/spout/CorrelationSpout.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/spout/CorrelationSpout.java
new file mode 100644
index 0000000..cd23405
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/spout/CorrelationSpout.java
@@ -0,0 +1,380 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+package org.apache.eagle.alert.engine.spout;
+
+import java.text.MessageFormat;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.eagle.alert.coordination.model.Kafka2TupleMetadata;
+import org.apache.eagle.alert.coordination.model.SpoutSpec;
+import org.apache.eagle.alert.engine.coordinator.IMetadataChangeNotifyService;
+import org.apache.eagle.alert.engine.coordinator.MetadataType;
+import org.apache.eagle.alert.engine.coordinator.StreamDefinition;
+import org.apache.eagle.alert.engine.router.SpoutSpecListener;
+import org.apache.eagle.alert.engine.serialization.PartitionedEventSerializer;
+import org.apache.eagle.alert.engine.serialization.SerializationMetadataProvider;
+import org.apache.eagle.alert.engine.serialization.Serializers;
+import org.apache.eagle.alert.utils.AlertConstants;
+import org.apache.eagle.alert.utils.StreamIdConversion;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import storm.kafka.BrokerHosts;
+import storm.kafka.KafkaSpoutMetric;
+import storm.kafka.KafkaSpoutWrapper;
+import storm.kafka.SpoutConfig;
+import storm.kafka.ZkHosts;
+import backtype.storm.spout.SchemeAsMultiScheme;
+import backtype.storm.spout.SpoutOutputCollector;
+import backtype.storm.task.TopologyContext;
+import backtype.storm.topology.OutputFieldsDeclarer;
+import backtype.storm.topology.base.BaseRichSpout;
+import backtype.storm.tuple.Fields;
+
+import com.typesafe.config.Config;
+
+/**
+ * wrap KafkaSpout to provide parallel processing of messages for multiple Kafka topics
+ *
+ * 1. onNewConfig() is interface for outside to update new metadata. Upon new metadata, this class will calculate if there is any new topic, removed topic or
+ * updated topic
+ *
+ */
+public class CorrelationSpout extends BaseRichSpout implements SpoutSpecListener,SerializationMetadataProvider {
+ private static final long serialVersionUID = -5280723341236671580L;
+ private static final Logger LOG = LoggerFactory.getLogger(CorrelationSpout.class);
+
+ public static final String DEFAULT_STORM_KAFKA_TRANSACTION_ZK_ROOT = "/consumers";
+ public static final String DEFAULT_STORM_KAFKA_TRANSACTION_ZK_RELATIVE_PATH = "/eagle_consumer";
+
+ // topic to KafkaSpoutWrapper
+ private volatile Map<String, KafkaSpoutWrapper> kafkaSpoutList = new HashMap<>();
+ private int numOfRouterBolts;
+
+ private SpoutSpec cachedSpoutSpec;
+
+ private transient KafkaSpoutMetric kafkaSpoutMetric;
+
+ @SuppressWarnings("rawtypes")
+ private Map conf;
+ private TopologyContext context;
+ private SpoutOutputCollector collector;
+ private final Config config;
+ private String topologyId;
+ private String spoutName;
+ private String routeBoltName;
+ @SuppressWarnings("unused")
+ private int taskIndex;
+ private IMetadataChangeNotifyService changeNotifyService;
+ private PartitionedEventSerializer serializer;
+ private volatile Map<String, StreamDefinition> sds;
+
+ /**
+ * FIXME one single changeNotifyService may have issues as possibly multiple spout tasks will register themselves and initialize service
+ * @param config
+ * @param topologyId
+ * @param changeNotifyService
+ * @param numOfRouterBolts
+ */
+ public CorrelationSpout(Config config, String topologyId, IMetadataChangeNotifyService changeNotifyService, int numOfRouterBolts){
+ this(config, topologyId, changeNotifyService, numOfRouterBolts, AlertConstants.DEFAULT_SPOUT_NAME, AlertConstants.DEFAULT_ROUTERBOLT_NAME);
+ }
+ /**
+ *
+ * @param config
+ * @param topologyId used for distinguishing kafka offset for different topologies
+ * @param numOfRouterBolts used for generating streamId and routing
+ * @param spoutName used for generating streamId between spout and router bolt
+ * @param routerBoltName used for generating streamId between spout and router bolt
+ */
+ public CorrelationSpout(Config config, String topologyId, IMetadataChangeNotifyService changeNotifyService, int numOfRouterBolts, String spoutName, String routerBoltName){
+ this.config = config;
+ this.topologyId = topologyId;
+ this.changeNotifyService = changeNotifyService;
+ this.numOfRouterBolts = numOfRouterBolts;
+ this.spoutName = spoutName;
+ this.routeBoltName = routerBoltName;
+ }
+
+ public String getSpoutName(){
+ return spoutName;
+ }
+
+ public String getRouteBoltName(){
+ return routeBoltName;
+ }
+
+ /**
+ * the only output field is for StreamEvent
+ * @param declarer
+ */
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+ for (int i = 0; i < numOfRouterBolts; i++) {
+ String streamId = StreamIdConversion.generateStreamIdBetween(spoutName, routeBoltName + i);
+ declarer.declareStream(streamId, new Fields(AlertConstants.FIELD_0));
+ LOG.info("declare stream between spout and streamRouterBolt " + streamId);
+ }
+ }
+
+ @SuppressWarnings("rawtypes")
+ @Override
+ public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("open method invoked");
+ }
+ this.conf = conf;
+ this.context = context;
+ this.collector = collector;
+ this.taskIndex = context.getThisTaskIndex();
+
+ // initialize an empty SpoutSpec
+ cachedSpoutSpec = new SpoutSpec(topologyId, new HashMap<>(), new HashMap<>(), new HashMap<>());
+
+ changeNotifyService.registerListener(this);
+ changeNotifyService.init(config, MetadataType.SPOUT);
+
+ // register KafkaSpout metric
+ kafkaSpoutMetric = new KafkaSpoutMetric();
+ context.registerMetric("kafkaSpout", kafkaSpoutMetric, 60);
+
+ this.serializer = Serializers.newPartitionedEventSerializer(this);
+ }
+
+ @Override
+ public void onSpoutSpecChange(SpoutSpec spec, Map<String, StreamDefinition> sds) {
+ LOG.info("new metadata is updated " + spec);
+ try{
+ onReload(spec, sds);
+ }catch(Exception ex){
+ LOG.error("error applying new SpoutSpec", ex);
+ }
+ }
+
+ @Override
+ public void nextTuple() {
+ for (KafkaSpoutWrapper wrapper : kafkaSpoutList.values()) {
+ wrapper.nextTuple();
+ }
+ }
+
+ /**
+ * find the correct wrapper to do ack that means msgId should be mapped to
+ * wrapper
+ *
+ * @param msgId
+ */
+ @Override
+ public void ack(Object msgId) {
+ // decode and get topic
+ KafkaMessageIdWrapper id = (KafkaMessageIdWrapper) msgId;
+ KafkaSpoutWrapper spout = kafkaSpoutList.get(id.topic);
+ spout.ack(id.id);
+ }
+
+ @Override
+ public void fail(Object msgId) {
+ // decode and get topic
+ KafkaMessageIdWrapper id = (KafkaMessageIdWrapper) msgId;
+ LOG.error("Failing message {}, with topic {}", msgId, id.topic);
+ KafkaSpoutWrapper spout = kafkaSpoutList.get(id.topic);
+ spout.fail(id.id);
+ }
+
+ @Override
+ public void deactivate() {
+ System.out.println("deactivate");
+ for (KafkaSpoutWrapper wrapper : kafkaSpoutList.values()) {
+ wrapper.deactivate();
+ }
+ }
+
+ @Override
+ public void close() {
+ System.out.println("close");
+ for (KafkaSpoutWrapper wrapper : kafkaSpoutList.values()) {
+ wrapper.close();
+ }
+ }
+
+ private List<String> getTopics(SpoutSpec spoutSpec) {
+ List<String> meta = new ArrayList<String>();
+ for (Kafka2TupleMetadata entry : spoutSpec.getKafka2TupleMetadataMap().values()) {
+ meta.add(entry.getTopic());
+ }
+ return meta;
+ }
+
+ @SuppressWarnings("unchecked")
+ public void onReload(final SpoutSpec newMeta, Map<String, StreamDefinition> sds) throws Exception {
+ // calculate topic create/remove/update
+ List<String> topics = getTopics(newMeta);
+ List<String> cachedTopcies = getTopics(cachedSpoutSpec);
+ Collection<String> newTopics = CollectionUtils.subtract(topics, cachedTopcies);
+ Collection<String> removeTopics = CollectionUtils.subtract(cachedTopcies, topics);
+ Collection<String> updateTopics = CollectionUtils.intersection(topics, cachedTopcies);
+
+ LOG.info("Topics were added={}, removed={}, modified={}", newTopics, removeTopics, updateTopics);
+
+ // build lookup table for scheme
+ Map<String, String> newSchemaName = new HashMap<String, String>();
+ for (Kafka2TupleMetadata ds : newMeta.getKafka2TupleMetadataMap().values()) {
+ newSchemaName.put(ds.getTopic(), ds.getSchemeCls());
+ }
+
+ // copy and swap
+ Map<String, KafkaSpoutWrapper> newKafkaSpoutList = new HashMap<>(this.kafkaSpoutList);
+ // iterate new topics and then create KafkaSpout
+ for(String topic : newTopics){
+ KafkaSpoutWrapper wrapper = newKafkaSpoutList.get(topic);
+ if (wrapper != null) {
+ LOG.warn(MessageFormat.format("try to create new topic {0}, but found in the active spout list, this may indicate some inconsistency", topic));
+ continue;
+ }
+ KafkaSpoutWrapper newWrapper = createKafkaSpout(conf, context, collector, topic, newSchemaName.get(topic), newMeta, sds);
+ newKafkaSpoutList.put(topic, newWrapper);
+ }
+ // iterate remove topics and then close KafkaSpout
+ for(String topic : removeTopics){
+ KafkaSpoutWrapper wrapper = newKafkaSpoutList.get(topic);
+ if (wrapper == null) {
+ LOG.warn(MessageFormat.format("try to remove topic {0}, but not found in the active spout list, this may indicate some inconsistency", topic));
+ continue;
+ }
+ removeKafkaSpout(wrapper);
+ newKafkaSpoutList.remove(topic);
+ }
+
+ // iterate update topic and then update metadata
+ for(String topic : updateTopics){
+ KafkaSpoutWrapper spoutWrapper = newKafkaSpoutList.get(topic);
+ if (spoutWrapper == null) {
+ LOG.warn(MessageFormat.format("try to update topic {0}, but not found in the active spout list, this may indicate some inconsistency", topic));
+ continue;
+ }
+ spoutWrapper.update(newMeta, sds);
+ }
+
+ // swap
+ this.cachedSpoutSpec = newMeta;
+ this.kafkaSpoutList = newKafkaSpoutList;
+ this.sds = sds;
+ }
+
+ /**
+ * make this method protected to make sure unit test can work well
+ * Q: Where to persist consumer state, i.e. what offset has been consumed for each topic and partition
+ * A: stormKafkaTransactionZkPath + "/" + consumerId + "/" + topic + "/" + topologyId + "/" + partitionId
+ * Note1: PartitionManager.committedPath for composing zkState path, _spoutConfig.zkRoot + "/" + _spoutConfig.id + "/" + _partition.getId();
+ * consumerId by default is EagleConsumer unless it is specified by "stormKafkaEagleConsumer"
+ * Note2: put topologyId as part of zkState because one topic by design can be consumed by multiple topologies so one topology needs to know
+ * processed offset for itself
+ *
+ * TODO: Should avoid use Config.get in deep calling stack, should generate config bean as early as possible
+ *
+ * @param conf
+ * @param context
+ * @param collector
+ * @param topic
+ * @param spoutSpec
+ * @return
+ */
+ @SuppressWarnings("rawtypes")
+ protected KafkaSpoutWrapper createKafkaSpout(Map conf, TopologyContext context, SpoutOutputCollector collector, final String topic,
+ String schemeClsName, SpoutSpec spoutSpec, Map<String, StreamDefinition> sds) throws Exception{
+ String kafkaBrokerZkQuorum = config.getString("spout.kafkaBrokerZkQuorum");
+ BrokerHosts hosts = new ZkHosts(kafkaBrokerZkQuorum);
+ String transactionZkRoot = DEFAULT_STORM_KAFKA_TRANSACTION_ZK_ROOT;
+ if(config.hasPath("spout.stormKafkaTransactionZkPath")) {
+ transactionZkRoot = config.getString("spout.stormKafkaTransactionZkPath");
+ }
+ // write partition offset etc. into zkRoot+id, see PartitionManager.committedPath
+ String zkStateTransactionRelPath = DEFAULT_STORM_KAFKA_TRANSACTION_ZK_RELATIVE_PATH;
+ if(config.hasPath("spout.stormKafkaEagleConsumer")){
+ zkStateTransactionRelPath = config.getString("spout.stormKafkaEagleConsumer");
+ }
+ SpoutConfig spoutConfig = new SpoutConfig(hosts, topic, transactionZkRoot, zkStateTransactionRelPath + "/" + topic + "/" + topologyId);
+ // transaction zkServers
+ boolean stormKafkaUseSameZkQuorumWithKafkaBroker = config.getBoolean("spout.stormKafkaUseSameZkQuorumWithKafkaBroker");
+ if(stormKafkaUseSameZkQuorumWithKafkaBroker){
+ ZkServerPortUtils utils = new ZkServerPortUtils(kafkaBrokerZkQuorum);
+ spoutConfig.zkServers = utils.getZkHosts();
+ spoutConfig.zkPort = utils.getZkPort();
+ }else{
+ ZkServerPortUtils utils = new ZkServerPortUtils(config.getString("spout.stormKafkaTransactionZkQuorum"));
+ spoutConfig.zkServers = utils.getZkHosts();
+ spoutConfig.zkPort = utils.getZkPort();
+ }
+ // transaction update interval
+ spoutConfig.stateUpdateIntervalMs = config.getLong("spout.stormKafkaStateUpdateIntervalMs");
+ // Kafka fetch size
+ spoutConfig.fetchSizeBytes = config.getInt("spout.stormKafkaFetchSizeBytes");
+ // "startOffsetTime" is for test usage, prod should not use this
+ if (config.hasPath("spout.stormKafkaStartOffsetTime")) {
+ spoutConfig.startOffsetTime = config.getInt("spout.stormKafkaStartOffsetTime");
+ }
+
+ spoutConfig.scheme = new SchemeAsMultiScheme(SchemeBuilder.buildFromClsName(schemeClsName, topic));
+ KafkaSpoutWrapper wrapper = new KafkaSpoutWrapper(spoutConfig, kafkaSpoutMetric);
+ SpoutOutputCollectorWrapper collectorWrapper = new SpoutOutputCollectorWrapper(this, collector, topic, spoutSpec, numOfRouterBolts, sds,this.serializer);
+ wrapper.open(conf, context, collectorWrapper);
+ return wrapper;
+ }
+
+ @Override
+ public StreamDefinition getStreamDefinition(String streamId) {
+ return sds.get(streamId);
+ }
+
+ /**
+ * utility to get list of zkServers and zkPort.(It is assumed that zkPort is same for all zkServers as storm-kafka library requires this though it is not efficient)
+ */
+ private static class ZkServerPortUtils{
+ private List<String> zkHosts = new ArrayList<>();
+ private Integer zkPort;
+ public ZkServerPortUtils(String zkQuorum){
+ String[] zkConnections = zkQuorum.split(",");
+ for (String zkConnection : zkConnections) {
+ zkHosts.add(zkConnection.split(":")[0]);
+ }
+ zkPort = Integer.valueOf(zkConnections[0].split(":")[1]);
+ }
+
+ public List<String> getZkHosts(){
+ return zkHosts;
+ }
+
+ public Integer getZkPort(){
+ return zkPort;
+ }
+ }
+
+ protected void removeKafkaSpout(KafkaSpoutWrapper wrapper){
+ try {
+ wrapper.close();
+ } catch (Exception e) {
+ LOG.error("Close wrapper failed. Ignore and continue!", e);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/spout/CreateTopicUtils.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/spout/CreateTopicUtils.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/spout/CreateTopicUtils.java
new file mode 100644
index 0000000..a2c9219
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/spout/CreateTopicUtils.java
@@ -0,0 +1,43 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+package org.apache.eagle.alert.engine.spout;
+
+import java.util.Properties;
+
+import kafka.admin.AdminUtils;
+import kafka.utils.ZKStringSerializer$;
+
+import org.I0Itec.zkclient.ZkClient;
+import org.slf4j.Logger;
+
+/**
+ * normally this is used in unit test for convenience
+ */
+public class CreateTopicUtils {
+ private static final Logger LOG = org.slf4j.LoggerFactory.getLogger(CreateTopicUtils.class);
+ private static final int partitions = 2;
+ private static final int replicationFactor = 1;
+ public static void ensureTopicReady(String zkQuorum, String topic){
+ ZkClient zkClient = new ZkClient(zkQuorum, 10000, 10000, ZKStringSerializer$.MODULE$);
+ if(!AdminUtils.topicExists(zkClient, topic)) {
+ LOG.info("create topic " + topic + " with partitions " + partitions + ", and replicationFactor " + replicationFactor);
+ AdminUtils.createTopic(zkClient, topic, partitions, replicationFactor, new Properties());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/spout/ISpoutSpecLCM.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/spout/ISpoutSpecLCM.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/spout/ISpoutSpecLCM.java
new file mode 100644
index 0000000..23e94c3
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/spout/ISpoutSpecLCM.java
@@ -0,0 +1,41 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.alert.engine.spout;
+
+import java.util.Map;
+
+import org.apache.eagle.alert.coordination.model.SpoutSpec;
+import org.apache.eagle.alert.engine.coordinator.StreamDefinition;
+
+/**
+ * topic to stream metadata lifecycle method
+ * one topic may spawn multiple streams, the metadata change includes
+ * 1. add/remove stream
+ * 2. for a specific stream, groupingstrategy is changed
+ * ex1, this stream has more alert bolts than before, then this spout would take more traffic
+ * ex2, this stream has less alert bolts than before, then this spout would take less traffic
+ */
+public interface ISpoutSpecLCM {
+ /**
+ * stream metadata is used for SPOUT to filter traffic and route traffic to following groupby bolts.
+ * @param metadata
+ */
+ void update(SpoutSpec metadata, Map<String, StreamDefinition> sds);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/spout/KafkaMessageIdWrapper.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/spout/KafkaMessageIdWrapper.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/spout/KafkaMessageIdWrapper.java
new file mode 100644
index 0000000..c786c01
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/spout/KafkaMessageIdWrapper.java
@@ -0,0 +1,42 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+package org.apache.eagle.alert.engine.spout;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+/**
+ * Created on 2/18/16.
+ */
+public class KafkaMessageIdWrapper {
+ public Object id;
+ public KafkaMessageIdWrapper(Object o){
+ this.id = o;
+ }
+ public String topic;
+ private final static ObjectMapper objectMapper = new ObjectMapper();
+
+ public String toString(){
+ try {
+ return String.format("KafkaMessageIdWrapper[topic=%s, id=%s]", topic, objectMapper.writeValueAsString(id));
+ } catch (JsonProcessingException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/spout/SchemeBuilder.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/spout/SchemeBuilder.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/spout/SchemeBuilder.java
new file mode 100644
index 0000000..223f1b5
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/spout/SchemeBuilder.java
@@ -0,0 +1,35 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.alert.engine.spout;
+
+import backtype.storm.spout.Scheme;
+
+
+/**
+ * All Scheme implementations should have the following conditions
+ * 1) implement Scheme interface
+ * 2) has one constructor with topic name as parameter
+ */
+public class SchemeBuilder {
+ public static Scheme buildFromClsName(String clsName, String topic) throws Exception{
+ Object o = Class.forName(clsName).getConstructor(String.class).newInstance(topic);
+ return (Scheme)o;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/spout/SpoutOutputCollectorWrapper.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/spout/SpoutOutputCollectorWrapper.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/spout/SpoutOutputCollectorWrapper.java
new file mode 100644
index 0000000..b37f7b3
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/spout/SpoutOutputCollectorWrapper.java
@@ -0,0 +1,220 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+package org.apache.eagle.alert.engine.spout;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+import org.apache.eagle.alert.coordination.model.SpoutSpec;
+import org.apache.eagle.alert.coordination.model.StreamRepartitionMetadata;
+import org.apache.eagle.alert.coordination.model.StreamRepartitionStrategy;
+import org.apache.eagle.alert.coordination.model.Tuple2StreamConverter;
+import org.apache.eagle.alert.engine.coordinator.StreamDefinition;
+import org.apache.eagle.alert.engine.coordinator.StreamPartition;
+import org.apache.eagle.alert.engine.model.PartitionedEvent;
+import org.apache.eagle.alert.engine.model.StreamEvent;
+import org.apache.eagle.alert.engine.serialization.PartitionedEventSerializer;
+import org.apache.eagle.alert.engine.serialization.SerializationMetadataProvider;
+import org.apache.eagle.alert.utils.StreamIdConversion;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import backtype.storm.spout.ISpoutOutputCollector;
+import backtype.storm.spout.SpoutOutputCollector;
+
+/**
+ * intercept the message sent from within KafkaSpout and select downstream bolts based on meta-data
+ * This is topic based. each topic will have one SpoutOutputCollectorWrapper
+ */
+public class SpoutOutputCollectorWrapper extends SpoutOutputCollector implements ISpoutSpecLCM,SerializationMetadataProvider {
+ private static final Logger LOG = LoggerFactory.getLogger(SpoutOutputCollectorWrapper.class);
+
+ private final ISpoutOutputCollector delegate;
+ private final String topic;
+ private final PartitionedEventSerializer serializer;
+ private int numOfRouterBolts;
+
+ private volatile List<StreamRepartitionMetadata> streamRepartitionMetadataList;
+ private volatile Tuple2StreamConverter converter;
+ private CorrelationSpout spout;
+ private volatile Map<String, StreamDefinition> sds;
+
+ /**
+ * @param delegate actual SpoutOutputCollector to send data to following bolts
+ * @param topic topic for this KafkaSpout to handle
+ * @param numGroupbyBolts bolts following this spout
+ * @param serializer
+ */
+ public SpoutOutputCollectorWrapper(CorrelationSpout spout,
+ ISpoutOutputCollector delegate,
+ String topic,
+ SpoutSpec spoutSpec,
+ int numGroupbyBolts,
+ Map<String, StreamDefinition> sds, PartitionedEventSerializer serializer) {
+ super(delegate);
+ this.spout = spout;
+ this.delegate = delegate;
+ this.topic = topic;
+ this.streamRepartitionMetadataList = spoutSpec.getStreamRepartitionMetadataMap().get(topic);
+ this.converter = new Tuple2StreamConverter(spoutSpec.getTuple2StreamMetadataMap().get(topic));
+ this.numOfRouterBolts = numGroupbyBolts;
+ this.sds = sds;
+ this.serializer = serializer;
+ }
+
+ /**
+ * How to assert that numTotalGroupbyBolts >= numOfRouterBolts, otherwise
+ * there is runtime issue by default, tuple includes 2 fields field 1: topic
+ * name field 2: map of key/value
+ */
+ @SuppressWarnings("rawtypes")
+ @Override
+ public List<Integer> emit(List<Object> tuple, Object messageId) {
+ if (!sanityCheck()) {
+ LOG.error(
+ "spout collector for topic {} see monitored metadata invalid, is this data source removed! Trigger message id {} ",
+ topic, messageId);
+ return null;
+ }
+
+ KafkaMessageIdWrapper newMessageId = new KafkaMessageIdWrapper(messageId);
+ newMessageId.topic = topic;
+ /**
+ phase 1: tuple to stream converter
+ if this topic multiplexes multiple streams, then retrieve the individual streams
+ */
+ List<Object> convertedTuple = converter.convert(tuple);
+ if(convertedTuple == null) {
+ LOG.warn("source data {} can't be converted to a stream, ignore this message", tuple);
+ spout.ack(newMessageId);
+ return null;
+ }
+ Map m = (Map)convertedTuple.get(3);
+ Object streamId = convertedTuple.get(1);
+
+ StreamDefinition sd = sds.get(streamId);
+ if(sd == null){
+ LOG.warn("StreamDefinition {} is not found within {}, ignore this message", streamId, sds);
+ spout.ack(newMessageId);
+ return null;
+ }
+
+ StreamEvent event = convertToStreamEventByStreamDefinition((Long)convertedTuple.get(2), m, sds.get(streamId));
+ /*
+ phase 2: stream repartition
+ */
+ for(StreamRepartitionMetadata md : streamRepartitionMetadataList) {
+ // one stream may have multiple group-by strategies, each strategy is for a specific group-by
+ for(StreamRepartitionStrategy groupingStrategy : md.groupingStrategies){
+ int hash = 0;
+ if(groupingStrategy.getPartition().getType().equals(StreamPartition.Type.GROUPBY)) {
+ hash = getRoutingHashByGroupingStrategy(m, groupingStrategy);
+ }else if(groupingStrategy.getPartition().getType().equals(StreamPartition.Type.SHUFFLE)){
+ hash = Math.abs((int)System.currentTimeMillis());
+ }
+ int mod = hash % groupingStrategy.numTotalParticipatingRouterBolts;
+ // filter out message
+ if (mod >= groupingStrategy.startSequence && mod < groupingStrategy.startSequence + numOfRouterBolts) {
+ // framework takes care of field grouping instead of using storm internal field grouping
+ String sid = StreamIdConversion.generateStreamIdBetween(spout.getSpoutName(), spout.getRouteBoltName()+ (hash % numOfRouterBolts));
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Emitted tuple: {} with message Id: {}, with topic {}, to streamId {}", convertedTuple, messageId, topic, sid);
+ }
+ // send message to StreamRouterBolt
+ PartitionedEvent pEvent = new PartitionedEvent(event, groupingStrategy.partition, hash);
+ if(this.serializer == null){
+ delegate.emit(sid, Collections.singletonList(pEvent), newMessageId);
+ }else {
+ try {
+ delegate.emit(sid, Collections.singletonList(serializer.serialize(pEvent)), newMessageId);
+ } catch (IOException e) {
+ LOG.error("Failed to serialize {}", pEvent, e);
+ throw new RuntimeException(e);
+ }
+ }
+ }else{
+ // ******* short-cut ack ********
+ // we should simply ack those messages which are not processed in this topology because KafkaSpout implementation requires _pending is empty
+ // before moving to next offsets.
+ if(LOG.isDebugEnabled()){
+ LOG.debug("Message filtered with mod {} not within range {} and {} for message {}", mod, groupingStrategy.startSequence,
+ groupingStrategy.startSequence+ numOfRouterBolts, tuple);
+ }
+ spout.ack(newMessageId);
+ }
+ }
+ }
+
+ return null;
+ }
+
+ @SuppressWarnings("rawtypes")
+ private int getRoutingHashByGroupingStrategy(Map data, StreamRepartitionStrategy gs){
+ // calculate hash value for values from group-by fields
+ HashCodeBuilder hashCodeBuilder = new HashCodeBuilder();
+ for(String groupingField : gs.partition.getColumns()) {
+ if(data.get(groupingField) != null){
+ hashCodeBuilder.append(data.get(groupingField));
+ } else {
+ LOG.warn("Required GroupBy fields {} not found: {}", gs.partition.getColumns(), data);
+ }
+ }
+ int hash = hashCodeBuilder.toHashCode();
+ hash = Math.abs(hash);
+ return hash;
+ }
+
+ private boolean sanityCheck() {
+ boolean isOk = true;
+ if (streamRepartitionMetadataList == null) {
+ LOG.error("streamRepartitionMetadataList is null!");
+ isOk = false;
+ }
+ if (converter == null) {
+ LOG.error("tuple2StreamMetadata is null!");
+ isOk = false;
+ }
+ return isOk;
+ }
+
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ private StreamEvent convertToStreamEventByStreamDefinition(long timestamp, Map m, StreamDefinition sd){
+ return StreamEvent.Builder().timestamep(timestamp).attributes(m,sd).build();
+ }
+
+ /**
+ * SpoutSpec may be changed, this class will respond to changes on tuple2StreamMetadataMap and streamRepartitionMetadataMap
+ * @param spoutSpec
+ * @param sds
+ */
+ @Override
+ public void update(SpoutSpec spoutSpec, Map<String, StreamDefinition> sds) {
+ this.streamRepartitionMetadataList = spoutSpec.getStreamRepartitionMetadataMap().get(topic);
+ this.converter = new Tuple2StreamConverter(spoutSpec.getTuple2StreamMetadataMap().get(topic));
+ this.sds = sds;
+ }
+
+ @Override
+ public StreamDefinition getStreamDefinition(String streamId) {
+ return this.sds.get(streamId);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/utils/CompressionUtils.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/utils/CompressionUtils.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/utils/CompressionUtils.java
new file mode 100644
index 0000000..f526cad
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/utils/CompressionUtils.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.utils;
+
+import com.google.common.io.ByteStreams;
+
+import java.io.*;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.GZIPOutputStream;
+
+
+public class CompressionUtils {
+ public static byte[] compress(byte[] source) throws IOException {
+ if (source == null || source.length == 0) {
+ return source;
+ }
+ ByteArrayInputStream sourceStream = new ByteArrayInputStream(source);
+ ByteArrayOutputStream outputStream = new ByteArrayOutputStream(source.length / 2);
+ try (OutputStream compressor = new GZIPOutputStream(outputStream)) {
+ ByteStreams.copy(sourceStream, compressor);
+ compressor.close();
+ }
+ try {
+ return outputStream.toByteArray();
+ } finally {
+ sourceStream.close();
+ outputStream.close();
+ }
+ }
+
+ public static byte[] decompress(byte[] compressed) throws IOException{
+ if (compressed == null || compressed.length == 0) {
+ return compressed;
+ }
+ ByteArrayInputStream sourceStream = new ByteArrayInputStream(compressed);
+ ByteArrayOutputStream outputStream = new ByteArrayOutputStream(compressed.length * 2);
+ try (GZIPInputStream compressor = new GZIPInputStream(sourceStream)) {
+ ByteStreams.copy(compressor, outputStream);
+ compressor.close();
+ }
+ try {
+ return outputStream.toByteArray();
+ } finally {
+ sourceStream.close();
+ outputStream.close();
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/utils/MetadataSerDeser.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/utils/MetadataSerDeser.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/utils/MetadataSerDeser.java
new file mode 100644
index 0000000..a576404
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/utils/MetadataSerDeser.java
@@ -0,0 +1,93 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.alert.engine.utils;
+
+import java.io.InputStream;
+
+import org.codehaus.jackson.JsonParser;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.type.TypeReference;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Since 5/6/16.
+ */
+public class MetadataSerDeser {
+ private static final Logger LOG = LoggerFactory.getLogger(MetadataSerDeser.class);
+
+ @SuppressWarnings("rawtypes")
+ public static <K> K deserialize(InputStream is, TypeReference typeRef){
+ ObjectMapper mapper = new ObjectMapper();
+ try {
+ K spec = mapper.readValue(is, typeRef);
+ return spec;
+ }catch(Exception ex){
+ LOG.error("error in deserializing metadata of type {} from input stream", new TypeReference<K>(){}.getType().getTypeName(), ex);
+ }
+ return null;
+ }
+
+ public static <K> K deserialize(InputStream is, Class<K> cls){
+ ObjectMapper mapper = new ObjectMapper();
+ mapper.configure(JsonParser.Feature.ALLOW_COMMENTS,true);
+ try {
+ K spec = mapper.readValue(is, cls);
+ return spec;
+ }catch(Exception ex){
+ LOG.error("Got error to deserialize metadata of type {} from input stream", new TypeReference<K>(){}.getType().getTypeName(), ex);
+ }
+ return null;
+ }
+
+ @SuppressWarnings("rawtypes")
+ public static <K> K deserialize(String json, TypeReference typeRef){
+ ObjectMapper mapper = new ObjectMapper();
+ try {
+ K spec = mapper.readValue(json, typeRef);
+ return spec;
+ }catch(Exception ex){
+ LOG.error("error in deserializing metadata of type {} from {}", new TypeReference<K>(){}.getType().getTypeName(), json, ex);
+ }
+ return null;
+ }
+
+ public static <K> K deserialize(String json, Class<K> cls){
+ ObjectMapper mapper = new ObjectMapper();
+ try {
+ K spec = mapper.readValue(json, cls);
+ return spec;
+ }catch(Exception ex){
+ LOG.error("error in deserializing metadata of type {} from {}", new TypeReference<K>(){}.getType().getTypeName(), json, ex);
+ }
+ return null;
+ }
+
+ public static <K> String serialize(K spec){
+ ObjectMapper mapper = new ObjectMapper();
+ try{
+ String json = mapper.writeValueAsString(spec);
+ return json;
+ }catch(Exception ex){
+ LOG.error("error in serializing object {} with type {}", spec, new TypeReference<K>(){}.getType().getTypeName(), ex);
+ }
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/utils/SerializableUtils.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/utils/SerializableUtils.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/utils/SerializableUtils.java
new file mode 100644
index 0000000..374de5d
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/utils/SerializableUtils.java
@@ -0,0 +1,112 @@
+package org.apache.eagle.alert.engine.utils;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.Serializable;
+
+import org.xerial.snappy.SnappyInputStream;
+import org.xerial.snappy.SnappyOutputStream;
+
+/**
+ * Utilities for working with Serializables.
+ *
+ * Derived from "com.google.cloud.dataflow.sdk.util.SerializableUtils":
+ * https://github.com/apache/incubator-beam/blob/master/sdks/java/core/src/main/java/com/google/cloud/dataflow/sdk/util/SerializableUtils.java
+ */
+public class SerializableUtils {
+ /**
+ * Serializes the argument into an array of bytes, and returns it.
+ *
+ * @throws IllegalArgumentException if there are errors when serializing
+ */
+ public static byte[] serializeToCompressedByteArray(Object value) {
+ try {
+ ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+ try (ObjectOutputStream oos = new ObjectOutputStream(new SnappyOutputStream(buffer))) {
+ oos.writeObject(value);
+ }
+ return buffer.toByteArray();
+ } catch (IOException exn) {
+ throw new IllegalArgumentException(
+ "unable to serialize " + value,
+ exn);
+ }
+ }
+
+ /**
+ * Serializes the argument into an array of bytes, and returns it.
+ *
+ * @throws IllegalArgumentException if there are errors when serializing
+ */
+ public static byte[] serializeToByteArray(Object value) {
+ try {
+ ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+ try (ObjectOutputStream oos = new ObjectOutputStream(buffer)) {
+ oos.writeObject(value);
+ }
+ return buffer.toByteArray();
+ } catch (IOException exn) {
+ throw new IllegalArgumentException("unable to serialize " + value, exn);
+ }
+ }
+
+ /**
+ * Deserializes an object from the given array of bytes, e.g., as
+ * serialized using {@link #serializeToCompressedByteArray}, and returns it.
+ *
+ * @throws IllegalArgumentException if there are errors when
+ * deserializing, using the provided description to identify what
+ * was being deserialized
+ */
+ public static Object deserializeFromByteArray(byte[] encodedValue,
+ String description) {
+ try {
+ try (ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(encodedValue))) {
+ return ois.readObject();
+ }
+ } catch (IOException | ClassNotFoundException exn) {
+ throw new IllegalArgumentException(
+ "unable to deserialize " + description,
+ exn);
+ }
+ }
+
+ /**
+ * Deserializes an object from the given array of bytes, e.g., as
+ * serialized using {@link #serializeToCompressedByteArray}, and returns it.
+ *
+ * @throws IllegalArgumentException if there are errors when
+ * deserializing, using the provided description to identify what
+ * was being deserialized
+ */
+ public static Object deserializeFromCompressedByteArray(byte[] encodedValue,
+ String description) {
+ try {
+ try (ObjectInputStream ois = new ObjectInputStream(
+ new SnappyInputStream(new ByteArrayInputStream(encodedValue)))) {
+ return ois.readObject();
+ }
+ } catch (IOException | ClassNotFoundException exn) {
+ throw new IllegalArgumentException(
+ "unable to deserialize " + description,
+ exn);
+ }
+ }
+
+ public static <T extends Serializable> T ensureSerializable(T value) {
+ @SuppressWarnings("unchecked")
+ T copy = (T) deserializeFromCompressedByteArray(serializeToCompressedByteArray(value),
+ value.toString());
+ return copy;
+ }
+
+ public static <T extends Serializable> T clone(T value) {
+ @SuppressWarnings("unchecked")
+ T copy = (T) deserializeFromCompressedByteArray(serializeToCompressedByteArray(value),
+ value.toString());
+ return copy;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/storm/kafka/KafkaSpoutMetric.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/storm/kafka/KafkaSpoutMetric.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/storm/kafka/KafkaSpoutMetric.java
new file mode 100644
index 0000000..2153c40
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/storm/kafka/KafkaSpoutMetric.java
@@ -0,0 +1,78 @@
+package storm.kafka;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import backtype.storm.metric.api.IMetric;
+
+/**
+ * Since 5/18/16.
+ * The original storm.kafka.KafkaSpout has some issues like the following
+ * 1) can only support one single topic
+ * 2) can only be initialized at open(), can't dynamically support another topic
+ */
+public class KafkaSpoutMetric implements IMetric {
+ @SuppressWarnings("unused")
+ private final static Logger LOG = LoggerFactory.getLogger(KafkaSpoutMetric.class);
+ private Map<String, KafkaSpoutMetricContext> metricContextMap = new ConcurrentHashMap<>();
+ private Map<String, KafkaUtils.KafkaOffsetMetric> offsetMetricMap = new ConcurrentHashMap<>();
+
+ public static class KafkaSpoutMetricContext {
+ SpoutConfig _spoutConfig;
+ DynamicPartitionConnections _connections;
+ PartitionCoordinator _coordinator;
+ }
+
+ public void addTopic(String topic, KafkaSpoutMetricContext context) {
+ // construct KafkaOffsetMetric
+ KafkaUtils.KafkaOffsetMetric kafkaOffsetMetric = new KafkaUtils.KafkaOffsetMetric(context._spoutConfig.topic, context._connections);
+ metricContextMap.put(topic, context);
+ offsetMetricMap.put(topic, kafkaOffsetMetric);
+ }
+
+ public void removeTopic(String topic) {
+ metricContextMap.remove(topic);
+ offsetMetricMap.remove(topic);
+ }
+
+ @SuppressWarnings({ "unchecked", "rawtypes" })
+ @Override
+ public Object getValueAndReset() {
+ HashMap spoutMetric = new HashMap();
+ for (Map.Entry<String, KafkaSpoutMetricContext> entry : metricContextMap.entrySet()) {
+ // construct offset metric
+ List<PartitionManager> pms = entry.getValue()._coordinator.getMyManagedPartitions();
+ Set<Partition> latestPartitions = new HashSet();
+ for (PartitionManager pm : pms) {
+ latestPartitions.add(pm.getPartition());
+ }
+
+ KafkaUtils.KafkaOffsetMetric offsetMetric = offsetMetricMap.get(entry.getKey());
+ offsetMetric.refreshPartitions(latestPartitions);
+ for (PartitionManager pm : pms) {
+ offsetMetric.setLatestEmittedOffset(pm.getPartition(), pm.lastCompletedOffset());
+ }
+ Object o = offsetMetric.getValueAndReset();
+ if(o != null) {
+ ((HashMap) o).forEach(
+ (k, v) -> spoutMetric.put(k + "_" + entry.getKey(), v)
+ );
+ }
+
+ // construct partition metric
+ for (PartitionManager pm : pms) {
+ pm.getMetricsDataMap().forEach(
+ (k, v) -> spoutMetric.put(k + "_" + entry.getKey(), v)
+ );
+ }
+ }
+ return spoutMetric;
+ }
+}
[19/19] incubator-eagle git commit: EAGLE-324: Init branch-v0.5
Posted by ra...@apache.org.
EAGLE-324: Init branch-v0.5
Author: ralphsu
Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/75a8265c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/75a8265c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/75a8265c
Branch: refs/heads/branch-0.5
Commit: 75a8265c6ea2b26fd542de7e103a7235feb935c0
Parents: fb00fa3
Author: Ralph, Su <su...@gmail.com>
Authored: Wed Jun 1 13:50:39 2016 +0800
Committer: Ralph, Su <su...@gmail.com>
Committed: Wed Jun 1 13:50:39 2016 +0800
----------------------------------------------------------------------
eagle-core/eagle-alert/alert/.gitignore | 80 +++
eagle-core/eagle-alert/alert/README.md | 151 +++++
.../eagle-alert/alert/alert-assembly/pom.xml | 86 +++
.../src/assembly/alert-assembly.xml | 41 ++
.../eagle-alert/alert/alert-common/.gitignore | 2 +
.../eagle-alert/alert/alert-common/pom.xml | 129 ++++
.../eagle/alert/config/ConfigBusBase.java | 40 ++
.../eagle/alert/config/ConfigBusConsumer.java | 40 ++
.../eagle/alert/config/ConfigBusProducer.java | 37 ++
.../alert/config/ConfigChangeCallback.java | 5 +
.../apache/eagle/alert/config/ConfigValue.java | 33 +
.../org/apache/eagle/alert/config/ZKConfig.java | 17 +
.../eagle/alert/config/ZKConfigBuilder.java | 38 ++
.../alert/coordination/model/AlertBoltSpec.java | 114 ++++
.../coordination/model/Kafka2TupleMetadata.java | 97 +++
.../coordination/model/PolicyWorkerQueue.java | 63 ++
.../alert/coordination/model/PublishSpec.java | 82 +++
.../alert/coordination/model/RouterSpec.java | 76 +++
.../alert/coordination/model/ScheduleState.java | 215 +++++++
.../alert/coordination/model/SpoutSpec.java | 124 ++++
.../coordination/model/StreamNameSelector.java | 15 +
.../model/StreamRepartitionMetadata.java | 53 ++
.../model/StreamRepartitionStrategy.java | 82 +++
.../coordination/model/StreamRouterSpec.java | 75 +++
.../model/Tuple2StreamConverter.java | 65 ++
.../model/Tuple2StreamMetadata.java | 71 +++
.../model/VersionedPolicyDefinition.java | 53 ++
.../model/VersionedStreamDefinition.java | 53 ++
.../alert/coordination/model/WorkSlot.java | 55 ++
.../model/internal/MonitoredStream.java | 91 +++
.../model/internal/PolicyAssignment.java | 69 ++
.../model/internal/StreamGroup.java | 82 +++
.../model/internal/StreamWorkSlotQueue.java | 124 ++++
.../coordination/model/internal/Topology.java | 170 +++++
.../eagle/alert/engine/codec/IStreamCodec.java | 30 +
.../alert/engine/codec/SherlockEventCodec.java | 41 ++
.../alert/engine/codec/SherlockMetricCodec.java | 39 ++
.../engine/coordinator/PolicyDefinition.java | 199 ++++++
.../alert/engine/coordinator/Publishment.java | 101 +++
.../engine/coordinator/PublishmentType.java | 83 +++
.../alert/engine/coordinator/StreamColumn.java | 128 ++++
.../engine/coordinator/StreamDefinition.java | 105 +++
.../engine/coordinator/StreamPartition.java | 143 +++++
.../engine/coordinator/StreamSortSpec.java | 98 +++
.../engine/coordinator/StreamingCluster.java | 89 +++
.../alert/engine/model/AlertStreamEvent.java | 89 +++
.../alert/engine/model/PartitionedEvent.java | 149 +++++
.../eagle/alert/engine/model/StreamEvent.java | 144 +++++
.../alert/engine/model/StreamEventBuilder.java | 72 +++
.../eagle/alert/metric/IMetricSystem.java | 64 ++
.../apache/eagle/alert/metric/MetricSystem.java | 116 ++++
.../eagle/alert/metric/entity/MetricEvent.java | 111 ++++
.../alert/metric/reporter/KafkaReporter.java | 197 ++++++
.../eagle/alert/metric/sink/ConsoleSink.java | 47 ++
.../alert/metric/sink/ElasticSearchSink.java | 74 +++
.../apache/eagle/alert/metric/sink/JmxSink.java | 48 ++
.../eagle/alert/metric/sink/KafkaSink.java | 65 ++
.../eagle/alert/metric/sink/MetricSink.java | 30 +
.../alert/metric/sink/MetricSinkRepository.java | 47 ++
.../eagle/alert/metric/sink/Slf4jSink.java | 74 +++
.../alert/metric/source/JVMMetricSource.java | 41 ++
.../eagle/alert/metric/source/MetricSource.java | 24 +
.../metric/source/MetricSourceWrapper.java | 39 ++
.../eagle/alert/resource/SimpleCORSFiler.java | 59 ++
.../alert/service/IMetadataServiceClient.java | 69 ++
.../service/MetadataServiceClientImpl.java | 235 +++++++
.../eagle/alert/utils/AlertConstants.java | 27 +
.../org/apache/eagle/alert/utils/ByteUtils.java | 172 +++++
.../apache/eagle/alert/utils/ConfigUtils.java | 31 +
.../apache/eagle/alert/utils/DateTimeUtil.java | 141 ++++
.../org/apache/eagle/alert/utils/HostUtils.java | 70 ++
.../org/apache/eagle/alert/utils/JsonUtils.java | 41 ++
.../eagle/alert/utils/StreamIdConversion.java | 36 ++
.../eagle/alert/utils/TimePeriodUtils.java | 62 ++
.../src/main/resources/log4j.properties | 21 +
.../eagle/alert/config/TestConfigBus.java | 61 ++
.../eagle/alert/metric/MetricSystemTest.java | 78 +++
.../service/TestMetadataServiceClientImpl.java | 40 ++
.../apache/eagle/correlation/meta/JsonTest.java | 30 +
.../src/test/resources/application.conf | 17 +
.../src/test/resources/log4j.properties | 21 +
.../alert/alert-coordinator/.gitignore | 1 +
.../eagle-alert/alert/alert-coordinator/pom.xml | 96 +++
.../eagle/alert/coordinator/Coordinator.java | 212 +++++++
.../alert/coordinator/CoordinatorConstants.java | 31 +
.../alert/coordinator/IPolicyScheduler.java | 34 +
.../alert/coordinator/IScheduleContext.java | 54 ++
.../coordinator/PolicySchedulerFactory.java | 30 +
.../eagle/alert/coordinator/ScheduleOption.java | 74 +++
.../alert/coordinator/TopologyMgmtService.java | 83 +++
.../coordinator/impl/GreedyPolicyScheduler.java | 340 ++++++++++
.../impl/MonitorMetadataGenerator.java | 287 +++++++++
.../alert/coordinator/impl/ScheduleResult.java | 42 ++
.../eagle/alert/coordinator/impl/WorkItem.java | 33 +
.../coordinator/impl/WorkQueueBuilder.java | 93 +++
.../impl/strategies/IWorkSlotStrategy.java | 32 +
.../strategies/SameTopologySlotStrategy.java | 164 +++++
.../alert/coordinator/model/AlertBoltUsage.java | 93 +++
.../alert/coordinator/model/GroupBoltUsage.java | 67 ++
.../alert/coordinator/model/TopologyUsage.java | 102 +++
.../provider/InMemScheduleConext.java | 148 +++++
.../provider/ScheduleContextBuilder.java | 400 ++++++++++++
.../resource/CoordinatorResource.java | 69 ++
.../trigger/DynamicPolicyLoader.java | 81 +++
.../trigger/PolicyChangeListener.java | 10 +
.../src/main/resources/application.conf | 27 +
.../src/main/resources/log4j.properties | 21 +
.../src/main/webapp/WEB-INF/web.xml | 87 +++
.../src/main/webapp/index.html | 18 +
.../alert/coordinator/CoordinatorTest.java | 124 ++++
.../coordinator/DynamicPolicyLoaderTest.java | 26 +
.../MetadataServiceClientImplTest.java | 57 ++
.../coordinator/ScheduleContextBuilderTest.java | 281 ++++++++
.../apache/alert/coordinator/SchedulerTest.java | 635 +++++++++++++++++++
.../alert/coordinator/WorkSlotStrategyTest.java | 162 +++++
.../mock/InMemMetadataServiceClient.java | 147 +++++
.../mock/TestTopologyMgmtService.java | 73 +++
.../src/test/resources/test-application.conf | 27 +
.../eagle-alert/alert/alert-devtools/.gitignore | 3 +
.../alert/alert-devtools/bin/kafka-producer.sh | 21 +
.../alert-devtools/bin/kafka-server-start.sh | 51 ++
.../alert-devtools/bin/kafka-server-status.sh | 24 +
.../alert-devtools/bin/kafka-server-stop.sh | 23 +
.../alert/alert-devtools/bin/kafka-topics.sh | 17 +
.../alert/alert-devtools/bin/run-class.sh | 112 ++++
.../alert-devtools/bin/start-coordinator.sh | 20 +
.../alert-devtools/bin/start-integration1.sh | 16 +
.../alert-devtools/bin/start-integration2.sh | 19 +
.../alert/alert-devtools/bin/start-metadata.sh | 20 +
.../alert-devtools/bin/start-sampleclient1.sh | 6 +
.../alert-devtools/bin/start-sampleclient2.sh | 6 +
.../alert/alert-devtools/bin/start-zk-kafka.sh | 28 +
.../alert/alert-devtools/bin/stop-zk-kafka.sh | 28 +
.../bin/zookeeper-server-start.sh | 50 ++
.../bin/zookeeper-server-status.sh | 24 +
.../alert-devtools/bin/zookeeper-server-stop.sh | 24 +
.../alert-devtools/conf/cli-log4j.properties | 21 +
.../alert-devtools/conf/kafka-server.properties | 115 ++++
.../alert/alert-devtools/conf/log4j.properties | 21 +
.../conf/zookeeper-server.properties | 20 +
.../eagle-alert/alert/alert-devtools/pom.xml | 96 +++
.../eagle/alert/tools/KafkaConsumerOffset.java | 27 +
.../alert/tools/KafkaConsumerOffsetFetcher.java | 66 ++
.../alert/tools/KafkaLatestOffsetFetcher.java | 97 +++
.../eagle/contrib/kafka/ProducerTool.scala | 230 +++++++
.../eagle/alert/tools/TestKafkaOffset.java | 69 ++
.../kafka-offset-test.application.conf | 9 +
.../src/test/resources/log4j.properties | 21 +
.../eagle-alert/alert/alert-engine/.gitignore | 1 +
.../alert-engine/alert-engine-base/pom.xml | 65 ++
.../alert/engine/AlertStreamCollector.java | 27 +
.../apache/eagle/alert/engine/Collector.java | 27 +
.../alert/engine/PartitionedEventCollector.java | 29 +
.../eagle/alert/engine/StreamContext.java | 27 +
.../eagle/alert/engine/StreamContextImpl.java | 26 +
.../eagle/alert/engine/UnitTopologyMain.java | 59 ++
.../IMetadataChangeNotifyService.java | 57 ++
.../alert/engine/coordinator/MetadataType.java | 30 +
.../StreamDefinitionNotFoundException.java | 25 +
.../AbstractMetadataChangeNotifyService.java | 113 ++++
.../impl/ZKMetadataChangeNotifyService.java | 200 ++++++
.../engine/evaluator/PolicyChangeListener.java | 29 +
.../engine/evaluator/PolicyGroupEvaluator.java | 45 ++
.../engine/evaluator/PolicyHandlerContext.java | 60 ++
.../engine/evaluator/PolicyStreamHandler.java | 27 +
.../engine/evaluator/PolicyStreamHandlers.java | 33 +
...ertBoltOutputCollectorThreadSafeWrapper.java | 118 ++++
.../impl/AlertBoltOutputCollectorWrapper.java | 55 ++
.../impl/PolicyGroupEvaluatorImpl.java | 172 +++++
.../evaluator/impl/SiddhiDefinitionAdapter.java | 142 +++++
.../evaluator/impl/SiddhiPolicyHandler.java | 148 +++++
.../engine/publisher/AlertDeduplicator.java | 31 +
.../engine/publisher/AlertPublishListener.java | 28 +
.../engine/publisher/AlertPublishPlugin.java | 50 ++
.../publisher/AlertPublishSpecListener.java | 26 +
.../alert/engine/publisher/AlertPublisher.java | 31 +
.../eagle/alert/engine/publisher/AlertSink.java | 33 +
.../engine/publisher/PublishConstants.java | 48 ++
.../publisher/email/AlertEmailConstants.java | 39 ++
.../publisher/email/AlertEmailContext.java | 71 +++
.../publisher/email/AlertEmailGenerator.java | 143 +++++
.../email/AlertEmailGeneratorBuilder.java | 59 ++
.../publisher/email/AlertEmailSender.java | 164 +++++
.../engine/publisher/email/EagleMailClient.java | 240 +++++++
.../impl/AlertEagleStorePersister.java | 60 ++
.../impl/AlertEagleStorePublisher.java | 113 ++++
.../publisher/impl/AlertEmailPublisher.java | 151 +++++
.../publisher/impl/AlertKafkaPublisher.java | 136 ++++
.../impl/AlertPublishPluginsFactory.java | 48 ++
.../publisher/impl/AlertPublisherImpl.java | 154 +++++
.../publisher/impl/DefaultDeduplicator.java | 113 ++++
.../alert/engine/publisher/impl/EventUniq.java | 57 ++
.../publisher/impl/KafkaProducerManager.java | 42 ++
.../engine/publisher/impl/PublishStatus.java | 27 +
.../engine/router/AlertBoltSpecListener.java | 32 +
.../alert/engine/router/SpoutSpecListener.java | 32 +
.../eagle/alert/engine/router/StreamRoute.java | 85 +++
.../router/StreamRoutePartitionFactory.java | 37 ++
.../engine/router/StreamRoutePartitioner.java | 29 +
.../engine/router/StreamRouteSpecListener.java | 30 +
.../eagle/alert/engine/router/StreamRouter.java | 30 +
.../router/StreamRouterBoltSpecListener.java | 33 +
.../alert/engine/router/StreamSortHandler.java | 42 ++
.../engine/router/StreamSortSpecListener.java | 28 +
.../impl/BasicStreamRoutePartitioner.java | 77 +++
.../router/impl/RoutePhysicalGrouping.java | 76 +++
.../engine/router/impl/ShuffleGrouping.java | 74 +++
.../impl/StreamRouterBoltOutputCollector.java | 229 +++++++
.../engine/router/impl/StreamRouterImpl.java | 163 +++++
.../alert/engine/runner/AbstractStreamBolt.java | 93 +++
.../eagle/alert/engine/runner/AlertBolt.java | 157 +++++
.../alert/engine/runner/AlertPublisherBolt.java | 113 ++++
.../alert/engine/runner/MapComparator.java | 72 +++
.../engine/runner/StormMetricConsumer.java | 136 ++++
.../runner/StormMetricTaggedConsumer.java | 156 +++++
.../alert/engine/runner/StreamRouterBolt.java | 201 ++++++
.../alert/engine/runner/UnitTopologyRunner.java | 215 +++++++
.../eagle/alert/engine/scheme/JsonScheme.java | 71 +++
.../scheme/JsonStringStreamNameSelector.java | 74 +++
.../alert/engine/scheme/PlainStringScheme.java | 67 ++
.../scheme/PlainStringStreamNameSelector.java | 49 ++
.../PartitionedEventDigestSerializer.java | 62 ++
.../PartitionedEventSerializer.java | 39 ++
.../SerializationMetadataProvider.java | 30 +
.../alert/engine/serialization/Serializer.java | 26 +
.../alert/engine/serialization/Serializers.java | 56 ++
.../serialization/impl/BooleanSerializer.java | 35 +
.../serialization/impl/DoubleSerializer.java | 35 +
.../serialization/impl/FloatSerializer.java | 35 +
.../serialization/impl/IntegerSerializer.java | 35 +
.../impl/JavaObjectSerializer.java | 42 ++
.../serialization/impl/LongSerializer.java | 35 +
.../impl/PartitionedEventSerializerImpl.java | 98 +++
.../impl/StreamEventSerializer.java | 87 +++
.../impl/StreamPartitionDigestSerializer.java | 83 +++
.../impl/StreamPartitionSerializer.java | 64 ++
.../serialization/impl/StringSerializer.java | 35 +
.../extension/AttributeCollectAggregator.java | 122 ++++
.../alert/engine/sorter/BaseStreamWindow.java | 183 ++++++
.../alert/engine/sorter/StreamTimeClock.java | 45 ++
.../engine/sorter/StreamTimeClockListener.java | 27 +
.../engine/sorter/StreamTimeClockManager.java | 45 ++
.../engine/sorter/StreamTimeClockTrigger.java | 61 ++
.../eagle/alert/engine/sorter/StreamWindow.java | 113 ++++
.../engine/sorter/StreamWindowManager.java | 64 ++
.../engine/sorter/StreamWindowRepository.java | 257 ++++++++
.../sorter/impl/CachedEventGroupSerializer.java | 114 ++++
.../impl/PartitionedEventGroupSerializer.java | 112 ++++
.../PartitionedEventTimeOrderingComparator.java | 50 ++
.../impl/StreamSortWindowHandlerImpl.java | 113 ++++
.../sorter/impl/StreamSortedWindowInMapDB.java | 145 +++++
.../sorter/impl/StreamSortedWindowOnHeap.java | 80 +++
.../impl/StreamTimeClockInLocalMemory.java | 64 ++
.../sorter/impl/StreamTimeClockManagerImpl.java | 171 +++++
.../sorter/impl/StreamWindowManagerImpl.java | 173 +++++
.../alert/engine/spout/CorrelationSpout.java | 380 +++++++++++
.../alert/engine/spout/CreateTopicUtils.java | 43 ++
.../eagle/alert/engine/spout/ISpoutSpecLCM.java | 41 ++
.../engine/spout/KafkaMessageIdWrapper.java | 42 ++
.../eagle/alert/engine/spout/SchemeBuilder.java | 35 +
.../spout/SpoutOutputCollectorWrapper.java | 220 +++++++
.../alert/engine/utils/CompressionUtils.java | 62 ++
.../alert/engine/utils/MetadataSerDeser.java | 93 +++
.../alert/engine/utils/SerializableUtils.java | 112 ++++
.../main/java/storm/kafka/KafkaSpoutMetric.java | 78 +++
.../java/storm/kafka/KafkaSpoutWrapper.java | 111 ++++
.../src/main/resources/ALERT_DEFAULT.vm | 267 ++++++++
.../src/main/resources/application.conf | 72 +++
.../src/main/resources/eagle.siddhiext | 19 +
.../src/main/resources/log4j.properties | 26 +
.../alert/engine/e2e/CoordinatorClient.java | 95 +++
.../eagle/alert/engine/e2e/Integration1.java | 236 +++++++
.../eagle/alert/engine/e2e/Integration2.java | 142 +++++
.../eagle/alert/engine/e2e/SampleClient1.java | 127 ++++
.../eagle/alert/engine/e2e/SampleClient2.java | 139 ++++
...oltOutputCollectorThreadSafeWrapperTest.java | 74 +++
.../SiddhiCEPPolicyEventHandlerTest.java | 161 +++++
.../integration/MockMetadataServiceClient.java | 124 ++++
.../engine/metric/MemoryUsageGaugeSetTest.java | 47 ++
.../engine/mock/MockPartitionedCollector.java | 55 ++
.../engine/mock/MockSampleMetadataFactory.java | 266 ++++++++
.../alert/engine/mock/MockStreamCollector.java | 51 ++
.../engine/mock/MockStreamMetadataService.java | 39 ++
.../alert/engine/mock/MockStreamReceiver.java | 79 +++
.../alert/engine/perf/TestSerDeserPer.java | 301 +++++++++
.../alert/engine/router/TestAlertBolt.java | 177 ++++++
.../engine/router/TestAlertPublisherBolt.java | 111 ++++
.../engine/runner/TestStreamRouterBolt.java | 259 ++++++++
.../serialization/JavaSerializationTest.java | 112 ++++
.../PartitionedEventSerializerTest.java | 220 +++++++
.../AttributeCollectAggregatorTest.java | 149 +++++
.../alert/engine/sorter/MapDBTestSuite.java | 45 ++
.../engine/sorter/StreamSortHandlerTest.java | 270 ++++++++
.../sorter/StreamWindowBenchmarkTest.java | 145 +++++
.../engine/sorter/StreamWindowTestSuite.java | 164 +++++
.../sorter/TreeMultisetComparatorTest.java | 94 +++
.../engine/topology/AlertTopologyTest.java | 138 ++++
.../CoordinatorSpoutIntegrationTest.java | 101 +++
.../engine/topology/CorrelationSpoutTest.java | 184 ++++++
.../engine/topology/FastWordCountTopology.java | 215 +++++++
.../MockMetadataChangeNotifyService.java | 111 ++++
.../engine/topology/SendData2KafkaTest.java | 103 +++
.../eagle/alert/engine/topology/TestBolt.java | 61 ++
.../alert/engine/topology/TestByteBuffer.java | 53 ++
.../topology/TestMetadataSpecSerDeser.java | 266 ++++++++
.../TestStormCustomGroupingRouting.java | 144 +++++
.../engine/topology/TestStormParallelism.java | 161 +++++
.../topology/TestStormStreamIdRouting.java | 140 ++++
.../topology/TestTuple2StreamConverter.java | 62 ++
.../engine/topology/TestUnitTopologyMain.java | 57 ++
.../engine/utils/CompressionUtilsTest.java | 43 ++
.../alert/engine/utils/TimePeriodUtilsTest.java | 91 +++
.../test/resources/application-integration.conf | 57 ++
.../test/resources/application-test-backup.conf | 71 +++
.../src/test/resources/application-test.conf | 68 ++
.../correlation/application-integration-2.conf | 57 ++
.../test/resources/correlation/datasources.json | 37 ++
.../test/resources/correlation/policies.json | 39 ++
.../resources/correlation/publishments.json | 17 +
.../correlation/streamdefinitions.json | 93 +++
.../test/resources/correlation/topologies.json | 31 +
.../src/test/resources/datasources.json | 19 +
.../src/test/resources/log4j.properties | 23 +
.../src/test/resources/policies.json | 54 ++
.../src/test/resources/publishments.json | 29 +
.../src/test/resources/publishments2.json | 19 +
.../src/test/resources/sample_perfmon_data.json | 3 +
.../src/test/resources/streamdefinitions.json | 44 ++
.../src/test/resources/testAlertBoltSpec.json | 92 +++
.../src/test/resources/testPublishSpec.json | 32 +
.../src/test/resources/testPublishSpec2.json | 32 +
.../src/test/resources/testSpoutSpec.json | 139 ++++
.../resources/testStreamDefinitionsSpec.json | 47 ++
.../resources/testStreamRouterBoltSpec.json | 123 ++++
.../src/test/resources/topic.json | 1 +
.../src/test/resources/topologies.json | 31 +
.../eagle-alert/alert/alert-engine/pom.xml | 110 ++++
.../alert-metadata-service/.gitignore | 2 +
.../alert-metadata-service/pom.xml | 113 ++++
.../metadata/resource/MetadataResource.java | 225 +++++++
.../topology/resource/TopologyMgmtResource.java | 69 ++
.../impl/TopologyMgmtResourceHelper.java | 48 ++
.../resource/impl/TopologyMgmtResourceImpl.java | 158 +++++
.../topology/resource/impl/TopologyStatus.java | 82 +++
.../src/main/resources/application.conf | 6 +
.../src/main/webapp/WEB-INF/web.xml | 81 +++
.../src/main/webapp/index.html | 18 +
.../impl/TopologyMgmtResourceImplTest.java | 55 ++
.../alert-metadata/.gitignore | 2 +
.../alert-metadata/pom.xml | 52 ++
.../metadata/impl/InMemMetadataDaoImpl.java | 333 ++++++++++
.../alert/metadata/impl/MetadataDaoFactory.java | 70 ++
.../metadata/impl/MongoMetadataDaoImpl.java | 380 +++++++++++
.../metadata/impl/jdbc/JdbcMetadataDaoImpl.java | 292 +++++++++
.../alert/metadata/resource/IMetadataDao.java | 86 +++
.../eagle/alert/metadata/resource/Models.java | 48 ++
.../eagle/alert/metadata/resource/OpResult.java | 28 +
.../src/main/resources/application.conf | 8 +
.../alert/resource/impl/InMemoryTest.java | 43 ++
.../alert/resource/impl/MongoImplTest.java | 187 ++++++
.../src/test/resources/application-mongo.conf | 6 +
.../src/test/resources/policy-sample.json | 32 +
.../alert/alert-metadata-parent/pom.xml | 29 +
eagle-core/eagle-alert/alert/pom.xml | 387 +++++++++++
364 files changed, 31608 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/.gitignore
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/.gitignore b/eagle-core/eagle-alert/alert/.gitignore
new file mode 100644
index 0000000..12af842
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/.gitignore
@@ -0,0 +1,80 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Eagle - Git Ignore Configuration
+#
+# See: https://github.com/github/gitignore/
+
+*.class
+*.out
+
+# Mobile Tools for Java (J2ME)
+.mtj.tmp/
+
+# Package Files #
+*.jar
+*.war
+*.ear
+
+.cache-main
+
+# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
+hs_err_pid*
+
+# Maven
+target/
+target/*
+*/target/*
+**/target/*
+pom.xml.tag
+pom.xml.releaseBackup
+pom.xml.versionsBackup
+pom.xml.next
+release.properties
+dependency-reduced-pom.xml
+buildNumber.properties
+eagle-security/eagle-security-mltraining/src/test/resources/outputDir/
+eagle-security/eagle-security-userprofile/src/test/resources/models/
+
+# IntelliJ IDEA
+.idea/
+.metadata/
+**.iml
+**.eml
+**.patch
+mobile.userlibraries
+
+# Eclipse
+build
+.metadata
+.classpath
+.project
+.settings
+.externalToolBuilders
+classes
+# bin
+
+# Temporary files
+logs/
+*.log*
+# Mac files
+.DS_Store
+
+*.cache-tests
+application-local.conf
+
+*.orig
+**/*.pyc
+*.bak
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/README.md
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/README.md b/eagle-core/eagle-alert/alert/README.md
new file mode 100644
index 0000000..54cd226
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/README.md
@@ -0,0 +1,151 @@
+
+## Prerequisites
+
+* [Apache Maven](https://maven.apache.org/)
+* [Java 8](http://www.oracle.com/technetwork/java/javase/downloads/jdk8-downloads-2133151.html)
+
+## Documentation
+
+
+## Build
+
+ mvn install
+
+## Setup
+The alert engine have three dependency module: Coordinator Service, Metadata Service, and engine runtime(storm topologies).
+
+####0. Dependencies
+> Alert engine need kafka as data source, ZK as coordination. Check alert-devtools/bin to start zk and kafka through start-zk-kafka.sh.
+
+####1. Start metadata service
+> For local dev, project alert-metadata-service packaging as a war, and enabled mvn jetty:run to run it. By default, metadata runs on localhost:8080
+
+> For deployment, after mvn install, a war is avaialble in alert-metadata-service/target
+
+####2. Start coordiantor service
+> For local dev, project alert-coordinator packaing as a war, and enabled mvn jetty:run to run it. By default, it runs in localhost:9090, and have dependency on metadata. See application.conf for coordinator.
+
+> For deployment, find war in alert-coordinator/target after mvn install
+
+####3. Start engine runtime.
+> The engine are the topologies that runs in any storm (local or remote) with configuration to connect to the ZK and metadata service. The alert engine runtime main as in UnitTopologyMain.java. The started storm bolt should have the same name described in alert-metadata. Example of the configuration is /alert-engine-base/src/main/resources/application.conf
+
+See below detailed steps.
+
+
+## Run
+* pre-requisites
+ * zookeeper
+ * storm
+ * kafka
+ * tomcat
+ * mongdb
+
+* Run Metadata service
+ 1. copy alert-metadata/target/alert-metadata-0.0.1-SNAPSHOT.war into tomcat webapps/alertmetadata.war
+ 2. check config under webapps/alertmetadata/WEB-INF/classes/application.conf
+ ```json
+ {
+ "datastore": {
+ "metadataDao": "org.apache.eagle.alert.metadata.impl.MongoMetadataDaoImpl",
+ "connection": "localhost:27017"
+ }
+ }
+ ```
+
+ 3. start tomcat
+
+* Run Coordinator service
+ 1. copy alert-coordinator/target/alert-coordinator-0.0.1-SNAPSHOT.war to tomcat webappes/coordinator.war
+ 2. check config under webapps/coordinator/WEB-INF/classes/application.conf
+ ```json
+ {
+ "coordinator" : {
+ "policiesPerBolt" : 5,
+ "boltParallelism" : 5,
+ "policyDefaultParallelism" : 5,
+ "boltLoadUpbound": 0.8,
+ "topologyLoadUpbound" : 0.8,
+ "numOfAlertBoltsPerTopology" : 5,
+ "zkConfig" : {
+ "zkQuorum" : "localhost:2181",
+ "zkRoot" : "/alert",
+ "zkSessionTimeoutMs" : 10000,
+ "connectionTimeoutMs" : 10000,
+ "zkRetryTimes" : 3,
+ "zkRetryInterval" : 3000
+ },
+ "metadataService" : {
+ "host" : "localhost",
+ "port" : 8080,
+ "context" : "/alertmetadata/api"
+ },
+ "metadataDynamicCheck" : {
+ "initDelayMillis" : 1000,
+ "delayMillis" : 30000
+ }
+ }
+ }
+ ```
+ 3. start tomcat
+
+* Run UnitTopologyMain
+ 1. copy alert-assembly/target/alert-engine-0.0.1-SNAPSHOT-alert-assembly.jar to somewhere close to your storm installation
+ 2. check config application.conf
+ ```json
+ {
+ "topology" : {
+ "name" : "alertUnitTopology_1",
+ "numOfTotalWorkers" : 2,
+ "numOfSpoutTasks" : 1,
+ "numOfRouterBolts" : 4,
+ "numOfAlertBolts" : 10,
+ "numOfPublishTasks" : 1,
+ "messageTimeoutSecs": 3600,
+ "localMode" : "true"
+ },
+ "spout" : {
+ "kafkaBrokerZkQuorum": "localhost:2181",
+ "kafkaBrokerZkBasePath": "/brokers",
+ "stormKafkaUseSameZkQuorumWithKafkaBroker": true,
+ "stormKafkaTransactionZkQuorum": "",
+ "stormKafkaTransactionZkPath": "/consumers",
+ "stormKafkaEagleConsumer": "eagle_consumer",
+ "stormKafkaStateUpdateIntervalMs": 2000,
+ "stormKafkaFetchSizeBytes": 1048586,
+ },
+ "zkConfig" : {
+ "zkQuorum" : "localhost:2181",
+ "zkRoot" : "/alert",
+ "zkSessionTimeoutMs" : 10000,
+ "connectionTimeoutMs" : 10000,
+ "zkRetryTimes" : 3,
+ "zkRetryInterval" : 3000
+ },
+ "dynamicConfigSource" : {
+ "initDelayMillis": 3000,
+ "delayMillis" : 10000
+ },
+ "metadataService": {
+ "context" : "/alertmetadata/api",
+ "host" : "localhost",
+ "port" : 8080
+ },
+ "coordinatorService": {
+ "host": "localhost",
+ "port": 8080,
+ "context" : "/coordinator/api"
+ }
+ "metric": {
+ "sink": {
+ "stdout": {}
+ }
+ }
+}
+```
+ Note: please make sure the above configuration is used by storm instead of the configuration within fat jar
+ 3. start storm
+ storm jar alert-engine-0.0.1-SNAPSHOT-alert-assembly.jar org.apache.eagle.alert.engine.UnitTopologyMain
+
+## Support
+
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-assembly/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-assembly/pom.xml b/eagle-core/eagle-alert/alert/alert-assembly/pom.xml
new file mode 100644
index 0000000..b48ee5b
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-assembly/pom.xml
@@ -0,0 +1,86 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.eagle</groupId>
+ <artifactId>alert-parent</artifactId>
+ <version>0.0.1-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>alert-assembly</artifactId>
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.eagle</groupId>
+ <artifactId>alert-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.eagle</groupId>
+ <artifactId>alert-coordinator</artifactId>
+ <version>${project.version}</version>
+ <type>war</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.eagle</groupId>
+ <artifactId>alert-metadata-service</artifactId>
+ <version>${project.version}</version>
+ <type>war</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.eagle</groupId>
+ <artifactId>alert-engine-base</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <!-- package test cases -->
+ <dependency>
+ <groupId>org.apache.eagle</groupId>
+ <artifactId>alert-common</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.eagle</groupId>
+ <artifactId>alert-coordinator</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.eagle</groupId>
+ <artifactId>alert-engine-base</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <resources>
+ <resource>
+ <directory>src/resources</directory>
+ </resource>
+ </resources>
+ <plugins>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <descriptor>src/assembly/alert-assembly.xml</descriptor>
+ <finalName>alert-engine-${project.version}</finalName>
+ </configuration>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <configuration>
+ <tarLongFileMode>posix</tarLongFileMode>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-assembly/src/assembly/alert-assembly.xml
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-assembly/src/assembly/alert-assembly.xml b/eagle-core/eagle-alert/alert/alert-assembly/src/assembly/alert-assembly.xml
new file mode 100644
index 0000000..556b2ad
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-assembly/src/assembly/alert-assembly.xml
@@ -0,0 +1,41 @@
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
+ <id>alert-assembly</id>
+ <formats>
+ <format>jar</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <dependencySets>
+ <dependencySet>
+ <outputDirectory>/</outputDirectory>
+ <useProjectArtifact>false</useProjectArtifact>
+ <unpack>true</unpack>
+ <scope>runtime</scope>
+ <unpackOptions>
+ </unpackOptions>
+ <excludes>
+ <!--<exclude>org.apache.storm:storm-core</exclude>
+ <exclude>org.slf4j:slf4j-log4j12</exclude>-->
+ <exclude>org.apache.storm:storm-core</exclude>
+ <exclude>org.slf4j:slf4j-api</exclude>
+ <exclude>org.slf4j:log4j-over-slf4j</exclude>
+ <exclude>org.slf4j:slf4j-log4j12</exclude>
+ <exclude>log4j:log4j</exclude>
+ <exclude>asm:asm</exclude>
+ <exclude>org.apache.log4j.wso2:log4j</exclude>
+ <exclude>log4j:apache-log4j-extras</exclude>
+ </excludes>
+ </dependencySet>
+ </dependencySets>
+ <fileSets>
+ <fileSet>
+ <directory>${project.build.outputDirectory}</directory>
+ <outputDirectory>/</outputDirectory>
+ <excludes>
+ </excludes>
+ <includes>
+ </includes>
+ </fileSet>
+ </fileSets>
+</assembly>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/.gitignore
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/.gitignore b/eagle-core/eagle-alert/alert/alert-common/.gitignore
new file mode 100644
index 0000000..1dd3331
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/.gitignore
@@ -0,0 +1,2 @@
+/target/
+/target/
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/pom.xml b/eagle-core/eagle-alert/alert/alert-common/pom.xml
new file mode 100644
index 0000000..129cff0
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/pom.xml
@@ -0,0 +1,129 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with ~
+ this work for additional information regarding copyright ownership. ~ The
+ ASF licenses this file to You under the Apache License, Version 2.0 ~ (the
+ "License"); you may not use this file except in compliance with ~ the License.
+ You may obtain a copy of the License at ~ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~ ~ Unless required by applicable law or agreed to in writing, software ~
+ distributed under the License is distributed on an "AS IS" BASIS, ~ WITHOUT
+ WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ~ See the
+ License for the specific language governing permissions and ~ limitations
+ under the License. -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.eagle</groupId>
+ <artifactId>alert-parent</artifactId>
+ <version>0.0.1-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>alert-common</artifactId>
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>com.typesafe</groupId>
+ <artifactId>config</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-collections</groupId>
+ <artifactId>commons-collections</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-framework</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-recipes</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-client</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-jaxrs</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>joda-time</groupId>
+ <artifactId>joda-time</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.storm</groupId>
+ <artifactId>storm-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch</groupId>
+ <artifactId>metrics-elasticsearch-reporter</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka-clients</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>${kafka.artifact.id}</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-jvm</artifactId>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <version>2.6</version>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/config/ConfigBusBase.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/config/ConfigBusBase.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/config/ConfigBusBase.java
new file mode 100644
index 0000000..7644d89
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/config/ConfigBusBase.java
@@ -0,0 +1,40 @@
+package org.apache.eagle.alert.config;
+
+import java.io.Closeable;
+
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.retry.RetryNTimes;
+
+
+/**
+ * Abstraction of asynchronized configuration management
+ * This is used for config change notification between processes, without this one process has to pull changes triggered by another process
+ *
+ * Config bus is similar to message bus, config change producer can publish config change(message) to config bus,
+ * while config change consumer can subscribe config change and do business logic in callback
+ * 1. use zookeeper as media to notify config consumer of config changes
+ * 2. each type of config is represented by topic
+ * 3. each config change can contain actual value or contain reference Id which consumer uses to retrieve actual value. This mechanism will reduce zookeeper overhed
+ *
+ */
+public class ConfigBusBase implements Closeable{
+ protected String zkRoot;
+ protected CuratorFramework curator;
+
+ public ConfigBusBase(ZKConfig config) {
+ this.zkRoot = config.zkRoot;
+ curator = CuratorFrameworkFactory.newClient(
+ config.zkQuorum,
+ config.zkSessionTimeoutMs,
+ config.connectionTimeoutMs,
+ new RetryNTimes(config.zkRetryTimes, config.zkRetryInterval)
+ );
+ curator.start();
+ }
+
+ @Override
+ public void close(){
+ curator.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/config/ConfigBusConsumer.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/config/ConfigBusConsumer.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/config/ConfigBusConsumer.java
new file mode 100644
index 0000000..11eea1f
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/config/ConfigBusConsumer.java
@@ -0,0 +1,40 @@
+package org.apache.eagle.alert.config;
+
+import org.apache.curator.framework.recipes.cache.NodeCache;
+import org.slf4j.Logger;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+/**
+ * 1. When consumer is started, it always get notified of config
+ * 2. When config is changed, consumer always get notified of config change
+ *
+ * Reliability issue:
+ * TODO How to ensure config change message is always delivered to consumer
+ */
+public class ConfigBusConsumer extends ConfigBusBase {
+ private static final Logger LOG = org.slf4j.LoggerFactory.getLogger(ConfigBusConsumer.class);
+
+ private NodeCache cache;
+ public ConfigBusConsumer(ZKConfig config, String topic, ConfigChangeCallback callback){
+ super(config);
+ String zkPath = zkRoot + "/" + topic;
+ LOG.info("monitor change for zkPath " + zkPath);
+ cache = new NodeCache(curator, zkPath);
+ cache.getListenable().addListener( () ->
+ {
+ // get node value and notify callback
+ byte[] value = curator.getData().forPath(zkPath);
+ ObjectMapper mapper = new ObjectMapper();
+ ConfigValue v = mapper.readValue(value, ConfigValue.class);
+ callback.onNewConfig(v);
+ }
+ );
+ try {
+ cache.start();
+ }catch(Exception ex){
+ LOG.error("error start NodeCache listener", ex);
+ throw new RuntimeException(ex);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/config/ConfigBusProducer.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/config/ConfigBusProducer.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/config/ConfigBusProducer.java
new file mode 100644
index 0000000..6dab51a
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/config/ConfigBusProducer.java
@@ -0,0 +1,37 @@
+package org.apache.eagle.alert.config;
+
+import org.apache.zookeeper.CreateMode;
+import org.slf4j.Logger;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+public class ConfigBusProducer extends ConfigBusBase {
+ private static final Logger LOG = org.slf4j.LoggerFactory.getLogger(ConfigBusProducer.class);
+
+ public ConfigBusProducer(ZKConfig config){
+ super(config);
+ }
+
+ /**
+ * @param topic
+ * @param config
+ */
+ public void send(String topic, ConfigValue config){
+ // check if topic exists, create this topic if not existing
+ String zkPath = zkRoot + "/" + topic;
+ try {
+ if (curator.checkExists().forPath(zkPath) == null) {
+ curator.create()
+ .creatingParentsIfNeeded()
+ .withMode(CreateMode.PERSISTENT)
+ .forPath(zkPath);
+ }
+ ObjectMapper mapper = new ObjectMapper();
+ byte[] content = mapper.writeValueAsBytes(config);
+ curator.setData().forPath(zkPath, content);
+ }catch(Exception ex){
+ LOG.error("error creating zkPath " + zkPath, ex);
+ throw new RuntimeException(ex);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/config/ConfigChangeCallback.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/config/ConfigChangeCallback.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/config/ConfigChangeCallback.java
new file mode 100644
index 0000000..60df605
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/config/ConfigChangeCallback.java
@@ -0,0 +1,5 @@
+package org.apache.eagle.alert.config;
+
+public interface ConfigChangeCallback {
+ void onNewConfig(ConfigValue value);
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/config/ConfigValue.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/config/ConfigValue.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/config/ConfigValue.java
new file mode 100644
index 0000000..f59c2b9
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/config/ConfigValue.java
@@ -0,0 +1,33 @@
+package org.apache.eagle.alert.config;
+
+
+/**
+ * Config body contains actual data for one topic
+ * this is serialized with json format into zookeeper
+ * value can be versionId which is used for referencing outside data
+ * or value can be actual config value
+ */
+public class ConfigValue {
+ private boolean isValueVersionId;
+ private Object value;
+
+ public boolean isValueVersionId() {
+ return isValueVersionId;
+ }
+
+ public void setValueVersionId(boolean valueVersionId) {
+ isValueVersionId = valueVersionId;
+ }
+
+ public Object getValue() {
+ return value;
+ }
+
+ public void setValue(Object value) {
+ this.value = value;
+ }
+
+ public String toString(){
+ return "isValueVersionId: " + isValueVersionId + ", value: " + value;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/config/ZKConfig.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/config/ZKConfig.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/config/ZKConfig.java
new file mode 100644
index 0000000..fe04b4b
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/config/ZKConfig.java
@@ -0,0 +1,17 @@
+package org.apache.eagle.alert.config;
+
+import java.io.Serializable;
+
+/**
+ * Memory representation of key zookeeper configurations
+ */
+public class ZKConfig implements Serializable{
+ private static final long serialVersionUID = -1287231022807492775L;
+
+ public String zkQuorum;
+ public String zkRoot;
+ public int zkSessionTimeoutMs;
+ public int connectionTimeoutMs;
+ public int zkRetryTimes;
+ public int zkRetryInterval;
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/config/ZKConfigBuilder.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/config/ZKConfigBuilder.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/config/ZKConfigBuilder.java
new file mode 100644
index 0000000..9d77a58
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/config/ZKConfigBuilder.java
@@ -0,0 +1,38 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.alert.config;
+
+import com.typesafe.config.Config;
+
+/**
+ * Since 4/28/16.
+ */
+public class ZKConfigBuilder {
+ public static ZKConfig getZKConfig(Config config){
+ ZKConfig zkConfig = new ZKConfig();
+ zkConfig.zkQuorum = config.getString("zkConfig.zkQuorum");
+ zkConfig.zkRoot = config.getString("zkConfig.zkRoot");
+ zkConfig.zkSessionTimeoutMs = config.getInt("zkConfig.zkSessionTimeoutMs");
+ zkConfig.connectionTimeoutMs = config.getInt("zkConfig.connectionTimeoutMs");
+ zkConfig.zkRetryTimes = config.getInt("zkConfig.zkRetryTimes");
+ zkConfig.zkRetryInterval = config.getInt("zkConfig.zkRetryInterval");
+ return zkConfig;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/AlertBoltSpec.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/AlertBoltSpec.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/AlertBoltSpec.java
new file mode 100644
index 0000000..83d307c
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/AlertBoltSpec.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.coordination.model;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.eagle.alert.engine.coordinator.PolicyDefinition;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+
+/**
+ * The alert specification for topology bolts.
+ *
+ * @since Apr 29, 2016
+ */
+public class AlertBoltSpec {
+ private String version;
+ private String topologyName;
+
+ // mapping from boltId to list of PolicyDefinitions
+ @JsonIgnore
+ private Map<String, List<PolicyDefinition>> boltPoliciesMap = new HashMap<String, List<PolicyDefinition>>();
+
+ // mapping from boltId to list of PolicyDefinition's Ids
+ private Map<String, List<String>> boltPolicyIdsMap = new HashMap<String, List<String>>();
+
+ public AlertBoltSpec() {
+ }
+
+ public AlertBoltSpec(String topo) {
+ this.topologyName = topo;
+ }
+
+ public String getVersion() {
+ return version;
+ }
+
+ public void setVersion(String version) {
+ this.version = version;
+ }
+
+ public String getTopologyName() {
+ return topologyName;
+ }
+
+ public void setTopologyName(String topologyName) {
+ this.topologyName = topologyName;
+ }
+
+// public List<PolicyDefinition> getBoltPolicy(String boltId) {
+// return boltPoliciesMap.get(boltId);
+// }
+//
+// public void addBoltPolicy(String boltId, PolicyDefinition pd) {
+// if (boltPoliciesMap.containsKey(boltId)) {
+// boltPoliciesMap.get(boltId).add(pd);
+// } else {
+// List<PolicyDefinition> list = new ArrayList<PolicyDefinition>();
+// boltPoliciesMap.put(boltId, list);
+// list.add(pd);
+// }
+// }
+
+ public void addBoltPolicy(String boltId, String policyName) {
+ if (boltPolicyIdsMap.containsKey(boltId)) {
+ boltPolicyIdsMap.get(boltId).add(policyName);
+ } else {
+ List<String> list = new ArrayList<String>();
+ boltPolicyIdsMap.put(boltId, list);
+ list.add(policyName);
+ }
+ }
+
+ @JsonIgnore
+ public Map<String, List<PolicyDefinition>> getBoltPoliciesMap() {
+ return boltPoliciesMap;
+ }
+
+ @JsonIgnore
+ public void setBoltPoliciesMap(Map<String, List<PolicyDefinition>> boltPoliciesMap) {
+ this.boltPoliciesMap = boltPoliciesMap;
+ }
+
+ public Map<String, List<String>> getBoltPolicyIdsMap() {
+ return boltPolicyIdsMap;
+ }
+
+ public void setBoltPolicyIdsMap(Map<String, List<String>> boltPolicyIdsMap) {
+ this.boltPolicyIdsMap = boltPolicyIdsMap;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("version:%s-topo:%s, boltPolicyIdsMap %s", version, topologyName, boltPolicyIdsMap);
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/Kafka2TupleMetadata.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/Kafka2TupleMetadata.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/Kafka2TupleMetadata.java
new file mode 100644
index 0000000..6c4f576
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/Kafka2TupleMetadata.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.coordination.model;
+
+import java.util.Map;
+
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+
+import com.google.common.base.Objects;
+
+/**
+ * @since Apr 5, 2016
+ * this metadata model controls how to convert kafka topic into tuple stream
+ */
+public class Kafka2TupleMetadata {
+ private String type;
+ private String name; // data source name
+ private Map<String, String> properties;
+ private String topic;
+ private String schemeCls;
+
+ private Tuple2StreamMetadata codec;
+
+ public String getType() {
+ return type;
+ }
+
+ public void setType(String type) {
+ this.type = type;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setTopic(String topic) {
+ this.topic = topic;
+ }
+
+ public void setSchemeCls(String schemeCls) {
+ this.schemeCls = schemeCls;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public Map<String, String> getProperties() {
+ return properties;
+ }
+
+ public void setProperties(Map<String, String> properties) {
+ this.properties = properties;
+ }
+
+ public Tuple2StreamMetadata getCodec() {
+ return codec;
+ }
+
+ public void setCodec(Tuple2StreamMetadata codec) {
+ this.codec = codec;
+ }
+
+ public String getTopic() {
+ return this.topic;
+ }
+ public String getSchemeCls() {
+ return this.schemeCls;
+ }
+
+ public int hashCode() {
+ return new HashCodeBuilder().append(name).append(type).build();
+ }
+
+ public boolean equals(Object obj) {
+ if (!(obj instanceof Kafka2TupleMetadata)) {
+ return false;
+ }
+ Kafka2TupleMetadata o = (Kafka2TupleMetadata) obj;
+ return Objects.equal(name, o.name) && Objects.equal(type, o.type);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/PolicyWorkerQueue.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/PolicyWorkerQueue.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/PolicyWorkerQueue.java
new file mode 100644
index 0000000..e5e5618
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/PolicyWorkerQueue.java
@@ -0,0 +1,63 @@
+package org.apache.eagle.alert.coordination.model;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.eagle.alert.engine.coordinator.StreamPartition;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+public class PolicyWorkerQueue {
+
+ private StreamPartition partition;
+ private List<WorkSlot> workers;
+
+ public PolicyWorkerQueue() {
+ workers = new ArrayList<>();
+ }
+
+ public PolicyWorkerQueue(List<WorkSlot> workers) {
+ this.workers = workers;
+ }
+
+ public PolicyWorkerQueue(StreamPartition partition, List<WorkSlot> workers) {
+ this.workers = workers;
+ this.partition = partition;
+ }
+
+ public StreamPartition getPartition() {
+ return partition;
+ }
+
+ public void setPartition(StreamPartition partition) {
+ this.partition = partition;
+ }
+
+ public List<WorkSlot> getWorkers() {
+ return workers;
+ }
+
+ public void setWorkers(List<WorkSlot> workers) {
+ this.workers = workers;
+ }
+
+ public String toString() {
+ return "[" + StringUtils.join(workers, ",") + "]";
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/PublishSpec.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/PublishSpec.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/PublishSpec.java
new file mode 100644
index 0000000..06e819a
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/PublishSpec.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.coordination.model;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.eagle.alert.engine.coordinator.Publishment;
+
+/**
+ *
+ * @since May 1, 2016
+ *
+ */
+public class PublishSpec {
+
+ private String topologyName;
+ // actually only publish spec for one topology
+ private String boltId;
+ private String version;
+
+ private List<Publishment> publishments = new ArrayList<Publishment>();
+
+ public PublishSpec() {
+ }
+
+ public PublishSpec(String topoName, String boltId) {
+ this.topologyName = topoName;
+ this.boltId = boltId;
+ }
+
+ public void addPublishment(Publishment p) {
+ this.publishments.add(p);
+ }
+
+ public String getTopologyName() {
+ return topologyName;
+ }
+
+ public String getBoltId() {
+ return boltId;
+ }
+
+ public List<Publishment> getPublishments() {
+ return publishments;
+ }
+
+ public String getVersion() {
+ return version;
+ }
+
+ public void setVersion(String version) {
+ this.version = version;
+ }
+
+ public void setTopologyName(String topologyName) {
+ this.topologyName = topologyName;
+ }
+
+ public void setBoltId(String boltId) {
+ this.boltId = boltId;
+ }
+
+ public void setPublishments(List<Publishment> publishments) {
+ this.publishments = publishments;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/RouterSpec.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/RouterSpec.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/RouterSpec.java
new file mode 100644
index 0000000..5241920
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/RouterSpec.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.coordination.model;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.codehaus.jackson.annotate.JsonIgnore;
+
+/**
+ * @since Apr 29, 2016
+ *
+ */
+public class RouterSpec {
+ private String version;
+ private String topologyName;
+
+ private List<StreamRouterSpec> routerSpecs;
+
+ public RouterSpec() {
+ routerSpecs = new ArrayList<StreamRouterSpec>();
+ }
+
+ public RouterSpec(String topoName) {
+ this();
+ this.topologyName = topoName;
+ }
+
+ public String getVersion() {
+ return version;
+ }
+
+ public void setVersion(String version) {
+ this.version = version;
+ }
+
+ public String getTopologyName() {
+ return topologyName;
+ }
+
+ public void setTopologyName(String topologyName) {
+ this.topologyName = topologyName;
+ }
+
+ @JsonIgnore
+ public void addRouterSpec(StreamRouterSpec routerSpec) {
+ routerSpecs.add(routerSpec);
+ }
+
+ public List<StreamRouterSpec> getRouterSpecs() {
+ return routerSpecs;
+ }
+
+ public void setRouterSpecs(List<StreamRouterSpec> routerSpecs) {
+ this.routerSpecs = routerSpecs;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("version:%s-topo:%s, boltSpec:%s", version, topologyName, routerSpecs);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/ScheduleState.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/ScheduleState.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/ScheduleState.java
new file mode 100644
index 0000000..6036f29
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/ScheduleState.java
@@ -0,0 +1,215 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.coordination.model;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.eagle.alert.coordination.model.internal.MonitoredStream;
+import org.apache.eagle.alert.coordination.model.internal.PolicyAssignment;
+import org.apache.eagle.alert.engine.coordinator.PolicyDefinition;
+import org.apache.eagle.alert.engine.coordinator.StreamDefinition;
+
+/**
+ * A global wise of schedule status <br/>
+ * <br/>
+ * TODO/FIXME: The persistence simply deserial ScheduleState to Json. One
+ * concern is that this string might become too big for store. <br/>
+ * <br/>
+ * The solution is in metadata resource, have specs/monitoredStreams/policy
+ * assignments stored in different table/collections with tage version.
+ *
+ *
+ * @since Apr 26, 2016
+ *
+ */
+public class ScheduleState {
+
+ // ScheduleSpec
+ private Map<String, SpoutSpec> spoutSpecs = new HashMap<String, SpoutSpec>();
+ private Map<String, AlertBoltSpec> alertSpecs = new HashMap<String, AlertBoltSpec>();
+ private Map<String, RouterSpec> groupSpecs = new HashMap<String, RouterSpec>();
+ private Map<String, PublishSpec> publishSpecs = new HashMap<String, PublishSpec>();
+
+ // ScheduleSnapshot
+ private List<VersionedPolicyDefinition> policySnapshots = new ArrayList<VersionedPolicyDefinition>();
+ private List<VersionedStreamDefinition> streamSnapshots = new ArrayList<VersionedStreamDefinition>();
+
+ // ScheduleResult
+ private List<MonitoredStream> monitoredStreams = new ArrayList<MonitoredStream>();
+ private List<PolicyAssignment> assignments = new ArrayList<PolicyAssignment>();
+
+ private String version;
+ // FIXME : should be date, can not make it simple in mongo..
+ private String generateTime;
+ private int code = 200;
+ private String message = "OK";
+
+ public ScheduleState() {
+ }
+
+ public ScheduleState(String version,
+ Map<String, SpoutSpec> topoSpoutSpecsMap,
+ Map<String, RouterSpec> groupSpecsMap,
+ Map<String, AlertBoltSpec> alertSpecsMap,
+ Map<String, PublishSpec> pubMap,
+ Collection<PolicyAssignment> assignments,
+ Collection<MonitoredStream> monitoredStreams,
+ Collection<PolicyDefinition> definitions,
+ Collection<StreamDefinition> streams) {
+ this.spoutSpecs = topoSpoutSpecsMap;
+ this.groupSpecs = groupSpecsMap;
+ this.alertSpecs = alertSpecsMap;
+ this.publishSpecs = pubMap;
+ this.version = version;
+ this.generateTime = String.valueOf(new Date().getTime());
+ this.assignments = new ArrayList<PolicyAssignment>(assignments);
+ this.monitoredStreams = new ArrayList<MonitoredStream>(monitoredStreams);
+ this.policySnapshots = new ArrayList<VersionedPolicyDefinition>();
+ this.streamSnapshots = new ArrayList<VersionedStreamDefinition>();
+
+ for (SpoutSpec ss : this.spoutSpecs.values()) {
+ ss.setVersion(version);
+ }
+
+ for (RouterSpec ss : this.groupSpecs.values()) {
+ ss.setVersion(version);
+ }
+
+ for (AlertBoltSpec ss : this.alertSpecs.values()) {
+ ss.setVersion(version);
+ }
+
+ for (PublishSpec ps : this.publishSpecs.values()) {
+ ps.setVersion(version);
+ }
+
+ for (MonitoredStream ms : this.monitoredStreams) {
+ ms.setVersion(version);
+ }
+ for (PolicyAssignment ps : this.assignments) {
+ ps.setVersion(version);
+ }
+ for (PolicyDefinition def : definitions) {
+ this.policySnapshots.add(new VersionedPolicyDefinition(version, def));
+ }
+ for (StreamDefinition sd :streams) {
+ this.streamSnapshots.add(new VersionedStreamDefinition(version, sd));
+ }
+ }
+
+ public Map<String, SpoutSpec> getSpoutSpecs() {
+ return spoutSpecs;
+ }
+
+ public void setSpoutSpecs(Map<String, SpoutSpec> spoutSpecs) {
+ this.spoutSpecs = spoutSpecs;
+ }
+
+ public Map<String, AlertBoltSpec> getAlertSpecs() {
+ return alertSpecs;
+ }
+
+ public void setAlertSpecs(Map<String, AlertBoltSpec> alertSpecs) {
+ this.alertSpecs = alertSpecs;
+ }
+
+ public Map<String, RouterSpec> getGroupSpecs() {
+ return groupSpecs;
+ }
+
+ public void setGroupSpecs(Map<String, RouterSpec> groupSpecs) {
+ this.groupSpecs = groupSpecs;
+ }
+
+ public Map<String, PublishSpec> getPublishSpecs() {
+ return publishSpecs;
+ }
+
+ public void setPublishSpecs(Map<String, PublishSpec> publishSpecs) {
+ this.publishSpecs = publishSpecs;
+ }
+
+ public String getVersion() {
+ return version;
+ }
+
+ public void setVersion(String version) {
+ this.version = version;
+ }
+
+ public int getCode() {
+ return code;
+ }
+
+ public void setCode(int code) {
+ this.code = code;
+ }
+
+ public String getMessage() {
+ return message;
+ }
+
+ public void setMessage(String message) {
+ this.message = message;
+ }
+
+ public String getGenerateTime() {
+ return generateTime;
+ }
+
+ public void setGenerateTime(String generateTime) {
+ this.generateTime = generateTime;
+ }
+
+ public List<MonitoredStream> getMonitoredStreams() {
+ return monitoredStreams;
+ }
+
+ public List<PolicyAssignment> getAssignments() {
+ return assignments;
+ }
+
+ public List<VersionedPolicyDefinition> getPolicySnapshots() {
+ return policySnapshots;
+ }
+
+ public void setPolicySnapshots(List<VersionedPolicyDefinition> policySnapshots) {
+ this.policySnapshots = policySnapshots;
+ }
+
+ public void setMonitoredStreams(List<MonitoredStream> monitoredStreams) {
+ this.monitoredStreams = monitoredStreams;
+ }
+
+ public void setAssignments(List<PolicyAssignment> assignments) {
+ this.assignments = assignments;
+ }
+
+ public List<VersionedStreamDefinition> getStreamSnapshots() {
+ return streamSnapshots;
+ }
+
+ public void setStreamSnapshots(List<VersionedStreamDefinition> streamSnapshots) {
+ this.streamSnapshots = streamSnapshots;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/SpoutSpec.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/SpoutSpec.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/SpoutSpec.java
new file mode 100644
index 0000000..a197858
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/SpoutSpec.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.coordination.model;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+
+/**
+ * SpoutSpec metadata control 3 phases for data transformation for one specific topic
+ * phase 1: kafka topic to tuple, controlled by Kafka2TupleMetadata, i.e. Scheme
+ * phase 2: tuple to stream, controlled by Tuple2StreamMetadata, i.e. stream name selector etc.
+ * phase 3: stream repartition, controlled by StreamRepartitionMetadata, i.e. groupby spec
+ * @since Apr 18, 2016
+ *
+ */
+public class SpoutSpec {
+ private String version;
+
+// private String spoutId;
+ private String topologyId;
+
+ // topicName -> kafka2TupleMetadata
+ private Map<String, Kafka2TupleMetadata> kafka2TupleMetadataMap = new HashMap<String, Kafka2TupleMetadata>();
+ // topicName -> Tuple2StreamMetadata
+ private Map<String, Tuple2StreamMetadata> tuple2StreamMetadataMap = new HashMap<String, Tuple2StreamMetadata>();
+ // topicName -> list of StreamRepartitionMetadata, here it is list because one topic(data source) may spawn multiple streams.
+ private Map<String, List<StreamRepartitionMetadata>> streamRepartitionMetadataMap = new HashMap<String, List<StreamRepartitionMetadata>>();
+
+ public SpoutSpec(){}
+
+ public SpoutSpec(
+ String topologyId,
+// String spoutId,
+ Map<String, List<StreamRepartitionMetadata>> streamRepartitionMetadataMap,
+ Map<String, Tuple2StreamMetadata> tuple2StreamMetadataMap,
+ Map<String, Kafka2TupleMetadata> kafka2TupleMetadataMap) {
+ this.topologyId = topologyId;
+// this.spoutId = spoutId;
+ this.streamRepartitionMetadataMap = streamRepartitionMetadataMap;
+ this.tuple2StreamMetadataMap = tuple2StreamMetadataMap;
+ this.kafka2TupleMetadataMap = kafka2TupleMetadataMap;
+ }
+
+// public String getSpoutId() {
+// return spoutId;
+// }
+// public void setSpoutId(String spoutId) {
+// this.spoutId = spoutId;
+// }
+
+ public String getTopologyId() {
+ return topologyId;
+ }
+
+ public Map<String, List<StreamRepartitionMetadata>> getStreamRepartitionMetadataMap() {
+ return streamRepartitionMetadataMap;
+ }
+
+ public Map<String, Tuple2StreamMetadata> getTuple2StreamMetadataMap(){
+ return this.tuple2StreamMetadataMap;
+ }
+
+ public Map<String, Kafka2TupleMetadata> getKafka2TupleMetadataMap() {
+ return kafka2TupleMetadataMap;
+ }
+
+ @org.codehaus.jackson.annotate.JsonIgnore
+ public StreamRepartitionMetadata getStream(String streamName) {
+ for (List<StreamRepartitionMetadata> meta : this.streamRepartitionMetadataMap.values()) {
+ Optional<StreamRepartitionMetadata> m = meta.stream().filter((t) -> t.getStreamId().equalsIgnoreCase(streamName)).findFirst();
+ if (m.isPresent()) {
+ return m.get();
+ }
+ }
+ return null;
+ }
+
+ public void setTopologyId(String topologyId) {
+ this.topologyId = topologyId;
+ }
+
+ public void setKafka2TupleMetadataMap(Map<String, Kafka2TupleMetadata> kafka2TupleMetadataMap) {
+ this.kafka2TupleMetadataMap = kafka2TupleMetadataMap;
+ }
+
+ public void setTuple2StreamMetadataMap(Map<String, Tuple2StreamMetadata> tuple2StreamMetadataMap) {
+ this.tuple2StreamMetadataMap = tuple2StreamMetadataMap;
+ }
+
+ public void setStreamRepartitionMetadataMap(Map<String, List<StreamRepartitionMetadata>> streamRepartitionMetadataMap) {
+ this.streamRepartitionMetadataMap = streamRepartitionMetadataMap;
+ }
+
+ public String getVersion() {
+ return version;
+ }
+
+ public void setVersion(String version) {
+ this.version = version;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("version:%s-topo:%s ", version, this.topologyId);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/StreamNameSelector.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/StreamNameSelector.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/StreamNameSelector.java
new file mode 100644
index 0000000..d421716
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/StreamNameSelector.java
@@ -0,0 +1,15 @@
+package org.apache.eagle.alert.coordination.model;
+
+import java.util.Map;
+
+/**
+ * This metadata controls how to figure out stream name from incoming tuple
+ */
+public interface StreamNameSelector {
+ /**
+ * field name to value mapping
+ * @param tuple
+ * @return
+ */
+ String getStreamName(Map<String, Object> tuple);
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/StreamRepartitionMetadata.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/StreamRepartitionMetadata.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/StreamRepartitionMetadata.java
new file mode 100644
index 0000000..5f8f689
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/StreamRepartitionMetadata.java
@@ -0,0 +1,53 @@
+package org.apache.eagle.alert.coordination.model;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.codehaus.jackson.annotate.JsonIgnore;
+
+/**
+ * @since Apr 25, 2016
+ * This meta-data controls how tuple streamId is repartitioned
+ */
+public class StreamRepartitionMetadata {
+ private String topicName;
+ private String streamId;
+ /**
+ * each stream may have multiple different grouping strategies,for example groupby some fields or even shuffling
+ */
+ public List<StreamRepartitionStrategy> groupingStrategies = new ArrayList<StreamRepartitionStrategy>();
+
+ public StreamRepartitionMetadata(){}
+
+ public StreamRepartitionMetadata(String topicName, String stream) {
+ this.topicName = topicName;
+ this.streamId = stream;
+ }
+
+ public String getStreamId() {
+ return streamId;
+ }
+ public void setStreamId(String streamId) {
+ this.streamId = streamId;
+ }
+
+ public String getTopicName() {
+ return topicName;
+ }
+ public void setTopicName(String topicName) {
+ this.topicName = topicName;
+ }
+
+ public List<StreamRepartitionStrategy> getGroupingStrategies() {
+ return groupingStrategies;
+ }
+
+ @JsonIgnore
+ public void addGroupStrategy(StreamRepartitionStrategy gs) {
+ this.groupingStrategies.add(gs);
+ }
+
+ public void setGroupingStrategies(List<StreamRepartitionStrategy> groupingStrategies) {
+ this.groupingStrategies = groupingStrategies;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/StreamRepartitionStrategy.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/StreamRepartitionStrategy.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/StreamRepartitionStrategy.java
new file mode 100644
index 0000000..203114e
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/coordination/model/StreamRepartitionStrategy.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.coordination.model;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.eagle.alert.engine.coordinator.StreamPartition;
+
+public class StreamRepartitionStrategy {
+ public StreamPartition partition ;
+
+ public int numTotalParticipatingRouterBolts = 0; // how many group-by bolts participate policy evaluation
+ public int startSequence = 0; // what is the sequence for the first bolt in this topology among all bolts
+ public List<String> totalTargetBoltIds = new ArrayList<String>();
+
+ public int hashCode() {
+ int hashcode = 1 * 31;
+ hashcode += partition.hashCode();
+ for (String str : totalTargetBoltIds) {
+ hashcode += str.hashCode();
+ }
+ return hashcode;
+ }
+
+ public boolean equals(Object obj) {
+ if (!(obj instanceof StreamRepartitionStrategy)) {
+ return false;
+ }
+ StreamRepartitionStrategy o = (StreamRepartitionStrategy) obj;
+ return partition.equals(o.partition)
+ && CollectionUtils.isEqualCollection(totalTargetBoltIds, o.totalTargetBoltIds);
+ }
+
+ public StreamPartition getPartition() {
+ return partition;
+ }
+
+ public void setPartition(StreamPartition partition) {
+ this.partition = partition;
+ }
+
+ public int getNumTotalParticipatingRouterBolts() {
+ return numTotalParticipatingRouterBolts;
+ }
+
+ public void setNumTotalParticipatingRouterBolts(int numTotalParticipatingRouterBolts) {
+ this.numTotalParticipatingRouterBolts = numTotalParticipatingRouterBolts;
+ }
+
+ public int getStartSequence() {
+ return startSequence;
+ }
+
+ public void setStartSequence(int startSequence) {
+ this.startSequence = startSequence;
+ }
+
+ public List<String> getTotalTargetBoltIds() {
+ return totalTargetBoltIds;
+ }
+
+ public void setTotalTargetBoltIds(List<String> totalTargetBoltIds) {
+ this.totalTargetBoltIds = totalTargetBoltIds;
+ }
+
+}
\ No newline at end of file
[10/19] incubator-eagle git commit: EAGLE-324: Init branch-v0.5
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/router/impl/StreamRouterImpl.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/router/impl/StreamRouterImpl.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/router/impl/StreamRouterImpl.java
new file mode 100644
index 0000000..5008dbf
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/router/impl/StreamRouterImpl.java
@@ -0,0 +1,163 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.router.impl;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.eagle.alert.engine.PartitionedEventCollector;
+import org.apache.eagle.alert.engine.StreamContext;
+import org.apache.eagle.alert.engine.coordinator.StreamPartition;
+import org.apache.eagle.alert.engine.coordinator.StreamSortSpec;
+import org.apache.eagle.alert.engine.model.PartitionedEvent;
+import org.apache.eagle.alert.engine.router.StreamRouter;
+import org.apache.eagle.alert.engine.router.StreamSortHandler;
+import org.apache.eagle.alert.engine.sorter.StreamTimeClockManager;
+import org.apache.eagle.alert.engine.sorter.impl.StreamSortWindowHandlerImpl;
+import org.apache.eagle.alert.engine.sorter.impl.StreamTimeClockManagerImpl;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class StreamRouterImpl implements StreamRouter {
+ private static final long serialVersionUID = -4640125063690900014L;
+ private final static Logger LOG = LoggerFactory.getLogger(StreamRouterImpl.class);
+ private final String name;
+ private volatile Map<StreamPartition,StreamSortHandler> streamSortHandlers;
+ private PartitionedEventCollector outputCollector;
+ private StreamTimeClockManager streamTimeClockManager;
+ private StreamContext context;
+
+ /**
+ * @param name This name should be formed by topologyId + router id, which is built by topology builder
+ */
+ public StreamRouterImpl(String name){
+ this.name = name;
+ }
+
+ public String getName(){
+ return this.name;
+ }
+
+ @Override
+ public void close() {
+ streamSortHandlers.values().forEach(StreamSortHandler::close);
+ streamTimeClockManager.close();
+ }
+
+ public void prepare(StreamContext context, PartitionedEventCollector outputCollector) {
+ this.streamTimeClockManager = new StreamTimeClockManagerImpl();
+ this.streamSortHandlers = new HashMap<>();
+ this.outputCollector = outputCollector;
+ this.context = context;
+ }
+
+ /**
+ * TODO: Potential improvement: if StreamSortHandler is expensive, we can use DISRUPTOR to buffer
+ *
+ * @param event StreamEvent
+ */
+ public void nextEvent(PartitionedEvent event) {
+ this.context.counter().scope("receive_count").incr();
+ if(!dispatchToSortHandler(event)) {
+ this.context.counter().scope("direct_count").incr();
+ // Pass through directly if no need to sort
+ outputCollector.emit(event);
+ }
+ this.context.counter().scope("sort_count").incr();
+ // Update stream clock time if moving forward and trigger all tick listeners
+ streamTimeClockManager.onTimeUpdate(event.getStreamId(),event.getTimestamp());
+ }
+
+ /**
+ * @param event input event
+ * @return whether sorted
+ */
+ private boolean dispatchToSortHandler(PartitionedEvent event){
+ if(event.getTimestamp() <= 0) return false;
+
+ StreamSortHandler sortHandler = streamSortHandlers.get(event.getPartition());
+ if(sortHandler == null){
+ if(event.isSortRequired()) {
+ LOG.warn("Stream sort handler required has not been loaded so emmit directly: {}", event);
+ this.context.counter().scope("miss_sort_count").incr();
+ }
+ return false;
+ } else {
+ sortHandler.nextEvent(event);
+ return true;
+ }
+ }
+
+ @Override
+ public void onStreamSortSpecChange(Map<StreamPartition, StreamSortSpec> added,
+ Map<StreamPartition, StreamSortSpec> removed,
+ Map<StreamPartition, StreamSortSpec> changed) {
+ synchronized (streamTimeClockManager) {
+ Map<StreamPartition, StreamSortHandler> copy = new HashMap<>(this.streamSortHandlers);
+ // add new StreamSortSpec
+ if (added != null && added.size() > 0) {
+ for (Entry<StreamPartition, StreamSortSpec> spec : added.entrySet()) {
+ StreamPartition tmp = spec.getKey();
+ if (copy.containsKey(tmp)) {
+ LOG.error("Metadata calculation error: Duplicated StreamSortSpec " + spec);
+ } else {
+ StreamSortHandler handler = new StreamSortWindowHandlerImpl();
+ handler.prepare(tmp.getStreamId(),spec.getValue(), this.outputCollector);
+ copy.put(tmp, handler);
+ streamTimeClockManager.registerListener(streamTimeClockManager.createStreamTimeClock(tmp.getStreamId()), handler);
+ }
+ }
+ }
+
+ // remove StreamSortSpec
+ if (removed != null && removed.size() > 0) {
+ for (Entry<StreamPartition, StreamSortSpec> spec : removed.entrySet()) {
+ StreamPartition tmp = spec.getKey();
+ if (copy.containsKey(tmp)) {
+ copy.get(tmp).close();
+ streamTimeClockManager.removeListener(copy.get(tmp));
+ copy.remove(tmp);
+ } else {
+ LOG.error("Metadata calculation error: remove nonexisting StreamSortSpec " + spec.getValue());
+ }
+ }
+ }
+
+ // modify StreamSortSpec
+ if (changed != null && changed.size() > 0) {
+ for (Entry<StreamPartition, StreamSortSpec> spec : changed.entrySet()) {
+ StreamPartition tmp = spec.getKey();
+ if (copy.containsKey(tmp)) {
+ copy.get(tmp).close();
+ streamTimeClockManager.removeListener(copy.get(tmp));
+ copy.remove(tmp);
+ StreamSortHandler handler = new StreamSortWindowHandlerImpl();
+ handler.prepare(tmp.getStreamId(), spec.getValue(), this.outputCollector);
+ copy.put(tmp, handler);
+ streamTimeClockManager.registerListener(tmp.getStreamId(), handler);
+ } else {
+ LOG.error("Metadata calculation error: modify non-existing StreamSortSpec " + spec.getValue());
+ }
+ }
+ }
+
+ // atomic switch
+ this.streamSortHandlers = copy;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/runner/AbstractStreamBolt.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/runner/AbstractStreamBolt.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/runner/AbstractStreamBolt.java
new file mode 100644
index 0000000..cc819ba
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/runner/AbstractStreamBolt.java
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.runner;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.eagle.alert.engine.coordinator.IMetadataChangeNotifyService;
+import org.apache.eagle.alert.utils.AlertConstants;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import backtype.storm.task.OutputCollector;
+import backtype.storm.task.TopologyContext;
+import backtype.storm.topology.OutputFieldsDeclarer;
+import backtype.storm.topology.base.BaseRichBolt;
+import backtype.storm.tuple.Fields;
+
+import com.google.common.base.Preconditions;
+import com.typesafe.config.Config;
+
+@SuppressWarnings({"rawtypes", "serial"})
+public abstract class AbstractStreamBolt extends BaseRichBolt {
+ private static final Logger LOG = LoggerFactory.getLogger(AbstractStreamBolt.class);
+ private IMetadataChangeNotifyService changeNotifyService;
+ private Config config;
+ private List<String> outputStreamIds;
+ protected OutputCollector collector;
+
+ public AbstractStreamBolt(IMetadataChangeNotifyService changeNotifyService, Config config){
+ this.changeNotifyService = changeNotifyService;
+ this.config = config;
+ }
+
+ public void declareOutputStreams(List<String> outputStreamIds){
+ this.outputStreamIds = outputStreamIds;
+ }
+
+ protected List<String> getOutputStreamIds(){
+ return this.outputStreamIds;
+ }
+
+ @Override
+ public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
+ Preconditions.checkNotNull(this.changeNotifyService, "IMetadataChangeNotifyService is not set yet");
+ this.collector = collector;
+ internalPrepare(collector,this.changeNotifyService,this.config,context);
+ }
+
+ /**
+ * subclass should implement more initialization for example
+ * 1) register metadata change
+ * @param collector
+ * @param metadataManager
+ * @param config
+ * @param context
+ */
+ public abstract void internalPrepare(
+ OutputCollector collector,
+ IMetadataChangeNotifyService metadataManager,
+ Config config, TopologyContext context);
+
+ @Override
+ public void cleanup() {
+ super.cleanup();
+ }
+
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+ if(this.outputStreamIds!=null){
+ LOG.info("declare streams: {} ", outputStreamIds);
+ for(String streamId:this.outputStreamIds){
+ declarer.declareStream(streamId,new Fields(AlertConstants.FIELD_0));
+ }
+ } else {
+ declarer.declare(new Fields(AlertConstants.FIELD_0));
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/runner/AlertBolt.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/runner/AlertBolt.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/runner/AlertBolt.java
new file mode 100755
index 0000000..30ff5f0
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/runner/AlertBolt.java
@@ -0,0 +1,157 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.runner;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.eagle.alert.coordination.model.AlertBoltSpec;
+import org.apache.eagle.alert.coordination.model.WorkSlot;
+import org.apache.eagle.alert.engine.AlertStreamCollector;
+import org.apache.eagle.alert.engine.StreamContext;
+import org.apache.eagle.alert.engine.StreamContextImpl;
+import org.apache.eagle.alert.engine.coordinator.IMetadataChangeNotifyService;
+import org.apache.eagle.alert.engine.coordinator.MetadataType;
+import org.apache.eagle.alert.engine.coordinator.PolicyDefinition;
+import org.apache.eagle.alert.engine.coordinator.StreamDefinition;
+import org.apache.eagle.alert.engine.evaluator.PolicyGroupEvaluator;
+import org.apache.eagle.alert.engine.evaluator.impl.AlertBoltOutputCollectorWrapper;
+import org.apache.eagle.alert.engine.model.PartitionedEvent;
+import org.apache.eagle.alert.engine.router.AlertBoltSpecListener;
+import org.apache.eagle.alert.engine.serialization.PartitionedEventSerializer;
+import org.apache.eagle.alert.engine.serialization.SerializationMetadataProvider;
+import org.apache.eagle.alert.engine.serialization.Serializers;
+import org.apache.eagle.alert.utils.AlertConstants;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import backtype.storm.metric.api.MultiCountMetric;
+import backtype.storm.task.OutputCollector;
+import backtype.storm.task.TopologyContext;
+import backtype.storm.topology.OutputFieldsDeclarer;
+import backtype.storm.tuple.Fields;
+import backtype.storm.tuple.Tuple;
+
+import com.typesafe.config.Config;
+
+/**
+ * Since 5/1/16.
+ * This is container for hosting all policies belonging to the same monitoredStream
+ * MonitoredStream refers to tuple of {dataSource, streamId, grouopby}
+ * The container is also called {@link WorkSlot}
+ */
+public class AlertBolt extends AbstractStreamBolt implements AlertBoltSpecListener,SerializationMetadataProvider {
+ private final static Logger LOG = LoggerFactory.getLogger(AlertBolt.class);
+ private static final long serialVersionUID = -4132297691448945672L;
+ private PolicyGroupEvaluator policyGroupEvaluator;
+ private AlertStreamCollector alertOutputCollector;
+ private String boltId;
+ private volatile Object outputLock;
+ // mapping from policy name to PolicyDefinition
+ private volatile Map<String, PolicyDefinition> cachedPolicies = new HashMap<>(); // for one streamGroup, there are multiple policies
+
+ private StreamContext streamContext;
+ private volatile Map<String, StreamDefinition> sdf;
+ private PartitionedEventSerializer serializer;
+
+ public AlertBolt(String boltId, PolicyGroupEvaluator policyGroupEvaluator, Config config, IMetadataChangeNotifyService changeNotifyService){
+ super(changeNotifyService, config);
+ this.boltId = boltId;
+ this.policyGroupEvaluator = policyGroupEvaluator;
+ }
+
+ PartitionedEvent deserialize(Object object) throws IOException {
+ // byte[] in higher priority
+ if(object instanceof byte[]) {
+ return serializer.deserialize((byte[]) object);
+ } else {
+ return (PartitionedEvent) object;
+ }
+ }
+
+ @Override
+ public void execute(Tuple input) {
+ this.streamContext.counter().scope("execute_count").incr();
+ try {
+ policyGroupEvaluator.nextEvent(deserialize(input.getValueByField(AlertConstants.FIELD_0)).withAnchor(input));
+ synchronized (outputLock) {
+ this.collector.ack(input);
+ }
+ this.streamContext.counter().scope("ack_count").incr();
+ }catch (Exception ex) {
+ LOG.error(ex.getMessage(),ex);
+ synchronized (outputLock) {
+ this.streamContext.counter().scope("fail_count").incr();
+ this.collector.fail(input);
+ }
+ } finally {
+ alertOutputCollector.flush();
+ }
+ }
+
+ @Override
+ public void internalPrepare(OutputCollector collector, IMetadataChangeNotifyService metadataChangeNotifyService, Config config, TopologyContext context) {
+ // instantiate output lock object
+ outputLock = new Object();
+ streamContext = new StreamContextImpl(config,context.registerMetric("eagle.evaluator",new MultiCountMetric(),60),context);
+ serializer = Serializers.newPartitionedEventSerializer(this);
+ alertOutputCollector = new AlertBoltOutputCollectorWrapper(collector, outputLock,streamContext);
+ policyGroupEvaluator.init(streamContext, alertOutputCollector);
+ metadataChangeNotifyService.registerListener(this);
+ metadataChangeNotifyService.init(config, MetadataType.ALERT_BOLT);
+ }
+
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+ declarer.declare(new Fields(AlertConstants.FIELD_0, AlertConstants.FIELD_1));
+ }
+
+ @Override
+ public void cleanup() {
+ policyGroupEvaluator.close();
+ alertOutputCollector.flush();
+ alertOutputCollector.close();
+ super.cleanup();
+ }
+
+ @Override
+ public void onAlertBoltSpecChange(AlertBoltSpec spec, Map<String, StreamDefinition> sds) {
+ List<PolicyDefinition> newPolicies = spec.getBoltPoliciesMap().get(boltId);
+ if(newPolicies == null) {
+ LOG.info("no policy with AlertBoltSpec {} for this bolt {}", spec, boltId);
+ return;
+ }
+
+ Map<String, PolicyDefinition> newPoliciesMap = new HashMap<>();
+ newPolicies.forEach(p -> newPoliciesMap.put(p.getName(), p));
+ MapComparator<String, PolicyDefinition> comparator = new MapComparator<>(newPoliciesMap, cachedPolicies);
+ comparator.compare();
+
+ policyGroupEvaluator.onPolicyChange(comparator.getAdded(), comparator.getRemoved(), comparator.getModified(), sds);
+
+ // switch
+ cachedPolicies = newPoliciesMap;
+ sdf = sds;
+ }
+
+ @Override
+ public StreamDefinition getStreamDefinition(String streamId) {
+ return sdf.get(streamId);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/runner/AlertPublisherBolt.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/runner/AlertPublisherBolt.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/runner/AlertPublisherBolt.java
new file mode 100644
index 0000000..0a239e2
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/runner/AlertPublisherBolt.java
@@ -0,0 +1,113 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.runner;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.eagle.alert.coordination.model.PublishSpec;
+import org.apache.eagle.alert.engine.StreamContextImpl;
+import org.apache.eagle.alert.engine.coordinator.IMetadataChangeNotifyService;
+import org.apache.eagle.alert.engine.coordinator.MetadataType;
+import org.apache.eagle.alert.engine.coordinator.Publishment;
+import org.apache.eagle.alert.engine.coordinator.StreamDefinition;
+import org.apache.eagle.alert.engine.model.AlertStreamEvent;
+import org.apache.eagle.alert.engine.publisher.AlertPublishSpecListener;
+import org.apache.eagle.alert.engine.publisher.AlertPublisher;
+import org.apache.eagle.alert.utils.AlertConstants;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import backtype.storm.metric.api.MultiCountMetric;
+import backtype.storm.task.OutputCollector;
+import backtype.storm.task.TopologyContext;
+import backtype.storm.topology.OutputFieldsDeclarer;
+import backtype.storm.tuple.Fields;
+import backtype.storm.tuple.Tuple;
+
+import com.typesafe.config.Config;
+
+@SuppressWarnings("serial")
+public class AlertPublisherBolt extends AbstractStreamBolt implements AlertPublishSpecListener {
+ private final static Logger LOG = LoggerFactory.getLogger(AlertPublisherBolt.class);
+ private final AlertPublisher alertPublisher;
+ private volatile Map<String, Publishment> cachedPublishments = new HashMap<>();
+ private StreamContextImpl streamContext;
+
+ public AlertPublisherBolt(AlertPublisher alertPublisher, Config config, IMetadataChangeNotifyService coordinatorService){
+ super(coordinatorService, config);
+ this.alertPublisher = alertPublisher;
+ this.alertPublisher.init(config);
+ }
+
+ @Override
+ public void internalPrepare(OutputCollector collector, IMetadataChangeNotifyService coordinatorService, Config config, TopologyContext context) {
+ coordinatorService.registerListener(this);
+ coordinatorService.init(config, MetadataType.ALERT_PUBLISH_BOLT);
+ streamContext = new StreamContextImpl(config,context.registerMetric("eagle.publisher",new MultiCountMetric(),60),context);
+ }
+
+ @Override
+ public void execute(Tuple input) {
+ try {
+ streamContext.counter().scope("receive_count");
+ alertPublisher.nextEvent((AlertStreamEvent) input.getValueByField(AlertConstants.FIELD_1));
+ this.collector.ack(input);
+ streamContext.counter().scope("ack_count");
+ } catch (Exception ex){
+ streamContext.counter().scope("fail_count");
+ LOG.error(ex.getMessage(),ex);
+ collector.reportError(ex);
+ }
+ }
+
+ @Override
+ public void cleanup() {
+ alertPublisher.close();
+ super.cleanup();
+ }
+
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+ declarer.declare(new Fields());
+ }
+
+ @Override
+ public void onAlertPublishSpecChange(PublishSpec pubSpec, Map<String, StreamDefinition> sds) {
+ if(pubSpec == null) return;
+
+ List<Publishment> newPublishments = pubSpec.getPublishments();
+ if(newPublishments == null) {
+ LOG.info("no publishments with PublishSpec {} for this topology", pubSpec);
+ return;
+ }
+
+ Map<String, Publishment> newPublishmentsMap = new HashMap<>();
+ newPublishments.forEach(p -> newPublishmentsMap.put(p.getName(), p));
+ MapComparator<String, Publishment> comparator = new MapComparator<>(newPublishmentsMap, cachedPublishments);
+ comparator.compare();
+
+ List<Publishment> beforeModified = new ArrayList<>();
+ comparator.getModified().forEach(p -> beforeModified.add(cachedPublishments.get(p.getName())));
+ alertPublisher.onPublishChange(comparator.getAdded(), comparator.getRemoved(), comparator.getModified(), beforeModified);
+
+ // switch
+ cachedPublishments = newPublishmentsMap;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/runner/MapComparator.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/runner/MapComparator.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/runner/MapComparator.java
new file mode 100644
index 0000000..04595b1
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/runner/MapComparator.java
@@ -0,0 +1,72 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.alert.engine.runner;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.collections.CollectionUtils;
+
+/**
+ * Since 5/2/16.
+ */
+public class MapComparator<K, V> {
+ private Map<K, V> map1;
+ private Map<K, V> map2;
+ private List<V> added = new ArrayList<>();
+ private List<V> removed = new ArrayList<>();
+ private List<V> modified = new ArrayList<>();
+ public MapComparator(Map<K, V> map1, Map<K, V> map2){
+ this.map1 = map1;
+ this.map2 = map2;
+ }
+
+ @SuppressWarnings("unchecked")
+ public void compare(){
+ Set<K> keys1 = map1.keySet();
+ Set<K> keys2 = map2.keySet();
+ Collection<K> addedKeys = CollectionUtils.subtract(keys1, keys2);
+ Collection<K> removedKeys = CollectionUtils.subtract(keys2, keys1);
+ Collection<K> modifiedKeys = CollectionUtils.intersection(keys1, keys2);
+
+ addedKeys.forEach(k -> added.add(map1.get(k)));
+ removedKeys.forEach(k -> removed.add(map2.get(k)));
+ modifiedKeys.forEach(k -> {
+ if(!map1.get(k).equals(map2.get(k))){
+ modified.add(map1.get(k));
+ }
+ });
+ }
+
+ public List<V> getAdded(){
+ return added;
+ }
+
+ public List<V> getRemoved(){
+ return removed;
+ }
+
+ public List<V> getModified(){
+ return modified;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/runner/StormMetricConsumer.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/runner/StormMetricConsumer.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/runner/StormMetricConsumer.java
new file mode 100644
index 0000000..1a6267b
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/runner/StormMetricConsumer.java
@@ -0,0 +1,136 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.runner;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.eagle.alert.metric.IMetricSystem;
+import org.apache.eagle.alert.metric.MetricSystem;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import backtype.storm.metric.api.IMetricsConsumer;
+import backtype.storm.task.IErrorReporter;
+import backtype.storm.task.TopologyContext;
+
+import com.codahale.metrics.Gauge;
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+import com.typesafe.config.ConfigParseOptions;
+
+/**
+ * Share same metric system
+ */
+public class StormMetricConsumer implements IMetricsConsumer {
+ public static final Logger LOG = LoggerFactory.getLogger(StormMetricConsumer.class);
+ private String topologyName;
+ private IMetricSystem metricSystem;
+ private String topologyId;
+
+ @SuppressWarnings({ "serial", "rawtypes" })
+ @Override
+ public void prepare(Map stormConf, Object registrationArgument, TopologyContext context, IErrorReporter errorReporter) {
+ Config config = ConfigFactory.parseString((String) registrationArgument, ConfigParseOptions.defaults());
+ topologyName = config.getString("topology.name");
+ topologyId = context.getStormId();
+ metricSystem = MetricSystem.load(config);
+ metricSystem.tags(new HashMap<String,Object>(){{
+ put("topologyName",topologyName);
+ put("topologyId",topologyId);
+ }});
+ metricSystem.start();
+ }
+
+ @SuppressWarnings({ "unchecked", "rawtypes" })
+ @Override
+ public void handleDataPoints(TaskInfo taskInfo, Collection<DataPoint> dataPoints) {
+ synchronized (metricSystem) {
+ List<String> metricList = new LinkedList<>();
+ for(DataPoint dataPoint:dataPoints){
+ if(dataPoint.value instanceof Map) {
+ Map<String,Object> values = (Map<String, Object>) dataPoint.value;
+ for(Map.Entry<String,Object> entry:values.entrySet()){
+ String metricName = buildMetricName(taskInfo, dataPoint.name, entry.getKey());
+ metricList.add(metricName);
+ Gauge gauge = metricSystem.registry().getGauges().get(metricName);
+ if(gauge == null) {
+ LOG.info("Register metric {}", metricName);
+ gauge = new DataPointGauge(entry.getValue());
+ metricSystem.registry().register(metricName,gauge);
+ }else{
+ ((DataPointGauge) gauge).setValue(entry.getValue());
+ }
+ }
+ } else {
+ String metricName = buildMetricName(taskInfo, dataPoint.name);
+ metricList.add(metricName);
+ LOG.info("Register metric {}",metricName);
+ Gauge gauge = metricSystem.registry().getGauges().get(metricName);
+ if(gauge == null) {
+ LOG.info("Register metric {}", metricName);
+ gauge = new DataPointGauge(dataPoint.value);
+ metricSystem.registry().register(metricName,gauge);
+ } else {
+ ((DataPointGauge) gauge).setValue(dataPoint.value);
+ }
+ }
+ }
+ metricSystem.registry().removeMatching((name, metric) -> metricList.indexOf(name) < 0);
+ metricSystem.report();
+ metricSystem.registry().getGauges().values().forEach((gauge -> {
+ if(gauge instanceof DataPointGauge){
+ ((DataPointGauge)gauge).reset();
+ }
+ }));
+ LOG.info("Reported {} metric data points from {} [{}]",dataPoints.size(),taskInfo.srcComponentId,taskInfo.srcTaskId);
+ }
+ }
+
+ private class DataPointGauge implements Gauge<Object> {
+ private Object value;
+ public DataPointGauge(Object initialValue){
+ this.value = initialValue;
+ }
+
+ @Override
+ public Object getValue() {
+ return value;
+ }
+
+ public void setValue(Object value){
+ this.value = value;
+ }
+
+ public void reset(){
+ this.value = 0;
+ }
+ }
+
+ private String buildMetricName(TaskInfo taskInfo,String ... name ){
+ return String.join(".",StringUtils.join(name,".").replace("/","."),taskInfo.srcComponentId,taskInfo.srcTaskId+"");
+ }
+
+ @Override
+ public void cleanup() {
+ metricSystem.stop();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/runner/StormMetricTaggedConsumer.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/runner/StormMetricTaggedConsumer.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/runner/StormMetricTaggedConsumer.java
new file mode 100644
index 0000000..c18a44f
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/runner/StormMetricTaggedConsumer.java
@@ -0,0 +1,156 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.runner;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.eagle.alert.metric.IMetricSystem;
+import org.apache.eagle.alert.metric.MetricSystem;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import backtype.storm.metric.api.IMetricsConsumer;
+import backtype.storm.task.IErrorReporter;
+import backtype.storm.task.TopologyContext;
+
+import com.codahale.metrics.Gauge;
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+import com.typesafe.config.ConfigParseOptions;
+
+/**
+ * Per MetricSystem instance per task
+ */
+public class StormMetricTaggedConsumer implements IMetricsConsumer {
+ public static final Logger LOG = LoggerFactory.getLogger(StormMetricTaggedConsumer.class);
+ private String topologyName;
+ private Map<String,MetricSystem> metricSystems;
+ private String stormId;
+ private Config config;
+
+ @SuppressWarnings("rawtypes")
+ @Override
+ public void prepare(Map stormConf, Object registrationArgument, TopologyContext context, IErrorReporter errorReporter) {
+ this.config = ConfigFactory.parseString((String) registrationArgument, ConfigParseOptions.defaults());
+ topologyName = config.getString("topology.name");
+ stormId = context.getStormId();
+ metricSystems = new HashMap<>();
+ }
+
+ @SuppressWarnings("serial")
+ @Override
+ public void handleDataPoints(TaskInfo taskInfo, Collection<DataPoint> dataPoints) {
+ synchronized (metricSystems) {
+ String uniqueTaskKey = buildUniqueTaskKey(taskInfo);
+ MetricSystem metricSystem = metricSystems.get(uniqueTaskKey);
+ if(metricSystem == null){
+ metricSystem = MetricSystem.load(config);
+ metricSystems.put(uniqueTaskKey,metricSystem);
+ metricSystem.tags(new HashMap<String,Object>(){{
+ put("topology",topologyName);
+ put("stormId",stormId);
+ put("component",taskInfo.srcComponentId);
+ put("task",taskInfo.srcTaskId);
+ }});
+ metricSystem.start();
+ LOG.info("Initialized metric reporter for {}",uniqueTaskKey);
+ }
+ report(metricSystem,taskInfo,dataPoints);
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("Reported {} metric points from {}", dataPoints.size(), uniqueTaskKey);
+ }
+ }
+ }
+
+ @SuppressWarnings({ "unchecked", "rawtypes" })
+ private void report(MetricSystem metricSystem,TaskInfo taskInfo,Collection<DataPoint> dataPoints){
+ List<String> metricList = new LinkedList<>();
+ for (DataPoint dataPoint : dataPoints) {
+ if (dataPoint.value instanceof Map) {
+ Map<String, Object> values = (Map<String, Object>) dataPoint.value;
+ for (Map.Entry<String, Object> entry : values.entrySet()) {
+ String metricName = buildSimpleMetricName(taskInfo, dataPoint.name, entry.getKey());
+ metricList.add(metricName);
+ Gauge gauge = metricSystem.registry().getGauges().get(metricName);
+ if (gauge == null) {
+ gauge = new DataPointGauge(entry.getValue());
+ metricSystem.registry().register(metricName, gauge);
+ LOG.debug("Register metric {}", metricName);
+ } else {
+ ((DataPointGauge) gauge).setValue(entry.getValue());
+ }
+ }
+ } else {
+ String metricName = buildSimpleMetricName(taskInfo, dataPoint.name);
+ metricList.add(metricName);
+ Gauge gauge = metricSystem.registry().getGauges().get(metricName);
+ if (gauge == null) {
+ gauge = new DataPointGauge(dataPoint.value);
+ metricSystem.registry().register(metricName, gauge);
+ LOG.debug("Register metric {}", metricName);
+ } else {
+ ((DataPointGauge) gauge).setValue(dataPoint.value);
+ }
+ }
+ }
+ metricSystem.registry().removeMatching((name, metric) -> metricList.indexOf(name) < 0);
+ metricSystem.report();
+ metricSystem.registry().getGauges().values().forEach((gauge -> {
+ if(gauge instanceof DataPointGauge){
+ ((DataPointGauge)gauge).reset();
+ }
+ }));
+ }
+
+ private static class DataPointGauge implements Gauge<Object> {
+ private Object value;
+ public DataPointGauge(Object initialValue){
+ this.setValue(initialValue);
+ }
+
+ @Override
+ public Object getValue() {
+ return value;
+ }
+
+ public void setValue(Object value){
+ this.value = value;
+ }
+
+ public void reset(){
+ this.value = 0;
+ }
+ }
+
+ private static String buildUniqueTaskKey(TaskInfo taskInfo){
+ return String.format("%s[%s]",taskInfo.srcComponentId,taskInfo.srcTaskId);
+ }
+
+ private static String buildSimpleMetricName(TaskInfo taskInfo,String ... name ){
+ return String.join(".",StringUtils.join(name,".").replace("/","."));
+ }
+
+ @Override
+ public void cleanup() {
+ metricSystems.values().forEach(IMetricSystem::stop);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/runner/StreamRouterBolt.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/runner/StreamRouterBolt.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/runner/StreamRouterBolt.java
new file mode 100644
index 0000000..942ef97
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/runner/StreamRouterBolt.java
@@ -0,0 +1,201 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.runner;
+
+import backtype.storm.metric.api.MultiCountMetric;
+import backtype.storm.task.OutputCollector;
+import backtype.storm.task.TopologyContext;
+import backtype.storm.tuple.Tuple;
+import com.typesafe.config.Config;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.eagle.alert.coordination.model.PolicyWorkerQueue;
+import org.apache.eagle.alert.coordination.model.RouterSpec;
+import org.apache.eagle.alert.coordination.model.StreamRouterSpec;
+import org.apache.eagle.alert.engine.StreamContext;
+import org.apache.eagle.alert.engine.StreamContextImpl;
+import org.apache.eagle.alert.engine.coordinator.*;
+import org.apache.eagle.alert.engine.model.PartitionedEvent;
+import org.apache.eagle.alert.engine.router.StreamRouter;
+import org.apache.eagle.alert.engine.router.StreamRouterBoltSpecListener;
+import org.apache.eagle.alert.engine.router.impl.StreamRouterBoltOutputCollector;
+import org.apache.eagle.alert.engine.serialization.PartitionedEventSerializer;
+import org.apache.eagle.alert.engine.serialization.SerializationMetadataProvider;
+import org.apache.eagle.alert.engine.serialization.Serializers;
+import org.apache.eagle.alert.utils.AlertConstants;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.*;
+
+public class StreamRouterBolt extends AbstractStreamBolt implements StreamRouterBoltSpecListener, SerializationMetadataProvider{
+ private final static Logger LOG = LoggerFactory.getLogger(StreamRouterBolt.class);
+ private static final long serialVersionUID = -7611470889316430372L;
+ private StreamRouter router;
+ private StreamRouterBoltOutputCollector routeCollector;
+ // mapping from StreamPartition to StreamSortSpec
+ private volatile Map<StreamPartition, StreamSortSpec> cachedSSS = new HashMap<>();
+ // mapping from StreamPartition(streamId, groupbyspec) to StreamRouterSpec
+ private volatile Map<StreamPartition, StreamRouterSpec> cachedSRS = new HashMap<>();
+ private volatile Map<String,StreamDefinition> sdf = new HashMap<>();
+ private PartitionedEventSerializer serializer;
+
+ public StreamRouterBolt(StreamRouter router, Config config, IMetadataChangeNotifyService changeNotifyService) {
+ super(changeNotifyService, config);
+ this.router = router;
+ }
+
+ private StreamContext streamContext;
+
+ @Override
+ public void internalPrepare(OutputCollector collector, IMetadataChangeNotifyService changeNotifyService, Config config, TopologyContext context) {
+ streamContext = new StreamContextImpl(config,context.registerMetric("eagle.router",new MultiCountMetric(),60),context);
+ serializer= Serializers.newPartitionedEventSerializer(this);
+ routeCollector = new StreamRouterBoltOutputCollector(this.router.getName(),collector,this.getOutputStreamIds(),streamContext,serializer);
+ router.prepare(streamContext, routeCollector);
+ changeNotifyService.registerListener(this);
+ changeNotifyService.init(config, MetadataType.STREAM_ROUTER_BOLT);
+ }
+
+ PartitionedEvent deserialize(Object object) throws IOException {
+ // byte[] in higher priority
+ if(object instanceof byte[]) {
+ return serializer.deserialize((byte[]) object);
+ } else {
+ return (PartitionedEvent) object;
+ }
+ }
+
+ @Override
+ public void execute(Tuple input) {
+ try {
+ this.streamContext.counter().scope("execute_count").incr();
+ this.router.nextEvent(deserialize(input.getValueByField(AlertConstants.FIELD_0)).withAnchor(input));
+ } catch (Exception ex) {
+ this.streamContext.counter().scope("fail_count").incr();
+ LOG.error(ex.getMessage(),ex);
+ this.collector.fail(input);
+ }
+ }
+
+ @Override
+ public void cleanup() {
+ this.router.close();
+ super.cleanup();
+ }
+
+ /**
+ * Compare with metadata snapshot cache to generate diff like added, removed and modified between different versions.
+ * @param spec
+ */
+ @SuppressWarnings("unchecked")
+ @Override
+ public void onStreamRouteBoltSpecChange(RouterSpec spec, Map<String, StreamDefinition> sds) {
+ sanityCheck(spec);
+
+ // figure out added, removed, modified StreamSortSpec
+ Map<StreamPartition, StreamSortSpec> newSSS = new HashMap<>();
+ spec.getRouterSpecs().forEach(t -> {
+ if (t.getPartition().getSortSpec() != null) {
+ newSSS.put(t.getPartition(), t.getPartition().getSortSpec());
+ }
+ });
+
+ Set<StreamPartition> newStreamIds = newSSS.keySet();
+ Set<StreamPartition> cachedStreamIds = cachedSSS.keySet();
+ Collection<StreamPartition> addedStreamIds = CollectionUtils.subtract(newStreamIds, cachedStreamIds);
+ Collection<StreamPartition> removedStreamIds = CollectionUtils.subtract(cachedStreamIds, newStreamIds);
+ Collection<StreamPartition> modifiedStreamIds = CollectionUtils.intersection(newStreamIds, cachedStreamIds);
+
+ Map<StreamPartition, StreamSortSpec> added = new HashMap<>();
+ Map<StreamPartition, StreamSortSpec> removed = new HashMap<>();
+ Map<StreamPartition, StreamSortSpec> modified = new HashMap<>();
+ addedStreamIds.forEach(s -> added.put(s, newSSS.get(s)));
+ removedStreamIds.forEach(s -> removed.put(s, cachedSSS.get(s)));
+ modifiedStreamIds.forEach(s -> {
+ if(!newSSS.get(s).equals(cachedSSS.get(s))){ // this means StreamSortSpec is changed for one specific streamId
+ modified.put(s, newSSS.get(s));
+ }
+ });
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("added StreamSortSpec " + added);
+ LOG.debug("removed StreamSortSpec " + removed);
+ LOG.debug("modified StreamSortSpec " + modified);
+ }
+ router.onStreamSortSpecChange(added, removed, modified);
+ // switch cache
+ cachedSSS = newSSS;
+
+ // figure out added, removed, modified StreamRouterSpec
+ Map<StreamPartition, StreamRouterSpec> newSRS = new HashMap<>();
+ spec.getRouterSpecs().forEach(t -> newSRS.put(t.getPartition(), t));
+
+ Set<StreamPartition> newStreamPartitions = newSRS.keySet();
+ Set<StreamPartition> cachedStreamPartitions = cachedSRS.keySet();
+
+ Collection<StreamPartition> addedStreamPartitions = CollectionUtils.subtract(newStreamPartitions, cachedStreamPartitions);
+ Collection<StreamPartition> removedStreamPartitions = CollectionUtils.subtract(cachedStreamPartitions, newStreamPartitions);
+ Collection<StreamPartition> modifiedStreamPartitions = CollectionUtils.intersection(newStreamPartitions, cachedStreamPartitions);
+
+ Collection<StreamRouterSpec> addedRouterSpecs = new ArrayList<>();
+ Collection<StreamRouterSpec> removedRouterSpecs = new ArrayList<>();
+ Collection<StreamRouterSpec> modifiedRouterSpecs = new ArrayList<>();
+ addedStreamPartitions.forEach(s -> addedRouterSpecs.add(newSRS.get(s)));
+ removedStreamPartitions.forEach(s -> removedRouterSpecs.add(cachedSRS.get(s)));
+ modifiedStreamPartitions.forEach(s -> {
+ if(!newSRS.get(s).equals(cachedSRS.get(s))){ // this means StreamRouterSpec is changed for one specific StreamPartition
+ modifiedRouterSpecs.add(newSRS.get(s));
+ }
+ });
+
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("added StreamRouterSpec " + addedRouterSpecs);
+ LOG.debug("removed StreamRouterSpec " + removedRouterSpecs);
+ LOG.debug("modified StreamRouterSpec " + modifiedRouterSpecs);
+ }
+
+ routeCollector.onStreamRouterSpecChange(addedRouterSpecs, removedRouterSpecs, modifiedRouterSpecs, sds);
+ // switch cache
+ cachedSRS = newSRS;
+ sdf = sds;
+ }
+
+ /**
+ * in correlation cases, multiple streams will go to the same queue for correlation policy
+ * @param spec
+ */
+ private void sanityCheck(RouterSpec spec){
+ Set<String> totalRequestedSlots = new HashSet<>();
+ for(StreamRouterSpec s : spec.getRouterSpecs()){
+ for(PolicyWorkerQueue q : s.getTargetQueue()){
+ List<String> workers = new ArrayList<>();
+ q.getWorkers().forEach(w -> workers.add(w.getBoltId()));
+ totalRequestedSlots.addAll(workers);
+ }
+ }
+ if(totalRequestedSlots.size() > getOutputStreamIds().size()){
+ String error = String.format("Requested slots are not consistent with provided slots, %s, %s", totalRequestedSlots, getOutputStreamIds());
+ LOG.error(error);
+ throw new IllegalStateException(error);
+ }
+ }
+
+ @Override
+ public StreamDefinition getStreamDefinition(String streamId) {
+ return this.sdf.get(streamId);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/runner/UnitTopologyRunner.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/runner/UnitTopologyRunner.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/runner/UnitTopologyRunner.java
new file mode 100755
index 0000000..cef94c7
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/runner/UnitTopologyRunner.java
@@ -0,0 +1,215 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.alert.engine.runner;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.eagle.alert.engine.coordinator.IMetadataChangeNotifyService;
+import org.apache.eagle.alert.engine.evaluator.impl.PolicyGroupEvaluatorImpl;
+import org.apache.eagle.alert.engine.publisher.impl.AlertPublisherImpl;
+import org.apache.eagle.alert.engine.router.impl.StreamRouterImpl;
+import org.apache.eagle.alert.engine.spout.CorrelationSpout;
+import org.apache.eagle.alert.utils.AlertConstants;
+import org.apache.eagle.alert.utils.StreamIdConversion;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import backtype.storm.LocalCluster;
+import backtype.storm.StormSubmitter;
+import backtype.storm.generated.StormTopology;
+import backtype.storm.topology.BoltDeclarer;
+import backtype.storm.topology.TopologyBuilder;
+import backtype.storm.tuple.Fields;
+import backtype.storm.utils.Utils;
+
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigRenderOptions;
+
+/**
+ * By default
+ * 1. one spout with multiple tasks
+ * 2. multiple router bolts with each bolt having exactly one task
+ * 3. multiple alert bolts with each bolt having exactly one task
+ * 4. one publish bolt with multiple tasks
+ */
+public class UnitTopologyRunner {
+ private static final Logger LOG = LoggerFactory.getLogger(UnitTopologyRunner.class);
+ public final static String spoutName = "alertEngineSpout";
+ private final static String streamRouterBoltNamePrefix = "streamRouterBolt";
+ private final static String alertBoltNamePrefix = "alertBolt";
+ public final static String alertPublishBoltName = "alertPublishBolt";
+
+ public final static String TOTAL_WORKER_NUM = "topology.numOfTotalWorkers";
+ public final static String SPOUT_TASK_NUM = "topology.numOfSpoutTasks";
+ public final static String ROUTER_TASK_NUM = "topology.numOfRouterBolts";
+ public final static String ALERT_TASK_NUM = "topology.numOfAlertBolts";
+ public final static String PUBLISH_TASK_NUM = "topology.numOfPublishTasks";
+ public final static String LOCAL_MODE = "topology.localMode";
+ public final static String MESSAGE_TIMEOUT_SECS = "topology.messageTimeoutSecs";
+ public final static int DEFAULT_MESSAGE_TIMEOUT_SECS = 3600;
+
+ private final IMetadataChangeNotifyService metadataChangeNotifyService;
+
+ public UnitTopologyRunner(IMetadataChangeNotifyService metadataChangeNotifyService){
+ this.metadataChangeNotifyService = metadataChangeNotifyService;
+ }
+
+ public StormTopology buildTopology(String topologyId,
+ int numOfSpoutTasks,
+ int numOfRouterBolts,
+ int numOfAlertBolts,
+ int numOfPublishTasks,
+ Config config) {
+
+ StreamRouterImpl[] routers = new StreamRouterImpl[numOfRouterBolts];
+ StreamRouterBolt[] routerBolts = new StreamRouterBolt[numOfRouterBolts];
+ PolicyGroupEvaluatorImpl[] evaluators = new PolicyGroupEvaluatorImpl[numOfAlertBolts];
+ AlertBolt[] alertBolts = new AlertBolt[numOfAlertBolts];
+ AlertPublisherImpl publisher;
+ AlertPublisherBolt publisherBolt;
+
+ TopologyBuilder builder = new TopologyBuilder();
+
+
+ // construct Spout object
+ CorrelationSpout spout = new CorrelationSpout(config, topologyId, getMetadataChangeNotifyService(), numOfRouterBolts, spoutName, streamRouterBoltNamePrefix);
+ builder.setSpout(spoutName, spout, numOfSpoutTasks).setNumTasks(numOfSpoutTasks);
+
+ // construct StreamRouterBolt objects
+ for(int i=0; i<numOfRouterBolts; i++){
+ routers[i] = new StreamRouterImpl(streamRouterBoltNamePrefix + i);
+ routerBolts[i] = new StreamRouterBolt(routers[i], config, getMetadataChangeNotifyService());
+ }
+
+ // construct AlertBolt objects
+ for(int i=0; i<numOfAlertBolts; i++){
+ evaluators[i] = new PolicyGroupEvaluatorImpl(alertBoltNamePrefix + i);
+ alertBolts[i] = new AlertBolt(alertBoltNamePrefix+i, evaluators[i], config, getMetadataChangeNotifyService());
+ }
+
+ // construct AlertPublishBolt object
+ publisher = new AlertPublisherImpl(alertPublishBoltName);
+ publisherBolt = new AlertPublisherBolt(publisher, config, getMetadataChangeNotifyService());
+
+ // connect spout and router bolt, also define output streams for downstreaming alert bolt
+ for(int i=0; i<numOfRouterBolts; i++){
+ String boltName = streamRouterBoltNamePrefix + i;
+
+ // define output streams, which are based on
+ String streamId = StreamIdConversion.generateStreamIdBetween(spoutName, boltName);
+ List<String> outputStreamIds = new ArrayList<>(numOfAlertBolts);
+ for(int j=0; j<numOfAlertBolts; j++){
+ String sid = StreamIdConversion.generateStreamIdBetween(boltName, alertBoltNamePrefix+j);
+ outputStreamIds.add(sid);
+ }
+ routerBolts[i].declareOutputStreams(outputStreamIds);
+
+ /**
+ * TODO potentially one route bolt may have multiple tasks, so that is field grouping by groupby fields
+ * that means we need a separate field to become groupby field
+ */
+ builder.setBolt(boltName, routerBolts[i]).fieldsGrouping(spoutName, streamId, new Fields());
+ }
+
+ // connect router bolt and alert bolt, also define output streams for downstreaming alert publish bolt
+ for(int i=0; i<numOfAlertBolts; i++){
+ String boltName = alertBoltNamePrefix + i;
+ BoltDeclarer boltDeclarer = builder.setBolt(boltName, alertBolts[i]);
+ for(int j=0; j<numOfRouterBolts; j++) {
+ String streamId = StreamIdConversion.generateStreamIdBetween(streamRouterBoltNamePrefix+j, boltName);
+ boltDeclarer.fieldsGrouping(streamRouterBoltNamePrefix+j, streamId, new Fields());
+ }
+ }
+
+ // connect alert bolt and alert publish bolt, this is the last bolt in the pipeline
+ BoltDeclarer boltDeclarer = builder.setBolt(alertPublishBoltName, publisherBolt).setNumTasks(numOfPublishTasks);
+ for(int i=0; i<numOfAlertBolts; i++) {
+ boltDeclarer.fieldsGrouping(alertBoltNamePrefix+i, new Fields(AlertConstants.FIELD_0));
+ }
+
+ return builder.createTopology();
+ }
+
+ public void run(String topologyId,
+ int numOfTotalWorkers,
+ int numOfSpoutTasks,
+ int numOfRouterBolts,
+ int numOfAlertBolts,
+ int numOfPublishTasks,
+ Config config,
+ boolean localMode) {
+ backtype.storm.Config stormConfig = new backtype.storm.Config();
+ // TODO: Configurable metric consumer instance number
+
+ int messageTimeoutSecs = config.hasPath(MESSAGE_TIMEOUT_SECS)?config.getInt(MESSAGE_TIMEOUT_SECS) : DEFAULT_MESSAGE_TIMEOUT_SECS;
+ LOG.info("Set topology.message.timeout.secs as {}",messageTimeoutSecs);
+ stormConfig.setMessageTimeoutSecs(messageTimeoutSecs);
+
+ if(config.hasPath("metric")) {
+ stormConfig.registerMetricsConsumer(StormMetricTaggedConsumer.class, config.root().render(ConfigRenderOptions.concise()),1);
+ }
+
+ stormConfig.setNumWorkers(numOfTotalWorkers);
+ StormTopology topology = buildTopology(topologyId, numOfSpoutTasks, numOfRouterBolts, numOfAlertBolts, numOfPublishTasks, config);
+
+ if(localMode) {
+ LOG.info("Submitting as local mode");
+ LocalCluster cluster = new LocalCluster();
+ cluster.submitTopology(topologyId, stormConfig, topology);
+ Utils.sleep(Long.MAX_VALUE);
+ }else{
+ LOG.info("Submitting as cluster mode");
+ try {
+ StormSubmitter.submitTopologyWithProgressBar(topologyId, stormConfig, topology);
+ } catch(Exception ex) {
+ LOG.error("fail submitting topology {}", topology, ex);
+ throw new IllegalStateException(ex);
+ }
+ }
+ }
+
+ public void run(Config config) {
+ String topologyId = config.getString("topology.name");
+ run(topologyId,config);
+ }
+
+ public void run(String topologyId,Config config) {
+ int numOfSpoutTasks = config.getInt(SPOUT_TASK_NUM);
+ int numOfRouterBolts = config.getInt(ROUTER_TASK_NUM);
+ int numOfAlertBolts = config.getInt(ALERT_TASK_NUM);
+ int numOfPublishTasks = config.getInt(PUBLISH_TASK_NUM);
+ boolean localMode = config.getBoolean(LOCAL_MODE);
+ int numOfTotalWorkers = config.getInt(TOTAL_WORKER_NUM);
+ run(topologyId,numOfTotalWorkers, numOfSpoutTasks,numOfRouterBolts,numOfAlertBolts,numOfPublishTasks,config, localMode);
+ }
+
+ public StormTopology buildTopology(String topologyId,Config config) {
+ int numOfSpoutTasks = config.getInt("topology.numOfSpoutTasks");
+ int numOfRouterBolts = config.getInt("topology.numOfRouterBolts");
+ int numOfAlertBolts = config.getInt("topology.numOfAlertBolts");
+ int numOfPublishTasks = config.getInt("topology.numOfPublishTasks");
+ return buildTopology(topologyId,numOfSpoutTasks,numOfRouterBolts,numOfAlertBolts,numOfPublishTasks,config);
+ }
+
+ public IMetadataChangeNotifyService getMetadataChangeNotifyService() {
+ return metadataChangeNotifyService;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/scheme/JsonScheme.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/scheme/JsonScheme.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/scheme/JsonScheme.java
new file mode 100644
index 0000000..a3487d3
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/scheme/JsonScheme.java
@@ -0,0 +1,71 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.alert.engine.scheme;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+import org.codehaus.jackson.map.ObjectMapper;
+import org.slf4j.Logger;
+
+import backtype.storm.spout.Scheme;
+import backtype.storm.tuple.Fields;
+
+/**
+ * Expects flat Json scheme
+ */
+public class JsonScheme implements Scheme {
+ private static final long serialVersionUID = -8352896475656975577L;
+ private static final Logger LOG = org.slf4j.LoggerFactory.getLogger(JsonScheme.class);
+ private static final ObjectMapper mapper = new ObjectMapper();
+
+ private String topic;
+
+ public JsonScheme(String topic) {
+ this.topic = topic;
+ }
+
+ @Override
+ public Fields getOutputFields() {
+ return new Fields("f1");
+ }
+
+ @Override
+ @SuppressWarnings("rawtypes")
+ public List<Object> deserialize(byte[] ser) {
+ try {
+ if(ser != null ) {
+ Map map = mapper.readValue(ser, Map.class);
+ return Arrays.asList(topic, map);
+ }else{
+ if(LOG.isDebugEnabled()) LOG.debug("Content is null, ignore");
+ }
+ } catch (IOException e) {
+ try {
+ LOG.error("Failed to deserialize as JSON: {}", new String(ser, "UTF-8"), e);
+ }catch(Exception ex){
+ LOG.error(ex.getMessage(), ex);
+ }
+ }
+ return null;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/scheme/JsonStringStreamNameSelector.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/scheme/JsonStringStreamNameSelector.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/scheme/JsonStringStreamNameSelector.java
new file mode 100644
index 0000000..1182e3f
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/scheme/JsonStringStreamNameSelector.java
@@ -0,0 +1,74 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.alert.engine.scheme;
+
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.eagle.alert.coordination.model.StreamNameSelector;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A strategy to get stream name from message tuple.
+ *
+ * Since 5/5/16.
+ */
+public class JsonStringStreamNameSelector implements StreamNameSelector {
+ private final static Logger LOG = LoggerFactory.getLogger(JsonStringStreamNameSelector.class);
+ private final static String USER_PROVIDED_STREAM_NAME_PROPERTY = "userProvidedStreamName";
+ private final static String FIELD_NAMES_TO_INFER_STREAM_NAME_PROPERTY = "fieldNamesToInferStreamName";
+ private final static String STREAM_NAME_FORMAT = "streamNameFormat";
+
+ private String userProvidedStreamName;
+ private String[] fieldNamesToInferStreamName;
+ private String streamNameFormat;
+
+ public JsonStringStreamNameSelector(Properties prop) {
+ userProvidedStreamName = prop.getProperty(USER_PROVIDED_STREAM_NAME_PROPERTY);
+ String fields = prop.getProperty(FIELD_NAMES_TO_INFER_STREAM_NAME_PROPERTY);
+ if (fields != null) {
+ fieldNamesToInferStreamName = fields.split(",");
+ }
+ streamNameFormat = prop.getProperty(STREAM_NAME_FORMAT);
+ if (streamNameFormat == null) {
+ LOG.warn("no stream name format found, this might cause default stream name be used which is dis-encouraged. Possibly this is a mis-configuration.");
+ }
+ }
+
+ @Override
+ public String getStreamName(Map<String, Object> tuple) {
+ if (userProvidedStreamName != null) {
+ return userProvidedStreamName;
+ } else if (fieldNamesToInferStreamName != null && streamNameFormat != null) {
+ Object[] args = new Object[fieldNamesToInferStreamName.length];
+ for (int i = 0; i < fieldNamesToInferStreamName.length; i++) {
+ Object colValue = tuple.get(fieldNamesToInferStreamName[i]);
+ args[i] = colValue;
+ }
+ return String.format(streamNameFormat, args);
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("can not find the stream name for data source. Use the default stream, possibly this means mis-configuration of datasource!");
+ }
+ return "defaultStringStream";
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/scheme/PlainStringScheme.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/scheme/PlainStringScheme.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/scheme/PlainStringScheme.java
new file mode 100644
index 0000000..89d2e76
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/scheme/PlainStringScheme.java
@@ -0,0 +1,67 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.alert.engine.scheme;
+
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import backtype.storm.spout.Scheme;
+import backtype.storm.tuple.Fields;
+import backtype.storm.tuple.Values;
+
+/**
+ * used for parsing plain string
+ */
+public class PlainStringScheme implements Scheme {
+ private static final long serialVersionUID = 5969724968671646310L;
+ @SuppressWarnings("unused")
+ private static final Logger LOG = LoggerFactory.getLogger(PlainStringScheme.class);
+ private String topic;
+
+ public PlainStringScheme(String topic){
+ this.topic = topic;
+ }
+
+ private static final Charset UTF8_CHARSET = StandardCharsets.UTF_8;
+ public static final String STRING_SCHEME_KEY = "str";
+
+ public static String deserializeString(byte[] buff) {
+ return new String(buff, UTF8_CHARSET);
+ }
+
+ public Fields getOutputFields() {
+ return new Fields(STRING_SCHEME_KEY);
+ }
+
+ @SuppressWarnings({ "unchecked", "rawtypes" })
+ @Override
+ public List<Object> deserialize(byte[] ser) {
+ Map m = new HashMap<>();
+ m.put("value", deserializeString(ser));
+ m.put("timestamp", System.currentTimeMillis());
+ return new Values(topic, m);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/scheme/PlainStringStreamNameSelector.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/scheme/PlainStringStreamNameSelector.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/scheme/PlainStringStreamNameSelector.java
new file mode 100644
index 0000000..61ec943
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/scheme/PlainStringStreamNameSelector.java
@@ -0,0 +1,49 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.alert.engine.scheme;
+
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.eagle.alert.coordination.model.StreamNameSelector;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Since 5/3/16.
+ */
+public class PlainStringStreamNameSelector implements StreamNameSelector {
+ @SuppressWarnings("unused")
+ private static final Logger LOG = LoggerFactory.getLogger(PlainStringStreamNameSelector.class);
+ private final static String USER_PROVIDED_STREAM_NAME_PROPERTY = "userProvidedStreamName";
+ private static final String DEFAULT_STRING_STREAM_NAME = "defaultStringStream";
+
+ private String streamName;
+
+ public PlainStringStreamNameSelector(Properties prop){
+ streamName = prop.getProperty(USER_PROVIDED_STREAM_NAME_PROPERTY);
+ if(streamName == null)
+ streamName = DEFAULT_STRING_STREAM_NAME;
+ }
+ @Override
+ public String getStreamName(Map<String, Object> tuple) {
+ return streamName;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/serialization/PartitionedEventDigestSerializer.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/serialization/PartitionedEventDigestSerializer.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/serialization/PartitionedEventDigestSerializer.java
new file mode 100644
index 0000000..5ba1080
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/serialization/PartitionedEventDigestSerializer.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.serialization;
+
+import org.apache.eagle.alert.engine.coordinator.StreamPartition;
+import org.apache.eagle.alert.engine.model.PartitionedEvent;
+import org.apache.eagle.alert.engine.model.StreamEvent;
+import org.apache.eagle.alert.engine.serialization.impl.StreamEventSerializer;
+import org.apache.eagle.alert.engine.serialization.impl.StreamPartitionDigestSerializer;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+/**
+ * TODO: Seams the complexity dosen't bring enough performance improve
+ *
+ * @see PartitionedEvent
+ */
+@Deprecated
+public class PartitionedEventDigestSerializer implements Serializer<PartitionedEvent> {
+ private final Serializer<StreamEvent> streamEventSerializer;
+ private final Serializer<StreamPartition> streamPartitionSerializer;
+
+ public PartitionedEventDigestSerializer(SerializationMetadataProvider serializationMetadataProvider){
+ this.streamEventSerializer = new StreamEventSerializer(serializationMetadataProvider);
+ this.streamPartitionSerializer = StreamPartitionDigestSerializer.INSTANCE;
+ }
+
+ @Override
+ public void serialize(PartitionedEvent entity, DataOutput dataOutput) throws IOException {
+ dataOutput.writeLong(entity.getPartitionKey());
+ streamEventSerializer.serialize(entity.getEvent(),dataOutput);
+ streamPartitionSerializer.serialize(entity.getPartition(),dataOutput);
+ }
+
+ @Override
+ public PartitionedEvent deserialize(DataInput dataInput) throws IOException {
+ PartitionedEvent event = new PartitionedEvent();
+ event.setPartitionKey(dataInput.readLong());
+ StreamEvent streamEvent = streamEventSerializer.deserialize(dataInput);
+ event.setEvent(streamEvent);
+ StreamPartition partition = streamPartitionSerializer.deserialize(dataInput);
+ partition.setStreamId(streamEvent.getStreamId());
+ event.setPartition(partition);
+ return event;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/serialization/PartitionedEventSerializer.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/serialization/PartitionedEventSerializer.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/serialization/PartitionedEventSerializer.java
new file mode 100644
index 0000000..c518e40
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/serialization/PartitionedEventSerializer.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.serialization;
+
+import org.apache.eagle.alert.engine.model.PartitionedEvent;
+
+import java.io.IOException;
+
+public interface PartitionedEventSerializer {
+ /**
+ *
+ * @param entity
+ * @return
+ * @throws IOException
+ */
+ byte[] serialize(PartitionedEvent entity) throws IOException;
+
+ /**
+ *
+ * @param bytes
+ * @return
+ * @throws IOException
+ */
+ PartitionedEvent deserialize(byte[] bytes) throws IOException;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/serialization/SerializationMetadataProvider.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/serialization/SerializationMetadataProvider.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/serialization/SerializationMetadataProvider.java
new file mode 100644
index 0000000..71b274d
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/serialization/SerializationMetadataProvider.java
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.serialization;
+
+import org.apache.eagle.alert.engine.coordinator.StreamDefinition;
+
+/**
+ * Integration interface to provide stream definition for serializer
+ */
+public interface SerializationMetadataProvider {
+ /**
+ * @param streamId
+ * @return StreamDefinition or null if not exist
+ */
+ StreamDefinition getStreamDefinition(String streamId);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/serialization/Serializer.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/serialization/Serializer.java b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/serialization/Serializer.java
new file mode 100644
index 0000000..c2f87d0
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-engine/alert-engine-base/src/main/java/org/apache/eagle/alert/engine/serialization/Serializer.java
@@ -0,0 +1,26 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.engine.serialization;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+public interface Serializer<V> {
+ void serialize(V value,DataOutput dataOutput) throws IOException;
+ V deserialize(DataInput dataInput) throws IOException;
+}
\ No newline at end of file
[16/19] incubator-eagle git commit: EAGLE-324: Init branch-v0.5
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/utils/StreamIdConversion.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/utils/StreamIdConversion.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/utils/StreamIdConversion.java
new file mode 100644
index 0000000..ccdb6f3
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/utils/StreamIdConversion.java
@@ -0,0 +1,36 @@
+package org.apache.eagle.alert.utils;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+public class StreamIdConversion {
+ private final static String STREAM_ID_TEMPLATE = "stream_%s_to_%s";
+ private final static String STREAM_ID_NUM_TEMPLATE = "stream_%s";
+ public static String generateStreamIdBetween(String sourceId, String targetId){
+ return String.format(STREAM_ID_TEMPLATE,sourceId,targetId);
+ }
+
+ /**
+ * Hard-coded stream format in stream_${partitionNum}
+ *
+ *
+ * @param partitionNum
+ * @return
+ */
+ public static String generateStreamIdByPartition(int partitionNum){
+ return String.format(STREAM_ID_NUM_TEMPLATE,partitionNum);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/utils/TimePeriodUtils.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/utils/TimePeriodUtils.java b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/utils/TimePeriodUtils.java
new file mode 100644
index 0000000..782188d
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/java/org/apache/eagle/alert/utils/TimePeriodUtils.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.utils;
+
+import org.joda.time.Period;
+import org.joda.time.Seconds;
+
+import scala.Int;
+
+public class TimePeriodUtils {
+ /**
+ * For example: timestamp stands for time: 1990/01/07 12:45 and period is PT30, then result is 1990/01/07 12:30
+ *
+ * @param seconds
+ * @param period
+ *
+ * @return formatted timestamp
+ */
+ public static long formatSecondsByPeriod(long seconds,Seconds period){
+ return seconds - (seconds % Int.int2long(period.getSeconds()));
+ }
+
+ /**
+ * @param seconds
+ * @param period
+ * @return
+ */
+ public static long formatSecondsByPeriod(long seconds,Period period){
+ return seconds - (seconds % Int.int2long(period.toStandardSeconds().getSeconds()));
+ }
+
+ /**
+ * @param milliseconds
+ * @param period
+ * @return milliseconds
+ */
+ public static long formatMillisecondsByPeriod(long milliseconds,Period period){
+ return formatSecondsByPeriod(milliseconds/1000,period)*1000;
+ }
+
+ public static int getSecondsOfPeriod(Period period){
+ return period.toStandardSeconds().getSeconds();
+ }
+
+ public static int getMillisecondsOfPeriod(Period period){
+ return getSecondsOfPeriod(period) * 1000;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/main/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/main/resources/log4j.properties b/eagle-core/eagle-alert/alert/alert-common/src/main/resources/log4j.properties
new file mode 100644
index 0000000..fb13ad5
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/main/resources/log4j.properties
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+log4j.rootLogger=DEBUG, stdout
+
+# standard output
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %p [%t] %c{2}[%L]: %m%n
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/test/java/org/apache/eagle/alert/config/TestConfigBus.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/test/java/org/apache/eagle/alert/config/TestConfigBus.java b/eagle-core/eagle-alert/alert/alert-common/src/test/java/org/apache/eagle/alert/config/TestConfigBus.java
new file mode 100644
index 0000000..8123f45
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/test/java/org/apache/eagle/alert/config/TestConfigBus.java
@@ -0,0 +1,61 @@
+package org.apache.eagle.alert.config;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.junit.Assert;
+import org.junit.Ignore;
+import org.junit.Test;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+public class TestConfigBus {
+
+ @Ignore
+ @Test
+ public void testConfigChange() throws Exception {
+ String topic = "spout";
+ ZKConfig config = new ZKConfig();
+ config.zkQuorum = "localhost:2181";
+ config.zkRoot = "/alert";
+ config.zkRetryInterval = 1000;
+ config.zkRetryTimes = 3;
+ config.connectionTimeoutMs = 3000;
+ config.zkSessionTimeoutMs = 10000;
+ ConfigBusProducer producer = new ConfigBusProducer(config);
+ final AtomicBoolean validate = new AtomicBoolean(false);
+ ConfigBusConsumer consumer = new ConfigBusConsumer(config, topic, value -> {
+ validate.set(true);
+ System.out.println("******** get notified of config " + value);
+ });
+ // first change
+ ConfigValue value = new ConfigValue();
+ value.setValueVersionId(false);
+ value.setValue("testvalue1");
+ producer.send(topic, value);
+
+ Thread.sleep(1000);
+
+ // second change
+ value.setValueVersionId(false);
+ value.setValue("testvalue2");
+ producer.send(topic, value);
+ producer.close();
+ Thread.sleep(1000);
+ consumer.close();
+ Assert.assertTrue(validate.get());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/test/java/org/apache/eagle/alert/metric/MetricSystemTest.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/test/java/org/apache/eagle/alert/metric/MetricSystemTest.java b/eagle-core/eagle-alert/alert/alert-common/src/test/java/org/apache/eagle/alert/metric/MetricSystemTest.java
new file mode 100644
index 0000000..e96ded9
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/test/java/org/apache/eagle/alert/metric/MetricSystemTest.java
@@ -0,0 +1,78 @@
+package org.apache.eagle.alert.metric;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.eagle.alert.metric.sink.KafkaSink;
+import org.apache.eagle.alert.metric.source.JVMMetricSource;
+import org.apache.eagle.alert.metric.source.MetricSource;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import com.codahale.metrics.Gauge;
+import com.codahale.metrics.JvmAttributeGaugeSet;
+import com.codahale.metrics.MetricRegistry;
+import com.typesafe.config.ConfigFactory;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+public class MetricSystemTest {
+ @Test @Ignore
+ public void testKafkaSink(){
+ KafkaSink sink = new KafkaSink();
+ MetricRegistry registry = new MetricRegistry();
+ registry.registerAll(new JvmAttributeGaugeSet());
+ sink.prepare(ConfigFactory.load().getConfig("metric.sink.kafka"),registry);
+ sink.report();
+ sink.stop();
+ }
+
+ @Test @Ignore
+ public void testMetricSystem() throws InterruptedException {
+ MetricSystem system = MetricSystem.load(ConfigFactory.load());
+ system.register(new JVMMetricSource());
+ system.register(new SampleMetricSource());
+ system.start();
+ system.report();
+ system.stop();
+ }
+
+ private class SampleMetricSource implements MetricSource {
+ private MetricRegistry registry = new MetricRegistry();
+
+ public SampleMetricSource(){
+ registry.register("sample.long", (Gauge<Long>) System::currentTimeMillis);
+ registry.register("sample.map", (Gauge<Map<String, Object>>) () -> new HashMap<String, Object>(){
+ private static final long serialVersionUID = 3948508906655117683L;
+ {
+ put("int",1234);
+ put("str","text");
+ put("bool",true);
+ }});
+ }
+
+ @Override
+ public String name() {
+ return "sampleSource";
+ }
+
+ @Override
+ public MetricRegistry registry() {
+ return registry;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/test/java/org/apache/eagle/alert/service/TestMetadataServiceClientImpl.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/test/java/org/apache/eagle/alert/service/TestMetadataServiceClientImpl.java b/eagle-core/eagle-alert/alert/alert-common/src/test/java/org/apache/eagle/alert/service/TestMetadataServiceClientImpl.java
new file mode 100644
index 0000000..4496b8e
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/test/java/org/apache/eagle/alert/service/TestMetadataServiceClientImpl.java
@@ -0,0 +1,40 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.alert.service;
+
+import java.util.List;
+
+import org.apache.eagle.alert.engine.coordinator.PolicyDefinition;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.junit.Ignore;
+import org.junit.Test;
+
+public class TestMetadataServiceClientImpl {
+ @SuppressWarnings("resource")
+ @Ignore
+ @Test
+ public void test() throws Exception{
+ MetadataServiceClientImpl impl = new MetadataServiceClientImpl("localhost", 58080, "/api/metadata/policies");
+ List<PolicyDefinition> policies = impl.listPolicies();
+ ObjectMapper mapper = new ObjectMapper();
+ String ret = mapper.writeValueAsString(policies);
+ System.out.println(ret);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/test/java/org/apache/eagle/correlation/meta/JsonTest.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/test/java/org/apache/eagle/correlation/meta/JsonTest.java b/eagle-core/eagle-alert/alert/alert-common/src/test/java/org/apache/eagle/correlation/meta/JsonTest.java
new file mode 100644
index 0000000..71cce51
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/test/java/org/apache/eagle/correlation/meta/JsonTest.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.correlation.meta;
+
+import org.junit.Test;
+
+/**
+ * Created on 3/11/16.
+ */
+public class JsonTest {
+
+ @Test
+ public void policyDefTest() {
+ // TODO
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/test/resources/application.conf
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/test/resources/application.conf b/eagle-core/eagle-alert/alert/alert-common/src/test/resources/application.conf
new file mode 100644
index 0000000..1f7bd01
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/test/resources/application.conf
@@ -0,0 +1,17 @@
+{
+ metric {
+ sink {
+ kafka {
+ "topic":"alert_metric_test"
+ "bootstrap.servers": "localhost:9092"
+ }
+ logger {
+ level = "INFO"
+ }
+ elasticsearch {
+ hosts = ["10.64.223.222:9200"]
+ index = "alert_metric_test"
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-common/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-common/src/test/resources/log4j.properties b/eagle-core/eagle-alert/alert/alert-common/src/test/resources/log4j.properties
new file mode 100644
index 0000000..fb13ad5
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-common/src/test/resources/log4j.properties
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+log4j.rootLogger=DEBUG, stdout
+
+# standard output
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %p [%t] %c{2}[%L]: %m%n
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-coordinator/.gitignore
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-coordinator/.gitignore b/eagle-core/eagle-alert/alert/alert-coordinator/.gitignore
new file mode 100644
index 0000000..b83d222
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-coordinator/.gitignore
@@ -0,0 +1 @@
+/target/
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-coordinator/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-coordinator/pom.xml b/eagle-core/eagle-alert/alert/alert-coordinator/pom.xml
new file mode 100644
index 0000000..26c743a
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-coordinator/pom.xml
@@ -0,0 +1,96 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>io.sherlock</groupId>
+ <artifactId>alert-parent</artifactId>
+ <version>0.0.1-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>alert-coordinator</artifactId>
+ <packaging>war</packaging>
+
+ <name>alert-coordinator</name>
+ <url>http://maven.apache.org</url>
+
+ <dependencies>
+ <dependency>
+ <groupId>io.sherlock</groupId>
+ <artifactId>alert-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-server</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-servlet</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.sun.jersey.contribs</groupId>
+ <artifactId>jersey-multipart</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-mapper-asl</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-jaxrs</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-xc</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.typesafe</groupId>
+ <artifactId>config</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tomcat.embed</groupId>
+ <artifactId>tomcat-embed-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>io.swagger</groupId>
+ <artifactId>swagger-jaxrs</artifactId>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-maven-plugin</artifactId>
+ <configuration>
+ <scanIntervalSeconds>5</scanIntervalSeconds>
+ <httpConnector>
+ <port>9090</port>
+ </httpConnector>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <version>2.6</version>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/Coordinator.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/Coordinator.java b/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/Coordinator.java
new file mode 100644
index 0000000..ccb4624
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/Coordinator.java
@@ -0,0 +1,212 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.coordinator;
+
+import java.text.MessageFormat;
+import java.util.Collection;
+import java.util.List;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.eagle.alert.config.ConfigBusProducer;
+import org.apache.eagle.alert.config.ConfigValue;
+import org.apache.eagle.alert.config.ZKConfig;
+import org.apache.eagle.alert.config.ZKConfigBuilder;
+import org.apache.eagle.alert.coordination.model.ScheduleState;
+import org.apache.eagle.alert.coordinator.provider.ScheduleContextBuilder;
+import org.apache.eagle.alert.coordinator.trigger.DynamicPolicyLoader;
+import org.apache.eagle.alert.coordinator.trigger.PolicyChangeListener;
+import org.apache.eagle.alert.engine.coordinator.PolicyDefinition;
+import org.apache.eagle.alert.service.IMetadataServiceClient;
+import org.apache.eagle.alert.service.MetadataServiceClientImpl;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+
+/**
+ * @since Mar 24, 2016 Coordinator is a standalone java application, which
+ * listens to policy changes and use schedule algorithm to distribute
+ * policies 1) reacting to shutdown events 2) start non-daemon thread to
+ * pull policies and figure out if polices are changed
+ */
+public class Coordinator {
+ private static final String COORDINATOR = "coordinator";
+ /**
+ * {@link ZKMetadataChangeNotifyService}
+ * /alert/{topologyName}/spout
+ * /router
+ * /alert
+ * /publisher
+ */
+ private static final String ZK_ALERT_CONFIG_SPOUT = "{0}/spout";
+ private static final String ZK_ALERT_CONFIG_ROUTER = "{0}/router";
+ private static final String ZK_ALERT_CONFIG_ALERT = "{0}/alert";
+ private static final String ZK_ALERT_CONFIG_PUBLISHER = "{0}/publisher";
+
+ private static final Logger LOG = LoggerFactory.getLogger(Coordinator.class);
+
+ private final static String METADATA_SERVICE_HOST = "metadataService.host";
+ private final static String METADATA_SERVICE_PORT = "metadataService.port";
+ private final static String METADATA_SERVICE_CONTEXT = "metadataService.context";
+ private final static String DYNAMIC_POLICY_LOADER_INIT_MILLS = "metadataDynamicCheck.initDelayMillis";
+ private final static String DYNAMIC_POLICY_LOADER_DELAY_MILLS = "metadataDynamicCheck.delayMillis";
+
+ private volatile ScheduleState currentState = null;
+ private final ConfigBusProducer producer;
+ private final IMetadataServiceClient client;
+ private Config config;
+
+ public Coordinator() {
+ config = ConfigFactory.load().getConfig(COORDINATOR);
+ ZKConfig zkConfig = ZKConfigBuilder.getZKConfig(config);
+ producer = new ConfigBusProducer(zkConfig);
+ client = new MetadataServiceClientImpl(config);
+ }
+
+ public Coordinator(Config config, ConfigBusProducer producer, IMetadataServiceClient client) {
+ this.config = config;
+ this.producer = producer;
+ this.client = client;
+ }
+
+ public ScheduleState schedule(ScheduleOption option) {
+ IScheduleContext context = new ScheduleContextBuilder(client).buildContext();
+ TopologyMgmtService mgmtService = new TopologyMgmtService();
+ IPolicyScheduler scheduler = PolicySchedulerFactory.createScheduler();
+
+ scheduler.init(context, mgmtService);
+ ScheduleState state = scheduler.schedule(option);
+
+ // persist & notify
+ postSchedule(client, state, producer);
+
+ currentState = state;
+ return state;
+ }
+
+ public static void postSchedule(IMetadataServiceClient client, ScheduleState state, ConfigBusProducer producer) {
+ // persist state
+ client.addScheduleState(state);
+ // TODO, see ScheduleState comments on how to better store these configs
+ // store policy assignment
+ // store monitored stream
+
+ // notify
+ ConfigValue value = new ConfigValue();
+ value.setValue(state.getVersion());
+ value.setValueVersionId(true);
+ for (String topo : state.getSpoutSpecs().keySet()) {
+ producer.send(MessageFormat.format(ZK_ALERT_CONFIG_SPOUT, topo), value);
+ }
+ for (String topo : state.getGroupSpecs().keySet()) {
+ producer.send(MessageFormat.format(ZK_ALERT_CONFIG_ROUTER, topo), value);
+ }
+ for (String topo : state.getAlertSpecs().keySet()) {
+ producer.send(MessageFormat.format(ZK_ALERT_CONFIG_ALERT, topo), value);
+ }
+ for (String topo : state.getPublishSpecs().keySet()) {
+ producer.send(MessageFormat.format(ZK_ALERT_CONFIG_PUBLISHER, topo), value);
+ }
+ }
+
+ public ScheduleState getState() {
+ return currentState;
+ }
+
+ /**
+ * shutdown background threads and release various resources
+ */
+ private static class CoordinatorShutdownHook implements Runnable {
+ private static final Logger LOG = LoggerFactory.getLogger(CoordinatorShutdownHook.class);
+ private ScheduledExecutorService executorSrv;
+
+ public CoordinatorShutdownHook(ScheduledExecutorService executorSrv) {
+ this.executorSrv = executorSrv;
+ }
+
+ @Override
+ public void run() {
+ LOG.info("start shutdown coordinator ...");
+ LOG.info("Step 1 shutdown dynamic policy loader thread ");
+ // we should catch every exception to make best effort for clean
+ // shutdown
+ try {
+ executorSrv.shutdown();
+ executorSrv.awaitTermination(2000, TimeUnit.MILLISECONDS);
+ } catch (Throwable t) {
+ LOG.error("error shutdown dynamic policy loader", t);
+ } finally {
+ executorSrv.shutdownNow();
+ }
+ }
+ }
+
+ private static class PolicyChangeHandler implements PolicyChangeListener {
+ private final static Logger LOG = LoggerFactory.getLogger(PolicyChangeHandler.class);
+ private Config config;
+ private IMetadataServiceClient client;
+
+ public PolicyChangeHandler(Config config) {
+ this.config = config;
+ this.client = new MetadataServiceClientImpl(config);
+ }
+
+ @Override
+ public void onPolicyChange(List<PolicyDefinition> allPolicies, Collection<String> addedPolicies,
+ Collection<String> removedPolicies, Collection<String> modifiedPolicies) {
+ LOG.info("policy changed ... ");
+ LOG.info("allPolicies: " + allPolicies + ", addedPolicies: " + addedPolicies + ", removedPolicies: "
+ + removedPolicies + ", modifiedPolicies: " + modifiedPolicies);
+
+ IScheduleContext context = new ScheduleContextBuilder(client).buildContext();
+ TopologyMgmtService mgmtService = new TopologyMgmtService();
+ IPolicyScheduler scheduler = PolicySchedulerFactory.createScheduler();
+
+ scheduler.init(context, mgmtService);
+
+ ScheduleState state = scheduler.schedule(new ScheduleOption());
+
+ ConfigBusProducer producer = new ConfigBusProducer(ZKConfigBuilder.getZKConfig(config));
+ postSchedule(client, state, producer);
+ producer.send("spout", new ConfigValue());
+ }
+ }
+
+ public static void main(String[] args) throws Exception {
+ Config config = ConfigFactory.load().getConfig(COORDINATOR);
+ // build dynamic policy loader
+ String host = config.getString(METADATA_SERVICE_HOST);
+ int port = config.getInt(METADATA_SERVICE_PORT);
+ String context = config.getString(METADATA_SERVICE_CONTEXT);
+ IMetadataServiceClient client = new MetadataServiceClientImpl(host, port, context);
+ DynamicPolicyLoader loader = new DynamicPolicyLoader(client);
+ loader.addPolicyChangeListener(new PolicyChangeHandler(config));
+
+ // schedule dynamic policy loader
+ long initDelayMillis = config.getLong(DYNAMIC_POLICY_LOADER_INIT_MILLS);
+ long delayMillis = config.getLong(DYNAMIC_POLICY_LOADER_DELAY_MILLS);
+ ScheduledExecutorService scheduleSrv = Executors.newScheduledThreadPool(1);
+ scheduleSrv.scheduleAtFixedRate(loader, initDelayMillis, delayMillis, TimeUnit.MILLISECONDS);
+ Runtime.getRuntime().addShutdownHook(new Thread(new CoordinatorShutdownHook(scheduleSrv)));
+ LOG.info("Eagle Coordinator started ...");
+
+ Thread.currentThread().join();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/CoordinatorConstants.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/CoordinatorConstants.java b/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/CoordinatorConstants.java
new file mode 100644
index 0000000..3458b3e
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/CoordinatorConstants.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.coordinator;
+
+/**
+ * @since Apr 22, 2016
+ *
+ */
+public class CoordinatorConstants {
+ public static final String CONFIG_ITEM_COORDINATOR = "coordinator";
+ public static final String CONFIG_ITEM_TOPOLOGY_LOAD_UPBOUND = "topologyLoadUpbound";
+ public static final String CONFIG_ITEM_BOLT_LOAD_UPBOUND = "boltLoadUpbound";
+ public static final String POLICY_DEFAULT_PARALLELISM = "policyDefaultParallelism";
+ public static final String BOLT_PARALLELISM = "boltParallelism";
+ public static final String NUM_OF_ALERT_BOLTS_PER_TOPOLOGY = "numOfAlertBoltsPerTopology";
+ public static final String POLICIES_PER_BOLT = "policiesPerBolt";
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/IPolicyScheduler.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/IPolicyScheduler.java b/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/IPolicyScheduler.java
new file mode 100644
index 0000000..5e61443
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/IPolicyScheduler.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.coordinator;
+
+import org.apache.eagle.alert.coordination.model.ScheduleState;
+
+/**
+ * @since Mar 24, 2016
+ *
+ */
+public interface IPolicyScheduler {
+
+ void init(IScheduleContext context, TopologyMgmtService mgmtService);
+
+ /**
+ * Build the assignments for all.
+ */
+ ScheduleState schedule(ScheduleOption option);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/IScheduleContext.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/IScheduleContext.java b/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/IScheduleContext.java
new file mode 100644
index 0000000..0cde22d
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/IScheduleContext.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.coordinator;
+
+import java.util.Map;
+
+import org.apache.eagle.alert.coordination.model.Kafka2TupleMetadata;
+import org.apache.eagle.alert.coordination.model.internal.MonitoredStream;
+import org.apache.eagle.alert.coordination.model.internal.PolicyAssignment;
+import org.apache.eagle.alert.coordination.model.internal.StreamGroup;
+import org.apache.eagle.alert.coordination.model.internal.Topology;
+import org.apache.eagle.alert.coordinator.model.TopologyUsage;
+import org.apache.eagle.alert.engine.coordinator.PolicyDefinition;
+import org.apache.eagle.alert.engine.coordinator.Publishment;
+import org.apache.eagle.alert.engine.coordinator.StreamDefinition;
+
+/**
+ * @since Mar 28, 2016
+ *
+ */
+public interface IScheduleContext {
+
+ Map<String, Topology> getTopologies();
+
+ Map<String, PolicyDefinition> getPolicies();
+
+ // data source
+ Map<String, Kafka2TupleMetadata> getDataSourceMetadata();
+
+ Map<String, StreamDefinition> getStreamSchemas();
+
+ Map<String, TopologyUsage> getTopologyUsages();
+
+ Map<String, PolicyAssignment> getPolicyAssignments();
+
+ Map<StreamGroup, MonitoredStream> getMonitoredStreams();
+
+ Map<String, Publishment> getPublishments();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/PolicySchedulerFactory.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/PolicySchedulerFactory.java b/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/PolicySchedulerFactory.java
new file mode 100644
index 0000000..8bccb53
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/PolicySchedulerFactory.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.coordinator;
+
+import org.apache.eagle.alert.coordinator.impl.GreedyPolicyScheduler;
+
+/**
+ * @since Mar 24, 2016
+ *
+ */
+public class PolicySchedulerFactory {
+
+ public static IPolicyScheduler createScheduler() {
+ return new GreedyPolicyScheduler();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/ScheduleOption.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/ScheduleOption.java b/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/ScheduleOption.java
new file mode 100644
index 0000000..6c04e61
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/ScheduleOption.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.coordinator;
+
+/**
+ * A runtime option for one schedule processing.
+ *
+ * Could used for configuration override.
+ *
+ * @since Apr 19, 2016
+ *
+ */
+public class ScheduleOption {
+ private int policiesPerBolt;
+ private int boltParallelism;
+ private int policyDefaultParallelism;
+ private double boltLoadUpbound;
+ private double topoLoadUpbound;
+
+ public int getPoliciesPerBolt() {
+ return policiesPerBolt;
+ }
+
+ public void setPoliciesPerBolt(int policiesPerBolt) {
+ this.policiesPerBolt = policiesPerBolt;
+ }
+
+ public int getBoltParallelism() {
+ return boltParallelism;
+ }
+
+ public void setBoltParallelism(int boltParallelism) {
+ this.boltParallelism = boltParallelism;
+ }
+
+ public int getPolicyDefaultParallelism() {
+ return policyDefaultParallelism;
+ }
+
+ public void setPolicyDefaultParallelism(int policyDefaultParallelism) {
+ this.policyDefaultParallelism = policyDefaultParallelism;
+ }
+
+ public double getBoltLoadUpbound() {
+ return boltLoadUpbound;
+ }
+
+ public void setBoltLoadUpbound(double boltLoadUpbound) {
+ this.boltLoadUpbound = boltLoadUpbound;
+ }
+
+ public double getTopoLoadUpbound() {
+ return topoLoadUpbound;
+ }
+
+ public void setTopoLoadUpbound(double topoLoadUpbound) {
+ this.topoLoadUpbound = topoLoadUpbound;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/TopologyMgmtService.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/TopologyMgmtService.java b/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/TopologyMgmtService.java
new file mode 100644
index 0000000..4ae07f5
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/TopologyMgmtService.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.coordinator;
+
+import static org.apache.eagle.alert.coordinator.CoordinatorConstants.CONFIG_ITEM_COORDINATOR;
+import static org.apache.eagle.alert.coordinator.CoordinatorConstants.NUM_OF_ALERT_BOLTS_PER_TOPOLOGY;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.eagle.alert.coordination.model.internal.Topology;
+import org.apache.eagle.alert.coordinator.model.TopologyUsage;
+
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+
+/**
+ * @since Mar 29, 2016
+ *
+ */
+public class TopologyMgmtService {
+
+ public static class TopologyMeta {
+ public String topologyId;
+ public Topology topology;
+ public TopologyUsage usage;
+
+ public String clusterId;
+ public String nimbusHost;
+ public String nimbusPort;
+
+ }
+
+ public static class StormClusterMeta {
+ public String clusterId;
+ public String nimbusHost;
+ public String nimbusPort;
+ public String stormVersion;
+ }
+
+ @SuppressWarnings("unused")
+ private int boltParallelism = 0;
+ private int numberOfBoltsPerTopology = 0;
+
+ public TopologyMgmtService() {
+ Config config = ConfigFactory.load().getConfig(CONFIG_ITEM_COORDINATOR);
+ boltParallelism = config.getInt(CoordinatorConstants.BOLT_PARALLELISM);
+ numberOfBoltsPerTopology = config.getInt(NUM_OF_ALERT_BOLTS_PER_TOPOLOGY);
+ }
+
+ public int getNumberOfAlertBoltsInTopology() {
+ return numberOfBoltsPerTopology;
+ }
+
+ /**
+ * TODO: call topology mgmt API to create a topology
+ *
+ * @return
+ */
+ public TopologyMeta creatTopology() {
+ // TODO
+ throw new UnsupportedOperationException("not supported yet!");
+ }
+
+ public List<TopologyMeta> listTopologies() {
+ // TODO
+ return Collections.emptyList();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/impl/GreedyPolicyScheduler.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/impl/GreedyPolicyScheduler.java b/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/impl/GreedyPolicyScheduler.java
new file mode 100644
index 0000000..a9b6c00
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/impl/GreedyPolicyScheduler.java
@@ -0,0 +1,340 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.coordinator.impl;
+
+import static org.apache.eagle.alert.coordinator.CoordinatorConstants.CONFIG_ITEM_BOLT_LOAD_UPBOUND;
+import static org.apache.eagle.alert.coordinator.CoordinatorConstants.CONFIG_ITEM_COORDINATOR;
+import static org.apache.eagle.alert.coordinator.CoordinatorConstants.POLICIES_PER_BOLT;
+import static org.apache.eagle.alert.coordinator.CoordinatorConstants.POLICY_DEFAULT_PARALLELISM;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.eagle.alert.coordination.model.ScheduleState;
+import org.apache.eagle.alert.coordination.model.WorkSlot;
+import org.apache.eagle.alert.coordination.model.internal.MonitoredStream;
+import org.apache.eagle.alert.coordination.model.internal.PolicyAssignment;
+import org.apache.eagle.alert.coordination.model.internal.StreamGroup;
+import org.apache.eagle.alert.coordination.model.internal.StreamWorkSlotQueue;
+import org.apache.eagle.alert.coordination.model.internal.Topology;
+import org.apache.eagle.alert.coordinator.IPolicyScheduler;
+import org.apache.eagle.alert.coordinator.IScheduleContext;
+import org.apache.eagle.alert.coordinator.ScheduleOption;
+import org.apache.eagle.alert.coordinator.TopologyMgmtService;
+import org.apache.eagle.alert.coordinator.model.AlertBoltUsage;
+import org.apache.eagle.alert.coordinator.model.TopologyUsage;
+import org.apache.eagle.alert.coordinator.provider.InMemScheduleConext;
+import org.apache.eagle.alert.engine.coordinator.PolicyDefinition;
+import org.apache.eagle.alert.engine.coordinator.StreamDefinition;
+import org.apache.eagle.alert.utils.JsonUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+
+/**
+ * A simple greedy assigner. <br/>
+ * A greedy assigner simply loop the policies, find the most suitable topology
+ * to locate the policy first, then assign the topics to corresponding
+ * spouts/group-by bolts.
+ *
+ * <br/>
+ * For each given policy, the greedy steps are
+ * <ul>
+ * <li>1. Find the same topology that already serve the policy without exceed the load</li>
+ * <li>2. Find the topology that already take the source traffic without exceed the load</li>
+ * <li>3. Find the topology that available to place source topic without exceed the load</li>
+ * <li>4. Create a new topology and locate the policy</li>
+ * <li>Route table generated after all policies assigned</li>
+ * <ul>
+ * <br/>
+ *
+ * @since Mar 24, 2016
+ *
+ */
+public class GreedyPolicyScheduler implements IPolicyScheduler {
+
+ private static final Logger LOG = LoggerFactory.getLogger(GreedyPolicyScheduler.class);
+
+ private int policiesPerBolt;
+ private int policyDefaultParallelism;
+ private int initialQueueSize;
+ private double boltLoadUpbound;
+
+ // copied context for scheduling
+ private IScheduleContext context;
+
+ private TopologyMgmtService mgmtService;
+
+ private ScheduleState state;
+
+ public GreedyPolicyScheduler() {
+ Config config = ConfigFactory.load().getConfig(CONFIG_ITEM_COORDINATOR);
+ policiesPerBolt = config.getInt(POLICIES_PER_BOLT);
+ policyDefaultParallelism = config.getInt(POLICY_DEFAULT_PARALLELISM);
+ initialQueueSize = policyDefaultParallelism;
+ boltLoadUpbound = config.getDouble(CONFIG_ITEM_BOLT_LOAD_UPBOUND);
+ }
+
+ public synchronized ScheduleState schedule(ScheduleOption option) {
+ // FIXME: never re-assign right now: sticky mode
+ // TODO: how to identify the over-heat nodes? not yet done #Scale of policies
+ // Answer: Use configured policiesPerBolt and configured bolt load up-bound
+ // FIXME: Here could be strategy to define the topology priorities
+ List<WorkItem> workSets = findWorkingSets();
+ /**
+ * <pre>
+ * <ul>
+ * <li>how to support take multiple "dumped" topology that consuming the same input as one available sets?</li>
+ * Answer: spout spec generated after policy assigned
+ * <li>How to define the input traffic partition?</li>
+ * Answer: input traffic might not be supported right now.
+ * <li>How to support traffic partition between topology?</li>
+ * Answer: two possible place: a global route table will be generated, those target not in current topology tuples will be dropped. This make the partition for tuple to alert
+ * <li>How to support add topology on demand by evaluate the available topology bandwidth(need topology level load)?</li>
+ * Answer: Use configured topology load up-bound, when topology load is available, will adopt
+ * <ul>
+ * <pre>
+ */
+ List<ScheduleResult> results = new ArrayList<ScheduleResult>();
+ Map<String, PolicyAssignment> newAssignments = new HashMap<String, PolicyAssignment>();
+ for (WorkItem item : workSets) {
+ ScheduleResult r = schedulePolicy(item, newAssignments);
+ results.add(r);
+ }
+
+ state = generateMonitorMetadata(workSets, newAssignments);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("calculated schedule state: {}", JsonUtils.writeValueAsString(state));
+ }
+ return state;
+ }
+
+ private List<WorkItem> findWorkingSets() {
+ // find the unassigned definition
+ List<WorkItem> workSets = new LinkedList<WorkItem>();
+ for (PolicyDefinition def : context.getPolicies().values()) {
+ int expectParal = def.getParallelismHint();
+ if (expectParal == 0) {
+ expectParal = policyDefaultParallelism;
+ }
+ // how to handle expand of an policy in a smooth transition manner
+ PolicyAssignment assignment = context.getPolicyAssignments().get(def.getName());
+ if (assignment != null) {
+ LOG.info("policy {} already allocated", def.getName());
+ continue;
+ }
+
+ WorkItem item = new WorkItem(def, expectParal);
+ workSets.add(item);
+ }
+ LOG.info("work set calculation: {}", workSets);
+ return workSets;
+ }
+
+ private ScheduleState generateMonitorMetadata(List<WorkItem> expandworkSets,
+ Map<String, PolicyAssignment> newAssignments) {
+ MonitorMetadataGenerator generator = new MonitorMetadataGenerator(context);
+ return generator.generate(expandworkSets);
+ }
+
+ private void placePolicy(PolicyDefinition def, AlertBoltUsage alertBoltUsage, Topology targetTopology,
+ TopologyUsage usage) {
+ String policyName = def.getName();
+
+ // topology usage update
+ alertBoltUsage.addPolicies(def);
+
+ // update alert policy
+ usage.getPolicies().add(policyName);
+
+ // update source topics
+ updateDataSource(usage, def);
+
+ // update group-by
+ updateGrouping(usage, def);
+ }
+
+ private void updateGrouping(TopologyUsage usage, PolicyDefinition def) {
+ // groupByMeta is removed since groupspec generate doesnt need it now.
+// List<StreamPartition> policyPartitionSpec = def.getPartitionSpec();
+// Map<String, List<StreamPartition>> groupByMeta = usage.getGroupByMeta();
+// for (StreamPartition par : policyPartitionSpec) {
+// List<StreamPartition> partitions = groupByMeta.get(par.getStreamId());
+// if (partitions == null) {
+// partitions = new ArrayList<StreamPartition>();
+// // de-dup of the partition on the list?
+// groupByMeta.put(par.getStreamId(), partitions);
+// }
+// if (!partitions.contains(par)) {
+// partitions.add(par);
+// }
+// }
+ }
+
+ private void updateDataSource(TopologyUsage usage, PolicyDefinition def) {
+ List<String> datasources = findDatasource(def);
+ usage.getDataSources().addAll(datasources);
+ }
+
+ private List<String> findDatasource(PolicyDefinition def) {
+ List<String> result = new ArrayList<String>();
+
+ List<String> inputStreams = def.getInputStreams();
+ Map<String, StreamDefinition> schemaMaps = context.getStreamSchemas();
+ for (String is : inputStreams) {
+ StreamDefinition ss = schemaMaps.get(is);
+ result.add(ss.getDataSource());
+ }
+ return result;
+ }
+
+ /**
+ * For each given policy, the greedy steps are
+ * <ul>
+ * <li>1. Find the same topology that already serve the policy</li>
+ * <li>2. Find the topology that already take the source traffic</li>
+ * <li>3. Find the topology that available to place source topic</li>
+ * <li>4. Create a new topology and locate the policy</li>
+ * <li>Route table generated after all policies assigned</li>
+ * <ul>
+ * <br/>
+ *
+ * @param newAssignments
+ */
+ private ScheduleResult schedulePolicy(WorkItem item, Map<String, PolicyAssignment> newAssignments) {
+ LOG.info(" schedule for {}", item );
+
+ String policyName = item.def.getName();
+ StreamGroup policyStreamPartition = new StreamGroup();
+ if (item.def.getPartitionSpec().isEmpty()) {
+ LOG.error(" policy {} partition spec is empty! ", policyName);
+ ScheduleResult result = new ScheduleResult();
+ result.policyName = policyName;
+ result.code = 400;
+ result.message = "policy doesn't have partition spec";
+ return result;
+ }
+ policyStreamPartition.addStreamPartitions(item.def.getPartitionSpec());
+
+ MonitoredStream targetdStream = context.getMonitoredStreams().get(policyStreamPartition);
+ if (targetdStream == null) {
+ targetdStream = new MonitoredStream(policyStreamPartition);
+ context.getMonitoredStreams().put(policyStreamPartition, targetdStream);
+ }
+
+ ScheduleResult result = new ScheduleResult();
+ result.policyName = policyName;
+
+ StreamWorkSlotQueue queue = findWorkSlotQueue(targetdStream, item.def);
+ if (queue == null) {
+ result.code = 400;
+ result.message = String.format("unable to allocate work queue resource for policy %s !", policyName);
+ } else {
+ placePolicyToQueue(item.def, queue, newAssignments);
+ result.code = 200;
+ result.message = "OK";
+ }
+
+ LOG.info(" schedule result : {}", result);
+ return result;
+ }
+
+ private void placePolicyToQueue(PolicyDefinition def, StreamWorkSlotQueue queue,
+ Map<String, PolicyAssignment> newAssignments) {
+ for (WorkSlot slot : queue.getWorkingSlots()) {
+ Topology targetTopology = context.getTopologies().get(slot.getTopologyName());
+ TopologyUsage usage = context.getTopologyUsages().get(slot.getTopologyName());
+ AlertBoltUsage alertBoltUsage = usage.getAlertBoltUsage(slot.getBoltId());
+ placePolicy(def, alertBoltUsage, targetTopology, usage);
+ }
+// queue.placePolicy(def);
+ PolicyAssignment assignment = new PolicyAssignment(def.getName(), queue.getQueueId());
+ context.getPolicyAssignments().put(def.getName(), assignment);
+ newAssignments.put(def.getName(), assignment);
+ }
+
+ private StreamWorkSlotQueue findWorkSlotQueue(MonitoredStream targetdStream, PolicyDefinition def) {
+ StreamWorkSlotQueue targetQueue = null;
+ for (StreamWorkSlotQueue queue : targetdStream.getQueues()) {
+ if (isQueueAvailable(queue, def)) {
+ targetQueue = queue;
+ break;
+ }
+ }
+
+ if (targetQueue == null) {
+ WorkQueueBuilder builder = new WorkQueueBuilder(context, mgmtService);
+ // TODO : get the properties from policy definiton
+ targetQueue = builder.createQueue(targetdStream, false, getQueueSize(def.getParallelismHint()),
+ new HashMap<String, Object>());
+ }
+ return targetQueue;
+ }
+
+ /**
+ * Some strategy to generate correct size in Startegy of queue builder
+ *
+ * @param hint
+ * @return
+ */
+ private int getQueueSize(int hint) {
+ return initialQueueSize;
+ }
+
+ private boolean isQueueAvailable(StreamWorkSlotQueue queue, PolicyDefinition def) {
+ if (queue.getQueueSize() < def.getParallelismHint()) {
+ return false;
+ }
+
+ for (WorkSlot slot : queue.getWorkingSlots()) {
+ TopologyUsage u = context.getTopologyUsages().get(slot.getTopologyName());
+ AlertBoltUsage usage = u.getAlertBoltUsage(slot.getBoltId());
+ if (!isBoltAvailable(usage, def)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private boolean isBoltAvailable(AlertBoltUsage boltUsage, PolicyDefinition def) {
+ // overload or over policy # or already contains
+ if (boltUsage == null || boltUsage.getLoad() > boltLoadUpbound
+ || boltUsage.getPolicies().size() > policiesPerBolt || boltUsage.getPolicies().contains(def.getName())) {
+ return false;
+ }
+ return true;
+ }
+
+ public void init(IScheduleContext context, TopologyMgmtService mgmtService) {
+ this.context = new InMemScheduleConext(context);
+ this.mgmtService = mgmtService;
+ }
+
+ public IScheduleContext getContext() {
+ return context;
+ }
+
+ public ScheduleState getState() {
+ return state;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/impl/MonitorMetadataGenerator.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/impl/MonitorMetadataGenerator.java b/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/impl/MonitorMetadataGenerator.java
new file mode 100644
index 0000000..40f16e9
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/impl/MonitorMetadataGenerator.java
@@ -0,0 +1,287 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.coordinator.impl;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.eagle.alert.coordination.model.AlertBoltSpec;
+import org.apache.eagle.alert.coordination.model.Kafka2TupleMetadata;
+import org.apache.eagle.alert.coordination.model.PolicyWorkerQueue;
+import org.apache.eagle.alert.coordination.model.PublishSpec;
+import org.apache.eagle.alert.coordination.model.RouterSpec;
+import org.apache.eagle.alert.coordination.model.ScheduleState;
+import org.apache.eagle.alert.coordination.model.SpoutSpec;
+import org.apache.eagle.alert.coordination.model.StreamRepartitionMetadata;
+import org.apache.eagle.alert.coordination.model.StreamRepartitionStrategy;
+import org.apache.eagle.alert.coordination.model.StreamRouterSpec;
+import org.apache.eagle.alert.coordination.model.Tuple2StreamMetadata;
+import org.apache.eagle.alert.coordination.model.internal.MonitoredStream;
+import org.apache.eagle.alert.coordination.model.internal.PolicyAssignment;
+import org.apache.eagle.alert.coordination.model.internal.StreamWorkSlotQueue;
+import org.apache.eagle.alert.coordination.model.internal.Topology;
+import org.apache.eagle.alert.coordinator.IScheduleContext;
+import org.apache.eagle.alert.coordinator.model.AlertBoltUsage;
+import org.apache.eagle.alert.coordinator.model.TopologyUsage;
+import org.apache.eagle.alert.engine.coordinator.PolicyDefinition;
+import org.apache.eagle.alert.engine.coordinator.Publishment;
+import org.apache.eagle.alert.engine.coordinator.StreamDefinition;
+import org.apache.eagle.alert.engine.coordinator.StreamPartition;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * @since Apr 26, 2016
+ * Given current policy placement, figure out monitor metadata
+ *
+ * TODO: refactor to eliminate the duplicate of stupid if-notInMap-then-create....
+ * FIXME: too many duplicated code logic : check null; add list to map; add to list..
+ */
+public class MonitorMetadataGenerator {
+
+ private static final Logger LOG = LoggerFactory.getLogger(MonitorMetadataGenerator.class);
+
+ private IScheduleContext context;
+
+ public MonitorMetadataGenerator(IScheduleContext context) {
+ this.context = context;
+ }
+
+ public ScheduleState generate(List<WorkItem> expandworkSets) {
+ // topologyId -> SpoutSpec
+ Map<String, SpoutSpec> topoSpoutSpecsMap = generateSpoutMonitorMetadata();
+
+ // grp-by meta spec(sort & grp)
+ Map<String, RouterSpec> groupSpecsMap = generateGroupbyMonitorMetadata();
+
+ // alert bolt spec
+ Map<String, AlertBoltSpec> alertSpecsMap = generateAlertMonitorMetadata();
+
+ Map<String, PublishSpec> publishSpecsMap = generatePublishMetadata();
+
+ String uniqueVersion = generateVersion();
+ ScheduleState status = new ScheduleState(uniqueVersion,
+ topoSpoutSpecsMap,
+ groupSpecsMap,
+ alertSpecsMap,
+ publishSpecsMap,
+ context.getPolicyAssignments().values(),
+ context.getMonitoredStreams().values(),
+ context.getPolicies().values(),
+ context.getStreamSchemas().values());
+ return status;
+ }
+
+ private Map<String, PublishSpec> generatePublishMetadata() {
+ Map<String, PublishSpec> pubSpecs = new HashMap<String, PublishSpec>();
+ // prebuild policy to publishment map
+ Map<String, List<Publishment>> policyToPub = new HashMap<String, List<Publishment>>();
+ for (Publishment pub : context.getPublishments().values()) {
+ for (String policyId : pub.getPolicyIds()) {
+ List<Publishment> policyPubs = policyToPub.get(policyId);
+ if (policyPubs == null) {
+ policyPubs = new ArrayList<>();
+ policyToPub.put(policyId, policyPubs);
+ }
+ policyPubs.add(pub);
+ }
+ }
+
+ // build per topology
+ for (TopologyUsage u : context.getTopologyUsages().values()) {
+ PublishSpec pubSpec = pubSpecs.get(u.getTopoName());
+ if (pubSpec == null) {
+ pubSpec = new PublishSpec(u.getTopoName(), context.getTopologies().get(u.getTopoName()).getPubBoltId());
+ pubSpecs.put(u.getTopoName(), pubSpec);
+ }
+
+ for (String p : u.getPolicies()) {
+ PolicyDefinition definition = context.getPolicies().get(p);
+ if (definition == null) {
+ continue;
+ }
+ if (policyToPub.containsKey(p)) {
+ for (Publishment pub : policyToPub.get(p)) {
+ pubSpec.addPublishment(pub);
+ }
+ }
+ }
+ }
+ return pubSpecs;
+ }
+
+ /**
+ * FIXME: add auto-increment version number?
+ */
+ private String generateVersion() {
+ return "spec_version_" + System.currentTimeMillis();
+ }
+
+ private Map<String, AlertBoltSpec> generateAlertMonitorMetadata() {
+ Map<String, AlertBoltSpec> alertSpecs = new HashMap<String, AlertBoltSpec>();
+ for (TopologyUsage u : context.getTopologyUsages().values()) {
+ AlertBoltSpec alertSpec = alertSpecs.get(u.getTopoName());
+ if (alertSpec == null) {
+ alertSpec = new AlertBoltSpec(u.getTopoName());
+ alertSpecs.put(u.getTopoName(), alertSpec);
+ }
+ for (AlertBoltUsage boltUsage : u.getAlertUsages().values()) {
+ for (String policyName : boltUsage.getPolicies()) {
+ PolicyDefinition definition = context.getPolicies().get(policyName);
+ alertSpec.addBoltPolicy(boltUsage.getBoltId(), definition.getName());
+ }
+ }
+ }
+ return alertSpecs;
+ }
+
+ private Map<String, RouterSpec> generateGroupbyMonitorMetadata() {
+ Map<String, RouterSpec> groupSpecsMap = new HashMap<String, RouterSpec>();
+ for (TopologyUsage u : context.getTopologyUsages().values()) {
+ RouterSpec spec = groupSpecsMap.get(u.getTopoName());
+ if (spec == null) {
+ spec = new RouterSpec(u.getTopoName());
+ groupSpecsMap.put(u.getTopoName(), spec);
+ }
+
+ for (MonitoredStream ms : u.getMonitoredStream()) {
+ // mutiple stream on the same policy group : for correlation group case:
+ for (StreamPartition partiton : ms.getStreamGroup().getStreamPartitions()) {
+ StreamRouterSpec routeSpec = new StreamRouterSpec();
+ routeSpec.setPartition(partiton);
+ routeSpec.setStreamId(partiton.getStreamId());
+
+ for (StreamWorkSlotQueue sq : ms.getQueues()) {
+ PolicyWorkerQueue queue = new PolicyWorkerQueue();
+ queue.setWorkers(sq.getWorkingSlots());
+ queue.setPartition(partiton);
+ routeSpec.addQueue(queue);
+ }
+
+ spec.addRouterSpec(routeSpec);
+ }
+ }
+ }
+
+ return groupSpecsMap;
+ }
+
+ private Map<String, SpoutSpec> generateSpoutMonitorMetadata() {
+ Map<String, StreamWorkSlotQueue> queueMap = buildQueueMap();
+
+ Map<String, SpoutSpec> topoSpoutSpecsMap = new HashMap<String, SpoutSpec>();
+ // streamName -> StreamDefinition
+ Map<String, StreamDefinition> streamSchemaMap = context.getStreamSchemas();
+ Map<String, Kafka2TupleMetadata> datasourcesMap = context.getDataSourceMetadata();
+ for (TopologyUsage usage : context.getTopologyUsages().values()) {
+ Topology topo = context.getTopologies().get(usage.getTopoName());
+
+ // based on data source schemas
+ // generate topic -> Kafka2TupleMetadata
+ // generate topic -> Tuple2StreamMetadata (actually the schema selector)
+ Map<String, Kafka2TupleMetadata> dss = new HashMap<String, Kafka2TupleMetadata>();
+ Map<String, Tuple2StreamMetadata> tss = new HashMap<String, Tuple2StreamMetadata>();
+ for (String dataSourceId : usage.getDataSources()) {
+ Kafka2TupleMetadata ds = datasourcesMap.get(dataSourceId);
+ dss.put(ds.getTopic(), ds);
+ tss.put(ds.getTopic(), ds.getCodec());
+ }
+
+ // generate topicId -> StreamRepartitionMetadata
+ Map<String, List<StreamRepartitionMetadata>> streamsMap = new HashMap<String, List<StreamRepartitionMetadata>>();
+ for (String policyName : usage.getPolicies()) {
+ PolicyDefinition def = context.getPolicies().get(policyName);
+
+ PolicyAssignment assignment = context.getPolicyAssignments().get(policyName);
+ if (assignment == null) {
+ LOG.error(" can not find assignment for policy {} ! ", policyName);
+ continue;
+ }
+
+ for (StreamPartition policyStreamPartition : def.getPartitionSpec()) {
+ String stream = policyStreamPartition.getStreamId();
+ StreamDefinition schema = streamSchemaMap.get(stream);
+ String topic = datasourcesMap.get(schema.getDataSource()).getTopic();
+
+ // add stream name to tuple metadata
+ if (tss.containsKey(topic)) {
+ Tuple2StreamMetadata tupleMetadata = tss.get(topic);
+ tupleMetadata.getActiveStreamNames().add(stream);
+ }
+
+ // grouping strategy
+ StreamRepartitionStrategy gs = new StreamRepartitionStrategy();
+ gs.partition = policyStreamPartition;
+ gs.numTotalParticipatingRouterBolts = queueMap.get(assignment.getQueueId()).getNumberOfGroupBolts();
+ gs.startSequence = queueMap.get(assignment.getQueueId()).getTopologyGroupStartIndex(topo.getName());
+ gs.totalTargetBoltIds = new ArrayList<String>(topo.getGroupNodeIds());
+
+ // add to map
+ addGroupingStrategy(streamsMap, stream, schema, topic, schema.getDataSource(), gs);
+ }
+ }
+
+ SpoutSpec spoutSpec = new SpoutSpec(topo.getName(), streamsMap, tss, dss);
+ topoSpoutSpecsMap.put(topo.getName(), spoutSpec);
+ }
+ return topoSpoutSpecsMap;
+ }
+
+ /**
+ * Work queue not a root level object, thus we need to build a map from
+ * MonitoredStream for later quick lookup
+ *
+ * @return
+ */
+ private Map<String, StreamWorkSlotQueue> buildQueueMap() {
+ Map<String, StreamWorkSlotQueue> queueMap = new HashMap<String, StreamWorkSlotQueue>();
+ for (MonitoredStream ms : context.getMonitoredStreams().values()) {
+ for (StreamWorkSlotQueue queue : ms.getQueues()) {
+ queueMap.put(queue.getQueueId(), queue);
+ }
+ }
+ return queueMap;
+ }
+
+ private void addGroupingStrategy(Map<String, List<StreamRepartitionMetadata>> streamsMap, String stream,
+ StreamDefinition schema, String topicName, String datasourceName, StreamRepartitionStrategy gs) {
+ List<StreamRepartitionMetadata> dsStreamMeta;
+ if (streamsMap.containsKey(topicName)) {
+ dsStreamMeta = streamsMap.get(topicName);
+ } else {
+ dsStreamMeta = new ArrayList<StreamRepartitionMetadata>();
+ streamsMap.put(topicName, dsStreamMeta);
+ }
+ StreamRepartitionMetadata targetSm = null;
+ for (StreamRepartitionMetadata sm : dsStreamMeta) {
+ if (stream.equalsIgnoreCase(sm.getStreamId())) {
+ targetSm = sm;
+ break;
+ }
+ }
+ if (targetSm == null) {
+ targetSm = new StreamRepartitionMetadata(datasourceName, schema.getStreamId());
+ dsStreamMeta.add(targetSm);
+ }
+ if (!targetSm.groupingStrategies.contains(gs)) {
+ targetSm.addGroupStrategy(gs);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/impl/ScheduleResult.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/impl/ScheduleResult.java b/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/impl/ScheduleResult.java
new file mode 100644
index 0000000..ea96d79
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/impl/ScheduleResult.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.coordinator.impl;
+
+import java.util.List;
+
+import org.apache.eagle.alert.coordination.model.internal.PolicyAssignment;
+import org.apache.eagle.alert.engine.coordinator.StreamPartition;
+
+/**
+ * Schedule result for one policy
+ *
+ *
+ * @since Apr 26, 2016
+ *
+ */
+public class ScheduleResult {
+ int code;
+ String message;
+ String policyName;
+ StreamPartition partition;
+ int index;
+ List<PolicyAssignment> topoliciesScheduled;
+
+ public String toString() {
+ return String.format("policy: %s, result code: %d ", policyName, code, message);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/impl/WorkItem.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/impl/WorkItem.java b/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/impl/WorkItem.java
new file mode 100644
index 0000000..baa489d
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/impl/WorkItem.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.coordinator.impl;
+
+import org.apache.eagle.alert.engine.coordinator.PolicyDefinition;
+
+public class WorkItem {
+ public final PolicyDefinition def;
+ public final int requestParallelism;
+
+ public WorkItem(PolicyDefinition def, int workNum) {
+ this.def = def;
+ this.requestParallelism = workNum;
+ }
+
+ public String toString() {
+ return "policy name: " + def.getName() + "(" + requestParallelism + ")";
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/impl/WorkQueueBuilder.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/impl/WorkQueueBuilder.java b/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/impl/WorkQueueBuilder.java
new file mode 100644
index 0000000..a32b8fb
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/impl/WorkQueueBuilder.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.coordinator.impl;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.eagle.alert.coordination.model.WorkSlot;
+import org.apache.eagle.alert.coordination.model.internal.MonitoredStream;
+import org.apache.eagle.alert.coordination.model.internal.StreamWorkSlotQueue;
+import org.apache.eagle.alert.coordinator.IScheduleContext;
+import org.apache.eagle.alert.coordinator.TopologyMgmtService;
+import org.apache.eagle.alert.coordinator.impl.strategies.IWorkSlotStrategy;
+import org.apache.eagle.alert.coordinator.impl.strategies.SameTopologySlotStrategy;
+import org.apache.eagle.alert.coordinator.model.AlertBoltUsage;
+import org.apache.eagle.alert.coordinator.model.TopologyUsage;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * @since Apr 27, 2016
+ *
+ */
+public class WorkQueueBuilder {
+
+ private static final Logger LOG = LoggerFactory.getLogger(WorkQueueBuilder.class);
+
+ private final IScheduleContext context;
+ private final TopologyMgmtService mgmtService;
+
+ public WorkQueueBuilder(IScheduleContext context, TopologyMgmtService mgmtService) {
+ this.context = context;
+ this.mgmtService = mgmtService;
+ }
+
+ public StreamWorkSlotQueue createQueue(MonitoredStream stream, boolean isDedicated, int size,
+ Map<String, Object> properties) {
+ // FIXME: make extensible and configurable
+ IWorkSlotStrategy strategy = new SameTopologySlotStrategy(context, stream.getStreamGroup(), mgmtService);
+ List<WorkSlot> slots = strategy.reserveWorkSlots(size, isDedicated, properties);
+ if (slots.size() < size) {
+ LOG.error("allocat stream work queue failed, required size");
+ return null;
+ }
+ StreamWorkSlotQueue queue = new StreamWorkSlotQueue(stream.getStreamGroup(), isDedicated, properties,
+ slots);
+ calculateGroupIndexAndCount(queue);
+ assignQueueSlots(stream, queue);// build reverse reference
+ stream.addQueues(queue);
+
+ return queue;
+ }
+
+ private void assignQueueSlots(MonitoredStream stream, StreamWorkSlotQueue queue) {
+ for (WorkSlot slot : queue.getWorkingSlots()) {
+ TopologyUsage u = context.getTopologyUsages().get(slot.getTopologyName());
+ AlertBoltUsage boltUsage = u.getAlertBoltUsage(slot.getBoltId());
+ boltUsage.addQueue(stream.getStreamGroup(), queue);
+ u.addMonitoredStream(stream);
+ }
+ }
+
+ private void calculateGroupIndexAndCount(StreamWorkSlotQueue queue) {
+ Map<String, Integer> result = new HashMap<String, Integer>();
+ int total = 0;
+ for (WorkSlot slot : queue.getWorkingSlots()) {
+ if (result.containsKey(slot.getTopologyName())) {
+ continue;
+ }
+ result.put(slot.getTopologyName(), total);
+ total += context.getTopologies().get(slot.getTopologyName()).getNumOfGroupBolt();
+ }
+
+ queue.setNumberOfGroupBolts(total);
+ queue.setTopoGroupStartIndex(result);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/75a8265c/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/impl/strategies/IWorkSlotStrategy.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/impl/strategies/IWorkSlotStrategy.java b/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/impl/strategies/IWorkSlotStrategy.java
new file mode 100644
index 0000000..28df3c4
--- /dev/null
+++ b/eagle-core/eagle-alert/alert/alert-coordinator/src/main/java/org/apache/eagle/alert/coordinator/impl/strategies/IWorkSlotStrategy.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.coordinator.impl.strategies;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.eagle.alert.coordination.model.WorkSlot;
+
+/**
+ * @since Apr 27, 2016
+ *
+ */
+public interface IWorkSlotStrategy {
+
+ List<WorkSlot> reserveWorkSlots(int size, boolean isDedicated, Map<String, Object> properties);
+
+}