You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@eagle.apache.org by ha...@apache.org on 2016/09/12 06:46:13 UTC

incubator-eagle git commit: [EAGLE-526] aggregation framework for history jobs

Repository: incubator-eagle
Updated Branches:
  refs/heads/master fcaf14a4e -> 2c30276ca


[EAGLE-526] aggregation framework for history jobs

Author: wujinhu <wu...@126.com>

Closes #430 from wujinhu/EAGLE-526.


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/2c30276c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/2c30276c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/2c30276c

Branch: refs/heads/master
Commit: 2c30276caebf7a920c4a3d09f84f80f0986b31c0
Parents: fcaf14a
Author: wujinhu <wu...@126.com>
Authored: Mon Sep 12 14:45:56 2016 +0800
Committer: Hao Chen <ha...@apache.org>
Committed: Mon Sep 12 14:45:56 2016 +0800

----------------------------------------------------------------------
 .../environment/impl/StormExecutionRuntime.java |   7 +
 eagle-jpm/eagle-jpm-aggregation/pom.xml         |  87 +++++++++++
 .../assembly/eagle-jpm-aggregation-assembly.xml |  65 ++++++++
 .../jpm/aggregation/AggregationApplication.java |  72 +++++++++
 .../AggregationApplicationProvider.java         |  26 ++++
 .../jpm/aggregation/AggregationConfig.java      | 135 ++++++++++++++++
 .../eagle/jpm/aggregation/AggregationMain.java  |  24 +++
 .../aggregation/common/AggregatorColumns.java   |  61 ++++++++
 .../aggregation/common/MetricAggregator.java    |  23 +++
 .../common/MetricsAggregateContainer.java       |  25 +++
 .../jpm/aggregation/mr/MRMetricAggregator.java  | 154 +++++++++++++++++++
 .../mr/MRMetricsAggregateContainer.java         |  88 +++++++++++
 .../state/AggregationTimeManager.java           | 103 +++++++++++++
 .../jpm/aggregation/storm/AggregationBolt.java  |  69 +++++++++
 .../jpm/aggregation/storm/AggregationSpout.java | 141 +++++++++++++++++
 .../src/main/resources/application.conf         |  82 ++++++++++
 .../org/apache/eagle/jpm/util/Constants.java    |   1 +
 eagle-jpm/pom.xml                               |   1 +
 18 files changed, 1164 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/2c30276c/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/impl/StormExecutionRuntime.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/impl/StormExecutionRuntime.java b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/impl/StormExecutionRuntime.java
index 7acfc62..1d0f812 100644
--- a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/impl/StormExecutionRuntime.java
+++ b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/impl/StormExecutionRuntime.java
@@ -58,10 +58,13 @@ public class StormExecutionRuntime implements ExecutionRuntime<StormEnvironment,
         return this.environment;
     }
 
+    public static final String TOPOLOGY_MESSAGE_TIMEOUT_SECS = "topology.message.timeout.secs";
+
     private static final String STORM_NIMBUS_HOST_CONF_PATH = "application.storm.nimbusHost";
     private static final String STORM_NIMBUS_HOST_DEFAULT = "localhost";
     private static final Integer STORM_NIMBUS_THRIFT_DEFAULT = 6627;
     private static final String STORM_NIMBUS_THRIFT_CONF_PATH = "application.storm.nimbusThriftPort";
+
     private static final String WORKERS = "workers";
 
     private backtype.storm.Config getStormConfig() {
@@ -92,6 +95,10 @@ public class StormExecutionRuntime implements ExecutionRuntime<StormEnvironment,
         if (environment.config().hasPath(WORKERS)) {
             conf.setNumWorkers(environment.config().getInt(WORKERS));
         }
+
+        if (environment.config().hasPath(TOPOLOGY_MESSAGE_TIMEOUT_SECS)) {
+            conf.put(TOPOLOGY_MESSAGE_TIMEOUT_SECS, environment.config().getInt(TOPOLOGY_MESSAGE_TIMEOUT_SECS));
+        }
         return conf;
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/2c30276c/eagle-jpm/eagle-jpm-aggregation/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-aggregation/pom.xml b/eagle-jpm/eagle-jpm-aggregation/pom.xml
new file mode 100644
index 0000000..80e2bc8
--- /dev/null
+++ b/eagle-jpm/eagle-jpm-aggregation/pom.xml
@@ -0,0 +1,87 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>eagle-jpm-parent</artifactId>
+        <groupId>org.apache.eagle</groupId>
+        <version>0.5.0-incubating-SNAPSHOT</version>
+        <relativePath>../pom.xml</relativePath>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>eagle-jpm-aggregation</artifactId>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.eagle</groupId>
+            <artifactId>eagle-jpm-util</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.eagle</groupId>
+            <artifactId>eagle-jpm-app</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.curator</groupId>
+            <artifactId>curator-recipes</artifactId>
+            <version>${curator.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.eagle</groupId>
+            <artifactId>eagle-jpm-entity</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <resources>
+            <resource>
+                <directory>src/main/resources</directory>
+            </resource>
+        </resources>
+        <plugins>
+            <plugin>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <configuration>
+                    <descriptor>src/assembly/eagle-jpm-aggregation-assembly.xml</descriptor>
+                    <finalName>eagle-jpm-aggregation-${project.version}</finalName>
+                </configuration>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                        <configuration>
+                            <tarLongFileMode>posix</tarLongFileMode>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+</project>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/2c30276c/eagle-jpm/eagle-jpm-aggregation/src/assembly/eagle-jpm-aggregation-assembly.xml
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-aggregation/src/assembly/eagle-jpm-aggregation-assembly.xml b/eagle-jpm/eagle-jpm-aggregation/src/assembly/eagle-jpm-aggregation-assembly.xml
new file mode 100644
index 0000000..66133a0
--- /dev/null
+++ b/eagle-jpm/eagle-jpm-aggregation/src/assembly/eagle-jpm-aggregation-assembly.xml
@@ -0,0 +1,65 @@
+<?xml version="1.0"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~    http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+          xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+    <id>assembly</id>
+    <formats>
+        <format>jar</format>
+    </formats>
+    <includeBaseDirectory>false</includeBaseDirectory>
+    <dependencySets>
+        <dependencySet>
+            <outputDirectory>/</outputDirectory>
+            <useProjectArtifact>false</useProjectArtifact>
+            <unpack>true</unpack>
+            <scope>runtime</scope>
+            <!--includes>
+                <include>org.apache.hadoop:hadoop-common</include>
+                <include>org.apache.hadoop:hadoop-hdfs</include>
+                <include>org.apache.hadoop:hadoop-client</include>
+                <include>org.apache.hadoop:hadoop-auth</include>
+                <include>org.apache.eagle:eagle-stream-process-api</include>
+                <include>org.apache.eagle:eagle-stream-process-base</include>
+                <include>org.jsoup:jsoup</include>
+            </includes-->
+            <excludes>
+                <exclude>org.wso2.orbit.com.lmax:disruptor</exclude>
+                <exclude>asm:asm</exclude>
+                <exclude>org.apache.storm:storm-core</exclude>
+            </excludes>
+        </dependencySet>
+    </dependencySets>
+    <fileSets>
+        <fileSet>
+            <directory>${project.build.outputDirectory}/</directory>
+            <outputDirectory>/</outputDirectory>
+            <!--<includes>-->
+            <!--<include>*.conf</include>-->
+            <!--<include>*.xml</include>-->
+            <!--<include>*.properties</include>-->
+            <!--<include>*.config</include>-->
+            <!--<include>classes/META-INF/*</include>-->
+            <!--</includes>-->
+
+            <excludes>
+                <exclude>*.yaml</exclude>
+            </excludes>
+        </fileSet>
+    </fileSets>
+</assembly>

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/2c30276c/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/AggregationApplication.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/AggregationApplication.java b/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/AggregationApplication.java
new file mode 100644
index 0000000..3c40f58
--- /dev/null
+++ b/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/AggregationApplication.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eagle.jpm.aggregation;
+
+import org.apache.eagle.app.StormApplication;
+import org.apache.eagle.app.environment.impl.StormEnvironment;
+import org.apache.eagle.jpm.aggregation.mr.MRMetricsAggregateContainer;
+import org.apache.eagle.jpm.aggregation.storm.AggregationBolt;
+import org.apache.eagle.jpm.aggregation.storm.AggregationSpout;
+import backtype.storm.generated.StormTopology;
+import backtype.storm.topology.TopologyBuilder;
+import com.typesafe.config.Config;
+
+import java.util.*;
+
+public class AggregationApplication extends StormApplication {
+    @Override
+    public StormTopology execute(Config config, StormEnvironment environment) {
+        AggregationConfig aggregationConfig = AggregationConfig.getInstance(config);
+
+        //TODO
+        List<String> metricNames = config.getStringList("aggregate.counters.metrics");
+        List<String> groupByColumns = config.getStringList("aggregate.counters.groupByColumns");
+        Map<String, List<List<String>>> metrics = new HashMap<>();
+        for (String metric : metricNames) {
+            metrics.put(metric, new ArrayList<>());
+            for (String cols : groupByColumns) {
+                metrics.get(metric).add(Arrays.asList(cols.replaceAll(" ", "").split(",")));
+            }
+        }
+
+        TopologyBuilder topologyBuilder = new TopologyBuilder();
+        String spoutName = "mrHistoryAggregationSpout";
+        String boltName = "mrHistoryAggregationBolt";
+        int parallelism = aggregationConfig.getConfig().getInt("envContextConfig.parallelismConfig." + spoutName);
+        int tasks = aggregationConfig.getConfig().getInt("envContextConfig.tasks." + spoutName);
+        if (parallelism > tasks) {
+            parallelism = tasks;
+        }
+        topologyBuilder.setSpout(
+            spoutName,
+            new AggregationSpout(config, new MRMetricsAggregateContainer(metrics)),
+            parallelism
+        ).setNumTasks(tasks);
+
+        parallelism = aggregationConfig.getConfig().getInt("envContextConfig.parallelismConfig." + boltName);
+        tasks = aggregationConfig.getConfig().getInt("envContextConfig.tasks." + boltName);
+        if (parallelism > tasks) {
+            parallelism = tasks;
+        }
+        topologyBuilder.setBolt(boltName,
+            new AggregationBolt(config, new MRMetricsAggregateContainer(metrics)),
+            parallelism).setNumTasks(tasks).shuffleGrouping(spoutName);
+
+        return topologyBuilder.createTopology();
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/2c30276c/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/AggregationApplicationProvider.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/AggregationApplicationProvider.java b/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/AggregationApplicationProvider.java
new file mode 100644
index 0000000..3e39fbf
--- /dev/null
+++ b/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/AggregationApplicationProvider.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eagle.jpm.aggregation;
+import org.apache.eagle.app.spi.AbstractApplicationProvider;
+
+public class AggregationApplicationProvider  extends AbstractApplicationProvider<AggregationApplication> {
+    @Override
+    public AggregationApplication getApplication() {
+        return new AggregationApplication();
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/2c30276c/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/AggregationConfig.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/AggregationConfig.java b/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/AggregationConfig.java
new file mode 100644
index 0000000..c50de92
--- /dev/null
+++ b/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/AggregationConfig.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eagle.jpm.aggregation;
+
+import com.typesafe.config.Config;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Serializable;
+
+public class AggregationConfig implements Serializable {
+    private static final Logger LOG = LoggerFactory.getLogger(AggregationConfig.class);
+
+    public String getEnv() {
+        return env;
+    }
+
+    private String env;
+
+    public ZKStateConfig getZkStateConfig() {
+        return zkStateConfig;
+    }
+
+    private ZKStateConfig zkStateConfig;
+
+    public JobExtractorConfig getJobExtractorConfig() {
+        return jobExtractorConfig;
+    }
+
+    private JobExtractorConfig jobExtractorConfig;
+
+    public EagleServiceConfig getEagleServiceConfig() {
+        return eagleServiceConfig;
+    }
+
+    private EagleServiceConfig eagleServiceConfig;
+
+    public Config getConfig() {
+        return config;
+    }
+
+    private Config config;
+
+    public static class ZKStateConfig implements Serializable {
+        public String zkQuorum;
+        public String zkRoot;
+        public int zkSessionTimeoutMs;
+        public int zkRetryTimes;
+        public int zkRetryInterval;
+        public String zkPort;
+    }
+
+    public static class JobExtractorConfig implements Serializable {
+        public String site;
+        public int readTimeoutSeconds;
+        public long aggregationDuration;
+    }
+
+    public static class EagleServiceConfig implements Serializable {
+        public String eagleServiceHost;
+        public int eagleServicePort;
+        public String username;
+        public String password;
+    }
+
+    private static AggregationConfig manager = new AggregationConfig();
+
+    private AggregationConfig() {
+        this.zkStateConfig = new ZKStateConfig();
+        this.jobExtractorConfig = new JobExtractorConfig();
+        this.eagleServiceConfig = new EagleServiceConfig();
+        this.config = null;
+    }
+
+    public static AggregationConfig getInstance(Config config) {
+        if (config != null && manager.config == null) {
+            manager.init(config);
+        }
+
+        return manager;
+    }
+
+    public static AggregationConfig get() {
+        return getInstance(null);
+    }
+
+    /**
+     * read configuration file and load hbase config etc.
+     */
+    private void init(Config config) {
+        this.config = config;
+        this.env = config.getString("envContextConfig.env");
+        //parse eagle job extractor
+        this.jobExtractorConfig.site = config.getString("jobExtractorConfig.site");
+        this.jobExtractorConfig.readTimeoutSeconds = config.getInt("jobExtractorConfig.readTimeOutSeconds");
+        this.jobExtractorConfig.aggregationDuration = config.getLong("jobExtractorConfig.aggregationDuration");
+
+        //parse eagle zk
+        this.zkStateConfig.zkQuorum = config.getString("zkStateConfig.zkQuorum");
+        this.zkStateConfig.zkPort = config.getString("zkStateConfig.zkPort");
+        this.zkStateConfig.zkSessionTimeoutMs = config.getInt("zkStateConfig.zkSessionTimeoutMs");
+        this.zkStateConfig.zkRetryTimes = config.getInt("zkStateConfig.zkRetryTimes");
+        this.zkStateConfig.zkRetryInterval = config.getInt("zkStateConfig.zkRetryInterval");
+        this.zkStateConfig.zkRoot = config.getString("zkStateConfig.zkRoot");
+
+        // parse eagle service endpoint
+        this.eagleServiceConfig.eagleServiceHost = config.getString("eagleProps.eagleService.host");
+        String port = config.getString("eagleProps.eagleService.port");
+        this.eagleServiceConfig.eagleServicePort = (port == null ? 8080 : Integer.parseInt(port));
+        this.eagleServiceConfig.username = config.getString("eagleProps.eagleService.username");
+        this.eagleServiceConfig.password = config.getString("eagleProps.eagleService.password");
+
+        LOG.info("Successfully initialized MRHistoryJobConfig");
+        LOG.info("env: " + this.env);
+        LOG.info("zookeeper.quorum: " + this.zkStateConfig.zkQuorum);
+        LOG.info("zookeeper.property.clientPort: " + this.zkStateConfig.zkPort);
+        LOG.info("eagle.service.host: " + this.eagleServiceConfig.eagleServiceHost);
+        LOG.info("eagle.service.port: " + this.eagleServiceConfig.eagleServicePort);
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/2c30276c/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/AggregationMain.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/AggregationMain.java b/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/AggregationMain.java
new file mode 100644
index 0000000..28fc79c
--- /dev/null
+++ b/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/AggregationMain.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eagle.jpm.aggregation;
+
+public class AggregationMain {
+    public static void main(String[] args) {
+        new AggregationApplication().run(args);
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/2c30276c/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/common/AggregatorColumns.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/common/AggregatorColumns.java b/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/common/AggregatorColumns.java
new file mode 100644
index 0000000..887de21
--- /dev/null
+++ b/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/common/AggregatorColumns.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.eagle.jpm.aggregation.common;
+
+import java.util.List;
+
+public class AggregatorColumns implements Comparable<AggregatorColumns> {
+    private List<String> columnNames;
+    private List<String> columnValues;
+
+    public AggregatorColumns(List<String> columnNames, List<String> columnValues) {
+        this.columnNames = columnNames;
+        this.columnValues = columnValues;
+    }
+
+    public List<String> getColumnNames() {
+        return columnNames;
+    }
+
+    public List<String> getColumnValues() {
+        return columnValues;
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder sb = new StringBuilder();
+        for (String columnValue : columnValues) {
+            sb.append(columnValue);
+            sb.append(",");
+        }
+
+        return sb.deleteCharAt(sb.length() - 1).toString();
+    }
+
+    @Override
+    public int compareTo(AggregatorColumns o) {
+        if (this.columnValues.size() > o.columnValues.size()) {
+            return 1;
+        } else if (this.columnValues.size() < o.columnValues.size()) {
+            return -1;
+        } else {
+            return this.toString().compareTo(o.toString());
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/2c30276c/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/common/MetricAggregator.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/common/MetricAggregator.java b/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/common/MetricAggregator.java
new file mode 100644
index 0000000..e12b6c7
--- /dev/null
+++ b/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/common/MetricAggregator.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.eagle.jpm.aggregation.common;
+
+public interface MetricAggregator {
+    boolean aggregate(long startTime, long endTime);
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/2c30276c/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/common/MetricsAggregateContainer.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/common/MetricsAggregateContainer.java b/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/common/MetricsAggregateContainer.java
new file mode 100644
index 0000000..c14e26b
--- /dev/null
+++ b/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/common/MetricsAggregateContainer.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.eagle.jpm.aggregation.common;
+
+public interface MetricsAggregateContainer {
+    long fetchLatestJobProcessTime();
+
+    boolean aggregate(long startTime, long endTime);
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/2c30276c/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/mr/MRMetricAggregator.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/mr/MRMetricAggregator.java b/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/mr/MRMetricAggregator.java
new file mode 100644
index 0000000..53ef968
--- /dev/null
+++ b/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/mr/MRMetricAggregator.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.eagle.jpm.aggregation.mr;
+
+import org.apache.eagle.jpm.aggregation.AggregationConfig;
+import org.apache.eagle.jpm.aggregation.common.AggregatorColumns;
+import org.apache.eagle.jpm.aggregation.common.MetricAggregator;
+import org.apache.eagle.jpm.util.Constants;
+import org.apache.eagle.log.entity.GenericMetricEntity;
+import org.apache.eagle.log.entity.GenericServiceAPIResponseEntity;
+import org.apache.eagle.service.client.IEagleServiceClient;
+import org.apache.eagle.service.client.impl.EagleServiceClientImpl;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Serializable;
+import java.util.*;
+
+public class MRMetricAggregator implements MetricAggregator, Serializable {
+    private static final Logger LOG = LoggerFactory.getLogger(MRMetricAggregator.class);
+
+    private String metric;
+    private List<List<String>> aggregateColumns;
+    //key is AggregatorColumns, value is a map(key is timeStamp, value is metric value)
+    private Map<AggregatorColumns, Map<Long, Long>> aggregateValues;
+
+    public MRMetricAggregator(String metric, List<List<String>> aggregateColumns) {
+        this.metric = metric;
+        this.aggregateColumns = aggregateColumns;
+        this.aggregateValues = new TreeMap<>();
+    }
+
+    @Override
+    public boolean aggregate(long startTime, long endTime) {
+        LOG.info("start to aggregate {} from {} to {}", metric, startTime, endTime);
+        IEagleServiceClient client = new EagleServiceClientImpl(
+            AggregationConfig.get().getEagleServiceConfig().eagleServiceHost,
+            AggregationConfig.get().getEagleServiceConfig().eagleServicePort,
+            AggregationConfig.get().getEagleServiceConfig().username,
+            AggregationConfig.get().getEagleServiceConfig().password);
+
+        String query = String.format("%s[@site=\"%s\"]{*}",
+            Constants.GENERIC_METRIC_SERVICE,
+            AggregationConfig.get().getJobExtractorConfig().site);
+
+        GenericServiceAPIResponseEntity response;
+        try {
+            response = client
+                .search(query)
+                .metricName(String.format(Constants.HADOOP_HISTORY_MINUTE_METRIC_FORMAT, Constants.JOB_LEVEL, metric))
+                .startTime(startTime)
+                .endTime(endTime)
+                .pageSize(Integer.MAX_VALUE)
+                .send();
+        } catch (Exception e) {
+            LOG.warn("{}", e);
+            return false;
+        }
+
+        List<GenericMetricEntity> entities = response.getObj();
+        for (GenericMetricEntity entity : entities) {
+            Map<String, String> tags = entity.getTags();
+            for (List<String> columnNames : this.aggregateColumns) {
+                List<String> columnValues = new ArrayList<>();
+                boolean allContains = true;
+                for (String columnName : columnNames) {
+                    if (tags.containsKey(columnName)) {
+                        columnValues.add(tags.get(columnName));
+                    } else {
+                        LOG.warn("metric entity tags do not contain {}", columnName);
+                        allContains = false;
+                        break;
+                    }
+                }
+
+                if (!allContains) {
+                    continue;
+                }
+
+                AggregatorColumns aggregatorColumns = new AggregatorColumns(columnNames, columnValues);
+                if (!this.aggregateValues.containsKey(aggregatorColumns)) {
+                    this.aggregateValues.put(aggregatorColumns, new HashMap<>());
+                }
+
+                if (!this.aggregateValues.get(aggregatorColumns).containsKey(entity.getTimestamp())) {
+                    this.aggregateValues.get(aggregatorColumns).put(entity.getTimestamp(), 0L);
+                }
+
+                Long previous = this.aggregateValues.get(aggregatorColumns).get(entity.getTimestamp());
+                this.aggregateValues.get(aggregatorColumns).put(entity.getTimestamp(), previous + (long)entity.getValue()[0]);
+            }
+        }
+        return flush();
+    }
+
+    private String buildMetricName(List<String> columnNames) {
+        return String.format(Constants.HADOOP_HISTORY_MINUTE_METRIC_FORMAT, columnNames.get(columnNames.size() - 1).toLowerCase(), metric);
+    }
+
+    private boolean flush() {
+        IEagleServiceClient client = new EagleServiceClientImpl(
+            AggregationConfig.get().getEagleServiceConfig().eagleServiceHost,
+            AggregationConfig.get().getEagleServiceConfig().eagleServicePort,
+            AggregationConfig.get().getEagleServiceConfig().username,
+            AggregationConfig.get().getEagleServiceConfig().password);
+
+        List<GenericMetricEntity> entities = new ArrayList<>();
+        for (AggregatorColumns aggregatorColumns : this.aggregateValues.keySet()) {
+            List<String> columnNames = aggregatorColumns.getColumnNames();
+            List<String> columnValues = aggregatorColumns.getColumnValues();
+            Map<String, String> baseTags = new HashMap<>();
+            for (int i = 0; i < columnNames.size(); ++i) {
+                baseTags.put(columnNames.get(i), columnValues.get(i));
+            }
+
+            Map<Long, Long> metricByMin = this.aggregateValues.get(aggregatorColumns);
+            for (Long timeStamp : metricByMin.keySet()) {
+                GenericMetricEntity metricEntity = new GenericMetricEntity();
+                metricEntity.setTimestamp(timeStamp);
+                metricEntity.setPrefix(buildMetricName(columnNames));
+                metricEntity.setValue(new double[] {metricByMin.get(timeStamp)});
+                metricEntity.setTags(baseTags);
+                entities.add(metricEntity);
+            }
+        }
+
+        try {
+            LOG.info("start flushing entities of total number " + entities.size());
+            client.create(entities);
+            LOG.info("finish flushing entities of total number " + entities.size());
+            entities.clear();
+        } catch (Exception e) {
+            LOG.warn("{}", e);
+            return false;
+        }
+        return true;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/2c30276c/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/mr/MRMetricsAggregateContainer.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/mr/MRMetricsAggregateContainer.java b/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/mr/MRMetricsAggregateContainer.java
new file mode 100644
index 0000000..d504bcf
--- /dev/null
+++ b/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/mr/MRMetricsAggregateContainer.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.eagle.jpm.aggregation.mr;
+
+import org.apache.eagle.jpm.aggregation.AggregationConfig;
+import org.apache.eagle.jpm.aggregation.common.MetricAggregator;
+import org.apache.eagle.jpm.aggregation.common.MetricsAggregateContainer;
+import org.apache.eagle.jpm.mr.historyentity.JobProcessTimeStampEntity;
+import org.apache.eagle.jpm.util.Constants;
+import org.apache.eagle.log.entity.GenericServiceAPIResponseEntity;
+import org.apache.eagle.service.client.IEagleServiceClient;
+import org.apache.eagle.service.client.impl.EagleServiceClientImpl;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class MRMetricsAggregateContainer implements MetricsAggregateContainer, Serializable {
+    private static final Logger LOG = LoggerFactory.getLogger(MRMetricsAggregateContainer.class);
+
+    private Map<String, MetricAggregator> metricAggregators;
+
+    public MRMetricsAggregateContainer(Map<String, List<List<String>>> metrics) {
+        this.metricAggregators = new HashMap<>();
+        //metric name, aggregate columns
+        for (String metric : metrics.keySet()) {
+            this.metricAggregators.put(metric, new MRMetricAggregator(metric, metrics.get(metric)));
+        }
+    }
+
+    @Override
+    public long fetchLatestJobProcessTime() {
+        try {
+            IEagleServiceClient client = new EagleServiceClientImpl(
+                AggregationConfig.get().getEagleServiceConfig().eagleServiceHost,
+                AggregationConfig.get().getEagleServiceConfig().eagleServicePort,
+                AggregationConfig.get().getEagleServiceConfig().username,
+                AggregationConfig.get().getEagleServiceConfig().password);
+
+            String query = String.format("%s[@site=\"%s\"]<@site>{max(currentTimeStamp)}",
+                Constants.JPA_JOB_PROCESS_TIME_STAMP_NAME,
+                AggregationConfig.get().getJobExtractorConfig().site);
+
+            GenericServiceAPIResponseEntity response = client
+                .search(query)
+                .startTime(0L)
+                .endTime(System.currentTimeMillis())
+                .pageSize(10)
+                .send();
+
+            List<JobProcessTimeStampEntity> entities = response.getObj();
+            return entities.get(0).getCurrentTimeStamp();
+        } catch (Exception e) {
+            LOG.warn("{}", e);
+        }
+        return 0L;
+    }
+
+    @Override
+    public boolean aggregate(long startTime, long endTime) {
+        for (String metric : this.metricAggregators.keySet()) {
+            MetricAggregator metricAggregator = this.metricAggregators.get(metric);
+            if (!metricAggregator.aggregate(startTime, endTime)) {
+                return false;
+            }
+        }
+        return true;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/2c30276c/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/state/AggregationTimeManager.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/state/AggregationTimeManager.java b/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/state/AggregationTimeManager.java
new file mode 100644
index 0000000..b650ba5
--- /dev/null
+++ b/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/state/AggregationTimeManager.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.eagle.jpm.aggregation.state;
+
+import org.apache.eagle.jpm.aggregation.AggregationConfig;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.retry.RetryNTimes;
+import org.apache.zookeeper.CreateMode;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class AggregationTimeManager {
+    private static final Logger LOG = LoggerFactory.getLogger(AggregationTimeManager.class);
+    private String zkRoot;
+    private CuratorFramework curator;
+
+    public static final String ZNODE_LAST_UPDATE = "lastUpdateTime";
+
+    private static AggregationTimeManager manager = new AggregationTimeManager();
+
+    private AggregationTimeManager() {
+    }
+
+    public static AggregationTimeManager instance() {
+        return manager;
+    }
+
+    private CuratorFramework newCurator(AggregationConfig.ZKStateConfig config) throws Exception {
+        return CuratorFrameworkFactory.newClient(
+            config.zkQuorum,
+            config.zkSessionTimeoutMs,
+            15000,
+            new RetryNTimes(config.zkRetryTimes, config.zkRetryInterval)
+        );
+    }
+
+    public void init(AggregationConfig.ZKStateConfig config) {
+        this.zkRoot = config.zkRoot;
+        try {
+            curator = newCurator(config);
+            curator.start();
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public void close() {
+        curator.close();
+        curator = null;
+    }
+
+    public long readLastFinishTime() throws Exception {
+        String path = zkRoot + "/" + ZNODE_LAST_UPDATE;
+        try {
+            if (curator.checkExists().forPath(path) != null) {
+                return Long.parseLong(new String(curator.getData().forPath(path), "UTF-8"));
+            } else {
+                curator.create()
+                    .creatingParentsIfNeeded()
+                    .withMode(CreateMode.PERSISTENT)
+                    .forPath(path, "0".getBytes("UTF-8"));
+                return 0L;
+            }
+        } catch (Exception e) {
+            LOG.error("fail reading lastUpdateTime {}", e);
+            throw e;
+        }
+    }
+
+    public void updateLastFinishTime(long lastUpdateTime) throws Exception {
+        String path = zkRoot + "/" + ZNODE_LAST_UPDATE;
+        try {
+            if (curator.checkExists().forPath(path) == null) {
+                curator.create()
+                    .creatingParentsIfNeeded()
+                    .withMode(CreateMode.PERSISTENT)
+                    .forPath(path, (lastUpdateTime + "").getBytes("UTF-8"));
+            } else {
+                curator.setData().forPath(path, (lastUpdateTime + "").getBytes("UTF-8"));
+            }
+        } catch (Exception e) {
+            LOG.error("fail update lastUpdateTime {}", e);
+            throw e;
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/2c30276c/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/storm/AggregationBolt.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/storm/AggregationBolt.java b/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/storm/AggregationBolt.java
new file mode 100644
index 0000000..c1facb9
--- /dev/null
+++ b/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/storm/AggregationBolt.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.eagle.jpm.aggregation.storm;
+
+import org.apache.eagle.jpm.aggregation.AggregationConfig;
+import org.apache.eagle.jpm.aggregation.common.MetricsAggregateContainer;
+import backtype.storm.task.OutputCollector;
+import backtype.storm.task.TopologyContext;
+import backtype.storm.topology.OutputFieldsDeclarer;
+import backtype.storm.topology.base.BaseRichBolt;
+import backtype.storm.tuple.Tuple;
+import com.typesafe.config.Config;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Map;
+
+public class AggregationBolt extends BaseRichBolt {
+    private static final Logger LOG = LoggerFactory.getLogger(AggregationBolt.class);
+    private Config config;
+    private OutputCollector collector;
+    private MetricsAggregateContainer metricsAggregateContainer;
+
+    public AggregationBolt(Config config, MetricsAggregateContainer metricsAggregateContainer) {
+        this.config = config;
+        this.metricsAggregateContainer = metricsAggregateContainer;
+    }
+
+    @Override
+    public void prepare(Map map, TopologyContext topologyContext, OutputCollector outputCollector) {
+        AggregationConfig.getInstance(config);
+        this.collector = outputCollector;
+    }
+
+    @Override
+    public void execute(Tuple tuple) {
+        Long startTime = tuple.getLongByField("startTime");
+        LOG.info("get startTime {}", startTime);
+        Long endTime = startTime + AggregationConfig.get().getJobExtractorConfig().aggregationDuration * 1000;
+
+        if (metricsAggregateContainer.aggregate(startTime, endTime)) {
+            collector.ack(tuple);
+            LOG.info("succeed startTime {}", startTime);
+        } else {
+            collector.fail(tuple);
+            LOG.warn("failed startTime {}", startTime);
+        }
+    }
+
+    @Override
+    public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) {
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/2c30276c/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/storm/AggregationSpout.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/storm/AggregationSpout.java b/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/storm/AggregationSpout.java
new file mode 100644
index 0000000..1f2cd46
--- /dev/null
+++ b/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/storm/AggregationSpout.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.eagle.jpm.aggregation.storm;
+
+import org.apache.eagle.jpm.aggregation.AggregationConfig;
+import org.apache.eagle.jpm.aggregation.common.MetricsAggregateContainer;
+import org.apache.eagle.jpm.aggregation.state.AggregationTimeManager;
+import org.apache.eagle.jpm.util.Utils;
+import backtype.storm.spout.SpoutOutputCollector;
+import backtype.storm.task.TopologyContext;
+import backtype.storm.topology.OutputFieldsDeclarer;
+import backtype.storm.topology.base.BaseRichSpout;
+import backtype.storm.tuple.Fields;
+import backtype.storm.tuple.Values;
+import com.typesafe.config.Config;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+public class AggregationSpout extends BaseRichSpout {
+    private static final Logger LOG = LoggerFactory.getLogger(AggregationSpout.class);
+    private static final Long MAX_WAIT_TIME = 12 * 60 * 60000L;//12 hours
+    private static final Long MAX_SAFE_TIME = 6 * 60 * 60000L;//6 hours
+
+    private Config config;
+    MetricsAggregateContainer jobProcessTime;
+
+    private SpoutOutputCollector collector;
+    private Set<Long> processStartTime;
+    private Long lastUpdateTime;
+
+    public AggregationSpout(Config config, MetricsAggregateContainer jobProcessTime) {
+        this.config = config;
+        this.jobProcessTime = jobProcessTime;
+        this.processStartTime = new HashSet<>();
+    }
+
+    @Override
+    public void open(Map conf, TopologyContext context, final SpoutOutputCollector collector) {
+        AggregationConfig.getInstance(config);
+        AggregationTimeManager.instance().init(AggregationConfig.get().getZkStateConfig());
+        this.collector = collector;
+    }
+
+    @Override
+    public void nextTuple() {
+        Utils.sleep(5);
+        if (LOG.isDebugEnabled()) {
+            LOG.debug("start to run");
+        }
+        try {
+            //0, check processStartTime size
+            if (this.processStartTime.size() > 0) {
+                //bolts are processing
+                return;
+            }
+
+            long currentJobTime = System.currentTimeMillis();//this.jobProcessTime.fetchLatestJobProcessTime();
+            //1, get last updateTime;
+            lastUpdateTime = AggregationTimeManager.instance().readLastFinishTime();
+            if (lastUpdateTime == 0L) {
+                //init state, just set to currentTime - 18 hours
+                lastUpdateTime = (currentJobTime - (MAX_SAFE_TIME + MAX_WAIT_TIME)) / 3600000 * 3600000;
+            }
+
+            if (currentJobTime - lastUpdateTime < MAX_WAIT_TIME) {
+                LOG.info("wait for max wait time, remaining {}ms", MAX_WAIT_TIME - currentJobTime + lastUpdateTime);
+                return;
+            }
+
+            for (Long startTime = lastUpdateTime; startTime < lastUpdateTime + MAX_SAFE_TIME;) {
+                collector.emit(new Values(startTime), startTime);
+                this.processStartTime.add(startTime);
+                startTime += AggregationConfig.get().getJobExtractorConfig().aggregationDuration * 1000;
+            }
+        } catch (Exception e) {
+            LOG.warn("{}", e);
+        }
+    }
+
+    @Override
+    public void declareOutputFields(OutputFieldsDeclarer declarer) {
+        declarer.declare(new Fields("startTime"));
+    }
+
+    @Override
+    public void ack(Object messageId) {
+        //remove from processStartTime, if size == 0, then update lastUpdateTime
+        Long startTime = (Long)messageId;
+        if (LOG.isDebugEnabled()) {
+            LOG.debug("succeed startTime {}", startTime);
+        }
+
+        this.processStartTime.remove(startTime);
+        if (this.processStartTime.size() == 0) {
+            while (true) {
+                try {
+                    LOG.info("all have finished, update lastUpdateTime to {}", this.lastUpdateTime + MAX_SAFE_TIME);
+                    AggregationTimeManager.instance().updateLastFinishTime(this.lastUpdateTime + MAX_SAFE_TIME);
+                    break;
+                } catch (Exception e) {
+                    Utils.sleep(3);
+                }
+            }
+        }
+    }
+
+    @Override
+    public void fail(Object messageId) {
+        Long startTime = (Long)messageId;
+        LOG.warn("failed startTime {}", startTime);
+        this.collector.emit(new Values(startTime), startTime);
+    }
+
+    @Override
+    public void deactivate() {
+    }
+
+    @Override
+    public void close() {
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/2c30276c/eagle-jpm/eagle-jpm-aggregation/src/main/resources/application.conf
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-aggregation/src/main/resources/application.conf b/eagle-jpm/eagle-jpm-aggregation/src/main/resources/application.conf
new file mode 100644
index 0000000..94e06b3
--- /dev/null
+++ b/eagle-jpm/eagle-jpm-aggregation/src/main/resources/application.conf
@@ -0,0 +1,82 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{
+  "envContextConfig" : {
+    "env" : "cluster",
+    "topologyName" : "mrHistoryAggregation",
+    "stormConfigFile" : "storm.yaml",
+    "parallelismConfig" : {
+      "mrHistoryAggregationSpout" : 1,
+      "mrHistoryAggregationBolt" : 6
+    },
+    "tasks" : {
+      "mrHistoryAggregationSpout" : 1,
+      "mrHistoryAggregationBolt" : 6
+    }
+  },
+
+  "jobExtractorConfig" : {
+    "site" : "sandbox",
+    "readTimeOutSeconds" : 10,
+    "aggregationDuration" : 3600 #seconds, each bolt process metrics from [start, start + aggregationDuration]
+  },
+
+  "zkStateConfig" : {
+    "zkQuorum" : "sandbox.hortonworks.com:2181",
+    "zkPort" : "2181",
+    "zkRoot" : "/aggregation/mr/sandbox",
+    "zkSessionTimeoutMs" : 15000,
+    "zkRetryTimes" : 3,
+    "zkRetryInterval" : 20000,
+  },
+
+  "eagleProps" : {
+    "mailHost" : "abc.com",
+    "mailDebug" : "true",
+    "eagleService": {
+      "host": "sandbox.hortonworks.com:2181",
+      "port": 9099,
+      "username": "admin",
+      "password": "secret"
+    }
+  },
+
+  "aggregate" : {
+    "counters" : {
+      "metrics": [
+        "cpu_milliseconds",
+        "file_bytes_read",
+        "file_bytes_written",
+        "hdfs_bytes_read",
+        "hdfs_bytes_written",
+        "hdfs_read_ops",
+        "hdfs_write_ops"
+      ],
+
+      "groupByColumns": [
+        "site, jobType",
+        "site, user",
+        "site"
+      ]
+    }
+  },
+
+  "appId":"mrHistoryAggregation",
+  "mode":"LOCAL",
+  "workers" : 3,
+  application.storm.nimbusHost=localhost
+  topology.message.timeout.secs=1800
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/2c30276c/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/Constants.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/Constants.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/Constants.java
index 5a60ee3..c9bb387 100644
--- a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/Constants.java
+++ b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/Constants.java
@@ -177,6 +177,7 @@ public class Constants {
     public static final String REDUCE_COUNT_RATIO = "reduce.count.ratio";
     public static final String JOB_LEVEL = "job";
     public static final String TASK_LEVEL = "task";
+    public static final String USER_LEVEL = "user";
     public static final String JOB_COUNT_PER_DAY = "hadoop.job.day.count";
 
     public static final String HADOOP_HISTORY_TOTAL_METRIC_FORMAT = "hadoop.%s.history.%s";

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/2c30276c/eagle-jpm/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-jpm/pom.xml b/eagle-jpm/pom.xml
index a96604d..702c89d 100644
--- a/eagle-jpm/pom.xml
+++ b/eagle-jpm/pom.xml
@@ -42,6 +42,7 @@
         <module>eagle-jpm-mr-running</module>
         <module>eagle-jpm-app</module>
         <module>eagle-jpm-service</module>
+        <module>eagle-jpm-aggregation</module>
     </modules>
 
     <dependencies>