You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@eagle.apache.org by ha...@apache.org on 2016/09/12 06:46:13 UTC
incubator-eagle git commit: [EAGLE-526] aggregation framework for
history jobs
Repository: incubator-eagle
Updated Branches:
refs/heads/master fcaf14a4e -> 2c30276ca
[EAGLE-526] aggregation framework for history jobs
Author: wujinhu <wu...@126.com>
Closes #430 from wujinhu/EAGLE-526.
Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/2c30276c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/2c30276c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/2c30276c
Branch: refs/heads/master
Commit: 2c30276caebf7a920c4a3d09f84f80f0986b31c0
Parents: fcaf14a
Author: wujinhu <wu...@126.com>
Authored: Mon Sep 12 14:45:56 2016 +0800
Committer: Hao Chen <ha...@apache.org>
Committed: Mon Sep 12 14:45:56 2016 +0800
----------------------------------------------------------------------
.../environment/impl/StormExecutionRuntime.java | 7 +
eagle-jpm/eagle-jpm-aggregation/pom.xml | 87 +++++++++++
.../assembly/eagle-jpm-aggregation-assembly.xml | 65 ++++++++
.../jpm/aggregation/AggregationApplication.java | 72 +++++++++
.../AggregationApplicationProvider.java | 26 ++++
.../jpm/aggregation/AggregationConfig.java | 135 ++++++++++++++++
.../eagle/jpm/aggregation/AggregationMain.java | 24 +++
.../aggregation/common/AggregatorColumns.java | 61 ++++++++
.../aggregation/common/MetricAggregator.java | 23 +++
.../common/MetricsAggregateContainer.java | 25 +++
.../jpm/aggregation/mr/MRMetricAggregator.java | 154 +++++++++++++++++++
.../mr/MRMetricsAggregateContainer.java | 88 +++++++++++
.../state/AggregationTimeManager.java | 103 +++++++++++++
.../jpm/aggregation/storm/AggregationBolt.java | 69 +++++++++
.../jpm/aggregation/storm/AggregationSpout.java | 141 +++++++++++++++++
.../src/main/resources/application.conf | 82 ++++++++++
.../org/apache/eagle/jpm/util/Constants.java | 1 +
eagle-jpm/pom.xml | 1 +
18 files changed, 1164 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/2c30276c/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/impl/StormExecutionRuntime.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/impl/StormExecutionRuntime.java b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/impl/StormExecutionRuntime.java
index 7acfc62..1d0f812 100644
--- a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/impl/StormExecutionRuntime.java
+++ b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/impl/StormExecutionRuntime.java
@@ -58,10 +58,13 @@ public class StormExecutionRuntime implements ExecutionRuntime<StormEnvironment,
return this.environment;
}
+ public static final String TOPOLOGY_MESSAGE_TIMEOUT_SECS = "topology.message.timeout.secs";
+
private static final String STORM_NIMBUS_HOST_CONF_PATH = "application.storm.nimbusHost";
private static final String STORM_NIMBUS_HOST_DEFAULT = "localhost";
private static final Integer STORM_NIMBUS_THRIFT_DEFAULT = 6627;
private static final String STORM_NIMBUS_THRIFT_CONF_PATH = "application.storm.nimbusThriftPort";
+
private static final String WORKERS = "workers";
private backtype.storm.Config getStormConfig() {
@@ -92,6 +95,10 @@ public class StormExecutionRuntime implements ExecutionRuntime<StormEnvironment,
if (environment.config().hasPath(WORKERS)) {
conf.setNumWorkers(environment.config().getInt(WORKERS));
}
+
+ if (environment.config().hasPath(TOPOLOGY_MESSAGE_TIMEOUT_SECS)) {
+ conf.put(TOPOLOGY_MESSAGE_TIMEOUT_SECS, environment.config().getInt(TOPOLOGY_MESSAGE_TIMEOUT_SECS));
+ }
return conf;
}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/2c30276c/eagle-jpm/eagle-jpm-aggregation/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-aggregation/pom.xml b/eagle-jpm/eagle-jpm-aggregation/pom.xml
new file mode 100644
index 0000000..80e2bc8
--- /dev/null
+++ b/eagle-jpm/eagle-jpm-aggregation/pom.xml
@@ -0,0 +1,87 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>eagle-jpm-parent</artifactId>
+ <groupId>org.apache.eagle</groupId>
+ <version>0.5.0-incubating-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>eagle-jpm-aggregation</artifactId>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.eagle</groupId>
+ <artifactId>eagle-jpm-util</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.eagle</groupId>
+ <artifactId>eagle-jpm-app</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-recipes</artifactId>
+ <version>${curator.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.eagle</groupId>
+ <artifactId>eagle-jpm-entity</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ </resource>
+ </resources>
+ <plugins>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <descriptor>src/assembly/eagle-jpm-aggregation-assembly.xml</descriptor>
+ <finalName>eagle-jpm-aggregation-${project.version}</finalName>
+ </configuration>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <configuration>
+ <tarLongFileMode>posix</tarLongFileMode>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/2c30276c/eagle-jpm/eagle-jpm-aggregation/src/assembly/eagle-jpm-aggregation-assembly.xml
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-aggregation/src/assembly/eagle-jpm-aggregation-assembly.xml b/eagle-jpm/eagle-jpm-aggregation/src/assembly/eagle-jpm-aggregation-assembly.xml
new file mode 100644
index 0000000..66133a0
--- /dev/null
+++ b/eagle-jpm/eagle-jpm-aggregation/src/assembly/eagle-jpm-aggregation-assembly.xml
@@ -0,0 +1,65 @@
+<?xml version="1.0"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+ <id>assembly</id>
+ <formats>
+ <format>jar</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <dependencySets>
+ <dependencySet>
+ <outputDirectory>/</outputDirectory>
+ <useProjectArtifact>false</useProjectArtifact>
+ <unpack>true</unpack>
+ <scope>runtime</scope>
+ <!--includes>
+ <include>org.apache.hadoop:hadoop-common</include>
+ <include>org.apache.hadoop:hadoop-hdfs</include>
+ <include>org.apache.hadoop:hadoop-client</include>
+ <include>org.apache.hadoop:hadoop-auth</include>
+ <include>org.apache.eagle:eagle-stream-process-api</include>
+ <include>org.apache.eagle:eagle-stream-process-base</include>
+ <include>org.jsoup:jsoup</include>
+ </includes-->
+ <excludes>
+ <exclude>org.wso2.orbit.com.lmax:disruptor</exclude>
+ <exclude>asm:asm</exclude>
+ <exclude>org.apache.storm:storm-core</exclude>
+ </excludes>
+ </dependencySet>
+ </dependencySets>
+ <fileSets>
+ <fileSet>
+ <directory>${project.build.outputDirectory}/</directory>
+ <outputDirectory>/</outputDirectory>
+ <!--<includes>-->
+ <!--<include>*.conf</include>-->
+ <!--<include>*.xml</include>-->
+ <!--<include>*.properties</include>-->
+ <!--<include>*.config</include>-->
+ <!--<include>classes/META-INF/*</include>-->
+ <!--</includes>-->
+
+ <excludes>
+ <exclude>*.yaml</exclude>
+ </excludes>
+ </fileSet>
+ </fileSets>
+</assembly>
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/2c30276c/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/AggregationApplication.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/AggregationApplication.java b/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/AggregationApplication.java
new file mode 100644
index 0000000..3c40f58
--- /dev/null
+++ b/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/AggregationApplication.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eagle.jpm.aggregation;
+
+import org.apache.eagle.app.StormApplication;
+import org.apache.eagle.app.environment.impl.StormEnvironment;
+import org.apache.eagle.jpm.aggregation.mr.MRMetricsAggregateContainer;
+import org.apache.eagle.jpm.aggregation.storm.AggregationBolt;
+import org.apache.eagle.jpm.aggregation.storm.AggregationSpout;
+import backtype.storm.generated.StormTopology;
+import backtype.storm.topology.TopologyBuilder;
+import com.typesafe.config.Config;
+
+import java.util.*;
+
+public class AggregationApplication extends StormApplication {
+ @Override
+ public StormTopology execute(Config config, StormEnvironment environment) {
+ AggregationConfig aggregationConfig = AggregationConfig.getInstance(config);
+
+ //TODO
+ List<String> metricNames = config.getStringList("aggregate.counters.metrics");
+ List<String> groupByColumns = config.getStringList("aggregate.counters.groupByColumns");
+ Map<String, List<List<String>>> metrics = new HashMap<>();
+ for (String metric : metricNames) {
+ metrics.put(metric, new ArrayList<>());
+ for (String cols : groupByColumns) {
+ metrics.get(metric).add(Arrays.asList(cols.replaceAll(" ", "").split(",")));
+ }
+ }
+
+ TopologyBuilder topologyBuilder = new TopologyBuilder();
+ String spoutName = "mrHistoryAggregationSpout";
+ String boltName = "mrHistoryAggregationBolt";
+ int parallelism = aggregationConfig.getConfig().getInt("envContextConfig.parallelismConfig." + spoutName);
+ int tasks = aggregationConfig.getConfig().getInt("envContextConfig.tasks." + spoutName);
+ if (parallelism > tasks) {
+ parallelism = tasks;
+ }
+ topologyBuilder.setSpout(
+ spoutName,
+ new AggregationSpout(config, new MRMetricsAggregateContainer(metrics)),
+ parallelism
+ ).setNumTasks(tasks);
+
+ parallelism = aggregationConfig.getConfig().getInt("envContextConfig.parallelismConfig." + boltName);
+ tasks = aggregationConfig.getConfig().getInt("envContextConfig.tasks." + boltName);
+ if (parallelism > tasks) {
+ parallelism = tasks;
+ }
+ topologyBuilder.setBolt(boltName,
+ new AggregationBolt(config, new MRMetricsAggregateContainer(metrics)),
+ parallelism).setNumTasks(tasks).shuffleGrouping(spoutName);
+
+ return topologyBuilder.createTopology();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/2c30276c/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/AggregationApplicationProvider.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/AggregationApplicationProvider.java b/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/AggregationApplicationProvider.java
new file mode 100644
index 0000000..3e39fbf
--- /dev/null
+++ b/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/AggregationApplicationProvider.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eagle.jpm.aggregation;
+import org.apache.eagle.app.spi.AbstractApplicationProvider;
+
+public class AggregationApplicationProvider extends AbstractApplicationProvider<AggregationApplication> {
+ @Override
+ public AggregationApplication getApplication() {
+ return new AggregationApplication();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/2c30276c/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/AggregationConfig.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/AggregationConfig.java b/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/AggregationConfig.java
new file mode 100644
index 0000000..c50de92
--- /dev/null
+++ b/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/AggregationConfig.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eagle.jpm.aggregation;
+
+import com.typesafe.config.Config;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Serializable;
+
+public class AggregationConfig implements Serializable {
+ private static final Logger LOG = LoggerFactory.getLogger(AggregationConfig.class);
+
+ public String getEnv() {
+ return env;
+ }
+
+ private String env;
+
+ public ZKStateConfig getZkStateConfig() {
+ return zkStateConfig;
+ }
+
+ private ZKStateConfig zkStateConfig;
+
+ public JobExtractorConfig getJobExtractorConfig() {
+ return jobExtractorConfig;
+ }
+
+ private JobExtractorConfig jobExtractorConfig;
+
+ public EagleServiceConfig getEagleServiceConfig() {
+ return eagleServiceConfig;
+ }
+
+ private EagleServiceConfig eagleServiceConfig;
+
+ public Config getConfig() {
+ return config;
+ }
+
+ private Config config;
+
+ public static class ZKStateConfig implements Serializable {
+ public String zkQuorum;
+ public String zkRoot;
+ public int zkSessionTimeoutMs;
+ public int zkRetryTimes;
+ public int zkRetryInterval;
+ public String zkPort;
+ }
+
+ public static class JobExtractorConfig implements Serializable {
+ public String site;
+ public int readTimeoutSeconds;
+ public long aggregationDuration;
+ }
+
+ public static class EagleServiceConfig implements Serializable {
+ public String eagleServiceHost;
+ public int eagleServicePort;
+ public String username;
+ public String password;
+ }
+
+ private static AggregationConfig manager = new AggregationConfig();
+
+ private AggregationConfig() {
+ this.zkStateConfig = new ZKStateConfig();
+ this.jobExtractorConfig = new JobExtractorConfig();
+ this.eagleServiceConfig = new EagleServiceConfig();
+ this.config = null;
+ }
+
+ public static AggregationConfig getInstance(Config config) {
+ if (config != null && manager.config == null) {
+ manager.init(config);
+ }
+
+ return manager;
+ }
+
+ public static AggregationConfig get() {
+ return getInstance(null);
+ }
+
+ /**
+ * read configuration file and load hbase config etc.
+ */
+ private void init(Config config) {
+ this.config = config;
+ this.env = config.getString("envContextConfig.env");
+ //parse eagle job extractor
+ this.jobExtractorConfig.site = config.getString("jobExtractorConfig.site");
+ this.jobExtractorConfig.readTimeoutSeconds = config.getInt("jobExtractorConfig.readTimeOutSeconds");
+ this.jobExtractorConfig.aggregationDuration = config.getLong("jobExtractorConfig.aggregationDuration");
+
+ //parse eagle zk
+ this.zkStateConfig.zkQuorum = config.getString("zkStateConfig.zkQuorum");
+ this.zkStateConfig.zkPort = config.getString("zkStateConfig.zkPort");
+ this.zkStateConfig.zkSessionTimeoutMs = config.getInt("zkStateConfig.zkSessionTimeoutMs");
+ this.zkStateConfig.zkRetryTimes = config.getInt("zkStateConfig.zkRetryTimes");
+ this.zkStateConfig.zkRetryInterval = config.getInt("zkStateConfig.zkRetryInterval");
+ this.zkStateConfig.zkRoot = config.getString("zkStateConfig.zkRoot");
+
+ // parse eagle service endpoint
+ this.eagleServiceConfig.eagleServiceHost = config.getString("eagleProps.eagleService.host");
+ String port = config.getString("eagleProps.eagleService.port");
+ this.eagleServiceConfig.eagleServicePort = (port == null ? 8080 : Integer.parseInt(port));
+ this.eagleServiceConfig.username = config.getString("eagleProps.eagleService.username");
+ this.eagleServiceConfig.password = config.getString("eagleProps.eagleService.password");
+
+ LOG.info("Successfully initialized MRHistoryJobConfig");
+ LOG.info("env: " + this.env);
+ LOG.info("zookeeper.quorum: " + this.zkStateConfig.zkQuorum);
+ LOG.info("zookeeper.property.clientPort: " + this.zkStateConfig.zkPort);
+ LOG.info("eagle.service.host: " + this.eagleServiceConfig.eagleServiceHost);
+ LOG.info("eagle.service.port: " + this.eagleServiceConfig.eagleServicePort);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/2c30276c/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/AggregationMain.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/AggregationMain.java b/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/AggregationMain.java
new file mode 100644
index 0000000..28fc79c
--- /dev/null
+++ b/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/AggregationMain.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eagle.jpm.aggregation;
+
+public class AggregationMain {
+ public static void main(String[] args) {
+ new AggregationApplication().run(args);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/2c30276c/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/common/AggregatorColumns.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/common/AggregatorColumns.java b/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/common/AggregatorColumns.java
new file mode 100644
index 0000000..887de21
--- /dev/null
+++ b/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/common/AggregatorColumns.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.eagle.jpm.aggregation.common;
+
+import java.util.List;
+
+public class AggregatorColumns implements Comparable<AggregatorColumns> {
+ private List<String> columnNames;
+ private List<String> columnValues;
+
+ public AggregatorColumns(List<String> columnNames, List<String> columnValues) {
+ this.columnNames = columnNames;
+ this.columnValues = columnValues;
+ }
+
+ public List<String> getColumnNames() {
+ return columnNames;
+ }
+
+ public List<String> getColumnValues() {
+ return columnValues;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ for (String columnValue : columnValues) {
+ sb.append(columnValue);
+ sb.append(",");
+ }
+
+ return sb.deleteCharAt(sb.length() - 1).toString();
+ }
+
+ @Override
+ public int compareTo(AggregatorColumns o) {
+ if (this.columnValues.size() > o.columnValues.size()) {
+ return 1;
+ } else if (this.columnValues.size() < o.columnValues.size()) {
+ return -1;
+ } else {
+ return this.toString().compareTo(o.toString());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/2c30276c/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/common/MetricAggregator.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/common/MetricAggregator.java b/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/common/MetricAggregator.java
new file mode 100644
index 0000000..e12b6c7
--- /dev/null
+++ b/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/common/MetricAggregator.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.eagle.jpm.aggregation.common;
+
+public interface MetricAggregator {
+ boolean aggregate(long startTime, long endTime);
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/2c30276c/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/common/MetricsAggregateContainer.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/common/MetricsAggregateContainer.java b/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/common/MetricsAggregateContainer.java
new file mode 100644
index 0000000..c14e26b
--- /dev/null
+++ b/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/common/MetricsAggregateContainer.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.eagle.jpm.aggregation.common;
+
+public interface MetricsAggregateContainer {
+ long fetchLatestJobProcessTime();
+
+ boolean aggregate(long startTime, long endTime);
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/2c30276c/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/mr/MRMetricAggregator.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/mr/MRMetricAggregator.java b/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/mr/MRMetricAggregator.java
new file mode 100644
index 0000000..53ef968
--- /dev/null
+++ b/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/mr/MRMetricAggregator.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.eagle.jpm.aggregation.mr;
+
+import org.apache.eagle.jpm.aggregation.AggregationConfig;
+import org.apache.eagle.jpm.aggregation.common.AggregatorColumns;
+import org.apache.eagle.jpm.aggregation.common.MetricAggregator;
+import org.apache.eagle.jpm.util.Constants;
+import org.apache.eagle.log.entity.GenericMetricEntity;
+import org.apache.eagle.log.entity.GenericServiceAPIResponseEntity;
+import org.apache.eagle.service.client.IEagleServiceClient;
+import org.apache.eagle.service.client.impl.EagleServiceClientImpl;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Serializable;
+import java.util.*;
+
+public class MRMetricAggregator implements MetricAggregator, Serializable {
+ private static final Logger LOG = LoggerFactory.getLogger(MRMetricAggregator.class);
+
+ private String metric;
+ private List<List<String>> aggregateColumns;
+ //key is AggregatorColumns, value is a map(key is timeStamp, value is metric value)
+ private Map<AggregatorColumns, Map<Long, Long>> aggregateValues;
+
+ public MRMetricAggregator(String metric, List<List<String>> aggregateColumns) {
+ this.metric = metric;
+ this.aggregateColumns = aggregateColumns;
+ this.aggregateValues = new TreeMap<>();
+ }
+
+ @Override
+ public boolean aggregate(long startTime, long endTime) {
+ LOG.info("start to aggregate {} from {} to {}", metric, startTime, endTime);
+ IEagleServiceClient client = new EagleServiceClientImpl(
+ AggregationConfig.get().getEagleServiceConfig().eagleServiceHost,
+ AggregationConfig.get().getEagleServiceConfig().eagleServicePort,
+ AggregationConfig.get().getEagleServiceConfig().username,
+ AggregationConfig.get().getEagleServiceConfig().password);
+
+ String query = String.format("%s[@site=\"%s\"]{*}",
+ Constants.GENERIC_METRIC_SERVICE,
+ AggregationConfig.get().getJobExtractorConfig().site);
+
+ GenericServiceAPIResponseEntity response;
+ try {
+ response = client
+ .search(query)
+ .metricName(String.format(Constants.HADOOP_HISTORY_MINUTE_METRIC_FORMAT, Constants.JOB_LEVEL, metric))
+ .startTime(startTime)
+ .endTime(endTime)
+ .pageSize(Integer.MAX_VALUE)
+ .send();
+ } catch (Exception e) {
+ LOG.warn("{}", e);
+ return false;
+ }
+
+ List<GenericMetricEntity> entities = response.getObj();
+ for (GenericMetricEntity entity : entities) {
+ Map<String, String> tags = entity.getTags();
+ for (List<String> columnNames : this.aggregateColumns) {
+ List<String> columnValues = new ArrayList<>();
+ boolean allContains = true;
+ for (String columnName : columnNames) {
+ if (tags.containsKey(columnName)) {
+ columnValues.add(tags.get(columnName));
+ } else {
+ LOG.warn("metric entity tags do not contain {}", columnName);
+ allContains = false;
+ break;
+ }
+ }
+
+ if (!allContains) {
+ continue;
+ }
+
+ AggregatorColumns aggregatorColumns = new AggregatorColumns(columnNames, columnValues);
+ if (!this.aggregateValues.containsKey(aggregatorColumns)) {
+ this.aggregateValues.put(aggregatorColumns, new HashMap<>());
+ }
+
+ if (!this.aggregateValues.get(aggregatorColumns).containsKey(entity.getTimestamp())) {
+ this.aggregateValues.get(aggregatorColumns).put(entity.getTimestamp(), 0L);
+ }
+
+ Long previous = this.aggregateValues.get(aggregatorColumns).get(entity.getTimestamp());
+ this.aggregateValues.get(aggregatorColumns).put(entity.getTimestamp(), previous + (long)entity.getValue()[0]);
+ }
+ }
+ return flush();
+ }
+
+ private String buildMetricName(List<String> columnNames) {
+ return String.format(Constants.HADOOP_HISTORY_MINUTE_METRIC_FORMAT, columnNames.get(columnNames.size() - 1).toLowerCase(), metric);
+ }
+
+ private boolean flush() {
+ IEagleServiceClient client = new EagleServiceClientImpl(
+ AggregationConfig.get().getEagleServiceConfig().eagleServiceHost,
+ AggregationConfig.get().getEagleServiceConfig().eagleServicePort,
+ AggregationConfig.get().getEagleServiceConfig().username,
+ AggregationConfig.get().getEagleServiceConfig().password);
+
+ List<GenericMetricEntity> entities = new ArrayList<>();
+ for (AggregatorColumns aggregatorColumns : this.aggregateValues.keySet()) {
+ List<String> columnNames = aggregatorColumns.getColumnNames();
+ List<String> columnValues = aggregatorColumns.getColumnValues();
+ Map<String, String> baseTags = new HashMap<>();
+ for (int i = 0; i < columnNames.size(); ++i) {
+ baseTags.put(columnNames.get(i), columnValues.get(i));
+ }
+
+ Map<Long, Long> metricByMin = this.aggregateValues.get(aggregatorColumns);
+ for (Long timeStamp : metricByMin.keySet()) {
+ GenericMetricEntity metricEntity = new GenericMetricEntity();
+ metricEntity.setTimestamp(timeStamp);
+ metricEntity.setPrefix(buildMetricName(columnNames));
+ metricEntity.setValue(new double[] {metricByMin.get(timeStamp)});
+ metricEntity.setTags(baseTags);
+ entities.add(metricEntity);
+ }
+ }
+
+ try {
+ LOG.info("start flushing entities of total number " + entities.size());
+ client.create(entities);
+ LOG.info("finish flushing entities of total number " + entities.size());
+ entities.clear();
+ } catch (Exception e) {
+ LOG.warn("{}", e);
+ return false;
+ }
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/2c30276c/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/mr/MRMetricsAggregateContainer.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/mr/MRMetricsAggregateContainer.java b/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/mr/MRMetricsAggregateContainer.java
new file mode 100644
index 0000000..d504bcf
--- /dev/null
+++ b/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/mr/MRMetricsAggregateContainer.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.eagle.jpm.aggregation.mr;
+
+import org.apache.eagle.jpm.aggregation.AggregationConfig;
+import org.apache.eagle.jpm.aggregation.common.MetricAggregator;
+import org.apache.eagle.jpm.aggregation.common.MetricsAggregateContainer;
+import org.apache.eagle.jpm.mr.historyentity.JobProcessTimeStampEntity;
+import org.apache.eagle.jpm.util.Constants;
+import org.apache.eagle.log.entity.GenericServiceAPIResponseEntity;
+import org.apache.eagle.service.client.IEagleServiceClient;
+import org.apache.eagle.service.client.impl.EagleServiceClientImpl;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class MRMetricsAggregateContainer implements MetricsAggregateContainer, Serializable {
+ private static final Logger LOG = LoggerFactory.getLogger(MRMetricsAggregateContainer.class);
+
+ private Map<String, MetricAggregator> metricAggregators;
+
+ public MRMetricsAggregateContainer(Map<String, List<List<String>>> metrics) {
+ this.metricAggregators = new HashMap<>();
+ //metric name, aggregate columns
+ for (String metric : metrics.keySet()) {
+ this.metricAggregators.put(metric, new MRMetricAggregator(metric, metrics.get(metric)));
+ }
+ }
+
+ @Override
+ public long fetchLatestJobProcessTime() {
+ try {
+ IEagleServiceClient client = new EagleServiceClientImpl(
+ AggregationConfig.get().getEagleServiceConfig().eagleServiceHost,
+ AggregationConfig.get().getEagleServiceConfig().eagleServicePort,
+ AggregationConfig.get().getEagleServiceConfig().username,
+ AggregationConfig.get().getEagleServiceConfig().password);
+
+ String query = String.format("%s[@site=\"%s\"]<@site>{max(currentTimeStamp)}",
+ Constants.JPA_JOB_PROCESS_TIME_STAMP_NAME,
+ AggregationConfig.get().getJobExtractorConfig().site);
+
+ GenericServiceAPIResponseEntity response = client
+ .search(query)
+ .startTime(0L)
+ .endTime(System.currentTimeMillis())
+ .pageSize(10)
+ .send();
+
+ List<JobProcessTimeStampEntity> entities = response.getObj();
+ return entities.get(0).getCurrentTimeStamp();
+ } catch (Exception e) {
+ LOG.warn("{}", e);
+ }
+ return 0L;
+ }
+
+ @Override
+ public boolean aggregate(long startTime, long endTime) {
+ for (String metric : this.metricAggregators.keySet()) {
+ MetricAggregator metricAggregator = this.metricAggregators.get(metric);
+ if (!metricAggregator.aggregate(startTime, endTime)) {
+ return false;
+ }
+ }
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/2c30276c/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/state/AggregationTimeManager.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/state/AggregationTimeManager.java b/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/state/AggregationTimeManager.java
new file mode 100644
index 0000000..b650ba5
--- /dev/null
+++ b/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/state/AggregationTimeManager.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.eagle.jpm.aggregation.state;
+
+import org.apache.eagle.jpm.aggregation.AggregationConfig;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.retry.RetryNTimes;
+import org.apache.zookeeper.CreateMode;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class AggregationTimeManager {
+ private static final Logger LOG = LoggerFactory.getLogger(AggregationTimeManager.class);
+ private String zkRoot;
+ private CuratorFramework curator;
+
+ public static final String ZNODE_LAST_UPDATE = "lastUpdateTime";
+
+ private static AggregationTimeManager manager = new AggregationTimeManager();
+
+ private AggregationTimeManager() {
+ }
+
+ public static AggregationTimeManager instance() {
+ return manager;
+ }
+
+ private CuratorFramework newCurator(AggregationConfig.ZKStateConfig config) throws Exception {
+ return CuratorFrameworkFactory.newClient(
+ config.zkQuorum,
+ config.zkSessionTimeoutMs,
+ 15000,
+ new RetryNTimes(config.zkRetryTimes, config.zkRetryInterval)
+ );
+ }
+
+ public void init(AggregationConfig.ZKStateConfig config) {
+ this.zkRoot = config.zkRoot;
+ try {
+ curator = newCurator(config);
+ curator.start();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public void close() {
+ curator.close();
+ curator = null;
+ }
+
+ public long readLastFinishTime() throws Exception {
+ String path = zkRoot + "/" + ZNODE_LAST_UPDATE;
+ try {
+ if (curator.checkExists().forPath(path) != null) {
+ return Long.parseLong(new String(curator.getData().forPath(path), "UTF-8"));
+ } else {
+ curator.create()
+ .creatingParentsIfNeeded()
+ .withMode(CreateMode.PERSISTENT)
+ .forPath(path, "0".getBytes("UTF-8"));
+ return 0L;
+ }
+ } catch (Exception e) {
+ LOG.error("fail reading lastUpdateTime {}", e);
+ throw e;
+ }
+ }
+
+ public void updateLastFinishTime(long lastUpdateTime) throws Exception {
+ String path = zkRoot + "/" + ZNODE_LAST_UPDATE;
+ try {
+ if (curator.checkExists().forPath(path) == null) {
+ curator.create()
+ .creatingParentsIfNeeded()
+ .withMode(CreateMode.PERSISTENT)
+ .forPath(path, (lastUpdateTime + "").getBytes("UTF-8"));
+ } else {
+ curator.setData().forPath(path, (lastUpdateTime + "").getBytes("UTF-8"));
+ }
+ } catch (Exception e) {
+ LOG.error("fail update lastUpdateTime {}", e);
+ throw e;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/2c30276c/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/storm/AggregationBolt.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/storm/AggregationBolt.java b/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/storm/AggregationBolt.java
new file mode 100644
index 0000000..c1facb9
--- /dev/null
+++ b/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/storm/AggregationBolt.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.eagle.jpm.aggregation.storm;
+
+import org.apache.eagle.jpm.aggregation.AggregationConfig;
+import org.apache.eagle.jpm.aggregation.common.MetricsAggregateContainer;
+import backtype.storm.task.OutputCollector;
+import backtype.storm.task.TopologyContext;
+import backtype.storm.topology.OutputFieldsDeclarer;
+import backtype.storm.topology.base.BaseRichBolt;
+import backtype.storm.tuple.Tuple;
+import com.typesafe.config.Config;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Map;
+
+public class AggregationBolt extends BaseRichBolt {
+ private static final Logger LOG = LoggerFactory.getLogger(AggregationBolt.class);
+ private Config config;
+ private OutputCollector collector;
+ private MetricsAggregateContainer metricsAggregateContainer;
+
+ public AggregationBolt(Config config, MetricsAggregateContainer metricsAggregateContainer) {
+ this.config = config;
+ this.metricsAggregateContainer = metricsAggregateContainer;
+ }
+
+ @Override
+ public void prepare(Map map, TopologyContext topologyContext, OutputCollector outputCollector) {
+ AggregationConfig.getInstance(config);
+ this.collector = outputCollector;
+ }
+
+ @Override
+ public void execute(Tuple tuple) {
+ Long startTime = tuple.getLongByField("startTime");
+ LOG.info("get startTime {}", startTime);
+ Long endTime = startTime + AggregationConfig.get().getJobExtractorConfig().aggregationDuration * 1000;
+
+ if (metricsAggregateContainer.aggregate(startTime, endTime)) {
+ collector.ack(tuple);
+ LOG.info("succeed startTime {}", startTime);
+ } else {
+ collector.fail(tuple);
+ LOG.warn("failed startTime {}", startTime);
+ }
+ }
+
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) {
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/2c30276c/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/storm/AggregationSpout.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/storm/AggregationSpout.java b/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/storm/AggregationSpout.java
new file mode 100644
index 0000000..1f2cd46
--- /dev/null
+++ b/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/storm/AggregationSpout.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.eagle.jpm.aggregation.storm;
+
+import org.apache.eagle.jpm.aggregation.AggregationConfig;
+import org.apache.eagle.jpm.aggregation.common.MetricsAggregateContainer;
+import org.apache.eagle.jpm.aggregation.state.AggregationTimeManager;
+import org.apache.eagle.jpm.util.Utils;
+import backtype.storm.spout.SpoutOutputCollector;
+import backtype.storm.task.TopologyContext;
+import backtype.storm.topology.OutputFieldsDeclarer;
+import backtype.storm.topology.base.BaseRichSpout;
+import backtype.storm.tuple.Fields;
+import backtype.storm.tuple.Values;
+import com.typesafe.config.Config;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+public class AggregationSpout extends BaseRichSpout {
+ private static final Logger LOG = LoggerFactory.getLogger(AggregationSpout.class);
+ private static final Long MAX_WAIT_TIME = 12 * 60 * 60000L;//12 hours
+ private static final Long MAX_SAFE_TIME = 6 * 60 * 60000L;//6 hours
+
+ private Config config;
+ MetricsAggregateContainer jobProcessTime;
+
+ private SpoutOutputCollector collector;
+ private Set<Long> processStartTime;
+ private Long lastUpdateTime;
+
+ public AggregationSpout(Config config, MetricsAggregateContainer jobProcessTime) {
+ this.config = config;
+ this.jobProcessTime = jobProcessTime;
+ this.processStartTime = new HashSet<>();
+ }
+
+ @Override
+ public void open(Map conf, TopologyContext context, final SpoutOutputCollector collector) {
+ AggregationConfig.getInstance(config);
+ AggregationTimeManager.instance().init(AggregationConfig.get().getZkStateConfig());
+ this.collector = collector;
+ }
+
+ @Override
+ public void nextTuple() {
+ Utils.sleep(5);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("start to run");
+ }
+ try {
+ //0, check processStartTime size
+ if (this.processStartTime.size() > 0) {
+ //bolts are processing
+ return;
+ }
+
+ long currentJobTime = System.currentTimeMillis();//this.jobProcessTime.fetchLatestJobProcessTime();
+ //1, get last updateTime;
+ lastUpdateTime = AggregationTimeManager.instance().readLastFinishTime();
+ if (lastUpdateTime == 0L) {
+ //init state, just set to currentTime - 18 hours
+ lastUpdateTime = (currentJobTime - (MAX_SAFE_TIME + MAX_WAIT_TIME)) / 3600000 * 3600000;
+ }
+
+ if (currentJobTime - lastUpdateTime < MAX_WAIT_TIME) {
+ LOG.info("wait for max wait time, remaining {}ms", MAX_WAIT_TIME - currentJobTime + lastUpdateTime);
+ return;
+ }
+
+ for (Long startTime = lastUpdateTime; startTime < lastUpdateTime + MAX_SAFE_TIME;) {
+ collector.emit(new Values(startTime), startTime);
+ this.processStartTime.add(startTime);
+ startTime += AggregationConfig.get().getJobExtractorConfig().aggregationDuration * 1000;
+ }
+ } catch (Exception e) {
+ LOG.warn("{}", e);
+ }
+ }
+
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+ declarer.declare(new Fields("startTime"));
+ }
+
+ @Override
+ public void ack(Object messageId) {
+ //remove from processStartTime, if size == 0, then update lastUpdateTime
+ Long startTime = (Long)messageId;
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("succeed startTime {}", startTime);
+ }
+
+ this.processStartTime.remove(startTime);
+ if (this.processStartTime.size() == 0) {
+ while (true) {
+ try {
+ LOG.info("all have finished, update lastUpdateTime to {}", this.lastUpdateTime + MAX_SAFE_TIME);
+ AggregationTimeManager.instance().updateLastFinishTime(this.lastUpdateTime + MAX_SAFE_TIME);
+ break;
+ } catch (Exception e) {
+ Utils.sleep(3);
+ }
+ }
+ }
+ }
+
+ @Override
+ public void fail(Object messageId) {
+ Long startTime = (Long)messageId;
+ LOG.warn("failed startTime {}", startTime);
+ this.collector.emit(new Values(startTime), startTime);
+ }
+
+ @Override
+ public void deactivate() {
+ }
+
+ @Override
+ public void close() {
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/2c30276c/eagle-jpm/eagle-jpm-aggregation/src/main/resources/application.conf
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-aggregation/src/main/resources/application.conf b/eagle-jpm/eagle-jpm-aggregation/src/main/resources/application.conf
new file mode 100644
index 0000000..94e06b3
--- /dev/null
+++ b/eagle-jpm/eagle-jpm-aggregation/src/main/resources/application.conf
@@ -0,0 +1,82 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{
+ "envContextConfig" : {
+ "env" : "cluster",
+ "topologyName" : "mrHistoryAggregation",
+ "stormConfigFile" : "storm.yaml",
+ "parallelismConfig" : {
+ "mrHistoryAggregationSpout" : 1,
+ "mrHistoryAggregationBolt" : 6
+ },
+ "tasks" : {
+ "mrHistoryAggregationSpout" : 1,
+ "mrHistoryAggregationBolt" : 6
+ }
+ },
+
+ "jobExtractorConfig" : {
+ "site" : "sandbox",
+ "readTimeOutSeconds" : 10,
+ "aggregationDuration" : 3600 #seconds, each bolt process metrics from [start, start + aggregationDuration]
+ },
+
+ "zkStateConfig" : {
+ "zkQuorum" : "sandbox.hortonworks.com:2181",
+ "zkPort" : "2181",
+ "zkRoot" : "/aggregation/mr/sandbox",
+ "zkSessionTimeoutMs" : 15000,
+ "zkRetryTimes" : 3,
+ "zkRetryInterval" : 20000,
+ },
+
+ "eagleProps" : {
+ "mailHost" : "abc.com",
+ "mailDebug" : "true",
+ "eagleService": {
+ "host": "sandbox.hortonworks.com:2181",
+ "port": 9099,
+ "username": "admin",
+ "password": "secret"
+ }
+ },
+
+ "aggregate" : {
+ "counters" : {
+ "metrics": [
+ "cpu_milliseconds",
+ "file_bytes_read",
+ "file_bytes_written",
+ "hdfs_bytes_read",
+ "hdfs_bytes_written",
+ "hdfs_read_ops",
+ "hdfs_write_ops"
+ ],
+
+ "groupByColumns": [
+ "site, jobType",
+ "site, user",
+ "site"
+ ]
+ }
+ },
+
+ "appId":"mrHistoryAggregation",
+ "mode":"LOCAL",
+ "workers" : 3,
+ application.storm.nimbusHost=localhost
+ topology.message.timeout.secs=1800
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/2c30276c/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/Constants.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/Constants.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/Constants.java
index 5a60ee3..c9bb387 100644
--- a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/Constants.java
+++ b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/Constants.java
@@ -177,6 +177,7 @@ public class Constants {
public static final String REDUCE_COUNT_RATIO = "reduce.count.ratio";
public static final String JOB_LEVEL = "job";
public static final String TASK_LEVEL = "task";
+ public static final String USER_LEVEL = "user";
public static final String JOB_COUNT_PER_DAY = "hadoop.job.day.count";
public static final String HADOOP_HISTORY_TOTAL_METRIC_FORMAT = "hadoop.%s.history.%s";
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/2c30276c/eagle-jpm/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-jpm/pom.xml b/eagle-jpm/pom.xml
index a96604d..702c89d 100644
--- a/eagle-jpm/pom.xml
+++ b/eagle-jpm/pom.xml
@@ -42,6 +42,7 @@
<module>eagle-jpm-mr-running</module>
<module>eagle-jpm-app</module>
<module>eagle-jpm-service</module>
+ <module>eagle-jpm-aggregation</module>
</modules>
<dependencies>