You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@eagle.apache.org by qi...@apache.org on 2016/10/20 11:13:16 UTC
incubator-eagle git commit: [EAGLE-646] AlertUnitTopology does not
rebuild the scheduler state after restarting
Repository: incubator-eagle
Updated Branches:
refs/heads/master d76fa311b -> 823c66dae
[EAGLE-646] AlertUnitTopology does not rebuild the scheduler state after restarting
https://issues.apache.org/jira/browse/EAGLE-646
1. Fix a bug that alertUnitTopology does not rebuild the scheduler state after restarting
2. Fix topology assembly dependency bug
Author: Zhao, Qingwen <qi...@apache.org>
Closes #538 from qingwen220/EAGLE-646.
Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/823c66da
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/823c66da
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/823c66da
Branch: refs/heads/master
Commit: 823c66dae5accefa9631b03bb6e3ab607a12bd55
Parents: d76fa31
Author: Zhao, Qingwen <qi...@apache.org>
Authored: Thu Oct 20 19:13:09 2016 +0800
Committer: Zhao, Qingwen <qi...@apache.org>
Committed: Thu Oct 20 19:13:09 2016 +0800
----------------------------------------------------------------------
.../eagle-alert-parent/eagle-alert-app/pom.xml | 5 ++++
.../alert/app/AlertUnitTopologyAppListener.java | 7 +++++
.../impl/storm/kafka/KafkaSpoutProvider.java | 4 ++-
eagle-topology-assembly/pom.xml | 29 +++-----------------
...org.apache.eagle.app.spi.ApplicationProvider | 7 ++++-
5 files changed, 25 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/823c66da/eagle-core/eagle-alert-parent/eagle-alert-app/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert-app/pom.xml b/eagle-core/eagle-alert-parent/eagle-alert-app/pom.xml
index 768709f..74f82d0 100644
--- a/eagle-core/eagle-alert-parent/eagle-alert-app/pom.xml
+++ b/eagle-core/eagle-alert-parent/eagle-alert-app/pom.xml
@@ -31,5 +31,10 @@
<artifactId>eagle-app-base</artifactId>
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.eagle</groupId>
+ <artifactId>alert-coordinator</artifactId>
+ <version>${project.version}</version>
+ </dependency>
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/823c66da/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/java/org/apache/eagle/alert/app/AlertUnitTopologyAppListener.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/java/org/apache/eagle/alert/app/AlertUnitTopologyAppListener.java b/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/java/org/apache/eagle/alert/app/AlertUnitTopologyAppListener.java
index fd3f9d2..0b19424 100644
--- a/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/java/org/apache/eagle/alert/app/AlertUnitTopologyAppListener.java
+++ b/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/java/org/apache/eagle/alert/app/AlertUnitTopologyAppListener.java
@@ -18,6 +18,7 @@
package org.apache.eagle.alert.app;
import org.apache.eagle.alert.coordination.model.internal.Topology;
+import org.apache.eagle.alert.coordinator.resource.CoordinatorResource;
import org.apache.eagle.alert.engine.runner.UnitTopologyRunner;
import org.apache.eagle.alert.metadata.IMetadataDao;
import org.apache.eagle.alert.metadata.resource.OpResult;
@@ -33,6 +34,7 @@ class AlertUnitTopologyAppListener implements ApplicationListener {
private static final Logger LOG = LoggerFactory.getLogger(AlertUnitTopologyAppListener.class);
@Inject private IMetadataDao metadataDao;
+ @Inject private CoordinatorResource coordinatorResource;
private ApplicationEntity applicationEntity;
@@ -73,6 +75,11 @@ class AlertUnitTopologyAppListener implements ApplicationListener {
LOG.error(result.message);
throw new IllegalStateException(result.message);
}
+ try {
+ coordinatorResource.build();
+ } catch (Exception e) {
+ LOG.error(e.getMessage(), e);
+ }
}
private void removeTopologyMetadata() {
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/823c66da/eagle-core/eagle-data-process/src/main/java/org/apache/eagle/dataproc/impl/storm/kafka/KafkaSpoutProvider.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/src/main/java/org/apache/eagle/dataproc/impl/storm/kafka/KafkaSpoutProvider.java b/eagle-core/eagle-data-process/src/main/java/org/apache/eagle/dataproc/impl/storm/kafka/KafkaSpoutProvider.java
index 7454cc2..e8c40ae 100644
--- a/eagle-core/eagle-data-process/src/main/java/org/apache/eagle/dataproc/impl/storm/kafka/KafkaSpoutProvider.java
+++ b/eagle-core/eagle-data-process/src/main/java/org/apache/eagle/dataproc/impl/storm/kafka/KafkaSpoutProvider.java
@@ -60,7 +60,7 @@ public class KafkaSpoutProvider implements StormSpoutProvider {
// Kafka broker zk connection
String zkConnString = context.getString("zkConnection");
// Kafka fetch size
- int fetchSize = context.hasPath("fetchSize") ? context.getInt("fetchSize") : 1048586;
+ int fetchSize = context.hasPath("fetchSize") ? context.getInt("fetchSize") : 1048576;
LOG.info(String.format("Use topic : %s, zkConnection : %s , fetchSize : %d", topic, zkConnString, fetchSize));
/*
@@ -97,6 +97,8 @@ public class KafkaSpoutProvider implements StormSpoutProvider {
spoutConfig.stateUpdateIntervalMs = context.hasPath("transactionStateUpdateMS") ? context.getLong("transactionStateUpdateMS") : 2000;
// Kafka fetch size
spoutConfig.fetchSizeBytes = fetchSize;
+ spoutConfig.startOffsetTime = kafka.api.OffsetRequest.LatestTime();
+
// "startOffsetTime" is for test usage, prod should not use this
if (context.hasPath("startOffsetTime")) {
spoutConfig.startOffsetTime = context.getInt("startOffsetTime");
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/823c66da/eagle-topology-assembly/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-topology-assembly/pom.xml b/eagle-topology-assembly/pom.xml
index 4ea2b0a..a9dddb0 100644
--- a/eagle-topology-assembly/pom.xml
+++ b/eagle-topology-assembly/pom.xml
@@ -45,33 +45,12 @@
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-server</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
</exclusions>
</dependency>
- <dependency>
- <groupId>org.apache.eagle</groupId>
- <artifactId>eagle-jpm-mr-history</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.eagle</groupId>
- <artifactId>eagle-jpm-mr-running</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.eagle</groupId>
- <artifactId>eagle-jpm-web</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.eagle</groupId>
- <artifactId>eagle-jpm-aggregation</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.eagle</groupId>
- <artifactId>eagle-hadoop-queue</artifactId>
- <version>${project.version}</version>
- </dependency>
</dependencies>
<profiles>
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/823c66da/eagle-topology-assembly/src/resources/META-INF/services/org.apache.eagle.app.spi.ApplicationProvider
----------------------------------------------------------------------
diff --git a/eagle-topology-assembly/src/resources/META-INF/services/org.apache.eagle.app.spi.ApplicationProvider b/eagle-topology-assembly/src/resources/META-INF/services/org.apache.eagle.app.spi.ApplicationProvider
index 8d35f31..86e5e0a 100644
--- a/eagle-topology-assembly/src/resources/META-INF/services/org.apache.eagle.app.spi.ApplicationProvider
+++ b/eagle-topology-assembly/src/resources/META-INF/services/org.apache.eagle.app.spi.ApplicationProvider
@@ -13,8 +13,13 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+org.apache.eagle.alert.app.AlertUnitTopologyAppProvider
+
+## DAM
org.apache.eagle.security.hbase.HBaseAuditLogAppProvider
-org.apache.eagle.app.example.ExampleApplicationProvider
+org.apache.eagle.security.auditlog.HdfsAuditLogAppProvider
+
+## JPM
org.apache.eagle.app.jpm.JPMWebApplicationProvider
org.apache.eagle.jpm.mr.history.MRHistoryJobApplicationProvider
org.apache.eagle.jpm.mr.running.MRRunningJobApplicationProvider