You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@eagle.apache.org by mw...@apache.org on 2016/07/25 09:37:02 UTC
[25/47] incubator-eagle git commit: [EAGLE-305] 306,
308 & 313: Fix several application manager minor bugs & update the doc
[EAGLE-305] 306, 308 & 313: Fix several application manager minor bugs & update the doc
https://issues.apache.org/jira/browse/EAGLE-305
https://issues.apache.org/jira/browse/EAGLE-306
https://issues.apache.org/jira/browse/EAGLE-308
https://issues.apache.org/jira/browse/EAGLE-313
Author: Zhao, Qingwen <qi...@ebay.com>
Author: jiljiang <ji...@ebay.com>
Author: Daniel <Da...@dataguise.com>
Closes #206 from qingwen220/EAGLE-305.
Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/ffd5fddd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/ffd5fddd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/ffd5fddd
Branch: refs/heads/master
Commit: ffd5fddd357effae7aaf2384a2a9b6d479f9c930
Parents: fd39e6e
Author: Zhao, Qingwen <qi...@ebay.com>
Authored: Mon May 30 14:53:03 2016 +0800
Committer: Hao Chen <ha...@apache.org>
Committed: Mon May 30 14:53:03 2016 +0800
----------------------------------------------------------------------
.../src/main/bin/eagle-topology-init.sh | 2 +
.../ApplicationManagementResource.java | 16 +++--
.../application/dao/ApplicationManagerDAO.java | 1 +
.../dao/ApplicationManagerDaoImpl.java | 18 +++++-
.../impl/StormExecutionPlatform.scala | 2 +-
.../tutorial/application_manager_tutorial.md | 68 +++++++++++---------
.../app/public/feature/topology/controller.js | 2 +-
.../src/main/webapp/app/public/js/app.config.js | 3 +-
8 files changed, 73 insertions(+), 39 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/ffd5fddd/eagle-assembly/src/main/bin/eagle-topology-init.sh
----------------------------------------------------------------------
diff --git a/eagle-assembly/src/main/bin/eagle-topology-init.sh b/eagle-assembly/src/main/bin/eagle-topology-init.sh
index be38a0d..dc9dd59 100755
--- a/eagle-assembly/src/main/bin/eagle-topology-init.sh
+++ b/eagle-assembly/src/main/bin/eagle-topology-init.sh
@@ -64,6 +64,8 @@ curl -silent -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Conten
curl -silent -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=FeatureDescService" -d '[{"prefix":"eagleFeatureDesc","tags":{"feature":"metrics"},"description":"Metrics dashboard","version":"v0.3.0"}]'
+curl -silent -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=FeatureDescService" -d '[{"prefix":"eagleFeatureDesc","tags":{"feature":"topology"},"description":"Application topology management feature","version":"v0.4.0"}]'
+
## AlertStreamService: alert streams generated from data source
echo ""
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/ffd5fddd/eagle-core/eagle-application-management/eagle-application-service/src/main/java/org/apache/eagle/service/application/ApplicationManagementResource.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-application-management/eagle-application-service/src/main/java/org/apache/eagle/service/application/ApplicationManagementResource.java b/eagle-core/eagle-application-management/eagle-application-service/src/main/java/org/apache/eagle/service/application/ApplicationManagementResource.java
index 779fbe5..6e4521d 100644
--- a/eagle-core/eagle-application-management/eagle-application-service/src/main/java/org/apache/eagle/service/application/ApplicationManagementResource.java
+++ b/eagle-core/eagle-application-management/eagle-application-service/src/main/java/org/apache/eagle/service/application/ApplicationManagementResource.java
@@ -21,6 +21,7 @@ package org.apache.eagle.service.application;
import org.apache.eagle.log.base.taggedlog.TaggedLogAPIEntity;
import org.apache.eagle.log.entity.GenericServiceAPIResponseEntity;
+import org.apache.eagle.policy.common.Constants;
import org.apache.eagle.service.application.dao.ApplicationManagerDAO;
import org.apache.eagle.service.application.dao.ApplicationManagerDaoImpl;
import org.apache.eagle.service.application.entity.TopologyExecutionStatus;
@@ -28,10 +29,7 @@ import org.apache.eagle.service.application.entity.TopologyOperationEntity;
import org.codehaus.jackson.map.ObjectMapper;
import org.codehaus.jackson.map.type.TypeFactory;
-import javax.ws.rs.Consumes;
-import javax.ws.rs.POST;
-import javax.ws.rs.Path;
-import javax.ws.rs.Produces;
+import javax.ws.rs.*;
import javax.ws.rs.core.MediaType;
import java.io.IOException;
import java.io.InputStream;
@@ -98,4 +96,14 @@ public class ApplicationManagementResource {
ObjectMapper objectMapper = new ObjectMapper();
return objectMapper.readValue(inputStream, TypeFactory.defaultInstance().constructCollectionType(LinkedList.class, TopologyOperationEntity.class));
}
+
+ @Path("topology")
+ @DELETE
+ @Consumes(MediaType.APPLICATION_JSON)
+ @Produces(MediaType.APPLICATION_JSON)
+ public GenericServiceAPIResponseEntity deleteTopology(@QueryParam("topology") String topology) {
+ return dao.deleteTopology(topology);
+ }
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/ffd5fddd/eagle-core/eagle-application-management/eagle-application-service/src/main/java/org/apache/eagle/service/application/dao/ApplicationManagerDAO.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-application-management/eagle-application-service/src/main/java/org/apache/eagle/service/application/dao/ApplicationManagerDAO.java b/eagle-core/eagle-application-management/eagle-application-service/src/main/java/org/apache/eagle/service/application/dao/ApplicationManagerDAO.java
index 189370b..dfa261b 100644
--- a/eagle-core/eagle-application-management/eagle-application-service/src/main/java/org/apache/eagle/service/application/dao/ApplicationManagerDAO.java
+++ b/eagle-core/eagle-application-management/eagle-application-service/src/main/java/org/apache/eagle/service/application/dao/ApplicationManagerDAO.java
@@ -29,4 +29,5 @@ public interface ApplicationManagerDAO {
String loadTopologyExecutionStatus(String site, String application, String topology);
int loadTopologyOperationsInRunning(String site, String application, String topology) throws Exception;
GenericServiceAPIResponseEntity createOperation(List<TopologyOperationEntity> entities) throws Exception;
+ GenericServiceAPIResponseEntity deleteTopology(String topology);
}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/ffd5fddd/eagle-core/eagle-application-management/eagle-application-service/src/main/java/org/apache/eagle/service/application/dao/ApplicationManagerDaoImpl.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-application-management/eagle-application-service/src/main/java/org/apache/eagle/service/application/dao/ApplicationManagerDaoImpl.java b/eagle-core/eagle-application-management/eagle-application-service/src/main/java/org/apache/eagle/service/application/dao/ApplicationManagerDaoImpl.java
index e21f624..4881cf4 100644
--- a/eagle-core/eagle-application-management/eagle-application-service/src/main/java/org/apache/eagle/service/application/dao/ApplicationManagerDaoImpl.java
+++ b/eagle-core/eagle-application-management/eagle-application-service/src/main/java/org/apache/eagle/service/application/dao/ApplicationManagerDaoImpl.java
@@ -18,11 +18,9 @@
package org.apache.eagle.service.application.dao;
-
import org.apache.eagle.log.entity.GenericServiceAPIResponseEntity;
import org.apache.eagle.policy.common.Constants;
import org.apache.eagle.service.application.entity.TopologyExecutionEntity;
-import org.apache.eagle.service.application.entity.TopologyExecutionStatus;
import org.apache.eagle.service.application.entity.TopologyOperationEntity;
import org.apache.eagle.service.generic.GenericEntityServiceResource;
import org.slf4j.Logger;
@@ -73,5 +71,21 @@ public class ApplicationManagerDaoImpl implements ApplicationManagerDAO {
return response;
}
+ @Override
+ public GenericServiceAPIResponseEntity deleteTopology(String topology) {
+ String topologyQuery = Constants.TOPOLOGY_DESCRIPTION_SERVICE_ENDPOINT_NAME+ "[@topology=\"" + topology + "\"]{*}";
+ String executionQuery = Constants.TOPOLOGY_EXECUTION_SERVICE_ENDPOINT_NAME + "[@topology=\"" + topology + "\"]{*}";
+ int pageSize = Integer.MAX_VALUE;
+
+ GenericServiceAPIResponseEntity response = resource.deleteByQuery(topologyQuery, null, null, pageSize, null, false, false, 0L, 0, true, 0, null, false);
+ if(response.isSuccess()) {
+ response = resource.deleteByQuery(executionQuery, null, null, pageSize, null, false, false, 0L, 0, true, 0, null, false);
+ }
+ if(!response.isSuccess()) {
+ LOG.error(response.getException());
+ }
+ return response;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/ffd5fddd/eagle-core/eagle-application-management/eagle-stream-application-manager/src/main/scala/org/apache/eagle/stream/application/impl/StormExecutionPlatform.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-application-management/eagle-stream-application-manager/src/main/scala/org/apache/eagle/stream/application/impl/StormExecutionPlatform.scala b/eagle-core/eagle-application-management/eagle-stream-application-manager/src/main/scala/org/apache/eagle/stream/application/impl/StormExecutionPlatform.scala
index 5b1bb48..af4cafa 100644
--- a/eagle-core/eagle-application-management/eagle-stream-application-manager/src/main/scala/org/apache/eagle/stream/application/impl/StormExecutionPlatform.scala
+++ b/eagle-core/eagle-application-management/eagle-stream-application-manager/src/main/scala/org/apache/eagle/stream/application/impl/StormExecutionPlatform.scala
@@ -166,7 +166,7 @@ class StormExecutionPlatform extends ExecutionPlatform {
case None =>
topologyExecution.setStatus(TopologyExecutionStatus.STOPPED)
topologyExecution.setUrl("")
- topologyExecution.setDescription(s"Fail to find topology: $name")
+ topologyExecution.setDescription("")
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/ffd5fddd/eagle-docs/tutorial/application_manager_tutorial.md
----------------------------------------------------------------------
diff --git a/eagle-docs/tutorial/application_manager_tutorial.md b/eagle-docs/tutorial/application_manager_tutorial.md
index 4690428..4eccfa4 100644
--- a/eagle-docs/tutorial/application_manager_tutorial.md
+++ b/eagle-docs/tutorial/application_manager_tutorial.md
@@ -15,55 +15,63 @@ See the License for the specific language governing permissions and
limitations under the License.
-->
-> Application manager aims to manage topology status on EAGLE UI. Users can easily start/start topologies remotely or locally without shell commands.
+> Application manager aims to manage topology status on EAGLE UI. Users can easily start/start topologies remotely or locally without any shell commands. At the same, it should be capable to sync the latest status of topologies on the execution platform (e.g., storm cluster).
-This tutorial will go through all parts of appliaction manager and then give an example to use it.
+This tutorial will go through all parts of application manager and then give an example to use it.
### Design
-Briefly speaking, Application manager consists of a deamon scheduler and the application manager. The scheduler loads user operations(start/stop), and the applicaiton manager is responsible to execute these operations. For more details, please refer to [here](https://cwiki.apache.org/confluence/display/EAG/Application+Management)
+Application manager consists of a daemon scheduler and an execution module. The scheduler periodically loads user operations(start/stop) from database, and the execution module executes these operations. For more details, please refer to [here](https://cwiki.apache.org/confluence/display/EAG/Application+Management)
-### Manual
+### Configurations
+The configuration file `eagle-scheduler.conf` defines scheduler parameters, execution platform settings and parts of default topology configuration.
-#### Step 1: configure the scheduler
-The configuration file `eagle-scheduler.conf` defines scheduler parameters, topology execution platform settings and parts of topology settings. Here are some important ones:
+* **Scheduler properties**
-* `envContextConfig.env`
+ **appCommandLoaderEnabled**: enable application manager. **TODO**: change it to true. <br />
+ **appCommandLoaderIntervalSecs**: defines the interval of the scheduler loads commands. Default value is 1 second. <br />
+ **appHealthCheckIntervalSecs**: define the interval of health check, which tries to sync the topology execution status to Eagle. <br /><br />
- application execution platform. Default value is storm
+* **Execution platform properties**
-* `envContextConfig.url`
-
- execution platform http url. Default is "http://sandbox.hortonworks.com:8744"
+ **envContextConfig.env**: application execution platform. Default value is storm. <br />
+ **envContextConfig.url**: execution platform http url. Default is "http://sandbox.hortonworks.com:8744". <br />
+ **envContextConfig.nimbusHost**: storm nimbus host. Default is "sandbox.hortonworks.com". <br />
+ **envContextConfig.nimbusThriftPort**: default is 6627.
+ **envContextConfig.jarFile**: storm fat jar path. **TODO**: change "/dir-to-jar/eagle-topology-0.3.0-incubating-assembly.jar" to your own jar path. <br /><br />
+
+* **Topology default properties**
+
+ Some default topology properties are defined here.
+
+Note: these configurations can be overridden in the topology configurations, which is shown below. The only difference is to add a prefix `.app`. For example, 'app.envContextConfig.jarFile' is to override 'envContextConfig.jarFile' in eagle-schedule.conf
-* `envContextConfig.nimbusHost`
- Storm nimbus host. Default is "sandbox.hortonworks.com"
-
-* `envContextConfig.nimbusThriftPort`
-
- Default is 6627
-
-* `envContextConfig.jarFile`
+#### Manual
+
+1. Editing eagle-scheduler.conf, and start Eagle service
- Storm fat jar path. The only setting users must specify "/dir-to-jar/eagle-topology-0.3.0-incubating-assembly.jar"
+ # enable application manager
+ appCommandLoaderEnabled=true
+ # provide jar path
+ envContextConfig.jarFile=/path-to-jar
-After the configuration is ready, start Eagle service `bin/eagle-service.sh start`.
-
-#### Step 2: add topologies on UI
-1. First of all, go to admin page
+ For more configurations, please back to **Configuration**. <br />
+ After the configuration is ready, start Eagle service `bin/eagle-service.sh start`.
+
+2. Go to admin page
-2. Go to management page, and create a topology description. There are three required fields
+3. Go to management page, and create a topology description. There are three required fields
* name: topology name
* type: topology type [CLASS, DYNAMIC]
- * execution entry: either the class which implement interface TopologyExecutable or eagle [DSL](https://github.com/apache/incubator-eagle/blob/master/eagle-assembly/src/main/conf/sandbox-hadoopjmx-pipeline.conf) based topology definition
+ * execution entry: either the class which implements interface TopologyExecutable or eagle [DSL](https://github.com/apache/incubator-eagle/blob/master/eagle-assembly/src/main/conf/sandbox-hadoopjmx-pipeline.conf) based topology definition
-3. Back to monitoring page, and choose the site/application to deploy the topology
+4. Back to monitoring page, and choose the site/application to deploy the topology
-4. Go to site page, and edit site->application and add some new configurations. Blow are some example configurations for [site=sandbox, applicatoin=hbaseSecurityLog]
+5. Go to site page, and edit site->application and add some new configurations. Blow are some example configurations for [site=sandbox, applicatoin=hbaseSecurityLog]
`These configurations have a higher priority than those in eagle-scheduler.conf`
classification.hbase.zookeeper.property.clientPort=2181
@@ -97,11 +105,11 @@ After the configuration is ready, start Eagle service `bin/eagle-service.sh star
-5. Go to monitoring page, and start topologies
+6. Go to monitoring page, and start topologies
-6. stop topologies on monitoring page
+7. stop topologies on monitoring page
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/ffd5fddd/eagle-webservice/src/main/webapp/app/public/feature/topology/controller.js
----------------------------------------------------------------------
diff --git a/eagle-webservice/src/main/webapp/app/public/feature/topology/controller.js b/eagle-webservice/src/main/webapp/app/public/feature/topology/controller.js
index 5504a77..94886c9 100644
--- a/eagle-webservice/src/main/webapp/app/public/feature/topology/controller.js
+++ b/eagle-webservice/src/main/webapp/app/public/feature/topology/controller.js
@@ -247,7 +247,7 @@
$scope.deleteTopology = function (topology) {
UI.deleteConfirm(topology.tags.topology).then(null, null, function(holder) {
- Entities.deleteEntities("TopologyDescriptionService", {topology: topology.tags.topology})._promise.then(function() {
+ Entities.delete("TopologyDescriptionService", {topology: topology.tags.topology})._promise.then(function() {
holder.closeFunc();
common.array.remove(topology, $scope.topologyList);
});
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/ffd5fddd/eagle-webservice/src/main/webapp/app/public/js/app.config.js
----------------------------------------------------------------------
diff --git a/eagle-webservice/src/main/webapp/app/public/js/app.config.js b/eagle-webservice/src/main/webapp/app/public/js/app.config.js
index bd6d62c..ec30ccd 100644
--- a/eagle-webservice/src/main/webapp/app/public/js/app.config.js
+++ b/eagle-webservice/src/main/webapp/app/public/js/app.config.js
@@ -43,7 +43,8 @@
DELETE_HOOK: {
FeatureDescService: 'rest/module/feature?feature=${feature}',
ApplicationDescService: 'rest/module/application?application=${application}',
- SiteDescService: 'rest/module/site?site=${site}'
+ SiteDescService: 'rest/module/site?site=${site}',
+ TopologyDescriptionService: 'rest/app/topology?topology=${topology}'
},
UPDATE_HOOK: {
SiteDescService: 'rest/module/siteApplication'