You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@eagle.apache.org by yo...@apache.org on 2016/08/15 19:46:42 UTC
[2/2] incubator-eagle git commit: EAGLE-468 clean up
eagle-application-service clean up eagle-application-service
EAGLE-468 clean up eagle-application-service
clean up eagle-application-service
https://issues.apache.org/jira/browse/EAGLE-468
Author: @yonzhang <yo...@apache.org>
Reviewer: @qingwen220 <qi...@ebay.com>
Closes: #350
Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/46afec39
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/46afec39
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/46afec39
Branch: refs/heads/develop
Commit: 46afec3953405592d9a74f1b051877fe07c14835
Parents: 18ae3bb
Author: yonzhang <yo...@gmail.com>
Authored: Mon Aug 15 12:50:33 2016 -0700
Committer: yonzhang <yo...@gmail.com>
Committed: Mon Aug 15 12:50:33 2016 -0700
----------------------------------------------------------------------
.../eagle-app/eagle-application-service/pom.xml | 54 ---
.../application/AppManagerConstants.java | 43 ---
.../ApplicationManagementResource.java | 108 ------
.../application/dao/ApplicationManagerDAO.java | 33 --
.../dao/ApplicationManagerDaoImpl.java | 91 -----
.../entity/ApplicationEntityRepo.java | 30 --
.../entity/TopologyDescriptionEntity.java | 104 ------
.../entity/TopologyExecutionEntity.java | 132 -------
.../entity/TopologyExecutionStatus.java | 38 --
.../entity/TopologyOperationEntity.java | 105 ------
eagle-core/eagle-app/pom.xml | 1 -
eagle-hadoop-metric/pom.xml | 14 +-
.../hadoop/metric/HadoopJmxApplication.java | 74 ----
.../eagle/hadoop/metric/JsonParserBolt.java | 62 ----
.../src/main/resources/application.conf | 51 ---
.../resources/capacityused-policy-import.sh | 51 ---
.../src/main/resources/datasources.json | 19 +
.../src/main/resources/eagle-env.sh | 44 ---
.../src/main/resources/hadoop-metric-init.sh | 180 ----------
.../src/main/resources/hadoopjmx.yaml | 18 -
.../src/main/resources/hastate-policy-import.sh | 51 ---
.../lastcheckpointtime-policy-import.sh | 51 ---
.../src/main/resources/log4j.properties | 35 --
.../resources/missingblock-policy-import.sh | 51 ---
.../main/resources/namenodelag-policy-import.sh | 49 ---
.../main/resources/nodecount-policy-import.sh | 51 ---
.../src/main/resources/policies.json | 169 +++++++++
.../src/main/resources/publishments.json | 33 ++
.../resources/safemodecheck-policy-import.sh | 51 ---
.../sanbox/capacityused-policy-import.sh | 51 ---
.../resources/sanbox/hastate-policy-import.sh | 51 ---
.../sanbox/lastcheckpointtime-policy-import.sh | 51 ---
.../sanbox/missingblock-policy-import.sh | 51 ---
.../sanbox/namenodehanoactive-policy-import.sh | 52 ---
...nodehawithmorethanoneactive-policy-import.sh | 53 ---
.../sanbox/namenodelag-policy-import.sh | 49 ---
.../resources/sanbox/nodecount-policy-import.sh | 51 ---
.../resourcemanagerhanoactive-policy-import.sh | 53 ---
...agerhawithmorethanoneactive-policy-import.sh | 54 ---
.../sanbox/safemodecheck-policy-import.sh | 51 ---
.../src/main/resources/streamdefinitions.json | 47 +++
.../metric/HadoopJmxMetricDeserializerTest.java | 36 --
.../hadoop/metric/TestHadoopMetricSiddhiQL.java | 354 -------------------
.../src/test/resources/cassandra.json | 13 -
44 files changed, 271 insertions(+), 2539 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/46afec39/eagle-core/eagle-app/eagle-application-service/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-app/eagle-application-service/pom.xml b/eagle-core/eagle-app/eagle-application-service/pom.xml
deleted file mode 100644
index 3e3e8f4..0000000
--- a/eagle-core/eagle-app/eagle-application-service/pom.xml
+++ /dev/null
@@ -1,54 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one or more
- ~ contributor license agreements. See the NOTICE file distributed with
- ~ this work for additional information regarding copyright ownership.
- ~ The ASF licenses this file to You under the Apache License, Version 2.0
- ~ (the "License"); you may not use this file except in compliance with
- ~ the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- ~
- -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>eagle-app-parent</artifactId>
- <groupId>org.apache.eagle</groupId>
- <version>0.5.0-incubating-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>eagle-application-service</artifactId>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.eagle</groupId>
- <artifactId>eagle-policy-base</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.eagle</groupId>
- <artifactId>eagle-service-base</artifactId>
- <version>${project.version}</version>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-plugin</artifactId>
- <configuration>
- <skipTests>true</skipTests>
- </configuration>
- </plugin>
- </plugins>
- </build>
-</project>
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/46afec39/eagle-core/eagle-app/eagle-application-service/src/main/java/org/apache/eagle/service/application/AppManagerConstants.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-app/eagle-application-service/src/main/java/org/apache/eagle/service/application/AppManagerConstants.java b/eagle-core/eagle-app/eagle-application-service/src/main/java/org/apache/eagle/service/application/AppManagerConstants.java
deleted file mode 100644
index 3aa3579..0000000
--- a/eagle-core/eagle-app/eagle-application-service/src/main/java/org/apache/eagle/service/application/AppManagerConstants.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.eagle.service.application;
-
-
-public class AppManagerConstants {
- public final static String SITE_TAG = "site";
- public final static String APPLICATION_TAG = "application";
- public final static String OPERATION_TAG = "operation";
- public final static String OPERATION_ID_TAG = "operationID";
- public final static String TOPOLOGY_TAG = "topology";
- public final static String FULLNAME = "fullName";
- public final static String APPLICATION_ID = "id";
-
- public final static String CLUSTER_ENV = "envContextConfig.env";
- public final static String CLUSTER_URL = "envContextConfig.url";
- public final static String DEFAULT_CLUSTER_URL = "http://sandbox.hortonworks.com:8744";
-
- public final static String RUNNING_MODE = "envContextConfig.mode";
- public final static String EAGLE_CLUSTER_STORM = "storm";
- public final static String EAGLE_CLUSTER_SPARK = "spark";
-
- public final static String APP_COMMAND_LOADER_ENABLED = "appCommandLoaderEnabled";
- public final static String APP_COMMAND_LOADER_INTERVAL_SECS = "appCommandLoaderIntervalSecs";
- public final static String APP_HEALTH_CHECK_INTERVAL_SECS = "appHealthCheckIntervalSecs";
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/46afec39/eagle-core/eagle-app/eagle-application-service/src/main/java/org/apache/eagle/service/application/ApplicationManagementResource.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-app/eagle-application-service/src/main/java/org/apache/eagle/service/application/ApplicationManagementResource.java b/eagle-core/eagle-app/eagle-application-service/src/main/java/org/apache/eagle/service/application/ApplicationManagementResource.java
deleted file mode 100644
index c651b6b..0000000
--- a/eagle-core/eagle-app/eagle-application-service/src/main/java/org/apache/eagle/service/application/ApplicationManagementResource.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.eagle.service.application;
-
-
-import org.apache.eagle.log.base.taggedlog.TaggedLogAPIEntity;
-import org.apache.eagle.log.entity.GenericServiceAPIResponseEntity;
-import org.apache.eagle.service.application.dao.ApplicationManagerDAO;
-import org.apache.eagle.service.application.dao.ApplicationManagerDaoImpl;
-import org.apache.eagle.service.application.entity.TopologyExecutionStatus;
-import org.apache.eagle.service.application.entity.TopologyOperationEntity;
-import org.codehaus.jackson.map.ObjectMapper;
-import org.codehaus.jackson.map.type.TypeFactory;
-
-import javax.ws.rs.*;
-import javax.ws.rs.core.MediaType;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.*;
-
-@Path(ApplicationManagementResource.ROOT_PATH)
-public class ApplicationManagementResource {
- private final static ApplicationManagerDAO dao = new ApplicationManagerDaoImpl();
- public final static String ROOT_PATH = "/app";
-
- @Path("operation")
- @POST
- @Consumes(MediaType.APPLICATION_JSON)
- @Produces(MediaType.APPLICATION_JSON)
- public GenericServiceAPIResponseEntity createOperation(InputStream inputStream) {
- GenericServiceAPIResponseEntity response = new GenericServiceAPIResponseEntity<>();
- List<TopologyOperationEntity> operations = new LinkedList<>();
- try {
- List<TopologyOperationEntity> entities = (List<TopologyOperationEntity>) unmarshalOperationEntities(inputStream);
- if (entities == null) {
- throw new IllegalArgumentException("inputStream cannot convert to TopologyOperationEntity");
- }
- for (TopologyOperationEntity entity : entities) {
- String status = dao.loadTopologyExecutionStatus(entity.getSite(), entity.getApplication(), entity.getTopology());
- if(status == null) {
- throw new Exception(String.format("Fail to fetch the topology execution status by site=%s, application=%s, topology=%s", entity.getSite(), entity.getApplication(), entity.getTopology()));
- }
- int operationsInRunning = dao.loadTopologyOperationsInRunning(entity.getSite(), entity.getApplication(), entity.getTopology());
- if(operationsInRunning !=0) {
- throw new Exception(operationsInRunning + "operations are running, please wait for a minute");
- }
- if (validateOperation(entity.getOperation(), status)) {
- Map<String, String> tags = entity.getTags();
- tags.put(AppManagerConstants.OPERATION_ID_TAG, UUID.randomUUID().toString());
- entity.setTags(tags);
- entity.setLastModifiedDate(System.currentTimeMillis());
- entity.setTimestamp(System.currentTimeMillis());
- operations.add(entity);
- } else {
- throw new Exception(String.format("%s is an invalid operation, as the topology's current status is %s", entity.getOperation(), status));
- }
- }
- response = dao.createOperation(operations);
- } catch (Exception e) {
- response.setSuccess(false);
- response.setException(e);
- }
- return response;
- }
-
- private boolean validateOperation(String operation, String status) {
- boolean ret = false;
- switch (operation) {
- case TopologyOperationEntity.OPERATION.START:
- return TopologyExecutionStatus.isReadyToStart(status);
- case TopologyOperationEntity.OPERATION.STOP:
- return TopologyExecutionStatus.isReadyToStop(status);
- default: break;
- }
- return ret;
- }
-
- private List<? extends TaggedLogAPIEntity> unmarshalOperationEntities(InputStream inputStream) throws IllegalAccessException, InstantiationException, IOException {
- ObjectMapper objectMapper = new ObjectMapper();
- return objectMapper.readValue(inputStream, TypeFactory.defaultInstance().constructCollectionType(LinkedList.class, TopologyOperationEntity.class));
- }
-
- @Path("topology")
- @DELETE
- @Consumes(MediaType.APPLICATION_JSON)
- @Produces(MediaType.APPLICATION_JSON)
- public GenericServiceAPIResponseEntity deleteTopology(@QueryParam("topology") String topology) {
- return dao.deleteTopology(topology);
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/46afec39/eagle-core/eagle-app/eagle-application-service/src/main/java/org/apache/eagle/service/application/dao/ApplicationManagerDAO.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-app/eagle-application-service/src/main/java/org/apache/eagle/service/application/dao/ApplicationManagerDAO.java b/eagle-core/eagle-app/eagle-application-service/src/main/java/org/apache/eagle/service/application/dao/ApplicationManagerDAO.java
deleted file mode 100644
index dfa261b..0000000
--- a/eagle-core/eagle-app/eagle-application-service/src/main/java/org/apache/eagle/service/application/dao/ApplicationManagerDAO.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.eagle.service.application.dao;
-
-
-import org.apache.eagle.log.entity.GenericServiceAPIResponseEntity;
-import org.apache.eagle.service.application.entity.TopologyExecutionStatus;
-import org.apache.eagle.service.application.entity.TopologyOperationEntity;
-
-import java.util.List;
-
-public interface ApplicationManagerDAO {
- String loadTopologyExecutionStatus(String site, String application, String topology);
- int loadTopologyOperationsInRunning(String site, String application, String topology) throws Exception;
- GenericServiceAPIResponseEntity createOperation(List<TopologyOperationEntity> entities) throws Exception;
- GenericServiceAPIResponseEntity deleteTopology(String topology);
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/46afec39/eagle-core/eagle-app/eagle-application-service/src/main/java/org/apache/eagle/service/application/dao/ApplicationManagerDaoImpl.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-app/eagle-application-service/src/main/java/org/apache/eagle/service/application/dao/ApplicationManagerDaoImpl.java b/eagle-core/eagle-app/eagle-application-service/src/main/java/org/apache/eagle/service/application/dao/ApplicationManagerDaoImpl.java
deleted file mode 100644
index 4881cf4..0000000
--- a/eagle-core/eagle-app/eagle-application-service/src/main/java/org/apache/eagle/service/application/dao/ApplicationManagerDaoImpl.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.eagle.service.application.dao;
-
-import org.apache.eagle.log.entity.GenericServiceAPIResponseEntity;
-import org.apache.eagle.policy.common.Constants;
-import org.apache.eagle.service.application.entity.TopologyExecutionEntity;
-import org.apache.eagle.service.application.entity.TopologyOperationEntity;
-import org.apache.eagle.service.generic.GenericEntityServiceResource;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-
-public class ApplicationManagerDaoImpl implements ApplicationManagerDAO {
- private static Logger LOG = LoggerFactory.getLogger(ApplicationManagerDaoImpl.class);
- GenericEntityServiceResource resource = new GenericEntityServiceResource();
-
- @Override
- public String loadTopologyExecutionStatus(String site, String application, String topology) {
- String query = String.format("%s[@site=\"%s\" AND @application=\"%s\" AND @topology=\"%s\"]{*}", Constants.TOPOLOGY_EXECUTION_SERVICE_ENDPOINT_NAME, site, application, topology);
- GenericServiceAPIResponseEntity<TopologyExecutionEntity> response = resource.search(query, null, null, Integer.MAX_VALUE, null, false, false, 0L, 0, false, 0, null, false);
- if(!response.isSuccess()) {
- LOG.error(response.getException());
- return null;
- }
- List<TopologyExecutionEntity> list = response.getObj();
- if(list == null || list.size() != 1) {
- LOG.error("ERROR: fetching 0 or more than 1 topology execution entities");
- return null;
- }
- return list.get(0).getStatus();
- }
-
- @Override
- public int loadTopologyOperationsInRunning(String site, String application, String topology) throws Exception {
- int ret = 0;
- String query = String.format("%s[@site=\"%s\" AND @application=\"%s\" AND @topology=\"%s\" AND (@status=\"%s\" OR @status=\"%s\")]{*}", Constants.TOPOLOGY_OPERATION_SERVICE_ENDPOINT_NAME, site, application, topology, TopologyOperationEntity.OPERATION_STATUS.INITIALIZED, TopologyOperationEntity.OPERATION_STATUS.PENDING);
- GenericServiceAPIResponseEntity<TopologyExecutionEntity> response = resource.search(query, null, null, Integer.MAX_VALUE, null, false, false, 0L, 0, false, 0, null, false);
- if(!response.isSuccess()) {
- throw new Exception(response.getException());
- }
- if(response.getObj() != null && response.getObj().size() != 0) {
- ret = response.getObj().size();
- }
- return ret;
- }
-
- @Override
- public GenericServiceAPIResponseEntity createOperation(List<TopologyOperationEntity> entities) throws Exception {
- if(entities.size() == 0) {
- LOG.info("TopologyOperationEntity set is empty.");
- }
- GenericServiceAPIResponseEntity response = resource.updateEntities(entities, Constants.TOPOLOGY_OPERATION_SERVICE_ENDPOINT_NAME);
- return response;
- }
-
- @Override
- public GenericServiceAPIResponseEntity deleteTopology(String topology) {
- String topologyQuery = Constants.TOPOLOGY_DESCRIPTION_SERVICE_ENDPOINT_NAME+ "[@topology=\"" + topology + "\"]{*}";
- String executionQuery = Constants.TOPOLOGY_EXECUTION_SERVICE_ENDPOINT_NAME + "[@topology=\"" + topology + "\"]{*}";
- int pageSize = Integer.MAX_VALUE;
-
- GenericServiceAPIResponseEntity response = resource.deleteByQuery(topologyQuery, null, null, pageSize, null, false, false, 0L, 0, true, 0, null, false);
- if(response.isSuccess()) {
- response = resource.deleteByQuery(executionQuery, null, null, pageSize, null, false, false, 0L, 0, true, 0, null, false);
- }
- if(!response.isSuccess()) {
- LOG.error(response.getException());
- }
- return response;
- }
-
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/46afec39/eagle-core/eagle-app/eagle-application-service/src/main/java/org/apache/eagle/service/application/entity/ApplicationEntityRepo.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-app/eagle-application-service/src/main/java/org/apache/eagle/service/application/entity/ApplicationEntityRepo.java b/eagle-core/eagle-app/eagle-application-service/src/main/java/org/apache/eagle/service/application/entity/ApplicationEntityRepo.java
deleted file mode 100644
index 3226650..0000000
--- a/eagle-core/eagle-app/eagle-application-service/src/main/java/org/apache/eagle/service/application/entity/ApplicationEntityRepo.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.eagle.service.application.entity;
-
-
-import org.apache.eagle.log.entity.repo.EntityRepository;
-
-public class ApplicationEntityRepo extends EntityRepository {
- public ApplicationEntityRepo() {
- this.registerEntity(TopologyDescriptionEntity.class);
- this.registerEntity(TopologyExecutionEntity.class);
- this.registerEntity(TopologyOperationEntity.class);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/46afec39/eagle-core/eagle-app/eagle-application-service/src/main/java/org/apache/eagle/service/application/entity/TopologyDescriptionEntity.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-app/eagle-application-service/src/main/java/org/apache/eagle/service/application/entity/TopologyDescriptionEntity.java b/eagle-core/eagle-app/eagle-application-service/src/main/java/org/apache/eagle/service/application/entity/TopologyDescriptionEntity.java
deleted file mode 100644
index 6442e6c..0000000
--- a/eagle-core/eagle-app/eagle-application-service/src/main/java/org/apache/eagle/service/application/entity/TopologyDescriptionEntity.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.eagle.service.application.entity;
-
-
-import org.apache.eagle.log.base.taggedlog.TaggedLogAPIEntity;
-import org.apache.eagle.log.entity.meta.*;
-import org.apache.eagle.policy.common.Constants;
-import org.apache.eagle.service.application.AppManagerConstants;
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-import java.util.HashMap;
-import java.util.Map;
-
-
-@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL)
-@Table("eagle_metadata")
-@ColumnFamily("f")
-@Prefix("topologyDescription")
-@Service(Constants.TOPOLOGY_DESCRIPTION_SERVICE_ENDPOINT_NAME)
-@JsonIgnoreProperties(ignoreUnknown = true)
-@TimeSeries(false)
-@Tags({"topology"})
-public class TopologyDescriptionEntity extends TaggedLogAPIEntity {
- @Column("a")
- private String exeClass;
- @Column("b")
- private String type;
- @Column("c")
- private String description;
- @Column("d")
- private String version;
- private String context;
- public String getContext() {
- return context;
- }
-
- public void setContext(String context) {
- this.context = context;
- }
-
- public String getExeClass() {
- return exeClass;
- }
-
- public void setExeClass(String exeClass) {
- this.exeClass = exeClass;
- valueChanged("exeClass");
- }
-
- public String getType() {
- return type;
- }
-
- public void setType(String type) {
- this.type = type;
- valueChanged("type");
- }
-
- public String getDescription() {
- return description;
- }
-
- public void setDescription(String description) {
- this.description = description;
- valueChanged("description");
- }
-
- public String getVersion() {
- return version;
- }
-
- public void setVersion(String version) {
- this.version = version;
- valueChanged("version");
- }
-
- public String getTopology() {
- return this.getTags().get(AppManagerConstants.TOPOLOGY_TAG);
- }
-
- public final static class TYPE {
- public final static String DYNAMIC = "DYNAMIC";
- public final static String CLASS = "CLASS";
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/46afec39/eagle-core/eagle-app/eagle-application-service/src/main/java/org/apache/eagle/service/application/entity/TopologyExecutionEntity.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-app/eagle-application-service/src/main/java/org/apache/eagle/service/application/entity/TopologyExecutionEntity.java b/eagle-core/eagle-app/eagle-application-service/src/main/java/org/apache/eagle/service/application/entity/TopologyExecutionEntity.java
deleted file mode 100644
index 9991d3b..0000000
--- a/eagle-core/eagle-app/eagle-application-service/src/main/java/org/apache/eagle/service/application/entity/TopologyExecutionEntity.java
+++ /dev/null
@@ -1,132 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.eagle.service.application.entity;
-
-import org.apache.eagle.log.base.taggedlog.TaggedLogAPIEntity;
-import org.apache.eagle.log.entity.meta.*;
-import org.apache.eagle.policy.common.Constants;
-
-import org.apache.eagle.service.application.AppManagerConstants;
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-import java.util.HashMap;
-import java.util.Map;
-
-
-@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL)
-@Table("eagle_metadata")
-@ColumnFamily("f")
-@Prefix("topologyExecution")
-@Service(Constants.TOPOLOGY_EXECUTION_SERVICE_ENDPOINT_NAME)
-@JsonIgnoreProperties(ignoreUnknown = true)
-@TimeSeries(false)
-@Tags({"site", "application", "topology"})
-public class TopologyExecutionEntity extends TaggedLogAPIEntity {
- @Column("a")
- private String fullName;
- @Column("b")
- private String url;
- @Column("c")
- private String description;
- @Column("d")
- private String status;
- @Column("e")
- private long lastModifiedDate;
- @Column("f")
- private String mode;
- @Column("g")
- private String environment;
-
- public String getEnvironment() {
- return environment;
- }
-
- public void setEnvironment(String environment) {
- this.environment = environment;
- valueChanged("environment");
- }
-
- public String getMode() {
- return mode;
- }
-
- public void setMode(String mode) {
- this.mode = mode;
- valueChanged("mode");
- }
-
- public String getFullName() {
- return fullName;
- }
-
- public void setFullName(String fullName) {
- this.fullName = fullName;
- valueChanged("fullName");
- }
-
- public String getUrl() {
- return url;
- }
-
- public void setUrl(String url) {
- this.url = url;
- valueChanged("url");
- }
-
- public String getDescription() {
- return description;
- }
-
- public void setDescription(String description) {
- this.description = description;
- valueChanged("description");
- }
-
- public String getStatus() {
- return status;
- }
-
- public void setStatus(String status) {
- this.status = status;
- valueChanged("status");
- }
-
- public long getLastModifiedDate() {
- return lastModifiedDate;
- }
-
- public void setLastModifiedDate(long lastModifiedDate) {
- this.lastModifiedDate = lastModifiedDate;
- valueChanged("lastModifiedDate");
- }
-
- public String getSite() {
- return this.getTags().get(AppManagerConstants.SITE_TAG);
- }
-
- public String getApplication() {
- return this.getTags().get(AppManagerConstants.APPLICATION_TAG);
- }
-
- public String getTopology() {
- return this.getTags().get(AppManagerConstants.TOPOLOGY_TAG);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/46afec39/eagle-core/eagle-app/eagle-application-service/src/main/java/org/apache/eagle/service/application/entity/TopologyExecutionStatus.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-app/eagle-application-service/src/main/java/org/apache/eagle/service/application/entity/TopologyExecutionStatus.java b/eagle-core/eagle-app/eagle-application-service/src/main/java/org/apache/eagle/service/application/entity/TopologyExecutionStatus.java
deleted file mode 100644
index f62ad8a..0000000
--- a/eagle-core/eagle-app/eagle-application-service/src/main/java/org/apache/eagle/service/application/entity/TopologyExecutionStatus.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.eagle.service.application.entity;
-
-
-public class TopologyExecutionStatus {
- public final static String STOPPED = "STOPPED";
- public final static String STARTED = "STARTED";
- public final static String STARTING = "STARTING";
- public final static String STOPPING = "STOPPING";
- public final static String NEW = "NEW";
-
- public static boolean isReadyToStart(String status){
- return status.equals(STOPPED) || status.equals(NEW);
- }
-
- public static boolean isReadyToStop(String status){
- return status.equals(STARTED);
- }
-
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/46afec39/eagle-core/eagle-app/eagle-application-service/src/main/java/org/apache/eagle/service/application/entity/TopologyOperationEntity.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-app/eagle-application-service/src/main/java/org/apache/eagle/service/application/entity/TopologyOperationEntity.java b/eagle-core/eagle-app/eagle-application-service/src/main/java/org/apache/eagle/service/application/entity/TopologyOperationEntity.java
deleted file mode 100644
index 6d8f1a0..0000000
--- a/eagle-core/eagle-app/eagle-application-service/src/main/java/org/apache/eagle/service/application/entity/TopologyOperationEntity.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.eagle.service.application.entity;
-
-import org.apache.eagle.log.base.taggedlog.TaggedLogAPIEntity;
-import org.apache.eagle.log.entity.meta.*;
-import org.apache.eagle.policy.common.Constants;
-
-import org.apache.eagle.service.application.AppManagerConstants;
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-import java.util.HashMap;
-import java.util.Map;
-
-
-@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL)
-@Table("eagle_metadata")
-@ColumnFamily("f")
-@Prefix("topologyOperation")
-@Service(Constants.TOPOLOGY_OPERATION_SERVICE_ENDPOINT_NAME)
-@JsonIgnoreProperties(ignoreUnknown = true)
-@TimeSeries(true)
-@Tags({"site", "application", "topology", "operationID", "operation"})
-public class TopologyOperationEntity extends TaggedLogAPIEntity {
- @Column("a")
- private String status;
- @Column("b")
- private String message;
- @Column("c")
- private long lastModifiedDate;
-
- public String getStatus() {
- return status;
- }
-
- public void setStatus(String status) {
- this.status = status;
- valueChanged("status");
- }
-
- public String getMessage() {
- return message;
- }
-
- public void setMessage(String message) {
- this.message = message;
- valueChanged("message");
- }
-
- public long getLastModifiedDate() {
- return lastModifiedDate;
- }
-
- public void setLastModifiedDate(long lastModifiedDate) {
- this.lastModifiedDate = lastModifiedDate;
- valueChanged("lastModifiedDate");
- }
-
- public final static class OPERATION {
- public final static String START = "START";
- public final static String STOP = "STOP";
- public final static String STATUS = "STATUS";
- }
-
- public final static class OPERATION_STATUS {
- public final static String PENDING = "PENDING";
- public final static String INITIALIZED = "INITIALIZED";
- public final static String SUCCESS = "SUCCESS";
- public final static String FAILED = "FAILED";
- }
-
- public String getSite() {
- return this.getTags().get(AppManagerConstants.SITE_TAG);
- }
-
- public String getApplication() {
- return this.getTags().get(AppManagerConstants.APPLICATION_TAG);
- }
-
- public String getTopology() {
- return this.getTags().get(AppManagerConstants.TOPOLOGY_TAG);
- }
-
- public String getOperation() {
- return this.getTags().get(AppManagerConstants.OPERATION_TAG);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/46afec39/eagle-core/eagle-app/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-app/pom.xml b/eagle-core/eagle-app/pom.xml
index 5637b01..7c95fe6 100644
--- a/eagle-core/eagle-app/pom.xml
+++ b/eagle-core/eagle-app/pom.xml
@@ -32,6 +32,5 @@
<modules>
<module>eagle-app-base</module>
- <module>eagle-application-service</module>
</modules>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/46afec39/eagle-hadoop-metric/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-hadoop-metric/pom.xml b/eagle-hadoop-metric/pom.xml
index 15eea00..b397e37 100644
--- a/eagle-hadoop-metric/pom.xml
+++ b/eagle-hadoop-metric/pom.xml
@@ -15,7 +15,8 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>eagle-parent</artifactId>
<groupId>org.apache.eagle</groupId>
@@ -25,15 +26,6 @@
<artifactId>eagle-hadoop-metric</artifactId>
<dependencies>
- <dependency>
- <groupId>org.apache.eagle</groupId>
- <artifactId>eagle-stream-process-api</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.eagle</groupId>
- <artifactId>eagle-app-base</artifactId>
- <version>${project.version}</version>
- </dependency>
</dependencies>
+
</project>
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/46afec39/eagle-hadoop-metric/src/main/java/org/apache/eagle/hadoop/metric/HadoopJmxApplication.java
----------------------------------------------------------------------
diff --git a/eagle-hadoop-metric/src/main/java/org/apache/eagle/hadoop/metric/HadoopJmxApplication.java b/eagle-hadoop-metric/src/main/java/org/apache/eagle/hadoop/metric/HadoopJmxApplication.java
deleted file mode 100644
index 40d1a24..0000000
--- a/eagle-hadoop-metric/src/main/java/org/apache/eagle/hadoop/metric/HadoopJmxApplication.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.eagle.hadoop.metric;
-
-import backtype.storm.generated.StormTopology;
-import backtype.storm.topology.BoltDeclarer;
-import backtype.storm.topology.IRichSpout;
-import backtype.storm.topology.TopologyBuilder;
-import backtype.storm.tuple.Fields;
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigFactory;
-import org.apache.eagle.app.StormApplication;
-import org.apache.eagle.app.environment.impl.StormEnvironment;
-import org.apache.eagle.app.sink.StormStreamSink;
-import org.apache.eagle.dataproc.impl.storm.kafka.KafkaSpoutProvider;
-import storm.kafka.StringScheme;
-
-/**
- * Since 8/12/16.
- * This application just pass through data from jmx metric
- * For persistence or alert purpose, it is not necessary to start application
- * But keep this application in case of future business process
- *
- * Note: this application should be run as multiple instances based on different topic for data source
- */
-public class HadoopJmxApplication extends StormApplication {
- public final static String SPOUT_TASK_NUM = "topology.numOfSpoutTasks";
- public final static String PARSER_TASK_NUM = "topology.numOfParserTasks";
- public final static String SINK_TASK_NUM = "topology.numOfSinkTasks";
-
- @Override
- public StormTopology execute(Config config, StormEnvironment environment) {
- int numOfSpoutTasks = config.getInt(SPOUT_TASK_NUM);
- int numOfParserTasks = config.getInt(PARSER_TASK_NUM);
- int numOfSinkTasks = config.getInt(SINK_TASK_NUM);
-
- TopologyBuilder builder = new TopologyBuilder();
-
- KafkaSpoutProvider provider = new KafkaSpoutProvider();
- IRichSpout spout = provider.getSpout(config);
- builder.setSpout("ingest", spout, numOfSpoutTasks);
-
- JsonParserBolt bolt = new JsonParserBolt();
- BoltDeclarer boltDeclarer = builder.setBolt("parserBolt", bolt, numOfParserTasks);
- boltDeclarer.fieldsGrouping("ingest", new Fields(StringScheme.STRING_SCHEME_KEY));
-
- StormStreamSink sinkBolt = environment.getStreamSink("hadoop_jmx_stream",config);
- BoltDeclarer kafkaBoltDeclarer = builder.setBolt("kafkaSink", sinkBolt, numOfSinkTasks);
- kafkaBoltDeclarer.fieldsGrouping("parserBolt", new Fields("f1"));
-
- return builder.createTopology();
- }
-
- public static void main(String[] args){
- Config config = ConfigFactory.load();
- HadoopJmxApplication app = new HadoopJmxApplication();
- app.run(config);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/46afec39/eagle-hadoop-metric/src/main/java/org/apache/eagle/hadoop/metric/JsonParserBolt.java
----------------------------------------------------------------------
diff --git a/eagle-hadoop-metric/src/main/java/org/apache/eagle/hadoop/metric/JsonParserBolt.java b/eagle-hadoop-metric/src/main/java/org/apache/eagle/hadoop/metric/JsonParserBolt.java
deleted file mode 100644
index 7ca5ba6..0000000
--- a/eagle-hadoop-metric/src/main/java/org/apache/eagle/hadoop/metric/JsonParserBolt.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.hadoop.metric;
-
-import backtype.storm.task.OutputCollector;
-import backtype.storm.task.TopologyContext;
-import backtype.storm.topology.OutputFieldsDeclarer;
-import backtype.storm.topology.base.BaseRichBolt;
-import backtype.storm.tuple.Fields;
-import backtype.storm.tuple.Tuple;
-import org.codehaus.jackson.map.ObjectMapper;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Arrays;
-import java.util.Map;
-
-/**
- * Since 8/14/16.
- */
-public class JsonParserBolt extends BaseRichBolt {
- private Logger LOG = LoggerFactory.getLogger(JsonParserBolt.class);
- private OutputCollector collector;
- private ObjectMapper mapper = new ObjectMapper();
-
- @Override
- public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
- this.collector = collector;
- }
-
- @Override
- public void execute(Tuple input) {
- String msg = input.getString(0);
- try {
- Map ret = mapper.readValue(msg, Map.class);
- collector.emit(Arrays.asList(ret));
- }catch(Exception ex){
- LOG.error("error in passing json message", ex);
- }finally{
- collector.ack(input);
- }
- }
-
- @Override
- public void declareOutputFields(OutputFieldsDeclarer declarer) {
- declarer.declare(new Fields("msg"));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/46afec39/eagle-hadoop-metric/src/main/resources/application.conf
----------------------------------------------------------------------
diff --git a/eagle-hadoop-metric/src/main/resources/application.conf b/eagle-hadoop-metric/src/main/resources/application.conf
deleted file mode 100644
index e75355f..0000000
--- a/eagle-hadoop-metric/src/main/resources/application.conf
+++ /dev/null
@@ -1,51 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-{
- "appId" : "HadoopJmxApplication",
- "mode" : "LOCAL",
- "siteId" : "testsite",
- "topology" : {
- "numOfSpoutTasks" : 2,
- "numOfParserTasks" : 2,
- "numOfSinkTasks" : 2
- },
- "dataSourceConfig": {
- "topic" : "jmx_metric",
- "zkConnection" : "server.eagle.apache.org:2181",
- "zkConnectionTimeoutMS" : 15000,
- "consumerGroupId" : "EagleConsumer",
- "fetchSize" : 1048586,
- "transactionZKServers" : "server.eagle.apache.org",
- "transactionZKPort" : 2181,
- "transactionZKRoot" : "/consumers",
- "transactionStateUpdateMS" : 2000
- "schemeCls" : "storm.kafka.StringScheme"
- },
- "eagleProps" : {
- "eagleService": {
- "host": "localhost",
- "port": 9090,
- "username": "admin",
- "password": "secret"
- }
- },
- "dataSinkConfig": {
- "topic" : "jmx_metric_parsed",
- "brokerList" : "server.eagle.apache.org:6667",
- "serializerClass" : "kafka.serializer.StringEncoder",
- "keySerializerClass" : "kafka.serializer.StringEncoder"
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/46afec39/eagle-hadoop-metric/src/main/resources/capacityused-policy-import.sh
----------------------------------------------------------------------
diff --git a/eagle-hadoop-metric/src/main/resources/capacityused-policy-import.sh b/eagle-hadoop-metric/src/main/resources/capacityused-policy-import.sh
deleted file mode 100644
index 71e2756..0000000
--- a/eagle-hadoop-metric/src/main/resources/capacityused-policy-import.sh
+++ /dev/null
@@ -1,51 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with`
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-source $(dirname $0)/eagle-env.sh
-source $(dirname $0)/hadoop-metric-init.sh
-
-
-##### add policies ##########
-echo ""
-echo "Importing policy: capacityUsedPolicy "
-curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' \
- "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=AlertDefinitionService" \
- -d '
- [
- {
- "prefix": "alertdef",
- "tags": {
- "site": "sandbox",
- "application": "hadoopJmxMetricDataSource",
- "policyId": "capacityUsedPolicy",
- "alertExecutorId": "hadoopJmxMetricAlertExecutor",
- "policyType": "siddhiCEPEngine"
- },
- "description": "jmx metric ",
- "policyDef": "{\"expression\":\"from hadoopJmxMetricEventStream[metric == \\\"hadoop.namenode.fsnamesystemstate.capacityused\\\" and convert(value, \\\"long\\\") > 0]#window.externalTime(timestamp ,10 min) select metric, host, value, timestamp, component, site insert into tmp; \",\"type\":\"siddhiCEPEngine\"}",
- "enabled": true,
- "dedupeDef": "{\"alertDedupIntervalMin\":10,\"emailDedupIntervalMin\":10}",
- "notificationDef": "[{\"sender\":\"eagle@apache.org\",\"recipients\":\"eagle@apache.org\",\"subject\":\"missing block found.\",\"flavor\":\"email\",\"id\":\"email_1\",\"tplFileName\":\"\"}]"
- }
- ]
- '
-
- ## Finished
-echo ""
-echo "Finished initialization for eagle topology"
-
-exit 0
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/46afec39/eagle-hadoop-metric/src/main/resources/datasources.json
----------------------------------------------------------------------
diff --git a/eagle-hadoop-metric/src/main/resources/datasources.json b/eagle-hadoop-metric/src/main/resources/datasources.json
new file mode 100644
index 0000000..5c10696
--- /dev/null
+++ b/eagle-hadoop-metric/src/main/resources/datasources.json
@@ -0,0 +1,19 @@
+[
+ {
+ "name": "hadoop_jmx_datasource",
+ "type": "KAFKA",
+ "properties": {
+ },
+ "topic": "hadoop_jmx_metrics",
+ "schemeCls": "org.apache.eagle.alert.engine.scheme.JsonScheme",
+ "codec": {
+ "streamNameSelectorProp": {
+ "userProvidedStreamName" : "hadoopJmxMetricEventStream",
+ "streamNameFormat":"%s"
+ },
+ "streamNameSelectorCls":"org.apache.eagle.alert.engine.scheme.JsonStringStreamNameSelector",
+ "timestampColumn": "timestamp",
+ "timestampFormat": ""
+ }
+ }
+]
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/46afec39/eagle-hadoop-metric/src/main/resources/eagle-env.sh
----------------------------------------------------------------------
diff --git a/eagle-hadoop-metric/src/main/resources/eagle-env.sh b/eagle-hadoop-metric/src/main/resources/eagle-env.sh
deleted file mode 100755
index 79ff5fa..0000000
--- a/eagle-hadoop-metric/src/main/resources/eagle-env.sh
+++ /dev/null
@@ -1,44 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# set EAGLE_HOME
-export EAGLE_HOME=$(dirname $0)/..
-
-# The java implementation to use. please use jdk 1.7 or later
-# export JAVA_HOME=${JAVA_HOME}
-# export JAVA_HOME=/usr/java/jdk1.7.0_80/
-
-# nimbus.host, default is localhost
-export EAGLE_NIMBUS_HOST=localhost
-
-# EAGLE_SERVICE_HOST, default is `hostname -f`
-export EAGLE_SERVICE_HOST=localhost
-
-# EAGLE_SERVICE_PORT, default is 9099
-export EAGLE_SERVICE_PORT=9099
-
-# EAGLE_SERVICE_USER
-export EAGLE_SERVICE_USER=admin
-
-# EAGLE_SERVICE_PASSWORD
-export EAGLE_SERVICE_PASSWD=secret
-
-export EAGLE_CLASSPATH=$EAGLE_HOME/conf
-# Add eagle shared library jars
-for file in $EAGLE_HOME/lib/share/*;do
- EAGLE_CLASSPATH=$EAGLE_CLASSPATH:$file
-done
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/46afec39/eagle-hadoop-metric/src/main/resources/hadoop-metric-init.sh
----------------------------------------------------------------------
diff --git a/eagle-hadoop-metric/src/main/resources/hadoop-metric-init.sh b/eagle-hadoop-metric/src/main/resources/hadoop-metric-init.sh
deleted file mode 100755
index 849f462..0000000
--- a/eagle-hadoop-metric/src/main/resources/hadoop-metric-init.sh
+++ /dev/null
@@ -1,180 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with`
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-source $(dirname $0)/eagle-env.sh
-
-#####################################################################
-# Import stream metadata for HDFS
-#####################################################################
-
-## AlertDataSource: data sources bound to sites
-echo "Importing AlertDataSourceService for persist... "
-
-curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' \
- "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=SiteApplicationService" \
- -d '
- [
- {
- "tags":{
- "site":"sandbox",
- "application":"hadoopJmxMetricDataSource"
- },
- "enabled": true,
- "config": "web.druid.coordinator=coordinatorHost:port\nweb.druid.broker=brokerHost:port"
- }
- ]
- '
-
-curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' \
- "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=ApplicationDescService" \
- -d '
- [
- {
- "tags":{
- "application":"hadoopJmxMetricDataSource"
- },
- "description":"hadoop jmx metric monitoring",
- "alias":"JmxMetricMonitor",
- "groupName":"METRIC",
- "config":"{}",
- "features":["common","metadata"]
- }
- ]
- '
-
-## AlertStreamService: alert streams generated from data source
-echo ""
-echo "Importing AlertStreamService for HDFS... "
-curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' \
- "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=AlertStreamService" \
- -d '
- [
- {
- "prefix":"alertStream",
- "tags":{
- "application":"hadoopJmxMetricDataSource",
- "streamName":"hadoopJmxMetricEventStream"
- },
- "description":"hadoop"
- }
- ]
- '
-
-## AlertExecutorService: what alert streams are consumed by alert executor
-echo ""
-echo "Importing AlertExecutorService for HDFS... "
-curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' \
- "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=AlertExecutorService" \
- -d '
- [
- {
- "prefix":"alertExecutor",
- "tags":{
- "application":"hadoopJmxMetricDataSource",
- "alertExecutorId":"hadoopJmxMetricAlertExecutor",
- "streamName":"hadoopJmxMetricEventStream"
- },
- "description":"aggregate executor for hadoop jmx metric event stream"
- }
- ]
- '
-
-## AlertStreamSchemaService: schema for event from alert stream
-echo ""
-echo "Importing AlertStreamSchemaService for HDFS... "
-curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' \
-"http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=AlertStreamSchemaService" \
- -d '
- [
- {
- "prefix": "alertStreamSchema",
- "tags": {
- "application": "hadoopJmxMetricDataSource",
- "streamName": "hadoopJmxMetricEventStream",
- "attrName": "host"
- },
- "attrDescription": "the host that current metric comes form",
- "attrType": "string",
- "category": "",
- "attrValueResolver": ""
- },
- {
- "prefix": "alertStreamSchema",
- "tags": {
- "application": "hadoopJmxMetricDataSource",
- "streamName": "hadoopJmxMetricEventStream",
- "attrName": "timestamp"
- },
- "attrDescription": "the metric timestamp",
- "attrType": "long",
- "category": "",
- "attrValueResolver": ""
- },
- {
- "prefix": "alertStreamSchema",
- "tags": {
- "application": "hadoopJmxMetricDataSource",
- "streamName": "hadoopJmxMetricEventStream",
- "attrName": "metric"
- },
- "attrDescription": "the metric name",
- "attrType": "string",
- "category": "",
- "attrValueResolver": ""
- },
- {
- "prefix": "alertStreamSchema",
- "tags": {
- "application": "hadoopJmxMetricDataSource",
- "streamName": "hadoopJmxMetricEventStream",
- "attrName": "component"
- },
- "attrDescription": "the component that the metric comes from",
- "attrType": "string",
- "category": "",
- "attrValueResolver": ""
- },
- {
- "prefix": "alertStreamSchema",
- "tags": {
- "application": "hadoopJmxMetricDataSource",
- "streamName": "hadoopJmxMetricEventStream",
- "attrName": "site"
- },
- "attrDescription": "the site that the metric belongs to",
- "attrType": "string",
- "category": "",
- "attrValueResolver": ""
- },
- {
- "prefix": "alertStreamSchema",
- "tags": {
- "application": "hadoopJmxMetricDataSource",
- "streamName": "hadoopJmxMetricEventStream",
- "attrName": "value"
- },
- "attrDescription": "the metric value in string presentation",
- "attrType": "double",
- "category": "",
- "attrValueResolver": ""
- }
- ]
- '
-
-## Finished
-echo ""
-echo "Finished initialization for eagle topology"
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/46afec39/eagle-hadoop-metric/src/main/resources/hadoopjmx.yaml
----------------------------------------------------------------------
diff --git a/eagle-hadoop-metric/src/main/resources/hadoopjmx.yaml b/eagle-hadoop-metric/src/main/resources/hadoopjmx.yaml
deleted file mode 100644
index a68a323..0000000
--- a/eagle-hadoop-metric/src/main/resources/hadoopjmx.yaml
+++ /dev/null
@@ -1,18 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-topology.workers: 1
-topology.acker.executors: 1
-topology.tasks: 1
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/46afec39/eagle-hadoop-metric/src/main/resources/hastate-policy-import.sh
----------------------------------------------------------------------
diff --git a/eagle-hadoop-metric/src/main/resources/hastate-policy-import.sh b/eagle-hadoop-metric/src/main/resources/hastate-policy-import.sh
deleted file mode 100644
index b125e37..0000000
--- a/eagle-hadoop-metric/src/main/resources/hastate-policy-import.sh
+++ /dev/null
@@ -1,51 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with`
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-source $(dirname $0)/eagle-env.sh
-source $(dirname $0)/hadoop-metric-init.sh
-
-
-##### add policies ##########
-echo ""
-echo "Importing policy: haStatePolicy "
-curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' \
- "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=AlertDefinitionService" \
- -d '
- [
- {
- "prefix": "alertdef",
- "tags": {
- "site": "sandbox",
- "application": "hadoopJmxMetricDataSource",
- "policyId": "haStatePolicy",
- "alertExecutorId": "hadoopJmxMetricAlertExecutor",
- "policyType": "siddhiCEPEngine"
- },
- "description": "jmx metric ",
- "policyDef": "{\"expression\":\"from every a = hadoopJmxMetricEventStream[metric==\\\"hadoop.namenode.fsnamesystem.hastate\\\"] -> b = hadoopJmxMetricEventStream[metric==a.metric and b.host == a.host and (convert(a.value, \\\"long\\\") != convert(value, \\\"long\\\"))] within 10 min select a.host, a.value as oldHaState, b.value as newHaState, b.timestamp as timestamp, b.metric as metric, b.component as component, b.site as site insert into tmp; \",\"type\":\"siddhiCEPEngine\"}",
- "enabled": true,
- "dedupeDef": "{\"alertDedupIntervalMin\":10,\"emailDedupIntervalMin\":10}",
- "notificationDef": "[{\"sender\":\"eagle@apache.org\",\"recipients\":\"eagle@apache.org\",\"subject\":\"missing block found.\",\"flavor\":\"email\",\"id\":\"email_1\",\"tplFileName\":\"\"}]"
- }
- ]
- '
-
- ## Finished
-echo ""
-echo "Finished initialization for eagle topology"
-
-exit 0
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/46afec39/eagle-hadoop-metric/src/main/resources/lastcheckpointtime-policy-import.sh
----------------------------------------------------------------------
diff --git a/eagle-hadoop-metric/src/main/resources/lastcheckpointtime-policy-import.sh b/eagle-hadoop-metric/src/main/resources/lastcheckpointtime-policy-import.sh
deleted file mode 100644
index 333367c..0000000
--- a/eagle-hadoop-metric/src/main/resources/lastcheckpointtime-policy-import.sh
+++ /dev/null
@@ -1,51 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with`
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-source $(dirname $0)/eagle-env.sh
-source $(dirname $0)/hadoop-metric-init.sh
-
-
-##### add policies ##########
-echo ""
-echo "Importing policy: lastCheckPointTimePolicy "
-curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' \
- "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=AlertDefinitionService" \
- -d '
- [
- {
- "prefix": "alertdef",
- "tags": {
- "site": "sandbox",
- "application": "hadoopJmxMetricDataSource",
- "policyId": "lastCheckPointTimePolicy",
- "alertExecutorId": "hadoopJmxMetricAlertExecutor",
- "policyType": "siddhiCEPEngine"
- },
- "description": "jmx metric ",
- "policyDef": "{\"expression\":\"from hadoopJmxMetricEventStream[metric == \\\"hadoop.namenode.dfs.lastcheckpointtime\\\" and (convert(value, \\\"long\\\") + 18000000) < timestamp]#window.externalTime(timestamp ,10 min) select metric, host, value, timestamp, component, site insert into tmp; \",\"type\":\"siddhiCEPEngine\"}",
- "enabled": true,
- "dedupeDef": "{\"alertDedupIntervalMin\":10,\"emailDedupIntervalMin\":10}",
- "notificationDef": "[{\"sender\":\"eagle@apache.org\",\"recipients\":\"eagle@apache.org\",\"subject\":\"last check point time lag found.\",\"flavor\":\"email\",\"id\":\"email_1\",\"tplFileName\":\"\"}]"
- }
- ]
- '
-
- ## Finished
-echo ""
-echo "Finished initialization for eagle topology"
-
-exit 0
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/46afec39/eagle-hadoop-metric/src/main/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/eagle-hadoop-metric/src/main/resources/log4j.properties b/eagle-hadoop-metric/src/main/resources/log4j.properties
deleted file mode 100644
index 149caa7..0000000
--- a/eagle-hadoop-metric/src/main/resources/log4j.properties
+++ /dev/null
@@ -1,35 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-log4j.rootLogger=INFO, stdout, DRFA
-
-eagle.log.dir=./logs
-eagle.log.file=eagle.log
-
-# standard output
-log4j.appender.stdout=org.apache.log4j.ConsoleAppender
-log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
-log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %p [%t] %c{2}[%L]: %m%n
-
-# Daily Rolling File Appender
-log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
-log4j.appender.DRFA.File=${eagle.log.dir}/${eagle.log.file}
-log4j.appender.DRFA.DatePattern=yyyy-MM-dd
-## 30-day backup
-# log4j.appender.DRFA.MaxBackupIndex=30
-log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
-
-# Pattern format: Date LogLevel LoggerName LogMessage
-log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p [%t] %c{2}[%L]: %m%n
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/46afec39/eagle-hadoop-metric/src/main/resources/missingblock-policy-import.sh
----------------------------------------------------------------------
diff --git a/eagle-hadoop-metric/src/main/resources/missingblock-policy-import.sh b/eagle-hadoop-metric/src/main/resources/missingblock-policy-import.sh
deleted file mode 100644
index 9a5e924..0000000
--- a/eagle-hadoop-metric/src/main/resources/missingblock-policy-import.sh
+++ /dev/null
@@ -1,51 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with`
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-source $(dirname $0)/eagle-env.sh
-source $(dirname $0)/hadoop-metric-init.sh
-
-
-##### add policies ##########
-echo ""
-echo "Importing policy: missingBlockPolicy "
-curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' \
- "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=AlertDefinitionService" \
- -d '
- [
- {
- "prefix": "alertdef",
- "tags": {
- "site": "sandbox",
- "application": "hadoopJmxMetricDataSource",
- "policyId": "missingBlockPolicy",
- "alertExecutorId": "hadoopJmxMetricAlertExecutor",
- "policyType": "siddhiCEPEngine"
- },
- "description": "jmx metric ",
- "policyDef": "{\"expression\":\"from hadoopJmxMetricEventStream[metric == \\\"hadoop.namenode.dfs.missingblocks\\\" and convert(value, \\\"long\\\") > 0]#window.externalTime(timestamp ,10 min) select metric, host, value, timestamp, component, site insert into tmp; \",\"type\":\"siddhiCEPEngine\"}",
- "enabled": true,
- "dedupeDef": "{\"alertDedupIntervalMin\":10,\"emailDedupIntervalMin\":10}",
- "notificationDef": "[{\"sender\":\"eagle@apache.org\",\"recipients\":\"eagle@apache.org\",\"subject\":\"missing block found.\",\"flavor\":\"email\",\"id\":\"email_1\",\"tplFileName\":\"\"}]"
- }
- ]
- '
-
- ## Finished
-echo ""
-echo "Finished initialization for eagle topology"
-
-exit 0
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/46afec39/eagle-hadoop-metric/src/main/resources/namenodelag-policy-import.sh
----------------------------------------------------------------------
diff --git a/eagle-hadoop-metric/src/main/resources/namenodelag-policy-import.sh b/eagle-hadoop-metric/src/main/resources/namenodelag-policy-import.sh
deleted file mode 100644
index 45de4b8..0000000
--- a/eagle-hadoop-metric/src/main/resources/namenodelag-policy-import.sh
+++ /dev/null
@@ -1,49 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with`
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-source $(dirname $0)/eagle-env.sh
-source $(dirname $0)/hadoop-metric-init.sh
-
-echo ""
-echo "Importing Policy: NameNodeLagPolicy"
-curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' \
- "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=AlertDefinitionService" \
- -d '
- [
- {
- "prefix": "alertdef",
- "tags": {
- "site": "sandbox",
- "application": "hadoopJmxMetricDataSource",
- "policyId": "NameNodeLagPolicy",
- "alertExecutorId": "hadoopJmxMetricAlertExecutor",
- "policyType": "siddhiCEPEngine"
- },
- "description": "jmx metric ",
- "policyDef": "{\"expression\":\"from every a = hadoopJmxMetricEventStream[metric==\\\"hadoop.namenode.journaltransaction.lastappliedorwrittentxid\\\"] -> b = hadoopJmxMetricEventStream[metric==a.metric and b.host != a.host and (max(convert(a.value, \\\"long\\\")) + 100) <= max(convert(value, \\\"long\\\"))] within 5 min select a.host as hostA, a.value as transactIdA, b.host as hostB, b.value as transactIdB insert into tmp; \",\"type\":\"siddhiCEPEngine\"}",
- "enabled": true,
- "dedupeDef": "{\"alertDedupIntervalMin\":10,\"emailDedupIntervalMin\":10}",
- "notificationDef": "[{\"sender\":\"eagle@apache.org\",\"recipients\":\"eagle@apache.org\",\"subject\":\"name node lag found.\",\"flavor\":\"email\",\"id\":\"email_1\",\"tplFileName\":\"\"}]"
- }
- ]
- '
-
- ## Finished
-echo ""
-echo "Finished initialization for eagle topology"
-
-exit 0
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/46afec39/eagle-hadoop-metric/src/main/resources/nodecount-policy-import.sh
----------------------------------------------------------------------
diff --git a/eagle-hadoop-metric/src/main/resources/nodecount-policy-import.sh b/eagle-hadoop-metric/src/main/resources/nodecount-policy-import.sh
deleted file mode 100644
index 589a44e..0000000
--- a/eagle-hadoop-metric/src/main/resources/nodecount-policy-import.sh
+++ /dev/null
@@ -1,51 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with`
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-source $(dirname $0)/eagle-env.sh
-source $(dirname $0)/hadoop-metric-init.sh
-
-
-##### add policies ##########
-echo ""
-echo "Importing policy: dataNodeCountPolicy "
-curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' \
- "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=AlertDefinitionService" \
- -d '
- [
- {
- "prefix": "alertdef",
- "tags": {
- "site": "sandbox",
- "application": "hadoopJmxMetricDataSource",
- "policyId": "dataNodeCountPolicy",
- "alertExecutorId": "hadoopJmxMetricAlertExecutor",
- "policyType": "siddhiCEPEngine"
- },
- "description": "jmx metric ",
- "policyDef": "{\"expression\":\"from every (e1 = hadoopJmxMetricEventStream[metric == \\\"hadoop.namenode.fsnamesystemstate.numlivedatanodes\\\" ]) -> e2 = hadoopJmxMetricEventStream[metric == e1.metric and host == e1.host and (convert(e1.value, \\\"long\\\") - 5) >= convert(value, \\\"long\\\") ] within 5 min select e1.metric, e1.host, e1.value as highNum, e1.timestamp as start, e2.value as lowNum, e2.timestamp as end insert into tmp; \",\"type\":\"siddhiCEPEngine\"}",
- "enabled": true,
- "dedupeDef": "{\"alertDedupIntervalMin\":10,\"emailDedupIntervalMin\":10}",
- "notificationDef": "[{\"sender\":\"eagle@apache.org\",\"recipients\":\"eagle@apache.org\",\"subject\":\"node count joggling found.\",\"flavor\":\"email\",\"id\":\"email_1\",\"tplFileName\":\"\"}]"
- }
- ]
- '
-
- ## Finished
-echo ""
-echo "Finished initialization for eagle topology"
-
-exit 0
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/46afec39/eagle-hadoop-metric/src/main/resources/policies.json
----------------------------------------------------------------------
diff --git a/eagle-hadoop-metric/src/main/resources/policies.json b/eagle-hadoop-metric/src/main/resources/policies.json
new file mode 100644
index 0000000..811ffb0
--- /dev/null
+++ b/eagle-hadoop-metric/src/main/resources/policies.json
@@ -0,0 +1,169 @@
+[
+ {
+ "name": "haStatePolicy",
+ "description": "ha state check for hadoop name node",
+ "definition": {
+ "inputStreams": [
+ "hadoopJmxMetricEventStream"
+ ],
+ "outputStreams": [
+ "tmp"
+ ],
+ "type": "siddhi",
+ "value": " from every a = hadoopJmxMetricEventStream[metric==\"hadoop.namenode.fsnamesystem.hastate\"] -> b = hadoopJmxMetricEventStream[metric==a.metric and b.host == a.host and (convert(a.value, \"long\\\") != convert(value, \"long\"))] within 10 min select a.host, a.value as oldHaState, b.value as newHaState, b.timestamp as timestamp, b.metric as metric, b.component as component, b.site as site insert into tmp; "
+ },
+ "partitionSpec": [
+ {
+ "streamId": "hadoopJmxMetricEventStream",
+ "type": "GROUPBY",
+ "columns": [
+ "host"
+ ]
+ }
+ ]
+ },
+
+ {
+ "name": "capacityUsedPolicy",
+ "description": "capacity usage check for hadoop cluster",
+ "definition": {
+ "inputStreams": [
+ "hadoopJmxMetricEventStream"
+ ],
+ "outputStreams": [
+ "tmp"
+ ],
+ "type": "siddhi",
+ "value": " from hadoopJmxMetricEventStream[metric == \"hadoop.namenode.fsnamesystemstate.capacityused\" and convert(value, \"long\") > 0]#window.externalTime(timestamp ,10 min) select metric, host, value, timestamp, component, site insert into tmp; "
+ },
+ "partitionSpec": [
+ {
+ "streamId": "hadoopJmxMetricEventStream",
+ "type": "GROUPBY",
+ "columns": [
+ "host"
+ ]
+ }
+ ]
+ },
+
+ {
+ "name": "lastCheckPointTimePolicy",
+ "description": "last check point interval check for hadoop name node pair",
+ "definition": {
+ "inputStreams": [
+ "hadoopJmxMetricEventStream"
+ ],
+ "outputStreams": [
+ "tmp"
+ ],
+ "type": "siddhi",
+ "value": " from hadoopJmxMetricEventStream[metric == \"hadoop.namenode.dfs.lastcheckpointtime\" and (convert(value, \"long\") + 18000000) < timestamp]#window.externalTime(timestamp ,10 min) select metric, host, value, timestamp, component, site insert into tmp; "
+ },
+ "partitionSpec": [
+ {
+ "streamId": "hadoopJmxMetricEventStream",
+ "type": "GROUPBY",
+ "columns": [
+ "host"
+ ]
+ }
+ ]
+ },
+
+ {
+ "name": "missingBlockPolicy",
+ "description": "missing block number check for hadoop cluster",
+ "definition": {
+ "inputStreams": [
+ "hadoopJmxMetricEventStream"
+ ],
+ "outputStreams": [
+ "tmp"
+ ],
+ "type": "siddhi",
+ "value": " from hadoopJmxMetricEventStream[metric == \"hadoop.namenode.dfs.missingblocks\" and convert(value, \"long\") > 0]#window.externalTime(timestamp ,10 min) select metric, host, value, timestamp, component, site insert into tmp; "
+ },
+ "partitionSpec": [
+ {
+ "streamId": "hadoopJmxMetricEventStream",
+ "type": "GROUPBY",
+ "columns": [
+ "host"
+ ]
+ }
+ ]
+ },
+
+ {
+ "name": "namenodeTxLagPolicy",
+ "description": "name node tx log lag check",
+ "definition": {
+ "inputStreams": [
+ "hadoopJmxMetricEventStream"
+ ],
+ "outputStreams": [
+ "tmp"
+ ],
+ "type": "siddhi",
+ "value": " from every a = hadoopJmxMetricEventStream[metric==\"hadoop.namenode.journaltransaction.lastappliedorwrittentxid\"] -> b = hadoopJmxMetricEventStream[metric==a.metric and b.host != a.host and (max(convert(a.value, \"long\")) + 100) <= max(convert(value, \"long\"))] within 5 min select a.host as hostA, a.value as transactIdA, b.host as hostB, b.value as transactIdB insert into tmp; "
+ },
+ "partitionSpec": [
+ {
+ "streamId": "hadoopJmxMetricEventStream",
+ "type": "GROUPBY",
+ "columns": [
+ "host"
+ ]
+ }
+ ]
+ },
+
+ {
+ "name": "nodecountPolicy",
+ "description": "data node number check for hadoop cluster",
+ "definition": {
+ "inputStreams": [
+ "hadoopJmxMetricEventStream"
+ ],
+ "outputStreams": [
+ "tmp"
+ ],
+ "type": "siddhi",
+ "value": " from every (e1 = hadoopJmxMetricEventStream[metric == \"hadoop.namenode.fsnamesystemstate.numlivedatanodes\" ]) -> e2 = hadoopJmxMetricEventStream[metric == e1.metric and host == e1.host and (convert(e1.value, \"long\") - 5) >= convert(value, \"long\") ] within 5 min select e1.metric, e1.host, e1.value as highNum, e1.timestamp as start, e2.value as lowNum, e2.timestamp as end insert into tmp; "
+ },
+ "partitionSpec": [
+ {
+ "streamId": "hadoopJmxMetricEventStream",
+ "type": "GROUPBY",
+ "columns": [
+ "host"
+ ]
+ }
+ ]
+ },
+
+ {
+ "name": "nameNodeSafeModeCheckPolicy",
+ "description": "safe mode check for name node",
+ "definition": {
+ "inputStreams": [
+ "hadoopJmxMetricEventStream"
+ ],
+ "outputStreams": [
+ "tmp"
+ ],
+ "type": "siddhi",
+ "value": " from hadoopJmxMetricEventStream[component==\"namenode\" and metric == \"hadoop.namenode.fsnamesystemstate.fsstate\" and convert(value, \"long\") > 0]#window.externalTime(timestamp ,10 min) select metric, host, value, timestamp, component, site insert into tmp; "
+ },
+ "partitionSpec": [
+ {
+ "streamId": "hadoopJmxMetricEventStream",
+ "type": "GROUPBY",
+ "columns": [
+ "host"
+ ]
+ }
+ ]
+ }
+]
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/46afec39/eagle-hadoop-metric/src/main/resources/publishments.json
----------------------------------------------------------------------
diff --git a/eagle-hadoop-metric/src/main/resources/publishments.json b/eagle-hadoop-metric/src/main/resources/publishments.json
new file mode 100644
index 0000000..10d16a4
--- /dev/null
+++ b/eagle-hadoop-metric/src/main/resources/publishments.json
@@ -0,0 +1,33 @@
+[
+ {
+ "name":"jmxStreamOutput_email",
+ "type":"org.apache.eagle.alert.engine.publisher.impl.AlertEmailPublisher",
+ "policyIds": [
+ "perfmon_cpu_host_check", "perfmon_cpu_pool_check"
+ ],
+ "properties": {
+ "subject":"Hadoop Jmx Metrics Alert",
+ "template":"",
+ "sender": "sender@corp.com",
+ "recipients": "services@corp.com",
+ "smtp.server":"smtp.mailhost.com",
+ "connection": "plaintext",
+ "smtp.port": "25"
+ },
+ "dedupIntervalMin" : "PT1M",
+ "serializer" : "org.apache.eagle.alert.engine.publisher.impl.StringEventSerializer"
+ },
+ {
+ "type": "org.apache.eagle.alert.engine.publisher.impl.AlertKafkaPublisher",
+ "name":"jmxStreamOutput_kafka",
+ "policyIds": [
+ "perfmon_cpu_host_check"
+ ],
+ "dedupIntervalMin": "PT1M",
+ "properties":{
+ "kafka_broker":"localhost:9092",
+ "topic":"jmx_alerts"
+ },
+ "serializer" : "org.apache.eagle.alert.engine.publisher.impl.StringEventSerializer"
+ }
+]