You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@eagle.apache.org by mw...@apache.org on 2016/10/26 08:46:51 UTC
incubator-eagle git commit: [EAGLE-685] adjust hadoop-queue
application configuration parameters according to classified style
Repository: incubator-eagle
Updated Branches:
refs/heads/master 8e00cdfe0 -> 278b0961c
[EAGLE-685] adjust hadoop-queue application configuration parameters according to classified style
application configuration has been divided into a few sections based on parameter type.
need to adjust code to fit for the change.
Author: anyway1021 <mw...@apache.org>
Closes #565 from anyway1021/adjust-params.
Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/278b0961
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/278b0961
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/278b0961
Branch: refs/heads/master
Commit: 278b0961c9c198ce68dc0596b7f9b71ffd89b038
Parents: 8e00cdf
Author: anyway1021 <mw...@apache.org>
Authored: Wed Oct 26 16:46:24 2016 +0800
Committer: anyway1021 <mw...@apache.org>
Committed: Wed Oct 26 16:46:24 2016 +0800
----------------------------------------------------------------------
.../common/config/EagleConfigConstants.java | 3 ++
.../service/client/EagleServiceConnector.java | 22 +++++++---
.../queue/HadoopQueueRunningAppConfig.java | 14 +++---
.../queue/common/HadoopClusterConstants.java | 1 +
.../crawler/ClusterMetricsParseListener.java | 2 +-
.../storm/HadoopQueueMetricPersistBolt.java | 4 +-
...doop.queue.HadoopQueueRunningAppProvider.xml | 46 ++++++--------------
.../src/main/resources/application.conf | 18 +++-----
.../src/test/resources/application.conf | 18 +++-----
9 files changed, 55 insertions(+), 73 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/278b0961/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/config/EagleConfigConstants.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/config/EagleConfigConstants.java b/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/config/EagleConfigConstants.java
index 693041d..7c0c257 100644
--- a/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/config/EagleConfigConstants.java
+++ b/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/config/EagleConfigConstants.java
@@ -18,8 +18,11 @@ package org.apache.eagle.common.config;
public final class EagleConfigConstants {
public static final String SERVICE_ENV = "service.env";
+ public static final String SERVICE = "service";
public static final String SERVICE_HOST = "service.host";
public static final String SERVICE_PORT = "service.port";
+ public static final String SERVICE_USERNAME = "service.username";
+ public static final String SERVICE_PASSWORD = "service.password";
public static final String SERVICE_HBASE_ZOOKEEPER_QUORUM = "storage.hbase.zookeeperQuorum";
public static final String SERVICE_HBASE_ZOOKEEPER_PROPERTY_CLIENTPORT = "storage.hbase.zookeeperPropertyClientPort";
public static final String SERVICE_ZOOKEEPER_ZNODE_PARENT = "storage.hbase.zookeeperZnodeParent";
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/278b0961/eagle-core/eagle-query/eagle-client-base/src/main/java/org/apache/eagle/service/client/EagleServiceConnector.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/eagle-client-base/src/main/java/org/apache/eagle/service/client/EagleServiceConnector.java b/eagle-core/eagle-query/eagle-client-base/src/main/java/org/apache/eagle/service/client/EagleServiceConnector.java
index 2b2b1ac..a34f552 100644
--- a/eagle-core/eagle-query/eagle-client-base/src/main/java/org/apache/eagle/service/client/EagleServiceConnector.java
+++ b/eagle-core/eagle-query/eagle-client-base/src/main/java/org/apache/eagle/service/client/EagleServiceConnector.java
@@ -22,6 +22,8 @@ import org.apache.eagle.common.config.EagleConfigConstants;
import java.io.Serializable;
+import static org.apache.eagle.common.config.EagleConfigConstants.SERVICE_ENV;
+
/**
* Some common codes to enable DAO through eagle service including service host/post, credential population etc.
*/
@@ -59,12 +61,20 @@ public class EagleServiceConnector implements Serializable{
}
public EagleServiceConnector(Config config){
- this.eagleServiceHost = config.getString(EagleConfigConstants.EAGLE_PROPS + "." + EagleConfigConstants.EAGLE_SERVICE + "." + EagleConfigConstants.HOST);
- this.eagleServicePort = config.getInt(EagleConfigConstants.EAGLE_PROPS + "." + EagleConfigConstants.EAGLE_SERVICE + "." + EagleConfigConstants.PORT);
- if (config.hasPath(EagleConfigConstants.EAGLE_PROPS + "." + EagleConfigConstants.EAGLE_SERVICE + "." + EagleConfigConstants.USERNAME) &&
- config.hasPath(EagleConfigConstants.EAGLE_PROPS + "." + EagleConfigConstants.EAGLE_SERVICE + "." + EagleConfigConstants.PASSWORD)) {
- this.username = config.getString(EagleConfigConstants.EAGLE_PROPS + "." + EagleConfigConstants.EAGLE_SERVICE + "." + EagleConfigConstants.USERNAME);
- this.password = config.getString(EagleConfigConstants.EAGLE_PROPS + "." + EagleConfigConstants.EAGLE_SERVICE + "." + EagleConfigConstants.PASSWORD);
+ if (config.hasPath(EagleConfigConstants.SERVICE)) {
+ this.eagleServiceHost = config.getString(EagleConfigConstants.SERVICE_HOST);
+ this.eagleServicePort = config.getInt(EagleConfigConstants.SERVICE_PORT);
+ if (config.hasPath(EagleConfigConstants.SERVICE_USERNAME) && config.hasPath(EagleConfigConstants.SERVICE_PASSWORD)) {
+ this.username = config.getString(EagleConfigConstants.SERVICE_USERNAME);
+ this.password = config.getString(EagleConfigConstants.SERVICE_PASSWORD);
+ }
+ } else {
+ this.eagleServiceHost = config.getString(EagleConfigConstants.EAGLE_PROPS + "." + EagleConfigConstants.EAGLE_SERVICE + "." + EagleConfigConstants.HOST);
+ this.eagleServicePort = config.getInt(EagleConfigConstants.EAGLE_PROPS + "." + EagleConfigConstants.EAGLE_SERVICE + "." + EagleConfigConstants.PORT);
+ if (config.hasPath(EagleConfigConstants.EAGLE_PROPS + "." + EagleConfigConstants.EAGLE_SERVICE + "." + EagleConfigConstants.USERNAME) && config.hasPath(EagleConfigConstants.EAGLE_PROPS + "." + EagleConfigConstants.EAGLE_SERVICE + "." + EagleConfigConstants.PASSWORD)) {
+ this.username = config.getString(EagleConfigConstants.EAGLE_PROPS + "." + EagleConfigConstants.EAGLE_SERVICE + "." + EagleConfigConstants.USERNAME);
+ this.password = config.getString(EagleConfigConstants.EAGLE_PROPS + "." + EagleConfigConstants.EAGLE_SERVICE + "." + EagleConfigConstants.PASSWORD);
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/278b0961/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/HadoopQueueRunningAppConfig.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/HadoopQueueRunningAppConfig.java b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/HadoopQueueRunningAppConfig.java
index 5d5d736..e370ce9 100644
--- a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/HadoopQueueRunningAppConfig.java
+++ b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/HadoopQueueRunningAppConfig.java
@@ -37,9 +37,7 @@ public class HadoopQueueRunningAppConfig implements Serializable {
}
public static class Topology implements Serializable {
- public boolean localMode;
public int numOfParserTasks;
- public String name;
}
public static class DataSourceConfig implements Serializable {
@@ -82,17 +80,15 @@ public class HadoopQueueRunningAppConfig implements Serializable {
private void init(Config config) {
this.config = config;
- this.topology.localMode = config.getBoolean("topology.localMode");
this.topology.numOfParserTasks = config.getInt("topology.numOfParserTasks");
- this.topology.name = config.getString("topology.name");
this.dataSourceConfig.rMEndPoints = config.getString("dataSourceConfig.rMEndPoints");
this.dataSourceConfig.fetchIntervalSec = config.getString("dataSourceConfig.fetchIntervalSec");
- this.eagleProps.site = config.getString("eagleProps.site");
- this.eagleProps.eagleService.host = config.getString("eagleProps.eagleService.host");
- this.eagleProps.eagleService.port = config.getInt("eagleProps.eagleService.port");
- this.eagleProps.eagleService.username = config.getString("eagleProps.eagleService.username");
- this.eagleProps.eagleService.password = config.getString("eagleProps.eagleService.password");
+ this.eagleProps.site = config.getString("siteId");
+ this.eagleProps.eagleService.host = config.getString("service.host");
+ this.eagleProps.eagleService.port = config.getInt("service.port");
+ this.eagleProps.eagleService.username = config.getString("service.username");
+ this.eagleProps.eagleService.password = config.getString("service.password");
}
}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/278b0961/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/common/HadoopClusterConstants.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/common/HadoopClusterConstants.java b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/common/HadoopClusterConstants.java
index c924455..4a63343 100644
--- a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/common/HadoopClusterConstants.java
+++ b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/common/HadoopClusterConstants.java
@@ -45,6 +45,7 @@ public class HadoopClusterConstants {
public static final String HADOOP_CLUSTER_AVAILABLE_MEMORY = "hadoop.cluster.availablememory";
public static final String HADOOP_CLUSTER_RESERVED_MEMORY = "hadoop.cluster.reservedmemory";
public static final String HADOOP_CLUSTER_TOTAL_VIRTUAL_CORES = "hadoop.cluster.totalvirtualcores";
+ public static final String HADOOP_CLUSTER_ALLOCATED_VIRTUAL_CORES = "hadoop.cluster.allocatedvcores";
// metrics from scheduler info
public static final String HADOOP_CLUSTER_CAPACITY = "hadoop.cluster.capacity";
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/278b0961/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/crawler/ClusterMetricsParseListener.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/crawler/ClusterMetricsParseListener.java b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/crawler/ClusterMetricsParseListener.java
index ed78465..d26e295 100644
--- a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/crawler/ClusterMetricsParseListener.java
+++ b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/crawler/ClusterMetricsParseListener.java
@@ -83,7 +83,7 @@ public class ClusterMetricsParseListener {
createMetric(MetricName.HADOOP_CLUSTER_TOTAL_MEMORY, currentTimestamp, metrics.getTotalMB(), AggregateFunc.MAX);
createMetric(MetricName.HADOOP_CLUSTER_AVAILABLE_MEMORY, currentTimestamp, metrics.getAvailableMB(), AggregateFunc.AVG);
createMetric(MetricName.HADOOP_CLUSTER_RESERVED_MEMORY, currentTimestamp, metrics.getReservedMB(), AggregateFunc.AVG);
- createMetric(MetricName.HADOOP_CLUSTER_TOTAL_VIRTUAL_CORES, currentTimestamp, metrics.getTotalVirtualCores(), AggregateFunc.MAX);
+ createMetric(MetricName.HADOOP_CLUSTER_TOTAL_VIRTUAL_CORES, currentTimestamp, metrics.getAllocatedVirtualCores(), AggregateFunc.MAX);
}
public void flush() {
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/278b0961/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/storm/HadoopQueueMetricPersistBolt.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/storm/HadoopQueueMetricPersistBolt.java b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/storm/HadoopQueueMetricPersistBolt.java
index 4edf27d..3609184 100644
--- a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/storm/HadoopQueueMetricPersistBolt.java
+++ b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/storm/HadoopQueueMetricPersistBolt.java
@@ -28,7 +28,6 @@ import org.apache.eagle.hadoop.queue.common.HadoopClusterConstants;
import org.apache.eagle.hadoop.queue.model.scheduler.RunningQueueAPIEntity;
import org.apache.eagle.log.entity.GenericMetricEntity;
import org.apache.eagle.log.entity.GenericServiceAPIResponseEntity;
-import org.apache.eagle.service.client.EagleServiceConnector;
import org.apache.eagle.service.client.IEagleServiceClient;
import org.apache.eagle.service.client.impl.EagleServiceClientImpl;
import org.slf4j.Logger;
@@ -51,7 +50,8 @@ public class HadoopQueueMetricPersistBolt extends BaseRichBolt {
@Override
public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
- this.client = new EagleServiceClientImpl(new EagleServiceConnector(this.config.getConfig()));
+ HadoopQueueRunningAppConfig.EagleProps.EagleService eagleService = config.eagleProps.eagleService;
+ this.client = new EagleServiceClientImpl(eagleService.host, eagleService.port, eagleService.username, eagleService.password);
this.collector = collector;
}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/278b0961/eagle-jpm/eagle-hadoop-queue/src/main/resources/META-INF/providers/org.apache.eagle.hadoop.queue.HadoopQueueRunningAppProvider.xml
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-hadoop-queue/src/main/resources/META-INF/providers/org.apache.eagle.hadoop.queue.HadoopQueueRunningAppProvider.xml b/eagle-jpm/eagle-hadoop-queue/src/main/resources/META-INF/providers/org.apache.eagle.hadoop.queue.HadoopQueueRunningAppProvider.xml
index c5e9ff9..43092d2 100644
--- a/eagle-jpm/eagle-hadoop-queue/src/main/resources/META-INF/providers/org.apache.eagle.hadoop.queue.HadoopQueueRunningAppProvider.xml
+++ b/eagle-jpm/eagle-hadoop-queue/src/main/resources/META-INF/providers/org.apache.eagle.hadoop.queue.HadoopQueueRunningAppProvider.xml
@@ -23,50 +23,30 @@
<configuration>
<!-- org.apache.eagle.hadoop.queue.HadoopQueueRunningAppConfig -->
<property>
- <name>workers</name>
- <displayName>storm worker number</displayName>
- <value>4</value>
+ <name>dataSourceConfig.rMEndPoints</name>
+ <displayName>Resource Manager End Point</displayName>
+ <description>end point of resource manager, e.g. http://sandbox.hortonworks.com:8088/</description>
+ <value>http://sandbox.hortonworks.com:8088/</value>
+ <required>true</required>
</property>
<property>
- <name>topology.localMode</name>
- <value>true</value>
+ <name>workers</name>
+ <displayName>Storm Worker Number</displayName>
+ <description>the number of storm worker</description>
+ <value>1</value>
</property>
<property>
<name>topology.numOfParserTasks</name>
+ <displayName>Parallel Tasks Per Bolt</displayName>
+ <description>the number of tasks that should be assigned to execute a bolt</description>
<value>2</value>
</property>
<property>
- <name>topology.name</name>
- <value>sandbox-running-queue-topology</value>
- </property>
- <property>
- <name>dataSourceConfig.rMEndPoints</name>
- <value>http://sandbox.hortonworks.com:8088/</value>
- </property>
- <property>
<name>dataSourceConfig.fetchIntervalSec</name>
+ <displayName>Fetch Interval in Seconds</displayName>
+ <description>fetching interval in seconds</description>
<value>10</value>
</property>
- <property>
- <name>eagleProps.site</name>
- <value>sandbox</value>
- </property>
- <property>
- <name>eagleProps.eagleService.host</name>
- <value>localhost</value>
- </property>
- <property>
- <name>eagleProps.eagleService.port</name>
- <value>9099</value>
- </property>
- <property>
- <name>eagleProps.eagleService.username</name>
- <value>admin</value>
- </property>
- <property>
- <name>eagleProps.eagleService.password</name>
- <value>secret</value>
- </property>
</configuration>
<docs>
<install>
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/278b0961/eagle-jpm/eagle-hadoop-queue/src/main/resources/application.conf
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-hadoop-queue/src/main/resources/application.conf b/eagle-jpm/eagle-hadoop-queue/src/main/resources/application.conf
index 807bd5b..e5c9b81 100644
--- a/eagle-jpm/eagle-hadoop-queue/src/main/resources/application.conf
+++ b/eagle-jpm/eagle-hadoop-queue/src/main/resources/application.conf
@@ -15,25 +15,21 @@
{
"topology" : {
- "localMode" : true,
"numOfParserTasks" : 2,
- "name" : "sandbox-running-queue-topology",
},
"dataSourceConfig": {
"rMEndPoints" : "http://sandbox.hortonworks.com:8088/",
"fetchIntervalSec": "10"
},
- "eagleProps" : {
- "site": "sandbox",
- "eagleService": {
- "host": "localhost",
- "port": 9099
- "username": "admin",
- "password": "secret"
- }
+ "siteId": "sandbox"
+ "service": {
+ "host": "localhost"
+ "port": "9099"
+ "username": "admin"
+ "password": "secret"
}
"appId":"hadoopQueueMonitorJob",
"mode":"LOCAL",
application.storm.nimbusHost=localhost,
- "workers":4,
+ "workers":1,
}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/278b0961/eagle-jpm/eagle-hadoop-queue/src/test/resources/application.conf
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-hadoop-queue/src/test/resources/application.conf b/eagle-jpm/eagle-hadoop-queue/src/test/resources/application.conf
index 807bd5b..e5c9b81 100644
--- a/eagle-jpm/eagle-hadoop-queue/src/test/resources/application.conf
+++ b/eagle-jpm/eagle-hadoop-queue/src/test/resources/application.conf
@@ -15,25 +15,21 @@
{
"topology" : {
- "localMode" : true,
"numOfParserTasks" : 2,
- "name" : "sandbox-running-queue-topology",
},
"dataSourceConfig": {
"rMEndPoints" : "http://sandbox.hortonworks.com:8088/",
"fetchIntervalSec": "10"
},
- "eagleProps" : {
- "site": "sandbox",
- "eagleService": {
- "host": "localhost",
- "port": 9099
- "username": "admin",
- "password": "secret"
- }
+ "siteId": "sandbox"
+ "service": {
+ "host": "localhost"
+ "port": "9099"
+ "username": "admin"
+ "password": "secret"
}
"appId":"hadoopQueueMonitorJob",
"mode":"LOCAL",
application.storm.nimbusHost=localhost,
- "workers":4,
+ "workers":1,
}